From c15fa3962270123aa01973a4efa2f0c2cc213092 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Wed, 16 Sep 2026 00:53:04 +0300 Subject: [PATCH 01/25] feat: add draft Kubernetes execution provider Implement a separately deployed provider/controller, confined executor, optional mecak8s mTLS client, charts, and mock kind qualification. Keep lifecycle, distributed ownership, and acceptance gaps explicit for draft review. Co-Authored-By: Mecatl --- Taskfile.yml | 18 + build/execution-provider/Dockerfile | 16 + build/execution-workload/Dockerfile | 17 + cmd/mecak8s/flags.go | 28 + cmd/mecak8s/main.go | 21 + cmd/mecatl-execution-provider/main.go | 182 +++++++ cmd/mecatl-executor/main.go | 81 +++ deploy/helm/mecak8s/templates/deployment.yaml | 21 + deploy/helm/mecak8s/values.schema.json | 25 + deploy/helm/mecak8s/values.yaml | 11 + deploy/helm/mecatl-execution/Chart.yaml | 6 + deploy/helm/mecatl-execution/chart_test.go | 42 ++ .../crds/executionenvironment.yaml | 115 ++++ .../mecatl-execution/templates/_helpers.tpl | 3 + .../mecatl-execution/templates/provider.yaml | 74 +++ .../helm/mecatl-execution/templates/rbac.yaml | 38 ++ .../helm/mecatl-execution/values.schema.json | 12 + deploy/helm/mecatl-execution/values.yaml | 27 + deploy/mecatl-execution-kind/local-path.yaml | 33 ++ .../mecatl-execution-kind/mecak8s-values.yaml | 37 ++ .../mock-script-reattach.json | 7 + deploy/mecatl-execution-kind/mock-script.json | 16 + deploy/mecatl-execution-kind/oidc.yaml | 56 ++ deploy/mecatl-execution-kind/run.sh | 160 ++++++ docs/adr/0027-cloud-native.md | 14 + docs/architecture.md | 20 + docs/design/IMPLEMENTATION-NOTES.md | 34 ++ e2e/k8s_execution/fixture/oidcissuer/main.go | 91 ++++ e2e/k8s_execution/fixture/pki/main.go | 88 +++ e2e/k8s_execution/qualification_test.go | 380 +++++++++++++ engine/agent/authority_evaluator_test.go | 49 ++ internal/adapter/executionclient/client.go | 511 ++++++++++++++++++ .../adapter/executionclient/client_test.go | 302 +++++++++++ .../adapter/executioncontroller/controller.go | 428 +++++++++++++++ .../executioncontroller/controller_test.go | 193 +++++++ .../adapter/executioncontroller/handler.go | 409 ++++++++++++++ .../executioncontroller/handler_test.go | 195 +++++++ .../adapter/executioncontroller/podexec.go | 120 ++++ .../executioncontroller/podexec_test.go | 36 ++ .../adapter/executioncontroller/profiles.go | 152 ++++++ .../executioncontroller/profiles_test.go | 94 ++++ internal/adapter/executioncontroller/store.go | 401 ++++++++++++++ .../adapter/executioncontroller/store_test.go | 117 ++++ internal/adapter/server/acp_placement.go | 2 +- internal/adapter/server/classification.go | 77 +-- internal/adapter/server/learning.go | 4 +- .../adapter/server/local_session_context.go | 2 +- internal/adapter/server/placement.go | 29 +- .../adapter/server/placement_discovery.go | 2 +- .../adapter/server/placement_successor.go | 2 +- .../server/sdk_typescript_release_test.go | 1 + internal/adapter/server/service.go | 161 +++--- internal/app/build.go | 112 +++- internal/app/catalog.go | 37 +- internal/app/placement.go | 28 + internal/app/remote_execution_test.go | 142 +++++ internal/executionenv/errors.go | 117 ++++ internal/executionenv/grant.go | 163 ++++++ internal/executionenv/protocol_test.go | 67 +++ internal/executionenv/types.go | 247 +++++++++ internal/executionexecutor/command_linux.go | 169 ++++++ .../executionexecutor/command_unsupported.go | 24 + internal/executionexecutor/executor.go | 248 +++++++++ internal/executionexecutor/executor_test.go | 164 ++++++ user-docs/building/deployment/mecak8s.md | 114 ++++ 65 files changed, 6423 insertions(+), 169 deletions(-) create mode 100644 build/execution-provider/Dockerfile create mode 100644 build/execution-workload/Dockerfile create mode 100644 cmd/mecatl-execution-provider/main.go create mode 100644 cmd/mecatl-executor/main.go create mode 100644 deploy/helm/mecatl-execution/Chart.yaml create mode 100644 deploy/helm/mecatl-execution/chart_test.go create mode 100644 deploy/helm/mecatl-execution/crds/executionenvironment.yaml create mode 100644 deploy/helm/mecatl-execution/templates/_helpers.tpl create mode 100644 deploy/helm/mecatl-execution/templates/provider.yaml create mode 100644 deploy/helm/mecatl-execution/templates/rbac.yaml create mode 100644 deploy/helm/mecatl-execution/values.schema.json create mode 100644 deploy/helm/mecatl-execution/values.yaml create mode 100644 deploy/mecatl-execution-kind/local-path.yaml create mode 100644 deploy/mecatl-execution-kind/mecak8s-values.yaml create mode 100644 deploy/mecatl-execution-kind/mock-script-reattach.json create mode 100644 deploy/mecatl-execution-kind/mock-script.json create mode 100644 deploy/mecatl-execution-kind/oidc.yaml create mode 100755 deploy/mecatl-execution-kind/run.sh create mode 100644 e2e/k8s_execution/fixture/oidcissuer/main.go create mode 100644 e2e/k8s_execution/fixture/pki/main.go create mode 100644 e2e/k8s_execution/qualification_test.go create mode 100644 internal/adapter/executionclient/client.go create mode 100644 internal/adapter/executionclient/client_test.go create mode 100644 internal/adapter/executioncontroller/controller.go create mode 100644 internal/adapter/executioncontroller/controller_test.go create mode 100644 internal/adapter/executioncontroller/handler.go create mode 100644 internal/adapter/executioncontroller/handler_test.go create mode 100644 internal/adapter/executioncontroller/podexec.go create mode 100644 internal/adapter/executioncontroller/podexec_test.go create mode 100644 internal/adapter/executioncontroller/profiles.go create mode 100644 internal/adapter/executioncontroller/profiles_test.go create mode 100644 internal/adapter/executioncontroller/store.go create mode 100644 internal/adapter/executioncontroller/store_test.go create mode 100644 internal/app/remote_execution_test.go create mode 100644 internal/executionenv/errors.go create mode 100644 internal/executionenv/grant.go create mode 100644 internal/executionenv/protocol_test.go create mode 100644 internal/executionenv/types.go create mode 100644 internal/executionexecutor/command_linux.go create mode 100644 internal/executionexecutor/command_unsupported.go create mode 100644 internal/executionexecutor/executor.go create mode 100644 internal/executionexecutor/executor_test.go diff --git a/Taskfile.yml b/Taskfile.yml index 0d298bfaea..66dd433b90 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -72,6 +72,9 @@ tasks: # mecak8s: the storage-free, k8s-native agent binary (ADR 0048; a thin peer # of mecated composing app.Build with Redis store + k8s lease + drain gate). - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecak8s ./cmd/mecak8s + # Optional native Kubernetes execution provider and its fixed workload helper. + - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecatl-execution-provider ./cmd/mecatl-execution-provider + - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecatl-executor ./cmd/mecatl-executor - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecatui ./cmd/mecatui # engine/ is its own Go module (consumed via the committed go.work). Building # it here keeps the importable-core module compiling alongside the binaries. @@ -503,6 +506,21 @@ tasks: esac GOWORK=off go test -tags kind_e2e -count=1 -timeout 30m ./e2e/k8s/... + e2e:k8s:execution: + desc: | + Run the focused mock-only Kind qualification for the optional Kubernetes + execution provider. It uses a unique cluster and explicit kubeconfig under + .scratch, builds/loads immutable local images, and never reads a provider + credential. The owned cluster is intentionally retained; the command prints + the exact human-confirmed cleanup command. NOT part of task test. + preconditions: + - sh: command -v toolbox >/dev/null 2>&1 + msg: "toolbox is required (dev and sre toolboxes)" + - sh: command -v kind >/dev/null 2>&1 && command -v podman >/dev/null 2>&1 + msg: "kind and rootless Podman are required" + cmds: + - deploy/mecatl-execution-kind/run.sh + e2e:k8s:podman: desc: Run e2e:k8s with rootless Podman (requires the k8s-file Podman log driver) cmds: diff --git a/build/execution-provider/Dockerfile b/build/execution-provider/Dockerfile new file mode 100644 index 0000000000..7c3574b94d --- /dev/null +++ b/build/execution-provider/Dockerfile @@ -0,0 +1,16 @@ +# Both bases are mandatory digest references supplied by the release workflow. +ARG GO_IMAGE +ARG RUNTIME_IMAGE +FROM ${GO_IMAGE} AS build +ARG GO_IMAGE +ARG RUNTIME_IMAGE +WORKDIR /src +COPY . . +RUN case "${GO_IMAGE}" in *@sha256:*) ;; *) echo "GO_IMAGE must be digest-pinned" >&2; exit 1;; esac \ + && case "${RUNTIME_IMAGE}" in *@sha256:*) ;; *) echo "RUNTIME_IMAGE must be digest-pinned" >&2; exit 1;; esac \ + && CGO_ENABLED=0 go build -trimpath -buildvcs=false -o /out/mecatl-execution-provider ./cmd/mecatl-execution-provider + +FROM ${RUNTIME_IMAGE} +COPY --from=build --chown=65532:65532 /out/mecatl-execution-provider /mecatl-execution-provider +USER 65532:65532 +ENTRYPOINT ["/mecatl-execution-provider"] diff --git a/build/execution-workload/Dockerfile b/build/execution-workload/Dockerfile new file mode 100644 index 0000000000..1654cf2acb --- /dev/null +++ b/build/execution-workload/Dockerfile @@ -0,0 +1,17 @@ +# GO_IMAGE is deliberately mandatory and must include a digest, for example: +# docker build --build-arg GO_IMAGE=golang:1.27@sha256: ... +ARG GO_IMAGE +FROM ${GO_IMAGE} AS build +ARG GO_IMAGE +WORKDIR /src +COPY . . +RUN case "${GO_IMAGE}" in *@sha256:*) ;; *) echo "GO_IMAGE must be digest-pinned" >&2; exit 1;; esac \ + && go build -trimpath -buildvcs=false -o /out/mecatl-executor ./cmd/mecatl-executor + +FROM ${GO_IMAGE} +COPY --from=build --chown=65532:65532 /out/mecatl-executor /mecatl-executor +RUN mkdir -p /workspace /tmp && chown 65532:65532 /workspace /tmp +USER 65532:65532 +WORKDIR /workspace +ENV HOME=/workspace TMPDIR=/tmp GOTMPDIR=/tmp +ENTRYPOINT ["/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"] diff --git a/cmd/mecak8s/flags.go b/cmd/mecak8s/flags.go index 3d86ca44b5..078102e32c 100644 --- a/cmd/mecak8s/flags.go +++ b/cmd/mecak8s/flags.go @@ -120,6 +120,12 @@ type config struct { httpShutdownTimeout time.Duration closeTimeout time.Duration workspace string + executionEnabled bool + executionEndpoint string + executionProfile string + executionTLSCA string + executionTLSCert string + executionTLSKey string model string defaultProvider string defaultModel string @@ -352,6 +358,12 @@ func parseFlags(argv []string) (config, error) { positiveDurationFlag(fs, &cfg.httpShutdownTimeout, "http-shutdown-timeout", defaultHTTPShutdownTimeout, "Maximum time for graceful HTTP and metrics shutdown") positiveDurationFlag(fs, &cfg.closeTimeout, "close-timeout", defaultCloseTimeout, "Maximum time for final application cleanup") fs.StringVar(&cfg.workspace, "workspace", "", "Shared agent workspace root, such as a mounted PVC. Empty gives every session a shell-less, file-less workspace; clients cannot choose another root") + fs.BoolVar(&cfg.executionEnabled, "execution-enabled", false, "Use an independently deployed Kubernetes execution provider for default sessions") + fs.StringVar(&cfg.executionEndpoint, "execution-endpoint", "", "HTTPS endpoint of the execution provider. Requires --execution-enabled") + fs.StringVar(&cfg.executionProfile, "execution-profile", "", "Operator-configured execution provider profile") + fs.StringVar(&cfg.executionTLSCA, "execution-tls-ca", "", "Mounted CA bundle used only by the execution client") + fs.StringVar(&cfg.executionTLSCert, "execution-tls-cert", "", "Mounted execution-provider mTLS client certificate") + fs.StringVar(&cfg.executionTLSKey, "execution-tls-key", "", "Mounted execution-provider mTLS client private key") fs.StringVar(&cfg.model, "model", "", "Model identifier sent to the provider. Empty uses the provider default") fs.StringVar(&cfg.defaultProvider, "default-provider", "", "Deployment-wide default provider ID, such as openai, openrouter, or anthropic. Invalid values prevent startup") fs.StringVar(&cfg.defaultModel, "default-model", "", "Deployment-wide default model ID for the default provider. Invalid values prevent startup") @@ -612,6 +624,21 @@ func parseFlags(argv []string) (config, error) { if cfg.workspace != "" && (!filepath.IsAbs(cfg.workspace) || filepath.Clean(cfg.workspace) != cfg.workspace) { return config{}, fmt.Errorf("--workspace %q must be a clean absolute path (a mounted filesystem root); leave it empty for a file-less deployment", cfg.workspace) } + if cfg.executionEnabled { + if cfg.executionEndpoint == "" || cfg.executionProfile == "" || cfg.executionTLSCA == "" || cfg.executionTLSCert == "" || cfg.executionTLSKey == "" { + return config{}, errors.New("--execution-enabled requires --execution-endpoint, --execution-profile, --execution-tls-ca, --execution-tls-cert, and --execution-tls-key") + } + if !cfg.oidc.Enabled() { + return config{}, errors.New("--execution-enabled requires OIDC caller ownership enforcement") + } + if cfg.workspace != "" || cfg.redisFilesystem { + return config{}, errors.New("--execution-enabled conflicts with --workspace and --redis-filesystem") + } + if cfg.enableParallel || cfg.enableTeams { + return config{}, errors.New("remote execution does not support --enable-parallel or --enable-teams") + } + cfg.noScheduler = true + } if cfg.redisFilesystem && cfg.workspace != "" { return config{}, errors.New("--redis-filesystem and --workspace are mutually exclusive") } @@ -659,6 +686,7 @@ func appConfig(cfg config, diag port.Diagnostics, obs observability) app.Config mcpAuthorityDefault = mcpauthority.Global } out := app.Config{ + RemoteExecution: cfg.executionEnabled, Workspace: cfg.workspace, Model: cfg.model, DefaultProvider: cfg.defaultProvider, diff --git a/cmd/mecak8s/main.go b/cmd/mecak8s/main.go index d1bcc0c430..c9a6ae6302 100644 --- a/cmd/mecak8s/main.go +++ b/cmd/mecak8s/main.go @@ -10,6 +10,7 @@ import ( "os" "time" + "github.com/stacklok/mecatl/internal/adapter/executionclient" "github.com/stacklok/mecatl/internal/adapter/mockscript" "github.com/stacklok/mecatl/internal/adapter/slogdiag" "github.com/stacklok/mecatl/internal/app" @@ -82,6 +83,26 @@ func run() error { } composition := appConfig(cfg, diag, obs) + if cfg.executionEnabled { + tlsConfig, tlsErr := executionclient.LoadTLSConfig(executionclient.TLSFiles{CA: cfg.executionTLSCA, Cert: cfg.executionTLSCert, Key: cfg.executionTLSKey}) + if tlsErr != nil { + flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) + return tlsErr + } + client, clientErr := executionclient.New(cfg.executionEndpoint, tlsConfig) + if clientErr != nil { + flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) + return clientErr + } + defer client.Close() + placement, placementErr := executionclient.NewProvider(client, cfg.executionProfile) + if placementErr != nil { + flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) + return placementErr + } + composition.PlacementProvider = placement + composition.PlacementScope = "remote-execution" + } built, err := app.Build(ctx, composition) if err != nil { flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) diff --git a/cmd/mecatl-execution-provider/main.go b/cmd/mecatl-execution-provider/main.go new file mode 100644 index 0000000000..e53b6bd25f --- /dev/null +++ b/cmd/mecatl-execution-provider/main.go @@ -0,0 +1,182 @@ +// Command mecatl-execution-provider runs the authenticated Kubernetes execution provider. +package main + +import ( + "context" + "crypto/ed25519" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "flag" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func main() { + if err := run(); err != nil { + slog.Error("execution provider stopped", "error", err) + os.Exit(1) + } +} +func run() error { + var addr, namespace, profilesPath, certPath, keyPath, caPath, signingPath, keyID, issuer, audience, clients, attesters, admins string + flag.StringVar(&addr, "listen", ":8443", "HTTPS listen address") + flag.StringVar(&namespace, "namespace", "", "managed Kubernetes namespace") + flag.StringVar(&profilesPath, "profiles", "/etc/mecatl-execution/profiles.yaml", "strict operator profile file") + flag.StringVar(&certPath, "tls-cert", "/etc/mecatl-execution/tls/tls.crt", "server certificate") + flag.StringVar(&keyPath, "tls-key", "/etc/mecatl-execution/tls/tls.key", "server private key") + flag.StringVar(&caPath, "client-ca", "/etc/mecatl-execution/tls/ca.crt", "client CA bundle") + flag.StringVar(&signingPath, "grant-signing-key", "/etc/mecatl-execution/grant/key.pem", "Ed25519 PKCS8 grant signing key") + flag.StringVar(&keyID, "grant-key-id", "current", "grant verification key id") + flag.StringVar(&issuer, "grant-issuer", "mecatl-execution-provider", "exact grant issuer") + flag.StringVar(&audience, "grant-audience", "mecatl-execution", "exact grant audience") + flag.StringVar(&clients, "client-uri-san", "", "comma-separated exact client URI SAN allowlist") + flag.StringVar(&attesters, "owner-attester-uri-san", "", "comma-separated client identities allowed to attest owners") + flag.StringVar(&admins, "administrator-uri-san", "", "comma-separated retirement administrators") + flag.Parse() + if namespace == "" || clients == "" { + return errors.New("--namespace and --client-uri-san are required") + } + profiles, err := executioncontroller.LoadProfiles(profilesPath) + if err != nil { + return err + } + serverCert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return fmt.Errorf("load provider TLS identity: %w", err) + } + caPEM, err := os.ReadFile(caPath) + if err != nil { + return fmt.Errorf("load client CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return errors.New("client CA has no certificates") + } + priv, err := loadSigningKey(signingPath) + if err != nil { + return err + } + policies, err := clientPolicies(clients, attesters, admins) + if err != nil { + return err + } + cfg, err := rest.InClusterConfig() + if err != nil { + return fmt.Errorf("build in-cluster Kubernetes config: %w", err) + } + kube, err := kubernetes.NewForConfig(cfg) + if err != nil { + return err + } + dyn, err := dynamic.NewForConfig(cfg) + if err != nil { + return err + } + podexec := executioncontroller.NewPodExecutor(cfg, kube, namespace) + store := executioncontroller.NewStore(dyn, namespace, profiles, podexec) + reconciler := executioncontroller.NewReconciler(dyn, kube, namespace, profiles) + pub := priv.Public().(ed25519.PublicKey) + verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{keyID: pub}, Issuer: issuer, Audience: audience, MaxLifetime: 5 * time.Minute} + handler := executioncontroller.NewHandler(executioncontroller.HandlerConfig{Clients: policies, Signer: executioncontroller.GrantSigner{KeyID: keyID, PrivateKey: priv, Issuer: issuer, Audience: audience, Lifetime: time.Minute}, Verifier: verifier, Ready: reconciler.Ready}, store) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + if err := reconciler.Initialize(ctx); err != nil { + return fmt.Errorf("initialize controller: %w", err) + } + controllerErr := make(chan error, 1) + go func() { controllerErr <- reconciler.Run(ctx) }() + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + tlsLn := tls.NewListener(ln, executioncontroller.TLSConfig(serverCert, pool)) + server := &http.Server{Handler: handler, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 35 * time.Minute, IdleTimeout: 60 * time.Second} + serveErr := make(chan error, 1) + go func() { serveErr <- server.Serve(tlsLn) }() + select { + case err := <-controllerErr: + cancel() + _ = server.Shutdown(context.Background()) + if err != nil { + return fmt.Errorf("controller: %w", err) + } + return nil + case err := <-serveErr: + cancel() + if !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil + case <-ctx.Done(): + shutdown, c := context.WithTimeout(context.Background(), 10*time.Second) + defer c() + return server.Shutdown(shutdown) + } +} +func loadSigningKey(path string) (ed25519.PrivateKey, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("load grant signing key: %w", err) + } + block, _ := pem.Decode(b) + if block == nil || block.Type != "PRIVATE KEY" { + return nil, errors.New("grant signing key must be PKCS8 PEM") + } + raw, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, errors.New("grant signing key is invalid") + } + key, ok := raw.(ed25519.PrivateKey) + if !ok { + return nil, errors.New("grant signing key is not Ed25519") + } + return key, nil +} +func clientPolicies(clients, attesters, admins string) (map[string]executioncontroller.ClientPolicy, error) { + out := map[string]executioncontroller.ClientPolicy{} + for _, id := range splitList(clients) { + out[id] = executioncontroller.ClientPolicy{} + } + for _, id := range splitList(attesters) { + p, ok := out[id] + if !ok { + return nil, fmt.Errorf("owner attester %q is not a client", id) + } + p.MayAttestOwner = true + out[id] = p + } + for _, id := range splitList(admins) { + p, ok := out[id] + if !ok { + return nil, fmt.Errorf("administrator %q is not a client", id) + } + p.Administrator = true + out[id] = p + } + return out, nil +} +func splitList(v string) []string { + out := []string{} + for _, x := range strings.Split(v, ",") { + if x = strings.TrimSpace(x); x != "" { + out = append(out, x) + } + } + return out +} diff --git a/cmd/mecatl-executor/main.go b/cmd/mecatl-executor/main.go new file mode 100644 index 0000000000..34c594783e --- /dev/null +++ b/cmd/mecatl-executor/main.go @@ -0,0 +1,81 @@ +// Command mecatl-executor runs the fixed credential-free workload helper. +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/signal" + "runtime" + "syscall" + "time" + + "github.com/stacklok/mecatl/internal/executionenv" + "github.com/stacklok/mecatl/internal/executionexecutor" +) + +const workspaceRoot = "/workspace" + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + if err := run(ctx, os.Stdin, os.Stdout); err != nil { + _, _ = fmt.Fprintln(os.Stderr, "mecatl-executor: protocol output failed") + os.Exit(1) + } +} + +func run(parent context.Context, in io.Reader, out io.Writer) error { + var resp executionenv.ExecutorResponse + var opErr error + if runtime.GOOS != "linux" { + opErr = errors.New("mecatl-executor requires Linux at runtime") + } else if err := executionexecutor.EnableCommandExecution(); err != nil { + opErr = err + } else { + reader := bufio.NewReader(&io.LimitedReader{R: in, N: executionenv.MaxJSONBody + 1}) + body, err := reader.ReadBytes('\n') + if err != nil && !errors.Is(err, io.EOF) { + opErr = err + } else if len(body) > executionenv.MaxJSONBody { + opErr = &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "executor request exceeds limit"} + } else { + var q executionenv.ExecutorRequest + if err := executionenv.DecodeStrict(body, &q); err != nil { + opErr = err + } else { + x, err := executionexecutor.New(workspaceRoot, executionexecutor.Limits{CommandTimeout: 30 * time.Minute}) + if err != nil { + opErr = err + } else { + defer func() { _ = x.Close() }() + opCtx, cancel := context.WithCancel(parent) + if q.Operation == executionenv.OpCommandStart { + go func() { + var one [1]byte + _, _ = reader.Read(one[:]) + cancel() + }() + } + resp, opErr = x.Execute(opCtx, q) + cancel() + } + } + } + } + envelope := executionenv.ExecutorEnvelope{} + if opErr != nil { + var pe *executionenv.Error + if !errors.As(opErr, &pe) { + pe = &executionenv.Error{Code: executionenv.CodeInternal, Message: "executor operation failed"} + } + envelope.Error = pe + } else { + envelope.Response = &resp + } + return json.NewEncoder(out).Encode(envelope) +} diff --git a/deploy/helm/mecak8s/templates/deployment.yaml b/deploy/helm/mecak8s/templates/deployment.yaml index 9fd32c5019..78a15d6edf 100644 --- a/deploy/helm/mecak8s/templates/deployment.yaml +++ b/deploy/helm/mecak8s/templates/deployment.yaml @@ -130,6 +130,14 @@ spec: {{- if .Values.workspace }} - {{ printf "--workspace=%s" .Values.workspace | quote }} {{- end }} + {{- if .Values.execution.enabled }} + - --execution-enabled + - {{ printf "--execution-endpoint=%s" .Values.execution.endpoint | quote }} + - {{ printf "--execution-profile=%s" .Values.execution.profile | quote }} + - {{ printf "--execution-tls-ca=/var/run/secrets/execution/%s" .Values.execution.caKey | quote }} + - {{ printf "--execution-tls-cert=/var/run/secrets/execution/%s" .Values.execution.certKey | quote }} + - {{ printf "--execution-tls-key=/var/run/secrets/execution/%s" .Values.execution.keyKey | quote }} + {{- end }} {{- if .Values.mockProvider }} - --mock {{- end }} @@ -294,6 +302,9 @@ spec: {{- if .Values.oidc.caSecret }} - {name: oidc-ca, mountPath: /var/run/secrets/oidc-ca, readOnly: true} {{- end }} + {{- if .Values.execution.enabled }} + - {name: execution-mtls, mountPath: /var/run/secrets/execution, readOnly: true} + {{- end }} {{- if .Values.learning.store.tls.caSecret }} - {name: learning-store-ca, mountPath: /var/run/secrets/learning-store-ca, readOnly: true} {{- end }} @@ -342,6 +353,16 @@ spec: items: - {key: {{ .Values.oidc.caKey | quote }}, path: {{ .Values.oidc.caKey | quote }}} {{- end }} + {{- if .Values.execution.enabled }} + - name: execution-mtls + secret: + secretName: {{ .Values.execution.tlsSecret }} + defaultMode: 0440 + items: + - {key: {{ .Values.execution.caKey | quote }}, path: {{ .Values.execution.caKey | quote }}} + - {key: {{ .Values.execution.certKey | quote }}, path: {{ .Values.execution.certKey | quote }}} + - {key: {{ .Values.execution.keyKey | quote }}, path: {{ .Values.execution.keyKey | quote }}} + {{- end }} {{- if .Values.learning.store.tls.caSecret }} - name: learning-store-ca secret: diff --git a/deploy/helm/mecak8s/values.schema.json b/deploy/helm/mecak8s/values.schema.json index b17f2cd3f0..66049e7396 100644 --- a/deploy/helm/mecak8s/values.schema.json +++ b/deploy/helm/mecak8s/values.schema.json @@ -116,6 +116,20 @@ } }, "workspace": {"type": "string"}, + "execution": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "endpoint", "profile", "tlsSecret", "caKey", "certKey", "keyKey"], + "properties": { + "enabled": {"type": "boolean"}, + "endpoint": {"type": "string", "anyOf": [{"const": ""}, {"pattern": "^https://[^@/\\s#]+(?:/[^\\s?#]*)?$"}]}, + "profile": {"type": "string"}, + "tlsSecret": {"type": "string"}, + "caKey": {"type": "string", "minLength": 1}, + "certKey": {"type": "string", "minLength": 1}, + "keyKey": {"type": "string", "minLength": 1} + } + }, "imagePullSecrets": {"type": "array", "items": {"type": "object", "additionalProperties": false, "required": ["name"], "properties": {"name": {"type": "string", "minLength": 1}}}}, "mcp": { "type": "object", @@ -371,6 +385,17 @@ "podAnnotations": {"type": "object"} }, "allOf": [ + { + "if": {"properties": {"execution": {"properties": {"enabled": {"const": true}}}}}, + "then": { + "properties": { + "workspace": {"maxLength": 0}, + "execution": {"properties": {"endpoint": {"minLength": 1}, "profile": {"minLength": 1}, "tlsSecret": {"minLength": 1}}}, + "redis": {"properties": {"filesystem": {"properties": {"enabled": {"const": false}}}}}, + "oidc": {"properties": {"enabled": {"const": true}}} + } + } + }, { "if": {"properties": {"redis": {"properties": {"filesystem": {"properties": {"enabled": {"const": true}}}}}}}, "then": {"properties": {"workspace": {"maxLength": 0}}} diff --git a/deploy/helm/mecak8s/values.yaml b/deploy/helm/mecak8s/values.yaml index 621c326c5e..dfad257386 100644 --- a/deploy/helm/mecak8s/values.yaml +++ b/deploy/helm/mecak8s/values.yaml @@ -160,6 +160,17 @@ podDisruptionBudget: # the volume yourself; the chart only passes the flag. workspace: "" +# Optional client for the separately installed execution-provider service. The +# provider chart is not a dependency of this chart. +execution: + enabled: false + endpoint: "" + profile: "" + tlsSecret: "" + caKey: ca.crt + certKey: tls.crt + keyKey: tls.key + redis: # A production release never creates Redis. Supply the managed endpoint and, # when the service needs a private CA or ACL credentials, a Secret holding diff --git a/deploy/helm/mecatl-execution/Chart.yaml b/deploy/helm/mecatl-execution/Chart.yaml new file mode 100644 index 0000000000..7d35df551f --- /dev/null +++ b/deploy/helm/mecatl-execution/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: mecatl-execution +version: 0.1.0-draft +appVersion: dev +description: Optional native Kubernetes execution provider (draft) +type: application diff --git a/deploy/helm/mecatl-execution/chart_test.go b/deploy/helm/mecatl-execution/chart_test.go new file mode 100644 index 0000000000..615c04d0cf --- /dev/null +++ b/deploy/helm/mecatl-execution/chart_test.go @@ -0,0 +1,42 @@ +package chart_test + +import ( + "os" + "strings" + "testing" +) + +func TestChartRetainsCRDAndDoesNotGrantSecretAPI(t *testing.T) { + crd, err := os.ReadFile("crds/executionenvironment.yaml") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(crd), "helm.sh/resource-policy: keep") { + t.Fatal("CRD is not retained on uninstall") + } + rbac, err := os.ReadFile("templates/rbac.yaml") + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(rbac), "secrets") || strings.Contains(string(rbac), "clusterrole") { + t.Fatal("provider RBAC must be namespaced and have no Secret access") + } +} +func TestChartHasNoDeletionHook(t *testing.T) { + entries, err := os.ReadDir("templates") + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + b, err := os.ReadFile("templates/" + entry.Name()) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(b), "helm.sh/hook") || strings.Contains(string(b), "PersistentVolumeClaim\nmetadata") { + t.Fatalf("unsafe lifecycle resource in %s", entry.Name()) + } + } +} diff --git a/deploy/helm/mecatl-execution/crds/executionenvironment.yaml b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml new file mode 100644 index 0000000000..b56b180926 --- /dev/null +++ b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml @@ -0,0 +1,115 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: executionenvironments.execution.mecatl.dev + annotations: + helm.sh/resource-policy: keep +spec: + group: execution.mecatl.dev + scope: Namespaced + names: + plural: executionenvironments + singular: executionenvironment + kind: ExecutionEnvironment + shortNames: [execenv] + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + spec: + type: object + x-kubernetes-validations: + - rule: "!has(oldSelf.allocationID) || self.allocationID == oldSelf.allocationID" + message: allocationID is immutable + - rule: "!has(oldSelf.revision) || self.revision == oldSelf.revision" + message: revision is immutable + - rule: "!has(oldSelf.ownerHash) || self.ownerHash == oldSelf.ownerHash" + message: ownerHash is immutable + - rule: "!has(oldSelf.clientHash) || self.clientHash == oldSelf.clientHash" + message: clientHash is immutable + - rule: "!has(oldSelf.bindingID) || self.bindingID == oldSelf.bindingID" + message: bindingID is immutable + - rule: "!has(oldSelf.requestFingerprint) || self.requestFingerprint == oldSelf.requestFingerprint" + message: requestFingerprint is immutable + - rule: "!has(oldSelf.profileDigest) || self.profileDigest == oldSelf.profileDigest" + message: profileDigest is immutable + - rule: "!has(oldSelf.profile) || self.profile == oldSelf.profile" + message: profile is immutable + - rule: "!has(oldSelf.image) || self.image == oldSelf.image" + message: image is immutable + - rule: "!has(oldSelf.storageClass) || self.storageClass == oldSelf.storageClass" + message: storageClass is immutable + - rule: "!has(oldSelf.storageSize) || self.storageSize == oldSelf.storageSize" + message: storageSize is immutable + - rule: "!has(oldSelf.resources) || self.resources == oldSelf.resources" + message: resources are immutable + required: [allocationID, revision, ownerHash, clientHash, bindingID, requestFingerprint, profile, profileDigest, image, storageClass, storageSize, resources, desired] + properties: + allocationID: {type: string, minLength: 1, maxLength: 63} + revision: {type: string, minLength: 1, maxLength: 64} + ownerHash: {type: string, pattern: "^[0-9a-f]{64}$"} + clientHash: {type: string, pattern: "^[0-9a-f]{64}$"} + bindingID: {type: string, minLength: 1, maxLength: 253} + requestFingerprint: {type: string, pattern: "^[0-9a-f]{64}$"} + profile: {type: string, minLength: 1, maxLength: 63} + profileDigest: {type: string, pattern: "^sha256:[0-9a-f]{64}$"} + image: {type: string, pattern: "@sha256:[0-9a-fA-F]{64}$"} + storageClass: {type: string, minLength: 1} + storageSize: {type: string, minLength: 1} + resources: + type: object + additionalProperties: false + required: [cpuRequest, memoryRequest, cpuLimit, memoryLimit] + properties: + cpuRequest: {type: string, minLength: 1} + memoryRequest: {type: string, minLength: 1} + cpuLimit: {type: string, minLength: 1} + memoryLimit: {type: string, minLength: 1} + desired: {type: string, enum: [Active, Retiring]} + status: + type: object + properties: + observedGeneration: {type: integer, format: int64} + epoch: {type: integer, format: int64, minimum: 1} + fenceState: {type: string, enum: [Healthy, FenceUnknown]} + references: + type: array + maxItems: 64 + x-kubernetes-list-type: set + items: {type: string, maxLength: 253} + pvc: + type: object + properties: + name: {type: string} + uid: {type: string} + pod: + type: object + properties: + name: {type: string} + uid: {type: string} + activeOperation: + type: object + required: [id, operation, startedAt] + properties: + id: {type: string} + operation: {type: string} + startedAt: {type: string, format: date-time} + conditions: + type: array + items: + type: object + required: [type, status, reason, message, observedGeneration, lastTransitionTime] + properties: + type: {type: string} + status: {type: string, enum: ["True", "False", Unknown]} + reason: {type: string} + message: {type: string, maxLength: 1024} + observedGeneration: {type: integer, format: int64} + lastTransitionTime: {type: string, format: date-time} diff --git a/deploy/helm/mecatl-execution/templates/_helpers.tpl b/deploy/helm/mecatl-execution/templates/_helpers.tpl new file mode 100644 index 0000000000..66568c818a --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/_helpers.tpl @@ -0,0 +1,3 @@ +{{- define "mecatl-execution.fullname" -}} +{{- default (printf "%s-mecatl-execution" .Release.Name) .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/deploy/helm/mecatl-execution/templates/provider.yaml b/deploy/helm/mecatl-execution/templates/provider.yaml new file mode 100644 index 0000000000..bbc9bf458c --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/provider.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "mecatl-execution.fullname" . }}-profiles +data: + profiles.yaml: | + profiles: +{{ toYaml .Values.profiles | indent 6 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + replicas: 1 + selector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + template: + metadata: + labels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + spec: + serviceAccountName: {{ include "mecatl-execution.fullname" . }} + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + containers: + - name: provider + image: {{ required "provider.image must be digest-pinned" .Values.provider.image | quote }} + imagePullPolicy: {{ .Values.provider.imagePullPolicy }} + args: + - --namespace={{ .Release.Namespace }} + - --profiles=/etc/mecatl-execution/profiles.yaml + - --tls-cert=/etc/mecatl-execution/tls/tls.crt + - --tls-key=/etc/mecatl-execution/tls/tls.key + - --client-ca=/etc/mecatl-execution/tls/ca.crt + - --grant-signing-key=/etc/mecatl-execution/grant/key.pem + - --grant-key-id={{ .Values.provider.grantKeyID }} + - --client-uri-san={{ join "," .Values.provider.clientURISANs }} + - --owner-attester-uri-san={{ join "," .Values.provider.ownerAttesterURISANs }} + - --administrator-uri-san={{ join "," .Values.provider.administratorURISANs }} + ports: + - {name: https, containerPort: 8443} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: ["ALL"]} + resources: + requests: {cpu: 100m, memory: 128Mi} + limits: {cpu: "1", memory: 512Mi} + volumeMounts: + - {name: profiles, mountPath: /etc/mecatl-execution/profiles.yaml, subPath: profiles.yaml, readOnly: true} + - {name: tls, mountPath: /etc/mecatl-execution/tls, readOnly: true} + - {name: grant, mountPath: /etc/mecatl-execution/grant, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: profiles + configMap: {name: {{ include "mecatl-execution.fullname" . }}-profiles} + - name: tls + secret: {secretName: {{ required "provider.tlsSecretName is required" .Values.provider.tlsSecretName }}} + - name: grant + secret: {secretName: {{ required "provider.grantSigningSecretName is required" .Values.provider.grantSigningSecretName }}} + - name: tmp + emptyDir: {sizeLimit: 64Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + selector: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + ports: + - {name: https, port: {{ .Values.service.port }}, targetPort: https} diff --git a/deploy/helm/mecatl-execution/templates/rbac.yaml b/deploy/helm/mecatl-execution/templates/rbac.yaml new file mode 100644 index 0000000000..7c561c8365 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/rbac.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "mecatl-execution.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "mecatl-execution.fullname" . }} +rules: +- apiGroups: ["execution.mecatl.dev"] + resources: ["executionenvironments"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +- apiGroups: ["execution.mecatl.dev"] + resources: ["executionenvironments/status"] + verbs: ["get", "update", "patch"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +- apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "mecatl-execution.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "mecatl-execution.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "mecatl-execution.fullname" . }} diff --git a/deploy/helm/mecatl-execution/values.schema.json b/deploy/helm/mecatl-execution/values.schema.json new file mode 100644 index 0000000000..adcc877382 --- /dev/null +++ b/deploy/helm/mecatl-execution/values.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": ["provider", "service", "profiles"], + "properties": { + "fullnameOverride": {"type":"string"}, + "provider": {"type":"object","additionalProperties":false,"required":["image","clientURISANs","ownerAttesterURISANs","tlsSecretName","grantSigningSecretName"],"properties":{"image":{"type":"string","pattern":"@sha256:[0-9a-fA-F]{64}$"},"imagePullPolicy":{"enum":["Always","IfNotPresent","Never"]},"replicas":{"const":1},"clientURISANs":{"type":"array","minItems":1,"uniqueItems":true,"items":{"type":"string","pattern":"^[a-z][a-z0-9+.-]*://"}},"ownerAttesterURISANs":{"type":"array","minItems":1,"uniqueItems":true,"items":{"type":"string"}},"administratorURISANs":{"type":"array","uniqueItems":true,"items":{"type":"string"}},"tlsSecretName":{"type":"string","minLength":1},"grantSigningSecretName":{"type":"string","minLength":1},"grantKeyID":{"type":"string","minLength":1}}}, + "service":{"type":"object","additionalProperties":false,"required":["port"],"properties":{"port":{"type":"integer","minimum":1,"maximum":65535}}}, + "profiles":{"type":"object","minProperties":1,"additionalProperties":{"type":"object","additionalProperties":false,"required":["image","storageClass","storageSize","cpuRequest","memoryRequest","cpuLimit","memoryLimit","maxFileBytes","maxCommandBytes","maxCommandDuration"],"properties":{"image":{"type":"string","pattern":"@sha256:[0-9a-fA-F]{64}$"},"storageClass":{"type":"string","minLength":1},"storageSize":{"type":"string","minLength":1},"cpuRequest":{"type":"string","minLength":1},"memoryRequest":{"type":"string","minLength":1},"cpuLimit":{"type":"string","minLength":1},"memoryLimit":{"type":"string","minLength":1},"maxFileBytes":{"type":"integer","minimum":1,"maximum":5242880},"maxCommandBytes":{"type":"integer","minimum":1,"maximum":1048576},"maxCommandDuration":{"type":"string","pattern":"^[1-9][0-9]*(ms|s|m)$"}}}} + } +} diff --git a/deploy/helm/mecatl-execution/values.yaml b/deploy/helm/mecatl-execution/values.yaml new file mode 100644 index 0000000000..bb00fe83a6 --- /dev/null +++ b/deploy/helm/mecatl-execution/values.yaml @@ -0,0 +1,27 @@ +fullnameOverride: "" +provider: + image: "" + imagePullPolicy: IfNotPresent + replicas: 1 + clientURISANs: [] + ownerAttesterURISANs: [] + administratorURISANs: [] + tlsSecretName: "" + grantSigningSecretName: "" + grantKeyID: current +service: + port: 8443 +profiles: {} +# Example profile (all fields are required): +# profiles: +# go: +# image: registry.example/workload@sha256: +# storageClass: standard +# storageSize: 2Gi +# cpuRequest: 100m +# memoryRequest: 128Mi +# cpuLimit: "1" +# memoryLimit: 1Gi +# maxFileBytes: 5242880 +# maxCommandBytes: 1048576 +# maxCommandDuration: 5m diff --git a/deploy/mecatl-execution-kind/local-path.yaml b/deploy/mecatl-execution-kind/local-path.yaml new file mode 100644 index 0000000000..bb7d4991fb --- /dev/null +++ b/deploy/mecatl-execution-kind/local-path.yaml @@ -0,0 +1,33 @@ +# Recovery fixture for a Kind local-path provisioner whose helper image was +# changed during an interrupted qualification setup. Fresh clusters do not need +# this manifest. The workload image supplies /bin/sh, mkdir, and chown. +apiVersion: v1 +kind: ConfigMap +metadata: + name: local-path-config + namespace: local-path-storage +data: + config.json: |- + {"nodePathMap":[{"node":"DEFAULT_PATH_FOR_NON_LISTED_NODES","paths":["/var/local-path-provisioner"]}]} + setup: |- + #!/bin/sh + set -eu + mkdir -m 0770 -p "$VOL_DIR" + chown 65532:65532 "$VOL_DIR" + teardown: |- + #!/bin/sh + echo "retained by execution qualification; no automatic deletion" + helperPod.yaml: |- + apiVersion: v1 + kind: Pod + metadata: + name: helper-pod + spec: + securityContext: {runAsUser: 0, runAsGroup: 0} + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: helper-pod + image: WORKLOAD_IMAGE + imagePullPolicy: IfNotPresent diff --git a/deploy/mecatl-execution-kind/mecak8s-values.yaml b/deploy/mecatl-execution-kind/mecak8s-values.yaml new file mode 100644 index 0000000000..dc684e06f1 --- /dev/null +++ b/deploy/mecatl-execution-kind/mecak8s-values.yaml @@ -0,0 +1,37 @@ +fullnameOverride: mecak8s +replicaCount: 1 +mockProvider: true +redis: + endpoint: redis:6379 + credentialsSecret: "" + local: + enabled: true +execution: + enabled: true + endpoint: https://mecatl-execution.execution-qualification.svc.cluster.local:8443 + profile: go + tlsSecret: execution-client-tls + caKey: ca.crt + certKey: tls.crt + keyKey: tls.key +oidc: + enabled: true + issuer: https://oidc-issuer.execution-qualification.svc.cluster.local:8443 + audience: mecatl + allowPrivateHTTPSIssuer: true + caSecret: execution-client-tls + caKey: ca.crt +extraArgs: +- --mock-script=/var/run/mecatl-execution/mock-script.json +- --no-soul +- --no-user-model +- --permissions-conventional=false +- --agents-conventional=false +extraVolumeMounts: +- name: execution-mock + mountPath: /var/run/mecatl-execution + readOnly: true +extraVolumes: +- name: execution-mock + configMap: + name: execution-mock diff --git a/deploy/mecatl-execution-kind/mock-script-reattach.json b/deploy/mecatl-execution-kind/mock-script-reattach.json new file mode 100644 index 0000000000..104f83ecf3 --- /dev/null +++ b/deploy/mecatl-execution-kind/mock-script-reattach.json @@ -0,0 +1,7 @@ +{ + "turns": [ + {"tool_calls":[{"id":"reattach-read","name":"Read","args":{"path":"proof.txt"}}]}, + {"tool_calls":[{"id":"reattach-shell","name":"Shell","args":{"command":"go test ./...","timeout_ms":120000}}]}, + {"text":"REMOTE_EXECUTION_REATTACH_COMPLETE"} + ] +} diff --git a/deploy/mecatl-execution-kind/mock-script.json b/deploy/mecatl-execution-kind/mock-script.json new file mode 100644 index 0000000000..16e59416b7 --- /dev/null +++ b/deploy/mecatl-execution-kind/mock-script.json @@ -0,0 +1,16 @@ +{ + "turns": [ + {"tool_calls":[{"id":"write-proof","name":"Write","args":{"path":"proof.txt","content":"alpha\n"}}]}, + {"tool_calls":[{"id":"read-before-edit","name":"Read","args":{"path":"proof.txt"}}]}, + {"tool_calls":[{"id":"edit-proof","name":"Edit","args":{"path":"proof.txt","old_string":"alpha","new_string":"beta"}}]}, + {"tool_calls":[{"id":"copy-proof","name":"Copy","args":{"source":"proof.txt","destination":"copy.txt"}}]}, + {"tool_calls":[{"id":"move-proof","name":"Move","args":{"source":"copy.txt","destination":"moved.txt"}}]}, + {"tool_calls":[{"id":"glob-proof","name":"Glob","args":{"pattern":"*.txt"}},{"id":"grep-proof","name":"Grep","args":{"pattern":"beta","path":"*.txt"}}]}, + {"tool_calls":[{"id":"write-mod","name":"Write","args":{"path":"go.mod","content":"module example.com/qualification\n\ngo 1.27\n"}}]}, + {"tool_calls":[{"id":"write-test","name":"Write","args":{"path":"remote_test.go","content":"package qualification\n\nimport (\"os\"; \"testing\")\n\nfunc TestRemoteWorkspace(t *testing.T) { b, err := os.ReadFile(\"proof.txt\"); if err != nil { t.Fatal(err) }; if string(b) != \"beta\\n\" { t.Fatalf(\"proof = %q\", b) } }\n"}}]}, + {"tool_calls":[{"id":"shell-go-test","name":"Shell","args":{"command":"go test ./...","timeout_ms":120000}}]}, + {"tool_calls":[{"id":"remove-moved","name":"Remove","args":{"path":"moved.txt"}}]}, + {"tool_calls":[{"id":"read-final","name":"Read","args":{"path":"proof.txt"}}]}, + {"text":"REMOTE_EXECUTION_QUALIFICATION_COMPLETE"} + ] +} diff --git a/deploy/mecatl-execution-kind/oidc.yaml b/deploy/mecatl-execution-kind/oidc.yaml new file mode 100644 index 0000000000..5f9775c216 --- /dev/null +++ b/deploy/mecatl-execution-kind/oidc.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oidc-issuer + namespace: execution-qualification +spec: + replicas: 1 + selector: + matchLabels: {app.kubernetes.io/name: oidc-issuer} + template: + metadata: + labels: {app.kubernetes.io/name: oidc-issuer} + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + containers: + - name: issuer + image: OIDC_IMAGE + imagePullPolicy: IfNotPresent + args: + - --tls-cert=/identity/tls.crt + - --tls-key=/identity/tls.key + - --signing-key=/identity/jwt-key.pem + - --issuer=https://oidc-issuer.execution-qualification.svc.cluster.local:8443 + - --audience=mecatl + ports: [{name: https, containerPort: 8443}] + readinessProbe: {httpGet: {scheme: HTTPS, path: /healthz, port: https}, periodSeconds: 2} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: ["ALL"]} + resources: + requests: {cpu: 10m, memory: 32Mi} + limits: {cpu: 200m, memory: 128Mi} + volumeMounts: + - {name: identity, mountPath: /identity, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: identity + secret: {secretName: oidc-fixture-identity, defaultMode: 0440} + - name: tmp + emptyDir: {sizeLimit: 16Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: oidc-issuer + namespace: execution-qualification +spec: + selector: {app.kubernetes.io/name: oidc-issuer} + ports: [{name: https, port: 8443, targetPort: https}] diff --git a/deploy/mecatl-execution-kind/run.sh b/deploy/mecatl-execution-kind/run.sh new file mode 100755 index 0000000000..765ec169fa --- /dev/null +++ b/deploy/mecatl-execution-kind/run.sh @@ -0,0 +1,160 @@ +#!/bin/sh +set -eu + +# Re-exec once with a deliberately small environment. In particular, no +# provider/API credential or ambient kube context can reach any subprocess. +if [ "${MECATL_EXECUTION_QUAL_CLEAN_ENV:-}" != 1 ]; then + exec env -i \ + HOME="$HOME" USER="${USER:-user}" PATH="$PATH" XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" \ + DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-}" CONTAINER_HOST="${CONTAINER_HOST:-}" DOCKER_HOST="${DOCKER_HOST:-}" \ + TOOLBOX_PATH="${TOOLBOX_PATH:-}" MECATL_EXECUTION_DEV_TOOLBOX="${MECATL_EXECUTION_DEV_TOOLBOX:-}" MECATL_EXECUTION_K8S_TOOLBOX="${MECATL_EXECUTION_K8S_TOOLBOX:-}" \ + MECATL_EXECUTION_QUAL_CLEAN_ENV=1 "$0" "$@" +fi + +[ "$#" -eq 0 ] || { echo "usage: $0" >&2; exit 2; } +root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +run_id=$(date -u +%Y%m%d%H%M%S)-$$ +cluster="mecatl-execution-qual-$run_id" +context="kind-$cluster" +state="$root/.scratch/k8s-execution/$run_id" +kubeconfig="$state/kubeconfig" +mkdir -p "$root/.scratch/k8s-execution" +umask 077 +mkdir "$state" +mkdir "$state/pki" "$state/images" +printf 'cluster=%s\ncontext=%s\nkubeconfig=%s\nnamespace=execution-qualification\nowner=%s\ncreated_at=%s\n' \ + "$cluster" "$context" "$kubeconfig" "${USER:-user}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$state/ownership" + +# Refuse collisions; never delete or reuse any cluster. +if kind get clusters 2>/dev/null | grep -Fx "$cluster" >/dev/null; then + echo "qualification cluster already exists: $cluster" >&2 + exit 1 +fi +printf '%s\n' "$state" >"$root/.scratch/k8s-execution/current" + +export KIND_EXPERIMENTAL_PROVIDER=podman +export KUBECONFIG="$kubeconfig" +printf '{}\n' >"$state/registry-auth.json" +chmod 600 "$state/registry-auth.json" +export REGISTRY_AUTH_FILE="$state/registry-auth.json" + +kind create cluster --name "$cluster" --image kindest/node:v1.35.8@sha256:07b2536e30b803ed61d1677a79df6115f798ce64c80f9e22f6ed45afd09323c0 --kubeconfig "$kubeconfig" + +dev() { + if [ -n "$MECATL_EXECUTION_DEV_TOOLBOX" ]; then + (cd "$root" && toolbox run -c "$MECATL_EXECUTION_DEV_TOOLBOX" "$@") + else + (cd "$root" && "$@") + fi +} +kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then + toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" kubectl --kubeconfig "$kubeconfig" --context "$context" "$@" + else + kubectl --kubeconfig "$kubeconfig" --context "$context" "$@" + fi +} +helm_kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then + toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@" + else + helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@" + fi +} + +# Synthetic keys are generated by the fixture only and never printed. Resume +# keeps the original PKI so mounted trust and running identities cannot drift. +if [ ! -s "$state/pki/ca.crt" ]; then + dev go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/pki "$state/pki" +fi + +# Build the three static Go images in the development toolbox. ko emits a +# content tag; after loading, the fixture resolves the node's actual manifest +# digest and creates a matching digest alias for Kubernetes. +build_ko() { + package=$1 repo=$2 + dev env KIND_EXPERIMENTAL_PROVIDER=podman KO_DOCKER_REPO="$repo" ko build --local --bare "$package" | tail -n 1 +} +provider_tag=$(build_ko ./cmd/mecatl-execution-provider ko.local/mecatl-execution-provider) +agent_tag=$(build_ko ./cmd/mecak8s ko.local/mecak8s) +oidc_tag=$(dev env KIND_EXPERIMENTAL_PROVIDER=podman GOFLAGS=-tags=kind_execution_e2e KO_DOCKER_REPO=ko.local/mecatl-oidc-fixture ko build --local --bare ./e2e/k8s_execution/fixture/oidcissuer | tail -n 1) + +# Resolve the public Go base from the local OCI store, then feed its observed +# digest to the production workload recipe without weakening its pin check. +podman image exists docker.io/library/golang:1.27 || podman pull docker.io/library/golang:1.27 >/dev/null +go_digest=$(podman image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') +go_image="docker.io/library/golang@${go_digest}" +workload_tag="localhost/mecatl-execution-workload:e2e" +podman build --build-arg GO_IMAGE="$go_image" -f "$root/build/execution-workload/Dockerfile" -t "$workload_tag" "$root" >/dev/null + +for item in "$provider_tag" "$agent_tag" "$oidc_tag" "$workload_tag"; do + archive="$state/images/$(printf '%s' "$item" | sha256sum | cut -c1-16).tar" + podman save "$item" -o "$archive" >/dev/null + kind load image-archive "$archive" --name "$cluster" +done +node="${cluster}-control-plane" +pin_loaded() { + tagged=$1 + digest=$(podman exec "$node" ctr -n k8s.io images ls | awk -v ref="$tagged" '$1 == ref {print $3; exit}') + [ -n "$digest" ] || { echo "loaded image digest unavailable for $tagged" >&2; exit 1; } + pinned="${tagged%:*}@${digest}" + podman exec "$node" ctr -n k8s.io images tag "$tagged" "$pinned" >/dev/null + printf '%s\n' "$pinned" +} +provider_image=$(pin_loaded "$provider_tag") +agent_image=$(pin_loaded "$agent_tag") +oidc_image=$(pin_loaded "$oidc_tag") +workload_image=$(pin_loaded "$workload_tag") +printf 'provider=%s\nagent=%s\noidc=%s\nworkload=%s\ngo_base=%s\n' "$provider_image" "$agent_image" "$oidc_image" "$workload_image" "$go_image" >"$state/images/proof" + +kube create namespace execution-qualification --dry-run=client -o yaml | kube apply -f - +kube label namespace execution-qualification pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/audit=restricted pod-security.kubernetes.io/warn=restricted --overwrite + +# Kind v0.33 installs its pinned local-path provisioner and default "standard" +# StorageClass as part of cluster creation. + +kube -n execution-qualification create secret generic execution-provider-tls \ + --from-file=ca.crt="$state/pki/ca.crt" --from-file=tls.crt="$state/pki/provider.crt" --from-file=tls.key="$state/pki/provider.key" --dry-run=client -o yaml | kube apply -f - +kube -n execution-qualification create secret generic execution-client-tls \ + --from-file=ca.crt="$state/pki/ca.crt" --from-file=tls.crt="$state/pki/mecak8s.crt" --from-file=tls.key="$state/pki/mecak8s.key" --dry-run=client -o yaml | kube apply -f - +kube -n execution-qualification create secret generic execution-intruder-tls \ + --from-file=ca.crt="$state/pki/ca.crt" --from-file=tls.crt="$state/pki/intruder.crt" --from-file=tls.key="$state/pki/intruder.key" --dry-run=client -o yaml | kube apply -f - +kube -n execution-qualification create secret generic execution-grant-key --from-file=key.pem="$state/pki/grant-key.pem" --dry-run=client -o yaml | kube apply -f - +kube -n execution-qualification create secret generic oidc-fixture-identity \ + --from-file=tls.crt="$state/pki/oidc.crt" --from-file=tls.key="$state/pki/oidc.key" --from-file=jwt-key.pem="$state/pki/oidc-jwt-key.pem" --dry-run=client -o yaml | kube apply -f - + +sed "s|OIDC_IMAGE|$oidc_image|g" "$root/deploy/mecatl-execution-kind/oidc.yaml" | kube apply -f - +kube -n execution-qualification rollout restart deployment/oidc-issuer +kube -n execution-qualification rollout status deployment/oidc-issuer --timeout=180s + +helm_kube upgrade --install mecatl-execution "$root/deploy/helm/mecatl-execution" \ + --namespace execution-qualification \ + --set fullnameOverride=mecatl-execution \ + --set-string provider.image="$provider_image" \ + --set provider.imagePullPolicy=IfNotPresent \ + --set provider.tlsSecretName=execution-provider-tls \ + --set provider.grantSigningSecretName=execution-grant-key \ + --set 'provider.clientURISANs={spiffe://mecatl.test/client/mecak8s,spiffe://mecatl.test/client/intruder}' \ + --set 'provider.ownerAttesterURISANs={spiffe://mecatl.test/client/mecak8s,spiffe://mecatl.test/client/intruder}' \ + --set 'provider.administratorURISANs={spiffe://mecatl.test/client/mecak8s}' \ + --set-string profiles.go.image="$workload_image" \ + --set profiles.go.storageClass=standard --set profiles.go.storageSize=1Gi \ + --set profiles.go.cpuRequest=50m --set profiles.go.memoryRequest=64Mi \ + --set-string profiles.go.cpuLimit=1 --set profiles.go.memoryLimit=512Mi \ + --set profiles.go.maxFileBytes=5242880 --set profiles.go.maxCommandBytes=1048576 \ + --set profiles.go.maxCommandDuration=5m --wait --timeout=4m + +kube -n execution-qualification create configmap execution-mock --from-file=mock-script.json="$root/deploy/mecatl-execution-kind/mock-script.json" --dry-run=client -o yaml | kube apply -f - +helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" \ + --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" \ + --wait --timeout=5m +kube -n execution-qualification rollout restart deployment/mecak8s +kube -n execution-qualification rollout status deployment/mecak8s --timeout=240s + +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_QUAL_STATE="$state" \ + go test -tags kind_execution_e2e -count=1 -timeout=15m ./e2e/k8s_execution + +echo "qualification cluster retained: $cluster" +echo "kubeconfig: $kubeconfig" +echo "cleanup (requires human confirmation): KIND_EXPERIMENTAL_PROVIDER=podman kind delete cluster --name $cluster" diff --git a/docs/adr/0027-cloud-native.md b/docs/adr/0027-cloud-native.md index c936159c64..d8de5f32b7 100644 --- a/docs/adr/0027-cloud-native.md +++ b/docs/adr/0027-cloud-native.md @@ -981,6 +981,19 @@ rather than papered over. | 72 | Per-generation isolated Redis follow client and bounded connection pool | each redisstore `clientGeneration`, owned by `clientGenerations` | one follow client per live or retiring credential generation; `PoolSize` and `MaxActiveConns` use the configured process-local follow-pool size | a rejected candidate closes both clients; generation retirement closes the pair after its leases release. `Store.Close` may asynchronously force-close only the follow client when an admitted read misses the cooperative join bound, while durability clients retain their lease-safe close rule | reconstructible from Redis connection settings and mounted credential files. A restart builds and probes a new pair; it retains no watch position or session authority | `internal/adapter/redisstore/generation.go` (`clientPair`, `buildClientPair`, `forceCloseFollow`); `internal/adapter/redisstore/reload.go` (`reloadCandidate`); ADR 0330 | | 73 | Redis follower admission, cancellation, and join registry | redisstore `Store` owns one `followerRegistry` | process; one entry per admitted `ReadAfter` iterator with `Follow:true`, capped by `MaxFollowers` | iterator release removes its entry on every exit. `Store.Close` linearizes against admission, rejects new followers, cancels the active snapshot, waits cooperatively, and retains bounded late-cleanup ownership after a follow-client force-close | reset-by-design and reattached by the client. The registry contains no cursor or durable session state; after restart the client resumes with its last processed cursor and unchanged filter | `internal/adapter/redisstore/followers.go` (`followerRegistry`); `internal/adapter/redisstore/cursoreventlog.go` (`ReadAfter`); `internal/adapter/redisstore/redisstore.go` (`Close`); ADR 0330 | | 74 | Prompt-free persisted-ask continuation contexts and detached relay join registry | `internal/adapter/server.Service` owns the post-acceptance context, exact resumed `agent.Run`, recorder/relay goroutine, and `detachedControlWG`; the unary caller owns only work before atomic acceptance | one process-local entry per accepted persisted ordinary-ask resolution | relay admission adds to the Service wait group before the run is published and is fenced before shutdown wait. `Service.Close` first closes admission, marks every already-durable awaiting handoff `preserveDurable` and every cancellation `cancelSignaled` under the persistence barrier, then cancels contexts/runs and waits with the engine-close bound. Graceful drain and lease loss use the same mark-before-signal order. A preserved terminal relay cannot overwrite the awaiting snapshot | **derived from durable state / reset local ownership**: the context, run, recorder, marker bits, and join count disappear on process loss. A replacement acquires the distributed session lease, reloads the authoritative snapshot after acquisition, and repeats exact run/state/ask/origin/placement validation before rehydrating. See List 2 row 47 | `internal/adapter/server/service.go` (`ResolveRunAsk`, `validatePersistedRunAsk`, `reserveDetachedControlRelay`, `relayDetachedControlRun`, `Close`, `GracefulDrain`, `cancelRegisteredRunState`); ADR 0347 | +| 75 | Experimental execution-provider HTTPS server and Kubernetes REST transports | `cmd/mecatl-execution-provider` | provider process | process cancellation shuts down the HTTP server and closes idle client connections; Kubernetes clients own no separate application goroutine | **derive** from explicit flags, mounted mTLS material, in-cluster configuration, and the live Kubernetes API. Disabled `mecak8s` composition creates none of these resources | `cmd/mecatl-execution-provider/main.go`; `internal/adapter/executioncontroller/handler.go` (`Handler`) | +| 76 | ExecutionEnvironment informer, rate-limited workqueue, and controller worker | `executioncontroller.Reconciler` | provider process, namespace-scoped | the Run context stops the informer and worker; `Run` shuts down the queue before return | **derive** by listing and watching the external Kubernetes CRs after restart. Startup marks any persisted active operation `FenceUnknown` before admitting mutations | `internal/adapter/executioncontroller/controller.go` (`Reconciler`, `Initialize`, `Run`) | +| 77 | ExecutionEnvironment CR, retained PVC, executor Pod, and one status-held active-operation claim | Kubernetes API and storage provider; the execution controller reconciles them | logical remote environment | explicit retirement removes a proven-terminal Pod but retains the PVC. CR deletion remains finalizer-blocked until retirement and quiescence; this candidate has no PVC-delete path | **external Kubernetes state**: CR status, PVC data, and current Pod references survive provider or `mecak8s` process restart according to cluster/storage durability. An interrupted active-operation claim becomes fail-closed `FenceUnknown`, not reconstructed authority | `internal/adapter/executioncontroller/controller.go` (`Reconcile`, `reconcileRetiring`, `reconcileDeletion`); `internal/adapter/executioncontroller/store.go` (`execute`) | +| 78 | In-memory grant signer/verifier configuration, short-lived signed grants, and revocation maps | execution-provider `Handler`; each remote `grantContext` caches one grant | provider process / bound session | grants expire; client reattachment refreshes them. Signing and verifier values leave process memory at shutdown. Full durable key rotation and revocation are not implemented | **reset-by-design / derive** from mounted signing configuration and exact external environment state. A restart does not preserve in-memory nonce/key revocation, so this draft is not production rotation or revocation | `internal/adapter/executioncontroller/handler.go` (`GrantSigner`, `HandlerConfig`, `sign`); `internal/executionenv/grant.go`; `internal/adapter/executionclient/client.go` (`grantContext`) | +| 79 | Mecak8s execution-provider mTLS client transport and per-binding remote Workspace/runner | `executionclient.Client` and the Service placement registry | process transport / session environment | `Client.Close` closes idle connections; session engines and environments follow existing Service teardown. Each authority lookup and file/command call has caller or adapter bounds | **derive** by exact reattachment from the persisted `session.EnvironmentRef` and owner-bound provider state. Missing, changed, or fenced environments fail closed with no local fallback | `internal/adapter/executionclient/client.go` (`Client`, `Provider`, `workspace`, `runner`) | + +**Experimental native-execution re-audit (List 1 / List 2).** List 1 rows +75–79 inventory the provider server and transports, controller informer and queue, +external Kubernetes runtime state, grant state, and `mecak8s` client transport. +List 2 row 48 records the split between the persisted exact environment reference, +external CR/PVC state, and process-local operation/grant state. The implementation +is draft: run-wide ownership, complete successor reference lifecycle, controlled +replacement/PVC deletion, and durable grant-key rotation remain unresolved. **Mecatui status-line re-audit (ADR 0247).** List 1 row 64 owns the local source lifecycle. It contains only display-safe input and generated presentation state, never session authority or transcript data, so List 2 gains no row. Shutdown is bounded rather than silently abandoning a live command tree; a process restart deliberately starts from fresh local status state. @@ -1228,6 +1241,7 @@ what is persisted), **reset-by-design** (documented, acceptable), | 45 | Session-scoped MCP broker incarnation, local attachment, and parked authorization (P10–P11; List 1 rows 68–70) | the snapshot persists `session.Session.ExternalBinding` plus the exact private `PendingAuthorization`; the process-owned broker holds logical state, grants/tokens, callback/replay state, timers, and prepared continuations in memory | same-process controls reattach only when the opaque binding matches exactly. **The binding value itself, not merely its persistence, is what defeats a restart**: it is `bindingPrefix + "." + generation`, where `bindingPrefix` is random per process and `generation` is an in-memory counter (`internal/adapter/mcpbroker/runtime.go`), so a byte-perfect restored value is structurally guaranteed stale the moment the broker process restarts — no persistence fix closes this alone. A process restart loses in-process broker authority and timers; a lease-gated status-unavailable observation pairs the parked call as interrupted and never replays the protected mutation. The one narrow exception is the pre-prompt workspace-enrollment seam (`rebindBrokerAttachment`, `internal/adapter/server/mcp_broker.go`), which adopts the live incarnation ONLY where nothing durable was built on the lost one — it must not widen to authorization control paths, which stay fail-closed. A future remote broker can return pending, granted, or a concrete terminal status through the same control flow | **durable pending / reset runtime / derive control**: the aggregate preserves exact effective and deferred calls, while timers and prepared runs are recreated only from authoritative live state. Public wire and client surfaces remain deferred to P12–P14 | P10, P11 | | 46 | Workspace-enrollment outer correlation versus ToolHive inner authorization storage (ADR 0311; List 1 rows 68–69 and 71) | the snapshot persists the exact `PendingWorkspaceEnrollment`; mecatl's runtime owns its process-local callback state/incarnation mapping, while ToolHive memory or Redis storage owns separate inner upstream authorization/token records | Redis-backed ToolHive records may survive restart, but the outer operation-to-session correlation does not. Rebinding the pre-prompt session to a fresh runtime only discards the exact old pending correlation and permits a fresh enrollment; it is not authority to search ToolHive storage or resume an old operation. A wrong replica has the same correlation loss | **persist aggregate / reset outer correlation / do not infer inner recovery**: terminal observations are repeatable in-process until the aggregate save succeeds; after process loss, discard only the exact old pre-prompt correlation and begin a fresh enrollment. Durable outer correlation, replica routing, and restart recovery remain deferred | ADR 0311 | | 47 | Prompt-free ordinary-ask handoff and acknowledgement-only resumed-run lifecycle (ADR 0347; List 1 row 74) | the existing session snapshot persists `StateAwaiting`, exact `RunID`, `PendingAsk`, mode, and `EnvironmentRef`; the accepting Service owns only the reconstructed run/context/relay described in List 1 | a restart or replica handoff loses local execution and join bookkeeping but retains the last authoritative awaiting or terminal snapshot. An accepting replica acquires the distributed lease, reloads after acquisition, and revalidates exact run, awaiting state, ask ID, non-plan origin, and placement before `ResumeApproval`; a stale pre-lease snapshot cannot replay allow-once work | **persist existing snapshot, reconstruct under fresh lease**: no new durable field is introduced. Before shutdown cancellation reaches a detached continuation, an already-persisted awaiting point is marked for preservation and the relay is joined boundedly; otherwise terminal cancellation is persisted so accepted work is not ambiguously replayed. Plan asks remain on their dedicated resolution path | 0347 | +| 48 | Experimental native Kubernetes environment identity, references, grants, and active operation | the session snapshot persists the exact `EnvironmentRef`; the Kubernetes `ExecutionEnvironment` CR/status and retained PVC hold provider-side identity, references, operation state, and workspace data; grants and client transports are process memory | restart reloads the session reference and exactly reattaches through the provider to the external CR. The controller derives its queue from an informer list/watch. A provider restart that observes an active operation clears the claim only by marking the environment `FenceUnknown`; it does not infer completion or takeover. Short-lived grants and in-memory revocation state reset | **persist-in-snapshot + external Kubernetes state + derive/reset-by-design**: derive the live Workspace/runner from exact reattachment; derive controller work from CRs; reset transports, queue state, and expired grants. Complete successor reference release, run-wide ownership, automated fenced recovery, controlled replacement/PVC deletion, and durable multi-key rotation are not implemented | experimental native execution; List 1 rows 75–79 | Two ledger observations worth stating in prose: diff --git a/docs/architecture.md b/docs/architecture.md index 1905dc6623..90a227b6c1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -70,6 +70,26 @@ This page is the overview and router; the big picture and the layering rule are - **[Session titles & durable token accounting](architecture/domain-model.md#session-titles-and-durable-token-accounting)** — operator renaming, opt-in asynchronous generation, and canonical token usage. - **[Deployment & server hardening](architecture/deployment-and-hardening.md)** +### Experimental native Kubernetes execution + +The candidate native execution path is opt-in and keeps the controller boundary +outside `mecak8s`. `cmd/mecatl-execution-provider` owns the private authenticated +API and the namespaced `ExecutionEnvironment` reconciler. It creates a retained +PVC and a credential-free executor Pod from an operator-defined, digest-pinned +profile. `cmd/mecatl-executor` serves bounded file operations and foreground +commands inside `/workspace`; the host-side `internal/adapter/executionclient` +adapts those operations to the existing `tool.Environment` contract. Mecak8s +persists only the exact private environment reference and uses an authenticated, +bounded RPC to obtain the executor's physically confined authority-resource +identity before policy evaluation. + +This remains a draft implementation. Run-wide ownership, complete successor +reference lifecycle, controlled executor replacement and PVC deletion, and full +grant-key rotation are not implemented. A lost terminal receipt or unproven Pod +termination leaves the environment fenced with no automated takeover. Disabled +composition builds no execution client, and the separate provider chart has no +dependency on the `mecak8s` chart. + ### Protected-resource discovery Both `mecated` and `mecak8s` use the shared OIDC profile flags. When configured, diff --git a/docs/design/IMPLEMENTATION-NOTES.md b/docs/design/IMPLEMENTATION-NOTES.md index 5d5d042560..a86be74b0d 100644 --- a/docs/design/IMPLEMENTATION-NOTES.md +++ b/docs/design/IMPLEMENTATION-NOTES.md @@ -6536,6 +6536,40 @@ deny-dominant (the inner fold runs first — a configured Deny or configured Ask reaches the checker). Default `false` is the byte-identical un-routed posture table. See `docs/adr/0080-guardrail-routed-escape-checking.md`. +### Experimental native Kubernetes execution provider + +The opt-in candidate composes a separately deployed provider through +`internal/adapter/executionclient/client.go` (`Provider`). The private +`/internal/execution/v1` mTLS API authenticates an exact URI SAN, binds short-lived +Ed25519 grants to client, owner hash, session binding, environment revision, +epoch, and a closed operation set, and sends credential-free requests to the +workload helper. `internal/adapter/executioncontroller/handler.go` (`Handler`) +owns authentication and grant checks; `internal/adapter/executioncontroller/store.go` +(`execute`) claims the one CR status operation before dispatch and marks +uncertain completion `FenceUnknown`. + +`internal/adapter/executioncontroller/controller.go` (`Reconciler`) watches the +namespaced `ExecutionEnvironment` CR and creates a retained PVC plus a +single executor Pod from an operator-only, digest-pinned profile. The Pod +has no service-account token and contains the fixed helper built by +`build/execution-workload/Dockerfile`. `internal/executionexecutor/executor.go` +(`Executor.Execute`) dispatches bounded file operations and foreground Shell in +`/workspace`. Authority-resource resolution is itself an authenticated +`file.resolve_authority` operation: the executor invokes the existing physical +`osfs` resolver, and the remote Workspace returns only the confined canonical +`/workspace` identity. Resolver, transport, authorization, or confinement errors +fail closed before the external authority evaluator runs; there is no lexical +client-side substitute. + +Composition selects this path only when `Config.RemoteExecution` is set. The +remote catalog excludes local project ingestion, schedules, SkillDraft, +Parallel, and Team, while explicit `profile:"no-fs"` still selects the existing +no-FS placement. The `mecatl-execution` and `mecak8s` charts remain independent. +The current candidate does not complete run-wide execution ownership, successor +binding/reference release, controlled executor replacement or PVC deletion, or +multi-key rotation/revocation. Unproven executor loss remains `FenceUnknown` and +has no automated takeover. + ### Server-owned session placement and worktree successors (ADR 0291) Placement is server-owned across embedded, loopback, remote, and cloud-native composition. diff --git a/e2e/k8s_execution/fixture/oidcissuer/main.go b/e2e/k8s_execution/fixture/oidcissuer/main.go new file mode 100644 index 0000000000..ddd1c3ce8c --- /dev/null +++ b/e2e/k8s_execution/fixture/oidcissuer/main.go @@ -0,0 +1,91 @@ +//go:build kind_execution_e2e + +// Command oidcissuer is a synthetic OIDC issuer for the offline Kind qualification. +package main + +import ( + "crypto/rsa" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "flag" + "fmt" + "log" + "math/big" + "net/http" + "os" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +func main() { + var certFile, keyFile, signingFile, issuer, audience string + flag.StringVar(&certFile, "tls-cert", "", "TLS certificate") + flag.StringVar(&keyFile, "tls-key", "", "TLS key") + flag.StringVar(&signingFile, "signing-key", "", "RSA PKCS8 signing key") + flag.StringVar(&issuer, "issuer", "", "exact issuer URL") + flag.StringVar(&audience, "audience", "mecatl", "token audience") + flag.Parse() + if certFile == "" || keyFile == "" || signingFile == "" || issuer == "" { + log.Fatal("all identity flags are required") + } + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + must(err) + raw, err := os.ReadFile(signingFile) + must(err) + block, _ := pem.Decode(raw) + if block == nil { + log.Fatal("invalid signing key") + } + parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) + must(err) + key, ok := parsed.(*rsa.PrivateKey) + if !ok { + log.Fatal("signing key is not RSA") + } + kidRaw := sha256.Sum256(x509.MarshalPKCS1PublicKey(&key.PublicKey)) + kid := base64.RawURLEncoding.EncodeToString(kidRaw[:12]) + mux := http.NewServeMux() + mux.HandleFunc("GET /.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, map[string]any{"issuer": issuer, "jwks_uri": issuer + "/keys", "id_token_signing_alg_values_supported": []string{"RS256"}}) + }) + mux.HandleFunc("GET /keys", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, map[string]any{"keys": []any{map[string]any{"kty": "RSA", "use": "sig", "alg": "RS256", "kid": kid, "n": base64.RawURLEncoding.EncodeToString(key.N.Bytes()), "e": base64.RawURLEncoding.EncodeToString(big.NewInt(int64(key.E)).Bytes())}}}) + }) + mux.HandleFunc("GET /token", func(w http.ResponseWriter, r *http.Request) { + sub := r.URL.Query().Get("sub") + if sub != "alice" && sub != "bob" { + http.Error(w, "unknown fixture subject", http.StatusBadRequest) + return + } + now := time.Now().UTC() + tok := jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{"iss": issuer, "aud": audience, "sub": sub, "iat": now.Unix(), "exp": now.Add(10 * time.Minute).Unix()}) + tok.Header["kid"] = kid + signed, err := tok.SignedString(key) + if err != nil { + http.Error(w, "signing failed", 500) + return + } + writeJSON(w, map[string]string{"id_token": signed}) + }) + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + server := &http.Server{Addr: ":8443", Handler: mux, ReadHeaderTimeout: 5 * time.Second, TLSConfig: &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{cert}}} + ln, err := tls.Listen("tcp", server.Addr, server.TLSConfig) + must(err) + must(server.Serve(ln)) +} +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(v); err != nil { + log.Print("response encoding failed") + } +} +func must(err error) { + if err != nil { + log.Fatal(fmt.Errorf("fixture startup: %w", err)) + } +} diff --git a/e2e/k8s_execution/fixture/pki/main.go b/e2e/k8s_execution/fixture/pki/main.go new file mode 100644 index 0000000000..0da0eacf7d --- /dev/null +++ b/e2e/k8s_execution/fixture/pki/main.go @@ -0,0 +1,88 @@ +//go:build kind_execution_e2e + +// Command pki generates synthetic, short-lived qualification identities. +package main + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net/url" + "os" + "path/filepath" + "time" +) + +func main() { + if len(os.Args) != 2 { + panic("usage: pki OUTPUT_DIRECTORY") + } + dir := os.Args[1] + if err := os.MkdirAll(dir, 0o700); err != nil { + panic(err) + } + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + must(err) + now := time.Now().UTC() + ca := &x509.Certificate{SerialNumber: serial(), Subject: pkix.Name{CommonName: "mecatl execution qualification CA"}, NotBefore: now.Add(-time.Minute), NotAfter: now.Add(6 * time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + must(err) + write(filepath.Join(dir, "ca.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), 0o600) + must(issue(dir, "provider", ca, caKey, []string{"mecatl-execution", "mecatl-execution.execution-qualification.svc", "mecatl-execution.execution-qualification.svc.cluster.local"}, "", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth})) + must(issue(dir, "mecak8s", ca, caKey, nil, "spiffe://mecatl.test/client/mecak8s", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})) + must(issue(dir, "intruder", ca, caKey, nil, "spiffe://mecatl.test/client/intruder", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})) + grantPub, grantKey, err := ed25519.GenerateKey(rand.Reader) + _ = grantPub + must(err) + grantDER, err := x509.MarshalPKCS8PrivateKey(grantKey) + must(err) + write(filepath.Join(dir, "grant-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: grantDER}), 0o600) + must(issue(dir, "oidc", ca, caKey, []string{"oidc-issuer", "oidc-issuer.execution-qualification.svc", "oidc-issuer.execution-qualification.svc.cluster.local"}, "", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth})) + jwtKey, err := rsa.GenerateKey(rand.Reader, 2048) + must(err) + jwtDER, err := x509.MarshalPKCS8PrivateKey(jwtKey) + must(err) + write(filepath.Join(dir, "oidc-jwt-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: jwtDER}), 0o600) +} + +func issue(dir, name string, ca *x509.Certificate, caKey *rsa.PrivateKey, dns []string, uri string, usages []x509.ExtKeyUsage) error { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + cert := &x509.Certificate{SerialNumber: serial(), Subject: pkix.Name{CommonName: name}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(6 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, ExtKeyUsage: usages, DNSNames: dns} + if uri != "" { + u, err := url.Parse(uri) + if err != nil { + return err + } + cert.URIs = []*url.URL{u} + } + der, err := x509.CreateCertificate(rand.Reader, cert, ca, &key.PublicKey, caKey) + if err != nil { + return err + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return err + } + write(filepath.Join(dir, name+".crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600) + write(filepath.Join(dir, name+".key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600) + return nil +} + +func serial() *big.Int { + n, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + must(err) + return n +} +func write(path string, data []byte, mode os.FileMode) { must(os.WriteFile(path, data, mode)) } +func must(err error) { + if err != nil { + panic(err) + } +} diff --git a/e2e/k8s_execution/qualification_test.go b/e2e/k8s_execution/qualification_test.go new file mode 100644 index 0000000000..a8950741d1 --- /dev/null +++ b/e2e/k8s_execution/qualification_test.go @@ -0,0 +1,380 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const namespace = "execution-qualification" + +func TestKindExecutionQualification(t *testing.T) { + state := os.Getenv("MECATL_EXECUTION_QUAL_STATE") + if state == "" || os.Getenv("MECATL_KUBE_CONTEXT") == "" { + t.Fatal("MECATL_EXECUTION_QUAL_STATE and MECATL_KUBE_CONTEXT are required; run task e2e:k8s:execution") + } + kubeconfig := filepath.Join(state, "kubeconfig") + pki := filepath.Join(state, "pki") + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + defer cancel() + + baseline := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev") + providerForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + providerTLS := loadTLS(t, pki, "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + providerClient, err := executionclient.New("https://"+providerForward.addr, providerTLS) + if err != nil { + t.Fatal(err) + } + defer providerClient.Close() + if _, err = providerClient.ValidateProfile(ctx, "go"); err != nil { + t.Fatalf("validate profile: %v", err) + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got != baseline { + t.Fatalf("profile validation changed environment count from %d to %d", baseline, got) + } + + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "alice"} + binding := fmt.Sprintf("direct-binding-%d", time.Now().UnixNano()) + first, err := providerClient.Ensure(ctx, binding, "go", owner) + if err != nil { + t.Fatalf("ensure: %v", err) + } + if first.Environment.ID == "" || first.Environment.Revision == "" { + t.Fatal("ensure returned an empty exact environment reference") + } + ready := waitReady(t, ctx, providerClient, owner, binding, first.Environment) + if ready.Environment != first.Environment { + t.Fatalf("ready reference drifted: got %+v want %+v", ready.Environment, first.Environment) + } + second, err := providerClient.Ensure(ctx, binding, "go", owner) + if err != nil { + t.Fatalf("repeat ensure: %v", err) + } + if second.Environment != first.Environment { + t.Fatalf("repeat ensure changed identity: got %+v want %+v", second.Environment, first.Environment) + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got != baseline+1 { + t.Fatalf("repeat ensure left %d environments, want baseline + 1 (%d)", got, baseline+1) + } + if got := resourceCount(t, ctx, kubeconfig, "pods -l execution.mecatl.dev/environment="+first.Environment.ID); got != 1 { + t.Fatalf("executor pods = %d, want 1", got) + } + if got := resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+first.Environment.ID); got != 1 { + t.Fatalf("workspace PVCs = %d, want 1", got) + } + + intruderTLS := loadTLS(t, pki, "intruder", "mecatl-execution.execution-qualification.svc.cluster.local") + intruder, err := executionclient.New("https://"+providerForward.addr, intruderTLS) + if err != nil { + t.Fatal(err) + } + defer intruder.Close() + _, err = intruder.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: first.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}) + if err == nil || (!isRemoteCode(err, executionenv.CodePermissionDenied) && !isRemoteCode(err, executionenv.CodeNotFound)) { + t.Fatalf("different client attach error = %v, want an existence-hiding denial", err) + } + for _, path := range []string{"/commands/status", "/commands/cancel", "/environments/retire"} { + request := map[string]any{"context": map[string]any{"environment": first.Environment, "owner": owner, "binding_id": binding, "epoch": ready.Epoch, "grant": ready.Grant}} + if strings.HasPrefix(path, "/commands/") { + request["command_id"] = "not-owned" + } + status := rawProviderPost(t, ctx, "https://"+providerForward.addr, intruderTLS, path, request) + if status != http.StatusForbidden { + t.Fatalf("different client %s status = %d, want 403", path, status) + } + } + providerForward.stop() + if _, err := providerClient.ValidateProfile(ctx, "go"); err == nil { + t.Fatal("provider endpoint loss unexpectedly fell back") + } + + tokenForward := portForward(t, ctx, kubeconfig, "service/oidc-issuer", 8443) + alice := fixtureToken(t, ctx, tokenForward.addr, pki, "alice") + bob := fixtureToken(t, ctx, tokenForward.addr, pki, "bob") + tokenForward.stop() + agentForward := portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + sessionID := createSession(t, ctx, agentForward.addr, alice) + body := prompt(t, ctx, agentForward.addr, sessionID, alice, "run the scripted remote qualification") + if err := os.WriteFile(filepath.Join(state, "mock-journey.sse"), body, 0o600); err != nil { + t.Fatal(err) + } + for _, marker := range []string{"write-proof", "read-before-edit", "edit-proof", "copy-proof", "move-proof", "write-mod", "write-test", "shell-go-test", "remove-moved", "read-final", "moved.txt", "proof.txt:1:beta", "REMOTE_EXECUTION_QUALIFICATION_COMPLETE", "ok"} { + if !bytes.Contains(body, []byte(marker)) { + t.Fatalf("SSE result omitted %q", marker) + } + } + if bytes.Contains(body, []byte(`"is_error":true`)) { + t.Fatal("scripted remote tool journey contained an error result") + } + if status := getSession(t, ctx, agentForward.addr, sessionID, bob); status != http.StatusNotFound { + t.Fatalf("different OIDC owner read status = %d, want 404", status) + } + agentForward.stop() + + applyReattachScript(t, ctx, kubeconfig, state) + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=180s") + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecak8s", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecak8s", "-n", namespace, "--timeout=240s") + agentForward = portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + defer agentForward.stop() + body = promptEventually(t, ctx, agentForward.addr, sessionID, alice, "verify exact reattachment after both deployments restarted") + for _, marker := range []string{"reattach-read", "reattach-shell", "beta", "REMOTE_EXECUTION_REATTACH_COMPLETE", "ok"} { + if !bytes.Contains(body, []byte(marker)) { + t.Fatalf("reattach SSE omitted %q", marker) + } + } + if bytes.Contains(body, []byte(`"is_error":true`)) { + t.Fatal("reattached remote operation contained an error result") + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got != baseline+2 { + t.Fatalf("post-restart environments = %d, want baseline + direct + session (%d)", got, baseline+2) + } +} + +func waitReady(t *testing.T, ctx context.Context, c *executionclient.Client, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) executionenv.AttachEnvironmentResponse { + t.Helper() + deadline := time.Now().Add(3 * time.Minute) + for time.Now().Before(deadline) { + out, err := c.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}) + if err == nil && out.Ready { + return out + } + time.Sleep(500 * time.Millisecond) + } + t.Fatal("execution environment did not become ready") + return executionenv.AttachEnvironmentResponse{} +} +func isRemoteCode(err error, code executionenv.ErrorCode) bool { + var remote *executionenv.Error + return errors.As(err, &remote) && remote.Code == code +} + +func loadTLS(t *testing.T, dir, name, serverName string) *tls.Config { + t.Helper() + cfg, err := executionclient.LoadTLSConfig(executionclient.TLSFiles{CA: filepath.Join(dir, "ca.crt"), Cert: filepath.Join(dir, name+".crt"), Key: filepath.Join(dir, name+".key")}) + if err != nil { + t.Fatal(err) + } + cfg.ServerName = serverName + return cfg +} + +type forward struct { + addr string + cmd *exec.Cmd +} + +func (f *forward) stop() { + if f == nil || f.cmd == nil || f.cmd.Process == nil { + return + } + _ = f.cmd.Process.Signal(os.Interrupt) + _, _ = f.cmd.Process.Wait() +} +func portForward(t *testing.T, ctx context.Context, kubeconfig, target string, remote int) *forward { + t.Helper() + local := freePort(t) + cmd := command(ctx, kubeconfig, "port-forward", "-n", namespace, target, fmt.Sprintf("%d:%d", local, remote)) + var stderr bytes.Buffer + cmd.Stdout = io.Discard + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + f := &forward{addr: fmt.Sprintf("127.0.0.1:%d", local), cmd: cmd} + t.Cleanup(f.stop) + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", f.addr, 200*time.Millisecond) + if err == nil { + _ = conn.Close() + return f + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("port-forward %s did not start: %s", target, stderr.String()) + return nil +} +func freePort(t *testing.T) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + return ln.Addr().(*net.TCPAddr).Port +} + +func cleanEnv() []string { + return []string{"HOME=" + os.Getenv("HOME"), "PATH=" + os.Getenv("PATH")} +} +func command(ctx context.Context, kubeconfig string, args ...string) *exec.Cmd { + contextName := os.Getenv("MECATL_KUBE_CONTEXT") + fullArgs := append([]string{"--kubeconfig", kubeconfig, "--context", contextName}, args...) + cmd := exec.CommandContext(ctx, "kubectl", fullArgs...) + cmd.Env = cleanEnv() + return cmd +} +func runKubectl(t *testing.T, ctx context.Context, kubeconfig string, args ...string) []byte { + t.Helper() + out, err := command(ctx, kubeconfig, args...).CombinedOutput() + if err != nil { + t.Fatalf("kubectl %s failed: %v: %s", args[0], err, out) + } + return out +} +func resourceCount(t *testing.T, ctx context.Context, kubeconfig, spec string) int { + t.Helper() + args := append([]string{"get"}, strings.Fields(spec)...) + args = append(args, "-n", namespace, "-o", "name") + out := runKubectl(t, ctx, kubeconfig, args...) + if len(bytes.TrimSpace(out)) == 0 { + return 0 + } + return len(bytes.Split(bytes.TrimSpace(out), []byte{'\n'})) +} + +func rawProviderPost(t *testing.T, ctx context.Context, endpoint string, cfg *tls.Config, path string, body any) int { + t.Helper() + raw, _ := json.Marshal(body) + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, endpoint+executionenv.BasePath+path, bytes.NewReader(raw)) + req.Header.Set("Content-Type", "application/json") + client := &http.Client{Transport: &http.Transport{TLSClientConfig: cfg}} + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16)) + return resp.StatusCode +} +func fixtureToken(t *testing.T, ctx context.Context, addr, pki, sub string) string { + t.Helper() + ca, err := os.ReadFile(filepath.Join(pki, "ca.crt")) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(ca) { + t.Fatal("fixture CA invalid") + } + tr := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: pool, ServerName: "oidc-issuer.execution-qualification.svc.cluster.local"}} + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+addr+"/token?sub="+sub, nil) + resp, err := (&http.Client{Transport: tr}).Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out struct { + IDToken string `json:"id_token"` + } + if resp.StatusCode != 200 || json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&out) != nil || out.IDToken == "" { + t.Fatal("fixture token issuance failed") + } + return out.IDToken +} +func createSession(t *testing.T, ctx context.Context, addr, token string) string { + t.Helper() + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions", token, []byte(`{"mode":"default"}`)) + if status != http.StatusCreated { + t.Fatalf("create session status=%d body=%s", status, body) + } + var out struct { + SessionID string `json:"session_id"` + } + if json.Unmarshal(body, &out) != nil || out.SessionID == "" { + t.Fatal("create session returned no id") + } + return out.SessionID +} +func prompt(t *testing.T, ctx context.Context, addr, id, token, text string) []byte { + t.Helper() + raw, _ := json.Marshal(map[string]string{"text": text}) + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+id+"/prompt", token, raw) + if status/100 != 2 { + t.Fatalf("prompt status=%d body=%s", status, body) + } + return body +} +func promptEventually(t *testing.T, ctx context.Context, addr, id, token, text string) []byte { + t.Helper() + raw, _ := json.Marshal(map[string]string{"text": text}) + deadline := time.Now().Add(90 * time.Second) + for { + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+id+"/prompt", token, raw) + if status/100 == 2 { + return body + } + if status != http.StatusConflict || time.Now().After(deadline) { + t.Fatalf("reattach prompt status=%d body=%s", status, body) + } + time.Sleep(time.Second) + } +} +func getSession(t *testing.T, ctx context.Context, addr, id, token string) int { + t.Helper() + status, _ := request(t, ctx, http.MethodGet, "http://"+addr+"/v1/sessions/"+id, token, nil) + return status +} +func request(t *testing.T, ctx context.Context, method, url, token string, body []byte) (int, []byte) { + t.Helper() + req, _ := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("Accept", "text/event-stream") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if err != nil { + t.Fatal(err) + } + return resp.StatusCode, raw +} +func applyReattachScript(t *testing.T, ctx context.Context, kubeconfig, state string) { + t.Helper() + root := filepath.Clean(filepath.Join(state, "..", "..", "..")) + _ = root + source := filepath.Join(repoRoot(t), "deploy", "mecatl-execution-kind", "mock-script-reattach.json") + cmd := command(ctx, kubeconfig, "create", "configmap", "execution-mock", "-n", namespace, "--from-file=mock-script.json="+source, "--dry-run=client", "-o", "yaml") + rendered, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + apply := command(ctx, kubeconfig, "apply", "-f", "-") + apply.Stdin = bytes.NewReader(rendered) + if out, err := apply.CombinedOutput(); err != nil { + t.Fatalf("update mock script: %v: %s", err, out) + } +} +func repoRoot(t *testing.T) string { + t.Helper() + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + return filepath.Clean(filepath.Join(wd, "..", "..")) +} diff --git a/engine/agent/authority_evaluator_test.go b/engine/agent/authority_evaluator_test.go index 70bec40d10..6cedc58805 100644 --- a/engine/agent/authority_evaluator_test.go +++ b/engine/agent/authority_evaluator_test.go @@ -3,6 +3,7 @@ package agent_test import ( "context" "encoding/json" + "errors" "strings" "sync" "sync/atomic" @@ -431,6 +432,54 @@ func TestADR_0233_AuthorityEvaluator_Scenario7_ResourceAttributeIsDerivedWithout }) } +type failingAuthorityWorkspace struct { + tool.Workspace + err error +} + +func (w failingAuthorityWorkspace) AuthorityResourcePath(string) (string, string, error) { + return "", "", w.err +} + +func TestAuthorityResourceResolutionFailureDeniesBeforeEvaluator(t *testing.T) { + for _, tc := range []struct { + name string + err error + }{ + {name: "remote resolver unavailable", err: context.DeadlineExceeded}, + {name: "physical symlink escape", err: errors.New("path is outside workspace")}, + } { + t.Run(tc.name, func(t *testing.T) { + read := &authorityTool{name: "Read"} + evaluator := &recordingAuthorityEvaluator{decision: port.AuthorityDecision{Allowed: true}} + eng := newEngine(agent.Deps{ + LLM: mockllm.New(mockllm.ToolCallTurn(toolCall("read", "Read", `{"path":"escape"}`))), + Catalog: catalogWith(t, read), + AuthorityEvaluator: evaluator, + }) + base := agent.MemEnv("/workspace") + env := tool.MustEnvironment(base.Ref(), failingAuthorityWorkspace{Workspace: base.Workspace(), err: tc.err}, base.ReadLedger(), nil) + sess := authoritySession(t, "Read") + if err := sess.Rehome(env.Ref()); err != nil { + t.Fatal(err) + } + events := drain(eng.Run(context.Background(), sess, env, agent.RunRequest{Text: "read"})) + if read.ran.Load() != 0 || evaluator.calls() != 0 { + t.Fatalf("tool executions=%d evaluator calls=%d, want 0/0", read.ran.Load(), evaluator.calls()) + } + for _, event := range events { + if event.ToolResult != nil && event.ToolResult.CallID == "read" { + if !event.ToolResult.IsError || !strings.Contains(event.ToolResult.Content, "authority resource") { + t.Fatalf("tool result=%+v, want fail-closed authority resolution error", event.ToolResult) + } + return + } + } + t.Fatal("missing fail-closed tool result") + }) + } +} + // sequencedAuthorityEvaluator returns one decision per call in order, so a test // can distinguish the source-resource request from the destination-resource // request in a dual-resource call (Copy/Move). diff --git a/internal/adapter/executionclient/client.go b/internal/adapter/executionclient/client.go new file mode 100644 index 0000000000..c2ed801659 --- /dev/null +++ b/internal/adapter/executionclient/client.go @@ -0,0 +1,511 @@ +// Package executionclient implements the private mTLS client and placement adapter +// for the independently deployed Kubernetes execution provider. +package executionclient + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + remoteRoot = "/workspace" + authorityResolveRequestTimeout = 10 * time.Second +) + +// TLSFiles names the runtime-only mTLS material mounted into mecak8s. +type TLSFiles struct{ CA, Cert, Key string } + +// LoadTLSConfig loads provider trust and client identity at process startup. +func LoadTLSConfig(files TLSFiles) (*tls.Config, error) { + if files.CA == "" || files.Cert == "" || files.Key == "" { + return nil, errors.New("execution client: CA, certificate, and key are required") + } + ca, err := os.ReadFile(files.CA) + if err != nil { + return nil, fmt.Errorf("execution client: load CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(ca) { + return nil, errors.New("execution client: CA contains no certificates") + } + cert, err := tls.LoadX509KeyPair(files.Cert, files.Key) + if err != nil { + return nil, fmt.Errorf("execution client: load client identity: %w", err) + } + return &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: pool, Certificates: []tls.Certificate{cert}}, nil +} + +// Client is a bounded client for the private execution-provider HTTP API. +type Client struct { + endpoint *url.URL + http *http.Client +} + +// New constructs a production mTLS execution-provider client. +func New(endpoint string, tlsConfig *tls.Config) (*Client, error) { + u, err := url.Parse(endpoint) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return nil, errors.New("execution client: endpoint must be an absolute HTTPS URL without credentials, query, or fragment") + } + if tlsConfig == nil || len(tlsConfig.Certificates) == 0 || tlsConfig.RootCAs == nil { + return nil, errors.New("execution client: production mTLS configuration is required") + } + u.Path = strings.TrimRight(u.Path, "/") + tr := &http.Transport{TLSClientConfig: tlsConfig.Clone(), DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext, TLSHandshakeTimeout: 10 * time.Second} + return &Client{endpoint: u, http: &http.Client{Transport: tr, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}}, nil +} + +// NewWithHTTPClient is a hermetic test seam; production callers use New. +func NewWithHTTPClient(endpoint string, hc *http.Client) (*Client, error) { + u, err := url.Parse(endpoint) + if err != nil || u.Scheme == "" || u.Host == "" || hc == nil { + return nil, errors.New("execution client: invalid endpoint or HTTP client") + } + u.Path = strings.TrimRight(u.Path, "/") + return &Client{endpoint: u, http: hc}, nil +} + +// Close releases pooled provider connections. +func (c *Client) Close() { + if c != nil && c.http != nil { + c.http.CloseIdleConnections() + } +} + +func (c *Client) post(ctx context.Context, path string, in, out any) error { + body, err := json.Marshal(in) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint.String()+executionenv.BasePath+path, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("execution provider request failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + data, err := io.ReadAll(io.LimitReader(resp.Body, executionenv.MaxJSONBody+1)) + if err != nil { + return fmt.Errorf("execution provider response failed: %w", err) + } + if len(data) > executionenv.MaxJSONBody { + return errors.New("execution provider response exceeds limit") + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + var envelope executionenv.ErrorResponse + if executionenv.DecodeStrict(data, &envelope) == nil && envelope.Error != nil { + return envelope.Error + } + return fmt.Errorf("execution provider returned HTTP %d", resp.StatusCode) + } + if err := executionenv.DecodeStrict(data, out); err != nil { + return fmt.Errorf("execution provider returned invalid response: %w", err) + } + return nil +} + +// ValidateProfile validates a provider profile without allocating an environment. +func (c *Client) ValidateProfile(ctx context.Context, profile string) (executionenv.ValidateProfileResponse, error) { + var out executionenv.ValidateProfileResponse + err := c.post(ctx, "/profiles/validate", executionenv.ValidateProfileRequest{Profile: profile}, &out) + return out, err +} + +// Ensure idempotently allocates or resolves the environment for a binding. +func (c *Client) Ensure(ctx context.Context, binding, profile string, owner executionenv.Owner) (executionenv.EnsureEnvironmentResponse, error) { + var out executionenv.EnsureEnvironmentResponse + err := c.post(ctx, "/environments/ensure", executionenv.EnsureEnvironmentRequest{BindingID: binding, Profile: profile, Owner: owner}, &out) + return out, err +} + +// Attach obtains a fresh grant for an exact environment reference. +func (c *Client) Attach(ctx context.Context, req executionenv.AttachEnvironmentRequest) (executionenv.AttachEnvironmentResponse, error) { + var out executionenv.AttachEnvironmentResponse + err := c.post(ctx, "/environments/attach", req, &out) + return out, err +} + +// File executes one authorized filesystem operation. +func (c *Client) File(ctx context.Context, req executionenv.FileRequest) (executionenv.FileResponse, error) { + var out executionenv.FileResponse + err := c.post(ctx, "/files", req, &out) + return out, err +} + +// StartCommand executes one foreground command. +func (c *Client) StartCommand(ctx context.Context, req executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + var out executionenv.CommandStartResponse + err := c.post(ctx, "/commands/start", req, &out) + return out, err +} + +// Provider adapts the execution service to server placement binding. +type Provider struct { + client *Client + profile string +} + +// NewProvider binds a client to one operator-selected profile. +func NewProvider(client *Client, profile string) (*Provider, error) { + if client == nil || profile == "" { + return nil, errors.New("execution client: client and profile are required") + } + return &Provider{client: client, profile: profile}, nil +} + +// ValidatePlacement performs side-effect-free profile preflight. +func (p *Provider) ValidatePlacement(ctx context.Context) error { + _, err := p.client.ValidateProfile(ctx, p.profile) + return mapPlacementError(err) +} + +func ownerOf(principal *session.Principal) (executionenv.Owner, error) { + if principal == nil || principal.Issuer == "" || principal.Subject == "" { + return executionenv.Owner{}, server.ErrPlacementNotFound + } + return executionenv.Owner{Issuer: principal.Issuer, Subject: principal.Subject}, nil +} + +// Bind allocates the environment for the final session binding. +func (p *Provider) Bind(ctx context.Context, req server.PlacementBindRequest) (server.PlacementBinding, error) { + if !req.Selector.IsDefault() || req.BindingID == "" { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return server.PlacementBinding{}, err + } + ensured, err := p.client.Ensure(ctx, string(req.BindingID), p.profile, owner) + if err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + if ensured.Environment.ID == "" || ensured.Environment.Revision == "" { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return p.waitForBinding(ctx, owner, string(req.BindingID), ensured.Environment) +} + +// Reattach exactly resolves a persisted remote environment for its binding. +func (p *Provider) Reattach(ctx context.Context, req server.PlacementReattachRequest) (server.PlacementBinding, error) { + if req.BindingID == "" { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return server.PlacementBinding{}, err + } + ref := executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision} + attached, err := p.client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: string(req.BindingID)}, Purpose: executionenv.PurposeSession}) + if err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + if toSessionRef(attached.Environment) != req.Ref { + return server.PlacementBinding{}, server.ErrPlacementChanged + } + return p.binding(owner, string(req.BindingID), attached) +} + +func (p *Provider) waitForBinding(ctx context.Context, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) (server.PlacementBinding, error) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + attached, err := p.client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}) + if err == nil && attached.Ready { + if attached.Environment != ref { + return server.PlacementBinding{}, server.ErrPlacementChanged + } + return p.binding(owner, binding, attached) + } + if err != nil { + var remote *executionenv.Error + if !errors.As(err, &remote) || (remote.Code != executionenv.CodeNotReady && !remote.Retryable) { + return server.PlacementBinding{}, mapPlacementError(err) + } + } + select { + case <-ctx.Done(): + return server.PlacementBinding{}, server.ErrPlacementUnavailable + case <-ticker.C: + } + } +} + +func (p *Provider) binding(owner executionenv.Owner, binding string, attached executionenv.AttachEnvironmentResponse) (server.PlacementBinding, error) { + if !attached.Ready || attached.Environment.ID == "" || attached.Environment.Revision == "" || attached.Epoch == 0 || attached.Grant == "" || attached.GrantExpiresAt.IsZero() { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + credentials := &grantContext{client: p.client, context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding, Epoch: attached.Epoch, Grant: attached.Grant}, expiresAt: attached.GrantExpiresAt} + ws := &workspace{credentials: credentials} + runner := &runner{credentials: credentials} + sref := toSessionRef(attached.Environment) + env, err := tool.NewEnvironment(sref, ws, memledger.New(), runner) + if err != nil { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return server.PlacementBinding{Environment: env, Ref: sref, Metadata: server.PlacementMetadata{Kind: "kubernetes", Label: "Remote Kubernetes workspace", Revision: attached.Environment.Revision}}, nil +} +func toSessionRef(ref executionenv.EnvironmentRef) session.EnvironmentRef { + return session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: ref.ID, Revision: ref.Revision} +} +func mapPlacementError(err error) error { + if err == nil { + return nil + } + var remote *executionenv.Error + if errors.As(err, &remote) { + switch remote.Code { + case executionenv.CodeNotFound, executionenv.CodePermissionDenied, executionenv.CodeUnauthenticated: + return server.ErrPlacementNotFound + case executionenv.CodeConflict, executionenv.CodeVersionMismatch: + return server.ErrPlacementChanged + case executionenv.CodeInvalidArgument: + return server.ErrInvalidPlacementSelection + } + } + return server.ErrPlacementUnavailable +} + +type grantContext struct { + mu sync.Mutex + client *Client + context executionenv.RequestContext + expiresAt time.Time +} + +func (g *grantContext) current(ctx context.Context) (executionenv.RequestContext, error) { + return g.renew(ctx, false) +} + +func (g *grantContext) refresh(ctx context.Context) (executionenv.RequestContext, error) { + return g.renew(ctx, true) +} + +func (g *grantContext) renew(ctx context.Context, force bool) (executionenv.RequestContext, error) { + g.mu.Lock() + defer g.mu.Unlock() + if !force && time.Until(g.expiresAt) > 10*time.Second { + return g.context, nil + } + attached, err := g.client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: g.context.Environment, Owner: g.context.Owner, BindingID: g.context.BindingID}, Purpose: executionenv.PurposeSession}) + if err != nil { + return executionenv.RequestContext{}, err + } + if !attached.Ready || attached.Environment != g.context.Environment || attached.Epoch != g.context.Epoch || attached.Grant == "" || attached.GrantExpiresAt.IsZero() { + return executionenv.RequestContext{}, server.ErrPlacementChanged + } + g.context.Grant = attached.Grant + g.expiresAt = attached.GrantExpiresAt + return g.context, nil +} + +// workspace is bound to one provider-issued environment grant. +type workspace struct { + credentials *grantContext + opMu sync.Mutex +} + +var _ tool.AuthorityResourceResolver = (*workspace)(nil) + +func (*workspace) Root() string { return remoteRoot } + +// AuthorityResourcePath asks the authenticated executor to derive the same +// physical, confined identity that the subsequent filesystem operation uses. +func (w *workspace) AuthorityResourcePath(path string) (target, root string, err error) { + ctx, cancel := context.WithTimeout(context.Background(), authorityResolveRequestTimeout) + defer cancel() + response, err := w.call(ctx, executionenv.OpFileResolveAuthority, path, "", "", nil, "") + if err != nil { + return "", "", err + } + if response.AuthorityWorkspace != remoteRoot || (response.AuthorityTarget != remoteRoot && !strings.HasPrefix(response.AuthorityTarget, remoteRoot+"/")) { + return "", "", errors.New("execution provider returned an invalid authority resource identity") + } + return response.AuthorityTarget, response.AuthorityWorkspace, nil +} + +func (w *workspace) call(ctx context.Context, op executionenv.Operation, path, dest, pattern string, data []byte, version string) (executionenv.FileResponse, error) { + // The provider's CRD deliberately carries one active operation. The engine may + // dispatch read-only sibling tools concurrently, so serialize this bound + // workspace before claiming that provider-side operation slot. + w.opMu.Lock() + defer w.opMu.Unlock() + credentials, err := w.credentials.current(ctx) + if err != nil { + return executionenv.FileResponse{}, mapFileError(path, err) + } + out, err := w.credentials.client.File(ctx, executionenv.FileRequest{Context: credentials, Operation: op, Path: path, Destination: dest, Pattern: pattern, Data: data, Version: version, Limit: executionenv.MaxListEntries}) + if isDefinitiveAuthDenial(err) && readOnlyFileOperation(op) { + credentials, refreshErr := w.credentials.refresh(ctx) + if refreshErr != nil { + return executionenv.FileResponse{}, mapFileError(path, refreshErr) + } + out, err = w.credentials.client.File(ctx, executionenv.FileRequest{Context: credentials, Operation: op, Path: path, Destination: dest, Pattern: pattern, Data: data, Version: version, Limit: executionenv.MaxListEntries}) + } + return out, mapFileError(path, err) +} + +func isDefinitiveAuthDenial(err error) bool { + var remote *executionenv.Error + return errors.As(err, &remote) && (remote.Code == executionenv.CodePermissionDenied || remote.Code == executionenv.CodeUnauthenticated) +} + +func readOnlyFileOperation(op executionenv.Operation) bool { + switch op { + case executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileList, executionenv.OpFileGlob, executionenv.OpFileGrep: + return true + default: + return false + } +} +func (w *workspace) Read(ctx context.Context, path string) ([]byte, error) { + r, e := w.call(ctx, executionenv.OpFileRead, path, "", "", nil, "") + return r.Data, e +} +func (w *workspace) ReadVersion(ctx context.Context, path string) ([]byte, tool.FileVersion, error) { + r, e := w.call(ctx, executionenv.OpFileRead, path, "", "", nil, "") + if e != nil { + return nil, tool.FileVersion{}, e + } + return r.Data, tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) Stat(ctx context.Context, path string) (tool.FileInfo, error) { + r, e := w.call(ctx, executionenv.OpFileStat, path, "", "", nil, "") + if e != nil { + return tool.FileInfo{}, e + } + if r.Info == nil { + return tool.FileInfo{}, errors.New("execution provider omitted file metadata") + } + return fileInfo(*r.Info), nil +} +func (w *workspace) CreateFile(ctx context.Context, path string, data []byte) (tool.FileVersion, error) { + r, e := w.call(ctx, executionenv.OpFileCreate, path, "", "", data, "") + if e != nil { + return tool.FileVersion{}, e + } + return tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) ReplaceFile(ctx context.Context, path string, old tool.FileVersion, data []byte) (tool.FileVersion, error) { + v, e := tool.EncodeFileVersion(old) + if e != nil { + return tool.FileVersion{}, e + } + r, e := w.call(ctx, executionenv.OpFileReplace, path, "", "", data, v) + if e != nil { + return tool.FileVersion{}, e + } + return tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) ReadDir(ctx context.Context, path string) ([]tool.FileInfo, error) { + r, e := w.call(ctx, executionenv.OpFileList, path, "", "", nil, "") + if e != nil { + return nil, e + } + out := make([]tool.FileInfo, len(r.Entries)) + for i := range r.Entries { + out[i] = fileInfo(r.Entries[i]) + } + return out, nil +} +func (w *workspace) Remove(ctx context.Context, path string) error { + _, e := w.call(ctx, executionenv.OpFileRemove, path, "", "", nil, "") + return e +} +func (w *workspace) Rename(ctx context.Context, a, b string) error { + _, e := w.call(ctx, executionenv.OpFileRename, a, b, "", nil, "") + return e +} +func (w *workspace) CopyFile(ctx context.Context, a, b string) (tool.FileVersion, error) { + r, e := w.call(ctx, executionenv.OpFileCopy, a, b, "", nil, "") + if e != nil { + return tool.FileVersion{}, e + } + return tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) Glob(ctx context.Context, pattern string) ([]string, error) { + r, e := w.call(ctx, executionenv.OpFileGlob, "", "", pattern, nil, "") + return r.Paths, e +} +func (w *workspace) Grep(ctx context.Context, pattern, pathGlob string) ([]tool.GrepMatch, error) { + r, e := w.call(ctx, executionenv.OpFileGrep, pathGlob, "", pattern, nil, "") + if e != nil { + return nil, e + } + out := make([]tool.GrepMatch, len(r.Matches)) + for i, m := range r.Matches { + out[i] = tool.GrepMatch{Path: m.Path, Line: m.Line, Text: m.Text} + } + return out, nil +} +func fileInfo(i executionenv.FileInfo) tool.FileInfo { + return tool.FileInfo{Name: i.Name, Size: i.Size, Mode: fs.FileMode(i.Mode), ModTime: i.ModTime, IsDir: i.IsDir} +} +func mapFileError(path string, err error) error { + if err == nil { + return nil + } + var remote *executionenv.Error + if errors.As(err, &remote) { + switch remote.Code { + case executionenv.CodeNotFound: + return fmt.Errorf("%s: %w", path, fs.ErrNotExist) + case executionenv.CodeAlreadyExists: + return fmt.Errorf("%s: %w", path, fs.ErrExist) + case executionenv.CodeVersionMismatch: + return &tool.VersionMismatchError{Path: path} + } + } + return err +} + +// runner deliberately implements only foreground CommandRunner. +type runner struct { + credentials *grantContext +} + +func (*runner) BoundWorkspaceRoot() string { return remoteRoot } +func (r *runner) Run(ctx context.Context, command string) (tool.CommandResult, error) { + credentials, err := r.credentials.current(ctx) + if err != nil { + return tool.CommandResult{}, err + } + resp, err := r.credentials.client.StartCommand(ctx, executionenv.CommandStartRequest{Context: credentials, Command: command}) + if err != nil { + return tool.CommandResult{}, err + } + result := resp.Result + out := tool.CommandResult{Stdout: session.ToValidUTF8(string(result.Stdout)), Stderr: session.ToValidUTF8(string(result.Stderr)), ExitCode: result.ExitCode} + switch result.State { + case executionenv.CommandSucceeded, executionenv.CommandFailed: + return out, nil + case executionenv.CommandCancelled: + return out, context.Canceled + case executionenv.CommandFenceUnknown: + return out, errors.New("execution provider lost terminal command receipt; environment fencing is unknown") + default: + return out, errors.New("execution provider returned a non-terminal foreground command") + } +} diff --git a/internal/adapter/executionclient/client_test.go b/internal/adapter/executionclient/client_test.go new file mode 100644 index 0000000000..828c52389c --- /dev/null +++ b/internal/adapter/executionclient/client_test.go @@ -0,0 +1,302 @@ +package executionclient + +import ( + "context" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "math/big" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type integrationBackend struct { + allocation executioncontroller.Allocation + ensureCalls int + attachCalls int + fileCalls int +} + +func (*integrationBackend) ValidateProfile(context.Context, string) (executioncontroller.Profile, error) { + return executioncontroller.Profile{Name: "coding", Digest: "sha256:test"}, nil +} +func (b *integrationBackend) Ensure(_ context.Context, client, owner, binding, _, _ string) (executioncontroller.Allocation, error) { + b.ensureCalls++ + b.allocation = executioncontroller.Allocation{Environment: executionenv.EnvironmentRef{ID: "env-real", Revision: "rev-1"}, Epoch: 1, OwnerHash: owner, BindingID: binding, Client: client, Ready: true} + return b.allocation, nil +} +func (b *integrationBackend) Attach(_ context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) (executioncontroller.Allocation, error) { + b.attachCalls++ + if ref != b.allocation.Environment || client != b.allocation.Client || owner != b.allocation.OwnerHash || binding != b.allocation.BindingID { + return executioncontroller.Allocation{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "not found"} + } + return b.allocation, nil +} +func (*integrationBackend) ReleaseReference(context.Context, executionenv.EnvironmentRef, string, string, string) error { + return nil +} +func (*integrationBackend) Retire(context.Context, executionenv.EnvironmentRef, string) error { + return nil +} +func (b *integrationBackend) File(_ context.Context, _, _ string, q executionenv.FileRequest) (executionenv.FileResponse, error) { + b.fileCalls++ + if q.Operation == executionenv.OpFileResolveAuthority { + return executionenv.FileResponse{AuthorityTarget: "/workspace/main.go", AuthorityWorkspace: "/workspace"}, nil + } + return executionenv.FileResponse{Data: []byte("from-real-handler"), Version: "v1"}, nil +} +func (*integrationBackend) StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + return executionenv.CommandStartResponse{}, nil +} +func (*integrationBackend) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, nil +} +func (*integrationBackend) CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, nil +} + +func certificate(t *testing.T, parent *x509.Certificate, parentKey *ecdsa.PrivateKey, serverName string, client bool) (tls.Certificate, *x509.Certificate, *ecdsa.PrivateKey) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: serverName}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, IsCA: parent == nil, BasicConstraintsValid: true} + if client { + tmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + tmpl.URIs = []*url.URL{{Scheme: "spiffe", Host: "example.test", Path: "/mecak8s"}} + } else if parent != nil { + tmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + tmpl.DNSNames = []string{"example.test"} + } + issuer, signer := tmpl, key + if parent != nil { + issuer, signer = parent, parentKey + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, issuer, &key.PublicKey, signer) + if err != nil { + t.Fatal(err) + } + parsed, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: parsed}, parsed, key +} + +func TestProviderThroughRealSignedHandlerRefreshesAndReattachesExactly(t *testing.T) { + _, ca, caKey := certificate(t, nil, nil, "ca", false) + serverTLS, _, _ := certificate(t, ca, caKey, "example.test", false) + clientTLS, _, _ := certificate(t, ca, caKey, "client", true) + _, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + backend := &integrationBackend{} + handler := executioncontroller.NewHandler(executioncontroller.HandlerConfig{ + Clients: map[string]executioncontroller.ClientPolicy{"spiffe://example.test/mecak8s": {MayAttestOwner: true}}, + Signer: executioncontroller.GrantSigner{KeyID: "k1", PrivateKey: private, Issuer: "provider", Audience: "executor", Lifetime: 5 * time.Second}, + Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": private.Public().(ed25519.PublicKey)}, Issuer: "provider", Audience: "executor", MaxLifetime: 10 * time.Second}, + }, backend) + clientCAs := x509.NewCertPool() + clientCAs.AddCert(ca) + ts := httptest.NewUnstartedServer(handler) + ts.TLS = &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{serverTLS}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientCAs} + ts.StartTLS() + defer ts.Close() + roots := x509.NewCertPool() + roots.AddCert(ca) + client, err := New(ts.URL, &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, Certificates: []tls.Certificate{clientTLS}, ServerName: "example.test"}) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, _ := NewProvider(client, "coding") + principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + binding, err := provider.Bind(context.Background(), server.PlacementBindRequest{Selector: server.DefaultPlacement(), Principal: principal, BindingID: "session-real"}) + if err != nil { + t.Fatal(err) + } + reattached, err := provider.Reattach(context.Background(), server.PlacementReattachRequest{Ref: binding.Ref, Principal: principal, BindingID: "session-real"}) + if err != nil { + t.Fatal(err) + } + resolver, ok := reattached.Environment.Workspace().(tool.AuthorityResourceResolver) + if !ok { + t.Fatal("remote workspace does not implement AuthorityResourceResolver") + } + target, workspace, err := resolver.AuthorityResourcePath("main.go") + if err != nil || target != "/workspace/main.go" || workspace != "/workspace" { + t.Fatalf("authority resource=(%q,%q) err=%v", target, workspace, err) + } + data, err := reattached.Environment.Workspace().Read(context.Background(), "main.go") + if err != nil || string(data) != "from-real-handler" { + t.Fatalf("read=%q err=%v", data, err) + } + if backend.ensureCalls != 1 || backend.attachCalls < 3 || backend.fileCalls != 2 { + t.Fatalf("ensure=%d attach=%d file=%d", backend.ensureCalls, backend.attachCalls, backend.fileCalls) + } +} + +func TestProviderMTLSWorkspaceAndForegroundRunner(t *testing.T) { + caTLS, ca, caKey := certificate(t, nil, nil, "ca", false) + _ = caTLS + serverTLS, _, _ := certificate(t, ca, caKey, "example.test", false) + clientTLS, _, _ := certificate(t, ca, caKey, "client", true) + clientPool := x509.NewCertPool() + clientPool.AddCert(ca) + calls := map[string]int{} + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if len(r.TLS.PeerCertificates) == 0 { + t.Error("request had no verified client certificate") + } + calls[r.URL.Path]++ + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case executionenv.BasePath + "/profiles/validate": + json.NewEncoder(w).Encode(executionenv.ValidateProfileResponse{Profile: "coding", Digest: "sha256:test"}) + case executionenv.BasePath + "/environments/ensure": + json.NewEncoder(w).Encode(executionenv.EnsureEnvironmentResponse{Environment: executionenv.EnvironmentRef{ID: "env-1", Revision: "rev-1"}, Epoch: 2, Ready: true, Grant: "grant-1", GrantExpiresAt: time.Now().Add(time.Minute)}) + case executionenv.BasePath + "/environments/attach": + json.NewEncoder(w).Encode(executionenv.AttachEnvironmentResponse{Environment: executionenv.EnvironmentRef{ID: "env-1", Revision: "rev-1"}, Epoch: 2, Ready: true, Grant: "grant-2", GrantExpiresAt: time.Now().Add(time.Minute)}) + case executionenv.BasePath + "/files": + var q executionenv.FileRequest + json.NewDecoder(r.Body).Decode(&q) + if q.Context.BindingID != "session-1" || q.Context.Grant != "grant-2" { + t.Errorf("file context = %+v", q.Context) + } + if q.Operation == executionenv.OpFileResolveAuthority { + if q.Path == "escape" { + w.WriteHeader(http.StatusForbidden) + json.NewEncoder(w).Encode(executionenv.ErrorResponse{Error: &executionenv.Error{Code: executionenv.CodePermissionDenied, Message: "path is outside workspace"}}) + } else { + json.NewEncoder(w).Encode(executionenv.FileResponse{AuthorityTarget: "/workspace/main.go", AuthorityWorkspace: "/workspace"}) + } + } else { + json.NewEncoder(w).Encode(executionenv.FileResponse{Data: []byte("remote"), Version: "v1"}) + } + case executionenv.BasePath + "/commands/start": + json.NewEncoder(w).Encode(executionenv.CommandStartResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Result: executionenv.CommandStatusResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Stdout: []byte("ok\n"), Stderr: []byte("warn\n")}}) + default: + http.NotFound(w, r) + } + }) + ts := httptest.NewUnstartedServer(h) + ts.TLS = &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{serverTLS}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientPool} + ts.StartTLS() + defer ts.Close() + roots := x509.NewCertPool() + roots.AddCert(ca) + client, err := New(ts.URL, &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, Certificates: []tls.Certificate{clientTLS}, ServerName: "example.test"}) + if err != nil { + t.Fatal(err) + } + provider, _ := NewProvider(client, "coding") + if err := provider.ValidatePlacement(context.Background()); err != nil { + t.Fatal(err) + } + principal := &session.Principal{Issuer: "https://issuer.example", Subject: "alice", GrantType: session.GrantTypeUser} + binding, err := provider.Bind(context.Background(), server.PlacementBindRequest{Selector: server.DefaultPlacement(), Principal: principal, Scope: "remote", Operation: server.PlacementOperationCreate, BindingID: "session-1"}) + if err != nil { + t.Fatal(err) + } + resolver := binding.Environment.Workspace().(tool.AuthorityResourceResolver) + target, root, err := resolver.AuthorityResourcePath("main.go") + if err != nil || target != "/workspace/main.go" || root != "/workspace" { + t.Fatalf("authority resource=(%q,%q) err=%v", target, root, err) + } + if target, root, err = resolver.AuthorityResourcePath("escape"); err == nil || target != "" || root != "" { + t.Fatalf("escaped authority resource=(%q,%q) err=%v, want fail closed", target, root, err) + } + data, _, err := binding.Environment.Workspace().ReadVersion(context.Background(), "main.go") + if err != nil || string(data) != "remote" { + t.Fatalf("read=%q err=%v", data, err) + } + result, err := binding.Environment.CommandRunner().Run(context.Background(), "go test ./...") + if err != nil || result.Stdout != "ok\n" || result.Stderr != "warn\n" { + t.Fatalf("command=%+v err=%v", result, err) + } + if _, streaming := binding.Environment.CommandRunner().(tool.CommandStreamer); streaming { + t.Fatal("remote runner unexpectedly advertises streaming") + } + if calls[executionenv.BasePath+"/profiles/validate"] != 1 || calls[executionenv.BasePath+"/environments/ensure"] != 1 { + t.Fatalf("calls=%v", calls) + } +} + +func TestProviderWaitsForAsyncReadinessWithoutReEnsuring(t *testing.T) { + var ensureCalls, attachCalls int + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + ref := executionenv.EnvironmentRef{ID: "env-pending", Revision: "rev-1"} + switch r.URL.Path { + case executionenv.BasePath + "/environments/ensure": + ensureCalls++ + json.NewEncoder(w).Encode(executionenv.EnsureEnvironmentResponse{Environment: ref, Epoch: 1, Ready: false, Grant: "initial", GrantExpiresAt: time.Now().Add(time.Minute)}) + case executionenv.BasePath + "/environments/attach": + attachCalls++ + json.NewEncoder(w).Encode(executionenv.AttachEnvironmentResponse{Environment: ref, Epoch: 1, Ready: attachCalls > 1, Grant: "fresh", GrantExpiresAt: time.Now().Add(time.Minute)}) + default: + http.NotFound(w, r) + } + }) + ts := httptest.NewServer(h) + defer ts.Close() + client, err := NewWithHTTPClient(ts.URL, ts.Client()) + if err != nil { + t.Fatal(err) + } + provider, _ := NewProvider(client, "coding") + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + if _, err := provider.Bind(ctx, server.PlacementBindRequest{Selector: server.DefaultPlacement(), Principal: principal, BindingID: "session-pending"}); err != nil { + t.Fatal(err) + } + if ensureCalls != 1 || attachCalls != 2 { + t.Fatalf("ensure=%d attach=%d, want 1/2", ensureCalls, attachCalls) + } +} + +func TestProviderReattachMissingNeverEnsures(t *testing.T) { + var ensureCalls int + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == executionenv.BasePath+"/environments/ensure" { + ensureCalls++ + } + w.WriteHeader(http.StatusNotFound) + json.NewEncoder(w).Encode(executionenv.ErrorResponse{Error: &executionenv.Error{Code: executionenv.CodeNotFound, Message: "environment not found"}}) + }) + ts := httptest.NewServer(h) + defer ts.Close() + client, _ := NewWithHTTPClient(ts.URL, ts.Client()) + provider, _ := NewProvider(client, "coding") + principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + _, err := provider.Reattach(context.Background(), server.PlacementReattachRequest{Ref: session.EnvironmentRef{Kind: "kubernetes", ID: "deleted", Revision: "rev-1"}, Principal: principal, BindingID: "session-1"}) + if err == nil || ensureCalls != 0 { + t.Fatalf("err=%v ensure=%d, want unavailable and zero ensure calls", err, ensureCalls) + } +} + +func TestProviderRequiresOwnerAndBinding(t *testing.T) { + p := &Provider{} + _, err := p.Bind(context.Background(), server.PlacementBindRequest{Selector: server.DefaultPlacement()}) + if err == nil { + t.Fatal("ownerless/bindingless allocation succeeded") + } +} diff --git a/internal/adapter/executioncontroller/controller.go b/internal/adapter/executioncontroller/controller.go new file mode 100644 index 0000000000..8688464eb7 --- /dev/null +++ b/internal/adapter/executioncontroller/controller.go @@ -0,0 +1,428 @@ +package executioncontroller + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + environmentFinalizer = "execution.mecatl.dev/retain-workspace" + fenceHealthy = "Healthy" +) + +// Reconciler maintains executor Pods and retained workspace PVCs. +type Reconciler struct { + dynamic dynamic.Interface + kube kubernetes.Interface + namespace string + profiles *Profiles + queue workqueue.TypedRateLimitingInterface[string] + initOnce sync.Once + initErr error + ready atomic.Bool +} + +// NewReconciler constructs a reconciler for one namespace. +func NewReconciler(d dynamic.Interface, k kubernetes.Interface, namespace string, p *Profiles) *Reconciler { + return &Reconciler{dynamic: d, kube: k, namespace: namespace, profiles: p, queue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[string]())} +} + +// Initialize completes startup fencing and informer synchronization before the +// provider may expose mutating APIs. +func (r *Reconciler) Initialize(ctx context.Context) error { + r.initOnce.Do(func() { + if err := r.fenceInterruptedOperations(ctx); err != nil { + r.initErr = err + return + } + factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(r.dynamic, 0, r.namespace, nil) + environments := factory.ForResource(ExecutionEnvironmentGVR).Informer() + _, err := environments.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: r.enqueue, UpdateFunc: r.enqueueUpdate}) + if err != nil { + r.initErr = err + return + } + runtimeFactory := informers.NewSharedInformerFactoryWithOptions(r.kube, 0, informers.WithNamespace(r.namespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) { + options.LabelSelector = "execution.mecatl.dev/environment" + })) + pods := runtimeFactory.Core().V1().Pods().Informer() + pvcs := runtimeFactory.Core().V1().PersistentVolumeClaims().Informer() + for _, informer := range []cache.SharedIndexInformer{pods, pvcs} { + if _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: r.enqueueRuntime, UpdateFunc: func(_, next any) { r.enqueueRuntime(next) }, DeleteFunc: r.enqueueRuntime}); err != nil { + r.initErr = err + return + } + } + factory.Start(ctx.Done()) + runtimeFactory.Start(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), environments.HasSynced, pods.HasSynced, pvcs.HasSynced) { + r.initErr = errors.New("execution environment informer failed to sync") + return + } + go wait.UntilWithContext(ctx, r.worker, time.Second) + r.ready.Store(true) + }) + return r.initErr +} + +// Ready reports whether startup fencing and informer synchronization completed. +func (r *Reconciler) Ready() bool { return r.ready.Load() } + +// Run initializes the reconciler and blocks until shutdown. +func (r *Reconciler) Run(ctx context.Context) error { + if err := r.Initialize(ctx); err != nil { + return err + } + <-ctx.Done() + r.queue.ShutDown() + return nil +} +func (r *Reconciler) enqueue(obj any) { + u, ok := obj.(*unstructured.Unstructured) + if ok { + r.queue.Add(u.GetName()) + } +} + +func (r *Reconciler) enqueueUpdate(oldObj, newObj any) { + oldEnv, oldOK := oldObj.(*unstructured.Unstructured) + newEnv, newOK := newObj.(*unstructured.Unstructured) + if !oldOK || !newOK || !reconcileInputsEqual(oldEnv, newEnv) { + r.enqueue(newObj) + } +} + +func reconcileInputsEqual(oldEnv, newEnv *unstructured.Unstructured) bool { + if oldEnv.GetGeneration() != newEnv.GetGeneration() || + !reflect.DeepEqual(oldEnv.GetDeletionTimestamp(), newEnv.GetDeletionTimestamp()) || + !reflect.DeepEqual(oldEnv.GetFinalizers(), newEnv.GetFinalizers()) { + return false + } + for _, path := range [][]string{{"spec"}, {"status", "activeOperation"}, {"status", "references"}, {"status", "fenceState"}} { + oldValue, _, _ := unstructured.NestedFieldNoCopy(oldEnv.Object, path...) + newValue, _, _ := unstructured.NestedFieldNoCopy(newEnv.Object, path...) + if !reflect.DeepEqual(oldValue, newValue) { + return false + } + } + return true +} + +func (r *Reconciler) enqueueRuntime(obj any) { + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + var labels map[string]string + switch value := obj.(type) { + case *corev1.Pod: + labels = value.Labels + case *corev1.PersistentVolumeClaim: + labels = value.Labels + } + if name := labels["execution.mecatl.dev/environment"]; name != "" { + r.queue.Add(name) + } +} +func (r *Reconciler) worker(ctx context.Context) { + for r.process(ctx) { + } +} +func (r *Reconciler) process(ctx context.Context) bool { + name, shutdown := r.queue.Get() + if shutdown { + return false + } + defer r.queue.Done(name) + if err := r.Reconcile(ctx, name); err != nil { + r.queue.AddRateLimited(name) + } else { + r.queue.Forget(name) + } + return true +} +func (r *Reconciler) fenceInterruptedOperations(ctx context.Context) error { + list, err := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("list execution environments for startup fencing: %w", err) + } + for i := range list.Items { + o := list.Items[i].DeepCopy() + if textNested(o.Object, "status", "activeOperation", "id") == "" { + continue + } + unstructured.RemoveNestedField(o.Object, "status", "activeOperation") + _ = unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") + if _, err := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace).UpdateStatus(ctx, o, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("fence interrupted environment %q: %w", o.GetName(), err) + } + } + return nil +} + +// Reconcile converges one ExecutionEnvironment and its runtime resources. +func (r *Reconciler) Reconcile(ctx context.Context, name string) error { + res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) + env, err := res.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + profileName := textNested(env.Object, "spec", "profile") + p, ok := r.profiles.get(profileName) + if !ok || p.Digest != textNested(env.Object, "spec", "profileDigest") { + return r.setCondition(ctx, env, "Ready", false, "InvalidProfile", "configured profile is unavailable or changed") + } + if env.GetDeletionTimestamp() != nil { + return r.reconcileDeletion(ctx, res, env) + } + if !contains(env.GetFinalizers(), environmentFinalizer) { + updated := env.DeepCopy() + updated.SetFinalizers(append(updated.GetFinalizers(), environmentFinalizer)) + if _, err := res.Update(ctx, updated, metav1.UpdateOptions{}); err != nil { + return err + } + return nil + } + if textNested(env.Object, "spec", "desired") == "Retiring" { + return r.reconcileRetiring(ctx, env) + } + pvcName := resourceName("workspace", name) + podName := resourceName("executor", name) + pvc, err := r.ensurePVC(ctx, env, p, pvcName) + if err != nil { + return r.setCondition(ctx, env, "Ready", false, "PVCUnavailable", "workspace PVC unavailable") + } + pod, err := r.ensurePod(ctx, env, p, podName, pvcName) + if err != nil { + return r.setCondition(ctx, env, "Ready", false, "ExecutorUnavailable", "executor Pod unavailable") + } + ready := podReady(pod) + if err := r.updateRuntimeStatus(ctx, env, pvc, pod, ready); err != nil { + return err + } + if !ready { + r.queue.AddAfter(name, time.Second) + } + return nil +} +func (r *Reconciler) ensurePVC(ctx context.Context, env *unstructured.Unstructured, p resolvedProfile, name string) (*corev1.PersistentVolumeClaim, error) { + pvcs := r.kube.CoreV1().PersistentVolumeClaims(r.namespace) + cur, err := pvcs.Get(ctx, name, metav1.GetOptions{}) + if err == nil { + if cur.Labels["execution.mecatl.dev/environment"] != env.GetName() { + return nil, errors.New("PVC ownership label mismatch") + } + return cur, nil + } + if !apierrors.IsNotFound(err) { + return nil, err + } + qty := p.StorageSize + mode := corev1.PersistentVolumeFilesystem + created, err := pvcs.Create(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/revision": textNested(env.Object, "spec", "revision")}}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, StorageClassName: &p.Spec.StorageClass, VolumeMode: &mode, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: qty}}}}, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return pvcs.Get(ctx, name, metav1.GetOptions{}) + } + return created, err +} +func (r *Reconciler) ensurePod(ctx context.Context, env *unstructured.Unstructured, p resolvedProfile, name, pvc string) (*corev1.Pod, error) { + pods := r.kube.CoreV1().Pods(r.namespace) + cur, err := pods.Get(ctx, name, metav1.GetOptions{}) + if err == nil { + if cur.Labels["execution.mecatl.dev/environment"] != env.GetName() { + return nil, errors.New("pod ownership label mismatch") + } + return cur, nil + } + if !apierrors.IsNotFound(err) { + return nil, err + } + if textNested(env.Object, "status", "pod", "uid") != "" { + _ = r.setFenceUnknown(ctx, env, "previous executor disappearance is not externally fenced") + return nil, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "previous executor termination unconfirmed"} + } + cpuReq := p.CPURequest + memReq := p.MemoryRequest + cpuLim := p.CPULimit + memLim := p.MemoryLimit + nonroot := true + uid := int64(65532) + noPriv := false + ro := true + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName()}, OwnerReferences: []metav1.OwnerReference{{APIVersion: env.GetAPIVersion(), Kind: env.GetKind(), Name: env.GetName(), UID: env.GetUID(), Controller: &nonroot}}}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noPriv, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &nonroot, RunAsUser: &uid, RunAsGroup: &uid, FSGroup: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "executor", Image: p.Spec.Image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &noPriv, ReadOnlyRootFilesystem: &ro, RunAsNonRoot: &nonroot, RunAsUser: &uid, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: cpuReq, corev1.ResourceMemory: memReq}, Limits: corev1.ResourceList{corev1.ResourceCPU: cpuLim, corev1.ResourceMemory: memLim}}, VolumeMounts: []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}}}, Volumes: []corev1.Volume{{Name: "workspace", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc}}}, {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: quantityPtr("1Gi")}}}}}} + created, err := pods.Create(ctx, pod, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return pods.Get(ctx, name, metav1.GetOptions{}) + } + return created, err +} +func (r *Reconciler) reconcileRetiring(ctx context.Context, env *unstructured.Unstructured) error { + if textNested(env.Object, "status", "activeOperation", "id") != "" || textNested(env.Object, "status", "fenceState") != fenceHealthy { + return r.setCondition(ctx, env, "Retired", false, "NotQuiescent", "retirement waits for proven quiescence") + } + refs, _, _ := unstructured.NestedStringSlice(env.Object, "status", "references") + if len(refs) > 0 { + return r.setCondition(ctx, env, "Retired", false, "Referenced", "retirement refused while references remain") + } + podName := textNested(env.Object, "status", "pod", "name") + if podName == "" { + return r.setCondition(ctx, env, "Retired", false, "FenceUnknown", "executor identity is unavailable; external fencing proof is required") + } + pod, err := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + if conditionTrue(env, "ExecutorTerminated") { + return r.setCondition(ctx, env, "Retired", true, "WorkspaceRetained", "verified terminal executor removed; PVC retained") + } + return r.setCondition(ctx, env, "Retired", false, "FenceUnknown", "executor disappeared without a verified terminal state; external fencing proof is required") + } + if err != nil { + return err + } + if pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed { + return r.setCondition(ctx, env, "Retired", false, "PendingExternalFence", "executor is not terminal; no automatic force retirement is permitted") + } + if err := r.setCondition(ctx, env, "ExecutorTerminated", true, "TerminalPod", "executor Pod reported a terminal phase"); err != nil { + return err + } + grace := int64(30) + if err := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, podName, metav1.DeleteOptions{GracePeriodSeconds: &grace}); err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} +func (r *Reconciler) reconcileDeletion(ctx context.Context, res dynamic.ResourceInterface, env *unstructured.Unstructured) error { + refs, _, _ := unstructured.NestedStringSlice(env.Object, "status", "references") + if textNested(env.Object, "spec", "desired") != "Retiring" || !conditionTrue(env, "Retired") || len(refs) > 0 || textNested(env.Object, "status", "activeOperation", "id") != "" || textNested(env.Object, "status", "fenceState") != fenceHealthy { + return r.setCondition(ctx, env, "DeletionBlocked", true, "RetentionSafety", "finalizer requires explicit completed retirement and proven quiescence") + } + pod := textNested(env.Object, "status", "pod", "name") + if pod != "" { + grace := int64(30) + if err := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, pod, metav1.DeleteOptions{GracePeriodSeconds: &grace}); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + updated := env.DeepCopy() + next := []string{} + for _, f := range updated.GetFinalizers() { + if f != environmentFinalizer { + next = append(next, f) + } + } + updated.SetFinalizers(next) + _, err := res.Update(ctx, updated, metav1.UpdateOptions{}) + return err +} +func (r *Reconciler) updateRuntimeStatus(ctx context.Context, env *unstructured.Unstructured, pvc *corev1.PersistentVolumeClaim, pod *corev1.Pod, ready bool) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pvc.Name, "uid": string(pvc.UID)}, "status", "pvc") + _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pod.Name, "uid": string(pod.UID)}, "status", "pod") + _ = unstructured.SetNestedField(o.Object, o.GetGeneration(), "status", "observedGeneration") + setConditionObject(o, "Ready", ready, "Reconciled", map[bool]string{true: "PVC and executor Pod are ready", false: "waiting for executor Pod readiness"}[ready]) + return nil + }) +} +func (r *Reconciler) setFenceUnknown(ctx context.Context, env *unstructured.Unstructured, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + _ = unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") + setConditionObject(o, "Ready", false, "FenceUnknown", msg) + return nil + }) +} +func (r *Reconciler) setCondition(ctx context.Context, env *unstructured.Unstructured, name string, status bool, reason, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { setConditionObject(o, name, status, reason, msg); return nil }) +} +func (r *Reconciler) updateStatus(ctx context.Context, env *unstructured.Unstructured, fn func(*unstructured.Unstructured) error) error { + res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) + for i := 0; i < 5; i++ { + cur, err := res.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + return err + } + before, _, _ := unstructured.NestedMap(cur.Object, "status") + if err := fn(cur); err != nil { + return err + } + after, _, _ := unstructured.NestedMap(cur.Object, "status") + if reflect.DeepEqual(before, after) { + return nil + } + if _, err = res.UpdateStatus(ctx, cur, metav1.UpdateOptions{}); err == nil { + return nil + } else if !apierrors.IsConflict(err) { + return err + } + } + return errors.New("status update conflict limit exceeded") +} +func setConditionObject(o *unstructured.Unstructured, name string, status bool, reason, msg string) { + conds, _, _ := unstructured.NestedSlice(o.Object, "status", "conditions") + st := "False" + if status { + st = "True" + } + replacement := map[string]any{"type": name, "status": st, "reason": reason, "message": msg, "observedGeneration": o.GetGeneration()} + next := make([]any, 0, len(conds)+1) + replaced := false + for _, value := range conds { + condition, ok := value.(map[string]any) + if !ok || text(condition, "type") != name { + next = append(next, value) + continue + } + if replaced { + continue + } + transitionTime := text(condition, "lastTransitionTime") + if text(condition, "status") != st || transitionTime == "" { + transitionTime = time.Now().UTC().Format(time.RFC3339) + } + replacement["lastTransitionTime"] = transitionTime + next = append(next, replacement) + replaced = true + } + if !replaced { + replacement["lastTransitionTime"] = time.Now().UTC().Format(time.RFC3339) + next = append(next, replacement) + } + _ = unstructured.SetNestedSlice(o.Object, next, "status", "conditions") +} +func podReady(p *corev1.Pod) bool { + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return true + } + } + return false +} +func resourceName(prefix, env string) string { + name := prefix + "-" + strings.TrimPrefix(env, "exec-") + if len(name) > 63 { + name = name[:63] + } + return strings.TrimRight(name, "-") +} +func quantityPtr(v string) *resource.Quantity { q := resource.MustParse(v); return &q } diff --git a/internal/adapter/executioncontroller/controller_test.go b/internal/adapter/executioncontroller/controller_test.go new file mode 100644 index 0000000000..1ba985e86f --- /dev/null +++ b/internal/adapter/executioncontroller/controller_test.go @@ -0,0 +1,193 @@ +package executioncontroller + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func TestReconcileCreatesTokenlessNonRootPodAndRetainedPVC(t *testing.T) { + ctx := context.Background() + env := testEnvironment() + scheme := runtime.NewScheme() + d := dynamicfake.NewSimpleDynamicClient(scheme, env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + pvcs, _ := k.CoreV1().PersistentVolumeClaims("ns").List(ctx, metav1.ListOptions{}) + if len(pvcs.Items) != 1 { + t.Fatalf("pvcs=%d", len(pvcs.Items)) + } + if len(pvcs.Items[0].OwnerReferences) != 0 { + t.Fatal("PVC must not have owner reference") + } + pods, _ := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if len(pods.Items) != 1 { + t.Fatalf("pods=%d", len(pods.Items)) + } + p := pods.Items[0] + if p.Spec.AutomountServiceAccountToken == nil || *p.Spec.AutomountServiceAccountToken { + t.Fatal("service account token mounted") + } + if p.Spec.SecurityContext == nil || p.Spec.SecurityContext.RunAsNonRoot == nil || !*p.Spec.SecurityContext.RunAsNonRoot { + t.Fatal("pod is not non-root") + } + c := p.Spec.Containers[0] + if c.SecurityContext == nil || c.SecurityContext.ReadOnlyRootFilesystem == nil || !*c.SecurityContext.ReadOnlyRootFilesystem || c.SecurityContext.AllowPrivilegeEscalation == nil || *c.SecurityContext.AllowPrivilegeEscalation { + t.Fatal("container security context is not hardened") + } +} +func TestRepeatedReconcileDoesNotRewriteUnchangedStatus(t *testing.T) { + ctx := context.Background() + env := testEnvironment() + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + writes := statusUpdateCount(d.Actions()) + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + transition := conditionTransition(got, "Ready") + if transition == "" { + t.Fatal("Ready condition has no transition timestamp") + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if extra := statusUpdateCount(d.Actions()) - writes; extra != 0 { + t.Fatalf("unchanged reconcile performed %d status writes", extra) + } + got, err = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if current := conditionTransition(got, "Ready"); current != transition { + t.Fatalf("lastTransitionTime changed from %q to %q", transition, current) + } + + pods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pods.Items) != 1 { + t.Fatalf("pods=%d err=%v", len(pods.Items), err) + } + pod := pods.Items[0].DeepCopy() + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := k.CoreV1().Pods("ns").UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + got, err = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(got, "Ready") { + t.Fatalf("Ready did not follow Pod readiness: %v", got.Object["status"]) + } + if statusUpdateCount(d.Actions()) != writes+1 { + t.Fatalf("status writes=%d, want %d", statusUpdateCount(d.Actions()), writes+1) + } +} + +func TestRuntimeAndRelevantStatusUpdatesEnqueue(t *testing.T) { + r := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kubefake.NewSimpleClientset(), "ns", testProfiles()) + r.enqueueRuntime(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"execution.mecatl.dev/environment": "exec-pod"}}}) + r.enqueueRuntime(&corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"execution.mecatl.dev/environment": "exec-pvc"}}}) + if r.queue.Len() != 2 { + t.Fatalf("runtime queue length=%d", r.queue.Len()) + } + + oldEnv := testEnvironment() + newEnv := oldEnv.DeepCopy() + _ = unstructured.SetNestedField(newEnv.Object, "op", "status", "activeOperation", "id") + r.enqueueUpdate(oldEnv, newEnv) + if r.queue.Len() != 3 { + t.Fatalf("relevant status update was not queued: len=%d", r.queue.Len()) + } + controllerStatus := newEnv.DeepCopy() + _ = unstructured.SetNestedField(controllerStatus.Object, "pod", "status", "pod", "name") + r.enqueueUpdate(newEnv, controllerStatus) + if r.queue.Len() != 3 { + t.Fatalf("controller-owned status update was queued: len=%d", r.queue.Len()) + } +} + +func statusUpdateCount(actions []k8stesting.Action) int { + count := 0 + for _, action := range actions { + if action.GetVerb() == "update" && action.GetSubresource() == "status" { + count++ + } + } + return count +} + +func conditionTransition(env *unstructured.Unstructured, name string) string { + conditions, _, _ := unstructured.NestedSlice(env.Object, "status", "conditions") + for _, value := range conditions { + condition, ok := value.(map[string]any) + if ok && text(condition, "type") == name { + return text(condition, "lastTransitionTime") + } + } + return "" +} + +func TestStartupFencesInterruptedOperationBeforeReady(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + env := testEnvironment() + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "old", "operation": "file.replace"}, "status", "activeOperation") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + r := NewReconciler(d, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if r.Ready() { + t.Fatal("reconciler reported ready before startup fencing") + } + if err := r.Initialize(ctx); err != nil { + t.Fatal(err) + } + if !r.Ready() { + t.Fatal("reconciler did not report ready after startup fencing and cache sync") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(got.Object, "status", "fenceState") != "FenceUnknown" || textNested(got.Object, "status", "activeOperation", "id") != "" { + t.Fatalf("status=%v", got.Object["status"]) + } +} +func testProfiles() *Profiles { + spec := ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute} + profile, err := validateProfile("go", spec) + if err != nil { + panic(err) + } + profile.Spec = spec + profile.Digest = "sha256:profile" + return &Profiles{byName: map[string]resolvedProfile{"go": profile}} +} +func testEnvironment() *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "exec-test", "namespace": "ns", "uid": string(types.UID("uid"))}, "spec": map[string]any{"profile": "go", "profileDigest": "sha256:profile", "revision": "rev", "desired": "Active"}, "status": map[string]any{"epoch": int64(1), "references": []any{}, "fenceState": "Healthy"}}} +} diff --git a/internal/adapter/executioncontroller/handler.go b/internal/adapter/executioncontroller/handler.go new file mode 100644 index 0000000000..2148d58ed8 --- /dev/null +++ b/internal/adapter/executioncontroller/handler.go @@ -0,0 +1,409 @@ +// Package executioncontroller implements the authenticated provider HTTP boundary. +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "sort" + "strings" + "time" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// ClientPolicy defines capabilities assigned to an authenticated provider client. +type ClientPolicy struct { + MayAttestOwner bool + Administrator bool +} + +// GrantSigner configures short-lived environment grant issuance. +type GrantSigner struct { + KeyID string + PrivateKey ed25519.PrivateKey + Issuer string + Audience string + Lifetime time.Duration +} + +// HandlerConfig configures authentication and capability grants. +type HandlerConfig struct { + Clients map[string]ClientPolicy + Signer GrantSigner + Verifier executionenv.GrantVerifier + Ready func() bool +} + +// Profile is the externally visible immutable execution profile. +type Profile struct { + Name string + Digest string + MaxFileBytes int64 + MaxCommandBytes int64 + MaxCommandDuration time.Duration + Capabilities []string +} + +// Allocation is an exact environment allocation and authorization binding. +type Allocation struct { + Environment executionenv.EnvironmentRef + Epoch uint64 + OwnerHash string + BindingID string + Client string + Ready bool +} + +// Backend implements provider-side authorization state and executor dispatch. +type Backend interface { + ValidateProfile(context.Context, string) (Profile, error) + Ensure(context.Context, string, string, string, string, string) (Allocation, error) // client, owner hash, binding, profile, immutable fingerprint + Attach(context.Context, executionenv.EnvironmentRef, string, string, string) (Allocation, error) + ReleaseReference(context.Context, executionenv.EnvironmentRef, string, string, string) error + Retire(context.Context, executionenv.EnvironmentRef, string) error + File(context.Context, string, string, executionenv.FileRequest) (executionenv.FileResponse, error) + StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) + CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) + CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) +} + +// Handler is the authenticated private execution-provider HTTP boundary. +type Handler struct { + cfg HandlerConfig + backend Backend + mux *http.ServeMux +} + +// NewHandler constructs the authenticated private HTTP boundary. +func NewHandler(cfg HandlerConfig, b Backend) *Handler { + h := &Handler{cfg: cfg, backend: b, mux: http.NewServeMux()} + h.mux.HandleFunc("POST "+executionenv.BasePath+"/profiles/validate", h.validate) + h.mux.HandleFunc("POST "+executionenv.BasePath+"/environments/ensure", h.ensure) + h.mux.HandleFunc("POST "+executionenv.BasePath+"/environments/attach", h.attach) + h.mux.HandleFunc("POST "+executionenv.BasePath+"/references/release", h.release) + h.mux.HandleFunc("POST "+executionenv.BasePath+"/environments/retire", h.retire) + h.mux.HandleFunc("POST "+executionenv.BasePath+"/files", h.file) + h.mux.HandleFunc("POST "+executionenv.BasePath+"/commands/start", h.startCommand) + h.mux.HandleFunc("POST "+executionenv.BasePath+"/commands/status", h.commandStatus) + h.mux.HandleFunc("POST "+executionenv.BasePath+"/commands/cancel", h.cancelCommand) + return h +} +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if h.cfg.Ready != nil && !h.cfg.Ready() { + writeError(w, http.StatusServiceUnavailable, executionenv.CodeNotReady, "provider startup fencing is incomplete", true) + return + } + id, policy, err := h.authenticate(r) + if err != nil { + writeError(w, http.StatusUnauthorized, executionenv.CodeUnauthenticated, "authenticated client certificate required", false) + return + } + ctx := context.WithValue(r.Context(), clientContextKey{}, authenticatedClient{id, policy}) + h.mux.ServeHTTP(w, r.WithContext(ctx)) +} + +type clientContextKey struct{} +type authenticatedClient struct { + id string + policy ClientPolicy +} + +func (h *Handler) authenticate(r *http.Request) (string, ClientPolicy, error) { + if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 { + return "", ClientPolicy{}, errors.New("no peer certificate") + } + id, err := canonicalClientIdentity(r.TLS.PeerCertificates[0]) + if err != nil { + return "", ClientPolicy{}, err + } + p, ok := h.cfg.Clients[id] + if !ok { + return "", ClientPolicy{}, errors.New("client not allowlisted") + } + return id, p, nil +} +func canonicalClientIdentity(c *x509.Certificate) (string, error) { + ids := make([]string, 0, len(c.URIs)) + for _, u := range c.URIs { + if u == nil || u.Scheme == "" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + continue + } + v := *u + v.Scheme = strings.ToLower(v.Scheme) + v.Host = strings.ToLower(v.Host) + ids = append(ids, v.String()) + } + if len(ids) == 0 { + return "", errors.New("certificate has no canonical URI SAN") + } + sort.Strings(ids) + return ids[0], nil +} + +func (h *Handler) validate(w http.ResponseWriter, r *http.Request) { + var q executionenv.ValidateProfileRequest + if !decode(w, r, &q) { + return + } + if q.Profile == "" || len(q.Profile) > 63 { + writeError(w, 400, executionenv.CodeInvalidArgument, "invalid profile", false) + return + } + if h.backend == nil { + writeError(w, 503, executionenv.CodeNotReady, "provider backend unavailable", true) + return + } + p, err := h.backend.ValidateProfile(r.Context(), q.Profile) + if err != nil { + writeBackendError(w, err) + return + } + writeJSON(w, 200, executionenv.ValidateProfileResponse{Profile: p.Name, Digest: p.Digest, Capabilities: p.Capabilities, MaxFileBytes: p.MaxFileBytes, MaxCommandBytes: p.MaxCommandBytes, MaxCommandDurationMillis: p.MaxCommandDuration.Milliseconds()}) +} +func (h *Handler) ensure(w http.ResponseWriter, r *http.Request) { + var q executionenv.EnsureEnvironmentRequest + if !decode(w, r, &q) { + return + } + c := r.Context().Value(clientContextKey{}).(authenticatedClient) + if !c.policy.MayAttestOwner || q.Owner.Issuer == "" || q.Owner.Subject == "" || len(q.Owner.Issuer) > executionenv.MaxIdentityBytes || len(q.Owner.Subject) > executionenv.MaxIdentityBytes || q.BindingID == "" || len(q.BindingID) > executionenv.MaxBindingBytes || q.Profile == "" || len(q.Profile) > 63 { + writeError(w, 403, executionenv.CodePermissionDenied, "owner attestation or required allocation identity missing", false) + return + } + owner := ownerHash(q.Owner) + fp := fingerprint(c.id, owner, q.BindingID, q.Profile) + a, err := h.backend.Ensure(r.Context(), c.id, owner, q.BindingID, q.Profile, fp) + if err != nil { + writeBackendError(w, err) + return + } + grant, expiresAt, err := h.sign(a, c.id) + if err != nil { + writeError(w, 500, executionenv.CodeInternal, "grant issuance failed", false) + return + } + writeJSON(w, 200, executionenv.EnsureEnvironmentResponse{Environment: a.Environment, Epoch: a.Epoch, Ready: a.Ready, Grant: grant, GrantExpiresAt: expiresAt}) +} +func (h *Handler) attach(w http.ResponseWriter, r *http.Request) { + var q executionenv.AttachEnvironmentRequest + if !decode(w, r, &q) { + return + } + if q.Purpose != executionenv.PurposeSession { + writeError(w, 400, executionenv.CodeInvalidArgument, "unsupported attachment purpose", false) + return + } + c := r.Context().Value(clientContextKey{}).(authenticatedClient) + if !c.policy.MayAttestOwner || q.Context.Owner.Issuer == "" || q.Context.Owner.Subject == "" || len(q.Context.Owner.Issuer) > executionenv.MaxIdentityBytes || len(q.Context.Owner.Subject) > executionenv.MaxIdentityBytes || q.Context.BindingID == "" || len(q.Context.BindingID) > executionenv.MaxBindingBytes || q.Context.Environment.ID == "" || q.Context.Environment.Revision == "" || len(q.Context.Environment.ID) > executionenv.MaxIdentityBytes || len(q.Context.Environment.Revision) > executionenv.MaxIdentityBytes { + writeError(w, 403, executionenv.CodePermissionDenied, "owner attestation or attachment identity missing", false) + return + } + owner := ownerHash(q.Context.Owner) + a, err := h.backend.Attach(r.Context(), q.Context.Environment, c.id, owner, q.Context.BindingID) + if err != nil { + writeBackendError(w, err) + return + } + grant, expiresAt, err := h.sign(a, c.id) + if err != nil { + writeError(w, 500, executionenv.CodeInternal, "grant issuance failed", false) + return + } + writeJSON(w, 200, executionenv.AttachEnvironmentResponse{Environment: a.Environment, Epoch: a.Epoch, Ready: a.Ready, Grant: grant, GrantExpiresAt: expiresAt}) +} +func (h *Handler) release(w http.ResponseWriter, r *http.Request) { + var q executionenv.ReferenceReleaseRequest + if !decode(w, r, &q) { + return + } + c, owner, ok := h.authorize(w, r, q.Context, executionenv.OpReferenceRelease) + if !ok { + return + } + if err := h.backend.ReleaseReference(r.Context(), q.Context.Environment, c.id, owner, q.Context.BindingID); err != nil { + writeBackendError(w, err) + return + } + writeJSON(w, 200, executionenv.EmptyResponse{}) +} +func (h *Handler) retire(w http.ResponseWriter, r *http.Request) { + var q executionenv.RetireEnvironmentRequest + if !decode(w, r, &q) { + return + } + c := r.Context().Value(clientContextKey{}).(authenticatedClient) + if !c.policy.Administrator { + writeError(w, 403, executionenv.CodePermissionDenied, "administrative identity required", false) + return + } + _, owner, ok := h.authorize(w, r, q.Context, executionenv.OpRetire) + if !ok { + return + } + if err := h.backend.Retire(r.Context(), q.Context.Environment, owner); err != nil { + writeBackendError(w, err) + return + } + writeJSON(w, 200, executionenv.EmptyResponse{}) +} +func (h *Handler) file(w http.ResponseWriter, r *http.Request) { + var q executionenv.FileRequest + if !decode(w, r, &q) { + return + } + if len(q.Data) > executionenv.MaxFileBytes || !q.Operation.Valid() || len(q.Path) > executionenv.MaxPathBytes || len(q.Destination) > executionenv.MaxPathBytes || len(q.Pattern) > executionenv.MaxPathBytes || q.Limit < 0 || q.Limit > executionenv.MaxListEntries { + writeError(w, 400, executionenv.CodeInvalidArgument, "invalid or oversized file request", false) + return + } + c, owner, ok := h.authorize(w, r, q.Context, q.Operation) + if !ok { + return + } + resp, err := h.backend.File(r.Context(), c.id, owner, q) + if err != nil { + writeBackendError(w, err) + return + } + writeJSON(w, 200, resp) +} +func (h *Handler) startCommand(w http.ResponseWriter, r *http.Request) { + var q executionenv.CommandStartRequest + if !decode(w, r, &q) { + return + } + if len(q.Command) == 0 || len(q.Command) > executionenv.MaxCommandBytes || q.TimeoutMillis < 0 { + writeError(w, 400, executionenv.CodeInvalidArgument, "invalid command size", false) + return + } + c, owner, ok := h.authorize(w, r, q.Context, executionenv.OpCommandStart) + if !ok { + return + } + v, err := h.backend.StartCommand(r.Context(), c.id, owner, q) + if err != nil { + writeBackendError(w, err) + return + } + writeJSON(w, 200, v) +} +func (h *Handler) commandStatus(w http.ResponseWriter, r *http.Request) { + h.commandQuery(w, r, executionenv.OpCommandStatus) +} +func (h *Handler) cancelCommand(w http.ResponseWriter, r *http.Request) { + h.commandQuery(w, r, executionenv.OpCommandCancel) +} +func (h *Handler) commandQuery(w http.ResponseWriter, r *http.Request, op executionenv.Operation) { + var q executionenv.CommandQueryRequest + if !decode(w, r, &q) { + return + } + c, owner, ok := h.authorize(w, r, q.Context, op) + if !ok { + return + } + var v executionenv.CommandStatusResponse + var err error + if op == executionenv.OpCommandStatus { + v, err = h.backend.CommandStatus(r.Context(), c.id, owner, q) + } else { + v, err = h.backend.CancelCommand(r.Context(), c.id, owner, q) + } + if err != nil { + writeBackendError(w, err) + return + } + writeJSON(w, 200, v) +} +func (h *Handler) authorize(w http.ResponseWriter, r *http.Request, rc executionenv.RequestContext, op executionenv.Operation) (authenticatedClient, string, bool) { + c := r.Context().Value(clientContextKey{}).(authenticatedClient) + if rc.Owner.Issuer == "" || rc.Owner.Subject == "" || len(rc.Owner.Issuer) > executionenv.MaxIdentityBytes || len(rc.Owner.Subject) > executionenv.MaxIdentityBytes || rc.BindingID == "" || len(rc.BindingID) > executionenv.MaxBindingBytes || rc.Environment.ID == "" || rc.Environment.Revision == "" || rc.Epoch == 0 || rc.Grant == "" || len(rc.Grant) > executionenv.MaxGrantBytes { + writeError(w, 403, executionenv.CodePermissionDenied, "request authorization denied", false) + return c, "", false + } + owner := ownerHash(rc.Owner) + if _, err := h.cfg.Verifier.Verify(rc.Grant, executionenv.GrantExpectation{Client: c.id, OwnerHash: owner, BindingID: rc.BindingID, Environment: rc.Environment, Epoch: rc.Epoch, Operation: op}); err != nil { + writeError(w, 403, executionenv.CodePermissionDenied, "request authorization denied", false) + return c, "", false + } + return c, owner, true +} +func (h *Handler) sign(a Allocation, client string) (string, time.Time, error) { + life := h.cfg.Signer.Lifetime + if life <= 0 { + life = time.Minute + } + now := time.Now().UTC() + expiresAt := now.Add(life) + nonce := make([]byte, 16) + if _, err := rand.Read(nonce); err != nil { + return "", time.Time{}, err + } + ops := []executionenv.Operation{executionenv.OpAttach, executionenv.OpReferenceRelease, executionenv.OpRetire, executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileCreate, executionenv.OpFileReplace, executionenv.OpFileList, executionenv.OpFileRemove, executionenv.OpFileRename, executionenv.OpFileCopy, executionenv.OpFileGlob, executionenv.OpFileGrep, executionenv.OpCommandStart, executionenv.OpCommandStatus, executionenv.OpCommandCancel} + grant, err := executionenv.SignGrant(h.cfg.Signer.PrivateKey, executionenv.GrantClaims{KeyID: h.cfg.Signer.KeyID, Issuer: h.cfg.Signer.Issuer, Audience: h.cfg.Signer.Audience, Client: client, OwnerHash: a.OwnerHash, BindingID: a.BindingID, Environment: a.Environment, Epoch: a.Epoch, Operations: ops, NotBefore: now, ExpiresAt: expiresAt, Nonce: hex.EncodeToString(nonce)}) + return grant, expiresAt, err +} +func ownerHash(o executionenv.Owner) string { + s := sha256.Sum256([]byte(o.Issuer + "\x00" + o.Subject)) + return hex.EncodeToString(s[:]) +} +func fingerprint(fields ...string) string { + h := sha256.New() + for _, f := range fields { + _, _ = h.Write([]byte{0}) + _, _ = io.WriteString(h, f) + } + return hex.EncodeToString(h.Sum(nil)) +} +func decode(w http.ResponseWriter, r *http.Request, dst any) bool { + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, executionenv.MaxJSONBody+1)) + if err != nil { + writeError(w, 413, executionenv.CodeResourceExhausted, "request body exceeds limit", false) + return false + } + if err := executionenv.DecodeStrict(body, dst); err != nil { + writeError(w, 400, executionenv.CodeInvalidArgument, "invalid request body", false) + return false + } + return true +} +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} +func writeError(w http.ResponseWriter, status int, code executionenv.ErrorCode, msg string, retry bool) { + writeJSON(w, status, executionenv.ErrorResponse{Error: &executionenv.Error{Code: code, Message: msg, Retryable: retry}}) +} +func writeBackendError(w http.ResponseWriter, err error) { + var e *executionenv.Error + if errors.As(err, &e) { + status := 400 + switch e.Code { + case executionenv.CodeNotFound: + status = 404 + case executionenv.CodeAlreadyExists, executionenv.CodeConflict, executionenv.CodeVersionMismatch: + status = 409 + case executionenv.CodeNotReady, executionenv.CodeFenceUnknown: + status = 503 + case executionenv.CodePermissionDenied: + status = 403 + } + writeError(w, status, e.Code, e.Message, e.Retryable) + return + } + writeError(w, 500, executionenv.CodeInternal, "provider operation failed", false) +} + +// TLSConfig returns the provider's TLS 1.3 mutual-authentication policy. +func TLSConfig(server tls.Certificate, clientCAs *x509.CertPool) *tls.Config { + return &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{server}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientCAs} +} diff --git a/internal/adapter/executioncontroller/handler_test.go b/internal/adapter/executioncontroller/handler_test.go new file mode 100644 index 0000000000..cf8a0abbb2 --- /dev/null +++ b/internal/adapter/executioncontroller/handler_test.go @@ -0,0 +1,195 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "math/big" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +type fakeBackend struct { + ensureCalls, created int + allocations map[string]Allocation +} + +func newFakeBackend() *fakeBackend { return &fakeBackend{allocations: map[string]Allocation{}} } +func (*fakeBackend) ValidateProfile(context.Context, string) (Profile, error) { + return Profile{Name: "go", Digest: "sha256:test", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute}, nil +} +func (f *fakeBackend) Ensure(_ context.Context, client, owner, binding, _, fp string) (Allocation, error) { + f.ensureCalls++ + if a, ok := f.allocations[fp]; ok { + return a, nil + } + f.created++ + a := Allocation{Environment: executionenv.EnvironmentRef{ID: "env-1", Revision: "rev-1"}, Epoch: 1, OwnerHash: owner, BindingID: binding, Client: client, Ready: true} + f.allocations[fp] = a + return a, nil +} +func (*fakeBackend) Attach(context.Context, executionenv.EnvironmentRef, string, string, string) (Allocation, error) { + panic("unused") +} +func (*fakeBackend) ReleaseReference(context.Context, executionenv.EnvironmentRef, string, string, string) error { + return nil +} +func (*fakeBackend) Retire(context.Context, executionenv.EnvironmentRef, string) error { return nil } +func (*fakeBackend) File(context.Context, string, string, executionenv.FileRequest) (executionenv.FileResponse, error) { + return executionenv.FileResponse{}, nil +} +func (*fakeBackend) StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + return executionenv.CommandStartResponse{}, nil +} +func (*fakeBackend) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, nil +} +func (*fakeBackend) CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, nil +} + +func TestHandlerBlocksAllRequestsUntilStartupReady(t *testing.T) { + backend := newFakeBackend() + h := NewHandler(HandlerConfig{Ready: func() bool { return false }}, backend) + r := httptest.NewRequest("POST", executionenv.BasePath+"/environments/ensure", strings.NewReader(`{"binding_id":"session-1","profile":"go","owner":{"issuer":"issuer","subject":"subject"}}`)) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("code=%d body=%s", w.Code, w.Body.String()) + } + if backend.ensureCalls != 0 { + t.Fatalf("backend calls before startup readiness = %d", backend.ensureCalls) + } +} + +func TestHandlerRequiresAllowlistedCanonicalURISAN(t *testing.T) { + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {MayAttestOwner: true}}}, nil) + r := httptest.NewRequest("POST", executionenv.BasePath+"/profiles/validate", strings.NewReader(`{"profile":"go"}`)) + r.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{{URIs: []*url.URL{{Scheme: "spiffe", Host: "cluster", Path: "/ns/other"}}}}} + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != 401 { + t.Fatalf("code=%d body=%s", w.Code, w.Body.String()) + } +} + +func TestHandlerValidateIsReadOnlyAndEnsureIdempotent(t *testing.T) { + pub, priv, _ := ed25519.GenerateKey(rand.Reader) + backend := newFakeBackend() + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {MayAttestOwner: true}}, Signer: GrantSigner{KeyID: "k1", PrivateKey: priv, Issuer: "provider", Audience: "execution", Lifetime: time.Minute}, Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "provider", Audience: "execution"}}, backend) + call := func(path, body string) *httptest.ResponseRecorder { + r := httptest.NewRequest("POST", executionenv.BasePath+path, strings.NewReader(body)) + r.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{{URIs: []*url.URL{{Scheme: "spiffe", Host: "cluster", Path: "/ns/mecak8s"}}}}} + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return w + } + if w := call("/profiles/validate", `{"profile":"go"}`); w.Code != 200 { + t.Fatalf("validate %d %s", w.Code, w.Body.String()) + } + if backend.ensureCalls != 0 { + t.Fatal("validation allocated") + } + body := `{"binding_id":"session-1","profile":"go","owner":{"issuer":"https://issuer","subject":"alice"}}` + w1 := call("/environments/ensure", body) + if w1.Code != 200 { + t.Fatalf("ensure %d %s", w1.Code, w1.Body.String()) + } + w2 := call("/environments/ensure", body) + if w2.Code != 200 { + t.Fatalf("ensure2 %d %s", w2.Code, w2.Body.String()) + } + if backend.created != 1 { + t.Fatalf("created=%d", backend.created) + } + var first, second executionenv.EnsureEnvironmentResponse + if err := json.Unmarshal(w1.Body.Bytes(), &first); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(w2.Body.Bytes(), &second); err != nil { + t.Fatal(err) + } + if first.Environment != second.Environment || first.Epoch != second.Epoch { + t.Fatalf("allocation identity changed") + } +} + +func TestCanonicalClientIdentityRejectsCNAndAmbiguousURIs(t *testing.T) { + if _, err := canonicalClientIdentity(&x509.Certificate{Subject: pkix.Name{CommonName: "trusted"}}); err == nil { + t.Fatal("CN accepted") + } + u1, _ := url.Parse("spiffe://b/x") + u2, _ := url.Parse("spiffe://a/x") + got, err := canonicalClientIdentity(&x509.Certificate{URIs: []*url.URL{u1, u2}}) + if err != nil || got != "spiffe://a/x" { + t.Fatalf("got %q err %v", got, err) + } +} + +func TestMTLSFixtureActuallyVerifiesClientCertificate(t *testing.T) { + clientCert, pool, err := testCertificate() + if err != nil { + t.Fatal(err) + } + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {}}}, newFakeBackend()) + srv := httptest.NewUnstartedServer(h) + srv.TLS = &tls.Config{MinVersion: tls.VersionTLS13, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: pool} + srv.StartTLS() + defer srv.Close() + if _, err := srv.Client().Post(srv.URL+executionenv.BasePath+"/profiles/validate", "application/json", strings.NewReader(`{"profile":"go"}`)); err == nil { + t.Fatal("server accepted connection without client certificate") + } + transport := srv.Client().Transport.(*http.Transport).Clone() + transport.TLSClientConfig = transport.TLSClientConfig.Clone() + transport.TLSClientConfig.Certificates = []tls.Certificate{clientCert} + client := &http.Client{Transport: transport} + resp, err := client.Post(srv.URL+executionenv.BasePath+"/profiles/validate", "application/json", strings.NewReader(`{"profile":"go"}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status=%d", resp.StatusCode) + } +} + +func testCertificate() (tls.Certificate, *x509.CertPool, error) { + caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader) + ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test-ca"}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, caPub, caPriv) + if err != nil { + return tls.Certificate{}, nil, err + } + clientPub, clientPriv, _ := ed25519.GenerateKey(rand.Reader) + identity, _ := url.Parse("spiffe://cluster/ns/mecak8s") + leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "ignored-cn"}, URIs: []*url.URL{identity}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + leafDER, err := x509.CreateCertificate(rand.Reader, leaf, ca, clientPub, caPriv) + if err != nil { + return tls.Certificate{}, nil, err + } + certPEM := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER})...) + keyDER, _ := x509.MarshalPKCS8PrivateKey(clientPriv) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return tls.Certificate{}, nil, err + } + pool := x509.NewCertPool() + parsedCA, err := x509.ParseCertificate(caDER) + if err != nil { + return tls.Certificate{}, nil, err + } + pool.AddCert(parsedCA) + return cert, pool, err +} diff --git a/internal/adapter/executioncontroller/podexec.go b/internal/adapter/executioncontroller/podexec.go new file mode 100644 index 0000000000..2ed4ce2995 --- /dev/null +++ b/internal/adapter/executioncontroller/podexec.go @@ -0,0 +1,120 @@ +package executioncontroller + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "sync" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// PodExecutor invokes the fixed helper in an already-selected executor Pod. +type PodExecutor struct { + config *rest.Config + kube kubernetes.Interface + namespace string +} + +// NewPodExecutor constructs a namespace-scoped Pod executor transport. +func NewPodExecutor(config *rest.Config, kube kubernetes.Interface, namespace string) *PodExecutor { + return &PodExecutor{config: rest.CopyConfig(config), kube: kube, namespace: namespace} +} + +// Execute runs one bounded helper request in pod. +func (p *PodExecutor) Execute(ctx context.Context, pod string, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + body, err := json.Marshal(q) + if err != nil { + return executionenv.ExecutorResponse{}, err + } + if len(body) > executionenv.MaxJSONBody { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "executor request exceeds limit"} + } + req := p.kube.CoreV1().RESTClient().Post().Resource("pods").Name(pod).Namespace(p.namespace).SubResource("exec").VersionedParams(&corev1.PodExecOptions{Container: "executor", Command: []string{"/mecatl-executor"}, Stdin: true, Stdout: true, Stderr: true, TTY: false}, scheme.ParameterCodec) + exec, err := remotecommand.NewSPDYExecutor(p.config, "POST", req.URL()) + if err != nil { + return executionenv.ExecutorResponse{}, fmt.Errorf("create executor stream: %w", err) + } + var stdout, stderr limitedBuffer + stdout.limit = executionenv.MaxJSONBody + stderr.limit = 64 << 10 + var stdin io.Reader = bytes.NewReader(body) + var closeInput func() + if q.Operation == executionenv.OpCommandStart { + stdin, closeInput = commandInput(ctx, body) + defer closeInput() + } + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{Stdin: stdin, Stdout: &stdout, Stderr: &stderr}) + if err != nil { + return executionenv.ExecutorResponse{}, fmt.Errorf("executor stream failed: %w", err) + } + var envelope executionenv.ExecutorEnvelope + if err := executionenv.DecodeStrict(stdout.Bytes(), &envelope); err != nil { + return executionenv.ExecutorResponse{}, fmt.Errorf("invalid executor response: %w", err) + } + if envelope.Error != nil && envelope.Response != nil { + return executionenv.ExecutorResponse{}, errors.New("executor returned both result and error") + } + if envelope.Error != nil { + return executionenv.ExecutorResponse{}, envelope.Error + } + if envelope.Response == nil { + return executionenv.ExecutorResponse{}, errors.New("executor returned no result") + } + return *envelope.Response, nil +} + +func commandInput(ctx context.Context, body []byte) (io.Reader, func()) { + reader, writer := io.Pipe() + done := make(chan struct{}) + go func() { + payload := append(append(make([]byte, 0, len(body)+1), body...), '\n') + if _, err := writer.Write(payload); err != nil { + _ = writer.CloseWithError(err) + } + }() + go func() { + select { + case <-ctx.Done(): + _ = writer.CloseWithError(ctx.Err()) + case <-done: + } + }() + var once sync.Once + return reader, func() { + once.Do(func() { + close(done) + _ = writer.Close() + _ = reader.Close() + }) + } +} + +type limitedBuffer struct { + bytes.Buffer + limit int + overflow bool +} + +func (b *limitedBuffer) Write(p []byte) (int, error) { + if b.Len()+len(p) > b.limit { + remaining := b.limit - b.Len() + if remaining > 0 { + _, _ = b.Buffer.Write(p[:remaining]) + } + b.overflow = true + return len(p), errors.New("executor output exceeds limit") + } + return b.Buffer.Write(p) +} + +var _ ExecutorTransport = (*PodExecutor)(nil) diff --git a/internal/adapter/executioncontroller/podexec_test.go b/internal/adapter/executioncontroller/podexec_test.go new file mode 100644 index 0000000000..4108f48c93 --- /dev/null +++ b/internal/adapter/executioncontroller/podexec_test.go @@ -0,0 +1,36 @@ +package executioncontroller + +import ( + "context" + "io" + "testing" + "time" +) + +func TestCommandInputClosesIndependentlyOnCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + reader, closeInput := commandInput(ctx, []byte(`{"operation":"command.start"}`)) + defer closeInput() + + line := make([]byte, len(`{"operation":"command.start"}`)+1) + if _, err := io.ReadFull(reader, line); err != nil { + t.Fatal(err) + } + cancel() + readDone := make(chan error, 1) + go func() { + var one [1]byte + _, err := reader.Read(one[:]) + readDone <- err + }() + deadline := time.NewTimer(2 * time.Second) + defer deadline.Stop() + select { + case err := <-readDone: + if err == nil { + t.Fatal("command control pipe remained open after cancellation") + } + case <-deadline.C: + t.Fatal("command control pipe did not close after cancellation") + } +} diff --git a/internal/adapter/executioncontroller/profiles.go b/internal/adapter/executioncontroller/profiles.go new file mode 100644 index 0000000000..d38eb36e18 --- /dev/null +++ b/internal/adapter/executioncontroller/profiles.go @@ -0,0 +1,152 @@ +package executioncontroller + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/goccy/go-yaml" + "k8s.io/apimachinery/pkg/api/resource" +) + +// ProfilesFile is the strict operator profile-file schema. +type ProfilesFile struct { + Profiles map[string]ProfileSpec `yaml:"profiles"` +} + +// ProfileSpec defines immutable image, storage, resource, and operation bounds. +type ProfileSpec struct { + Image string `yaml:"image"` + StorageClass string `yaml:"storageClass"` + StorageSize string `yaml:"storageSize"` + CPURequest string `yaml:"cpuRequest"` + MemoryRequest string `yaml:"memoryRequest"` + CPULimit string `yaml:"cpuLimit"` + MemoryLimit string `yaml:"memoryLimit"` + MaxFileBytes int64 `yaml:"maxFileBytes"` + MaxCommandBytes int64 `yaml:"maxCommandBytes"` + MaxCommandDuration time.Duration `yaml:"maxCommandDuration"` +} + +// Profiles is a validated immutable profile registry. +type Profiles struct{ byName map[string]resolvedProfile } +type resolvedProfile struct { + Spec ProfileSpec + Digest string + StorageSize resource.Quantity + CPURequest resource.Quantity + MemoryRequest resource.Quantity + CPULimit resource.Quantity + MemoryLimit resource.Quantity +} + +// LoadProfiles reads and strictly validates an operator profile file. +func LoadProfiles(path string) (*Profiles, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read profiles: %w", err) + } + var f ProfilesFile + if err := yaml.UnmarshalWithOptions(b, &f, yaml.DisallowUnknownField()); err != nil { + return nil, fmt.Errorf("decode profiles: %w", err) + } + if len(f.Profiles) == 0 { + return nil, errors.New("profiles file has no profiles") + } + p := &Profiles{byName: map[string]resolvedProfile{}} + for name, s := range f.Profiles { + quantities, err := validateProfile(name, s) + if err != nil { + return nil, err + } + canonical, err := yaml.Marshal(s) + if err != nil { + return nil, err + } + sum := sha256.Sum256(canonical) + quantities.Spec = s + quantities.Digest = "sha256:" + hex.EncodeToString(sum[:]) + p.byName[name] = quantities + } + return p, nil +} +func validateProfile(name string, s ProfileSpec) (resolvedProfile, error) { + if name == "" || strings.ContainsAny(name, "/\\") { + return resolvedProfile{}, fmt.Errorf("invalid profile name %q", name) + } + if !strings.Contains(s.Image, "@sha256:") { + return resolvedProfile{}, fmt.Errorf("profile %q image must be digest-pinned", name) + } + if s.StorageClass == "" || s.StorageSize == "" { + return resolvedProfile{}, fmt.Errorf("profile %q requires explicit storage class and size", name) + } + if s.CPURequest == "" || s.MemoryRequest == "" || s.CPULimit == "" || s.MemoryLimit == "" { + return resolvedProfile{}, fmt.Errorf("profile %q requires explicit resource requests and limits", name) + } + quantities, err := resolveProfileQuantities(name, s) + if err != nil { + return resolvedProfile{}, err + } + if !validExecutionBounds(s) { + return resolvedProfile{}, fmt.Errorf("profile %q has invalid execution bounds", name) + } + return quantities, nil +} + +func resolveProfileQuantities(name string, s ProfileSpec) (resolvedProfile, error) { + storage, err := positiveQuantity(name, "storageSize", s.StorageSize) + if err != nil { + return resolvedProfile{}, err + } + cpuRequest, err := positiveQuantity(name, "cpuRequest", s.CPURequest) + if err != nil { + return resolvedProfile{}, err + } + memoryRequest, err := positiveQuantity(name, "memoryRequest", s.MemoryRequest) + if err != nil { + return resolvedProfile{}, err + } + cpuLimit, err := positiveQuantity(name, "cpuLimit", s.CPULimit) + if err != nil { + return resolvedProfile{}, err + } + memoryLimit, err := positiveQuantity(name, "memoryLimit", s.MemoryLimit) + if err != nil { + return resolvedProfile{}, err + } + if cpuRequest.Cmp(cpuLimit) > 0 { + return resolvedProfile{}, fmt.Errorf("profile %q cpuRequest exceeds cpuLimit", name) + } + if memoryRequest.Cmp(memoryLimit) > 0 { + return resolvedProfile{}, fmt.Errorf("profile %q memoryRequest exceeds memoryLimit", name) + } + return resolvedProfile{StorageSize: storage, CPURequest: cpuRequest, MemoryRequest: memoryRequest, CPULimit: cpuLimit, MemoryLimit: memoryLimit}, nil +} + +func validExecutionBounds(s ProfileSpec) bool { + return s.MaxFileBytes > 0 && s.MaxFileBytes <= 5<<20 && + s.MaxCommandBytes > 0 && s.MaxCommandBytes <= 1<<20 && + s.MaxCommandDuration > 0 && s.MaxCommandDuration <= 30*time.Minute +} + +func positiveQuantity(profile, field, value string) (resource.Quantity, error) { + quantity, err := resource.ParseQuantity(value) + if err != nil { + return resource.Quantity{}, fmt.Errorf("profile %q %s is invalid: %w", profile, field, err) + } + if quantity.Sign() <= 0 { + return resource.Quantity{}, fmt.Errorf("profile %q %s must be positive", profile, field) + } + return quantity, nil +} +func (p *Profiles) get(name string) (resolvedProfile, bool) { + if p == nil { + return resolvedProfile{}, false + } + v, ok := p.byName[name] + return v, ok +} diff --git a/internal/adapter/executioncontroller/profiles_test.go b/internal/adapter/executioncontroller/profiles_test.go new file mode 100644 index 0000000000..a1e25b35a8 --- /dev/null +++ b/internal/adapter/executioncontroller/profiles_test.go @@ -0,0 +1,94 @@ +package executioncontroller + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadProfilesStrictAndDigestPinned(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "profiles.yaml") + good := validProfileYAML() + if err := os.WriteFile(path, []byte(good), 0o600); err != nil { + t.Fatal(err) + } + p, err := LoadProfiles(path) + if err != nil { + t.Fatal(err) + } + v, ok := p.get("go") + if !ok || !strings.HasPrefix(v.Digest, "sha256:") { + t.Fatalf("profile=%+v", v) + } + if err := os.WriteFile(path, []byte(good+" unknown: true\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadProfiles(path); err == nil { + t.Fatal("unknown field accepted") + } + unpinned := strings.Replace(good, "@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "", 1) + _ = os.WriteFile(path, []byte(unpinned), 0o600) + if _, err := LoadProfiles(path); err == nil { + t.Fatal("unpinned image accepted") + } +} + +func TestLoadProfilesRejectsInvalidQuantities(t *testing.T) { + cases := map[string]string{ + "malformed": strings.Replace(validProfileYAML(), "cpuRequest: 100m", "cpuRequest: invalid", 1), + "zero": strings.Replace(validProfileYAML(), "memoryRequest: 128Mi", `memoryRequest: "0"`, 1), + "negative": strings.Replace(validProfileYAML(), `cpuLimit: "1"`, `cpuLimit: "-1"`, 1), + "cpu request over limit": strings.Replace(validProfileYAML(), "cpuRequest: 100m", "cpuRequest: 2", 1), + "memory request over limit": strings.Replace(validProfileYAML(), "memoryLimit: 1Gi", "memoryLimit: 64Mi", 1), + } + for name, content := range cases { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "profiles.yaml") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadProfiles(path); err == nil { + t.Fatal("invalid quantity accepted") + } + }) + } +} + +func TestLoadProfilesAcceptsQuantityUnitVariants(t *testing.T) { + content := validProfileYAML() + content = strings.Replace(content, "storageSize: 2Gi", "storageSize: 500M", 1) + content = strings.Replace(content, "cpuRequest: 100m", "cpuRequest: 0.1", 1) + content = strings.Replace(content, `cpuLimit: "1"`, "cpuLimit: 250m", 1) + content = strings.Replace(content, "memoryRequest: 128Mi", "memoryRequest: 1M", 1) + content = strings.Replace(content, "memoryLimit: 1Gi", "memoryLimit: 2M", 1) + path := filepath.Join(t.TempDir(), "profiles.yaml") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + profiles, err := LoadProfiles(path) + if err != nil { + t.Fatal(err) + } + profile, ok := profiles.get("go") + if !ok || profile.CPURequest.String() != "100m" || profile.MemoryRequest.String() != "1M" || profile.StorageSize.String() != "500M" { + t.Fatalf("parsed quantities=%+v", profile) + } +} + +func validProfileYAML() string { + return `profiles: + go: + image: ghcr.io/example/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 2Gi + cpuRequest: 100m + memoryRequest: 128Mi + cpuLimit: "1" + memoryLimit: 1Gi + maxFileBytes: 1048576 + maxCommandBytes: 65536 + maxCommandDuration: 1m +` +} diff --git a/internal/adapter/executioncontroller/store.go b/internal/adapter/executioncontroller/store.go new file mode 100644 index 0000000000..44738fe95e --- /dev/null +++ b/internal/adapter/executioncontroller/store.go @@ -0,0 +1,401 @@ +package executioncontroller + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "sort" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// ExecutionEnvironmentGVR identifies the private execution-environment CRD. +var ExecutionEnvironmentGVR = schema.GroupVersionResource{Group: "execution.mecatl.dev", Version: "v1alpha1", Resource: "executionenvironments"} + +const ( + maxReferences = 64 + environmentNotFoundMessage = "environment not found" +) + +// ExecutorTransport dispatches one request to a fixed workload helper. +type ExecutorTransport interface { + Execute(context.Context, string, executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) +} + +// Store persists authorization and fencing state in ExecutionEnvironment status. +type Store struct { + resources dynamic.ResourceInterface + profiles *Profiles + executor ExecutorTransport +} + +// NewStore constructs a namespace-scoped execution state store. +func NewStore(client dynamic.Interface, namespace string, profiles *Profiles, executor ExecutorTransport) *Store { + return &Store{resources: client.Resource(ExecutionEnvironmentGVR).Namespace(namespace), profiles: profiles, executor: executor} +} + +// ValidateProfile returns one validated immutable profile. +func (s *Store) ValidateProfile(_ context.Context, name string) (Profile, error) { + p, ok := s.profiles.get(name) + if !ok { + return Profile{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "profile not found"} + } + return Profile{Name: name, Digest: p.Digest, MaxFileBytes: p.Spec.MaxFileBytes, MaxCommandBytes: p.Spec.MaxCommandBytes, MaxCommandDuration: p.Spec.MaxCommandDuration, Capabilities: []string{"filesystem", "foreground-command"}}, nil +} + +// Ensure idempotently allocates an environment for an immutable request fingerprint. +func (s *Store) Ensure(ctx context.Context, client, owner, binding, profile, fp string) (Allocation, error) { + p, ok := s.profiles.get(profile) + if !ok { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "profile not found"} + } + name := allocationName(client, owner, binding) + cur, err := s.resources.Get(ctx, name, metav1.GetOptions{}) + if err == nil { + if err := s.initializeStatus(ctx, cur, binding); err != nil { + return Allocation{}, err + } + cur, err = s.resources.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return Allocation{}, err + } + return allocationFrom(cur, client, owner, binding, profile, fp) + } + if !apierrors.IsNotFound(err) { + return Allocation{}, fmt.Errorf("get execution environment: %w", err) + } + revision, err := randomID() + if err != nil { + return Allocation{}, err + } + obj := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": name, "finalizers": []any{environmentFinalizer}}, "spec": map[string]any{"allocationID": name, "revision": revision, "ownerHash": owner, "clientHash": hashText(client), "bindingID": binding, "requestFingerprint": fp, "profile": profile, "profileDigest": p.Digest, "image": p.Spec.Image, "storageClass": p.Spec.StorageClass, "storageSize": p.Spec.StorageSize, "resources": map[string]any{"cpuRequest": p.Spec.CPURequest, "memoryRequest": p.Spec.MemoryRequest, "cpuLimit": p.Spec.CPULimit, "memoryLimit": p.Spec.MemoryLimit}, "desired": "Active"}}} + created, err := s.resources.Create(ctx, obj, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + created, err = s.resources.Get(ctx, name, metav1.GetOptions{}) + } + if err != nil { + return Allocation{}, fmt.Errorf("create execution environment: %w", err) + } + if err := s.initializeStatus(ctx, created, binding); err != nil { + return Allocation{}, err + } + created, err = s.resources.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return Allocation{}, err + } + return allocationFrom(created, client, owner, binding, profile, fp) +} +func (s *Store) initializeStatus(ctx context.Context, env *unstructured.Unstructured, binding string) error { + if intNested(env.Object, "status", "epoch") > 0 { + return nil + } + return s.retryUpdateStatus(ctx, env.GetName(), func(o *unstructured.Unstructured) error { + if intNested(o.Object, "status", "epoch") > 0 { + return nil + } + if err := unstructured.SetNestedField(o.Object, int64(1), "status", "epoch"); err != nil { + return err + } + if err := unstructured.SetNestedStringSlice(o.Object, []string{binding}, "status", "references"); err != nil { + return err + } + return unstructured.SetNestedField(o.Object, fenceHealthy, "status", "fenceState") + }) +} +func allocationName(client, owner, binding string) string { + sum := sha256.Sum256([]byte(client + "\x00" + owner + "\x00" + binding)) + return "exec-" + hex.EncodeToString(sum[:20]) +} +func allocationFrom(o *unstructured.Unstructured, client, owner, binding, profile, fp string) (Allocation, error) { + spec, _, _ := unstructured.NestedMap(o.Object, "spec") + if text(spec, "ownerHash") != owner || text(spec, "clientHash") != hashText(client) || text(spec, "bindingID") != binding || text(spec, "profile") != profile || text(spec, "requestFingerprint") != fp { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "allocation identity conflicts with existing environment"} + } + return exactAllocation(o, client, owner, binding) +} +func exactAllocation(o *unstructured.Unstructured, client, owner, binding string) (Allocation, error) { + spec, _, _ := unstructured.NestedMap(o.Object, "spec") + if text(spec, "ownerHash") != owner || text(spec, "clientHash") != hashText(client) { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + revision := text(spec, "revision") + if revision == "" { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment identity is incomplete"} + } + epoch, ok := epochValue(o) + if !ok { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment epoch is invalid"} + } + ready := conditionTrue(o, "Ready") && textNested(o.Object, "status", "fenceState") == fenceHealthy + return Allocation{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: revision}, Epoch: epoch, OwnerHash: owner, BindingID: binding, Client: client, Ready: ready}, nil +} + +// Attach adds a binding reference only to the exact healthy environment. +func (s *Store) Attach(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) (Allocation, error) { + var out Allocation + err := s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + a, err := exactAllocation(o, client, owner, binding) + if err != nil { + return err + } + if a.Environment != ref || textNested(o.Object, "spec", "desired") != "Active" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment revision unavailable"} + } + if !a.Ready { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment is not ready at requested epoch"} + } + refs, _, _ := unstructured.NestedStringSlice(o.Object, "status", "references") + if !contains(refs, binding) { + if len(refs) >= maxReferences { + return &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "environment reference limit reached"} + } + refs = append(refs, binding) + sort.Strings(refs) + if err := unstructured.SetNestedStringSlice(o.Object, refs, "status", "references"); err != nil { + return err + } + } + out = a + return nil + }) + return out, err +} + +// ReleaseReference removes one binding reference without deleting the environment. +func (s *Store) ReleaseReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + a, err := exactAllocation(o, client, owner, binding) + if err != nil { + return err + } + if a.Environment != ref { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, _, _ := unstructured.NestedStringSlice(o.Object, "status", "references") + next := refs[:0] + for _, v := range refs { + if v != binding { + next = append(next, v) + } + } + return unstructured.SetNestedStringSlice(o.Object, next, "status", "references") + }) +} + +// Retire requests controlled retirement of an unreferenced healthy environment. +func (s *Store) Retire(ctx context.Context, ref executionenv.EnvironmentRef, owner string) error { + cur, err := s.resources.Get(ctx, ref.ID, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if err != nil { + return err + } + if textNested(cur.Object, "spec", "ownerHash") != owner || textNested(cur.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, _, _ := unstructured.NestedStringSlice(cur.Object, "status", "references") + if len(refs) != 0 || textNested(cur.Object, "status", "activeOperation", "id") != "" || textNested(cur.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is referenced or not quiescent"} + } + if err := unstructured.SetNestedField(cur.Object, "Retiring", "spec", "desired"); err != nil { + return err + } + _, err = s.resources.Update(ctx, cur, metav1.UpdateOptions{}) + if apierrors.IsConflict(err) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment changed concurrently", Retryable: true} + } + return err +} + +// File performs one claimed and fenced filesystem operation. +func (s *Store) File(ctx context.Context, client, owner string, q executionenv.FileRequest) (executionenv.FileResponse, error) { + req := executionenv.ExecutorRequest{Operation: q.Operation, Path: q.Path, Destination: q.Destination, Pattern: q.Pattern, Data: q.Data, Version: q.Version, Limit: q.Limit} + resp, err := s.execute(ctx, client, owner, q.Context, req) + return resp.FileResponse, err +} + +// StartCommand performs one claimed foreground command through terminal receipt. +func (s *Store) StartCommand(ctx context.Context, client, owner string, q executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + id, err := randomID() + if err != nil { + return executionenv.CommandStartResponse{}, err + } + resp, err := s.execute(ctx, client, owner, q.Context, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, Command: q.Command, CommandID: id, TimeoutMillis: q.TimeoutMillis}) + if err != nil { + return executionenv.CommandStartResponse{}, err + } + if resp.Command == nil { + return executionenv.CommandStartResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "executor returned no terminal command receipt"} + } + return executionenv.CommandStartResponse{CommandID: id, State: resp.Command.State, Result: *resp.Command}, nil +} + +// CommandStatus reports that foreground-only commands have no detached status API. +func (*Store) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "foreground commands have no resumable status stream"} +} + +// CancelCommand refuses cancellation without an attached foreground control stream. +func (*Store) CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "no active foreground stream is attached; external fencing is required"} +} +func (s *Store) execute(ctx context.Context, client, owner string, rc executionenv.RequestContext, req executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { //nolint:gocyclo // Atomic claim, dispatch, and fencing remain one auditable transaction. + if s.executor == nil { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeNotReady, Message: "executor transport unavailable", Retryable: true} + } + opID, err := randomID() + if err != nil { + return executionenv.ExecutorResponse{}, err + } + var pod string + var maxFileBytes, maxCommandBytes int64 + err = s.retryUpdateStatus(ctx, rc.Environment.ID, func(o *unstructured.Unstructured) error { + profile, ok := s.profiles.get(textNested(o.Object, "spec", "profile")) + if !ok || profile.Digest != textNested(o.Object, "spec", "profileDigest") { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment profile is unavailable"} + } + maxFileBytes, maxCommandBytes = profile.Spec.MaxFileBytes, profile.Spec.MaxCommandBytes + if int64(len(req.Data)) > maxFileBytes || int64(len(req.Command)) > maxCommandBytes { + return &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "operation exceeds profile bounds"} + } + if req.Operation == executionenv.OpCommandStart { + maxMillis := profile.Spec.MaxCommandDuration.Milliseconds() + if req.TimeoutMillis <= 0 { + req.TimeoutMillis = maxMillis + } else if req.TimeoutMillis > maxMillis { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "command timeout exceeds profile bound"} + } + } + refs, _, _ := unstructured.NestedStringSlice(o.Object, "status", "references") + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || !contains(refs, rc.BindingID) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if textNested(o.Object, "spec", "revision") != rc.Environment.Revision || !epochMatches(o, rc.Epoch) || textNested(o.Object, "spec", "desired") != "Active" || !conditionTrue(o, "Ready") || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment is not ready at requested epoch"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment already has an active operation", Retryable: true} + } + pod = textNested(o.Object, "status", "pod", "name") + if pod == "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "executor pod is unavailable", Retryable: true} + } + return unstructured.SetNestedMap(o.Object, map[string]any{"id": opID, "operation": string(req.Operation), "startedAt": time.Now().UTC().Format(time.RFC3339Nano)}, "status", "activeOperation") + }) + if err != nil { + return executionenv.ExecutorResponse{}, err + } + current, err := s.resources.Get(ctx, rc.Environment.ID, metav1.GetOptions{}) + if err != nil || textNested(current.Object, "spec", "ownerHash") != owner || textNested(current.Object, "spec", "clientHash") != hashText(client) || textNested(current.Object, "spec", "revision") != rc.Environment.Revision || !epochMatches(current, rc.Epoch) || textNested(current.Object, "status", "activeOperation", "id") != opID { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation identity changed before executor dispatch"} + } + resp, dispatchErr := s.executor.Execute(ctx, pod, req) + if dispatchErr == nil && (int64(len(resp.Data)) > maxFileBytes || resp.Command != nil && int64(len(resp.Command.Stdout)+len(resp.Command.Stderr)) > maxCommandBytes) { + dispatchErr = &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "executor response exceeds profile bound"} + } + var controlledErr *executionenv.Error + controlledTerminal := errors.As(dispatchErr, &controlledErr) && controlledErr.Code != executionenv.CodeFenceUnknown + terminal := controlledTerminal || dispatchErr == nil && (req.Operation != executionenv.OpCommandStart || resp.Command != nil && resp.Command.TerminalReceipt != "") + finishErr := s.retryUpdateStatus(context.WithoutCancel(ctx), rc.Environment.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "activeOperation", "id") != opID { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation ownership changed before completion"} + } + if terminal { + unstructured.RemoveNestedField(o.Object, "status", "activeOperation") + return nil + } + unstructured.RemoveNestedField(o.Object, "status", "activeOperation") + return unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") + }) + if finishErr != nil { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "could not persist terminal operation state"} + } + if controlledTerminal { + return executionenv.ExecutorResponse{}, controlledErr + } + if !terminal { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "executor termination is unconfirmed; environment fenced", Retryable: false} + } + return resp, nil +} +func (s *Store) retryUpdateStatus(ctx context.Context, name string, mutate func(*unstructured.Unstructured) error) error { + for i := 0; i < 5; i++ { + cur, err := s.resources.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if err != nil { + return err + } + if err := mutate(cur); err != nil { + return err + } + if _, err = s.resources.UpdateStatus(ctx, cur, metav1.UpdateOptions{}); err == nil { + return nil + } else if !apierrors.IsConflict(err) { + return err + } + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment changed concurrently", Retryable: true} +} +func epochValue(o *unstructured.Unstructured) (uint64, bool) { + epoch, found, err := unstructured.NestedInt64(o.Object, "status", "epoch") + if err != nil || !found || epoch <= 0 { + return 0, false + } + return uint64(epoch), true +} + +func epochMatches(o *unstructured.Unstructured, expected uint64) bool { + epoch, ok := epochValue(o) + return ok && epoch == expected +} + +func conditionTrue(o *unstructured.Unstructured, name string) bool { + conds, _, _ := unstructured.NestedSlice(o.Object, "status", "conditions") + for _, raw := range conds { + m, ok := raw.(map[string]any) + if ok && text(m, "type") == name && text(m, "status") == "True" { + return true + } + } + return false +} +func text(m map[string]any, k string) string { v, _ := m[k].(string); return v } +func textNested(m map[string]any, p ...string) string { + v, _, _ := unstructured.NestedString(m, p...) + return v +} +func intNested(m map[string]any, p ...string) int64 { + v, _, _ := unstructured.NestedInt64(m, p...) + return v +} +func hashText(v string) string { sum := sha256.Sum256([]byte(v)); return hex.EncodeToString(sum[:]) } +func randomID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} +func contains(v []string, s string) bool { + for _, x := range v { + if x == s { + return true + } + } + return false +} + +var _ Backend = (*Store)(nil) diff --git a/internal/adapter/executioncontroller/store_test.go b/internal/adapter/executioncontroller/store_test.go new file mode 100644 index 0000000000..08a8c8602b --- /dev/null +++ b/internal/adapter/executioncontroller/store_test.go @@ -0,0 +1,117 @@ +package executioncontroller + +import ( + "context" + "errors" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestStoreEnsureUsesStableLookupAndRejectsFingerprintDrift(t *testing.T) { + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + profiles := &Profiles{byName: map[string]resolvedProfile{"go": {Digest: "sha256:profile", Spec: ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute}}, "other": {Digest: "sha256:other", Spec: ProfileSpec{Image: "example@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute}}}} + s := NewStore(client, "ns", profiles, nil) + ctx := context.Background() + a, err := s.Ensure(ctx, "spiffe://client", "owner", "binding", "go", "fp1") + if err != nil { + t.Fatal(err) + } + b, err := s.Ensure(ctx, "spiffe://client", "owner", "binding", "go", "fp1") + if err != nil { + t.Fatal(err) + } + if a.Environment != b.Environment || a.Epoch != b.Epoch { + t.Fatalf("identity changed: %+v %+v", a, b) + } + if _, err := s.Ensure(ctx, "spiffe://client", "owner", "binding", "other", "fp2"); err == nil { + t.Fatal("profile drift adopted existing allocation") + } +} +func TestAllocationNameExcludesMutableProfile(t *testing.T) { + a := allocationName("client", "owner", "binding") + b := allocationName("client", "owner", "binding") + if a != b { + t.Fatal("unstable allocation name") + } +} + +type terminalErrorExecutor struct{ called int } + +type blockingExecutor struct { + started chan struct{} + release chan struct{} +} + +func (e *blockingExecutor) Execute(context.Context, string, executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + close(e.started) + <-e.release + return executionenv.ExecutorResponse{}, nil +} + +func (e *terminalErrorExecutor) Execute(context.Context, string, executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + e.called++ + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "path not found"} +} +func TestCancelledStoreOperationStaysActiveUntilBackendStopsThenFences(t *testing.T) { + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"epoch": int64(1), "fenceState": "Healthy", "references": []any{"binding"}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + exec := &blockingExecutor{started: make(chan struct{}), release: make(chan struct{})} + s := NewStore(client, "ns", testProfiles(), exec) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := s.StartCommand(ctx, "client", "owner", executionenv.CommandStartRequest{Context: executionenv.RequestContext{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", Epoch: 1}, Command: "sleep 10"}) + done <- err + }() + <-exec.started + cancel() + active, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(active.Object, "status", "activeOperation", "id") == "" { + t.Fatal("active operation was released while executor was still live") + } + close(exec.release) + var pe *executionenv.Error + if err := <-done; !errors.As(err, &pe) || pe.Code != executionenv.CodeFenceUnknown { + t.Fatalf("completion error=%v", err) + } + fenced, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(fenced.Object, "status", "activeOperation", "id") != "" || textNested(fenced.Object, "status", "fenceState") != "FenceUnknown" { + t.Fatalf("status=%v", fenced.Object["status"]) + } + _, err = s.Attach(context.Background(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "binding") + if !errors.As(err, &pe) || pe.Code != executionenv.CodeNotReady { + t.Fatalf("attach after fencing error=%v", err) + } +} + +func TestControlledExecutorErrorClearsOperationWithoutFencing(t *testing.T) { + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"epoch": int64(1), "fenceState": "Healthy", "references": []any{"binding"}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + exec := &terminalErrorExecutor{} + s := NewStore(client, "ns", testProfiles(), exec) + _, err := s.File(context.Background(), "client", "owner", executionenv.FileRequest{Context: executionenv.RequestContext{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", Epoch: 1}, Operation: executionenv.OpFileRead, Path: "missing"}) + var pe *executionenv.Error + if !errors.As(err, &pe) || pe.Code != executionenv.CodeNotFound { + t.Fatalf("error=%v", err) + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if exec.called != 1 || textNested(got.Object, "status", "activeOperation", "id") != "" || textNested(got.Object, "status", "fenceState") != "Healthy" { + t.Fatalf("status=%v calls=%d", got.Object["status"], exec.called) + } +} diff --git a/internal/adapter/server/acp_placement.go b/internal/adapter/server/acp_placement.go index 5c95b60bac..7ae85573f3 100644 --- a/internal/adapter/server/acp_placement.go +++ b/internal/adapter/server/acp_placement.go @@ -59,7 +59,7 @@ func (s *Service) LoadACPSession(ctx context.Context, id session.SessionID, cwd if !persisted.EnvironmentRef.Valid() { return nil, fmt.Errorf("%w: session has no exact placement", ErrFailedPrecondition) } - binding, err := s.ReattachPlacement(ctx, persisted.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, persisted.EnvironmentRef, persisted.ID) if err != nil { return nil, err } diff --git a/internal/adapter/server/classification.go b/internal/adapter/server/classification.go index 91f30b0287..4798c8fb2a 100644 --- a/internal/adapter/server/classification.go +++ b/internal/adapter/server/classification.go @@ -250,44 +250,45 @@ func requireCallerOwnedContext(surface string, t reflect.Type, table map[string] // TestInvariant_owned_access_is_classified fails, naming the method. var serviceAccessTable = map[string]ClassificationEntry{ // --- caller-owned: sessions --- - "CreateSession": {KindCallerOwned, "binds the verified context principal as owner atomically with visibility (reserveSessionID)"}, - "CreateSessionWithProvider": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, - "CreateSessionWithProfile": {KindCallerOwned, "atomic owner bind at creation (reserveSessionID); ForkSession/carryover sources are authorized via authorizeSession before copying history"}, - "CreateSessionWithMCP": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, - "CreateACPSession": {KindCallerOwned, "binds the trusted composition default before treating ACP cwd solely as an equality assertion"}, - "LoadACPSession": {KindCallerOwned, "owner-authorizes and reattaches the exact persisted placement before checking ACP cwd or mounting client MCP"}, - "BindPlacement": {KindCallerOwned, "passes the verified context principal and trusted composition scope to the provider's single atomic authorization-and-resolution operation"}, - "ReattachPlacement": {KindCallerOwned, "passes the exact persisted ref, verified context principal, and trusted scope to the provider without falling back to Bind"}, - "ReattachPlacementInScope": {KindCallerOwned, "requires the durable schedule scope to equal trusted composition scope, then reauthorizes the exact persisted ref as the verified owner without default fallback"}, - "ListCommandsForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and command discovery"}, - "ListWorktreesForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and scoped selector issuance"}, - "ClearSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing an empty-history placed successor"}, - "ForkSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing a history-carrying placed successor"}, - "GetSession": {KindCallerOwned, "authorizeSession: owner mismatch or absence both return ErrNotFound"}, - "WithAuthorizedSession": {KindCallerOwned, "ownership preflight excludes foreign lock contention; authoritative reload under runEntryMu precedes the caller-owned effect"}, - "GetTranscript": {KindCallerOwned, "one SessionStore.Load followed by authorizeSession; no run-entry side effects"}, - "LoadSession": {KindCallerOwned, "authorizeSession before rehydration"}, - "LoadSessionWithMCP": {KindCallerOwned, "delegates to LoadSession's authorizeSession before mounting client MCP"}, - "SetMode": {KindCallerOwned, "authorizes via GetSession before changing the session's permission mode"}, - "RenameSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before persisting"}, - "CompactSession": {KindCallerOwned, "binds the verified caller, then revalidates ownership, chat purpose, state, liveness, and lease under runEntryMu before saving and appending events"}, - "DeleteSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before physical deletion"}, - "EndSession": {KindCallerOwned, "authorizes via GetSession before CloseSession"}, - "ListSessions": {KindCallerOwned, "filters to the caller's own rows before any pagination/count is computed"}, - "ListSessionPage": {KindCallerOwned, "passes caller ownership into the store query before keyset page formation and counting"}, - "StorageHealth": {KindCallerOwned, "gates on the trusted-context management authorizer, NOT caller ownership — the aggregate it reads is store-wide (every session), never scoped to the caller's own rows; see the AccessKind doc comment note on management-authority boundaries"}, - "PlanSessionMigration": {KindCallerOwned, "gates on management authorization over the ENTIRE store, not the caller's own sessions; only the returned generation handle is bound to the verified caller for later Apply/Resume/Cancel binding"}, - "ApplySessionMigration": {KindCallerOwned, "gates on management authorization over the entire store; only the created job is caller-bound, so a foreign-caller job lookup is denied identically to a missing job"}, - "ResumeSessionMigration": {KindCallerOwned, "gates on management authorization; the same caller-bound job-handle check applies before processing another bounded batch — the underlying migration data remains store-wide, never caller-owned"}, - "CancelSessionMigration": {KindCallerOwned, "gates on management authorization; the same caller-bound job-handle check applies before stopping future items — the underlying migration data remains store-wide, never caller-owned"}, - "SessionMigrationJob": {KindCallerOwned, "gates on management authorization and conceals missing and cross-caller job handles identically; the underlying migration data is store-wide, not the caller's own sessions"}, - "PlanSessionCleanup": {KindCallerOwned, "gates on management authority, NOT caller ownership — the metadata pager plans over the ENTIRE store (owner scope is nil); the decision resolved here is 'is this caller a management principal', never per-session ownership"}, - "ApplySessionCleanup": {KindCallerOwned, "gates on management authority over the entire store; only the confirmation token/plan is bound to the verified caller, so a stolen or foreign token is rejected before any deletion"}, - "CancelSessionCleanup": {KindCallerOwned, "gates on management authority; matches the verified principal against the bounded job registry — the underlying cleanup scope is store-wide, never caller-owned"}, - "SessionCleanupJob": {KindCallerOwned, "gates on management authority and returns only a caller-bound sanitized job projection; the underlying cleanup data is store-wide, not the caller's own sessions"}, - "StreamSessionEvents": {KindCallerOwned, "event log/live stream resolves through the owning session's authorizeSession check"}, - "WatchSessionEvents": {KindCallerOwned, "durable replay-then-follow watch (ADR 0250); watchLog resolves ownership through the same GetSession check StreamSessionEvents uses, EAGERLY — before any envelope is yielded — because the durable log holds the whole transcript"}, - "Subscribe": {KindCallerOwned, "authorizes via GetSession before registering a live subscriber (issue #368)"}, + "CreateSession": {KindCallerOwned, "binds the verified context principal as owner atomically with visibility (reserveSessionID)"}, + "CreateSessionWithProvider": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, + "CreateSessionWithProfile": {KindCallerOwned, "atomic owner bind at creation (reserveSessionID); ForkSession/carryover sources are authorized via authorizeSession before copying history"}, + "CreateSessionWithMCP": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, + "CreateACPSession": {KindCallerOwned, "binds the trusted composition default before treating ACP cwd solely as an equality assertion"}, + "LoadACPSession": {KindCallerOwned, "owner-authorizes and reattaches the exact persisted placement before checking ACP cwd or mounting client MCP"}, + "BindPlacement": {KindCallerOwned, "passes the verified context principal and trusted composition scope to the provider's single atomic authorization-and-resolution operation"}, + "ReattachPlacement": {KindCallerOwned, "passes the exact persisted ref, verified context principal, and trusted scope to the provider without falling back to Bind"}, + "ReattachPlacementForBinding": {KindCallerOwned, "passes the exact persisted ref and session binding with the verified context principal to the provider"}, + "ReattachPlacementInScope": {KindCallerOwned, "requires the durable schedule scope to equal trusted composition scope, then reauthorizes the exact persisted ref as the verified owner without default fallback"}, + "ListCommandsForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and command discovery"}, + "ListWorktreesForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and scoped selector issuance"}, + "ClearSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing an empty-history placed successor"}, + "ForkSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing a history-carrying placed successor"}, + "GetSession": {KindCallerOwned, "authorizeSession: owner mismatch or absence both return ErrNotFound"}, + "WithAuthorizedSession": {KindCallerOwned, "ownership preflight excludes foreign lock contention; authoritative reload under runEntryMu precedes the caller-owned effect"}, + "GetTranscript": {KindCallerOwned, "one SessionStore.Load followed by authorizeSession; no run-entry side effects"}, + "LoadSession": {KindCallerOwned, "authorizeSession before rehydration"}, + "LoadSessionWithMCP": {KindCallerOwned, "delegates to LoadSession's authorizeSession before mounting client MCP"}, + "SetMode": {KindCallerOwned, "authorizes via GetSession before changing the session's permission mode"}, + "RenameSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before persisting"}, + "CompactSession": {KindCallerOwned, "binds the verified caller, then revalidates ownership, chat purpose, state, liveness, and lease under runEntryMu before saving and appending events"}, + "DeleteSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before physical deletion"}, + "EndSession": {KindCallerOwned, "authorizes via GetSession before CloseSession"}, + "ListSessions": {KindCallerOwned, "filters to the caller's own rows before any pagination/count is computed"}, + "ListSessionPage": {KindCallerOwned, "passes caller ownership into the store query before keyset page formation and counting"}, + "StorageHealth": {KindCallerOwned, "gates on the trusted-context management authorizer, NOT caller ownership — the aggregate it reads is store-wide (every session), never scoped to the caller's own rows; see the AccessKind doc comment note on management-authority boundaries"}, + "PlanSessionMigration": {KindCallerOwned, "gates on management authorization over the ENTIRE store, not the caller's own sessions; only the returned generation handle is bound to the verified caller for later Apply/Resume/Cancel binding"}, + "ApplySessionMigration": {KindCallerOwned, "gates on management authorization over the entire store; only the created job is caller-bound, so a foreign-caller job lookup is denied identically to a missing job"}, + "ResumeSessionMigration": {KindCallerOwned, "gates on management authorization; the same caller-bound job-handle check applies before processing another bounded batch — the underlying migration data remains store-wide, never caller-owned"}, + "CancelSessionMigration": {KindCallerOwned, "gates on management authorization; the same caller-bound job-handle check applies before stopping future items — the underlying migration data remains store-wide, never caller-owned"}, + "SessionMigrationJob": {KindCallerOwned, "gates on management authorization and conceals missing and cross-caller job handles identically; the underlying migration data is store-wide, not the caller's own sessions"}, + "PlanSessionCleanup": {KindCallerOwned, "gates on management authority, NOT caller ownership — the metadata pager plans over the ENTIRE store (owner scope is nil); the decision resolved here is 'is this caller a management principal', never per-session ownership"}, + "ApplySessionCleanup": {KindCallerOwned, "gates on management authority over the entire store; only the confirmation token/plan is bound to the verified caller, so a stolen or foreign token is rejected before any deletion"}, + "CancelSessionCleanup": {KindCallerOwned, "gates on management authority; matches the verified principal against the bounded job registry — the underlying cleanup scope is store-wide, never caller-owned"}, + "SessionCleanupJob": {KindCallerOwned, "gates on management authority and returns only a caller-bound sanitized job projection; the underlying cleanup data is store-wide, not the caller's own sessions"}, + "StreamSessionEvents": {KindCallerOwned, "event log/live stream resolves through the owning session's authorizeSession check"}, + "WatchSessionEvents": {KindCallerOwned, "durable replay-then-follow watch (ADR 0250); watchLog resolves ownership through the same GetSession check StreamSessionEvents uses, EAGERLY — before any envelope is yielded — because the durable log holds the whole transcript"}, + "Subscribe": {KindCallerOwned, "authorizes via GetSession before registering a live subscriber (issue #368)"}, // --- caller-owned: live run verbs --- "RetryFailedRun": {KindCallerOwned, "authorizes and reloads under runEntryMu before failed-step retry eligibility and launch"}, diff --git a/internal/adapter/server/learning.go b/internal/adapter/server/learning.go index 0118f42746..bc9fa4f2a5 100644 --- a/internal/adapter/server/learning.go +++ b/internal/adapter/server/learning.go @@ -82,7 +82,7 @@ func (s *Service) ReflectSession(ctx context.Context, id session.SessionID) (*me return nil, fmt.Errorf("%w: reflection requires a completed session", ErrFailedPrecondition) } if s.placementBinder != nil { - binding, bindErr := s.ReattachPlacement(ctx, sess.EnvironmentRef) + binding, bindErr := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if bindErr != nil { return nil, bindErr } @@ -570,7 +570,7 @@ func (s *Service) learningWorkspace(ctx context.Context, sess *session.Session) if s.placementBinder == nil { return "", true } - binding, err := s.ReattachPlacement(ctx, sess.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if err != nil { return "", false } diff --git a/internal/adapter/server/local_session_context.go b/internal/adapter/server/local_session_context.go index 37b98e24ba..6ef0b8ecf0 100644 --- a/internal/adapter/server/local_session_context.go +++ b/internal/adapter/server/local_session_context.go @@ -47,7 +47,7 @@ func (s *Service) localSessionContextRoot(ctx context.Context, id session.Sessio if persisted.EnvironmentRef.Kind != session.EnvKindLocal { return "", ErrFailedPrecondition } - binding, err := s.ReattachPlacement(ctx, persisted.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, persisted.EnvironmentRef, persisted.ID) if err != nil { return "", err } diff --git a/internal/adapter/server/placement.go b/internal/adapter/server/placement.go index cc8ccb81ba..fec39dbe51 100644 --- a/internal/adapter/server/placement.go +++ b/internal/adapter/server/placement.go @@ -103,6 +103,9 @@ type PlacementBindRequest struct { Principal *session.Principal Scope PlacementScope Operation PlacementOperation + // BindingID is the final server-minted session identity. Providers that + // allocate durable environments use it as their idempotency/reference key. + BindingID session.SessionID } // PlacementMetadata is the bounded, display-safe provider projection returned @@ -159,6 +162,14 @@ type PlacementReattachRequest struct { Ref session.EnvironmentRef Principal *session.Principal Scope PlacementScope + // BindingID identifies the durable reference being reattached. + BindingID session.SessionID +} + +// PlacementValidator performs side-effect-free startup validation. Providers +// whose Bind allocates resources implement this seam so preflight never binds. +type PlacementValidator interface { + ValidatePlacement(context.Context) error } // PlacementBinder is the server-owned choke point around one deployment @@ -241,10 +252,14 @@ func configuredPlacementBinder(ctx context.Context, cfg Config) (*PlacementBinde if err != nil { return nil, err } - // NewService runs before a listener can serve. Binding the configured - // default proves that its current record is authorized, available, - // revision-stable, and capable of constructing a complete environment. The - // result is deliberately not cached. + if validator, ok := cfg.PlacementProvider.(PlacementValidator); ok { + if err := validator.ValidatePlacement(ctx); err != nil { + return nil, fmt.Errorf("server: validate default placement: %w", sanitizePlacementProviderError(err)) + } + return binder, nil + } + // Legacy providers validate by binding the configured default. Providers whose + // Bind allocates must implement PlacementValidator above. validation, err := binder.Bind(ctx, PlacementBindRequest{ Selector: DefaultPlacement(), Scope: cfg.PlacementScope, Operation: PlacementOperationCreate, @@ -258,14 +273,14 @@ func configuredPlacementBinder(ctx context.Context, cfg Config) (*PlacementBinde return binder, nil } -func (s *Service) bindPlacementForCreate(ctx context.Context, profile SessionProfile, owner *session.Principal) (string, *PlacementBinding, error) { +func (s *Service) bindPlacementForCreate(ctx context.Context, profile SessionProfile, owner *session.Principal, bindingID session.SessionID) (string, *PlacementBinding, error) { selector := DefaultPlacement() if profile == ProfileNoFS { selector = NoFSPlacement() } binding, err := s.placementBinder.Bind(ctx, PlacementBindRequest{ Selector: selector, Principal: owner, Scope: s.cfg.PlacementScope, - Operation: PlacementOperationCreate, + Operation: PlacementOperationCreate, BindingID: bindingID, }) if err != nil { return "", nil, err @@ -318,7 +333,7 @@ func (s *Service) privateWorkspace(ctx context.Context, sess *session.Session) ( if ok && env.Ref() == sess.EnvironmentRef && env.Workspace() != nil { return env.Workspace().Root(), nil } - binding, err := s.ReattachPlacement(ctx, sess.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if err != nil { return "", err } diff --git a/internal/adapter/server/placement_discovery.go b/internal/adapter/server/placement_discovery.go index 3dbc4cb752..b9949cdfec 100644 --- a/internal/adapter/server/placement_discovery.go +++ b/internal/adapter/server/placement_discovery.go @@ -37,7 +37,7 @@ func (s *Service) ownedSessionEnvironment(ctx context.Context, id session.Sessio if sess.EnvironmentRef.Kind == session.EnvKindNoFS { return sess, tool.Environment{}, nil } - binding, err := s.ReattachPlacement(ctx, sess.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if err != nil { return nil, tool.Environment{}, err } diff --git a/internal/adapter/server/placement_successor.go b/internal/adapter/server/placement_successor.go index 5aca1d5883..81744044a2 100644 --- a/internal/adapter/server/placement_successor.go +++ b/internal/adapter/server/placement_successor.go @@ -293,7 +293,7 @@ func successorProviderSelector(source *session.Session, req ForkSuccessorRequest func (s *Service) successorPlacement(ctx context.Context, source *session.Session, requested SuccessorPlacement) (PlacementBinding, error) { if requested.Selector == "" { - return s.ReattachPlacement(ctx, source.EnvironmentRef) + return s.ReattachPlacementForBinding(ctx, source.EnvironmentRef, source.ID) } if source.EnvironmentRef.Kind == session.EnvKindNoFS { return PlacementBinding{}, ErrPlacementNotFound diff --git a/internal/adapter/server/sdk_typescript_release_test.go b/internal/adapter/server/sdk_typescript_release_test.go index 2aa455a359..d5b47268bf 100644 --- a/internal/adapter/server/sdk_typescript_release_test.go +++ b/internal/adapter/server/sdk_typescript_release_test.go @@ -236,6 +236,7 @@ func TestSDKTypescriptRelease_Scenario1_PublicServiceProjectionParity(t *testing "ProviderStatuses", "PublishSessionEvent", "ReattachPlacement", + "ReattachPlacementForBinding", "ReattachPlacementInScope", "ReconcileLeaseLossTombstone", "RecoverNotice", diff --git a/internal/adapter/server/service.go b/internal/adapter/server/service.go index 0b62f31cec..b3a7b34730 100644 --- a/internal/adapter/server/service.go +++ b/internal/adapter/server/service.go @@ -1553,11 +1553,21 @@ func (s *Service) BindPlacement(ctx context.Context, selector PlacementSelector, // ReattachPlacement authorizes and resolves the exact persisted environment // identity. It never invokes Bind and therefore cannot follow a changed default. func (s *Service) ReattachPlacement(ctx context.Context, ref session.EnvironmentRef) (PlacementBinding, error) { + return s.reattachPlacement(ctx, ref, "") +} + +// ReattachPlacementForBinding supplies the durable session reference identity +// required by allocating remote providers. +func (s *Service) ReattachPlacementForBinding(ctx context.Context, ref session.EnvironmentRef, bindingID session.SessionID) (PlacementBinding, error) { + return s.reattachPlacement(ctx, ref, bindingID) +} + +func (s *Service) reattachPlacement(ctx context.Context, ref session.EnvironmentRef, bindingID session.SessionID) (PlacementBinding, error) { if s == nil || s.placementBinder == nil { return PlacementBinding{}, fmt.Errorf("%w: no PlacementProvider is configured", ErrFailedPrecondition) } binding, err := s.placementBinder.Reattach(ctx, PlacementReattachRequest{ - Ref: ref, Principal: session.PrincipalFromContext(ctx), Scope: s.cfg.PlacementScope, + Ref: ref, Principal: session.PrincipalFromContext(ctx), Scope: s.cfg.PlacementScope, BindingID: bindingID, }) if err != nil { s.logPlacementProviderError(ctx, "reattach", err) @@ -2095,53 +2105,61 @@ func (s *Service) classifyCreateWinner(existing *session.Session, owner *session return existing, nil } -// reserveSessionID validates a caller-chosen session id (WithSessionID, ADR 0059 -// decision #7 Phase-2) against THREE collision sources and reserves it for the -// duration of the create, returning a release func the caller MUST defer: -// -// 1. a LIVE per-session engine (sessionEngines — a collision would shadow an -// in-flight session); -// 2. an in-flight create holding the id (reservedIDs — closes the old TOCTOU: -// the prior check released s.mu before the slow factory call and the separate -// registration, so two concurrent creates on the same id both passed); -// 3. a PERSISTED session already in the store (a completed prior create is NOT -// in sessionEngines — e.g. the shared-engine fast path never registers there). -// -// The in-memory reservation (1)+(2) is taken under a single s.mu hold; the store -// probe (3) runs after (no I/O under the mutex). On a collision or an infra probe -// fault the reservation is released before returning the error. Once the session -// is registered (per-session) or persisted (shared) the durable collision sources -// take over, so the reservation only needs to live for the create. -func (s *Service) reserveSessionID(ctx context.Context, id session.SessionID, owner *session.Principal, request createRequest) (existing *session.Session, release func(), err error) { +func (s *Service) reserveCreateID(ctx context.Context, id session.SessionID, owner *session.Principal) (*session.Session, func(), error) { s.mu.Lock() _, liveEngine := s.sessionEngines[id] _, reserved := s.reservedIDs[id] if liveEngine || reserved { s.mu.Unlock() - // Accurate for BOTH cases: a live per-session engine (liveEngine) OR a - // concurrent in-flight create holding the id (reserved). return nil, nil, fmt.Errorf("%w: session id %q is already in use", ErrInvalidArgument, id) } s.reservedIDs[id] = struct{}{} s.mu.Unlock() - release = func() { + release := func() { s.mu.Lock() delete(s.reservedIDs, id) s.mu.Unlock() } - // Probe the store for a persisted session under this id. A not-found error - // means the id is clear; any other infrastructure fault fails closed and is - // exposed only through a content-free public category. - if existing, lerr := s.cfg.Store.Load(ctx, id); lerr == nil && existing != nil { - release() - winner, classifyErr := s.classifyCreateWinner(existing, owner, request) - return winner, nil, classifyErr - } else if lerr != nil && !errors.Is(lerr, port.ErrSessionNotFound) { + existing, err := s.cfg.Store.Load(ctx, id) + if errors.Is(err, port.ErrSessionNotFound) { + return nil, release, nil + } + if err != nil { release() - s.logDiscoveryError(ctx, "probe session placement", lerr) return nil, nil, fmt.Errorf("%w: placement storage failed", ErrInternal) } - return nil, release, nil + sameOwner := existing != nil && (existing.Owner == nil && owner == nil || existing.Owner.SameIdentity(owner)) + if !sameOwner { + release() + return nil, nil, fmt.Errorf("%w: %q", ErrNotFound, id) + } + return existing, release, nil +} + +func (s *Service) reserveGeneratedSessionID(ctx context.Context, id session.SessionID) (func(), error) { + s.mu.Lock() + _, liveEngine := s.sessionEngines[id] + _, reserved := s.reservedIDs[id] + if liveEngine || reserved { + s.mu.Unlock() + return nil, fmt.Errorf("%w: session id %q is already in use", ErrInvalidArgument, id) + } + s.reservedIDs[id] = struct{}{} + s.mu.Unlock() + release := func() { + s.mu.Lock() + delete(s.reservedIDs, id) + s.mu.Unlock() + } + if existing, err := s.cfg.Store.Load(ctx, id); err == nil && existing != nil { + release() + return nil, port.ErrSessionAlreadyExists + } else if err != nil && !errors.Is(err, port.ErrSessionNotFound) { + release() + s.logDiscoveryError(ctx, "probe generated session placement", err) + return nil, fmt.Errorf("%w: placement storage failed", ErrInternal) + } + return release, nil } func (s *Service) persistNewSession(ctx context.Context, sess *session.Session) error { @@ -2260,6 +2278,27 @@ func (s *Service) createSession(ctx context.Context, mode session.PermissionMode // The owner stamped on the new session: the explicit WithOwner injection, else // the verified principal on the context, else nil (the ownerless no-auth path). owner := resolveOwner(ctx, opts) + // Durable placement allocation is keyed by the final session id. Mint it once + // before Bind; the id is never replaced after a remote allocation succeeds. + finalID := opts.id + generatedID := !opts.idSet + if opts.idSet && finalID == "" { + return nil, fmt.Errorf("%w: session id must not be empty", ErrInvalidArgument) + } + if !opts.idSet { + finalID = s.cfg.NewID() + } + var existingCreate *session.Session + var releaseCreate func() + if generatedID { + releaseCreate, err = s.reserveGeneratedSessionID(ctx, finalID) + } else { + existingCreate, releaseCreate, err = s.reserveCreateID(ctx, finalID, owner) + } + if err != nil { + return nil, err + } + defer releaseCreate() var placement *PlacementBinding if opts.placement != nil { if err := validatePlacementBinding(*opts.placement); err != nil { @@ -2271,7 +2310,7 @@ func (s *Service) createSession(ctx context.Context, mode session.PermissionMode return nil, ErrInvalidPlacementBinding } } else { - workspace, placement, err = s.bindPlacementForCreate(ctx, profile, owner) + workspace, placement, err = s.bindPlacementForCreate(ctx, profile, owner, finalID) if err != nil { return nil, err } @@ -2286,57 +2325,17 @@ func (s *Service) createSession(ctx context.Context, mode session.PermissionMode return nil, err } - // Resolve the session id: the caller's override (WithSessionID, ADR 0059 - // decision #7 Phase-2) wins; otherwise the Service's NewID generator mints a - // fresh one (the byte-identical pre-Phase-2 path). WithSessionID with an EMPTY - // id is rejected (the doc promises it), distinguished from "never called" by - // idSet. A caller-chosen id is validated + reserved by reserveSessionID (see - // its doc for the three collision sources); the reservation is released on - // EVERY exit path. + // Reserve the final id for the rest of creation. Placement allocation above is + // idempotent on this same key, so a retry cannot allocate a second environment. + request := newCreateRequest(placement.Ref, mode, limits, sel, profile, opts.sourceSessionID, opts) var retryRequest *createRequest - mintID := s.cfg.NewID - if opts.idSet { - if opts.id == "" { - return nil, fmt.Errorf("%w: session id must not be empty", ErrInvalidArgument) - } - request := newCreateRequest(placement.Ref, mode, limits, sel, profile, opts.sourceSessionID, opts) + if !generatedID { retryRequest = &request - existing, release, err := s.reserveSessionID(ctx, opts.id, owner, request) - if err != nil { - return nil, err - } - if existing != nil { - return existing, nil - } - defer release() - mintID = func() session.SessionID { return opts.id } - } - // A broker attachment is keyed by the canonical persisted identity. Mint and - // reserve generated IDs before any attachment or catalogue construction. - if s.cfg.MCPBroker != nil && !opts.idSet { - id := mintID() - request := newCreateRequest(placement.Ref, mode, limits, sel, profile, opts.sourceSessionID, opts) - // Populate the outer retryRequest too (not just the local var used for - // reserveSessionID above): persistCreatedSession's collision-retry path - // (resolveCreateCollision) needs a non-nil *createRequest to classify an - // idempotent-retry winner on this generated-id branch, exactly as the - // opts.idSet branch above already does. Before this fix, retryRequest - // stayed nil here (the "request" identifier above is a fresh local, not - // the outer var), so resolveCreateCollision's request==nil guard always - // short-circuited and a genuine ErrSessionAlreadyExists from persistNewSession - // always hard-failed instead of resolving to the existing winner. - retryRequest = &request - existing, release, reserveErr := s.reserveSessionID(ctx, id, owner, request) - if reserveErr != nil { - return nil, reserveErr - } - if existing != nil { - release() - return nil, fmt.Errorf("%w: generated session id %q already exists", ErrInvalidArgument, id) + if existingCreate != nil { + return s.classifyCreateWinner(existingCreate, owner, request) } - defer release() - mintID = func() session.SessionID { return id } } + mintID := func() session.SessionID { return finalID } // Issue #20 (model-switch context carryover): when a source session is // named, validate it (turn-boundary) and snapshot its conversation ONCE @@ -5446,7 +5445,7 @@ func (s *Service) engineAndEnvironmentFor(ctx context.Context, sess *session.Ses if !sess.EnvironmentRef.Valid() { return nil, tool.Environment{}, ErrInvalidPlacementSelection } - verified, err := s.ReattachPlacement(ctx, sess.EnvironmentRef) + verified, err := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if err != nil { return nil, tool.Environment{}, err } diff --git a/internal/app/build.go b/internal/app/build.go index ae170d437d..18b8d6a405 100644 --- a/internal/app/build.go +++ b/internal/app/build.go @@ -144,6 +144,9 @@ type Config struct { // PlacementScope is the trusted authorization scope passed to the provider. // Empty defaults to the process deployment scope. PlacementScope server.PlacementScope + // RemoteExecution selects the remote-environment catalog attenuation and + // model-visible posture. It is set only by an explicit composition root. + RemoteExecution bool // ClientMCPOnCreate permits client-provided MCP servers on a session-creating // API request (issue #821, ADR 0237 applied to outbound MCP). It is a // deployment policy the cmd/ main decides from its listener topology and Build passes through verbatim; the zero value fails @@ -2155,18 +2158,21 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { sessionReadLedger = redisBackend.ReadLedger } worktreeLister := buildWorktreeLister(cfg) + selectorIssuer, err := server.NewWorktreeSelectorIssuer(placementSelectorKey[:]) + if err != nil { + return nil, fmt.Errorf("initialize placement selector issuer: %w", err) + } + localProvider := &localPlacementProvider{ + scope: placementScope, root: cfg.Workspace, workspace: workspaceFactory, + runnerForRoot: func(root string) tool.CommandRunner { + return buildCommandRunnerForRoot(cfg, root) + }, + worktrees: worktreeLister, selectors: selectorIssuer, + } if placementProvider == nil { - selectorIssuer, err := server.NewWorktreeSelectorIssuer(placementSelectorKey[:]) - if err != nil { - return nil, fmt.Errorf("initialize placement selector issuer: %w", err) - } - placementProvider = &localPlacementProvider{ - scope: placementScope, root: cfg.Workspace, workspace: workspaceFactory, - runnerForRoot: func(root string) tool.CommandRunner { - return buildCommandRunnerForRoot(cfg, root) - }, - worktrees: worktreeLister, selectors: selectorIssuer, - } + placementProvider = localProvider + } else if cfg.RemoteExecution { + placementProvider = &profilePlacementProvider{remote: placementProvider, local: localProvider} } attempts := startAttemptRecovery(ctx, cfg, reg, store, eventLog, assets.attemptRepository, assets.reflectionRepository, assets, placementProvider, placementScope) if attempts != nil { @@ -3010,6 +3016,19 @@ func sessionEngineFactory( } } +func remoteSessionConfiguration(cfg Config, profile server.SessionProfile, workspace string, instructions prompt.InstructionAssembler, policy port.PermissionPolicy) (string, bool, prompt.InstructionAssembler, port.PermissionPolicy) { + if !cfg.RemoteExecution || profile == server.ProfileNoFS { + return workspace, false, instructions, policy + } + if variants, ok := instructions.(sessionInstructionSet); ok { + instructions = variants.remote + } + if variants, ok := policy.(sessionPermissionPolicies); ok { + policy = variants.remote + } + return "", true, instructions, policy +} + func sessionEngineFactoryWithTools( cfg Config, reg *providerRegistry, @@ -3034,7 +3053,8 @@ func sessionEngineFactoryWithTools( // is scoped to this one session's assembly; the SHARED engine keeps the // build-time pin over cfg.Workspace (the server's own root — the one // the shared engine was assembled for). - cfg.childPermResolver = childPermResolverFor(cfg, workspace) + projectWorkspace, remote, sessionInstructions, sessionPolicy := remoteSessionConfiguration(cfg, profile, workspace, instructions, policy) + cfg.childPermResolver = childPermResolverFor(cfg, projectWorkspace) // The NO-FS profile (issue #55): the service routes every no-fs session // through this factory unconditionally (the shared engine has the FS tools // baked in), and the profile selects the no-FS catalog assembly + the @@ -3203,7 +3223,7 @@ func sessionEngineFactoryWithTools( // Caller-scoped lazy hydration: rebuild this authenticated session's global // and admitted project generations from durable state before binding the // Skill tool. A failed authoritative read clears only these partitions. - skillPartitions := hydrateLearnedSkillPartitions(ctx, cfg, assets, workspace) + skillPartitions := hydrateLearnedSkillPartitions(ctx, cfg, assets, projectWorkspace) cat, closeFn, clientToolNames := assembleCatalog(ctx, cfg, reg, store, hooks, &assets, catalogSession{ provider: resolvedProvider, @@ -3212,6 +3232,7 @@ func sessionEngineFactoryWithTools( clientMgr: mgr, narrate: false, noFS: noFS, + remote: remote, mode: mode, skillPartitions: skillPartitions, sessionTools: sessionTools, @@ -3223,13 +3244,14 @@ func sessionEngineFactoryWithTools( // the shared wiring, so no collaborator is silently dropped and a non-default // provider never contaminates compaction/counting. learningCfg := cfg - learningCfg.Workspace = workspace - learningCfg.LearningMode, learningCfg.LearningSensitivity, learningCfg.SkillActivationPolicy = learningPolicyForWorkspace(cfg, workspace) + learningCfg.Workspace = projectWorkspace + learningCfg.LearningMode, learningCfg.LearningSensitivity, learningCfg.SkillActivationPolicy = learningPolicyForWorkspace(cfg, projectWorkspace) learningCfg.Model = resolvedModel learningCfg.attemptRepository = assets.attemptRepository learningCfg.automaticAdmissionLedger = assets.automaticAdmissionLedger learningCfg.learningSourceStore = store - deps := engineDepsForProvider(cfg, resolvedProvider, resolvedModel, windowFn, store, policy, hooks, mcpProvider, instructions) + deps := engineDepsForProvider(cfg, resolvedProvider, resolvedModel, windowFn, store, sessionPolicy, hooks, mcpProvider, sessionInstructions) + deps.PromptConfig = applyRemoteExecutionPosture(deps.PromptConfig, remote) attachOperatorProfile(&deps, assets.userModelStore) deps.LearningMode = learningCfg.LearningMode deps.LearningObserver = bindMaterializationLifecycle(buildReflectionObserver(learningCfg, resolvedProvider, learningCfg.Model, assets.userModelStore, assets.memStore, assets.reflectionRepository, assets.reflectionCoordinator, assets.learningAdmission, buildProcedureProcessor(learningCfg, assets)), assets.reflectionLifecycle) @@ -4106,7 +4128,10 @@ func buildEngine(ctx context.Context, cfg Config, reg *providerRegistry, provide // the tier-0 project MemoryIndexAssembler. Operator facts no longer ride a // turn-0 user fragment; the same user store is loaded per request into the // volatile system-prompt suffix below. - instructions := buildInstructionAssembler(rulesSrc, soulSrc, memStore, userModelStore, !projectIngestionAdmitted(cfg)) + instructions := sessionInstructionSet{ + standard: buildInstructionAssembler(rulesSrc, soulSrc, memStore, userModelStore, !projectIngestionAdmitted(cfg)), + remote: buildInstructionAssembler(nil, soulSrc, memStore, userModelStore, true), + } // Guardrails decorate the ordinary hook chain. Completion learning has its own // synchronous engine seam and no longer shares Stop-hook ownership. @@ -4140,7 +4165,11 @@ func buildEngine(ctx context.Context, cfg Config, reg *providerRegistry, provide // recursion guard and operator-tier-only config carry over); the option is a // no-op at any non-auto posture or with no checker, so yolo/strict/trusted // and the un-knobbed auto stay byte-identical. - sharedPolicy := escapePolicyForConfig(cfg, reg, provider, policy) + sharedPolicy := sessionPermissionPolicies{ + standard: escapePolicyForConfig(cfg, reg, provider, policy), + remote: escapePolicyForConfig(cfg, reg, provider, + permpolicy.NewPolicyWithResolver(mainRules(cfg), learned, nil, mainEvaluatorOptions(cfg)...)), + } var learningAdmission *learningAdmission if cfg.operatorLearningMode != learning.Off { learningAdmission = newLearningAdmission(cfg.UserModelReviewInterval) @@ -4219,6 +4248,27 @@ func attachOperatorProfile(deps *agent.Deps, store tool.MemoryStore) { } } +type sessionPermissionPolicies struct { + standard port.PermissionPolicy + remote port.PermissionPolicy +} + +func (p sessionPermissionPolicies) Evaluate(ctx context.Context, id session.SessionID, mode session.PermissionMode, call session.ToolCall, ws tool.WorkspaceReader) governance.PermissionDecision { + return p.standard.Evaluate(ctx, id, mode, call, ws) +} +func (p sessionPermissionPolicies) Learn(id session.SessionID, call session.ToolCall) { + p.standard.Learn(id, call) +} + +type sessionInstructionSet struct { + standard prompt.InstructionAssembler + remote prompt.InstructionAssembler +} + +func (s sessionInstructionSet) Assemble(ctx context.Context, ws tool.Workspace) ([]session.Message, error) { + return s.standard.Assemble(ctx, ws) +} + // buildInstructionAssembler composes the ephemeral turn-0 instruction fragments: // RootAssembler (project instructions), rules, soul, then the project memory // index. userModelStore remains in the internal signature to keep existing @@ -5355,7 +5405,7 @@ func compactionDecision(cfg Config) diagFact { // here with the composition-resolved provider (or the not-configured sentinel), // NOT as a zero-value tools.All()/NoFS() entry — an only-when-configured tool that // vanishes would be the silent-disable this harness avoids. -func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS bool, searchProvider tool.SearchProvider) { +func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS, remote bool, searchProvider tool.SearchProvider) { if noFS { for _, t := range tools.NoFS() { cat.MustRegister(t) @@ -5367,7 +5417,8 @@ func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS bool, searchProv cat.MustRegister(t) } cat.MustRegister(tools.NewWebSearchTool(searchProvider)) - if runner := buildCommandRunner(cfg); runner != nil { + runner := buildCommandRunner(cfg) + if runner != nil || remote { // The AGENT-loop Shell tool (not the fstools one): foreground byte-identical, // plus the `background: true` detach over the run's child registry. Its // companion ShellStatus — the SOLE status/collect/cancel channel for those @@ -5377,7 +5428,11 @@ func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS bool, searchProv cat.MustRegister(agent.NewShellTool()) cat.MustRegister(agent.NewShellStatusTool()) if log { - cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool ENABLED", "shell", cfg.Shell, "cwd", cfg.Workspace) + if remote { + cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool ENABLED", "shell", "remote execution") + } else { + cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool ENABLED", "shell", cfg.Shell, "cwd", cfg.Workspace) + } } } else if log { cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool DISABLED (shell-less mode): the agent has no command execution", @@ -5687,6 +5742,7 @@ func buildCatalog(ctx context.Context, cfg Config, reg *providerRegistry, provid provider: provider, providerID: reg.Default(), model: cfg.Model, + remote: cfg.RemoteExecution, narrate: true, }) // Aggregate the build-time Subagent per-def INLINE MCP managers' teardown into @@ -8176,6 +8232,20 @@ func applyRedisWorkspacePosture(pc prompt.Config, enabled bool) prompt.Config { const workspaceRootForPrompt = "/workspace" +const remoteExecutionPostureNote = "This session uses a persistent remote Kubernetes workspace. Use the filesystem tools and foreground Shell for work in /workspace. Local project instructions, rules, project-scoped skills, commands, schedules, SkillDraft, Parallel, Team, and Subagent delegation are unavailable; operator-global skills, MCP, memory, and web tools remain available. Never assume harness-local files are part of this workspace." + +func applyRemoteExecutionPosture(pc prompt.Config, enabled bool) prompt.Config { + if !enabled { + return pc + } + pc.Env.Cwd, pc.Env.Shell, pc.Env.GitStatus = workspaceRootForPrompt, "remote foreground shell", "" + if pc.Role == "" { + pc.Role = prompt.DefaultRole() + } + pc.Role += "\n\n" + remoteExecutionPostureNote + return pc +} + func applyDebugSessionPosture(pc prompt.Config, target session.SessionID, selectedServers []string) prompt.Config { if pc.Role == "" { pc.Role = prompt.DefaultRole() diff --git a/internal/app/catalog.go b/internal/app/catalog.go index 3d407ff7dd..f11b5833ae 100644 --- a/internal/app/catalog.go +++ b/internal/app/catalog.go @@ -167,7 +167,8 @@ type catalogSession struct { // EXACTLY as in the default profile — guarded by TestNoFSCatalogProfile, // which pins the EXACT name-set delta. Always false for the build-time shared // catalog (a process always has a default-profile shared engine). - noFS bool + noFS bool + remote bool // mode is the session's permission mode (the per-session factory passes the // session's resolved mode; the build-time shared catalog leaves it // ModeDefault). The Schedule registration reads it to register the @@ -214,7 +215,7 @@ func assembleCatalog(ctx context.Context, cfg Config, reg *providerRegistry, sto classified := newClassifiedCatalog() cat := classified.catalog classified.captureEach(coreToolClassification, func() { - registerCoreTools(cfg, cat, s.narrate, s.noFS, a.searchProvider) + registerCoreTools(cfg, cat, s.narrate, s.noFS, s.remote, a.searchProvider) }) for _, sessionTool := range s.sessionTools { // A direct global manager retains ownership of its existing query wrapper. @@ -285,27 +286,31 @@ func assembleCatalog(ctx context.Context, cfg Config, reg *providerRegistry, sto refMgr = s.clientMgr } var subagentClose func() error - classified.captureEach(delegationToolClassification, func() { - subagentClose = registerSubagentTrio(ctx, cfg, cat, reg, store, hooks, *a, s, refMgr) - }) - // Parallel is ABSENT under the no-FS profile (not merely disarmed): every - // branch is a force-copy filesystem fork and the deliverable is a preserved - // fork PATH — both meaningless without a filesystem. - if !s.noFS { + if !s.remote { + classified.captureEach(delegationToolClassification, func() { + subagentClose = registerSubagentTrio(ctx, cfg, cat, reg, store, hooks, *a, s, refMgr) + }) + } + // Parallel is ABSENT under no-FS and remote execution profiles. + if !s.noFS && !s.remote { classified.captureEach(delegationToolClassification, func() { registerParallelTool(ctx, cfg, cat, reg, store, hooks, *a, s) }) } - classified.captureEach(delegationToolClassification, func() { - registerTeamTools(ctx, cfg, cat, reg, store, *a, s, refMgr) - }) + if !s.remote { + classified.captureEach(delegationToolClassification, func() { + registerTeamTools(ctx, cfg, cat, reg, store, *a, s, refMgr) + }) + } classified.capture(server.ClassificationEntry{Kind: server.KindCallerOwned, Rationale: "memory tools resolve the verified caller through the caller-partitioned store"}, func() { registerMemoryFamilies(ctx, cfg, cat, *a) }) - classified.captureEach(scheduleToolClassification, func() { - registerScheduleTool(ctx, cfg, cat, a, s) - }) + if !s.remote { + classified.captureEach(scheduleToolClassification, func() { + registerScheduleTool(ctx, cfg, cat, a, s) + }) + } classified.captureEach(func(t tool.Tool) (server.ClassificationEntry, bool) { return skillToolClassification(t, *a, s) }, func() { @@ -646,7 +651,7 @@ func registerSkillFamily(ctx context.Context, cfg Config, cat *tool.Catalog, a c cfg.diag().Log(ctx, port.LevelWarn, "registering skills failed; Skill tool disabled", "err", err) } } - if !s.noFS { + if !s.noFS && !s.remote { if a.learnedSkills != nil && cfg.SkillsDraftDir != "" { inventory := make([]learning.SkillInventoryItem, 0, len(a.skills)) for _, meta := range a.skills { diff --git a/internal/app/placement.go b/internal/app/placement.go index fbecac9187..c538458083 100644 --- a/internal/app/placement.go +++ b/internal/app/placement.go @@ -18,6 +18,34 @@ const ( defaultPlacementScope = server.PlacementScope("deployment") ) +type profilePlacementProvider struct { + remote server.PlacementProvider + local *localPlacementProvider +} + +func (p *profilePlacementProvider) ValidatePlacement(ctx context.Context) error { + if validator, ok := p.remote.(server.PlacementValidator); ok { + return validator.ValidatePlacement(ctx) + } + return nil +} +func (p *profilePlacementProvider) Bind(ctx context.Context, req server.PlacementBindRequest) (server.PlacementBinding, error) { + if req.Selector.IsNoFS() { + return p.local.Bind(ctx, req) + } + return p.remote.Bind(ctx, req) +} +func (p *profilePlacementProvider) Reattach(ctx context.Context, req server.PlacementReattachRequest) (server.PlacementBinding, error) { + if req.Ref.Kind == session.EnvKindNoFS { + return p.local.Reattach(ctx, req) + } + remote, ok := p.remote.(server.PlacementReattacher) + if !ok { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return remote.Reattach(ctx, req) +} + // localPlacementProvider is the trusted composition default. It owns exactly // one configured local record plus the no-FS attenuation; it has no inventory // registry or path-derived public identifier. diff --git a/internal/app/remote_execution_test.go b/internal/app/remote_execution_test.go new file mode 100644 index 0000000000..aeb39b2c2f --- /dev/null +++ b/internal/app/remote_execution_test.go @@ -0,0 +1,142 @@ +package app + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/stacklok/mecatl/engine/adapter/memfs" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +type remoteFactoryPlacement struct { + env tool.Environment + binds *int +} + +func (remoteFactoryPlacement) ValidatePlacement(context.Context) error { return nil } +func (p remoteFactoryPlacement) Bind(_ context.Context, req server.PlacementBindRequest) (server.PlacementBinding, error) { + if p.binds != nil { + *p.binds++ + } + if req.BindingID == "" || req.Principal == nil { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + return server.PlacementBinding{Environment: p.env, Ref: p.env.Ref(), Metadata: server.PlacementMetadata{Label: "Remote Kubernetes workspace"}}, nil +} +func (p remoteFactoryPlacement) Reattach(_ context.Context, req server.PlacementReattachRequest) (server.PlacementBinding, error) { + if req.BindingID == "" || req.Ref != p.env.Ref() { + return server.PlacementBinding{}, server.ErrPlacementNotFound + } + return server.PlacementBinding{Environment: p.env, Ref: p.env.Ref()}, nil +} + +func TestRemoteDeploymentNoFSUsesLocalAttenuationWithoutProviderCall(t *testing.T) { + binds := 0 + remoteWS := memfs.NewWorkspace("/workspace") + remoteRef := session.EnvironmentRef{Kind: "kubernetes", ID: "env-1", Revision: "rev-1"} + remoteEnv := tool.MustEnvironment(remoteRef, remoteWS, memledger.New(), nil) + var captured port.LLMRequest + provider := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { captured = req })}, + mockllm.ToolCallTurn(session.ToolCall{ID: "read", Name: "Read", Args: json.RawMessage(`{"path":"secret"}`)}), + mockllm.TextTurn("done"), + ) + built, err := buildIsolated(t, context.Background(), Config{MockProvider: provider, PlacementProvider: remoteFactoryPlacement{env: remoteEnv, binds: &binds}, PlacementScope: "remote", RemoteExecution: true, NoSoul: true, SchedulerEnabled: false}) + if err != nil { + t.Fatal(err) + } + defer built.Close() + ctx := session.WithPrincipal(context.Background(), &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser}) + sess, err := built.Service.CreateSessionWithProfile(ctx, session.ModeAccept, session.Limits{}, server.ProviderSelector{}, server.ProfileNoFS) + if err != nil { + t.Fatal(err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "no files") + if err != nil { + t.Fatal(err) + } + for range run.Events() { + } + built.Service.FinishRun(sess.ID, run) + if binds != 0 { + t.Fatalf("remote provider Bind calls = %d, want zero", binds) + } + if !strings.Contains(captured.System.StablePrefix, noFSPostureNote) || strings.Contains(captured.System.StablePrefix, remoteExecutionPostureNote) { + t.Fatalf("no-fs posture was replaced by remote posture: %q", captured.System.StablePrefix) + } + for _, spec := range captured.Tools { + if spec.Name == "Read" || spec.Name == "Shell" { + t.Fatalf("no-fs catalog exposed %q", spec.Name) + } + } +} + +func TestRemoteExecutionRealFactoryCarriesPostureAndAttenuatedCatalog(t *testing.T) { + ws := memfs.NewWorkspace("/workspace") + if _, err := ws.CreateFile(context.Background(), "main.go", []byte("package main\n")); err != nil { + t.Fatal(err) + } + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env-1", Revision: "rev-1"} + env := tool.MustEnvironment(ref, ws, memledger.New(), nil) + var mu sync.Mutex + var captured port.LLMRequest + provider := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { mu.Lock(); captured = req; mu.Unlock() })}, + mockllm.ToolCallTurn(session.ToolCall{ID: "unknown", Name: "NoSuchTool", Args: json.RawMessage(`{}`)}), + mockllm.ToolCallTurn(session.ToolCall{ID: "read", Name: "Read", Args: json.RawMessage(`{"path":"main.go"}`)}), + mockllm.TextTurn(""), + mockllm.TextTurn("done"), + ) + localProject := t.TempDir() + if err := os.WriteFile(filepath.Join(localProject, "AGENTS.md"), []byte("LOCAL_PROJECT_MARKER_DO_NOT_LOAD\n"), 0o600); err != nil { + t.Fatal(err) + } + built, err := buildIsolated(t, context.Background(), Config{MockProvider: provider, PlacementProvider: remoteFactoryPlacement{env: env}, PlacementScope: "remote", RemoteExecution: true, Workspace: localProject, TrustProject: true, NoSoul: true, SchedulerEnabled: false}) + if err != nil { + t.Fatal(err) + } + defer built.Close() + principal := &session.Principal{Issuer: "https://issuer.example", Subject: "alice", GrantType: session.GrantTypeUser} + ctx := session.WithPrincipal(context.Background(), principal) + sess, err := built.Service.CreateSession(ctx, session.ModeAccept, session.Limits{}) + if err != nil { + t.Fatal(err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "read the remote fixture") + if err != nil { + t.Fatal(err) + } + for range run.Events() { + } + built.Service.FinishRun(sess.ID, run) + mu.Lock() + defer mu.Unlock() + if !strings.Contains(captured.System.StablePrefix, remoteExecutionPostureNote) { + t.Fatal("real per-session factory omitted remote execution posture") + } + if strings.Contains(captured.System.Render(), "LOCAL_PROJECT_MARKER_DO_NOT_LOAD") { + t.Fatal("remote session ingested host-local project instructions") + } + names := map[string]bool{} + for _, spec := range captured.Tools { + names[spec.Name] = true + } + for _, name := range []string{"Read", "Write", "WebFetch"} { + if !names[name] { + t.Errorf("remote catalog omitted compatible tool %q", name) + } + } + for _, name := range []string{"Subagent", "SubagentStatus", "InspectSubagent", "Parallel", "Team", "InspectMember", "SkillDraft", "Schedule"} { + if names[name] { + t.Errorf("remote catalog advertised unsupported tool %q", name) + } + } +} diff --git a/internal/executionenv/errors.go b/internal/executionenv/errors.go new file mode 100644 index 0000000000..8a75d95f82 --- /dev/null +++ b/internal/executionenv/errors.go @@ -0,0 +1,117 @@ +package executionenv + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "strings" +) + +// ErrorCode is a stable private-protocol error classification. +type ErrorCode string + +// Private-protocol error codes. +const ( + CodeInvalidArgument ErrorCode = "invalid_argument" + CodeUnauthenticated ErrorCode = "unauthenticated" + CodePermissionDenied ErrorCode = "permission_denied" + CodeNotFound ErrorCode = "not_found" + CodeAlreadyExists ErrorCode = "already_exists" + CodeConflict ErrorCode = "conflict" + CodeVersionMismatch ErrorCode = "version_mismatch" + CodeNotReady ErrorCode = "not_ready" + CodeFenceUnknown ErrorCode = "fence_unknown" + CodeResourceExhausted ErrorCode = "resource_exhausted" + CodeInternal ErrorCode = "internal" +) + +// Error is a bounded structured private-protocol failure. +type Error struct { + Code ErrorCode `json:"code"` + Message string `json:"message"` + Retryable bool `json:"retryable,omitempty"` +} + +func (e *Error) Error() string { return string(e.Code) + ": " + e.Message } + +// ErrorResponse is the bounded wire envelope returned for every failed request. +type ErrorResponse struct { + Error *Error `json:"error"` +} + +// DecodeStrict decodes exactly one bounded JSON value without unknown or duplicate fields. +func DecodeStrict(data []byte, dst any) error { + if len(data) > MaxJSONBody { + return &Error{Code: CodeResourceExhausted, Message: "request body exceeds limit"} + } + if err := rejectDuplicateKeys(data); err != nil { + return &Error{Code: CodeInvalidArgument, Message: "invalid JSON: " + err.Error()} + } + d := json.NewDecoder(strings.NewReader(string(data))) + d.DisallowUnknownFields() + if err := d.Decode(dst); err != nil { + return &Error{Code: CodeInvalidArgument, Message: "invalid JSON: " + err.Error()} + } + if err := d.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return &Error{Code: CodeInvalidArgument, Message: "request must contain exactly one JSON value"} + } + return nil +} + +func rejectDuplicateKeys(data []byte) error { + d := json.NewDecoder(strings.NewReader(string(data))) + if err := walkJSON(d); err != nil { + return err + } + if _, err := d.Token(); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("trailing JSON value") + } + return err + } + return nil +} +func walkJSON(d *json.Decoder) error { + tok, err := d.Token() + if err != nil { + return err + } + delim, ok := tok.(json.Delim) + if !ok { + return nil + } + switch delim { + case '{': + seen := map[string]struct{}{} + for d.More() { + t, err := d.Token() + if err != nil { + return err + } + k, ok := t.(string) + if !ok { + return errors.New("object key is not a string") + } + if _, ok := seen[k]; ok { + return fmt.Errorf("duplicate field %q", k) + } + seen[k] = struct{}{} + if err := walkJSON(d); err != nil { + return err + } + } + _, err = d.Token() + return err + case '[': + for d.More() { + if err := walkJSON(d); err != nil { + return err + } + } + _, err = d.Token() + return err + default: + return errors.New("unexpected delimiter") + } +} diff --git a/internal/executionenv/grant.go b/internal/executionenv/grant.go new file mode 100644 index 0000000000..4480225f88 --- /dev/null +++ b/internal/executionenv/grant.go @@ -0,0 +1,163 @@ +package executionenv + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +const grantAlgorithm = "Ed25519" + +type grantHeader struct { + Algorithm string `json:"alg"` + KeyID string `json:"kid"` + Type string `json:"typ"` +} + +// GrantClaims binds a short-lived capability to one client, owner, environment, and epoch. +type GrantClaims struct { + KeyID string `json:"-"` + Issuer string `json:"iss"` + Audience string `json:"aud"` + Client string `json:"client"` + OwnerHash string `json:"owner"` + BindingID string `json:"binding_id"` + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + Operations []Operation `json:"operations"` + NotBefore time.Time `json:"not_before"` + ExpiresAt time.Time `json:"expires_at"` + Nonce string `json:"nonce"` +} + +// GrantExpectation defines the exact binding and operation required by a request. +type GrantExpectation struct { + Client string + OwnerHash string + BindingID string + Environment EnvironmentRef + Epoch uint64 + Operation Operation +} + +// GrantVerifier verifies signed grants against configured trust and revocation state. +type GrantVerifier struct { + Keys map[string]ed25519.PublicKey + RevokedKeys map[string]struct{} + RevokedNonces map[string]struct{} + Issuer string + Audience string + MaxLifetime time.Duration + Now func() time.Time +} + +// SignGrant validates and signs claims with Ed25519. +func SignGrant(key ed25519.PrivateKey, c GrantClaims) (string, error) { + if len(key) != ed25519.PrivateKeySize { + return "", errors.New("invalid Ed25519 private key") + } + if err := validateClaims(c); err != nil { + return "", err + } + h, _ := json.Marshal(grantHeader{Algorithm: grantAlgorithm, KeyID: c.KeyID, Type: "MECATL-GRANT"}) + p, _ := json.Marshal(c) + unsigned := rawURL(h) + "." + rawURL(p) + sig := ed25519.Sign(key, []byte(unsigned)) + return unsigned + "." + rawURL(sig), nil +} + +// Verify validates a grant and its exact request binding. +func (v GrantVerifier) Verify(token string, e GrantExpectation) (GrantClaims, error) { //nolint:gocyclo // Fail-closed claim validation is intentionally linear. + var c GrantClaims + parts := strings.Split(token, ".") + if len(parts) != 3 { + return c, errors.New("malformed grant") + } + hb, err := decodeURL(parts[0]) + if err != nil { + return c, errors.New("malformed grant header") + } + var h grantHeader + if err := DecodeStrict(hb, &h); err != nil { + return c, errors.New("malformed grant header") + } + if h.Algorithm != grantAlgorithm || h.Type != "MECATL-GRANT" || h.KeyID == "" { + return c, errors.New("unsupported grant header") + } + if _, ok := v.RevokedKeys[h.KeyID]; ok { + return c, errors.New("grant key revoked") + } + key, ok := v.Keys[h.KeyID] + if !ok || len(key) != ed25519.PublicKeySize { + return c, errors.New("unknown grant key") + } + sig, err := decodeURL(parts[2]) + if err != nil || !ed25519.Verify(key, []byte(parts[0]+"."+parts[1]), sig) { + return c, errors.New("invalid grant signature") + } + pb, err := decodeURL(parts[1]) + if err != nil { + return c, errors.New("malformed grant claims") + } + if err := DecodeStrict(pb, &c); err != nil { + return c, errors.New("malformed grant claims") + } + c.KeyID = h.KeyID + if err := validateClaims(c); err != nil { + return c, err + } + now := time.Now().UTC() + if v.Now != nil { + now = v.Now().UTC() + } + if c.Issuer != v.Issuer || c.Audience != v.Audience { + return c, errors.New("grant issuer or audience mismatch") + } + if now.Before(c.NotBefore) || !now.Before(c.ExpiresAt) { + return c, errors.New("grant is not currently valid") + } + maxLifetime := v.MaxLifetime + if maxLifetime <= 0 { + maxLifetime = 5 * time.Minute + } + if c.ExpiresAt.Sub(c.NotBefore) > maxLifetime { + return c, errors.New("grant lifetime exceeds limit") + } + if _, ok := v.RevokedNonces[c.Nonce]; ok { + return c, errors.New("grant revoked") + } + if c.Client != e.Client || c.OwnerHash != e.OwnerHash || c.BindingID != e.BindingID || c.Environment != e.Environment || c.Epoch != e.Epoch { + return c, errors.New("grant binding mismatch") + } + for _, op := range c.Operations { + if op == e.Operation { + return c, nil + } + } + return c, errors.New("grant does not authorize operation") +} +func validateClaims(c GrantClaims) error { + if c.KeyID == "" || c.Issuer == "" || c.Audience == "" || c.Client == "" || c.OwnerHash == "" || c.BindingID == "" || c.Environment.ID == "" || c.Environment.Revision == "" || c.Epoch == 0 || c.Nonce == "" || c.NotBefore.IsZero() || c.ExpiresAt.IsZero() || !c.ExpiresAt.After(c.NotBefore) { + return errors.New("incomplete grant claims") + } + if len(c.Operations) == 0 || len(c.Operations) > 32 { + return errors.New("invalid grant operations") + } + seen := map[Operation]struct{}{} + for _, op := range c.Operations { + if !op.Valid() { + return fmt.Errorf("invalid grant operation %q", op) + } + if _, ok := seen[op]; ok { + return errors.New("duplicate grant operation") + } + seen[op] = struct{}{} + } + return nil +} +func rawURL(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) } +func decodeURL(s string) ([]byte, error) { return base64.RawURLEncoding.Strict().DecodeString(s) } diff --git a/internal/executionenv/protocol_test.go b/internal/executionenv/protocol_test.go new file mode 100644 index 0000000000..e94b6af648 --- /dev/null +++ b/internal/executionenv/protocol_test.go @@ -0,0 +1,67 @@ +package executionenv + +import ( + "crypto/ed25519" + "crypto/rand" + "testing" + "time" +) + +func TestGrantRoundTripAndExactBindings(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + now := time.Unix(1_800_000_000, 0).UTC() + claims := GrantClaims{KeyID: "k1", Issuer: "execution-provider", Audience: "mecatl-execution", Client: "spiffe://cluster/ns/mecak8s", OwnerHash: "owner", BindingID: "binding", Environment: EnvironmentRef{ID: "env", Revision: "rev"}, Epoch: 7, Operations: []Operation{OpFileRead}, NotBefore: now.Add(-time.Second), ExpiresAt: now.Add(time.Minute), Nonce: "n1"} + token, err := SignGrant(priv, claims) + if err != nil { + t.Fatal(err) + } + v := GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: 5 * time.Minute, Now: func() time.Time { return now }} + if _, err := v.Verify(token, GrantExpectation{Client: claims.Client, OwnerHash: "owner", BindingID: claims.BindingID, Environment: claims.Environment, Epoch: 7, Operation: OpFileRead}); err != nil { + t.Fatalf("verify: %v", err) + } + for name, mutate := range map[string]func(*GrantExpectation){ + "client": func(e *GrantExpectation) { e.Client = "spiffe://other" }, + "owner": func(e *GrantExpectation) { e.OwnerHash = "other" }, + "binding": func(e *GrantExpectation) { e.BindingID = "other" }, + "revision": func(e *GrantExpectation) { e.Environment.Revision = "other" }, + "epoch": func(e *GrantExpectation) { e.Epoch++ }, + "operation": func(e *GrantExpectation) { e.Operation = OpFileReplace }, + } { + t.Run(name, func(t *testing.T) { + e := GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: claims.BindingID, Environment: claims.Environment, Epoch: claims.Epoch, Operation: OpFileRead} + mutate(&e) + if _, err := v.Verify(token, e); err == nil { + t.Fatal("expected rejection") + } + }) + } +} + +func TestGrantRejectsExpiryRevocationAndUnknownFields(t *testing.T) { + pub, priv, _ := ed25519.GenerateKey(rand.Reader) + now := time.Unix(1_800_000_000, 0).UTC() + c := GrantClaims{KeyID: "k1", Issuer: "i", Audience: "a", Client: "spiffe://c", OwnerHash: "o", BindingID: "b", Environment: EnvironmentRef{ID: "e", Revision: "r"}, Epoch: 1, Operations: []Operation{OpAttach}, NotBefore: now.Add(-time.Minute), ExpiresAt: now.Add(-time.Second), Nonce: "n"} + tok, _ := SignGrant(priv, c) + v := GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "i", Audience: "a", Now: func() time.Time { return now }} + if _, err := v.Verify(tok, GrantExpectation{Client: c.Client, OwnerHash: c.OwnerHash, BindingID: c.BindingID, Environment: c.Environment, Epoch: 1, Operation: OpAttach}); err == nil { + t.Fatal("expired grant accepted") + } + c.ExpiresAt = now.Add(time.Minute) + tok, _ = SignGrant(priv, c) + v.RevokedNonces = map[string]struct{}{"n": {}} + if _, err := v.Verify(tok, GrantExpectation{Client: c.Client, OwnerHash: c.OwnerHash, BindingID: c.BindingID, Environment: c.Environment, Epoch: 1, Operation: OpAttach}); err == nil { + t.Fatal("revoked nonce accepted") + } +} + +func TestDecodeStrictRejectsUnknownTrailingAndOversize(t *testing.T) { + var req ValidateProfileRequest + for _, body := range []string{`{"profile":"p","extra":true}`, `{"profile":"p"}{}`, `{"profile":"` + string(make([]byte, MaxJSONBody)) + `"}`} { + if err := DecodeStrict([]byte(body), &req); err == nil { + t.Fatalf("accepted invalid body") + } + } +} diff --git a/internal/executionenv/types.go b/internal/executionenv/types.go new file mode 100644 index 0000000000..8708c6d770 --- /dev/null +++ b/internal/executionenv/types.go @@ -0,0 +1,247 @@ +// Package executionenv defines the private, versioned host-side execution-provider protocol. +// It is intentionally not part of the public Harness API. +package executionenv + +import "time" + +// Private protocol bounds. +const ( + BasePath = "/internal/execution/v1" + MaxJSONBody = 8 << 20 + MaxFileBytes = 5 << 20 + MaxCommandBytes = 1 << 20 + MaxPathBytes = 4096 + MaxIdentityBytes = 1024 + MaxBindingBytes = 253 + MaxGrantBytes = 16 << 10 + MaxListEntries = 10_000 +) + +// Operation identifies one provider or workload-helper operation. +type Operation string + +// Supported private-protocol operations. +const ( + OpAttach Operation = "attach" + OpReferenceRelease Operation = "reference.release" + OpRetire Operation = "retire" + OpFileRead Operation = "file.read" + OpFileResolveAuthority Operation = "file.resolve_authority" + OpFileStat Operation = "file.stat" + OpFileCreate Operation = "file.create" + OpFileReplace Operation = "file.replace" + OpFileList Operation = "file.list" + OpFileRemove Operation = "file.remove" + OpFileRename Operation = "file.rename" + OpFileCopy Operation = "file.copy" + OpFileGlob Operation = "file.glob" + OpFileGrep Operation = "file.grep" + OpCommandStart Operation = "command.start" + OpCommandStatus Operation = "command.status" + OpCommandCancel Operation = "command.cancel" + OpCommandStream Operation = "command.stream" +) + +// Valid reports whether the operation belongs to the closed protocol vocabulary. +func (o Operation) Valid() bool { + switch o { + case OpAttach, OpReferenceRelease, OpRetire, OpFileRead, OpFileResolveAuthority, OpFileStat, OpFileCreate, OpFileReplace, OpFileList, OpFileRemove, OpFileRename, OpFileCopy, OpFileGlob, OpFileGrep, OpCommandStart, OpCommandStatus, OpCommandCancel, OpCommandStream: + return true + } + return false +} + +// EnvironmentRef is the provider-private durable environment identity. +type EnvironmentRef struct { + ID string `json:"id"` + Revision string `json:"revision"` +} + +// Owner is an authenticated principal attestation. +type Owner struct { + Issuer string `json:"issuer"` + Subject string `json:"subject"` +} + +// RequestContext carries the exact authorization binding for an operation. +type RequestContext struct { + Environment EnvironmentRef `json:"environment"` + Owner Owner `json:"owner"` + BindingID string `json:"binding_id"` + Epoch uint64 `json:"epoch"` + Grant string `json:"grant"` +} + +// ValidateProfileRequest selects an operator-defined profile for validation. +type ValidateProfileRequest struct { + Profile string `json:"profile"` +} + +// ValidateProfileResponse reports immutable profile capabilities and bounds. +type ValidateProfileResponse struct { + Profile string `json:"profile"` + Digest string `json:"digest"` + Capabilities []string `json:"capabilities"` + MaxFileBytes int64 `json:"max_file_bytes"` + MaxCommandBytes int64 `json:"max_command_bytes"` + MaxCommandDurationMillis int64 `json:"max_command_duration_ms"` +} + +// EnsureEnvironmentRequest requests an idempotent environment allocation. +type EnsureEnvironmentRequest struct { + BindingID string `json:"binding_id"` + Profile string `json:"profile"` + Owner Owner `json:"owner"` +} + +// EnsureEnvironmentResponse returns allocation identity, readiness, and a short-lived grant. +type EnsureEnvironmentResponse struct { + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + Ready bool `json:"ready"` + Grant string `json:"grant"` + GrantExpiresAt time.Time `json:"grant_expires_at"` +} + +// AttachEnvironmentRequest requests exact reattachment to an existing environment. +type AttachEnvironmentRequest struct { + Context RequestContext `json:"context"` + Purpose string `json:"purpose"` +} + +// PurposeSession is the supported attachment purpose. +const PurposeSession = "session" + +// AttachEnvironmentResponse returns exact attachment state and a refreshed grant. +type AttachEnvironmentResponse struct { + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + Ready bool `json:"ready"` + Grant string `json:"grant"` + GrantExpiresAt time.Time `json:"grant_expires_at"` +} + +// ReferenceReleaseRequest releases one durable binding reference. +type ReferenceReleaseRequest struct { + Context RequestContext `json:"context"` +} + +// RetireEnvironmentRequest requests controlled retirement without PVC deletion. +type RetireEnvironmentRequest struct { + Context RequestContext `json:"context"` +} + +// EmptyResponse is the successful response for operations without a payload. +type EmptyResponse struct{} + +// FileRequest carries one bounded, authorized filesystem operation. +type FileRequest struct { + Context RequestContext `json:"context"` + Operation Operation `json:"operation"` + Path string `json:"path"` + Destination string `json:"destination,omitempty"` + Pattern string `json:"pattern,omitempty"` + Data []byte `json:"data,omitempty"` + Version string `json:"version,omitempty"` + Limit int `json:"limit,omitempty"` +} + +// FileInfo is bounded filesystem metadata returned by the helper. +type FileInfo struct { + Name string `json:"name"` + Size int64 `json:"size"` + Mode uint32 `json:"mode"` + ModTime time.Time `json:"mod_time"` + IsDir bool `json:"is_dir"` +} + +// GrepMatch is one bounded textual grep result. +type GrepMatch struct { + Path string `json:"path"` + Line int `json:"line"` + Text string `json:"text"` +} + +// FileResponse carries the result of one filesystem operation. +type FileResponse struct { + Data []byte `json:"data,omitempty"` + Version string `json:"version,omitempty"` + Info *FileInfo `json:"info,omitempty"` + Entries []FileInfo `json:"entries,omitempty"` + Paths []string `json:"paths,omitempty"` + Matches []GrepMatch `json:"matches,omitempty"` + AuthorityTarget string `json:"authority_target,omitempty"` + AuthorityWorkspace string `json:"authority_workspace,omitempty"` +} + +// CommandStartRequest starts one bounded foreground shell command. +type CommandStartRequest struct { + Context RequestContext `json:"context"` + Command string `json:"command"` + TimeoutMillis int64 `json:"timeout_ms,omitempty"` +} + +// CommandStartResponse returns the command identity and terminal result. +type CommandStartResponse struct { + CommandID string `json:"command_id"` + State CommandState `json:"state"` + Result CommandStatusResponse `json:"result"` +} + +// CommandState classifies command lifecycle and fencing outcomes. +type CommandState string + +// Command states. +const ( + CommandRunning CommandState = "running" + CommandSucceeded CommandState = "succeeded" + CommandFailed CommandState = "failed" + CommandCancelled CommandState = "cancelled" + CommandFenceUnknown CommandState = "fence_unknown" +) + +// CommandQueryRequest identifies one command for status or cancellation. +type CommandQueryRequest struct { + Context RequestContext `json:"context"` + CommandID string `json:"command_id"` + Offset int64 `json:"offset,omitempty"` +} + +// CommandStatusResponse carries bounded output and terminal proof. +type CommandStatusResponse struct { + CommandID string `json:"command_id"` + State CommandState `json:"state"` + ExitCode int `json:"exit_code,omitempty"` + Stdout []byte `json:"stdout,omitempty"` + Stderr []byte `json:"stderr,omitempty"` + NextOffset int64 `json:"next_offset,omitempty"` + Truncated bool `json:"truncated,omitempty"` + TerminalReceipt string `json:"terminal_receipt,omitempty"` +} + +// ExecutorRequest is the credential-free provider-to-workload stdin protocol. +// It deliberately cannot select an environment, root, Pod, image, argv, or process environment. +type ExecutorRequest struct { + Operation Operation `json:"operation"` + Path string `json:"path,omitempty"` + Destination string `json:"destination,omitempty"` + Pattern string `json:"pattern,omitempty"` + Data []byte `json:"data,omitempty"` + Version string `json:"version,omitempty"` + Limit int `json:"limit,omitempty"` + Command string `json:"command,omitempty"` + CommandID string `json:"command_id,omitempty"` + TimeoutMillis int64 `json:"timeout_ms,omitempty"` +} + +// ExecutorResponse carries one credential-free helper result. +type ExecutorResponse struct { + FileResponse + Command *CommandStatusResponse `json:"command,omitempty"` +} + +// ExecutorEnvelope provides a terminal, credential-free result over pods/exec. +type ExecutorEnvelope struct { + Response *ExecutorResponse `json:"response,omitempty"` + Error *Error `json:"error,omitempty"` +} diff --git a/internal/executionexecutor/command_linux.go b/internal/executionexecutor/command_linux.go new file mode 100644 index 0000000000..8d431da744 --- /dev/null +++ b/internal/executionexecutor/command_linux.go @@ -0,0 +1,169 @@ +//go:build linux + +package executionexecutor + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +const descendantCleanupTimeout = 2 * time.Second + +var subreaperEnabled atomic.Bool + +type commandResult struct { + stdout []byte + stderr []byte + exitCode int + truncated bool +} + +// EnableCommandExecution makes this process a Linux child subreaper. It must be +// called by the dedicated executor helper before it starts any other process. +func EnableCommandExecution() error { + if err := unix.Prctl(unix.PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0); err != nil { + return fmt.Errorf("enable executor child subreaper: %w", err) + } + subreaperEnabled.Store(true) + return nil +} + +func runIsolatedCommand(ctx context.Context, dir, command string, outputLimit int) (commandResult, bool, error) { + if !subreaperEnabled.Load() { + return commandResult{}, false, errors.New("command execution requires the dedicated subreaper helper") + } + var stdout, stderr boundedWriter + stdout.limit = outputLimit + stderr.limit = outputLimit + cmd := exec.Command("/bin/sh", "-c", command) // #nosec G204 -- the command is the authenticated tool payload. + cmd.Dir = dir + cmd.Env = []string{"HOME=/workspace", "PATH=/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin", "TMPDIR=/tmp", "GOTMPDIR=/tmp"} + cmd.Stdout = &stdout + cmd.Stderr = &stderr + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.WaitDelay = 250 * time.Millisecond + if err := cmd.Start(); err != nil { + return commandResult{}, true, err + } + + waited := make(chan error, 1) + go func() { waited <- cmd.Wait() }() + var waitErr error + select { + case waitErr = <-waited: + case <-ctx.Done(): + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + _ = cmd.Process.Kill() + waitErr = <-waited + } + + // The shell may have exited successfully while background descendants remain. + // Kill its original process group, then repeatedly kill every process reparented + // to this dedicated subreaper. ECHILD is the only clean-completion proof. + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + cleanupCtx, cancel := context.WithTimeout(context.Background(), descendantCleanupTimeout) + defer cancel() + proven := reapAllChildren(cleanupCtx) + + result := commandResult{stdout: stdout.Bytes(), stderr: stderr.Bytes(), exitCode: exitCode(waitErr), truncated: stdout.truncated || stderr.truncated} + if ctx.Err() != nil { + return result, proven, ctx.Err() + } + return result, proven, waitErr +} + +func reapAllChildren(ctx context.Context) bool { + for { + for _, pid := range directChildren() { + _ = syscall.Kill(pid, syscall.SIGKILL) + } + for { + var status syscall.WaitStatus + pid, err := syscall.Wait4(-1, &status, syscall.WNOHANG, nil) + switch { + case errors.Is(err, syscall.ECHILD): + return true + case err != nil: + return false + case pid <= 0: + goto wait + } + } + wait: + select { + case <-ctx.Done(): + return false + case <-time.After(5 * time.Millisecond): + } + } +} + +func directChildren() []int { + tasks, err := os.ReadDir("/proc/self/task") + if err != nil { + return nil + } + seen := make(map[int]struct{}) + for _, task := range tasks { + data, err := os.ReadFile(filepath.Join("/proc/self/task", task.Name(), "children")) + if err != nil { + continue + } + for _, field := range strings.Fields(string(data)) { + pid, err := strconv.Atoi(field) + if err == nil && pid > 0 { + seen[pid] = struct{}{} + } + } + } + out := make([]int, 0, len(seen)) + for pid := range seen { + out = append(out, pid) + } + return out +} + +func exitCode(err error) int { + if err == nil { + return 0 + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode() + } + return -1 +} + +type boundedWriter struct { + buf bytes.Buffer + limit int + truncated bool +} + +func (w *boundedWriter) Write(p []byte) (int, error) { + original := len(p) + if remaining := w.limit - w.buf.Len(); remaining > 0 { + if len(p) > remaining { + p = p[:remaining] + w.truncated = true + } + _, _ = w.buf.Write(p) + } else if original > 0 { + w.truncated = true + } + return original, nil +} + +func (w *boundedWriter) Bytes() []byte { return append([]byte(nil), w.buf.Bytes()...) } diff --git a/internal/executionexecutor/command_unsupported.go b/internal/executionexecutor/command_unsupported.go new file mode 100644 index 0000000000..177168a4fb --- /dev/null +++ b/internal/executionexecutor/command_unsupported.go @@ -0,0 +1,24 @@ +//go:build !linux + +package executionexecutor + +import ( + "context" + "errors" +) + +type commandResult struct { + stdout []byte + stderr []byte + exitCode int + truncated bool +} + +// EnableCommandExecution reports that the workload helper is Linux-only. +func EnableCommandExecution() error { + return errors.New("command execution requires Linux child-subreaper support") +} + +func runIsolatedCommand(context.Context, string, string, int) (commandResult, bool, error) { + return commandResult{}, false, errors.New("command execution requires Linux child-subreaper support") +} diff --git a/internal/executionexecutor/executor.go b/internal/executionexecutor/executor.go new file mode 100644 index 0000000000..f6e13a7aa0 --- /dev/null +++ b/internal/executionexecutor/executor.go @@ -0,0 +1,248 @@ +// Package executionexecutor is the fixed, credential-free workload helper. +package executionexecutor + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io/fs" + "strings" + "time" + "unicode/utf8" + + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/osfs" + "github.com/stacklok/mecatl/internal/executionenv" +) + +// Limits bounds one executor operation. +type Limits struct { + MaxFileBytes int + MaxEntries int + CommandTimeout time.Duration +} + +// Executor serves one credential-free workload-helper process. +type Executor struct { + workspace *osfs.Workspace + limits Limits +} + +// New constructs an executor confined to root. +func New(root string, limits Limits) (*Executor, error) { + if limits.MaxFileBytes <= 0 || limits.MaxFileBytes > executionenv.MaxFileBytes { + limits.MaxFileBytes = executionenv.MaxFileBytes + } + if limits.MaxEntries <= 0 || limits.MaxEntries > executionenv.MaxListEntries { + limits.MaxEntries = executionenv.MaxListEntries + } + if limits.CommandTimeout <= 0 { + limits.CommandTimeout = 30 * time.Second + } + ws, err := osfs.NewWorkspace(root) + if err != nil { + return nil, err + } + return &Executor{workspace: ws, limits: limits}, nil +} + +// Close releases executor resources. +func (*Executor) Close() error { return nil } + +// Execute performs one bounded executor operation. +func (e *Executor) Execute(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { //nolint:gocyclo // The operation switch keeps validation and dispatch in one boundary. + if !q.Operation.Valid() { + return executionenv.ExecutorResponse{}, bad("unsupported executor operation") + } + if len(q.Data) > e.limits.MaxFileBytes || len(q.Path) > executionenv.MaxPathBytes || len(q.Destination) > executionenv.MaxPathBytes || len(q.Pattern) > executionenv.MaxPathBytes || q.Limit < 0 || q.Limit > executionenv.MaxListEntries { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "executor request exceeds configured bounds") + } + switch q.Operation { + case executionenv.OpFileRead: + return e.read(ctx, q) + case executionenv.OpFileResolveAuthority: + target, workspace, err := e.workspace.AuthorityResourcePath(q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{AuthorityTarget: target, AuthorityWorkspace: workspace}}, nil + case executionenv.OpFileStat: + return e.stat(ctx, q) + case executionenv.OpFileCreate: + return e.create(ctx, q) + case executionenv.OpFileReplace: + return e.replace(ctx, q) + case executionenv.OpFileList: + return e.list(ctx, q) + case executionenv.OpFileRemove: + return executionenv.ExecutorResponse{}, mapErr(e.workspace.Remove(ctx, q.Path)) + case executionenv.OpFileRename: + return executionenv.ExecutorResponse{}, mapErr(e.workspace.Rename(ctx, q.Path, q.Destination)) + case executionenv.OpFileCopy: + v, err := e.workspace.CopyFile(ctx, q.Path, q.Destination) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Version: s}}, nil + case executionenv.OpFileGlob: + paths, err := e.workspace.Glob(ctx, q.Pattern) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(paths) > e.resultLimit(q.Limit) { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "glob result exceeds configured limit") + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Paths: paths}}, nil + case executionenv.OpFileGrep: + matches, err := e.workspace.Grep(ctx, q.Pattern, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(matches) > e.resultLimit(q.Limit) { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "grep result exceeds configured limit") + } + out := make([]executionenv.GrepMatch, len(matches)) + for i, m := range matches { + out[i] = executionenv.GrepMatch{Path: m.Path, Line: m.Line, Text: strings.ToValidUTF8(m.Text, "�")} + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Matches: out}}, nil + case executionenv.OpCommandStart: + return e.command(ctx, q) + default: + return executionenv.ExecutorResponse{}, bad("operation is handled by provider, not workload") + } +} +func (e *Executor) read(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + b, v, err := e.workspace.ReadVersion(ctx, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(b) > e.limits.MaxFileBytes { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "file exceeds configured limit") + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Data: b, Version: s}}, nil +} +func (e *Executor) stat(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + i, err := e.workspace.Stat(ctx, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + x := convertInfo(i) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Info: &x}}, nil +} +func (e *Executor) create(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + v, err := e.workspace.CreateFile(ctx, q.Path, q.Data) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Version: s}}, nil +} +func (e *Executor) replace(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + v, err := e.workspace.ReplaceFile(ctx, q.Path, tool.DecodeFileVersion(q.Version), q.Data) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Version: s}}, nil +} +func (e *Executor) list(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + items, err := e.workspace.ReadDir(ctx, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(items) > e.resultLimit(q.Limit) { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "directory exceeds configured limit") + } + out := make([]executionenv.FileInfo, len(items)) + for i, v := range items { + out[i] = convertInfo(v) + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Entries: out}}, nil +} +func (e *Executor) command(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + if q.CommandID == "" || len(q.Command) == 0 || len(q.Command) > executionenv.MaxCommandBytes { + return executionenv.ExecutorResponse{}, bad("command id and bounded command are required") + } + d := e.limits.CommandTimeout + if q.TimeoutMillis > 0 { + requested := time.Duration(q.TimeoutMillis) * time.Millisecond + if requested < d { + d = requested + } + } + runCtx, cancel := context.WithTimeout(ctx, d) + defer cancel() + result, cleanupProven, err := runIsolatedCommand(runCtx, e.workspace.Root(), q.Command, e.limits.MaxFileBytes) + stdout, stderr, truncated := capStreams(result.stdout, result.stderr, e.limits.MaxFileBytes) + truncated = truncated || result.truncated + if !cleanupProven { + return executionenv.ExecutorResponse{Command: &executionenv.CommandStatusResponse{CommandID: q.CommandID, State: executionenv.CommandFenceUnknown, Stdout: stdout, Stderr: stderr, Truncated: truncated}}, nil + } + state := executionenv.CommandSucceeded + if errors.Is(runCtx.Err(), context.Canceled) || errors.Is(runCtx.Err(), context.DeadlineExceeded) { + state = executionenv.CommandCancelled + } else if err != nil || result.exitCode != 0 { + state = executionenv.CommandFailed + } + receiptInput := append(append([]byte(q.CommandID+"\x00"), stdout...), 0) + receiptHash := sha256.Sum256(append(receiptInput, stderr...)) + return executionenv.ExecutorResponse{Command: &executionenv.CommandStatusResponse{CommandID: q.CommandID, State: state, ExitCode: result.exitCode, Stdout: stdout, Stderr: stderr, Truncated: truncated, TerminalReceipt: hex.EncodeToString(receiptHash[:])}}, nil +} +func convertInfo(i tool.FileInfo) executionenv.FileInfo { + return executionenv.FileInfo{Name: i.Name, Size: i.Size, Mode: uint32(i.Mode), ModTime: i.ModTime, IsDir: i.IsDir} +} +func (e *Executor) resultLimit(requested int) int { + if requested > 0 && requested < e.limits.MaxEntries { + return requested + } + return e.limits.MaxEntries +} +func capStreams(stdout, stderr []byte, n int) ([]byte, []byte, bool) { + original := len(stdout) + len(stderr) + stdout = capOutput(stdout, n) + remaining := n - len(stdout) + if remaining < 0 { + remaining = 0 + } + stderr = capOutput(stderr, remaining) + return stdout, stderr, original > len(stdout)+len(stderr) +} + +func capOutput(b []byte, n int) []byte { + b = []byte(strings.ToValidUTF8(string(b), "�")) + if len(b) > n { + b = b[:n] + for !utf8.Valid(b) { + b = b[:len(b)-1] + } + } + return b +} +func bad(msg string) error { return coded(executionenv.CodeInvalidArgument, msg) } +func coded(code executionenv.ErrorCode, msg string) error { + return &executionenv.Error{Code: code, Message: msg} +} +func mapErr(err error) error { + if err == nil { + return nil + } + var mismatch *tool.VersionMismatchError + switch { + case errors.As(err, &mismatch): + return coded(executionenv.CodeVersionMismatch, "file version mismatch") + case errors.Is(err, fs.ErrNotExist): + return coded(executionenv.CodeNotFound, "path not found") + case errors.Is(err, fs.ErrExist): + return coded(executionenv.CodeAlreadyExists, "destination already exists") + case errors.Is(err, tool.ErrDirectoryNotEmpty): + return coded(executionenv.CodeConflict, "directory is not empty") + case errors.Is(err, osfs.ErrPathEscape): + return coded(executionenv.CodePermissionDenied, "path is outside workspace") + default: + return err + } +} diff --git a/internal/executionexecutor/executor_test.go b/internal/executionexecutor/executor_test.go new file mode 100644 index 0000000000..9d68f3765c --- /dev/null +++ b/internal/executionexecutor/executor_test.go @@ -0,0 +1,164 @@ +package executionexecutor + +import ( + "context" + "encoding/json" + "errors" + "io/fs" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestFileOperationsAreConfinedAndVersioned(t *testing.T) { + root := t.TempDir() + x, err := New(root, Limits{MaxFileBytes: 1024, MaxEntries: 100, CommandTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer x.Close() + ctx := context.Background() + created, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: "a/file.txt", Data: []byte("one")}) + if err != nil { + t.Fatal(err) + } + read, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileRead, Path: "a/file.txt"}) + if err != nil || string(read.Data) != "one" || read.Version != created.Version { + t.Fatalf("read=%+v err=%v", read, err) + } + resolved, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileResolveAuthority, Path: "a/file.txt"}) + if err != nil || resolved.AuthorityWorkspace != root || resolved.AuthorityTarget != filepath.Join(root, "a/file.txt") { + t.Fatalf("resolved authority=%+v err=%v", resolved.FileResponse, err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileReplace, Path: "a/file.txt", Data: []byte("two"), Version: "wrong"}) + var pe *executionenv.Error + if !errors.As(err, &pe) || pe.Code != executionenv.CodeVersionMismatch { + t.Fatalf("expected version mismatch: %v", err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileRead, Path: "../../etc/passwd"}) + if err == nil { + t.Fatal("escape accepted") + } + outside := filepath.Join(root, "..", "outside") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil { + t.Fatal(err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileRead, Path: "escape"}) + if err == nil || errors.Is(err, fs.ErrNotExist) { + t.Fatalf("symlink escape not distinctly rejected: %v", err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileResolveAuthority, Path: "escape"}) + var denied *executionenv.Error + if !errors.As(err, &denied) || denied.Code != executionenv.CodePermissionDenied { + t.Fatalf("authority resolver accepted symlink escape: %v", err) + } +} + +func TestRenameAndCopyNeverClobber(t *testing.T) { + x, _ := New(t.TempDir(), Limits{MaxFileBytes: 1024, MaxEntries: 100}) + defer x.Close() + ctx := context.Background() + _, _ = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: "a", Data: []byte("a")}) + _, _ = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: "b", Data: []byte("b")}) + for _, op := range []executionenv.Operation{executionenv.OpFileCopy, executionenv.OpFileRename} { + if _, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: op, Path: "a", Destination: "b"}); err == nil { + t.Fatalf("%s clobbered", op) + } else { + var pe *executionenv.Error + if !errors.As(err, &pe) || pe.Code != executionenv.CodeAlreadyExists { + t.Fatalf("%s wrong error: %v", op, err) + } + } + } +} + +func TestForegroundCommandReturnsReceiptAndTimeoutTerminatesDescendants(t *testing.T) { + if testing.Short() { + t.Skip("subprocess regression test") + } + root := t.TempDir() + ok := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "c1", Command: "printf hello", TimeoutMillis: 500}) + if ok.Command == nil || ok.Command.State != executionenv.CommandSucceeded || ok.Command.TerminalReceipt == "" || string(ok.Command.Stdout) != "hello" { + t.Fatalf("response=%+v", ok) + } + timed := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "c2", Command: "sleep 10 & wait", TimeoutMillis: 20}) + if timed.Command == nil || timed.Command.State != executionenv.CommandCancelled || timed.Command.TerminalReceipt == "" { + t.Fatalf("response=%+v", timed) + } +} + +func TestSuccessfulDetachedChildIsKilledBeforeTerminalReceipt(t *testing.T) { + root := t.TempDir() + result := filepath.Join(root, "result") + response := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "detached", Command: "nohup sh -c 'sleep 0.2; echo late > result' >/dev/null 2>&1 &", TimeoutMillis: 1000}) + if response.Command == nil || response.Command.State != executionenv.CommandSucceeded || response.Command.TerminalReceipt == "" { + t.Fatalf("response=%+v", response) + } + timer := time.NewTimer(400 * time.Millisecond) + defer timer.Stop() + <-timer.C + if _, err := os.Stat(result); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("detached writer survived terminal receipt: %v", err) + } +} + +func runExecutorHelper(t *testing.T, root string, request executionenv.ExecutorRequest) executionenv.ExecutorResponse { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^TestExecutorCommandHelper$") + cmd.Env = append(os.Environ(), "MECATL_EXECUTOR_TEST_HELPER=1", "MECATL_EXECUTOR_TEST_ROOT="+root) + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + if err := json.NewEncoder(stdin).Encode(request); err != nil { + t.Fatal(err) + } + _ = stdin.Close() + var response executionenv.ExecutorResponse + if err := json.NewDecoder(stdout).Decode(&response); err != nil { + t.Fatal(err) + } + if err := cmd.Wait(); err != nil { + t.Fatalf("executor helper: %v", err) + } + return response +} + +func TestExecutorCommandHelper(_ *testing.T) { + if os.Getenv("MECATL_EXECUTOR_TEST_HELPER") != "1" { + return + } + if err := EnableCommandExecution(); err != nil { + os.Exit(2) + } + x, err := New(os.Getenv("MECATL_EXECUTOR_TEST_ROOT"), Limits{MaxFileBytes: 1024, MaxEntries: 100, CommandTimeout: time.Second}) + if err != nil { + os.Exit(2) + } + var request executionenv.ExecutorRequest + if err := json.NewDecoder(os.Stdin).Decode(&request); err != nil { + os.Exit(2) + } + response, err := x.Execute(context.Background(), request) + if err != nil { + os.Exit(2) + } + if err := json.NewEncoder(os.Stdout).Encode(response); err != nil { + os.Exit(2) + } + os.Exit(0) +} diff --git a/user-docs/building/deployment/mecak8s.md b/user-docs/building/deployment/mecak8s.md index e20bbd3180..b3727fbfcd 100644 --- a/user-docs/building/deployment/mecak8s.md +++ b/user-docs/building/deployment/mecak8s.md @@ -151,6 +151,120 @@ delegation cannot add filesystem access that the parent lacks. See [Execution environments](/features/execution-environments.md) for the shared placement, no-FS, child-environment, and reattachment model. +### Experimental Kubernetes execution provider + +The repository contains a draft, experimental execution provider for an +operator-controlled Kubernetes cluster. It runs as a separate service and +controller with its own `mecatl-execution` chart. `mecak8s` remains a client: it +has no Pod, PVC, custom-resource, or controller management permissions. + +This candidate is suitable for offline evaluation only. It does not yet provide +run-wide execution ownership, complete successor reference binding and release, +controlled executor replacement or PVC deletion, or complete signing-key +rotation and revocation. Do not use it for hostile multitenancy or claim high +availability from this configuration. + +Before you install it, prepare these values through your trusted image and +Secret delivery system: + +- A digest-pinned provider image. +- A digest-pinned workload image that contains `/mecatl-executor`, `/bin/sh`, + and the toolchain needed by foreground commands. +- A server mTLS Secret containing `ca.crt`, `tls.crt`, and `tls.key`. +- A separate Ed25519 grant-signing Secret containing `key.pem`. +- A `mecak8s` client mTLS Secret containing `ca.crt`, `tls.crt`, and `tls.key`. +- Exact client URI SANs for the `mecak8s` identity. The identity that attests + OIDC owners must also appear in `provider.ownerAttesterURISANs`. + +The following profile shows every required chart key. Save it as +`execution-values.yaml` and replace each placeholder: + +```yaml +provider: + image: @sha256: + imagePullPolicy: IfNotPresent + replicas: 1 + clientURISANs: + - spiffe:///client/mecak8s + ownerAttesterURISANs: + - spiffe:///client/mecak8s + administratorURISANs: [] + tlsSecretName: + grantSigningSecretName: + grantKeyID: current + +service: + port: 8443 + +profiles: + coding: + image: @sha256: + storageClass: + storageSize: 2Gi + cpuRequest: 100m + memoryRequest: 128Mi + cpuLimit: "1" + memoryLimit: 1Gi + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m +``` + +Install the provider chart separately from `mecak8s`: + +```sh +helm upgrade --install mecatl-execution ./deploy/helm/mecatl-execution \ + --namespace \ + --values execution-values.yaml +kubectl rollout status deployment/mecatl-execution --namespace +``` + +Add the following block to the existing `mecak8s` values. The endpoint must use +the provider certificate's DNS identity. Keep OIDC caller authentication enabled; +the remote binding is scoped to the verified issuer and subject. + +```yaml +execution: + enabled: true + endpoint: https://mecatl-execution..svc:8443 + profile: coding + tlsSecret: + caKey: ca.crt + certKey: tls.crt + keyKey: tls.key +``` + +An enabled client conflicts with `workspace`, `redis.filesystem.enabled`, +Parallel, and Team. Remote sessions receive the filesystem tools and foreground +Shell, but do not ingest project instructions, rules, skills, or source from the +remote PVC. Schedules, SkillDraft, background Shell, and delegated filesystem +execution are outside this draft. + +The provider retains one PVC per logical environment. Session deletion and chart +uninstall do not delete committed workspace data. Retirement requires no live +references and a terminal executor; even then, this candidate retains the PVC. +A missing executor or lost terminal receipt moves the environment to +`FenceUnknown` and requires external operator fencing. There is no automated +recovery from that state. + +Foreground command cancellation is cooperative and bounded. The helper attempts +to terminate the command process group and reports a terminal receipt. If it +cannot prove termination, the environment is fenced instead of admitting more +work. There is no detached command status or cancellation API in this draft. + +The focused qualification task is `task e2e:k8s:execution`. It requires a generic +local Go, `ko`, Podman, Kind, Helm, and `kubectl` toolchain. Optional toolbox use +requires explicit `MECATL_EXECUTION_DEV_TOOLBOX` and +`MECATL_EXECUTION_K8S_TOOLBOX` values. The task uses a synthetic OIDC issuer and +the mock model provider, creates a unique state directory and Kind cluster, and +retains both for inspection. It performs no cleanup. Kind's default kindnet does +not enforce NetworkPolicy, so this flow provides no NetworkPolicy isolation +evidence. + +When `execution.enabled` is `false`, the `mecak8s` chart mounts no execution mTLS +Secret and passes no execution-provider flags. The separate provider chart and +any environments it owns continue independently. + The no-FS default is intentional. A standard mecak8s pod is storage-free and has no authoritative filesystem root, so the server binds omitted/default profile to its configured no-FS placement. Clients never send a workspace path; explicit From cd1eddf466b6dc0109426f4fe839ecc40674550f Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Wed, 16 Sep 2026 01:08:42 +0300 Subject: [PATCH 02/25] test: synchronize running-team cleanup assertions Hold the actual model stream until both cleanup guards are checked instead of racing a fast finite mock stream. Co-Authored-By: Mecatl --- internal/adapter/server/team_test.go | 56 +++++++++++++++++++++------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/internal/adapter/server/team_test.go b/internal/adapter/server/team_test.go index 32121cd3b7..0f626f0e6c 100644 --- a/internal/adapter/server/team_test.go +++ b/internal/adapter/server/team_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "iter" "strings" "sync" "testing" @@ -54,9 +55,9 @@ func (r *teamLivenessTracker) IsLive(id session.SessionID) bool { } // teamService builds a team-enabled Service whose per-member Engine uses the -// supplied mockllm provider (shared by all members) and carries that member's +// supplied provider (shared by all members) and carries that member's // coordination tools. -func teamService(t *testing.T, llm *mockllm.Provider) *server.Service { +func teamService(t *testing.T, llm port.LLMProvider) *server.Service { t.Helper() allow := permpolicy.NewPolicy(permpolicy.AllowAllFloorRules(), nil) memberEngine := func(tm *team.Team, spec agent.MemberSpec, _ string) agent.MemberBuild { @@ -723,13 +724,45 @@ func TestCreateTeamMaxTeams(t *testing.T) { } } +// cleanupTeamBlockingProvider holds a member's actual stream after it has entered +// the runner, making the running-phase probe deterministic. +type cleanupTeamBlockingProvider struct { + entered chan<- struct{} + release <-chan struct{} +} + +func (p cleanupTeamBlockingProvider) Capabilities() port.ProviderCapabilities { + return port.ProviderCapabilities{} +} + +func (p cleanupTeamBlockingProvider) Stream(ctx context.Context, _ port.LLMRequest) (iter.Seq2[port.Chunk, error], error) { + return func(yield func(port.Chunk, error) bool) { + if !yield(port.Chunk{Kind: port.ChunkText, Text: "thinking"}, nil) { + return + } + select { + case p.entered <- struct{}{}: + case <-ctx.Done(): + return + } + select { + case <-p.release: + case <-ctx.Done(): + } + }, nil +} + // TestCleanupTeamRejectsRunning asserts Fix D: CleanupTeam on a running team is // rejected with FailedPrecondition (deleting it would orphan the live supervisor), // while a created or done team can be cleaned up and frees its slot. func TestCleanupTeamRejectsRunning(t *testing.T) { - // A member whose single turn blocks until ctx is cancelled keeps the team in the - // running phase for the duration of the probe. - llm := mockllm.New(mockllm.ChunksTurn(blockingChunks()...)) + // The provider signals only after the real member stream has started, then + // remains blocked until the probe has observed CleanupTeam's running guard. + entered := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + defer releaseOnce.Do(func() { close(release) }) + llm := cleanupTeamBlockingProvider{entered: entered, release: release} svc := teamService(t, llm) h := server.NewHarnessServer(svc) ctx := context.Background() @@ -755,22 +788,16 @@ func TestCleanupTeamRejectsRunning(t *testing.T) { runCtx, cancelRun := context.WithCancel(context.Background()) defer cancelRun() - firstEvent := make(chan struct{}, 1) runDone := make(chan struct{}) go func() { defer close(runDone) - _, _ = svc.RunTeam(runCtx, teamID, func(agent.TeamEvent) { - select { - case firstEvent <- struct{}{}: - default: - } - }) + _, _ = svc.RunTeam(runCtx, teamID, func(agent.TeamEvent) {}) }() select { - case <-firstEvent: + case <-entered: case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for the team run to start") + t.Fatal("timed out waiting for the member stream to start") } // CleanupTeam on the running team is rejected (FailedPrecondition). @@ -781,6 +808,7 @@ func TestCleanupTeamRejectsRunning(t *testing.T) { t.Fatalf("Service.CleanupTeam(running): err = %v, want ErrTeamRunning", serr) } + releaseOnce.Do(func() { close(release) }) cancelRun() select { case <-runDone: From e4b177249c468e1d30ecf597d79a1e9f31b11423 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Wed, 16 Sep 2026 01:20:54 +0300 Subject: [PATCH 03/25] test: remove unused cleanup fixture receiver Co-Authored-By: Mecatl --- internal/adapter/server/team_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/adapter/server/team_test.go b/internal/adapter/server/team_test.go index 0f626f0e6c..448e465f4b 100644 --- a/internal/adapter/server/team_test.go +++ b/internal/adapter/server/team_test.go @@ -731,7 +731,7 @@ type cleanupTeamBlockingProvider struct { release <-chan struct{} } -func (p cleanupTeamBlockingProvider) Capabilities() port.ProviderCapabilities { +func (cleanupTeamBlockingProvider) Capabilities() port.ProviderCapabilities { return port.ProviderCapabilities{} } From 041331202b14590b83be9b20f26de51a4c7266ef Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Wed, 16 Sep 2026 21:31:54 +0200 Subject: [PATCH 04/25] feat: use gRPC for execution provider and qualify live OpenRouter Replace the private REST protocol with typed protobuf RPCs over mTLS. Add a real-key kind coding smoke with runtime-only credential loading, UID-pinned Secret cleanup, mock restoration, and independent gRPC artifact and command verification. Co-Authored-By: Mecatl --- Taskfile.yml | 19 +- cmd/mecak8s/flags.go | 2 +- cmd/mecatl-execution-provider/main.go | 33 +- .../go/mecatl/execution/v1/execution.pb.go | 1929 +++++++++++++++++ .../mecatl/execution/v1/execution_grpc.pb.go | 433 ++++ .../proto/mecatl/execution/v1/execution.proto | 132 ++ deploy/helm/mecak8s/values.schema.json | 2 +- .../mecatl-execution/templates/provider.yaml | 14 +- deploy/mecatl-execution-kind/live.sh | 199 ++ .../mecatl-execution-kind/mecak8s-values.yaml | 2 +- deploy/mecatl-execution-kind/run.sh | 5 +- docs/adr/0027-cloud-native.md | 4 +- docs/adr/0349-native-kubernetes-execution.md | 8 +- docs/architecture.md | 2 +- docs/design/IMPLEMENTATION-NOTES.md | 3 +- .../fixture/credentialloader/main.go | 279 +++ .../fixture/credentialloader/main_test.go | 188 ++ e2e/k8s_execution/live_qualification_test.go | 381 ++++ .../live_qualification_unit_test.go | 100 + e2e/k8s_execution/qualification_test.go | 82 +- internal/adapter/executionclient/client.go | 346 ++- .../adapter/executionclient/client_test.go | 316 +-- .../adapter/executioncontroller/handler.go | 585 ++--- .../executioncontroller/handler_test.go | 252 ++- internal/executionenv/errors.go | 20 +- internal/executionenv/grant.go | 14 +- internal/executionenv/protocol_test.go | 6 +- internal/executionenv/types.go | 5 +- user-docs/building/deployment/mecak8s.md | 8 +- user-docs/features/execution-environments.md | 24 + 30 files changed, 4747 insertions(+), 646 deletions(-) create mode 100644 contracts/gen/go/mecatl/execution/v1/execution.pb.go create mode 100644 contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go create mode 100644 contracts/proto/mecatl/execution/v1/execution.proto create mode 100755 deploy/mecatl-execution-kind/live.sh create mode 100644 e2e/k8s_execution/fixture/credentialloader/main.go create mode 100644 e2e/k8s_execution/fixture/credentialloader/main_test.go create mode 100644 e2e/k8s_execution/live_qualification_test.go create mode 100644 e2e/k8s_execution/live_qualification_unit_test.go diff --git a/Taskfile.yml b/Taskfile.yml index 66dd433b90..f33f5d7e02 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -514,13 +514,28 @@ tasks: credential. The owned cluster is intentionally retained; the command prints the exact human-confirmed cleanup command. NOT part of task test. preconditions: - - sh: command -v toolbox >/dev/null 2>&1 - msg: "toolbox is required (dev and sre toolboxes)" - sh: command -v kind >/dev/null 2>&1 && command -v podman >/dev/null 2>&1 msg: "kind and rootless Podman are required" cmds: - deploy/mecatl-execution-kind/run.sh + e2e:k8s:execution:live: + desc: | + EXPLICIT OPT-IN: run mock qualification followed by one bounded real-provider + coding smoke on the approved retained Kind cluster. Requires absolute + MECATL_EXECUTION_CREDENTIAL_FILE and MECATL_EXECUTION_QUAL_STATE paths. A + trusted loader stages a run-scoped Secret without exposing its value, the + test independently verifies files and go test through typed gRPC, then the + mock profile is restored and only that run-scoped Secret is removed. NOT + part of task test. + preconditions: + - sh: test -n "${MECATL_EXECUTION_CREDENTIAL_FILE:-}" + msg: "MECATL_EXECUTION_CREDENTIAL_FILE is required" + - sh: test -n "${MECATL_EXECUTION_QUAL_STATE:-}" + msg: "MECATL_EXECUTION_QUAL_STATE is required" + cmds: + - deploy/mecatl-execution-kind/live.sh + e2e:k8s:podman: desc: Run e2e:k8s with rootless Podman (requires the k8s-file Podman log driver) cmds: diff --git a/cmd/mecak8s/flags.go b/cmd/mecak8s/flags.go index 078102e32c..505bfbbb78 100644 --- a/cmd/mecak8s/flags.go +++ b/cmd/mecak8s/flags.go @@ -359,7 +359,7 @@ func parseFlags(argv []string) (config, error) { positiveDurationFlag(fs, &cfg.closeTimeout, "close-timeout", defaultCloseTimeout, "Maximum time for final application cleanup") fs.StringVar(&cfg.workspace, "workspace", "", "Shared agent workspace root, such as a mounted PVC. Empty gives every session a shell-less, file-less workspace; clients cannot choose another root") fs.BoolVar(&cfg.executionEnabled, "execution-enabled", false, "Use an independently deployed Kubernetes execution provider for default sessions") - fs.StringVar(&cfg.executionEndpoint, "execution-endpoint", "", "HTTPS endpoint of the execution provider. Requires --execution-enabled") + fs.StringVar(&cfg.executionEndpoint, "execution-endpoint", "", "Host:port endpoint of the mTLS gRPC execution provider. Requires --execution-enabled") fs.StringVar(&cfg.executionProfile, "execution-profile", "", "Operator-configured execution provider profile") fs.StringVar(&cfg.executionTLSCA, "execution-tls-ca", "", "Mounted CA bundle used only by the execution client") fs.StringVar(&cfg.executionTLSCert, "execution-tls-cert", "", "Mounted execution-provider mTLS client certificate") diff --git a/cmd/mecatl-execution-provider/main.go b/cmd/mecatl-execution-provider/main.go index e53b6bd25f..872651c3ef 100644 --- a/cmd/mecatl-execution-provider/main.go +++ b/cmd/mecatl-execution-provider/main.go @@ -12,17 +12,19 @@ import ( "fmt" "log/slog" "net" - "net/http" "os" "os/signal" "strings" "syscall" "time" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" "github.com/stacklok/mecatl/internal/adapter/executioncontroller" "github.com/stacklok/mecatl/internal/executionenv" ) @@ -33,9 +35,9 @@ func main() { os.Exit(1) } } -func run() error { +func run() error { //nolint:gocyclo // Startup validation and owned-resource shutdown stay in one composition root. var addr, namespace, profilesPath, certPath, keyPath, caPath, signingPath, keyID, issuer, audience, clients, attesters, admins string - flag.StringVar(&addr, "listen", ":8443", "HTTPS listen address") + flag.StringVar(&addr, "listen", ":8443", "gRPC listen address") flag.StringVar(&namespace, "namespace", "", "managed Kubernetes namespace") flag.StringVar(&profilesPath, "profiles", "/etc/mecatl-execution/profiles.yaml", "strict operator profile file") flag.StringVar(&certPath, "tls-cert", "/etc/mecatl-execution/tls/tls.crt", "server certificate") @@ -105,28 +107,37 @@ func run() error { if err != nil { return err } - tlsLn := tls.NewListener(ln, executioncontroller.TLSConfig(serverCert, pool)) - server := &http.Server{Handler: handler, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 35 * time.Minute, IdleTimeout: 60 * time.Second} + server := grpc.NewServer( + grpc.Creds(credentials.NewTLS(executioncontroller.TLSConfig(serverCert, pool))), + grpc.MaxRecvMsgSize(executionenv.MaxMessageBytes), + grpc.MaxSendMsgSize(executionenv.MaxMessageBytes), + ) + executionv1.RegisterExecutionProviderServiceServer(server, handler) serveErr := make(chan error, 1) - go func() { serveErr <- server.Serve(tlsLn) }() + go func() { serveErr <- server.Serve(ln) }() select { case err := <-controllerErr: cancel() - _ = server.Shutdown(context.Background()) + server.GracefulStop() if err != nil { return fmt.Errorf("controller: %w", err) } return nil case err := <-serveErr: cancel() - if !errors.Is(err, http.ErrServerClosed) { + if err != nil { return err } return nil case <-ctx.Done(): - shutdown, c := context.WithTimeout(context.Background(), 10*time.Second) - defer c() - return server.Shutdown(shutdown) + done := make(chan struct{}) + go func() { server.GracefulStop(); close(done) }() + select { + case <-done: + case <-time.After(10 * time.Second): + server.Stop() + } + return nil } } func loadSigningKey(path string) (ed25519.PrivateKey, error) { diff --git a/contracts/gen/go/mecatl/execution/v1/execution.pb.go b/contracts/gen/go/mecatl/execution/v1/execution.pb.go new file mode 100644 index 0000000000..0f6c60c343 --- /dev/null +++ b/contracts/gen/go/mecatl/execution/v1/execution.pb.go @@ -0,0 +1,1929 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: LicenseRef-Stacklok-Proprietary + +// Private, mutually authenticated execution-provider protocol. This is not part +// of the public Harness API. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.4 +// protoc (unknown) +// source: mecatl/execution/v1/execution.proto + +package executionv1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type FileOperation int32 + +const ( + FileOperation_FILE_OPERATION_UNSPECIFIED FileOperation = 0 + FileOperation_FILE_OPERATION_READ FileOperation = 1 + FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY FileOperation = 2 + FileOperation_FILE_OPERATION_STAT FileOperation = 3 + FileOperation_FILE_OPERATION_CREATE FileOperation = 4 + FileOperation_FILE_OPERATION_REPLACE FileOperation = 5 + FileOperation_FILE_OPERATION_LIST FileOperation = 6 + FileOperation_FILE_OPERATION_REMOVE FileOperation = 7 + FileOperation_FILE_OPERATION_RENAME FileOperation = 8 + FileOperation_FILE_OPERATION_COPY FileOperation = 9 + FileOperation_FILE_OPERATION_GLOB FileOperation = 10 + FileOperation_FILE_OPERATION_GREP FileOperation = 11 +) + +// Enum value maps for FileOperation. +var ( + FileOperation_name = map[int32]string{ + 0: "FILE_OPERATION_UNSPECIFIED", + 1: "FILE_OPERATION_READ", + 2: "FILE_OPERATION_RESOLVE_AUTHORITY", + 3: "FILE_OPERATION_STAT", + 4: "FILE_OPERATION_CREATE", + 5: "FILE_OPERATION_REPLACE", + 6: "FILE_OPERATION_LIST", + 7: "FILE_OPERATION_REMOVE", + 8: "FILE_OPERATION_RENAME", + 9: "FILE_OPERATION_COPY", + 10: "FILE_OPERATION_GLOB", + 11: "FILE_OPERATION_GREP", + } + FileOperation_value = map[string]int32{ + "FILE_OPERATION_UNSPECIFIED": 0, + "FILE_OPERATION_READ": 1, + "FILE_OPERATION_RESOLVE_AUTHORITY": 2, + "FILE_OPERATION_STAT": 3, + "FILE_OPERATION_CREATE": 4, + "FILE_OPERATION_REPLACE": 5, + "FILE_OPERATION_LIST": 6, + "FILE_OPERATION_REMOVE": 7, + "FILE_OPERATION_RENAME": 8, + "FILE_OPERATION_COPY": 9, + "FILE_OPERATION_GLOB": 10, + "FILE_OPERATION_GREP": 11, + } +) + +func (x FileOperation) Enum() *FileOperation { + p := new(FileOperation) + *p = x + return p +} + +func (x FileOperation) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FileOperation) Descriptor() protoreflect.EnumDescriptor { + return file_mecatl_execution_v1_execution_proto_enumTypes[0].Descriptor() +} + +func (FileOperation) Type() protoreflect.EnumType { + return &file_mecatl_execution_v1_execution_proto_enumTypes[0] +} + +func (x FileOperation) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use FileOperation.Descriptor instead. +func (FileOperation) EnumDescriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{0} +} + +type CommandState int32 + +const ( + CommandState_COMMAND_STATE_UNSPECIFIED CommandState = 0 + CommandState_COMMAND_STATE_RUNNING CommandState = 1 + CommandState_COMMAND_STATE_SUCCEEDED CommandState = 2 + CommandState_COMMAND_STATE_FAILED CommandState = 3 + CommandState_COMMAND_STATE_CANCELLED CommandState = 4 + CommandState_COMMAND_STATE_FENCE_UNKNOWN CommandState = 5 +) + +// Enum value maps for CommandState. +var ( + CommandState_name = map[int32]string{ + 0: "COMMAND_STATE_UNSPECIFIED", + 1: "COMMAND_STATE_RUNNING", + 2: "COMMAND_STATE_SUCCEEDED", + 3: "COMMAND_STATE_FAILED", + 4: "COMMAND_STATE_CANCELLED", + 5: "COMMAND_STATE_FENCE_UNKNOWN", + } + CommandState_value = map[string]int32{ + "COMMAND_STATE_UNSPECIFIED": 0, + "COMMAND_STATE_RUNNING": 1, + "COMMAND_STATE_SUCCEEDED": 2, + "COMMAND_STATE_FAILED": 3, + "COMMAND_STATE_CANCELLED": 4, + "COMMAND_STATE_FENCE_UNKNOWN": 5, + } +) + +func (x CommandState) Enum() *CommandState { + p := new(CommandState) + *p = x + return p +} + +func (x CommandState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CommandState) Descriptor() protoreflect.EnumDescriptor { + return file_mecatl_execution_v1_execution_proto_enumTypes[1].Descriptor() +} + +func (CommandState) Type() protoreflect.EnumType { + return &file_mecatl_execution_v1_execution_proto_enumTypes[1] +} + +func (x CommandState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CommandState.Descriptor instead. +func (CommandState) EnumDescriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{1} +} + +type EnvironmentRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Revision string `protobuf:"bytes,2,opt,name=revision,proto3" json:"revision,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnvironmentRef) Reset() { + *x = EnvironmentRef{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnvironmentRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnvironmentRef) ProtoMessage() {} + +func (x *EnvironmentRef) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnvironmentRef.ProtoReflect.Descriptor instead. +func (*EnvironmentRef) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{0} +} + +func (x *EnvironmentRef) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *EnvironmentRef) GetRevision() string { + if x != nil { + return x.Revision + } + return "" +} + +type Owner struct { + state protoimpl.MessageState `protogen:"open.v1"` + Issuer string `protobuf:"bytes,1,opt,name=issuer,proto3" json:"issuer,omitempty"` + Subject string `protobuf:"bytes,2,opt,name=subject,proto3" json:"subject,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Owner) Reset() { + *x = Owner{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Owner) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Owner) ProtoMessage() {} + +func (x *Owner) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Owner.ProtoReflect.Descriptor instead. +func (*Owner) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{1} +} + +func (x *Owner) GetIssuer() string { + if x != nil { + return x.Issuer + } + return "" +} + +func (x *Owner) GetSubject() string { + if x != nil { + return x.Subject + } + return "" +} + +type RequestContext struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + Epoch uint64 `protobuf:"varint,4,opt,name=epoch,proto3" json:"epoch,omitempty"` + Grant string `protobuf:"bytes,5,opt,name=grant,proto3" json:"grant,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestContext) Reset() { + *x = RequestContext{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestContext) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestContext) ProtoMessage() {} + +func (x *RequestContext) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestContext.ProtoReflect.Descriptor instead. +func (*RequestContext) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{2} +} + +func (x *RequestContext) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RequestContext) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RequestContext) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *RequestContext) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *RequestContext) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +type ValidateProfileRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Profile string `protobuf:"bytes,1,opt,name=profile,proto3" json:"profile,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ValidateProfileRequest) Reset() { + *x = ValidateProfileRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ValidateProfileRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ValidateProfileRequest) ProtoMessage() {} + +func (x *ValidateProfileRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ValidateProfileRequest.ProtoReflect.Descriptor instead. +func (*ValidateProfileRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{3} +} + +func (x *ValidateProfileRequest) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +type ValidateProfileResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Profile string `protobuf:"bytes,1,opt,name=profile,proto3" json:"profile,omitempty"` + Digest string `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"` + Capabilities []string `protobuf:"bytes,3,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + MaxFileBytes int64 `protobuf:"varint,4,opt,name=max_file_bytes,json=maxFileBytes,proto3" json:"max_file_bytes,omitempty"` + MaxCommandBytes int64 `protobuf:"varint,5,opt,name=max_command_bytes,json=maxCommandBytes,proto3" json:"max_command_bytes,omitempty"` + MaxCommandDurationMillis int64 `protobuf:"varint,6,opt,name=max_command_duration_millis,json=maxCommandDurationMillis,proto3" json:"max_command_duration_millis,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ValidateProfileResponse) Reset() { + *x = ValidateProfileResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ValidateProfileResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ValidateProfileResponse) ProtoMessage() {} + +func (x *ValidateProfileResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ValidateProfileResponse.ProtoReflect.Descriptor instead. +func (*ValidateProfileResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{4} +} + +func (x *ValidateProfileResponse) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +func (x *ValidateProfileResponse) GetDigest() string { + if x != nil { + return x.Digest + } + return "" +} + +func (x *ValidateProfileResponse) GetCapabilities() []string { + if x != nil { + return x.Capabilities + } + return nil +} + +func (x *ValidateProfileResponse) GetMaxFileBytes() int64 { + if x != nil { + return x.MaxFileBytes + } + return 0 +} + +func (x *ValidateProfileResponse) GetMaxCommandBytes() int64 { + if x != nil { + return x.MaxCommandBytes + } + return 0 +} + +func (x *ValidateProfileResponse) GetMaxCommandDurationMillis() int64 { + if x != nil { + return x.MaxCommandDurationMillis + } + return 0 +} + +type EnsureEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + BindingId string `protobuf:"bytes,1,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + Profile string `protobuf:"bytes,2,opt,name=profile,proto3" json:"profile,omitempty"` + Owner *Owner `protobuf:"bytes,3,opt,name=owner,proto3" json:"owner,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnsureEnvironmentRequest) Reset() { + *x = EnsureEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnsureEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnsureEnvironmentRequest) ProtoMessage() {} + +func (x *EnsureEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnsureEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*EnsureEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{5} +} + +func (x *EnsureEnvironmentRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *EnsureEnvironmentRequest) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +func (x *EnsureEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +type EnsureEnvironmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` + Ready bool `protobuf:"varint,3,opt,name=ready,proto3" json:"ready,omitempty"` + Grant string `protobuf:"bytes,4,opt,name=grant,proto3" json:"grant,omitempty"` + GrantExpiresAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=grant_expires_at,json=grantExpiresAt,proto3" json:"grant_expires_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnsureEnvironmentResponse) Reset() { + *x = EnsureEnvironmentResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnsureEnvironmentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnsureEnvironmentResponse) ProtoMessage() {} + +func (x *EnsureEnvironmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnsureEnvironmentResponse.ProtoReflect.Descriptor instead. +func (*EnsureEnvironmentResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{6} +} + +func (x *EnsureEnvironmentResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *EnsureEnvironmentResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *EnsureEnvironmentResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *EnsureEnvironmentResponse) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *EnsureEnvironmentResponse) GetGrantExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.GrantExpiresAt + } + return nil +} + +type AttachEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + Purpose string `protobuf:"bytes,2,opt,name=purpose,proto3" json:"purpose,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttachEnvironmentRequest) Reset() { + *x = AttachEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttachEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttachEnvironmentRequest) ProtoMessage() {} + +func (x *AttachEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttachEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*AttachEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{7} +} + +func (x *AttachEnvironmentRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *AttachEnvironmentRequest) GetPurpose() string { + if x != nil { + return x.Purpose + } + return "" +} + +type AttachEnvironmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` + Ready bool `protobuf:"varint,3,opt,name=ready,proto3" json:"ready,omitempty"` + Grant string `protobuf:"bytes,4,opt,name=grant,proto3" json:"grant,omitempty"` + GrantExpiresAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=grant_expires_at,json=grantExpiresAt,proto3" json:"grant_expires_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttachEnvironmentResponse) Reset() { + *x = AttachEnvironmentResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttachEnvironmentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttachEnvironmentResponse) ProtoMessage() {} + +func (x *AttachEnvironmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttachEnvironmentResponse.ProtoReflect.Descriptor instead. +func (*AttachEnvironmentResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{8} +} + +func (x *AttachEnvironmentResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *AttachEnvironmentResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *AttachEnvironmentResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *AttachEnvironmentResponse) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *AttachEnvironmentResponse) GetGrantExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.GrantExpiresAt + } + return nil +} + +type ReleaseReferenceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReleaseReferenceRequest) Reset() { + *x = ReleaseReferenceRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReleaseReferenceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReleaseReferenceRequest) ProtoMessage() {} + +func (x *ReleaseReferenceRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReleaseReferenceRequest.ProtoReflect.Descriptor instead. +func (*ReleaseReferenceRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{9} +} + +func (x *ReleaseReferenceRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +type RetireEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RetireEnvironmentRequest) Reset() { + *x = RetireEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RetireEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RetireEnvironmentRequest) ProtoMessage() {} + +func (x *RetireEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RetireEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*RetireEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{10} +} + +func (x *RetireEnvironmentRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +type FileRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + Operation FileOperation `protobuf:"varint,2,opt,name=operation,proto3,enum=mecatl.execution.v1.FileOperation" json:"operation,omitempty"` + Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` + Destination string `protobuf:"bytes,4,opt,name=destination,proto3" json:"destination,omitempty"` + Pattern string `protobuf:"bytes,5,opt,name=pattern,proto3" json:"pattern,omitempty"` + Data []byte `protobuf:"bytes,6,opt,name=data,proto3" json:"data,omitempty"` + Version []byte `protobuf:"bytes,7,opt,name=version,proto3" json:"version,omitempty"` + Limit int32 `protobuf:"varint,8,opt,name=limit,proto3" json:"limit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileRequest) Reset() { + *x = FileRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileRequest) ProtoMessage() {} + +func (x *FileRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileRequest.ProtoReflect.Descriptor instead. +func (*FileRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{11} +} + +func (x *FileRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *FileRequest) GetOperation() FileOperation { + if x != nil { + return x.Operation + } + return FileOperation_FILE_OPERATION_UNSPECIFIED +} + +func (x *FileRequest) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *FileRequest) GetDestination() string { + if x != nil { + return x.Destination + } + return "" +} + +func (x *FileRequest) GetPattern() string { + if x != nil { + return x.Pattern + } + return "" +} + +func (x *FileRequest) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *FileRequest) GetVersion() []byte { + if x != nil { + return x.Version + } + return nil +} + +func (x *FileRequest) GetLimit() int32 { + if x != nil { + return x.Limit + } + return 0 +} + +type FileInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + Mode uint32 `protobuf:"varint,3,opt,name=mode,proto3" json:"mode,omitempty"` + ModTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=mod_time,json=modTime,proto3" json:"mod_time,omitempty"` + IsDir bool `protobuf:"varint,5,opt,name=is_dir,json=isDir,proto3" json:"is_dir,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileInfo) Reset() { + *x = FileInfo{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileInfo) ProtoMessage() {} + +func (x *FileInfo) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileInfo.ProtoReflect.Descriptor instead. +func (*FileInfo) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{12} +} + +func (x *FileInfo) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *FileInfo) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *FileInfo) GetMode() uint32 { + if x != nil { + return x.Mode + } + return 0 +} + +func (x *FileInfo) GetModTime() *timestamppb.Timestamp { + if x != nil { + return x.ModTime + } + return nil +} + +func (x *FileInfo) GetIsDir() bool { + if x != nil { + return x.IsDir + } + return false +} + +type GrepMatch struct { + state protoimpl.MessageState `protogen:"open.v1"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + Line int32 `protobuf:"varint,2,opt,name=line,proto3" json:"line,omitempty"` + Text string `protobuf:"bytes,3,opt,name=text,proto3" json:"text,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GrepMatch) Reset() { + *x = GrepMatch{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GrepMatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GrepMatch) ProtoMessage() {} + +func (x *GrepMatch) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GrepMatch.ProtoReflect.Descriptor instead. +func (*GrepMatch) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{13} +} + +func (x *GrepMatch) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *GrepMatch) GetLine() int32 { + if x != nil { + return x.Line + } + return 0 +} + +func (x *GrepMatch) GetText() string { + if x != nil { + return x.Text + } + return "" +} + +type FileResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + Version []byte `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Info *FileInfo `protobuf:"bytes,3,opt,name=info,proto3" json:"info,omitempty"` + Entries []*FileInfo `protobuf:"bytes,4,rep,name=entries,proto3" json:"entries,omitempty"` + Paths []string `protobuf:"bytes,5,rep,name=paths,proto3" json:"paths,omitempty"` + Matches []*GrepMatch `protobuf:"bytes,6,rep,name=matches,proto3" json:"matches,omitempty"` + AuthorityTarget string `protobuf:"bytes,7,opt,name=authority_target,json=authorityTarget,proto3" json:"authority_target,omitempty"` + AuthorityWorkspace string `protobuf:"bytes,8,opt,name=authority_workspace,json=authorityWorkspace,proto3" json:"authority_workspace,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileResponse) Reset() { + *x = FileResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileResponse) ProtoMessage() {} + +func (x *FileResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileResponse.ProtoReflect.Descriptor instead. +func (*FileResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{14} +} + +func (x *FileResponse) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *FileResponse) GetVersion() []byte { + if x != nil { + return x.Version + } + return nil +} + +func (x *FileResponse) GetInfo() *FileInfo { + if x != nil { + return x.Info + } + return nil +} + +func (x *FileResponse) GetEntries() []*FileInfo { + if x != nil { + return x.Entries + } + return nil +} + +func (x *FileResponse) GetPaths() []string { + if x != nil { + return x.Paths + } + return nil +} + +func (x *FileResponse) GetMatches() []*GrepMatch { + if x != nil { + return x.Matches + } + return nil +} + +func (x *FileResponse) GetAuthorityTarget() string { + if x != nil { + return x.AuthorityTarget + } + return "" +} + +func (x *FileResponse) GetAuthorityWorkspace() string { + if x != nil { + return x.AuthorityWorkspace + } + return "" +} + +type CommandStartRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + Command string `protobuf:"bytes,2,opt,name=command,proto3" json:"command,omitempty"` + TimeoutMillis int64 `protobuf:"varint,3,opt,name=timeout_millis,json=timeoutMillis,proto3" json:"timeout_millis,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandStartRequest) Reset() { + *x = CommandStartRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandStartRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandStartRequest) ProtoMessage() {} + +func (x *CommandStartRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandStartRequest.ProtoReflect.Descriptor instead. +func (*CommandStartRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{15} +} + +func (x *CommandStartRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *CommandStartRequest) GetCommand() string { + if x != nil { + return x.Command + } + return "" +} + +func (x *CommandStartRequest) GetTimeoutMillis() int64 { + if x != nil { + return x.TimeoutMillis + } + return 0 +} + +type CommandStartResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"` + State CommandState `protobuf:"varint,2,opt,name=state,proto3,enum=mecatl.execution.v1.CommandState" json:"state,omitempty"` + Result *CommandStatusResponse `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandStartResponse) Reset() { + *x = CommandStartResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandStartResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandStartResponse) ProtoMessage() {} + +func (x *CommandStartResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandStartResponse.ProtoReflect.Descriptor instead. +func (*CommandStartResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{16} +} + +func (x *CommandStartResponse) GetCommandId() string { + if x != nil { + return x.CommandId + } + return "" +} + +func (x *CommandStartResponse) GetState() CommandState { + if x != nil { + return x.State + } + return CommandState_COMMAND_STATE_UNSPECIFIED +} + +func (x *CommandStartResponse) GetResult() *CommandStatusResponse { + if x != nil { + return x.Result + } + return nil +} + +type CommandQueryRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + CommandId string `protobuf:"bytes,2,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"` + Offset int64 `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandQueryRequest) Reset() { + *x = CommandQueryRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandQueryRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandQueryRequest) ProtoMessage() {} + +func (x *CommandQueryRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandQueryRequest.ProtoReflect.Descriptor instead. +func (*CommandQueryRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{17} +} + +func (x *CommandQueryRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *CommandQueryRequest) GetCommandId() string { + if x != nil { + return x.CommandId + } + return "" +} + +func (x *CommandQueryRequest) GetOffset() int64 { + if x != nil { + return x.Offset + } + return 0 +} + +type CommandStatusResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"` + State CommandState `protobuf:"varint,2,opt,name=state,proto3,enum=mecatl.execution.v1.CommandState" json:"state,omitempty"` + ExitCode int32 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` + Stdout []byte `protobuf:"bytes,4,opt,name=stdout,proto3" json:"stdout,omitempty"` + Stderr []byte `protobuf:"bytes,5,opt,name=stderr,proto3" json:"stderr,omitempty"` + NextOffset int64 `protobuf:"varint,6,opt,name=next_offset,json=nextOffset,proto3" json:"next_offset,omitempty"` + Truncated bool `protobuf:"varint,7,opt,name=truncated,proto3" json:"truncated,omitempty"` + TerminalReceipt string `protobuf:"bytes,8,opt,name=terminal_receipt,json=terminalReceipt,proto3" json:"terminal_receipt,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandStatusResponse) Reset() { + *x = CommandStatusResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandStatusResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandStatusResponse) ProtoMessage() {} + +func (x *CommandStatusResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandStatusResponse.ProtoReflect.Descriptor instead. +func (*CommandStatusResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{18} +} + +func (x *CommandStatusResponse) GetCommandId() string { + if x != nil { + return x.CommandId + } + return "" +} + +func (x *CommandStatusResponse) GetState() CommandState { + if x != nil { + return x.State + } + return CommandState_COMMAND_STATE_UNSPECIFIED +} + +func (x *CommandStatusResponse) GetExitCode() int32 { + if x != nil { + return x.ExitCode + } + return 0 +} + +func (x *CommandStatusResponse) GetStdout() []byte { + if x != nil { + return x.Stdout + } + return nil +} + +func (x *CommandStatusResponse) GetStderr() []byte { + if x != nil { + return x.Stderr + } + return nil +} + +func (x *CommandStatusResponse) GetNextOffset() int64 { + if x != nil { + return x.NextOffset + } + return 0 +} + +func (x *CommandStatusResponse) GetTruncated() bool { + if x != nil { + return x.Truncated + } + return false +} + +func (x *CommandStatusResponse) GetTerminalReceipt() string { + if x != nil { + return x.TerminalReceipt + } + return "" +} + +// ErrorDetail is a stable machine-readable classification. Status messages are +// intentionally generic so backend/Kubernetes details never cross the boundary. +type ErrorDetail struct { + state protoimpl.MessageState `protogen:"open.v1"` + Code string `protobuf:"bytes,1,opt,name=code,proto3" json:"code,omitempty"` + Retryable bool `protobuf:"varint,2,opt,name=retryable,proto3" json:"retryable,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ErrorDetail) Reset() { + *x = ErrorDetail{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ErrorDetail) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ErrorDetail) ProtoMessage() {} + +func (x *ErrorDetail) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ErrorDetail.ProtoReflect.Descriptor instead. +func (*ErrorDetail) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{19} +} + +func (x *ErrorDetail) GetCode() string { + if x != nil { + return x.Code + } + return "" +} + +func (x *ErrorDetail) GetRetryable() bool { + if x != nil { + return x.Retryable + } + return false +} + +var File_mecatl_execution_v1_execution_proto protoreflect.FileDescriptor + +var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ + 0x0a, 0x23, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x13, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, + 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3c, 0x0a, 0x0e, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, + 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, + 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x39, 0x0a, 0x05, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x12, + 0x16, 0x0a, 0x06, 0x69, 0x73, 0x73, 0x75, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x69, 0x73, 0x73, 0x75, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, + 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, + 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, + 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x22, 0x32, 0x0a, 0x16, 0x56, 0x61, 0x6c, 0x69, + 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x80, 0x02, 0x0a, + 0x17, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, + 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, + 0x6c, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x61, + 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x24, + 0x0a, 0x0e, 0x6d, 0x61, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x0f, 0x6d, 0x61, 0x78, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, + 0x12, 0x3d, 0x0a, 0x1b, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, + 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x18, 0x6d, 0x61, 0x78, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, + 0x85, 0x01, 0x0a, 0x18, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, + 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x70, + 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, + 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, + 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x22, 0xea, 0x01, 0x0a, 0x19, 0x45, 0x6e, 0x73, 0x75, + 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, + 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, + 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x44, + 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, + 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x45, 0x78, 0x70, 0x69, 0x72, + 0x65, 0x73, 0x41, 0x74, 0x22, 0x73, 0x0a, 0x18, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x70, 0x75, 0x72, 0x70, 0x6f, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x70, 0x75, 0x72, 0x70, 0x6f, 0x73, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x19, 0x41, 0x74, + 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, + 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, + 0x70, 0x6f, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, + 0x61, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, + 0x12, 0x44, 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, + 0x73, 0x5f, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x45, 0x78, 0x70, + 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x22, 0x58, 0x0a, 0x17, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x22, 0x59, 0x0a, 0x18, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0xa2, 0x02, 0x0a, 0x0b, + 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x40, 0x0a, 0x09, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, + 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x12, 0x0a, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, + 0x6d, 0x69, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, + 0x22, 0x94, 0x01, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x6d, 0x6f, 0x64, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65, + 0x12, 0x15, 0x0a, 0x06, 0x69, 0x73, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x05, 0x69, 0x73, 0x44, 0x69, 0x72, 0x22, 0x47, 0x0a, 0x09, 0x47, 0x72, 0x65, 0x70, 0x4d, + 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x74, 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, + 0x22, 0xd4, 0x02, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x31, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, + 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x70, + 0x61, 0x74, 0x68, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, + 0x73, 0x12, 0x38, 0x0a, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x65, 0x70, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x52, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x61, + 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2f, 0x0a, 0x13, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x69, 0x74, 0x79, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x12, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x57, 0x6f, + 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x95, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x18, + 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, + 0xb2, 0x01, 0x0a, 0x14, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, + 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x12, 0x42, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x06, 0x72, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x22, 0x8b, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, + 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, + 0x65, 0x74, 0x22, 0xa6, 0x02, 0x0a, 0x15, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, + 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x37, 0x0a, 0x05, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, + 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, + 0x65, 0x72, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, + 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6e, 0x65, 0x78, 0x74, 0x4f, 0x66, 0x66, 0x73, + 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, + 0x12, 0x29, 0x0a, 0x10, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x63, + 0x65, 0x69, 0x70, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x72, 0x6d, + 0x69, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, 0x3f, 0x0a, 0x0b, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, + 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x1c, + 0x0a, 0x09, 0x72, 0x65, 0x74, 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x09, 0x72, 0x65, 0x74, 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x2a, 0xd8, 0x02, 0x0a, + 0x0d, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, + 0x0a, 0x1a, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, + 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, + 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, + 0x5f, 0x52, 0x45, 0x41, 0x44, 0x10, 0x01, 0x12, 0x24, 0x0a, 0x20, 0x46, 0x49, 0x4c, 0x45, 0x5f, + 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x53, 0x4f, 0x4c, 0x56, + 0x45, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x54, 0x59, 0x10, 0x02, 0x12, 0x17, 0x0a, + 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, + 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, + 0x04, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, 0x05, 0x12, 0x17, 0x0a, + 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, + 0x4c, 0x49, 0x53, 0x54, 0x10, 0x06, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, + 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x10, + 0x07, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x08, 0x12, 0x17, 0x0a, 0x13, + 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x43, + 0x4f, 0x50, 0x59, 0x10, 0x09, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, + 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x47, 0x4c, 0x4f, 0x42, 0x10, 0x0a, 0x12, 0x17, + 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, + 0x5f, 0x47, 0x52, 0x45, 0x50, 0x10, 0x0b, 0x2a, 0xbd, 0x01, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x43, 0x4f, 0x4d, 0x4d, + 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x43, 0x4f, 0x4d, 0x4d, 0x41, + 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, + 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x45, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x45, 0x44, 0x45, 0x44, 0x10, 0x02, 0x12, + 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x03, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4d, + 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, + 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, + 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x45, 0x4e, 0x43, 0x45, 0x5f, 0x55, 0x4e, + 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x05, 0x32, 0xa7, 0x07, 0x0a, 0x18, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, + 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, + 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, + 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x72, 0x0a, 0x11, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, + 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x73, + 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x11, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, + 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x10, 0x52, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2c, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5a, 0x0a, 0x11, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x12, 0x4c, 0x0a, 0x05, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, + 0x0a, 0x0c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, + 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x61, + 0x6e, 0x63, 0x65, 0x6c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x42, 0xe4, 0x01, 0x0a, 0x17, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x45, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, + 0x4b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, 0x61, 0x63, + 0x6b, 0x6c, 0x6f, 0x6b, 0x2f, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x31, + 0x3b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, + 0x45, 0x58, 0xaa, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0xe2, 0x02, + 0x1f, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0xea, 0x02, 0x15, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x3a, 0x3a, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +}) + +var ( + file_mecatl_execution_v1_execution_proto_rawDescOnce sync.Once + file_mecatl_execution_v1_execution_proto_rawDescData []byte +) + +func file_mecatl_execution_v1_execution_proto_rawDescGZIP() []byte { + file_mecatl_execution_v1_execution_proto_rawDescOnce.Do(func() { + file_mecatl_execution_v1_execution_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mecatl_execution_v1_execution_proto_rawDesc), len(file_mecatl_execution_v1_execution_proto_rawDesc))) + }) + return file_mecatl_execution_v1_execution_proto_rawDescData +} + +var file_mecatl_execution_v1_execution_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_mecatl_execution_v1_execution_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_mecatl_execution_v1_execution_proto_goTypes = []any{ + (FileOperation)(0), // 0: mecatl.execution.v1.FileOperation + (CommandState)(0), // 1: mecatl.execution.v1.CommandState + (*EnvironmentRef)(nil), // 2: mecatl.execution.v1.EnvironmentRef + (*Owner)(nil), // 3: mecatl.execution.v1.Owner + (*RequestContext)(nil), // 4: mecatl.execution.v1.RequestContext + (*ValidateProfileRequest)(nil), // 5: mecatl.execution.v1.ValidateProfileRequest + (*ValidateProfileResponse)(nil), // 6: mecatl.execution.v1.ValidateProfileResponse + (*EnsureEnvironmentRequest)(nil), // 7: mecatl.execution.v1.EnsureEnvironmentRequest + (*EnsureEnvironmentResponse)(nil), // 8: mecatl.execution.v1.EnsureEnvironmentResponse + (*AttachEnvironmentRequest)(nil), // 9: mecatl.execution.v1.AttachEnvironmentRequest + (*AttachEnvironmentResponse)(nil), // 10: mecatl.execution.v1.AttachEnvironmentResponse + (*ReleaseReferenceRequest)(nil), // 11: mecatl.execution.v1.ReleaseReferenceRequest + (*RetireEnvironmentRequest)(nil), // 12: mecatl.execution.v1.RetireEnvironmentRequest + (*FileRequest)(nil), // 13: mecatl.execution.v1.FileRequest + (*FileInfo)(nil), // 14: mecatl.execution.v1.FileInfo + (*GrepMatch)(nil), // 15: mecatl.execution.v1.GrepMatch + (*FileResponse)(nil), // 16: mecatl.execution.v1.FileResponse + (*CommandStartRequest)(nil), // 17: mecatl.execution.v1.CommandStartRequest + (*CommandStartResponse)(nil), // 18: mecatl.execution.v1.CommandStartResponse + (*CommandQueryRequest)(nil), // 19: mecatl.execution.v1.CommandQueryRequest + (*CommandStatusResponse)(nil), // 20: mecatl.execution.v1.CommandStatusResponse + (*ErrorDetail)(nil), // 21: mecatl.execution.v1.ErrorDetail + (*timestamppb.Timestamp)(nil), // 22: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 23: google.protobuf.Empty +} +var file_mecatl_execution_v1_execution_proto_depIdxs = []int32{ + 2, // 0: mecatl.execution.v1.RequestContext.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 1: mecatl.execution.v1.RequestContext.owner:type_name -> mecatl.execution.v1.Owner + 3, // 2: mecatl.execution.v1.EnsureEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 3: mecatl.execution.v1.EnsureEnvironmentResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 22, // 4: mecatl.execution.v1.EnsureEnvironmentResponse.grant_expires_at:type_name -> google.protobuf.Timestamp + 4, // 5: mecatl.execution.v1.AttachEnvironmentRequest.context:type_name -> mecatl.execution.v1.RequestContext + 2, // 6: mecatl.execution.v1.AttachEnvironmentResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 22, // 7: mecatl.execution.v1.AttachEnvironmentResponse.grant_expires_at:type_name -> google.protobuf.Timestamp + 4, // 8: mecatl.execution.v1.ReleaseReferenceRequest.context:type_name -> mecatl.execution.v1.RequestContext + 4, // 9: mecatl.execution.v1.RetireEnvironmentRequest.context:type_name -> mecatl.execution.v1.RequestContext + 4, // 10: mecatl.execution.v1.FileRequest.context:type_name -> mecatl.execution.v1.RequestContext + 0, // 11: mecatl.execution.v1.FileRequest.operation:type_name -> mecatl.execution.v1.FileOperation + 22, // 12: mecatl.execution.v1.FileInfo.mod_time:type_name -> google.protobuf.Timestamp + 14, // 13: mecatl.execution.v1.FileResponse.info:type_name -> mecatl.execution.v1.FileInfo + 14, // 14: mecatl.execution.v1.FileResponse.entries:type_name -> mecatl.execution.v1.FileInfo + 15, // 15: mecatl.execution.v1.FileResponse.matches:type_name -> mecatl.execution.v1.GrepMatch + 4, // 16: mecatl.execution.v1.CommandStartRequest.context:type_name -> mecatl.execution.v1.RequestContext + 1, // 17: mecatl.execution.v1.CommandStartResponse.state:type_name -> mecatl.execution.v1.CommandState + 20, // 18: mecatl.execution.v1.CommandStartResponse.result:type_name -> mecatl.execution.v1.CommandStatusResponse + 4, // 19: mecatl.execution.v1.CommandQueryRequest.context:type_name -> mecatl.execution.v1.RequestContext + 1, // 20: mecatl.execution.v1.CommandStatusResponse.state:type_name -> mecatl.execution.v1.CommandState + 5, // 21: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:input_type -> mecatl.execution.v1.ValidateProfileRequest + 7, // 22: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:input_type -> mecatl.execution.v1.EnsureEnvironmentRequest + 9, // 23: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:input_type -> mecatl.execution.v1.AttachEnvironmentRequest + 11, // 24: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:input_type -> mecatl.execution.v1.ReleaseReferenceRequest + 12, // 25: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:input_type -> mecatl.execution.v1.RetireEnvironmentRequest + 13, // 26: mecatl.execution.v1.ExecutionProviderService.Files:input_type -> mecatl.execution.v1.FileRequest + 17, // 27: mecatl.execution.v1.ExecutionProviderService.StartCommand:input_type -> mecatl.execution.v1.CommandStartRequest + 19, // 28: mecatl.execution.v1.ExecutionProviderService.CommandStatus:input_type -> mecatl.execution.v1.CommandQueryRequest + 19, // 29: mecatl.execution.v1.ExecutionProviderService.CancelCommand:input_type -> mecatl.execution.v1.CommandQueryRequest + 6, // 30: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:output_type -> mecatl.execution.v1.ValidateProfileResponse + 8, // 31: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:output_type -> mecatl.execution.v1.EnsureEnvironmentResponse + 10, // 32: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:output_type -> mecatl.execution.v1.AttachEnvironmentResponse + 23, // 33: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:output_type -> google.protobuf.Empty + 23, // 34: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:output_type -> google.protobuf.Empty + 16, // 35: mecatl.execution.v1.ExecutionProviderService.Files:output_type -> mecatl.execution.v1.FileResponse + 18, // 36: mecatl.execution.v1.ExecutionProviderService.StartCommand:output_type -> mecatl.execution.v1.CommandStartResponse + 20, // 37: mecatl.execution.v1.ExecutionProviderService.CommandStatus:output_type -> mecatl.execution.v1.CommandStatusResponse + 20, // 38: mecatl.execution.v1.ExecutionProviderService.CancelCommand:output_type -> mecatl.execution.v1.CommandStatusResponse + 30, // [30:39] is the sub-list for method output_type + 21, // [21:30] is the sub-list for method input_type + 21, // [21:21] is the sub-list for extension type_name + 21, // [21:21] is the sub-list for extension extendee + 0, // [0:21] is the sub-list for field type_name +} + +func init() { file_mecatl_execution_v1_execution_proto_init() } +func file_mecatl_execution_v1_execution_proto_init() { + if File_mecatl_execution_v1_execution_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_mecatl_execution_v1_execution_proto_rawDesc), len(file_mecatl_execution_v1_execution_proto_rawDesc)), + NumEnums: 2, + NumMessages: 20, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_mecatl_execution_v1_execution_proto_goTypes, + DependencyIndexes: file_mecatl_execution_v1_execution_proto_depIdxs, + EnumInfos: file_mecatl_execution_v1_execution_proto_enumTypes, + MessageInfos: file_mecatl_execution_v1_execution_proto_msgTypes, + }.Build() + File_mecatl_execution_v1_execution_proto = out.File + file_mecatl_execution_v1_execution_proto_goTypes = nil + file_mecatl_execution_v1_execution_proto_depIdxs = nil +} diff --git a/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go b/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go new file mode 100644 index 0000000000..e7826e1923 --- /dev/null +++ b/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go @@ -0,0 +1,433 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: LicenseRef-Stacklok-Proprietary + +// Private, mutually authenticated execution-provider protocol. This is not part +// of the public Harness API. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc (unknown) +// source: mecatl/execution/v1/execution.proto + +package executionv1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + emptypb "google.golang.org/protobuf/types/known/emptypb" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + ExecutionProviderService_ValidateProfile_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ValidateProfile" + ExecutionProviderService_EnsureEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/EnsureEnvironment" + ExecutionProviderService_AttachEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/AttachEnvironment" + ExecutionProviderService_ReleaseReference_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReleaseReference" + ExecutionProviderService_RetireEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RetireEnvironment" + ExecutionProviderService_Files_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/Files" + ExecutionProviderService_StartCommand_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/StartCommand" + ExecutionProviderService_CommandStatus_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CommandStatus" + ExecutionProviderService_CancelCommand_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CancelCommand" +) + +// ExecutionProviderServiceClient is the client API for ExecutionProviderService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ExecutionProviderServiceClient interface { + ValidateProfile(ctx context.Context, in *ValidateProfileRequest, opts ...grpc.CallOption) (*ValidateProfileResponse, error) + EnsureEnvironment(ctx context.Context, in *EnsureEnvironmentRequest, opts ...grpc.CallOption) (*EnsureEnvironmentResponse, error) + AttachEnvironment(ctx context.Context, in *AttachEnvironmentRequest, opts ...grpc.CallOption) (*AttachEnvironmentResponse, error) + ReleaseReference(ctx context.Context, in *ReleaseReferenceRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + RetireEnvironment(ctx context.Context, in *RetireEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + Files(ctx context.Context, in *FileRequest, opts ...grpc.CallOption) (*FileResponse, error) + StartCommand(ctx context.Context, in *CommandStartRequest, opts ...grpc.CallOption) (*CommandStartResponse, error) + CommandStatus(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) + CancelCommand(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) +} + +type executionProviderServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewExecutionProviderServiceClient(cc grpc.ClientConnInterface) ExecutionProviderServiceClient { + return &executionProviderServiceClient{cc} +} + +func (c *executionProviderServiceClient) ValidateProfile(ctx context.Context, in *ValidateProfileRequest, opts ...grpc.CallOption) (*ValidateProfileResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ValidateProfileResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_ValidateProfile_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) EnsureEnvironment(ctx context.Context, in *EnsureEnvironmentRequest, opts ...grpc.CallOption) (*EnsureEnvironmentResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(EnsureEnvironmentResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_EnsureEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) AttachEnvironment(ctx context.Context, in *AttachEnvironmentRequest, opts ...grpc.CallOption) (*AttachEnvironmentResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AttachEnvironmentResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_AttachEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ReleaseReference(ctx context.Context, in *ReleaseReferenceRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReleaseReference_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RetireEnvironment(ctx context.Context, in *RetireEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_RetireEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) Files(ctx context.Context, in *FileRequest, opts ...grpc.CallOption) (*FileResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(FileResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_Files_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) StartCommand(ctx context.Context, in *CommandStartRequest, opts ...grpc.CallOption) (*CommandStartResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CommandStartResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_StartCommand_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CommandStatus(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CommandStatusResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_CommandStatus_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CancelCommand(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CommandStatusResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_CancelCommand_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ExecutionProviderServiceServer is the server API for ExecutionProviderService service. +// All implementations must embed UnimplementedExecutionProviderServiceServer +// for forward compatibility. +type ExecutionProviderServiceServer interface { + ValidateProfile(context.Context, *ValidateProfileRequest) (*ValidateProfileResponse, error) + EnsureEnvironment(context.Context, *EnsureEnvironmentRequest) (*EnsureEnvironmentResponse, error) + AttachEnvironment(context.Context, *AttachEnvironmentRequest) (*AttachEnvironmentResponse, error) + ReleaseReference(context.Context, *ReleaseReferenceRequest) (*emptypb.Empty, error) + RetireEnvironment(context.Context, *RetireEnvironmentRequest) (*emptypb.Empty, error) + Files(context.Context, *FileRequest) (*FileResponse, error) + StartCommand(context.Context, *CommandStartRequest) (*CommandStartResponse, error) + CommandStatus(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) + CancelCommand(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) + mustEmbedUnimplementedExecutionProviderServiceServer() +} + +// UnimplementedExecutionProviderServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedExecutionProviderServiceServer struct{} + +func (UnimplementedExecutionProviderServiceServer) ValidateProfile(context.Context, *ValidateProfileRequest) (*ValidateProfileResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ValidateProfile not implemented") +} +func (UnimplementedExecutionProviderServiceServer) EnsureEnvironment(context.Context, *EnsureEnvironmentRequest) (*EnsureEnvironmentResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method EnsureEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) AttachEnvironment(context.Context, *AttachEnvironmentRequest) (*AttachEnvironmentResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AttachEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ReleaseReference(context.Context, *ReleaseReferenceRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReleaseReference not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RetireEnvironment(context.Context, *RetireEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method RetireEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) Files(context.Context, *FileRequest) (*FileResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Files not implemented") +} +func (UnimplementedExecutionProviderServiceServer) StartCommand(context.Context, *CommandStartRequest) (*CommandStartResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method StartCommand not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CommandStatus(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CommandStatus not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CancelCommand(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CancelCommand not implemented") +} +func (UnimplementedExecutionProviderServiceServer) mustEmbedUnimplementedExecutionProviderServiceServer() { +} +func (UnimplementedExecutionProviderServiceServer) testEmbeddedByValue() {} + +// UnsafeExecutionProviderServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ExecutionProviderServiceServer will +// result in compilation errors. +type UnsafeExecutionProviderServiceServer interface { + mustEmbedUnimplementedExecutionProviderServiceServer() +} + +func RegisterExecutionProviderServiceServer(s grpc.ServiceRegistrar, srv ExecutionProviderServiceServer) { + // If the following call pancis, it indicates UnimplementedExecutionProviderServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&ExecutionProviderService_ServiceDesc, srv) +} + +func _ExecutionProviderService_ValidateProfile_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ValidateProfileRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ValidateProfile(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ValidateProfile_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ValidateProfile(ctx, req.(*ValidateProfileRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_EnsureEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EnsureEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).EnsureEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_EnsureEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).EnsureEnvironment(ctx, req.(*EnsureEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_AttachEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AttachEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).AttachEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_AttachEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).AttachEnvironment(ctx, req.(*AttachEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ReleaseReference_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReleaseReferenceRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReleaseReference(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReleaseReference_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReleaseReference(ctx, req.(*ReleaseReferenceRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RetireEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RetireEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RetireEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RetireEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RetireEnvironment(ctx, req.(*RetireEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_Files_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FileRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).Files(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_Files_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).Files(ctx, req.(*FileRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_StartCommand_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CommandStartRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).StartCommand(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_StartCommand_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).StartCommand(ctx, req.(*CommandStartRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CommandStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CommandQueryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CommandStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CommandStatus_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CommandStatus(ctx, req.(*CommandQueryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CancelCommand_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CommandQueryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CancelCommand(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CancelCommand_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CancelCommand(ctx, req.(*CommandQueryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// ExecutionProviderService_ServiceDesc is the grpc.ServiceDesc for ExecutionProviderService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ExecutionProviderService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "mecatl.execution.v1.ExecutionProviderService", + HandlerType: (*ExecutionProviderServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ValidateProfile", + Handler: _ExecutionProviderService_ValidateProfile_Handler, + }, + { + MethodName: "EnsureEnvironment", + Handler: _ExecutionProviderService_EnsureEnvironment_Handler, + }, + { + MethodName: "AttachEnvironment", + Handler: _ExecutionProviderService_AttachEnvironment_Handler, + }, + { + MethodName: "ReleaseReference", + Handler: _ExecutionProviderService_ReleaseReference_Handler, + }, + { + MethodName: "RetireEnvironment", + Handler: _ExecutionProviderService_RetireEnvironment_Handler, + }, + { + MethodName: "Files", + Handler: _ExecutionProviderService_Files_Handler, + }, + { + MethodName: "StartCommand", + Handler: _ExecutionProviderService_StartCommand_Handler, + }, + { + MethodName: "CommandStatus", + Handler: _ExecutionProviderService_CommandStatus_Handler, + }, + { + MethodName: "CancelCommand", + Handler: _ExecutionProviderService_CancelCommand_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "mecatl/execution/v1/execution.proto", +} diff --git a/contracts/proto/mecatl/execution/v1/execution.proto b/contracts/proto/mecatl/execution/v1/execution.proto new file mode 100644 index 0000000000..b7be635a88 --- /dev/null +++ b/contracts/proto/mecatl/execution/v1/execution.proto @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: LicenseRef-Stacklok-Proprietary + +// Private, mutually authenticated execution-provider protocol. This is not part +// of the public Harness API. +syntax = "proto3"; + +package mecatl.execution.v1; + +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1;executionv1"; + +service ExecutionProviderService { + rpc ValidateProfile(ValidateProfileRequest) returns (ValidateProfileResponse); + rpc EnsureEnvironment(EnsureEnvironmentRequest) returns (EnsureEnvironmentResponse); + rpc AttachEnvironment(AttachEnvironmentRequest) returns (AttachEnvironmentResponse); + rpc ReleaseReference(ReleaseReferenceRequest) returns (google.protobuf.Empty); + rpc RetireEnvironment(RetireEnvironmentRequest) returns (google.protobuf.Empty); + rpc Files(FileRequest) returns (FileResponse); + rpc StartCommand(CommandStartRequest) returns (CommandStartResponse); + rpc CommandStatus(CommandQueryRequest) returns (CommandStatusResponse); + rpc CancelCommand(CommandQueryRequest) returns (CommandStatusResponse); +} + +message EnvironmentRef { string id = 1; string revision = 2; } +message Owner { string issuer = 1; string subject = 2; } +message RequestContext { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + uint64 epoch = 4; + string grant = 5; +} + +message ValidateProfileRequest { string profile = 1; } +message ValidateProfileResponse { + string profile = 1; + string digest = 2; + repeated string capabilities = 3; + int64 max_file_bytes = 4; + int64 max_command_bytes = 5; + int64 max_command_duration_millis = 6; +} +message EnsureEnvironmentRequest { string binding_id = 1; string profile = 2; Owner owner = 3; } +message EnsureEnvironmentResponse { + EnvironmentRef environment = 1; + uint64 epoch = 2; + bool ready = 3; + string grant = 4; + google.protobuf.Timestamp grant_expires_at = 5; +} +message AttachEnvironmentRequest { RequestContext context = 1; string purpose = 2; } +message AttachEnvironmentResponse { + EnvironmentRef environment = 1; + uint64 epoch = 2; + bool ready = 3; + string grant = 4; + google.protobuf.Timestamp grant_expires_at = 5; +} +message ReleaseReferenceRequest { RequestContext context = 1; } +message RetireEnvironmentRequest { RequestContext context = 1; } + +enum FileOperation { + FILE_OPERATION_UNSPECIFIED = 0; + FILE_OPERATION_READ = 1; + FILE_OPERATION_RESOLVE_AUTHORITY = 2; + FILE_OPERATION_STAT = 3; + FILE_OPERATION_CREATE = 4; + FILE_OPERATION_REPLACE = 5; + FILE_OPERATION_LIST = 6; + FILE_OPERATION_REMOVE = 7; + FILE_OPERATION_RENAME = 8; + FILE_OPERATION_COPY = 9; + FILE_OPERATION_GLOB = 10; + FILE_OPERATION_GREP = 11; +} +message FileRequest { + RequestContext context = 1; + FileOperation operation = 2; + string path = 3; + string destination = 4; + string pattern = 5; + bytes data = 6; + bytes version = 7; + int32 limit = 8; +} +message FileInfo { + string name = 1; + int64 size = 2; + uint32 mode = 3; + google.protobuf.Timestamp mod_time = 4; + bool is_dir = 5; +} +message GrepMatch { string path = 1; int32 line = 2; string text = 3; } +message FileResponse { + bytes data = 1; + bytes version = 2; + FileInfo info = 3; + repeated FileInfo entries = 4; + repeated string paths = 5; + repeated GrepMatch matches = 6; + string authority_target = 7; + string authority_workspace = 8; +} + +enum CommandState { + COMMAND_STATE_UNSPECIFIED = 0; + COMMAND_STATE_RUNNING = 1; + COMMAND_STATE_SUCCEEDED = 2; + COMMAND_STATE_FAILED = 3; + COMMAND_STATE_CANCELLED = 4; + COMMAND_STATE_FENCE_UNKNOWN = 5; +} +message CommandStartRequest { RequestContext context = 1; string command = 2; int64 timeout_millis = 3; } +message CommandStartResponse { string command_id = 1; CommandState state = 2; CommandStatusResponse result = 3; } +message CommandQueryRequest { RequestContext context = 1; string command_id = 2; int64 offset = 3; } +message CommandStatusResponse { + string command_id = 1; + CommandState state = 2; + int32 exit_code = 3; + bytes stdout = 4; + bytes stderr = 5; + int64 next_offset = 6; + bool truncated = 7; + string terminal_receipt = 8; +} + +// ErrorDetail is a stable machine-readable classification. Status messages are +// intentionally generic so backend/Kubernetes details never cross the boundary. +message ErrorDetail { string code = 1; bool retryable = 2; } diff --git a/deploy/helm/mecak8s/values.schema.json b/deploy/helm/mecak8s/values.schema.json index 66049e7396..ab7daa5a15 100644 --- a/deploy/helm/mecak8s/values.schema.json +++ b/deploy/helm/mecak8s/values.schema.json @@ -122,7 +122,7 @@ "required": ["enabled", "endpoint", "profile", "tlsSecret", "caKey", "certKey", "keyKey"], "properties": { "enabled": {"type": "boolean"}, - "endpoint": {"type": "string", "anyOf": [{"const": ""}, {"pattern": "^https://[^@/\\s#]+(?:/[^\\s?#]*)?$"}]}, + "endpoint": {"type": "string", "anyOf": [{"const": ""}, {"pattern": "^(?:[A-Za-z0-9.-]+|\\[[0-9A-Fa-f:]+\\]):[0-9]+$"}]}, "profile": {"type": "string"}, "tlsSecret": {"type": "string"}, "caKey": {"type": "string", "minLength": 1}, diff --git a/deploy/helm/mecatl-execution/templates/provider.yaml b/deploy/helm/mecatl-execution/templates/provider.yaml index bbc9bf458c..fb1bed6b5c 100644 --- a/deploy/helm/mecatl-execution/templates/provider.yaml +++ b/deploy/helm/mecatl-execution/templates/provider.yaml @@ -41,7 +41,17 @@ spec: - --owner-attester-uri-san={{ join "," .Values.provider.ownerAttesterURISANs }} - --administrator-uri-san={{ join "," .Values.provider.administratorURISANs }} ports: - - {name: https, containerPort: 8443} + - {name: grpc, containerPort: 8443} + startupProbe: + tcpSocket: {port: grpc} + failureThreshold: 30 + periodSeconds: 2 + readinessProbe: + tcpSocket: {port: grpc} + periodSeconds: 5 + livenessProbe: + tcpSocket: {port: grpc} + periodSeconds: 10 securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -71,4 +81,4 @@ metadata: spec: selector: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} ports: - - {name: https, port: {{ .Values.service.port }}, targetPort: https} + - {name: grpc, port: {{ .Values.service.port }}, targetPort: grpc} diff --git a/deploy/mecatl-execution-kind/live.sh b/deploy/mecatl-execution-kind/live.sh new file mode 100755 index 0000000000..a9332fc814 --- /dev/null +++ b/deploy/mecatl-execution-kind/live.sh @@ -0,0 +1,199 @@ +#!/bin/sh +set -eu + +# Explicit opt-in live qualification against one already-owned Kind cluster. +# Re-exec with only non-secret runtime selectors plus the credential FILE path. +if [ "${MECATL_EXECUTION_LIVE_CLEAN_ENV:-}" != 1 ]; then + exec env -i HOME="$HOME" USER="${USER:-user}" PATH="$PATH" XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" \ + DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-}" CONTAINER_HOST="${CONTAINER_HOST:-}" DOCKER_HOST="${DOCKER_HOST:-}" \ + TOOLBOX_PATH="${TOOLBOX_PATH:-}" MECATL_EXECUTION_DEV_TOOLBOX="${MECATL_EXECUTION_DEV_TOOLBOX:-}" MECATL_EXECUTION_K8S_TOOLBOX="${MECATL_EXECUTION_K8S_TOOLBOX:-}" \ + MECATL_EXECUTION_CREDENTIAL_FILE="${MECATL_EXECUTION_CREDENTIAL_FILE:-}" MECATL_EXECUTION_QUAL_STATE="${MECATL_EXECUTION_QUAL_STATE:-}" \ + MECATL_EXECUTION_LIVE_CLEAN_ENV=1 "$0" "$@" +fi +[ "$#" -eq 0 ] || { echo "usage: MECATL_EXECUTION_CREDENTIAL_FILE=/absolute/file MECATL_EXECUTION_QUAL_STATE=/owned/state $0" >&2; exit 2; } +[ -n "$MECATL_EXECUTION_CREDENTIAL_FILE" ] || { echo "MECATL_EXECUTION_CREDENTIAL_FILE is required" >&2; exit 2; } +[ -n "$MECATL_EXECUTION_QUAL_STATE" ] || { echo "MECATL_EXECUTION_QUAL_STATE is required" >&2; exit 2; } + +root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd -P) +cd "$root" +owned_root="$root/.scratch/k8s-execution" +case "${MECATL_EXECUTION_DEV_TOOLBOX:-}" in ''|dev) ;; *) echo "MECATL_EXECUTION_DEV_TOOLBOX must be empty or dev" >&2; exit 2 ;; esac +case "${MECATL_EXECUTION_K8S_TOOLBOX:-}" in ''|sre) ;; *) echo "MECATL_EXECUTION_K8S_TOOLBOX must be empty or sre" >&2; exit 2 ;; esac +state=$(CDPATH= cd -- "$MECATL_EXECUTION_QUAL_STATE" && pwd -P) || { echo "qualification state unavailable" >&2; exit 1; } +case "$state" in + "$owned_root"/*) ;; + *) echo "refusing qualification state outside owned root" >&2; exit 1 ;; +esac +ownership="$state/ownership" +[ -f "$ownership" ] && [ ! -L "$ownership" ] || { echo "owned qualification record unavailable" >&2; exit 1; } +owned_value() { + awk -F= -v key="$1" ' + $1 == key { value=substr($0, length(key)+2); count++ } + END { if (count != 1 || value == "") exit 1; print value } + ' "$ownership" +} +optional_owned_value() { + awk -F= -v key="$1" ' + $1 == key { value=substr($0, length(key)+2); count++ } + END { if (count > 1) exit 1; if (count == 1) print value } + ' "$ownership" +} +cluster=$(owned_value cluster) || { echo "owned cluster identity unavailable" >&2; exit 1; } +case "$cluster" in + [a-z0-9]|[a-z0-9]*[a-z0-9]) ;; + *) echo "owned cluster identity invalid" >&2; exit 1 ;; +esac +case "$cluster" in *[!a-z0-9-]*) echo "owned cluster identity invalid" >&2; exit 1 ;; esac +[ "${#cluster}" -le 63 ] || { echo "owned cluster identity invalid" >&2; exit 1; } +owned_namespace=$(owned_value namespace) || { echo "owned namespace unavailable" >&2; exit 1; } +[ "$owned_namespace" = execution-qualification ] || { echo "owned namespace mismatch" >&2; exit 1; } +recorded_owner=$(owned_value owner) || { echo "owned user identity unavailable" >&2; exit 1; } +[ "$recorded_owner" = "${USER:-user}" ] || { echo "owned user identity mismatch" >&2; exit 1; } +kubeconfig="$state/kubeconfig" +recorded_kubeconfig=$(owned_value kubeconfig) || { echo "owned kubeconfig identity unavailable" >&2; exit 1; } +[ "$recorded_kubeconfig" = "$kubeconfig" ] || { echo "owned kubeconfig mismatch" >&2; exit 1; } +[ -f "$kubeconfig" ] && [ ! -L "$kubeconfig" ] && [ "$(stat -c '%a' "$kubeconfig")" = 600 ] || { echo "owned kubeconfig must be a private regular file" >&2; exit 1; } +context=$(optional_owned_value context) || { echo "owned context identity invalid" >&2; exit 1; } +[ -n "$context" ] || context="kind-$cluster" +[ "$context" = "kind-$cluster" ] || { echo "owned context mismatch" >&2; exit 1; } +label=$(podman inspect "${cluster}-control-plane" --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}') +[ "$label" = "$cluster" ] || { echo "owned Kind container label mismatch" >&2; exit 1; } + +export KIND_EXPERIMENTAL_PROVIDER=podman +export KUBECONFIG=$kubeconfig +export REGISTRY_AUTH_FILE="$state/registry-auth.json" +dev() { + if [ -n "$MECATL_EXECUTION_DEV_TOOLBOX" ]; then toolbox run -c "$MECATL_EXECUTION_DEV_TOOLBOX" "$@"; else "$@"; fi +} +kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" kubectl --kubeconfig "$kubeconfig" --context "$context" "$@"; else kubectl --kubeconfig "$kubeconfig" --context "$context" "$@"; fi +} +helm_kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@"; else helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@"; fi +} +kube_jq() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" jq "$@"; else jq "$@"; fi +} +[ "$(kube config current-context)" = "$context" ] || { echo "owned kube context mismatch" >&2; exit 1; } +build_ko() { package=$1 repo=$2; dev env KIND_EXPERIMENTAL_PROVIDER=podman KO_DOCKER_REPO="$repo" ko build --local --bare "$package" | tail -n 1; } +pin_loaded() { + tagged=$1 + digest=$(podman exec "${cluster}-control-plane" ctr -n k8s.io images ls | awk -v ref="$tagged" '$1 == ref {print $3; exit}') + [ -n "$digest" ] || { echo "loaded image digest unavailable" >&2; exit 1; } + pinned="${tagged%:*}@${digest}" + podman exec "${cluster}-control-plane" ctr -n k8s.io images tag "$tagged" "$pinned" >/dev/null + printf '%s\n' "$pinned" +} +load_image() { + tagged=$1 + archive="$state/images/live-$(printf '%s' "$tagged" | sha256sum | cut -c1-16).tar" + podman save "$tagged" -o "$archive" >/dev/null + kind load image-archive "$archive" --name "$cluster" + pin_loaded "$tagged" +} + +provider_tag=$(build_ko ./cmd/mecatl-execution-provider ko.local/mecatl-execution-provider) +agent_tag=$(build_ko ./cmd/mecak8s ko.local/mecak8s) +podman image exists docker.io/library/golang:1.27 || podman pull docker.io/library/golang:1.27 >/dev/null +go_digest=$(podman image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') +go_image="docker.io/library/golang@${go_digest}" +workload_tag=localhost/mecatl-execution-workload:e2e +podman build --build-arg GO_IMAGE="$go_image" -f "$root/build/execution-workload/Dockerfile" -t "$workload_tag" "$root" >/dev/null +provider_image=$(load_image "$provider_tag") +agent_image=$(load_image "$agent_tag") +workload_image=$(load_image "$workload_tag") +printf 'provider=%s\nagent=%s\nworkload=%s\ngo_base=%s\n' "$provider_image" "$agent_image" "$workload_image" "$go_image" >"$state/images/live-proof" + +# This owned Kind fixture uses the workload image as local-path's helper image so +# rootless Podman never needs a second unpinned helper pull. Keep it in lockstep. +kube -n local-path-storage get configmap local-path-config -o json \ + | kube_jq --arg image "$workload_image" '.data["helperPod.yaml"] |= sub("image: [^\\n]+"; "image: " + $image)' \ + | kube replace -f - + +# Qualification identities are synthetic and intentionally short-lived. Rotate +# them before any rollout so an owned cluster can be safely reused hours later. +dev env -i HOME="$HOME" PATH="$PATH" go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/pki "$state/pki" +kube -n execution-qualification create secret generic execution-provider-tls \ + --from-file=ca.crt="$state/pki/ca.crt" --from-file=tls.crt="$state/pki/provider.crt" --from-file=tls.key="$state/pki/provider.key" --dry-run=client -o yaml | kube apply -f - +kube -n execution-qualification create secret generic execution-client-tls \ + --from-file=ca.crt="$state/pki/ca.crt" --from-file=tls.crt="$state/pki/mecak8s.crt" --from-file=tls.key="$state/pki/mecak8s.key" --dry-run=client -o yaml | kube apply -f - +kube -n execution-qualification create secret generic execution-intruder-tls \ + --from-file=ca.crt="$state/pki/ca.crt" --from-file=tls.crt="$state/pki/intruder.crt" --from-file=tls.key="$state/pki/intruder.key" --dry-run=client -o yaml | kube apply -f - +kube -n execution-qualification create secret generic execution-grant-key --from-file=key.pem="$state/pki/grant-key.pem" --dry-run=client -o yaml | kube apply -f - +kube -n execution-qualification create secret generic oidc-fixture-identity \ + --from-file=tls.crt="$state/pki/oidc.crt" --from-file=tls.key="$state/pki/oidc.key" --from-file=jwt-key.pem="$state/pki/oidc-jwt-key.pem" --dry-run=client -o yaml | kube apply -f - +kube -n execution-qualification rollout restart deployment/oidc-issuer +kube -n execution-qualification rollout status deployment/oidc-issuer --timeout=180s + +# First qualify the migrated typed-gRPC path under the deterministic mock model. +kube -n execution-qualification create configmap execution-mock --from-file=mock-script.json="$root/deploy/mecatl-execution-kind/mock-script.json" --dry-run=client -o yaml | kube apply -f - +helm_kube upgrade --install mecatl-execution "$root/deploy/helm/mecatl-execution" --namespace execution-qualification \ + --set fullnameOverride=mecatl-execution --set-string provider.image="$provider_image" --set provider.imagePullPolicy=IfNotPresent \ + --set provider.tlsSecretName=execution-provider-tls --set provider.grantSigningSecretName=execution-grant-key \ + --set 'provider.clientURISANs={spiffe://mecatl.test/client/mecak8s,spiffe://mecatl.test/client/intruder}' \ + --set 'provider.ownerAttesterURISANs={spiffe://mecatl.test/client/mecak8s,spiffe://mecatl.test/client/intruder}' \ + --set 'provider.administratorURISANs={spiffe://mecatl.test/client/mecak8s}' --set-string profiles.go.image="$workload_image" --reuse-values --wait --timeout=4m +kube -n execution-qualification rollout restart deployment/mecatl-execution +kube -n execution-qualification rollout status deployment/mecatl-execution --timeout=240s +helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" --wait --timeout=5m +kube -n execution-qualification rollout restart deployment/mecak8s +kube -n execution-qualification rollout status deployment/mecak8s --timeout=240s +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_QUAL_STATE="$state" \ + go test -tags kind_execution_e2e -run '^TestKindExecutionQualification$' -count=1 -timeout=15m ./e2e/k8s_execution + +secret="mecak8s-live-$(date -u +%Y%m%d%H%M%S)-$$" +receipt="$state/live-secret-$secret.receipt.json" +restore_needed=0 +restore() { + status=$? + trap - EXIT INT TERM + cleanup_failed=0 + mock_restored=0 + secret_cleaned=0 + if [ "$restore_needed" -eq 1 ]; then + if helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" --wait --timeout=5m >/dev/null 2>&1; then + mock_restored=1 + else + echo "mock profile restoration failed; cluster retained for inspection" >&2 + cleanup_failed=1 + fi + fi + if [ -f "$receipt" ]; then + if dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" \ + go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/credentialloader delete "$kubeconfig" "$context" "$secret" "$receipt"; then + secret_cleaned=1 + else + echo "UID-pinned run-scoped provider Secret cleanup pending; cluster retained for inspection" >&2 + cleanup_failed=1 + fi + else + secret_cleaned=1 + fi + if [ "$mock_restored" -eq 1 ] && [ "$secret_cleaned" -eq 1 ]; then + echo "cleanup verification passed: mock harness restored; run-scoped Secret receipt cleared" + fi + if [ "$status" -eq 0 ] && [ "$cleanup_failed" -ne 0 ]; then + status=1 + fi + exit "$status" +} +trap restore EXIT INT TERM + +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_CREDENTIAL_FILE="$MECATL_EXECUTION_CREDENTIAL_FILE" \ + go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/credentialloader stage "$kubeconfig" "$context" "$secret" "$receipt" +restore_needed=1 +helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" \ + --set mockProvider=false --set security.allowUnsafeRealProvider=true --set defaultProvider=openrouter --set-string model=anthropic/claude-haiku-4.5 --set maxRunTokens=32000 \ + --set-json 'extraArgs=["--no-soul","--no-user-model","--permissions-conventional=false","--agents-conventional=false"]' \ + --set extraVolumes=null --set extraVolumeMounts=null \ + --set-string extraEnv[0].name=OPENROUTER_API_KEY --set-string extraEnv[0].valueFrom.secretKeyRef.name="$secret" --set-string extraEnv[0].valueFrom.secretKeyRef.key=OPENROUTER_API_KEY \ + --wait --timeout=5m +kube -n execution-qualification rollout status deployment/mecak8s --timeout=240s +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_QUAL_STATE="$state" MECATL_EXECUTION_LIVE=1 MECATL_EXECUTION_LIVE_SECRET="$secret" MECATL_EXECUTION_LIVE_CLUSTER="$cluster" \ + go test -tags kind_execution_e2e -run '^TestKindExecutionLiveQualification$' -count=1 -timeout=10m ./e2e/k8s_execution + +echo "live qualification passed; sanitized summary: $state/live-summary.json" +echo "qualification cluster retained: $cluster" diff --git a/deploy/mecatl-execution-kind/mecak8s-values.yaml b/deploy/mecatl-execution-kind/mecak8s-values.yaml index dc684e06f1..07d7cef7f5 100644 --- a/deploy/mecatl-execution-kind/mecak8s-values.yaml +++ b/deploy/mecatl-execution-kind/mecak8s-values.yaml @@ -8,7 +8,7 @@ redis: enabled: true execution: enabled: true - endpoint: https://mecatl-execution.execution-qualification.svc.cluster.local:8443 + endpoint: mecatl-execution.execution-qualification.svc.cluster.local:8443 profile: go tlsSecret: execution-client-tls caKey: ca.crt diff --git a/deploy/mecatl-execution-kind/run.sh b/deploy/mecatl-execution-kind/run.sh index 765ec169fa..ae289962a5 100755 --- a/deploy/mecatl-execution-kind/run.sh +++ b/deploy/mecatl-execution-kind/run.sh @@ -12,7 +12,10 @@ if [ "${MECATL_EXECUTION_QUAL_CLEAN_ENV:-}" != 1 ]; then fi [ "$#" -eq 0 ] || { echo "usage: $0" >&2; exit 2; } -root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +case "${MECATL_EXECUTION_DEV_TOOLBOX:-}" in ''|dev) ;; *) echo "MECATL_EXECUTION_DEV_TOOLBOX must be empty or dev" >&2; exit 2 ;; esac +case "${MECATL_EXECUTION_K8S_TOOLBOX:-}" in ''|sre) ;; *) echo "MECATL_EXECUTION_K8S_TOOLBOX must be empty or sre" >&2; exit 2 ;; esac +root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd -P) +cd "$root" run_id=$(date -u +%Y%m%d%H%M%S)-$$ cluster="mecatl-execution-qual-$run_id" context="kind-$cluster" diff --git a/docs/adr/0027-cloud-native.md b/docs/adr/0027-cloud-native.md index d8de5f32b7..29618909bb 100644 --- a/docs/adr/0027-cloud-native.md +++ b/docs/adr/0027-cloud-native.md @@ -981,11 +981,11 @@ rather than papered over. | 72 | Per-generation isolated Redis follow client and bounded connection pool | each redisstore `clientGeneration`, owned by `clientGenerations` | one follow client per live or retiring credential generation; `PoolSize` and `MaxActiveConns` use the configured process-local follow-pool size | a rejected candidate closes both clients; generation retirement closes the pair after its leases release. `Store.Close` may asynchronously force-close only the follow client when an admitted read misses the cooperative join bound, while durability clients retain their lease-safe close rule | reconstructible from Redis connection settings and mounted credential files. A restart builds and probes a new pair; it retains no watch position or session authority | `internal/adapter/redisstore/generation.go` (`clientPair`, `buildClientPair`, `forceCloseFollow`); `internal/adapter/redisstore/reload.go` (`reloadCandidate`); ADR 0330 | | 73 | Redis follower admission, cancellation, and join registry | redisstore `Store` owns one `followerRegistry` | process; one entry per admitted `ReadAfter` iterator with `Follow:true`, capped by `MaxFollowers` | iterator release removes its entry on every exit. `Store.Close` linearizes against admission, rejects new followers, cancels the active snapshot, waits cooperatively, and retains bounded late-cleanup ownership after a follow-client force-close | reset-by-design and reattached by the client. The registry contains no cursor or durable session state; after restart the client resumes with its last processed cursor and unchanged filter | `internal/adapter/redisstore/followers.go` (`followerRegistry`); `internal/adapter/redisstore/cursoreventlog.go` (`ReadAfter`); `internal/adapter/redisstore/redisstore.go` (`Close`); ADR 0330 | | 74 | Prompt-free persisted-ask continuation contexts and detached relay join registry | `internal/adapter/server.Service` owns the post-acceptance context, exact resumed `agent.Run`, recorder/relay goroutine, and `detachedControlWG`; the unary caller owns only work before atomic acceptance | one process-local entry per accepted persisted ordinary-ask resolution | relay admission adds to the Service wait group before the run is published and is fenced before shutdown wait. `Service.Close` first closes admission, marks every already-durable awaiting handoff `preserveDurable` and every cancellation `cancelSignaled` under the persistence barrier, then cancels contexts/runs and waits with the engine-close bound. Graceful drain and lease loss use the same mark-before-signal order. A preserved terminal relay cannot overwrite the awaiting snapshot | **derived from durable state / reset local ownership**: the context, run, recorder, marker bits, and join count disappear on process loss. A replacement acquires the distributed session lease, reloads the authoritative snapshot after acquisition, and repeats exact run/state/ask/origin/placement validation before rehydrating. See List 2 row 47 | `internal/adapter/server/service.go` (`ResolveRunAsk`, `validatePersistedRunAsk`, `reserveDetachedControlRelay`, `relayDetachedControlRun`, `Close`, `GracefulDrain`, `cancelRegisteredRunState`); ADR 0347 | -| 75 | Experimental execution-provider HTTPS server and Kubernetes REST transports | `cmd/mecatl-execution-provider` | provider process | process cancellation shuts down the HTTP server and closes idle client connections; Kubernetes clients own no separate application goroutine | **derive** from explicit flags, mounted mTLS material, in-cluster configuration, and the live Kubernetes API. Disabled `mecak8s` composition creates none of these resources | `cmd/mecatl-execution-provider/main.go`; `internal/adapter/executioncontroller/handler.go` (`Handler`) | +| 75 | Experimental execution-provider bounded mTLS gRPC server and Kubernetes REST transports | `cmd/mecatl-execution-provider` | provider process | process cancellation gracefully stops the gRPC server (then hard-stops after the shutdown bound); Kubernetes clients own no separate application goroutine | **derive** from explicit flags, mounted mTLS material, in-cluster configuration, and the live Kubernetes API. Disabled `mecak8s` composition creates none of these resources | `cmd/mecatl-execution-provider/main.go`; `internal/adapter/executioncontroller/handler.go` (`Handler`) | | 76 | ExecutionEnvironment informer, rate-limited workqueue, and controller worker | `executioncontroller.Reconciler` | provider process, namespace-scoped | the Run context stops the informer and worker; `Run` shuts down the queue before return | **derive** by listing and watching the external Kubernetes CRs after restart. Startup marks any persisted active operation `FenceUnknown` before admitting mutations | `internal/adapter/executioncontroller/controller.go` (`Reconciler`, `Initialize`, `Run`) | | 77 | ExecutionEnvironment CR, retained PVC, executor Pod, and one status-held active-operation claim | Kubernetes API and storage provider; the execution controller reconciles them | logical remote environment | explicit retirement removes a proven-terminal Pod but retains the PVC. CR deletion remains finalizer-blocked until retirement and quiescence; this candidate has no PVC-delete path | **external Kubernetes state**: CR status, PVC data, and current Pod references survive provider or `mecak8s` process restart according to cluster/storage durability. An interrupted active-operation claim becomes fail-closed `FenceUnknown`, not reconstructed authority | `internal/adapter/executioncontroller/controller.go` (`Reconcile`, `reconcileRetiring`, `reconcileDeletion`); `internal/adapter/executioncontroller/store.go` (`execute`) | | 78 | In-memory grant signer/verifier configuration, short-lived signed grants, and revocation maps | execution-provider `Handler`; each remote `grantContext` caches one grant | provider process / bound session | grants expire; client reattachment refreshes them. Signing and verifier values leave process memory at shutdown. Full durable key rotation and revocation are not implemented | **reset-by-design / derive** from mounted signing configuration and exact external environment state. A restart does not preserve in-memory nonce/key revocation, so this draft is not production rotation or revocation | `internal/adapter/executioncontroller/handler.go` (`GrantSigner`, `HandlerConfig`, `sign`); `internal/executionenv/grant.go`; `internal/adapter/executionclient/client.go` (`grantContext`) | -| 79 | Mecak8s execution-provider mTLS client transport and per-binding remote Workspace/runner | `executionclient.Client` and the Service placement registry | process transport / session environment | `Client.Close` closes idle connections; session engines and environments follow existing Service teardown. Each authority lookup and file/command call has caller or adapter bounds | **derive** by exact reattachment from the persisted `session.EnvironmentRef` and owner-bound provider state. Missing, changed, or fenced environments fail closed with no local fallback | `internal/adapter/executionclient/client.go` (`Client`, `Provider`, `workspace`, `runner`) | +| 79 | Mecak8s execution-provider bounded mTLS gRPC client connection and per-binding remote Workspace/runner | `executionclient.Client` and the Service placement registry | process transport / session environment | `Client.Close` closes its gRPC connection; session engines and environments follow existing Service teardown. Each authority lookup and file/command call has caller or adapter bounds | **derive** by exact reattachment from the persisted `session.EnvironmentRef` and owner-bound provider state. Missing, changed, or fenced environments fail closed with no local fallback | `internal/adapter/executionclient/client.go` (`Client`, `Provider`, `workspace`, `runner`) | **Experimental native-execution re-audit (List 1 / List 2).** List 1 rows 75–79 inventory the provider server and transports, controller informer and queue, diff --git a/docs/adr/0349-native-kubernetes-execution.md b/docs/adr/0349-native-kubernetes-execution.md index a0ea302551..06cc92c637 100644 --- a/docs/adr/0349-native-kubernetes-execution.md +++ b/docs/adr/0349-native-kubernetes-execution.md @@ -62,9 +62,11 @@ Shell writes retain their documented ability to bypass Workspace CAS stronger remote filesystem transaction. Introduce a side-effect-free configuration/profile validation operation distinct from idempotent -allocation. Candidate private operations include profile validation, ensure, exact attach, command -start/stream/status/cancel, and retirement. Their precise Go signatures and private wire protocol -are deliberately undecided and block promotion of this ADR. +allocation. The candidate private wire transport is typed protobuf gRPC over mandatory +mTLS (not HTTP/JSON and not a JSON/bytes envelope). Candidate operations include profile +validation, ensure, exact attach, file operations, command start/stream/status/cancel, and +retirement. Message details, lifecycle semantics, and precise Go signatures remain deliberately +undecided in this planning ADR and block promotion; the draft implementation is not approval. ### 3. Make installation and connection explicitly optional diff --git a/docs/architecture.md b/docs/architecture.md index 90a227b6c1..e284618175 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -74,7 +74,7 @@ This page is the overview and router; the big picture and the layering rule are The candidate native execution path is opt-in and keeps the controller boundary outside `mecak8s`. `cmd/mecatl-execution-provider` owns the private authenticated -API and the namespaced `ExecutionEnvironment` reconciler. It creates a retained +API (`mecatl.execution.v1.ExecutionProviderService`, protocol `execution-grpc/1`) over bounded mTLS gRPC and the namespaced `ExecutionEnvironment` reconciler. It creates a retained PVC and a credential-free executor Pod from an operator-defined, digest-pinned profile. `cmd/mecatl-executor` serves bounded file operations and foreground commands inside `/workspace`; the host-side `internal/adapter/executionclient` diff --git a/docs/design/IMPLEMENTATION-NOTES.md b/docs/design/IMPLEMENTATION-NOTES.md index a86be74b0d..a3549e831e 100644 --- a/docs/design/IMPLEMENTATION-NOTES.md +++ b/docs/design/IMPLEMENTATION-NOTES.md @@ -6540,7 +6540,8 @@ reaches the checker). Default `false` is the byte-identical un-routed posture ta The opt-in candidate composes a separately deployed provider through `internal/adapter/executionclient/client.go` (`Provider`). The private -`/internal/execution/v1` mTLS API authenticates an exact URI SAN, binds short-lived +`mecatl.execution.v1.ExecutionProviderService` gRPC API uses private protocol +`execution-grpc/1`. It requires mTLS, authenticates exactly one allowlisted URI SAN, binds short-lived Ed25519 grants to client, owner hash, session binding, environment revision, epoch, and a closed operation set, and sends credential-free requests to the workload helper. `internal/adapter/executioncontroller/handler.go` (`Handler`) diff --git a/e2e/k8s_execution/fixture/credentialloader/main.go b/e2e/k8s_execution/fixture/credentialloader/main.go new file mode 100644 index 0000000000..7f69f1882e --- /dev/null +++ b/e2e/k8s_execution/fixture/credentialloader/main.go @@ -0,0 +1,279 @@ +//go:build kind_execution_e2e + +// Command credentialloader is the only qualification component permitted to read +// a real provider credential. It creates a new, narrowly labelled Secret and +// never reads Secret data back from Kubernetes. +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilvalidation "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + namespace = "execution-qualification" + secretKey = "OPENROUTER_API_KEY" + maxCredentialBytes = 16 << 10 + maxReceiptBytes = 4 << 10 +) + +type cleanupReceipt struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + UID types.UID `json:"uid"` + Context string `json:"context"` + Kubeconfig string `json:"kubeconfig"` +} + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, errorMessage(err)) + os.Exit(1) + } +} + +func errorMessage(err error) string { + return "credential loader: " + classify(err) +} + +func run(args []string) error { + if len(args) != 5 { + return errors.New("usage") + } + mode, kubeconfig, contextName, name, receiptPath := args[0], args[1], args[2], args[3], args[4] + if (mode != "stage" && mode != "delete") || !strings.HasPrefix(name, "mecak8s-live-") || len(utilvalidation.IsDNS1123Subdomain(name)) != 0 { + return errors.New("invalid arguments") + } + kubeconfig, receiptPath, err := validatePrivatePaths(kubeconfig, receiptPath, name) + if err != nil { + return err + } + + // Resolve kube configuration (including any auth plugin) and prove API access + // before the credential file is opened. + cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, + &clientcmd.ConfigOverrides{CurrentContext: contextName}, + ).ClientConfig() + if err != nil { + return fmt.Errorf("kube configuration: %w", err) + } + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return fmt.Errorf("kube client: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if _, err = client.Discovery().ServerVersion(); err != nil { + return fmt.Errorf("kube access: %w", err) + } + + secrets := client.CoreV1().Secrets(namespace) + if mode == "delete" { + if err := deleteStagedSecret(ctx, secrets, receiptPath, kubeconfig, contextName, name); err != nil { + return err + } + fmt.Println(`{"status":"deleted"}`) + return nil + } + path := os.Getenv("MECATL_EXECUTION_CREDENTIAL_FILE") + credential, err := loadCredential(path) + if err != nil { + return err + } + defer clear(credential) + if err := stageSecret(ctx, secrets, receiptPath, kubeconfig, contextName, name, credential); err != nil { + return err + } + fmt.Println(`{"status":"staged","provider":"openrouter"}`) + return nil +} + +func stageSecret(ctx context.Context, secrets v1.SecretInterface, receiptPath, kubeconfig, contextName, name string, credential []byte) error { + created, err := secrets.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: map[string]string{"app.kubernetes.io/managed-by": "mecatl-execution-live-qualification"}}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{secretKey: credential}, + }, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("secret stage: %w", err) + } + if created == nil || created.UID == "" { + return errors.New("secret staged without UID; cleanup pending") + } + receipt := cleanupReceipt{Name: name, Namespace: namespace, UID: created.UID, Context: contextName, Kubeconfig: kubeconfig} + if err := writeReceipt(receiptPath, receipt); err != nil { + uid := receipt.UID + _ = secrets.Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + return errors.New("cleanup receipt creation failed; cleanup status ambiguous") + } + return nil +} + +func deleteStagedSecret(ctx context.Context, secrets v1.SecretInterface, receiptPath, kubeconfig, contextName, name string) error { + receipt, err := readReceipt(receiptPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("cleanup receipt: %w", err) + } + if receipt.Name != name || receipt.Namespace != namespace || receipt.Context != contextName || receipt.Kubeconfig != kubeconfig { + return errors.New("cleanup receipt identity mismatch") + } + if receipt.UID == "" { + return errors.New("secret staged without UID; cleanup pending") + } + uid := receipt.UID + err = secrets.Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("secret cleanup: %w", err) + } + if err := os.Remove(receiptPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return errors.New("cleanup receipt removal failed") + } + return nil +} + +func validatePrivatePaths(kubeconfig, receiptPath, name string) (string, string, error) { + if !filepath.IsAbs(kubeconfig) || !filepath.IsAbs(receiptPath) { + return "", "", errors.New("state paths must be absolute") + } + kubeconfig, err := filepath.EvalSymlinks(kubeconfig) + if err != nil { + return "", "", errors.New("kubeconfig unavailable") + } + kubeInfo, err := os.Stat(kubeconfig) + if err != nil || !kubeInfo.Mode().IsRegular() || kubeInfo.Mode().Perm()&0o077 != 0 { + return "", "", errors.New("kubeconfig must be a private regular file") + } + stateDir, err := filepath.EvalSymlinks(filepath.Dir(receiptPath)) + if err != nil || stateDir != filepath.Dir(kubeconfig) { + return "", "", errors.New("receipt path must be in kubeconfig state directory") + } + info, err := os.Stat(stateDir) + if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 { + return "", "", errors.New("state directory must be private") + } + base := filepath.Base(receiptPath) + if base != "live-secret-"+name+".receipt.json" || filepath.Clean(receiptPath) != filepath.Join(stateDir, base) { + return "", "", errors.New("invalid receipt path") + } + return kubeconfig, filepath.Join(stateDir, base), nil +} + +func writeReceipt(path string, receipt cleanupReceipt) error { + blob, err := json.Marshal(receipt) + if err != nil { + return err + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + ok := false + defer func() { + _ = f.Close() + if !ok { + _ = os.Remove(path) + } + }() + if _, err := f.Write(append(blob, '\n')); err != nil { + return err + } + if err := f.Sync(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + ok = true + return nil +} + +func readReceipt(path string) (cleanupReceipt, error) { + var receipt cleanupReceipt + f, err := os.Open(path) + if err != nil { + return receipt, err + } + defer f.Close() + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 || info.Size() > maxReceiptBytes { + return receipt, errors.New("receipt must be a bounded private regular file") + } + dec := json.NewDecoder(io.LimitReader(f, maxReceiptBytes+1)) + dec.DisallowUnknownFields() + if err := dec.Decode(&receipt); err != nil { + return receipt, errors.New("invalid cleanup receipt") + } + var extra any + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + return receipt, errors.New("invalid cleanup receipt") + } + return receipt, nil +} + +func loadCredential(path string) ([]byte, error) { + if path == "" || !filepath.IsAbs(path) { + return nil, errors.New("credential file must be an absolute path") + } + f, err := os.Open(path) + if err != nil { + return nil, errors.New("credential file unavailable") + } + defer f.Close() + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() { + return nil, errors.New("credential file must be regular") + } + if info.Mode().Perm()&0o077 != 0 { + return nil, errors.New("credential file permissions must exclude group and others") + } + b, err := io.ReadAll(io.LimitReader(f, maxCredentialBytes+1)) + if err != nil { + return nil, errors.New("credential file read failed") + } + defer clear(b) + if len(b) > maxCredentialBytes { + return nil, errors.New("credential file exceeds limit") + } + trimmed := bytes.TrimSpace(b) + if len(trimmed) == 0 { + return nil, errors.New("credential file is empty") + } + return append([]byte(nil), trimmed...), nil +} + +func classify(err error) string { + s := err.Error() + for _, allowed := range []string{ + "usage", "invalid arguments", "state paths must be absolute", "kubeconfig unavailable", "kubeconfig must be a private regular file", + "receipt path must be in kubeconfig state directory", "state directory must be private", "invalid receipt path", + "credential file must be an absolute path", "credential file unavailable", "credential file must be regular", + "credential file permissions must exclude group and others", "credential file read failed", "credential file exceeds limit", "credential file is empty", + "cleanup receipt creation failed; cleanup status ambiguous", "secret staged without UID; cleanup pending", "cleanup receipt identity mismatch", + } { + if s == allowed { + return allowed + } + } + return "operation failed (details redacted)" +} diff --git a/e2e/k8s_execution/fixture/credentialloader/main_test.go b/e2e/k8s_execution/fixture/credentialloader/main_test.go new file mode 100644 index 0000000000..3a49282812 --- /dev/null +++ b/e2e/k8s_execution/fixture/credentialloader/main_test.go @@ -0,0 +1,188 @@ +//go:build kind_execution_e2e + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +func TestLoadCredentialAndRedaction(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "credential") + sentinel := "synthetic-provider-secret-never-print" + if err := os.WriteFile(path, []byte(" "+sentinel+"\n"), 0o600); err != nil { + t.Fatal(err) + } + got, err := loadCredential(path) + if err != nil { + t.Fatal(err) + } + if string(got) != sentinel { + t.Fatal("credential was not trimmed") + } + clear(got) + if out := classify(errors.New("request rejected: " + sentinel)); strings.Contains(out, sentinel) || out != "operation failed (details redacted)" { + t.Fatalf("classification was not strictly redacted: %q", out) + } + if stderr := errorMessage(errors.New("provider body: " + sentinel)); strings.Contains(stderr, sentinel) || stderr != "credential loader: operation failed (details redacted)" { + t.Fatalf("stderr was not strictly redacted: %q", stderr) + } +} + +func TestLoadCredentialRejectsBroadPermissions(t *testing.T) { + path := filepath.Join(t.TempDir(), "credential") + if err := os.WriteFile(path, []byte("synthetic"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := loadCredential(path); err == nil { + t.Fatal("broad permissions accepted") + } +} + +func TestDeletePinsReceiptUIDAndLeavesReplacement(t *testing.T) { + state, receipt, kubeconfig := privateState(t) + _ = state + const name = "mecak8s-live-replacement-test" + originalUID := types.UID("original-synthetic-uid") + client := fake.NewSimpleClientset() + client.PrependReactor("create", "secrets", func(action ktesting.Action) (bool, runtime.Object, error) { + created := action.(ktesting.CreateAction).GetObject().(*corev1.Secret).DeepCopy() + created.UID = originalUID + return true, created, nil + }) + if err := stageSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name, []byte("synthetic-sentinel")); err != nil { + t.Fatal(err) + } + + replacementUID := types.UID("replacement-synthetic-uid") + client.PrependReactor("delete", "secrets", func(action ktesting.Action) (bool, runtime.Object, error) { + deleteAction := action.(ktesting.DeleteAction) + preconditions := deleteAction.GetDeleteOptions().Preconditions + if preconditions == nil || preconditions.UID == nil || *preconditions.UID != originalUID { + t.Fatal("cleanup did not pin the staged UID") + } + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "secrets"}, name, errors.New("UID precondition does not match")) + }) + if err := deleteStagedSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name); err == nil { + t.Fatal("replacement UID mismatch was accepted") + } + if _, err := os.Stat(receipt); err != nil { + t.Fatal("receipt was removed despite UID mismatch") + } + _ = replacementUID // The simulated replacement identity must never be targeted. + assertNoSecretReads(t, client.Actions()) +} + +func TestDeleteTreatsNotFoundAsCleaned(t *testing.T) { + _, receipt, kubeconfig := privateState(t) + const name = "mecak8s-live-not-found-test" + uid := types.UID("staged-synthetic-uid") + if err := writeReceipt(receipt, cleanupReceipt{Name: name, Namespace: namespace, UID: uid, Context: "kind-owned", Kubeconfig: kubeconfig}); err != nil { + t.Fatal(err) + } + client := fake.NewSimpleClientset() + client.PrependReactor("delete", "secrets", func(action ktesting.Action) (bool, runtime.Object, error) { + preconditions := action.(ktesting.DeleteAction).GetDeleteOptions().Preconditions + if preconditions == nil || preconditions.UID == nil || *preconditions.UID != uid { + t.Fatal("cleanup did not pin the staged UID") + } + return true, nil, apierrors.NewNotFound(schema.GroupResource{Resource: "secrets"}, name) + }) + if err := deleteStagedSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(receipt); !errors.Is(err, os.ErrNotExist) { + t.Fatal("successful cleanup retained its receipt") + } + assertNoSecretReads(t, client.Actions()) +} + +func TestReceiptIsPrivateAndCreateOnly(t *testing.T) { + path := filepath.Join(t.TempDir(), "receipt") + receipt := cleanupReceipt{Name: "mecak8s-live-receipt-test", Namespace: namespace, UID: "synthetic-uid", Context: "kind-owned", Kubeconfig: "/private/kubeconfig"} + if err := writeReceipt(path, receipt); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatal("receipt was not created with mode 0600") + } + if err := writeReceipt(path, receipt); err == nil { + t.Fatal("receipt overwrite was accepted") + } +} + +func TestStageFailureDoesNotDeleteOrWriteReceipt(t *testing.T) { + _, receipt, kubeconfig := privateState(t) + client := fake.NewSimpleClientset() + client.PrependReactor("create", "secrets", func(ktesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewAlreadyExists(schema.GroupResource{Resource: "secrets"}, "mecak8s-live-stage-failure") + }) + err := stageSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", "mecak8s-live-stage-failure", []byte("synthetic-sentinel")) + if err == nil { + t.Fatal("stage failure was accepted") + } + if _, err := os.Stat(receipt); !errors.Is(err, os.ErrNotExist) { + t.Fatal("stage failure wrote a cleanup receipt") + } + for _, action := range client.Actions() { + if action.GetVerb() == "delete" { + t.Fatal("stage failure attempted deletion") + } + } + assertNoSecretReads(t, client.Actions()) +} + +func TestMissingUIDDoesNotWriteReceiptOrDeleteByName(t *testing.T) { + _, receipt, kubeconfig := privateState(t) + client := fake.NewSimpleClientset(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "unrelated", Namespace: namespace}}) + name := "mecak8s-live-ambiguous-test" + if err := stageSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name, []byte("synthetic-sentinel")); err == nil || err.Error() != "secret staged without UID; cleanup pending" { + t.Fatalf("missing UID result = %v", err) + } + if _, err := os.Stat(receipt); !errors.Is(err, os.ErrNotExist) { + t.Fatal("missing UID wrote an invalid cleanup receipt") + } + for _, action := range client.Actions() { + if action.GetVerb() == "delete" { + t.Fatal("missing UID triggered name-only deletion") + } + } + assertNoSecretReads(t, client.Actions()) +} + +func privateState(t *testing.T) (string, string, string) { + t.Helper() + state := t.TempDir() + if err := os.Chmod(state, 0o700); err != nil { + t.Fatal(err) + } + kubeconfig := filepath.Join(state, "kubeconfig") + if err := os.WriteFile(kubeconfig, []byte("synthetic"), 0o600); err != nil { + t.Fatal(err) + } + return state, filepath.Join(state, "live-secret-test.receipt.json"), kubeconfig +} + +func assertNoSecretReads(t *testing.T, actions []ktesting.Action) { + t.Helper() + for _, action := range actions { + if action.GetResource().Resource == "secrets" && (action.GetVerb() == "get" || action.GetVerb() == "list") { + t.Fatalf("unexpected Secret %s action", action.GetVerb()) + } + } +} diff --git a/e2e/k8s_execution/live_qualification_test.go b/e2e/k8s_execution/live_qualification_test.go new file mode 100644 index 0000000000..382de55c57 --- /dev/null +++ b/e2e/k8s_execution/live_qualification_test.go @@ -0,0 +1,381 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + liveModel = "anthropic/claude-haiku-4.5" + liveCredentialKey = "OPENROUTER_API_KEY" +) + +func TestKindExecutionLiveQualification(t *testing.T) { + if os.Getenv("MECATL_EXECUTION_LIVE") != "1" { + t.Skip("explicit live qualification only") + } + state := os.Getenv("MECATL_EXECUTION_QUAL_STATE") + kubeconfig := filepath.Join(state, "kubeconfig") + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute) + defer cancel() + + tokenForward := portForward(t, ctx, kubeconfig, "service/oidc-issuer", 8443) + alice := fixtureToken(t, ctx, tokenForward.addr, filepath.Join(state, "pki"), "alice") + tokenForward.stop() + agentForward := portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + defer agentForward.stop() + + created := createLiveSession(t, ctx, agentForward.addr, alice) + secretName := os.Getenv("MECATL_EXECUTION_LIVE_SECRET") + if !strings.HasPrefix(secretName, "mecak8s-live-") { + t.Fatal("run-scoped provider Secret name is unavailable") + } + assertCredentialScope(t, ctx, kubeconfig, secretName) + if created.ResolvedModel == nil || created.ResolvedModel.ProviderId != "openrouter" || created.ResolvedModel.ModelId != liveModel { + t.Fatal("live session resolved an unexpected provider or model") + } + nonce := fmt.Sprintf("qual_%d", time.Now().UnixNano()) + promptText := "Create a small Go 1.27 module with arithmetic/sum.go defining Sum(a, b int) int and a table-driven arithmetic/sum_test.go covering positive, negative, and zero inputs. Include the comment // qualification: " + nonce + " in sum.go. Use the Write tool to create the files, then use Shell with the exact command `go test ./...`. Fix any failures and finish with a concise result only after that exact command passes. Do not add external dependencies." + events := livePrompt(t, ctx, agentForward.addr, created.SessionId, alice, promptText) + + calls := map[string]bool{} + modelShellPassed := modelRanExactShell(events, "go test ./...") + var usage *mecatlv1.Usage + stop := "" + for _, ev := range events { + if ev.ToolCall != nil { + calls[ev.ToolCall.Name] = true + } + if ev.ToolResult != nil { + if ev.ToolResult.IsError { + t.Fatal("live coding smoke had a tool error") + } + } + if ev.Result != nil { + stop = ev.Result.Stop + usage = ev.Result.Usage + } + } + if stop != string(session.StopEndTurn) { + t.Fatalf("real provider run did not finish successfully (stop=%s)", stop) + } + if !calls["Write"] { + t.Fatal("real provider did not call a filesystem write tool") + } + if !calls["Shell"] { + t.Fatal("real provider did not call Shell") + } + if !modelShellPassed { + t.Fatal("real provider Shell result did not report exit code 0") + } + if usage == nil || usage.InputTokens <= 0 || usage.OutputTokens <= 0 { + t.Fatal("real provider returned no positive token usage") + } + + providerForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + defer providerForward.stop() + client, err := executionclient.New(providerForward.addr, loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local")) + if err != nil { + t.Fatal("typed execution client setup failed") + } + defer client.Close() + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "alice"} + ensured, err := client.Ensure(ctx, created.SessionId, "go", owner) + if err != nil { + t.Fatal("typed execution reattachment lookup failed") + } + attached := waitReady(t, ctx, client, owner, created.SessionId, ensured.Environment) + rc := executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: created.SessionId, Epoch: attached.Epoch, Grant: attached.Grant} + artifactCount := 0 + file, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: "arithmetic/sum.go"}) + if err != nil || !strings.Contains(string(file.Data), "return a + b") || !strings.Contains(string(file.Data), nonce) { + t.Fatal("typed gRPC file verification failed") + } + artifactCount++ + testFile, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: "arithmetic/sum_test.go"}) + if err != nil || len(testFile.Data) == 0 { + t.Fatal("typed gRPC test-file verification failed") + } + artifactCount++ + command, err := client.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go test ./...", TimeoutMillis: 120000}) + if err != nil || command.State != executionenv.CommandSucceeded || command.Result.ExitCode != 0 { + t.Fatal("typed gRPC independent go test failed") + } + + observedTools := make([]string, 0, 4) + for _, name := range []string{"Write", "Read", "Edit", "Shell"} { + if calls[name] { + observedTools = append(observedTools, name) + } + } + cluster := os.Getenv("MECATL_EXECUTION_LIVE_CLUSTER") + if cluster == "" { + t.Fatal("verified qualification cluster identity is unavailable") + } + summary := liveSummary{ + Cluster: cluster, + Provider: created.ResolvedModel.ProviderId, + Model: created.ResolvedModel.ModelId, + InputTokens: usage.InputTokens, + OutputTokens: usage.OutputTokens, + ToolNames: observedTools, + ArtifactCount: artifactCount, + ExitCodes: []int{command.Result.ExitCode}, + Timestamp: time.Now().UTC().Format(time.RFC3339), + } + blob, err := json.MarshalIndent(summary, "", " ") + if err != nil { + t.Fatal("marshal sanitized summary failed") + } + if err := os.WriteFile(filepath.Join(state, "live-summary.json"), append(blob, '\n'), 0o600); err != nil { + t.Fatal("write sanitized summary failed") + } +} + +func modelRanExactShell(events []*mecatlv1.Event, command string) bool { + matching := map[string]struct{}{} + for _, ev := range events { + if ev.ToolCall != nil && ev.ToolCall.Name == "Shell" { + var args map[string]json.RawMessage + if json.Unmarshal([]byte(ev.ToolCall.Args), &args) != nil { + continue + } + var requested string + if json.Unmarshal(args["command"], &requested) == nil && strings.TrimSpace(requested) == command { + matching[ev.ToolCall.Id] = struct{}{} + } + } + if ev.ToolResult != nil { + if _, ok := matching[ev.ToolResult.CallId]; ok && !ev.ToolResult.IsError && strings.Contains(ev.ToolResult.Content, "[exit code: 0]") { + return true + } + } + } + return false +} + +type liveSummary struct { + Cluster string `json:"cluster"` + Provider string `json:"provider"` + Model string `json:"model"` + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + ToolNames []string `json:"tool_names"` + ArtifactCount int `json:"artifact_count"` + ExitCodes []int `json:"exit_codes"` + Timestamp string `json:"timestamp"` +} + +func assertCredentialScope(t *testing.T, ctx context.Context, kubeconfig, secretName string) { + t.Helper() + var agent appsv1.Deployment + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "deployment/mecak8s", "-n", namespace, "-o", "json"), &agent); err != nil { + t.Fatal("decode mecak8s credential-scope metadata failed") + } + assertRealProviderArgs(t, agent.Spec.Template.Spec) + want := secretReference{Container: "agent", Source: "env", Name: secretName, Key: liveCredentialKey, EnvName: liveCredentialKey} + refs := referencesNamed(secretReferences(agent.Spec.Template.Spec), secretName) + if len(refs) != 1 || refs[0] != want { + t.Fatal("run-scoped provider credential is not confined to the mecak8s agent environment") + } + + var provider appsv1.Deployment + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "deployment/mecatl-execution", "-n", namespace, "-o", "json"), &provider); err != nil { + t.Fatal("decode provider credential-scope metadata failed") + } + var executors corev1.PodList + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment", "-o", "json"), &executors); err != nil { + t.Fatal("decode executor credential-scope metadata failed") + } + if containsSecretReference(secretReferences(provider.Spec.Template.Spec), secretName) { + t.Fatal("provider credential escaped into the execution provider") + } + for _, pod := range executors.Items { + if containsSecretReference(secretReferences(pod.Spec), secretName) { + t.Fatal("provider credential escaped into an executor pod") + } + } +} + +func assertRealProviderArgs(t *testing.T, spec corev1.PodSpec) { + t.Helper() + if err := realProviderConfigError(spec); err != nil { + t.Fatal(err) + } +} + +func realProviderConfigError(spec corev1.PodSpec) error { + for _, volume := range spec.Volumes { + if volume.Name == "execution-mock" || (volume.ConfigMap != nil && volume.ConfigMap.Name == "execution-mock") { + return errors.New("live deployment retained the execution-mock volume") + } + if volume.Projected != nil { + for _, source := range volume.Projected.Sources { + if source.ConfigMap != nil && source.ConfigMap.Name == "execution-mock" { + return errors.New("live deployment retained a projected execution-mock source") + } + } + } + } + for _, container := range spec.Containers { + for _, mount := range container.VolumeMounts { + if mount.Name == "execution-mock" { + return errors.New("live deployment retained the execution-mock mount") + } + } + if container.Name != "agent" { + continue + } + seen := map[string]bool{} + for _, arg := range container.Args { + seen[arg] = true + if arg == "--mock" || strings.HasPrefix(arg, "--mock-script=") { + return errors.New("live deployment retained mock-provider arguments") + } + if strings.Contains(arg, "base-url") { + return errors.New("live deployment overrides the provider endpoint") + } + } + for _, required := range []string{"--default-provider=openrouter", "--model=" + liveModel, "--max-run-tokens=32000"} { + if !seen[required] { + return fmt.Errorf("live deployment is missing required argument %s", required) + } + } + return nil + } + return errors.New("live deployment has no agent container") +} + +type secretReference struct { + Container string + Source string + Name string + Key string + EnvName string +} + +func secretReferences(spec corev1.PodSpec) []secretReference { + var refs []secretReference + collectContainer := func(container corev1.Container, kind string) { + for _, env := range container.Env { + if env.ValueFrom != nil && env.ValueFrom.SecretKeyRef != nil { + refs = append(refs, secretReference{Container: kind + container.Name, Source: "env", Name: env.ValueFrom.SecretKeyRef.Name, Key: env.ValueFrom.SecretKeyRef.Key, EnvName: env.Name}) + } + } + for _, envFrom := range container.EnvFrom { + if envFrom.SecretRef != nil { + refs = append(refs, secretReference{Container: kind + container.Name, Source: "envFrom", Name: envFrom.SecretRef.Name}) + } + } + } + for _, container := range spec.Containers { + collectContainer(container, "") + } + for _, container := range spec.InitContainers { + collectContainer(container, "init:") + } + for _, volume := range spec.Volumes { + if volume.Secret != nil { + refs = append(refs, secretReference{Source: "volume", Name: volume.Secret.SecretName}) + } + if volume.Projected != nil { + for _, source := range volume.Projected.Sources { + if source.Secret != nil { + refs = append(refs, secretReference{Source: "projected", Name: source.Secret.Name}) + } + } + } + } + for _, pullSecret := range spec.ImagePullSecrets { + refs = append(refs, secretReference{Source: "imagePullSecret", Name: pullSecret.Name}) + } + return refs +} + +func referencesNamed(refs []secretReference, name string) []secretReference { + matched := make([]secretReference, 0, 1) + for _, ref := range refs { + if ref.Name == name { + matched = append(matched, ref) + } + } + return matched +} + +func containsSecretReference(refs []secretReference, name string) bool { + return len(referencesNamed(refs, name)) != 0 +} + +type liveCreateResponse struct { + SessionId string `json:"session_id"` + ResolvedModel *mecatlv1.ResolvedModel `json:"resolved_model"` +} + +func createLiveSession(t *testing.T, ctx context.Context, addr, token string) liveCreateResponse { + t.Helper() + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions", token, []byte(`{"mode":"default","limits":{"max_turns":8,"max_tool_calls":20,"max_consecutive_failures":3}}`)) + if status != http.StatusCreated { + t.Fatalf("live session creation failed (HTTP %d)", status) + } + var out liveCreateResponse + if json.Unmarshal(body, &out) != nil || out.SessionId == "" { + t.Fatal("live session response invalid") + } + return out +} + +func livePrompt(t *testing.T, ctx context.Context, addr, id, token, text string) []*mecatlv1.Event { + t.Helper() + raw, _ := json.Marshal(map[string]string{"text": text}) + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+id+"/prompt", strings.NewReader(string(raw))) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "text/event-stream") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal("live prompt transport failed") + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 2<<20)) + t.Fatalf("live prompt rejected (HTTP %d; body redacted)", resp.StatusCode) + } + var events []*mecatlv1.Event + s := bufio.NewScanner(io.LimitReader(resp.Body, 4<<20)) + s.Buffer(make([]byte, 64<<10), 1<<20) + for s.Scan() { + line := s.Text() + if !strings.HasPrefix(line, "data: ") { + continue + } + var ev mecatlv1.Event + if json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &ev) != nil { + t.Fatal("invalid live SSE event") + } + events = append(events, &ev) + } + if s.Err() != nil { + t.Fatal("live SSE stream failed") + } + return events +} diff --git a/e2e/k8s_execution/live_qualification_unit_test.go b/e2e/k8s_execution/live_qualification_unit_test.go new file mode 100644 index 0000000000..a73a99d4bf --- /dev/null +++ b/e2e/k8s_execution/live_qualification_unit_test.go @@ -0,0 +1,100 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" +) + +func TestModelRanExactShellRequiresDecodedCommandAndMatchingCallID(t *testing.T) { + call := func(id, args string) *mecatlv1.Event { + return &mecatlv1.Event{ToolCall: &mecatlv1.ToolCall{Id: id, Name: "Shell", Args: args}} + } + result := func(id string) *mecatlv1.Event { + return &mecatlv1.Event{ToolResult: &mecatlv1.ToolResult{CallId: id, Content: "[exit code: 0]"}} + } + for name, events := range map[string][]*mecatlv1.Event{ + "echo": {call("shell", `{"command":"echo ok"}`), result("shell")}, + "invalid JSON": {call("shell", `{"command":`), result("shell")}, + "wrong result ID": {call("shell", `{"command":"go test ./..."}`), result("other")}, + "unrelated success": {call("echo", `{"command":"echo ok"}`), result("echo"), call("test", `{"command":"go test ./..."}`)}, + } { + t.Run(name, func(t *testing.T) { + if modelRanExactShell(events, "go test ./...") { + t.Fatal("unqualified Shell call counted") + } + }) + } + if !modelRanExactShell([]*mecatlv1.Event{call("test", `{"command":" go test ./... "}`), result("test")}, "go test ./...") { + t.Fatal("matching successful Shell call was not counted") + } +} + +func TestRealProviderArgsRejectExecutionMockSources(t *testing.T) { + tests := []corev1.PodSpec{ + {Volumes: []corev1.Volume{{Name: "execution-mock"}}}, + {Volumes: []corev1.Volume{{Name: "other", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "execution-mock"}}}}}}, + { + Volumes: []corev1.Volume{{ + Name: "other", + VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{Sources: []corev1.VolumeProjection{{ + ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "execution-mock"}}, + }}}}, + }}, + }, + {Containers: []corev1.Container{{Name: "sidecar", VolumeMounts: []corev1.VolumeMount{{Name: "execution-mock"}}}}}, + } + for i, spec := range tests { + if err := realProviderConfigError(spec); err == nil { + t.Fatalf("case %d did not reject execution-mock source", i) + } + } +} + +func TestSecretReferencesFindEveryCredentialEscape(t *testing.T) { + const sentinel = "synthetic-live-secret-sentinel" + tests := map[string]corev1.PodSpec{ + "sidecar env": { + Containers: []corev1.Container{{Name: "sidecar", Env: []corev1.EnvVar{{Name: "TOKEN", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}, Key: "token"}}}}}}, + }, + "sidecar envFrom": { + Containers: []corev1.Container{{Name: "sidecar", EnvFrom: []corev1.EnvFromSource{{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}}}}}}, + }, + "init container env": { + InitContainers: []corev1.Container{{Name: "setup", Env: []corev1.EnvVar{{Name: "TOKEN", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}, Key: "token"}}}}}}, + }, + "secret volume": { + Volumes: []corev1.Volume{{Name: "credentials", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: sentinel}}}}, + }, + "projected secret volume": { + Volumes: []corev1.Volume{{Name: "credentials", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{Sources: []corev1.VolumeProjection{{Secret: &corev1.SecretProjection{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}}}}}}}}, + }, + } + for name, spec := range tests { + t.Run(name, func(t *testing.T) { + if !containsSecretReference(secretReferences(spec), sentinel) { + t.Fatal("credential escape was not detected") + } + }) + } +} + +func TestExpectedHarnessCredentialReferenceIsExact(t *testing.T) { + const sentinel = "synthetic-live-secret-sentinel" + spec := corev1.PodSpec{Containers: []corev1.Container{{ + Name: "agent", + Env: []corev1.EnvVar{{Name: liveCredentialKey, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}, + Key: liveCredentialKey, + }}}}, + }}} + want := secretReference{Container: "agent", Source: "env", Name: sentinel, Key: liveCredentialKey, EnvName: liveCredentialKey} + refs := secretReferences(spec) + if len(refs) != 1 || refs[0] != want { + t.Fatal("expected harness credential reference was not projected exactly") + } +} diff --git a/e2e/k8s_execution/qualification_test.go b/e2e/k8s_execution/qualification_test.go index a8950741d1..8be2aa0b8c 100644 --- a/e2e/k8s_execution/qualification_test.go +++ b/e2e/k8s_execution/qualification_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" "github.com/stacklok/mecatl/internal/adapter/executionclient" "github.com/stacklok/mecatl/internal/executionenv" ) @@ -39,7 +40,7 @@ func TestKindExecutionQualification(t *testing.T) { baseline := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev") providerForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) providerTLS := loadTLS(t, pki, "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") - providerClient, err := executionclient.New("https://"+providerForward.addr, providerTLS) + providerClient, err := executionclient.New(providerForward.addr, providerTLS) if err != nil { t.Fatal(err) } @@ -82,7 +83,7 @@ func TestKindExecutionQualification(t *testing.T) { } intruderTLS := loadTLS(t, pki, "intruder", "mecatl-execution.execution-qualification.svc.cluster.local") - intruder, err := executionclient.New("https://"+providerForward.addr, intruderTLS) + intruder, err := executionclient.New(providerForward.addr, intruderTLS) if err != nil { t.Fatal(err) } @@ -91,14 +92,22 @@ func TestKindExecutionQualification(t *testing.T) { if err == nil || (!isRemoteCode(err, executionenv.CodePermissionDenied) && !isRemoteCode(err, executionenv.CodeNotFound)) { t.Fatalf("different client attach error = %v, want an existence-hiding denial", err) } - for _, path := range []string{"/commands/status", "/commands/cancel", "/environments/retire"} { - request := map[string]any{"context": map[string]any{"environment": first.Environment, "owner": owner, "binding_id": binding, "epoch": ready.Epoch, "grant": ready.Grant}} - if strings.HasPrefix(path, "/commands/") { - request["command_id"] = "not-owned" - } - status := rawProviderPost(t, ctx, "https://"+providerForward.addr, intruderTLS, path, request) - if status != http.StatusForbidden { - t.Fatalf("different client %s status = %d, want 403", path, status) + rc := executionenv.RequestContext{Environment: first.Environment, Owner: owner, BindingID: binding, Epoch: ready.Epoch, Grant: ready.Grant} + for name, call := range map[string]func() error{ + "command status": func() error { + _, err := intruder.CommandStatus(ctx, executionenv.CommandQueryRequest{Context: rc, CommandID: "not-owned"}) + return err + }, + "command cancel": func() error { + _, err := intruder.CancelCommand(ctx, executionenv.CommandQueryRequest{Context: rc, CommandID: "not-owned"}) + return err + }, + "environment retire": func() error { + return intruder.RetireEnvironment(ctx, executionenv.RetireEnvironmentRequest{Context: rc}) + }, + } { + if err := call(); err == nil || (!isRemoteCode(err, executionenv.CodePermissionDenied) && !isRemoteCode(err, executionenv.CodeUnauthenticated)) { + t.Fatalf("different client %s error = %v, want denial", name, err) } } providerForward.stop() @@ -118,7 +127,7 @@ func TestKindExecutionQualification(t *testing.T) { } for _, marker := range []string{"write-proof", "read-before-edit", "edit-proof", "copy-proof", "move-proof", "write-mod", "write-test", "shell-go-test", "remove-moved", "read-final", "moved.txt", "proof.txt:1:beta", "REMOTE_EXECUTION_QUALIFICATION_COMPLETE", "ok"} { if !bytes.Contains(body, []byte(marker)) { - t.Fatalf("SSE result omitted %q", marker) + t.Fatalf("SSE result omitted %q (%s)", marker, mockSSEStatus(body)) } } if bytes.Contains(body, []byte(`"is_error":true`)) { @@ -150,6 +159,41 @@ func TestKindExecutionQualification(t *testing.T) { } } +func mockSSEStatus(body []byte) string { + var names []string + errorClasses := map[string]int{} + stop := "missing" + for _, line := range bytes.Split(body, []byte("\n")) { + if !bytes.HasPrefix(line, []byte("data: ")) { + continue + } + var ev mecatlv1.Event + if json.Unmarshal(bytes.TrimPrefix(line, []byte("data: ")), &ev) != nil { + continue + } + if ev.ToolCall != nil { + names = append(names, ev.ToolCall.Name) + } + if ev.ToolResult != nil && ev.ToolResult.IsError { + errorClasses[classifyMockToolError(ev.ToolResult.Content)]++ + } + if ev.Result != nil { + stop = ev.Result.Stop + } + } + return fmt.Sprintf("tools=%s error_classes=%v stop=%s", strings.Join(names, ","), errorClasses, stop) +} + +func classifyMockToolError(content string) string { + content = strings.ToLower(content) + for _, class := range []string{"grant", "permission", "owner", "stale", "unavailable", "not found", "version", "placement", "environment", "connection", "timeout", "binding", "argument", "request", "profile", "reference", "invalid", "read before", "already exists"} { + if strings.Contains(content, class) { + return strings.ReplaceAll(class, " ", "_") + } + } + return "other_redacted" +} + func waitReady(t *testing.T, ctx context.Context, c *executionclient.Client, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) executionenv.AttachEnvironmentResponse { t.Helper() deadline := time.Now().Add(3 * time.Minute) @@ -253,20 +297,6 @@ func resourceCount(t *testing.T, ctx context.Context, kubeconfig, spec string) i return len(bytes.Split(bytes.TrimSpace(out), []byte{'\n'})) } -func rawProviderPost(t *testing.T, ctx context.Context, endpoint string, cfg *tls.Config, path string, body any) int { - t.Helper() - raw, _ := json.Marshal(body) - req, _ := http.NewRequestWithContext(ctx, http.MethodPost, endpoint+executionenv.BasePath+path, bytes.NewReader(raw)) - req.Header.Set("Content-Type", "application/json") - client := &http.Client{Transport: &http.Transport{TLSClientConfig: cfg}} - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16)) - return resp.StatusCode -} func fixtureToken(t *testing.T, ctx context.Context, addr, pki, sub string) string { t.Helper() ca, err := os.ReadFile(filepath.Join(pki, "ca.crt")) @@ -294,7 +324,7 @@ func fixtureToken(t *testing.T, ctx context.Context, addr, pki, sub string) stri } func createSession(t *testing.T, ctx context.Context, addr, token string) string { t.Helper() - status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions", token, []byte(`{"mode":"default"}`)) + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions", token, []byte(`{"mode":"default","limits":{"max_turns":16,"max_tool_calls":24,"max_consecutive_failures":3}}`)) if status != http.StatusCreated { t.Fatalf("create session status=%d body=%s", status, body) } diff --git a/internal/adapter/executionclient/client.go b/internal/adapter/executionclient/client.go index c2ed801659..e39b01774a 100644 --- a/internal/adapter/executionclient/client.go +++ b/internal/adapter/executionclient/client.go @@ -3,23 +3,24 @@ package executionclient import ( - "bytes" "context" "crypto/tls" "crypto/x509" - "encoding/json" "errors" "fmt" - "io" "io/fs" "net" - "net/http" - "net/url" "os" "strings" "sync" "time" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + grpccredentials "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" "github.com/stacklok/mecatl/engine/adapter/memledger" "github.com/stacklok/mecatl/engine/session" "github.com/stacklok/mecatl/engine/tool" @@ -55,111 +56,292 @@ func LoadTLSConfig(files TLSFiles) (*tls.Config, error) { return &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: pool, Certificates: []tls.Certificate{cert}}, nil } -// Client is a bounded client for the private execution-provider HTTP API. +// Client is a bounded client for the private execution-provider gRPC API. type Client struct { - endpoint *url.URL - http *http.Client + conn *grpc.ClientConn + rpc executionv1.ExecutionProviderServiceClient } -// New constructs a production mTLS execution-provider client. +// New constructs a production mTLS execution-provider client. Endpoint is a +// plain host:port authority; URI schemes and alternate resolvers are rejected. func New(endpoint string, tlsConfig *tls.Config) (*Client, error) { - u, err := url.Parse(endpoint) - if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { - return nil, errors.New("execution client: endpoint must be an absolute HTTPS URL without credentials, query, or fragment") + if strings.Contains(endpoint, "://") || strings.ContainsAny(endpoint, "/?#@") { + return nil, errors.New("execution client: endpoint must be host:port") + } + host, port, err := net.SplitHostPort(endpoint) + if err != nil || host == "" || port == "" { + return nil, errors.New("execution client: endpoint must be host:port") } if tlsConfig == nil || len(tlsConfig.Certificates) == 0 || tlsConfig.RootCAs == nil { return nil, errors.New("execution client: production mTLS configuration is required") } - u.Path = strings.TrimRight(u.Path, "/") - tr := &http.Transport{TLSClientConfig: tlsConfig.Clone(), DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext, TLSHandshakeTimeout: 10 * time.Second} - return &Client{endpoint: u, http: &http.Client{Transport: tr, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}}, nil -} - -// NewWithHTTPClient is a hermetic test seam; production callers use New. -func NewWithHTTPClient(endpoint string, hc *http.Client) (*Client, error) { - u, err := url.Parse(endpoint) - if err != nil || u.Scheme == "" || u.Host == "" || hc == nil { - return nil, errors.New("execution client: invalid endpoint or HTTP client") + cfg := tlsConfig.Clone() + if cfg.MinVersion < tls.VersionTLS13 { + cfg.MinVersion = tls.VersionTLS13 } - u.Path = strings.TrimRight(u.Path, "/") - return &Client{endpoint: u, http: hc}, nil + conn, err := grpc.NewClient(endpoint, + grpc.WithTransportCredentials(grpccredentials.NewTLS(cfg)), + grpc.WithDisableRetry(), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(executionenv.MaxMessageBytes), grpc.MaxCallSendMsgSize(executionenv.MaxMessageBytes)), + ) + if err != nil { + return nil, fmt.Errorf("execution client: connect: %w", err) + } + return &Client{conn: conn, rpc: executionv1.NewExecutionProviderServiceClient(conn)}, nil } -// Close releases pooled provider connections. +// Close releases the provider connection. func (c *Client) Close() { - if c != nil && c.http != nil { - c.http.CloseIdleConnections() + if c != nil && c.conn != nil { + _ = c.conn.Close() } } -func (c *Client) post(ctx context.Context, path string, in, out any) error { - body, err := json.Marshal(in) - if err != nil { - return err - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint.String()+executionenv.BasePath+path, bytes.NewReader(body)) +// ValidateProfile validates a provider profile without allocating an environment. +func (c *Client) ValidateProfile(ctx context.Context, profile string) (executionenv.ValidateProfileResponse, error) { + v, err := c.rpc.ValidateProfile(ctx, &executionv1.ValidateProfileRequest{Profile: profile}) if err != nil { - return err + return executionenv.ValidateProfileResponse{}, decodeError(ctx, err) } - req.Header.Set("Content-Type", "application/json") - resp, err := c.http.Do(req) + return executionenv.ValidateProfileResponse{Profile: v.Profile, Digest: v.Digest, Capabilities: v.Capabilities, MaxFileBytes: v.MaxFileBytes, MaxCommandBytes: v.MaxCommandBytes, MaxCommandDurationMillis: v.MaxCommandDurationMillis}, nil +} + +// Ensure idempotently allocates or resolves the environment for a binding. +func (c *Client) Ensure(ctx context.Context, binding, profile string, owner executionenv.Owner) (executionenv.EnsureEnvironmentResponse, error) { + v, err := c.rpc.EnsureEnvironment(ctx, &executionv1.EnsureEnvironmentRequest{BindingId: binding, Profile: profile, Owner: ownerToProto(owner)}) if err != nil { - return fmt.Errorf("execution provider request failed: %w", err) + return executionenv.EnsureEnvironmentResponse{}, decodeError(ctx, err) } - defer func() { _ = resp.Body.Close() }() - data, err := io.ReadAll(io.LimitReader(resp.Body, executionenv.MaxJSONBody+1)) + return ensureFromProto(v) +} + +// Attach obtains a fresh grant for an exact environment reference. +func (c *Client) Attach(ctx context.Context, req executionenv.AttachEnvironmentRequest) (executionenv.AttachEnvironmentResponse, error) { + v, err := c.rpc.AttachEnvironment(ctx, &executionv1.AttachEnvironmentRequest{Context: contextToProto(req.Context), Purpose: req.Purpose}) if err != nil { - return fmt.Errorf("execution provider response failed: %w", err) + return executionenv.AttachEnvironmentResponse{}, decodeError(ctx, err) } - if len(data) > executionenv.MaxJSONBody { - return errors.New("execution provider response exceeds limit") + return attachFromProto(v) +} + +// File executes one authorized filesystem operation. +func (c *Client) File(ctx context.Context, req executionenv.FileRequest) (executionenv.FileResponse, error) { + if req.Limit < 0 || req.Limit > executionenv.MaxListEntries { + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "invalid file request limit"} } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - var envelope executionenv.ErrorResponse - if executionenv.DecodeStrict(data, &envelope) == nil && envelope.Error != nil { - return envelope.Error - } - return fmt.Errorf("execution provider returned HTTP %d", resp.StatusCode) + v, err := c.rpc.Files(ctx, &executionv1.FileRequest{Context: contextToProto(req.Context), Operation: fileOperationToProto(req.Operation), Path: req.Path, Destination: req.Destination, Pattern: req.Pattern, Data: req.Data, Version: []byte(req.Version), Limit: int32(req.Limit)}) //nolint:gosec // bounded above + if err != nil { + return executionenv.FileResponse{}, decodeError(ctx, err) } - if err := executionenv.DecodeStrict(data, out); err != nil { - return fmt.Errorf("execution provider returned invalid response: %w", err) + return fileFromProto(v), nil +} + +// StartCommand executes one foreground command. +func (c *Client) StartCommand(ctx context.Context, req executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + v, err := c.rpc.StartCommand(ctx, &executionv1.CommandStartRequest{Context: contextToProto(req.Context), Command: req.Command, TimeoutMillis: req.TimeoutMillis}) + if err != nil { + return executionenv.CommandStartResponse{}, decodeError(ctx, err) } - return nil + return commandStartFromProto(v) } -// ValidateProfile validates a provider profile without allocating an environment. -func (c *Client) ValidateProfile(ctx context.Context, profile string) (executionenv.ValidateProfileResponse, error) { - var out executionenv.ValidateProfileResponse - err := c.post(ctx, "/profiles/validate", executionenv.ValidateProfileRequest{Profile: profile}, &out) - return out, err +// CommandStatus queries a command after the request has been authorized. +func (c *Client) CommandStatus(ctx context.Context, req executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + v, err := c.rpc.CommandStatus(ctx, &executionv1.CommandQueryRequest{Context: contextToProto(req.Context), CommandId: req.CommandID, Offset: req.Offset}) + if err != nil { + return executionenv.CommandStatusResponse{}, decodeError(ctx, err) + } + return commandStatusFromProto(v) } -// Ensure idempotently allocates or resolves the environment for a binding. -func (c *Client) Ensure(ctx context.Context, binding, profile string, owner executionenv.Owner) (executionenv.EnsureEnvironmentResponse, error) { - var out executionenv.EnsureEnvironmentResponse - err := c.post(ctx, "/environments/ensure", executionenv.EnsureEnvironmentRequest{BindingID: binding, Profile: profile, Owner: owner}, &out) - return out, err +// CancelCommand cancels a command after the request has been authorized. +func (c *Client) CancelCommand(ctx context.Context, req executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + v, err := c.rpc.CancelCommand(ctx, &executionv1.CommandQueryRequest{Context: contextToProto(req.Context), CommandId: req.CommandID, Offset: req.Offset}) + if err != nil { + return executionenv.CommandStatusResponse{}, decodeError(ctx, err) + } + return commandStatusFromProto(v) } -// Attach obtains a fresh grant for an exact environment reference. -func (c *Client) Attach(ctx context.Context, req executionenv.AttachEnvironmentRequest) (executionenv.AttachEnvironmentResponse, error) { - var out executionenv.AttachEnvironmentResponse - err := c.post(ctx, "/environments/attach", req, &out) - return out, err +// ReleaseReference releases one durable binding reference. +func (c *Client) ReleaseReference(ctx context.Context, req executionenv.ReferenceReleaseRequest) error { + _, err := c.rpc.ReleaseReference(ctx, &executionv1.ReleaseReferenceRequest{Context: contextToProto(req.Context)}) + return decodeError(ctx, err) } -// File executes one authorized filesystem operation. -func (c *Client) File(ctx context.Context, req executionenv.FileRequest) (executionenv.FileResponse, error) { - var out executionenv.FileResponse - err := c.post(ctx, "/files", req, &out) - return out, err +// RetireEnvironment requests administrative retirement. +func (c *Client) RetireEnvironment(ctx context.Context, req executionenv.RetireEnvironmentRequest) error { + _, err := c.rpc.RetireEnvironment(ctx, &executionv1.RetireEnvironmentRequest{Context: contextToProto(req.Context)}) + return decodeError(ctx, err) } -// StartCommand executes one foreground command. -func (c *Client) StartCommand(ctx context.Context, req executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { - var out executionenv.CommandStartResponse - err := c.post(ctx, "/commands/start", req, &out) - return out, err +func decodeError(ctx context.Context, err error) error { + if err == nil { + return nil + } + if ctx != nil && ctx.Err() != nil { + return ctx.Err() + } + st, ok := status.FromError(err) + if !ok || len(st.Details()) != 1 { + return sanitizedProviderError() + } + detail, ok := st.Details()[0].(*executionv1.ErrorDetail) + if !ok { + return sanitizedProviderError() + } + code := executionenv.ErrorCode(detail.Code) + if !code.Valid() || st.Code() != grpcCodeForError(code) || (detail.Retryable && code != executionenv.CodeNotReady && code != executionenv.CodeUnauthenticated) { + return sanitizedProviderError() + } + return &executionenv.Error{Code: code, Message: "execution provider request failed", Retryable: detail.Retryable} +} + +func sanitizedProviderError() error { + return &executionenv.Error{Code: executionenv.CodeInternal, Message: "execution provider request failed"} +} + +func grpcCodeForError(code executionenv.ErrorCode) codes.Code { + switch code { + case executionenv.CodeInvalidArgument: + return codes.InvalidArgument + case executionenv.CodeUnauthenticated: + return codes.Unauthenticated + case executionenv.CodePermissionDenied: + return codes.PermissionDenied + case executionenv.CodeNotFound: + return codes.NotFound + case executionenv.CodeAlreadyExists: + return codes.AlreadyExists + case executionenv.CodeConflict, executionenv.CodeVersionMismatch: + return codes.Aborted + case executionenv.CodeNotReady, executionenv.CodeFenceUnknown: + return codes.Unavailable + case executionenv.CodeResourceExhausted: + return codes.ResourceExhausted + default: + return codes.Internal + } +} +func ownerToProto(o executionenv.Owner) *executionv1.Owner { + return &executionv1.Owner{Issuer: o.Issuer, Subject: o.Subject} +} +func refToProto(r executionenv.EnvironmentRef) *executionv1.EnvironmentRef { + return &executionv1.EnvironmentRef{Id: r.ID, Revision: r.Revision} +} +func contextToProto(v executionenv.RequestContext) *executionv1.RequestContext { + return &executionv1.RequestContext{Environment: refToProto(v.Environment), Owner: ownerToProto(v.Owner), BindingId: v.BindingID, Epoch: v.Epoch, Grant: v.Grant} +} +func refFromProto(r *executionv1.EnvironmentRef) executionenv.EnvironmentRef { + return executionenv.EnvironmentRef{ID: r.GetId(), Revision: r.GetRevision()} +} +func ensureFromProto(v *executionv1.EnsureEnvironmentResponse) (executionenv.EnsureEnvironmentResponse, error) { + expiry, err := checkedTime(v.GetGrantExpiresAt()) + return executionenv.EnsureEnvironmentResponse{Environment: refFromProto(v.GetEnvironment()), Epoch: v.GetEpoch(), Ready: v.GetReady(), Grant: v.GetGrant(), GrantExpiresAt: expiry}, err +} +func attachFromProto(v *executionv1.AttachEnvironmentResponse) (executionenv.AttachEnvironmentResponse, error) { + expiry, err := checkedTime(v.GetGrantExpiresAt()) + return executionenv.AttachEnvironmentResponse{Environment: refFromProto(v.GetEnvironment()), Epoch: v.GetEpoch(), Ready: v.GetReady(), Grant: v.GetGrant(), GrantExpiresAt: expiry}, err +} +func checkedTime(v interface { + CheckValid() error + AsTime() time.Time +}) (time.Time, error) { + if v == nil { + return time.Time{}, errors.New("execution provider omitted timestamp") + } + if err := v.CheckValid(); err != nil { + return time.Time{}, errors.New("execution provider returned invalid timestamp") + } + return v.AsTime(), nil +} +func fileOperationToProto(op executionenv.Operation) executionv1.FileOperation { + switch op { + case executionenv.OpFileRead: + return executionv1.FileOperation_FILE_OPERATION_READ + case executionenv.OpFileResolveAuthority: + return executionv1.FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY + case executionenv.OpFileStat: + return executionv1.FileOperation_FILE_OPERATION_STAT + case executionenv.OpFileCreate: + return executionv1.FileOperation_FILE_OPERATION_CREATE + case executionenv.OpFileReplace: + return executionv1.FileOperation_FILE_OPERATION_REPLACE + case executionenv.OpFileList: + return executionv1.FileOperation_FILE_OPERATION_LIST + case executionenv.OpFileRemove: + return executionv1.FileOperation_FILE_OPERATION_REMOVE + case executionenv.OpFileRename: + return executionv1.FileOperation_FILE_OPERATION_RENAME + case executionenv.OpFileCopy: + return executionv1.FileOperation_FILE_OPERATION_COPY + case executionenv.OpFileGlob: + return executionv1.FileOperation_FILE_OPERATION_GLOB + case executionenv.OpFileGrep: + return executionv1.FileOperation_FILE_OPERATION_GREP + } + return executionv1.FileOperation_FILE_OPERATION_UNSPECIFIED +} +func fileFromProto(v *executionv1.FileResponse) executionenv.FileResponse { + r := executionenv.FileResponse{Data: v.GetData(), Version: string(v.GetVersion()), Paths: v.GetPaths(), AuthorityTarget: v.GetAuthorityTarget(), AuthorityWorkspace: v.GetAuthorityWorkspace()} + if v.Info != nil { + x := fileInfoFromProto(v.Info) + r.Info = &x + } + for _, x := range v.Entries { + r.Entries = append(r.Entries, fileInfoFromProto(x)) + } + for _, x := range v.Matches { + r.Matches = append(r.Matches, executionenv.GrepMatch{Path: x.Path, Line: int(x.Line), Text: x.Text}) + } + return r +} +func fileInfoFromProto(v *executionv1.FileInfo) executionenv.FileInfo { + var mt time.Time + if v.ModTime != nil && v.ModTime.CheckValid() == nil { + mt = v.ModTime.AsTime() + } + return executionenv.FileInfo{Name: v.Name, Size: v.Size, Mode: v.Mode, ModTime: mt, IsDir: v.IsDir} +} +func commandStartFromProto(v *executionv1.CommandStartResponse) (executionenv.CommandStartResponse, error) { + if v == nil { + return executionenv.CommandStartResponse{}, sanitizedProviderError() + } + state, ok := commandStateFromProto(v.GetState()) + if !ok { + return executionenv.CommandStartResponse{}, sanitizedProviderError() + } + result, err := commandStatusFromProto(v.GetResult()) + if err != nil { + return executionenv.CommandStartResponse{}, err + } + return executionenv.CommandStartResponse{CommandID: v.GetCommandId(), State: state, Result: result}, nil +} +func commandStatusFromProto(v *executionv1.CommandStatusResponse) (executionenv.CommandStatusResponse, error) { + if v == nil { + return executionenv.CommandStatusResponse{}, sanitizedProviderError() + } + state, ok := commandStateFromProto(v.State) + if !ok { + return executionenv.CommandStatusResponse{}, sanitizedProviderError() + } + return executionenv.CommandStatusResponse{CommandID: v.CommandId, State: state, ExitCode: int(v.ExitCode), Stdout: v.Stdout, Stderr: v.Stderr, NextOffset: v.NextOffset, Truncated: v.Truncated, TerminalReceipt: v.TerminalReceipt}, nil +} +func commandStateFromProto(v executionv1.CommandState) (executionenv.CommandState, bool) { + switch v { + case executionv1.CommandState_COMMAND_STATE_RUNNING: + return executionenv.CommandRunning, true + case executionv1.CommandState_COMMAND_STATE_SUCCEEDED: + return executionenv.CommandSucceeded, true + case executionv1.CommandState_COMMAND_STATE_FAILED: + return executionenv.CommandFailed, true + case executionv1.CommandState_COMMAND_STATE_CANCELLED: + return executionenv.CommandCancelled, true + case executionv1.CommandState_COMMAND_STATE_FENCE_UNKNOWN: + return executionenv.CommandFenceUnknown, true + default: + return "", false + } } // Provider adapts the execution service to server placement binding. @@ -356,20 +538,26 @@ func (w *workspace) call(ctx context.Context, op executionenv.Operation, path, d if err != nil { return executionenv.FileResponse{}, mapFileError(path, err) } - out, err := w.credentials.client.File(ctx, executionenv.FileRequest{Context: credentials, Operation: op, Path: path, Destination: dest, Pattern: pattern, Data: data, Version: version, Limit: executionenv.MaxListEntries}) + limit := 0 + if op == executionenv.OpFileList || op == executionenv.OpFileGlob || op == executionenv.OpFileGrep { + limit = executionenv.MaxListEntries + } + request := executionenv.FileRequest{Context: credentials, Operation: op, Path: path, Destination: dest, Pattern: pattern, Data: data, Version: version, Limit: limit} + out, err := w.credentials.client.File(ctx, request) if isDefinitiveAuthDenial(err) && readOnlyFileOperation(op) { credentials, refreshErr := w.credentials.refresh(ctx) if refreshErr != nil { return executionenv.FileResponse{}, mapFileError(path, refreshErr) } - out, err = w.credentials.client.File(ctx, executionenv.FileRequest{Context: credentials, Operation: op, Path: path, Destination: dest, Pattern: pattern, Data: data, Version: version, Limit: executionenv.MaxListEntries}) + request.Context = credentials + out, err = w.credentials.client.File(ctx, request) } return out, mapFileError(path, err) } func isDefinitiveAuthDenial(err error) bool { var remote *executionenv.Error - return errors.As(err, &remote) && (remote.Code == executionenv.CodePermissionDenied || remote.Code == executionenv.CodeUnauthenticated) + return errors.As(err, &remote) && remote.Code == executionenv.CodeUnauthenticated && remote.Retryable } func readOnlyFileOperation(op executionenv.Operation) bool { diff --git a/internal/adapter/executionclient/client_test.go b/internal/adapter/executionclient/client_test.go index 828c52389c..24adbc2e81 100644 --- a/internal/adapter/executionclient/client_test.go +++ b/internal/adapter/executionclient/client_test.go @@ -9,14 +9,20 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" - "encoding/json" + "errors" "math/big" - "net/http" - "net/http/httptest" + "net" "net/url" + "strings" "testing" "time" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" "github.com/stacklok/mecatl/engine/session" "github.com/stacklok/mecatl/engine/tool" "github.com/stacklok/mecatl/internal/adapter/executioncontroller" @@ -25,10 +31,9 @@ import ( ) type integrationBackend struct { - allocation executioncontroller.Allocation - ensureCalls int - attachCalls int - fileCalls int + allocation executioncontroller.Allocation + ensureCalls, attachCalls, fileCalls int + expireNextRead bool } func (*integrationBackend) ValidateProfile(context.Context, string) (executioncontroller.Profile, error) { @@ -54,19 +59,23 @@ func (*integrationBackend) Retire(context.Context, executionenv.EnvironmentRef, } func (b *integrationBackend) File(_ context.Context, _, _ string, q executionenv.FileRequest) (executionenv.FileResponse, error) { b.fileCalls++ + if b.expireNextRead && q.Operation == executionenv.OpFileRead { + b.expireNextRead = false + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "expired grant", Retryable: true} + } if q.Operation == executionenv.OpFileResolveAuthority { return executionenv.FileResponse{AuthorityTarget: "/workspace/main.go", AuthorityWorkspace: "/workspace"}, nil } - return executionenv.FileResponse{Data: []byte("from-real-handler"), Version: "v1"}, nil + return executionenv.FileResponse{Data: []byte("from-grpc"), Version: string([]byte{0xff, 0, 1})}, nil } func (*integrationBackend) StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { - return executionenv.CommandStartResponse{}, nil + return executionenv.CommandStartResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Result: executionenv.CommandStatusResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Stdout: []byte("ok\n")}}, nil } func (*integrationBackend) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { - return executionenv.CommandStatusResponse{}, nil + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeInternal, Message: "unimplemented"} } func (*integrationBackend) CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { - return executionenv.CommandStatusResponse{}, nil + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeInternal, Message: "unimplemented"} } func certificate(t *testing.T, parent *x509.Certificate, parentKey *ecdsa.PrivateKey, serverName string, client bool) (tls.Certificate, *x509.Certificate, *ecdsa.PrivateKey) { @@ -78,7 +87,8 @@ func certificate(t *testing.T, parent *x509.Certificate, parentKey *ecdsa.Privat tmpl := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: serverName}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, IsCA: parent == nil, BasicConstraintsValid: true} if client { tmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} - tmpl.URIs = []*url.URL{{Scheme: "spiffe", Host: "example.test", Path: "/mecak8s"}} + u, _ := url.Parse("spiffe://example.test/mecak8s") + tmpl.URIs = []*url.URL{u} } else if parent != nil { tmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} tmpl.DNSNames = []string{"example.test"} @@ -98,29 +108,38 @@ func certificate(t *testing.T, parent *x509.Certificate, parentKey *ecdsa.Privat return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: parsed}, parsed, key } -func TestProviderThroughRealSignedHandlerRefreshesAndReattachesExactly(t *testing.T) { +type grpcFixture struct { + endpoint string + clientTLS *tls.Config + stop func() +} + +func startFixture(t *testing.T, backend executioncontroller.Backend, ready func() bool) grpcFixture { + t.Helper() _, ca, caKey := certificate(t, nil, nil, "ca", false) - serverTLS, _, _ := certificate(t, ca, caKey, "example.test", false) - clientTLS, _, _ := certificate(t, ca, caKey, "client", true) - _, private, err := ed25519.GenerateKey(rand.Reader) + serverCert, _, _ := certificate(t, ca, caKey, "example.test", false) + clientCert, _, _ := certificate(t, ca, caKey, "client", true) + _, private, _ := ed25519.GenerateKey(rand.Reader) + h := executioncontroller.NewHandler(executioncontroller.HandlerConfig{Clients: map[string]executioncontroller.ClientPolicy{"spiffe://example.test/mecak8s": {MayAttestOwner: true}}, Signer: executioncontroller.GrantSigner{KeyID: "k1", PrivateKey: private, Issuer: "provider", Audience: "executor", Lifetime: time.Minute}, Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": private.Public().(ed25519.PublicKey)}, Issuer: "provider", Audience: "executor", MaxLifetime: 2 * time.Minute}, Ready: ready}, backend) + pool := x509.NewCertPool() + pool.AddCert(ca) + ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } - backend := &integrationBackend{} - handler := executioncontroller.NewHandler(executioncontroller.HandlerConfig{ - Clients: map[string]executioncontroller.ClientPolicy{"spiffe://example.test/mecak8s": {MayAttestOwner: true}}, - Signer: executioncontroller.GrantSigner{KeyID: "k1", PrivateKey: private, Issuer: "provider", Audience: "executor", Lifetime: 5 * time.Second}, - Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": private.Public().(ed25519.PublicKey)}, Issuer: "provider", Audience: "executor", MaxLifetime: 10 * time.Second}, - }, backend) - clientCAs := x509.NewCertPool() - clientCAs.AddCert(ca) - ts := httptest.NewUnstartedServer(handler) - ts.TLS = &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{serverTLS}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientCAs} - ts.StartTLS() - defer ts.Close() + s := grpc.NewServer(grpc.Creds(credentials.NewTLS(executioncontroller.TLSConfig(serverCert, pool))), grpc.MaxRecvMsgSize(executionenv.MaxMessageBytes), grpc.MaxSendMsgSize(executionenv.MaxMessageBytes)) + executionv1.RegisterExecutionProviderServiceServer(s, h) + go func() { _ = s.Serve(ln) }() roots := x509.NewCertPool() roots.AddCert(ca) - client, err := New(ts.URL, &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, Certificates: []tls.Certificate{clientTLS}, ServerName: "example.test"}) + return grpcFixture{endpoint: ln.Addr().String(), clientTLS: &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, Certificates: []tls.Certificate{clientCert}, ServerName: "example.test"}, stop: func() { s.Stop(); _ = ln.Close() }} +} + +func TestProviderThroughRealGRPCSignedHandlerRefreshesAndReattachesExactly(t *testing.T) { + backend := &integrationBackend{} + fx := startFixture(t, backend, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) if err != nil { t.Fatal(err) } @@ -135,168 +154,151 @@ func TestProviderThroughRealSignedHandlerRefreshesAndReattachesExactly(t *testin if err != nil { t.Fatal(err) } - resolver, ok := reattached.Environment.Workspace().(tool.AuthorityResourceResolver) - if !ok { - t.Fatal("remote workspace does not implement AuthorityResourceResolver") - } - target, workspace, err := resolver.AuthorityResourcePath("main.go") - if err != nil || target != "/workspace/main.go" || workspace != "/workspace" { - t.Fatalf("authority resource=(%q,%q) err=%v", target, workspace, err) + resolver := reattached.Environment.Workspace().(tool.AuthorityResourceResolver) + target, root, err := resolver.AuthorityResourcePath("main.go") + if err != nil || target != "/workspace/main.go" || root != "/workspace" { + t.Fatalf("authority=(%q,%q) err=%v", target, root, err) } - data, err := reattached.Environment.Workspace().Read(context.Background(), "main.go") - if err != nil || string(data) != "from-real-handler" { + backend.expireNextRead = true + data, version, err := reattached.Environment.Workspace().ReadVersion(context.Background(), "main.go") + if err != nil || string(data) != "from-grpc" { t.Fatalf("read=%q err=%v", data, err) } - if backend.ensureCalls != 1 || backend.attachCalls < 3 || backend.fileCalls != 2 { + encoded, err := tool.EncodeFileVersion(version) + if err != nil || encoded != string([]byte{0xff, 0, 1}) { + t.Fatalf("opaque version=%q err=%v", encoded, err) + } + result, err := reattached.Environment.CommandRunner().Run(context.Background(), "go test ./...") + if err != nil || result.Stdout != "ok\n" { + t.Fatalf("result=%+v err=%v", result, err) + } + if backend.ensureCalls != 1 || backend.attachCalls < 3 || backend.fileCalls != 3 { t.Fatalf("ensure=%d attach=%d file=%d", backend.ensureCalls, backend.attachCalls, backend.fileCalls) } } -func TestProviderMTLSWorkspaceAndForegroundRunner(t *testing.T) { - caTLS, ca, caKey := certificate(t, nil, nil, "ca", false) - _ = caTLS - serverTLS, _, _ := certificate(t, ca, caKey, "example.test", false) - clientTLS, _, _ := certificate(t, ca, caKey, "client", true) - clientPool := x509.NewCertPool() - clientPool.AddCert(ca) - calls := map[string]int{} - h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if len(r.TLS.PeerCertificates) == 0 { - t.Error("request had no verified client certificate") - } - calls[r.URL.Path]++ - w.Header().Set("Content-Type", "application/json") - switch r.URL.Path { - case executionenv.BasePath + "/profiles/validate": - json.NewEncoder(w).Encode(executionenv.ValidateProfileResponse{Profile: "coding", Digest: "sha256:test"}) - case executionenv.BasePath + "/environments/ensure": - json.NewEncoder(w).Encode(executionenv.EnsureEnvironmentResponse{Environment: executionenv.EnvironmentRef{ID: "env-1", Revision: "rev-1"}, Epoch: 2, Ready: true, Grant: "grant-1", GrantExpiresAt: time.Now().Add(time.Minute)}) - case executionenv.BasePath + "/environments/attach": - json.NewEncoder(w).Encode(executionenv.AttachEnvironmentResponse{Environment: executionenv.EnvironmentRef{ID: "env-1", Revision: "rev-1"}, Epoch: 2, Ready: true, Grant: "grant-2", GrantExpiresAt: time.Now().Add(time.Minute)}) - case executionenv.BasePath + "/files": - var q executionenv.FileRequest - json.NewDecoder(r.Body).Decode(&q) - if q.Context.BindingID != "session-1" || q.Context.Grant != "grant-2" { - t.Errorf("file context = %+v", q.Context) - } - if q.Operation == executionenv.OpFileResolveAuthority { - if q.Path == "escape" { - w.WriteHeader(http.StatusForbidden) - json.NewEncoder(w).Encode(executionenv.ErrorResponse{Error: &executionenv.Error{Code: executionenv.CodePermissionDenied, Message: "path is outside workspace"}}) - } else { - json.NewEncoder(w).Encode(executionenv.FileResponse{AuthorityTarget: "/workspace/main.go", AuthorityWorkspace: "/workspace"}) - } - } else { - json.NewEncoder(w).Encode(executionenv.FileResponse{Data: []byte("remote"), Version: "v1"}) - } - case executionenv.BasePath + "/commands/start": - json.NewEncoder(w).Encode(executionenv.CommandStartResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Result: executionenv.CommandStatusResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Stdout: []byte("ok\n"), Stderr: []byte("warn\n")}}) - default: - http.NotFound(w, r) - } - }) - ts := httptest.NewUnstartedServer(h) - ts.TLS = &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{serverTLS}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientPool} - ts.StartTLS() - defer ts.Close() - roots := x509.NewCertPool() - roots.AddCert(ca) - client, err := New(ts.URL, &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, Certificates: []tls.Certificate{clientTLS}, ServerName: "example.test"}) +func TestCommandStatusIsUnimplementedOnlyAfterAuthorization(t *testing.T) { + backend := &integrationBackend{} + fx := startFixture(t, backend, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) if err != nil { t.Fatal(err) } - provider, _ := NewProvider(client, "coding") - if err := provider.ValidatePlacement(context.Background()); err != nil { + defer client.Close() + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + ensured, err := client.Ensure(context.Background(), "binding", "coding", owner) + if err != nil { t.Fatal(err) } - principal := &session.Principal{Issuer: "https://issuer.example", Subject: "alice", GrantType: session.GrantTypeUser} - binding, err := provider.Bind(context.Background(), server.PlacementBindRequest{Selector: server.DefaultPlacement(), Principal: principal, Scope: "remote", Operation: server.PlacementOperationCreate, BindingID: "session-1"}) + attached, err := client.Attach(context.Background(), executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ensured.Environment, Owner: owner, BindingID: "binding"}, Purpose: executionenv.PurposeSession}) if err != nil { t.Fatal(err) } - resolver := binding.Environment.Workspace().(tool.AuthorityResourceResolver) - target, root, err := resolver.AuthorityResourcePath("main.go") - if err != nil || target != "/workspace/main.go" || root != "/workspace" { - t.Fatalf("authority resource=(%q,%q) err=%v", target, root, err) + rc := executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: "binding", Epoch: attached.Epoch, Grant: attached.Grant} + _, err = client.rpc.CommandStatus(context.Background(), &executionv1.CommandQueryRequest{Context: contextToProto(rc), CommandId: "c1"}) + if status.Code(err) != codes.Unimplemented { + t.Fatalf("authorized status code=%v", status.Code(err)) } - if target, root, err = resolver.AuthorityResourcePath("escape"); err == nil || target != "" || root != "" { - t.Fatalf("escaped authority resource=(%q,%q) err=%v, want fail closed", target, root, err) + rc.Owner.Subject = "mallory" + _, err = client.rpc.CommandStatus(context.Background(), &executionv1.CommandQueryRequest{Context: contextToProto(rc), CommandId: "c1"}) + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("wrong-owner status code=%v", status.Code(err)) } - data, _, err := binding.Environment.Workspace().ReadVersion(context.Background(), "main.go") - if err != nil || string(data) != "remote" { - t.Fatalf("read=%q err=%v", data, err) +} + +func TestGRPCReadinessAndMTLSAreMandatory(t *testing.T) { + fx := startFixture(t, &integrationBackend{}, func() bool { return false }) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) } - result, err := binding.Environment.CommandRunner().Run(context.Background(), "go test ./...") - if err != nil || result.Stdout != "ok\n" || result.Stderr != "warn\n" { - t.Fatalf("command=%+v err=%v", result, err) + defer client.Close() + _, err = client.ValidateProfile(context.Background(), "coding") + var remote *executionenv.Error + if !errors.As(err, &remote) || remote.Code != executionenv.CodeNotReady { + t.Fatalf("error=%v", err) } - if _, streaming := binding.Environment.CommandRunner().(tool.CommandStreamer); streaming { - t.Fatal("remote runner unexpectedly advertises streaming") + noCert := fx.clientTLS.Clone() + noCert.Certificates = nil + conn, err := grpc.NewClient(fx.endpoint, grpc.WithTransportCredentials(credentials.NewTLS(noCert))) + if err != nil { + t.Fatal(err) } - if calls[executionenv.BasePath+"/profiles/validate"] != 1 || calls[executionenv.BasePath+"/environments/ensure"] != 1 { - t.Fatalf("calls=%v", calls) + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := executionv1.NewExecutionProviderServiceClient(conn).ValidateProfile(ctx, &executionv1.ValidateProfileRequest{Profile: "coding"}); err == nil { + t.Fatal("server accepted client without certificate") } } -func TestProviderWaitsForAsyncReadinessWithoutReEnsuring(t *testing.T) { - var ensureCalls, attachCalls int - h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - ref := executionenv.EnvironmentRef{ID: "env-pending", Revision: "rev-1"} - switch r.URL.Path { - case executionenv.BasePath + "/environments/ensure": - ensureCalls++ - json.NewEncoder(w).Encode(executionenv.EnsureEnvironmentResponse{Environment: ref, Epoch: 1, Ready: false, Grant: "initial", GrantExpiresAt: time.Now().Add(time.Minute)}) - case executionenv.BasePath + "/environments/attach": - attachCalls++ - json.NewEncoder(w).Encode(executionenv.AttachEnvironmentResponse{Environment: ref, Epoch: 1, Ready: attachCalls > 1, Grant: "fresh", GrantExpiresAt: time.Now().Add(time.Minute)}) - default: - http.NotFound(w, r) +func TestDecodeErrorRejectsUntrustedOrContradictoryMetadata(t *testing.T) { + const marker = "/var/run/secrets/provider-key" + typed := func(code executionenv.ErrorCode, retry bool, grpcCode codes.Code) error { + st, err := status.New(grpcCode, marker).WithDetails(&executionv1.ErrorDetail{Code: string(code), Retryable: retry}) + if err != nil { + t.Fatal(err) } - }) - ts := httptest.NewServer(h) - defer ts.Close() - client, err := NewWithHTTPClient(ts.URL, ts.Client()) + return st.Err() + } + multiple, err := status.New(codes.NotFound, marker).WithDetails( + &executionv1.ErrorDetail{Code: string(executionenv.CodeNotFound)}, + &executionv1.ErrorDetail{Code: string(executionenv.CodePermissionDenied)}, + ) if err != nil { t.Fatal(err) } - provider, _ := NewProvider(client, "coding") - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} - if _, err := provider.Bind(ctx, server.PlacementBindRequest{Selector: server.DefaultPlacement(), Principal: principal, BindingID: "session-pending"}); err != nil { + cases := []error{ + errors.New(marker), + status.Error(codes.Internal, marker), + typed("", false, codes.Internal), + typed(executionenv.ErrorCode("future"), false, codes.Internal), + typed(executionenv.CodeNotFound, false, codes.PermissionDenied), + typed(executionenv.CodeNotFound, true, codes.NotFound), + multiple.Err(), + } + for i, input := range cases { + got := decodeError(context.Background(), input) + var remote *executionenv.Error + if !errors.As(got, &remote) || remote.Code != executionenv.CodeInternal || remote.Retryable || strings.Contains(got.Error(), marker) { + t.Fatalf("case %d escaped or misclassified: %v", i, got) + } + } +} + +func TestDecodeErrorAcceptsClosedTypedErrorAndLocalCancellation(t *testing.T) { + st, err := status.New(codes.Unauthenticated, "peer text").WithDetails(&executionv1.ErrorDetail{Code: string(executionenv.CodeUnauthenticated), Retryable: true}) + if err != nil { t.Fatal(err) } - if ensureCalls != 1 || attachCalls != 2 { - t.Fatalf("ensure=%d attach=%d, want 1/2", ensureCalls, attachCalls) + var remote *executionenv.Error + if got := decodeError(context.Background(), st.Err()); !errors.As(got, &remote) || remote.Code != executionenv.CodeUnauthenticated || !remote.Retryable { + t.Fatalf("typed error=%v", got) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if got := decodeError(ctx, status.Error(codes.Canceled, "peer text")); !errors.Is(got, context.Canceled) { + t.Fatalf("local cancellation=%v", got) } } -func TestProviderReattachMissingNeverEnsures(t *testing.T) { - var ensureCalls int - h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - if r.URL.Path == executionenv.BasePath+"/environments/ensure" { - ensureCalls++ - } - w.WriteHeader(http.StatusNotFound) - json.NewEncoder(w).Encode(executionenv.ErrorResponse{Error: &executionenv.Error{Code: executionenv.CodeNotFound, Message: "environment not found"}}) - }) - ts := httptest.NewServer(h) - defer ts.Close() - client, _ := NewWithHTTPClient(ts.URL, ts.Client()) - provider, _ := NewProvider(client, "coding") - principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} - _, err := provider.Reattach(context.Background(), server.PlacementReattachRequest{Ref: session.EnvironmentRef{Kind: "kubernetes", ID: "deleted", Revision: "rev-1"}, Principal: principal, BindingID: "session-1"}) - if err == nil || ensureCalls != 0 { - t.Fatalf("err=%v ensure=%d, want unavailable and zero ensure calls", err, ensureCalls) +func TestCommandResponsesRejectUnknownStates(t *testing.T) { + if _, err := commandStartFromProto(&executionv1.CommandStartResponse{State: executionv1.CommandState_COMMAND_STATE_UNSPECIFIED, Result: &executionv1.CommandStatusResponse{State: executionv1.CommandState_COMMAND_STATE_SUCCEEDED}}); err == nil { + t.Fatal("accepted unspecified command-start state") + } + if _, err := commandStatusFromProto(&executionv1.CommandStatusResponse{State: executionv1.CommandState(99), TerminalReceipt: "clean"}); err == nil { + t.Fatal("accepted unknown command-status state") } } -func TestProviderRequiresOwnerAndBinding(t *testing.T) { - p := &Provider{} - _, err := p.Bind(context.Background(), server.PlacementBindRequest{Selector: server.DefaultPlacement()}) - if err == nil { - t.Fatal("ownerless/bindingless allocation succeeded") +func TestEndpointRejectsHTTPAndAlternateResolvers(t *testing.T) { + cfg := &tls.Config{RootCAs: x509.NewCertPool(), Certificates: []tls.Certificate{{Certificate: [][]byte{{1}}}}} + for _, endpoint := range []string{"https://provider:8443", "dns:///provider:8443", "provider"} { + if c, err := New(endpoint, cfg); err == nil { + c.Close() + t.Fatalf("accepted %q", endpoint) + } } } diff --git a/internal/adapter/executioncontroller/handler.go b/internal/adapter/executioncontroller/handler.go index 2148d58ed8..0db2cec4b7 100644 --- a/internal/adapter/executioncontroller/handler.go +++ b/internal/adapter/executioncontroller/handler.go @@ -1,4 +1,4 @@ -// Package executioncontroller implements the authenticated provider HTTP boundary. +// Package executioncontroller implements the authenticated provider gRPC boundary. package executioncontroller import ( @@ -9,30 +9,33 @@ import ( "crypto/tls" "crypto/x509" "encoding/hex" - "encoding/json" "errors" "io" - "net/http" - "sort" + "math" "strings" "time" + "unicode/utf8" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" "github.com/stacklok/mecatl/internal/executionenv" ) // ClientPolicy defines capabilities assigned to an authenticated provider client. -type ClientPolicy struct { - MayAttestOwner bool - Administrator bool -} +type ClientPolicy struct{ MayAttestOwner, Administrator bool } // GrantSigner configures short-lived environment grant issuance. type GrantSigner struct { - KeyID string - PrivateKey ed25519.PrivateKey - Issuer string - Audience string - Lifetime time.Duration + KeyID string + PrivateKey ed25519.PrivateKey + Issuer, Audience string + Lifetime time.Duration } // HandlerConfig configures authentication and capability grants. @@ -45,28 +48,24 @@ type HandlerConfig struct { // Profile is the externally visible immutable execution profile. type Profile struct { - Name string - Digest string - MaxFileBytes int64 - MaxCommandBytes int64 - MaxCommandDuration time.Duration - Capabilities []string + Name, Digest string + MaxFileBytes, MaxCommandBytes int64 + MaxCommandDuration time.Duration + Capabilities []string } // Allocation is an exact environment allocation and authorization binding. type Allocation struct { - Environment executionenv.EnvironmentRef - Epoch uint64 - OwnerHash string - BindingID string - Client string - Ready bool + Environment executionenv.EnvironmentRef + Epoch uint64 + OwnerHash, BindingID, Client string + Ready bool } // Backend implements provider-side authorization state and executor dispatch. type Backend interface { ValidateProfile(context.Context, string) (Profile, error) - Ensure(context.Context, string, string, string, string, string) (Allocation, error) // client, owner hash, binding, profile, immutable fingerprint + Ensure(context.Context, string, string, string, string, string) (Allocation, error) Attach(context.Context, executionenv.EnvironmentRef, string, string, string) (Allocation, error) ReleaseReference(context.Context, executionenv.EnvironmentRef, string, string, string) error Retire(context.Context, executionenv.EnvironmentRef, string) error @@ -76,280 +75,368 @@ type Backend interface { CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) } -// Handler is the authenticated private execution-provider HTTP boundary. +// Handler is the authenticated private execution-provider gRPC service. type Handler struct { + executionv1.UnimplementedExecutionProviderServiceServer cfg HandlerConfig backend Backend - mux *http.ServeMux -} - -// NewHandler constructs the authenticated private HTTP boundary. -func NewHandler(cfg HandlerConfig, b Backend) *Handler { - h := &Handler{cfg: cfg, backend: b, mux: http.NewServeMux()} - h.mux.HandleFunc("POST "+executionenv.BasePath+"/profiles/validate", h.validate) - h.mux.HandleFunc("POST "+executionenv.BasePath+"/environments/ensure", h.ensure) - h.mux.HandleFunc("POST "+executionenv.BasePath+"/environments/attach", h.attach) - h.mux.HandleFunc("POST "+executionenv.BasePath+"/references/release", h.release) - h.mux.HandleFunc("POST "+executionenv.BasePath+"/environments/retire", h.retire) - h.mux.HandleFunc("POST "+executionenv.BasePath+"/files", h.file) - h.mux.HandleFunc("POST "+executionenv.BasePath+"/commands/start", h.startCommand) - h.mux.HandleFunc("POST "+executionenv.BasePath+"/commands/status", h.commandStatus) - h.mux.HandleFunc("POST "+executionenv.BasePath+"/commands/cancel", h.cancelCommand) - return h -} -func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if h.cfg.Ready != nil && !h.cfg.Ready() { - writeError(w, http.StatusServiceUnavailable, executionenv.CodeNotReady, "provider startup fencing is incomplete", true) - return - } - id, policy, err := h.authenticate(r) - if err != nil { - writeError(w, http.StatusUnauthorized, executionenv.CodeUnauthenticated, "authenticated client certificate required", false) - return - } - ctx := context.WithValue(r.Context(), clientContextKey{}, authenticatedClient{id, policy}) - h.mux.ServeHTTP(w, r.WithContext(ctx)) } -type clientContextKey struct{} +// NewHandler constructs the authenticated private gRPC service. +func NewHandler(cfg HandlerConfig, b Backend) *Handler { return &Handler{cfg: cfg, backend: b} } + type authenticatedClient struct { id string policy ClientPolicy } -func (h *Handler) authenticate(r *http.Request) (string, ClientPolicy, error) { - if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 { - return "", ClientPolicy{}, errors.New("no peer certificate") +func (h *Handler) client(ctx context.Context) (authenticatedClient, error) { + if h.backend == nil || (h.cfg.Ready != nil && !h.cfg.Ready()) { + return authenticatedClient{}, wireError(executionenv.CodeNotReady, true) } - id, err := canonicalClientIdentity(r.TLS.PeerCertificates[0]) + p, ok := peer.FromContext(ctx) + if !ok { + return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) + } + tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) + if !ok || len(tlsInfo.State.VerifiedChains) == 0 || len(tlsInfo.State.PeerCertificates) == 0 { + return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) + } + id, err := canonicalClientIdentity(tlsInfo.State.PeerCertificates[0]) if err != nil { - return "", ClientPolicy{}, err + return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) } - p, ok := h.cfg.Clients[id] + policy, ok := h.cfg.Clients[id] if !ok { - return "", ClientPolicy{}, errors.New("client not allowlisted") + return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) } - return id, p, nil + return authenticatedClient{id: id, policy: policy}, nil } + func canonicalClientIdentity(c *x509.Certificate) (string, error) { - ids := make([]string, 0, len(c.URIs)) + if len(c.URIs) != 1 { + return "", errors.New("certificate must have exactly one URI SAN") + } + ids := make([]string, 0, 1) for _, u := range c.URIs { if u == nil || u.Scheme == "" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { continue } v := *u - v.Scheme = strings.ToLower(v.Scheme) - v.Host = strings.ToLower(v.Host) + v.Scheme, v.Host = strings.ToLower(v.Scheme), strings.ToLower(v.Host) ids = append(ids, v.String()) } - if len(ids) == 0 { - return "", errors.New("certificate has no canonical URI SAN") + if len(ids) != 1 { + return "", errors.New("certificate must have exactly one canonical URI SAN") } - sort.Strings(ids) return ids[0], nil } -func (h *Handler) validate(w http.ResponseWriter, r *http.Request) { - var q executionenv.ValidateProfileRequest - if !decode(w, r, &q) { - return +// ValidateProfile validates one operator-defined execution profile. +func (h *Handler) ValidateProfile(ctx context.Context, q *executionv1.ValidateProfileRequest) (*executionv1.ValidateProfileResponse, error) { + if _, err := h.client(ctx); err != nil { + return nil, err } - if q.Profile == "" || len(q.Profile) > 63 { - writeError(w, 400, executionenv.CodeInvalidArgument, "invalid profile", false) - return + if q == nil || q.Profile == "" || len(q.Profile) > 63 { + return nil, wireError(executionenv.CodeInvalidArgument, false) } - if h.backend == nil { - writeError(w, 503, executionenv.CodeNotReady, "provider backend unavailable", true) - return - } - p, err := h.backend.ValidateProfile(r.Context(), q.Profile) + p, err := h.backend.ValidateProfile(ctx, q.Profile) if err != nil { - writeBackendError(w, err) - return + return nil, backendError(err) } - writeJSON(w, 200, executionenv.ValidateProfileResponse{Profile: p.Name, Digest: p.Digest, Capabilities: p.Capabilities, MaxFileBytes: p.MaxFileBytes, MaxCommandBytes: p.MaxCommandBytes, MaxCommandDurationMillis: p.MaxCommandDuration.Milliseconds()}) + return &executionv1.ValidateProfileResponse{Profile: valid(p.Name), Digest: valid(p.Digest), Capabilities: validAll(p.Capabilities), MaxFileBytes: p.MaxFileBytes, MaxCommandBytes: p.MaxCommandBytes, MaxCommandDurationMillis: p.MaxCommandDuration.Milliseconds()}, nil } -func (h *Handler) ensure(w http.ResponseWriter, r *http.Request) { - var q executionenv.EnsureEnvironmentRequest - if !decode(w, r, &q) { - return + +// EnsureEnvironment idempotently resolves or allocates an environment. +func (h *Handler) EnsureEnvironment(ctx context.Context, q *executionv1.EnsureEnvironmentRequest) (*executionv1.EnsureEnvironmentResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err } - c := r.Context().Value(clientContextKey{}).(authenticatedClient) - if !c.policy.MayAttestOwner || q.Owner.Issuer == "" || q.Owner.Subject == "" || len(q.Owner.Issuer) > executionenv.MaxIdentityBytes || len(q.Owner.Subject) > executionenv.MaxIdentityBytes || q.BindingID == "" || len(q.BindingID) > executionenv.MaxBindingBytes || q.Profile == "" || len(q.Profile) > 63 { - writeError(w, 403, executionenv.CodePermissionDenied, "owner attestation or required allocation identity missing", false) - return + owner, ok := ownerFromProto(q.GetOwner()) + if q == nil || !c.policy.MayAttestOwner || !ok || q.BindingId == "" || len(q.BindingId) > executionenv.MaxBindingBytes || q.Profile == "" || len(q.Profile) > 63 { + return nil, wireError(executionenv.CodePermissionDenied, false) } - owner := ownerHash(q.Owner) - fp := fingerprint(c.id, owner, q.BindingID, q.Profile) - a, err := h.backend.Ensure(r.Context(), c.id, owner, q.BindingID, q.Profile, fp) + oh := ownerHash(owner) + a, err := h.backend.Ensure(ctx, c.id, oh, q.BindingId, q.Profile, fingerprint(c.id, oh, q.BindingId, q.Profile)) if err != nil { - writeBackendError(w, err) - return + return nil, backendError(err) } - grant, expiresAt, err := h.sign(a, c.id) + grant, expiry, err := h.sign(a, c.id) if err != nil { - writeError(w, 500, executionenv.CodeInternal, "grant issuance failed", false) - return + return nil, wireError(executionenv.CodeInternal, false) } - writeJSON(w, 200, executionenv.EnsureEnvironmentResponse{Environment: a.Environment, Epoch: a.Epoch, Ready: a.Ready, Grant: grant, GrantExpiresAt: expiresAt}) + return ensureResponse(a, grant, expiry), nil } -func (h *Handler) attach(w http.ResponseWriter, r *http.Request) { - var q executionenv.AttachEnvironmentRequest - if !decode(w, r, &q) { - return - } - if q.Purpose != executionenv.PurposeSession { - writeError(w, 400, executionenv.CodeInvalidArgument, "unsupported attachment purpose", false) - return + +// AttachEnvironment exactly reattaches and refreshes a short-lived grant. +func (h *Handler) AttachEnvironment(ctx context.Context, q *executionv1.AttachEnvironmentRequest) (*executionv1.AttachEnvironmentResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err } - c := r.Context().Value(clientContextKey{}).(authenticatedClient) - if !c.policy.MayAttestOwner || q.Context.Owner.Issuer == "" || q.Context.Owner.Subject == "" || len(q.Context.Owner.Issuer) > executionenv.MaxIdentityBytes || len(q.Context.Owner.Subject) > executionenv.MaxIdentityBytes || q.Context.BindingID == "" || len(q.Context.BindingID) > executionenv.MaxBindingBytes || q.Context.Environment.ID == "" || q.Context.Environment.Revision == "" || len(q.Context.Environment.ID) > executionenv.MaxIdentityBytes || len(q.Context.Environment.Revision) > executionenv.MaxIdentityBytes { - writeError(w, 403, executionenv.CodePermissionDenied, "owner attestation or attachment identity missing", false) - return + rc, ok := attachContext(q.GetContext()) + if q == nil || q.Purpose != executionenv.PurposeSession || !c.policy.MayAttestOwner || !ok { + return nil, wireError(executionenv.CodePermissionDenied, false) } - owner := ownerHash(q.Context.Owner) - a, err := h.backend.Attach(r.Context(), q.Context.Environment, c.id, owner, q.Context.BindingID) + oh := ownerHash(rc.Owner) + a, err := h.backend.Attach(ctx, rc.Environment, c.id, oh, rc.BindingID) if err != nil { - writeBackendError(w, err) - return + return nil, backendError(err) } - grant, expiresAt, err := h.sign(a, c.id) + grant, expiry, err := h.sign(a, c.id) if err != nil { - writeError(w, 500, executionenv.CodeInternal, "grant issuance failed", false) - return + return nil, wireError(executionenv.CodeInternal, false) } - writeJSON(w, 200, executionenv.AttachEnvironmentResponse{Environment: a.Environment, Epoch: a.Epoch, Ready: a.Ready, Grant: grant, GrantExpiresAt: expiresAt}) + return attachResponse(a, grant, expiry), nil } -func (h *Handler) release(w http.ResponseWriter, r *http.Request) { - var q executionenv.ReferenceReleaseRequest - if !decode(w, r, &q) { - return + +// ReleaseReference releases one authorized durable binding reference. +func (h *Handler) ReleaseReference(ctx context.Context, q *executionv1.ReleaseReferenceRequest) (*emptypb.Empty, error) { + c, owner, rc, err := h.authorize(ctx, q.GetContext(), executionenv.OpReferenceRelease, false) + if err != nil { + return nil, err } - c, owner, ok := h.authorize(w, r, q.Context, executionenv.OpReferenceRelease) - if !ok { - return + if err := h.backend.ReleaseReference(ctx, rc.Environment, c.id, owner, rc.BindingID); err != nil { + return nil, backendError(err) } - if err := h.backend.ReleaseReference(r.Context(), q.Context.Environment, c.id, owner, q.Context.BindingID); err != nil { - writeBackendError(w, err) - return + return &emptypb.Empty{}, nil +} + +// RetireEnvironment requests authorized administrative retirement. +func (h *Handler) RetireEnvironment(ctx context.Context, q *executionv1.RetireEnvironmentRequest) (*emptypb.Empty, error) { + c, owner, rc, err := h.authorize(ctx, q.GetContext(), executionenv.OpRetire, true) + if err != nil { + return nil, err + } + _ = c + if err := h.backend.Retire(ctx, rc.Environment, owner); err != nil { + return nil, backendError(err) } - writeJSON(w, 200, executionenv.EmptyResponse{}) + return &emptypb.Empty{}, nil } -func (h *Handler) retire(w http.ResponseWriter, r *http.Request) { - var q executionenv.RetireEnvironmentRequest - if !decode(w, r, &q) { - return + +// Files executes one bounded and authorized filesystem operation. +func (h *Handler) Files(ctx context.Context, q *executionv1.FileRequest) (*executionv1.FileResponse, error) { + if _, err := h.client(ctx); err != nil { + return nil, err } - c := r.Context().Value(clientContextKey{}).(authenticatedClient) - if !c.policy.Administrator { - writeError(w, 403, executionenv.CodePermissionDenied, "administrative identity required", false) - return + op, ok := operationFromProto(q.GetOperation()) + if q == nil || !ok || !validFileRequest(q) { + return nil, wireError(executionenv.CodeInvalidArgument, false) } - _, owner, ok := h.authorize(w, r, q.Context, executionenv.OpRetire) - if !ok { - return + c, owner, rc, err := h.authorize(ctx, q.Context, op, false) + if err != nil { + return nil, err } - if err := h.backend.Retire(r.Context(), q.Context.Environment, owner); err != nil { - writeBackendError(w, err) - return + req := executionenv.FileRequest{Context: rc, Operation: op, Path: q.Path, Destination: q.Destination, Pattern: q.Pattern, Data: q.Data, Version: string(q.Version), Limit: int(q.Limit)} + out, err := h.backend.File(ctx, c.id, owner, req) + if err != nil { + return nil, backendError(err) } - writeJSON(w, 200, executionenv.EmptyResponse{}) + return fileResponse(out), nil } -func (h *Handler) file(w http.ResponseWriter, r *http.Request) { - var q executionenv.FileRequest - if !decode(w, r, &q) { - return + +// StartCommand executes one authorized foreground command. +func (h *Handler) StartCommand(ctx context.Context, q *executionv1.CommandStartRequest) (*executionv1.CommandStartResponse, error) { + if _, err := h.client(ctx); err != nil { + return nil, err } - if len(q.Data) > executionenv.MaxFileBytes || !q.Operation.Valid() || len(q.Path) > executionenv.MaxPathBytes || len(q.Destination) > executionenv.MaxPathBytes || len(q.Pattern) > executionenv.MaxPathBytes || q.Limit < 0 || q.Limit > executionenv.MaxListEntries { - writeError(w, 400, executionenv.CodeInvalidArgument, "invalid or oversized file request", false) - return + if q == nil || len(q.Command) == 0 || len(q.Command) > executionenv.MaxCommandBytes || q.TimeoutMillis < 0 { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + c, owner, rc, err := h.authorize(ctx, q.Context, executionenv.OpCommandStart, false) + if err != nil { + return nil, err } - c, owner, ok := h.authorize(w, r, q.Context, q.Operation) + out, err := h.backend.StartCommand(ctx, c.id, owner, executionenv.CommandStartRequest{Context: rc, Command: q.Command, TimeoutMillis: q.TimeoutMillis}) + if err != nil { + return nil, backendError(err) + } + response, ok := commandStartResponse(out) if !ok { - return + return nil, wireError(executionenv.CodeInternal, false) } - resp, err := h.backend.File(r.Context(), c.id, owner, q) + return response, nil +} + +// CommandStatus authorizes then reports the detached API as unsupported. +func (h *Handler) CommandStatus(ctx context.Context, q *executionv1.CommandQueryRequest) (*executionv1.CommandStatusResponse, error) { + return h.commandQuery(ctx, q, executionenv.OpCommandStatus) +} + +// CancelCommand authorizes then reports the detached API as unsupported. +func (h *Handler) CancelCommand(ctx context.Context, q *executionv1.CommandQueryRequest) (*executionv1.CommandStatusResponse, error) { + return h.commandQuery(ctx, q, executionenv.OpCommandCancel) +} +func (h *Handler) commandQuery(ctx context.Context, q *executionv1.CommandQueryRequest, op executionenv.Operation) (*executionv1.CommandStatusResponse, error) { + _, _, _, err := h.authorize(ctx, q.GetContext(), op, false) if err != nil { - writeBackendError(w, err) - return + return nil, err } - writeJSON(w, 200, resp) + if q.CommandId == "" || q.Offset < 0 { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + return nil, status.Error(codes.Unimplemented, "foreground commands have no detached control API") } -func (h *Handler) startCommand(w http.ResponseWriter, r *http.Request) { - var q executionenv.CommandStartRequest - if !decode(w, r, &q) { - return + +func (h *Handler) authorize(ctx context.Context, p *executionv1.RequestContext, op executionenv.Operation, admin bool) (authenticatedClient, string, executionenv.RequestContext, error) { + c, err := h.client(ctx) + if err != nil { + return c, "", executionenv.RequestContext{}, err } - if len(q.Command) == 0 || len(q.Command) > executionenv.MaxCommandBytes || q.TimeoutMillis < 0 { - writeError(w, 400, executionenv.CodeInvalidArgument, "invalid command size", false) - return + if admin && !c.policy.Administrator { + return c, "", executionenv.RequestContext{}, wireError(executionenv.CodePermissionDenied, false) } - c, owner, ok := h.authorize(w, r, q.Context, executionenv.OpCommandStart) + rc, ok := requestContextFromProto(p) if !ok { - return + return c, "", rc, wireError(executionenv.CodePermissionDenied, false) } - v, err := h.backend.StartCommand(r.Context(), c.id, owner, q) - if err != nil { - writeBackendError(w, err) - return + oh := ownerHash(rc.Owner) + if _, err := h.cfg.Verifier.Verify(rc.Grant, executionenv.GrantExpectation{Client: c.id, OwnerHash: oh, BindingID: rc.BindingID, Environment: rc.Environment, Epoch: rc.Epoch, Operation: op}); err != nil { + if errors.Is(err, executionenv.ErrGrantExpired) { + return c, "", rc, wireError(executionenv.CodeUnauthenticated, true) + } + return c, "", rc, wireError(executionenv.CodePermissionDenied, false) + } + return c, oh, rc, nil +} + +func requestContextFromProto(p *executionv1.RequestContext) (executionenv.RequestContext, bool) { + owner, ownerOK := ownerFromProto(p.GetOwner()) + ref := refFromProto(p.GetEnvironment()) + rc := executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: p.GetBindingId(), Epoch: p.GetEpoch(), Grant: p.GetGrant()} + ok := p != nil && ownerOK && ref.ID != "" && ref.Revision != "" && len(ref.ID) <= executionenv.MaxIdentityBytes && len(ref.Revision) <= executionenv.MaxIdentityBytes && rc.BindingID != "" && len(rc.BindingID) <= executionenv.MaxBindingBytes && rc.Epoch != 0 && rc.Grant != "" && len(rc.Grant) <= executionenv.MaxGrantBytes + return rc, ok +} +func attachContext(p *executionv1.RequestContext) (executionenv.RequestContext, bool) { + owner, ownerOK := ownerFromProto(p.GetOwner()) + ref := refFromProto(p.GetEnvironment()) + rc := executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: p.GetBindingId()} + return rc, p != nil && ownerOK && ref.ID != "" && ref.Revision != "" && len(ref.ID) <= executionenv.MaxIdentityBytes && len(ref.Revision) <= executionenv.MaxIdentityBytes && rc.BindingID != "" && len(rc.BindingID) <= executionenv.MaxBindingBytes +} +func ownerFromProto(p *executionv1.Owner) (executionenv.Owner, bool) { + o := executionenv.Owner{Issuer: p.GetIssuer(), Subject: p.GetSubject()} + return o, p != nil && o.Issuer != "" && o.Subject != "" && len(o.Issuer) <= executionenv.MaxIdentityBytes && len(o.Subject) <= executionenv.MaxIdentityBytes +} +func refFromProto(p *executionv1.EnvironmentRef) executionenv.EnvironmentRef { + return executionenv.EnvironmentRef{ID: p.GetId(), Revision: p.GetRevision()} +} +func refToProto(r executionenv.EnvironmentRef) *executionv1.EnvironmentRef { + return &executionv1.EnvironmentRef{Id: valid(r.ID), Revision: valid(r.Revision)} +} + +func validFileRequest(q *executionv1.FileRequest) bool { //nolint:gocyclo // Closed operation/field matrix is intentionally explicit. + if len(q.Data) > executionenv.MaxFileBytes || len(q.Path) > executionenv.MaxPathBytes || len(q.Destination) > executionenv.MaxPathBytes || len(q.Pattern) > executionenv.MaxPathBytes || q.Limit < 0 || q.Limit > executionenv.MaxListEntries { + return false + } + path, dest, pattern, data, version, limit := q.Path != "", q.Destination != "", q.Pattern != "", len(q.Data) != 0, len(q.Version) != 0, q.Limit != 0 + switch q.Operation { + case executionv1.FileOperation_FILE_OPERATION_READ, executionv1.FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY, executionv1.FileOperation_FILE_OPERATION_STAT, executionv1.FileOperation_FILE_OPERATION_REMOVE: + return path && !dest && !pattern && !data && !version && !limit + case executionv1.FileOperation_FILE_OPERATION_CREATE: + return path && !dest && !pattern && !version && !limit + case executionv1.FileOperation_FILE_OPERATION_REPLACE: + return path && !dest && !pattern && version && !limit + case executionv1.FileOperation_FILE_OPERATION_LIST: + return path && !dest && !pattern && !data && !version + case executionv1.FileOperation_FILE_OPERATION_RENAME, executionv1.FileOperation_FILE_OPERATION_COPY: + return path && dest && !pattern && !data && !version && !limit + case executionv1.FileOperation_FILE_OPERATION_GLOB: + return !path && !dest && pattern && !data && !version + case executionv1.FileOperation_FILE_OPERATION_GREP: + return path && !dest && pattern && !data && !version + default: + return false } - writeJSON(w, 200, v) } -func (h *Handler) commandStatus(w http.ResponseWriter, r *http.Request) { - h.commandQuery(w, r, executionenv.OpCommandStatus) + +func operationFromProto(op executionv1.FileOperation) (executionenv.Operation, bool) { + m := map[executionv1.FileOperation]executionenv.Operation{executionv1.FileOperation_FILE_OPERATION_READ: executionenv.OpFileRead, executionv1.FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY: executionenv.OpFileResolveAuthority, executionv1.FileOperation_FILE_OPERATION_STAT: executionenv.OpFileStat, executionv1.FileOperation_FILE_OPERATION_CREATE: executionenv.OpFileCreate, executionv1.FileOperation_FILE_OPERATION_REPLACE: executionenv.OpFileReplace, executionv1.FileOperation_FILE_OPERATION_LIST: executionenv.OpFileList, executionv1.FileOperation_FILE_OPERATION_REMOVE: executionenv.OpFileRemove, executionv1.FileOperation_FILE_OPERATION_RENAME: executionenv.OpFileRename, executionv1.FileOperation_FILE_OPERATION_COPY: executionenv.OpFileCopy, executionv1.FileOperation_FILE_OPERATION_GLOB: executionenv.OpFileGlob, executionv1.FileOperation_FILE_OPERATION_GREP: executionenv.OpFileGrep} + v, ok := m[op] + return v, ok +} + +func ensureResponse(a Allocation, grant string, expiry time.Time) *executionv1.EnsureEnvironmentResponse { + return &executionv1.EnsureEnvironmentResponse{Environment: refToProto(a.Environment), Epoch: a.Epoch, Ready: a.Ready, Grant: valid(grant), GrantExpiresAt: timestamppb.New(expiry)} } -func (h *Handler) cancelCommand(w http.ResponseWriter, r *http.Request) { - h.commandQuery(w, r, executionenv.OpCommandCancel) +func attachResponse(a Allocation, grant string, expiry time.Time) *executionv1.AttachEnvironmentResponse { + return &executionv1.AttachEnvironmentResponse{Environment: refToProto(a.Environment), Epoch: a.Epoch, Ready: a.Ready, Grant: valid(grant), GrantExpiresAt: timestamppb.New(expiry)} } -func (h *Handler) commandQuery(w http.ResponseWriter, r *http.Request, op executionenv.Operation) { - var q executionenv.CommandQueryRequest - if !decode(w, r, &q) { - return +func fileResponse(v executionenv.FileResponse) *executionv1.FileResponse { + r := &executionv1.FileResponse{Data: v.Data, Version: []byte(v.Version), Paths: validAll(v.Paths), AuthorityTarget: valid(v.AuthorityTarget), AuthorityWorkspace: valid(v.AuthorityWorkspace)} + if v.Info != nil { + r.Info = fileInfoToProto(*v.Info) + } + for _, x := range v.Entries { + r.Entries = append(r.Entries, fileInfoToProto(x)) + } + for _, x := range v.Matches { + r.Matches = append(r.Matches, &executionv1.GrepMatch{Path: valid(x.Path), Line: boundedInt32(x.Line), Text: valid(x.Text)}) } - c, owner, ok := h.authorize(w, r, q.Context, op) + return r +} +func fileInfoToProto(v executionenv.FileInfo) *executionv1.FileInfo { + return &executionv1.FileInfo{Name: valid(v.Name), Size: v.Size, Mode: v.Mode, ModTime: timestamppb.New(v.ModTime), IsDir: v.IsDir} +} +func commandStartResponse(v executionenv.CommandStartResponse) (*executionv1.CommandStartResponse, bool) { + state, ok := commandStateToProto(v.State) if !ok { - return + return nil, false } - var v executionenv.CommandStatusResponse - var err error - if op == executionenv.OpCommandStatus { - v, err = h.backend.CommandStatus(r.Context(), c.id, owner, q) - } else { - v, err = h.backend.CancelCommand(r.Context(), c.id, owner, q) + result, ok := commandStatusResponse(v.Result) + if !ok { + return nil, false } - if err != nil { - writeBackendError(w, err) - return + return &executionv1.CommandStartResponse{CommandId: valid(v.CommandID), State: state, Result: result}, true +} +func commandStatusResponse(v executionenv.CommandStatusResponse) (*executionv1.CommandStatusResponse, bool) { + state, ok := commandStateToProto(v.State) + if !ok { + return nil, false } - writeJSON(w, 200, v) + return &executionv1.CommandStatusResponse{CommandId: valid(v.CommandID), State: state, ExitCode: boundedInt32(v.ExitCode), Stdout: v.Stdout, Stderr: v.Stderr, NextOffset: v.NextOffset, Truncated: v.Truncated, TerminalReceipt: valid(v.TerminalReceipt)}, true } -func (h *Handler) authorize(w http.ResponseWriter, r *http.Request, rc executionenv.RequestContext, op executionenv.Operation) (authenticatedClient, string, bool) { - c := r.Context().Value(clientContextKey{}).(authenticatedClient) - if rc.Owner.Issuer == "" || rc.Owner.Subject == "" || len(rc.Owner.Issuer) > executionenv.MaxIdentityBytes || len(rc.Owner.Subject) > executionenv.MaxIdentityBytes || rc.BindingID == "" || len(rc.BindingID) > executionenv.MaxBindingBytes || rc.Environment.ID == "" || rc.Environment.Revision == "" || rc.Epoch == 0 || rc.Grant == "" || len(rc.Grant) > executionenv.MaxGrantBytes { - writeError(w, 403, executionenv.CodePermissionDenied, "request authorization denied", false) - return c, "", false +func boundedInt32(v int) int32 { + if v > math.MaxInt32 { + return math.MaxInt32 } - owner := ownerHash(rc.Owner) - if _, err := h.cfg.Verifier.Verify(rc.Grant, executionenv.GrantExpectation{Client: c.id, OwnerHash: owner, BindingID: rc.BindingID, Environment: rc.Environment, Epoch: rc.Epoch, Operation: op}); err != nil { - writeError(w, 403, executionenv.CodePermissionDenied, "request authorization denied", false) - return c, "", false + if v < math.MinInt32 { + return math.MinInt32 } - return c, owner, true + return int32(v) //nolint:gosec // bounds checked above } + +func commandStateToProto(v executionenv.CommandState) (executionv1.CommandState, bool) { + switch v { + case executionenv.CommandRunning: + return executionv1.CommandState_COMMAND_STATE_RUNNING, true + case executionenv.CommandSucceeded: + return executionv1.CommandState_COMMAND_STATE_SUCCEEDED, true + case executionenv.CommandFailed: + return executionv1.CommandState_COMMAND_STATE_FAILED, true + case executionenv.CommandCancelled: + return executionv1.CommandState_COMMAND_STATE_CANCELLED, true + case executionenv.CommandFenceUnknown: + return executionv1.CommandState_COMMAND_STATE_FENCE_UNKNOWN, true + default: + return executionv1.CommandState_COMMAND_STATE_UNSPECIFIED, false + } +} + func (h *Handler) sign(a Allocation, client string) (string, time.Time, error) { life := h.cfg.Signer.Lifetime if life <= 0 { life = time.Minute } - now := time.Now().UTC() - expiresAt := now.Add(life) + now, expiry := time.Now().UTC(), time.Now().UTC().Add(life) nonce := make([]byte, 16) if _, err := rand.Read(nonce); err != nil { return "", time.Time{}, err } ops := []executionenv.Operation{executionenv.OpAttach, executionenv.OpReferenceRelease, executionenv.OpRetire, executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileCreate, executionenv.OpFileReplace, executionenv.OpFileList, executionenv.OpFileRemove, executionenv.OpFileRename, executionenv.OpFileCopy, executionenv.OpFileGlob, executionenv.OpFileGrep, executionenv.OpCommandStart, executionenv.OpCommandStatus, executionenv.OpCommandCancel} - grant, err := executionenv.SignGrant(h.cfg.Signer.PrivateKey, executionenv.GrantClaims{KeyID: h.cfg.Signer.KeyID, Issuer: h.cfg.Signer.Issuer, Audience: h.cfg.Signer.Audience, Client: client, OwnerHash: a.OwnerHash, BindingID: a.BindingID, Environment: a.Environment, Epoch: a.Epoch, Operations: ops, NotBefore: now, ExpiresAt: expiresAt, Nonce: hex.EncodeToString(nonce)}) - return grant, expiresAt, err + grant, err := executionenv.SignGrant(h.cfg.Signer.PrivateKey, executionenv.GrantClaims{KeyID: h.cfg.Signer.KeyID, Issuer: h.cfg.Signer.Issuer, Audience: h.cfg.Signer.Audience, Client: client, OwnerHash: a.OwnerHash, BindingID: a.BindingID, Environment: a.Environment, Epoch: a.Epoch, Operations: ops, NotBefore: now, ExpiresAt: expiry, Nonce: hex.EncodeToString(nonce)}) + return grant, expiry, err } func ownerHash(o executionenv.Owner) string { s := sha256.Sum256([]byte(o.Issuer + "\x00" + o.Subject)) @@ -363,44 +450,56 @@ func fingerprint(fields ...string) string { } return hex.EncodeToString(h.Sum(nil)) } -func decode(w http.ResponseWriter, r *http.Request, dst any) bool { - body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, executionenv.MaxJSONBody+1)) - if err != nil { - writeError(w, 413, executionenv.CodeResourceExhausted, "request body exceeds limit", false) - return false - } - if err := executionenv.DecodeStrict(body, dst); err != nil { - writeError(w, 400, executionenv.CodeInvalidArgument, "invalid request body", false) - return false + +func backendError(err error) error { + var e *executionenv.Error + if errors.As(err, &e) && e.Code.Valid() { + return wireError(e.Code, e.Retryable) } - return true + return wireError(executionenv.CodeInternal, false) } -func writeJSON(w http.ResponseWriter, status int, v any) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(v) +func wireError(code executionenv.ErrorCode, retry bool) error { + if !code.Valid() || (retry && code != executionenv.CodeNotReady && code != executionenv.CodeUnauthenticated) { + code, retry = executionenv.CodeInternal, false + } + grpcCode := codes.Internal + switch code { + case executionenv.CodeInvalidArgument: + grpcCode = codes.InvalidArgument + case executionenv.CodeUnauthenticated: + grpcCode = codes.Unauthenticated + case executionenv.CodePermissionDenied: + grpcCode = codes.PermissionDenied + case executionenv.CodeNotFound: + grpcCode = codes.NotFound + case executionenv.CodeAlreadyExists: + grpcCode = codes.AlreadyExists + case executionenv.CodeConflict, executionenv.CodeVersionMismatch: + grpcCode = codes.Aborted + case executionenv.CodeNotReady, executionenv.CodeFenceUnknown: + grpcCode = codes.Unavailable + case executionenv.CodeResourceExhausted: + grpcCode = codes.ResourceExhausted + } + st := status.New(grpcCode, "execution provider request failed") + with, err := st.WithDetails(&executionv1.ErrorDetail{Code: string(code), Retryable: retry}) + if err == nil { + st = with + } + return st.Err() } -func writeError(w http.ResponseWriter, status int, code executionenv.ErrorCode, msg string, retry bool) { - writeJSON(w, status, executionenv.ErrorResponse{Error: &executionenv.Error{Code: code, Message: msg, Retryable: retry}}) +func valid(s string) string { + if utf8.ValidString(s) { + return s + } + return strings.ToValidUTF8(s, "�") } -func writeBackendError(w http.ResponseWriter, err error) { - var e *executionenv.Error - if errors.As(err, &e) { - status := 400 - switch e.Code { - case executionenv.CodeNotFound: - status = 404 - case executionenv.CodeAlreadyExists, executionenv.CodeConflict, executionenv.CodeVersionMismatch: - status = 409 - case executionenv.CodeNotReady, executionenv.CodeFenceUnknown: - status = 503 - case executionenv.CodePermissionDenied: - status = 403 - } - writeError(w, status, e.Code, e.Message, e.Retryable) - return +func validAll(in []string) []string { + out := make([]string, len(in)) + for i := range in { + out[i] = valid(in[i]) } - writeError(w, 500, executionenv.CodeInternal, "provider operation failed", false) + return out } // TLSConfig returns the provider's TLS 1.3 mutual-authentication policy. diff --git a/internal/adapter/executioncontroller/handler_test.go b/internal/adapter/executioncontroller/handler_test.go index cf8a0abbb2..08d099d9e6 100644 --- a/internal/adapter/executioncontroller/handler_test.go +++ b/internal/adapter/executioncontroller/handler_test.go @@ -7,22 +7,23 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" - "encoding/json" - "encoding/pem" - "math/big" - "net/http" - "net/http/httptest" "net/url" - "strings" "testing" "time" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" "github.com/stacklok/mecatl/internal/executionenv" ) type fakeBackend struct { ensureCalls, created int allocations map[string]Allocation + commandState executionenv.CommandState } func newFakeBackend() *fakeBackend { return &fakeBackend{allocations: map[string]Allocation{}} } @@ -49,8 +50,12 @@ func (*fakeBackend) Retire(context.Context, executionenv.EnvironmentRef, string) func (*fakeBackend) File(context.Context, string, string, executionenv.FileRequest) (executionenv.FileResponse, error) { return executionenv.FileResponse{}, nil } -func (*fakeBackend) StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { - return executionenv.CommandStartResponse{}, nil +func (f *fakeBackend) StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + state := f.commandState + if state == "" { + state = executionenv.CommandSucceeded + } + return executionenv.CommandStartResponse{CommandID: "command", State: state, Result: executionenv.CommandStatusResponse{CommandID: "command", State: state}}, nil } func (*fakeBackend) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { return executionenv.CommandStatusResponse{}, nil @@ -59,28 +64,34 @@ func (*fakeBackend) CancelCommand(context.Context, string, string, executionenv. return executionenv.CommandStatusResponse{}, nil } +func authenticatedContext(id string) context.Context { + u, _ := url.Parse(id) + cert := &x509.Certificate{URIs: []*url.URL{u}} + return peer.NewContext(context.Background(), &peer.Peer{AuthInfo: credentials.TLSInfo{State: structTLSState(cert)}}) +} +func structTLSState(cert *x509.Certificate) (s tls.ConnectionState) { + s.PeerCertificates = []*x509.Certificate{cert} + s.VerifiedChains = [][]*x509.Certificate{{cert}} + return s +} + func TestHandlerBlocksAllRequestsUntilStartupReady(t *testing.T) { backend := newFakeBackend() h := NewHandler(HandlerConfig{Ready: func() bool { return false }}, backend) - r := httptest.NewRequest("POST", executionenv.BasePath+"/environments/ensure", strings.NewReader(`{"binding_id":"session-1","profile":"go","owner":{"issuer":"issuer","subject":"subject"}}`)) - w := httptest.NewRecorder() - h.ServeHTTP(w, r) - if w.Code != http.StatusServiceUnavailable { - t.Fatalf("code=%d body=%s", w.Code, w.Body.String()) + _, err := h.EnsureEnvironment(authenticatedContext("spiffe://cluster/ns/mecak8s"), &executionv1.EnsureEnvironmentRequest{}) + if status.Code(err) != codes.Unavailable { + t.Fatalf("code=%v", status.Code(err)) } if backend.ensureCalls != 0 { - t.Fatalf("backend calls before startup readiness = %d", backend.ensureCalls) + t.Fatalf("backend calls before readiness=%d", backend.ensureCalls) } } func TestHandlerRequiresAllowlistedCanonicalURISAN(t *testing.T) { - h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {MayAttestOwner: true}}}, nil) - r := httptest.NewRequest("POST", executionenv.BasePath+"/profiles/validate", strings.NewReader(`{"profile":"go"}`)) - r.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{{URIs: []*url.URL{{Scheme: "spiffe", Host: "cluster", Path: "/ns/other"}}}}} - w := httptest.NewRecorder() - h.ServeHTTP(w, r) - if w.Code != 401 { - t.Fatalf("code=%d body=%s", w.Code, w.Body.String()) + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {}}}, newFakeBackend()) + _, err := h.ValidateProfile(authenticatedContext("spiffe://cluster/ns/other"), &executionv1.ValidateProfileRequest{Profile: "go"}) + if status.Code(err) != codes.Unauthenticated { + t.Fatalf("code=%v", status.Code(err)) } } @@ -88,40 +99,126 @@ func TestHandlerValidateIsReadOnlyAndEnsureIdempotent(t *testing.T) { pub, priv, _ := ed25519.GenerateKey(rand.Reader) backend := newFakeBackend() h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {MayAttestOwner: true}}, Signer: GrantSigner{KeyID: "k1", PrivateKey: priv, Issuer: "provider", Audience: "execution", Lifetime: time.Minute}, Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "provider", Audience: "execution"}}, backend) - call := func(path, body string) *httptest.ResponseRecorder { - r := httptest.NewRequest("POST", executionenv.BasePath+path, strings.NewReader(body)) - r.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{{URIs: []*url.URL{{Scheme: "spiffe", Host: "cluster", Path: "/ns/mecak8s"}}}}} - w := httptest.NewRecorder() - h.ServeHTTP(w, r) - return w - } - if w := call("/profiles/validate", `{"profile":"go"}`); w.Code != 200 { - t.Fatalf("validate %d %s", w.Code, w.Body.String()) + ctx := authenticatedContext("spiffe://cluster/ns/mecak8s") + if _, err := h.ValidateProfile(ctx, &executionv1.ValidateProfileRequest{Profile: "go"}); err != nil { + t.Fatal(err) } if backend.ensureCalls != 0 { t.Fatal("validation allocated") } - body := `{"binding_id":"session-1","profile":"go","owner":{"issuer":"https://issuer","subject":"alice"}}` - w1 := call("/environments/ensure", body) - if w1.Code != 200 { - t.Fatalf("ensure %d %s", w1.Code, w1.Body.String()) + q := &executionv1.EnsureEnvironmentRequest{BindingId: "session-1", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}} + first, err := h.EnsureEnvironment(ctx, q) + if err != nil { + t.Fatal(err) } - w2 := call("/environments/ensure", body) - if w2.Code != 200 { - t.Fatalf("ensure2 %d %s", w2.Code, w2.Body.String()) + second, err := h.EnsureEnvironment(ctx, q) + if err != nil { + t.Fatal(err) } - if backend.created != 1 { + if backend.created != 1 || first.Environment.Id != second.Environment.Id || first.Epoch != second.Epoch { t.Fatalf("created=%d", backend.created) } - var first, second executionenv.EnsureEnvironmentResponse - if err := json.Unmarshal(w1.Body.Bytes(), &first); err != nil { - t.Fatal(err) +} + +func TestHandlerWithNilBackendRejectsEveryRPCWithoutPanicking(t *testing.T) { + h := NewHandler(HandlerConfig{}, nil) + ctx := context.Background() + calls := []func(bool) error{ + func(valid bool) error { + var q *executionv1.ValidateProfileRequest + if valid { + q = &executionv1.ValidateProfileRequest{Profile: "go"} + } + _, err := h.ValidateProfile(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.EnsureEnvironmentRequest + if valid { + q = &executionv1.EnsureEnvironmentRequest{BindingId: "binding", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}} + } + _, err := h.EnsureEnvironment(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.AttachEnvironmentRequest + if valid { + q = &executionv1.AttachEnvironmentRequest{Purpose: executionenv.PurposeSession} + } + _, err := h.AttachEnvironment(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.ReleaseReferenceRequest + if valid { + q = &executionv1.ReleaseReferenceRequest{Context: &executionv1.RequestContext{}} + } + _, err := h.ReleaseReference(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.RetireEnvironmentRequest + if valid { + q = &executionv1.RetireEnvironmentRequest{Context: &executionv1.RequestContext{}} + } + _, err := h.RetireEnvironment(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.FileRequest + if valid { + q = &executionv1.FileRequest{Operation: executionv1.FileOperation_FILE_OPERATION_READ, Path: "x"} + } + _, err := h.Files(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.CommandStartRequest + if valid { + q = &executionv1.CommandStartRequest{Command: "true"} + } + _, err := h.StartCommand(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.CommandQueryRequest + if valid { + q = &executionv1.CommandQueryRequest{CommandId: "command"} + } + _, err := h.CommandStatus(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.CommandQueryRequest + if valid { + q = &executionv1.CommandQueryRequest{CommandId: "command"} + } + _, err := h.CancelCommand(ctx, q) + return err + }, + } + for i, call := range calls { + for _, valid := range []bool{false, true} { + if err := call(valid); status.Code(err) != codes.Unavailable { + t.Fatalf("RPC %d valid=%t code=%v", i, valid, status.Code(err)) + } + } } - if err := json.Unmarshal(w2.Body.Bytes(), &second); err != nil { +} + +func TestHandlerRejectsUnknownBackendCommandState(t *testing.T) { + pub, priv, _ := ed25519.GenerateKey(rand.Reader) + backend := newFakeBackend() + backend.commandState = executionenv.CommandState("future") + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {MayAttestOwner: true}}, Signer: GrantSigner{KeyID: "k1", PrivateKey: priv, Issuer: "provider", Audience: "execution", Lifetime: time.Minute}, Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "provider", Audience: "execution"}}, backend) + ctx := authenticatedContext("spiffe://cluster/ns/mecak8s") + ensured, err := h.EnsureEnvironment(ctx, &executionv1.EnsureEnvironmentRequest{BindingId: "session-1", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}}) + if err != nil { t.Fatal(err) } - if first.Environment != second.Environment || first.Epoch != second.Epoch { - t.Fatalf("allocation identity changed") + q := &executionv1.CommandStartRequest{Context: &executionv1.RequestContext{Environment: ensured.Environment, Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, BindingId: "session-1", Epoch: ensured.Epoch, Grant: ensured.Grant}, Command: "true"} + if _, err := h.StartCommand(ctx, q); status.Code(err) != codes.Internal || status.Convert(err).Message() != "execution provider request failed" { + t.Fatalf("unexpected invalid-state result: code=%v message=%q", status.Code(err), status.Convert(err).Message()) } } @@ -131,65 +228,22 @@ func TestCanonicalClientIdentityRejectsCNAndAmbiguousURIs(t *testing.T) { } u1, _ := url.Parse("spiffe://b/x") u2, _ := url.Parse("spiffe://a/x") - got, err := canonicalClientIdentity(&x509.Certificate{URIs: []*url.URL{u1, u2}}) - if err != nil || got != "spiffe://a/x" { - t.Fatalf("got %q err %v", got, err) - } -} - -func TestMTLSFixtureActuallyVerifiesClientCertificate(t *testing.T) { - clientCert, pool, err := testCertificate() - if err != nil { - t.Fatal(err) - } - h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {}}}, newFakeBackend()) - srv := httptest.NewUnstartedServer(h) - srv.TLS = &tls.Config{MinVersion: tls.VersionTLS13, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: pool} - srv.StartTLS() - defer srv.Close() - if _, err := srv.Client().Post(srv.URL+executionenv.BasePath+"/profiles/validate", "application/json", strings.NewReader(`{"profile":"go"}`)); err == nil { - t.Fatal("server accepted connection without client certificate") - } - transport := srv.Client().Transport.(*http.Transport).Clone() - transport.TLSClientConfig = transport.TLSClientConfig.Clone() - transport.TLSClientConfig.Certificates = []tls.Certificate{clientCert} - client := &http.Client{Transport: transport} - resp, err := client.Post(srv.URL+executionenv.BasePath+"/profiles/validate", "application/json", strings.NewReader(`{"profile":"go"}`)) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status=%d", resp.StatusCode) + if _, err := canonicalClientIdentity(&x509.Certificate{URIs: []*url.URL{u1, u2}}); err == nil { + t.Fatal("ambiguous URI SANs accepted") } } -func testCertificate() (tls.Certificate, *x509.CertPool, error) { - caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader) - ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test-ca"}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature} - caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, caPub, caPriv) - if err != nil { - return tls.Certificate{}, nil, err +func TestWireErrorDoesNotExposeBackendMessage(t *testing.T) { + err := backendError(&executionenv.Error{Code: executionenv.CodeInternal, Message: "kubectl exec --token=secret"}) + if got := status.Convert(err).Message(); got != "execution provider request failed" { + t.Fatalf("message=%q", got) } - clientPub, clientPriv, _ := ed25519.GenerateKey(rand.Reader) - identity, _ := url.Parse("spiffe://cluster/ns/mecak8s") - leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "ignored-cn"}, URIs: []*url.URL{identity}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} - leafDER, err := x509.CreateCertificate(rand.Reader, leaf, ca, clientPub, caPriv) - if err != nil { - return tls.Certificate{}, nil, err - } - certPEM := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER})...) - keyDER, _ := x509.MarshalPKCS8PrivateKey(clientPriv) - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) - cert, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - return tls.Certificate{}, nil, err + unknown := backendError(&executionenv.Error{Code: executionenv.ErrorCode("future"), Message: "sensitive path"}) + st := status.Convert(unknown) + if st.Code() != codes.Internal || st.Message() != "execution provider request failed" { + t.Fatalf("unknown error escaped: code=%v message=%q", st.Code(), st.Message()) } - pool := x509.NewCertPool() - parsedCA, err := x509.ParseCertificate(caDER) - if err != nil { - return tls.Certificate{}, nil, err + if details := st.Details(); len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeInternal) { + t.Fatalf("unknown error detail=%v", details) } - pool.AddCert(parsedCA) - return cert, pool, err } diff --git a/internal/executionenv/errors.go b/internal/executionenv/errors.go index 8a75d95f82..e5ab5847c7 100644 --- a/internal/executionenv/errors.go +++ b/internal/executionenv/errors.go @@ -26,6 +26,18 @@ const ( CodeInternal ErrorCode = "internal" ) +// Valid reports whether c is part of the closed private-protocol vocabulary. +func (c ErrorCode) Valid() bool { + switch c { + case CodeInvalidArgument, CodeUnauthenticated, CodePermissionDenied, CodeNotFound, + CodeAlreadyExists, CodeConflict, CodeVersionMismatch, CodeNotReady, CodeFenceUnknown, + CodeResourceExhausted, CodeInternal: + return true + default: + return false + } +} + // Error is a bounded structured private-protocol failure. type Error struct { Code ErrorCode `json:"code"` @@ -35,12 +47,8 @@ type Error struct { func (e *Error) Error() string { return string(e.Code) + ": " + e.Message } -// ErrorResponse is the bounded wire envelope returned for every failed request. -type ErrorResponse struct { - Error *Error `json:"error"` -} - -// DecodeStrict decodes exactly one bounded JSON value without unknown or duplicate fields. +// DecodeStrict decodes exactly one bounded JSON value for the credential-free +// provider-to-workload stdin protocol, without unknown or duplicate fields. func DecodeStrict(data []byte, dst any) error { if len(data) > MaxJSONBody { return &Error{Code: CodeResourceExhausted, Message: "request body exceeds limit"} diff --git a/internal/executionenv/grant.go b/internal/executionenv/grant.go index 4480225f88..6dab8a2ba0 100644 --- a/internal/executionenv/grant.go +++ b/internal/executionenv/grant.go @@ -12,6 +12,13 @@ import ( const grantAlgorithm = "Ed25519" +var ( + // ErrGrantExpired identifies the only grant failure eligible for a read-only refresh. + ErrGrantExpired = errors.New("grant expired") + // ErrGrantNotYetValid identifies a grant whose validity window has not started. + ErrGrantNotYetValid = errors.New("grant not yet valid") +) + type grantHeader struct { Algorithm string `json:"alg"` KeyID string `json:"kid"` @@ -117,8 +124,11 @@ func (v GrantVerifier) Verify(token string, e GrantExpectation) (GrantClaims, er if c.Issuer != v.Issuer || c.Audience != v.Audience { return c, errors.New("grant issuer or audience mismatch") } - if now.Before(c.NotBefore) || !now.Before(c.ExpiresAt) { - return c, errors.New("grant is not currently valid") + if now.Before(c.NotBefore) { + return c, ErrGrantNotYetValid + } + if !now.Before(c.ExpiresAt) { + return c, ErrGrantExpired } maxLifetime := v.MaxLifetime if maxLifetime <= 0 { diff --git a/internal/executionenv/protocol_test.go b/internal/executionenv/protocol_test.go index e94b6af648..223b2d1241 100644 --- a/internal/executionenv/protocol_test.go +++ b/internal/executionenv/protocol_test.go @@ -57,9 +57,9 @@ func TestGrantRejectsExpiryRevocationAndUnknownFields(t *testing.T) { } } -func TestDecodeStrictRejectsUnknownTrailingAndOversize(t *testing.T) { - var req ValidateProfileRequest - for _, body := range []string{`{"profile":"p","extra":true}`, `{"profile":"p"}{}`, `{"profile":"` + string(make([]byte, MaxJSONBody)) + `"}`} { +func TestExecutorStdinDecodeStrictRejectsUnknownTrailingAndOversize(t *testing.T) { + var req ExecutorRequest + for _, body := range []string{`{"operation":"file.read","extra":true}`, `{"operation":"file.read"}{}`, `{"path":"` + string(make([]byte, MaxJSONBody)) + `"}`} { if err := DecodeStrict([]byte(body), &req); err == nil { t.Fatalf("accepted invalid body") } diff --git a/internal/executionenv/types.go b/internal/executionenv/types.go index 8708c6d770..df6f5b2387 100644 --- a/internal/executionenv/types.go +++ b/internal/executionenv/types.go @@ -6,8 +6,9 @@ import "time" // Private protocol bounds. const ( - BasePath = "/internal/execution/v1" - MaxJSONBody = 8 << 20 + ProtocolVersion = "execution-grpc/1" + MaxMessageBytes = 8 << 20 + MaxJSONBody = 8 << 20 // credential-free provider-to-workload stdin framing only MaxFileBytes = 5 << 20 MaxCommandBytes = 1 << 20 MaxPathBytes = 4096 diff --git a/user-docs/building/deployment/mecak8s.md b/user-docs/building/deployment/mecak8s.md index b3727fbfcd..85698d5380 100644 --- a/user-docs/building/deployment/mecak8s.md +++ b/user-docs/building/deployment/mecak8s.md @@ -219,14 +219,16 @@ helm upgrade --install mecatl-execution ./deploy/helm/mecatl-execution \ kubectl rollout status deployment/mecatl-execution --namespace ``` -Add the following block to the existing `mecak8s` values. The endpoint must use -the provider certificate's DNS identity. Keep OIDC caller authentication enabled; +Add the following block to the existing `mecak8s` values. The endpoint is a +`host:port` gRPC target (no URL scheme) and must use the provider certificate's +DNS identity. The private `mecatl.execution.v1.ExecutionProviderService` uses +protocol `execution-grpc/1` with mandatory mTLS. Keep OIDC caller authentication enabled; the remote binding is scoped to the verified issuer and subject. ```yaml execution: enabled: true - endpoint: https://mecatl-execution..svc:8443 + endpoint: mecatl-execution..svc:8443 profile: coding tlsSecret: caKey: ca.crt diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index f63297997d..08bb5cff69 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -78,6 +78,30 @@ reattached only when the deployment supplies an `EnvironmentResolver`; a missing resolver, mismatched identity, or nil workspace returns an error instead of using a local workspace. +## Live qualification (experimental) + +The repository keeps real-provider qualification separate from the default mock +suite. It is explicit opt-in, uses an already-owned retained Kind cluster, and is +not part of `task test`: + +```sh +MECATL_EXECUTION_CREDENTIAL_FILE=/absolute/path/to/provider-key \ +MECATL_EXECUTION_QUAL_STATE=/absolute/path/to/owned-state \ +task e2e:k8s:execution:live +``` + +The credential file must be a private regular file (no group or other access). +A trusted helper loads it only at runtime and creates a run-scoped Kubernetes +Secret through the API; it never renders the value into a manifest or reads the +Secret back. Only the `mecak8s` harness container receives the OpenRouter key. +The execution provider, controller, OIDC fixture, and executor workloads do not. +The task first runs the deterministic mock qualification, then runs one bounded +real-model coding smoke against `https://openrouter.ai/api/v1`. It verifies +positive token usage, required file and shell tool calls, file contents, and a +successful `go test` independently through the typed gRPC execution service. +Finally it restores the mock deployment and removes only its run-scoped Secret; +the owned cluster and execution workspaces remain for inspection. + ## Limitations - No-FS sessions cannot use local file tools, shell commands, workspace forks, From 36c4360c1397cb0a310c9f9411264a24d3fac584 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 17 Sep 2026 04:21:35 +0200 Subject: [PATCH 05/25] feat: complete execution ownership lifecycle and security controls Add run claims, transactional reference reconciliation, UID-fenced executor replacement and retirement, schema migration, durable grant revocation, reloadable TLS/keyring authority, deployment limits and release definitions. Qualification and independent review remain in progress. Co-Authored-By: Mecatl --- .github/workflows/k8s-e2e.yml | 16 + .github/workflows/release.yml | 114 + cmd/mecatl-execution-provider/main.go | 150 +- .../go/mecatl/execution/v1/execution.pb.go | 2390 ++++++++++++++--- .../mecatl/execution/v1/execution_grpc.pb.go | 588 +++- .../proto/mecatl/execution/v1/execution.proto | 132 +- deploy/helm/mecatl-execution/chart_test.go | 28 + .../crds/executionenvironment.yaml | 77 +- .../templates/network-policy.yaml | 99 + .../helm/mecatl-execution/templates/pdb.yaml | 10 + .../mecatl-execution/templates/provider.yaml | 66 +- .../helm/mecatl-execution/templates/rbac.yaml | 8 +- .../helm/mecatl-execution/values.schema.json | 27 +- deploy/helm/mecatl-execution/values.yaml | 43 +- e2e/k8s_execution/qualification_test.go | 2 +- internal/adapter/executionclient/client.go | 394 ++- .../adapter/executionclient/client_test.go | 61 +- .../reference_ambiguity_test.go | 60 + .../service_integration_test.go | 140 + .../executioncontroller/admin_lifecycle.go | 273 ++ .../adapter/executioncontroller/controller.go | 69 +- .../controller_lifecycle.go | 237 ++ .../executioncontroller/controller_test.go | 28 +- .../adapter/executioncontroller/handler.go | 433 ++- .../executioncontroller/handler_test.go | 48 +- .../executioncontroller/operation_lease.go | 50 + .../production_lifecycle_test.go | 207 ++ .../adapter/executioncontroller/profiles.go | 66 +- .../executioncontroller/profiles_test.go | 17 +- .../adapter/executioncontroller/security.go | 567 ++++ .../executioncontroller/security_limit.go | 83 + .../security_limit_test.go | 29 + .../executioncontroller/security_test.go | 188 ++ internal/adapter/executioncontroller/store.go | 216 +- .../executioncontroller/store_lifecycle.go | 259 ++ .../store_lifecycle_test.go | 142 + .../executioncontroller/store_revoke.go | 32 + .../executioncontroller/store_revoke_test.go | 77 + .../adapter/executioncontroller/store_run.go | 176 ++ .../adapter/executioncontroller/store_test.go | 14 +- internal/adapter/server/classification.go | 1 + .../server/native_execution_lifecycle_test.go | 93 + internal/adapter/server/placement.go | 80 +- .../adapter/server/placement_successor.go | 31 +- .../server/reference_intent_reconcile_test.go | 86 + .../server/sdk_typescript_release_test.go | 1 + internal/adapter/server/service.go | 252 +- internal/app/build.go | 4 + internal/app/placement.go | 71 + internal/app/reference_intent_reconcile.go | 36 + internal/executionenv/grant.go | 48 +- internal/executionenv/protocol_test.go | 12 +- internal/executionenv/types.go | 147 +- user-docs/building/deployment/mecak8s.md | 59 +- user-docs/features/execution-environments.md | 80 + 55 files changed, 7814 insertions(+), 803 deletions(-) create mode 100644 deploy/helm/mecatl-execution/templates/network-policy.yaml create mode 100644 deploy/helm/mecatl-execution/templates/pdb.yaml create mode 100644 internal/adapter/executionclient/reference_ambiguity_test.go create mode 100644 internal/adapter/executionclient/service_integration_test.go create mode 100644 internal/adapter/executioncontroller/admin_lifecycle.go create mode 100644 internal/adapter/executioncontroller/controller_lifecycle.go create mode 100644 internal/adapter/executioncontroller/operation_lease.go create mode 100644 internal/adapter/executioncontroller/production_lifecycle_test.go create mode 100644 internal/adapter/executioncontroller/security.go create mode 100644 internal/adapter/executioncontroller/security_limit.go create mode 100644 internal/adapter/executioncontroller/security_limit_test.go create mode 100644 internal/adapter/executioncontroller/security_test.go create mode 100644 internal/adapter/executioncontroller/store_lifecycle.go create mode 100644 internal/adapter/executioncontroller/store_lifecycle_test.go create mode 100644 internal/adapter/executioncontroller/store_revoke.go create mode 100644 internal/adapter/executioncontroller/store_revoke_test.go create mode 100644 internal/adapter/executioncontroller/store_run.go create mode 100644 internal/adapter/server/native_execution_lifecycle_test.go create mode 100644 internal/adapter/server/reference_intent_reconcile_test.go create mode 100644 internal/app/reference_intent_reconcile.go diff --git a/.github/workflows/k8s-e2e.yml b/.github/workflows/k8s-e2e.yml index c610233e43..6d3a3c92d7 100644 --- a/.github/workflows/k8s-e2e.yml +++ b/.github/workflows/k8s-e2e.yml @@ -37,6 +37,14 @@ on: - internal/adapter/server/grpc.go - internal/adapter/server/http.go - deploy/helm/mecak8s/** + - deploy/helm/mecatl-execution/** + - cmd/mecatl-execution-provider/** + - cmd/mecatl-executor/** + - internal/adapter/executioncontroller/** + - internal/adapter/executionclient/** + - internal/executionenv/** + - contracts/proto/mecatl/execution/** + - e2e/k8s_execution/** - e2e/k8s/** - go.mod - go.sum @@ -54,6 +62,14 @@ on: - internal/adapter/server/grpc.go - internal/adapter/server/http.go - deploy/helm/mecak8s/** + - deploy/helm/mecatl-execution/** + - cmd/mecatl-execution-provider/** + - cmd/mecatl-executor/** + - internal/adapter/executioncontroller/** + - internal/adapter/executionclient/** + - internal/executionenv/** + - contracts/proto/mecatl/execution/** + - e2e/k8s_execution/** - e2e/k8s/** - go.mod - go.sum diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c606297e2a..40681578c2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -562,6 +562,120 @@ jobs: # silently failing at it. create-storage-record: false + publish-execution-images: + needs: guard + name: Build, publish, sign, attest execution ${{ matrix.name }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + packages: write + id-token: write + attestations: write + strategy: + fail-fast: false + matrix: + include: + - {name: provider, dockerfile: build/execution-provider/Dockerfile, image: mecatl-execution-provider} + - {name: workload, dockerfile: build/execution-workload/Dockerfile, image: mecatl-execution-workload} + env: + VERSION: ${{ inputs.tag || github.ref_name }} + IMAGE: ghcr.io/${{ github.repository }}/${{ matrix.image }} + GO_IMAGE: ${{ vars.EXECUTION_GO_IMAGE }} + PROVIDER_RUNTIME_IMAGE: ${{ vars.EXECUTION_PROVIDER_RUNTIME_IMAGE }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: {persist-credentials: false, fetch-depth: 1, ref: "${{ env.VERSION }}"} + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v4.0.0 + with: {registry: ghcr.io, username: "${{ github.actor }}", password: "${{ secrets.GITHUB_TOKEN }}"} + - name: Validate immutable base inputs + run: | + case "$GO_IMAGE" in *@sha256:*) ;; *) echo "EXECUTION_GO_IMAGE must be digest-pinned" >&2; exit 1;; esac + if [ "${{ matrix.name }}" = provider ]; then + case "$PROVIDER_RUNTIME_IMAGE" in *@sha256:*) ;; *) echo "EXECUTION_PROVIDER_RUNTIME_IMAGE must be digest-pinned" >&2; exit 1;; esac + fi + - name: Build and push image + id: build + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64,linux/arm64 + push: true + provenance: false + sbom: false + build-args: | + GO_IMAGE=${{ env.GO_IMAGE }} + RUNTIME_IMAGE=${{ matrix.name == 'provider' && env.PROVIDER_RUNTIME_IMAGE || env.GO_IMAGE }} + tags: | + ${{ env.IMAGE }}:${{ env.VERSION }} + ${{ env.IMAGE }}:latest + - name: Sign image + run: cosign sign --yes "${IMAGE}@${{ steps.build.outputs.digest }}" + - name: Generate SBOM + uses: anchore/sbom-action@3ad7283483fc7af8ff2b4ea19663c2d5ca935e26 # v0.24.2 + with: {image: "${{ env.IMAGE }}@${{ steps.build.outputs.digest }}", format: spdx-json, output-file: sbom.spdx.json, upload-artifact: false} + - name: Attest SBOM + run: cosign attest --yes --predicate sbom.spdx.json --type spdxjson "${IMAGE}@${{ steps.build.outputs.digest }}" + - name: Attest build provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + create-storage-record: false + + publish-execution-chart: + needs: [guard, publish-execution-images] + name: Package, publish, sign, attest execution Helm chart + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + packages: write + id-token: write + attestations: write + env: + VERSION: ${{ inputs.tag || github.ref_name }} + OCI_REGISTRY: ghcr.io/${{ github.repository }}/charts + CHART: mecatl-execution + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: {persist-credentials: false, fetch-depth: 1, ref: "${{ env.VERSION }}"} + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: {version: v3.18.4} + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v4.0.0 + with: {registry: ghcr.io, username: "${{ github.actor }}", password: "${{ secrets.GITHUB_TOKEN }}"} + - name: Package chart + id: package + run: | + set -euo pipefail + chart_version="${VERSION#v}" + mkdir -p dist + helm package deploy/helm/mecatl-execution --version "$chart_version" --app-version "$VERSION" --destination dist + echo "chart_version=$chart_version" >> "$GITHUB_OUTPUT" + - name: Push chart + id: push + run: | + set -euo pipefail + helm push "dist/${CHART}-${{ steps.package.outputs.chart_version }}.tgz" "oci://${OCI_REGISTRY}" > push-output.txt 2>&1 + cat push-output.txt + digest="$(awk '/^Digest:/ {print $2}' push-output.txt)" + case "$digest" in sha256:*) ;; *) echo "could not parse chart digest" >&2; exit 1;; esac + echo "digest=$digest" >> "$GITHUB_OUTPUT" + - name: Sign chart + run: cosign sign --yes "${OCI_REGISTRY}/${CHART}@${{ steps.push.outputs.digest }}" + - name: Attest chart provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.OCI_REGISTRY }}/${{ env.CHART }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true + create-storage-record: false + # Not a Go binary, so this doesn't go through ko like the jobs above — a # plain docker/build-push-action build of the existing dev/Compose # Dockerfile. The bot's TypeScript source still lives under diff --git a/cmd/mecatl-execution-provider/main.go b/cmd/mecatl-execution-provider/main.go index 872651c3ef..6edcdf1c78 100644 --- a/cmd/mecatl-execution-provider/main.go +++ b/cmd/mecatl-execution-provider/main.go @@ -3,23 +3,20 @@ package main import ( "context" - "crypto/ed25519" - "crypto/tls" - "crypto/x509" - "encoding/pem" "errors" "flag" "fmt" "log/slog" "net" + "net/http" "os" "os/signal" - "strings" "syscall" "time" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -36,48 +33,28 @@ func main() { } } func run() error { //nolint:gocyclo // Startup validation and owned-resource shutdown stay in one composition root. - var addr, namespace, profilesPath, certPath, keyPath, caPath, signingPath, keyID, issuer, audience, clients, attesters, admins string + var addr, healthAddr, namespace, profilesPath, manifestPath, keyDirectory, authorityConfigMap string + var reloadInterval time.Duration + var maxConcurrentStreams, maxConcurrentRPCs, maxConcurrentRPCsPerClient int flag.StringVar(&addr, "listen", ":8443", "gRPC listen address") flag.StringVar(&namespace, "namespace", "", "managed Kubernetes namespace") flag.StringVar(&profilesPath, "profiles", "/etc/mecatl-execution/profiles.yaml", "strict operator profile file") - flag.StringVar(&certPath, "tls-cert", "/etc/mecatl-execution/tls/tls.crt", "server certificate") - flag.StringVar(&keyPath, "tls-key", "/etc/mecatl-execution/tls/tls.key", "server private key") - flag.StringVar(&caPath, "client-ca", "/etc/mecatl-execution/tls/ca.crt", "client CA bundle") - flag.StringVar(&signingPath, "grant-signing-key", "/etc/mecatl-execution/grant/key.pem", "Ed25519 PKCS8 grant signing key") - flag.StringVar(&keyID, "grant-key-id", "current", "grant verification key id") - flag.StringVar(&issuer, "grant-issuer", "mecatl-execution-provider", "exact grant issuer") - flag.StringVar(&audience, "grant-audience", "mecatl-execution", "exact grant audience") - flag.StringVar(&clients, "client-uri-san", "", "comma-separated exact client URI SAN allowlist") - flag.StringVar(&attesters, "owner-attester-uri-san", "", "comma-separated client identities allowed to attest owners") - flag.StringVar(&admins, "administrator-uri-san", "", "comma-separated retirement administrators") + flag.StringVar(&healthAddr, "health-listen", ":8081", "operational HTTP health listen address; empty disables") + flag.StringVar(&manifestPath, "grant-keyring-manifest", "/etc/mecatl-execution/security/manifest.json", "versioned security manifest") + flag.StringVar(&keyDirectory, "grant-key-directory", "/etc/mecatl-execution/security", "projected security key and TLS directory") + flag.StringVar(&authorityConfigMap, "security-authority-configmap", "", "provider-owned security generation high-water ConfigMap") + flag.DurationVar(&reloadInterval, "security-reload-interval", 2*time.Second, "security material reload interval") + flag.IntVar(&maxConcurrentStreams, "max-concurrent-streams", 64, "maximum concurrent HTTP/2 streams per connection") + flag.IntVar(&maxConcurrentRPCs, "max-concurrent-rpcs", 128, "maximum active provider RPCs") + flag.IntVar(&maxConcurrentRPCsPerClient, "max-concurrent-rpcs-per-client", 32, "maximum active provider RPCs per authorized client") flag.Parse() - if namespace == "" || clients == "" { - return errors.New("--namespace and --client-uri-san are required") + if namespace == "" || authorityConfigMap == "" || maxConcurrentStreams < 1 || maxConcurrentStreams > 1024 || maxConcurrentRPCs < 1 || maxConcurrentRPCs > 4096 || maxConcurrentRPCsPerClient < 1 || maxConcurrentRPCsPerClient > maxConcurrentRPCs { + return errors.New("required identity or RPC concurrency bounds are invalid") } profiles, err := executioncontroller.LoadProfiles(profilesPath) if err != nil { return err } - serverCert, err := tls.LoadX509KeyPair(certPath, keyPath) - if err != nil { - return fmt.Errorf("load provider TLS identity: %w", err) - } - caPEM, err := os.ReadFile(caPath) - if err != nil { - return fmt.Errorf("load client CA: %w", err) - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(caPEM) { - return errors.New("client CA has no certificates") - } - priv, err := loadSigningKey(signingPath) - if err != nil { - return err - } - policies, err := clientPolicies(clients, attesters, admins) - if err != nil { - return err - } cfg, err := rest.InClusterConfig() if err != nil { return fmt.Errorf("build in-cluster Kubernetes config: %w", err) @@ -90,14 +67,17 @@ func run() error { //nolint:gocyclo // Startup validation and owned-resource shu if err != nil { return err } + security := executioncontroller.NewSecurityManager(manifestPath, keyDirectory, namespace, authorityConfigMap, kube) + if err := security.Reload(context.Background()); err != nil { + return fmt.Errorf("load security material: %w", err) + } podexec := executioncontroller.NewPodExecutor(cfg, kube, namespace) - store := executioncontroller.NewStore(dyn, namespace, profiles, podexec) + store := executioncontroller.NewStore(dyn, namespace, profiles, podexec).WithKubeClient(kube) reconciler := executioncontroller.NewReconciler(dyn, kube, namespace, profiles) - pub := priv.Public().(ed25519.PublicKey) - verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{keyID: pub}, Issuer: issuer, Audience: audience, MaxLifetime: 5 * time.Minute} - handler := executioncontroller.NewHandler(executioncontroller.HandlerConfig{Clients: policies, Signer: executioncontroller.GrantSigner{KeyID: keyID, PrivateKey: priv, Issuer: issuer, Audience: audience, Lifetime: time.Minute}, Verifier: verifier, Ready: reconciler.Ready}, store) + handler := executioncontroller.NewHandler(executioncontroller.HandlerConfig{Security: security, Ready: func() bool { return reconciler.Ready() && security.Ready() }}, store) ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() + go security.Run(ctx, reloadInterval) if err := reconciler.Initialize(ctx); err != nil { return fmt.Errorf("initialize controller: %w", err) } @@ -107,12 +87,45 @@ func run() error { //nolint:gocyclo // Startup validation and owned-resource shu if err != nil { return err } + limiter, err := executioncontroller.NewRPCLimiter(security, maxConcurrentRPCs, maxConcurrentRPCsPerClient) + if err != nil { + return err + } server := grpc.NewServer( - grpc.Creds(credentials.NewTLS(executioncontroller.TLSConfig(serverCert, pool))), + grpc.Creds(credentials.NewTLS(security.TLSConfig())), + grpc.ChainUnaryInterceptor(limiter.UnaryInterceptor), grpc.MaxRecvMsgSize(executionenv.MaxMessageBytes), grpc.MaxSendMsgSize(executionenv.MaxMessageBytes), + grpc.MaxConcurrentStreams(uint32(maxConcurrentStreams)), //nolint:gosec // validated to 1..1024 above. + grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: 30 * time.Minute, MaxConnectionAgeGrace: 2 * time.Minute, Time: 2 * time.Minute, Timeout: 20 * time.Second}), + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{MinTime: 30 * time.Second, PermitWithoutStream: false}), ) executionv1.RegisterExecutionProviderServiceServer(server, handler) + var healthServer *http.Server + if healthAddr != "" { + mux := http.NewServeMux() + mux.HandleFunc("GET /live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + mux.HandleFunc("GET /ready", func(w http.ResponseWriter, r *http.Request) { + if !reconciler.Ready() || !security.CheckReady(r.Context()) { + http.Error(w, "not ready", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + }) + healthServer = &http.Server{Addr: healthAddr, Handler: mux, ReadHeaderTimeout: 2 * time.Second, IdleTimeout: 30 * time.Second} + go func() { + if err := healthServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + cancel() + } + }() + } + defer func() { + if healthServer != nil { + shutdown, stop := context.WithTimeout(context.Background(), 3*time.Second) + defer stop() + _ = healthServer.Shutdown(shutdown) + } + }() serveErr := make(chan error, 1) go func() { serveErr <- server.Serve(ln) }() select { @@ -140,54 +153,3 @@ func run() error { //nolint:gocyclo // Startup validation and owned-resource shu return nil } } -func loadSigningKey(path string) (ed25519.PrivateKey, error) { - b, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("load grant signing key: %w", err) - } - block, _ := pem.Decode(b) - if block == nil || block.Type != "PRIVATE KEY" { - return nil, errors.New("grant signing key must be PKCS8 PEM") - } - raw, err := x509.ParsePKCS8PrivateKey(block.Bytes) - if err != nil { - return nil, errors.New("grant signing key is invalid") - } - key, ok := raw.(ed25519.PrivateKey) - if !ok { - return nil, errors.New("grant signing key is not Ed25519") - } - return key, nil -} -func clientPolicies(clients, attesters, admins string) (map[string]executioncontroller.ClientPolicy, error) { - out := map[string]executioncontroller.ClientPolicy{} - for _, id := range splitList(clients) { - out[id] = executioncontroller.ClientPolicy{} - } - for _, id := range splitList(attesters) { - p, ok := out[id] - if !ok { - return nil, fmt.Errorf("owner attester %q is not a client", id) - } - p.MayAttestOwner = true - out[id] = p - } - for _, id := range splitList(admins) { - p, ok := out[id] - if !ok { - return nil, fmt.Errorf("administrator %q is not a client", id) - } - p.Administrator = true - out[id] = p - } - return out, nil -} -func splitList(v string) []string { - out := []string{} - for _, x := range strings.Split(v, ",") { - if x = strings.TrimSpace(x); x != "" { - out = append(out, x) - } - } - return out -} diff --git a/contracts/gen/go/mecatl/execution/v1/execution.pb.go b/contracts/gen/go/mecatl/execution/v1/execution.pb.go index 0f6c60c343..970598b79f 100644 --- a/contracts/gen/go/mecatl/execution/v1/execution.pb.go +++ b/contracts/gen/go/mecatl/execution/v1/execution.pb.go @@ -268,14 +268,17 @@ func (x *Owner) GetSubject() string { } type RequestContext struct { - state protoimpl.MessageState `protogen:"open.v1"` - Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` - Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` - BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` - Epoch uint64 `protobuf:"varint,4,opt,name=epoch,proto3" json:"epoch,omitempty"` - Grant string `protobuf:"bytes,5,opt,name=grant,proto3" json:"grant,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + Epoch uint64 `protobuf:"varint,4,opt,name=epoch,proto3" json:"epoch,omitempty"` + Grant string `protobuf:"bytes,5,opt,name=grant,proto3" json:"grant,omitempty"` + RunId string `protobuf:"bytes,6,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,7,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + GrantGeneration uint64 `protobuf:"varint,8,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RequestContext) Reset() { @@ -343,6 +346,27 @@ func (x *RequestContext) GetGrant() string { return "" } +func (x *RequestContext) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *RequestContext) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *RequestContext) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + type ValidateProfileRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Profile string `protobuf:"bytes,1,opt,name=profile,proto3" json:"profile,omitempty"` @@ -476,6 +500,7 @@ type EnsureEnvironmentRequest struct { BindingId string `protobuf:"bytes,1,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` Profile string `protobuf:"bytes,2,opt,name=profile,proto3" json:"profile,omitempty"` Owner *Owner `protobuf:"bytes,3,opt,name=owner,proto3" json:"owner,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -510,53 +535,1261 @@ func (*EnsureEnvironmentRequest) Descriptor() ([]byte, []int) { return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{5} } -func (x *EnsureEnvironmentRequest) GetBindingId() string { +func (x *EnsureEnvironmentRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *EnsureEnvironmentRequest) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +func (x *EnsureEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *EnsureEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type EnsureEnvironmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` + Ready bool `protobuf:"varint,3,opt,name=ready,proto3" json:"ready,omitempty"` + Grant string `protobuf:"bytes,4,opt,name=grant,proto3" json:"grant,omitempty"` + GrantExpiresAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=grant_expires_at,json=grantExpiresAt,proto3" json:"grant_expires_at,omitempty"` + GrantGeneration uint64 `protobuf:"varint,6,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnsureEnvironmentResponse) Reset() { + *x = EnsureEnvironmentResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnsureEnvironmentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnsureEnvironmentResponse) ProtoMessage() {} + +func (x *EnsureEnvironmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnsureEnvironmentResponse.ProtoReflect.Descriptor instead. +func (*EnsureEnvironmentResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{6} +} + +func (x *EnsureEnvironmentResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *EnsureEnvironmentResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *EnsureEnvironmentResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *EnsureEnvironmentResponse) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *EnsureEnvironmentResponse) GetGrantExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.GrantExpiresAt + } + return nil +} + +func (x *EnsureEnvironmentResponse) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type AttachEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + Purpose string `protobuf:"bytes,2,opt,name=purpose,proto3" json:"purpose,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttachEnvironmentRequest) Reset() { + *x = AttachEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttachEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttachEnvironmentRequest) ProtoMessage() {} + +func (x *AttachEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttachEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*AttachEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{7} +} + +func (x *AttachEnvironmentRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *AttachEnvironmentRequest) GetPurpose() string { + if x != nil { + return x.Purpose + } + return "" +} + +type AttachEnvironmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` + Ready bool `protobuf:"varint,3,opt,name=ready,proto3" json:"ready,omitempty"` + Grant string `protobuf:"bytes,4,opt,name=grant,proto3" json:"grant,omitempty"` + GrantExpiresAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=grant_expires_at,json=grantExpiresAt,proto3" json:"grant_expires_at,omitempty"` + GrantGeneration uint64 `protobuf:"varint,6,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttachEnvironmentResponse) Reset() { + *x = AttachEnvironmentResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttachEnvironmentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttachEnvironmentResponse) ProtoMessage() {} + +func (x *AttachEnvironmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttachEnvironmentResponse.ProtoReflect.Descriptor instead. +func (*AttachEnvironmentResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{8} +} + +func (x *AttachEnvironmentResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *AttachEnvironmentResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *AttachEnvironmentResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *AttachEnvironmentResponse) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *AttachEnvironmentResponse) GetGrantExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.GrantExpiresAt + } + return nil +} + +func (x *AttachEnvironmentResponse) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type AcquireRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,4,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + OperationId string `protobuf:"bytes,5,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + TtlMillis int64 `protobuf:"varint,6,opt,name=ttl_millis,json=ttlMillis,proto3" json:"ttl_millis,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AcquireRunRequest) Reset() { + *x = AcquireRunRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AcquireRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AcquireRunRequest) ProtoMessage() {} + +func (x *AcquireRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AcquireRunRequest.ProtoReflect.Descriptor instead. +func (*AcquireRunRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{9} +} + +func (x *AcquireRunRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *AcquireRunRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *AcquireRunRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *AcquireRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *AcquireRunRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *AcquireRunRequest) GetTtlMillis() int64 { + if x != nil { + return x.TtlMillis + } + return 0 +} + +type RenewRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,4,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,5,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + Epoch uint64 `protobuf:"varint,6,opt,name=epoch,proto3" json:"epoch,omitempty"` + OperationId string `protobuf:"bytes,7,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + TtlMillis int64 `protobuf:"varint,8,opt,name=ttl_millis,json=ttlMillis,proto3" json:"ttl_millis,omitempty"` + GrantGeneration uint64 `protobuf:"varint,9,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RenewRunRequest) Reset() { + *x = RenewRunRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RenewRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RenewRunRequest) ProtoMessage() {} + +func (x *RenewRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RenewRunRequest.ProtoReflect.Descriptor instead. +func (*RenewRunRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{10} +} + +func (x *RenewRunRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RenewRunRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RenewRunRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *RenewRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *RenewRunRequest) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *RenewRunRequest) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *RenewRunRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *RenewRunRequest) GetTtlMillis() int64 { + if x != nil { + return x.TtlMillis + } + return 0 +} + +func (x *RenewRunRequest) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type ReleaseRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,4,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,5,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + Epoch uint64 `protobuf:"varint,6,opt,name=epoch,proto3" json:"epoch,omitempty"` + OperationId string `protobuf:"bytes,7,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + GrantGeneration uint64 `protobuf:"varint,8,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReleaseRunRequest) Reset() { + *x = ReleaseRunRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReleaseRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReleaseRunRequest) ProtoMessage() {} + +func (x *ReleaseRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReleaseRunRequest.ProtoReflect.Descriptor instead. +func (*ReleaseRunRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{11} +} + +func (x *ReleaseRunRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReleaseRunRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReleaseRunRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *ReleaseRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *ReleaseRunRequest) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *ReleaseRunRequest) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *ReleaseRunRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *ReleaseRunRequest) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type RunClaimResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + BindingId string `protobuf:"bytes,2,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,3,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,4,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + Epoch uint64 `protobuf:"varint,5,opt,name=epoch,proto3" json:"epoch,omitempty"` + GrantGeneration uint64 `protobuf:"varint,6,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + Grant string `protobuf:"bytes,7,opt,name=grant,proto3" json:"grant,omitempty"` + ExpiresAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RunClaimResponse) Reset() { + *x = RunClaimResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RunClaimResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunClaimResponse) ProtoMessage() {} + +func (x *RunClaimResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunClaimResponse.ProtoReflect.Descriptor instead. +func (*RunClaimResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{12} +} + +func (x *RunClaimResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RunClaimResponse) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *RunClaimResponse) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *RunClaimResponse) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *RunClaimResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *RunClaimResponse) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +func (x *RunClaimResponse) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *RunClaimResponse) GetExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.ExpiresAt + } + return nil +} + +type ReferenceMutationRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReferenceMutationRequest) Reset() { + *x = ReferenceMutationRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReferenceMutationRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReferenceMutationRequest) ProtoMessage() {} + +func (x *ReferenceMutationRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReferenceMutationRequest.ProtoReflect.Descriptor instead. +func (*ReferenceMutationRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{13} +} + +func (x *ReferenceMutationRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReferenceMutationRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReferenceMutationRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *ReferenceMutationRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type ReserveSuccessorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + SourceBindingId string `protobuf:"bytes,3,opt,name=source_binding_id,json=sourceBindingId,proto3" json:"source_binding_id,omitempty"` + DestinationBindingId string `protobuf:"bytes,4,opt,name=destination_binding_id,json=destinationBindingId,proto3" json:"destination_binding_id,omitempty"` + OperationId string `protobuf:"bytes,5,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReserveSuccessorRequest) Reset() { + *x = ReserveSuccessorRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReserveSuccessorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReserveSuccessorRequest) ProtoMessage() {} + +func (x *ReserveSuccessorRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReserveSuccessorRequest.ProtoReflect.Descriptor instead. +func (*ReserveSuccessorRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{14} +} + +func (x *ReserveSuccessorRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReserveSuccessorRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReserveSuccessorRequest) GetSourceBindingId() string { + if x != nil { + return x.SourceBindingId + } + return "" +} + +func (x *ReserveSuccessorRequest) GetDestinationBindingId() string { + if x != nil { + return x.DestinationBindingId + } + return "" +} + +func (x *ReserveSuccessorRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type ReferenceReservationResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReferenceReservationResponse) Reset() { + *x = ReferenceReservationResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReferenceReservationResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReferenceReservationResponse) ProtoMessage() {} + +func (x *ReferenceReservationResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReferenceReservationResponse.ProtoReflect.Descriptor instead. +func (*ReferenceReservationResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{15} +} + +func (x *ReferenceReservationResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +type ListReferenceIntentsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Owner *Owner `protobuf:"bytes,1,opt,name=owner,proto3" json:"owner,omitempty"` + Limit int32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListReferenceIntentsRequest) Reset() { + *x = ListReferenceIntentsRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListReferenceIntentsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListReferenceIntentsRequest) ProtoMessage() {} + +func (x *ListReferenceIntentsRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListReferenceIntentsRequest.ProtoReflect.Descriptor instead. +func (*ListReferenceIntentsRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{16} +} + +func (x *ListReferenceIntentsRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ListReferenceIntentsRequest) GetLimit() int32 { + if x != nil { + return x.Limit + } + return 0 +} + +type ReferenceIntent struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + BindingId string `protobuf:"bytes,2,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + State string `protobuf:"bytes,3,opt,name=state,proto3" json:"state,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + SourceBindingId string `protobuf:"bytes,5,opt,name=source_binding_id,json=sourceBindingId,proto3" json:"source_binding_id,omitempty"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + Owner *Owner `protobuf:"bytes,7,opt,name=owner,proto3" json:"owner,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReferenceIntent) Reset() { + *x = ReferenceIntent{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReferenceIntent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReferenceIntent) ProtoMessage() {} + +func (x *ReferenceIntent) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReferenceIntent.ProtoReflect.Descriptor instead. +func (*ReferenceIntent) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{17} +} + +func (x *ReferenceIntent) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReferenceIntent) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *ReferenceIntent) GetState() string { + if x != nil { + return x.State + } + return "" +} + +func (x *ReferenceIntent) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *ReferenceIntent) GetSourceBindingId() string { + if x != nil { + return x.SourceBindingId + } + return "" +} + +func (x *ReferenceIntent) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +func (x *ReferenceIntent) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +type ListReferenceIntentsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Intents []*ReferenceIntent `protobuf:"bytes,1,rep,name=intents,proto3" json:"intents,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListReferenceIntentsResponse) Reset() { + *x = ListReferenceIntentsResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListReferenceIntentsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListReferenceIntentsResponse) ProtoMessage() {} + +func (x *ListReferenceIntentsResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListReferenceIntentsResponse.ProtoReflect.Descriptor instead. +func (*ListReferenceIntentsResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{18} +} + +func (x *ListReferenceIntentsResponse) GetIntents() []*ReferenceIntent { + if x != nil { + return x.Intents + } + return nil +} + +type ReleaseReferenceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReleaseReferenceRequest) Reset() { + *x = ReleaseReferenceRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReleaseReferenceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReleaseReferenceRequest) ProtoMessage() {} + +func (x *ReleaseReferenceRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReleaseReferenceRequest.ProtoReflect.Descriptor instead. +func (*ReleaseReferenceRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{19} +} + +func (x *ReleaseReferenceRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +type RetireEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedExecutionEpoch uint64 `protobuf:"varint,3,opt,name=expected_execution_epoch,json=expectedExecutionEpoch,proto3" json:"expected_execution_epoch,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RetireEnvironmentRequest) Reset() { + *x = RetireEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RetireEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RetireEnvironmentRequest) ProtoMessage() {} + +func (x *RetireEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RetireEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*RetireEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{20} +} + +func (x *RetireEnvironmentRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RetireEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RetireEnvironmentRequest) GetExpectedExecutionEpoch() uint64 { + if x != nil { + return x.ExpectedExecutionEpoch + } + return 0 +} + +func (x *RetireEnvironmentRequest) GetExpectedPodUid() string { + if x != nil { + return x.ExpectedPodUid + } + return "" +} + +func (x *RetireEnvironmentRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *RetireEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type ReplaceExecutorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedExecutionEpoch uint64 `protobuf:"varint,3,opt,name=expected_execution_epoch,json=expectedExecutionEpoch,proto3" json:"expected_execution_epoch,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReplaceExecutorRequest) Reset() { + *x = ReplaceExecutorRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReplaceExecutorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReplaceExecutorRequest) ProtoMessage() {} + +func (x *ReplaceExecutorRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReplaceExecutorRequest.ProtoReflect.Descriptor instead. +func (*ReplaceExecutorRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{21} +} + +func (x *ReplaceExecutorRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReplaceExecutorRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReplaceExecutorRequest) GetExpectedExecutionEpoch() uint64 { if x != nil { - return x.BindingId + return x.ExpectedExecutionEpoch + } + return 0 +} + +func (x *ReplaceExecutorRequest) GetExpectedPodUid() string { + if x != nil { + return x.ExpectedPodUid } return "" } -func (x *EnsureEnvironmentRequest) GetProfile() string { +func (x *ReplaceExecutorRequest) GetExpectedPvcUid() string { if x != nil { - return x.Profile + return x.ExpectedPvcUid } return "" } -func (x *EnsureEnvironmentRequest) GetOwner() *Owner { +func (x *ReplaceExecutorRequest) GetOperationId() string { if x != nil { - return x.Owner + return x.OperationId } - return nil + return "" } -type EnsureEnvironmentResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` - Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` - Ready bool `protobuf:"varint,3,opt,name=ready,proto3" json:"ready,omitempty"` - Grant string `protobuf:"bytes,4,opt,name=grant,proto3" json:"grant,omitempty"` - GrantExpiresAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=grant_expires_at,json=grantExpiresAt,proto3" json:"grant_expires_at,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache +type RecoverEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedExecutionEpoch uint64 `protobuf:"varint,3,opt,name=expected_execution_epoch,json=expectedExecutionEpoch,proto3" json:"expected_execution_epoch,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *EnsureEnvironmentResponse) Reset() { - *x = EnsureEnvironmentResponse{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[6] +func (x *RecoverEnvironmentRequest) Reset() { + *x = RecoverEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *EnsureEnvironmentResponse) String() string { +func (x *RecoverEnvironmentRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*EnsureEnvironmentResponse) ProtoMessage() {} +func (*RecoverEnvironmentRequest) ProtoMessage() {} -func (x *EnsureEnvironmentResponse) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[6] +func (x *RecoverEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -567,69 +1800,78 @@ func (x *EnsureEnvironmentResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use EnsureEnvironmentResponse.ProtoReflect.Descriptor instead. -func (*EnsureEnvironmentResponse) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{6} +// Deprecated: Use RecoverEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*RecoverEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{22} } -func (x *EnsureEnvironmentResponse) GetEnvironment() *EnvironmentRef { +func (x *RecoverEnvironmentRequest) GetEnvironment() *EnvironmentRef { if x != nil { return x.Environment } return nil } -func (x *EnsureEnvironmentResponse) GetEpoch() uint64 { +func (x *RecoverEnvironmentRequest) GetOwner() *Owner { if x != nil { - return x.Epoch + return x.Owner + } + return nil +} + +func (x *RecoverEnvironmentRequest) GetExpectedExecutionEpoch() uint64 { + if x != nil { + return x.ExpectedExecutionEpoch } return 0 } -func (x *EnsureEnvironmentResponse) GetReady() bool { +func (x *RecoverEnvironmentRequest) GetExpectedPodUid() string { if x != nil { - return x.Ready + return x.ExpectedPodUid } - return false + return "" } -func (x *EnsureEnvironmentResponse) GetGrant() string { +func (x *RecoverEnvironmentRequest) GetExpectedPvcUid() string { if x != nil { - return x.Grant + return x.ExpectedPvcUid } return "" } -func (x *EnsureEnvironmentResponse) GetGrantExpiresAt() *timestamppb.Timestamp { +func (x *RecoverEnvironmentRequest) GetOperationId() string { if x != nil { - return x.GrantExpiresAt + return x.OperationId } - return nil + return "" } -type AttachEnvironmentRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` - Purpose string `protobuf:"bytes,2,opt,name=purpose,proto3" json:"purpose,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache +type DeleteRetiredEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,3,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *AttachEnvironmentRequest) Reset() { - *x = AttachEnvironmentRequest{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[7] +func (x *DeleteRetiredEnvironmentRequest) Reset() { + *x = DeleteRetiredEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *AttachEnvironmentRequest) String() string { +func (x *DeleteRetiredEnvironmentRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*AttachEnvironmentRequest) ProtoMessage() {} +func (*DeleteRetiredEnvironmentRequest) ProtoMessage() {} -func (x *AttachEnvironmentRequest) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[7] +func (x *DeleteRetiredEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -640,51 +1882,63 @@ func (x *AttachEnvironmentRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use AttachEnvironmentRequest.ProtoReflect.Descriptor instead. -func (*AttachEnvironmentRequest) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{7} +// Deprecated: Use DeleteRetiredEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*DeleteRetiredEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{23} } -func (x *AttachEnvironmentRequest) GetContext() *RequestContext { +func (x *DeleteRetiredEnvironmentRequest) GetEnvironment() *EnvironmentRef { if x != nil { - return x.Context + return x.Environment } return nil } -func (x *AttachEnvironmentRequest) GetPurpose() string { +func (x *DeleteRetiredEnvironmentRequest) GetOwner() *Owner { if x != nil { - return x.Purpose + return x.Owner + } + return nil +} + +func (x *DeleteRetiredEnvironmentRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid } return "" } -type AttachEnvironmentResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` - Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` - Ready bool `protobuf:"varint,3,opt,name=ready,proto3" json:"ready,omitempty"` - Grant string `protobuf:"bytes,4,opt,name=grant,proto3" json:"grant,omitempty"` - GrantExpiresAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=grant_expires_at,json=grantExpiresAt,proto3" json:"grant_expires_at,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache +func (x *DeleteRetiredEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" } -func (x *AttachEnvironmentResponse) Reset() { - *x = AttachEnvironmentResponse{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[8] +type RevokeEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedGrantGeneration uint64 `protobuf:"varint,3,opt,name=expected_grant_generation,json=expectedGrantGeneration,proto3" json:"expected_grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RevokeEnvironmentRequest) Reset() { + *x = RevokeEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *AttachEnvironmentResponse) String() string { +func (x *RevokeEnvironmentRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*AttachEnvironmentResponse) ProtoMessage() {} +func (*RevokeEnvironmentRequest) ProtoMessage() {} -func (x *AttachEnvironmentResponse) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[8] +func (x *RevokeEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -695,68 +1949,54 @@ func (x *AttachEnvironmentResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use AttachEnvironmentResponse.ProtoReflect.Descriptor instead. -func (*AttachEnvironmentResponse) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{8} +// Deprecated: Use RevokeEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*RevokeEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{24} } -func (x *AttachEnvironmentResponse) GetEnvironment() *EnvironmentRef { +func (x *RevokeEnvironmentRequest) GetEnvironment() *EnvironmentRef { if x != nil { return x.Environment } return nil } -func (x *AttachEnvironmentResponse) GetEpoch() uint64 { - if x != nil { - return x.Epoch - } - return 0 -} - -func (x *AttachEnvironmentResponse) GetReady() bool { - if x != nil { - return x.Ready - } - return false -} - -func (x *AttachEnvironmentResponse) GetGrant() string { +func (x *RevokeEnvironmentRequest) GetOwner() *Owner { if x != nil { - return x.Grant + return x.Owner } - return "" + return nil } -func (x *AttachEnvironmentResponse) GetGrantExpiresAt() *timestamppb.Timestamp { +func (x *RevokeEnvironmentRequest) GetExpectedGrantGeneration() uint64 { if x != nil { - return x.GrantExpiresAt + return x.ExpectedGrantGeneration } - return nil + return 0 } -type ReleaseReferenceRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache +type RevokeEnvironmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + GrantGeneration uint64 `protobuf:"varint,1,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *ReleaseReferenceRequest) Reset() { - *x = ReleaseReferenceRequest{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[9] +func (x *RevokeEnvironmentResponse) Reset() { + *x = RevokeEnvironmentResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *ReleaseReferenceRequest) String() string { +func (x *RevokeEnvironmentResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ReleaseReferenceRequest) ProtoMessage() {} +func (*RevokeEnvironmentResponse) ProtoMessage() {} -func (x *ReleaseReferenceRequest) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[9] +func (x *RevokeEnvironmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -767,40 +2007,45 @@ func (x *ReleaseReferenceRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ReleaseReferenceRequest.ProtoReflect.Descriptor instead. -func (*ReleaseReferenceRequest) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{9} +// Deprecated: Use RevokeEnvironmentResponse.ProtoReflect.Descriptor instead. +func (*RevokeEnvironmentResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{25} } -func (x *ReleaseReferenceRequest) GetContext() *RequestContext { +func (x *RevokeEnvironmentResponse) GetGrantGeneration() uint64 { if x != nil { - return x.Context + return x.GrantGeneration } - return nil + return 0 } -type RetireEnvironmentRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache +type MigrateEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedSchemaVersion uint32 `protobuf:"varint,3,opt,name=expected_schema_version,json=expectedSchemaVersion,proto3" json:"expected_schema_version,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *RetireEnvironmentRequest) Reset() { - *x = RetireEnvironmentRequest{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[10] +func (x *MigrateEnvironmentRequest) Reset() { + *x = MigrateEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *RetireEnvironmentRequest) String() string { +func (x *MigrateEnvironmentRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*RetireEnvironmentRequest) ProtoMessage() {} +func (*MigrateEnvironmentRequest) ProtoMessage() {} -func (x *RetireEnvironmentRequest) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[10] +func (x *MigrateEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -811,18 +2056,53 @@ func (x *RetireEnvironmentRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use RetireEnvironmentRequest.ProtoReflect.Descriptor instead. -func (*RetireEnvironmentRequest) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{10} +// Deprecated: Use MigrateEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*MigrateEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{26} } -func (x *RetireEnvironmentRequest) GetContext() *RequestContext { +func (x *MigrateEnvironmentRequest) GetEnvironment() *EnvironmentRef { if x != nil { - return x.Context + return x.Environment + } + return nil +} + +func (x *MigrateEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner } return nil } +func (x *MigrateEnvironmentRequest) GetExpectedSchemaVersion() uint32 { + if x != nil { + return x.ExpectedSchemaVersion + } + return 0 +} + +func (x *MigrateEnvironmentRequest) GetExpectedPodUid() string { + if x != nil { + return x.ExpectedPodUid + } + return "" +} + +func (x *MigrateEnvironmentRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *MigrateEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + type FileRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` @@ -839,7 +2119,7 @@ type FileRequest struct { func (x *FileRequest) Reset() { *x = FileRequest{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[11] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -851,7 +2131,7 @@ func (x *FileRequest) String() string { func (*FileRequest) ProtoMessage() {} func (x *FileRequest) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[11] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -864,7 +2144,7 @@ func (x *FileRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FileRequest.ProtoReflect.Descriptor instead. func (*FileRequest) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{11} + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{27} } func (x *FileRequest) GetContext() *RequestContext { @@ -936,7 +2216,7 @@ type FileInfo struct { func (x *FileInfo) Reset() { *x = FileInfo{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[12] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -948,7 +2228,7 @@ func (x *FileInfo) String() string { func (*FileInfo) ProtoMessage() {} func (x *FileInfo) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[12] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -961,7 +2241,7 @@ func (x *FileInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use FileInfo.ProtoReflect.Descriptor instead. func (*FileInfo) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{12} + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{28} } func (x *FileInfo) GetName() string { @@ -1010,7 +2290,7 @@ type GrepMatch struct { func (x *GrepMatch) Reset() { *x = GrepMatch{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[13] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1022,7 +2302,7 @@ func (x *GrepMatch) String() string { func (*GrepMatch) ProtoMessage() {} func (x *GrepMatch) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[13] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1035,7 +2315,7 @@ func (x *GrepMatch) ProtoReflect() protoreflect.Message { // Deprecated: Use GrepMatch.ProtoReflect.Descriptor instead. func (*GrepMatch) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{13} + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{29} } func (x *GrepMatch) GetPath() string { @@ -1075,7 +2355,7 @@ type FileResponse struct { func (x *FileResponse) Reset() { *x = FileResponse{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[14] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1087,7 +2367,7 @@ func (x *FileResponse) String() string { func (*FileResponse) ProtoMessage() {} func (x *FileResponse) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[14] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1100,7 +2380,7 @@ func (x *FileResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FileResponse.ProtoReflect.Descriptor instead. func (*FileResponse) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{14} + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{30} } func (x *FileResponse) GetData() []byte { @@ -1170,7 +2450,7 @@ type CommandStartRequest struct { func (x *CommandStartRequest) Reset() { *x = CommandStartRequest{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[15] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1182,7 +2462,7 @@ func (x *CommandStartRequest) String() string { func (*CommandStartRequest) ProtoMessage() {} func (x *CommandStartRequest) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[15] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1195,7 +2475,7 @@ func (x *CommandStartRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CommandStartRequest.ProtoReflect.Descriptor instead. func (*CommandStartRequest) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{15} + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{31} } func (x *CommandStartRequest) GetContext() *RequestContext { @@ -1230,7 +2510,7 @@ type CommandStartResponse struct { func (x *CommandStartResponse) Reset() { *x = CommandStartResponse{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[16] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1242,7 +2522,7 @@ func (x *CommandStartResponse) String() string { func (*CommandStartResponse) ProtoMessage() {} func (x *CommandStartResponse) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[16] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1255,7 +2535,7 @@ func (x *CommandStartResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CommandStartResponse.ProtoReflect.Descriptor instead. func (*CommandStartResponse) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{16} + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{32} } func (x *CommandStartResponse) GetCommandId() string { @@ -1290,7 +2570,7 @@ type CommandQueryRequest struct { func (x *CommandQueryRequest) Reset() { *x = CommandQueryRequest{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[17] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1302,7 +2582,7 @@ func (x *CommandQueryRequest) String() string { func (*CommandQueryRequest) ProtoMessage() {} func (x *CommandQueryRequest) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[17] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1315,7 +2595,7 @@ func (x *CommandQueryRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CommandQueryRequest.ProtoReflect.Descriptor instead. func (*CommandQueryRequest) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{17} + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{33} } func (x *CommandQueryRequest) GetContext() *RequestContext { @@ -1355,7 +2635,7 @@ type CommandStatusResponse struct { func (x *CommandStatusResponse) Reset() { *x = CommandStatusResponse{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[18] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1367,7 +2647,7 @@ func (x *CommandStatusResponse) String() string { func (*CommandStatusResponse) ProtoMessage() {} func (x *CommandStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[18] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1380,7 +2660,7 @@ func (x *CommandStatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CommandStatusResponse.ProtoReflect.Descriptor instead. func (*CommandStatusResponse) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{18} + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{34} } func (x *CommandStatusResponse) GetCommandId() string { @@ -1451,7 +2731,7 @@ type ErrorDetail struct { func (x *ErrorDetail) Reset() { *x = ErrorDetail{} - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[19] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1463,7 +2743,7 @@ func (x *ErrorDetail) String() string { func (*ErrorDetail) ProtoMessage() {} func (x *ErrorDetail) ProtoReflect() protoreflect.Message { - mi := &file_mecatl_execution_v1_execution_proto_msgTypes[19] + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1476,7 +2756,7 @@ func (x *ErrorDetail) ProtoReflect() protoreflect.Message { // Deprecated: Use ErrorDetail.ProtoReflect.Descriptor instead. func (*ErrorDetail) Descriptor() ([]byte, []int) { - return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{19} + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{35} } func (x *ErrorDetail) GetCode() string { @@ -1511,7 +2791,7 @@ var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ 0x16, 0x0a, 0x06, 0x69, 0x73, 0x73, 0x75, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x73, 0x73, 0x75, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x22, 0xb1, 0x02, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, @@ -1524,35 +2804,43 @@ var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x22, 0x32, 0x0a, 0x16, 0x56, 0x61, 0x6c, 0x69, - 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x80, 0x02, 0x0a, - 0x17, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, - 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, - 0x6c, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x61, - 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x24, - 0x0a, 0x0e, 0x6d, 0x61, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x42, - 0x79, 0x74, 0x65, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, - 0x61, 0x6e, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x0f, 0x6d, 0x61, 0x78, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, - 0x12, 0x3d, 0x0a, 0x1b, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, - 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x18, 0x6d, 0x61, 0x78, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, - 0x64, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, - 0x85, 0x01, 0x0a, 0x18, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, - 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, - 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x70, - 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, - 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, - 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x22, 0xea, 0x01, 0x0a, 0x19, 0x45, 0x6e, 0x73, 0x75, + 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, + 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, + 0x19, 0x0a, 0x08, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, + 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x32, 0x0a, 0x16, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x80, 0x02, 0x0a, 0x17, 0x56, 0x61, + 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, + 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, + 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x6d, + 0x61, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x42, 0x79, 0x74, 0x65, + 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x6d, 0x61, + 0x78, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x3d, 0x0a, + 0x1b, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x64, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x18, 0x6d, 0x61, 0x78, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x44, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, 0xa8, 0x01, 0x0a, + 0x18, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, + 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, + 0x6c, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x95, 0x02, 0x0a, 0x19, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, @@ -1567,40 +2855,303 @@ var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x45, 0x78, 0x70, 0x69, 0x72, - 0x65, 0x73, 0x41, 0x74, 0x22, 0x73, 0x0a, 0x18, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, - 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, - 0x18, 0x0a, 0x07, 0x70, 0x75, 0x72, 0x70, 0x6f, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x70, 0x75, 0x72, 0x70, 0x6f, 0x73, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x19, 0x41, 0x74, - 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x65, 0x73, 0x41, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, + 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x73, 0x0a, 0x18, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x75, + 0x72, 0x70, 0x6f, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x75, 0x72, + 0x70, 0x6f, 0x73, 0x65, 0x22, 0x95, 0x02, 0x0a, 0x19, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, + 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, + 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, + 0x72, 0x65, 0x61, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x44, 0x0a, 0x10, 0x67, + 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x61, 0x74, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x0e, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, + 0x74, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, + 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x84, 0x02, 0x0a, + 0x11, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, + 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, + 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, + 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x74, 0x6c, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, + 0x69, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x74, 0x6c, 0x4d, 0x69, 0x6c, + 0x6c, 0x69, 0x73, 0x22, 0xde, 0x02, 0x0a, 0x0f, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, - 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, - 0x70, 0x6f, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, - 0x61, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, - 0x12, 0x44, 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, - 0x73, 0x5f, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x45, 0x78, 0x70, - 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x22, 0x58, 0x0a, 0x17, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, - 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, + 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, + 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, + 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x5f, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x49, + 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x74, + 0x6c, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, + 0x74, 0x74, 0x6c, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, + 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc1, 0x02, 0x0a, 0x11, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, + 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, + 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6c, 0x61, + 0x69, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, 0x61, + 0x69, 0x6d, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x29, 0x0a, + 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbc, 0x02, 0x0a, 0x10, 0x52, 0x75, 0x6e, + 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, + 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x22, 0x59, 0x0a, 0x18, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, - 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, + 0x67, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6c, + 0x61, 0x69, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, + 0x61, 0x69, 0x6d, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x29, 0x0a, 0x10, 0x67, + 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x39, 0x0a, 0x0a, + 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x78, + 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x22, 0xd5, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, + 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, + 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, + 0x97, 0x02, 0x0a, 0x17, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, + 0x12, 0x34, 0x0a, 0x16, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x14, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x65, 0x0a, 0x1c, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x22, 0x65, 0x0a, 0x1b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, + 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0xc9, 0x02, 0x0a, 0x0f, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, + 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x64, 0x5f, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, + 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, + 0x6e, 0x65, 0x72, 0x22, 0x5e, 0x0a, 0x1c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x07, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x52, 0x07, 0x69, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x73, 0x22, 0x58, 0x0a, 0x17, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, + 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0xc4, 0x02, + 0x0a, 0x18, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, + 0x6e, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x45, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x28, 0x0a, + 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, + 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, + 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xc2, 0x02, 0x0a, 0x16, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, + 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, + 0x70, 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, + 0x63, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, + 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, + 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, + 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xc5, 0x02, 0x0a, 0x19, 0x52, 0x65, + 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, + 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0xa2, 0x02, 0x0a, 0x0b, + 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, + 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, + 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, + 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, + 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x64, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, + 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, + 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, + 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x28, + 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, + 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xcf, 0x01, 0x0a, 0x18, + 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, + 0x72, 0x12, 0x3a, 0x0a, 0x19, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x67, 0x72, + 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x17, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x47, 0x72, + 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x46, 0x0a, + 0x19, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, + 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc3, 0x02, 0x0a, 0x19, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, + 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, + 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x36, 0x0a, 0x17, + 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, + 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xa2, 0x02, 0x0a, 0x0b, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, @@ -1740,7 +3291,7 @@ var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x45, 0x4e, 0x43, 0x45, 0x5f, 0x55, 0x4e, - 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x05, 0x32, 0xa7, 0x07, 0x0a, 0x18, 0x45, 0x78, 0x65, 0x63, + 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x05, 0x32, 0xe4, 0x12, 0x0a, 0x18, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, @@ -1763,57 +3314,149 @@ var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x10, 0x52, 0x65, - 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2c, - 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, - 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5a, 0x0a, 0x11, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, - 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5b, 0x0a, 0x0a, 0x41, 0x63, + 0x71, 0x75, 0x69, 0x72, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, + 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x25, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x08, 0x52, 0x65, 0x6e, 0x65, 0x77, + 0x52, 0x75, 0x6e, 0x12, 0x24, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x52, + 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, - 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, - 0x12, 0x4c, 0x0a, 0x05, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x4c, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x26, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x75, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x58, + 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x62, 0x6f, 0x72, + 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x12, 0x73, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x6f, 0x72, 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5f, 0x0a, 0x16, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, + 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5f, 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x72, 0x6d, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5e, 0x0a, 0x15, 0x43, 0x61, 0x6e, 0x63, + 0x65, 0x6c, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x7b, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, + 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, + 0x12, 0x30, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x10, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, - 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x65, - 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, - 0x0a, 0x0c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, + 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, + 0x5a, 0x0a, 0x11, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x74, 0x69, 0x72, + 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x56, 0x0a, 0x0f, 0x52, + 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, - 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, - 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x5c, 0x0a, 0x12, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x12, 0x68, 0x0a, 0x18, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, + 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, + 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5c, 0x0a, 0x12, 0x4d, + 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x72, 0x0a, 0x11, 0x52, 0x65, 0x76, + 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x61, - 0x6e, 0x63, 0x65, 0x6c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, + 0x05, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, + 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x0c, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, - 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x42, 0xe4, 0x01, 0x0a, 0x17, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, - 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x45, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, - 0x4b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, 0x61, 0x63, - 0x6b, 0x6c, 0x6f, 0x6b, 0x2f, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, 0x63, - 0x61, 0x74, 0x6c, 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x31, - 0x3b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, - 0x45, 0x58, 0xaa, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, - 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0xe2, 0x02, - 0x1f, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0xea, 0x02, 0x15, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x3a, 0x3a, 0x45, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, + 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x61, 0x6e, 0x63, 0x65, + 0x6c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0xe4, + 0x01, 0x0a, 0x17, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x4b, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x6c, 0x6f, + 0x6b, 0x2f, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, + 0x74, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x3b, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, 0x45, 0x58, 0xaa, + 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, 0x45, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x1f, 0x4d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5c, 0x56, + 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x15, + 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x3a, 0x3a, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, }) var ( @@ -1829,78 +3472,153 @@ func file_mecatl_execution_v1_execution_proto_rawDescGZIP() []byte { } var file_mecatl_execution_v1_execution_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_mecatl_execution_v1_execution_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_mecatl_execution_v1_execution_proto_msgTypes = make([]protoimpl.MessageInfo, 36) var file_mecatl_execution_v1_execution_proto_goTypes = []any{ - (FileOperation)(0), // 0: mecatl.execution.v1.FileOperation - (CommandState)(0), // 1: mecatl.execution.v1.CommandState - (*EnvironmentRef)(nil), // 2: mecatl.execution.v1.EnvironmentRef - (*Owner)(nil), // 3: mecatl.execution.v1.Owner - (*RequestContext)(nil), // 4: mecatl.execution.v1.RequestContext - (*ValidateProfileRequest)(nil), // 5: mecatl.execution.v1.ValidateProfileRequest - (*ValidateProfileResponse)(nil), // 6: mecatl.execution.v1.ValidateProfileResponse - (*EnsureEnvironmentRequest)(nil), // 7: mecatl.execution.v1.EnsureEnvironmentRequest - (*EnsureEnvironmentResponse)(nil), // 8: mecatl.execution.v1.EnsureEnvironmentResponse - (*AttachEnvironmentRequest)(nil), // 9: mecatl.execution.v1.AttachEnvironmentRequest - (*AttachEnvironmentResponse)(nil), // 10: mecatl.execution.v1.AttachEnvironmentResponse - (*ReleaseReferenceRequest)(nil), // 11: mecatl.execution.v1.ReleaseReferenceRequest - (*RetireEnvironmentRequest)(nil), // 12: mecatl.execution.v1.RetireEnvironmentRequest - (*FileRequest)(nil), // 13: mecatl.execution.v1.FileRequest - (*FileInfo)(nil), // 14: mecatl.execution.v1.FileInfo - (*GrepMatch)(nil), // 15: mecatl.execution.v1.GrepMatch - (*FileResponse)(nil), // 16: mecatl.execution.v1.FileResponse - (*CommandStartRequest)(nil), // 17: mecatl.execution.v1.CommandStartRequest - (*CommandStartResponse)(nil), // 18: mecatl.execution.v1.CommandStartResponse - (*CommandQueryRequest)(nil), // 19: mecatl.execution.v1.CommandQueryRequest - (*CommandStatusResponse)(nil), // 20: mecatl.execution.v1.CommandStatusResponse - (*ErrorDetail)(nil), // 21: mecatl.execution.v1.ErrorDetail - (*timestamppb.Timestamp)(nil), // 22: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 23: google.protobuf.Empty + (FileOperation)(0), // 0: mecatl.execution.v1.FileOperation + (CommandState)(0), // 1: mecatl.execution.v1.CommandState + (*EnvironmentRef)(nil), // 2: mecatl.execution.v1.EnvironmentRef + (*Owner)(nil), // 3: mecatl.execution.v1.Owner + (*RequestContext)(nil), // 4: mecatl.execution.v1.RequestContext + (*ValidateProfileRequest)(nil), // 5: mecatl.execution.v1.ValidateProfileRequest + (*ValidateProfileResponse)(nil), // 6: mecatl.execution.v1.ValidateProfileResponse + (*EnsureEnvironmentRequest)(nil), // 7: mecatl.execution.v1.EnsureEnvironmentRequest + (*EnsureEnvironmentResponse)(nil), // 8: mecatl.execution.v1.EnsureEnvironmentResponse + (*AttachEnvironmentRequest)(nil), // 9: mecatl.execution.v1.AttachEnvironmentRequest + (*AttachEnvironmentResponse)(nil), // 10: mecatl.execution.v1.AttachEnvironmentResponse + (*AcquireRunRequest)(nil), // 11: mecatl.execution.v1.AcquireRunRequest + (*RenewRunRequest)(nil), // 12: mecatl.execution.v1.RenewRunRequest + (*ReleaseRunRequest)(nil), // 13: mecatl.execution.v1.ReleaseRunRequest + (*RunClaimResponse)(nil), // 14: mecatl.execution.v1.RunClaimResponse + (*ReferenceMutationRequest)(nil), // 15: mecatl.execution.v1.ReferenceMutationRequest + (*ReserveSuccessorRequest)(nil), // 16: mecatl.execution.v1.ReserveSuccessorRequest + (*ReferenceReservationResponse)(nil), // 17: mecatl.execution.v1.ReferenceReservationResponse + (*ListReferenceIntentsRequest)(nil), // 18: mecatl.execution.v1.ListReferenceIntentsRequest + (*ReferenceIntent)(nil), // 19: mecatl.execution.v1.ReferenceIntent + (*ListReferenceIntentsResponse)(nil), // 20: mecatl.execution.v1.ListReferenceIntentsResponse + (*ReleaseReferenceRequest)(nil), // 21: mecatl.execution.v1.ReleaseReferenceRequest + (*RetireEnvironmentRequest)(nil), // 22: mecatl.execution.v1.RetireEnvironmentRequest + (*ReplaceExecutorRequest)(nil), // 23: mecatl.execution.v1.ReplaceExecutorRequest + (*RecoverEnvironmentRequest)(nil), // 24: mecatl.execution.v1.RecoverEnvironmentRequest + (*DeleteRetiredEnvironmentRequest)(nil), // 25: mecatl.execution.v1.DeleteRetiredEnvironmentRequest + (*RevokeEnvironmentRequest)(nil), // 26: mecatl.execution.v1.RevokeEnvironmentRequest + (*RevokeEnvironmentResponse)(nil), // 27: mecatl.execution.v1.RevokeEnvironmentResponse + (*MigrateEnvironmentRequest)(nil), // 28: mecatl.execution.v1.MigrateEnvironmentRequest + (*FileRequest)(nil), // 29: mecatl.execution.v1.FileRequest + (*FileInfo)(nil), // 30: mecatl.execution.v1.FileInfo + (*GrepMatch)(nil), // 31: mecatl.execution.v1.GrepMatch + (*FileResponse)(nil), // 32: mecatl.execution.v1.FileResponse + (*CommandStartRequest)(nil), // 33: mecatl.execution.v1.CommandStartRequest + (*CommandStartResponse)(nil), // 34: mecatl.execution.v1.CommandStartResponse + (*CommandQueryRequest)(nil), // 35: mecatl.execution.v1.CommandQueryRequest + (*CommandStatusResponse)(nil), // 36: mecatl.execution.v1.CommandStatusResponse + (*ErrorDetail)(nil), // 37: mecatl.execution.v1.ErrorDetail + (*timestamppb.Timestamp)(nil), // 38: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 39: google.protobuf.Empty } var file_mecatl_execution_v1_execution_proto_depIdxs = []int32{ 2, // 0: mecatl.execution.v1.RequestContext.environment:type_name -> mecatl.execution.v1.EnvironmentRef 3, // 1: mecatl.execution.v1.RequestContext.owner:type_name -> mecatl.execution.v1.Owner 3, // 2: mecatl.execution.v1.EnsureEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner 2, // 3: mecatl.execution.v1.EnsureEnvironmentResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef - 22, // 4: mecatl.execution.v1.EnsureEnvironmentResponse.grant_expires_at:type_name -> google.protobuf.Timestamp + 38, // 4: mecatl.execution.v1.EnsureEnvironmentResponse.grant_expires_at:type_name -> google.protobuf.Timestamp 4, // 5: mecatl.execution.v1.AttachEnvironmentRequest.context:type_name -> mecatl.execution.v1.RequestContext 2, // 6: mecatl.execution.v1.AttachEnvironmentResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef - 22, // 7: mecatl.execution.v1.AttachEnvironmentResponse.grant_expires_at:type_name -> google.protobuf.Timestamp - 4, // 8: mecatl.execution.v1.ReleaseReferenceRequest.context:type_name -> mecatl.execution.v1.RequestContext - 4, // 9: mecatl.execution.v1.RetireEnvironmentRequest.context:type_name -> mecatl.execution.v1.RequestContext - 4, // 10: mecatl.execution.v1.FileRequest.context:type_name -> mecatl.execution.v1.RequestContext - 0, // 11: mecatl.execution.v1.FileRequest.operation:type_name -> mecatl.execution.v1.FileOperation - 22, // 12: mecatl.execution.v1.FileInfo.mod_time:type_name -> google.protobuf.Timestamp - 14, // 13: mecatl.execution.v1.FileResponse.info:type_name -> mecatl.execution.v1.FileInfo - 14, // 14: mecatl.execution.v1.FileResponse.entries:type_name -> mecatl.execution.v1.FileInfo - 15, // 15: mecatl.execution.v1.FileResponse.matches:type_name -> mecatl.execution.v1.GrepMatch - 4, // 16: mecatl.execution.v1.CommandStartRequest.context:type_name -> mecatl.execution.v1.RequestContext - 1, // 17: mecatl.execution.v1.CommandStartResponse.state:type_name -> mecatl.execution.v1.CommandState - 20, // 18: mecatl.execution.v1.CommandStartResponse.result:type_name -> mecatl.execution.v1.CommandStatusResponse - 4, // 19: mecatl.execution.v1.CommandQueryRequest.context:type_name -> mecatl.execution.v1.RequestContext - 1, // 20: mecatl.execution.v1.CommandStatusResponse.state:type_name -> mecatl.execution.v1.CommandState - 5, // 21: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:input_type -> mecatl.execution.v1.ValidateProfileRequest - 7, // 22: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:input_type -> mecatl.execution.v1.EnsureEnvironmentRequest - 9, // 23: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:input_type -> mecatl.execution.v1.AttachEnvironmentRequest - 11, // 24: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:input_type -> mecatl.execution.v1.ReleaseReferenceRequest - 12, // 25: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:input_type -> mecatl.execution.v1.RetireEnvironmentRequest - 13, // 26: mecatl.execution.v1.ExecutionProviderService.Files:input_type -> mecatl.execution.v1.FileRequest - 17, // 27: mecatl.execution.v1.ExecutionProviderService.StartCommand:input_type -> mecatl.execution.v1.CommandStartRequest - 19, // 28: mecatl.execution.v1.ExecutionProviderService.CommandStatus:input_type -> mecatl.execution.v1.CommandQueryRequest - 19, // 29: mecatl.execution.v1.ExecutionProviderService.CancelCommand:input_type -> mecatl.execution.v1.CommandQueryRequest - 6, // 30: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:output_type -> mecatl.execution.v1.ValidateProfileResponse - 8, // 31: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:output_type -> mecatl.execution.v1.EnsureEnvironmentResponse - 10, // 32: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:output_type -> mecatl.execution.v1.AttachEnvironmentResponse - 23, // 33: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:output_type -> google.protobuf.Empty - 23, // 34: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:output_type -> google.protobuf.Empty - 16, // 35: mecatl.execution.v1.ExecutionProviderService.Files:output_type -> mecatl.execution.v1.FileResponse - 18, // 36: mecatl.execution.v1.ExecutionProviderService.StartCommand:output_type -> mecatl.execution.v1.CommandStartResponse - 20, // 37: mecatl.execution.v1.ExecutionProviderService.CommandStatus:output_type -> mecatl.execution.v1.CommandStatusResponse - 20, // 38: mecatl.execution.v1.ExecutionProviderService.CancelCommand:output_type -> mecatl.execution.v1.CommandStatusResponse - 30, // [30:39] is the sub-list for method output_type - 21, // [21:30] is the sub-list for method input_type - 21, // [21:21] is the sub-list for extension type_name - 21, // [21:21] is the sub-list for extension extendee - 0, // [0:21] is the sub-list for field type_name + 38, // 7: mecatl.execution.v1.AttachEnvironmentResponse.grant_expires_at:type_name -> google.protobuf.Timestamp + 2, // 8: mecatl.execution.v1.AcquireRunRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 9: mecatl.execution.v1.AcquireRunRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 10: mecatl.execution.v1.RenewRunRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 11: mecatl.execution.v1.RenewRunRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 12: mecatl.execution.v1.ReleaseRunRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 13: mecatl.execution.v1.ReleaseRunRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 14: mecatl.execution.v1.RunClaimResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 38, // 15: mecatl.execution.v1.RunClaimResponse.expires_at:type_name -> google.protobuf.Timestamp + 2, // 16: mecatl.execution.v1.ReferenceMutationRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 17: mecatl.execution.v1.ReferenceMutationRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 18: mecatl.execution.v1.ReserveSuccessorRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 19: mecatl.execution.v1.ReserveSuccessorRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 20: mecatl.execution.v1.ReferenceReservationResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 21: mecatl.execution.v1.ListReferenceIntentsRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 22: mecatl.execution.v1.ReferenceIntent.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 38, // 23: mecatl.execution.v1.ReferenceIntent.created_at:type_name -> google.protobuf.Timestamp + 3, // 24: mecatl.execution.v1.ReferenceIntent.owner:type_name -> mecatl.execution.v1.Owner + 19, // 25: mecatl.execution.v1.ListReferenceIntentsResponse.intents:type_name -> mecatl.execution.v1.ReferenceIntent + 4, // 26: mecatl.execution.v1.ReleaseReferenceRequest.context:type_name -> mecatl.execution.v1.RequestContext + 2, // 27: mecatl.execution.v1.RetireEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 28: mecatl.execution.v1.RetireEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 29: mecatl.execution.v1.ReplaceExecutorRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 30: mecatl.execution.v1.ReplaceExecutorRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 31: mecatl.execution.v1.RecoverEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 32: mecatl.execution.v1.RecoverEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 33: mecatl.execution.v1.DeleteRetiredEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 34: mecatl.execution.v1.DeleteRetiredEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 35: mecatl.execution.v1.RevokeEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 36: mecatl.execution.v1.RevokeEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 37: mecatl.execution.v1.MigrateEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 38: mecatl.execution.v1.MigrateEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 4, // 39: mecatl.execution.v1.FileRequest.context:type_name -> mecatl.execution.v1.RequestContext + 0, // 40: mecatl.execution.v1.FileRequest.operation:type_name -> mecatl.execution.v1.FileOperation + 38, // 41: mecatl.execution.v1.FileInfo.mod_time:type_name -> google.protobuf.Timestamp + 30, // 42: mecatl.execution.v1.FileResponse.info:type_name -> mecatl.execution.v1.FileInfo + 30, // 43: mecatl.execution.v1.FileResponse.entries:type_name -> mecatl.execution.v1.FileInfo + 31, // 44: mecatl.execution.v1.FileResponse.matches:type_name -> mecatl.execution.v1.GrepMatch + 4, // 45: mecatl.execution.v1.CommandStartRequest.context:type_name -> mecatl.execution.v1.RequestContext + 1, // 46: mecatl.execution.v1.CommandStartResponse.state:type_name -> mecatl.execution.v1.CommandState + 36, // 47: mecatl.execution.v1.CommandStartResponse.result:type_name -> mecatl.execution.v1.CommandStatusResponse + 4, // 48: mecatl.execution.v1.CommandQueryRequest.context:type_name -> mecatl.execution.v1.RequestContext + 1, // 49: mecatl.execution.v1.CommandStatusResponse.state:type_name -> mecatl.execution.v1.CommandState + 5, // 50: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:input_type -> mecatl.execution.v1.ValidateProfileRequest + 7, // 51: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:input_type -> mecatl.execution.v1.EnsureEnvironmentRequest + 9, // 52: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:input_type -> mecatl.execution.v1.AttachEnvironmentRequest + 11, // 53: mecatl.execution.v1.ExecutionProviderService.AcquireRun:input_type -> mecatl.execution.v1.AcquireRunRequest + 12, // 54: mecatl.execution.v1.ExecutionProviderService.RenewRun:input_type -> mecatl.execution.v1.RenewRunRequest + 13, // 55: mecatl.execution.v1.ExecutionProviderService.ReleaseRun:input_type -> mecatl.execution.v1.ReleaseRunRequest + 15, // 56: mecatl.execution.v1.ExecutionProviderService.CommitReference:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 57: mecatl.execution.v1.ExecutionProviderService.AbortReference:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 16, // 58: mecatl.execution.v1.ExecutionProviderService.ReserveSuccessor:input_type -> mecatl.execution.v1.ReserveSuccessorRequest + 15, // 59: mecatl.execution.v1.ExecutionProviderService.PrepareReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 60: mecatl.execution.v1.ExecutionProviderService.ConfirmReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 61: mecatl.execution.v1.ExecutionProviderService.CancelReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 18, // 62: mecatl.execution.v1.ExecutionProviderService.ListReferenceIntents:input_type -> mecatl.execution.v1.ListReferenceIntentsRequest + 21, // 63: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:input_type -> mecatl.execution.v1.ReleaseReferenceRequest + 22, // 64: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:input_type -> mecatl.execution.v1.RetireEnvironmentRequest + 23, // 65: mecatl.execution.v1.ExecutionProviderService.ReplaceExecutor:input_type -> mecatl.execution.v1.ReplaceExecutorRequest + 24, // 66: mecatl.execution.v1.ExecutionProviderService.RecoverEnvironment:input_type -> mecatl.execution.v1.RecoverEnvironmentRequest + 25, // 67: mecatl.execution.v1.ExecutionProviderService.DeleteRetiredEnvironment:input_type -> mecatl.execution.v1.DeleteRetiredEnvironmentRequest + 28, // 68: mecatl.execution.v1.ExecutionProviderService.MigrateEnvironment:input_type -> mecatl.execution.v1.MigrateEnvironmentRequest + 26, // 69: mecatl.execution.v1.ExecutionProviderService.RevokeEnvironment:input_type -> mecatl.execution.v1.RevokeEnvironmentRequest + 29, // 70: mecatl.execution.v1.ExecutionProviderService.Files:input_type -> mecatl.execution.v1.FileRequest + 33, // 71: mecatl.execution.v1.ExecutionProviderService.StartCommand:input_type -> mecatl.execution.v1.CommandStartRequest + 35, // 72: mecatl.execution.v1.ExecutionProviderService.CommandStatus:input_type -> mecatl.execution.v1.CommandQueryRequest + 35, // 73: mecatl.execution.v1.ExecutionProviderService.CancelCommand:input_type -> mecatl.execution.v1.CommandQueryRequest + 6, // 74: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:output_type -> mecatl.execution.v1.ValidateProfileResponse + 8, // 75: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:output_type -> mecatl.execution.v1.EnsureEnvironmentResponse + 10, // 76: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:output_type -> mecatl.execution.v1.AttachEnvironmentResponse + 14, // 77: mecatl.execution.v1.ExecutionProviderService.AcquireRun:output_type -> mecatl.execution.v1.RunClaimResponse + 14, // 78: mecatl.execution.v1.ExecutionProviderService.RenewRun:output_type -> mecatl.execution.v1.RunClaimResponse + 39, // 79: mecatl.execution.v1.ExecutionProviderService.ReleaseRun:output_type -> google.protobuf.Empty + 39, // 80: mecatl.execution.v1.ExecutionProviderService.CommitReference:output_type -> google.protobuf.Empty + 39, // 81: mecatl.execution.v1.ExecutionProviderService.AbortReference:output_type -> google.protobuf.Empty + 17, // 82: mecatl.execution.v1.ExecutionProviderService.ReserveSuccessor:output_type -> mecatl.execution.v1.ReferenceReservationResponse + 39, // 83: mecatl.execution.v1.ExecutionProviderService.PrepareReferenceDelete:output_type -> google.protobuf.Empty + 39, // 84: mecatl.execution.v1.ExecutionProviderService.ConfirmReferenceDelete:output_type -> google.protobuf.Empty + 39, // 85: mecatl.execution.v1.ExecutionProviderService.CancelReferenceDelete:output_type -> google.protobuf.Empty + 20, // 86: mecatl.execution.v1.ExecutionProviderService.ListReferenceIntents:output_type -> mecatl.execution.v1.ListReferenceIntentsResponse + 39, // 87: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:output_type -> google.protobuf.Empty + 39, // 88: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:output_type -> google.protobuf.Empty + 39, // 89: mecatl.execution.v1.ExecutionProviderService.ReplaceExecutor:output_type -> google.protobuf.Empty + 39, // 90: mecatl.execution.v1.ExecutionProviderService.RecoverEnvironment:output_type -> google.protobuf.Empty + 39, // 91: mecatl.execution.v1.ExecutionProviderService.DeleteRetiredEnvironment:output_type -> google.protobuf.Empty + 39, // 92: mecatl.execution.v1.ExecutionProviderService.MigrateEnvironment:output_type -> google.protobuf.Empty + 27, // 93: mecatl.execution.v1.ExecutionProviderService.RevokeEnvironment:output_type -> mecatl.execution.v1.RevokeEnvironmentResponse + 32, // 94: mecatl.execution.v1.ExecutionProviderService.Files:output_type -> mecatl.execution.v1.FileResponse + 34, // 95: mecatl.execution.v1.ExecutionProviderService.StartCommand:output_type -> mecatl.execution.v1.CommandStartResponse + 36, // 96: mecatl.execution.v1.ExecutionProviderService.CommandStatus:output_type -> mecatl.execution.v1.CommandStatusResponse + 36, // 97: mecatl.execution.v1.ExecutionProviderService.CancelCommand:output_type -> mecatl.execution.v1.CommandStatusResponse + 74, // [74:98] is the sub-list for method output_type + 50, // [50:74] is the sub-list for method input_type + 50, // [50:50] is the sub-list for extension type_name + 50, // [50:50] is the sub-list for extension extendee + 0, // [0:50] is the sub-list for field type_name } func init() { file_mecatl_execution_v1_execution_proto_init() } @@ -1914,7 +3632,7 @@ func file_mecatl_execution_v1_execution_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_mecatl_execution_v1_execution_proto_rawDesc), len(file_mecatl_execution_v1_execution_proto_rawDesc)), NumEnums: 2, - NumMessages: 20, + NumMessages: 36, NumExtensions: 0, NumServices: 1, }, diff --git a/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go b/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go index e7826e1923..abfd835935 100644 --- a/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go +++ b/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go @@ -26,15 +26,30 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - ExecutionProviderService_ValidateProfile_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ValidateProfile" - ExecutionProviderService_EnsureEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/EnsureEnvironment" - ExecutionProviderService_AttachEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/AttachEnvironment" - ExecutionProviderService_ReleaseReference_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReleaseReference" - ExecutionProviderService_RetireEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RetireEnvironment" - ExecutionProviderService_Files_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/Files" - ExecutionProviderService_StartCommand_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/StartCommand" - ExecutionProviderService_CommandStatus_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CommandStatus" - ExecutionProviderService_CancelCommand_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CancelCommand" + ExecutionProviderService_ValidateProfile_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ValidateProfile" + ExecutionProviderService_EnsureEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/EnsureEnvironment" + ExecutionProviderService_AttachEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/AttachEnvironment" + ExecutionProviderService_AcquireRun_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/AcquireRun" + ExecutionProviderService_RenewRun_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RenewRun" + ExecutionProviderService_ReleaseRun_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReleaseRun" + ExecutionProviderService_CommitReference_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CommitReference" + ExecutionProviderService_AbortReference_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/AbortReference" + ExecutionProviderService_ReserveSuccessor_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReserveSuccessor" + ExecutionProviderService_PrepareReferenceDelete_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/PrepareReferenceDelete" + ExecutionProviderService_ConfirmReferenceDelete_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ConfirmReferenceDelete" + ExecutionProviderService_CancelReferenceDelete_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CancelReferenceDelete" + ExecutionProviderService_ListReferenceIntents_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ListReferenceIntents" + ExecutionProviderService_ReleaseReference_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReleaseReference" + ExecutionProviderService_RetireEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RetireEnvironment" + ExecutionProviderService_ReplaceExecutor_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReplaceExecutor" + ExecutionProviderService_RecoverEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RecoverEnvironment" + ExecutionProviderService_DeleteRetiredEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/DeleteRetiredEnvironment" + ExecutionProviderService_MigrateEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/MigrateEnvironment" + ExecutionProviderService_RevokeEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RevokeEnvironment" + ExecutionProviderService_Files_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/Files" + ExecutionProviderService_StartCommand_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/StartCommand" + ExecutionProviderService_CommandStatus_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CommandStatus" + ExecutionProviderService_CancelCommand_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CancelCommand" ) // ExecutionProviderServiceClient is the client API for ExecutionProviderService service. @@ -44,8 +59,23 @@ type ExecutionProviderServiceClient interface { ValidateProfile(ctx context.Context, in *ValidateProfileRequest, opts ...grpc.CallOption) (*ValidateProfileResponse, error) EnsureEnvironment(ctx context.Context, in *EnsureEnvironmentRequest, opts ...grpc.CallOption) (*EnsureEnvironmentResponse, error) AttachEnvironment(ctx context.Context, in *AttachEnvironmentRequest, opts ...grpc.CallOption) (*AttachEnvironmentResponse, error) + AcquireRun(ctx context.Context, in *AcquireRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) + RenewRun(ctx context.Context, in *RenewRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) + ReleaseRun(ctx context.Context, in *ReleaseRunRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + CommitReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + AbortReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ReserveSuccessor(ctx context.Context, in *ReserveSuccessorRequest, opts ...grpc.CallOption) (*ReferenceReservationResponse, error) + PrepareReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ConfirmReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + CancelReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ListReferenceIntents(ctx context.Context, in *ListReferenceIntentsRequest, opts ...grpc.CallOption) (*ListReferenceIntentsResponse, error) ReleaseReference(ctx context.Context, in *ReleaseReferenceRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) RetireEnvironment(ctx context.Context, in *RetireEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ReplaceExecutor(ctx context.Context, in *ReplaceExecutorRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + RecoverEnvironment(ctx context.Context, in *RecoverEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + DeleteRetiredEnvironment(ctx context.Context, in *DeleteRetiredEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + MigrateEnvironment(ctx context.Context, in *MigrateEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + RevokeEnvironment(ctx context.Context, in *RevokeEnvironmentRequest, opts ...grpc.CallOption) (*RevokeEnvironmentResponse, error) Files(ctx context.Context, in *FileRequest, opts ...grpc.CallOption) (*FileResponse, error) StartCommand(ctx context.Context, in *CommandStartRequest, opts ...grpc.CallOption) (*CommandStartResponse, error) CommandStatus(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) @@ -90,6 +120,106 @@ func (c *executionProviderServiceClient) AttachEnvironment(ctx context.Context, return out, nil } +func (c *executionProviderServiceClient) AcquireRun(ctx context.Context, in *AcquireRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RunClaimResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_AcquireRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RenewRun(ctx context.Context, in *RenewRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RunClaimResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_RenewRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ReleaseRun(ctx context.Context, in *ReleaseRunRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReleaseRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CommitReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_CommitReference_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) AbortReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_AbortReference_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ReserveSuccessor(ctx context.Context, in *ReserveSuccessorRequest, opts ...grpc.CallOption) (*ReferenceReservationResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReferenceReservationResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReserveSuccessor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) PrepareReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_PrepareReferenceDelete_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ConfirmReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ConfirmReferenceDelete_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CancelReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_CancelReferenceDelete_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ListReferenceIntents(ctx context.Context, in *ListReferenceIntentsRequest, opts ...grpc.CallOption) (*ListReferenceIntentsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListReferenceIntentsResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_ListReferenceIntents_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *executionProviderServiceClient) ReleaseReference(ctx context.Context, in *ReleaseReferenceRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(emptypb.Empty) @@ -110,6 +240,56 @@ func (c *executionProviderServiceClient) RetireEnvironment(ctx context.Context, return out, nil } +func (c *executionProviderServiceClient) ReplaceExecutor(ctx context.Context, in *ReplaceExecutorRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReplaceExecutor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RecoverEnvironment(ctx context.Context, in *RecoverEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_RecoverEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) DeleteRetiredEnvironment(ctx context.Context, in *DeleteRetiredEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_DeleteRetiredEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) MigrateEnvironment(ctx context.Context, in *MigrateEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_MigrateEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RevokeEnvironment(ctx context.Context, in *RevokeEnvironmentRequest, opts ...grpc.CallOption) (*RevokeEnvironmentResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RevokeEnvironmentResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_RevokeEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *executionProviderServiceClient) Files(ctx context.Context, in *FileRequest, opts ...grpc.CallOption) (*FileResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(FileResponse) @@ -157,8 +337,23 @@ type ExecutionProviderServiceServer interface { ValidateProfile(context.Context, *ValidateProfileRequest) (*ValidateProfileResponse, error) EnsureEnvironment(context.Context, *EnsureEnvironmentRequest) (*EnsureEnvironmentResponse, error) AttachEnvironment(context.Context, *AttachEnvironmentRequest) (*AttachEnvironmentResponse, error) + AcquireRun(context.Context, *AcquireRunRequest) (*RunClaimResponse, error) + RenewRun(context.Context, *RenewRunRequest) (*RunClaimResponse, error) + ReleaseRun(context.Context, *ReleaseRunRequest) (*emptypb.Empty, error) + CommitReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + AbortReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + ReserveSuccessor(context.Context, *ReserveSuccessorRequest) (*ReferenceReservationResponse, error) + PrepareReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + ConfirmReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + CancelReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + ListReferenceIntents(context.Context, *ListReferenceIntentsRequest) (*ListReferenceIntentsResponse, error) ReleaseReference(context.Context, *ReleaseReferenceRequest) (*emptypb.Empty, error) RetireEnvironment(context.Context, *RetireEnvironmentRequest) (*emptypb.Empty, error) + ReplaceExecutor(context.Context, *ReplaceExecutorRequest) (*emptypb.Empty, error) + RecoverEnvironment(context.Context, *RecoverEnvironmentRequest) (*emptypb.Empty, error) + DeleteRetiredEnvironment(context.Context, *DeleteRetiredEnvironmentRequest) (*emptypb.Empty, error) + MigrateEnvironment(context.Context, *MigrateEnvironmentRequest) (*emptypb.Empty, error) + RevokeEnvironment(context.Context, *RevokeEnvironmentRequest) (*RevokeEnvironmentResponse, error) Files(context.Context, *FileRequest) (*FileResponse, error) StartCommand(context.Context, *CommandStartRequest) (*CommandStartResponse, error) CommandStatus(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) @@ -182,12 +377,57 @@ func (UnimplementedExecutionProviderServiceServer) EnsureEnvironment(context.Con func (UnimplementedExecutionProviderServiceServer) AttachEnvironment(context.Context, *AttachEnvironmentRequest) (*AttachEnvironmentResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method AttachEnvironment not implemented") } +func (UnimplementedExecutionProviderServiceServer) AcquireRun(context.Context, *AcquireRunRequest) (*RunClaimResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AcquireRun not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RenewRun(context.Context, *RenewRunRequest) (*RunClaimResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RenewRun not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ReleaseRun(context.Context, *ReleaseRunRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReleaseRun not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CommitReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method CommitReference not implemented") +} +func (UnimplementedExecutionProviderServiceServer) AbortReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method AbortReference not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ReserveSuccessor(context.Context, *ReserveSuccessorRequest) (*ReferenceReservationResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReserveSuccessor not implemented") +} +func (UnimplementedExecutionProviderServiceServer) PrepareReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method PrepareReferenceDelete not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ConfirmReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ConfirmReferenceDelete not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CancelReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method CancelReferenceDelete not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ListReferenceIntents(context.Context, *ListReferenceIntentsRequest) (*ListReferenceIntentsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListReferenceIntents not implemented") +} func (UnimplementedExecutionProviderServiceServer) ReleaseReference(context.Context, *ReleaseReferenceRequest) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method ReleaseReference not implemented") } func (UnimplementedExecutionProviderServiceServer) RetireEnvironment(context.Context, *RetireEnvironmentRequest) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method RetireEnvironment not implemented") } +func (UnimplementedExecutionProviderServiceServer) ReplaceExecutor(context.Context, *ReplaceExecutorRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReplaceExecutor not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RecoverEnvironment(context.Context, *RecoverEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method RecoverEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) DeleteRetiredEnvironment(context.Context, *DeleteRetiredEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteRetiredEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) MigrateEnvironment(context.Context, *MigrateEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method MigrateEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RevokeEnvironment(context.Context, *RevokeEnvironmentRequest) (*RevokeEnvironmentResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RevokeEnvironment not implemented") +} func (UnimplementedExecutionProviderServiceServer) Files(context.Context, *FileRequest) (*FileResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Files not implemented") } @@ -276,6 +516,186 @@ func _ExecutionProviderService_AttachEnvironment_Handler(srv interface{}, ctx co return interceptor(ctx, in, info, handler) } +func _ExecutionProviderService_AcquireRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AcquireRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).AcquireRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_AcquireRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).AcquireRun(ctx, req.(*AcquireRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RenewRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RenewRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RenewRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RenewRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RenewRun(ctx, req.(*RenewRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ReleaseRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReleaseRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReleaseRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReleaseRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReleaseRun(ctx, req.(*ReleaseRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CommitReference_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CommitReference(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CommitReference_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CommitReference(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_AbortReference_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).AbortReference(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_AbortReference_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).AbortReference(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ReserveSuccessor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReserveSuccessorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReserveSuccessor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReserveSuccessor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReserveSuccessor(ctx, req.(*ReserveSuccessorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_PrepareReferenceDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).PrepareReferenceDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_PrepareReferenceDelete_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).PrepareReferenceDelete(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ConfirmReferenceDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ConfirmReferenceDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ConfirmReferenceDelete_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ConfirmReferenceDelete(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CancelReferenceDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CancelReferenceDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CancelReferenceDelete_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CancelReferenceDelete(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ListReferenceIntents_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListReferenceIntentsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ListReferenceIntents(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ListReferenceIntents_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ListReferenceIntents(ctx, req.(*ListReferenceIntentsRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _ExecutionProviderService_ReleaseReference_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ReleaseReferenceRequest) if err := dec(in); err != nil { @@ -312,6 +732,96 @@ func _ExecutionProviderService_RetireEnvironment_Handler(srv interface{}, ctx co return interceptor(ctx, in, info, handler) } +func _ExecutionProviderService_ReplaceExecutor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReplaceExecutorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReplaceExecutor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReplaceExecutor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReplaceExecutor(ctx, req.(*ReplaceExecutorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RecoverEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RecoverEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RecoverEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RecoverEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RecoverEnvironment(ctx, req.(*RecoverEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_DeleteRetiredEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteRetiredEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).DeleteRetiredEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_DeleteRetiredEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).DeleteRetiredEnvironment(ctx, req.(*DeleteRetiredEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_MigrateEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MigrateEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).MigrateEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_MigrateEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).MigrateEnvironment(ctx, req.(*MigrateEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RevokeEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RevokeEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RevokeEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RevokeEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RevokeEnvironment(ctx, req.(*RevokeEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _ExecutionProviderService_Files_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(FileRequest) if err := dec(in); err != nil { @@ -403,6 +913,46 @@ var ExecutionProviderService_ServiceDesc = grpc.ServiceDesc{ MethodName: "AttachEnvironment", Handler: _ExecutionProviderService_AttachEnvironment_Handler, }, + { + MethodName: "AcquireRun", + Handler: _ExecutionProviderService_AcquireRun_Handler, + }, + { + MethodName: "RenewRun", + Handler: _ExecutionProviderService_RenewRun_Handler, + }, + { + MethodName: "ReleaseRun", + Handler: _ExecutionProviderService_ReleaseRun_Handler, + }, + { + MethodName: "CommitReference", + Handler: _ExecutionProviderService_CommitReference_Handler, + }, + { + MethodName: "AbortReference", + Handler: _ExecutionProviderService_AbortReference_Handler, + }, + { + MethodName: "ReserveSuccessor", + Handler: _ExecutionProviderService_ReserveSuccessor_Handler, + }, + { + MethodName: "PrepareReferenceDelete", + Handler: _ExecutionProviderService_PrepareReferenceDelete_Handler, + }, + { + MethodName: "ConfirmReferenceDelete", + Handler: _ExecutionProviderService_ConfirmReferenceDelete_Handler, + }, + { + MethodName: "CancelReferenceDelete", + Handler: _ExecutionProviderService_CancelReferenceDelete_Handler, + }, + { + MethodName: "ListReferenceIntents", + Handler: _ExecutionProviderService_ListReferenceIntents_Handler, + }, { MethodName: "ReleaseReference", Handler: _ExecutionProviderService_ReleaseReference_Handler, @@ -411,6 +961,26 @@ var ExecutionProviderService_ServiceDesc = grpc.ServiceDesc{ MethodName: "RetireEnvironment", Handler: _ExecutionProviderService_RetireEnvironment_Handler, }, + { + MethodName: "ReplaceExecutor", + Handler: _ExecutionProviderService_ReplaceExecutor_Handler, + }, + { + MethodName: "RecoverEnvironment", + Handler: _ExecutionProviderService_RecoverEnvironment_Handler, + }, + { + MethodName: "DeleteRetiredEnvironment", + Handler: _ExecutionProviderService_DeleteRetiredEnvironment_Handler, + }, + { + MethodName: "MigrateEnvironment", + Handler: _ExecutionProviderService_MigrateEnvironment_Handler, + }, + { + MethodName: "RevokeEnvironment", + Handler: _ExecutionProviderService_RevokeEnvironment_Handler, + }, { MethodName: "Files", Handler: _ExecutionProviderService_Files_Handler, diff --git a/contracts/proto/mecatl/execution/v1/execution.proto b/contracts/proto/mecatl/execution/v1/execution.proto index b7be635a88..c95c0c34d8 100644 --- a/contracts/proto/mecatl/execution/v1/execution.proto +++ b/contracts/proto/mecatl/execution/v1/execution.proto @@ -16,8 +16,23 @@ service ExecutionProviderService { rpc ValidateProfile(ValidateProfileRequest) returns (ValidateProfileResponse); rpc EnsureEnvironment(EnsureEnvironmentRequest) returns (EnsureEnvironmentResponse); rpc AttachEnvironment(AttachEnvironmentRequest) returns (AttachEnvironmentResponse); + rpc AcquireRun(AcquireRunRequest) returns (RunClaimResponse); + rpc RenewRun(RenewRunRequest) returns (RunClaimResponse); + rpc ReleaseRun(ReleaseRunRequest) returns (google.protobuf.Empty); + rpc CommitReference(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc AbortReference(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc ReserveSuccessor(ReserveSuccessorRequest) returns (ReferenceReservationResponse); + rpc PrepareReferenceDelete(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc ConfirmReferenceDelete(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc CancelReferenceDelete(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc ListReferenceIntents(ListReferenceIntentsRequest) returns (ListReferenceIntentsResponse); rpc ReleaseReference(ReleaseReferenceRequest) returns (google.protobuf.Empty); rpc RetireEnvironment(RetireEnvironmentRequest) returns (google.protobuf.Empty); + rpc ReplaceExecutor(ReplaceExecutorRequest) returns (google.protobuf.Empty); + rpc RecoverEnvironment(RecoverEnvironmentRequest) returns (google.protobuf.Empty); + rpc DeleteRetiredEnvironment(DeleteRetiredEnvironmentRequest) returns (google.protobuf.Empty); + rpc MigrateEnvironment(MigrateEnvironmentRequest) returns (google.protobuf.Empty); + rpc RevokeEnvironment(RevokeEnvironmentRequest) returns (RevokeEnvironmentResponse); rpc Files(FileRequest) returns (FileResponse); rpc StartCommand(CommandStartRequest) returns (CommandStartResponse); rpc CommandStatus(CommandQueryRequest) returns (CommandStatusResponse); @@ -32,6 +47,9 @@ message RequestContext { string binding_id = 3; uint64 epoch = 4; string grant = 5; + string run_id = 6; + string claim_id = 7; + uint64 grant_generation = 8; } message ValidateProfileRequest { string profile = 1; } @@ -43,13 +61,14 @@ message ValidateProfileResponse { int64 max_command_bytes = 5; int64 max_command_duration_millis = 6; } -message EnsureEnvironmentRequest { string binding_id = 1; string profile = 2; Owner owner = 3; } +message EnsureEnvironmentRequest { string binding_id = 1; string profile = 2; Owner owner = 3; string operation_id = 4; } message EnsureEnvironmentResponse { EnvironmentRef environment = 1; uint64 epoch = 2; bool ready = 3; string grant = 4; google.protobuf.Timestamp grant_expires_at = 5; + uint64 grant_generation = 6; } message AttachEnvironmentRequest { RequestContext context = 1; string purpose = 2; } message AttachEnvironmentResponse { @@ -58,9 +77,118 @@ message AttachEnvironmentResponse { bool ready = 3; string grant = 4; google.protobuf.Timestamp grant_expires_at = 5; + uint64 grant_generation = 6; } +message AcquireRunRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string run_id = 4; + string operation_id = 5; + int64 ttl_millis = 6; +} +message RenewRunRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string run_id = 4; + string claim_id = 5; + uint64 epoch = 6; + string operation_id = 7; + int64 ttl_millis = 8; + uint64 grant_generation = 9; +} +message ReleaseRunRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string run_id = 4; + string claim_id = 5; + uint64 epoch = 6; + string operation_id = 7; + uint64 grant_generation = 8; +} +message RunClaimResponse { + EnvironmentRef environment = 1; + string binding_id = 2; + string run_id = 3; + string claim_id = 4; + uint64 epoch = 5; + uint64 grant_generation = 6; + string grant = 7; + google.protobuf.Timestamp expires_at = 8; +} +message ReferenceMutationRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string operation_id = 4; +} +message ReserveSuccessorRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string source_binding_id = 3; + string destination_binding_id = 4; + string operation_id = 5; +} +message ReferenceReservationResponse { EnvironmentRef environment = 1; } +message ListReferenceIntentsRequest { Owner owner = 1; int32 limit = 2; } +message ReferenceIntent { + EnvironmentRef environment = 1; + string binding_id = 2; + string state = 3; + string operation_id = 4; + string source_binding_id = 5; + google.protobuf.Timestamp created_at = 6; + Owner owner = 7; +} +message ListReferenceIntentsResponse { repeated ReferenceIntent intents = 1; } message ReleaseReferenceRequest { RequestContext context = 1; } -message RetireEnvironmentRequest { RequestContext context = 1; } +message RetireEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_execution_epoch = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} +message ReplaceExecutorRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_execution_epoch = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} +message RecoverEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_execution_epoch = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} +message DeleteRetiredEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string expected_pvc_uid = 3; + string operation_id = 4; +} +message RevokeEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_grant_generation = 3; +} +message RevokeEnvironmentResponse { uint64 grant_generation = 1; } + +message MigrateEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint32 expected_schema_version = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} enum FileOperation { FILE_OPERATION_UNSPECIFIED = 0; diff --git a/deploy/helm/mecatl-execution/chart_test.go b/deploy/helm/mecatl-execution/chart_test.go index 615c04d0cf..67373f2271 100644 --- a/deploy/helm/mecatl-execution/chart_test.go +++ b/deploy/helm/mecatl-execution/chart_test.go @@ -40,3 +40,31 @@ func TestChartHasNoDeletionHook(t *testing.T) { } } } + +func TestProductionHardeningIsFailClosed(t *testing.T) { + provider, err := os.ReadFile("templates/provider.yaml") + if err != nil { + t.Fatal(err) + } + network, err := os.ReadFile("templates/network-policy.yaml") + if err != nil { + t.Fatal(err) + } + values, err := os.ReadFile("values.schema.json") + if err != nil { + t.Fatal(err) + } + for _, required := range []string{"runAsNonRoot: true", "seccompProfile", "capabilities: {drop: [\"ALL\"]}", "readOnlyRootFilesystem: true", "httpGet: {path: /ready", "security-authority-configmap", "projected:", "maxUnavailable: 0"} { + if !strings.Contains(string(provider), required) { + t.Fatalf("provider hardening missing %q", required) + } + } + for _, required := range []string{"workload-default-deny", "ingress: []", "egress: []", "ResourceQuota", "LimitRange", "count/executionenvironments.execution.mecatl.dev"} { + if !strings.Contains(string(network), required) { + t.Fatalf("network/resource hardening missing %q", required) + } + } + if strings.Contains(string(network), "0.0.0.0/0") || !strings.Contains(string(values), `"minimum":2`) { + t.Fatal("chart permits an unsafe replica or egress default") + } +} diff --git a/deploy/helm/mecatl-execution/crds/executionenvironment.yaml b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml index b56b180926..296505a7cd 100644 --- a/deploy/helm/mecatl-execution/crds/executionenvironment.yaml +++ b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml @@ -32,6 +32,10 @@ spec: message: revision is immutable - rule: "!has(oldSelf.ownerHash) || self.ownerHash == oldSelf.ownerHash" message: ownerHash is immutable + - rule: "!has(oldSelf.ownerIssuer) || self.ownerIssuer == oldSelf.ownerIssuer" + message: ownerIssuer is immutable + - rule: "!has(oldSelf.ownerSubject) || self.ownerSubject == oldSelf.ownerSubject" + message: ownerSubject is immutable - rule: "!has(oldSelf.clientHash) || self.clientHash == oldSelf.clientHash" message: clientHash is immutable - rule: "!has(oldSelf.bindingID) || self.bindingID == oldSelf.bindingID" @@ -52,9 +56,12 @@ spec: message: resources are immutable required: [allocationID, revision, ownerHash, clientHash, bindingID, requestFingerprint, profile, profileDigest, image, storageClass, storageSize, resources, desired] properties: + schemaVersion: {type: integer, format: int64, enum: [0, 1, 2]} allocationID: {type: string, minLength: 1, maxLength: 63} revision: {type: string, minLength: 1, maxLength: 64} ownerHash: {type: string, pattern: "^[0-9a-f]{64}$"} + ownerIssuer: {type: string, minLength: 1, maxLength: 1024} + ownerSubject: {type: string, minLength: 1, maxLength: 1024} clientHash: {type: string, pattern: "^[0-9a-f]{64}$"} bindingID: {type: string, minLength: 1, maxLength: 253} requestFingerprint: {type: string, pattern: "^[0-9a-f]{64}$"} @@ -76,14 +83,25 @@ spec: status: type: object properties: + schemaVersion: {type: integer, format: int64, enum: [0, 1, 2]} observedGeneration: {type: integer, format: int64} epoch: {type: integer, format: int64, minimum: 1} fenceState: {type: string, enum: [Healthy, FenceUnknown]} + grantGeneration: {type: integer, format: int64, minimum: 1} references: type: array maxItems: 64 - x-kubernetes-list-type: set - items: {type: string, maxLength: 253} + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [bindingID] + items: + type: object + required: [bindingID, state, operationID, createdAt] + properties: + bindingID: {type: string, minLength: 1, maxLength: 253} + state: {type: string, enum: [PendingCreate, Published, PendingDelete]} + operationID: {type: string, minLength: 1, maxLength: 1024} + sourceBindingID: {type: string, maxLength: 253} + createdAt: {type: string, format: date-time} pvc: type: object properties: @@ -94,13 +112,62 @@ spec: properties: name: {type: string} uid: {type: string} + activeRun: + type: object + required: [bindingID, runID, claimID, operationID, ownerHash, clientHash, epoch, expiresAt] + properties: + bindingID: {type: string, maxLength: 253} + runID: {type: string, maxLength: 1024} + claimID: {type: string, maxLength: 1024} + operationID: {type: string, maxLength: 1024} + ownerHash: {type: string, pattern: "^[0-9a-f]{64}$"} + clientHash: {type: string, pattern: "^[0-9a-f]{64}$"} + epoch: {type: integer, format: int64, minimum: 1} + expiresAt: {type: string, format: date-time} + lastRenewOperationID: {type: string, maxLength: 1024} activeOperation: type: object - required: [id, operation, startedAt] + required: [id, operation, startedAt, claimID, runID, epoch, holderID, renewedAt, expiresAt] properties: - id: {type: string} - operation: {type: string} + id: {type: string, minLength: 1, maxLength: 1024} + operation: {type: string, minLength: 1, maxLength: 128} startedAt: {type: string, format: date-time} + claimID: {type: string, maxLength: 1024} + runID: {type: string, maxLength: 1024} + epoch: {type: integer, format: int64, minimum: 1} + holderID: {type: string, minLength: 1, maxLength: 1024} + renewedAt: {type: string, format: date-time} + expiresAt: {type: string, format: date-time} + lifecycleOperation: + type: object + required: [id, type, phase, createdAt] + properties: + id: {type: string, minLength: 1, maxLength: 1024} + type: {type: string, enum: [ReplaceExecutor, RetireEnvironment, DeleteRetiredEnvironment]} + phase: {type: string, enum: [Quiescing, WaitingForTermination, RemovingPodFinalizer, WaitingForPodDeletion, CreatingReplacement, DeletingPVC]} + expectedEpoch: {type: integer, format: int64, minimum: 1} + expectedPodUID: {type: string, maxLength: 128} + expectedPVCUID: {type: string, minLength: 1, maxLength: 128} + createdAt: {type: string, format: date-time} + terminationProof: + type: object + required: [operationID, podUID, pvcUID, epoch, podPhase, observedAt] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + podUID: {type: string, minLength: 1, maxLength: 128} + pvcUID: {type: string, minLength: 1, maxLength: 128} + epoch: {type: integer, format: int64, minimum: 1} + podPhase: {type: string, enum: [Succeeded, Failed]} + observedAt: {type: string, format: date-time} + migrationOperation: + type: object + required: [id, fromSchema, expectedPodUID, expectedPVCUID] + properties: + id: {type: string, minLength: 1, maxLength: 1024} + fromSchema: {type: integer, format: int64, enum: [0, 1]} + expectedPodUID: {type: string, minLength: 1, maxLength: 128} + expectedPVCUID: {type: string, minLength: 1, maxLength: 128} + lastMigrationOperationID: {type: string, maxLength: 1024} conditions: type: array items: diff --git a/deploy/helm/mecatl-execution/templates/network-policy.yaml b/deploy/helm/mecatl-execution/templates/network-policy.yaml new file mode 100644 index 0000000000..1cffe177a4 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/network-policy.yaml @@ -0,0 +1,99 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "mecatl-execution.fullname" . }}-provider +spec: + podSelector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + policyTypes: [Ingress, Egress] + ingress: + - from: +{{- range .Values.provider.clientIngressSelectors }} + - namespaceSelector: + matchLabels: +{{ toYaml .namespaceLabels | indent 12 }} + podSelector: + matchLabels: +{{ toYaml .podLabels | indent 12 }} +{{- end }} + ports: + - {protocol: TCP, port: {{ .Values.service.port }}} + egress: +{{- range .Values.provider.apiServerCIDRs }} + - to: [{ipBlock: {cidr: {{ . | quote }}}}] + ports: +{{- range $.Values.networkPolicy.apiServerPorts }} + - {protocol: TCP, port: {{ . }}} +{{- end }} +{{- end }} +{{- range .Values.provider.dnsCIDRs }} + - to: [{ipBlock: {cidr: {{ . | quote }}}}] + ports: +{{- range $.Values.networkPolicy.dnsPorts }} + - {protocol: UDP, port: {{ . }}} + - {protocol: TCP, port: {{ . }}} +{{- end }} +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "mecatl-execution.fullname" . }}-workload-default-deny +spec: + podSelector: + matchExpressions: + - {key: execution.mecatl.dev/environment, operator: Exists} + policyTypes: [Ingress, Egress] + ingress: [] + egress: [] +{{- range $name, $profile := .Values.networkPolicy.workloadProfiles }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "mecatl-execution.fullname" $ }}-profile-{{ sha256sum $name | trunc 16 }} +spec: + podSelector: + matchLabels: + execution.mecatl.dev/profile: {{ sha256sum $name | trunc 16 }} + policyTypes: [Egress] + egress: +{{- range $profile.egress }} + - to: [{ipBlock: {cidr: {{ .cidr | quote }}}}] + ports: +{{- range .ports }} + - {protocol: {{ default "TCP" .protocol }}, port: {{ .port }}} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +--- +{{- if .Values.resourceGovernance.enabled }} +apiVersion: v1 +kind: ResourceQuota +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + hard: + pods: {{ .Values.resourceGovernance.pods | quote }} + persistentvolumeclaims: {{ .Values.resourceGovernance.persistentVolumeClaims | quote }} + count/executionenvironments.execution.mecatl.dev: {{ .Values.resourceGovernance.executionEnvironments | quote }} + requests.cpu: {{ .Values.resourceGovernance.requestsCPU | quote }} + requests.memory: {{ .Values.resourceGovernance.requestsMemory | quote }} + requests.storage: {{ .Values.resourceGovernance.requestsStorage | quote }} + limits.cpu: {{ .Values.resourceGovernance.limitsCPU | quote }} + limits.memory: {{ .Values.resourceGovernance.limitsMemory | quote }} +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + limits: + - type: Container + defaultRequest: {cpu: 50m, memory: 64Mi, ephemeral-storage: 32Mi} + default: {cpu: "1", memory: 1Gi, ephemeral-storage: 1Gi} + - type: PersistentVolumeClaim + max: {storage: {{ .Values.resourceGovernance.requestsStorage | quote }}} +{{- end }} diff --git a/deploy/helm/mecatl-execution/templates/pdb.yaml b/deploy/helm/mecatl-execution/templates/pdb.yaml new file mode 100644 index 0000000000..43ff755726 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/pdb.yaml @@ -0,0 +1,10 @@ +{{- if gt (int .Values.provider.replicas) 1 }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + minAvailable: 1 + selector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} +{{- end }} diff --git a/deploy/helm/mecatl-execution/templates/provider.yaml b/deploy/helm/mecatl-execution/templates/provider.yaml index fb1bed6b5c..caf2e0b02b 100644 --- a/deploy/helm/mecatl-execution/templates/provider.yaml +++ b/deploy/helm/mecatl-execution/templates/provider.yaml @@ -7,12 +7,29 @@ data: profiles: {{ toYaml .Values.profiles | indent 6 }} --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "mecatl-execution.fullname" . }}-security-manifest +data: + manifest.json: | +{{ required "provider.securityManifest is required" .Values.provider.securityManifest | indent 4 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "mecatl-execution.fullname" . }}-security-authority +data: {} +--- apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "mecatl-execution.fullname" . }} spec: - replicas: 1 + replicas: {{ .Values.provider.replicas }} + strategy: + type: RollingUpdate + rollingUpdate: {maxUnavailable: 0, maxSurge: 1} selector: matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} template: @@ -20,6 +37,13 @@ spec: labels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} spec: serviceAccountName: {{ include "mecatl-execution.fullname" . }} + automountServiceAccountToken: true + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} securityContext: runAsNonRoot: true runAsUser: 65532 @@ -32,45 +56,47 @@ spec: args: - --namespace={{ .Release.Namespace }} - --profiles=/etc/mecatl-execution/profiles.yaml - - --tls-cert=/etc/mecatl-execution/tls/tls.crt - - --tls-key=/etc/mecatl-execution/tls/tls.key - - --client-ca=/etc/mecatl-execution/tls/ca.crt - - --grant-signing-key=/etc/mecatl-execution/grant/key.pem - - --grant-key-id={{ .Values.provider.grantKeyID }} - - --client-uri-san={{ join "," .Values.provider.clientURISANs }} - - --owner-attester-uri-san={{ join "," .Values.provider.ownerAttesterURISANs }} - - --administrator-uri-san={{ join "," .Values.provider.administratorURISANs }} + - --grant-keyring-manifest=/etc/mecatl-execution/security/manifest.json + - --grant-key-directory=/etc/mecatl-execution/security + - --security-authority-configmap={{ include "mecatl-execution.fullname" . }}-security-authority + - --security-reload-interval={{ .Values.provider.securityReloadInterval }} + - --max-concurrent-streams={{ .Values.provider.maxConcurrentStreams }} + - --max-concurrent-rpcs={{ .Values.provider.maxConcurrentRPCs }} + - --max-concurrent-rpcs-per-client={{ .Values.provider.maxConcurrentRPCsPerClient }} + - --health-listen=:{{ .Values.provider.healthPort }} ports: - {name: grpc, containerPort: 8443} + - {name: health, containerPort: {{ .Values.provider.healthPort }}} startupProbe: - tcpSocket: {port: grpc} + httpGet: {path: /ready, port: health} failureThreshold: 30 periodSeconds: 2 readinessProbe: - tcpSocket: {port: grpc} + httpGet: {path: /ready, port: health} periodSeconds: 5 livenessProbe: - tcpSocket: {port: grpc} + httpGet: {path: /live, port: health} periodSeconds: 10 securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true capabilities: {drop: ["ALL"]} resources: - requests: {cpu: 100m, memory: 128Mi} - limits: {cpu: "1", memory: 512Mi} +{{ toYaml .Values.provider.resources | indent 10 }} volumeMounts: - {name: profiles, mountPath: /etc/mecatl-execution/profiles.yaml, subPath: profiles.yaml, readOnly: true} - - {name: tls, mountPath: /etc/mecatl-execution/tls, readOnly: true} - - {name: grant, mountPath: /etc/mecatl-execution/grant, readOnly: true} + - {name: security, mountPath: /etc/mecatl-execution/security, readOnly: true} - {name: tmp, mountPath: /tmp} volumes: - name: profiles configMap: {name: {{ include "mecatl-execution.fullname" . }}-profiles} - - name: tls - secret: {secretName: {{ required "provider.tlsSecretName is required" .Values.provider.tlsSecretName }}} - - name: grant - secret: {secretName: {{ required "provider.grantSigningSecretName is required" .Values.provider.grantSigningSecretName }}} + - name: security + projected: + sources: + - configMap: + name: {{ include "mecatl-execution.fullname" . }}-security-manifest + - secret: + name: {{ required "provider.securitySecretName is required" .Values.provider.securitySecretName }} - name: tmp emptyDir: {sizeLimit: 64Mi} --- diff --git a/deploy/helm/mecatl-execution/templates/rbac.yaml b/deploy/helm/mecatl-execution/templates/rbac.yaml index 7c561c8365..d1687be383 100644 --- a/deploy/helm/mecatl-execution/templates/rbac.yaml +++ b/deploy/helm/mecatl-execution/templates/rbac.yaml @@ -10,16 +10,20 @@ metadata: rules: - apiGroups: ["execution.mecatl.dev"] resources: ["executionenvironments"] - verbs: ["get", "list", "watch", "create", "update", "patch"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: ["execution.mecatl.dev"] resources: ["executionenvironments/status"] verbs: ["get", "update", "patch"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: [{{ printf "%s-security-authority" (include "mecatl-execution.fullname" .) | quote }}] + verbs: ["get", "update"] - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "create", "update", "patch"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["pods/exec"] verbs: ["create"] diff --git a/deploy/helm/mecatl-execution/values.schema.json b/deploy/helm/mecatl-execution/values.schema.json index adcc877382..2d2fa3fcd3 100644 --- a/deploy/helm/mecatl-execution/values.schema.json +++ b/deploy/helm/mecatl-execution/values.schema.json @@ -2,11 +2,32 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "additionalProperties": false, - "required": ["provider", "service", "profiles"], + "required": ["provider", "service", "profiles", "networkPolicy", "resourceGovernance"], "properties": { "fullnameOverride": {"type":"string"}, - "provider": {"type":"object","additionalProperties":false,"required":["image","clientURISANs","ownerAttesterURISANs","tlsSecretName","grantSigningSecretName"],"properties":{"image":{"type":"string","pattern":"@sha256:[0-9a-fA-F]{64}$"},"imagePullPolicy":{"enum":["Always","IfNotPresent","Never"]},"replicas":{"const":1},"clientURISANs":{"type":"array","minItems":1,"uniqueItems":true,"items":{"type":"string","pattern":"^[a-z][a-z0-9+.-]*://"}},"ownerAttesterURISANs":{"type":"array","minItems":1,"uniqueItems":true,"items":{"type":"string"}},"administratorURISANs":{"type":"array","uniqueItems":true,"items":{"type":"string"}},"tlsSecretName":{"type":"string","minLength":1},"grantSigningSecretName":{"type":"string","minLength":1},"grantKeyID":{"type":"string","minLength":1}}}, + "provider": { + "type":"object", "additionalProperties":false, + "required":["image","replicas","securitySecretName","securityManifest","maxConcurrentStreams","maxConcurrentRPCs","maxConcurrentRPCsPerClient","apiServerCIDRs","dnsCIDRs","resources"], + "properties": { + "image":{"type":"string","pattern":"@sha256:[0-9a-fA-F]{64}$"}, + "imagePullPolicy":{"enum":["Always","IfNotPresent","Never"]}, + "replicas":{"type":"integer","minimum":2,"maximum":32}, + "securitySecretName":{"type":"string","minLength":1}, + "securityManifest":{"type":"string","minLength":2}, + "securityReloadInterval":{"type":"string","pattern":"^[1-9][0-9]*(ms|s|m)$"}, + "maxConcurrentStreams":{"type":"integer","minimum":1,"maximum":1024}, + "maxConcurrentRPCs":{"type":"integer","minimum":1,"maximum":4096}, + "maxConcurrentRPCsPerClient":{"type":"integer","minimum":1,"maximum":4096}, + "healthPort":{"type":"integer","minimum":1,"maximum":65535}, + "clientIngressSelectors":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["namespaceLabels","podLabels"],"properties":{"namespaceLabels":{"type":"object","minProperties":1,"additionalProperties":{"type":"string"}},"podLabels":{"type":"object","minProperties":1,"additionalProperties":{"type":"string"}}}}}, + "apiServerCIDRs":{"type":"array","minItems":1,"uniqueItems":true,"items":{"type":"string","pattern":"^[0-9a-fA-F:.]+/[0-9]+$"}}, + "dnsCIDRs":{"type":"array","minItems":1,"uniqueItems":true,"items":{"type":"string","pattern":"^[0-9a-fA-F:.]+/[0-9]+$"}}, + "resources":{"type":"object","required":["requests","limits"],"properties":{"requests":{"type":"object","required":["cpu","memory","ephemeral-storage"]},"limits":{"type":"object","required":["cpu","memory","ephemeral-storage"]}}} + } + }, "service":{"type":"object","additionalProperties":false,"required":["port"],"properties":{"port":{"type":"integer","minimum":1,"maximum":65535}}}, - "profiles":{"type":"object","minProperties":1,"additionalProperties":{"type":"object","additionalProperties":false,"required":["image","storageClass","storageSize","cpuRequest","memoryRequest","cpuLimit","memoryLimit","maxFileBytes","maxCommandBytes","maxCommandDuration"],"properties":{"image":{"type":"string","pattern":"@sha256:[0-9a-fA-F]{64}$"},"storageClass":{"type":"string","minLength":1},"storageSize":{"type":"string","minLength":1},"cpuRequest":{"type":"string","minLength":1},"memoryRequest":{"type":"string","minLength":1},"cpuLimit":{"type":"string","minLength":1},"memoryLimit":{"type":"string","minLength":1},"maxFileBytes":{"type":"integer","minimum":1,"maximum":5242880},"maxCommandBytes":{"type":"integer","minimum":1,"maximum":1048576},"maxCommandDuration":{"type":"string","pattern":"^[1-9][0-9]*(ms|s|m)$"}}}} + "networkPolicy":{"type":"object","additionalProperties":false,"required":["enabled","dnsPorts","apiServerPorts","workloadProfiles"],"properties":{"enabled":{"const":true},"dnsPorts":{"type":"array","minItems":1,"items":{"type":"integer","minimum":1,"maximum":65535}},"apiServerPorts":{"type":"array","minItems":1,"items":{"type":"integer","minimum":1,"maximum":65535}},"workloadProfiles":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"required":["egress"],"properties":{"egress":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["cidr","ports"],"properties":{"cidr":{"type":"string","pattern":"^[0-9a-fA-F:.]+/[0-9]+$"},"ports":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["port"],"properties":{"port":{"type":"integer","minimum":1,"maximum":65535},"protocol":{"enum":["TCP","UDP"]}}}}}}}}}}}}, + "resourceGovernance":{"type":"object","additionalProperties":false,"required":["enabled","pods","persistentVolumeClaims","executionEnvironments","requestsCPU","requestsMemory","requestsStorage","limitsCPU","limitsMemory"],"properties":{"enabled":{"const":true},"pods":{"type":"string","minLength":1},"persistentVolumeClaims":{"type":"string","minLength":1},"executionEnvironments":{"type":"string","minLength":1},"requestsCPU":{"type":"string","minLength":1},"requestsMemory":{"type":"string","minLength":1},"requestsStorage":{"type":"string","minLength":1},"limitsCPU":{"type":"string","minLength":1},"limitsMemory":{"type":"string","minLength":1}}}, + "profiles":{"type":"object","minProperties":1,"additionalProperties":{"type":"object","additionalProperties":false,"required":["image","storageClass","storageSize","cpuRequest","memoryRequest","cpuLimit","memoryLimit","ephemeralStorageRequest","ephemeralStorageLimit","tmpSizeLimit","runtimeClassName","maxFileBytes","maxCommandBytes","maxCommandDuration","maxEnvironments"],"properties":{"image":{"type":"string","pattern":"@sha256:[0-9a-fA-F]{64}$"},"storageClass":{"type":"string","minLength":1},"storageSize":{"type":"string","minLength":1},"cpuRequest":{"type":"string","minLength":1},"memoryRequest":{"type":"string","minLength":1},"cpuLimit":{"type":"string","minLength":1},"memoryLimit":{"type":"string","minLength":1},"ephemeralStorageRequest":{"type":"string","minLength":1},"ephemeralStorageLimit":{"type":"string","minLength":1},"tmpSizeLimit":{"type":"string","minLength":1},"runtimeClassName":{"type":"string","minLength":1},"maxFileBytes":{"type":"integer","minimum":1,"maximum":5242880},"maxCommandBytes":{"type":"integer","minimum":1,"maximum":1048576},"maxCommandDuration":{"type":"string","pattern":"^[1-9][0-9]*(ms|s|m)$"},"maxEnvironments":{"type":"integer","minimum":1,"maximum":10000}}}} } } diff --git a/deploy/helm/mecatl-execution/values.yaml b/deploy/helm/mecatl-execution/values.yaml index bb00fe83a6..c25e896942 100644 --- a/deploy/helm/mecatl-execution/values.yaml +++ b/deploy/helm/mecatl-execution/values.yaml @@ -2,16 +2,40 @@ fullnameOverride: "" provider: image: "" imagePullPolicy: IfNotPresent - replicas: 1 - clientURISANs: [] - ownerAttesterURISANs: [] - administratorURISANs: [] - tlsSecretName: "" - grantSigningSecretName: "" - grantKeyID: current + replicas: 2 + securitySecretName: "" + securityManifest: "" + securityReloadInterval: 2s + maxConcurrentStreams: 64 + maxConcurrentRPCs: 128 + maxConcurrentRPCsPerClient: 32 + healthPort: 8081 + clientIngressSelectors: [] + apiServerCIDRs: [] + dnsCIDRs: [] + resources: + requests: {cpu: 100m, memory: 128Mi, ephemeral-storage: 64Mi} + limits: {cpu: "1", memory: 512Mi, ephemeral-storage: 256Mi} + # Deprecated single-key/TLS settings are intentionally unsupported. service: port: 8443 +networkPolicy: + enabled: true + dnsPorts: [53] + apiServerPorts: [443] + workloadProfiles: {} +resourceGovernance: + enabled: true + pods: "200" + persistentVolumeClaims: "100" + executionEnvironments: "100" + requestsCPU: "20" + requestsMemory: 40Gi + requestsStorage: 500Gi + limitsCPU: "40" + limitsMemory: 80Gi profiles: {} + # Example profile (all fields are required): # profiles: # go: @@ -22,6 +46,11 @@ profiles: {} # memoryRequest: 128Mi # cpuLimit: "1" # memoryLimit: 1Gi +# ephemeralStorageRequest: 64Mi +# ephemeralStorageLimit: 1Gi +# tmpSizeLimit: 256Mi +# runtimeClassName: gvisor # maxFileBytes: 5242880 # maxCommandBytes: 1048576 # maxCommandDuration: 5m +# maxEnvironments: 100 diff --git a/e2e/k8s_execution/qualification_test.go b/e2e/k8s_execution/qualification_test.go index 8be2aa0b8c..3898f2a205 100644 --- a/e2e/k8s_execution/qualification_test.go +++ b/e2e/k8s_execution/qualification_test.go @@ -103,7 +103,7 @@ func TestKindExecutionQualification(t *testing.T) { return err }, "environment retire": func() error { - return intruder.RetireEnvironment(ctx, executionenv.RetireEnvironmentRequest{Context: rc}) + return intruder.RetireEnvironment(ctx, executionenv.RetireEnvironmentRequest{Environment: rc.Environment, Owner: rc.Owner, ExpectedEpoch: rc.Epoch, ExpectedPodUID: "wrong-pod", ExpectedPVCUID: "wrong-pvc", OperationID: "unauthorized-retire"}) }, } { if err := call(); err == nil || (!isRemoteCode(err, executionenv.CodePermissionDenied) && !isRemoteCode(err, executionenv.CodeUnauthenticated)) { diff --git a/internal/adapter/executionclient/client.go b/internal/adapter/executionclient/client.go index e39b01774a..be625a258e 100644 --- a/internal/adapter/executionclient/client.go +++ b/internal/adapter/executionclient/client.go @@ -1,11 +1,15 @@ // Package executionclient implements the private mTLS client and placement adapter // for the independently deployed Kubernetes execution provider. +// +//nolint:revive // Private protocol methods are intentionally explicit across adapter boundaries. package executionclient import ( "context" + "crypto/rand" "crypto/tls" "crypto/x509" + "encoding/hex" "errors" "fmt" "io/fs" @@ -19,6 +23,7 @@ import ( "google.golang.org/grpc/codes" grpccredentials "google.golang.org/grpc/credentials" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" "github.com/stacklok/mecatl/engine/adapter/memledger" @@ -106,9 +111,21 @@ func (c *Client) ValidateProfile(ctx context.Context, profile string) (execution return executionenv.ValidateProfileResponse{Profile: v.Profile, Digest: v.Digest, Capabilities: v.Capabilities, MaxFileBytes: v.MaxFileBytes, MaxCommandBytes: v.MaxCommandBytes, MaxCommandDurationMillis: v.MaxCommandDurationMillis}, nil } +func newOperationID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + // Ensure idempotently allocates or resolves the environment for a binding. -func (c *Client) Ensure(ctx context.Context, binding, profile string, owner executionenv.Owner) (executionenv.EnsureEnvironmentResponse, error) { - v, err := c.rpc.EnsureEnvironment(ctx, &executionv1.EnsureEnvironmentRequest{BindingId: binding, Profile: profile, Owner: ownerToProto(owner)}) +func (c *Client) Ensure(ctx context.Context, binding, profile string, owner executionenv.Owner, operationIDs ...string) (executionenv.EnsureEnvironmentResponse, error) { + operationID := "" + if len(operationIDs) == 1 { + operationID = operationIDs[0] + } + v, err := c.rpc.EnsureEnvironment(ctx, &executionv1.EnsureEnvironmentRequest{BindingId: binding, Profile: profile, Owner: ownerToProto(owner), OperationId: operationID}) if err != nil { return executionenv.EnsureEnvironmentResponse{}, decodeError(ctx, err) } @@ -124,6 +141,86 @@ func (c *Client) Attach(ctx context.Context, req executionenv.AttachEnvironmentR return attachFromProto(v) } +// AcquireRun acquires one environment-wide run claim. +func (c *Client) AcquireRun(ctx context.Context, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + v, err := c.rpc.AcquireRun(ctx, &executionv1.AcquireRunRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, RunId: req.RunID, OperationId: req.OperationID, TtlMillis: req.TTL.Milliseconds()}) + if err != nil { + return executionenv.RunClaim{}, decodeError(ctx, err) + } + return runClaimFromProto(v) +} +func (c *Client) RenewRun(ctx context.Context, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + v, err := c.rpc.RenewRun(ctx, &executionv1.RenewRunRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, RunId: req.RunID, ClaimId: req.ClaimID, Epoch: req.Epoch, GrantGeneration: req.GrantGeneration, OperationId: req.OperationID, TtlMillis: req.TTL.Milliseconds()}) + if err != nil { + return executionenv.RunClaim{}, decodeError(ctx, err) + } + return runClaimFromProto(v) +} +func (c *Client) ReleaseRun(ctx context.Context, req executionenv.RunClaimRequest) error { + _, err := c.rpc.ReleaseRun(ctx, &executionv1.ReleaseRunRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, RunId: req.RunID, ClaimId: req.ClaimID, Epoch: req.Epoch, GrantGeneration: req.GrantGeneration, OperationId: req.OperationID}) + return decodeError(ctx, err) +} +func (c *Client) referenceMutation(ctx context.Context, req executionenv.ReferenceRequest, call func(context.Context, *executionv1.ReferenceMutationRequest, ...grpc.CallOption) (*emptypb.Empty, error)) error { + _, err := call(ctx, &executionv1.ReferenceMutationRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} +func (c *Client) CommitReference(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.CommitReference) +} +func (c *Client) AbortReference(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.AbortReference) +} +func (c *Client) PrepareReferenceDelete(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.PrepareReferenceDelete) +} +func (c *Client) ConfirmReferenceDelete(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.ConfirmReferenceDelete) +} +func (c *Client) CancelReferenceDelete(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.CancelReferenceDelete) +} +func (c *Client) ReserveSuccessor(ctx context.Context, req executionenv.ReferenceRequest) error { + _, err := c.rpc.ReserveSuccessor(ctx, &executionv1.ReserveSuccessorRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), SourceBindingId: req.SourceBindingID, DestinationBindingId: req.BindingID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} + +func (c *Client) ListReferenceIntents(ctx context.Context, owner executionenv.Owner) ([]executionenv.ReferenceIntent, error) { + return c.listReferenceIntents(ctx, &executionv1.ListReferenceIntentsRequest{Owner: ownerToProto(owner), Limit: 64}) +} + +func (c *Client) ListAllReferenceIntents(ctx context.Context) ([]executionenv.ReferenceIntent, error) { + return c.listReferenceIntents(ctx, &executionv1.ListReferenceIntentsRequest{Limit: 64}) +} + +func (c *Client) listReferenceIntents(ctx context.Context, req *executionv1.ListReferenceIntentsRequest) ([]executionenv.ReferenceIntent, error) { + v, err := c.rpc.ListReferenceIntents(ctx, req) + if err != nil { + return nil, decodeError(ctx, err) + } + out := make([]executionenv.ReferenceIntent, 0, len(v.GetIntents())) + for _, intent := range v.GetIntents() { + created, timeErr := checkedTime(intent.GetCreatedAt()) + if timeErr != nil { + return nil, sanitizedProviderError() + } + state := executionenv.ReferenceState(intent.GetState()) + if state != executionenv.ReferencePendingCreate && state != executionenv.ReferencePendingDelete { + return nil, sanitizedProviderError() + } + owner := executionenv.Owner{} + ownerOK := intent.GetOwner() == nil + if intent.GetOwner() != nil { + owner = executionenv.Owner{Issuer: intent.GetOwner().GetIssuer(), Subject: intent.GetOwner().GetSubject()} + ownerOK = owner.Issuer != "" && owner.Subject != "" && len(owner.Issuer) <= executionenv.MaxIdentityBytes && len(owner.Subject) <= executionenv.MaxIdentityBytes + } + if !ownerOK { + return nil, sanitizedProviderError() + } + out = append(out, executionenv.ReferenceIntent{Environment: refFromProto(intent.GetEnvironment()), Owner: owner, BindingID: intent.GetBindingId(), State: state, OperationID: intent.GetOperationId(), SourceBindingID: intent.GetSourceBindingId(), CreatedAt: created}) + } + return out, nil +} + // File executes one authorized filesystem operation. func (c *Client) File(ctx context.Context, req executionenv.FileRequest) (executionenv.FileResponse, error) { if req.Limit < 0 || req.Limit > executionenv.MaxListEntries { @@ -171,7 +268,7 @@ func (c *Client) ReleaseReference(ctx context.Context, req executionenv.Referenc // RetireEnvironment requests administrative retirement. func (c *Client) RetireEnvironment(ctx context.Context, req executionenv.RetireEnvironmentRequest) error { - _, err := c.rpc.RetireEnvironment(ctx, &executionv1.RetireEnvironmentRequest{Context: contextToProto(req.Context)}) + _, err := c.rpc.RetireEnvironment(ctx, &executionv1.RetireEnvironmentRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), ExpectedExecutionEpoch: req.ExpectedEpoch, ExpectedPodUid: req.ExpectedPodUID, ExpectedPvcUid: req.ExpectedPVCUID, OperationId: req.OperationID}) return decodeError(ctx, err) } @@ -230,18 +327,30 @@ func refToProto(r executionenv.EnvironmentRef) *executionv1.EnvironmentRef { return &executionv1.EnvironmentRef{Id: r.ID, Revision: r.Revision} } func contextToProto(v executionenv.RequestContext) *executionv1.RequestContext { - return &executionv1.RequestContext{Environment: refToProto(v.Environment), Owner: ownerToProto(v.Owner), BindingId: v.BindingID, Epoch: v.Epoch, Grant: v.Grant} + return &executionv1.RequestContext{Environment: refToProto(v.Environment), Owner: ownerToProto(v.Owner), BindingId: v.BindingID, RunId: v.RunID, ClaimId: v.ClaimID, Epoch: v.Epoch, GrantGeneration: v.GrantGeneration, Grant: v.Grant} } func refFromProto(r *executionv1.EnvironmentRef) executionenv.EnvironmentRef { return executionenv.EnvironmentRef{ID: r.GetId(), Revision: r.GetRevision()} } +func runClaimFromProto(v *executionv1.RunClaimResponse) (executionenv.RunClaim, error) { + expiry, err := checkedTime(v.GetExpiresAt()) + if err != nil || v.GetClaimId() == "" || v.GetRunId() == "" || v.GetEpoch() == 0 || v.GetGrantGeneration() == 0 || v.GetGrant() == "" { + return executionenv.RunClaim{}, sanitizedProviderError() + } + return executionenv.RunClaim{Environment: refFromProto(v.GetEnvironment()), BindingID: v.GetBindingId(), RunID: v.GetRunId(), ClaimID: v.GetClaimId(), Epoch: v.GetEpoch(), GrantGeneration: v.GetGrantGeneration(), Grant: v.GetGrant(), ExpiresAt: expiry}, nil +} + func ensureFromProto(v *executionv1.EnsureEnvironmentResponse) (executionenv.EnsureEnvironmentResponse, error) { - expiry, err := checkedTime(v.GetGrantExpiresAt()) - return executionenv.EnsureEnvironmentResponse{Environment: refFromProto(v.GetEnvironment()), Epoch: v.GetEpoch(), Ready: v.GetReady(), Grant: v.GetGrant(), GrantExpiresAt: expiry}, err + if v == nil { + return executionenv.EnsureEnvironmentResponse{}, sanitizedProviderError() + } + return executionenv.EnsureEnvironmentResponse{Environment: refFromProto(v.GetEnvironment()), Epoch: v.GetEpoch(), Ready: v.GetReady(), GrantGeneration: v.GetGrantGeneration()}, nil } func attachFromProto(v *executionv1.AttachEnvironmentResponse) (executionenv.AttachEnvironmentResponse, error) { - expiry, err := checkedTime(v.GetGrantExpiresAt()) - return executionenv.AttachEnvironmentResponse{Environment: refFromProto(v.GetEnvironment()), Epoch: v.GetEpoch(), Ready: v.GetReady(), Grant: v.GetGrant(), GrantExpiresAt: expiry}, err + if v == nil { + return executionenv.AttachEnvironmentResponse{}, sanitizedProviderError() + } + return executionenv.AttachEnvironmentResponse{Environment: refFromProto(v.GetEnvironment()), Epoch: v.GetEpoch(), Ready: v.GetReady(), GrantGeneration: v.GetGrantGeneration()}, nil } func checkedTime(v interface { CheckValid() error @@ -358,6 +467,204 @@ func NewProvider(client *Client, profile string) (*Provider, error) { return &Provider{client: client, profile: profile}, nil } +type referenceDeleteHandle struct { + client *Client + req executionenv.ReferenceRequest +} + +func (h *referenceDeleteHandle) Confirm(ctx context.Context) error { + return h.client.ConfirmReferenceDelete(ctx, h.req) +} +func (h *referenceDeleteHandle) Cancel(ctx context.Context) error { + return h.client.CancelReferenceDelete(ctx, h.req) +} +func (p *Provider) PrepareReferenceDelete(ctx context.Context, req server.PlacementSuccessorRequest) (server.ReferenceDeleteHandle, error) { + if !p.Applies(req.Ref) || req.SourceBindingID == "" { + return nil, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return nil, err + } + op, err := newOperationID() + if err != nil { + return nil, server.ErrPlacementUnavailable + } + mutation := executionenv.ReferenceRequest{Environment: executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision}, Owner: owner, BindingID: string(req.SourceBindingID), OperationID: op} + if err := p.client.PrepareReferenceDelete(ctx, mutation); err != nil { + return nil, mapPlacementError(err) + } + return &referenceDeleteHandle{client: p.client, req: mutation}, nil +} + +func (p *Provider) ReserveSuccessor(ctx context.Context, req server.PlacementSuccessorRequest) (server.PlacementBinding, error) { + if !p.Applies(req.Ref) || req.SourceBindingID == "" || req.DestinationBindingID == "" { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return server.PlacementBinding{}, err + } + op, err := newOperationID() + if err != nil { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + ref := executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision} + if err := p.client.ReserveSuccessor(ctx, executionenv.ReferenceRequest{Environment: ref, Owner: owner, SourceBindingID: string(req.SourceBindingID), BindingID: string(req.DestinationBindingID), OperationID: op}); err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + attached, err := p.client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: string(req.SourceBindingID)}, Purpose: executionenv.PurposeSession}) + if err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + base, err := p.binding(owner, string(req.DestinationBindingID), attached) + if err != nil { + return server.PlacementBinding{}, err + } + return p.withReferenceTransaction(base, owner, string(req.DestinationBindingID), op), nil +} + +func (p *Provider) Applies(ref session.EnvironmentRef) bool { + return ref.Kind == session.EnvironmentKind("kubernetes") +} + +type runHandle struct { + provider *Provider + owner executionenv.Owner + claim executionenv.RunClaimRequest + credentials *grantContext + environment tool.Environment + mu sync.Mutex +} + +func (h *runHandle) Environment() tool.Environment { return h.environment } +func (h *runHandle) Renew(ctx context.Context) error { + h.mu.Lock() + defer h.mu.Unlock() + op, err := newOperationID() + if err != nil { + return err + } + h.claim.OperationID = op + claim, err := h.provider.client.RenewRun(ctx, h.claim) + if err != nil { + return err + } + h.claim.Epoch, h.claim.ClaimID = claim.Epoch, claim.ClaimID + h.credentials.replace(claim) + return nil +} +func (h *runHandle) Release(ctx context.Context) error { + h.mu.Lock() + defer h.mu.Unlock() + op, err := newOperationID() + if err != nil { + return err + } + h.claim.OperationID = op + return h.provider.client.ReleaseRun(ctx, h.claim) +} + +func (p *Provider) ListReferenceIntents(ctx context.Context, limit int) ([]server.ReferenceIntent, error) { + if limit <= 0 || limit > 64 { + limit = 64 + } + intents, err := p.client.ListAllReferenceIntents(ctx) + if err != nil { + return nil, mapPlacementError(err) + } + if len(intents) > limit { + intents = intents[:limit] + } + out := make([]server.ReferenceIntent, 0, len(intents)) + for _, intent := range intents { + if intent.Owner.Issuer == "" || intent.Owner.Subject == "" || intent.BindingID == "" || intent.OperationID == "" { + return nil, server.ErrPlacementUnavailable + } + out = append(out, server.ReferenceIntent{ + Ref: toSessionRef(intent.Environment), + Principal: &session.Principal{Issuer: intent.Owner.Issuer, Subject: intent.Owner.Subject}, + BindingID: session.SessionID(intent.BindingID), + SourceBindingID: session.SessionID(intent.SourceBindingID), + OperationID: intent.OperationID, + PendingDelete: intent.State == executionenv.ReferencePendingDelete, + }) + } + return out, nil +} + +func (p *Provider) referenceIntentRequest(intent server.ReferenceIntent) (executionenv.ReferenceRequest, error) { + owner, err := ownerOf(intent.Principal) + if err != nil || !p.Applies(intent.Ref) || intent.BindingID == "" || intent.OperationID == "" { + return executionenv.ReferenceRequest{}, server.ErrInvalidPlacementSelection + } + return executionenv.ReferenceRequest{Environment: executionenv.EnvironmentRef{ID: intent.Ref.ID, Revision: intent.Ref.Revision}, Owner: owner, BindingID: string(intent.BindingID), SourceBindingID: string(intent.SourceBindingID), OperationID: intent.OperationID}, nil +} + +func (p *Provider) CommitReferenceIntent(ctx context.Context, intent server.ReferenceIntent) error { + req, err := p.referenceIntentRequest(intent) + if err != nil { + return err + } + return mapPlacementError(p.client.CommitReference(ctx, req)) +} +func (p *Provider) ConfirmReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + req, err := p.referenceIntentRequest(intent) + if err != nil { + return err + } + return mapPlacementError(p.client.ConfirmReferenceDelete(ctx, req)) +} +func (p *Provider) CancelReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + req, err := p.referenceIntentRequest(intent) + if err != nil { + return err + } + return mapPlacementError(p.client.CancelReferenceDelete(ctx, req)) +} + +func (p *Provider) AcquireRun(ctx context.Context, req server.ExecutionRunRequest) (server.ExecutionRunHandle, error) { + if !p.Applies(req.Ref) || req.BindingID == "" || req.RunID == "" { + return nil, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return nil, err + } + intents, err := p.client.ListReferenceIntents(ctx, owner) + if err != nil { + return nil, mapPlacementError(err) + } + for _, intent := range intents { + if intent.Environment == (executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision}) && intent.BindingID == string(req.BindingID) && intent.State == executionenv.ReferencePendingCreate { + if err := p.client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: intent.Environment, Owner: owner, BindingID: intent.BindingID, OperationID: intent.OperationID}); err != nil { + return nil, mapPlacementError(err) + } + } + } + op, err := newOperationID() + if err != nil { + return nil, server.ErrPlacementUnavailable + } + claimReq := executionenv.RunClaimRequest{Environment: executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision}, Owner: owner, BindingID: string(req.BindingID), RunID: req.RunID, OperationID: op, TTL: executionenv.DefaultRunTTL} + claim, err := p.client.AcquireRun(ctx, claimReq) + if err != nil { + return nil, mapPlacementError(err) + } + if toSessionRef(claim.Environment) != req.Ref || claim.BindingID != string(req.BindingID) || claim.RunID != req.RunID { + return nil, server.ErrPlacementChanged + } + credentials := &grantContext{client: p.client, context: executionenv.RequestContext{Environment: claim.Environment, Owner: owner, BindingID: claim.BindingID, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant}, expiresAt: claim.ExpiresAt} + ws := &workspace{credentials: credentials} + runner := &runner{credentials: credentials} + env, envErr := tool.NewEnvironment(req.Ref, ws, memledger.New(), runner) + if envErr != nil { + return nil, server.ErrPlacementUnavailable + } + claimReq.ClaimID, claimReq.Epoch, claimReq.GrantGeneration = claim.ClaimID, claim.Epoch, claim.GrantGeneration + return &runHandle{provider: p, owner: owner, claim: claimReq, credentials: credentials, environment: env}, nil +} + // ValidatePlacement performs side-effect-free profile preflight. func (p *Provider) ValidatePlacement(ctx context.Context) error { _, err := p.client.ValidateProfile(ctx, p.profile) @@ -380,14 +687,18 @@ func (p *Provider) Bind(ctx context.Context, req server.PlacementBindRequest) (s if err != nil { return server.PlacementBinding{}, err } - ensured, err := p.client.Ensure(ctx, string(req.BindingID), p.profile, owner) + operationID, err := newOperationID() + if err != nil { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + ensured, err := p.client.Ensure(ctx, string(req.BindingID), p.profile, owner, operationID) if err != nil { return server.PlacementBinding{}, mapPlacementError(err) } if ensured.Environment.ID == "" || ensured.Environment.Revision == "" { return server.PlacementBinding{}, server.ErrPlacementUnavailable } - return p.waitForBinding(ctx, owner, string(req.BindingID), ensured.Environment) + return p.waitForBinding(ctx, owner, string(req.BindingID), ensured.Environment, operationID) } // Reattach exactly resolves a persisted remote environment for its binding. @@ -410,7 +721,7 @@ func (p *Provider) Reattach(ctx context.Context, req server.PlacementReattachReq return p.binding(owner, string(req.BindingID), attached) } -func (p *Provider) waitForBinding(ctx context.Context, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) (server.PlacementBinding, error) { +func (p *Provider) waitForBinding(ctx context.Context, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef, operationID string) (server.PlacementBinding, error) { ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { @@ -419,7 +730,11 @@ func (p *Provider) waitForBinding(ctx context.Context, owner executionenv.Owner, if attached.Environment != ref { return server.PlacementBinding{}, server.ErrPlacementChanged } - return p.binding(owner, binding, attached) + base, bindErr := p.binding(owner, binding, attached) + if bindErr != nil { + return server.PlacementBinding{}, bindErr + } + return p.withReferenceTransaction(base, owner, binding, operationID), nil } if err != nil { var remote *executionenv.Error @@ -436,10 +751,10 @@ func (p *Provider) waitForBinding(ctx context.Context, owner executionenv.Owner, } func (p *Provider) binding(owner executionenv.Owner, binding string, attached executionenv.AttachEnvironmentResponse) (server.PlacementBinding, error) { - if !attached.Ready || attached.Environment.ID == "" || attached.Environment.Revision == "" || attached.Epoch == 0 || attached.Grant == "" || attached.GrantExpiresAt.IsZero() { + if !attached.Ready || attached.Environment.ID == "" || attached.Environment.Revision == "" || attached.Epoch == 0 { return server.PlacementBinding{}, server.ErrPlacementUnavailable } - credentials := &grantContext{client: p.client, context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding, Epoch: attached.Epoch, Grant: attached.Grant}, expiresAt: attached.GrantExpiresAt} + credentials := &grantContext{client: p.client, context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding, Epoch: attached.Epoch, GrantGeneration: attached.GrantGeneration}, expiresAt: time.Time{}} ws := &workspace{credentials: credentials} runner := &runner{credentials: credentials} sref := toSessionRef(attached.Environment) @@ -449,6 +764,35 @@ func (p *Provider) binding(owner executionenv.Owner, binding string, attached ex } return server.PlacementBinding{Environment: env, Ref: sref, Metadata: server.PlacementMetadata{Kind: "kubernetes", Label: "Remote Kubernetes workspace", Revision: attached.Environment.Revision}}, nil } +func (p *Provider) withReferenceTransaction(binding server.PlacementBinding, owner executionenv.Owner, bindingID, operationID string) server.PlacementBinding { + ref := executionenv.EnvironmentRef{ID: binding.Ref.ID, Revision: binding.Ref.Revision} + var mu sync.Mutex + finalized := false + binding.Commit = func(ctx context.Context) error { + mu.Lock() + defer mu.Unlock() + if finalized { + return nil + } + // Once dispatched, a missing response is ambiguous: the provider may have + // committed. Close must never turn that uncertainty into an abort. + finalized = true + return p.client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: ref, Owner: owner, BindingID: bindingID, OperationID: operationID}) + } + binding.Close = func() error { + mu.Lock() + defer mu.Unlock() + if finalized { + return nil + } + finalized = true + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return p.client.AbortReference(ctx, executionenv.ReferenceRequest{Environment: ref, Owner: owner, BindingID: bindingID, OperationID: operationID}) + } + return binding +} + func toSessionRef(ref executionenv.EnvironmentRef) session.EnvironmentRef { return session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: ref.ID, Revision: ref.Revision} } @@ -485,21 +829,19 @@ func (g *grantContext) refresh(ctx context.Context) (executionenv.RequestContext return g.renew(ctx, true) } -func (g *grantContext) renew(ctx context.Context, force bool) (executionenv.RequestContext, error) { +func (g *grantContext) replace(claim executionenv.RunClaim) { g.mu.Lock() defer g.mu.Unlock() - if !force && time.Until(g.expiresAt) > 10*time.Second { - return g.context, nil - } - attached, err := g.client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: g.context.Environment, Owner: g.context.Owner, BindingID: g.context.BindingID}, Purpose: executionenv.PurposeSession}) - if err != nil { - return executionenv.RequestContext{}, err - } - if !attached.Ready || attached.Environment != g.context.Environment || attached.Epoch != g.context.Epoch || attached.Grant == "" || attached.GrantExpiresAt.IsZero() { - return executionenv.RequestContext{}, server.ErrPlacementChanged + g.context.RunID, g.context.ClaimID, g.context.Epoch, g.context.GrantGeneration, g.context.Grant = claim.RunID, claim.ClaimID, claim.Epoch, claim.GrantGeneration, claim.Grant + g.expiresAt = claim.ExpiresAt +} + +func (g *grantContext) renew(_ context.Context, _ bool) (executionenv.RequestContext, error) { + g.mu.Lock() + defer g.mu.Unlock() + if g.context.Grant == "" || !time.Now().Before(g.expiresAt) { + return executionenv.RequestContext{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "run grant unavailable"} } - g.context.Grant = attached.Grant - g.expiresAt = attached.GrantExpiresAt return g.context, nil } diff --git a/internal/adapter/executionclient/client_test.go b/internal/adapter/executionclient/client_test.go index 24adbc2e81..f4cf4c335e 100644 --- a/internal/adapter/executionclient/client_test.go +++ b/internal/adapter/executionclient/client_test.go @@ -1,3 +1,4 @@ +//nolint:revive // Test doubles mirror the private protocol's complete method set. package executionclient import ( @@ -78,6 +79,40 @@ func (*integrationBackend) CancelCommand(context.Context, string, string, execut return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeInternal, Message: "unimplemented"} } +func (b *integrationBackend) EnsurePending(ctx context.Context, client, owner, binding, profile, fp, _ string) (executioncontroller.Allocation, error) { + return b.Ensure(ctx, client, owner, binding, profile, fp) +} +func (b *integrationBackend) AcquireRun(_ context.Context, ref executionenv.EnvironmentRef, client, owner, binding, run, _ string, ttl time.Duration) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: run, ClaimID: "claim", Epoch: b.allocation.Epoch + 1, GrantGeneration: 1, ExpiresAt: time.Now().Add(ttl)}, nil +} +func (b *integrationBackend) RenewRun(_ context.Context, _ executionenv.EnvironmentRef, _ string, _ string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: req.ClaimID, Epoch: req.Epoch, GrantGeneration: 1, ExpiresAt: time.Now().Add(req.TTL)}, nil +} +func (*integrationBackend) ReleaseRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) error { + return nil +} +func (*integrationBackend) CommitReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) AbortReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) ReserveSuccessor(context.Context, executionenv.EnvironmentRef, string, string, string, string, string) error { + return nil +} +func (*integrationBackend) PrepareReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) ConfirmReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) CancelReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) { + return nil, nil +} + func certificate(t *testing.T, parent *x509.Certificate, parentKey *ecdsa.PrivateKey, serverName string, client bool) (tls.Certificate, *x509.Certificate, *ecdsa.PrivateKey) { t.Helper() key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) @@ -150,17 +185,27 @@ func TestProviderThroughRealGRPCSignedHandlerRefreshesAndReattachesExactly(t *te if err != nil { t.Fatal(err) } + if binding.Commit != nil { + if err := binding.Commit(context.Background()); err != nil { + t.Fatal(err) + } + } reattached, err := provider.Reattach(context.Background(), server.PlacementReattachRequest{Ref: binding.Ref, Principal: principal, BindingID: "session-real"}) if err != nil { t.Fatal(err) } - resolver := reattached.Environment.Workspace().(tool.AuthorityResourceResolver) + handle, err := provider.AcquireRun(context.Background(), server.ExecutionRunRequest{Ref: reattached.Ref, Principal: principal, BindingID: "session-real", RunID: "run-real"}) + if err != nil { + t.Fatal(err) + } + defer handle.Release(context.Background()) + runEnv := handle.Environment() + resolver := runEnv.Workspace().(tool.AuthorityResourceResolver) target, root, err := resolver.AuthorityResourcePath("main.go") if err != nil || target != "/workspace/main.go" || root != "/workspace" { t.Fatalf("authority=(%q,%q) err=%v", target, root, err) } - backend.expireNextRead = true - data, version, err := reattached.Environment.Workspace().ReadVersion(context.Background(), "main.go") + data, version, err := runEnv.Workspace().ReadVersion(context.Background(), "main.go") if err != nil || string(data) != "from-grpc" { t.Fatalf("read=%q err=%v", data, err) } @@ -168,11 +213,11 @@ func TestProviderThroughRealGRPCSignedHandlerRefreshesAndReattachesExactly(t *te if err != nil || encoded != string([]byte{0xff, 0, 1}) { t.Fatalf("opaque version=%q err=%v", encoded, err) } - result, err := reattached.Environment.CommandRunner().Run(context.Background(), "go test ./...") + result, err := runEnv.CommandRunner().Run(context.Background(), "go test ./...") if err != nil || result.Stdout != "ok\n" { t.Fatalf("result=%+v err=%v", result, err) } - if backend.ensureCalls != 1 || backend.attachCalls < 3 || backend.fileCalls != 3 { + if backend.ensureCalls != 1 || backend.attachCalls < 2 || backend.fileCalls != 2 { t.Fatalf("ensure=%d attach=%d file=%d", backend.ensureCalls, backend.attachCalls, backend.fileCalls) } } @@ -187,15 +232,15 @@ func TestCommandStatusIsUnimplementedOnlyAfterAuthorization(t *testing.T) { } defer client.Close() owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} - ensured, err := client.Ensure(context.Background(), "binding", "coding", owner) + ensured, err := client.Ensure(context.Background(), "binding", "coding", owner, "create") if err != nil { t.Fatal(err) } - attached, err := client.Attach(context.Background(), executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ensured.Environment, Owner: owner, BindingID: "binding"}, Purpose: executionenv.PurposeSession}) + claim, err := client.AcquireRun(context.Background(), executionenv.RunClaimRequest{Environment: ensured.Environment, Owner: owner, BindingID: "binding", RunID: "run", OperationID: "acquire", TTL: time.Minute}) if err != nil { t.Fatal(err) } - rc := executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: "binding", Epoch: attached.Epoch, Grant: attached.Grant} + rc := executionenv.RequestContext{Environment: claim.Environment, Owner: owner, BindingID: "binding", RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant} _, err = client.rpc.CommandStatus(context.Background(), &executionv1.CommandQueryRequest{Context: contextToProto(rc), CommandId: "c1"}) if status.Code(err) != codes.Unimplemented { t.Fatalf("authorized status code=%v", status.Code(err)) diff --git a/internal/adapter/executionclient/reference_ambiguity_test.go b/internal/adapter/executionclient/reference_ambiguity_test.go new file mode 100644 index 0000000000..0e8929e769 --- /dev/null +++ b/internal/adapter/executionclient/reference_ambiguity_test.go @@ -0,0 +1,60 @@ +package executionclient + +import ( + "context" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type droppedCommitRPC struct { + executionv1.ExecutionProviderServiceClient + commits int + aborts int +} + +func (r *droppedCommitRPC) CommitReference(context.Context, *executionv1.ReferenceMutationRequest, ...grpc.CallOption) (*emptypb.Empty, error) { + r.commits++ + return nil, status.Error(codes.Unavailable, "response lost") +} +func (r *droppedCommitRPC) AbortReference(context.Context, *executionv1.ReferenceMutationRequest, ...grpc.CallOption) (*emptypb.Empty, error) { + r.aborts++ + return &emptypb.Empty{}, nil +} + +func TestDefinitePreCommitFailureAbortsReservation(t *testing.T) { + rpc := &droppedCommitRPC{} + provider := &Provider{client: &Client{rpc: rpc}, profile: "coding"} + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + binding := provider.withReferenceTransaction(server.PlacementBinding{Ref: ref}, executionenv.Owner{Issuer: "issuer", Subject: "alice"}, "session", "operation") + if err := binding.Close(); err != nil { + t.Fatal(err) + } + if rpc.commits != 0 || rpc.aborts != 1 { + t.Fatalf("commits=%d aborts=%d", rpc.commits, rpc.aborts) + } +} + +func TestAmbiguousCommitIsRetainedAndNeverAbortedByCleanup(t *testing.T) { + rpc := &droppedCommitRPC{} + provider := &Provider{client: &Client{rpc: rpc}, profile: "coding"} + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + binding := provider.withReferenceTransaction(server.PlacementBinding{Ref: ref}, executionenv.Owner{Issuer: "issuer", Subject: "alice"}, "session", "operation") + if err := binding.Commit(t.Context()); err == nil { + t.Fatal("commit transport ambiguity was hidden") + } + if err := binding.Close(); err != nil { + t.Fatal(err) + } + if rpc.commits != 1 || rpc.aborts != 0 { + t.Fatalf("commits=%d aborts=%d", rpc.commits, rpc.aborts) + } +} diff --git a/internal/adapter/executionclient/service_integration_test.go b/internal/adapter/executionclient/service_integration_test.go new file mode 100644 index 0000000000..9a8381bccc --- /dev/null +++ b/internal/adapter/executionclient/service_integration_test.go @@ -0,0 +1,140 @@ +package executionclient + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/server" + adaptertools "github.com/stacklok/mecatl/internal/adapter/tools" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type recordingExecutor struct{ calls atomic.Int64 } + +func (e *recordingExecutor) Execute(_ context.Context, _ string, req executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + e.calls.Add(1) + switch req.Operation { + case executionenv.OpFileResolveAuthority: + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{AuthorityTarget: "/workspace/main.go", AuthorityWorkspace: "/workspace"}}, nil + case executionenv.OpFileRead: + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Data: []byte("package main\n"), Version: "v1"}}, nil + case executionenv.OpCommandStart: + return executionenv.ExecutorResponse{Command: &executionenv.CommandStatusResponse{CommandID: req.CommandID, State: executionenv.CommandSucceeded, Stdout: []byte("ok\n"), TerminalReceipt: "complete"}}, nil + default: + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "unsupported test operation"} + } +} + +func TestServiceUsesRealMTLSProviderStoreAndReleasesOnlyAfterDrain(t *testing.T) { + profilePath := filepath.Join(t.TempDir(), "profiles.yaml") + profileYAML := []byte("profiles:\n coding:\n image: example.test/executor@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n storageClass: standard\n storageSize: 1Gi\n cpuRequest: 100m\n memoryRequest: 128Mi\n cpuLimit: 1\n memoryLimit: 1Gi\n ephemeralStorageRequest: 256Mi\n ephemeralStorageLimit: 1Gi\n tmpSizeLimit: 128Mi\n runtimeClassName: gvisor\n maxFileBytes: 5242880\n maxCommandBytes: 1048576\n maxCommandDuration: 1m\n maxEnvironments: 10\n") + if err := os.WriteFile(profilePath, profileYAML, 0o600); err != nil { + t.Fatal(err) + } + profiles, err := executioncontroller.LoadProfiles(profilePath) + if err != nil { + t.Fatal(err) + } + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{executioncontroller.ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + executor := &recordingExecutor{} + controllerStore := executioncontroller.NewStore(dyn, "ns", profiles, executor) + profile, err := controllerStore.ValidateProfile(t.Context(), "coding") + if err != nil { + t.Fatal(err) + } + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + ownerSum := sha256.Sum256([]byte(owner.Issuer + "\x00" + owner.Subject)) + clientSum := sha256.Sum256([]byte("spiffe://example.test/mecak8s")) + now := time.Now().UTC() + envObj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env-real", "namespace": "ns"}, + "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev-1", "ownerHash": hex.EncodeToString(ownerSum[:]), "ownerIssuer": owner.Issuer, "ownerSubject": owner.Subject, "clientHash": hex.EncodeToString(clientSum[:]), "bindingID": "service-session", "profile": "coding", "profileDigest": profile.Digest, "desired": "Active"}, + "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "pod": map[string]any{"name": "executor-pod"}, "references": []any{map[string]any{"bindingID": "service-session", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}, + }} + if _, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Create(t.Context(), envObj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + fx := startFixture(t, controllerStore, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + catalog := tool.NewCatalog() + catalog.MustRegister(adaptertools.ReadTool{}) + catalog.MustRegister(adaptertools.NewShellTool()) + llm := mockllm.New( + mockllm.ToolCallTurn(session.NewToolCall("read", "Read", json.RawMessage(`{"path":"main.go"}`))), + mockllm.ToolCallTurn(session.NewToolCall("shell", "Shell", json.RawMessage(`{"command":"true"}`))), + mockllm.TextTurn("done"), + mockllm.TextTurn("continued"), + ) + engine := agent.NewEngine(agent.Deps{LLM: llm, Catalog: catalog, Policy: permpolicy.NewPolicy(permpolicy.AllowAllFloorRules(), nil), Model: "test"}) + sessions := memstore.New() + svc, err := server.NewService(server.Config{Engine: engine, Store: sessions, PlacementProvider: provider, PlacementScope: "test", ExecutionAccess: provider, SharedEngineRoot: "/workspace", DefaultLimits: session.Limits{MaxTurns: 5}}) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + principal := &session.Principal{Issuer: owner.Issuer, Subject: owner.Subject, GrantType: session.GrantTypeUser} + sess := session.New("service-session", session.ModeDefault, session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env-real", Revision: "rev-1"}, session.Limits{MaxTurns: 5}, now) + if err := sess.RestoreLabels(principal, session.Authority{}); err != nil { + t.Fatal(err) + } + if err := sessions.Save(t.Context(), sess); err != nil { + t.Fatal(err) + } + ctx := session.WithPrincipal(t.Context(), principal) + run, err := svc.StartRunContent(ctx, sess.ID, "inspect", nil) + if err != nil { + t.Fatal(err) + } + for range run.Events() { + } + if executor.calls.Load() != 3 { + t.Fatalf("executor calls=%d, want authority+read+shell", executor.calls.Load()) + } + if _, err := svc.StartRunContent(ctx, sess.ID, "early", nil); err == nil { + t.Fatal("replacement run acquired before relay release") + } + svc.FinishRun(sess.ID, run) + stored, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env-real", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if _, found, err := unstructured.NestedMap(stored.Object, "status", "activeRun"); err != nil || found { + t.Fatalf("native claim remained after actual drain release: found=%t err=%v", found, err) + } + continued, err := svc.StartRunContent(ctx, sess.ID, "continue", nil) + if err != nil { + t.Fatal(err) + } + for range continued.Events() { + } + svc.FinishRun(sess.ID, continued) +} diff --git a/internal/adapter/executioncontroller/admin_lifecycle.go b/internal/adapter/executioncontroller/admin_lifecycle.go new file mode 100644 index 0000000000..c394ea486f --- /dev/null +++ b/internal/adapter/executioncontroller/admin_lifecycle.go @@ -0,0 +1,273 @@ +package executioncontroller + +import ( + "context" + "math" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +type adminLifecycleRequest struct { + Environment executionenv.EnvironmentRef + OwnerHash string + Client string + ExpectedEpoch uint64 + ExpectedPodUID string + ExpectedPVCUID string + OperationID string + ExpectedSchema int64 +} + +// ReplaceExecutor starts an exact, crash-recoverable executor replacement. +func (s *Store) ReplaceExecutor(ctx context.Context, q adminLifecycleRequest) error { + return s.startLifecycle(ctx, q, "ReplaceExecutor", false) +} + +// RetireExact starts exact executor retirement while retaining its workspace. +func (s *Store) RetireExact(ctx context.Context, q adminLifecycleRequest) error { + return s.startLifecycle(ctx, q, "RetireEnvironment", true) +} + +func (s *Store) startLifecycle(ctx context.Context, q adminLifecycleRequest, kind string, requireNoRefs bool) error { + if q.ExpectedEpoch == 0 || q.ExpectedEpoch > math.MaxInt64 || q.ExpectedPodUID == "" || q.ExpectedPVCUID == "" || q.OperationID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "exact lifecycle identity is required"} + } + return s.retryUpdateStatus(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := exactAdminSubject(o, q); err != nil { + return err + } + if existing, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation"); found { + if text(existing, "id") == q.OperationID && text(existing, "type") == kind && text(existing, "expectedPodUID") == q.ExpectedPodUID && text(existing, "expectedPVCUID") == q.ExpectedPVCUID && intNested(existing, "expectedEpoch") == int64(q.ExpectedEpoch) { //nolint:gosec // validated above. + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another lifecycle operation is active"} + } + if textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy || !conditionTrue(o, "Ready") { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not healthy and idle"} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + if requireNoRefs && len(refs) != 0 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment still has references"} + } + operation := map[string]any{"id": q.OperationID, "type": kind, "phase": "Quiescing", "expectedEpoch": int64(q.ExpectedEpoch), "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID, "createdAt": s.now().Format(time.RFC3339Nano)} //nolint:gosec // validated above. + setConditionObject(o, "Ready", false, "Quiescing", "administrator lifecycle operation is quiescing the executor") + return unstructured.SetNestedMap(o.Object, operation, "status", "lifecycleOperation") + }) +} + +func exactAdminSubject(o *unstructured.Unstructured, q adminLifecycleRequest) error { + if textNested(o.Object, "spec", "ownerHash") != q.OwnerHash || textNested(o.Object, "spec", "clientHash") != hashText(q.Client) || textNested(o.Object, "spec", "revision") != q.Environment.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if !epochMatches(o, q.ExpectedEpoch) || textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment execution identity changed"} + } + return nil +} + +// RecoverEnvironment clears uncertainty only after built-in exact terminal verification. +func (s *Store) RecoverEnvironment(ctx context.Context, q adminLifecycleRequest) error { + if s.kube == nil || q.ExpectedPodUID == "" || q.ExpectedPVCUID == "" || q.OperationID == "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "built-in recovery verifier is unavailable"} + } + podName, pvcName := "", "" + if err := s.retryUpdateStatus(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := exactAdminSubject(o, q); err != nil { + return err + } + if textNested(o.Object, "status", "fenceState") != "FenceUnknown" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not fenced"} + } + podName, pvcName = textNested(o.Object, "status", "pod", "name"), textNested(o.Object, "status", "pvc", "name") + return nil + }); err != nil { + return err + } + pod, err := s.kube.CoreV1().Pods(s.namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil || string(pod.UID) != q.ExpectedPodUID || !podTerminal(pod) { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "terminal executor proof unavailable; use the external-fencing operator runbook"} + } + pvc, err := s.kube.CoreV1().PersistentVolumeClaims(s.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil || string(pvc.UID) != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "workspace identity cannot be verified"} + } + return s.retryUpdateStatus(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := exactAdminSubject(o, q); err != nil { + return err + } + proof := terminationProof(q, pod) + if err := unstructured.SetNestedMap(o.Object, proof, "status", "terminationProof"); err != nil { + return err + } + unstructured.RemoveNestedField(o.Object, "status", "activeOperation") + unstructured.RemoveNestedField(o.Object, "status", "activeRun") + return unstructured.SetNestedField(o.Object, fenceHealthy, "status", "fenceState") + }) +} + +func terminationProof(q adminLifecycleRequest, pod *corev1.Pod) map[string]any { + return map[string]any{"operationID": q.OperationID, "podUID": q.ExpectedPodUID, "pvcUID": q.ExpectedPVCUID, "epoch": int64(q.ExpectedEpoch), "podPhase": string(pod.Status.Phase), "observedAt": time.Now().UTC().Format(time.RFC3339Nano)} //nolint:gosec // epoch is validated by caller. +} + +func podTerminal(pod *corev1.Pod) bool { + if pod == nil || pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed || len(pod.Status.ContainerStatuses) != len(pod.Spec.Containers) || len(pod.Spec.Containers) == 0 { + return false + } + for _, status := range pod.Status.ContainerStatuses { + if status.State.Terminated == nil { + return false + } + } + return true +} + +// DeleteRetiredEnvironment starts exact deletion of an explicitly retained workspace. +func (s *Store) DeleteRetiredEnvironment(ctx context.Context, q adminLifecycleRequest) error { + if q.ExpectedPVCUID == "" || q.OperationID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "exact retained workspace identity is required"} + } + return s.retryUpdateStatus(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "spec", "ownerHash") != q.OwnerHash || textNested(o.Object, "spec", "clientHash") != hashText(q.Client) || textNested(o.Object, "spec", "revision") != q.Environment.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "retained workspace identity changed"} + } + if existing, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation"); found { + if text(existing, "id") == q.OperationID && text(existing, "type") == "DeleteRetiredEnvironment" { + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another lifecycle operation is active"} + } + refs, err := referenceRecords(o) + if err != nil || len(refs) != 0 || textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || !conditionTrue(o, "Retired") || !conditionTrue(o, "ExecutorTerminated") { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not safely retained and unreferenced"} + } + return unstructured.SetNestedMap(o.Object, map[string]any{"id": q.OperationID, "type": "DeleteRetiredEnvironment", "phase": "DeletingPVC", "expectedPVCUID": q.ExpectedPVCUID, "createdAt": s.now().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + }) +} + +// MigrateEnvironment explicitly upgrades a recognized, verified prototype schema. +func (s *Store) MigrateEnvironment(ctx context.Context, q adminLifecycleRequest) error { //nolint:gocyclo // Two Kubernetes subresources require a durable phased migration. + if s.kube == nil || q.ExpectedSchema < 0 || q.ExpectedSchema > 1 || q.OperationID == "" || q.ExpectedPodUID == "" || q.ExpectedPVCUID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "recognized prototype schema and exact runtime identities are required"} + } + if err := s.retryUpdateStatusRaw(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "spec", "ownerHash") != q.OwnerHash || textNested(o.Object, "spec", "clientHash") != hashText(q.Client) || textNested(o.Object, "spec", "revision") != q.Environment.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if intNested(o.Object, "spec", "schemaVersion") == currentSchemaVersion && intNested(o.Object, "status", "schemaVersion") == currentSchemaVersion && textNested(o.Object, "status", "lastMigrationOperationID") == q.OperationID { + return nil + } + if migrationID := textNested(o.Object, "status", "migrationOperation", "id"); migrationID != "" { + if migrationID != q.OperationID || intNested(o.Object, "status", "migrationOperation", "fromSchema") != q.ExpectedSchema { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another schema migration is active"} + } + return nil + } + if intNested(o.Object, "spec", "schemaVersion") != q.ExpectedSchema || intNested(o.Object, "status", "schemaVersion") != q.ExpectedSchema || textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "lifecycleOperation", "id") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype environment is not eligible for migration"} + } + if textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype runtime identity mismatch"} + } + if err := s.verifyRuntimeUIDs(ctx, o, q); err != nil { + return err + } + refs, err := migrateReferenceRecords(o, q.OperationID, s.now()) + if err != nil { + return err + } + if err := setReferenceRecords(o, refs); err != nil { + return err + } + return unstructured.SetNestedMap(o.Object, map[string]any{"id": q.OperationID, "fromSchema": q.ExpectedSchema, "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID}, "status", "migrationOperation") + }); err != nil { + return err + } + for attempt := 0; attempt < 5; attempt++ { + o, err := s.resources.Get(ctx, q.Environment.ID, metav1.GetOptions{}) + if err != nil { + return err + } + if intNested(o.Object, "spec", "schemaVersion") == currentSchemaVersion { + break + } + if textNested(o.Object, "status", "migrationOperation", "id") != q.OperationID || intNested(o.Object, "spec", "schemaVersion") != q.ExpectedSchema { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "schema migration changed concurrently"} + } + _ = unstructured.SetNestedField(o.Object, currentSchemaVersion, "spec", "schemaVersion") + if _, err = s.resources.Update(ctx, o, metav1.UpdateOptions{}); err == nil { + break + } else if !apierrors.IsConflict(err) { + return err + } else if attempt == 4 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "schema migration conflict", Retryable: true} + } + } + return s.retryUpdateStatusRaw(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion || textNested(o.Object, "status", "migrationOperation", "id") != q.OperationID || intNested(o.Object, "status", "schemaVersion") != q.ExpectedSchema { + if intNested(o.Object, "status", "schemaVersion") == currentSchemaVersion && textNested(o.Object, "status", "lastMigrationOperationID") == q.OperationID { + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "schema migration phase mismatch"} + } + epoch, ok := epochValue(o) + if !ok || epoch >= math.MaxInt64 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype epoch is invalid"} + } + _ = unstructured.SetNestedField(o.Object, int64(epoch+1), "status", "epoch") //nolint:gosec // checked above. + _ = unstructured.SetNestedField(o.Object, currentSchemaVersion, "status", "schemaVersion") + _ = unstructured.SetNestedField(o.Object, q.OperationID, "status", "lastMigrationOperationID") + unstructured.RemoveNestedField(o.Object, "status", "migrationOperation") + return nil + }) +} + +func (s *Store) verifyRuntimeUIDs(ctx context.Context, o *unstructured.Unstructured, q adminLifecycleRequest) error { + pod, err := s.kube.CoreV1().Pods(s.namespace).Get(ctx, textNested(o.Object, "status", "pod", "name"), metav1.GetOptions{}) + if err != nil || string(pod.UID) != q.ExpectedPodUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype executor identity is not observable"} + } + pvc, err := s.kube.CoreV1().PersistentVolumeClaims(s.namespace).Get(ctx, textNested(o.Object, "status", "pvc", "name"), metav1.GetOptions{}) + if err != nil || string(pvc.UID) != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype workspace identity is not observable"} + } + return nil +} + +func migrateReferenceRecords(o *unstructured.Unstructured, operationID string, now time.Time) ([]referenceRecord, error) { + raw, found, err := unstructured.NestedSlice(o.Object, "status", "references") + if err != nil || !found { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + out := make([]referenceRecord, 0, len(raw)) + for _, value := range raw { + switch item := value.(type) { + case string: + if item == "" { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + out = append(out, referenceRecord{BindingID: item, State: executionenv.ReferencePublished, OperationID: "migration-" + operationID, CreatedAt: now}) + case map[string]any: + created, parseErr := time.Parse(time.RFC3339Nano, text(item, "createdAt")) + r := referenceRecord{BindingID: text(item, "bindingID"), State: executionenv.ReferenceState(text(item, "state")), OperationID: text(item, "operationID"), SourceBindingID: text(item, "sourceBindingID"), CreatedAt: created} + if parseErr != nil || r.BindingID == "" || r.OperationID == "" || r.State != executionenv.ReferencePendingCreate && r.State != executionenv.ReferencePublished && r.State != executionenv.ReferencePendingDelete { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + out = append(out, r) + default: + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + } + return out, nil +} diff --git a/internal/adapter/executioncontroller/controller.go b/internal/adapter/executioncontroller/controller.go index 8688464eb7..c4750b19d9 100644 --- a/internal/adapter/executioncontroller/controller.go +++ b/internal/adapter/executioncontroller/controller.go @@ -3,7 +3,6 @@ package executioncontroller import ( "context" "errors" - "fmt" "reflect" "strings" "sync" @@ -12,7 +11,6 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" @@ -28,6 +26,7 @@ import ( const ( environmentFinalizer = "execution.mecatl.dev/retain-workspace" + executorFinalizer = "execution.mecatl.dev/verify-termination" fenceHealthy = "Healthy" ) @@ -41,21 +40,18 @@ type Reconciler struct { initOnce sync.Once initErr error ready atomic.Bool + now func() time.Time } // NewReconciler constructs a reconciler for one namespace. func NewReconciler(d dynamic.Interface, k kubernetes.Interface, namespace string, p *Profiles) *Reconciler { - return &Reconciler{dynamic: d, kube: k, namespace: namespace, profiles: p, queue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[string]())} + return &Reconciler{dynamic: d, kube: k, namespace: namespace, profiles: p, queue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[string]()), now: func() time.Time { return time.Now().UTC() }} } -// Initialize completes startup fencing and informer synchronization before the -// provider may expose mutating APIs. +// Initialize synchronizes informer caches before the provider may expose mutating APIs. +// Startup never mutates operation ownership: another replica may still be its live holder. func (r *Reconciler) Initialize(ctx context.Context) error { r.initOnce.Do(func() { - if err := r.fenceInterruptedOperations(ctx); err != nil { - r.initErr = err - return - } factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(r.dynamic, 0, r.namespace, nil) environments := factory.ForResource(ExecutionEnvironmentGVR).Informer() _, err := environments.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: r.enqueue, UpdateFunc: r.enqueueUpdate}) @@ -161,24 +157,6 @@ func (r *Reconciler) process(ctx context.Context) bool { } return true } -func (r *Reconciler) fenceInterruptedOperations(ctx context.Context) error { - list, err := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - return fmt.Errorf("list execution environments for startup fencing: %w", err) - } - for i := range list.Items { - o := list.Items[i].DeepCopy() - if textNested(o.Object, "status", "activeOperation", "id") == "" { - continue - } - unstructured.RemoveNestedField(o.Object, "status", "activeOperation") - _ = unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") - if _, err := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace).UpdateStatus(ctx, o, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("fence interrupted environment %q: %w", o.GetName(), err) - } - } - return nil -} // Reconcile converges one ExecutionEnvironment and its runtime resources. func (r *Reconciler) Reconcile(ctx context.Context, name string) error { @@ -190,6 +168,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, name string) error { if err != nil { return err } + if requireCurrentSchema(env) != nil { + return r.setCondition(ctx, env, "Ready", false, "IncompatibleSchema", "explicit administrator migration to schema version 2 is required") + } + if expires := textNested(env.Object, "status", "activeOperation", "expiresAt"); expires != "" { + deadline, parseErr := time.Parse(time.RFC3339Nano, expires) + if parseErr != nil || !r.now().Before(deadline) { + return r.setFenceUnknown(ctx, env, "operation holder lease expired; operation identity retained for recovery") + } + } profileName := textNested(env.Object, "spec", "profile") p, ok := r.profiles.get(profileName) if !ok || p.Digest != textNested(env.Object, "spec", "profileDigest") { @@ -206,6 +193,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, name string) error { } return nil } + if textNested(env.Object, "status", "lifecycleOperation", "id") != "" { + return r.reconcileLifecycle(ctx, env) + } if textNested(env.Object, "spec", "desired") == "Retiring" { return r.reconcileRetiring(ctx, env) } @@ -232,14 +222,17 @@ func (r *Reconciler) ensurePVC(ctx context.Context, env *unstructured.Unstructur pvcs := r.kube.CoreV1().PersistentVolumeClaims(r.namespace) cur, err := pvcs.Get(ctx, name, metav1.GetOptions{}) if err == nil { - if cur.Labels["execution.mecatl.dev/environment"] != env.GetName() { - return nil, errors.New("PVC ownership label mismatch") + if cur.Labels["execution.mecatl.dev/environment"] != env.GetName() || (textNested(env.Object, "status", "pvc", "uid") != "" && textNested(env.Object, "status", "pvc", "uid") != string(cur.UID)) { + return nil, errors.New("PVC ownership identity mismatch") } return cur, nil } if !apierrors.IsNotFound(err) { return nil, err } + if textNested(env.Object, "status", "pvc", "uid") != "" { + return nil, errors.New("authoritative PVC disappeared; replacement is forbidden") + } qty := p.StorageSize mode := corev1.PersistentVolumeFilesystem created, err := pvcs.Create(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/revision": textNested(env.Object, "spec", "revision")}}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, StorageClassName: &p.Spec.StorageClass, VolumeMode: &mode, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: qty}}}}, metav1.CreateOptions{}) @@ -252,8 +245,8 @@ func (r *Reconciler) ensurePod(ctx context.Context, env *unstructured.Unstructur pods := r.kube.CoreV1().Pods(r.namespace) cur, err := pods.Get(ctx, name, metav1.GetOptions{}) if err == nil { - if cur.Labels["execution.mecatl.dev/environment"] != env.GetName() { - return nil, errors.New("pod ownership label mismatch") + if cur.Labels["execution.mecatl.dev/environment"] != env.GetName() || (textNested(env.Object, "status", "pod", "uid") != "" && textNested(env.Object, "status", "pod", "uid") != string(cur.UID)) { + return nil, errors.New("pod ownership identity mismatch") } return cur, nil } @@ -268,11 +261,14 @@ func (r *Reconciler) ensurePod(ctx context.Context, env *unstructured.Unstructur memReq := p.MemoryRequest cpuLim := p.CPULimit memLim := p.MemoryLimit + ephemeralReq := p.EphemeralStorageRequest + ephemeralLim := p.EphemeralStorageLimit + tmpLim := p.TmpSizeLimit nonroot := true uid := int64(65532) noPriv := false ro := true - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName()}, OwnerReferences: []metav1.OwnerReference{{APIVersion: env.GetAPIVersion(), Kind: env.GetKind(), Name: env.GetName(), UID: env.GetUID(), Controller: &nonroot}}}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noPriv, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &nonroot, RunAsUser: &uid, RunAsGroup: &uid, FSGroup: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "executor", Image: p.Spec.Image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &noPriv, ReadOnlyRootFilesystem: &ro, RunAsNonRoot: &nonroot, RunAsUser: &uid, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: cpuReq, corev1.ResourceMemory: memReq}, Limits: corev1.ResourceList{corev1.ResourceCPU: cpuLim, corev1.ResourceMemory: memLim}}, VolumeMounts: []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}}}, Volumes: []corev1.Volume{{Name: "workspace", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc}}}, {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: quantityPtr("1Gi")}}}}}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/profile": hashText(textNested(env.Object, "spec", "profile"))[:16]}, Finalizers: []string{executorFinalizer}, OwnerReferences: []metav1.OwnerReference{{APIVersion: env.GetAPIVersion(), Kind: env.GetKind(), Name: env.GetName(), UID: env.GetUID(), Controller: &nonroot}}}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noPriv, RuntimeClassName: &p.Spec.RuntimeClassName, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &nonroot, RunAsUser: &uid, RunAsGroup: &uid, FSGroup: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "executor", Image: p.Spec.Image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &noPriv, ReadOnlyRootFilesystem: &ro, RunAsNonRoot: &nonroot, RunAsUser: &uid, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: cpuReq, corev1.ResourceMemory: memReq, corev1.ResourceEphemeralStorage: ephemeralReq}, Limits: corev1.ResourceList{corev1.ResourceCPU: cpuLim, corev1.ResourceMemory: memLim, corev1.ResourceEphemeralStorage: ephemeralLim}}, VolumeMounts: []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}}}, Volumes: []corev1.Volume{{Name: "workspace", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc}}}, {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &tmpLim}}}}}} created, err := pods.Create(ctx, pod, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return pods.Get(ctx, name, metav1.GetOptions{}) @@ -283,8 +279,8 @@ func (r *Reconciler) reconcileRetiring(ctx context.Context, env *unstructured.Un if textNested(env.Object, "status", "activeOperation", "id") != "" || textNested(env.Object, "status", "fenceState") != fenceHealthy { return r.setCondition(ctx, env, "Retired", false, "NotQuiescent", "retirement waits for proven quiescence") } - refs, _, _ := unstructured.NestedStringSlice(env.Object, "status", "references") - if len(refs) > 0 { + refs, refsErr := referenceRecords(env) + if refsErr != nil || len(refs) > 0 || textNested(env.Object, "status", "activeRun", "claimID") != "" { return r.setCondition(ctx, env, "Retired", false, "Referenced", "retirement refused while references remain") } podName := textNested(env.Object, "status", "pod", "name") @@ -314,8 +310,8 @@ func (r *Reconciler) reconcileRetiring(ctx context.Context, env *unstructured.Un return nil } func (r *Reconciler) reconcileDeletion(ctx context.Context, res dynamic.ResourceInterface, env *unstructured.Unstructured) error { - refs, _, _ := unstructured.NestedStringSlice(env.Object, "status", "references") - if textNested(env.Object, "spec", "desired") != "Retiring" || !conditionTrue(env, "Retired") || len(refs) > 0 || textNested(env.Object, "status", "activeOperation", "id") != "" || textNested(env.Object, "status", "fenceState") != fenceHealthy { + refs, refsErr := referenceRecords(env) + if refsErr != nil || textNested(env.Object, "spec", "desired") != "Retiring" || !conditionTrue(env, "Retired") || len(refs) > 0 || textNested(env.Object, "status", "activeRun", "claimID") != "" || textNested(env.Object, "status", "activeOperation", "id") != "" || textNested(env.Object, "status", "fenceState") != fenceHealthy { return r.setCondition(ctx, env, "DeletionBlocked", true, "RetentionSafety", "finalizer requires explicit completed retirement and proven quiescence") } pod := textNested(env.Object, "status", "pod", "name") @@ -425,4 +421,3 @@ func resourceName(prefix, env string) string { } return strings.TrimRight(name, "-") } -func quantityPtr(v string) *resource.Quantity { q := resource.MustParse(v); return &q } diff --git a/internal/adapter/executioncontroller/controller_lifecycle.go b/internal/adapter/executioncontroller/controller_lifecycle.go new file mode 100644 index 0000000000..4483dab02b --- /dev/null +++ b/internal/adapter/executioncontroller/controller_lifecycle.go @@ -0,0 +1,237 @@ +package executioncontroller + +import ( + "context" + "errors" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.Unstructured) error { //nolint:gocyclo // Durable phases are deliberately explicit. + op, found, err := unstructured.NestedMap(env.Object, "status", "lifecycleOperation") + if err != nil || !found { + return nil + } + kind, phase := text(op, "type"), text(op, "phase") + operationID := text(op, "id") + podUID, pvcUID := text(op, "expectedPodUID"), text(op, "expectedPVCUID") + podName, pvcName := textNested(env.Object, "status", "pod", "name"), textNested(env.Object, "status", "pvc", "name") + if operationID == "" || pvcUID == "" { + return r.setFenceUnknown(ctx, env, "lifecycle operation identity is malformed") + } + if kind == "DeleteRetiredEnvironment" { + return r.reconcileRetainedDelete(ctx, env, op, pvcName, pvcUID) + } + epoch := intNested(op, "expectedEpoch") + if podUID == "" || epoch <= 0 || textNested(env.Object, "status", "pod", "uid") != podUID || textNested(env.Object, "status", "pvc", "uid") != pvcUID || intNested(env.Object, "status", "epoch") != epoch { + return r.setFenceUnknown(ctx, env, "lifecycle operation no longer matches the exact executor and workspace") + } + pvc, err := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil || string(pvc.UID) != pvcUID { + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return r.setFenceUnknown(ctx, env, "authoritative workspace identity is unavailable") + } + proofMatches := textNested(env.Object, "status", "terminationProof", "operationID") == operationID && textNested(env.Object, "status", "terminationProof", "podUID") == podUID && textNested(env.Object, "status", "terminationProof", "pvcUID") == pvcUID && intNested(env.Object, "status", "terminationProof", "epoch") == epoch + switch phase { + case "Quiescing": + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if getErr != nil || string(pod.UID) != podUID { + if getErr != nil && !apierrors.IsNotFound(getErr) { + return getErr + } + return r.setFenceUnknown(ctx, env, "executor disappeared before durable terminal proof") + } + uid := types.UID(podUID) + grace := int64(30) + if deleteErr := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, podName, metav1.DeleteOptions{GracePeriodSeconds: &grace, Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return deleteErr + } + return r.setLifecyclePhase(ctx, env, operationID, "WaitingForTermination") + case "WaitingForTermination": + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if apierrors.IsNotFound(getErr) { + return r.setFenceUnknown(ctx, env, "executor disappeared before durable terminal proof") + } + if getErr != nil { + return getErr + } + if string(pod.UID) != podUID { + return r.setFenceUnknown(ctx, env, "executor UID changed during quiescing") + } + if !podTerminal(pod) { + return r.setCondition(ctx, env, "Ready", false, "AwaitingTerminalExecutor", "waiting for kubelet terminal phase and terminated container states") + } + q := adminLifecycleRequest{ExpectedEpoch: uint64(epoch), ExpectedPodUID: podUID, ExpectedPVCUID: pvcUID, OperationID: operationID} //nolint:gosec // positivity checked above. + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + current, ok, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !ok || text(current, "id") != operationID || text(current, "phase") != phase { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "lifecycle operation changed concurrently"} + } + if err := unstructured.SetNestedMap(o.Object, terminationProof(q, pod), "status", "terminationProof"); err != nil { + return err + } + current["phase"] = "RemovingPodFinalizer" + return unstructured.SetNestedMap(o.Object, current, "status", "lifecycleOperation") + }) + case "RemovingPodFinalizer": + if !proofMatches { + return r.setFenceUnknown(ctx, env, "durable terminal proof is missing or mismatched") + } + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if getErr == nil { + if string(pod.UID) != podUID || !podTerminal(pod) { + return r.setFenceUnknown(ctx, env, "terminal executor identity changed before finalizer removal") + } + pod = pod.DeepCopy() + pod.Finalizers = withoutString(pod.Finalizers, executorFinalizer) + if _, updateErr := r.kube.CoreV1().Pods(r.namespace).Update(ctx, pod, metav1.UpdateOptions{}); updateErr != nil { + return updateErr + } + } else if !apierrors.IsNotFound(getErr) { + return getErr + } + return r.setLifecyclePhase(ctx, env, operationID, "WaitingForPodDeletion") + case "WaitingForPodDeletion": + if !proofMatches { + return r.setFenceUnknown(ctx, env, "durable terminal proof is missing or mismatched") + } + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if getErr == nil { + if string(pod.UID) != podUID { + return r.setFenceUnknown(ctx, env, "unexpected executor occupies the retained name") + } + uid := types.UID(podUID) + if deleteErr := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, podName, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return deleteErr + } + return nil + } + if !apierrors.IsNotFound(getErr) { + return getErr + } + if kind == "RetireEnvironment" { + return r.finishRetirement(ctx, env, operationID) + } + return r.prepareReplacement(ctx, env, operationID) + case "CreatingReplacement": + return r.finishReplacement(ctx, env, operationID) + default: + return r.setFenceUnknown(ctx, env, "unknown lifecycle phase") + } +} + +func (r *Reconciler) setLifecyclePhase(ctx context.Context, env *unstructured.Unstructured, operationID, phase string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, err := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if err != nil || !found || text(op, "id") != operationID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "lifecycle operation changed concurrently"} + } + op["phase"] = phase + return unstructured.SetNestedMap(o.Object, op, "status", "lifecycleOperation") + }) +} + +func (r *Reconciler) finishRetirement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !found || text(op, "id") != operationID { + return errors.New("lifecycle operation changed concurrently") + } + unstructured.RemoveNestedField(o.Object, "status", "lifecycleOperation") + setConditionObject(o, "ExecutorTerminated", true, "TerminalPodProof", "terminal executor proof is durable") + setConditionObject(o, "Retired", true, "WorkspaceRetained", "executor retired; workspace PVC retained") + setConditionObject(o, "Ready", false, "Retained", "environment is retired with retained storage") + return nil + }) +} + +func (r *Reconciler) prepareReplacement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !found || text(op, "id") != operationID || text(op, "phase") != "WaitingForPodDeletion" { + return errors.New("lifecycle operation changed concurrently") + } + unstructured.RemoveNestedField(o.Object, "status", "pod") + op["phase"] = "CreatingReplacement" + return unstructured.SetNestedMap(o.Object, op, "status", "lifecycleOperation") + }) +} + +func (r *Reconciler) finishReplacement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { + profile, ok := r.profiles.get(textNested(env.Object, "spec", "profile")) + if !ok || profile.Digest != textNested(env.Object, "spec", "profileDigest") { + return r.setFenceUnknown(ctx, env, "replacement profile is unavailable") + } + pvcName := textNested(env.Object, "status", "pvc", "name") + pod, err := r.ensurePod(ctx, env, profile, resourceName("executor", env.GetName()), pvcName) + if err != nil { + return err + } + if !podReady(pod) { + return r.setCondition(ctx, env, "Ready", false, "ReplacementStarting", "waiting for replacement executor readiness") + } + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !found || text(op, "id") != operationID || text(op, "phase") != "CreatingReplacement" { + return errors.New("lifecycle operation changed concurrently") + } + epoch := intNested(o.Object, "status", "epoch") + if epoch <= 0 { + return errors.New("execution epoch cannot advance") + } + _ = unstructured.SetNestedField(o.Object, epoch+1, "status", "epoch") + _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pod.Name, "uid": string(pod.UID)}, "status", "pod") + unstructured.RemoveNestedField(o.Object, "status", "lifecycleOperation") + setConditionObject(o, "Ready", true, "ReplacementReady", "replacement executor is ready on the retained workspace") + return nil + }) +} + +func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructured.Unstructured, op map[string]any, pvcName, pvcUID string) error { + if text(op, "phase") != "DeletingPVC" || !conditionTrue(env, "Retired") || !conditionTrue(env, "ExecutorTerminated") { + return r.setFenceUnknown(ctx, env, "retained deletion state is invalid") + } + pvc, err := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err == nil { + if string(pvc.UID) != pvcUID { + return r.setFenceUnknown(ctx, env, "retained PVC UID changed") + } + uid := types.UID(pvcUID) + if deleteErr := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Delete(ctx, pvcName, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return deleteErr + } + return nil + } + if !apierrors.IsNotFound(err) { + return err + } + res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) + current, err := res.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + return err + } + current.SetFinalizers(withoutString(current.GetFinalizers(), environmentFinalizer)) + updated, err := res.Update(ctx, current, metav1.UpdateOptions{}) + if err != nil { + return err + } + uid := updated.GetUID() + return res.Delete(ctx, updated.GetName(), metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) +} + +func withoutString(values []string, remove string) []string { + out := make([]string, 0, len(values)) + for _, value := range values { + if value != remove { + out = append(out, value) + } + } + return out +} diff --git a/internal/adapter/executioncontroller/controller_test.go b/internal/adapter/executioncontroller/controller_test.go index 1ba985e86f..50a9c7455e 100644 --- a/internal/adapter/executioncontroller/controller_test.go +++ b/internal/adapter/executioncontroller/controller_test.go @@ -50,6 +50,12 @@ func TestReconcileCreatesTokenlessNonRootPodAndRetainedPVC(t *testing.T) { if c.SecurityContext == nil || c.SecurityContext.ReadOnlyRootFilesystem == nil || !*c.SecurityContext.ReadOnlyRootFilesystem || c.SecurityContext.AllowPrivilegeEscalation == nil || *c.SecurityContext.AllowPrivilegeEscalation { t.Fatal("container security context is not hardened") } + if p.Spec.RuntimeClassName == nil || *p.Spec.RuntimeClassName != "sandboxed" || c.Resources.Requests.Cpu().IsZero() || c.Resources.Requests.Memory().IsZero() || c.Resources.Requests.StorageEphemeral().IsZero() || c.Resources.Limits.StorageEphemeral().IsZero() { + t.Fatal("runtime class or resource bounds are missing") + } + if p.Spec.Volumes[1].EmptyDir == nil || p.Spec.Volumes[1].EmptyDir.SizeLimit == nil || p.Spec.Volumes[1].EmptyDir.SizeLimit.String() != "256Mi" { + t.Fatal("tmp volume is not profile-bounded") + } } func TestRepeatedReconcileDoesNotRewriteUnchangedStatus(t *testing.T) { ctx := context.Background() @@ -154,32 +160,36 @@ func conditionTransition(env *unstructured.Unstructured, name string) string { return "" } -func TestStartupFencesInterruptedOperationBeforeReady(t *testing.T) { +func TestStartupDoesNotFenceLivePeerOperation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() env := testEnvironment() _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "old", "operation": "file.replace"}, "status", "activeOperation") d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) r := NewReconciler(d, kubefake.NewSimpleClientset(), "ns", testProfiles()) - if r.Ready() { - t.Fatal("reconciler reported ready before startup fencing") + peer := NewReconciler(d, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if r.Ready() || peer.Ready() { + t.Fatal("reconciler reported ready before cache synchronization") } if err := r.Initialize(ctx); err != nil { t.Fatal(err) } - if !r.Ready() { - t.Fatal("reconciler did not report ready after startup fencing and cache sync") + if err := peer.Initialize(ctx); err != nil { + t.Fatal(err) + } + if !r.Ready() || !peer.Ready() { + t.Fatal("reconcilers did not report ready after cache synchronization") } got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) if err != nil { t.Fatal(err) } - if textNested(got.Object, "status", "fenceState") != "FenceUnknown" || textNested(got.Object, "status", "activeOperation", "id") != "" { - t.Fatalf("status=%v", got.Object["status"]) + if textNested(got.Object, "status", "fenceState") != "Healthy" || textNested(got.Object, "status", "activeOperation", "id") != "old" { + t.Fatalf("startup mutated a potentially live peer operation: status=%v", got.Object["status"]) } } func testProfiles() *Profiles { - spec := ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute} + spec := ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100} profile, err := validateProfile("go", spec) if err != nil { panic(err) @@ -189,5 +199,5 @@ func testProfiles() *Profiles { return &Profiles{byName: map[string]resolvedProfile{"go": profile}} } func testEnvironment() *unstructured.Unstructured { - return &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "exec-test", "namespace": "ns", "uid": string(types.UID("uid"))}, "spec": map[string]any{"profile": "go", "profileDigest": "sha256:profile", "revision": "rev", "desired": "Active"}, "status": map[string]any{"epoch": int64(1), "references": []any{}, "fenceState": "Healthy"}}} + return &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "exec-test", "namespace": "ns", "uid": string(types.UID("uid"))}, "spec": map[string]any{"schemaVersion": int64(2), "profile": "go", "profileDigest": "sha256:profile", "revision": "rev", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "references": []any{}, "fenceState": "Healthy"}}} } diff --git a/internal/adapter/executioncontroller/handler.go b/internal/adapter/executioncontroller/handler.go index 0db2cec4b7..994d86133d 100644 --- a/internal/adapter/executioncontroller/handler.go +++ b/internal/adapter/executioncontroller/handler.go @@ -1,4 +1,6 @@ // Package executioncontroller implements the authenticated provider gRPC boundary. +// +//nolint:revive // Generated gRPC method names define this private handler surface. package executioncontroller import ( @@ -17,8 +19,6 @@ import ( "unicode/utf8" "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/peer" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" @@ -44,6 +44,7 @@ type HandlerConfig struct { Signer GrantSigner Verifier executionenv.GrantVerifier Ready func() bool + Security *SecurityManager } // Profile is the externally visible immutable execution profile. @@ -57,7 +58,7 @@ type Profile struct { // Allocation is an exact environment allocation and authorization binding. type Allocation struct { Environment executionenv.EnvironmentRef - Epoch uint64 + Epoch, GrantGeneration uint64 OwnerHash, BindingID, Client string Ready bool } @@ -68,22 +69,59 @@ type Backend interface { Ensure(context.Context, string, string, string, string, string) (Allocation, error) Attach(context.Context, executionenv.EnvironmentRef, string, string, string) (Allocation, error) ReleaseReference(context.Context, executionenv.EnvironmentRef, string, string, string) error - Retire(context.Context, executionenv.EnvironmentRef, string) error File(context.Context, string, string, executionenv.FileRequest) (executionenv.FileResponse, error) StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) } +type runClaimValidator interface { + ValidateRunClaim(context.Context, string, string, executionenv.RequestContext) error +} + +type lifecycleBackend interface { + EnsurePending(context.Context, string, string, string, string, string, string) (Allocation, error) + AcquireRun(context.Context, executionenv.EnvironmentRef, string, string, string, string, string, time.Duration) (executionenv.RunClaim, error) + RenewRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) (executionenv.RunClaim, error) + ReleaseRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) error + CommitReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + AbortReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + ReserveSuccessor(context.Context, executionenv.EnvironmentRef, string, string, string, string, string) error + PrepareReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + ConfirmReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + CancelReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) +} + +type ownerIntentBackend interface { + EnsurePendingOwned(context.Context, string, string, executionenv.Owner, string, string, string, string) (Allocation, error) + ListReferenceIntentsForClient(context.Context, string, int) ([]executionenv.ReferenceIntent, error) +} + +type adminLifecycleBackend interface { + ReplaceExecutor(context.Context, adminLifecycleRequest) error + RetireExact(context.Context, adminLifecycleRequest) error + RecoverEnvironment(context.Context, adminLifecycleRequest) error + DeleteRetiredEnvironment(context.Context, adminLifecycleRequest) error + MigrateEnvironment(context.Context, adminLifecycleRequest) error + RevokeEnvironment(context.Context, executionenv.EnvironmentRef, string, string, uint64) (uint64, error) +} + // Handler is the authenticated private execution-provider gRPC service. type Handler struct { executionv1.UnimplementedExecutionProviderServiceServer - cfg HandlerConfig - backend Backend + cfg HandlerConfig + backend Backend + lifecycleBackend lifecycleBackend + adminLifecycleBackend adminLifecycleBackend } // NewHandler constructs the authenticated private gRPC service. -func NewHandler(cfg HandlerConfig, b Backend) *Handler { return &Handler{cfg: cfg, backend: b} } +func NewHandler(cfg HandlerConfig, b Backend) *Handler { + lifecycle, _ := b.(lifecycleBackend) + adminLifecycle, _ := b.(adminLifecycleBackend) + return &Handler{cfg: cfg, backend: b, lifecycleBackend: lifecycle, adminLifecycleBackend: adminLifecycle} +} type authenticatedClient struct { id string @@ -94,15 +132,18 @@ func (h *Handler) client(ctx context.Context) (authenticatedClient, error) { if h.backend == nil || (h.cfg.Ready != nil && !h.cfg.Ready()) { return authenticatedClient{}, wireError(executionenv.CodeNotReady, true) } - p, ok := peer.FromContext(ctx) - if !ok { + leaf, handshakeVerified, err := clientCertificate(ctx) + if err != nil || h.cfg.Security == nil && !handshakeVerified { return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) } - tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) - if !ok || len(tlsInfo.State.VerifiedChains) == 0 || len(tlsInfo.State.PeerCertificates) == 0 { - return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) + if h.cfg.Security != nil { + id, policy, err := h.cfg.Security.authorize(ctx, leaf) + if err != nil { + return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) + } + return authenticatedClient{id: id, policy: policy}, nil } - id, err := canonicalClientIdentity(tlsInfo.State.PeerCertificates[0]) + id, err := canonicalClientIdentity(leaf) if err != nil { return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) } @@ -132,6 +173,13 @@ func canonicalClientIdentity(c *x509.Certificate) (string, error) { return ids[0], nil } +func (h *Handler) lifecycle() (lifecycleBackend, error) { + if h.lifecycleBackend == nil { + return nil, wireError(executionenv.CodeNotReady, false) + } + return h.lifecycleBackend, nil +} + // ValidateProfile validates one operator-defined execution profile. func (h *Handler) ValidateProfile(ctx context.Context, q *executionv1.ValidateProfileRequest) (*executionv1.ValidateProfileResponse, error) { if _, err := h.client(ctx); err != nil { @@ -154,19 +202,25 @@ func (h *Handler) EnsureEnvironment(ctx context.Context, q *executionv1.EnsureEn return nil, err } owner, ok := ownerFromProto(q.GetOwner()) - if q == nil || !c.policy.MayAttestOwner || !ok || q.BindingId == "" || len(q.BindingId) > executionenv.MaxBindingBytes || q.Profile == "" || len(q.Profile) > 63 { + if q == nil || !c.policy.MayAttestOwner || !ok || q.BindingId == "" || len(q.BindingId) > executionenv.MaxBindingBytes || q.Profile == "" || len(q.Profile) > 63 || !validOperationID(q.OperationId) { return nil, wireError(executionenv.CodePermissionDenied, false) } oh := ownerHash(owner) - a, err := h.backend.Ensure(ctx, c.id, oh, q.BindingId, q.Profile, fingerprint(c.id, oh, q.BindingId, q.Profile)) - if err != nil { - return nil, backendError(err) + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + fp := fingerprint(c.id, oh, q.BindingId, q.Profile) + var a Allocation + if owned, ok := h.backend.(ownerIntentBackend); ok { + a, err = owned.EnsurePendingOwned(ctx, c.id, oh, owner, q.BindingId, q.Profile, fp, q.OperationId) + } else { + a, err = lifecycle.EnsurePending(ctx, c.id, oh, q.BindingId, q.Profile, fp, q.OperationId) } - grant, expiry, err := h.sign(a, c.id) if err != nil { - return nil, wireError(executionenv.CodeInternal, false) + return nil, backendError(err) } - return ensureResponse(a, grant, expiry), nil + return ensureResponse(a, "", time.Time{}), nil } // AttachEnvironment exactly reattaches and refreshes a short-lived grant. @@ -184,16 +238,196 @@ func (h *Handler) AttachEnvironment(ctx context.Context, q *executionv1.AttachEn if err != nil { return nil, backendError(err) } - grant, expiry, err := h.sign(a, c.id) + return attachResponse(a, "", time.Time{}), nil +} + +// AcquireRun takes environment-wide execution ownership and issues a run-bound grant. +func (h *Handler) AcquireRun(ctx context.Context, q *executionv1.AcquireRunRequest) (*executionv1.RunClaimResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + owner, ok := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if q == nil || !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(q.BindingId) || !validIdentity(q.RunId) || !validOperationID(q.OperationId) || q.TtlMillis < executionenv.MinRunTTL.Milliseconds() || q.TtlMillis > executionenv.MaxRunTTL.Milliseconds() { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + claim, err := lifecycle.AcquireRun(ctx, ref, c.id, ownerHash(owner), q.BindingId, q.RunId, q.OperationId, time.Duration(q.TtlMillis)*time.Millisecond) + if err != nil { + return nil, backendError(err) + } + return h.runClaimResponse(ctx, claim, c.id, ownerHash(owner)) +} + +func (h *Handler) RenewRun(ctx context.Context, q *executionv1.RenewRunRequest) (*executionv1.RunClaimResponse, error) { + c, owner, req, err := h.runClaimRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetBindingId(), q.GetRunId(), q.GetClaimId(), q.GetEpoch(), q.GetGrantGeneration(), q.GetOperationId(), q.GetTtlMillis()) + if err != nil { + return nil, err + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + claim, backendErr := lifecycle.RenewRun(ctx, req.Environment, c.id, owner, req) + if backendErr != nil { + return nil, backendError(backendErr) + } + return h.runClaimResponse(ctx, claim, c.id, owner) +} + +func (h *Handler) ReleaseRun(ctx context.Context, q *executionv1.ReleaseRunRequest) (*emptypb.Empty, error) { + c, owner, req, err := h.runClaimRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetBindingId(), q.GetRunId(), q.GetClaimId(), q.GetEpoch(), q.GetGrantGeneration(), q.GetOperationId(), executionenv.DefaultRunTTL.Milliseconds()) + if err != nil { + return nil, err + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + if backendErr := lifecycle.ReleaseRun(ctx, req.Environment, c.id, owner, req); backendErr != nil { + return nil, backendError(backendErr) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) runClaimRequest(ctx context.Context, pRef *executionv1.EnvironmentRef, pOwner *executionv1.Owner, binding, runID, claimID string, epoch, generation uint64, operationID string, ttlMillis int64) (authenticatedClient, string, executionenv.RunClaimRequest, error) { + c, err := h.client(ctx) + if err != nil { + return c, "", executionenv.RunClaimRequest{}, err + } + owner, ok := ownerFromProto(pOwner) + ref := refFromProto(pRef) + if !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(binding) || !validIdentity(runID) || !validIdentity(claimID) || epoch == 0 || epoch > math.MaxInt64 || generation == 0 || generation > math.MaxInt64 || !validOperationID(operationID) || ttlMillis < executionenv.MinRunTTL.Milliseconds() || ttlMillis > executionenv.MaxRunTTL.Milliseconds() { + return c, "", executionenv.RunClaimRequest{}, wireError(executionenv.CodeInvalidArgument, false) + } + return c, ownerHash(owner), executionenv.RunClaimRequest{Environment: ref, Owner: owner, BindingID: binding, RunID: runID, ClaimID: claimID, Epoch: epoch, GrantGeneration: generation, OperationID: operationID, TTL: time.Duration(ttlMillis) * time.Millisecond}, nil +} + +func (h *Handler) runClaimResponse(ctx context.Context, claim executionenv.RunClaim, client, owner string) (*executionv1.RunClaimResponse, error) { + grant, expiry, err := h.signClaim(ctx, claim, client, owner) if err != nil { return nil, wireError(executionenv.CodeInternal, false) } - return attachResponse(a, grant, expiry), nil + return &executionv1.RunClaimResponse{Environment: refToProto(claim.Environment), BindingId: valid(claim.BindingID), RunId: valid(claim.RunID), ClaimId: valid(claim.ClaimID), Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: valid(grant), ExpiresAt: timestamppb.New(expiry)}, nil +} + +func (h *Handler) referenceRequest(ctx context.Context, q *executionv1.ReferenceMutationRequest) (authenticatedClient, string, executionenv.EnvironmentRef, string, string, error) { + c, err := h.client(ctx) + if err != nil { + return c, "", executionenv.EnvironmentRef{}, "", "", err + } + owner, ok := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if q == nil || !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(q.BindingId) || !validOperationID(q.OperationId) { + return c, "", ref, "", "", wireError(executionenv.CodeInvalidArgument, false) + } + return c, ownerHash(owner), ref, q.BindingId, q.OperationId, nil +} +func (h *Handler) referenceMutation(ctx context.Context, q *executionv1.ReferenceMutationRequest, operation executionenv.Operation) (*emptypb.Empty, error) { + c, owner, ref, binding, operationID, err := h.referenceRequest(ctx, q) + if err != nil { + return nil, err + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + switch operation { + case executionenv.OpReferenceCommit: + err = lifecycle.CommitReference(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceAbort: + err = lifecycle.AbortReference(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceDeletePrepare: + err = lifecycle.PrepareReferenceDelete(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceDeleteConfirm: + err = lifecycle.ConfirmReferenceDelete(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceDeleteCancel: + err = lifecycle.CancelReferenceDelete(ctx, ref, c.id, owner, binding, operationID) + default: + err = &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "invalid reference operation"} + } + if err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} +func (h *Handler) CommitReference(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceCommit) +} +func (h *Handler) AbortReference(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceAbort) +} +func (h *Handler) PrepareReferenceDelete(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceDeletePrepare) +} +func (h *Handler) ConfirmReferenceDelete(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceDeleteConfirm) +} +func (h *Handler) CancelReferenceDelete(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceDeleteCancel) +} + +func (h *Handler) ReserveSuccessor(ctx context.Context, q *executionv1.ReserveSuccessorRequest) (*executionv1.ReferenceReservationResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + owner, ok := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if q == nil || !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(q.SourceBindingId) || !validBinding(q.DestinationBindingId) || q.SourceBindingId == q.DestinationBindingId || !validOperationID(q.OperationId) { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + if err = lifecycle.ReserveSuccessor(ctx, ref, c.id, ownerHash(owner), q.SourceBindingId, q.DestinationBindingId, q.OperationId); err != nil { + return nil, backendError(err) + } + return &executionv1.ReferenceReservationResponse{Environment: refToProto(ref)}, nil +} + +func (h *Handler) ListReferenceIntents(ctx context.Context, q *executionv1.ListReferenceIntentsRequest) (*executionv1.ListReferenceIntentsResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + if q == nil || !c.policy.MayAttestOwner || q.Limit < 0 || q.Limit > maxReferences { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + var intents []executionenv.ReferenceIntent + if q.Owner != nil { + owner, ok := ownerFromProto(q.Owner) + if !ok { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + intents, err = lifecycle.ListReferenceIntents(ctx, c.id, ownerHash(owner), int(q.Limit)) + } else if owned, ok := h.backend.(ownerIntentBackend); ok { + intents, err = owned.ListReferenceIntentsForClient(ctx, c.id, int(q.Limit)) + } else { + return nil, wireError(executionenv.CodeNotReady, true) + } + if err != nil { + return nil, backendError(err) + } + out := &executionv1.ListReferenceIntentsResponse{} + for _, intent := range intents { + out.Intents = append(out.Intents, &executionv1.ReferenceIntent{Environment: refToProto(intent.Environment), BindingId: valid(intent.BindingID), State: valid(string(intent.State)), OperationId: valid(intent.OperationID), SourceBindingId: valid(intent.SourceBindingID), CreatedAt: timestamppb.New(intent.CreatedAt), Owner: &executionv1.Owner{Issuer: valid(intent.Owner.Issuer), Subject: valid(intent.Owner.Subject)}}) + } + return out, nil } // ReleaseReference releases one authorized durable binding reference. func (h *Handler) ReleaseReference(ctx context.Context, q *executionv1.ReleaseReferenceRequest) (*emptypb.Empty, error) { - c, owner, rc, err := h.authorize(ctx, q.GetContext(), executionenv.OpReferenceRelease, false) + c, owner, rc, err := h.authorize(ctx, q.GetContext(), executionenv.OpReferenceRelease) if err != nil { return nil, err } @@ -203,19 +437,107 @@ func (h *Handler) ReleaseReference(ctx context.Context, q *executionv1.ReleaseRe return &emptypb.Empty{}, nil } -// RetireEnvironment requests authorized administrative retirement. +// RetireEnvironment starts exact, proof-producing executor retirement while retaining the PVC. func (h *Handler) RetireEnvironment(ctx context.Context, q *executionv1.RetireEnvironmentRequest) (*emptypb.Empty, error) { - c, owner, rc, err := h.authorize(ctx, q.GetContext(), executionenv.OpRetire, true) + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetExpectedExecutionEpoch(), q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if err := h.adminLifecycleBackend.RetireExact(ctx, req); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) ReplaceExecutor(ctx context.Context, q *executionv1.ReplaceExecutorRequest) (*emptypb.Empty, error) { + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetExpectedExecutionEpoch(), q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if err := h.adminLifecycleBackend.ReplaceExecutor(ctx, req); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) RecoverEnvironment(ctx context.Context, q *executionv1.RecoverEnvironmentRequest) (*emptypb.Empty, error) { + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetExpectedExecutionEpoch(), q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if err := h.adminLifecycleBackend.RecoverEnvironment(ctx, req); err != nil { + var controlled *executionenv.Error + if errors.As(err, &controlled) && controlled.Code == executionenv.CodeFenceUnknown { + return nil, status.Error(codes.FailedPrecondition, "execution provider request failed; independent termination proof or external fencing is required") + } + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) DeleteRetiredEnvironment(ctx context.Context, q *executionv1.DeleteRetiredEnvironmentRequest) (*emptypb.Empty, error) { + c, err := h.client(ctx) + owner, ownerOK := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) if err != nil { return nil, err } - _ = c - if err := h.backend.Retire(ctx, rc.Environment, owner); err != nil { + if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) || !validIdentity(q.GetExpectedPvcUid()) || !validOperationID(q.GetOperationId()) { + return nil, wireError(executionenv.CodePermissionDenied, false) + } + req := adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, ExpectedPVCUID: q.GetExpectedPvcUid(), OperationID: q.GetOperationId()} + if err := h.adminLifecycleBackend.DeleteRetiredEnvironment(ctx, req); err != nil { return nil, backendError(err) } return &emptypb.Empty{}, nil } +func (h *Handler) MigrateEnvironment(ctx context.Context, q *executionv1.MigrateEnvironmentRequest) (*emptypb.Empty, error) { + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), 1, q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if q.GetExpectedSchemaVersion() > 1 { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + req.ExpectedEpoch = 0 + req.ExpectedSchema = int64(q.GetExpectedSchemaVersion()) + if err := h.adminLifecycleBackend.MigrateEnvironment(ctx, req); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) RevokeEnvironment(ctx context.Context, q *executionv1.RevokeEnvironmentRequest) (*executionv1.RevokeEnvironmentResponse, error) { + c, err := h.client(ctx) + owner, ownerOK := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if err != nil { + return nil, err + } + if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) || q.GetExpectedGrantGeneration() == 0 || q.GetExpectedGrantGeneration() > math.MaxInt64 { + return nil, wireError(executionenv.CodePermissionDenied, false) + } + generation, err := h.adminLifecycleBackend.RevokeEnvironment(ctx, ref, c.id, ownerHash(owner), q.GetExpectedGrantGeneration()) + if err != nil { + return nil, backendError(err) + } + return &executionv1.RevokeEnvironmentResponse{GrantGeneration: generation}, nil +} + +func (h *Handler) adminRequest(ctx context.Context, pRef *executionv1.EnvironmentRef, pOwner *executionv1.Owner, epoch uint64, podUID, pvcUID, operationID string) (adminLifecycleRequest, error) { + c, err := h.client(ctx) + if err != nil { + return adminLifecycleRequest{}, err + } + owner, ownerOK := ownerFromProto(pOwner) + ref := refFromProto(pRef) + if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) || epoch == 0 || epoch > math.MaxInt64 || !validIdentity(podUID) || !validIdentity(pvcUID) || !validOperationID(operationID) { + return adminLifecycleRequest{}, wireError(executionenv.CodePermissionDenied, false) + } + return adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, ExpectedEpoch: epoch, ExpectedPodUID: podUID, ExpectedPVCUID: pvcUID, OperationID: operationID}, nil +} + // Files executes one bounded and authorized filesystem operation. func (h *Handler) Files(ctx context.Context, q *executionv1.FileRequest) (*executionv1.FileResponse, error) { if _, err := h.client(ctx); err != nil { @@ -225,7 +547,7 @@ func (h *Handler) Files(ctx context.Context, q *executionv1.FileRequest) (*execu if q == nil || !ok || !validFileRequest(q) { return nil, wireError(executionenv.CodeInvalidArgument, false) } - c, owner, rc, err := h.authorize(ctx, q.Context, op, false) + c, owner, rc, err := h.authorize(ctx, q.Context, op) if err != nil { return nil, err } @@ -245,7 +567,7 @@ func (h *Handler) StartCommand(ctx context.Context, q *executionv1.CommandStartR if q == nil || len(q.Command) == 0 || len(q.Command) > executionenv.MaxCommandBytes || q.TimeoutMillis < 0 { return nil, wireError(executionenv.CodeInvalidArgument, false) } - c, owner, rc, err := h.authorize(ctx, q.Context, executionenv.OpCommandStart, false) + c, owner, rc, err := h.authorize(ctx, q.Context, executionenv.OpCommandStart) if err != nil { return nil, err } @@ -270,7 +592,7 @@ func (h *Handler) CancelCommand(ctx context.Context, q *executionv1.CommandQuery return h.commandQuery(ctx, q, executionenv.OpCommandCancel) } func (h *Handler) commandQuery(ctx context.Context, q *executionv1.CommandQueryRequest, op executionenv.Operation) (*executionv1.CommandStatusResponse, error) { - _, _, _, err := h.authorize(ctx, q.GetContext(), op, false) + _, _, _, err := h.authorize(ctx, q.GetContext(), op) if err != nil { return nil, err } @@ -280,33 +602,43 @@ func (h *Handler) commandQuery(ctx context.Context, q *executionv1.CommandQueryR return nil, status.Error(codes.Unimplemented, "foreground commands have no detached control API") } -func (h *Handler) authorize(ctx context.Context, p *executionv1.RequestContext, op executionenv.Operation, admin bool) (authenticatedClient, string, executionenv.RequestContext, error) { +func (h *Handler) authorize(ctx context.Context, p *executionv1.RequestContext, op executionenv.Operation) (authenticatedClient, string, executionenv.RequestContext, error) { c, err := h.client(ctx) if err != nil { return c, "", executionenv.RequestContext{}, err } - if admin && !c.policy.Administrator { - return c, "", executionenv.RequestContext{}, wireError(executionenv.CodePermissionDenied, false) - } rc, ok := requestContextFromProto(p) if !ok { return c, "", rc, wireError(executionenv.CodePermissionDenied, false) } oh := ownerHash(rc.Owner) - if _, err := h.cfg.Verifier.Verify(rc.Grant, executionenv.GrantExpectation{Client: c.id, OwnerHash: oh, BindingID: rc.BindingID, Environment: rc.Environment, Epoch: rc.Epoch, Operation: op}); err != nil { + verifier := h.cfg.Verifier + if h.cfg.Security != nil { + material, materialErr := h.cfg.Security.material(ctx) + if materialErr != nil { + return c, "", rc, wireError(executionenv.CodeNotReady, true) + } + verifier = material.verifier + } + if _, err := verifier.Verify(rc.Grant, executionenv.GrantExpectation{Client: c.id, OwnerHash: oh, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: rc.GrantGeneration, Operation: op}); err != nil { if errors.Is(err, executionenv.ErrGrantExpired) { return c, "", rc, wireError(executionenv.CodeUnauthenticated, true) } return c, "", rc, wireError(executionenv.CodePermissionDenied, false) } + if validator, ok := h.backend.(runClaimValidator); ok { + if err := validator.ValidateRunClaim(ctx, c.id, oh, rc); err != nil { + return c, "", rc, backendError(err) + } + } return c, oh, rc, nil } func requestContextFromProto(p *executionv1.RequestContext) (executionenv.RequestContext, bool) { owner, ownerOK := ownerFromProto(p.GetOwner()) ref := refFromProto(p.GetEnvironment()) - rc := executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: p.GetBindingId(), Epoch: p.GetEpoch(), Grant: p.GetGrant()} - ok := p != nil && ownerOK && ref.ID != "" && ref.Revision != "" && len(ref.ID) <= executionenv.MaxIdentityBytes && len(ref.Revision) <= executionenv.MaxIdentityBytes && rc.BindingID != "" && len(rc.BindingID) <= executionenv.MaxBindingBytes && rc.Epoch != 0 && rc.Grant != "" && len(rc.Grant) <= executionenv.MaxGrantBytes + rc := executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: p.GetBindingId(), RunID: p.GetRunId(), ClaimID: p.GetClaimId(), Epoch: p.GetEpoch(), GrantGeneration: p.GetGrantGeneration(), Grant: p.GetGrant()} + ok := p != nil && ownerOK && ref.ID != "" && ref.Revision != "" && len(ref.ID) <= executionenv.MaxIdentityBytes && len(ref.Revision) <= executionenv.MaxIdentityBytes && rc.BindingID != "" && len(rc.BindingID) <= executionenv.MaxBindingBytes && validIdentity(rc.RunID) && validIdentity(rc.ClaimID) && rc.Epoch != 0 && rc.GrantGeneration != 0 && rc.Grant != "" && len(rc.Grant) <= executionenv.MaxGrantBytes return rc, ok } func attachContext(p *executionv1.RequestContext) (executionenv.RequestContext, bool) { @@ -358,10 +690,10 @@ func operationFromProto(op executionv1.FileOperation) (executionenv.Operation, b } func ensureResponse(a Allocation, grant string, expiry time.Time) *executionv1.EnsureEnvironmentResponse { - return &executionv1.EnsureEnvironmentResponse{Environment: refToProto(a.Environment), Epoch: a.Epoch, Ready: a.Ready, Grant: valid(grant), GrantExpiresAt: timestamppb.New(expiry)} + return &executionv1.EnsureEnvironmentResponse{Environment: refToProto(a.Environment), Epoch: a.Epoch, Ready: a.Ready, GrantGeneration: a.GrantGeneration, Grant: valid(grant), GrantExpiresAt: timestamppb.New(expiry)} } func attachResponse(a Allocation, grant string, expiry time.Time) *executionv1.AttachEnvironmentResponse { - return &executionv1.AttachEnvironmentResponse{Environment: refToProto(a.Environment), Epoch: a.Epoch, Ready: a.Ready, Grant: valid(grant), GrantExpiresAt: timestamppb.New(expiry)} + return &executionv1.AttachEnvironmentResponse{Environment: refToProto(a.Environment), Epoch: a.Epoch, Ready: a.Ready, GrantGeneration: a.GrantGeneration, Grant: valid(grant), GrantExpiresAt: timestamppb.New(expiry)} } func fileResponse(v executionenv.FileResponse) *executionv1.FileResponse { r := &executionv1.FileResponse{Data: v.Data, Version: []byte(v.Version), Paths: validAll(v.Paths), AuthorityTarget: valid(v.AuthorityTarget), AuthorityWorkspace: valid(v.AuthorityWorkspace)} @@ -424,8 +756,23 @@ func commandStateToProto(v executionenv.CommandState) (executionv1.CommandState, } } -func (h *Handler) sign(a Allocation, client string) (string, time.Time, error) { - life := h.cfg.Signer.Lifetime +func validIdentity(v string) bool { return v != "" && len(v) <= executionenv.MaxIdentityBytes } +func validBinding(v string) bool { return v != "" && len(v) <= executionenv.MaxBindingBytes } +func validOperationID(v string) bool { return validIdentity(v) } +func validRef(v executionenv.EnvironmentRef) bool { + return validIdentity(v.ID) && validIdentity(v.Revision) +} + +func (h *Handler) signClaim(ctx context.Context, claim executionenv.RunClaim, client, owner string) (string, time.Time, error) { + signer := h.cfg.Signer + if h.cfg.Security != nil { + material, err := h.cfg.Security.material(ctx) + if err != nil { + return "", time.Time{}, err + } + signer = material.signer + } + life := signer.Lifetime if life <= 0 { life = time.Minute } @@ -434,8 +781,8 @@ func (h *Handler) sign(a Allocation, client string) (string, time.Time, error) { if _, err := rand.Read(nonce); err != nil { return "", time.Time{}, err } - ops := []executionenv.Operation{executionenv.OpAttach, executionenv.OpReferenceRelease, executionenv.OpRetire, executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileCreate, executionenv.OpFileReplace, executionenv.OpFileList, executionenv.OpFileRemove, executionenv.OpFileRename, executionenv.OpFileCopy, executionenv.OpFileGlob, executionenv.OpFileGrep, executionenv.OpCommandStart, executionenv.OpCommandStatus, executionenv.OpCommandCancel} - grant, err := executionenv.SignGrant(h.cfg.Signer.PrivateKey, executionenv.GrantClaims{KeyID: h.cfg.Signer.KeyID, Issuer: h.cfg.Signer.Issuer, Audience: h.cfg.Signer.Audience, Client: client, OwnerHash: a.OwnerHash, BindingID: a.BindingID, Environment: a.Environment, Epoch: a.Epoch, Operations: ops, NotBefore: now, ExpiresAt: expiry, Nonce: hex.EncodeToString(nonce)}) + ops := []executionenv.Operation{executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileCreate, executionenv.OpFileReplace, executionenv.OpFileList, executionenv.OpFileRemove, executionenv.OpFileRename, executionenv.OpFileCopy, executionenv.OpFileGlob, executionenv.OpFileGrep, executionenv.OpCommandStart, executionenv.OpCommandStatus, executionenv.OpCommandCancel} + grant, err := executionenv.SignGrant(signer.PrivateKey, executionenv.GrantClaims{KeyID: signer.KeyID, Issuer: signer.Issuer, Audience: signer.Audience, Client: client, OwnerHash: owner, BindingID: claim.BindingID, RunID: claim.RunID, ClaimID: claim.ClaimID, Environment: claim.Environment, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Operations: ops, NotBefore: now, ExpiresAt: expiry, Nonce: hex.EncodeToString(nonce)}) return grant, expiry, err } func ownerHash(o executionenv.Owner) string { diff --git a/internal/adapter/executioncontroller/handler_test.go b/internal/adapter/executioncontroller/handler_test.go index 08d099d9e6..2d0ce56b55 100644 --- a/internal/adapter/executioncontroller/handler_test.go +++ b/internal/adapter/executioncontroller/handler_test.go @@ -64,6 +64,40 @@ func (*fakeBackend) CancelCommand(context.Context, string, string, executionenv. return executionenv.CommandStatusResponse{}, nil } +func (f *fakeBackend) EnsurePending(ctx context.Context, client, owner, binding, profile, fp, _ string) (Allocation, error) { + return f.Ensure(ctx, client, owner, binding, profile, fp) +} +func (*fakeBackend) AcquireRun(_ context.Context, ref executionenv.EnvironmentRef, _ string, _ string, binding, run, _ string, ttl time.Duration) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: run, ClaimID: "claim", Epoch: 2, GrantGeneration: 1, ExpiresAt: time.Now().Add(ttl)}, nil +} +func (*fakeBackend) RenewRun(_ context.Context, _ executionenv.EnvironmentRef, _ string, _ string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: req.ClaimID, Epoch: req.Epoch, GrantGeneration: 1, ExpiresAt: time.Now().Add(req.TTL)}, nil +} +func (*fakeBackend) ReleaseRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) error { + return nil +} +func (*fakeBackend) CommitReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) AbortReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) ReserveSuccessor(context.Context, executionenv.EnvironmentRef, string, string, string, string, string) error { + return nil +} +func (*fakeBackend) PrepareReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) ConfirmReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) CancelReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) { + return nil, nil +} + func authenticatedContext(id string) context.Context { u, _ := url.Parse(id) cert := &x509.Certificate{URIs: []*url.URL{u}} @@ -106,7 +140,7 @@ func TestHandlerValidateIsReadOnlyAndEnsureIdempotent(t *testing.T) { if backend.ensureCalls != 0 { t.Fatal("validation allocated") } - q := &executionv1.EnsureEnvironmentRequest{BindingId: "session-1", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}} + q := &executionv1.EnsureEnvironmentRequest{BindingId: "session-1", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, OperationId: "create"} first, err := h.EnsureEnvironment(ctx, q) if err != nil { t.Fatal(err) @@ -135,7 +169,7 @@ func TestHandlerWithNilBackendRejectsEveryRPCWithoutPanicking(t *testing.T) { func(valid bool) error { var q *executionv1.EnsureEnvironmentRequest if valid { - q = &executionv1.EnsureEnvironmentRequest{BindingId: "binding", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}} + q = &executionv1.EnsureEnvironmentRequest{BindingId: "binding", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, OperationId: "create"} } _, err := h.EnsureEnvironment(ctx, q) return err @@ -159,7 +193,7 @@ func TestHandlerWithNilBackendRejectsEveryRPCWithoutPanicking(t *testing.T) { func(valid bool) error { var q *executionv1.RetireEnvironmentRequest if valid { - q = &executionv1.RetireEnvironmentRequest{Context: &executionv1.RequestContext{}} + q = &executionv1.RetireEnvironmentRequest{Environment: &executionv1.EnvironmentRef{}, Owner: &executionv1.Owner{}} } _, err := h.RetireEnvironment(ctx, q) return err @@ -212,11 +246,15 @@ func TestHandlerRejectsUnknownBackendCommandState(t *testing.T) { backend.commandState = executionenv.CommandState("future") h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {MayAttestOwner: true}}, Signer: GrantSigner{KeyID: "k1", PrivateKey: priv, Issuer: "provider", Audience: "execution", Lifetime: time.Minute}, Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "provider", Audience: "execution"}}, backend) ctx := authenticatedContext("spiffe://cluster/ns/mecak8s") - ensured, err := h.EnsureEnvironment(ctx, &executionv1.EnsureEnvironmentRequest{BindingId: "session-1", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}}) + ensured, err := h.EnsureEnvironment(ctx, &executionv1.EnsureEnvironmentRequest{BindingId: "session-1", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, OperationId: "create"}) + if err != nil { + t.Fatal(err) + } + claim, err := h.AcquireRun(ctx, &executionv1.AcquireRunRequest{Environment: ensured.Environment, Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, BindingId: "session-1", RunId: "run", OperationId: "acquire", TtlMillis: time.Minute.Milliseconds()}) if err != nil { t.Fatal(err) } - q := &executionv1.CommandStartRequest{Context: &executionv1.RequestContext{Environment: ensured.Environment, Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, BindingId: "session-1", Epoch: ensured.Epoch, Grant: ensured.Grant}, Command: "true"} + q := &executionv1.CommandStartRequest{Context: &executionv1.RequestContext{Environment: claim.Environment, Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, BindingId: "session-1", RunId: claim.RunId, ClaimId: claim.ClaimId, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant}, Command: "true"} if _, err := h.StartCommand(ctx, q); status.Code(err) != codes.Internal || status.Convert(err).Message() != "execution provider request failed" { t.Fatalf("unexpected invalid-state result: code=%v message=%q", status.Code(err), status.Convert(err).Message()) } diff --git a/internal/adapter/executioncontroller/operation_lease.go b/internal/adapter/executioncontroller/operation_lease.go new file mode 100644 index 0000000000..5f0bcfe38f --- /dev/null +++ b/internal/adapter/executioncontroller/operation_lease.go @@ -0,0 +1,50 @@ +package executioncontroller + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func operationMatches(o *unstructured.Unstructured, operationID, holderID, claimID string, epoch uint64) bool { + m, found, err := unstructured.NestedMap(o.Object, "status", "activeOperation") + return err == nil && found && text(m, "id") == operationID && text(m, "holderID") == holderID && text(m, "claimID") == claimID && intNested(m, "epoch") == int64(epoch) //nolint:gosec // epochs are bounded before persistence. +} + +func (s *Store) renewOperationLease(ctx context.Context, environment, operationID, claimID string, epoch uint64) error { + interval := s.opTTL / 3 + if interval <= 0 { + interval = operationLeaseTTL / 3 + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + err := s.retryUpdateStatus(ctx, environment, func(o *unstructured.Unstructured) error { + if !operationMatches(o, operationID, s.holderID, claimID, epoch) { + return incompatibleOperationError() + } + m, _, _ := unstructured.NestedMap(o.Object, "status", "activeOperation") + now := s.now() + m["renewedAt"] = now.Format(time.RFC3339Nano) + m["expiresAt"] = now.Add(s.opTTL).Format(time.RFC3339Nano) + return unstructured.SetNestedMap(o.Object, m, "status", "activeOperation") + }) + if err != nil { + return err + } + } + } +} + +func incompatibleOperationError() error { + return &operationLeaseError{} +} + +type operationLeaseError struct{} + +func (*operationLeaseError) Error() string { return "operation lease ownership changed" } diff --git a/internal/adapter/executioncontroller/production_lifecycle_test.go b/internal/adapter/executioncontroller/production_lifecycle_test.go new file mode 100644 index 0000000000..d4b2a83e5b --- /dev/null +++ b/internal/adapter/executioncontroller/production_lifecycle_test.go @@ -0,0 +1,207 @@ +package executioncontroller + +import ( + "context" + "errors" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func lifecycleAdminEnvironment(schema int64, refs []any) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", + "metadata": map[string]any{"name": "env", "namespace": "ns", "uid": "env-uid", "finalizers": []any{environmentFinalizer}}, + "spec": map[string]any{"schemaVersion": schema, "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, + "status": map[string]any{"schemaVersion": schema, "epoch": int64(4), "grantGeneration": int64(1), "fenceState": fenceHealthy, "references": refs, + "pvc": map[string]any{"name": "workspace", "uid": "pvc-uid"}, "pod": map[string]any{"name": "executor", "uid": "pod-uid"}, + "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}, + }} +} + +func terminalExecutor() *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "executor", Namespace: "ns", UID: types.UID("pod-uid"), Finalizers: []string{executorFinalizer}, Labels: map[string]string{"execution.mecatl.dev/environment": "env"}}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "executor"}}}, Status: corev1.PodStatus{Phase: corev1.PodFailed, ContainerStatuses: []corev1.ContainerStatus{{Name: "executor", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 137}}}}}} +} + +func retainedPVC() *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "workspace", Namespace: "ns", UID: types.UID("pvc-uid"), Labels: map[string]string{"execution.mecatl.dev/environment": "env"}}} +} + +func adminRequestFixture() adminLifecycleRequest { + return adminLifecycleRequest{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, OwnerHash: "owner", Client: "client", ExpectedEpoch: 4, ExpectedPodUID: "pod-uid", ExpectedPVCUID: "pvc-uid", OperationID: "admin-operation"} +} + +func TestOldSchemaFailsClosedAndExplicitMigrationConvertsReferences(t *testing.T) { + env := lifecycleAdminEnvironment(0, []any{"session-a"}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kube := kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC()) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kube) + _, err := store.Attach(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "session-a") + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeNotReady { + t.Fatalf("old schema attach error=%v", err) + } + q := adminRequestFixture() + q.ExpectedSchema = 0 + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatalf("migration retry was not idempotent: %v", err) + } + got, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if intNested(got.Object, "spec", "schemaVersion") != 2 || intNested(got.Object, "status", "schemaVersion") != 2 || intNested(got.Object, "status", "epoch") != 5 { + t.Fatalf("migration did not advance schema and epoch: %v", got.Object) + } + refs, err := referenceRecords(got) + if err != nil || len(refs) != 1 || refs[0].BindingID != "session-a" || refs[0].State != executionenv.ReferencePublished { + t.Fatalf("migrated refs=%+v err=%v", refs, err) + } +} + +func TestUnknownSchemaMigrationRejectedWithoutRewrite(t *testing.T) { + env := lifecycleAdminEnvironment(7, []any{}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + q.ExpectedSchema = 7 + if err := store.MigrateEnvironment(t.Context(), q); err == nil { + t.Fatal("unknown schema migrated") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if intNested(got.Object, "spec", "schemaVersion") != 7 || intNested(got.Object, "status", "epoch") != 4 { + t.Fatalf("unknown schema was rewritten: %v", got.Object) + } +} + +func TestExpiredOperationLeaseFencesWithoutClearingIdentity(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "peer-op", "operation": "file.replace", "claimID": "claim", "runID": "run", "epoch": int64(4), "holderID": "peer", "renewedAt": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano), "expiresAt": time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano)}, "status", "activeOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + r := NewReconciler(dynamicClient, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != "FenceUnknown" || textNested(got.Object, "status", "activeOperation", "id") != "peer-op" { + t.Fatalf("expired operation was not retained and fenced: %v", got.Object["status"]) + } +} + +func TestRecoverRequiresExactTerminalPodAndPersistsProof(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "uncertain", "claimID": "claim", "epoch": int64(4)}, "status", "activeOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + q.OperationID = "recover-op" + if err := store.RecoverEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != fenceHealthy || textNested(got.Object, "status", "activeOperation", "id") != "" || textNested(got.Object, "status", "terminationProof", "podUID") != "pod-uid" { + t.Fatalf("recovery proof/status=%v", got.Object["status"]) + } +} + +func TestRecoverMissingPodRemainsFenceUnknown(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(retainedPVC())) + if err := store.RecoverEnvironment(t.Context(), adminRequestFixture()); err == nil { + t.Fatal("missing executor accepted as termination proof") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != "FenceUnknown" || textNested(got.Object, "status", "terminationProof", "podUID") != "" { + t.Fatalf("missing executor changed fenced state: %v", got.Object["status"]) + } +} + +func TestReplacementQuiescesAndPendingReferenceBlocksRetirement(t *testing.T) { + published := map[string]any{"bindingID": "session", "state": "Published", "operationID": "publish", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} + env := lifecycleAdminEnvironment(2, []any{published}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + q := adminRequestFixture() + if err := store.ReplaceExecutor(t.Context(), q); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "phase") != "Quiescing" || conditionTrue(got, "Ready") { + t.Fatalf("replacement did not consume admission: %v", got.Object["status"]) + } + if _, err := store.AcquireRun(t.Context(), q.Environment, "client", "owner", "session", "run", "acquire", time.Minute); err == nil { + t.Fatal("run admitted during replacement") + } + + pending := map[string]any{"bindingID": "pending", "state": "PendingDelete", "operationID": "delete", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} + env = lifecycleAdminEnvironment(2, []any{pending}) + store = NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), "ns", testProfiles(), nil) + if err := store.RetireExact(t.Context(), q); err == nil { + t.Fatal("pending reference did not block retirement") + } +} + +func TestReplacementPersistsTerminalProofBeforeRemovingPodFinalizer(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "replace", "type": "ReplaceExecutor", "phase": "WaitingForTermination", "expectedEpoch": int64(4), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kube := kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC()) + r := NewReconciler(dynamicClient, kube, "ns", testProfiles()) + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "terminationProof", "podUID") != "pod-uid" || textNested(got.Object, "status", "lifecycleOperation", "phase") != "RemovingPodFinalizer" { + t.Fatalf("terminal proof was not persisted first: %v", got.Object["status"]) + } + pod, _ := kube.CoreV1().Pods("ns").Get(t.Context(), "executor", metav1.GetOptions{}) + if !contains(pod.Finalizers, executorFinalizer) { + t.Fatal("executor finalizer was removed in the proof-persistence transition") + } + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + pod, err := kube.CoreV1().Pods("ns").Get(t.Context(), "executor", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if contains(pod.Finalizers, executorFinalizer) { + t.Fatal("executor finalizer remained after durable exact proof") + } + got, _ = dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "phase") != "WaitingForPodDeletion" { + t.Fatalf("phase=%v", got.Object["status"]) + } +} + +func TestWrongPVCUIDCannotStartRetainedDeletion(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + q := adminRequestFixture() + q.ExpectedPVCUID = "wrong" + if err := store.DeleteRetiredEnvironment(t.Context(), q); err == nil { + t.Fatal("wrong PVC UID admitted for deletion") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "id") != "" { + t.Fatalf("wrong UID created delete operation: %v", got.Object["status"]) + } +} diff --git a/internal/adapter/executioncontroller/profiles.go b/internal/adapter/executioncontroller/profiles.go index d38eb36e18..abc3fcae24 100644 --- a/internal/adapter/executioncontroller/profiles.go +++ b/internal/adapter/executioncontroller/profiles.go @@ -20,28 +20,36 @@ type ProfilesFile struct { // ProfileSpec defines immutable image, storage, resource, and operation bounds. type ProfileSpec struct { - Image string `yaml:"image"` - StorageClass string `yaml:"storageClass"` - StorageSize string `yaml:"storageSize"` - CPURequest string `yaml:"cpuRequest"` - MemoryRequest string `yaml:"memoryRequest"` - CPULimit string `yaml:"cpuLimit"` - MemoryLimit string `yaml:"memoryLimit"` - MaxFileBytes int64 `yaml:"maxFileBytes"` - MaxCommandBytes int64 `yaml:"maxCommandBytes"` - MaxCommandDuration time.Duration `yaml:"maxCommandDuration"` + Image string `yaml:"image"` + StorageClass string `yaml:"storageClass"` + StorageSize string `yaml:"storageSize"` + CPURequest string `yaml:"cpuRequest"` + MemoryRequest string `yaml:"memoryRequest"` + CPULimit string `yaml:"cpuLimit"` + MemoryLimit string `yaml:"memoryLimit"` + EphemeralStorageRequest string `yaml:"ephemeralStorageRequest"` + EphemeralStorageLimit string `yaml:"ephemeralStorageLimit"` + TmpSizeLimit string `yaml:"tmpSizeLimit"` + RuntimeClassName string `yaml:"runtimeClassName"` + MaxFileBytes int64 `yaml:"maxFileBytes"` + MaxCommandBytes int64 `yaml:"maxCommandBytes"` + MaxCommandDuration time.Duration `yaml:"maxCommandDuration"` + MaxEnvironments int `yaml:"maxEnvironments"` } // Profiles is a validated immutable profile registry. type Profiles struct{ byName map[string]resolvedProfile } type resolvedProfile struct { - Spec ProfileSpec - Digest string - StorageSize resource.Quantity - CPURequest resource.Quantity - MemoryRequest resource.Quantity - CPULimit resource.Quantity - MemoryLimit resource.Quantity + Spec ProfileSpec + Digest string + StorageSize resource.Quantity + CPURequest resource.Quantity + MemoryRequest resource.Quantity + CPULimit resource.Quantity + MemoryLimit resource.Quantity + EphemeralStorageRequest resource.Quantity + EphemeralStorageLimit resource.Quantity + TmpSizeLimit resource.Quantity } // LoadProfiles reads and strictly validates an operator profile file. @@ -84,8 +92,8 @@ func validateProfile(name string, s ProfileSpec) (resolvedProfile, error) { if s.StorageClass == "" || s.StorageSize == "" { return resolvedProfile{}, fmt.Errorf("profile %q requires explicit storage class and size", name) } - if s.CPURequest == "" || s.MemoryRequest == "" || s.CPULimit == "" || s.MemoryLimit == "" { - return resolvedProfile{}, fmt.Errorf("profile %q requires explicit resource requests and limits", name) + if s.CPURequest == "" || s.MemoryRequest == "" || s.CPULimit == "" || s.MemoryLimit == "" || s.EphemeralStorageRequest == "" || s.EphemeralStorageLimit == "" || s.TmpSizeLimit == "" || s.RuntimeClassName == "" { + return resolvedProfile{}, fmt.Errorf("profile %q requires explicit resource requests, limits, tmp size, and runtime class", name) } quantities, err := resolveProfileQuantities(name, s) if err != nil { @@ -118,19 +126,35 @@ func resolveProfileQuantities(name string, s ProfileSpec) (resolvedProfile, erro if err != nil { return resolvedProfile{}, err } + ephemeralRequest, err := positiveQuantity(name, "ephemeralStorageRequest", s.EphemeralStorageRequest) + if err != nil { + return resolvedProfile{}, err + } + ephemeralLimit, err := positiveQuantity(name, "ephemeralStorageLimit", s.EphemeralStorageLimit) + if err != nil { + return resolvedProfile{}, err + } + tmpLimit, err := positiveQuantity(name, "tmpSizeLimit", s.TmpSizeLimit) + if err != nil { + return resolvedProfile{}, err + } if cpuRequest.Cmp(cpuLimit) > 0 { return resolvedProfile{}, fmt.Errorf("profile %q cpuRequest exceeds cpuLimit", name) } if memoryRequest.Cmp(memoryLimit) > 0 { return resolvedProfile{}, fmt.Errorf("profile %q memoryRequest exceeds memoryLimit", name) } - return resolvedProfile{StorageSize: storage, CPURequest: cpuRequest, MemoryRequest: memoryRequest, CPULimit: cpuLimit, MemoryLimit: memoryLimit}, nil + if ephemeralRequest.Cmp(ephemeralLimit) > 0 || tmpLimit.Cmp(ephemeralLimit) > 0 { + return resolvedProfile{}, fmt.Errorf("profile %q ephemeral storage request or tmp limit exceeds ephemeral storage limit", name) + } + return resolvedProfile{StorageSize: storage, CPURequest: cpuRequest, MemoryRequest: memoryRequest, CPULimit: cpuLimit, MemoryLimit: memoryLimit, EphemeralStorageRequest: ephemeralRequest, EphemeralStorageLimit: ephemeralLimit, TmpSizeLimit: tmpLimit}, nil } func validExecutionBounds(s ProfileSpec) bool { return s.MaxFileBytes > 0 && s.MaxFileBytes <= 5<<20 && s.MaxCommandBytes > 0 && s.MaxCommandBytes <= 1<<20 && - s.MaxCommandDuration > 0 && s.MaxCommandDuration <= 30*time.Minute + s.MaxCommandDuration > 0 && s.MaxCommandDuration <= 30*time.Minute && + s.MaxEnvironments > 0 && s.MaxEnvironments <= 10_000 } func positiveQuantity(profile, field, value string) (resource.Quantity, error) { diff --git a/internal/adapter/executioncontroller/profiles_test.go b/internal/adapter/executioncontroller/profiles_test.go index a1e25b35a8..563f95ea20 100644 --- a/internal/adapter/executioncontroller/profiles_test.go +++ b/internal/adapter/executioncontroller/profiles_test.go @@ -37,11 +37,13 @@ func TestLoadProfilesStrictAndDigestPinned(t *testing.T) { func TestLoadProfilesRejectsInvalidQuantities(t *testing.T) { cases := map[string]string{ - "malformed": strings.Replace(validProfileYAML(), "cpuRequest: 100m", "cpuRequest: invalid", 1), - "zero": strings.Replace(validProfileYAML(), "memoryRequest: 128Mi", `memoryRequest: "0"`, 1), - "negative": strings.Replace(validProfileYAML(), `cpuLimit: "1"`, `cpuLimit: "-1"`, 1), - "cpu request over limit": strings.Replace(validProfileYAML(), "cpuRequest: 100m", "cpuRequest: 2", 1), - "memory request over limit": strings.Replace(validProfileYAML(), "memoryLimit: 1Gi", "memoryLimit: 64Mi", 1), + "malformed": strings.Replace(validProfileYAML(), "cpuRequest: 100m", "cpuRequest: invalid", 1), + "zero": strings.Replace(validProfileYAML(), "memoryRequest: 128Mi", `memoryRequest: "0"`, 1), + "negative": strings.Replace(validProfileYAML(), `cpuLimit: "1"`, `cpuLimit: "-1"`, 1), + "cpu request over limit": strings.Replace(validProfileYAML(), "cpuRequest: 100m", "cpuRequest: 2", 1), + "memory request over limit": strings.Replace(validProfileYAML(), "memoryLimit: 1Gi", "memoryLimit: 64Mi", 1), + "ephemeral request over limit": strings.Replace(validProfileYAML(), "ephemeralStorageLimit: 1Gi", "ephemeralStorageLimit: 32Mi", 1), + "tmp over ephemeral limit": strings.Replace(validProfileYAML(), "tmpSizeLimit: 256Mi", "tmpSizeLimit: 2Gi", 1), } for name, content := range cases { t.Run(name, func(t *testing.T) { @@ -87,8 +89,13 @@ func validProfileYAML() string { memoryRequest: 128Mi cpuLimit: "1" memoryLimit: 1Gi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: sandboxed maxFileBytes: 1048576 maxCommandBytes: 65536 maxCommandDuration: 1m + maxEnvironments: 100 ` } diff --git a/internal/adapter/executioncontroller/security.go b/internal/adapter/executioncontroller/security.go new file mode 100644 index 0000000000..cc3571273c --- /dev/null +++ b/internal/adapter/executioncontroller/security.go @@ -0,0 +1,567 @@ +package executioncontroller + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "math" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync/atomic" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + securityManifestVersion = 1 + maxSecurityKeys = 64 + maxSecurityClients = 256 + securityStateDataKey = "state.json" +) + +type securityKeyManifest struct { + ID string `json:"id"` + Version uint64 `json:"version"` + File string `json:"file"` + PublicSHA256 string `json:"publicKeySHA256"` + ActivateAt time.Time `json:"activateAt"` + VerifyUntil time.Time `json:"verifyUntil"` + State string `json:"state"` +} + +type securityClientManifest struct { + URI string `json:"uri"` + MayAttestOwner bool `json:"mayAttestOwner"` + Administrator bool `json:"administrator"` +} + +type securityTLSManifest struct { + CertificateFile string `json:"certificateFile"` + PrivateKeyFile string `json:"privateKeyFile"` + ClientCAFile string `json:"clientCAFile"` +} + +type securityManifest struct { + Version int `json:"version"` + Generation uint64 `json:"generation"` + Issuer string `json:"issuer"` + Audience string `json:"audience"` + ActiveKeyID string `json:"activeKeyID"` + GrantTTL time.Duration `json:"-"` + GrantTTLText string `json:"grantTTL"` + ClockSkewText string `json:"clockSkew"` + Keys []securityKeyManifest `json:"keys"` + TLS securityTLSManifest `json:"tls"` + Clients []securityClientManifest `json:"clients"` +} + +type securitySnapshot struct { + generation uint64 + signer GrantSigner + verifier executionenv.GrantVerifier + tlsConfig *tls.Config + clientCAs *x509.CertPool + clients map[string]ClientPolicy + validUntil time.Time + fingerprints map[string]string +} + +type securityLedger struct { + Generation uint64 `json:"generation"` + Fingerprints map[string]string `json:"fingerprints"` +} + +// SecurityManager reloads one immutable, versioned security snapshot from +// projected files and binds its generation to a durable ConfigMap high-water mark. +type SecurityManager struct { + manifestPath, keyDirectory, namespace, configMap string + kube kubernetes.Interface + now func() time.Time + current atomic.Pointer[securitySnapshot] + ready atomic.Bool +} + +// NewSecurityManager constructs a fail-closed security material reloader. +func NewSecurityManager(manifestPath, keyDirectory, namespace, configMap string, kube kubernetes.Interface) *SecurityManager { + return &SecurityManager{manifestPath: manifestPath, keyDirectory: keyDirectory, namespace: namespace, configMap: configMap, kube: kube, now: func() time.Time { return time.Now().UTC() }} +} + +// Ready reports whether the current snapshot is authoritative and unexpired. +func (m *SecurityManager) Ready() bool { + s := m.current.Load() + return m.ready.Load() && s != nil && m.now().Before(s.validUntil) +} + +// CheckReady verifies that the loaded snapshot still matches durable authority. +func (m *SecurityManager) CheckReady(ctx context.Context) bool { + _, err := m.authoritative(ctx) + return err == nil +} + +// Reload atomically validates and publishes a complete security snapshot. +func (m *SecurityManager) Reload(ctx context.Context) error { + candidate, err := m.load() + if err != nil { + m.ready.Store(false) + return err + } + if err := m.publishGeneration(ctx, candidate); err != nil { + m.ready.Store(false) + return err + } + m.current.Store(candidate) + m.ready.Store(true) + return nil +} + +// Run periodically reloads security material until ctx is cancelled. +func (m *SecurityManager) Run(ctx context.Context, interval time.Duration) { + if interval <= 0 { + interval = 2 * time.Second + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _ = m.Reload(ctx) + } + } +} + +// TLSConfig returns a dynamic TLS configuration backed by the current snapshot. +func (m *SecurityManager) TLSConfig() *tls.Config { + return &tls.Config{MinVersion: tls.VersionTLS13, ClientAuth: tls.RequireAnyClientCert, GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + s := m.current.Load() + if !m.Ready() { + return nil, errors.New("security material is not ready") + } + return s.tlsConfig.Clone(), nil + }} +} + +func (m *SecurityManager) authorize(ctx context.Context, leaf *x509.Certificate) (string, ClientPolicy, error) { + s, err := m.authoritative(ctx) + if err != nil { + return "", ClientPolicy{}, err + } + opts := x509.VerifyOptions{Roots: s.clientCAs, Intermediates: x509.NewCertPool(), CurrentTime: m.now(), KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + if _, err := leaf.Verify(opts); err != nil { + return "", ClientPolicy{}, err + } + id, err := canonicalClientIdentity(leaf) + if err != nil { + return "", ClientPolicy{}, err + } + policy, ok := s.clients[id] + if !ok { + return "", ClientPolicy{}, errors.New("client is not authorized") + } + return id, policy, nil +} + +func (m *SecurityManager) material(ctx context.Context) (*securitySnapshot, error) { + return m.authoritative(ctx) +} + +func (m *SecurityManager) authoritative(ctx context.Context) (*securitySnapshot, error) { + s := m.current.Load() + if !m.Ready() || s == nil || m.kube == nil { + return nil, errors.New("security material is not ready") + } + cm, err := m.kube.CoreV1().ConfigMaps(m.namespace).Get(ctx, m.configMap, metav1.GetOptions{}) + if err != nil { + m.ready.Store(false) + return nil, err + } + var ledger securityLedger + if err := json.Unmarshal([]byte(cm.Data[securityStateDataKey]), &ledger); err != nil || ledger.Generation != s.generation || !sameFingerprints(ledger.Fingerprints, s.fingerprints) { + m.ready.Store(false) + return nil, errors.New("loaded security generation is not authoritative") + } + return s, nil +} + +func (m *SecurityManager) load() (*securitySnapshot, error) { + mf, ttl, skew, err := m.loadManifest() + if err != nil { + return nil, err + } + root, err := os.OpenRoot(m.keyDirectory) + if err != nil { + return nil, fmt.Errorf("open security directory: %w", err) + } + defer func() { _ = root.Close() }() + + keys, active, fingerprints, err := m.loadGrantKeys(root, mf) + if err != nil { + return nil, err + } + tlsConfig, clientCAs, validUntil, err := m.loadTLS(root, mf.TLS) + if err != nil { + return nil, err + } + clients, err := clientPolicies(mf.Clients) + if err != nil { + return nil, err + } + return &securitySnapshot{ + generation: mf.Generation, + signer: GrantSigner{ + KeyID: mf.ActiveKeyID, PrivateKey: active, Issuer: mf.Issuer, + Audience: mf.Audience, Lifetime: ttl, + }, + verifier: executionenv.GrantVerifier{ + Keys: keys, Issuer: mf.Issuer, Audience: mf.Audience, MaxLifetime: ttl + skew, + }, + tlsConfig: tlsConfig, clientCAs: clientCAs, clients: clients, + validUntil: validUntil, fingerprints: fingerprints, + }, nil +} + +func (m *SecurityManager) loadManifest() (securityManifest, time.Duration, time.Duration, error) { + manifestBytes, err := os.ReadFile(m.manifestPath) + if err != nil { + return securityManifest{}, 0, 0, fmt.Errorf("read security manifest: %w", err) + } + dec := json.NewDecoder(bytes.NewReader(manifestBytes)) + dec.DisallowUnknownFields() + var mf securityManifest + if err := dec.Decode(&mf); err != nil { + return securityManifest{}, 0, 0, fmt.Errorf("decode security manifest: %w", err) + } + if err := ensureJSONEOF(dec); err != nil { + return securityManifest{}, 0, 0, err + } + if mf.Version != securityManifestVersion || mf.Generation == 0 || mf.Generation > math.MaxInt64 || mf.Issuer == "" || mf.Audience == "" || mf.ActiveKeyID == "" || len(mf.Keys) == 0 || len(mf.Keys) > maxSecurityKeys || len(mf.Clients) == 0 || len(mf.Clients) > maxSecurityClients { + return securityManifest{}, 0, 0, errors.New("security manifest identity or bounds are invalid") + } + ttl, err := time.ParseDuration(mf.GrantTTLText) + if err != nil || ttl <= 0 || ttl > 5*time.Minute { + return securityManifest{}, 0, 0, errors.New("grantTTL must be positive and at most 5m") + } + skew, err := time.ParseDuration(mf.ClockSkewText) + if err != nil || skew < 0 || skew >= ttl { + return securityManifest{}, 0, 0, errors.New("clockSkew must be non-negative and less than grantTTL") + } + return mf, ttl, skew, nil +} + +func (m *SecurityManager) loadGrantKeys(root *os.Root, mf securityManifest) (map[string]ed25519.PublicKey, ed25519.PrivateKey, map[string]string, error) { + keys := make(map[string]ed25519.PublicKey, len(mf.Keys)) + fingerprints := make(map[string]string, len(mf.Keys)) + seenVersion := map[uint64]bool{} + seenID := map[string]bool{} + var active ed25519.PrivateKey + now := m.now() + for _, km := range mf.Keys { + if !validKeyManifest(km, seenID, seenVersion) { + return nil, nil, nil, errors.New("security key entry is invalid or duplicated") + } + seenID[km.ID] = true + seenVersion[km.Version] = true + key, err := readEd25519(root, km.File) + if err != nil { + return nil, nil, nil, fmt.Errorf("load security key %q: %w", km.ID, err) + } + pub := key.Public().(ed25519.PublicKey) + sum := sha256.Sum256(pub) + fp := hex.EncodeToString(sum[:]) + if !strings.EqualFold(fp, km.PublicSHA256) { + return nil, nil, nil, fmt.Errorf("security key %q fingerprint mismatch", km.ID) + } + fingerprints[km.ID+":"+strconv.FormatUint(km.Version, 10)] = fp + if km.State != "revoked" && !now.Before(km.ActivateAt) && now.Before(km.VerifyUntil) { + keys[km.ID] = pub + } + if km.ID == mf.ActiveKeyID { + if km.State != "active" || now.Before(km.ActivateAt) || !now.Before(km.VerifyUntil) { + return nil, nil, nil, errors.New("active grant key is outside its activation window") + } + active = key + } + } + if active == nil { + return nil, nil, nil, errors.New("active grant key is missing") + } + return keys, active, fingerprints, nil +} + +func validKeyManifest(km securityKeyManifest, seenID map[string]bool, seenVersion map[uint64]bool) bool { + return validKeyID(km.ID) && !seenID[km.ID] && km.Version != 0 && !seenVersion[km.Version] && validBaseName(km.File) && + (km.State == "active" || km.State == "verify-only" || km.State == "revoked") && + !km.VerifyUntil.IsZero() && km.VerifyUntil.After(km.ActivateAt) +} + +func (m *SecurityManager) loadTLS(root *os.Root, manifest securityTLSManifest) (*tls.Config, *x509.CertPool, time.Time, error) { + certPEM, err := readRootFile(root, manifest.CertificateFile) + if err != nil { + return nil, nil, time.Time{}, err + } + keyPEM, err := readRootFile(root, manifest.PrivateKeyFile) + if err != nil { + return nil, nil, time.Time{}, err + } + serverCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, nil, time.Time{}, fmt.Errorf("load TLS identity: %w", err) + } + leaf, err := x509.ParseCertificate(serverCert.Certificate[0]) + if err != nil { + return nil, nil, time.Time{}, err + } + now := m.now() + if now.Before(leaf.NotBefore) || !now.Before(leaf.NotAfter) || !hasUsage(leaf.ExtKeyUsage, x509.ExtKeyUsageServerAuth) { + return nil, nil, time.Time{}, errors.New("server certificate is not currently valid for server authentication") + } + serverCert.Leaf = leaf + caPEM, err := readRootFile(root, manifest.ClientCAFile) + if err != nil { + return nil, nil, time.Time{}, err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, nil, time.Time{}, errors.New("client CA bundle has no certificates") + } + cfg := &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{serverCert}, ClientAuth: tls.RequireAnyClientCert} + return cfg, pool, leaf.NotAfter, nil +} + +func clientPolicies(entries []securityClientManifest) (map[string]ClientPolicy, error) { + clients := make(map[string]ClientPolicy, len(entries)) + for _, entry := range entries { + uri, err := url.Parse(entry.URI) + if err != nil { + return nil, errors.New("client URI is invalid") + } + canonical, err := canonicalClientIdentity(&x509.Certificate{URIs: []*url.URL{uri}}) + if err != nil || canonical != entry.URI { + return nil, errors.New("client URI is not canonical") + } + if _, exists := clients[entry.URI]; exists { + return nil, errors.New("client URI is duplicated") + } + clients[entry.URI] = ClientPolicy{MayAttestOwner: entry.MayAttestOwner, Administrator: entry.Administrator} + } + return clients, nil +} + +func (m *SecurityManager) publishGeneration(ctx context.Context, s *securitySnapshot) error { + if m.kube == nil || m.configMap == "" { + return errors.New("security authority ConfigMap is required") + } + cms := m.kube.CoreV1().ConfigMaps(m.namespace) + for range 5 { + cm, exists, ledger, err := m.readSecurityLedger(ctx, cms) + if err != nil { + return err + } + changed, err := advanceSecurityLedger(&ledger, s) + if err != nil || !changed { + return err + } + raw, err := json.Marshal(ledger) + if err != nil { + return err + } + if cm.Data == nil { + cm.Data = map[string]string{} + } + cm.Data[securityStateDataKey] = string(raw) + if !exists { + _, err = cms.Create(ctx, cm, metav1.CreateOptions{}) + } else { + _, err = cms.Update(ctx, cm, metav1.UpdateOptions{}) + } + if err == nil { + return nil + } + if !apierrors.IsAlreadyExists(err) && !apierrors.IsConflict(err) { + return err + } + } + return errors.New("security authority state changed concurrently") +} + +func (m *SecurityManager) readSecurityLedger(ctx context.Context, cms corev1client.ConfigMapInterface) (*corev1.ConfigMap, bool, securityLedger, error) { + cm, err := cms.Get(ctx, m.configMap, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + cm = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: m.configMap, Namespace: m.namespace}, Data: map[string]string{}} + return cm, false, securityLedger{Fingerprints: map[string]string{}}, nil + } + if err != nil { + return nil, false, securityLedger{}, err + } + ledger := securityLedger{Fingerprints: map[string]string{}} + if raw := cm.Data[securityStateDataKey]; raw != "" { + if err := json.Unmarshal([]byte(raw), &ledger); err != nil || ledger.Fingerprints == nil { + return nil, false, securityLedger{}, errors.New("security authority state is invalid") + } + } + return cm, true, ledger, nil +} + +func advanceSecurityLedger(ledger *securityLedger, s *securitySnapshot) (bool, error) { + if s.generation < ledger.Generation { + return false, errors.New("security manifest generation rollback rejected") + } + if s.generation == ledger.Generation && ledger.Generation != 0 { + if !containsFingerprints(ledger.Fingerprints, s.fingerprints) { + return false, errors.New("security generation fingerprint drift rejected") + } + s.fingerprints = cloneFingerprints(ledger.Fingerprints) + return false, nil + } + for identity, fp := range s.fingerprints { + old, exists := ledger.Fingerprints[identity] + if exists && old != fp { + return false, errors.New("security key identity reuse rejected") + } + if !exists && !keyVersionAdvances(ledger.Fingerprints, identity) { + return false, errors.New("security key version rollback rejected") + } + ledger.Fingerprints[identity] = fp + } + if len(ledger.Fingerprints) > maxSecurityKeys { + return false, errors.New("security key tombstone limit reached; rotate issuer") + } + ledger.Generation = s.generation + s.fingerprints = cloneFingerprints(ledger.Fingerprints) + return true, nil +} + +func readRootFile(root *os.Root, name string) ([]byte, error) { + if !validBaseName(name) { + return nil, errors.New("security filename must be a basename") + } + f, err := root.Open(name) + if err != nil { + return nil, err + } + b, readErr := io.ReadAll(io.LimitReader(f, 1<<20)) + return b, errors.Join(readErr, f.Close()) +} +func readEd25519(root *os.Root, name string) (ed25519.PrivateKey, error) { + b, err := readRootFile(root, name) + if err != nil { + return nil, err + } + block, rest := pem.Decode(b) + if block == nil || block.Type != "PRIVATE KEY" || len(bytes.TrimSpace(rest)) != 0 { + return nil, errors.New("key must be one PKCS8 PEM block") + } + raw, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, errors.New("invalid PKCS8 key") + } + key, ok := raw.(ed25519.PrivateKey) + if !ok { + return nil, errors.New("key is not Ed25519") + } + return key, nil +} +func validBaseName(v string) bool { return v != "" && filepath.Base(v) == v && v != "." && v != ".." } +func validKeyID(v string) bool { + if len(v) == 0 || len(v) > 64 { + return false + } + for _, r := range v { + if r != '-' && r != '_' && (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') { + return false + } + } + return true +} +func ensureJSONEOF(dec *json.Decoder) error { + var extra any + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + return errors.New("security manifest has trailing data") + } + return nil +} +func hasUsage(usages []x509.ExtKeyUsage, wanted x509.ExtKeyUsage) bool { + for _, u := range usages { + if u == wanted || u == x509.ExtKeyUsageAny { + return true + } + } + return false +} +func containsFingerprints(authority, candidate map[string]string) bool { + for identity, fingerprint := range candidate { + if authority[identity] != fingerprint { + return false + } + } + return true +} + +func cloneFingerprints(source map[string]string) map[string]string { + clone := make(map[string]string, len(source)) + for identity, fingerprint := range source { + clone[identity] = fingerprint + } + return clone +} + +func keyVersionAdvances(authority map[string]string, identity string) bool { + separator := strings.LastIndexByte(identity, ':') + if separator <= 0 { + return false + } + id := identity[:separator] + version, err := strconv.ParseUint(identity[separator+1:], 10, 64) + if err != nil { + return false + } + for previous := range authority { + previousSeparator := strings.LastIndexByte(previous, ':') + if previousSeparator <= 0 || previous[:previousSeparator] != id { + continue + } + previousVersion, err := strconv.ParseUint(previous[previousSeparator+1:], 10, 64) + if err != nil || version <= previousVersion { + return false + } + } + return true +} + +func sameFingerprints(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + keys := make([]string, 0, len(a)) + for k := range a { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if a[k] != b[k] { + return false + } + } + return true +} diff --git a/internal/adapter/executioncontroller/security_limit.go b/internal/adapter/executioncontroller/security_limit.go new file mode 100644 index 0000000000..301475ef5f --- /dev/null +++ b/internal/adapter/executioncontroller/security_limit.go @@ -0,0 +1,83 @@ +package executioncontroller + +import ( + "context" + "crypto/x509" + "errors" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +// RPCLimiter bounds active unary RPCs globally and per authorized client. +type RPCLimiter struct { + security *SecurityManager + globalMax int + clientMax int + mu sync.Mutex + global int + clients map[string]int +} + +// NewRPCLimiter constructs a limiter whose client map is bounded by the security allowlist. +func NewRPCLimiter(security *SecurityManager, globalMax, clientMax int) (*RPCLimiter, error) { + if security == nil || globalMax < 1 || clientMax < 1 || clientMax > globalMax { + return nil, status.Error(codes.InvalidArgument, "invalid RPC concurrency limits") + } + return &RPCLimiter{security: security, globalMax: globalMax, clientMax: clientMax, clients: map[string]int{}}, nil +} + +// UnaryInterceptor authenticates against current policy before reserving capacity. +func (l *RPCLimiter) UnaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + client, _, err := clientCertificate(ctx) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "execution provider request failed") + } + id, _, err := l.security.authorize(ctx, client) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "execution provider request failed") + } + if !l.acquire(id) { + return nil, status.Error(codes.ResourceExhausted, "execution provider is at its concurrency limit") + } + defer l.release(id) + return handler(ctx, req) +} + +func clientCertificate(ctx context.Context) (*x509.Certificate, bool, error) { + p, ok := peer.FromContext(ctx) + if !ok { + return nil, false, errors.New("peer is unavailable") + } + tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) + if !ok || len(tlsInfo.State.PeerCertificates) == 0 { + return nil, false, errors.New("client certificate is unavailable") + } + return tlsInfo.State.PeerCertificates[0], len(tlsInfo.State.VerifiedChains) != 0, nil +} + +func (l *RPCLimiter) acquire(id string) bool { + l.mu.Lock() + defer l.mu.Unlock() + if l.global >= l.globalMax || l.clients[id] >= l.clientMax { + return false + } + l.global++ + l.clients[id]++ + return true +} + +func (l *RPCLimiter) release(id string) { + l.mu.Lock() + defer l.mu.Unlock() + l.global-- + if l.clients[id] == 1 { + delete(l.clients, id) + } else { + l.clients[id]-- + } +} diff --git a/internal/adapter/executioncontroller/security_limit_test.go b/internal/adapter/executioncontroller/security_limit_test.go new file mode 100644 index 0000000000..4600fce13c --- /dev/null +++ b/internal/adapter/executioncontroller/security_limit_test.go @@ -0,0 +1,29 @@ +package executioncontroller + +import "testing" + +func TestRPCLimiterBoundsGlobalPerClientAndIdentityMap(t *testing.T) { + limiter, err := NewRPCLimiter(&SecurityManager{}, 2, 1) + if err != nil { + t.Fatal(err) + } + if !limiter.acquire("client-a") || limiter.acquire("client-a") { + t.Fatal("per-client bound was not enforced") + } + if !limiter.acquire("client-b") || limiter.acquire("client-c") { + t.Fatal("global bound was not enforced") + } + limiter.release("client-a") + limiter.release("client-b") + if len(limiter.clients) != 0 || limiter.global != 0 { + t.Fatalf("released limiter retained cardinality: global=%d clients=%v", limiter.global, limiter.clients) + } +} + +func TestRPCLimiterRejectsInvalidBounds(t *testing.T) { + for _, limits := range [][2]int{{0, 1}, {1, 0}, {1, 2}} { + if _, err := NewRPCLimiter(&SecurityManager{}, limits[0], limits[1]); err == nil { + t.Fatalf("accepted limits %v", limits) + } + } +} diff --git a/internal/adapter/executioncontroller/security_test.go b/internal/adapter/executioncontroller/security_test.go new file mode 100644 index 0000000000..e427b65621 --- /dev/null +++ b/internal/adapter/executioncontroller/security_test.go @@ -0,0 +1,188 @@ +package executioncontroller + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/json" + "encoding/pem" + "math/big" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestSecurityManagerReloadRollbackAndRecovery(t *testing.T) { + dir := t.TempDir() + data := filepath.Join(dir, "..data") + if err := os.Mkdir(data, 0o700); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(data, "grant.pem"), grantPriv) + caPEM, serverCert, serverKey := syntheticPKI(t, now) + if err := os.WriteFile(filepath.Join(data, "server.crt"), serverCert, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(data, "server.key"), serverKey, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(data, "clients.pem"), caPEM, 0o600); err != nil { + t.Fatal(err) + } + for _, name := range []string{"grant.pem", "server.crt", "server.key", "clients.pem"} { + if err := os.Symlink(filepath.Join("..data", name), filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true, Administrator: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if !manager.Ready() { + t.Fatal("valid snapshot is not ready") + } + manifest.Generation = 0 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("invalid candidate retained authorization readiness") + } + manifest.Generation = 2 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("recovery reload: ready=%v err=%v", manager.Ready(), err) + } + firstCertificate := append([]byte(nil), manager.current.Load().tlsConfig.Certificates[0].Certificate[0]...) + cm, err := kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + originalState := cm.Data[securityStateDataKey] + cm.Data[securityStateDataKey] = `{"generation":2,"fingerprints":{"k1:1":"forged"}}` + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := manager.material(t.Context()); err == nil || manager.Ready() { + t.Fatal("authoritative fingerprint drift retained authorization") + } + cm, err = kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + cm.Data[securityStateDataKey] = originalState + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("restore authority: %v", err) + } + newCAPEM, newServerCert, newServerKey := syntheticPKI(t, now) + for name, contents := range map[string][]byte{"server.crt": newServerCert, "server.key": newServerKey, "clients.pem": newCAPEM} { + if err := os.WriteFile(filepath.Join(data, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + manifest.Generation = 3 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("TLS/CA correction did not recover: ready=%v err=%v", manager.Ready(), err) + } + if string(firstCertificate) == string(manager.current.Load().tlsConfig.Certificates[0].Certificate[0]) { + t.Fatal("TLS identity did not rotate atomically") + } + manifest.Generation = 1 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("generation rollback accepted") + } +} + +func TestSecurityLedgerRejectsKeyVersionRollbackAcrossRestart(t *testing.T) { + ledger := securityLedger{Generation: 2, Fingerprints: map[string]string{"key:2": "old"}} + candidate := &securitySnapshot{generation: 3, fingerprints: map[string]string{"key:1": "new"}} + if _, err := advanceSecurityLedger(&ledger, candidate); err == nil { + t.Fatal("key version rollback was accepted") + } + candidate.fingerprints = map[string]string{"key:3": "new"} + changed, err := advanceSecurityLedger(&ledger, candidate) + if err != nil || !changed || candidate.fingerprints["key:2"] != "old" { + t.Fatalf("monotonic rotation failed: changed=%v err=%v fingerprints=%v", changed, err, candidate.fingerprints) + } +} + +func TestSecurityManagerRejectsEscapingProjectedSymlink(t *testing.T) { + dir := t.TempDir() + if err := os.Symlink("../../outside", filepath.Join(dir, "grant.pem")); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := root.Close(); err != nil { + t.Errorf("close root: %v", err) + } + }) + if _, err := readRootFile(root, "grant.pem"); err == nil { + t.Fatal("escaping symlink was read") + } +} + +func writeManifest(t *testing.T, path string, manifest securityManifest) { + t.Helper() + b, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatal(err) + } +} +func writePKCS8(t *testing.T, path string, key ed25519.PrivateKey) { + t.Helper() + b, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: b}), 0o600); err != nil { + t.Fatal(err) + } +} +func syntheticPKI(t *testing.T, now time.Time) ([]byte, []byte, []byte) { + t.Helper() + caPub, caKey, _ := ed25519.GenerateKey(rand.Reader) + ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test-ca"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(2 * time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, caPub, caKey) + if err != nil { + t.Fatal(err) + } + serverPub, serverKey, _ := ed25519.GenerateKey(rand.Reader) + uri, _ := url.Parse("spiffe://example/provider") + server := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "provider"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, URIs: []*url.URL{uri}} + serverDER, err := x509.CreateCertificate(rand.Reader, server, ca, serverPub, caKey) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(serverKey) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) +} diff --git a/internal/adapter/executioncontroller/store.go b/internal/adapter/executioncontroller/store.go index 44738fe95e..d5b130f032 100644 --- a/internal/adapter/executioncontroller/store.go +++ b/internal/adapter/executioncontroller/store.go @@ -1,3 +1,4 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. package executioncontroller import ( @@ -7,7 +8,7 @@ import ( "encoding/hex" "errors" "fmt" - "sort" + "math" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -15,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" "github.com/stacklok/mecatl/internal/executionenv" ) @@ -25,6 +27,9 @@ var ExecutionEnvironmentGVR = schema.GroupVersionResource{Group: "execution.meca const ( maxReferences = 64 environmentNotFoundMessage = "environment not found" + currentSchemaVersion = int64(2) + operationLeaseTTL = 30 * time.Second + transitioningMessage = "environment is transitioning" ) // ExecutorTransport dispatches one request to a fixed workload helper. @@ -37,11 +42,26 @@ type Store struct { resources dynamic.ResourceInterface profiles *Profiles executor ExecutorTransport + kube kubernetes.Interface + namespace string + holderID string + now func() time.Time + opTTL time.Duration } // NewStore constructs a namespace-scoped execution state store. func NewStore(client dynamic.Interface, namespace string, profiles *Profiles, executor ExecutorTransport) *Store { - return &Store{resources: client.Resource(ExecutionEnvironmentGVR).Namespace(namespace), profiles: profiles, executor: executor} + holder, err := randomID() + if err != nil { + holder = "unavailable" + } + return &Store{resources: client.Resource(ExecutionEnvironmentGVR).Namespace(namespace), profiles: profiles, executor: executor, namespace: namespace, holderID: holder, now: func() time.Time { return time.Now().UTC() }, opTTL: 30 * time.Second} +} + +// WithKubeClient enables provider-owned executor and retained-volume lifecycle operations. +func (s *Store) WithKubeClient(kube kubernetes.Interface) *Store { + s.kube = kube + return s } // ValidateProfile returns one validated immutable profile. @@ -55,6 +75,22 @@ func (s *Store) ValidateProfile(_ context.Context, name string) (Profile, error) // Ensure idempotently allocates an environment for an immutable request fingerprint. func (s *Store) Ensure(ctx context.Context, client, owner, binding, profile, fp string) (Allocation, error) { + return s.EnsurePending(ctx, client, owner, binding, profile, fp, "legacy-"+fp) +} + +func (s *Store) EnsurePending(ctx context.Context, client, owner, binding, profile, fp, operationID string) (Allocation, error) { + return s.ensurePending(ctx, client, owner, executionenv.Owner{}, binding, profile, fp, operationID) +} + +// EnsurePendingOwned persists the attested principal needed for owner-safe intent reconciliation. +func (s *Store) EnsurePendingOwned(ctx context.Context, client, expectedOwnerHash string, owner executionenv.Owner, binding, profile, fp, operationID string) (Allocation, error) { + if owner.Issuer == "" || owner.Subject == "" || ownerHash(owner) != expectedOwnerHash { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "owner attestation is invalid"} + } + return s.ensurePending(ctx, client, expectedOwnerHash, owner, binding, profile, fp, operationID) +} + +func (s *Store) ensurePending(ctx context.Context, client, owner string, attested executionenv.Owner, binding, profile, fp, operationID string) (Allocation, error) { p, ok := s.profiles.get(profile) if !ok { return Allocation{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "profile not found"} @@ -62,23 +98,46 @@ func (s *Store) Ensure(ctx context.Context, client, owner, binding, profile, fp name := allocationName(client, owner, binding) cur, err := s.resources.Get(ctx, name, metav1.GetOptions{}) if err == nil { - if err := s.initializeStatus(ctx, cur, binding); err != nil { + if err := s.initializeStatus(ctx, cur, binding, operationID); err != nil { return Allocation{}, err } cur, err = s.resources.Get(ctx, name, metav1.GetOptions{}) if err != nil { return Allocation{}, err } + if !referenceOperationMatches(cur, binding, operationID) { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference operation conflicts with existing environment"} + } return allocationFrom(cur, client, owner, binding, profile, fp) } if !apierrors.IsNotFound(err) { return Allocation{}, fmt.Errorf("get execution environment: %w", err) } + if s.kube != nil { + all, listErr := s.resources.List(ctx, metav1.ListOptions{}) + if listErr != nil { + return Allocation{}, fmt.Errorf("count execution environments: %w", listErr) + } + profileCount := 0 + for i := range all.Items { + if textNested(all.Items[i].Object, "spec", "profile") == profile && textNested(all.Items[i].Object, "spec", "desired") != "Retired" { + profileCount++ + } + } + if profileCount >= p.Spec.MaxEnvironments { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "profile environment limit reached"} + } + } revision, err := randomID() if err != nil { return Allocation{}, err } - obj := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": name, "finalizers": []any{environmentFinalizer}}, "spec": map[string]any{"allocationID": name, "revision": revision, "ownerHash": owner, "clientHash": hashText(client), "bindingID": binding, "requestFingerprint": fp, "profile": profile, "profileDigest": p.Digest, "image": p.Spec.Image, "storageClass": p.Spec.StorageClass, "storageSize": p.Spec.StorageSize, "resources": map[string]any{"cpuRequest": p.Spec.CPURequest, "memoryRequest": p.Spec.MemoryRequest, "cpuLimit": p.Spec.CPULimit, "memoryLimit": p.Spec.MemoryLimit}, "desired": "Active"}}} + spec := map[string]any{"schemaVersion": currentSchemaVersion, "allocationID": name, "revision": revision, "ownerHash": owner, "clientHash": hashText(client), "bindingID": binding, "requestFingerprint": fp, "profile": profile, "profileDigest": p.Digest, "image": p.Spec.Image, "storageClass": p.Spec.StorageClass, "storageSize": p.Spec.StorageSize, "resources": map[string]any{"cpuRequest": p.Spec.CPURequest, "memoryRequest": p.Spec.MemoryRequest, "cpuLimit": p.Spec.CPULimit, "memoryLimit": p.Spec.MemoryLimit}, "desired": "Active"} + if attested.Issuer != "" && attested.Subject != "" { + spec["ownerIssuer"] = attested.Issuer + spec["ownerSubject"] = attested.Subject + } + obj := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": name, "finalizers": []any{environmentFinalizer}}, "spec": spec}} created, err := s.resources.Create(ctx, obj, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { created, err = s.resources.Get(ctx, name, metav1.GetOptions{}) @@ -86,7 +145,7 @@ func (s *Store) Ensure(ctx context.Context, client, owner, binding, profile, fp if err != nil { return Allocation{}, fmt.Errorf("create execution environment: %w", err) } - if err := s.initializeStatus(ctx, created, binding); err != nil { + if err := s.initializeStatus(ctx, created, binding, operationID); err != nil { return Allocation{}, err } created, err = s.resources.Get(ctx, name, metav1.GetOptions{}) @@ -95,18 +154,31 @@ func (s *Store) Ensure(ctx context.Context, client, owner, binding, profile, fp } return allocationFrom(created, client, owner, binding, profile, fp) } -func (s *Store) initializeStatus(ctx context.Context, env *unstructured.Unstructured, binding string) error { +func (s *Store) initializeStatus(ctx context.Context, env *unstructured.Unstructured, binding, operationID string) error { + if operationID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "operation identity is required"} + } if intNested(env.Object, "status", "epoch") > 0 { - return nil + return requireCurrentSchema(env) } - return s.retryUpdateStatus(ctx, env.GetName(), func(o *unstructured.Unstructured) error { + return s.retryUpdateStatusRaw(ctx, env.GetName(), func(o *unstructured.Unstructured) error { if intNested(o.Object, "status", "epoch") > 0 { - return nil + return requireCurrentSchema(o) + } + if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion { + return incompatibleSchemaError() + } + if err := unstructured.SetNestedField(o.Object, currentSchemaVersion, "status", "schemaVersion"); err != nil { + return err } if err := unstructured.SetNestedField(o.Object, int64(1), "status", "epoch"); err != nil { return err } - if err := unstructured.SetNestedStringSlice(o.Object, []string{binding}, "status", "references"); err != nil { + if err := unstructured.SetNestedField(o.Object, int64(1), "status", "grantGeneration"); err != nil { + return err + } + ref := map[string]any{"bindingID": binding, "state": string(executionenv.ReferencePendingCreate), "operationID": operationID, "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} + if err := unstructured.SetNestedSlice(o.Object, []any{ref}, "status", "references"); err != nil { return err } return unstructured.SetNestedField(o.Object, fenceHealthy, "status", "fenceState") @@ -136,8 +208,12 @@ func exactAllocation(o *unstructured.Unstructured, client, owner, binding string if !ok { return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment epoch is invalid"} } + generation := intNested(o.Object, "status", "grantGeneration") + if generation <= 0 { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "grant generation is invalid"} + } ready := conditionTrue(o, "Ready") && textNested(o.Object, "status", "fenceState") == fenceHealthy - return Allocation{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: revision}, Epoch: epoch, OwnerHash: owner, BindingID: binding, Client: client, Ready: ready}, nil + return Allocation{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: revision}, Epoch: epoch, GrantGeneration: uint64(generation), OwnerHash: owner, BindingID: binding, Client: client, Ready: ready}, nil } // Attach adds a binding reference only to the exact healthy environment. @@ -154,16 +230,12 @@ func (s *Store) Attach(ctx context.Context, ref executionenv.EnvironmentRef, cli if !a.Ready { return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment is not ready at requested epoch"} } - refs, _, _ := unstructured.NestedStringSlice(o.Object, "status", "references") - if !contains(refs, binding) { - if len(refs) >= maxReferences { - return &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "environment reference limit reached"} - } - refs = append(refs, binding) - sort.Strings(refs) - if err := unstructured.SetNestedStringSlice(o.Object, refs, "status", "references"); err != nil { - return err - } + refs, refsErr := referenceRecords(o) + if refsErr != nil { + return refsErr + } + if !attachedReference(refs, binding) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} } out = a return nil @@ -174,6 +246,9 @@ func (s *Store) Attach(ctx context.Context, ref executionenv.EnvironmentRef, cli // ReleaseReference removes one binding reference without deleting the environment. func (s *Store) ReleaseReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) error { return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } a, err := exactAllocation(o, client, owner, binding) if err != nil { return err @@ -181,43 +256,22 @@ func (s *Store) ReleaseReference(ctx context.Context, ref executionenv.Environme if a.Environment != ref { return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} } - refs, _, _ := unstructured.NestedStringSlice(o.Object, "status", "references") - next := refs[:0] - for _, v := range refs { - if v != binding { - next = append(next, v) + refs, refsErr := referenceRecords(o) + if refsErr != nil { + return refsErr + } + for i := range refs { + if refs[i].BindingID == binding { + if refs[i].State != executionenv.ReferencePublished { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference has a pending transaction"} + } + return setReferenceRecords(o, append(refs[:i], refs[i+1:]...)) } } - return unstructured.SetNestedStringSlice(o.Object, next, "status", "references") + return nil }) } -// Retire requests controlled retirement of an unreferenced healthy environment. -func (s *Store) Retire(ctx context.Context, ref executionenv.EnvironmentRef, owner string) error { - cur, err := s.resources.Get(ctx, ref.ID, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} - } - if err != nil { - return err - } - if textNested(cur.Object, "spec", "ownerHash") != owner || textNested(cur.Object, "spec", "revision") != ref.Revision { - return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} - } - refs, _, _ := unstructured.NestedStringSlice(cur.Object, "status", "references") - if len(refs) != 0 || textNested(cur.Object, "status", "activeOperation", "id") != "" || textNested(cur.Object, "status", "fenceState") != fenceHealthy { - return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is referenced or not quiescent"} - } - if err := unstructured.SetNestedField(cur.Object, "Retiring", "spec", "desired"); err != nil { - return err - } - _, err = s.resources.Update(ctx, cur, metav1.UpdateOptions{}) - if apierrors.IsConflict(err) { - return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment changed concurrently", Retryable: true} - } - return err -} - // File performs one claimed and fenced filesystem operation. func (s *Store) File(ctx context.Context, client, owner string, q executionenv.FileRequest) (executionenv.FileResponse, error) { req := executionenv.ExecutorRequest{Operation: q.Operation, Path: q.Path, Destination: q.Destination, Pattern: q.Pattern, Data: q.Data, Version: q.Version, Limit: q.Limit} @@ -251,6 +305,10 @@ func (*Store) CancelCommand(context.Context, string, string, executionenv.Comman return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "no active foreground stream is attached; external fencing is required"} } func (s *Store) execute(ctx context.Context, client, owner string, rc executionenv.RequestContext, req executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { //nolint:gocyclo // Atomic claim, dispatch, and fencing remain one auditable transaction. + if rc.Epoch == 0 || rc.Epoch > math.MaxInt64 || rc.GrantGeneration == 0 || rc.GrantGeneration > math.MaxInt64 { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "invalid execution epoch or grant generation"} + } + epochStatus := int64(rc.Epoch) //nolint:gosec // bounded above by MaxInt64. if s.executor == nil { return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeNotReady, Message: "executor transport unavailable", Retryable: true} } @@ -261,6 +319,9 @@ func (s *Store) execute(ctx context.Context, client, owner string, rc executione var pod string var maxFileBytes, maxCommandBytes int64 err = s.retryUpdateStatus(ctx, rc.Environment.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } profile, ok := s.profiles.get(textNested(o.Object, "spec", "profile")) if !ok || profile.Digest != textNested(o.Object, "spec", "profileDigest") { return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment profile is unavailable"} @@ -277,10 +338,17 @@ func (s *Store) execute(ctx context.Context, client, owner string, rc executione return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "command timeout exceeds profile bound"} } } - refs, _, _ := unstructured.NestedStringSlice(o.Object, "status", "references") - if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || !contains(refs, rc.BindingID) { + refs, refsErr := referenceRecords(o) + if refsErr != nil { + return refsErr + } + claim, _, expiry, claimOK := activeRunFrom(o) + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || !publishedReference(refs, rc.BindingID) { return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} } + if !claimOK || !time.Now().UTC().Before(expiry) || claim.BindingID != rc.BindingID || claim.RunID != rc.RunID || claim.ClaimID != rc.ClaimID || claim.Epoch != rc.Epoch || claim.GrantGeneration != rc.GrantGeneration || !generationMatches(o, rc.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim is not current"} + } if textNested(o.Object, "spec", "revision") != rc.Environment.Revision || !epochMatches(o, rc.Epoch) || textNested(o.Object, "spec", "desired") != "Active" || !conditionTrue(o, "Ready") || textNested(o.Object, "status", "fenceState") != fenceHealthy { return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment is not ready at requested epoch"} } @@ -291,16 +359,25 @@ func (s *Store) execute(ctx context.Context, client, owner string, rc executione if pod == "" { return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "executor pod is unavailable", Retryable: true} } - return unstructured.SetNestedMap(o.Object, map[string]any{"id": opID, "operation": string(req.Operation), "startedAt": time.Now().UTC().Format(time.RFC3339Nano)}, "status", "activeOperation") + now := s.now() + return unstructured.SetNestedMap(o.Object, map[string]any{"id": opID, "operation": string(req.Operation), "startedAt": now.Format(time.RFC3339Nano), "claimID": rc.ClaimID, "runID": rc.RunID, "epoch": epochStatus, "holderID": s.holderID, "renewedAt": now.Format(time.RFC3339Nano), "expiresAt": now.Add(s.opTTL).Format(time.RFC3339Nano)}, "status", "activeOperation") }) if err != nil { return executionenv.ExecutorResponse{}, err } current, err := s.resources.Get(ctx, rc.Environment.ID, metav1.GetOptions{}) - if err != nil || textNested(current.Object, "spec", "ownerHash") != owner || textNested(current.Object, "spec", "clientHash") != hashText(client) || textNested(current.Object, "spec", "revision") != rc.Environment.Revision || !epochMatches(current, rc.Epoch) || textNested(current.Object, "status", "activeOperation", "id") != opID { + if err != nil || textNested(current.Object, "spec", "ownerHash") != owner || textNested(current.Object, "spec", "clientHash") != hashText(client) || textNested(current.Object, "spec", "revision") != rc.Environment.Revision || !epochMatches(current, rc.Epoch) || !operationMatches(current, opID, s.holderID, rc.ClaimID, rc.Epoch) { return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation identity changed before executor dispatch"} } - resp, dispatchErr := s.executor.Execute(ctx, pod, req) + opCtx, stopLease := context.WithCancel(ctx) + leaseDone := make(chan error, 1) + go func() { leaseDone <- s.renewOperationLease(opCtx, rc.Environment.ID, opID, rc.ClaimID, rc.Epoch) }() + resp, dispatchErr := s.executor.Execute(opCtx, pod, req) + stopLease() + leaseErr := <-leaseDone + if leaseErr != nil && dispatchErr == nil { + dispatchErr = &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation lease renewal failed"} + } if dispatchErr == nil && (int64(len(resp.Data)) > maxFileBytes || resp.Command != nil && int64(len(resp.Command.Stdout)+len(resp.Command.Stderr)) > maxCommandBytes) { dispatchErr = &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "executor response exceeds profile bound"} } @@ -308,14 +385,13 @@ func (s *Store) execute(ctx context.Context, client, owner string, rc executione controlledTerminal := errors.As(dispatchErr, &controlledErr) && controlledErr.Code != executionenv.CodeFenceUnknown terminal := controlledTerminal || dispatchErr == nil && (req.Operation != executionenv.OpCommandStart || resp.Command != nil && resp.Command.TerminalReceipt != "") finishErr := s.retryUpdateStatus(context.WithoutCancel(ctx), rc.Environment.ID, func(o *unstructured.Unstructured) error { - if textNested(o.Object, "status", "activeOperation", "id") != opID { + if !operationMatches(o, opID, s.holderID, rc.ClaimID, rc.Epoch) { return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation ownership changed before completion"} } if terminal { unstructured.RemoveNestedField(o.Object, "status", "activeOperation") return nil } - unstructured.RemoveNestedField(o.Object, "status", "activeOperation") return unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") }) if finishErr != nil { @@ -330,6 +406,15 @@ func (s *Store) execute(ctx context.Context, client, owner string, rc executione return resp, nil } func (s *Store) retryUpdateStatus(ctx context.Context, name string, mutate func(*unstructured.Unstructured) error) error { + return s.retryUpdateStatusRaw(ctx, name, func(o *unstructured.Unstructured) error { + if err := requireCurrentSchema(o); err != nil { + return err + } + return mutate(o) + }) +} + +func (s *Store) retryUpdateStatusRaw(ctx context.Context, name string, mutate func(*unstructured.Unstructured) error) error { for i := 0; i < 5; i++ { cur, err := s.resources.Get(ctx, name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { @@ -349,6 +434,17 @@ func (s *Store) retryUpdateStatus(ctx context.Context, name string, mutate func( } return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment changed concurrently", Retryable: true} } +func requireCurrentSchema(o *unstructured.Unstructured) error { + if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion || intNested(o.Object, "status", "schemaVersion") != currentSchemaVersion { + return incompatibleSchemaError() + } + return nil +} + +func incompatibleSchemaError() error { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment schema is incompatible; explicit migration is required"} +} + func epochValue(o *unstructured.Unstructured) (uint64, bool) { epoch, found, err := unstructured.NestedInt64(o.Object, "status", "epoch") if err != nil || !found || epoch <= 0 { diff --git a/internal/adapter/executioncontroller/store_lifecycle.go b/internal/adapter/executioncontroller/store_lifecycle.go new file mode 100644 index 0000000000..0c0e51b49e --- /dev/null +++ b/internal/adapter/executioncontroller/store_lifecycle.go @@ -0,0 +1,259 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. +package executioncontroller + +import ( + "context" + "sort" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +type referenceRecord struct { + BindingID string + State executionenv.ReferenceState + OperationID string + SourceBindingID string + CreatedAt time.Time +} + +func referenceRecords(o *unstructured.Unstructured) ([]referenceRecord, error) { + raw, found, err := unstructured.NestedSlice(o.Object, "status", "references") + if err != nil || !found { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment references are invalid"} + } + out := make([]referenceRecord, 0, len(raw)) + for _, item := range raw { + m, ok := item.(map[string]any) + if !ok { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "legacy environment references require explicit migration"} + } + created, parseErr := time.Parse(time.RFC3339Nano, text(m, "createdAt")) + r := referenceRecord{BindingID: text(m, "bindingID"), State: executionenv.ReferenceState(text(m, "state")), OperationID: text(m, "operationID"), SourceBindingID: text(m, "sourceBindingID"), CreatedAt: created} + if parseErr != nil || r.BindingID == "" || r.OperationID == "" || (r.State != executionenv.ReferencePendingCreate && r.State != executionenv.ReferencePublished && r.State != executionenv.ReferencePendingDelete) { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment references are invalid"} + } + out = append(out, r) + } + return out, nil +} + +func setReferenceRecords(o *unstructured.Unstructured, refs []referenceRecord) error { + sort.Slice(refs, func(i, j int) bool { return refs[i].BindingID < refs[j].BindingID }) + raw := make([]any, len(refs)) + for i, r := range refs { + raw[i] = map[string]any{"bindingID": r.BindingID, "state": string(r.State), "operationID": r.OperationID, "sourceBindingID": r.SourceBindingID, "createdAt": r.CreatedAt.UTC().Format(time.RFC3339Nano)} + } + return unstructured.SetNestedSlice(o.Object, raw, "status", "references") +} + +func referenceOperationMatches(o *unstructured.Unstructured, binding, operation string) bool { + refs, err := referenceRecords(o) + if err != nil { + return false + } + for _, r := range refs { + if r.BindingID == binding { + return r.OperationID == operation && (r.State == executionenv.ReferencePendingCreate || r.State == executionenv.ReferencePublished) + } + } + return false +} + +func attachedReference(refs []referenceRecord, binding string) bool { + for _, r := range refs { + if r.BindingID == binding && (r.State == executionenv.ReferencePendingCreate || r.State == executionenv.ReferencePublished) { + return true + } + } + return false +} + +func publishedReference(refs []referenceRecord, binding string) bool { + for _, r := range refs { + if r.BindingID == binding && r.State == executionenv.ReferencePublished { + return true + } + } + return false +} + +func (s *Store) mutateReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string, from, to executionenv.ReferenceState, remove bool) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + for i := range refs { + if refs[i].BindingID != binding { + continue + } + if refs[i].OperationID == operation && refs[i].State == to { + return nil + } + if refs[i].OperationID == operation && refs[i].State == from { + if remove { + refs = append(refs[:i], refs[i+1:]...) + } else { + refs[i].State = to + } + return setReferenceRecords(o, refs) + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference transaction mismatch"} + } + if remove { + return nil + } + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference not found"} + }) +} + +func (s *Store) CommitReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingCreate, executionenv.ReferencePublished, false) +} +func (s *Store) AbortReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingCreate, "", true) +} +func (s *Store) PrepareReferenceDelete(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + for i := range refs { + if refs[i].BindingID != binding { + continue + } + if refs[i].State == executionenv.ReferencePendingDelete && refs[i].OperationID == operation { + return nil + } + if refs[i].State != executionenv.ReferencePublished { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference transaction mismatch"} + } + refs[i].State, refs[i].OperationID, refs[i].CreatedAt = executionenv.ReferencePendingDelete, operation, time.Now().UTC() + return setReferenceRecords(o, refs) + } + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference not found"} + }) +} +func (s *Store) ConfirmReferenceDelete(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingDelete, "", true) +} +func (s *Store) CancelReferenceDelete(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingDelete, executionenv.ReferencePublished, false) +} + +func (s *Store) ReserveSuccessor(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, source, destination, operation string) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + if !publishedReference(refs, source) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "source reference not found"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not quiescent"} + } + for _, r := range refs { + if r.BindingID == destination { + if r.State == executionenv.ReferencePendingCreate && r.OperationID == operation && r.SourceBindingID == source { + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "destination reference conflicts"} + } + } + if len(refs) >= maxReferences { + return &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "environment reference limit reached"} + } + refs = append(refs, referenceRecord{BindingID: destination, State: executionenv.ReferencePendingCreate, OperationID: operation, SourceBindingID: source, CreatedAt: time.Now().UTC()}) + return setReferenceRecords(o, refs) + }) +} + +func (s *Store) ListReferenceIntentsForClient(ctx context.Context, client string, limit int) ([]executionenv.ReferenceIntent, error) { + if limit <= 0 || limit > maxReferences { + limit = maxReferences + } + list, err := s.resources.List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + out := make([]executionenv.ReferenceIntent, 0) + for i := range list.Items { + o := &list.Items[i] + if textNested(o.Object, "spec", "clientHash") != hashText(client) { + continue + } + owner := executionenv.Owner{Issuer: textNested(o.Object, "spec", "ownerIssuer"), Subject: textNested(o.Object, "spec", "ownerSubject")} + if owner.Issuer == "" || owner.Subject == "" || ownerHash(owner) != textNested(o.Object, "spec", "ownerHash") { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference intent owner attestation is unavailable"} + } + refs, parseErr := referenceRecords(o) + if parseErr != nil { + return nil, parseErr + } + for _, r := range refs { + if r.State == executionenv.ReferencePublished { + continue + } + out = append(out, executionenv.ReferenceIntent{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: textNested(o.Object, "spec", "revision")}, Owner: owner, BindingID: r.BindingID, State: r.State, OperationID: r.OperationID, SourceBindingID: r.SourceBindingID, CreatedAt: r.CreatedAt}) + if len(out) == limit { + return out, nil + } + } + } + return out, nil +} + +func (s *Store) ListReferenceIntents(ctx context.Context, client, owner string, limit int) ([]executionenv.ReferenceIntent, error) { + if limit <= 0 || limit > maxReferences { + limit = maxReferences + } + list, err := s.resources.List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + out := make([]executionenv.ReferenceIntent, 0) + for i := range list.Items { + o := &list.Items[i] + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) { + continue + } + refs, parseErr := referenceRecords(o) + if parseErr != nil { + return nil, parseErr + } + for _, r := range refs { + if r.State == executionenv.ReferencePublished { + continue + } + out = append(out, executionenv.ReferenceIntent{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: textNested(o.Object, "spec", "revision")}, BindingID: r.BindingID, State: r.State, OperationID: r.OperationID, SourceBindingID: r.SourceBindingID, CreatedAt: r.CreatedAt}) + if len(out) == limit { + return out, nil + } + } + } + return out, nil +} diff --git a/internal/adapter/executioncontroller/store_lifecycle_test.go b/internal/adapter/executioncontroller/store_lifecycle_test.go new file mode 100644 index 0000000000..697cc3e63e --- /dev/null +++ b/internal/adapter/executioncontroller/store_lifecycle_test.go @@ -0,0 +1,142 @@ +package executioncontroller + +import ( + "context" + "errors" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func lifecycleEnvironment(now time.Time) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", + "metadata": map[string]any{"name": "env", "namespace": "ns"}, + "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, + "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "references": []any{ + map[string]any{"bindingID": "source", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}, + }, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}, + }} +} + +func TestRunClaimIsEnvironmentWideAndExact(t *testing.T) { + now := time.Now().UTC() + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), lifecycleEnvironment(now)) + store := NewStore(client, "ns", testProfiles(), nil) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + first, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-a", "op-a", time.Minute) + if err != nil { + t.Fatal(err) + } + if first.Epoch != 2 || first.ClaimID == "" || first.GrantGeneration != 1 { + t.Fatalf("claim=%+v", first) + } + repeat, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-a", "op-a", time.Minute) + if err != nil || repeat.ClaimID != first.ClaimID || repeat.Epoch != first.Epoch { + t.Fatalf("repeat=%+v err=%v", repeat, err) + } + if _, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-b", "op-b", time.Minute); err == nil { + t.Fatal("overlapping claim admitted") + } + stale := executionenv.RunClaimRequest{Environment: ref, BindingID: "source", RunID: "run-a", ClaimID: "wrong", Epoch: first.Epoch, GrantGeneration: first.GrantGeneration, OperationID: "release", TTL: time.Minute} + if err := store.ReleaseRun(t.Context(), ref, "client", "owner", stale); err == nil { + t.Fatal("stale release admitted") + } + stale.ClaimID = first.ClaimID + if err := store.ReleaseRun(t.Context(), ref, "client", "owner", stale); err != nil { + t.Fatal(err) + } + second, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-b", "op-b", time.Minute) + if err != nil || second.Epoch != first.Epoch+1 { + t.Fatalf("second=%+v err=%v", second, err) + } +} + +func TestExpiredClaimWithActiveOperationCannotBeReplaced(t *testing.T) { + now := time.Now().UTC() + env := lifecycleEnvironment(now) + statusMap, _, _ := unstructured.NestedMap(env.Object, "status") + statusMap["epoch"] = int64(4) + statusMap["activeRun"] = map[string]any{"bindingID": "source", "runID": "old", "claimID": "old-claim", "operationID": "old-acquire", "epoch": int64(4), "grantGeneration": int64(1), "expiresAt": now.Add(-time.Minute).Format(time.RFC3339Nano)} + statusMap["activeOperation"] = map[string]any{"id": "still-running", "claimID": "old-claim", "runID": "old", "epoch": int64(4)} + env.Object["status"] = statusMap + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + _, err := store.AcquireRun(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "source", "new", "new-op", time.Minute) + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeFenceUnknown { + t.Fatalf("error=%v", err) + } +} + +func TestClientScopedIntentListReturnsAttestedOwnerOnlyToOwningClient(t *testing.T) { + now := time.Now().UTC() + env := lifecycleEnvironment(now) + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + if err := unstructured.SetNestedField(env.Object, ownerHash(owner), "spec", "ownerHash"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(env.Object, owner.Issuer, "spec", "ownerIssuer"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(env.Object, owner.Subject, "spec", "ownerSubject"); err != nil { + t.Fatal(err) + } + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + if err := store.ReserveSuccessor(t.Context(), ref, "client", ownerHash(owner), "source", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + intents, err := store.ListReferenceIntentsForClient(t.Context(), "client", 64) + if err != nil || len(intents) != 1 || intents[0].Owner != owner || intents[0].BindingID != "destination" { + t.Fatalf("intents=%+v err=%v", intents, err) + } + other, err := store.ListReferenceIntentsForClient(t.Context(), "other-client", 64) + if err != nil || len(other) != 0 { + t.Fatalf("other-client intents=%+v err=%v", other, err) + } +} + +func TestReferenceTransactionsRetainUnknownAndNeverChangeSource(t *testing.T) { + now := time.Now().UTC() + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), lifecycleEnvironment(now)) + store := NewStore(client, "ns", testProfiles(), nil) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + if err := store.ReserveSuccessor(t.Context(), ref, "client", "owner", "source", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + if err := store.ReserveSuccessor(t.Context(), ref, "client", "owner", "source", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + intents, err := store.ListReferenceIntents(t.Context(), "client", "owner", 64) + if err != nil || len(intents) != 1 || intents[0].BindingID != "destination" || intents[0].State != executionenv.ReferencePendingCreate { + t.Fatalf("intents=%+v err=%v", intents, err) + } + if err := store.CommitReference(t.Context(), ref, "client", "owner", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + if err := store.PrepareReferenceDelete(t.Context(), ref, "client", "owner", "destination", "delete-op"); err != nil { + t.Fatal(err) + } + if err := store.CancelReferenceDelete(t.Context(), ref, "client", "owner", "destination", "delete-op"); err != nil { + t.Fatal(err) + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + refs, err := referenceRecords(got) + if err != nil { + t.Fatal(err) + } + if !publishedReference(refs, "source") || !publishedReference(refs, "destination") { + t.Fatalf("references=%+v", refs) + } +} diff --git a/internal/adapter/executioncontroller/store_revoke.go b/internal/adapter/executioncontroller/store_revoke.go new file mode 100644 index 0000000000..1a97dc5399 --- /dev/null +++ b/internal/adapter/executioncontroller/store_revoke.go @@ -0,0 +1,32 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. +package executioncontroller + +import ( + "context" + "math" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// RevokeEnvironment atomically invalidates every issued grant without claiming +// that an already-dispatched executor operation has stopped. +func (s *Store) RevokeEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, client, owner string, expected uint64) (uint64, error) { + if expected == 0 || expected >= math.MaxInt64 { + return 0, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "grant generation is invalid"} + } + var next uint64 + err := s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + current := intNested(o.Object, "status", "grantGeneration") + if current <= 0 || uint64(current) != expected { //nolint:gosec // positivity is checked before conversion. + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "grant generation changed concurrently"} + } + next = expected + 1 + return unstructured.SetNestedField(o.Object, int64(next), "status", "grantGeneration") //nolint:gosec // expected is bounded below MaxInt64. + }) + return next, err +} diff --git a/internal/adapter/executioncontroller/store_revoke_test.go b/internal/adapter/executioncontroller/store_revoke_test.go new file mode 100644 index 0000000000..c04e4a382d --- /dev/null +++ b/internal/adapter/executioncontroller/store_revoke_test.go @@ -0,0 +1,77 @@ +package executioncontroller + +import ( + "context" + "errors" + "math" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestRevokeEnvironmentFencesOldClaimWithoutChangingExecutionEpoch(t *testing.T) { + env := runFixtureEnvironment(7, 3) + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), &terminalErrorExecutor{}) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + next, err := store.RevokeEnvironment(t.Context(), ref, "client", "owner", 3) + if err != nil || next != 4 { + t.Fatalf("revoke=(%d,%v)", next, err) + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if intNested(got.Object, "status", "epoch") != 7 || intNested(got.Object, "status", "grantGeneration") != 4 { + t.Fatalf("status=%v", got.Object["status"]) + } + otherReplica := NewStore(client, "ns", testProfiles(), &terminalErrorExecutor{}) + _, err = otherReplica.File(t.Context(), "client", "owner", executionenv.FileRequest{Context: executionenv.RequestContext{Environment: ref, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3}, Operation: executionenv.OpFileRead, Path: "x"}) + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("old grant error=%v", err) + } + if _, err := store.RenewRun(t.Context(), ref, "client", "owner", executionenv.RunClaimRequest{Environment: ref, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: time.Minute}); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("old renewal error=%v", err) + } + if _, err := store.RevokeEnvironment(t.Context(), ref, "client", "owner", 3); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("stale CAS error=%v", err) + } +} + +func TestRevokeEnvironmentOverflowFailsClosed(t *testing.T) { + env := runFixtureEnvironment(1, math.MaxInt64) + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + if _, err := store.RevokeEnvironment(context.Background(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", math.MaxInt64); err == nil { + t.Fatal("overflow accepted") + } +} + +func TestProfileEnvironmentAdmissionLimit(t *testing.T) { + env := lifecycleEnvironment(time.Now().UTC()) + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}, env) + profiles := testProfiles() + profile := profiles.byName["go"] + profile.Spec.MaxEnvironments = 1 + profiles.byName["go"] = profile + store := NewStore(client, "ns", profiles, nil).WithKubeClient(kubernetesfake.NewSimpleClientset()) + _, err := store.Ensure(t.Context(), "client", "owner", "new-binding", "go", "new-fingerprint") + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeResourceExhausted { + t.Fatalf("limit error=%v", err) + } +} + +func runFixtureEnvironment(epoch, generation uint64) *unstructured.Unstructured { + now := time.Now().UTC() + return &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(epoch), "grantGeneration": int64(generation), "fenceState": "Healthy", "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "epoch": int64(epoch), "grantGeneration": int64(generation), "expiresAt": now.Add(time.Minute).Format(time.RFC3339Nano)}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} +} diff --git a/internal/adapter/executioncontroller/store_run.go b/internal/adapter/executioncontroller/store_run.go new file mode 100644 index 0000000000..a5fc13bab2 --- /dev/null +++ b/internal/adapter/executioncontroller/store_run.go @@ -0,0 +1,176 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. +package executioncontroller + +import ( + "context" + "math" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func boundedRunTTL(ttl time.Duration) (time.Duration, error) { + if ttl < executionenv.MinRunTTL || ttl > executionenv.MaxRunTTL { + return 0, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "run claim ttl is outside the operator bound"} + } + return ttl, nil +} + +func generationMatches(o *unstructured.Unstructured, generation uint64) bool { + return generation > 0 && generation <= math.MaxInt64 && intNested(o.Object, "status", "grantGeneration") == int64(generation) +} + +// ValidateRunClaim rechecks durable generation and ownership before any grant-authorized operation. +func (s *Store) ValidateRunClaim(ctx context.Context, client, owner string, rc executionenv.RequestContext) error { + if rc.Epoch == 0 || rc.Epoch > math.MaxInt64 || rc.GrantGeneration == 0 || rc.GrantGeneration > math.MaxInt64 { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "run claim identity is invalid"} + } + o, err := s.resources.Get(ctx, rc.Environment.ID, metav1.GetOptions{}) + if err != nil { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if err := requireCurrentSchema(o); err != nil { + return err + } + claim, _, expiry, ok := activeRunFrom(o) + if !ok || !s.now().Before(expiry) || claim.Environment != rc.Environment || claim.BindingID != rc.BindingID || claim.RunID != rc.RunID || claim.ClaimID != rc.ClaimID || claim.Epoch != rc.Epoch || claim.GrantGeneration != rc.GrantGeneration || !generationMatches(o, rc.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim is not current"} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + return nil +} + +func activeRunFrom(o *unstructured.Unstructured) (executionenv.RunClaim, string, time.Time, bool) { + m, found, _ := unstructured.NestedMap(o.Object, "status", "activeRun") + if !found { + return executionenv.RunClaim{}, "", time.Time{}, false + } + expires, err := time.Parse(time.RFC3339Nano, text(m, "expiresAt")) + epoch, epochOK := epochValue(o) + generationValue := intNested(m, "grantGeneration") + if generationValue <= 0 { + return executionenv.RunClaim{}, "", time.Time{}, false + } + generation := uint64(generationValue) //nolint:gosec // positivity is checked immediately above. + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: textNested(o.Object, "spec", "revision")}, BindingID: text(m, "bindingID"), RunID: text(m, "runID"), ClaimID: text(m, "claimID"), Epoch: epoch, GrantGeneration: generation, ExpiresAt: expires} + return claim, text(m, "operationID"), expires, err == nil && epochOK && generation > 0 && claim.BindingID != "" && claim.RunID != "" && claim.ClaimID != "" +} + +//nolint:gocyclo // Run ownership admission keeps every CAS precondition in one auditable transition. +func (s *Store) AcquireRun(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, runID, operationID string, ttl time.Duration) (executionenv.RunClaim, error) { + ttl, err := boundedRunTTL(ttl) + if err != nil { + return executionenv.RunClaim{}, err + } + var out executionenv.RunClaim + err = s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, parseErr := referenceRecords(o) + if parseErr != nil { + return parseErr + } + if !publishedReference(refs, binding) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "published reference not found"} + } + if textNested(o.Object, "spec", "desired") != "Active" || !conditionTrue(o, "Ready") || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment is not ready"} + } + if current, currentOp, expires, found := activeRunFrom(o); found { + if current.BindingID == binding && current.RunID == runID && currentOp == operationID { + if !generationMatches(o, current.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim was revoked"} + } + out = current + return nil + } + if time.Now().UTC().Before(expires) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment has an active run"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "expired run still has an active operation"} + } + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment has an active operation"} + } + epoch, ok := epochValue(o) + if !ok || epoch >= math.MaxInt64 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment epoch is invalid"} + } + epoch++ + epochStatus := int64(epoch) //nolint:gosec // the persisted epoch was checked below MaxInt64 before increment. + claimID, idErr := randomID() + if idErr != nil { + return idErr + } + expires := time.Now().UTC().Add(ttl) + generation := intNested(o.Object, "status", "grantGeneration") + if generation <= 0 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "grant generation is invalid"} + } + if setErr := unstructured.SetNestedField(o.Object, epochStatus, "status", "epoch"); setErr != nil { + return setErr + } + if setErr := unstructured.SetNestedMap(o.Object, map[string]any{"bindingID": binding, "runID": runID, "claimID": claimID, "operationID": operationID, "ownerHash": owner, "clientHash": hashText(client), "epoch": epochStatus, "grantGeneration": generation, "expiresAt": expires.Format(time.RFC3339Nano)}, "status", "activeRun"); setErr != nil { + return setErr + } + out = executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: runID, ClaimID: claimID, Epoch: epoch, GrantGeneration: uint64(generation), ExpiresAt: expires} + return nil + }) + return out, err +} + +func (s *Store) RenewRun(ctx context.Context, ref executionenv.EnvironmentRef, client, owner string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + ttl, err := boundedRunTTL(req.TTL) + if err != nil { + return executionenv.RunClaim{}, err + } + var out executionenv.RunClaim + err = s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + cur, _, _, ok := activeRunFrom(o) + if !ok || textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || cur.Environment != ref || cur.BindingID != req.BindingID || cur.RunID != req.RunID || cur.ClaimID != req.ClaimID || cur.Epoch != req.Epoch || cur.GrantGeneration != req.GrantGeneration || !generationMatches(o, req.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim mismatch"} + } + expires := time.Now().UTC().Add(ttl) + m, _, _ := unstructured.NestedMap(o.Object, "status", "activeRun") + m["expiresAt"] = expires.Format(time.RFC3339Nano) + m["lastRenewOperationID"] = req.OperationID + if setErr := unstructured.SetNestedMap(o.Object, m, "status", "activeRun"); setErr != nil { + return setErr + } + cur.ExpiresAt = expires + out = cur + return nil + }) + return out, err +} + +func (s *Store) ReleaseRun(ctx context.Context, ref executionenv.EnvironmentRef, client, owner string, req executionenv.RunClaimRequest) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + cur, _, _, ok := activeRunFrom(o) + if !ok { + return nil + } + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || cur.Environment != ref || cur.BindingID != req.BindingID || cur.RunID != req.RunID || cur.ClaimID != req.ClaimID || cur.Epoch != req.Epoch || cur.GrantGeneration != req.GrantGeneration || !generationMatches(o, req.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim mismatch"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "run has an active operation"} + } + unstructured.RemoveNestedField(o.Object, "status", "activeRun") + return nil + }) +} diff --git a/internal/adapter/executioncontroller/store_test.go b/internal/adapter/executioncontroller/store_test.go index 08a8c8602b..197224b96f 100644 --- a/internal/adapter/executioncontroller/store_test.go +++ b/internal/adapter/executioncontroller/store_test.go @@ -16,7 +16,7 @@ import ( func TestStoreEnsureUsesStableLookupAndRejectsFingerprintDrift(t *testing.T) { client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) - profiles := &Profiles{byName: map[string]resolvedProfile{"go": {Digest: "sha256:profile", Spec: ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute}}, "other": {Digest: "sha256:other", Spec: ProfileSpec{Image: "example@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute}}}} + profiles := &Profiles{byName: map[string]resolvedProfile{"go": {Digest: "sha256:profile", Spec: ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100}}, "other": {Digest: "sha256:other", Spec: ProfileSpec{Image: "example@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100}}}} s := NewStore(client, "ns", profiles, nil) ctx := context.Background() a, err := s.Ensure(ctx, "spiffe://client", "owner", "binding", "go", "fp1") @@ -60,14 +60,14 @@ func (e *terminalErrorExecutor) Execute(context.Context, string, executionenv.Ex return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "path not found"} } func TestCancelledStoreOperationStaysActiveUntilBackendStopsThenFences(t *testing.T) { - env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"epoch": int64(1), "fenceState": "Healthy", "references": []any{"binding"}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "epoch": int64(1), "grantGeneration": int64(1), "expiresAt": time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) exec := &blockingExecutor{started: make(chan struct{}), release: make(chan struct{})} s := NewStore(client, "ns", testProfiles(), exec) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { - _, err := s.StartCommand(ctx, "client", "owner", executionenv.CommandStartRequest{Context: executionenv.RequestContext{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", Epoch: 1}, Command: "sleep 10"}) + _, err := s.StartCommand(ctx, "client", "owner", executionenv.CommandStartRequest{Context: executionenv.RequestContext{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1}, Command: "sleep 10"}) done <- err }() <-exec.started @@ -88,8 +88,8 @@ func TestCancelledStoreOperationStaysActiveUntilBackendStopsThenFences(t *testin if err != nil { t.Fatal(err) } - if textNested(fenced.Object, "status", "activeOperation", "id") != "" || textNested(fenced.Object, "status", "fenceState") != "FenceUnknown" { - t.Fatalf("status=%v", fenced.Object["status"]) + if textNested(fenced.Object, "status", "activeOperation", "id") == "" || textNested(fenced.Object, "status", "fenceState") != "FenceUnknown" { + t.Fatalf("uncertain operation identity was not retained: status=%v", fenced.Object["status"]) } _, err = s.Attach(context.Background(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "binding") if !errors.As(err, &pe) || pe.Code != executionenv.CodeNotReady { @@ -98,11 +98,11 @@ func TestCancelledStoreOperationStaysActiveUntilBackendStopsThenFences(t *testin } func TestControlledExecutorErrorClearsOperationWithoutFencing(t *testing.T) { - env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"epoch": int64(1), "fenceState": "Healthy", "references": []any{"binding"}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "epoch": int64(1), "grantGeneration": int64(1), "expiresAt": time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) exec := &terminalErrorExecutor{} s := NewStore(client, "ns", testProfiles(), exec) - _, err := s.File(context.Background(), "client", "owner", executionenv.FileRequest{Context: executionenv.RequestContext{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", Epoch: 1}, Operation: executionenv.OpFileRead, Path: "missing"}) + _, err := s.File(context.Background(), "client", "owner", executionenv.FileRequest{Context: executionenv.RequestContext{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1}, Operation: executionenv.OpFileRead, Path: "missing"}) var pe *executionenv.Error if !errors.As(err, &pe) || pe.Code != executionenv.CodeNotFound { t.Fatalf("error=%v", err) diff --git a/internal/adapter/server/classification.go b/internal/adapter/server/classification.go index 4798c8fb2a..ba03490ab3 100644 --- a/internal/adapter/server/classification.go +++ b/internal/adapter/server/classification.go @@ -344,6 +344,7 @@ var serviceAccessTable = map[string]ClassificationEntry{ "LookupRun": {KindDerived, "in-memory run registry read; every caller-facing entry point (Cancel, Persist, Approve*, MaybeAutoApprovePlan) authorizes the session FIRST and only then consults this"}, "IsLive": {KindDerived, "combined Service-run and engine-child process-local registry; consumed by destructive maintenance, not a caller-facing verb"}, "MaintenanceMutationAvailable": {KindDerived, "read-only capability truth consumed by composition before scheduling automatic retention"}, + "ReconcileReferenceIntents": {KindDerived, "composition-owned maintenance joins provider-owned exact intents to durable sessions and invokes only exact retained callbacks"}, "FinishRun": {KindDerived, "deregisters an id the wire adapter already finished draining from its own authorized run"}, "PublishSessionEvent": {KindDerived, "publishes to subscribers already registered via the (caller-owned) Subscribe for this id; PublishSessionEvent itself takes no ctx and makes no independent decision"}, "RecoverNotice": {KindDerived, "pops a notice keyed by id that only the relay's own immediately-preceding, already-authorized StartRunContent call could have set"}, diff --git a/internal/adapter/server/native_execution_lifecycle_test.go b/internal/adapter/server/native_execution_lifecycle_test.go new file mode 100644 index 0000000000..bc11cea4e1 --- /dev/null +++ b/internal/adapter/server/native_execution_lifecycle_test.go @@ -0,0 +1,93 @@ +package server_test + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memfs" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +type nativeRunProvider struct { + ref session.EnvironmentRef + acquires atomic.Int64 + releases atomic.Int64 +} + +func (p *nativeRunProvider) Bind(context.Context, server.PlacementBindRequest) (server.PlacementBinding, error) { + return p.binding(), nil +} +func (p *nativeRunProvider) Reattach(context.Context, server.PlacementReattachRequest) (server.PlacementBinding, error) { + return p.binding(), nil +} +func (p *nativeRunProvider) Applies(ref session.EnvironmentRef) bool { return ref == p.ref } +func (p *nativeRunProvider) AcquireRun(context.Context, server.ExecutionRunRequest) (server.ExecutionRunHandle, error) { + p.acquires.Add(1) + return &nativeRunHandle{provider: p, env: p.binding().Environment}, nil +} +func (p *nativeRunProvider) binding() server.PlacementBinding { + return server.PlacementBinding{Ref: p.ref, Environment: tool.MustEnvironment(p.ref, memfs.NewWorkspace("/native"), memledger.New(), nil)} +} + +type nativeRunHandle struct { + provider *nativeRunProvider + env tool.Environment +} + +func (h *nativeRunHandle) Environment() tool.Environment { return h.env } +func (*nativeRunHandle) Renew(context.Context) error { return nil } +func (h *nativeRunHandle) Release(context.Context) error { h.provider.releases.Add(1); return nil } + +func TestNativeRunClaimHeldUntilActualDrainRelease(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + provider := &nativeRunProvider{ref: ref} + llm := mockllm.New(mockllm.TextTurn("first"), mockllm.TextTurn("second")) + engine := agent.NewEngine(agent.Deps{LLM: llm, Catalog: tool.NewCatalog(), Policy: permpolicy.NewPolicy(nil, nil), Model: "test"}) + store := memstore.New() + svc, err := newPlacementTestService(server.Config{Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", ExecutionAccess: provider, SharedEngineRoot: "/native", DefaultLimits: session.Limits{MaxTurns: 2}, Now: func() time.Time { return time.Unix(1, 0) }}) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + sess := session.New("native-session", session.ModeDefault, ref, session.Limits{MaxTurns: 2}, time.Unix(1, 0)) + if err := store.Save(t.Context(), sess); err != nil { + t.Fatal(err) + } + + first, err := svc.StartRunContent(t.Context(), sess.ID, "one", nil) + if err != nil { + t.Fatal(err) + } + for range first.Events() { + } + if _, err := svc.StartRunContent(t.Context(), sess.ID, "too early", nil); err == nil { + t.Fatal("native continuation started before the prior relay released its claim") + } + if provider.acquires.Load() != 1 || provider.releases.Load() != 0 { + t.Fatalf("before FinishRun acquires=%d releases=%d", provider.acquires.Load(), provider.releases.Load()) + } + + svc.FinishRun(sess.ID, first) + if provider.releases.Load() != 1 { + t.Fatalf("release count=%d, want 1", provider.releases.Load()) + } + second, err := svc.StartRunContent(t.Context(), sess.ID, "two", nil) + if err != nil { + t.Fatal(err) + } + for range second.Events() { + } + svc.FinishRun(sess.ID, second) + if provider.acquires.Load() != 2 || provider.releases.Load() != 2 { + t.Fatalf("after continuation acquires=%d releases=%d", provider.acquires.Load(), provider.releases.Load()) + } +} diff --git a/internal/adapter/server/placement.go b/internal/adapter/server/placement.go index fec39dbe51..58245c0fbb 100644 --- a/internal/adapter/server/placement.go +++ b/internal/adapter/server/placement.go @@ -1,3 +1,4 @@ +//nolint:revive // Host-only lifecycle seams are exported solely for internal adapter composition. package server import ( @@ -5,6 +6,7 @@ import ( "errors" "fmt" "strings" + "time" "unicode" "unicode/utf8" @@ -122,9 +124,29 @@ type PlacementBinding struct { Environment tool.Environment Ref session.EnvironmentRef Metadata PlacementMetadata - // Close releases provisional provider resources. It is called after creation - // because ordinary bindings are reattached fresh at run entry. - Close func() error + // Commit publishes a provisional durable reference after the session store + // definitively publishes the matching binding. Close aborts it only while Commit + // has not succeeded; an ambiguous commit must remain recoverable. + Commit func(context.Context) error + Close func() error +} + +type ExecutionRunRequest struct { + Ref session.EnvironmentRef + Principal *session.Principal + BindingID session.SessionID + RunID string +} + +type ExecutionRunHandle interface { + Environment() tool.Environment + Renew(context.Context) error + Release(context.Context) error +} + +type ExecutionAccess interface { + Applies(session.EnvironmentRef) bool + AcquireRun(context.Context, ExecutionRunRequest) (ExecutionRunHandle, error) } // PlacementProvider owns placement authorization, atomic binding resolution, @@ -133,6 +155,46 @@ type PlacementProvider interface { Bind(context.Context, PlacementBindRequest) (PlacementBinding, error) } +type PlacementSuccessorRequest struct { + Ref session.EnvironmentRef + Principal *session.Principal + SourceBindingID session.SessionID + DestinationBindingID session.SessionID +} + +type ReferenceDeleteHandle interface { + Confirm(context.Context) error + Cancel(context.Context) error +} +type ReferenceLifecycle interface { + Applies(session.EnvironmentRef) bool + PrepareReferenceDelete(context.Context, PlacementSuccessorRequest) (ReferenceDeleteHandle, error) +} + +// ReferenceIntent is the owner-attested, exact provider operation retained after +// an ambiguous publication or deletion outcome. +type ReferenceIntent struct { + Ref session.EnvironmentRef + Principal *session.Principal + BindingID session.SessionID + SourceBindingID session.SessionID + OperationID string + PendingDelete bool +} + +// ReferenceIntentLifecycle is the optional host reconciliation seam. Its list is +// scoped by the provider to the authenticated mTLS client and bounded by limit. +type ReferenceIntentLifecycle interface { + ListReferenceIntents(context.Context, int) ([]ReferenceIntent, error) + CommitReferenceIntent(context.Context, ReferenceIntent) error + ConfirmReferenceIntentDelete(context.Context, ReferenceIntent) error + CancelReferenceIntentDelete(context.Context, ReferenceIntent) error +} + +type PlacementSuccessorReservoir interface { + ReserveSuccessor(context.Context, PlacementSuccessorRequest) (PlacementBinding, error) +} + // PlacementDiscoveryRequest scopes alternate-worktree discovery to an owned // source and its exact current placement. type PlacementDiscoveryRequest struct { @@ -299,7 +361,17 @@ func (s *Service) persistPlacedCreatedSession(ctx context.Context, sess *session if !sess.EnvironmentRef.Valid() { return nil, fmt.Errorf("%w: placement did not provide an exact environment ref", ErrInvalidPlacementBinding) } - return s.persistCreatedSession(ctx, sess, owner, request) + persisted, err := s.persistCreatedSession(ctx, sess, owner, request) + if err != nil || persisted != sess || placement == nil || placement.Commit == nil { + return persisted, err + } + commitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if err := placement.Commit(commitCtx); err != nil { + s.logPlacementProviderError(ctx, "commit reference", err) + return sess, fmt.Errorf("%w: session %q persisted but reference publication must be retried", ErrInternal, sess.ID) + } + return sess, nil } func (s *Service) resolveSchedulePlacement(ctx context.Context, ref session.EnvironmentRef, profile SessionProfile) (session.EnvironmentRef, string, SessionProfile, error) { diff --git a/internal/adapter/server/placement_successor.go b/internal/adapter/server/placement_successor.go index 81744044a2..0f85e8c0c2 100644 --- a/internal/adapter/server/placement_successor.go +++ b/internal/adapter/server/placement_successor.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/stacklok/mecatl/engine/port" "github.com/stacklok/mecatl/engine/session" @@ -57,7 +58,7 @@ func (s *Service) ClearSessionSuccessor(ctx context.Context, source session.Sess // non-destructive gate. Exact inherited placement stays on the lease-protected // path below because reattachment may itself wait for lease loss/cancellation. if placement.Selector != "" { - preflight, placementErr := s.successorPlacement(ctx, lockedSource, placement) + preflight, placementErr := s.successorPlacement(ctx, lockedSource, placement, "") if placementErr != nil { return "", placementErr } @@ -153,11 +154,17 @@ func (s *Service) createPlacedSuccessorLocked(ctx context.Context, req ForkSucce if err != nil || absent { return "", err } + destinationID := s.cfg.NewID() + releaseDestination, reserveErr := s.reserveGeneratedSessionID(mutationCtx, destinationID) + if reserveErr != nil { + return "", reserveErr + } + defer releaseDestination() selector, err := successorProviderSelector(source, req) if err != nil { return "", err } - binding, err := s.successorPlacement(mutationCtx, source, req.Placement) + binding, err := s.successorPlacement(mutationCtx, source, req.Placement, destinationID) if err != nil { return "", err } @@ -176,7 +183,7 @@ func (s *Service) createPlacedSuccessorLocked(ctx context.Context, req ForkSucce } } - created := session.New(s.cfg.NewID(), source.Mode, binding.Ref, source.Limits, s.cfg.Now()) + created := session.New(destinationID, source.Mode, binding.Ref, source.Limits, s.cfg.Now()) created.Placement = canonicalPlacementMetadata(binding) authority, bound := source.BoundAuthority() if !bound { @@ -261,6 +268,15 @@ func (s *Service) createPlacedSuccessorLocked(ctx context.Context, req ForkSucce s.logDiscoveryError(ctx, "persist successor placement", err) return "", fmt.Errorf("%w: placement storage failed", ErrInternal) } + if binding.Commit != nil { + commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(mutationCtx), 10*time.Second) + commitErr := binding.Commit(commitCtx) + cancelCommit() + if commitErr != nil { + s.logPlacementProviderError(ctx, "commit successor reference", commitErr) + return created.ID, fmt.Errorf("%w: session %q persisted but reference publication must be retried", ErrInternal, created.ID) + } + } if broker != nil { commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(mutationCtx), engineCloseTimeout) commitErr := s.commitBrokerAttachment(commitCtx, created.ID, broker) @@ -291,8 +307,15 @@ func successorProviderSelector(source *session.Session, req ForkSuccessorRequest return selector, nil } -func (s *Service) successorPlacement(ctx context.Context, source *session.Session, requested SuccessorPlacement) (PlacementBinding, error) { +func (s *Service) successorPlacement(ctx context.Context, source *session.Session, requested SuccessorPlacement, destination session.SessionID) (PlacementBinding, error) { if requested.Selector == "" { + if destination != "" && s.cfg.ExecutionAccess != nil && s.cfg.ExecutionAccess.Applies(source.EnvironmentRef) { + reservoir, ok := s.cfg.PlacementProvider.(PlacementSuccessorReservoir) + if !ok { + return PlacementBinding{}, ErrPlacementUnavailable + } + return reservoir.ReserveSuccessor(ctx, PlacementSuccessorRequest{Ref: source.EnvironmentRef, Principal: source.Owner, SourceBindingID: source.ID, DestinationBindingID: destination}) + } return s.ReattachPlacementForBinding(ctx, source.EnvironmentRef, source.ID) } if source.EnvironmentRef.Kind == session.EnvKindNoFS { diff --git a/internal/adapter/server/reference_intent_reconcile_test.go b/internal/adapter/server/reference_intent_reconcile_test.go new file mode 100644 index 0000000000..6933cb2a87 --- /dev/null +++ b/internal/adapter/server/reference_intent_reconcile_test.go @@ -0,0 +1,86 @@ +package server_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +type intentLifecycle struct { + intents []server.ReferenceIntent + mu sync.Mutex + committed, confirmed, reset []session.SessionID +} + +func (l *intentLifecycle) ListReferenceIntents(context.Context, int) ([]server.ReferenceIntent, error) { + return append([]server.ReferenceIntent(nil), l.intents...), nil +} +func (l *intentLifecycle) CommitReferenceIntent(_ context.Context, i server.ReferenceIntent) error { + l.mu.Lock() + defer l.mu.Unlock() + l.committed = append(l.committed, i.BindingID) + return nil +} +func (l *intentLifecycle) ConfirmReferenceIntentDelete(_ context.Context, i server.ReferenceIntent) error { + l.mu.Lock() + defer l.mu.Unlock() + l.confirmed = append(l.confirmed, i.BindingID) + return nil +} +func (l *intentLifecycle) CancelReferenceIntentDelete(_ context.Context, i server.ReferenceIntent) error { + l.mu.Lock() + defer l.mu.Unlock() + l.reset = append(l.reset, i.BindingID) + return nil +} + +func TestReferenceIntentReconciliationIsOwnerAndRefExact(t *testing.T) { + owner := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + otherRef := session.EnvironmentRef{Kind: ref.Kind, ID: ref.ID, Revision: "new-revision"} + store := memstore.New() + persist := func(id session.SessionID, placement session.EnvironmentRef) { + sess := session.New(id, session.ModeDefault, placement, session.Limits{MaxTurns: 1}, time.Unix(1, 0)) + if err := sess.RestoreLabels(owner, session.Authority{}); err != nil { + t.Fatal(err) + } + if err := store.Save(t.Context(), sess); err != nil { + t.Fatal(err) + } + } + persist("create-match", ref) + persist("delete-restored", ref) + persist("reused-id", otherRef) + lifecycle := &intentLifecycle{intents: []server.ReferenceIntent{ + {Ref: ref, Principal: owner, BindingID: "create-match", OperationID: "create-op"}, + {Ref: ref, Principal: owner, BindingID: "delete-gone", OperationID: "delete-op", PendingDelete: true}, + {Ref: ref, Principal: owner, BindingID: "delete-restored", OperationID: "delete-restore-op", PendingDelete: true}, + {Ref: ref, Principal: owner, BindingID: "reused-id", OperationID: "old-delete-op", PendingDelete: true}, + }} + engine := agent.NewEngine(agent.Deps{LLM: mockllm.New(), Catalog: tool.NewCatalog(), Policy: permpolicy.NewPolicy(nil, nil)}) + svc, err := newPlacementTestService(server.Config{Engine: engine, Store: store, PlacementProvider: resolverPlacementProvider{}, PlacementScope: "test", ReferenceIntents: lifecycle}) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + + svc.ReconcileReferenceIntents(t.Context()) + if len(lifecycle.committed) != 1 || lifecycle.committed[0] != "create-match" { + t.Fatalf("committed=%v", lifecycle.committed) + } + if len(lifecycle.confirmed) != 1 || lifecycle.confirmed[0] != "delete-gone" { + t.Fatalf("confirmed=%v", lifecycle.confirmed) + } + if len(lifecycle.reset) != 1 || lifecycle.reset[0] != "delete-restored" { + t.Fatalf("cancelled=%v", lifecycle.reset) + } +} diff --git a/internal/adapter/server/sdk_typescript_release_test.go b/internal/adapter/server/sdk_typescript_release_test.go index d5b47268bf..3c1ad07d9c 100644 --- a/internal/adapter/server/sdk_typescript_release_test.go +++ b/internal/adapter/server/sdk_typescript_release_test.go @@ -229,6 +229,7 @@ func TestSDKTypescriptRelease_Scenario1_PublicServiceProjectionParity(t *testing "LostOwnershipCandidates", "MaintenanceMutationAvailable", "ManualDreamCapabilities", + "ReconcileReferenceIntents", "MaybeAutoApprovePlan", "OwnershipEnforced", "Persist", diff --git a/internal/adapter/server/service.go b/internal/adapter/server/service.go index b3a7b34730..ae6c16fd55 100644 --- a/internal/adapter/server/service.go +++ b/internal/adapter/server/service.go @@ -343,6 +343,12 @@ type Config struct { // PlacementScope is the trusted deployment scope supplied to every provider // Bind. It must be non-empty when PlacementProvider is configured. PlacementScope PlacementScope + // ExecutionAccess is an optional host-only run-ownership seam. Local and no-FS + // placements leave it nil and perform no lifecycle RPCs. + ExecutionAccess ExecutionAccess + // ReferenceIntents enables owner-safe reconciliation of provider-retained + // publication/deletion outcomes. Nil is fully inert. + ReferenceIntents ReferenceIntentLifecycle // SessionReadLedger optionally selects a durable read-before-write ledger for // each session independently of its placement's content backend. The returned // handle must be non-nil; nil fails the run closed. @@ -902,6 +908,8 @@ type Service struct { // refresh — background or on-demand — re-projects it). Never nil after // NewService. providerStatus atomic.Pointer[[]*mecatlv1.ProviderStatus] + // referenceIntentWarned bounds diagnostics for retained ambiguous intents. + referenceIntentWarned atomic.Bool // modelsRefresher is the OPTIONAL composition-supplied closure ListModels // calls before returning its snapshot (issue #262, R1.4: a proxy started @@ -1293,8 +1301,10 @@ type runState struct { admissionCancel context.CancelFunc // runContextStop releases the lease-linked launch context after the promoted // run has settled. It must outlive the run-entry call itself. - runContextStop context.CancelFunc - awaiting atomic.Bool + runContextStop context.CancelFunc + execution ExecutionRunHandle + executionRenewCancel context.CancelFunc + awaiting atomic.Bool // titleRevision is the last title metadata revision successfully persisted and // published for this run. It starts from the admitted durable snapshot so // prompt-ingress changes publish only after their save succeeds. @@ -2545,6 +2555,15 @@ func (s *Service) createPerSessionEngine(ctx context.Context, mintID func() sess } return persisted, perr } + if placement != nil && placement.Commit != nil { + commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + commitErr := placement.Commit(commitCtx) + cancelCommit() + if commitErr != nil { + s.logPlacementProviderError(ctx, "commit reference", commitErr) + return sess, fmt.Errorf("%w: session %q persisted but reference publication must be retried", ErrInternal, sess.ID) + } + } if broker != nil { commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(ctx), engineCloseTimeout) commitErr := s.commitBrokerAttachment(commitCtx, id, broker) @@ -3336,10 +3355,97 @@ func (s *Service) StorageReady(ctx context.Context) bool { return p.Ping(ctx) == nil } +// ReconcileReferenceIntents resolves the provider's bounded, client-scoped +// ambiguous publication and deletion outcomes against exact durable sessions. +func (s *Service) ReconcileReferenceIntents(ctx context.Context) { + lifecycle := s.cfg.ReferenceIntents + if lifecycle == nil { + return + } + intents, err := lifecycle.ListReferenceIntents(ctx, 64) + if err != nil { + s.warnReferenceIntentRetention(ctx) + return + } + retained := false + for _, intent := range intents { + if intent.BindingID == "" || intent.OperationID == "" || !intent.Ref.Valid() || intent.Principal == nil { + retained = true + continue + } + sess, loadErr := s.cfg.Store.Load(ctx, intent.BindingID) + if loadErr != nil { + if errors.Is(loadErr, port.ErrSessionNotFound) && intent.PendingDelete { + if err := lifecycle.ConfirmReferenceIntentDelete(ctx, intent); err != nil { + retained = true + } + } else { + // A missing pending create may have been published by another replica + // after this read. It is never safe to abort or TTL-collect it here. + retained = true + } + continue + } + matching := sess != nil && sess.ID == intent.BindingID && sess.EnvironmentRef == intent.Ref && sess.Owner != nil && sess.Owner.SameIdentity(intent.Principal) + if !matching { + retained = true + continue + } + if intent.PendingDelete { + err = lifecycle.CancelReferenceIntentDelete(ctx, intent) + } else { + err = lifecycle.CommitReferenceIntent(ctx, intent) + } + if err != nil { + retained = true + } + } + if retained { + s.warnReferenceIntentRetention(ctx) + } else { + s.referenceIntentWarned.Store(false) + } +} + +func (s *Service) warnReferenceIntentRetention(ctx context.Context) { + if s.referenceIntentWarned.CompareAndSwap(false, true) { + s.cfg.Diagnostics.Log(ctx, port.LevelWarn, "execution reference reconciliation retained ambiguous intents") + } +} + func (s *Service) saveSession(ctx context.Context, sess *session.Session) error { return s.cfg.MutationCapability.GuardStore(s.cfg.Store).Save(ctx, sess) } +func (s *Service) prepareReferenceDelete(ctx context.Context, sess *session.Session) (ReferenceDeleteHandle, error) { + if sess == nil || s.cfg.ExecutionAccess == nil || !s.cfg.ExecutionAccess.Applies(sess.EnvironmentRef) { + return nil, nil + } + lifecycle, ok := s.cfg.PlacementProvider.(ReferenceLifecycle) + if !ok { + return nil, ErrPlacementUnavailable + } + return lifecycle.PrepareReferenceDelete(ctx, PlacementSuccessorRequest{Ref: sess.EnvironmentRef, Principal: sess.Owner, SourceBindingID: sess.ID}) +} +func (s *Service) cancelReferenceDelete(ctx context.Context, handle ReferenceDeleteHandle) { + if handle == nil { + return + } + cancelCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if err := handle.Cancel(cancelCtx); err != nil { + s.logPlacementProviderError(ctx, "cancel reference delete", err) + } +} +func (*Service) confirmReferenceDelete(ctx context.Context, handle ReferenceDeleteHandle) error { + if handle == nil { + return nil + } + confirmCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + return handle.Confirm(confirmCtx) +} + func (s *Service) deleteSessionFamily(ctx context.Context, id session.SessionID, store port.PrunableStore) error { if !s.mutationLeaseHeld(id) { return fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) @@ -3562,10 +3668,17 @@ func (s *Service) DeleteSession(ctx context.Context, id session.SessionID) error if err != nil || absent { return err } + referenceDelete, err := s.prepareReferenceDelete(ctx, sess) + if err != nil { + return err + } unlockBroker := s.brokerMu.lock(id) defer unlockBroker() if err := s.deleteSessionFamily(ctx, sess.ID, prunable); err != nil { if errors.Is(err, port.ErrSessionNotFound) { + if confirmErr := s.confirmReferenceDelete(ctx, referenceDelete); confirmErr != nil { + return fmt.Errorf("%w: confirm deleted session reference: %v", ErrInternal, confirmErr) + } // The durable record is already gone: still attempt broker cleanup // (best-effort) before reporting success, so a locally-retained // broker handle is never orphaned by an already-completed delete. @@ -3576,11 +3689,15 @@ func (s *Service) DeleteSession(ctx context.Context, id session.SessionID) error s.closeSessionLocal(id) return nil } + s.cancelReferenceDelete(ctx, referenceDelete) if errors.Is(err, port.ErrPruneUnsupported) { return ErrSessionDeleteUnsupported } return fmt.Errorf("%w: delete session: %v", ErrInternal, err) } + if err := s.confirmReferenceDelete(ctx, referenceDelete); err != nil { + return fmt.Errorf("%w: session deleted but reference confirmation must be retried", ErrInternal) + } // The durable record is gone; broker cleanup is now best-effort. Reordered // deliberately (I-8): deleting broker state FIRST left an unrecoverable // partial-deletion window if the durable delete then failed — the snapshot @@ -3632,6 +3749,10 @@ func (s *Service) DeleteSessionForRetentionCandidate(ctx context.Context, candid if !retentionCandidateMatches(sess, candidate) { return errRetentionCandidateChanged } + referenceDelete, err := s.prepareReferenceDelete(ctx, sess) + if err != nil { + return err + } unlockBroker := s.brokerMu.lock(candidate.ID) defer unlockBroker() if !s.mutationLeaseHeld(candidate.ID) { @@ -3639,14 +3760,19 @@ func (s *Service) DeleteSessionForRetentionCandidate(ctx context.Context, candid } deleted, err := deleter.DeleteSessionIfUnchanged(ctx, candidate) if err != nil { + s.cancelReferenceDelete(ctx, referenceDelete) if errors.Is(err, port.ErrPruneUnsupported) { return ErrSessionDeleteUnsupported } return fmt.Errorf("%w: delete retention candidate: %v", ErrInternal, err) } if !deleted { + s.cancelReferenceDelete(ctx, referenceDelete) return errRetentionCandidateChanged } + if err := s.confirmReferenceDelete(ctx, referenceDelete); err != nil { + return fmt.Errorf("%w: retention deleted session but reference confirmation must be retried", ErrInternal) + } // The durable record is gone; broker cleanup is now best-effort (I-8: see // deleteBrokerSessionLocked's doc comment for the ordering rationale). if err := s.deleteBrokerSessionLocked(ctx, candidate.ID); err != nil { @@ -3672,6 +3798,8 @@ func retentionCandidateMatches(sess *session.Session, candidate port.SessionDisc // chats, but it still serializes against run entry, acquires the cross-process // mutation lease, and revalidates durable taxonomy and lifecycle after acquiring // that lease. It is an internal composition callback, never a wire operation. +// +//nolint:gocyclo // Retention keeps lease, taxonomy, reference intent, and deletion in one transaction. func (s *Service) DeleteSessionForRetention(ctx context.Context, id session.SessionID) error { unlock := s.runEntryMu.lock(id) defer unlock() @@ -3702,10 +3830,17 @@ func (s *Service) DeleteSessionForRetention(ctx context.Context, id session.Sess if sess.State == session.StateRunning || sess.State == session.StateAwaiting || sess.State == session.StateAuthorizing || s.IsLive(id) { return fmt.Errorf("%w: retention candidate is active or awaiting approval", ErrFailedPrecondition) } + referenceDelete, err := s.prepareReferenceDelete(ctx, sess) + if err != nil { + return err + } unlockBroker := s.brokerMu.lock(id) defer unlockBroker() if err := s.deleteSessionFamily(ctx, id, prunable); err != nil { if errors.Is(err, port.ErrSessionNotFound) { + if confirmErr := s.confirmReferenceDelete(ctx, referenceDelete); confirmErr != nil { + return fmt.Errorf("%w: confirm deleted retention reference: %v", ErrInternal, confirmErr) + } if brokerErr := s.deleteBrokerSessionLocked(ctx, id); brokerErr != nil { s.cfg.Diagnostics.Log(context.WithoutCancel(ctx), port.LevelWarn, "broker cleanup after already-deleted retention candidate failed", "session", string(id), "err", brokerErr.Error()) @@ -3713,11 +3848,15 @@ func (s *Service) DeleteSessionForRetention(ctx context.Context, id session.Sess s.closeSessionLocal(id) return nil } + s.cancelReferenceDelete(ctx, referenceDelete) if errors.Is(err, port.ErrPruneUnsupported) { return ErrSessionDeleteUnsupported } return fmt.Errorf("%w: delete retention candidate: %v", ErrInternal, err) } + if err := s.confirmReferenceDelete(ctx, referenceDelete); err != nil { + return fmt.Errorf("%w: retention deleted session but reference confirmation must be retried", ErrInternal) + } // The durable record is gone; broker cleanup is now best-effort (I-8: see // deleteBrokerSessionLocked's doc comment for the ordering rationale). if err := s.deleteBrokerSessionLocked(ctx, id); err != nil { @@ -4395,11 +4534,8 @@ func (s *Service) RetryFailedRun(ctx context.Context, id session.SessionID) (*ag if err := admitRunPurpose(sess, runPurposeChat); err != nil { return nil, fmt.Errorf("%w: session %q is not eligible for chat retry", ErrFailedStepRetryIneligible, id) } - if registered, ok := s.LookupRun(id); ok { - if !sess.State.IsTerminal() { - return nil, fmt.Errorf("%w: session %q already has an active run", ErrFailedStepRetryIneligible, id) - } - s.deregister(id, registered) + if _, ok := s.LookupRun(id); ok { + return nil, fmt.Errorf("%w: session %q still has an undrained run", ErrFailedStepRetryIneligible, id) } eligibleFailure, err := failedStepRetryEligibility(sess) if err != nil { @@ -4439,6 +4575,10 @@ func (s *Service) RetryFailedRun(ctx context.Context, id session.SessionID) (*ag if !leaseHeld() { return nil, fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } + env, err = s.acquireExecution(ctx, st, sess, sess.RunID(), env) + if err != nil { + return nil, err + } ctx = memory.WithWorkspace(ctx, env.Workspace().Root()) run, err := s.promoteRunAdmission(id, st, stopAdmission, func() *agent.Run { return engine.RetryFailedStep(ctx, sess, env) @@ -4447,6 +4587,7 @@ func (s *Service) RetryFailedRun(ctx context.Context, id session.SessionID) (*ag return nil, err } promoted = true + s.startExecutionRenewal(id, st, run) return run, nil } @@ -4560,12 +4701,19 @@ func (s *Service) startRunContent(ctx context.Context, id session.SessionID, tex } // The run registry is the authoritative same-process single-run gate while the // loaded aggregate is non-terminal. A terminal snapshot means the registered - // run has finished driving but its relay has not called FinishRun yet. Remove - // that exact run before reopening so the finished relay's later FinishRun - // cannot deregister the continuation that replaces it. + // run has finished driving but its relay has not called FinishRun yet. Local + // and no-FS runs retain the historical fast handoff: remove that exact run + // before reopening so its later FinishRun cannot deregister the continuation. + // Native execution is different: its runState owns the provider claim until + // FinishRun after relay drain, so an undrained native run must remain registered + // and block a replacement claim. if registered, ok := s.LookupRun(id); ok { - if !sess.State.IsTerminal() { - return nil, fmt.Errorf("%w: session %q already has an active run", ErrFailedPrecondition, id) + s.mu.Lock() + st := s.runs[id] + nativeUndrained := st != nil && st.run == registered && st.execution != nil + s.mu.Unlock() + if !sess.State.IsTerminal() || nativeUndrained { + return nil, fmt.Errorf("%w: session %q still has an active or undrained run", ErrFailedPrecondition, id) } s.deregister(id, registered) } @@ -4680,6 +4828,10 @@ func (s *Service) startRunContent(ctx context.Context, id session.SessionID, tex // one mint site for a new run. runID := newRunID() sess.BeginRun(runID) + env, err = s.acquireExecution(ctx, st, sess, runID, env) + if err != nil { + return nil, err + } if !leaseHeld() { return nil, fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } @@ -4696,6 +4848,7 @@ func (s *Service) startRunContent(ctx context.Context, id session.SessionID, tex *interruptedContinuationOwned = true } promoted = true + s.startExecutionRenewal(id, st, run) return run, nil } @@ -6245,6 +6398,10 @@ func (s *Service) resumeFromAwaiting(ctx context.Context, id session.SessionID, if !leaseHeld() { return nil, fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } + env, err = s.acquireExecution(ctx, st, sess, sess.RunID(), env) + if err != nil { + return nil, err + } ctx = memory.WithWorkspace(ctx, env.Workspace().Root()) run, err := s.promoteRunAdmission(id, st, stopAdmission, func() *agent.Run { return engine.ResumeApproval(ctx, sess, env, askID, verdict) @@ -6253,6 +6410,7 @@ func (s *Service) resumeFromAwaiting(ctx context.Context, id session.SessionID, return nil, err } promoted = true + s.startExecutionRenewal(id, st, run) return run, nil } @@ -6352,7 +6510,7 @@ func (s *Service) approvePlan(ctx context.Context, id session.SessionID, targetM var resumedStop session.StopReason if resumed != nil { resumedStop = s.forwardRunEvents(ctx, id, resumed, out) - s.deregister(id, resumed) + s.FinishRun(id, resumed) } // (5) Atomic continuation (allow paths only). A deny leaves the session // in plan mode with no continuation run — the model re-plans on the next @@ -6388,7 +6546,7 @@ func (s *Service) approvePlan(ctx context.Context, id session.SessionID, targetM return } s.forwardRunEvents(ctx, id, cont, out) - s.deregister(id, cont) + s.FinishRun(id, cont) }() return out, nil @@ -6709,7 +6867,7 @@ func (s *Service) completeRelay(ctx context.Context, id session.SessionID, run * // disconnected client cannot lose the terminal snapshot, then release the run. func (s *Service) finishRelayRun(ctx context.Context, id session.SessionID, run *agent.Run) { s.completeRelay(ctx, id, run) - s.deregister(id, run) + s.FinishRun(id, run) } // appendEvent durably records one projected relay event to the configured @@ -7002,7 +7160,7 @@ func (s *Service) autoApproveContinuation(ctx context.Context, id session.Sessio recorder.Observe(ev) } recorder.Close() - s.deregister(id, cont) + s.FinishRun(id, cont) } // autoApproveWaitTimeout bounds how long autoApproveContinuation waits for the @@ -7823,6 +7981,13 @@ func (s *Service) cleanupRunAdmission(id session.SessionID, st *runState, promot return } s.cancelRegisteredRunState(id, st, nil, true) + if st.execution != nil { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := st.execution.Release(ctx); err != nil { + s.logDiscoveryError(context.Background(), "execution admission rollback failed", err) + } + cancel() + } s.removeRunState(id, st) } @@ -7903,6 +8068,51 @@ func (s *Service) deregister(id session.SessionID, run *agent.Run) { } } +func (s *Service) acquireExecution(ctx context.Context, st *runState, sess *session.Session, runID string, fallback tool.Environment) (tool.Environment, error) { + access := s.cfg.ExecutionAccess + if access == nil || !access.Applies(sess.EnvironmentRef) { + return fallback, nil + } + handle, err := access.AcquireRun(ctx, ExecutionRunRequest{Ref: sess.EnvironmentRef, Principal: sess.Owner, BindingID: sess.ID, RunID: runID}) + if err != nil { + return tool.Environment{}, err + } + env := handle.Environment() + if env.Ref() != sess.EnvironmentRef || env.Workspace() == nil { + _ = handle.Release(context.WithoutCancel(ctx)) + return tool.Environment{}, ErrInvalidPlacementBinding + } + st.execution = handle + return env, nil +} + +func (s *Service) startExecutionRenewal(_ session.SessionID, st *runState, run *agent.Run) { + if st.execution == nil { + return + } + ctx, cancel := context.WithCancel(context.Background()) + st.executionRenewCancel = cancel + go func() { + ticker := time.NewTicker(20 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + renewCtx, stop := context.WithTimeout(ctx, 10*time.Second) + err := st.execution.Renew(renewCtx) + stop() + if err != nil { + s.logDiscoveryError(context.Background(), "execution ownership renewal failed", err) + run.Cancel() + return + } + } + } + }() +} + // FinishRun removes run from the in-flight registry for id. It is the EXPORTED // counterpart of register that every wire adapter must call (typically via // `defer`) once it has finished draining run.Events(), so a completed run does @@ -7922,6 +8132,16 @@ func (s *Service) FinishRun(id session.SessionID, run *agent.Run) { if parked && st.sess != nil { pending, pendingOK = st.sess.PendingAuthorization() } + if st.executionRenewCancel != nil { + st.executionRenewCancel() + } + if st.execution != nil { + releaseCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := st.execution.Release(releaseCtx); err != nil { + s.logDiscoveryError(context.Background(), "execution ownership release failed", err) + } + cancel() + } s.removeRunState(id, st) if parked { s.scheduleAuthorizationExpiry(id, pending, pendingOK) diff --git a/internal/app/build.go b/internal/app/build.go index 18b8d6a405..108e73e8d6 100644 --- a/internal/app/build.go +++ b/internal/app/build.go @@ -2213,6 +2213,8 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { PlacementProvider: placementProvider, PlacementScope: placementScope, + ExecutionAccess: executionAccess(placementProvider), + ReferenceIntents: referenceIntentLifecycle(placementProvider), SessionReadLedger: sessionReadLedger, RootAuthority: func(kind session.SessionKind) session.Authority { return mintRootAuthority(assets.rootCatalog, mcpResourceCapabilities(assets.globalMgr), kind) @@ -2720,6 +2722,7 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { // subagent-*/parallel-*/team-* children the run-entry funnel's own repair // (Step 3) never sees. See internal/app/session_reconcile.go. staleSessionReconcileClose := startStaleSessionReconcile(cfg, svc) + referenceIntentReconcileClose := startReferenceIntentReconcile(ctx, svcCfg.ReferenceIntents, svc) // Close tears down the main MCP manager AND any per-session client-MCP engines // still registered (svc.Close), so a process exit leaks neither. It also cancels @@ -2730,6 +2733,7 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { if assets.reflectionLifecycle != nil { assets.reflectionLifecycle.close() } + referenceIntentReconcileClose() staleSessionReconcileClose() childGCClose() managedTempWorkerClose() diff --git a/internal/app/placement.go b/internal/app/placement.go index c538458083..2a6cf94380 100644 --- a/internal/app/placement.go +++ b/internal/app/placement.go @@ -18,6 +18,20 @@ const ( defaultPlacementScope = server.PlacementScope("deployment") ) +func referenceIntentLifecycle(provider server.PlacementProvider) server.ReferenceIntentLifecycle { + if profiled, ok := provider.(*profilePlacementProvider); ok { + lifecycle, _ := profiled.remote.(server.ReferenceIntentLifecycle) + return lifecycle + } + lifecycle, _ := provider.(server.ReferenceIntentLifecycle) + return lifecycle +} + +func executionAccess(provider server.PlacementProvider) server.ExecutionAccess { + access, _ := provider.(server.ExecutionAccess) + return access +} + type profilePlacementProvider struct { remote server.PlacementProvider local *localPlacementProvider @@ -46,6 +60,63 @@ func (p *profilePlacementProvider) Reattach(ctx context.Context, req server.Plac return remote.Reattach(ctx, req) } +func (p *profilePlacementProvider) PrepareReferenceDelete(ctx context.Context, req server.PlacementSuccessorRequest) (server.ReferenceDeleteHandle, error) { + lifecycle, ok := p.remote.(server.ReferenceLifecycle) + if !ok || !lifecycle.Applies(req.Ref) { + return nil, server.ErrPlacementUnavailable + } + return lifecycle.PrepareReferenceDelete(ctx, req) +} + +func (p *profilePlacementProvider) ListReferenceIntents(ctx context.Context, limit int) ([]server.ReferenceIntent, error) { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return nil, server.ErrPlacementUnavailable + } + return lifecycle.ListReferenceIntents(ctx, limit) +} +func (p *profilePlacementProvider) CommitReferenceIntent(ctx context.Context, intent server.ReferenceIntent) error { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return server.ErrPlacementUnavailable + } + return lifecycle.CommitReferenceIntent(ctx, intent) +} +func (p *profilePlacementProvider) ConfirmReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return server.ErrPlacementUnavailable + } + return lifecycle.ConfirmReferenceIntentDelete(ctx, intent) +} +func (p *profilePlacementProvider) CancelReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return server.ErrPlacementUnavailable + } + return lifecycle.CancelReferenceIntentDelete(ctx, intent) +} + +func (p *profilePlacementProvider) ReserveSuccessor(ctx context.Context, req server.PlacementSuccessorRequest) (server.PlacementBinding, error) { + reservoir, ok := p.remote.(server.PlacementSuccessorReservoir) + if !ok { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return reservoir.ReserveSuccessor(ctx, req) +} + +func (p *profilePlacementProvider) Applies(ref session.EnvironmentRef) bool { + access, ok := p.remote.(server.ExecutionAccess) + return ok && access.Applies(ref) +} +func (p *profilePlacementProvider) AcquireRun(ctx context.Context, req server.ExecutionRunRequest) (server.ExecutionRunHandle, error) { + access, ok := p.remote.(server.ExecutionAccess) + if !ok || !access.Applies(req.Ref) { + return nil, server.ErrPlacementUnavailable + } + return access.AcquireRun(ctx, req) +} + // localPlacementProvider is the trusted composition default. It owns exactly // one configured local record plus the no-FS attenuation; it has no inventory // registry or path-derived public identifier. diff --git a/internal/app/reference_intent_reconcile.go b/internal/app/reference_intent_reconcile.go new file mode 100644 index 0000000000..03fd722a8a --- /dev/null +++ b/internal/app/reference_intent_reconcile.go @@ -0,0 +1,36 @@ +package app + +import ( + "context" + "time" + + "github.com/stacklok/mecatl/internal/adapter/server" +) + +const referenceIntentReconcileInterval = time.Minute + +func startReferenceIntentReconcile(ctx context.Context, lifecycle server.ReferenceIntentLifecycle, svc *server.Service) func() { + if lifecycle == nil || svc == nil { + return func() {} + } + workerCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + defer close(done) + svc.ReconcileReferenceIntents(workerCtx) + ticker := time.NewTicker(referenceIntentReconcileInterval) + defer ticker.Stop() + for { + select { + case <-workerCtx.Done(): + return + case <-ticker.C: + svc.ReconcileReferenceIntents(workerCtx) + } + } + }() + return func() { + cancel() + <-done + } +} diff --git a/internal/executionenv/grant.go b/internal/executionenv/grant.go index 6dab8a2ba0..7da228d2e2 100644 --- a/internal/executionenv/grant.go +++ b/internal/executionenv/grant.go @@ -27,28 +27,34 @@ type grantHeader struct { // GrantClaims binds a short-lived capability to one client, owner, environment, and epoch. type GrantClaims struct { - KeyID string `json:"-"` - Issuer string `json:"iss"` - Audience string `json:"aud"` - Client string `json:"client"` - OwnerHash string `json:"owner"` - BindingID string `json:"binding_id"` - Environment EnvironmentRef `json:"environment"` - Epoch uint64 `json:"epoch"` - Operations []Operation `json:"operations"` - NotBefore time.Time `json:"not_before"` - ExpiresAt time.Time `json:"expires_at"` - Nonce string `json:"nonce"` + KeyID string `json:"-"` + Issuer string `json:"iss"` + Audience string `json:"aud"` + Client string `json:"client"` + OwnerHash string `json:"owner"` + BindingID string `json:"binding_id"` + RunID string `json:"run_id"` + ClaimID string `json:"claim_id"` + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + GrantGeneration uint64 `json:"grant_generation"` + Operations []Operation `json:"operations"` + NotBefore time.Time `json:"not_before"` + ExpiresAt time.Time `json:"expires_at"` + Nonce string `json:"nonce"` } // GrantExpectation defines the exact binding and operation required by a request. type GrantExpectation struct { - Client string - OwnerHash string - BindingID string - Environment EnvironmentRef - Epoch uint64 - Operation Operation + Client string + OwnerHash string + BindingID string + RunID string + ClaimID string + Environment EnvironmentRef + Epoch uint64 + GrantGeneration uint64 + Operation Operation } // GrantVerifier verifies signed grants against configured trust and revocation state. @@ -140,7 +146,7 @@ func (v GrantVerifier) Verify(token string, e GrantExpectation) (GrantClaims, er if _, ok := v.RevokedNonces[c.Nonce]; ok { return c, errors.New("grant revoked") } - if c.Client != e.Client || c.OwnerHash != e.OwnerHash || c.BindingID != e.BindingID || c.Environment != e.Environment || c.Epoch != e.Epoch { + if c.Client != e.Client || c.OwnerHash != e.OwnerHash || c.BindingID != e.BindingID || c.RunID != e.RunID || c.ClaimID != e.ClaimID || c.Environment != e.Environment || c.Epoch != e.Epoch || c.GrantGeneration != e.GrantGeneration { return c, errors.New("grant binding mismatch") } for _, op := range c.Operations { @@ -150,8 +156,10 @@ func (v GrantVerifier) Verify(token string, e GrantExpectation) (GrantClaims, er } return c, errors.New("grant does not authorize operation") } + +//nolint:gocyclo // Complete fail-closed claim validation is deliberately linear. func validateClaims(c GrantClaims) error { - if c.KeyID == "" || c.Issuer == "" || c.Audience == "" || c.Client == "" || c.OwnerHash == "" || c.BindingID == "" || c.Environment.ID == "" || c.Environment.Revision == "" || c.Epoch == 0 || c.Nonce == "" || c.NotBefore.IsZero() || c.ExpiresAt.IsZero() || !c.ExpiresAt.After(c.NotBefore) { + if c.KeyID == "" || c.Issuer == "" || c.Audience == "" || c.Client == "" || c.OwnerHash == "" || c.BindingID == "" || c.RunID == "" || c.ClaimID == "" || c.Environment.ID == "" || c.Environment.Revision == "" || c.Epoch == 0 || c.GrantGeneration == 0 || c.Nonce == "" || c.NotBefore.IsZero() || c.ExpiresAt.IsZero() || !c.ExpiresAt.After(c.NotBefore) { return errors.New("incomplete grant claims") } if len(c.Operations) == 0 || len(c.Operations) > 32 { diff --git a/internal/executionenv/protocol_test.go b/internal/executionenv/protocol_test.go index 223b2d1241..862780f625 100644 --- a/internal/executionenv/protocol_test.go +++ b/internal/executionenv/protocol_test.go @@ -13,13 +13,13 @@ func TestGrantRoundTripAndExactBindings(t *testing.T) { t.Fatal(err) } now := time.Unix(1_800_000_000, 0).UTC() - claims := GrantClaims{KeyID: "k1", Issuer: "execution-provider", Audience: "mecatl-execution", Client: "spiffe://cluster/ns/mecak8s", OwnerHash: "owner", BindingID: "binding", Environment: EnvironmentRef{ID: "env", Revision: "rev"}, Epoch: 7, Operations: []Operation{OpFileRead}, NotBefore: now.Add(-time.Second), ExpiresAt: now.Add(time.Minute), Nonce: "n1"} + claims := GrantClaims{KeyID: "k1", Issuer: "execution-provider", Audience: "mecatl-execution", Client: "spiffe://cluster/ns/mecak8s", OwnerHash: "owner", BindingID: "binding", RunID: "run", ClaimID: "claim", Environment: EnvironmentRef{ID: "env", Revision: "rev"}, Epoch: 7, GrantGeneration: 1, Operations: []Operation{OpFileRead}, NotBefore: now.Add(-time.Second), ExpiresAt: now.Add(time.Minute), Nonce: "n1"} token, err := SignGrant(priv, claims) if err != nil { t.Fatal(err) } v := GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: 5 * time.Minute, Now: func() time.Time { return now }} - if _, err := v.Verify(token, GrantExpectation{Client: claims.Client, OwnerHash: "owner", BindingID: claims.BindingID, Environment: claims.Environment, Epoch: 7, Operation: OpFileRead}); err != nil { + if _, err := v.Verify(token, GrantExpectation{Client: claims.Client, OwnerHash: "owner", BindingID: claims.BindingID, RunID: claims.RunID, ClaimID: claims.ClaimID, Environment: claims.Environment, Epoch: 7, GrantGeneration: claims.GrantGeneration, Operation: OpFileRead}); err != nil { t.Fatalf("verify: %v", err) } for name, mutate := range map[string]func(*GrantExpectation){ @@ -31,7 +31,7 @@ func TestGrantRoundTripAndExactBindings(t *testing.T) { "operation": func(e *GrantExpectation) { e.Operation = OpFileReplace }, } { t.Run(name, func(t *testing.T) { - e := GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: claims.BindingID, Environment: claims.Environment, Epoch: claims.Epoch, Operation: OpFileRead} + e := GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: claims.BindingID, RunID: claims.RunID, ClaimID: claims.ClaimID, Environment: claims.Environment, Epoch: claims.Epoch, GrantGeneration: claims.GrantGeneration, Operation: OpFileRead} mutate(&e) if _, err := v.Verify(token, e); err == nil { t.Fatal("expected rejection") @@ -43,16 +43,16 @@ func TestGrantRoundTripAndExactBindings(t *testing.T) { func TestGrantRejectsExpiryRevocationAndUnknownFields(t *testing.T) { pub, priv, _ := ed25519.GenerateKey(rand.Reader) now := time.Unix(1_800_000_000, 0).UTC() - c := GrantClaims{KeyID: "k1", Issuer: "i", Audience: "a", Client: "spiffe://c", OwnerHash: "o", BindingID: "b", Environment: EnvironmentRef{ID: "e", Revision: "r"}, Epoch: 1, Operations: []Operation{OpAttach}, NotBefore: now.Add(-time.Minute), ExpiresAt: now.Add(-time.Second), Nonce: "n"} + c := GrantClaims{KeyID: "k1", Issuer: "i", Audience: "a", Client: "spiffe://c", OwnerHash: "o", BindingID: "b", RunID: "run", ClaimID: "claim", Environment: EnvironmentRef{ID: "e", Revision: "r"}, Epoch: 1, GrantGeneration: 1, Operations: []Operation{OpAttach}, NotBefore: now.Add(-time.Minute), ExpiresAt: now.Add(-time.Second), Nonce: "n"} tok, _ := SignGrant(priv, c) v := GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "i", Audience: "a", Now: func() time.Time { return now }} - if _, err := v.Verify(tok, GrantExpectation{Client: c.Client, OwnerHash: c.OwnerHash, BindingID: c.BindingID, Environment: c.Environment, Epoch: 1, Operation: OpAttach}); err == nil { + if _, err := v.Verify(tok, GrantExpectation{Client: c.Client, OwnerHash: c.OwnerHash, BindingID: c.BindingID, RunID: c.RunID, ClaimID: c.ClaimID, Environment: c.Environment, Epoch: 1, GrantGeneration: c.GrantGeneration, Operation: OpAttach}); err == nil { t.Fatal("expired grant accepted") } c.ExpiresAt = now.Add(time.Minute) tok, _ = SignGrant(priv, c) v.RevokedNonces = map[string]struct{}{"n": {}} - if _, err := v.Verify(tok, GrantExpectation{Client: c.Client, OwnerHash: c.OwnerHash, BindingID: c.BindingID, Environment: c.Environment, Epoch: 1, Operation: OpAttach}); err == nil { + if _, err := v.Verify(tok, GrantExpectation{Client: c.Client, OwnerHash: c.OwnerHash, BindingID: c.BindingID, RunID: c.RunID, ClaimID: c.ClaimID, Environment: c.Environment, Epoch: 1, GrantGeneration: c.GrantGeneration, Operation: OpAttach}); err == nil { t.Fatal("revoked nonce accepted") } } diff --git a/internal/executionenv/types.go b/internal/executionenv/types.go index df6f5b2387..b451344cf1 100644 --- a/internal/executionenv/types.go +++ b/internal/executionenv/types.go @@ -1,5 +1,7 @@ // Package executionenv defines the private, versioned host-side execution-provider protocol. // It is intentionally not part of the public Harness API. +// +//nolint:revive // Private protocol types are exported only across internal adapter packages. package executionenv import "time" @@ -23,30 +25,39 @@ type Operation string // Supported private-protocol operations. const ( - OpAttach Operation = "attach" - OpReferenceRelease Operation = "reference.release" - OpRetire Operation = "retire" - OpFileRead Operation = "file.read" - OpFileResolveAuthority Operation = "file.resolve_authority" - OpFileStat Operation = "file.stat" - OpFileCreate Operation = "file.create" - OpFileReplace Operation = "file.replace" - OpFileList Operation = "file.list" - OpFileRemove Operation = "file.remove" - OpFileRename Operation = "file.rename" - OpFileCopy Operation = "file.copy" - OpFileGlob Operation = "file.glob" - OpFileGrep Operation = "file.grep" - OpCommandStart Operation = "command.start" - OpCommandStatus Operation = "command.status" - OpCommandCancel Operation = "command.cancel" - OpCommandStream Operation = "command.stream" + OpAttach Operation = "attach" + OpAcquireRun Operation = "run.acquire" + OpRenewRun Operation = "run.renew" + OpReleaseRun Operation = "run.release" + OpReferenceCommit Operation = "reference.commit" + OpReferenceAbort Operation = "reference.abort" + OpReferenceReserve Operation = "reference.reserve" + OpReferenceDeletePrepare Operation = "reference.delete.prepare" + OpReferenceDeleteConfirm Operation = "reference.delete.confirm" + OpReferenceDeleteCancel Operation = "reference.delete.cancel" + OpReferenceRelease Operation = "reference.release" + OpRetire Operation = "retire" + OpFileRead Operation = "file.read" + OpFileResolveAuthority Operation = "file.resolve_authority" + OpFileStat Operation = "file.stat" + OpFileCreate Operation = "file.create" + OpFileReplace Operation = "file.replace" + OpFileList Operation = "file.list" + OpFileRemove Operation = "file.remove" + OpFileRename Operation = "file.rename" + OpFileCopy Operation = "file.copy" + OpFileGlob Operation = "file.glob" + OpFileGrep Operation = "file.grep" + OpCommandStart Operation = "command.start" + OpCommandStatus Operation = "command.status" + OpCommandCancel Operation = "command.cancel" + OpCommandStream Operation = "command.stream" ) // Valid reports whether the operation belongs to the closed protocol vocabulary. func (o Operation) Valid() bool { switch o { - case OpAttach, OpReferenceRelease, OpRetire, OpFileRead, OpFileResolveAuthority, OpFileStat, OpFileCreate, OpFileReplace, OpFileList, OpFileRemove, OpFileRename, OpFileCopy, OpFileGlob, OpFileGrep, OpCommandStart, OpCommandStatus, OpCommandCancel, OpCommandStream: + case OpAttach, OpAcquireRun, OpRenewRun, OpReleaseRun, OpReferenceCommit, OpReferenceAbort, OpReferenceReserve, OpReferenceDeletePrepare, OpReferenceDeleteConfirm, OpReferenceDeleteCancel, OpReferenceRelease, OpRetire, OpFileRead, OpFileResolveAuthority, OpFileStat, OpFileCreate, OpFileReplace, OpFileList, OpFileRemove, OpFileRename, OpFileCopy, OpFileGlob, OpFileGrep, OpCommandStart, OpCommandStatus, OpCommandCancel, OpCommandStream: return true } return false @@ -66,11 +77,14 @@ type Owner struct { // RequestContext carries the exact authorization binding for an operation. type RequestContext struct { - Environment EnvironmentRef `json:"environment"` - Owner Owner `json:"owner"` - BindingID string `json:"binding_id"` - Epoch uint64 `json:"epoch"` - Grant string `json:"grant"` + Environment EnvironmentRef `json:"environment"` + Owner Owner `json:"owner"` + BindingID string `json:"binding_id"` + RunID string `json:"run_id"` + ClaimID string `json:"claim_id"` + Epoch uint64 `json:"epoch"` + GrantGeneration uint64 `json:"grant_generation"` + Grant string `json:"grant"` } // ValidateProfileRequest selects an operator-defined profile for validation. @@ -97,11 +111,12 @@ type EnsureEnvironmentRequest struct { // EnsureEnvironmentResponse returns allocation identity, readiness, and a short-lived grant. type EnsureEnvironmentResponse struct { - Environment EnvironmentRef `json:"environment"` - Epoch uint64 `json:"epoch"` - Ready bool `json:"ready"` - Grant string `json:"grant"` - GrantExpiresAt time.Time `json:"grant_expires_at"` + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + Ready bool `json:"ready"` + GrantGeneration uint64 `json:"grant_generation"` + Grant string `json:"grant"` + GrantExpiresAt time.Time `json:"grant_expires_at"` } // AttachEnvironmentRequest requests exact reattachment to an existing environment. @@ -115,11 +130,68 @@ const PurposeSession = "session" // AttachEnvironmentResponse returns exact attachment state and a refreshed grant. type AttachEnvironmentResponse struct { - Environment EnvironmentRef `json:"environment"` - Epoch uint64 `json:"epoch"` - Ready bool `json:"ready"` - Grant string `json:"grant"` - GrantExpiresAt time.Time `json:"grant_expires_at"` + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + Ready bool `json:"ready"` + GrantGeneration uint64 `json:"grant_generation"` + Grant string `json:"grant"` + GrantExpiresAt time.Time `json:"grant_expires_at"` +} + +const ( + MinRunTTL = 30 * time.Second + MaxRunTTL = 5 * time.Minute + DefaultRunTTL = time.Minute + DefaultRenewInterval = 20 * time.Second +) + +type RunClaimRequest struct { + Environment EnvironmentRef + Owner Owner + BindingID string + RunID string + ClaimID string + Epoch uint64 + GrantGeneration uint64 + OperationID string + TTL time.Duration +} + +type RunClaim struct { + Environment EnvironmentRef + BindingID string + RunID string + ClaimID string + Epoch uint64 + GrantGeneration uint64 + Grant string + ExpiresAt time.Time +} + +type ReferenceState string + +const ( + ReferencePendingCreate ReferenceState = "PendingCreate" + ReferencePublished ReferenceState = "Published" + ReferencePendingDelete ReferenceState = "PendingDelete" +) + +type ReferenceRequest struct { + Environment EnvironmentRef + Owner Owner + BindingID string + SourceBindingID string + OperationID string +} + +type ReferenceIntent struct { + Environment EnvironmentRef + Owner Owner + BindingID string + State ReferenceState + OperationID string + SourceBindingID string + CreatedAt time.Time } // ReferenceReleaseRequest releases one durable binding reference. @@ -129,7 +201,12 @@ type ReferenceReleaseRequest struct { // RetireEnvironmentRequest requests controlled retirement without PVC deletion. type RetireEnvironmentRequest struct { - Context RequestContext `json:"context"` + Environment EnvironmentRef `json:"environment"` + Owner Owner `json:"owner"` + ExpectedEpoch uint64 `json:"expected_execution_epoch"` + ExpectedPodUID string `json:"expected_pod_uid"` + ExpectedPVCUID string `json:"expected_pvc_uid"` + OperationID string `json:"operation_id"` } // EmptyResponse is the successful response for operations without a payload. diff --git a/user-docs/building/deployment/mecak8s.md b/user-docs/building/deployment/mecak8s.md index 85698d5380..626dd04b7c 100644 --- a/user-docs/building/deployment/mecak8s.md +++ b/user-docs/building/deployment/mecak8s.md @@ -158,11 +158,11 @@ operator-controlled Kubernetes cluster. It runs as a separate service and controller with its own `mecatl-execution` chart. `mecak8s` remains a client: it has no Pod, PVC, custom-resource, or controller management permissions. -This candidate is suitable for offline evaluation only. It does not yet provide -run-wide execution ownership, complete successor reference binding and release, -controlled executor replacement or PVC deletion, or complete signing-key -rotation and revocation. Do not use it for hostile multitenancy or claim high -availability from this configuration. +This production candidate provides run-wide ownership, transactional references, +controlled replacement and retirement, durable grant revocation, and reloadable +TLS and signing material. Its default-deny network policy still requires an +enforcing CNI; Kind's default kindnet is not isolation evidence. A configured +RuntimeClass is an operator input, not by itself a hostile-workload guarantee. Before you install it, prepare these values through your trusted image and Secret delivery system: @@ -170,11 +170,13 @@ Secret delivery system: - A digest-pinned provider image. - A digest-pinned workload image that contains `/mecatl-executor`, `/bin/sh`, and the toolchain needed by foreground commands. -- A server mTLS Secret containing `ca.crt`, `tls.crt`, and `tls.key`. -- A separate Ed25519 grant-signing Secret containing `key.pem`. +- One projected security Secret containing the provider certificate/key, client + CA bundle, and versioned Ed25519 grant keys. The chart never generates them. +- A versioned `provider.securityManifest` as documented in + [Execution environments](/features/execution-environments.md#production-security-material). - A `mecak8s` client mTLS Secret containing `ca.crt`, `tls.crt`, and `tls.key`. -- Exact client URI SANs for the `mecak8s` identity. The identity that attests - OIDC owners must also appear in `provider.ownerAttesterURISANs`. +- Explicit provider-client pod/namespace selectors, API-server CIDRs, and DNS + resolver CIDRs. Do not use `0.0.0.0/0`. The following profile shows every required chart key. Save it as `execution-values.yaml` and replace each placeholder: @@ -183,15 +185,14 @@ The following profile shows every required chart key. Save it as provider: image: @sha256: imagePullPolicy: IfNotPresent - replicas: 1 - clientURISANs: - - spiffe:///client/mecak8s - ownerAttesterURISANs: - - spiffe:///client/mecak8s - administratorURISANs: [] - tlsSecretName: - grantSigningSecretName: - grantKeyID: current + replicas: 2 + securitySecretName: + securityManifest: '' + clientIngressSelectors: + - namespaceLabels: {kubernetes.io/metadata.name: } + podLabels: {app.kubernetes.io/name: mecak8s} + apiServerCIDRs: [/32] + dnsCIDRs: [/32] service: port: 8443 @@ -205,9 +206,31 @@ profiles: memoryRequest: 128Mi cpuLimit: "1" memoryLimit: 1Gi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: maxFileBytes: 5242880 maxCommandBytes: 1048576 maxCommandDuration: 5m + maxEnvironments: 100 + +networkPolicy: + enabled: true + dnsPorts: [53] + apiServerPorts: [443] + workloadProfiles: {} # default deny; add explicit coding egress only if required + +resourceGovernance: + enabled: true + pods: "200" + persistentVolumeClaims: "100" + executionEnvironments: "100" + requestsCPU: "20" + requestsMemory: 40Gi + requestsStorage: 500Gi + limitsCPU: "40" + limitsMemory: 80Gi ``` Install the provider chart separately from `mecak8s`: diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index 08bb5cff69..26878039fd 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -78,6 +78,86 @@ reattached only when the deployment supplies an `EnvironmentResolver`; a missing resolver, mismatched identity, or nil workspace returns an error instead of using a local workspace. +## Native Kubernetes lifecycle and retention + +The optional Kubernetes execution provider stores environment ownership in a +namespaced `ExecutionEnvironment`. Provider replicas coordinate through +Kubernetes resource-version compare-and-swap; a replica restart does not clear +another replica's operation. Operation lease expiry fences the environment and +retains the unresolved operation identity for administrator recovery. + +Workspace PVCs are retained by default. Removing the last session reference, +deleting a session, uninstalling the chart, or deleting the provider does not +delete a workspace PVC. Executor replacement and retirement are private, +mutually authenticated administrator operations. They require exact environment, +execution-epoch, Pod-UID, and PVC-UID values. The provider waits for a kubelet +terminal Pod phase and terminated state for every container before removing its +Pod finalizer. A missing Pod or unreachable kubelet is not termination proof and +leaves the environment fenced. + +Retirement stops the executor but retains the PVC. PVC deletion requires the +separate `DeleteRetiredEnvironment` administrator RPC with the exact retained +PVC UID and no references or claims. Deleting the custom resource outside this +provider workflow is unsupported. Do not remove the retention or executor +finalizers manually; follow the operator's external-fencing runbook when the +provider cannot independently observe terminal compute. + +Persisted prototype resources use an explicit administrator migration. Normal +operations reject schema versions other than 2. `MigrateEnvironment` accepts +only recognized versions 0 and 1, verifies the exact live Pod and PVC identities, +and requires a healthy, idle environment. Unknown or malformed state is retained +unchanged rather than reset. + +### Production security material + +The production chart requires one projected Secret containing the TLS identity, +client CA bundle, and Ed25519 grant keys, plus `provider.securityManifest`. The +chart does not generate keys or certificates. The manifest is strict JSON with +this shape: + +```json +{"version":1,"generation":7,"issuer":"https://issuer.example","audience":"mecatl-execution","activeKeyID":"grant-2026-09","grantTTL":"1m","clockSkew":"5s","keys":[{"id":"grant-2026-09","version":7,"file":"grant-2026-09.pem","publicKeySHA256":"","activateAt":"2026-09-17T00:00:00Z","verifyUntil":"2026-09-17T01:00:00Z","state":"active"}],"tls":{"certificateFile":"tls.crt","privateKeyFile":"tls.key","clientCAFile":"client-ca.pem"},"clients":[{"uri":"spiffe://example/mecatl","mayAttestOwner":true,"administrator":false}]} +``` + +Use only basename file names. Kubernetes projected-volume `..data` symlinks are +supported, but paths escaping the mounted directory are rejected. Increase +`generation` for every change. Key IDs and `(id, version)` fingerprints cannot be +reused; the provider persists a bounded high-water ledger in its authority +ConfigMap. Keep retired keys as `verify-only` until all grants expire, then mark +them `revoked`. Invalid, incomplete, rolled-back, or newly expired material makes +readiness fail and denies new RPC authorization until corrected. The provider +re-verifies the peer certificate and URI policy against the current client CA on +every RPC, including RPCs on an existing HTTP/2 connection. + +`RevokeEnvironment` is an administrator-only, exact-reference CAS. Supply the +current positive grant generation; success increments it without changing the +execution epoch. Old grants then cannot authorize another operation or renewal. +An operation already accepted may still finish and clean up, so revocation is not +termination proof and a new run needs a fresh claim after the old claim is +released or fenced. + +The gRPC port remains the only execution data plane. `/live` and `/ready` are +separate, unauthenticated operational HTTP endpoints intended only for Kubernetes +probes; do not expose that port outside the cluster. Readiness requires current +security material, the authoritative generation, synchronized controller caches, +and Kubernetes access. Liveness reports process health only. + +Production values require two or more provider replicas, explicit provider-client +selectors, API-server and DNS CIDRs, and finite ResourceQuota/LimitRange values. +Executor pods are default-deny for ingress and egress. Add only profile-specific +CIDR and port egress under `networkPolicy.workloadProfiles`; there is no implicit +DNS, metadata-service, API-server, provider, or peer access. Each profile requires +an explicit RuntimeClass supplied by the cluster, but do not treat a RuntimeClass +name alone as a hostile-workload isolation guarantee. CPU, memory, ephemeral +storage, and `/tmp` are explicitly bounded per profile. Each profile also requires +`maxEnvironments`; the namespace quota is the final global bound. + +For an existing single-key installation, first create the authority ConfigMap and +a generation-1 manifest that names the existing TLS and grant files, then upgrade +the chart. Confirm `/ready` before rotating. Do not reuse the old key ID with new +key bytes. Schema-2 environment migration is independent of keyring migration: +complete both before enabling new sessions. + ## Live qualification (experimental) The repository keeps real-provider qualification separate from the default mock From fd39f223192a8161c84054a4f31055fe02b17f93 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 17 Sep 2026 06:54:05 +0200 Subject: [PATCH 06/25] fix: harden production lifecycle and add enforcing kind qualification Close ownership, schema pruning, rotation and termination-proof review findings. Add Calico-backed lifecycle, quota, rotation and failure scenarios, strict release provenance, and operational runbooks. Local production qualification remains blocked by host inotify capacity; CI proof is pending. Co-Authored-By: Mecatl --- .github/scripts/release-tag-guard.sh | 66 ++ .github/scripts/release-tag-guard_test.sh | 72 ++ .github/workflows/k8s-e2e.yml | 51 +- .github/workflows/release.yml | 62 +- Taskfile.yml | 23 +- cmd/mecatl-execution-provider/main.go | 12 +- .../go/mecatl/execution/v1/execution.pb.go | 644 ++++++------ .../proto/mecatl/execution/v1/execution.proto | 3 +- deploy/helm/mecatl-execution/chart_test.go | 14 +- .../crds/executionenvironment.yaml | 15 +- .../templates/network-policy.yaml | 2 + .../mecatl-execution/templates/provider.yaml | 6 + .../helm/mecatl-execution/templates/rbac.yaml | 37 +- .../helm/mecatl-execution/values.schema.json | 2 +- deploy/helm/mecatl-execution/values.yaml | 2 + .../execution-values.yaml | 59 ++ deploy/mecatl-execution-kind/live.sh | 48 +- deploy/mecatl-execution-kind/run.sh | 208 +++- .../mecatl-execution-kind/runtimeclass.yaml | 5 + .../acceptance/native-kubernetes-execution.md | 43 +- docs/adr/0027-cloud-native.md | 11 + docs/architecture.md | 40 +- docs/design/IMPLEMENTATION-NOTES.md | 28 +- e2e/k8s_execution/fixture/netprobe/main.go | 37 + e2e/k8s_execution/fixture/pki/main.go | 16 +- e2e/k8s_execution/fixture/rotation/main.go | 132 +++ e2e/k8s_execution/live_qualification_test.go | 5 +- .../production_qualification_test.go | 943 ++++++++++++++++++ e2e/k8s_execution/qualification_test.go | 123 ++- internal/adapter/executionclient/client.go | 114 ++- .../executioncontroller/admin_lifecycle.go | 61 +- .../adapter/executioncontroller/controller.go | 150 +-- .../controller_lifecycle.go | 7 +- .../executioncontroller/controller_test.go | 73 +- .../executioncontroller/crd_schema_test.go | 49 + .../adapter/executioncontroller/handler.go | 45 +- .../executioncontroller/handler_test.go | 50 + .../production_lifecycle_test.go | 52 + .../adapter/executioncontroller/profiles.go | 22 + .../adapter/executioncontroller/security.go | 216 ++-- .../executioncontroller/security_test.go | 246 +++++ internal/adapter/executioncontroller/store.go | 121 ++- .../store_concurrency_test.go | 157 +++ .../executioncontroller/store_lifecycle.go | 4 +- .../store_lifecycle_test.go | 51 + .../executioncontroller/store_revoke.go | 39 +- .../executioncontroller/store_revoke_test.go | 10 +- .../adapter/executioncontroller/store_run.go | 14 +- .../adapter/executioncontroller/store_test.go | 19 + .../server/native_execution_lifecycle_test.go | 3 + .../server/reference_intent_reconcile_test.go | 4 +- internal/adapter/server/service.go | 84 +- user-docs/building/deployment/mecak8s.md | 196 +++- user-docs/features/execution-environments.md | 56 +- 54 files changed, 3806 insertions(+), 746 deletions(-) create mode 100755 .github/scripts/release-tag-guard.sh create mode 100755 .github/scripts/release-tag-guard_test.sh create mode 100644 deploy/mecatl-execution-kind/execution-values.yaml create mode 100644 deploy/mecatl-execution-kind/runtimeclass.yaml create mode 100644 e2e/k8s_execution/fixture/netprobe/main.go create mode 100644 e2e/k8s_execution/fixture/rotation/main.go create mode 100644 e2e/k8s_execution/production_qualification_test.go create mode 100644 internal/adapter/executioncontroller/crd_schema_test.go create mode 100644 internal/adapter/executioncontroller/store_concurrency_test.go diff --git a/.github/scripts/release-tag-guard.sh b/.github/scripts/release-tag-guard.sh new file mode 100755 index 0000000000..0e14bfdecc --- /dev/null +++ b/.github/scripts/release-tag-guard.sh @@ -0,0 +1,66 @@ +#!/bin/sh +set -eu + +: "${VERSION:?VERSION is required}" +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" + +if ! printf '%s\n' "$VERSION" | grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'; then + echo "release tag must match vMAJOR.MINOR.PATCH" >&2 + exit 1 +fi + +tag_ref="refs/tags/$VERSION" +if ! git show-ref --verify --quiet "$tag_ref"; then + echo "release tag does not exist: $VERSION" >&2 + exit 1 +fi +commit=$(git rev-parse --verify "$tag_ref^{commit}") +if ! printf '%s\n' "$commit" | grep -Eq '^[0-9a-f]{40}$'; then + echo "release tag did not peel to a commit" >&2 + exit 1 +fi + +case "${GITHUB_EVENT_NAME:-}" in + push) + if [ "${GITHUB_REF:-}" != "$tag_ref" ]; then + echo "push ref does not match the release tag" >&2 + exit 1 + fi + if ! printf '%s\n' "${GITHUB_SHA:-}" | grep -Eq '^[0-9a-f]{40}$'; then + echo "push commit is not a full SHA" >&2 + exit 1 + fi + event_commit=$(git rev-parse --verify "${GITHUB_SHA}^{commit}" 2>/dev/null) || { + echo "push commit did not peel to a commit" >&2 + exit 1 + } + if [ "$event_commit" != "$commit" ]; then + echo "push commit does not match the release tag" >&2 + exit 1 + fi + ;; + workflow_dispatch) + # A dispatch runs from the selected workflow branch, not from the tag. + ;; + *) + echo "unsupported release event: ${GITHUB_EVENT_NAME:-unset}" >&2 + exit 1 + ;; +esac +if ! git merge-base --is-ancestor "$commit" origin/main; then + echo "release tag commit is not on origin/main" >&2 + exit 1 +fi + +git checkout --detach "$commit" >/dev/null 2>&1 +head=$(git rev-parse --verify HEAD) +if [ "$head" != "$commit" ]; then + echo "checked-out commit does not match the verified release tag" >&2 + exit 1 +fi + +{ + echo "version=$VERSION" + echo "commit=$commit" +} >>"$GITHUB_OUTPUT" +printf '%s is verified at %s\n' "$VERSION" "$commit" diff --git a/.github/scripts/release-tag-guard_test.sh b/.github/scripts/release-tag-guard_test.sh new file mode 100755 index 0000000000..9718ad4b38 --- /dev/null +++ b/.github/scripts/release-tag-guard_test.sh @@ -0,0 +1,72 @@ +#!/bin/sh +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +script="$script_dir/release-tag-guard.sh" +work=${TEST_TMPDIR:-.scratch}/release-tag-guard-$$ +mkdir -p "$work" +work=$(CDPATH= cd -- "$work" && pwd -P) + +repo="$work/repo" +mkdir "$repo" +repo=$(CDPATH= cd -- "$repo" && pwd -P) +git -C "$repo" init -q -b main +git -C "$repo" config user.name test +git -C "$repo" config user.email test@example.com +git -C "$repo" config commit.gpgSign false +git -C "$repo" config tag.gpgSign false +printf 'release\n' >"$repo/file" +git -C "$repo" add file +git -C "$repo" commit -qm release +release_commit=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" tag v1.2.3 +printf 'main\n' >>"$repo/file" +git -C "$repo" commit -qam main +main=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" remote add origin "$repo" +git -C "$repo" fetch -q origin main:refs/remotes/origin/main + +run_ok() { + output="$work/output" + : >"$output" + (cd "$repo" && VERSION="$1" GITHUB_EVENT_NAME="$2" GITHUB_REF="$3" GITHUB_SHA="$4" GITHUB_OUTPUT="$output" "$script") >/dev/null + grep -Fx "version=$1" "$output" >/dev/null + grep -Fx "commit=$5" "$output" >/dev/null +} +run_fail() { + output="$work/output" + : >"$output" + if (cd "$repo" && VERSION="$1" GITHUB_EVENT_NAME="$2" GITHUB_REF="$3" GITHUB_SHA="$4" GITHUB_OUTPUT="$output" "$script") >/dev/null 2>&1; then + echo "unexpectedly accepted: $1" >&2 + exit 1 + fi +} + +# Dispatch starts at main HEAD, but must publish the selected older tag commit. +run_ok v1.2.3 workflow_dispatch refs/heads/main "$main" "$release_commit" +run_fail main workflow_dispatch refs/heads/main "$main" +run_fail refs/heads/main workflow_dispatch refs/heads/main "$main" +marker="$work/pwned" +run_fail "v1.2.3; touch $marker" workflow_dispatch refs/heads/main "$main" +[ ! -e "$marker" ] +run_fail v01.2.3 workflow_dispatch refs/heads/main "$main" +run_fail v9.9.9 workflow_dispatch refs/heads/main "$main" +run_fail v1.2.3 push refs/heads/main "$release_commit" +run_fail v1.2.3 push refs/tags/v1.2.3 0000000000000000000000000000000000000000 +run_ok v1.2.3 push refs/tags/v1.2.3 "$release_commit" "$release_commit" + +# An annotated tag push reports its tag-object SHA; both sides must peel it. +git -C "$repo" tag -a v1.2.4 -m annotated "$release_commit" +annotated_tag=$(git -C "$repo" rev-parse v1.2.4) +run_ok v1.2.4 push refs/tags/v1.2.4 "$annotated_tag" "$release_commit" + +git -C "$repo" switch -q --detach "$release_commit" +printf 'branch\n' >>"$repo/file" +git -C "$repo" commit -qam branch +off_main=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" tag v2.0.0 +run_fail v2.0.0 workflow_dispatch refs/heads/main "$main" +run_fail v2.0.0 push refs/tags/v2.0.0 "$off_main" +run_fail v1.2.3 schedule refs/heads/main "$main" + +printf 'release tag guard tests passed\n' diff --git a/.github/workflows/k8s-e2e.yml b/.github/workflows/k8s-e2e.yml index 6d3a3c92d7..9134f1c4f8 100644 --- a/.github/workflows/k8s-e2e.yml +++ b/.github/workflows/k8s-e2e.yml @@ -45,6 +45,9 @@ on: - internal/executionenv/** - contracts/proto/mecatl/execution/** - e2e/k8s_execution/** + - deploy/mecatl-execution-kind/** + - build/execution-provider/Dockerfile + - build/execution-workload/Dockerfile - e2e/k8s/** - go.mod - go.sum @@ -70,6 +73,9 @@ on: - internal/executionenv/** - contracts/proto/mecatl/execution/** - e2e/k8s_execution/** + - deploy/mecatl-execution-kind/** + - build/execution-provider/Dockerfile + - build/execution-workload/Dockerfile - e2e/k8s/** - go.mod - go.sum @@ -139,9 +145,9 @@ jobs: # kind: pinned version, installed via `go install` (no third-party action — # the house idiom for go-installable tools, matching actionlint/govulncheck - # in ci.yml). v0.32.0 is the current release. + # in ci.yml). v0.33.0 is the current release. - name: Install kind - run: go install sigs.k8s.io/kind@v0.32.0 + run: go install sigs.k8s.io/kind@v0.33.0 # ko: pinned version, same `go install` idiom. v0.18.1 builds the mecak8s # image from .ko.yaml onto the distroless base. KO_DOCKER_REPO=ko.local below @@ -182,3 +188,44 @@ jobs: path: k8s-e2e-output.log retention-days: 7 if-no-files-found: warn + + execution-production-e2e: + name: execution provider production profile (kind) + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: | + go.sum + engine/go.sum + - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v3.18.4 + - name: Install pinned Kind and ko + run: | + go install sigs.k8s.io/kind@v0.33.0 + go install github.com/google/ko@v0.18.1 + - name: Run production execution qualification + shell: bash + env: + CONTAINER_ENGINE: docker + MECATL_EXECUTION_QUAL_CI: "1" + run: task e2e:k8s:execution:production + - name: Upload bounded failure artifact + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: execution-e2e-artifacts-${{ github.run_id }} + path: .scratch/k8s-execution/*/production-failure-artifact.txt + retention-days: 7 + if-no-files-found: warn diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40681578c2..95399b0da3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,7 @@ jobs: # KEEPS its own inline copy as well (like the pin gate it repeats): it is the one job that # writes outside this repository, so it re-verifies rather than inheriting. guard: - name: Ensure the release tag is on main + name: Verify immutable release tag runs-on: ubuntu-24.04 # 10, not 5: the v0.0.35 release was LOST because an unfiltered depth-0 clone # took 5m18s and tripped a 5-minute cap, skipping all six publish jobs. The @@ -57,6 +57,9 @@ jobs: timeout-minutes: 10 permissions: contents: read + outputs: + version: ${{ steps.verify.outputs.version }} + commit: ${{ steps.verify.outputs.commit }} env: # On workflow_dispatch github.ref_name is the default branch, so prefer the # explicit input; on a tag push github.ref_name is the tag. @@ -65,26 +68,12 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # 0 so origin/main is present to compare against. NOT `git fetch origin main`: - # persist-credentials is false, so a fetch has no credentials on this repo. fetch-depth: 0 - # This job reads NO file contents — only `git merge-base` and `git rev-parse` - # against the commit graph. An unfiltered depth-0 clone here cost ~5 minutes - # (306 MB, 184 branches, 123 tags) in front of every publish job, since all - # six `needs: guard`. blob:none fetches the graph without the blobs. filter: blob:none - ref: ${{ env.VERSION }} - # HEAD, not $GITHUB_SHA: on workflow_dispatch $GITHUB_SHA is the branch head, while - # HEAD is the tag commit actually checked out above. - - name: Ensure the release tag is on main - run: | - set -euo pipefail - if ! git merge-base --is-ancestor HEAD origin/main; then - echo "::error::release tags must be cut from a commit on main; $(git rev-parse HEAD) is not on main" - exit 1 - fi - echo "$(git rev-parse HEAD) is on main" + - name: Verify and check out the immutable release tag + id: verify + run: .github/scripts/release-tag-guard.sh publish: needs: guard @@ -102,14 +91,14 @@ jobs: KO_DOCKER_REPO: ghcr.io/${{ github.repository }} # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # ko reads the base image digest from .ko.yaml; no deep history needed. fetch-depth: 1 - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: @@ -262,14 +251,14 @@ jobs: KO_DOCKER_REPO: ghcr.io/${{ github.repository }}/mecatui # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # ko reads the base image digest from .ko.yaml; no deep history needed. fetch-depth: 1 - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: @@ -422,14 +411,14 @@ jobs: KO_DOCKER_REPO: ghcr.io/${{ github.repository }}/mecak8s # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # ko reads the base image digest from .ko.yaml; no deep history needed. fetch-depth: 1 - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: @@ -579,14 +568,16 @@ jobs: - {name: provider, dockerfile: build/execution-provider/Dockerfile, image: mecatl-execution-provider} - {name: workload, dockerfile: build/execution-workload/Dockerfile, image: mecatl-execution-workload} env: - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} IMAGE: ghcr.io/${{ github.repository }}/${{ matrix.image }} GO_IMAGE: ${{ vars.EXECUTION_GO_IMAGE }} PROVIDER_RUNTIME_IMAGE: ${{ vars.EXECUTION_PROVIDER_RUNTIME_IMAGE }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: {persist-credentials: false, fetch-depth: 1, ref: "${{ env.VERSION }}"} + with: {persist-credentials: false, fetch-depth: 1, ref: "${{ needs.guard.outputs.commit }}"} - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + with: + version: v0.30.1 - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v4.0.0 with: {registry: ghcr.io, username: "${{ github.actor }}", password: "${{ secrets.GITHUB_TOKEN }}"} @@ -638,12 +629,12 @@ jobs: id-token: write attestations: write env: - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} OCI_REGISTRY: ghcr.io/${{ github.repository }}/charts CHART: mecatl-execution steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: {persist-credentials: false, fetch-depth: 1, ref: "${{ env.VERSION }}"} + with: {persist-credentials: false, fetch-depth: 1, ref: "${{ needs.guard.outputs.commit }}"} - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: {version: v3.18.4} - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 @@ -668,6 +659,9 @@ jobs: echo "digest=$digest" >> "$GITHUB_OUTPUT" - name: Sign chart run: cosign sign --yes "${OCI_REGISTRY}/${CHART}@${{ steps.push.outputs.digest }}" + # The chart has no chart dependencies or bundled binaries. Its packaged + # templates and metadata are the artifact, so image SBOMs cover executable + # contents while this immutable chart digest receives provenance directly. - name: Attest chart provenance uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: @@ -696,13 +690,13 @@ jobs: IMAGE: ghcr.io/${{ github.repository }}/slack-bot # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 1 - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} - uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0 - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 @@ -790,13 +784,13 @@ jobs: CHART_PATH: deploy/helm/mecak8s # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 1 - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: @@ -937,7 +931,7 @@ jobs: env: # On workflow_dispatch github.ref_name is the default branch, so prefer the # explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -958,7 +952,7 @@ jobs: # credentials for a lazy blob fetch, so do not add a step here that reads a # HISTORICAL file's contents. filter: blob:none - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} # Eight cross-compiles (2 binaries x 4 platforms) of this dependency graph # exhaust the runner's disk: the first v0.0.30 attempt died 18 minutes in diff --git a/Taskfile.yml b/Taskfile.yml index f33f5d7e02..e8c6c10493 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -511,14 +511,28 @@ tasks: Run the focused mock-only Kind qualification for the optional Kubernetes execution provider. It uses a unique cluster and explicit kubeconfig under .scratch, builds/loads immutable local images, and never reads a provider - credential. The owned cluster is intentionally retained; the command prints - the exact human-confirmed cleanup command. NOT part of task test. + credential. Docker is the default; set CONTAINER_ENGINE=podman for rootless + Podman. The owned cluster is intentionally retained; the command prints the + exact human-confirmed cleanup command. NOT part of task test. preconditions: - - sh: command -v kind >/dev/null 2>&1 && command -v podman >/dev/null 2>&1 - msg: "kind and rootless Podman are required" + - sh: command -v kind >/dev/null 2>&1 + msg: "kind is required" + - sh: | + runtime="${CONTAINER_ENGINE:-docker}" + case "$runtime" in docker|podman) command -v "$runtime" >/dev/null 2>&1 ;; *) exit 1 ;; esac + msg: "CONTAINER_ENGINE must be docker or podman and available on PATH" cmds: - deploy/mecatl-execution-kind/run.sh + e2e:k8s:execution:production: + desc: | + Run the deterministic production-network and lifecycle execution-provider + qualification. CI sets MECATL_EXECUTION_QUAL_CI=1 so its uniquely owned Kind + cluster is removed automatically; local runs retain it for inspection and + print the human-confirmed cleanup command. No model credential is used. + cmds: + - MECATL_EXECUTION_QUAL_PROFILE=production task e2e:k8s:execution + e2e:k8s:execution:live: desc: | EXPLICIT OPT-IN: run mock qualification followed by one bounded real-provider @@ -911,6 +925,7 @@ tasks: - task: lint:action-templates - bash .github/scripts/ensure-perf-dashboard-noindex_test.sh - bash .github/scripts/deslop-workflow_test.sh + - sh .github/scripts/release-tag-guard_test.sh lint:action-templates: desc: | diff --git a/cmd/mecatl-execution-provider/main.go b/cmd/mecatl-execution-provider/main.go index 6edcdf1c78..afad7568f6 100644 --- a/cmd/mecatl-execution-provider/main.go +++ b/cmd/mecatl-execution-provider/main.go @@ -48,8 +48,8 @@ func run() error { //nolint:gocyclo // Startup validation and owned-resource shu flag.IntVar(&maxConcurrentRPCs, "max-concurrent-rpcs", 128, "maximum active provider RPCs") flag.IntVar(&maxConcurrentRPCsPerClient, "max-concurrent-rpcs-per-client", 32, "maximum active provider RPCs per authorized client") flag.Parse() - if namespace == "" || authorityConfigMap == "" || maxConcurrentStreams < 1 || maxConcurrentStreams > 1024 || maxConcurrentRPCs < 1 || maxConcurrentRPCs > 4096 || maxConcurrentRPCsPerClient < 1 || maxConcurrentRPCsPerClient > maxConcurrentRPCs { - return errors.New("required identity or RPC concurrency bounds are invalid") + if namespace == "" || authorityConfigMap == "" || reloadInterval <= 0 || reloadInterval > time.Minute || maxConcurrentStreams < 1 || maxConcurrentStreams > 1024 || maxConcurrentRPCs < 1 || maxConcurrentRPCs > 4096 || maxConcurrentRPCsPerClient < 1 || maxConcurrentRPCsPerClient > maxConcurrentRPCs { + return errors.New("required identity, security reload interval, or RPC concurrency bounds are invalid") } profiles, err := executioncontroller.LoadProfiles(profilesPath) if err != nil { @@ -106,8 +106,12 @@ func run() error { //nolint:gocyclo // Startup validation and owned-resource shu mux := http.NewServeMux() mux.HandleFunc("GET /live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) mux.HandleFunc("GET /ready", func(w http.ResponseWriter, r *http.Request) { - if !reconciler.Ready() || !security.CheckReady(r.Context()) { - http.Error(w, "not ready", http.StatusServiceUnavailable) + if !reconciler.Ready() { + http.Error(w, "not ready: "+reconciler.ReadinessReason(), http.StatusServiceUnavailable) + return + } + if !security.CheckReady(r.Context()) { + http.Error(w, "not ready: security-authority-or-expiry", http.StatusServiceUnavailable) return } w.WriteHeader(http.StatusNoContent) diff --git a/contracts/gen/go/mecatl/execution/v1/execution.pb.go b/contracts/gen/go/mecatl/execution/v1/execution.pb.go index 970598b79f..2e59488d94 100644 --- a/contracts/gen/go/mecatl/execution/v1/execution.pb.go +++ b/contracts/gen/go/mecatl/execution/v1/execution.pb.go @@ -1920,6 +1920,7 @@ type RevokeEnvironmentRequest struct { Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` ExpectedGrantGeneration uint64 `protobuf:"varint,3,opt,name=expected_grant_generation,json=expectedGrantGeneration,proto3" json:"expected_grant_generation,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1975,6 +1976,13 @@ func (x *RevokeEnvironmentRequest) GetExpectedGrantGeneration() uint64 { return 0 } +func (x *RevokeEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + type RevokeEnvironmentResponse struct { state protoimpl.MessageState `protogen:"open.v1"` GrantGeneration uint64 `protobuf:"varint,1,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` @@ -2023,7 +2031,7 @@ type MigrateEnvironmentRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` - ExpectedSchemaVersion uint32 `protobuf:"varint,3,opt,name=expected_schema_version,json=expectedSchemaVersion,proto3" json:"expected_schema_version,omitempty"` + ExpectedSchemaVersion *uint32 `protobuf:"varint,3,opt,name=expected_schema_version,json=expectedSchemaVersion,proto3,oneof" json:"expected_schema_version,omitempty"` ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` @@ -2076,8 +2084,8 @@ func (x *MigrateEnvironmentRequest) GetOwner() *Owner { } func (x *MigrateEnvironmentRequest) GetExpectedSchemaVersion() uint32 { - if x != nil { - return x.ExpectedSchemaVersion + if x != nil && x.ExpectedSchemaVersion != nil { + return *x.ExpectedSchemaVersion } return 0 } @@ -3113,7 +3121,7 @@ var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xcf, 0x01, 0x0a, 0x18, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xf2, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, @@ -3126,337 +3134,342 @@ var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ 0x72, 0x12, 0x3a, 0x0a, 0x19, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x17, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x47, 0x72, - 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x46, 0x0a, - 0x19, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, - 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc3, 0x02, 0x0a, 0x19, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, - 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, - 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, - 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, - 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, - 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x36, 0x0a, 0x17, - 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x65, - 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, - 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, - 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, - 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, - 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, - 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xa2, 0x02, 0x0a, 0x0b, - 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, + 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, + 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, + 0x22, 0x46, 0x0a, 0x19, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, + 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xe4, 0x02, 0x0a, 0x19, 0x4d, 0x69, 0x67, + 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, + 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, + 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x40, 0x0a, 0x09, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, - 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, - 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, - 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x12, 0x0a, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, - 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, - 0x6d, 0x69, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, - 0x22, 0x94, 0x01, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x6d, 0x6f, 0x64, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65, - 0x12, 0x15, 0x0a, 0x06, 0x69, 0x73, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x05, 0x69, 0x73, 0x44, 0x69, 0x72, 0x22, 0x47, 0x0a, 0x09, 0x47, 0x72, 0x65, 0x70, 0x4d, - 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x74, 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, - 0x22, 0xd4, 0x02, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, - 0x31, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, - 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, - 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, - 0x66, 0x6f, 0x52, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x70, - 0x61, 0x74, 0x68, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, - 0x73, 0x12, 0x38, 0x0a, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x65, 0x70, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x52, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x61, - 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2f, 0x0a, 0x13, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, - 0x69, 0x74, 0x79, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x12, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x57, 0x6f, - 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x95, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, 0x6d, - 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, + 0x3b, 0x0a, 0x17, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, + 0x48, 0x00, 0x52, 0x15, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x53, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x28, 0x0a, 0x10, + 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, + 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x49, 0x64, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, + 0xa2, 0x02, 0x0a, 0x0b, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x18, - 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, - 0xb2, 0x01, 0x0a, 0x14, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, - 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, - 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, - 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x42, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x06, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x22, 0x8b, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, - 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, - 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, - 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, - 0x65, 0x74, 0x22, 0xa6, 0x02, 0x0a, 0x15, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x40, + 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x22, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x70, 0x61, 0x74, 0x68, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, + 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, + 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, + 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6c, + 0x69, 0x6d, 0x69, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x35, 0x0a, + 0x08, 0x6d, 0x6f, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x6d, 0x6f, 0x64, + 0x54, 0x69, 0x6d, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x69, 0x73, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x69, 0x73, 0x44, 0x69, 0x72, 0x22, 0x47, 0x0a, 0x09, 0x47, + 0x72, 0x65, 0x70, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, + 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x74, 0x65, 0x78, 0x74, 0x22, 0xd4, 0x02, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, + 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, + 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, + 0x70, 0x61, 0x74, 0x68, 0x73, 0x12, 0x38, 0x0a, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, + 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x65, + 0x70, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, + 0x29, 0x0a, 0x10, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x75, 0x74, 0x68, 0x6f, + 0x72, 0x69, 0x74, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2f, 0x0a, 0x13, 0x61, 0x75, + 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, + 0x74, 0x79, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x95, 0x01, 0x0a, 0x13, + 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x25, 0x0a, 0x0e, + 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x69, 0x6c, + 0x6c, 0x69, 0x73, 0x22, 0xb2, 0x01, 0x0a, 0x14, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, - 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, - 0x65, 0x72, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, - 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6e, 0x65, 0x78, 0x74, 0x4f, 0x66, 0x66, 0x73, - 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, - 0x12, 0x29, 0x0a, 0x10, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x63, - 0x65, 0x69, 0x70, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x72, 0x6d, - 0x69, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, 0x3f, 0x0a, 0x0b, 0x45, - 0x72, 0x72, 0x6f, 0x72, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, - 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x1c, - 0x0a, 0x09, 0x72, 0x65, 0x74, 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x09, 0x72, 0x65, 0x74, 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x2a, 0xd8, 0x02, 0x0a, - 0x0d, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, - 0x0a, 0x1a, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, - 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, - 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, - 0x5f, 0x52, 0x45, 0x41, 0x44, 0x10, 0x01, 0x12, 0x24, 0x0a, 0x20, 0x46, 0x49, 0x4c, 0x45, 0x5f, - 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x53, 0x4f, 0x4c, 0x56, - 0x45, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x54, 0x59, 0x10, 0x02, 0x12, 0x17, 0x0a, - 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, - 0x53, 0x54, 0x41, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, - 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, - 0x04, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, - 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, 0x05, 0x12, 0x17, 0x0a, - 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, - 0x4c, 0x49, 0x53, 0x54, 0x10, 0x06, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, - 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x10, - 0x07, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, - 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x08, 0x12, 0x17, 0x0a, 0x13, - 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x43, - 0x4f, 0x50, 0x59, 0x10, 0x09, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, - 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x47, 0x4c, 0x4f, 0x42, 0x10, 0x0a, 0x12, 0x17, - 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, - 0x5f, 0x47, 0x52, 0x45, 0x50, 0x10, 0x0b, 0x2a, 0xbd, 0x01, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, 0x6d, - 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x43, 0x4f, 0x4d, 0x4d, - 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, - 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x43, 0x4f, 0x4d, 0x4d, 0x41, - 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, - 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, - 0x41, 0x54, 0x45, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x45, 0x44, 0x45, 0x44, 0x10, 0x02, 0x12, - 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, - 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x03, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4d, - 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, - 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, - 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x45, 0x4e, 0x43, 0x45, 0x5f, 0x55, 0x4e, - 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x05, 0x32, 0xe4, 0x12, 0x0a, 0x18, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, - 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, - 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, - 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, - 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x72, 0x0a, 0x11, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, - 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, - 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, - 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x73, - 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x11, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x42, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, + 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x8b, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, + 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x16, + 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, + 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0xa6, 0x02, 0x0a, 0x15, 0x43, 0x6f, 0x6d, 0x6d, 0x61, + 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, + 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, + 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x65, 0x78, 0x69, + 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x12, 0x16, 0x0a, + 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, + 0x74, 0x64, 0x65, 0x72, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x6f, 0x66, + 0x66, 0x73, 0x65, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6e, 0x65, 0x78, 0x74, + 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, + 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, + 0x61, 0x74, 0x65, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, + 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, + 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, + 0x3f, 0x0a, 0x0b, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x12, + 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, + 0x64, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x74, 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x74, 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, + 0x2a, 0xd8, 0x02, 0x0a, 0x0d, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, + 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, + 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x10, 0x01, 0x12, 0x24, 0x0a, 0x20, 0x46, + 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, + 0x53, 0x4f, 0x4c, 0x56, 0x45, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x54, 0x59, 0x10, + 0x02, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, + 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x43, 0x52, 0x45, + 0x41, 0x54, 0x45, 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, + 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, + 0x05, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x06, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, + 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4d, + 0x4f, 0x56, 0x45, 0x10, 0x07, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, + 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x08, + 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, + 0x4f, 0x4e, 0x5f, 0x43, 0x4f, 0x50, 0x59, 0x10, 0x09, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, + 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x47, 0x4c, 0x4f, 0x42, + 0x10, 0x0a, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, + 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x47, 0x52, 0x45, 0x50, 0x10, 0x0b, 0x2a, 0xbd, 0x01, 0x0a, 0x0c, + 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, + 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x43, + 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4e, + 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, + 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x45, 0x44, 0x45, + 0x44, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x03, 0x12, 0x1b, 0x0a, + 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x43, + 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, + 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x45, 0x4e, 0x43, + 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x05, 0x32, 0xe4, 0x12, 0x0a, 0x18, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, + 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x2b, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, + 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x11, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x31, 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5b, 0x0a, 0x0a, 0x41, 0x63, - 0x71, 0x75, 0x69, 0x72, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, - 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, - 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x25, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x08, 0x52, 0x65, 0x6e, 0x65, 0x77, - 0x52, 0x75, 0x6e, 0x12, 0x24, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x52, - 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, 0x63, 0x61, - 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, - 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x4c, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x26, + 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x11, 0x41, 0x74, + 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x75, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x58, - 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x62, 0x6f, 0x72, - 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, - 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x12, 0x73, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x6f, 0x72, 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, - 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5f, 0x0a, 0x16, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, - 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5f, 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x72, 0x6d, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5e, 0x0a, 0x15, 0x43, 0x61, 0x6e, 0x63, - 0x65, 0x6c, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x7b, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, - 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, - 0x12, 0x30, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x10, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, - 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5b, + 0x0a, 0x0a, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6c, + 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x08, 0x52, + 0x65, 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, 0x12, 0x24, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, + 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x12, 0x58, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0e, + 0x41, 0x62, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2d, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, + 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x73, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, - 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x5a, 0x0a, 0x11, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, - 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x74, 0x69, 0x72, - 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5f, 0x0a, 0x16, 0x50, 0x72, + 0x65, 0x70, 0x61, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x56, 0x0a, 0x0f, 0x52, - 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x12, 0x2b, - 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5f, 0x0a, 0x16, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5e, 0x0a, 0x15, + 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x7b, 0x0a, 0x14, + 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x73, 0x12, 0x30, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x10, 0x52, 0x65, 0x6c, + 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2c, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x12, 0x5c, 0x0a, 0x12, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, - 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, - 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, - 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x12, 0x68, 0x0a, 0x18, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, - 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x2e, + 0x70, 0x74, 0x79, 0x12, 0x5a, 0x0a, 0x11, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, + 0x56, 0x0a, 0x0f, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x6f, 0x72, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5c, 0x0a, 0x12, 0x52, 0x65, 0x63, 0x6f, 0x76, + 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, - 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5c, 0x0a, 0x12, 0x4d, - 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, - 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x68, 0x0a, 0x18, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, + 0x65, 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x12, 0x34, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, + 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, + 0x5c, 0x0a, 0x12, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x67, 0x72, + 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x72, 0x0a, + 0x11, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x72, 0x0a, 0x11, 0x52, 0x65, 0x76, - 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, - 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, - 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, - 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, - 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, - 0x05, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, - 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, - 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x0c, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, - 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x4c, 0x0a, 0x05, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x63, 0x0a, 0x0c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, + 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, - 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, - 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x65, - 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x61, 0x6e, 0x63, 0x65, - 0x6c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, - 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0xe4, - 0x01, 0x0a, 0x17, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x4b, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x6c, 0x6f, - 0x6b, 0x2f, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, - 0x74, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, - 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x3b, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, 0x45, 0x58, 0xaa, - 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, 0x45, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x1f, 0x4d, 0x65, - 0x63, 0x61, 0x74, 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5c, 0x56, - 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x15, - 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x3a, 0x3a, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, + 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x42, 0xe4, 0x01, 0x0a, 0x17, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x0e, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, + 0x5a, 0x4b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, 0x61, + 0x63, 0x6b, 0x6c, 0x6f, 0x6b, 0x2f, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, + 0x31, 0x3b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x76, 0x31, 0xa2, 0x02, 0x03, + 0x4d, 0x45, 0x58, 0xaa, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x45, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0xe2, + 0x02, 0x1f, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0xea, 0x02, 0x15, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x3a, 0x3a, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, }) var ( @@ -3626,6 +3639,7 @@ func file_mecatl_execution_v1_execution_proto_init() { if File_mecatl_execution_v1_execution_proto != nil { return } + file_mecatl_execution_v1_execution_proto_msgTypes[26].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/contracts/proto/mecatl/execution/v1/execution.proto b/contracts/proto/mecatl/execution/v1/execution.proto index c95c0c34d8..36ca47912a 100644 --- a/contracts/proto/mecatl/execution/v1/execution.proto +++ b/contracts/proto/mecatl/execution/v1/execution.proto @@ -178,13 +178,14 @@ message RevokeEnvironmentRequest { EnvironmentRef environment = 1; Owner owner = 2; uint64 expected_grant_generation = 3; + string operation_id = 4; } message RevokeEnvironmentResponse { uint64 grant_generation = 1; } message MigrateEnvironmentRequest { EnvironmentRef environment = 1; Owner owner = 2; - uint32 expected_schema_version = 3; + optional uint32 expected_schema_version = 3; string expected_pod_uid = 4; string expected_pvc_uid = 5; string operation_id = 6; diff --git a/deploy/helm/mecatl-execution/chart_test.go b/deploy/helm/mecatl-execution/chart_test.go index 67373f2271..a3da6bdbad 100644 --- a/deploy/helm/mecatl-execution/chart_test.go +++ b/deploy/helm/mecatl-execution/chart_test.go @@ -18,8 +18,16 @@ func TestChartRetainsCRDAndDoesNotGrantSecretAPI(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(string(rbac), "secrets") || strings.Contains(string(rbac), "clusterrole") { - t.Fatal("provider RBAC must be namespaced and have no Secret access") + if strings.Contains(string(rbac), "secrets") { + t.Fatal("provider RBAC must have no Secret access") + } + for _, required := range []string{"kind: ClusterRole", `resources: ["runtimeclasses"]`, `resources: ["storageclasses"]`, "resourceNames:", `verbs: ["get"]`} { + if !strings.Contains(string(rbac), required) { + t.Fatalf("profile preflight RBAC missing %q", required) + } + } + if strings.Contains(string(rbac), `resources: ["*"]`) { + t.Fatal("cluster-scoped profile preflight RBAC must not grant wildcard resources") } } func TestChartHasNoDeletionHook(t *testing.T) { @@ -59,7 +67,7 @@ func TestProductionHardeningIsFailClosed(t *testing.T) { t.Fatalf("provider hardening missing %q", required) } } - for _, required := range []string{"workload-default-deny", "ingress: []", "egress: []", "ResourceQuota", "LimitRange", "count/executionenvironments.execution.mecatl.dev"} { + for _, required := range []string{"workload-default-deny", "ingress: []", "egress: []", "ResourceQuota", "LimitRange", "count/executionenvironments.execution.mecatl.dev", "requests.ephemeral-storage", "limits.ephemeral-storage"} { if !strings.Contains(string(network), required) { t.Fatalf("network/resource hardening missing %q", required) } diff --git a/deploy/helm/mecatl-execution/crds/executionenvironment.yaml b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml index 296505a7cd..7c3433a9f3 100644 --- a/deploy/helm/mecatl-execution/crds/executionenvironment.yaml +++ b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml @@ -88,6 +88,18 @@ spec: epoch: {type: integer, format: int64, minimum: 1} fenceState: {type: string, enum: [Healthy, FenceUnknown]} grantGeneration: {type: integer, format: int64, minimum: 1} + revocationReceipts: + type: array + maxItems: 32 + items: + type: object + additionalProperties: false + required: [operationID, fingerprint, expectedGrantGeneration, grantGeneration] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + fingerprint: {type: string, pattern: "^[0-9a-f]{64}$"} + expectedGrantGeneration: {type: integer, format: int64, minimum: 1} + grantGeneration: {type: integer, format: int64, minimum: 1} references: type: array maxItems: 64 @@ -114,7 +126,7 @@ spec: uid: {type: string} activeRun: type: object - required: [bindingID, runID, claimID, operationID, ownerHash, clientHash, epoch, expiresAt] + required: [bindingID, runID, claimID, operationID, ownerHash, clientHash, epoch, grantGeneration, expiresAt] properties: bindingID: {type: string, maxLength: 253} runID: {type: string, maxLength: 1024} @@ -123,6 +135,7 @@ spec: ownerHash: {type: string, pattern: "^[0-9a-f]{64}$"} clientHash: {type: string, pattern: "^[0-9a-f]{64}$"} epoch: {type: integer, format: int64, minimum: 1} + grantGeneration: {type: integer, format: int64, minimum: 1} expiresAt: {type: string, format: date-time} lastRenewOperationID: {type: string, maxLength: 1024} activeOperation: diff --git a/deploy/helm/mecatl-execution/templates/network-policy.yaml b/deploy/helm/mecatl-execution/templates/network-policy.yaml index 1cffe177a4..62b3f04578 100644 --- a/deploy/helm/mecatl-execution/templates/network-policy.yaml +++ b/deploy/helm/mecatl-execution/templates/network-policy.yaml @@ -82,8 +82,10 @@ spec: requests.cpu: {{ .Values.resourceGovernance.requestsCPU | quote }} requests.memory: {{ .Values.resourceGovernance.requestsMemory | quote }} requests.storage: {{ .Values.resourceGovernance.requestsStorage | quote }} + requests.ephemeral-storage: {{ .Values.resourceGovernance.requestsEphemeralStorage | quote }} limits.cpu: {{ .Values.resourceGovernance.limitsCPU | quote }} limits.memory: {{ .Values.resourceGovernance.limitsMemory | quote }} + limits.ephemeral-storage: {{ .Values.resourceGovernance.limitsEphemeralStorage | quote }} --- apiVersion: v1 kind: LimitRange diff --git a/deploy/helm/mecatl-execution/templates/provider.yaml b/deploy/helm/mecatl-execution/templates/provider.yaml index caf2e0b02b..7d2f0d058b 100644 --- a/deploy/helm/mecatl-execution/templates/provider.yaml +++ b/deploy/helm/mecatl-execution/templates/provider.yaml @@ -21,6 +21,12 @@ metadata: name: {{ include "mecatl-execution.fullname" . }}-security-authority data: {} --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mecatl-execution-profile-allocations +data: {} +--- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deploy/helm/mecatl-execution/templates/rbac.yaml b/deploy/helm/mecatl-execution/templates/rbac.yaml index d1687be383..ef3dab3635 100644 --- a/deploy/helm/mecatl-execution/templates/rbac.yaml +++ b/deploy/helm/mecatl-execution/templates/rbac.yaml @@ -16,7 +16,9 @@ rules: verbs: ["get", "update", "patch"] - apiGroups: [""] resources: ["configmaps"] - resourceNames: [{{ printf "%s-security-authority" (include "mecatl-execution.fullname" .) | quote }}] + resourceNames: + - {{ printf "%s-security-authority" (include "mecatl-execution.fullname" .) | quote }} + - "mecatl-execution-profile-allocations" verbs: ["get", "update"] - apiGroups: [""] resources: ["pods"] @@ -40,3 +42,36 @@ roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: {{ include "mecatl-execution.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ printf "%s-%s-profile-preflight" .Release.Namespace (include "mecatl-execution.fullname" .) }} +rules: +- apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + resourceNames: +{{- range $name, $profile := .Values.profiles }} + - {{ $profile.runtimeClassName | quote }} +{{- end }} + verbs: ["get"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + resourceNames: +{{- range $name, $profile := .Values.profiles }} + - {{ $profile.storageClass | quote }} +{{- end }} + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ printf "%s-%s-profile-preflight" .Release.Namespace (include "mecatl-execution.fullname" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "mecatl-execution.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ printf "%s-%s-profile-preflight" .Release.Namespace (include "mecatl-execution.fullname" .) }} diff --git a/deploy/helm/mecatl-execution/values.schema.json b/deploy/helm/mecatl-execution/values.schema.json index 2d2fa3fcd3..fc37d2ea8a 100644 --- a/deploy/helm/mecatl-execution/values.schema.json +++ b/deploy/helm/mecatl-execution/values.schema.json @@ -27,7 +27,7 @@ }, "service":{"type":"object","additionalProperties":false,"required":["port"],"properties":{"port":{"type":"integer","minimum":1,"maximum":65535}}}, "networkPolicy":{"type":"object","additionalProperties":false,"required":["enabled","dnsPorts","apiServerPorts","workloadProfiles"],"properties":{"enabled":{"const":true},"dnsPorts":{"type":"array","minItems":1,"items":{"type":"integer","minimum":1,"maximum":65535}},"apiServerPorts":{"type":"array","minItems":1,"items":{"type":"integer","minimum":1,"maximum":65535}},"workloadProfiles":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"required":["egress"],"properties":{"egress":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["cidr","ports"],"properties":{"cidr":{"type":"string","pattern":"^[0-9a-fA-F:.]+/[0-9]+$"},"ports":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["port"],"properties":{"port":{"type":"integer","minimum":1,"maximum":65535},"protocol":{"enum":["TCP","UDP"]}}}}}}}}}}}}, - "resourceGovernance":{"type":"object","additionalProperties":false,"required":["enabled","pods","persistentVolumeClaims","executionEnvironments","requestsCPU","requestsMemory","requestsStorage","limitsCPU","limitsMemory"],"properties":{"enabled":{"const":true},"pods":{"type":"string","minLength":1},"persistentVolumeClaims":{"type":"string","minLength":1},"executionEnvironments":{"type":"string","minLength":1},"requestsCPU":{"type":"string","minLength":1},"requestsMemory":{"type":"string","minLength":1},"requestsStorage":{"type":"string","minLength":1},"limitsCPU":{"type":"string","minLength":1},"limitsMemory":{"type":"string","minLength":1}}}, + "resourceGovernance":{"type":"object","additionalProperties":false,"required":["enabled","pods","persistentVolumeClaims","executionEnvironments","requestsCPU","requestsMemory","requestsStorage","requestsEphemeralStorage","limitsCPU","limitsMemory","limitsEphemeralStorage"],"properties":{"enabled":{"const":true},"pods":{"type":"string","minLength":1},"persistentVolumeClaims":{"type":"string","minLength":1},"executionEnvironments":{"type":"string","minLength":1},"requestsCPU":{"type":"string","minLength":1},"requestsMemory":{"type":"string","minLength":1},"requestsStorage":{"type":"string","minLength":1},"requestsEphemeralStorage":{"type":"string","minLength":1},"limitsCPU":{"type":"string","minLength":1},"limitsMemory":{"type":"string","minLength":1},"limitsEphemeralStorage":{"type":"string","minLength":1}}}, "profiles":{"type":"object","minProperties":1,"additionalProperties":{"type":"object","additionalProperties":false,"required":["image","storageClass","storageSize","cpuRequest","memoryRequest","cpuLimit","memoryLimit","ephemeralStorageRequest","ephemeralStorageLimit","tmpSizeLimit","runtimeClassName","maxFileBytes","maxCommandBytes","maxCommandDuration","maxEnvironments"],"properties":{"image":{"type":"string","pattern":"@sha256:[0-9a-fA-F]{64}$"},"storageClass":{"type":"string","minLength":1},"storageSize":{"type":"string","minLength":1},"cpuRequest":{"type":"string","minLength":1},"memoryRequest":{"type":"string","minLength":1},"cpuLimit":{"type":"string","minLength":1},"memoryLimit":{"type":"string","minLength":1},"ephemeralStorageRequest":{"type":"string","minLength":1},"ephemeralStorageLimit":{"type":"string","minLength":1},"tmpSizeLimit":{"type":"string","minLength":1},"runtimeClassName":{"type":"string","minLength":1},"maxFileBytes":{"type":"integer","minimum":1,"maximum":5242880},"maxCommandBytes":{"type":"integer","minimum":1,"maximum":1048576},"maxCommandDuration":{"type":"string","pattern":"^[1-9][0-9]*(ms|s|m)$"},"maxEnvironments":{"type":"integer","minimum":1,"maximum":10000}}}} } } diff --git a/deploy/helm/mecatl-execution/values.yaml b/deploy/helm/mecatl-execution/values.yaml index c25e896942..d97077f19f 100644 --- a/deploy/helm/mecatl-execution/values.yaml +++ b/deploy/helm/mecatl-execution/values.yaml @@ -32,8 +32,10 @@ resourceGovernance: requestsCPU: "20" requestsMemory: 40Gi requestsStorage: 500Gi + requestsEphemeralStorage: 40Gi limitsCPU: "40" limitsMemory: 80Gi + limitsEphemeralStorage: 80Gi profiles: {} # Example profile (all fields are required): diff --git a/deploy/mecatl-execution-kind/execution-values.yaml b/deploy/mecatl-execution-kind/execution-values.yaml new file mode 100644 index 0000000000..a6ed5a8bc9 --- /dev/null +++ b/deploy/mecatl-execution-kind/execution-values.yaml @@ -0,0 +1,59 @@ +provider: + clientIngressSelectors: + - namespaceLabels: + kubernetes.io/metadata.name: execution-qualification + podLabels: + app.kubernetes.io/name: mecak8s + apiServerCIDRs: [172.16.0.0/12] + dnsCIDRs: [10.96.0.10/32] +profiles: + go: + image: placeholder.invalid/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 1Gi + cpuRequest: 50m + memoryRequest: 64Mi + cpuLimit: "1" + memoryLimit: 512Mi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: qualification-runc + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m + maxEnvironments: 20 + quota-cas: + image: placeholder.invalid/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 1Gi + cpuRequest: 50m + memoryRequest: 64Mi + cpuLimit: "1" + memoryLimit: 512Mi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: qualification-runc + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m + maxEnvironments: 1 + quota-kube: + image: placeholder.invalid/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 1Gi + cpuRequest: 50m + memoryRequest: 64Mi + cpuLimit: "1" + memoryLimit: 512Mi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: qualification-runc + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m + maxEnvironments: 4 +networkPolicy: + workloadProfiles: {} diff --git a/deploy/mecatl-execution-kind/live.sh b/deploy/mecatl-execution-kind/live.sh index a9332fc814..99a222f01c 100755 --- a/deploy/mecatl-execution-kind/live.sh +++ b/deploy/mecatl-execution-kind/live.sh @@ -56,10 +56,14 @@ recorded_kubeconfig=$(owned_value kubeconfig) || { echo "owned kubeconfig identi context=$(optional_owned_value context) || { echo "owned context identity invalid" >&2; exit 1; } [ -n "$context" ] || context="kind-$cluster" [ "$context" = "kind-$cluster" ] || { echo "owned context mismatch" >&2; exit 1; } -label=$(podman inspect "${cluster}-control-plane" --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}') +runtime=$(owned_value runtime) || { echo "owned container runtime unavailable" >&2; exit 1; } +[ "$runtime" = docker ] || [ "$runtime" = podman ] || { echo "owned container runtime invalid" >&2; exit 1; } +profile=$(owned_value profile) || { echo "owned qualification profile unavailable" >&2; exit 1; } +[ "$profile" = production ] || { echo "live qualification requires a completed production-state fixture" >&2; exit 1; } +label=$($runtime inspect "${cluster}-control-plane" --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}') [ "$label" = "$cluster" ] || { echo "owned Kind container label mismatch" >&2; exit 1; } -export KIND_EXPERIMENTAL_PROVIDER=podman +if [ "$runtime" = podman ]; then export KIND_EXPERIMENTAL_PROVIDER=podman; else unset KIND_EXPERIMENTAL_PROVIDER; fi export KUBECONFIG=$kubeconfig export REGISTRY_AUTH_FILE="$state/registry-auth.json" dev() { @@ -75,30 +79,30 @@ kube_jq() { if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" jq "$@"; else jq "$@"; fi } [ "$(kube config current-context)" = "$context" ] || { echo "owned kube context mismatch" >&2; exit 1; } -build_ko() { package=$1 repo=$2; dev env KIND_EXPERIMENTAL_PROVIDER=podman KO_DOCKER_REPO="$repo" ko build --local --bare "$package" | tail -n 1; } +build_ko() { package=$1 repo=$2; dev env KIND_EXPERIMENTAL_PROVIDER="${KIND_EXPERIMENTAL_PROVIDER:-}" KO_DOCKER_REPO="$repo" ko build --local --bare "$package" | tail -n 1; } pin_loaded() { tagged=$1 - digest=$(podman exec "${cluster}-control-plane" ctr -n k8s.io images ls | awk -v ref="$tagged" '$1 == ref {print $3; exit}') + digest=$($runtime exec "${cluster}-control-plane" ctr -n k8s.io images ls | awk -v ref="$tagged" '$1 == ref {print $3; exit}') [ -n "$digest" ] || { echo "loaded image digest unavailable" >&2; exit 1; } pinned="${tagged%:*}@${digest}" - podman exec "${cluster}-control-plane" ctr -n k8s.io images tag "$tagged" "$pinned" >/dev/null + $runtime exec "${cluster}-control-plane" ctr -n k8s.io images tag "$tagged" "$pinned" >/dev/null printf '%s\n' "$pinned" } load_image() { tagged=$1 archive="$state/images/live-$(printf '%s' "$tagged" | sha256sum | cut -c1-16).tar" - podman save "$tagged" -o "$archive" >/dev/null + $runtime save "$tagged" -o "$archive" >/dev/null kind load image-archive "$archive" --name "$cluster" pin_loaded "$tagged" } provider_tag=$(build_ko ./cmd/mecatl-execution-provider ko.local/mecatl-execution-provider) agent_tag=$(build_ko ./cmd/mecak8s ko.local/mecak8s) -podman image exists docker.io/library/golang:1.27 || podman pull docker.io/library/golang:1.27 >/dev/null -go_digest=$(podman image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') +$runtime image inspect docker.io/library/golang:1.27 >/dev/null 2>&1 || $runtime pull docker.io/library/golang:1.27 >/dev/null +go_digest=$($runtime image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') go_image="docker.io/library/golang@${go_digest}" workload_tag=localhost/mecatl-execution-workload:e2e -podman build --build-arg GO_IMAGE="$go_image" -f "$root/build/execution-workload/Dockerfile" -t "$workload_tag" "$root" >/dev/null +$runtime build --build-arg GO_IMAGE="$go_image" -f "$root/build/execution-workload/Dockerfile" -t "$workload_tag" "$root" >/dev/null provider_image=$(load_image "$provider_tag") agent_image=$(load_image "$agent_tag") workload_image=$(load_image "$workload_tag") @@ -110,30 +114,10 @@ kube -n local-path-storage get configmap local-path-config -o json \ | kube_jq --arg image "$workload_image" '.data["helperPod.yaml"] |= sub("image: [^\\n]+"; "image: " + $image)' \ | kube replace -f - -# Qualification identities are synthetic and intentionally short-lived. Rotate -# them before any rollout so an owned cluster can be safely reused hours later. -dev env -i HOME="$HOME" PATH="$PATH" go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/pki "$state/pki" -kube -n execution-qualification create secret generic execution-provider-tls \ - --from-file=ca.crt="$state/pki/ca.crt" --from-file=tls.crt="$state/pki/provider.crt" --from-file=tls.key="$state/pki/provider.key" --dry-run=client -o yaml | kube apply -f - -kube -n execution-qualification create secret generic execution-client-tls \ - --from-file=ca.crt="$state/pki/ca.crt" --from-file=tls.crt="$state/pki/mecak8s.crt" --from-file=tls.key="$state/pki/mecak8s.key" --dry-run=client -o yaml | kube apply -f - -kube -n execution-qualification create secret generic execution-intruder-tls \ - --from-file=ca.crt="$state/pki/ca.crt" --from-file=tls.crt="$state/pki/intruder.crt" --from-file=tls.key="$state/pki/intruder.key" --dry-run=client -o yaml | kube apply -f - -kube -n execution-qualification create secret generic execution-grant-key --from-file=key.pem="$state/pki/grant-key.pem" --dry-run=client -o yaml | kube apply -f - -kube -n execution-qualification create secret generic oidc-fixture-identity \ - --from-file=tls.crt="$state/pki/oidc.crt" --from-file=tls.key="$state/pki/oidc.key" --from-file=jwt-key.pem="$state/pki/oidc-jwt-key.pem" --dry-run=client -o yaml | kube apply -f - -kube -n execution-qualification rollout restart deployment/oidc-issuer -kube -n execution-qualification rollout status deployment/oidc-issuer --timeout=180s - -# First qualify the migrated typed-gRPC path under the deterministic mock model. +# Consume the already-qualified synthetic security state. Live mode never +# regenerates or replaces the execution Secret/keyring, and never reads it back. +# A real provider credential is staged only after this deterministic rerun passes. kube -n execution-qualification create configmap execution-mock --from-file=mock-script.json="$root/deploy/mecatl-execution-kind/mock-script.json" --dry-run=client -o yaml | kube apply -f - -helm_kube upgrade --install mecatl-execution "$root/deploy/helm/mecatl-execution" --namespace execution-qualification \ - --set fullnameOverride=mecatl-execution --set-string provider.image="$provider_image" --set provider.imagePullPolicy=IfNotPresent \ - --set provider.tlsSecretName=execution-provider-tls --set provider.grantSigningSecretName=execution-grant-key \ - --set 'provider.clientURISANs={spiffe://mecatl.test/client/mecak8s,spiffe://mecatl.test/client/intruder}' \ - --set 'provider.ownerAttesterURISANs={spiffe://mecatl.test/client/mecak8s,spiffe://mecatl.test/client/intruder}' \ - --set 'provider.administratorURISANs={spiffe://mecatl.test/client/mecak8s}' --set-string profiles.go.image="$workload_image" --reuse-values --wait --timeout=4m -kube -n execution-qualification rollout restart deployment/mecatl-execution kube -n execution-qualification rollout status deployment/mecatl-execution --timeout=240s helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" --wait --timeout=5m diff --git a/deploy/mecatl-execution-kind/run.sh b/deploy/mecatl-execution-kind/run.sh index ae289962a5..cc52e3b305 100755 --- a/deploy/mecatl-execution-kind/run.sh +++ b/deploy/mecatl-execution-kind/run.sh @@ -7,6 +7,7 @@ if [ "${MECATL_EXECUTION_QUAL_CLEAN_ENV:-}" != 1 ]; then exec env -i \ HOME="$HOME" USER="${USER:-user}" PATH="$PATH" XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" \ DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-}" CONTAINER_HOST="${CONTAINER_HOST:-}" DOCKER_HOST="${DOCKER_HOST:-}" \ + CONTAINER_ENGINE="${CONTAINER_ENGINE:-}" MECATL_EXECUTION_QUAL_CI="${MECATL_EXECUTION_QUAL_CI:-}" MECATL_EXECUTION_QUAL_PROFILE="${MECATL_EXECUTION_QUAL_PROFILE:-}" \ TOOLBOX_PATH="${TOOLBOX_PATH:-}" MECATL_EXECUTION_DEV_TOOLBOX="${MECATL_EXECUTION_DEV_TOOLBOX:-}" MECATL_EXECUTION_K8S_TOOLBOX="${MECATL_EXECUTION_K8S_TOOLBOX:-}" \ MECATL_EXECUTION_QUAL_CLEAN_ENV=1 "$0" "$@" fi @@ -14,6 +15,8 @@ fi [ "$#" -eq 0 ] || { echo "usage: $0" >&2; exit 2; } case "${MECATL_EXECUTION_DEV_TOOLBOX:-}" in ''|dev) ;; *) echo "MECATL_EXECUTION_DEV_TOOLBOX must be empty or dev" >&2; exit 2 ;; esac case "${MECATL_EXECUTION_K8S_TOOLBOX:-}" in ''|sre) ;; *) echo "MECATL_EXECUTION_K8S_TOOLBOX must be empty or sre" >&2; exit 2 ;; esac +runtime=${CONTAINER_ENGINE:-docker} +case "$runtime" in docker|podman) command -v "$runtime" >/dev/null 2>&1 ;; *) echo "CONTAINER_ENGINE must be docker or podman" >&2; exit 2 ;; esac root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd -P) cd "$root" run_id=$(date -u +%Y%m%d%H%M%S)-$$ @@ -25,8 +28,8 @@ mkdir -p "$root/.scratch/k8s-execution" umask 077 mkdir "$state" mkdir "$state/pki" "$state/images" -printf 'cluster=%s\ncontext=%s\nkubeconfig=%s\nnamespace=execution-qualification\nowner=%s\ncreated_at=%s\n' \ - "$cluster" "$context" "$kubeconfig" "${USER:-user}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$state/ownership" +printf 'cluster=%s\ncontext=%s\nkubeconfig=%s\nnamespace=execution-qualification\nowner=%s\nruntime=%s\nprofile=%s\ncreated_at=%s\n' \ + "$cluster" "$context" "$kubeconfig" "${USER:-user}" "$runtime" "${MECATL_EXECUTION_QUAL_PROFILE:-development}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$state/ownership" # Refuse collisions; never delete or reuse any cluster. if kind get clusters 2>/dev/null | grep -Fx "$cluster" >/dev/null; then @@ -35,13 +38,48 @@ if kind get clusters 2>/dev/null | grep -Fx "$cluster" >/dev/null; then fi printf '%s\n' "$state" >"$root/.scratch/k8s-execution/current" -export KIND_EXPERIMENTAL_PROVIDER=podman +if [ "$runtime" = podman ]; then + export KIND_EXPERIMENTAL_PROVIDER=podman +else + unset KIND_EXPERIMENTAL_PROVIDER +fi +if [ "${MECATL_EXECUTION_QUAL_CI:-}" = 1 ]; then + cleanup_cluster() { + status=$? + artifact="$state/production-failure-artifact.txt" + if [ "$status" -ne 0 ]; then + kubectl --kubeconfig "$kubeconfig" --context "$context" get events -n execution-qualification \ + -o 'custom-columns=OBJECT:.involvedObject.name,REASON:.reason' --no-headers >"$artifact" 2>/dev/null || printf 'qualification_setup_failed\n' >"$artifact" + head -c 1048576 "$artifact" >"$artifact.bounded" + mv "$artifact.bounded" "$artifact" + fi + kind delete cluster --name "$cluster" + return "$status" + } + trap cleanup_cluster EXIT +fi export KUBECONFIG="$kubeconfig" printf '{}\n' >"$state/registry-auth.json" chmod 600 "$state/registry-auth.json" export REGISTRY_AUTH_FILE="$state/registry-auth.json" -kind create cluster --name "$cluster" --image kindest/node:v1.35.8@sha256:07b2536e30b803ed61d1677a79df6115f798ce64c80f9e22f6ed45afd09323c0 --kubeconfig "$kubeconfig" +kind_config= +if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then + kind_config="$state/kind.yaml" + cat >"$kind_config" <<'EOF' +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + disableDefaultCNI: true + podSubnet: 192.168.0.0/16 + serviceSubnet: 10.96.0.0/12 +EOF +fi +if [ -n "$kind_config" ]; then + kind create cluster --name "$cluster" --image kindest/node:v1.35.8@sha256:07b2536e30b803ed61d1677a79df6115f798ce64c80f9e22f6ed45afd09323c0 --config "$kind_config" --kubeconfig "$kubeconfig" +else + kind create cluster --name "$cluster" --image kindest/node:v1.35.8@sha256:07b2536e30b803ed61d1677a79df6115f798ce64c80f9e22f6ed45afd09323c0 --kubeconfig "$kubeconfig" +fi dev() { if [ -n "$MECATL_EXECUTION_DEV_TOOLBOX" ]; then @@ -65,6 +103,23 @@ helm_kube() { fi } +if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then + cni_manifest="$state/calico-v3.32.2.yaml" + curl --fail --location --proto '=https' --tlsv1.2 \ + https://raw.githubusercontent.com/projectcalico/calico/v3.32.2/manifests/calico.yaml \ + --output "$cni_manifest" + printf '%s %s\n' a8c828a06a87c629a282ebbc424895b77f3a030251993e41ea400a743675bb02 "$cni_manifest" | sha256sum -c - + sed -i \ + -e 's|quay.io/calico/cni:v3.32.2|quay.io/calico/cni@sha256:0ef740bc587f25565905adf1d1f61a7faff0d571c449c6bdd789feed743d3ef7|g' \ + -e 's|quay.io/calico/kube-controllers:v3.32.2|quay.io/calico/kube-controllers@sha256:7870b67ebb13fabc3005252b44fe6e78b21635649bd3072b80afa1684b6565d0|g' \ + -e 's|quay.io/calico/node:v3.32.2|quay.io/calico/node@sha256:99b03fe91e8bfbcb153ae65ef4b701b24ce541ffdd74ff314eb041096008f7fd|g' \ + "$cni_manifest" + kube apply -f "$cni_manifest" + kube -n kube-system rollout status daemonset/calico-node --timeout=5m + kube -n kube-system rollout status deployment/calico-kube-controllers --timeout=5m + kube wait --for=condition=Ready node --all --timeout=3m +fi + # Synthetic keys are generated by the fixture only and never printed. Resume # keeps the original PKI so mounted trust and running identities cannot drift. if [ ! -s "$state/pki/ca.crt" ]; then @@ -76,53 +131,106 @@ fi # digest and creates a matching digest alias for Kubernetes. build_ko() { package=$1 repo=$2 - dev env KIND_EXPERIMENTAL_PROVIDER=podman KO_DOCKER_REPO="$repo" ko build --local --bare "$package" | tail -n 1 + dev env KO_DOCKER_REPO="$repo" ko build --local --bare "$package" | tail -n 1 } provider_tag=$(build_ko ./cmd/mecatl-execution-provider ko.local/mecatl-execution-provider) agent_tag=$(build_ko ./cmd/mecak8s ko.local/mecak8s) -oidc_tag=$(dev env KIND_EXPERIMENTAL_PROVIDER=podman GOFLAGS=-tags=kind_execution_e2e KO_DOCKER_REPO=ko.local/mecatl-oidc-fixture ko build --local --bare ./e2e/k8s_execution/fixture/oidcissuer | tail -n 1) +oidc_tag=$(dev env GOFLAGS=-tags=kind_execution_e2e KO_DOCKER_REPO=ko.local/mecatl-oidc-fixture ko build --local --bare ./e2e/k8s_execution/fixture/oidcissuer | tail -n 1) +netprobe_tag=$(dev env GOFLAGS=-tags=kind_execution_e2e KO_DOCKER_REPO=ko.local/mecatl-netprobe-fixture ko build --local --bare ./e2e/k8s_execution/fixture/netprobe | tail -n 1) # Resolve the public Go base from the local OCI store, then feed its observed # digest to the production workload recipe without weakening its pin check. -podman image exists docker.io/library/golang:1.27 || podman pull docker.io/library/golang:1.27 >/dev/null -go_digest=$(podman image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') +if [ "$runtime" = podman ]; then + podman image exists docker.io/library/golang:1.27 || podman pull docker.io/library/golang:1.27 >/dev/null + go_digest=$(podman image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') +else + docker image inspect docker.io/library/golang:1.27 >/dev/null 2>&1 || docker pull docker.io/library/golang:1.27 >/dev/null + go_ref=$(docker image inspect docker.io/library/golang:1.27 --format '{{index .RepoDigests 0}}') + go_digest=${go_ref#*@} +fi go_image="docker.io/library/golang@${go_digest}" workload_tag="localhost/mecatl-execution-workload:e2e" -podman build --build-arg GO_IMAGE="$go_image" -f "$root/build/execution-workload/Dockerfile" -t "$workload_tag" "$root" >/dev/null +"$runtime" build --build-arg GO_IMAGE="$go_image" -f "$root/build/execution-workload/Dockerfile" -t "$workload_tag" "$root" >/dev/null -for item in "$provider_tag" "$agent_tag" "$oidc_tag" "$workload_tag"; do +for item in "$provider_tag" "$agent_tag" "$oidc_tag" "$netprobe_tag" "$workload_tag"; do archive="$state/images/$(printf '%s' "$item" | sha256sum | cut -c1-16).tar" - podman save "$item" -o "$archive" >/dev/null + "$runtime" save "$item" -o "$archive" >/dev/null kind load image-archive "$archive" --name "$cluster" done node="${cluster}-control-plane" pin_loaded() { tagged=$1 - digest=$(podman exec "$node" ctr -n k8s.io images ls | awk -v ref="$tagged" '$1 == ref {print $3; exit}') + digest=$("$runtime" exec "$node" ctr -n k8s.io images ls | awk -v ref="$tagged" '$1 == ref {print $3; exit}') [ -n "$digest" ] || { echo "loaded image digest unavailable for $tagged" >&2; exit 1; } pinned="${tagged%:*}@${digest}" - podman exec "$node" ctr -n k8s.io images tag "$tagged" "$pinned" >/dev/null + "$runtime" exec "$node" ctr -n k8s.io images tag "$tagged" "$pinned" >/dev/null printf '%s\n' "$pinned" } provider_image=$(pin_loaded "$provider_tag") agent_image=$(pin_loaded "$agent_tag") oidc_image=$(pin_loaded "$oidc_tag") +netprobe_image=$(pin_loaded "$netprobe_tag") workload_image=$(pin_loaded "$workload_tag") -printf 'provider=%s\nagent=%s\noidc=%s\nworkload=%s\ngo_base=%s\n' "$provider_image" "$agent_image" "$oidc_image" "$workload_image" "$go_image" >"$state/images/proof" +printf 'provider=%s\nagent=%s\noidc=%s\nnetprobe=%s\nworkload=%s\ngo_base=%s\n' "$provider_image" "$agent_image" "$oidc_image" "$netprobe_image" "$workload_image" "$go_image" >"$state/images/proof" kube create namespace execution-qualification --dry-run=client -o yaml | kube apply -f - kube label namespace execution-qualification pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/audit=restricted pod-security.kubernetes.io/warn=restricted --overwrite # Kind v0.33 installs its pinned local-path provisioner and default "standard" -# StorageClass as part of cluster creation. +# StorageClass as part of cluster creation. The fixture RuntimeClass names Kind's +# existing runc handler so provider startup can exercise the production preflight. +kube apply -f "$root/deploy/mecatl-execution-kind/runtimeclass.yaml" -kube -n execution-qualification create secret generic execution-provider-tls \ - --from-file=ca.crt="$state/pki/ca.crt" --from-file=tls.crt="$state/pki/provider.crt" --from-file=tls.key="$state/pki/provider.key" --dry-run=client -o yaml | kube apply -f - +if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then + cat >"$state/network-fixtures.yaml" <&2; exit 1 ;; + :*|*:) echo "missing discovered network-policy endpoint" >&2; exit 1 ;; + esac + runtime_values="$state/production-runtime-values.yaml" + cat >"$runtime_values" </dev/null 2>&1 & i=0; trap '' HUP TERM; while [ $i -lt 90 ]; do i=$((i+1)); printf '%s\n' "$i" > holder-loss-nonce; sleep 1; done`, TimeoutMillis: 110000}) + commandDone <- commandErr + }() + waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.ActiveOperationID != "" }) + runKubectl(t, ctx, kubeconfig, "delete", "pod/"+holderPod, "-n", namespace, "--wait=true", "--timeout=90s") + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=180s") + + if _, err := serviceClient.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "second-writer", Data: []byte("must-not-run")}); err == nil { + t.Fatal("surviving replica admitted an overlapping writer after holder loss") + } + if _, err := serviceClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "overlap-run", OperationID: "overlap-run-acquire", TTL: time.Minute}); err == nil { + t.Fatal("surviving replica admitted a new run while old operation ownership was unresolved") + } + fenced := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { + return s.FenceState == "FenceUnknown" && s.ActiveOperationID != "" && !s.Ready + }) + if fenced.ReadyReason != "FenceUnknown" { + t.Fatalf("holder loss reason=%q, want bounded FenceUnknown", fenced.ReadyReason) + } + select { + case <-commandDone: + case <-time.After(5 * time.Second): + // A disconnected exec stream is permitted to remain locally blocked; the + // durable operation identity above, not RPC completion, is the proof. + } + + waitExecutorTerminal(t, ctx, kubeconfig, attached.Environment.ID) + recovery := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: initial.Epoch, ExpectedPodUID: initial.PodUID, ExpectedPVCUID: initial.PVCUID, OperationID: "recover-holder-loss"} + if err := serviceClient.RecoverEnvironment(ctx, recovery); err != nil { + t.Fatalf("recover exact terminal executor: %v", err) + } + if err := serviceClient.ReplaceExecutor(ctx, executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: initial.Epoch, ExpectedPodUID: initial.PodUID, ExpectedPVCUID: initial.PVCUID, OperationID: "replace-holder-loss"}); err != nil { + t.Fatalf("replace recovered executor: %v", err) + } + replaced := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Ready && s.PodUID != "" && s.PodUID != initial.PodUID }) + if replaced.PVCUID != initial.PVCUID { + t.Fatal("holder-loss recovery replaced the retained workspace") + } + reattached := waitReady(t, ctx, serviceClient, owner, binding, attached.Environment) + verify, release := acquireRun(t, ctx, serviceClient, owner, binding, reattached, "holder-loss-verify") + if _, err := serviceClient.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "holder-loss-nonce"}); err != nil { + t.Fatalf("post-recovery sentinel unavailable: %v", err) + } + if _, err := serviceClient.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "second-writer"}); err == nil { + t.Fatal("denied overlapping writer nevertheless reached the executor") + } + release() +} + +func waitExecutorTerminal(t *testing.T, ctx context.Context, kubeconfig, environmentID string) { + t.Helper() + deadline := time.Now().Add(2 * time.Minute) + for time.Now().Before(deadline) { + var pods corev1.PodList + raw := runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment="+environmentID, "-o", "json") + if json.Unmarshal(raw, &pods) == nil && len(pods.Items) == 1 && (pods.Items[0].Status.Phase == corev1.PodFailed || pods.Items[0].Status.Phase == corev1.PodSucceeded) { + terminated := len(pods.Items[0].Status.ContainerStatuses) == 1 && pods.Items[0].Status.ContainerStatuses[0].State.Terminated != nil + if terminated { + return + } + } + time.Sleep(time.Second) + } + t.Fatal("test-owned executor did not produce terminal kubelet evidence") +} + +func TestKindExecutionProductionQuotaSaturation(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + + // First prove the provider's profile-wide CAS authority under concurrent + // requests. Exactly one distinct allocation may consume quota-cas's sole slot. + type ensureResult struct { + owner executionenv.Owner + binding string + out executionenv.EnsureEnvironmentResponse + err error + } + start := make(chan struct{}) + results := make(chan ensureResult, 2) + for i := 0; i < 2; i++ { + i := i + go func() { + <-start + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: fmt.Sprintf("quota-cas-%d", i)} + binding := fmt.Sprintf("quota-cas-%d-%d", i, time.Now().UnixNano()) + out, err := client.Ensure(ctx, binding, "quota-cas", owner, "ensure-"+binding) + results <- ensureResult{owner: owner, binding: binding, out: out, err: err} + }() + } + close(start) + var accepted *ensureResult + denied := 0 + for range 2 { + result := <-results + if result.err == nil { + copy := result + accepted = © + } else if isRemoteCode(result.err, executionenv.CodeResourceExhausted) { + denied++ + } else { + t.Fatalf("quota CAS returned unexpected error: %v", result.err) + } + } + if accepted == nil || denied != 1 { + t.Fatalf("quota CAS accepted=%v resource_exhausted=%d, want one each", accepted != nil, denied) + } + if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: accepted.out.Environment, Owner: accepted.owner, BindingID: accepted.binding, OperationID: "commit-" + accepted.binding}); err != nil { + t.Fatal(err) + } + waitReady(t, ctx, client, accepted.owner, accepted.binding, accepted.out.Environment) + + // Then exercise Kubernetes admission itself. Tighten only the fixture-owned + // namespace's PVC count to one additional claim and restore the known chart + // value before converging and explicitly deleting these synthetic allocations. + pvcBaseline := resourceCount(t, ctx, kubeconfig, "pvc") + runKubectl(t, ctx, kubeconfig, "patch", "resourcequota/mecatl-execution", "-n", namespace, "--type=merge", "-p", fmt.Sprintf(`{"spec":{"hard":{"persistentvolumeclaims":"%d"}}}`, pvcBaseline+1)) + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute) + defer cleanupCancel() + _ = command(cleanupCtx, kubeconfig, "patch", "resourcequota/mecatl-execution", "-n", namespace, "--type=merge", "-p", `{"spec":{"hard":{"persistentvolumeclaims":"100"}}}`).Run() + }) + + allocations := make([]ensureResult, 0, 2) + for i := 0; i < 2; i++ { + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: fmt.Sprintf("quota-kube-%d", i)} + binding := fmt.Sprintf("quota-kube-%d-%d", i, time.Now().UnixNano()) + out, err := client.Ensure(ctx, binding, "quota-kube", owner, "ensure-"+binding) + if err != nil { + t.Fatalf("create quota admission candidate: %v", err) + } + allocations = append(allocations, ensureResult{owner: owner, binding: binding, out: out}) + } + deadline := time.Now().Add(2 * time.Minute) + var blocked ensureResult + for time.Now().Before(deadline) { + for _, allocation := range allocations { + status := readExecutionStatus(t, ctx, kubeconfig, allocation.out.Environment.ID) + if !status.Ready && status.ReadyReason == "PVCUnavailable" && resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+allocation.out.Environment.ID) == 0 { + blocked = allocation + } + } + if blocked.binding != "" { + break + } + time.Sleep(time.Second) + } + if blocked.binding == "" { + t.Fatal("Kubernetes PVC quota did not leave one allocation NotReady with reason PVCUnavailable") + } + if resourceCount(t, ctx, kubeconfig, "pvc") != pvcBaseline+1 { + t.Fatal("PVC quota admitted more than one additional workspace") + } + runKubectl(t, ctx, kubeconfig, "patch", "resourcequota/mecatl-execution", "-n", namespace, "--type=merge", "-p", `{"spec":{"hard":{"persistentvolumeclaims":"100"}}}`) + + all := append(allocations, *accepted) + for _, allocation := range all { + if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: allocation.out.Environment, Owner: allocation.owner, BindingID: allocation.binding, OperationID: "commit-" + allocation.binding}); err != nil { + t.Fatal(err) + } + waitReady(t, ctx, client, allocation.owner, allocation.binding, allocation.out.Environment) + retireSyntheticEnvironment(t, ctx, client, kubeconfig, allocation.owner, allocation.binding, allocation.out.Environment) + } +} + +func retireSyntheticEnvironment(t *testing.T, ctx context.Context, client *executionclient.Client, kubeconfig string, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) { + t.Helper() + deleteRef := executionenv.ReferenceRequest{Environment: ref, Owner: owner, BindingID: binding, OperationID: "delete-ref-" + binding} + if err := client.PrepareReferenceDelete(ctx, deleteRef); err != nil { + t.Fatal(err) + } + if err := client.ConfirmReferenceDelete(ctx, deleteRef); err != nil { + t.Fatal(err) + } + status := readExecutionStatus(t, ctx, kubeconfig, ref.ID) + request := executionenv.RetireEnvironmentRequest{Environment: ref, Owner: owner, ExpectedEpoch: status.Epoch, ExpectedPodUID: status.PodUID, ExpectedPVCUID: status.PVCUID, OperationID: "retire-" + binding} + if err := client.RetireEnvironment(ctx, request); err != nil { + t.Fatal(err) + } + retired := waitExecutionStatus(t, ctx, kubeconfig, ref.ID, func(s executionStatus) bool { return s.Retired && s.ExecutorTerminated }) + if err := client.DeleteRetiredEnvironment(ctx, ref, owner, retired.PVCUID, "delete-retired-"+binding); err != nil { + t.Fatal(err) + } + waitResourceAbsent(t, ctx, kubeconfig, "executionenvironment", ref.ID) +} + +func TestKindExecutionProductionPendingDeleteOutageRecovery(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, client, "pending-delete") + request := executionenv.ReferenceRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, OperationID: "pending-delete-outage"} + if err := client.PrepareReferenceDelete(ctx, request); err != nil { + t.Fatal(err) + } + if got := executionReferences(t, ctx, kubeconfig, attached.Environment.ID)[binding]; got != string(executionenv.ReferencePendingDelete) { + t.Fatalf("prepared reference state=%q", got) + } + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=240s") + intents, err := client.ListReferenceIntents(ctx, owner) + if err != nil { + t.Fatal(err) + } + found := false + for _, intent := range intents { + if intent.Environment == attached.Environment && intent.BindingID == binding && intent.State == executionenv.ReferencePendingDelete && intent.OperationID == request.OperationID { + found = true + } + } + if !found { + t.Fatal("pending-delete intent was not durable across provider outage") + } + intruderTLS := loadTLS(t, filepath.Join(state, "pki"), "intruder", "mecatl-execution.execution-qualification.svc.cluster.local") + forward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + intruder, err := executionclient.New(forward.addr, intruderTLS) + if err != nil { + t.Fatal(err) + } + defer intruder.Close() + private, err := intruder.ListReferenceIntents(ctx, owner) + if err != nil { + t.Fatal(err) + } + if len(private) != 0 { + t.Fatal("pending reference intent crossed the mTLS client privacy boundary") + } + if err := client.ConfirmReferenceDelete(ctx, request); err != nil { + t.Fatal(err) + } + waitReferenceSet(t, ctx, kubeconfig, attached.Environment.ID) + if resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+attached.Environment.ID) != 1 { + t.Fatal("pending-delete recovery removed retained storage") + } + status := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + retire := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: status.Epoch, ExpectedPodUID: status.PodUID, ExpectedPVCUID: status.PVCUID, OperationID: "retire-pending-delete"} + if err := client.RetireEnvironment(ctx, retire); err != nil { + t.Fatal(err) + } + retired := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Retired && s.ExecutorTerminated }) + if err := client.DeleteRetiredEnvironment(ctx, attached.Environment, owner, retired.PVCUID, "delete-pending-delete"); err != nil { + t.Fatal(err) + } +} + +func TestKindExecutionProductionCompatiblePrototypeMigration(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, client, "migration") + before := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + rc, release := acquireRun(t, ctx, client, owner, binding, attached, "migration-seed") + if _, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "migration-sentinel", Data: []byte("prototype-data\n")}); err != nil { + t.Fatal(err) + } + release() + + // The supported prototype profile is deliberately narrow: the stored CR, + // Pod, PVC, finalizer, ownership labels, and immutable security shape already + // match production; only schema markers and legacy reference encoding predate + // v2. Foreign or legacy-insecure Pods are never adopted by migration. + runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+attached.Environment.ID, "-n", namespace, "--type=merge", "-p", `{"spec":{"schemaVersion":1}}`) + runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+attached.Environment.ID, "-n", namespace, "--subresource=status", "--type=merge", "-p", `{"status":{"schemaVersion":1}}`) + waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return !s.Ready && s.ReadyReason == "IncompatibleSchema" }) + if _, err := client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); err == nil { + t.Fatal("prototype schema attached before explicit migration") + } + if err := client.MigrateEnvironment(ctx, attached.Environment, owner, 1, "foreign-pod-uid", before.PVCUID, "migration-wrong-uid"); err == nil { + t.Fatal("migration adopted a foreign Pod UID") + } + if err := client.MigrateEnvironment(ctx, attached.Environment, owner, 1, "", before.PVCUID, "migration-missing-uid"); err == nil { + t.Fatal("migration accepted a missing runtime UID") + } + if err := client.MigrateEnvironment(ctx, attached.Environment, owner, 2, before.PodUID, before.PVCUID, "migration-unknown-version"); err == nil { + t.Fatal("migration accepted an unrecognized source schema") + } + if err := client.MigrateEnvironment(ctx, attached.Environment, owner, 1, before.PodUID, before.PVCUID, "migration-compatible-v1"); err != nil { + t.Fatalf("migrate compatible prototype: %v", err) + } + after := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Ready && s.SpecSchema == 2 && s.StatusSchema == 2 }) + if after.PodUID != before.PodUID || after.PVCUID != before.PVCUID || after.Epoch != before.Epoch+1 { + t.Fatal("migration changed runtime identity or failed to advance only the fence epoch") + } + if got := executionReferences(t, ctx, kubeconfig, attached.Environment.ID); got[binding] != string(executionenv.ReferencePublished) { + t.Fatal("migration changed the published source reference") + } + reattached := waitReady(t, ctx, client, owner, binding, attached.Environment) + verify, verifyRelease := acquireRun(t, ctx, client, owner, binding, reattached, "migration-verify") + got, err := client.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "migration-sentinel"}) + if err != nil || string(got.Data) != "prototype-data\n" { + t.Fatal("migration lost prototype workspace data") + } + verifyRelease() +} + +func TestKindExecutionProductionClearForkLifecycle(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + provider, _ := productionClient(t, ctx, state, kubeconfig) + + tokenForward := portForward(t, ctx, kubeconfig, "service/oidc-issuer", 8443) + alice := fixtureToken(t, ctx, tokenForward.addr, filepath.Join(state, "pki"), "alice") + bob := fixtureToken(t, ctx, tokenForward.addr, filepath.Join(state, "pki"), "bob") + tokenForward.stop() + agent := portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + defer agent.stop() + + source := createSession(t, ctx, agent.addr, alice) + assertMockJourney(t, prompt(t, ctx, agent.addr, source, alice, "run the scripted remote qualification")) + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "alice"} + allocation, err := provider.Ensure(ctx, source, "go", owner, "clear-fork-lookup-"+source) + if err != nil { + t.Fatalf("resolve source placement: %v", err) + } + attached := waitReady(t, ctx, provider, owner, source, allocation.Environment) + before := readExecutionStatus(t, ctx, kubeconfig, allocation.Environment.ID) + + forkID := successorSession(t, ctx, agent.addr, source, alice, "fork") + clearID := successorSession(t, ctx, agent.addr, source, alice, "clear") + if forkID == source || clearID == source || forkID == clearID { + t.Fatal("Clear/Fork did not publish distinct session identities") + } + if got := getSession(t, ctx, agent.addr, forkID, bob); got != http.StatusNotFound { + t.Fatalf("unknown owner observed fork: status=%d", got) + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got < 1 { + t.Fatal("successor publication lost its source environment") + } + waitReferenceSet(t, ctx, kubeconfig, allocation.Environment.ID, source, forkID, clearID) + after := readExecutionStatus(t, ctx, kubeconfig, allocation.Environment.ID) + if after.PVCUID != before.PVCUID || after.PodUID != before.PodUID || after.Epoch != before.Epoch { + t.Fatal("Clear/Fork changed the exact remote execution identity") + } + for _, binding := range []string{forkID, clearID} { + if _, err := provider.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: allocation.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); err != nil { + t.Fatalf("successor %s did not attach to source environment: %v", binding, err) + } + } + + claim, err := provider.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: source, RunID: "source-blocker", OperationID: "source-blocker-acquire", TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + if _, err := provider.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: clearID, RunID: "clear-overlap", OperationID: "clear-overlap-acquire", TTL: time.Minute}); err == nil || !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("second session admitted an overlapping run: %v", err) + } + if err := provider.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: source, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, OperationID: "source-blocker-release"}); err != nil { + t.Fatal(err) + } + clearAttached := waitReady(t, ctx, provider, owner, clearID, allocation.Environment) + _, release := acquireRun(t, ctx, provider, owner, clearID, clearAttached, "clear-after-release") + release() + + for _, id := range []string{forkID, clearID, source} { + status, body := request(t, ctx, http.MethodPost, "http://"+agent.addr+"/v1/sessions/"+id+"/delete", alice, nil) + if status != http.StatusNoContent { + t.Fatalf("delete session %s status=%d body=%s", id, status, body) + } + } + waitReferenceSet(t, ctx, kubeconfig, allocation.Environment.ID) + final := readExecutionStatus(t, ctx, kubeconfig, allocation.Environment.ID) + if final.PVCUID != before.PVCUID || resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+allocation.Environment.ID) != 1 { + t.Fatal("session deletion removed or replaced the retained workspace") + } +} + +func successorSession(t *testing.T, ctx context.Context, addr, source, token, operation string) string { + t.Helper() + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+source+"/"+operation, token, []byte(`{}`)) + if status != http.StatusCreated { + t.Fatalf("%s session status=%d body=%s", operation, status, body) + } + var out struct { + SessionID string `json:"session_id"` + } + if json.Unmarshal(body, &out) != nil || out.SessionID == "" { + t.Fatalf("%s session returned no identity", operation) + } + return out.SessionID +} + +func waitReferenceSet(t *testing.T, ctx context.Context, kubeconfig, environmentID string, want ...string) { + t.Helper() + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + got := executionReferences(t, ctx, kubeconfig, environmentID) + if len(got) == len(want) { + matched := true + for _, binding := range want { + if got[binding] != string(executionenv.ReferencePublished) { + matched = false + } + } + if matched { + return + } + } + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("environment references did not converge to published set %v", want) +} + +func executionReferences(t *testing.T, ctx context.Context, kubeconfig, environmentID string) map[string]string { + t.Helper() + var object struct { + Status struct { + References []struct { + BindingID string `json:"bindingID"` + State string `json:"state"` + } `json:"references"` + } `json:"status"` + } + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "executionenvironment", environmentID, "-n", namespace, "-o", "json"), &object); err != nil { + t.Fatal(err) + } + out := make(map[string]string, len(object.Status.References)) + for _, ref := range object.Status.References { + out[ref.BindingID] = ref.State + } + return out +} + +func TestKindExecutionProductionFailureArtifactBoundary(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + artifact := runKubectl(t, ctx, kubeconfig, "get", "events", "-n", namespace, "-o", "custom-columns=OBJECT:.involvedObject.name,REASON:.reason", "--no-headers") + if len(artifact) > 1<<20 { + t.Fatal("sanitized failure artifact exceeds one MiB") + } + for _, forbidden := range []string{"kubeconfig", "BEGIN PRIVATE KEY", "Bearer ", "grant-key", "ownerSubject", "clientHash", "Secret/data"} { + if strings.Contains(string(artifact), forbidden) { + t.Fatalf("sanitized failure artifact contains forbidden class %q", forbidden) + } + } + if err := os.WriteFile(filepath.Join(state, "production-failure-artifact.txt"), artifact, 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/e2e/k8s_execution/qualification_test.go b/e2e/k8s_execution/qualification_test.go index 3898f2a205..24b698f16c 100644 --- a/e2e/k8s_execution/qualification_test.go +++ b/e2e/k8s_execution/qualification_test.go @@ -54,18 +54,25 @@ func TestKindExecutionQualification(t *testing.T) { owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "alice"} binding := fmt.Sprintf("direct-binding-%d", time.Now().UnixNano()) - first, err := providerClient.Ensure(ctx, binding, "go", owner) + operationID := fmt.Sprintf("direct-ensure-%d", time.Now().UnixNano()) + first, err := providerClient.Ensure(ctx, binding, "go", owner, operationID) if err != nil { t.Fatalf("ensure: %v", err) } + if err := providerClient.CommitReference(ctx, executionenv.ReferenceRequest{Environment: first.Environment, Owner: owner, BindingID: binding, OperationID: "commit-" + operationID}); err != nil { + t.Fatalf("commit reference: %v", err) + } if first.Environment.ID == "" || first.Environment.Revision == "" { t.Fatal("ensure returned an empty exact environment reference") } ready := waitReady(t, ctx, providerClient, owner, binding, first.Environment) + if os.Getenv("MECATL_EXECUTION_QUAL_PROFILE") == "production" { + assertProductionExecutorPod(t, ctx, kubeconfig, first.Environment.ID) + } if ready.Environment != first.Environment { t.Fatalf("ready reference drifted: got %+v want %+v", ready.Environment, first.Environment) } - second, err := providerClient.Ensure(ctx, binding, "go", owner) + second, err := providerClient.Ensure(ctx, binding, "go", owner, operationID) if err != nil { t.Fatalf("repeat ensure: %v", err) } @@ -120,19 +127,35 @@ func TestKindExecutionQualification(t *testing.T) { bob := fixtureToken(t, ctx, tokenForward.addr, pki, "bob") tokenForward.stop() agentForward := portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + beforeNoFS := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev") + status, noFSBody := request(t, ctx, http.MethodPost, "http://"+agentForward.addr+"/v1/sessions", alice, []byte(`{"profile":"no-fs","mode":"default"}`)) + if status != http.StatusCreated { + t.Fatalf("create no-fs session status=%d body=%s", status, noFSBody) + } + if afterNoFS := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); afterNoFS != beforeNoFS { + t.Fatalf("no-fs session allocated execution resources: before=%d after=%d", beforeNoFS, afterNoFS) + } sessionID := createSession(t, ctx, agentForward.addr, alice) body := prompt(t, ctx, agentForward.addr, sessionID, alice, "run the scripted remote qualification") if err := os.WriteFile(filepath.Join(state, "mock-journey.sse"), body, 0o600); err != nil { t.Fatal(err) } - for _, marker := range []string{"write-proof", "read-before-edit", "edit-proof", "copy-proof", "move-proof", "write-mod", "write-test", "shell-go-test", "remove-moved", "read-final", "moved.txt", "proof.txt:1:beta", "REMOTE_EXECUTION_QUALIFICATION_COMPLETE", "ok"} { - if !bytes.Contains(body, []byte(marker)) { - t.Fatalf("SSE result omitted %q (%s)", marker, mockSSEStatus(body)) - } + assertMockJourney(t, body) + lookup, err := providerClient.Ensure(ctx, sessionID, "go", owner, "independent-lookup-"+sessionID) + if err != nil { + t.Fatalf("resolve harness environment: %v", err) } - if bytes.Contains(body, []byte(`"is_error":true`)) { - t.Fatal("scripted remote tool journey contained an error result") + attached := waitReady(t, ctx, providerClient, owner, sessionID, lookup.Environment) + rc, releaseVerification := acquireRun(t, ctx, providerClient, owner, sessionID, attached, fmt.Sprintf("independent-verify-%d", time.Now().UnixNano())) + proof, err := providerClient.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: "proof.txt"}) + if err != nil || string(proof.Data) != "beta\n" { + t.Fatalf("independent file proof failed: err=%v", err) + } + verification, err := providerClient.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go test ./...", TimeoutMillis: 120000}) + if err != nil || verification.State != executionenv.CommandSucceeded || verification.Result.ExitCode != 0 { + t.Fatalf("independent go test proof failed: err=%v state=%s", err, verification.State) } + releaseVerification() if status := getSession(t, ctx, agentForward.addr, sessionID, bob); status != http.StatusNotFound { t.Fatalf("different OIDC owner read status = %d, want 404", status) } @@ -159,6 +182,73 @@ func TestKindExecutionQualification(t *testing.T) { } } +func assertMockJourney(t *testing.T, body []byte) { + t.Helper() + expected := map[string]struct { + name string + key string + want string + }{ + "write-proof": {"Write", "path", "proof.txt"}, "read-before-edit": {"Read", "path", "proof.txt"}, + "edit-proof": {"Edit", "path", "proof.txt"}, "copy-proof": {"Copy", "destination", "copy.txt"}, + "move-proof": {"Move", "destination", "moved.txt"}, "glob-proof": {"Glob", "pattern", "*.txt"}, + "grep-proof": {"Grep", "pattern", "beta"}, "write-mod": {"Write", "path", "go.mod"}, + "write-test": {"Write", "path", "remote_test.go"}, "shell-go-test": {"Shell", "command", "go test ./..."}, + "remove-moved": {"Remove", "path", "moved.txt"}, "read-final": {"Read", "path", "proof.txt"}, + } + calls := map[string]string{} + results := map[string]*mecatlv1.ToolResult{} + stop := "" + for _, line := range bytes.Split(body, []byte("\n")) { + if !bytes.HasPrefix(line, []byte("data: ")) { + continue + } + var ev mecatlv1.Event + if err := json.Unmarshal(bytes.TrimPrefix(line, []byte("data: ")), &ev); err != nil { + t.Fatalf("decode harness SSE event: %v", err) + } + if ev.ToolCall != nil { + want, ok := expected[ev.ToolCall.Id] + if !ok { + t.Fatalf("unexpected tool call id %q", ev.ToolCall.Id) + } + if ev.ToolCall.Name != want.name { + t.Fatalf("tool call %s name=%s want=%s", ev.ToolCall.Id, ev.ToolCall.Name, want.name) + } + var args map[string]json.RawMessage + var got string + if json.Unmarshal([]byte(ev.ToolCall.Args), &args) != nil || json.Unmarshal(args[want.key], &got) != nil || got != want.want { + t.Fatalf("tool call %s did not carry expected structured argument", ev.ToolCall.Id) + } + calls[ev.ToolCall.Id] = ev.ToolCall.Name + } + if ev.ToolResult != nil { + results[ev.ToolResult.CallId] = ev.ToolResult + } + if ev.Result != nil { + stop = ev.Result.Stop + } + } + for id := range expected { + if calls[id] == "" { + t.Fatalf("missing structured tool call %s (%s)", id, mockSSEStatus(body)) + } + result := results[id] + if result == nil || result.IsError { + t.Fatalf("tool call %s has no correlated successful result", id) + } + } + if result := results["shell-go-test"]; !strings.Contains(result.Content, "[exit code: 0]") { + t.Fatal("model-issued exact go test did not return exit code 0") + } + if result := results["read-final"]; !strings.Contains(result.Content, "beta") { + t.Fatal("final Read result did not contain persisted proof") + } + if stop != "end_turn" { + t.Fatalf("mock harness stop=%q, want end_turn", stop) + } +} + func mockSSEStatus(body []byte) string { var names []string errorClasses := map[string]int{} @@ -194,6 +284,23 @@ func classifyMockToolError(content string) string { return "other_redacted" } +func acquireRun(t *testing.T, ctx context.Context, c *executionclient.Client, owner executionenv.Owner, binding string, attached executionenv.AttachEnvironmentResponse, runID string) (executionenv.RequestContext, func()) { + t.Helper() + operationID := "acquire-" + runID + claim, err := c.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: runID, OperationID: operationID, TTL: time.Minute}) + if err != nil { + t.Fatalf("acquire run: %v", err) + } + rc := executionenv.RequestContext{Environment: claim.Environment, Owner: owner, BindingID: binding, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant} + release := func() { + err := c.ReleaseRun(context.WithoutCancel(ctx), executionenv.RunClaimRequest{Environment: claim.Environment, Owner: owner, BindingID: binding, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, OperationID: "release-" + runID}) + if err != nil { + t.Errorf("release run: %v", err) + } + } + return rc, release +} + func waitReady(t *testing.T, ctx context.Context, c *executionclient.Client, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) executionenv.AttachEnvironmentResponse { t.Helper() deadline := time.Now().Add(3 * time.Minute) diff --git a/internal/adapter/executionclient/client.go b/internal/adapter/executionclient/client.go index be625a258e..e697f64054 100644 --- a/internal/adapter/executionclient/client.go +++ b/internal/adapter/executionclient/client.go @@ -120,11 +120,7 @@ func newOperationID() (string, error) { } // Ensure idempotently allocates or resolves the environment for a binding. -func (c *Client) Ensure(ctx context.Context, binding, profile string, owner executionenv.Owner, operationIDs ...string) (executionenv.EnsureEnvironmentResponse, error) { - operationID := "" - if len(operationIDs) == 1 { - operationID = operationIDs[0] - } +func (c *Client) Ensure(ctx context.Context, binding, profile string, owner executionenv.Owner, operationID string) (executionenv.EnsureEnvironmentResponse, error) { v, err := c.rpc.EnsureEnvironment(ctx, &executionv1.EnsureEnvironmentRequest{BindingId: binding, Profile: profile, Owner: ownerToProto(owner), OperationId: operationID}) if err != nil { return executionenv.EnsureEnvironmentResponse{}, decodeError(ctx, err) @@ -266,12 +262,45 @@ func (c *Client) ReleaseReference(ctx context.Context, req executionenv.Referenc return decodeError(ctx, err) } +// ReplaceExecutor requests UID-checked administrative executor replacement. +func (c *Client) ReplaceExecutor(ctx context.Context, req executionenv.RetireEnvironmentRequest) error { + _, err := c.rpc.ReplaceExecutor(ctx, &executionv1.ReplaceExecutorRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), ExpectedExecutionEpoch: req.ExpectedEpoch, ExpectedPodUid: req.ExpectedPodUID, ExpectedPvcUid: req.ExpectedPVCUID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} + // RetireEnvironment requests administrative retirement. func (c *Client) RetireEnvironment(ctx context.Context, req executionenv.RetireEnvironmentRequest) error { _, err := c.rpc.RetireEnvironment(ctx, &executionv1.RetireEnvironmentRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), ExpectedExecutionEpoch: req.ExpectedEpoch, ExpectedPodUid: req.ExpectedPodUID, ExpectedPvcUid: req.ExpectedPVCUID, OperationId: req.OperationID}) return decodeError(ctx, err) } +// DeleteRetiredEnvironment deletes retained storage after retirement proof. +func (c *Client) DeleteRetiredEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, owner executionenv.Owner, expectedPVCUID, operationID string) error { + _, err := c.rpc.DeleteRetiredEnvironment(ctx, &executionv1.DeleteRetiredEnvironmentRequest{Environment: refToProto(ref), Owner: ownerToProto(owner), ExpectedPvcUid: expectedPVCUID, OperationId: operationID}) + return decodeError(ctx, err) +} + +// RevokeEnvironment atomically fences grants and replays a matching operation receipt. +func (c *Client) RevokeEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, owner executionenv.Owner, expectedGeneration uint64, operationID string) (uint64, error) { + out, err := c.rpc.RevokeEnvironment(ctx, &executionv1.RevokeEnvironmentRequest{Environment: refToProto(ref), Owner: ownerToProto(owner), ExpectedGrantGeneration: expectedGeneration, OperationId: operationID}) + if err != nil { + return 0, decodeError(ctx, err) + } + return out.GetGrantGeneration(), nil +} + +// MigrateEnvironment upgrades one explicitly identified legacy schema. +func (c *Client) MigrateEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, owner executionenv.Owner, expectedSchema uint32, podUID, pvcUID, operationID string) error { + _, err := c.rpc.MigrateEnvironment(ctx, &executionv1.MigrateEnvironmentRequest{Environment: refToProto(ref), Owner: ownerToProto(owner), ExpectedSchemaVersion: &expectedSchema, ExpectedPodUid: podUID, ExpectedPvcUid: pvcUID, OperationId: operationID}) + return decodeError(ctx, err) +} + +// RecoverEnvironment clears a fence only after the provider verifies exact terminal evidence. +func (c *Client) RecoverEnvironment(ctx context.Context, req executionenv.RetireEnvironmentRequest) error { + _, err := c.rpc.RecoverEnvironment(ctx, &executionv1.RecoverEnvironmentRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), ExpectedExecutionEpoch: req.ExpectedEpoch, ExpectedPodUid: req.ExpectedPodUID, ExpectedPvcUid: req.ExpectedPVCUID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} + func decodeError(ctx context.Context, err error) error { if err == nil { return nil @@ -312,8 +341,10 @@ func grpcCodeForError(code executionenv.ErrorCode) codes.Code { return codes.AlreadyExists case executionenv.CodeConflict, executionenv.CodeVersionMismatch: return codes.Aborted - case executionenv.CodeNotReady, executionenv.CodeFenceUnknown: + case executionenv.CodeNotReady: return codes.Unavailable + case executionenv.CodeFenceUnknown: + return codes.FailedPrecondition case executionenv.CodeResourceExhausted: return codes.ResourceExhausted default: @@ -486,9 +517,22 @@ func (p *Provider) PrepareReferenceDelete(ctx context.Context, req server.Placem if err != nil { return nil, err } - op, err := newOperationID() + op := "" + intents, err := p.client.ListReferenceIntents(ctx, owner) if err != nil { - return nil, server.ErrPlacementUnavailable + return nil, mapPlacementError(err) + } + for _, intent := range intents { + if intent.Environment.ID == req.Ref.ID && intent.Environment.Revision == req.Ref.Revision && intent.BindingID == string(req.SourceBindingID) && intent.State == executionenv.ReferencePendingDelete { + op = intent.OperationID + break + } + } + if op == "" { + op, err = newOperationID() + if err != nil { + return nil, server.ErrPlacementUnavailable + } } mutation := executionenv.ReferenceRequest{Environment: executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision}, Owner: owner, BindingID: string(req.SourceBindingID), OperationID: op} if err := p.client.PrepareReferenceDelete(ctx, mutation); err != nil { @@ -662,7 +706,9 @@ func (p *Provider) AcquireRun(ctx context.Context, req server.ExecutionRunReques return nil, server.ErrPlacementUnavailable } claimReq.ClaimID, claimReq.Epoch, claimReq.GrantGeneration = claim.ClaimID, claim.Epoch, claim.GrantGeneration - return &runHandle{provider: p, owner: owner, claim: claimReq, credentials: credentials, environment: env}, nil + handle := &runHandle{provider: p, owner: owner, claim: claimReq, credentials: credentials, environment: env} + credentials.renewClaim = handle.Renew + return handle, nil } // ValidatePlacement performs side-effect-free profile preflight. @@ -815,18 +861,45 @@ func mapPlacementError(err error) error { } type grantContext struct { - mu sync.Mutex - client *Client - context executionenv.RequestContext - expiresAt time.Time + mu sync.Mutex + client *Client + context executionenv.RequestContext + expiresAt time.Time + renewClaim func(context.Context) error } -func (g *grantContext) current(ctx context.Context) (executionenv.RequestContext, error) { - return g.renew(ctx, false) +func (g *grantContext) current(_ context.Context) (executionenv.RequestContext, error) { + g.mu.Lock() + defer g.mu.Unlock() + if g.context.Grant == "" || !time.Now().Before(g.expiresAt) { + return executionenv.RequestContext{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "run grant unavailable", Retryable: true} + } + return g.context, nil } func (g *grantContext) refresh(ctx context.Context) (executionenv.RequestContext, error) { - return g.renew(ctx, true) + g.mu.Lock() + before := g.context.Grant + if before == "" || !time.Now().Before(g.expiresAt) { + g.mu.Unlock() + return executionenv.RequestContext{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "active run grant expired; retry the run", Retryable: true} + } + renew := g.renewClaim + g.mu.Unlock() + g.mu.Lock() + if g.context.Grant != before && time.Now().Before(g.expiresAt) { + out := g.context + g.mu.Unlock() + return out, nil + } + g.mu.Unlock() + if renew == nil { + return executionenv.RequestContext{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "active run grant cannot be refreshed", Retryable: true} + } + if err := renew(ctx); err != nil { + return executionenv.RequestContext{}, err + } + return g.current(ctx) } func (g *grantContext) replace(claim executionenv.RunClaim) { @@ -836,15 +909,6 @@ func (g *grantContext) replace(claim executionenv.RunClaim) { g.expiresAt = claim.ExpiresAt } -func (g *grantContext) renew(_ context.Context, _ bool) (executionenv.RequestContext, error) { - g.mu.Lock() - defer g.mu.Unlock() - if g.context.Grant == "" || !time.Now().Before(g.expiresAt) { - return executionenv.RequestContext{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "run grant unavailable"} - } - return g.context, nil -} - // workspace is bound to one provider-issued environment grant. type workspace struct { credentials *grantContext diff --git a/internal/adapter/executioncontroller/admin_lifecycle.go b/internal/adapter/executioncontroller/admin_lifecycle.go index c394ea486f..b551e4b975 100644 --- a/internal/adapter/executioncontroller/admin_lifecycle.go +++ b/internal/adapter/executioncontroller/admin_lifecycle.go @@ -48,7 +48,7 @@ func (s *Store) startLifecycle(ctx context.Context, q adminLifecycleRequest, kin } return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another lifecycle operation is active"} } - if textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy || !conditionTrue(o, "Ready") { + if lifecycleAdmissionBlocked(o, q) { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not healthy and idle"} } refs, err := referenceRecords(o) @@ -64,6 +64,19 @@ func (s *Store) startLifecycle(ctx context.Context, q adminLifecycleRequest, kin }) } +func lifecycleAdmissionBlocked(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + return textNested(o.Object, "status", "activeRun", "claimID") != "" || + textNested(o.Object, "status", "activeOperation", "id") != "" || + textNested(o.Object, "status", "fenceState") != fenceHealthy || + !conditionTrue(o, "Ready") && !exactTerminationProofMatches(o, q) +} + +func exactTerminationProofMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + return textNested(o.Object, "status", "terminationProof", "podUID") == q.ExpectedPodUID && + textNested(o.Object, "status", "terminationProof", "pvcUID") == q.ExpectedPVCUID && + intNested(o.Object, "status", "terminationProof", "epoch") == int64(q.ExpectedEpoch) //nolint:gosec // q.ExpectedEpoch is bounded above. +} + func exactAdminSubject(o *unstructured.Unstructured, q adminLifecycleRequest) error { if textNested(o.Object, "spec", "ownerHash") != q.OwnerHash || textNested(o.Object, "spec", "clientHash") != hashText(q.Client) || textNested(o.Object, "spec", "revision") != q.Environment.Revision { return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} @@ -115,19 +128,53 @@ func (s *Store) RecoverEnvironment(ctx context.Context, q adminLifecycleRequest) } func terminationProof(q adminLifecycleRequest, pod *corev1.Pod) map[string]any { - return map[string]any{"operationID": q.OperationID, "podUID": q.ExpectedPodUID, "pvcUID": q.ExpectedPVCUID, "epoch": int64(q.ExpectedEpoch), "podPhase": string(pod.Status.Phase), "observedAt": time.Now().UTC().Format(time.RFC3339Nano)} //nolint:gosec // epoch is validated by caller. + return map[string]any{operationIDField: q.OperationID, "podUID": q.ExpectedPodUID, "pvcUID": q.ExpectedPVCUID, "epoch": int64(q.ExpectedEpoch), "podPhase": string(pod.Status.Phase), "observedAt": time.Now().UTC().Format(time.RFC3339Nano)} //nolint:gosec // epoch is validated by caller. } func podTerminal(pod *corev1.Pod) bool { - if pod == nil || pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed || len(pod.Status.ContainerStatuses) != len(pod.Spec.Containers) || len(pod.Spec.Containers) == 0 { + if pod == nil || pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed || len(pod.Spec.Containers) == 0 { + return false + } + if !allDeclaredContainersTerminated(pod.Spec.InitContainers, pod.Status.InitContainerStatuses) || !allDeclaredContainersTerminated(pod.Spec.Containers, pod.Status.ContainerStatuses) { + return false + } + return allDeclaredContainersTerminated(pod.Spec.EphemeralContainers, pod.Status.EphemeralContainerStatuses) +} + +func allDeclaredContainersTerminated[T interface { + corev1.Container | corev1.EphemeralContainer +}](declared []T, statuses []corev1.ContainerStatus) bool { + if len(statuses) != len(declared) { return false } - for _, status := range pod.Status.ContainerStatuses { - if status.State.Terminated == nil { + names := make(map[string]struct{}, len(declared)) + for _, container := range declared { + var name string + switch c := any(container).(type) { + case corev1.Container: + name = c.Name + case corev1.EphemeralContainer: + name = c.Name + } + if name == "" { + return false + } + names[name] = struct{}{} + } + if len(names) != len(declared) { + return false + } + seen := make(map[string]struct{}, len(statuses)) + for _, status := range statuses { + if _, declared := names[status.Name]; !declared || status.State.Terminated == nil { + return false + } + if _, duplicate := seen[status.Name]; duplicate { return false } + seen[status.Name] = struct{}{} } - return true + return len(seen) == len(names) } // DeleteRetiredEnvironment starts exact deletion of an explicitly retained workspace. @@ -260,7 +307,7 @@ func migrateReferenceRecords(o *unstructured.Unstructured, operationID string, n out = append(out, referenceRecord{BindingID: item, State: executionenv.ReferencePublished, OperationID: "migration-" + operationID, CreatedAt: now}) case map[string]any: created, parseErr := time.Parse(time.RFC3339Nano, text(item, "createdAt")) - r := referenceRecord{BindingID: text(item, "bindingID"), State: executionenv.ReferenceState(text(item, "state")), OperationID: text(item, "operationID"), SourceBindingID: text(item, "sourceBindingID"), CreatedAt: created} + r := referenceRecord{BindingID: text(item, "bindingID"), State: executionenv.ReferenceState(text(item, "state")), OperationID: text(item, operationIDField), SourceBindingID: text(item, "sourceBindingID"), CreatedAt: created} if parseErr != nil || r.BindingID == "" || r.OperationID == "" || r.State != executionenv.ReferencePendingCreate && r.State != executionenv.ReferencePublished && r.State != executionenv.ReferencePendingDelete { return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} } diff --git a/internal/adapter/executioncontroller/controller.go b/internal/adapter/executioncontroller/controller.go index c4750b19d9..2bee571f6a 100644 --- a/internal/adapter/executioncontroller/controller.go +++ b/internal/adapter/executioncontroller/controller.go @@ -3,6 +3,7 @@ package executioncontroller import ( "context" "errors" + "fmt" "reflect" "strings" "sync" @@ -28,6 +29,7 @@ const ( environmentFinalizer = "execution.mecatl.dev/retain-workspace" executorFinalizer = "execution.mecatl.dev/verify-termination" fenceHealthy = "Healthy" + statusField = "status" ) // Reconciler maintains executor Pods and retained workspace PVCs. @@ -52,6 +54,10 @@ func NewReconciler(d dynamic.Interface, k kubernetes.Interface, namespace string // Startup never mutates operation ownership: another replica may still be its live holder. func (r *Reconciler) Initialize(ctx context.Context) error { r.initOnce.Do(func() { + if err := r.preflightProfiles(ctx); err != nil { + r.initErr = err + return + } factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(r.dynamic, 0, r.namespace, nil) environments := factory.ForResource(ExecutionEnvironmentGVR).Informer() _, err := environments.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: r.enqueue, UpdateFunc: r.enqueueUpdate}) @@ -82,9 +88,38 @@ func (r *Reconciler) Initialize(ctx context.Context) error { return r.initErr } -// Ready reports whether startup fencing and informer synchronization completed. +func (r *Reconciler) preflightProfiles(ctx context.Context) error { + runtimeClasses, storageClasses := r.profiles.clusterResources() + for _, name := range runtimeClasses { + if _, err := r.kube.NodeV1().RuntimeClasses().Get(ctx, name, metav1.GetOptions{}); err != nil { + return fmt.Errorf("profile preflight: RuntimeClass %q unavailable: %w", name, err) + } + } + for _, name := range storageClasses { + if _, err := r.kube.StorageV1().StorageClasses().Get(ctx, name, metav1.GetOptions{}); err != nil { + return fmt.Errorf("profile preflight: StorageClass %q unavailable: %w", name, err) + } + } + return nil +} + +// Ready reports whether startup fencing, profile preflight, and informer synchronization completed. func (r *Reconciler) Ready() bool { return r.ready.Load() } +// ReadinessReason returns a bounded operator-facing reason class. +func (r *Reconciler) ReadinessReason() string { + if r.ready.Load() { + return "ready" + } + if r.initErr != nil && strings.HasPrefix(r.initErr.Error(), "profile preflight:") { + return "profile-resource-unavailable" + } + if r.initErr != nil { + return "controller-cache-unavailable" + } + return "controller-starting" +} + // Run initializes the reconciler and blocks until shutdown. func (r *Reconciler) Run(ctx context.Context) error { if err := r.Initialize(ctx); err != nil { @@ -115,7 +150,7 @@ func reconcileInputsEqual(oldEnv, newEnv *unstructured.Unstructured) bool { !reflect.DeepEqual(oldEnv.GetFinalizers(), newEnv.GetFinalizers()) { return false } - for _, path := range [][]string{{"spec"}, {"status", "activeOperation"}, {"status", "references"}, {"status", "fenceState"}} { + for _, path := range [][]string{{"spec"}, {statusField, "activeOperation"}, {statusField, "lifecycleOperation"}, {statusField, "migrationOperation"}, {statusField, "references"}, {statusField, "fenceState"}} { oldValue, _, _ := unstructured.NestedFieldNoCopy(oldEnv.Object, path...) newValue, _, _ := unstructured.NestedFieldNoCopy(newEnv.Object, path...) if !reflect.DeepEqual(oldValue, newValue) { @@ -197,7 +232,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, name string) error { return r.reconcileLifecycle(ctx, env) } if textNested(env.Object, "spec", "desired") == "Retiring" { - return r.reconcileRetiring(ctx, env) + if conditionTrue(env, "Retired") { + return nil + } + return r.setCondition(ctx, env, "Ready", false, "UnsupportedRetirementRequest", "retirement requires the exact administrator lifecycle operation") } pvcName := resourceName("workspace", name) podName := resourceName("executor", name) @@ -222,10 +260,7 @@ func (r *Reconciler) ensurePVC(ctx context.Context, env *unstructured.Unstructur pvcs := r.kube.CoreV1().PersistentVolumeClaims(r.namespace) cur, err := pvcs.Get(ctx, name, metav1.GetOptions{}) if err == nil { - if cur.Labels["execution.mecatl.dev/environment"] != env.GetName() || (textNested(env.Object, "status", "pvc", "uid") != "" && textNested(env.Object, "status", "pvc", "uid") != string(cur.UID)) { - return nil, errors.New("PVC ownership identity mismatch") - } - return cur, nil + return cur, validatePVC(env, p, cur) } if !apierrors.IsNotFound(err) { return nil, err @@ -235,20 +270,20 @@ func (r *Reconciler) ensurePVC(ctx context.Context, env *unstructured.Unstructur } qty := p.StorageSize mode := corev1.PersistentVolumeFilesystem - created, err := pvcs.Create(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/revision": textNested(env.Object, "spec", "revision")}}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, StorageClassName: &p.Spec.StorageClass, VolumeMode: &mode, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: qty}}}}, metav1.CreateOptions{}) + created, err := pvcs.Create(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/revision": textNested(env.Object, "spec", "revision"), "execution.mecatl.dev/allocation-uid": string(env.GetUID())}}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, StorageClassName: &p.Spec.StorageClass, VolumeMode: &mode, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: qty}}}}, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { - return pvcs.Get(ctx, name, metav1.GetOptions{}) + created, err = pvcs.Get(ctx, name, metav1.GetOptions{}) + } + if err != nil { + return nil, err } - return created, err + return created, validatePVC(env, p, created) } func (r *Reconciler) ensurePod(ctx context.Context, env *unstructured.Unstructured, p resolvedProfile, name, pvc string) (*corev1.Pod, error) { pods := r.kube.CoreV1().Pods(r.namespace) cur, err := pods.Get(ctx, name, metav1.GetOptions{}) if err == nil { - if cur.Labels["execution.mecatl.dev/environment"] != env.GetName() || (textNested(env.Object, "status", "pod", "uid") != "" && textNested(env.Object, "status", "pod", "uid") != string(cur.UID)) { - return nil, errors.New("pod ownership identity mismatch") - } - return cur, nil + return cur, validatePod(env, p, pvc, cur) } if !apierrors.IsNotFound(err) { return nil, err @@ -271,66 +306,57 @@ func (r *Reconciler) ensurePod(ctx context.Context, env *unstructured.Unstructur pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/profile": hashText(textNested(env.Object, "spec", "profile"))[:16]}, Finalizers: []string{executorFinalizer}, OwnerReferences: []metav1.OwnerReference{{APIVersion: env.GetAPIVersion(), Kind: env.GetKind(), Name: env.GetName(), UID: env.GetUID(), Controller: &nonroot}}}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noPriv, RuntimeClassName: &p.Spec.RuntimeClassName, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &nonroot, RunAsUser: &uid, RunAsGroup: &uid, FSGroup: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "executor", Image: p.Spec.Image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &noPriv, ReadOnlyRootFilesystem: &ro, RunAsNonRoot: &nonroot, RunAsUser: &uid, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: cpuReq, corev1.ResourceMemory: memReq, corev1.ResourceEphemeralStorage: ephemeralReq}, Limits: corev1.ResourceList{corev1.ResourceCPU: cpuLim, corev1.ResourceMemory: memLim, corev1.ResourceEphemeralStorage: ephemeralLim}}, VolumeMounts: []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}}}, Volumes: []corev1.Volume{{Name: "workspace", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc}}}, {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &tmpLim}}}}}} created, err := pods.Create(ctx, pod, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { - return pods.Get(ctx, name, metav1.GetOptions{}) + created, err = pods.Get(ctx, name, metav1.GetOptions{}) } - return created, err + if err != nil { + return nil, err + } + return created, validatePod(env, p, pvc, created) } -func (r *Reconciler) reconcileRetiring(ctx context.Context, env *unstructured.Unstructured) error { - if textNested(env.Object, "status", "activeOperation", "id") != "" || textNested(env.Object, "status", "fenceState") != fenceHealthy { - return r.setCondition(ctx, env, "Retired", false, "NotQuiescent", "retirement waits for proven quiescence") + +func validatePVC(env *unstructured.Unstructured, p resolvedProfile, pvc *corev1.PersistentVolumeClaim) error { + storedUID := textNested(env.Object, "status", "pvc", "uid") + qty, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + if pvc.Labels["execution.mecatl.dev/environment"] != env.GetName() || pvc.Labels["execution.mecatl.dev/revision"] != textNested(env.Object, "spec", "revision") || pvc.Labels["execution.mecatl.dev/allocation-uid"] != string(env.GetUID()) || len(pvc.OwnerReferences) != 0 || (storedUID != "" && storedUID != string(pvc.UID)) || pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != p.Spec.StorageClass || len(pvc.Spec.AccessModes) != 1 || pvc.Spec.AccessModes[0] != corev1.ReadWriteOnce || pvc.Spec.VolumeMode == nil || *pvc.Spec.VolumeMode != corev1.PersistentVolumeFilesystem || !ok || qty.Cmp(p.StorageSize) != 0 { + return errors.New("PVC ownership or immutable specification mismatch") } - refs, refsErr := referenceRecords(env) - if refsErr != nil || len(refs) > 0 || textNested(env.Object, "status", "activeRun", "claimID") != "" { - return r.setCondition(ctx, env, "Retired", false, "Referenced", "retirement refused while references remain") + return nil +} + +func validatePod(env *unstructured.Unstructured, p resolvedProfile, pvcName string, pod *corev1.Pod) error { //nolint:gocyclo // Every security-sensitive immutable field is checked explicitly. + storedUID := textNested(env.Object, "status", "pod", "uid") + if pod.Labels["execution.mecatl.dev/environment"] != env.GetName() || pod.Labels["execution.mecatl.dev/profile"] != hashText(textNested(env.Object, "spec", "profile"))[:16] || (storedUID != "" && storedUID != string(pod.UID)) || len(pod.OwnerReferences) != 1 { + return errors.New("pod ownership identity mismatch") } - podName := textNested(env.Object, "status", "pod", "name") - if podName == "" { - return r.setCondition(ctx, env, "Retired", false, "FenceUnknown", "executor identity is unavailable; external fencing proof is required") + owner := pod.OwnerReferences[0] + if owner.APIVersion != env.GetAPIVersion() || owner.Kind != env.GetKind() || owner.Name != env.GetName() || owner.UID != env.GetUID() || owner.Controller == nil || !*owner.Controller || !reflect.DeepEqual(pod.Finalizers, []string{executorFinalizer}) { + return errors.New("pod controller ownership mismatch") } - pod, err := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - if conditionTrue(env, "ExecutorTerminated") { - return r.setCondition(ctx, env, "Retired", true, "WorkspaceRetained", "verified terminal executor removed; PVC retained") - } - return r.setCondition(ctx, env, "Retired", false, "FenceUnknown", "executor disappeared without a verified terminal state; external fencing proof is required") + if pod.Spec.RestartPolicy != corev1.RestartPolicyNever || pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken || pod.Spec.RuntimeClassName == nil || *pod.Spec.RuntimeClassName != p.Spec.RuntimeClassName || len(pod.Spec.Containers) != 1 || len(pod.Spec.InitContainers) != 0 || len(pod.Spec.EphemeralContainers) != 0 || len(pod.Spec.Volumes) != 2 { + return errors.New("pod immutable specification mismatch") } - if err != nil { - return err + psc := pod.Spec.SecurityContext + if psc == nil || psc.RunAsNonRoot == nil || !*psc.RunAsNonRoot || psc.RunAsUser == nil || *psc.RunAsUser != 65532 || psc.RunAsGroup == nil || *psc.RunAsGroup != 65532 || psc.FSGroup == nil || *psc.FSGroup != 65532 || psc.SeccompProfile == nil || psc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + return errors.New("pod security context mismatch") } - if pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed { - return r.setCondition(ctx, env, "Retired", false, "PendingExternalFence", "executor is not terminal; no automatic force retirement is permitted") + c := pod.Spec.Containers[0] + if c.Name != "executor" || c.Image != p.Spec.Image || !reflect.DeepEqual(c.Command, []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}) || !reflect.DeepEqual(c.VolumeMounts, []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}) { + return errors.New("executor container specification mismatch") } - if err := r.setCondition(ctx, env, "ExecutorTerminated", true, "TerminalPod", "executor Pod reported a terminal phase"); err != nil { - return err + cs := c.SecurityContext + if cs == nil || cs.AllowPrivilegeEscalation == nil || *cs.AllowPrivilegeEscalation || cs.ReadOnlyRootFilesystem == nil || !*cs.ReadOnlyRootFilesystem || cs.RunAsNonRoot == nil || !*cs.RunAsNonRoot || cs.RunAsUser == nil || *cs.RunAsUser != 65532 || cs.Capabilities == nil || !reflect.DeepEqual(cs.Capabilities.Drop, []corev1.Capability{"ALL"}) { + return errors.New("executor container security mismatch") } - grace := int64(30) - if err := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, podName, metav1.DeleteOptions{GracePeriodSeconds: &grace}); err != nil && !apierrors.IsNotFound(err) { - return err + wantRequests := corev1.ResourceList{corev1.ResourceCPU: p.CPURequest, corev1.ResourceMemory: p.MemoryRequest, corev1.ResourceEphemeralStorage: p.EphemeralStorageRequest} + wantLimits := corev1.ResourceList{corev1.ResourceCPU: p.CPULimit, corev1.ResourceMemory: p.MemoryLimit, corev1.ResourceEphemeralStorage: p.EphemeralStorageLimit} + if !reflect.DeepEqual(c.Resources.Requests, wantRequests) || !reflect.DeepEqual(c.Resources.Limits, wantLimits) || pod.Spec.Volumes[0].Name != "workspace" || pod.Spec.Volumes[0].PersistentVolumeClaim == nil || pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName != pvcName || pod.Spec.Volumes[1].Name != "tmp" || pod.Spec.Volumes[1].EmptyDir == nil || pod.Spec.Volumes[1].EmptyDir.SizeLimit == nil || pod.Spec.Volumes[1].EmptyDir.SizeLimit.Cmp(p.TmpSizeLimit) != 0 { + return errors.New("executor resources or volumes mismatch") } return nil } -func (r *Reconciler) reconcileDeletion(ctx context.Context, res dynamic.ResourceInterface, env *unstructured.Unstructured) error { - refs, refsErr := referenceRecords(env) - if refsErr != nil || textNested(env.Object, "spec", "desired") != "Retiring" || !conditionTrue(env, "Retired") || len(refs) > 0 || textNested(env.Object, "status", "activeRun", "claimID") != "" || textNested(env.Object, "status", "activeOperation", "id") != "" || textNested(env.Object, "status", "fenceState") != fenceHealthy { - return r.setCondition(ctx, env, "DeletionBlocked", true, "RetentionSafety", "finalizer requires explicit completed retirement and proven quiescence") - } - pod := textNested(env.Object, "status", "pod", "name") - if pod != "" { - grace := int64(30) - if err := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, pod, metav1.DeleteOptions{GracePeriodSeconds: &grace}); err != nil && !apierrors.IsNotFound(err) { - return err - } - } - updated := env.DeepCopy() - next := []string{} - for _, f := range updated.GetFinalizers() { - if f != environmentFinalizer { - next = append(next, f) - } - } - updated.SetFinalizers(next) - _, err := res.Update(ctx, updated, metav1.UpdateOptions{}) - return err + +func (r *Reconciler) reconcileDeletion(ctx context.Context, _ dynamic.ResourceInterface, env *unstructured.Unstructured) error { + return r.setCondition(ctx, env, "DeletionBlocked", true, "ExactLifecycleRequired", "deletion requires the exact retained-environment lifecycle operation") } func (r *Reconciler) updateRuntimeStatus(ctx context.Context, env *unstructured.Unstructured, pvc *corev1.PersistentVolumeClaim, pod *corev1.Pod, ready bool) error { return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { diff --git a/internal/adapter/executioncontroller/controller_lifecycle.go b/internal/adapter/executioncontroller/controller_lifecycle.go index 4483dab02b..5d5511cace 100644 --- a/internal/adapter/executioncontroller/controller_lifecycle.go +++ b/internal/adapter/executioncontroller/controller_lifecycle.go @@ -38,7 +38,7 @@ func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.U } return r.setFenceUnknown(ctx, env, "authoritative workspace identity is unavailable") } - proofMatches := textNested(env.Object, "status", "terminationProof", "operationID") == operationID && textNested(env.Object, "status", "terminationProof", "podUID") == podUID && textNested(env.Object, "status", "terminationProof", "pvcUID") == pvcUID && intNested(env.Object, "status", "terminationProof", "epoch") == epoch + proofMatches := textNested(env.Object, "status", "terminationProof", operationIDField) == operationID && textNested(env.Object, "status", "terminationProof", "podUID") == podUID && textNested(env.Object, "status", "terminationProof", "pvcUID") == pvcUID && intNested(env.Object, "status", "terminationProof", "epoch") == epoch switch phase { case "Quiescing": pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) @@ -223,7 +223,10 @@ func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructu return err } uid := updated.GetUID() - return res.Delete(ctx, updated.GetName(), metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + if err := res.Delete(ctx, updated.GetName(), metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return err + } + return releaseProfileSlot(ctx, r.kube, r.namespace, textNested(updated.Object, "spec", "profile"), updated.GetName()) } func withoutString(values []string, remove string) []string { diff --git a/internal/adapter/executioncontroller/controller_test.go b/internal/adapter/executioncontroller/controller_test.go index 50a9c7455e..7d90454547 100644 --- a/internal/adapter/executioncontroller/controller_test.go +++ b/internal/adapter/executioncontroller/controller_test.go @@ -2,10 +2,13 @@ package executioncontroller import ( "context" + "strings" "testing" "time" corev1 "k8s.io/api/core/v1" + nodev1 "k8s.io/api/node/v1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -15,6 +18,30 @@ import ( k8stesting "k8s.io/client-go/testing" ) +func TestReconcileRefusesForeignExistingPVCWithoutPersistingUID(t *testing.T) { + ctx := context.Background() + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + foreign := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "workspace-test", Namespace: "ns", UID: types.UID("foreign"), Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/revision": "rev"}}} + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(foreign) + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if uid := textNested(got.Object, "status", "pvc", "uid"); uid != "" { + t.Fatalf("foreign PVC UID persisted as authoritative: %q", uid) + } + pods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pods.Items) != 0 { + t.Fatalf("executor created over foreign PVC: pods=%d err=%v", len(pods.Items), err) + } +} + func TestReconcileCreatesTokenlessNonRootPodAndRetainedPVC(t *testing.T) { ctx := context.Background() env := testEnvironment() @@ -131,9 +158,23 @@ func TestRuntimeAndRelevantStatusUpdatesEnqueue(t *testing.T) { if r.queue.Len() != 3 { t.Fatalf("relevant status update was not queued: len=%d", r.queue.Len()) } - controllerStatus := newEnv.DeepCopy() + lifecycle := newEnv.DeepCopy() + _ = unstructured.SetNestedMap(lifecycle.Object, map[string]any{"id": "retire", "phase": "Quiescing"}, "status", "lifecycleOperation") + lifecycleQueue := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kubefake.NewSimpleClientset(), "ns", testProfiles()) + lifecycleQueue.enqueueUpdate(newEnv, lifecycle) + if lifecycleQueue.queue.Len() != 1 { + t.Fatalf("lifecycle start was not queued: len=%d", lifecycleQueue.queue.Len()) + } + advanced := lifecycle.DeepCopy() + _ = unstructured.SetNestedField(advanced.Object, "WaitingForTermination", "status", "lifecycleOperation", "phase") + phaseQueue := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kubefake.NewSimpleClientset(), "ns", testProfiles()) + phaseQueue.enqueueUpdate(lifecycle, advanced) + if phaseQueue.queue.Len() != 1 { + t.Fatalf("lifecycle phase change was not queued: len=%d", phaseQueue.queue.Len()) + } + controllerStatus := advanced.DeepCopy() _ = unstructured.SetNestedField(controllerStatus.Object, "pod", "status", "pod", "name") - r.enqueueUpdate(newEnv, controllerStatus) + r.enqueueUpdate(advanced, controllerStatus) if r.queue.Len() != 3 { t.Fatalf("controller-owned status update was queued: len=%d", r.queue.Len()) } @@ -166,8 +207,8 @@ func TestStartupDoesNotFenceLivePeerOperation(t *testing.T) { env := testEnvironment() _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "old", "operation": "file.replace"}, "status", "activeOperation") d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) - r := NewReconciler(d, kubefake.NewSimpleClientset(), "ns", testProfiles()) - peer := NewReconciler(d, kubefake.NewSimpleClientset(), "ns", testProfiles()) + r := NewReconciler(d, profileResourceClient(), "ns", testProfiles()) + peer := NewReconciler(d, profileResourceClient(), "ns", testProfiles()) if r.Ready() || peer.Ready() { t.Fatal("reconciler reported ready before cache synchronization") } @@ -188,6 +229,30 @@ func TestStartupDoesNotFenceLivePeerOperation(t *testing.T) { t.Fatalf("startup mutated a potentially live peer operation: status=%v", got.Object["status"]) } } +func TestInitializeRefusesMissingRuntimeClassBeforeCreatingPods(t *testing.T) { + kube := kubefake.NewSimpleClientset(&storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "standard"}}) + r := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kube, "ns", testProfiles()) + err := r.Initialize(t.Context()) + if err == nil || !strings.Contains(err.Error(), `profile preflight: RuntimeClass "sandboxed" unavailable`) { + t.Fatalf("Initialize error = %v", err) + } + if r.Ready() { + t.Fatal("reconciler reported ready after failed profile preflight") + } + for _, action := range kube.Actions() { + if action.GetVerb() == "create" && action.GetResource().Resource == "pods" { + t.Fatal("profile preflight created a Pod") + } + } +} + +func profileResourceClient() *kubefake.Clientset { + return kubefake.NewSimpleClientset( + &nodev1.RuntimeClass{ObjectMeta: metav1.ObjectMeta{Name: "sandboxed"}}, + &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "standard"}}, + ) +} + func testProfiles() *Profiles { spec := ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100} profile, err := validateProfile("go", spec) diff --git a/internal/adapter/executioncontroller/crd_schema_test.go b/internal/adapter/executioncontroller/crd_schema_test.go new file mode 100644 index 0000000000..ef5a834992 --- /dev/null +++ b/internal/adapter/executioncontroller/crd_schema_test.go @@ -0,0 +1,49 @@ +package executioncontroller + +import ( + "os" + "testing" + + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +func TestExecutionEnvironmentCRDPreservesControllerStatus(t *testing.T) { + raw, err := os.ReadFile("../../../deploy/helm/mecatl-execution/crds/executionenvironment.yaml") + if err != nil { + t.Fatal(err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(raw, &crd); err != nil { + t.Fatal(err) + } + var internal apiextensions.JSONSchemaProps + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(crd.Spec.Versions[0].Schema.OpenAPIV3Schema, &internal, nil); err != nil { + t.Fatal(err) + } + structural, err := schema.NewStructural(&internal) + if err != nil { + t.Fatal(err) + } + obj := map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env"}, "spec": map[string]any{}, "status": map[string]any{ + "schemaVersion": int64(2), "observedGeneration": int64(9), "epoch": int64(3), "grantGeneration": int64(4), "fenceState": "Healthy", + "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "op", "sourceBindingID": "source", "createdAt": "2026-09-17T00:00:00Z"}}, + "pvc": map[string]any{"name": "pvc", "uid": "pvc-uid"}, "pod": map[string]any{"name": "pod", "uid": "pod-uid"}, + "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "ownerHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "clientHash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "epoch": int64(3), "grantGeneration": int64(4), "expiresAt": "2026-09-17T00:01:00Z", "lastRenewOperationID": "renew"}, + "activeOperation": map[string]any{"id": "file", "operation": "file.read", "startedAt": "2026-09-17T00:00:00Z", "claimID": "claim", "runID": "run", "epoch": int64(3), "holderID": "holder", "renewedAt": "2026-09-17T00:00:01Z", "expiresAt": "2026-09-17T00:01:00Z"}, + "lifecycleOperation": map[string]any{"id": "life", "type": "ReplaceExecutor", "phase": "Quiescing", "expectedEpoch": int64(3), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid", "createdAt": "2026-09-17T00:00:00Z"}, + "terminationProof": map[string]any{"operationID": "life", "podUID": "pod-uid", "pvcUID": "pvc-uid", "epoch": int64(3), "podPhase": "Failed", "observedAt": "2026-09-17T00:00:00Z"}, + "migrationOperation": map[string]any{"id": "migration", "fromSchema": int64(1), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid"}, "lastMigrationOperationID": "migration", + "conditions": []any{map[string]any{"type": "Ready", "status": "True", "reason": "Reconciled", "message": "ready", "observedGeneration": int64(9), "lastTransitionTime": "2026-09-17T00:00:00Z"}}, + }} + pruning.Prune(obj, structural, true) + for _, path := range [][]string{{"status", "schemaVersion"}, {"status", "observedGeneration"}, {"status", "epoch"}, {"status", "grantGeneration"}, {"status", "references"}, {"status", "pvc"}, {"status", "pod"}, {"status", "activeRun", "grantGeneration"}, {"status", "activeOperation"}, {"status", "lifecycleOperation"}, {"status", "terminationProof"}, {"status", "migrationOperation"}, {"status", "lastMigrationOperationID"}, {"status", "conditions"}} { + if _, found, err := unstructured.NestedFieldNoCopy(obj, path...); err != nil || !found { + t.Fatalf("CRD pruning removed controller field %v (found=%t err=%v)", path, found, err) + } + } +} diff --git a/internal/adapter/executioncontroller/handler.go b/internal/adapter/executioncontroller/handler.go index 994d86133d..4205472036 100644 --- a/internal/adapter/executioncontroller/handler.go +++ b/internal/adapter/executioncontroller/handler.go @@ -104,7 +104,7 @@ type adminLifecycleBackend interface { RecoverEnvironment(context.Context, adminLifecycleRequest) error DeleteRetiredEnvironment(context.Context, adminLifecycleRequest) error MigrateEnvironment(context.Context, adminLifecycleRequest) error - RevokeEnvironment(context.Context, executionenv.EnvironmentRef, string, string, uint64) (uint64, error) + RevokeEnvironment(context.Context, executionenv.EnvironmentRef, string, string, uint64, string) (uint64, error) } // Handler is the authenticated private execution-provider gRPC service. @@ -137,6 +137,9 @@ func (h *Handler) client(ctx context.Context) (authenticatedClient, error) { return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) } if h.cfg.Security != nil { + if !h.cfg.Security.Ready() { + return authenticatedClient{}, wireError(executionenv.CodeNotReady, true) + } id, policy, err := h.cfg.Security.authorize(ctx, leaf) if err != nil { return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) @@ -468,7 +471,7 @@ func (h *Handler) RecoverEnvironment(ctx context.Context, q *executionv1.Recover if err := h.adminLifecycleBackend.RecoverEnvironment(ctx, req); err != nil { var controlled *executionenv.Error if errors.As(err, &controlled) && controlled.Code == executionenv.CodeFenceUnknown { - return nil, status.Error(codes.FailedPrecondition, "execution provider request failed; independent termination proof or external fencing is required") + return nil, wireError(executionenv.CodeFenceUnknown, false) } return nil, backendError(err) } @@ -497,7 +500,7 @@ func (h *Handler) MigrateEnvironment(ctx context.Context, q *executionv1.Migrate if err != nil { return nil, err } - if q.GetExpectedSchemaVersion() > 1 { + if q == nil || q.ExpectedSchemaVersion == nil || q.GetExpectedSchemaVersion() > 1 { return nil, wireError(executionenv.CodeInvalidArgument, false) } req.ExpectedEpoch = 0 @@ -515,10 +518,13 @@ func (h *Handler) RevokeEnvironment(ctx context.Context, q *executionv1.RevokeEn if err != nil { return nil, err } - if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) || q.GetExpectedGrantGeneration() == 0 || q.GetExpectedGrantGeneration() > math.MaxInt64 { + if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) { return nil, wireError(executionenv.CodePermissionDenied, false) } - generation, err := h.adminLifecycleBackend.RevokeEnvironment(ctx, ref, c.id, ownerHash(owner), q.GetExpectedGrantGeneration()) + if q.GetExpectedGrantGeneration() == 0 || q.GetExpectedGrantGeneration() > math.MaxInt64 || !validOperationID(q.GetOperationId()) { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + generation, err := h.adminLifecycleBackend.RevokeEnvironment(ctx, ref, c.id, ownerHash(owner), q.GetExpectedGrantGeneration(), q.GetOperationId()) if err != nil { return nil, backendError(err) } @@ -614,11 +620,12 @@ func (h *Handler) authorize(ctx context.Context, p *executionv1.RequestContext, oh := ownerHash(rc.Owner) verifier := h.cfg.Verifier if h.cfg.Security != nil { - material, materialErr := h.cfg.Security.material(ctx) + now := h.cfg.Security.now() + material, materialErr := h.cfg.Security.authoritativeAt(ctx, now) if materialErr != nil { return c, "", rc, wireError(executionenv.CodeNotReady, true) } - verifier = material.verifier + verifier = material.verifierAt(now) } if _, err := verifier.Verify(rc.Grant, executionenv.GrantExpectation{Client: c.id, OwnerHash: oh, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: rc.GrantGeneration, Operation: op}); err != nil { if errors.Is(err, executionenv.ErrGrantExpired) { @@ -765,18 +772,28 @@ func validRef(v executionenv.EnvironmentRef) bool { func (h *Handler) signClaim(ctx context.Context, claim executionenv.RunClaim, client, owner string) (string, time.Time, error) { signer := h.cfg.Signer + now := time.Now().UTC() + var signingDeadline time.Time if h.cfg.Security != nil { - material, err := h.cfg.Security.material(ctx) + now = h.cfg.Security.now() + material, err := h.cfg.Security.authoritativeAt(ctx, now) if err != nil { return "", time.Time{}, err } signer = material.signer + signingDeadline = material.activeWindow.verifyUntil } life := signer.Lifetime if life <= 0 { life = time.Minute } - now, expiry := time.Now().UTC(), time.Now().UTC().Add(life) + expiry := now.Add(life) + if !signingDeadline.IsZero() && signingDeadline.Before(expiry) { + expiry = signingDeadline + } + if !now.Before(expiry) { + return "", time.Time{}, errors.New("active signing key is expired") + } nonce := make([]byte, 16) if _, err := rand.Read(nonce); err != nil { return "", time.Time{}, err @@ -823,12 +840,18 @@ func wireError(code executionenv.ErrorCode, retry bool) error { grpcCode = codes.AlreadyExists case executionenv.CodeConflict, executionenv.CodeVersionMismatch: grpcCode = codes.Aborted - case executionenv.CodeNotReady, executionenv.CodeFenceUnknown: + case executionenv.CodeNotReady: grpcCode = codes.Unavailable + case executionenv.CodeFenceUnknown: + grpcCode = codes.FailedPrecondition case executionenv.CodeResourceExhausted: grpcCode = codes.ResourceExhausted } - st := status.New(grpcCode, "execution provider request failed") + message := "execution provider request failed" + if code == executionenv.CodeFenceUnknown { + message = "execution provider request failed; verify termination independently or use external fencing" + } + st := status.New(grpcCode, message) with, err := st.WithDetails(&executionv1.ErrorDetail{Code: string(code), Retryable: retry}) if err == nil { st = with diff --git a/internal/adapter/executioncontroller/handler_test.go b/internal/adapter/executioncontroller/handler_test.go index 2d0ce56b55..7a67cc1e59 100644 --- a/internal/adapter/executioncontroller/handler_test.go +++ b/internal/adapter/executioncontroller/handler_test.go @@ -8,6 +8,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "net/url" + "strings" "testing" "time" @@ -98,6 +99,27 @@ func (*fakeBackend) ListReferenceIntents(context.Context, string, string, int) ( return nil, nil } +type adminFakeBackend struct { + *fakeBackend + migratedSchema int64 +} + +func (*adminFakeBackend) ReplaceExecutor(context.Context, adminLifecycleRequest) error { return nil } +func (*adminFakeBackend) RetireExact(context.Context, adminLifecycleRequest) error { return nil } +func (*adminFakeBackend) RecoverEnvironment(context.Context, adminLifecycleRequest) error { + return nil +} +func (*adminFakeBackend) DeleteRetiredEnvironment(context.Context, adminLifecycleRequest) error { + return nil +} +func (b *adminFakeBackend) MigrateEnvironment(_ context.Context, req adminLifecycleRequest) error { + b.migratedSchema = req.ExpectedSchema + return nil +} +func (*adminFakeBackend) RevokeEnvironment(context.Context, executionenv.EnvironmentRef, string, string, uint64, string) (uint64, error) { + return 2, nil +} + func authenticatedContext(id string) context.Context { u, _ := url.Parse(id) cert := &x509.Certificate{URIs: []*url.URL{u}} @@ -109,6 +131,27 @@ func structTLSState(cert *x509.Certificate) (s tls.ConnectionState) { return s } +func TestMigrateEnvironmentRequiresExpectedSchemaPresence(t *testing.T) { + backend := &adminFakeBackend{fakeBackend: newFakeBackend()} + id := "spiffe://cluster/ns/admin" + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{id: {Administrator: true}}}, backend) + ctx := authenticatedContext(id) + base := &executionv1.MigrateEnvironmentRequest{Environment: &executionv1.EnvironmentRef{Id: "env", Revision: "rev"}, Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, ExpectedPodUid: "pod", ExpectedPvcUid: "pvc", OperationId: "migrate"} + if _, err := h.MigrateEnvironment(ctx, base); status.Code(err) != codes.InvalidArgument { + t.Fatalf("omitted schema code=%v", status.Code(err)) + } + zero := uint32(0) + base.ExpectedSchemaVersion = &zero + if _, err := h.MigrateEnvironment(ctx, base); err != nil || backend.migratedSchema != 0 { + t.Fatalf("explicit zero rejected: schema=%d err=%v", backend.migratedSchema, err) + } + unknown := uint32(2) + base.ExpectedSchemaVersion = &unknown + if _, err := h.MigrateEnvironment(ctx, base); status.Code(err) != codes.InvalidArgument { + t.Fatalf("unknown schema code=%v", status.Code(err)) + } +} + func TestHandlerBlocksAllRequestsUntilStartupReady(t *testing.T) { backend := newFakeBackend() h := NewHandler(HandlerConfig{Ready: func() bool { return false }}, backend) @@ -284,4 +327,11 @@ func TestWireErrorDoesNotExposeBackendMessage(t *testing.T) { if details := st.Details(); len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeInternal) { t.Fatalf("unknown error detail=%v", details) } + fenced := status.Convert(backendError(&executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "/var/run/provider/private-key"})) + if fenced.Code() != codes.FailedPrecondition || strings.Contains(fenced.Message(), "/var/") { + t.Fatalf("fence error leaked or mapped incorrectly: code=%v message=%q", fenced.Code(), fenced.Message()) + } + if details := fenced.Details(); len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeFenceUnknown) { + t.Fatalf("fence error detail=%v", details) + } } diff --git a/internal/adapter/executioncontroller/production_lifecycle_test.go b/internal/adapter/executioncontroller/production_lifecycle_test.go index d4b2a83e5b..f3c3186469 100644 --- a/internal/adapter/executioncontroller/production_lifecycle_test.go +++ b/internal/adapter/executioncontroller/production_lifecycle_test.go @@ -17,6 +17,36 @@ import ( "github.com/stacklok/mecatl/internal/executionenv" ) +func TestPodTerminalRequiresEveryDeclaredContainerByName(t *testing.T) { + terminated := corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}} + base := &corev1.Pod{Spec: corev1.PodSpec{InitContainers: []corev1.Container{{Name: "init"}}, Containers: []corev1.Container{{Name: "executor"}}}, Status: corev1.PodStatus{Phase: corev1.PodSucceeded, InitContainerStatuses: []corev1.ContainerStatus{{Name: "init", State: terminated}}, ContainerStatuses: []corev1.ContainerStatus{{Name: "executor", State: terminated}}}} + if !podTerminal(base) { + t.Fatal("exact terminated init and regular containers were rejected") + } + cases := map[string]func(*corev1.Pod){ + "init waiting": func(p *corev1.Pod) { + p.Status.InitContainerStatuses[0].State = corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{}} + }, + "init running": func(p *corev1.Pod) { + p.Status.InitContainerStatuses[0].State = corev1.ContainerState{Running: &corev1.ContainerStateRunning{}} + }, + "missing init": func(p *corev1.Pod) { p.Status.InitContainerStatuses = nil }, + "duplicate regular": func(p *corev1.Pod) { + p.Status.ContainerStatuses = append(p.Status.ContainerStatuses, p.Status.ContainerStatuses[0]) + }, + "wrong regular name": func(p *corev1.Pod) { p.Status.ContainerStatuses[0].Name = "other" }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + pod := base.DeepCopy() + mutate(pod) + if podTerminal(pod) { + t.Fatal("invalid container evidence proved terminal") + } + }) + } +} + func lifecycleAdminEnvironment(schema int64, refs []any) *unstructured.Unstructured { return &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", @@ -117,6 +147,28 @@ func TestRecoverRequiresExactTerminalPodAndPersistsProof(t *testing.T) { } } +func TestRecoveredTerminalProofCanStartExactReplacement(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + setConditionObject(env, "Ready", false, "FenceUnknown", "holder lost") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "uncertain", "claimID": "claim", "epoch": int64(4)}, "status", "activeOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + q.OperationID = "recover-op" + if err := store.RecoverEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + q.OperationID = "replace-op" + if err := store.ReplaceExecutor(t.Context(), q); err != nil { + t.Fatalf("exact replacement after terminal recovery: %v", err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "id") != "replace-op" { + t.Fatalf("replacement operation missing after recovery: %v", got.Object["status"]) + } +} + func TestRecoverMissingPodRemainsFenceUnknown(t *testing.T) { env := lifecycleAdminEnvironment(2, []any{}) _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") diff --git a/internal/adapter/executioncontroller/profiles.go b/internal/adapter/executioncontroller/profiles.go index abc3fcae24..ea9a46d2cd 100644 --- a/internal/adapter/executioncontroller/profiles.go +++ b/internal/adapter/executioncontroller/profiles.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "os" + "sort" "strings" "time" @@ -174,3 +175,24 @@ func (p *Profiles) get(name string) (resolvedProfile, bool) { v, ok := p.byName[name] return v, ok } + +func (p *Profiles) clusterResources() (runtimeClasses, storageClasses []string) { + if p == nil { + return nil, nil + } + runtimes := make(map[string]struct{}, len(p.byName)) + storage := make(map[string]struct{}, len(p.byName)) + for _, profile := range p.byName { + runtimes[profile.Spec.RuntimeClassName] = struct{}{} + storage[profile.Spec.StorageClass] = struct{}{} + } + for name := range runtimes { + runtimeClasses = append(runtimeClasses, name) + } + for name := range storage { + storageClasses = append(storageClasses, name) + } + sort.Strings(runtimeClasses) + sort.Strings(storageClasses) + return runtimeClasses, storageClasses +} diff --git a/internal/adapter/executioncontroller/security.go b/internal/adapter/executioncontroller/security.go index cc3571273c..d8f4e69272 100644 --- a/internal/adapter/executioncontroller/security.go +++ b/internal/adapter/executioncontroller/security.go @@ -13,11 +13,12 @@ import ( "errors" "fmt" "io" + "maps" "math" "net/url" "os" "path/filepath" - "sort" + "slices" "strconv" "strings" "sync/atomic" @@ -37,6 +38,7 @@ const ( maxSecurityKeys = 64 maxSecurityClients = 256 securityStateDataKey = "state.json" + keyStateRevoked = "revoked" ) type securityKeyManifest struct { @@ -75,10 +77,18 @@ type securityManifest struct { Clients []securityClientManifest `json:"clients"` } +type keyValidity struct { + activateAt, verifyUntil time.Time + state string +} + type securitySnapshot struct { generation uint64 + digest string signer GrantSigner verifier executionenv.GrantVerifier + keyValidity map[string]keyValidity + activeWindow keyValidity tlsConfig *tls.Config clientCAs *x509.CertPool clients map[string]ClientPolicy @@ -88,6 +98,7 @@ type securitySnapshot struct { type securityLedger struct { Generation uint64 `json:"generation"` + Digest string `json:"digest"` Fingerprints map[string]string `json:"fingerprints"` } @@ -108,8 +119,9 @@ func NewSecurityManager(manifestPath, keyDirectory, namespace, configMap string, // Ready reports whether the current snapshot is authoritative and unexpired. func (m *SecurityManager) Ready() bool { + now := m.now() s := m.current.Load() - return m.ready.Load() && s != nil && m.now().Before(s.validUntil) + return m.ready.Load() && snapshotValidAt(s, now) } // CheckReady verifies that the loaded snapshot still matches durable authority. @@ -139,6 +151,9 @@ func (m *SecurityManager) Run(ctx context.Context, interval time.Duration) { if interval <= 0 { interval = 2 * time.Second } + if interval > time.Minute { + interval = time.Minute + } ticker := time.NewTicker(interval) defer ticker.Stop() for { @@ -163,11 +178,12 @@ func (m *SecurityManager) TLSConfig() *tls.Config { } func (m *SecurityManager) authorize(ctx context.Context, leaf *x509.Certificate) (string, ClientPolicy, error) { - s, err := m.authoritative(ctx) + now := m.now() + s, err := m.authoritativeAt(ctx, now) if err != nil { return "", ClientPolicy{}, err } - opts := x509.VerifyOptions{Roots: s.clientCAs, Intermediates: x509.NewCertPool(), CurrentTime: m.now(), KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + opts := x509.VerifyOptions{Roots: s.clientCAs, Intermediates: x509.NewCertPool(), CurrentTime: now, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} if _, err := leaf.Verify(opts); err != nil { return "", ClientPolicy{}, err } @@ -187,8 +203,12 @@ func (m *SecurityManager) material(ctx context.Context) (*securitySnapshot, erro } func (m *SecurityManager) authoritative(ctx context.Context) (*securitySnapshot, error) { + return m.authoritativeAt(ctx, m.now()) +} + +func (m *SecurityManager) authoritativeAt(ctx context.Context, now time.Time) (*securitySnapshot, error) { s := m.current.Load() - if !m.Ready() || s == nil || m.kube == nil { + if !m.ready.Load() || !snapshotValidAt(s, now) || m.kube == nil { return nil, errors.New("security material is not ready") } cm, err := m.kube.CoreV1().ConfigMaps(m.namespace).Get(ctx, m.configMap, metav1.GetOptions{}) @@ -197,13 +217,29 @@ func (m *SecurityManager) authoritative(ctx context.Context) (*securitySnapshot, return nil, err } var ledger securityLedger - if err := json.Unmarshal([]byte(cm.Data[securityStateDataKey]), &ledger); err != nil || ledger.Generation != s.generation || !sameFingerprints(ledger.Fingerprints, s.fingerprints) { + if err := executionenv.DecodeStrict([]byte(cm.Data[securityStateDataKey]), &ledger); err != nil || ledger.Generation != s.generation || ledger.Digest != s.digest || !maps.Equal(ledger.Fingerprints, s.fingerprints) { m.ready.Store(false) return nil, errors.New("loaded security generation is not authoritative") } return s, nil } +func snapshotValidAt(s *securitySnapshot, now time.Time) bool { + return s != nil && !now.Before(s.activeWindow.activateAt) && now.Before(s.activeWindow.verifyUntil) && now.Before(s.validUntil) +} + +func (s *securitySnapshot) verifierAt(now time.Time) executionenv.GrantVerifier { + v := s.verifier + v.Keys = make(map[string]ed25519.PublicKey, len(s.verifier.Keys)) + for id, key := range s.verifier.Keys { + window := s.keyValidity[id] + if window.state != keyStateRevoked && !now.Before(window.activateAt) && now.Before(window.verifyUntil) { + v.Keys[id] = key + } + } + return v +} + func (m *SecurityManager) load() (*securitySnapshot, error) { mf, ttl, skew, err := m.loadManifest() if err != nil { @@ -215,11 +251,11 @@ func (m *SecurityManager) load() (*securitySnapshot, error) { } defer func() { _ = root.Close() }() - keys, active, fingerprints, err := m.loadGrantKeys(root, mf) + keys, active, fingerprints, windows, activeWindow, err := m.loadGrantKeys(root, mf) if err != nil { return nil, err } - tlsConfig, clientCAs, validUntil, err := m.loadTLS(root, mf.TLS) + tlsConfig, clientCAs, tlsValidUntil, caFingerprints, serverFingerprint, err := m.loadTLS(root, mf.TLS) if err != nil { return nil, err } @@ -227,8 +263,22 @@ func (m *SecurityManager) load() (*securitySnapshot, error) { if err != nil { return nil, err } + digest, err := authorityDigest(mf, ttl, skew, fingerprints, caFingerprints, serverFingerprint) + if err != nil { + return nil, err + } + validUntil := tlsValidUntil + now := m.now() + for _, window := range windows { + if window.state != keyStateRevoked && window.verifyUntil.Before(validUntil) { + validUntil = window.verifyUntil + } + if window.state != keyStateRevoked && now.Before(window.activateAt) && window.activateAt.Before(validUntil) { + validUntil = window.activateAt + } + } return &securitySnapshot{ - generation: mf.Generation, + generation: mf.Generation, digest: digest, signer: GrantSigner{ KeyID: mf.ActiveKeyID, PrivateKey: active, Issuer: mf.Issuer, Audience: mf.Audience, Lifetime: ttl, @@ -236,6 +286,7 @@ func (m *SecurityManager) load() (*securitySnapshot, error) { verifier: executionenv.GrantVerifier{ Keys: keys, Issuer: mf.Issuer, Audience: mf.Audience, MaxLifetime: ttl + skew, }, + keyValidity: windows, activeWindow: activeWindow, tlsConfig: tlsConfig, clientCAs: clientCAs, clients: clients, validUntil: validUntil, fingerprints: fingerprints, }, nil @@ -246,15 +297,10 @@ func (m *SecurityManager) loadManifest() (securityManifest, time.Duration, time. if err != nil { return securityManifest{}, 0, 0, fmt.Errorf("read security manifest: %w", err) } - dec := json.NewDecoder(bytes.NewReader(manifestBytes)) - dec.DisallowUnknownFields() var mf securityManifest - if err := dec.Decode(&mf); err != nil { + if err := executionenv.DecodeStrict(manifestBytes, &mf); err != nil { return securityManifest{}, 0, 0, fmt.Errorf("decode security manifest: %w", err) } - if err := ensureJSONEOF(dec); err != nil { - return securityManifest{}, 0, 0, err - } if mf.Version != securityManifestVersion || mf.Generation == 0 || mf.Generation > math.MaxInt64 || mf.Issuer == "" || mf.Audience == "" || mf.ActiveKeyID == "" || len(mf.Keys) == 0 || len(mf.Keys) > maxSecurityKeys || len(mf.Clients) == 0 || len(mf.Clients) > maxSecurityClients { return securityManifest{}, 0, 0, errors.New("security manifest identity or bounds are invalid") } @@ -269,84 +315,114 @@ func (m *SecurityManager) loadManifest() (securityManifest, time.Duration, time. return mf, ttl, skew, nil } -func (m *SecurityManager) loadGrantKeys(root *os.Root, mf securityManifest) (map[string]ed25519.PublicKey, ed25519.PrivateKey, map[string]string, error) { +func (m *SecurityManager) loadGrantKeys(root *os.Root, mf securityManifest) (map[string]ed25519.PublicKey, ed25519.PrivateKey, map[string]string, map[string]keyValidity, keyValidity, error) { keys := make(map[string]ed25519.PublicKey, len(mf.Keys)) fingerprints := make(map[string]string, len(mf.Keys)) + windows := make(map[string]keyValidity, len(mf.Keys)) seenVersion := map[uint64]bool{} seenID := map[string]bool{} var active ed25519.PrivateKey + var activeWindow keyValidity now := m.now() for _, km := range mf.Keys { if !validKeyManifest(km, seenID, seenVersion) { - return nil, nil, nil, errors.New("security key entry is invalid or duplicated") + return nil, nil, nil, nil, keyValidity{}, errors.New("security key entry is invalid or duplicated") } seenID[km.ID] = true seenVersion[km.Version] = true key, err := readEd25519(root, km.File) if err != nil { - return nil, nil, nil, fmt.Errorf("load security key %q: %w", km.ID, err) + return nil, nil, nil, nil, keyValidity{}, fmt.Errorf("load security key %q: %w", km.ID, err) } pub := key.Public().(ed25519.PublicKey) sum := sha256.Sum256(pub) fp := hex.EncodeToString(sum[:]) if !strings.EqualFold(fp, km.PublicSHA256) { - return nil, nil, nil, fmt.Errorf("security key %q fingerprint mismatch", km.ID) + return nil, nil, nil, nil, keyValidity{}, fmt.Errorf("security key %q fingerprint mismatch", km.ID) } fingerprints[km.ID+":"+strconv.FormatUint(km.Version, 10)] = fp - if km.State != "revoked" && !now.Before(km.ActivateAt) && now.Before(km.VerifyUntil) { + window := keyValidity{activateAt: km.ActivateAt, verifyUntil: km.VerifyUntil, state: km.State} + windows[km.ID] = window + if km.State != keyStateRevoked { keys[km.ID] = pub } if km.ID == mf.ActiveKeyID { if km.State != "active" || now.Before(km.ActivateAt) || !now.Before(km.VerifyUntil) { - return nil, nil, nil, errors.New("active grant key is outside its activation window") + return nil, nil, nil, nil, keyValidity{}, errors.New("active grant key is outside its activation window") } active = key + activeWindow = window } } if active == nil { - return nil, nil, nil, errors.New("active grant key is missing") + return nil, nil, nil, nil, keyValidity{}, errors.New("active grant key is missing") } - return keys, active, fingerprints, nil + return keys, active, fingerprints, windows, activeWindow, nil } func validKeyManifest(km securityKeyManifest, seenID map[string]bool, seenVersion map[uint64]bool) bool { return validKeyID(km.ID) && !seenID[km.ID] && km.Version != 0 && !seenVersion[km.Version] && validBaseName(km.File) && - (km.State == "active" || km.State == "verify-only" || km.State == "revoked") && + (km.State == "active" || km.State == "verify-only" || km.State == keyStateRevoked) && !km.VerifyUntil.IsZero() && km.VerifyUntil.After(km.ActivateAt) } -func (m *SecurityManager) loadTLS(root *os.Root, manifest securityTLSManifest) (*tls.Config, *x509.CertPool, time.Time, error) { +func (m *SecurityManager) loadTLS(root *os.Root, manifest securityTLSManifest) (*tls.Config, *x509.CertPool, time.Time, []string, string, error) { certPEM, err := readRootFile(root, manifest.CertificateFile) if err != nil { - return nil, nil, time.Time{}, err + return nil, nil, time.Time{}, nil, "", err } keyPEM, err := readRootFile(root, manifest.PrivateKeyFile) if err != nil { - return nil, nil, time.Time{}, err + return nil, nil, time.Time{}, nil, "", err } serverCert, err := tls.X509KeyPair(certPEM, keyPEM) if err != nil { - return nil, nil, time.Time{}, fmt.Errorf("load TLS identity: %w", err) + return nil, nil, time.Time{}, nil, "", fmt.Errorf("load TLS identity: %w", err) } leaf, err := x509.ParseCertificate(serverCert.Certificate[0]) if err != nil { - return nil, nil, time.Time{}, err + return nil, nil, time.Time{}, nil, "", err } now := m.now() if now.Before(leaf.NotBefore) || !now.Before(leaf.NotAfter) || !hasUsage(leaf.ExtKeyUsage, x509.ExtKeyUsageServerAuth) { - return nil, nil, time.Time{}, errors.New("server certificate is not currently valid for server authentication") + return nil, nil, time.Time{}, nil, "", errors.New("server certificate is not currently valid for server authentication") } serverCert.Leaf = leaf caPEM, err := readRootFile(root, manifest.ClientCAFile) if err != nil { - return nil, nil, time.Time{}, err + return nil, nil, time.Time{}, nil, "", err } pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(caPEM) { - return nil, nil, time.Time{}, errors.New("client CA bundle has no certificates") + caFingerprints, err := appendCAs(pool, caPEM) + if err != nil { + return nil, nil, time.Time{}, nil, "", err } + serverSum := sha256.Sum256(leaf.Raw) cfg := &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{serverCert}, ClientAuth: tls.RequireAnyClientCert} - return cfg, pool, leaf.NotAfter, nil + return cfg, pool, leaf.NotAfter, caFingerprints, hex.EncodeToString(serverSum[:]), nil +} + +func appendCAs(pool *x509.CertPool, pemBytes []byte) ([]string, error) { + var fingerprints []string + for len(bytes.TrimSpace(pemBytes)) != 0 { + block, rest := pem.Decode(pemBytes) + if block == nil || block.Type != "CERTIFICATE" { + return nil, errors.New("client CA bundle is invalid") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil || !cert.IsCA { + return nil, errors.New("client CA bundle contains an invalid CA certificate") + } + pool.AddCert(cert) + sum := sha256.Sum256(cert.Raw) + fingerprints = append(fingerprints, hex.EncodeToString(sum[:])) + pemBytes = rest + } + if len(fingerprints) == 0 { + return nil, errors.New("client CA bundle has no certificates") + } + slices.Sort(fingerprints) + return fingerprints, nil } func clientPolicies(entries []securityClientManifest) (map[string]ClientPolicy, error) { @@ -368,6 +444,35 @@ func clientPolicies(entries []securityClientManifest) (map[string]ClientPolicy, return clients, nil } +func authorityDigest(mf securityManifest, ttl, skew time.Duration, keyFingerprints map[string]string, caFingerprints []string, serverFingerprint string) (string, error) { + keys := slices.Clone(mf.Keys) + slices.SortFunc(keys, func(a, b securityKeyManifest) int { return strings.Compare(a.ID, b.ID) }) + clients := slices.Clone(mf.Clients) + slices.SortFunc(clients, func(a, b securityClientManifest) int { return strings.Compare(a.URI, b.URI) }) + canonical := struct { + Version int + Generation uint64 + Issuer, Audience, ActiveKeyID string + GrantTTL, ClockSkew int64 + Keys []securityKeyManifest + TLS securityTLSManifest + Clients []securityClientManifest + KeyFingerprints map[string]string + CAFingerprints []string + ServerFingerprint string + }{ + Version: mf.Version, Generation: mf.Generation, Issuer: mf.Issuer, Audience: mf.Audience, ActiveKeyID: mf.ActiveKeyID, + GrantTTL: int64(ttl), ClockSkew: int64(skew), Keys: keys, TLS: mf.TLS, Clients: clients, + KeyFingerprints: keyFingerprints, CAFingerprints: slices.Clone(caFingerprints), ServerFingerprint: serverFingerprint, + } + raw, err := json.Marshal(canonical) + if err != nil { + return "", err + } + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]), nil +} + func (m *SecurityManager) publishGeneration(ctx context.Context, s *securitySnapshot) error { if m.kube == nil || m.configMap == "" { return errors.New("security authority ConfigMap is required") @@ -416,7 +521,7 @@ func (m *SecurityManager) readSecurityLedger(ctx context.Context, cms corev1clie } ledger := securityLedger{Fingerprints: map[string]string{}} if raw := cm.Data[securityStateDataKey]; raw != "" { - if err := json.Unmarshal([]byte(raw), &ledger); err != nil || ledger.Fingerprints == nil { + if err := executionenv.DecodeStrict([]byte(raw), &ledger); err != nil || ledger.Generation == 0 || ledger.Digest == "" || ledger.Fingerprints == nil || len(ledger.Fingerprints) > maxSecurityKeys { return nil, false, securityLedger{}, errors.New("security authority state is invalid") } } @@ -428,10 +533,10 @@ func advanceSecurityLedger(ledger *securityLedger, s *securitySnapshot) (bool, e return false, errors.New("security manifest generation rollback rejected") } if s.generation == ledger.Generation && ledger.Generation != 0 { - if !containsFingerprints(ledger.Fingerprints, s.fingerprints) { - return false, errors.New("security generation fingerprint drift rejected") + if ledger.Digest != s.digest || !containsFingerprints(ledger.Fingerprints, s.fingerprints) { + return false, errors.New("security generation authority drift rejected") } - s.fingerprints = cloneFingerprints(ledger.Fingerprints) + s.fingerprints = maps.Clone(ledger.Fingerprints) return false, nil } for identity, fp := range s.fingerprints { @@ -448,7 +553,8 @@ func advanceSecurityLedger(ledger *securityLedger, s *securitySnapshot) (bool, e return false, errors.New("security key tombstone limit reached; rotate issuer") } ledger.Generation = s.generation - s.fingerprints = cloneFingerprints(ledger.Fingerprints) + ledger.Digest = s.digest + s.fingerprints = maps.Clone(ledger.Fingerprints) return true, nil } @@ -494,13 +600,6 @@ func validKeyID(v string) bool { } return true } -func ensureJSONEOF(dec *json.Decoder) error { - var extra any - if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { - return errors.New("security manifest has trailing data") - } - return nil -} func hasUsage(usages []x509.ExtKeyUsage, wanted x509.ExtKeyUsage) bool { for _, u := range usages { if u == wanted || u == x509.ExtKeyUsageAny { @@ -518,14 +617,6 @@ func containsFingerprints(authority, candidate map[string]string) bool { return true } -func cloneFingerprints(source map[string]string) map[string]string { - clone := make(map[string]string, len(source)) - for identity, fingerprint := range source { - clone[identity] = fingerprint - } - return clone -} - func keyVersionAdvances(authority map[string]string, identity string) bool { separator := strings.LastIndexByte(identity, ':') if separator <= 0 { @@ -548,20 +639,3 @@ func keyVersionAdvances(authority map[string]string, identity string) bool { } return true } - -func sameFingerprints(a, b map[string]string) bool { - if len(a) != len(b) { - return false - } - keys := make([]string, 0, len(a)) - for k := range a { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - if a[k] != b[k] { - return false - } - } - return true -} diff --git a/internal/adapter/executioncontroller/security_test.go b/internal/adapter/executioncontroller/security_test.go index e427b65621..e0e22321fb 100644 --- a/internal/adapter/executioncontroller/security_test.go +++ b/internal/adapter/executioncontroller/security_test.go @@ -4,21 +4,34 @@ import ( "crypto/ed25519" "crypto/rand" "crypto/sha256" + "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/hex" "encoding/json" "encoding/pem" "math/big" + "net" "net/url" "os" "path/filepath" + "slices" + "strings" "testing" "time" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/fake" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" ) func TestSecurityManagerReloadRollbackAndRecovery(t *testing.T) { @@ -58,6 +71,16 @@ func TestSecurityManagerReloadRollbackAndRecovery(t *testing.T) { if !manager.Ready() { t.Fatal("valid snapshot is not ready") } + manifest.Clients[0].Administrator = false + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("same-generation client policy drift was accepted") + } + manifest.Clients[0].Administrator = true + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("restore same generation: %v", err) + } manifest.Generation = 0 writeManifest(t, manifestPath, manifest) if err := manager.Reload(t.Context()); err == nil || manager.Ready() { @@ -126,6 +149,229 @@ func TestSecurityLedgerRejectsKeyVersionRollbackAcrossRestart(t *testing.T) { } } +func TestSecurityManagerGuardsEveryRPCOnExistingConnection(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + caPEM, serverPEM, serverKeyPEM, clientCert := rpcSecurityPKI(t, now, "spiffe://example/client") + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + store := NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), "ns", testProfiles(), nil) + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig()))) + executionv1.RegisterExecutionProviderServiceServer(server, NewHandler(HandlerConfig{Security: manager}, store)) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(caPEM) + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "provider.test", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + client := executionv1.NewExecutionProviderServiceClient(conn) + call := func() error { + _, err := client.ValidateProfile(t.Context(), &executionv1.ValidateProfileRequest{Profile: "go"}) + return err + } + if err := call(); err != nil { + t.Fatalf("initial RPC: %v", err) + } + cm, err := kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + authoritativeState := cm.Data[securityStateDataKey] + var drifted securityLedger + if err := executionenv.DecodeStrict([]byte(authoritativeState), &drifted); err != nil { + t.Fatal(err) + } + drifted.Digest = strings.Repeat("0", 64) + raw, _ := json.Marshal(drifted) + cm.Data[securityStateDataKey] = string(raw) + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := call(); status.Code(err) != codes.Unauthenticated || manager.Ready() { + t.Fatalf("old snapshot survived durable authority drift: ready=%v err=%v", manager.Ready(), err) + } + cm, _ = kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + cm.Data[securityStateDataKey] = authoritativeState + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + newCAPEM, newServerPEM, newServerKeyPEM, _ := rpcSecurityPKI(t, now, "spiffe://example/other") + for name, contents := range map[string][]byte{"server.crt": newServerPEM, "server.key": newServerKeyPEM, "clients.pem": newCAPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + manifest.Generation = 2 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if err := call(); status.Code(err) != codes.Unauthenticated { + t.Fatalf("old connection survived CA rotation: %v", err) + } + if err := os.WriteFile(manifestPath, []byte(`{"version":2}`), 0o600); err != nil { + t.Fatal(err) + } + if err := manager.Reload(t.Context()); err == nil { + t.Fatal("invalid manifest reloaded") + } + if err := call(); status.Code(err) != codes.Unavailable { + t.Fatalf("RPC remained authorized during invalid manifest: %v", err) + } + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + manifest.Generation = 3 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || call() != nil { + t.Fatalf("valid generation did not recover existing connection: %v", err) + } +} + +func rpcSecurityPKI(t *testing.T, now time.Time, clientURI string) ([]byte, []byte, []byte, tls.Certificate) { + t.Helper() + caPub, caKey, _ := ed25519.GenerateKey(rand.Reader) + ca := &x509.Certificate{SerialNumber: big.NewInt(101), NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, caPub, caKey) + if err != nil { + t.Fatal(err) + } + issue := func(serial int64, uriText string, usages []x509.ExtKeyUsage, dns []string) ([]byte, ed25519.PrivateKey) { + pub, key, _ := ed25519.GenerateKey(rand.Reader) + tmpl := &x509.Certificate{SerialNumber: big.NewInt(serial), NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: usages, DNSNames: dns} + if uriText != "" { + u, _ := url.Parse(uriText) + tmpl.URIs = []*url.URL{u} + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca, pub, caKey) + if err != nil { + t.Fatal(err) + } + return der, key + } + serverDER, serverKey := issue(102, "spiffe://example/provider", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, []string{"provider.test"}) + clientDER, clientKey := issue(103, clientURI, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, nil) + encodeKey := func(key ed25519.PrivateKey) []byte { + raw, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: raw}) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), encodeKey(serverKey), tls.Certificate{Certificate: [][]byte{clientDER}, PrivateKey: clientKey} +} + +func TestAuthorityDigestCoversSecurityMeaningfulState(t *testing.T) { + base := securityManifest{Version: 1, Generation: 7, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "key.pem", PublicSHA256: "fingerprint", ActivateAt: time.Unix(1, 0).UTC(), VerifyUntil: time.Unix(100, 0).UTC(), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "ca.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client"}}} + digest := func(m securityManifest, ca, server string) string { + t.Helper() + ttl, err := time.ParseDuration(m.GrantTTLText) + if err != nil { + t.Fatal(err) + } + skew, err := time.ParseDuration(m.ClockSkewText) + if err != nil { + t.Fatal(err) + } + got, err := authorityDigest(m, ttl, skew, map[string]string{"k1:1": "fingerprint"}, []string{ca}, server) + if err != nil { + t.Fatal(err) + } + return got + } + original := digest(base, "ca-1", "server-1") + mutations := map[string]func(*securityManifest){ + "capability": func(m *securityManifest) { m.Clients[0].Administrator = true }, + "key revoke": func(m *securityManifest) { m.Keys[0].State = "revoked" }, + "active key": func(m *securityManifest) { m.ActiveKeyID = "k2" }, + "activation time": func(m *securityManifest) { m.Keys[0].ActivateAt = m.Keys[0].ActivateAt.Add(time.Second) }, + "grant ttl": func(m *securityManifest) { m.GrantTTLText = "2m" }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + candidate := base + candidate.Keys = slices.Clone(base.Keys) + candidate.Clients = slices.Clone(base.Clients) + mutate(&candidate) + if digest(candidate, "ca-1", "server-1") == original { + t.Fatal("mutation did not change authority digest") + } + }) + } + if digest(base, "ca-2", "server-1") == original || digest(base, "ca-1", "server-2") == original { + t.Fatal("CA or server identity swap did not change authority digest") + } +} + +func TestSecurityManagerSigningWindowIsCheckedOnEveryRequest(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + caPEM, serverCert, serverKey := syntheticPKI(t, now) + for name, contents := range map[string][]byte{"server.crt": serverCert, "server.key": serverKey, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(10 * time.Second), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + clock := now + manager.now = func() time.Time { return clock } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + h := NewHandler(HandlerConfig{Security: manager}, &fakeBackend{}) + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1} + if _, expiry, err := h.signClaim(t.Context(), claim, "spiffe://example/client", "owner"); err != nil || expiry.After(manifest.Keys[0].VerifyUntil) { + t.Fatalf("initial sign expiry=%v err=%v", expiry, err) + } + clock = manifest.Keys[0].VerifyUntil + if manager.Ready() { + t.Fatal("expired active key remained ready between reload ticks") + } + if _, _, err := h.signClaim(t.Context(), claim, "spiffe://example/client", "owner"); err == nil { + t.Fatal("expired active key signed between reload ticks") + } + manifest.Generation = 2 + manifest.Keys[0].VerifyUntil = clock.Add(time.Hour) + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("corrected generation did not recover: ready=%v err=%v", manager.Ready(), err) + } +} + func TestSecurityManagerRejectsEscapingProjectedSymlink(t *testing.T) { dir := t.TempDir() if err := os.Symlink("../../outside", filepath.Join(dir, "grant.pem")); err != nil { diff --git a/internal/adapter/executioncontroller/store.go b/internal/adapter/executioncontroller/store.go index d5b130f032..fe22fd5638 100644 --- a/internal/adapter/executioncontroller/store.go +++ b/internal/adapter/executioncontroller/store.go @@ -6,11 +6,14 @@ import ( "crypto/rand" "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "math" + "slices" "time" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -30,6 +33,7 @@ const ( currentSchemaVersion = int64(2) operationLeaseTTL = 30 * time.Second transitioningMessage = "environment is transitioning" + operationIDField = "operationID" ) // ExecutorTransport dispatches one request to a fixed workload helper. @@ -114,18 +118,8 @@ func (s *Store) ensurePending(ctx context.Context, client, owner string, atteste return Allocation{}, fmt.Errorf("get execution environment: %w", err) } if s.kube != nil { - all, listErr := s.resources.List(ctx, metav1.ListOptions{}) - if listErr != nil { - return Allocation{}, fmt.Errorf("count execution environments: %w", listErr) - } - profileCount := 0 - for i := range all.Items { - if textNested(all.Items[i].Object, "spec", "profile") == profile && textNested(all.Items[i].Object, "spec", "desired") != "Retired" { - profileCount++ - } - } - if profileCount >= p.Spec.MaxEnvironments { - return Allocation{}, &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "profile environment limit reached"} + if err := s.reserveProfileSlot(ctx, profile, name, p.Spec.MaxEnvironments); err != nil { + return Allocation{}, err } } revision, err := randomID() @@ -154,6 +148,107 @@ func (s *Store) ensurePending(ctx context.Context, client, owner string, atteste } return allocationFrom(created, client, owner, binding, profile, fp) } + +const profileAllocationConfigMap = "mecatl-execution-profile-allocations" + +func profileAllocationKey(profile string) string { + keySum := sha256.Sum256([]byte(profile)) + return "profile-" + hex.EncodeToString(keySum[:16]) + ".json" +} + +func (s *Store) reserveProfileSlot(ctx context.Context, profile, allocationID string, limit int) error { + cms := s.kube.CoreV1().ConfigMaps(s.namespace) + key := profileAllocationKey(profile) + for range 8 { + cm, err := cms.Get(ctx, profileAllocationConfigMap, metav1.GetOptions{}) + exists := true + if apierrors.IsNotFound(err) { + exists = false + cm = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: s.namespace}, Data: map[string]string{}} + } else if err != nil { + return fmt.Errorf("read profile allocation authority: %w", err) + } + var slots []string + if raw := cm.Data[key]; raw != "" { + if err := executionenv.DecodeStrict([]byte(raw), &slots); err != nil || len(slots) > limit { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "profile allocation authority is invalid"} + } + } + if slices.Contains(slots, allocationID) { + return nil + } + all, err := s.resources.List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("list profile allocations: %w", err) + } + for i := range all.Items { + if textNested(all.Items[i].Object, "spec", "profile") == profile && textNested(all.Items[i].Object, "spec", "desired") != "Retired" && !slices.Contains(slots, all.Items[i].GetName()) { + slots = append(slots, all.Items[i].GetName()) + } + } + if len(slots) >= limit { + return &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "profile environment limit reached"} + } + slots = append(slots, allocationID) + slices.Sort(slots) + raw, err := json.Marshal(slots) + if err != nil { + return err + } + if cm.Data == nil { + cm.Data = map[string]string{} + } + cm.Data[key] = string(raw) + if exists { + _, err = cms.Update(ctx, cm, metav1.UpdateOptions{}) + } else { + _, err = cms.Create(ctx, cm, metav1.CreateOptions{}) + } + if err == nil { + return nil + } + if !apierrors.IsConflict(err) && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("reserve profile allocation: %w", err) + } + } + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "profile allocation authority changed concurrently", Retryable: true} +} + +func releaseProfileSlot(ctx context.Context, kube kubernetes.Interface, namespace, profile, allocationID string) error { + cms := kube.CoreV1().ConfigMaps(namespace) + key := profileAllocationKey(profile) + for range 8 { + cm, err := cms.Get(ctx, profileAllocationConfigMap, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + var slots []string + if err := executionenv.DecodeStrict([]byte(cm.Data[key]), &slots); err != nil { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "profile allocation authority is invalid"} + } + index := slices.Index(slots, allocationID) + if index < 0 { + return nil + } + slots = append(slots[:index], slots[index+1:]...) + raw, err := json.Marshal(slots) + if err != nil { + return err + } + cm.Data[key] = string(raw) + if _, err = cms.Update(ctx, cm, metav1.UpdateOptions{}); err == nil { + return nil + } + if !apierrors.IsConflict(err) { + return err + } + } + return errors.New("profile allocation authority changed concurrently") +} + func (s *Store) initializeStatus(ctx context.Context, env *unstructured.Unstructured, binding, operationID string) error { if operationID == "" { return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "operation identity is required"} @@ -177,7 +272,7 @@ func (s *Store) initializeStatus(ctx context.Context, env *unstructured.Unstruct if err := unstructured.SetNestedField(o.Object, int64(1), "status", "grantGeneration"); err != nil { return err } - ref := map[string]any{"bindingID": binding, "state": string(executionenv.ReferencePendingCreate), "operationID": operationID, "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} + ref := map[string]any{"bindingID": binding, "state": string(executionenv.ReferencePendingCreate), operationIDField: operationID, "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} if err := unstructured.SetNestedSlice(o.Object, []any{ref}, "status", "references"); err != nil { return err } diff --git a/internal/adapter/executioncontroller/store_concurrency_test.go b/internal/adapter/executioncontroller/store_concurrency_test.go new file mode 100644 index 0000000000..e8c1409359 --- /dev/null +++ b/internal/adapter/executioncontroller/store_concurrency_test.go @@ -0,0 +1,157 @@ +package executioncontroller + +import ( + "context" + "errors" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestConcurrentProfileReservationsEnforceHardLimit(t *testing.T) { + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + ledger := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns", ResourceVersion: "1"}, Data: map[string]string{}} + kubes := []*kubefake.Clientset{kubefake.NewSimpleClientset(ledger), kubefake.NewSimpleClientset(ledger)} + var mu sync.Mutex + current := ledger.DeepCopy() + barrier := make(chan struct{}) + var reads atomic.Int32 + for _, kube := range kubes { + kube.PrependReactor("get", "configmaps", func(k8stesting.Action) (bool, runtime.Object, error) { + mu.Lock() + out := current.DeepCopy() + mu.Unlock() + if reads.Add(1) == 2 { + close(barrier) + } + if reads.Load() <= 2 { + <-barrier + } + return true, out, nil + }) + kube.PrependReactor("update", "configmaps", func(action k8stesting.Action) (bool, runtime.Object, error) { + candidate := action.(k8stesting.UpdateAction).GetObject().(*corev1.ConfigMap) + mu.Lock() + defer mu.Unlock() + if candidate.ResourceVersion != current.ResourceVersion { + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "configmaps"}, candidate.Name, errors.New("resource version changed")) + } + rv, _ := strconv.Atoi(current.ResourceVersion) + current = candidate.DeepCopy() + current.ResourceVersion = strconv.Itoa(rv + 1) + return true, current.DeepCopy(), nil + }) + } + profiles := testProfiles() + profile := profiles.byName["go"] + profile.Spec.MaxEnvironments = 1 + profiles.byName["go"] = profile + stores := []*Store{NewStore(dynamicClient, "ns", profiles, nil).WithKubeClient(kubes[0]), NewStore(dynamicClient, "ns", profiles, nil).WithKubeClient(kubes[1])} + results := make(chan error, 2) + for i := range stores { + go func(i int) { + _, err := stores[i].EnsurePending(context.Background(), "client", "owner", "binding-"+strconv.Itoa(i), "go", "fp-"+strconv.Itoa(i), "op-"+strconv.Itoa(i)) + results <- err + }(i) + } + admitted, exhausted := 0, 0 + for range 2 { + err := <-results + var controlled *executionenv.Error + if err == nil { + admitted++ + } else if errors.As(err, &controlled) && controlled.Code == executionenv.CodeResourceExhausted { + exhausted++ + } else { + t.Fatalf("unexpected result: %v", err) + } + } + if admitted != 1 || exhausted != 1 { + t.Fatalf("admitted=%d exhausted=%d", admitted, exhausted) + } +} + +func TestAcquireRunAndRevokeUseResourceVersionCAS(t *testing.T) { + now := time.Date(2026, 9, 17, 5, 0, 0, 0, time.UTC) + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns", "resourceVersion": "1"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": fenceHealthy, "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} + clients := []*dynamicfake.FakeDynamicClient{ + dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), + dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), + } + var mu sync.Mutex + current := env.DeepCopy() + firstReads := make(chan struct{}) + var reads, conflicts atomic.Int32 + for _, client := range clients { + client.PrependReactor("get", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + mu.Lock() + out := current.DeepCopy() + mu.Unlock() + if reads.Add(1) == 2 { + close(firstReads) + } + if reads.Load() <= 2 { + <-firstReads + } + return true, out, nil + }) + client.PrependReactor("update", "executionenvironments", func(action k8stesting.Action) (bool, runtime.Object, error) { + candidate := action.(k8stesting.UpdateAction).GetObject().(*unstructured.Unstructured) + mu.Lock() + defer mu.Unlock() + if candidate.GetResourceVersion() != current.GetResourceVersion() { + conflicts.Add(1) + return true, nil, apierrors.NewConflict(schema.GroupResource{Group: ExecutionEnvironmentGVR.Group, Resource: ExecutionEnvironmentGVR.Resource}, candidate.GetName(), errors.New("resource version changed")) + } + rv, _ := strconv.Atoi(current.GetResourceVersion()) + candidate = candidate.DeepCopy() + candidate.SetResourceVersion(strconv.Itoa(rv + 1)) + current = candidate + return true, candidate.DeepCopy(), nil + }) + } + first := NewStore(clients[0], "ns", testProfiles(), nil) + second := NewStore(clients[1], "ns", testProfiles(), nil) + first.now, second.now = func() time.Time { return now }, func() time.Time { return now } + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + results := make(chan error, 2) + go func() { + _, err := first.AcquireRun(context.Background(), ref, "client", "owner", "binding", "run", "acquire", executionenv.MinRunTTL) + results <- err + }() + go func() { + _, err := second.RevokeEnvironment(context.Background(), ref, "client", "owner", 1, "revoke") + results <- err + }() + for range 2 { + if err := <-results; err != nil { + t.Fatalf("CAS retry did not converge: %v", err) + } + } + if conflicts.Load() == 0 { + t.Fatal("test did not force a resource-version conflict") + } + mu.Lock() + final := current.DeepCopy() + mu.Unlock() + if generation := intNested(final.Object, "status", "grantGeneration"); generation != 2 { + t.Fatalf("grant generation=%d, want 2", generation) + } + if claim, _, _, ok := activeRunFrom(final); ok && generationMatches(final, claim.GrantGeneration) && claim.GrantGeneration != 2 { + t.Fatalf("stale claim remained current after revoke: %+v", claim) + } +} diff --git a/internal/adapter/executioncontroller/store_lifecycle.go b/internal/adapter/executioncontroller/store_lifecycle.go index 0c0e51b49e..537edb78f3 100644 --- a/internal/adapter/executioncontroller/store_lifecycle.go +++ b/internal/adapter/executioncontroller/store_lifecycle.go @@ -32,7 +32,7 @@ func referenceRecords(o *unstructured.Unstructured) ([]referenceRecord, error) { return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "legacy environment references require explicit migration"} } created, parseErr := time.Parse(time.RFC3339Nano, text(m, "createdAt")) - r := referenceRecord{BindingID: text(m, "bindingID"), State: executionenv.ReferenceState(text(m, "state")), OperationID: text(m, "operationID"), SourceBindingID: text(m, "sourceBindingID"), CreatedAt: created} + r := referenceRecord{BindingID: text(m, "bindingID"), State: executionenv.ReferenceState(text(m, "state")), OperationID: text(m, operationIDField), SourceBindingID: text(m, "sourceBindingID"), CreatedAt: created} if parseErr != nil || r.BindingID == "" || r.OperationID == "" || (r.State != executionenv.ReferencePendingCreate && r.State != executionenv.ReferencePublished && r.State != executionenv.ReferencePendingDelete) { return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment references are invalid"} } @@ -45,7 +45,7 @@ func setReferenceRecords(o *unstructured.Unstructured, refs []referenceRecord) e sort.Slice(refs, func(i, j int) bool { return refs[i].BindingID < refs[j].BindingID }) raw := make([]any, len(refs)) for i, r := range refs { - raw[i] = map[string]any{"bindingID": r.BindingID, "state": string(r.State), "operationID": r.OperationID, "sourceBindingID": r.SourceBindingID, "createdAt": r.CreatedAt.UTC().Format(time.RFC3339Nano)} + raw[i] = map[string]any{"bindingID": r.BindingID, "state": string(r.State), operationIDField: r.OperationID, "sourceBindingID": r.SourceBindingID, "createdAt": r.CreatedAt.UTC().Format(time.RFC3339Nano)} } return unstructured.SetNestedSlice(o.Object, raw, "status", "references") } diff --git a/internal/adapter/executioncontroller/store_lifecycle_test.go b/internal/adapter/executioncontroller/store_lifecycle_test.go index 697cc3e63e..1f1ba14770 100644 --- a/internal/adapter/executioncontroller/store_lifecycle_test.go +++ b/internal/adapter/executioncontroller/store_lifecycle_test.go @@ -3,6 +3,7 @@ package executioncontroller import ( "context" "errors" + "reflect" "testing" "time" @@ -75,6 +76,56 @@ func TestExpiredClaimWithActiveOperationCannotBeReplaced(t *testing.T) { } } +func TestClientScopedIntentReconciliationRejectsEachOwnerMutationWithoutSideEffects(t *testing.T) { + now := time.Now().UTC() + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + mutations := map[string]func(*unstructured.Unstructured){ + "owner hash": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, hashText("other-owner"), "spec", "ownerHash") + }, + "owner issuer": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, "other-issuer", "spec", "ownerIssuer") + }, + "owner subject": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, "mallory", "spec", "ownerSubject") + }, + "client binding": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, hashText("other-client"), "spec", "clientHash") + }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + env := lifecycleEnvironment(now) + _ = unstructured.SetNestedField(env.Object, ownerHash(owner), "spec", "ownerHash") + _ = unstructured.SetNestedField(env.Object, owner.Issuer, "spec", "ownerIssuer") + _ = unstructured.SetNestedField(env.Object, owner.Subject, "spec", "ownerSubject") + refs, err := referenceRecords(env) + if err != nil { + t.Fatal(err) + } + refs = append(refs, referenceRecord{BindingID: "pending", State: executionenv.ReferencePendingDelete, OperationID: "delete", CreatedAt: now}) + if err := setReferenceRecords(env, refs); err != nil { + t.Fatal(err) + } + mutate(env) + before := env.DeepCopy() + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + intents, listErr := store.ListReferenceIntentsForClient(t.Context(), "client", 64) + if listErr == nil && len(intents) != 0 { + t.Fatalf("mutated identity exposed intents: %+v", intents) + } + after, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before.Object, after.Object) { + t.Fatal("negative reconciliation mutated durable state") + } + }) + } +} + func TestClientScopedIntentListReturnsAttestedOwnerOnlyToOwningClient(t *testing.T) { now := time.Now().UTC() env := lifecycleEnvironment(now) diff --git a/internal/adapter/executioncontroller/store_revoke.go b/internal/adapter/executioncontroller/store_revoke.go index 1a97dc5399..12ffe95897 100644 --- a/internal/adapter/executioncontroller/store_revoke.go +++ b/internal/adapter/executioncontroller/store_revoke.go @@ -3,6 +3,7 @@ package executioncontroller import ( "context" + "fmt" "math" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -12,20 +13,52 @@ import ( // RevokeEnvironment atomically invalidates every issued grant without claiming // that an already-dispatched executor operation has stopped. -func (s *Store) RevokeEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, client, owner string, expected uint64) (uint64, error) { - if expected == 0 || expected >= math.MaxInt64 { - return 0, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "grant generation is invalid"} +func (s *Store) RevokeEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, client, owner string, expected uint64, operationID string) (uint64, error) { + if expected == 0 || expected >= math.MaxInt64 || operationID == "" || len(operationID) > executionenv.MaxIdentityBytes { + return 0, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "grant generation or operation identity is invalid"} } + requestFingerprint := fingerprint(ref.ID, ref.Revision, client, owner, fmt.Sprint(expected)) var next uint64 err := s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} } + receipts, found, err := unstructured.NestedSlice(o.Object, "status", "revocationReceipts") + if err != nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "revocation receipt ledger is invalid"} + } + if found { + for _, raw := range receipts { + receipt, ok := raw.(map[string]any) + if !ok { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "revocation receipt ledger is invalid"} + } + if text(receipt, operationIDField) != operationID { + continue + } + if text(receipt, "fingerprint") != requestFingerprint { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "operation identity was reused for a different revocation"} + } + generation, ok := receipt["grantGeneration"].(int64) + if !ok || generation <= 0 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "revocation receipt ledger is invalid"} + } + next = uint64(generation) + return nil + } + } current := intNested(o.Object, "status", "grantGeneration") if current <= 0 || uint64(current) != expected { //nolint:gosec // positivity is checked before conversion. return &executionenv.Error{Code: executionenv.CodeConflict, Message: "grant generation changed concurrently"} } next = expected + 1 + receipts = append(receipts, map[string]any{operationIDField: operationID, "fingerprint": requestFingerprint, "expectedGrantGeneration": int64(expected), "grantGeneration": int64(next)}) //nolint:gosec // values are bounded above. + if len(receipts) > 32 { + receipts = receipts[len(receipts)-32:] + } + if err := unstructured.SetNestedSlice(o.Object, receipts, "status", "revocationReceipts"); err != nil { + return err + } return unstructured.SetNestedField(o.Object, int64(next), "status", "grantGeneration") //nolint:gosec // expected is bounded below MaxInt64. }) return next, err diff --git a/internal/adapter/executioncontroller/store_revoke_test.go b/internal/adapter/executioncontroller/store_revoke_test.go index c04e4a382d..5a68dfd8c9 100644 --- a/internal/adapter/executioncontroller/store_revoke_test.go +++ b/internal/adapter/executioncontroller/store_revoke_test.go @@ -22,7 +22,7 @@ func TestRevokeEnvironmentFencesOldClaimWithoutChangingExecutionEpoch(t *testing client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) store := NewStore(client, "ns", testProfiles(), &terminalErrorExecutor{}) ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} - next, err := store.RevokeEnvironment(t.Context(), ref, "client", "owner", 3) + next, err := store.RevokeEnvironment(t.Context(), ref, "client", "owner", 3, "revoke-1") if err != nil || next != 4 { t.Fatalf("revoke=(%d,%v)", next, err) } @@ -42,7 +42,11 @@ func TestRevokeEnvironmentFencesOldClaimWithoutChangingExecutionEpoch(t *testing if _, err := store.RenewRun(t.Context(), ref, "client", "owner", executionenv.RunClaimRequest{Environment: ref, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: time.Minute}); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { t.Fatalf("old renewal error=%v", err) } - if _, err := store.RevokeEnvironment(t.Context(), ref, "client", "owner", 3); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + replayed, err := store.RevokeEnvironment(t.Context(), ref, "client", "owner", 3, "revoke-1") + if err != nil || replayed != 4 { + t.Fatalf("replayed revoke=(%d,%v)", replayed, err) + } + if _, err := store.RevokeEnvironment(t.Context(), ref, "client", "owner", 3, "revoke-2"); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { t.Fatalf("stale CAS error=%v", err) } } @@ -51,7 +55,7 @@ func TestRevokeEnvironmentOverflowFailsClosed(t *testing.T) { env := runFixtureEnvironment(1, math.MaxInt64) client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) store := NewStore(client, "ns", testProfiles(), nil) - if _, err := store.RevokeEnvironment(context.Background(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", math.MaxInt64); err == nil { + if _, err := store.RevokeEnvironment(context.Background(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", math.MaxInt64, "overflow"); err == nil { t.Fatal("overflow accepted") } } diff --git a/internal/adapter/executioncontroller/store_run.go b/internal/adapter/executioncontroller/store_run.go index a5fc13bab2..bcc1521619 100644 --- a/internal/adapter/executioncontroller/store_run.go +++ b/internal/adapter/executioncontroller/store_run.go @@ -58,7 +58,7 @@ func activeRunFrom(o *unstructured.Unstructured) (executionenv.RunClaim, string, } generation := uint64(generationValue) //nolint:gosec // positivity is checked immediately above. claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: textNested(o.Object, "spec", "revision")}, BindingID: text(m, "bindingID"), RunID: text(m, "runID"), ClaimID: text(m, "claimID"), Epoch: epoch, GrantGeneration: generation, ExpiresAt: expires} - return claim, text(m, "operationID"), expires, err == nil && epochOK && generation > 0 && claim.BindingID != "" && claim.RunID != "" && claim.ClaimID != "" + return claim, text(m, operationIDField), expires, err == nil && epochOK && generation > 0 && claim.BindingID != "" && claim.RunID != "" && claim.ClaimID != "" } //nolint:gocyclo // Run ownership admission keeps every CAS precondition in one auditable transition. @@ -93,7 +93,7 @@ func (s *Store) AcquireRun(ctx context.Context, ref executionenv.EnvironmentRef, out = current return nil } - if time.Now().UTC().Before(expires) { + if s.now().Before(expires) { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment has an active run"} } if textNested(o.Object, "status", "activeOperation", "id") != "" { @@ -113,7 +113,7 @@ func (s *Store) AcquireRun(ctx context.Context, ref executionenv.EnvironmentRef, if idErr != nil { return idErr } - expires := time.Now().UTC().Add(ttl) + expires := s.now().Add(ttl) generation := intNested(o.Object, "status", "grantGeneration") if generation <= 0 { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "grant generation is invalid"} @@ -121,7 +121,7 @@ func (s *Store) AcquireRun(ctx context.Context, ref executionenv.EnvironmentRef, if setErr := unstructured.SetNestedField(o.Object, epochStatus, "status", "epoch"); setErr != nil { return setErr } - if setErr := unstructured.SetNestedMap(o.Object, map[string]any{"bindingID": binding, "runID": runID, "claimID": claimID, "operationID": operationID, "ownerHash": owner, "clientHash": hashText(client), "epoch": epochStatus, "grantGeneration": generation, "expiresAt": expires.Format(time.RFC3339Nano)}, "status", "activeRun"); setErr != nil { + if setErr := unstructured.SetNestedMap(o.Object, map[string]any{"bindingID": binding, "runID": runID, "claimID": claimID, operationIDField: operationID, "ownerHash": owner, "clientHash": hashText(client), "epoch": epochStatus, "grantGeneration": generation, "expiresAt": expires.Format(time.RFC3339Nano)}, "status", "activeRun"); setErr != nil { return setErr } out = executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: runID, ClaimID: claimID, Epoch: epoch, GrantGeneration: uint64(generation), ExpiresAt: expires} @@ -137,11 +137,11 @@ func (s *Store) RenewRun(ctx context.Context, ref executionenv.EnvironmentRef, c } var out executionenv.RunClaim err = s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { - cur, _, _, ok := activeRunFrom(o) - if !ok || textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || cur.Environment != ref || cur.BindingID != req.BindingID || cur.RunID != req.RunID || cur.ClaimID != req.ClaimID || cur.Epoch != req.Epoch || cur.GrantGeneration != req.GrantGeneration || !generationMatches(o, req.GrantGeneration) { + cur, _, expiry, ok := activeRunFrom(o) + if !ok || !s.now().Before(expiry) || textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || cur.Environment != ref || cur.BindingID != req.BindingID || cur.RunID != req.RunID || cur.ClaimID != req.ClaimID || cur.Epoch != req.Epoch || cur.GrantGeneration != req.GrantGeneration || !generationMatches(o, req.GrantGeneration) { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim mismatch"} } - expires := time.Now().UTC().Add(ttl) + expires := s.now().Add(ttl) m, _, _ := unstructured.NestedMap(o.Object, "status", "activeRun") m["expiresAt"] = expires.Format(time.RFC3339Nano) m["lastRenewOperationID"] = req.OperationID diff --git a/internal/adapter/executioncontroller/store_test.go b/internal/adapter/executioncontroller/store_test.go index 197224b96f..db0a0171c4 100644 --- a/internal/adapter/executioncontroller/store_test.go +++ b/internal/adapter/executioncontroller/store_test.go @@ -14,6 +14,25 @@ import ( "github.com/stacklok/mecatl/internal/executionenv" ) +func TestRenewRunRejectsExpiredClaimAtBoundary(t *testing.T) { + now := time.Date(2026, 9, 17, 4, 0, 0, 0, time.UTC) + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(7), "grantGeneration": int64(3), "fenceState": fenceHealthy, "references": []any{}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "epoch": int64(7), "grantGeneration": int64(3), "expiresAt": now.Format(time.RFC3339Nano)}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + store.now = func() time.Time { return now } + req := executionenv.RunClaimRequest{BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: executionenv.MinRunTTL} + if _, err := store.RenewRun(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", req); err == nil { + t.Fatal("renewal at the expiry boundary resurrected the claim") + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if expiry := textNested(got.Object, "status", "activeRun", "expiresAt"); expiry != now.Format(time.RFC3339Nano) { + t.Fatalf("expired claim changed to %q", expiry) + } +} + func TestStoreEnsureUsesStableLookupAndRejectsFingerprintDrift(t *testing.T) { client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) profiles := &Profiles{byName: map[string]resolvedProfile{"go": {Digest: "sha256:profile", Spec: ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100}}, "other": {Digest: "sha256:other", Spec: ProfileSpec{Image: "example@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100}}}} diff --git a/internal/adapter/server/native_execution_lifecycle_test.go b/internal/adapter/server/native_execution_lifecycle_test.go index bc11cea4e1..3dc5999b03 100644 --- a/internal/adapter/server/native_execution_lifecycle_test.go +++ b/internal/adapter/server/native_execution_lifecycle_test.go @@ -87,6 +87,9 @@ func TestNativeRunClaimHeldUntilActualDrainRelease(t *testing.T) { for range second.Events() { } svc.FinishRun(sess.ID, second) + svc.FinishRun(sess.ID, second) + svc.Close() + svc.Close() if provider.acquires.Load() != 2 || provider.releases.Load() != 2 { t.Fatalf("after continuation acquires=%d releases=%d", provider.acquires.Load(), provider.releases.Load()) } diff --git a/internal/adapter/server/reference_intent_reconcile_test.go b/internal/adapter/server/reference_intent_reconcile_test.go index 6933cb2a87..fd0d694113 100644 --- a/internal/adapter/server/reference_intent_reconcile_test.go +++ b/internal/adapter/server/reference_intent_reconcile_test.go @@ -80,7 +80,7 @@ func TestReferenceIntentReconciliationIsOwnerAndRefExact(t *testing.T) { if len(lifecycle.confirmed) != 1 || lifecycle.confirmed[0] != "delete-gone" { t.Fatalf("confirmed=%v", lifecycle.confirmed) } - if len(lifecycle.reset) != 1 || lifecycle.reset[0] != "delete-restored" { - t.Fatalf("cancelled=%v", lifecycle.reset) + if len(lifecycle.reset) != 0 { + t.Fatalf("ambiguous present delete was cancelled=%v", lifecycle.reset) } } diff --git a/internal/adapter/server/service.go b/internal/adapter/server/service.go index ae6c16fd55..68fa873f40 100644 --- a/internal/adapter/server/service.go +++ b/internal/adapter/server/service.go @@ -1303,7 +1303,11 @@ type runState struct { // run has settled. It must outlive the run-entry call itself. runContextStop context.CancelFunc execution ExecutionRunHandle + executionMu sync.Mutex + executionClosed bool executionRenewCancel context.CancelFunc + executionRenewDone chan struct{} + executionTeardown sync.Once awaiting atomic.Bool // titleRevision is the last title metadata revision successfully persisted and // published for this run. It starts from the admitted durable snapshot so @@ -3027,6 +3031,7 @@ func (s *Service) Close() { } else if admissionCancel != nil { admissionCancel() } + s.teardownExecution(rs) } s.waitDetachedControlRelays() @@ -3392,10 +3397,14 @@ func (s *Service) ReconcileReferenceIntents(ctx context.Context) { continue } if intent.PendingDelete { - err = lifecycle.CancelReferenceIntentDelete(ctx, intent) - } else { - err = lifecycle.CommitReferenceIntent(ctx, intent) + // Presence after an earlier ambiguous delete attempt is not proof that the + // attempt had no effect: a delayed commit may still remove this incarnation. + // Keep the exact pending intent until absence proves deletion or an explicit + // retry obtains a deterministic outcome. + retained = true + continue } + err = lifecycle.CommitReferenceIntent(ctx, intent) if err != nil { retained = true } @@ -3689,7 +3698,6 @@ func (s *Service) DeleteSession(ctx context.Context, id session.SessionID) error s.closeSessionLocal(id) return nil } - s.cancelReferenceDelete(ctx, referenceDelete) if errors.Is(err, port.ErrPruneUnsupported) { return ErrSessionDeleteUnsupported } @@ -3760,7 +3768,6 @@ func (s *Service) DeleteSessionForRetentionCandidate(ctx context.Context, candid } deleted, err := deleter.DeleteSessionIfUnchanged(ctx, candidate) if err != nil { - s.cancelReferenceDelete(ctx, referenceDelete) if errors.Is(err, port.ErrPruneUnsupported) { return ErrSessionDeleteUnsupported } @@ -3848,7 +3855,6 @@ func (s *Service) DeleteSessionForRetention(ctx context.Context, id session.Sess s.closeSessionLocal(id) return nil } - s.cancelReferenceDelete(ctx, referenceDelete) if errors.Is(err, port.ErrPruneUnsupported) { return ErrSessionDeleteUnsupported } @@ -4710,7 +4716,7 @@ func (s *Service) startRunContent(ctx context.Context, id session.SessionID, tex if registered, ok := s.LookupRun(id); ok { s.mu.Lock() st := s.runs[id] - nativeUndrained := st != nil && st.run == registered && st.execution != nil + nativeUndrained := st != nil && st.run == registered && s.cfg.ExecutionAccess != nil && s.cfg.ExecutionAccess.Applies(sess.EnvironmentRef) s.mu.Unlock() if !sess.State.IsTerminal() || nativeUndrained { return nil, fmt.Errorf("%w: session %q still has an active or undrained run", ErrFailedPrecondition, id) @@ -7981,13 +7987,7 @@ func (s *Service) cleanupRunAdmission(id session.SessionID, st *runState, promot return } s.cancelRegisteredRunState(id, st, nil, true) - if st.execution != nil { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - if err := st.execution.Release(ctx); err != nil { - s.logDiscoveryError(context.Background(), "execution admission rollback failed", err) - } - cancel() - } + s.teardownExecution(st) s.removeRunState(id, st) } @@ -8082,17 +8082,31 @@ func (s *Service) acquireExecution(ctx context.Context, st *runState, sess *sess _ = handle.Release(context.WithoutCancel(ctx)) return tool.Environment{}, ErrInvalidPlacementBinding } + st.executionMu.Lock() + if st.executionClosed { + st.executionMu.Unlock() + _ = handle.Release(context.WithoutCancel(ctx)) + return tool.Environment{}, ErrPlacementUnavailable + } st.execution = handle + st.executionMu.Unlock() return env, nil } func (s *Service) startExecutionRenewal(_ session.SessionID, st *runState, run *agent.Run) { - if st.execution == nil { + st.executionMu.Lock() + if st.execution == nil || st.executionClosed { + st.executionMu.Unlock() return } ctx, cancel := context.WithCancel(context.Background()) st.executionRenewCancel = cancel + st.executionRenewDone = make(chan struct{}) + done := st.executionRenewDone + handle := st.execution + st.executionMu.Unlock() go func() { + defer close(done) ticker := time.NewTicker(20 * time.Second) defer ticker.Stop() for { @@ -8101,7 +8115,7 @@ func (s *Service) startExecutionRenewal(_ session.SessionID, st *runState, run * return case <-ticker.C: renewCtx, stop := context.WithTimeout(ctx, 10*time.Second) - err := st.execution.Renew(renewCtx) + err := handle.Renew(renewCtx) stop() if err != nil { s.logDiscoveryError(context.Background(), "execution ownership renewal failed", err) @@ -8113,6 +8127,33 @@ func (s *Service) startExecutionRenewal(_ session.SessionID, st *runState, run * }() } +func (s *Service) teardownExecution(st *runState) { + if st == nil { + return + } + st.executionTeardown.Do(func() { + st.executionMu.Lock() + st.executionClosed = true + cancelRenew := st.executionRenewCancel + done := st.executionRenewDone + handle := st.execution + st.executionMu.Unlock() + if cancelRenew != nil { + cancelRenew() + } + if done != nil { + <-done + } + if handle != nil { + releaseCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := handle.Release(releaseCtx); err != nil { + s.logDiscoveryError(context.Background(), "execution ownership release failed", err) + } + cancel() + } + }) +} + // FinishRun removes run from the in-flight registry for id. It is the EXPORTED // counterpart of register that every wire adapter must call (typically via // `defer`) once it has finished draining run.Events(), so a completed run does @@ -8132,16 +8173,7 @@ func (s *Service) FinishRun(id session.SessionID, run *agent.Run) { if parked && st.sess != nil { pending, pendingOK = st.sess.PendingAuthorization() } - if st.executionRenewCancel != nil { - st.executionRenewCancel() - } - if st.execution != nil { - releaseCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - if err := st.execution.Release(releaseCtx); err != nil { - s.logDiscoveryError(context.Background(), "execution ownership release failed", err) - } - cancel() - } + s.teardownExecution(st) s.removeRunState(id, st) if parked { s.scheduleAuthorizationExpiry(id, pending, pendingOK) diff --git a/user-docs/building/deployment/mecak8s.md b/user-docs/building/deployment/mecak8s.md index 626dd04b7c..1f76649e62 100644 --- a/user-docs/building/deployment/mecak8s.md +++ b/user-docs/building/deployment/mecak8s.md @@ -151,32 +151,33 @@ delegation cannot add filesystem access that the parent lacks. See [Execution environments](/features/execution-environments.md) for the shared placement, no-FS, child-environment, and reattachment model. -### Experimental Kubernetes execution provider - -The repository contains a draft, experimental execution provider for an -operator-controlled Kubernetes cluster. It runs as a separate service and -controller with its own `mecatl-execution` chart. `mecak8s` remains a client: it -has no Pod, PVC, custom-resource, or controller management permissions. - -This production candidate provides run-wide ownership, transactional references, -controlled replacement and retirement, durable grant revocation, and reloadable -TLS and signing material. Its default-deny network policy still requires an -enforcing CNI; Kind's default kindnet is not isolation evidence. A configured -RuntimeClass is an operator input, not by itself a hostile-workload guarantee. - -Before you install it, prepare these values through your trusted image and -Secret delivery system: - -- A digest-pinned provider image. -- A digest-pinned workload image that contains `/mecatl-executor`, `/bin/sh`, - and the toolchain needed by foreground commands. -- One projected security Secret containing the provider certificate/key, client - CA bundle, and versioned Ed25519 grant keys. The chart never generates them. -- A versioned `provider.securityManifest` as documented in - [Execution environments](/features/execution-environments.md#production-security-material). +### Kubernetes execution provider + +The optional execution provider runs as a separate service and controller with its +own `mecatl-execution` chart. `mecak8s` remains a client and receives no Pod, +PVC, custom-resource, or controller management permissions. + +The provider supplies run-wide ownership, transactional references, controlled +replacement and retirement, durable grant revocation, and reloadable TLS and +signing material. Production isolation requires a CNI that enforces NetworkPolicy +and a RuntimeClass that supplies the isolation promised by your platform. + +Before installation, provide these values through your trusted image and Secret +delivery system: + +- Digest-pinned provider and workload images. The workload image must contain + `/mecatl-executor`, `/bin/sh`, and the toolchain used by foreground commands. +- A projected security Secret with the keys `grant-k1.pem`, `tls.crt`, `tls.key`, + and `clients.pem`. The Secret bytes remain externally managed. Mount rotation + uses the projected volume's `..data` link, without `subPath`. - A `mecak8s` client mTLS Secret containing `ca.crt`, `tls.crt`, and `tls.key`. -- Explicit provider-client pod/namespace selectors, API-server CIDRs, and DNS - resolver CIDRs. Do not use `0.0.0.0/0`. +- Explicit provider-client pod and namespace selectors, API-server CIDRs, and DNS + resolver CIDRs. + +The provider validates every configured RuntimeClass and StorageClass with +cluster-scoped `get` requests before it becomes ready. The chart grants those +requests only for the names present in `profiles`; it grants no cluster-wide list +or watch access. The following profile shows every required chart key. Save it as `execution-values.yaml` and replace each placeholder: @@ -187,7 +188,39 @@ provider: imagePullPolicy: IfNotPresent replicas: 2 securitySecretName: - securityManifest: '' + securityManifest: | + { + "version": 1, + "generation": 42, + "issuer": "https://execution.example.com", + "audience": "mecatl-execution", + "activeKeyID": "k1", + "grantTTL": "1m", + "clockSkew": "5s", + "keys": [{ + "id": "k1", + "version": 1, + "file": "grant-k1.pem", + "publicKeySHA256": "0000000000000000000000000000000000000000000000000000000000000000", + "activateAt": "2027-01-01T00:00:00Z", + "verifyUntil": "2027-01-02T00:00:00Z", + "state": "active" + }], + "tls": { + "certificateFile": "tls.crt", + "privateKeyFile": "tls.key", + "clientCAFile": "clients.pem" + }, + "clients": [{ + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/mecak8s", + "mayAttestOwner": true, + "administrator": false + }, { + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/execution-admin", + "mayAttestOwner": false, + "administrator": true + }] + } clientIngressSelectors: - namespaceLabels: {kubernetes.io/metadata.name: } podLabels: {app.kubernetes.io/name: mecak8s} @@ -229,10 +262,18 @@ resourceGovernance: requestsCPU: "20" requestsMemory: 40Gi requestsStorage: 500Gi + requestsEphemeralStorage: 40Gi limitsCPU: "40" limitsMemory: 80Gi + limitsEphemeralStorage: 80Gi ``` +Replace the all-zero `publicKeySHA256` with the SHA-256 fingerprint of the +Ed25519 public key corresponding to `grant-k1.pem`. Replace the sample activation +and verification dates with a current, reviewed rotation window. Increase +`generation` for every authority change, including a CA, client policy, +issuer/audience, key state, key window, or TLS identity change. + Install the provider chart separately from `mecak8s`: ```sh @@ -269,22 +310,111 @@ The provider retains one PVC per logical environment. Session deletion and chart uninstall do not delete committed workspace data. Retirement requires no live references and a terminal executor; even then, this candidate retains the PVC. A missing executor or lost terminal receipt moves the environment to -`FenceUnknown` and requires external operator fencing. There is no automated -recovery from that state. +`FenceUnknown` and requires external operator fencing. The built-in recovery RPC +accepts only a still-observable Pod in `Succeeded` or `Failed` phase with every +container terminated and the exact PVC still present. If that proof is missing, +use your platform's external fencing runbook; there is no acknowledgement flag. + +#### Run an administrative lifecycle operation + +Administrative RPCs are client-scoped. The administrator certificate must present +the same canonical URI identity that originated the environment and the request +must carry the exact owner. You can issue a separate short-lived administrator +certificate with that URI and mark the URI `administrator: true` during the +maintenance window. A namespace-wide administrator policy is not part of this +contract. + +First capture the private identity while the environment still has a reference: + +```sh +kubectl --namespace get executionenvironment \ + -o jsonpath='{.spec.revision}{"\n"}{.spec.ownerIssuer}{"\n"}{.spec.ownerSubject}{"\n"}{.status.epoch}{"\n"}{.status.pod.uid}{"\n"}{.status.pvc.uid}{"\n"}{.status.grantGeneration}{"\n"}' +``` + +Record those seven lines as ``, ``, ``, +``, ``, ``, and ``. The owner hash is +not reversible, so retain the bounded `spec.ownerIssuer` and `spec.ownerSubject` +attestation in your authorized operations record before removing the final +reference. + +Set file references to trusted, mounted mTLS material. Keep private-key bytes out +of shell arguments and manifests: + +```sh +EXECUTION_ENDPOINT=mecatl-execution..svc:8443 +CA_FILE=/var/run/secrets/mecatl-admin/ca.crt +CERT_FILE=/var/run/secrets/mecatl-admin/tls.crt +KEY_FILE=/var/run/secrets/mecatl-admin/tls.key +PROTO=contracts/proto/mecatl/execution/v1/execution.proto +``` + +Replace one executor by reusing the same operation ID for every retry: + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedExecutionEpoch":"","expectedPodUid":"","expectedPvcUid":"","operationId":"replace-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/ReplaceExecutor +``` + +To retire and then delete retained storage, call `RetireEnvironment` with the same +identity fields and a new stable operation ID. Poll until +`.status.conditions[?(@.type=="Retired")].status` is `True`, then call +`DeleteRetiredEnvironment` with the retained PVC UID and another stable operation +ID. The provider refuses either request while references, claims, identity proof, +or UID checks are incomplete. + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedExecutionEpoch":"","expectedPodUid":"","expectedPvcUid":"","operationId":"retire-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/RetireEnvironment +kubectl --namespace wait executionenvironment/ \ + --for='jsonpath={.status.conditions[?(@.type=="Retired")].status}=True' \ + --timeout=10m +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedPvcUid":"","operationId":"delete-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/DeleteRetiredEnvironment +``` + +Revoke grants with the current generation and a stable operation ID. The response +returns the new generation; retrying the identical request returns the same +receipt. An old grant is denied after this CAS succeeds. + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedGrantGeneration":"","operationId":"revoke-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/RevokeEnvironment +``` + +Migration requires proto presence for `expectedSchemaVersion`. Supply exactly `0` +or `1`, plus the exact observable Pod and PVC UIDs. Omission, an unknown version, +or missing proof is refused without changing the resource. + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedSchemaVersion":1,"expectedPodUid":"","expectedPvcUid":"","operationId":"migrate-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/MigrateEnvironment +``` Foreground command cancellation is cooperative and bounded. The helper attempts to terminate the command process group and reports a terminal receipt. If it cannot prove termination, the environment is fenced instead of admitting more work. There is no detached command status or cancellation API in this draft. -The focused qualification task is `task e2e:k8s:execution`. It requires a generic -local Go, `ko`, Podman, Kind, Helm, and `kubectl` toolchain. Optional toolbox use +The focused qualification task is `task e2e:k8s:execution`. It requires Go, +`ko`, Docker or rootless Podman, Kind, Helm, and `kubectl`. Optional toolbox use requires explicit `MECATL_EXECUTION_DEV_TOOLBOX` and `MECATL_EXECUTION_K8S_TOOLBOX` values. The task uses a synthetic OIDC issuer and the mock model provider, creates a unique state directory and Kind cluster, and -retains both for inspection. It performs no cleanup. Kind's default kindnet does -not enforce NetworkPolicy, so this flow provides no NetworkPolicy isolation -evidence. +retains both for inspection. CI invokes +`task e2e:k8s:execution:production` and automatically removes only its uniquely +owned cluster. The production target is the home for the enforcing-CNI network +and lifecycle fixture; until that fixture passes, Kind qualification is not +NetworkPolicy isolation evidence. When `execution.enabled` is `false`, the `mecak8s` chart mounts no execution mTLS Secret and passes no execution-provider flags. The separate provider chart and diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index 26878039fd..1c374de050 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -116,18 +116,64 @@ chart does not generate keys or certificates. The manifest is strict JSON with this shape: ```json -{"version":1,"generation":7,"issuer":"https://issuer.example","audience":"mecatl-execution","activeKeyID":"grant-2026-09","grantTTL":"1m","clockSkew":"5s","keys":[{"id":"grant-2026-09","version":7,"file":"grant-2026-09.pem","publicKeySHA256":"","activateAt":"2026-09-17T00:00:00Z","verifyUntil":"2026-09-17T01:00:00Z","state":"active"}],"tls":{"certificateFile":"tls.crt","privateKeyFile":"tls.key","clientCAFile":"client-ca.pem"},"clients":[{"uri":"spiffe://example/mecatl","mayAttestOwner":true,"administrator":false}]} +{ + "version": 1, + "generation": 42, + "issuer": "https://execution.example.com", + "audience": "mecatl-execution", + "activeKeyID": "k1", + "grantTTL": "1m", + "clockSkew": "5s", + "keys": [ + { + "id": "k1", + "version": 1, + "file": "grant-k1.pem", + "publicKeySHA256": "0000000000000000000000000000000000000000000000000000000000000000", + "activateAt": "2027-01-01T00:00:00Z", + "verifyUntil": "2027-01-02T00:00:00Z", + "state": "active" + } + ], + "tls": { + "certificateFile": "tls.crt", + "privateKeyFile": "tls.key", + "clientCAFile": "clients.pem" + }, + "clients": [ + { + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/mecak8s", + "mayAttestOwner": true, + "administrator": false + }, + { + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/execution-admin", + "mayAttestOwner": false, + "administrator": true + } + ] +} ``` -Use only basename file names. Kubernetes projected-volume `..data` symlinks are -supported, but paths escaping the mounted directory are rejected. Increase -`generation` for every change. Key IDs and `(id, version)` fingerprints cannot be +The all-zero fingerprint and 2027 dates are non-secret example values. Replace +them with the public-key fingerprint and a reviewed active window before use. + +Use only basename file names. The projected Secret keys in this example are +`grant-k1.pem`, `tls.crt`, `tls.key`, and `clients.pem`; an external secret manager +owns their bytes. Kubernetes projected-volume `..data` symlinks are supported, +but paths escaping the mounted directory are rejected. Increase `generation` for +every authority change. Key IDs and `(id, version)` fingerprints cannot be reused; the provider persists a bounded high-water ledger in its authority ConfigMap. Keep retired keys as `verify-only` until all grants expire, then mark them `revoked`. Invalid, incomplete, rolled-back, or newly expired material makes readiness fail and denies new RPC authorization until corrected. The provider re-verifies the peer certificate and URI policy against the current client CA on -every RPC, including RPCs on an existing HTTP/2 connection. +every RPC, including RPCs on an existing HTTP/2 connection. Profile-resource or +controller-cache startup failures terminate the provider with that bounded reason +class before it accepts traffic. A failed `/ready` response reports +`security-authority-or-expiry`. Inspect provider logs and the named RuntimeClass, +StorageClass, and authority ConfigMap metadata; the endpoint never returns key or +certificate contents. `RevokeEnvironment` is an administrator-only, exact-reference CAS. Supply the current positive grant generation; success increments it without changing the From 8d63b95ecc9b69a9f210c2100391f33bd0d08c3c Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 17 Sep 2026 10:22:10 +0200 Subject: [PATCH 07/25] fix: close lifecycle edge cases and qualify legacy migration Fix CRD admission, termination and replay edges, bounded renewal, exact intent lookup and release provenance. Add real API-server legacy migration and replica/network proof corrections. Final local production qualification is blocked by kernel keyring quota, not reported as passed. Co-Authored-By: Mecatl --- .github/scripts/release-tag-guard_test.sh | 11 + .github/workflows/release.yml | 15 +- .../go/mecatl/execution/v1/execution.pb.go | 1043 +++++++++-------- .../proto/mecatl/execution/v1/execution.proto | 7 +- .../crds/executionenvironment.yaml | 25 +- deploy/mecatl-execution-kind/run.sh | 40 +- .../aa_production_migration_test.go | 9 + .../fixture/legacyfixture/crd.yaml | 34 + .../fixture/legacyfixture/main.go | 52 + e2e/k8s_execution/fixture/netprobe/main.go | 2 +- .../production_qualification_test.go | 324 +++-- e2e/k8s_execution/qualification_test.go | 11 +- go.work.sum | 1 + internal/adapter/executionclient/client.go | 41 +- .../adapter/executionclient/client_test.go | 3 + .../executioncontroller/admin_lifecycle.go | 84 +- .../adapter/executioncontroller/controller.go | 6 + .../controller_lifecycle.go | 23 +- .../executioncontroller/controller_test.go | 34 + .../executioncontroller/crd_schema_test.go | 26 +- .../adapter/executioncontroller/handler.go | 16 +- .../executioncontroller/handler_test.go | 3 + .../legacy_fixture_kind.go | 158 +++ .../production_lifecycle_test.go | 117 +- internal/adapter/executioncontroller/store.go | 40 +- .../store_concurrency_test.go | 80 ++ .../executioncontroller/store_lifecycle.go | 24 + .../store_lifecycle_test.go | 49 + .../adapter/executioncontroller/store_run.go | 54 +- .../adapter/executioncontroller/store_test.go | 29 + .../adapter/server/execution_renewal_test.go | 33 + internal/adapter/server/service.go | 47 +- user-docs/building/deployment/mecak8s.md | 17 + 33 files changed, 1823 insertions(+), 635 deletions(-) create mode 100644 e2e/k8s_execution/aa_production_migration_test.go create mode 100644 e2e/k8s_execution/fixture/legacyfixture/crd.yaml create mode 100644 e2e/k8s_execution/fixture/legacyfixture/main.go create mode 100644 internal/adapter/executioncontroller/legacy_fixture_kind.go create mode 100644 internal/adapter/server/execution_renewal_test.go diff --git a/.github/scripts/release-tag-guard_test.sh b/.github/scripts/release-tag-guard_test.sh index 9718ad4b38..af73be63a4 100755 --- a/.github/scripts/release-tag-guard_test.sh +++ b/.github/scripts/release-tag-guard_test.sh @@ -69,4 +69,15 @@ run_fail v2.0.0 workflow_dispatch refs/heads/main "$main" run_fail v2.0.0 push refs/tags/v2.0.0 "$off_main" run_fail v1.2.3 schedule refs/heads/main "$main" +workflow="$script_dir/../workflows/release.yml" +# The privileged graph consumes only the commit emitted by a guard implementation +# checked out from protected main; event-selected refs remain untrusted data. +grep -F "if: github.event_name == 'push' || github.ref == 'refs/heads/main'" "$workflow" >/dev/null +grep -F 'ref: refs/heads/main' "$workflow" >/dev/null +if grep -F 'org.opencontainers.image.revision=${{ github.sha }}' "$workflow" >/dev/null; then + echo "release image revision still uses the event workflow SHA" >&2 + exit 1 +fi +[ "$(grep -Fc 'org.opencontainers.image.revision=${{ needs.guard.outputs.commit }}' "$workflow")" -eq 5 ] + printf 'release tag guard tests passed\n' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 95399b0da3..036daddefd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,6 +48,7 @@ jobs: # KEEPS its own inline copy as well (like the pin gate it repeats): it is the one job that # writes outside this repository, so it re-verifies rather than inheriting. guard: + if: github.event_name == 'push' || github.ref == 'refs/heads/main' name: Verify immutable release tag runs-on: ubuntu-24.04 # 10, not 5: the v0.0.35 release was LOST because an unfiltered depth-0 clone @@ -70,6 +71,9 @@ jobs: persist-credentials: false fetch-depth: 0 filter: blob:none + # The guard implementation is trusted only from protected main. The tag + # and dispatch input remain data verified by that implementation. + ref: refs/heads/main - name: Verify and check out the immutable release tag id: verify @@ -167,7 +171,7 @@ jobs: --image-refs=image-refs.txt \ --image-label=org.opencontainers.image.source=https://github.com/${{ github.repository }} \ --image-label=org.opencontainers.image.title=mecated \ - --image-label=org.opencontainers.image.revision=${{ github.sha }} \ + --image-label=org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} \ --image-label=org.opencontainers.image.vendor=Stacklok \ ./cmd/mecated # The digest ref is the same image regardless of tag; take the first. @@ -329,7 +333,7 @@ jobs: --image-label=org.stacklok.broodbox.agent=/var/run/ko/agent.yaml \ --image-label=org.opencontainers.image.source=https://github.com/${{ github.repository }} \ --image-label=org.opencontainers.image.title=mecatui \ - --image-label=org.opencontainers.image.revision=${{ github.sha }} \ + --image-label=org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} \ --image-label=org.opencontainers.image.vendor=Stacklok \ ./cmd/mecatui # The digest ref is the same image regardless of tag; take the first. @@ -486,7 +490,7 @@ jobs: --image-refs=image-refs.txt \ --image-label=org.opencontainers.image.source=https://github.com/${{ github.repository }} \ --image-label=org.opencontainers.image.title=mecak8s \ - --image-label=org.opencontainers.image.revision=${{ github.sha }} \ + --image-label=org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} \ --image-label=org.opencontainers.image.vendor=Stacklok \ ./cmd/mecak8s # The digest ref is the same image regardless of tag; take the first. @@ -603,6 +607,9 @@ jobs: tags: | ${{ env.IMAGE }}:${{ env.VERSION }} ${{ env.IMAGE }}:latest + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} - name: Sign image run: cosign sign --yes "${IMAGE}@${{ steps.build.outputs.digest }}" - name: Generate SBOM @@ -725,7 +732,7 @@ jobs: labels: | org.opencontainers.image.source=https://github.com/${{ github.repository }} org.opencontainers.image.title=slack-bot - org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} org.opencontainers.image.vendor=Stacklok # Sign/SBOM/attest the pushed manifest list BY DIGEST, matching the diff --git a/contracts/gen/go/mecatl/execution/v1/execution.pb.go b/contracts/gen/go/mecatl/execution/v1/execution.pb.go index 2e59488d94..83f59d3ff6 100644 --- a/contracts/gen/go/mecatl/execution/v1/execution.pb.go +++ b/contracts/gen/go/mecatl/execution/v1/execution.pb.go @@ -1367,6 +1367,8 @@ type ListReferenceIntentsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Owner *Owner `protobuf:"bytes,1,opt,name=owner,proto3" json:"owner,omitempty"` Limit int32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + Environment *EnvironmentRef `protobuf:"bytes,3,opt,name=environment,proto3" json:"environment,omitempty"` + BindingId string `protobuf:"bytes,4,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1415,6 +1417,20 @@ func (x *ListReferenceIntentsRequest) GetLimit() int32 { return 0 } +func (x *ListReferenceIntentsRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ListReferenceIntentsRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + type ReferenceIntent struct { state protoimpl.MessageState `protogen:"open.v1"` Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` @@ -3007,142 +3023,73 @@ var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, - 0x22, 0x65, 0x0a, 0x1b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, - 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0xc9, 0x02, 0x0a, 0x0f, 0x52, 0x65, 0x66, 0x65, - 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, - 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x69, 0x6e, - 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x64, 0x5f, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, - 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, - 0x6e, 0x65, 0x72, 0x22, 0x5e, 0x0a, 0x1c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x07, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x52, 0x07, 0x69, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x73, 0x22, 0x58, 0x0a, 0x17, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, - 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, - 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0xc4, 0x02, - 0x0a, 0x18, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, - 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, - 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, - 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, - 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, - 0x6e, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x45, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x28, 0x0a, - 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, - 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, - 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, - 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xc2, 0x02, 0x0a, 0x16, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, - 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, - 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, - 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, - 0x63, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, - 0x70, 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, - 0x63, 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, - 0x63, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, - 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, - 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, - 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, - 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xc5, 0x02, 0x0a, 0x19, 0x52, 0x65, - 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, - 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, - 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, - 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x22, 0xcb, 0x01, 0x0a, 0x1b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, + 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, + 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, - 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, - 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, - 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, - 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, - 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, - 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, + 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x22, 0xc9, + 0x02, 0x0a, 0x0f, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, + 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x64, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, - 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, - 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, - 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, - 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, - 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, + 0x64, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x62, 0x69, 0x6e, 0x64, + 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x39, 0x0a, + 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, + 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, + 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x22, 0x5e, 0x0a, 0x1c, 0x4c, 0x69, + 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x07, 0x69, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x28, + 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x52, 0x07, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x58, 0x0a, 0x17, 0x52, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x22, 0xc4, 0x02, 0x0a, 0x18, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, + 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, + 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, + 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, + 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, + 0x70, 0x6f, 0x63, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, - 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xf2, 0x01, 0x0a, 0x18, - 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, - 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, - 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, - 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, - 0x72, 0x12, 0x3a, 0x0a, 0x19, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x67, 0x72, - 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x17, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x47, 0x72, - 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, - 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, - 0x22, 0x46, 0x0a, 0x19, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, - 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, - 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xe4, 0x02, 0x0a, 0x19, 0x4d, 0x69, 0x67, - 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xc2, 0x02, 0x0a, 0x16, + 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, @@ -3151,325 +3098,400 @@ var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, - 0x3b, 0x0a, 0x17, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, - 0x48, 0x00, 0x52, 0x15, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x53, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x28, 0x0a, 0x10, - 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, - 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, - 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, - 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x64, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, - 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, - 0xa2, 0x02, 0x0a, 0x0b, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x40, - 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x22, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x70, 0x61, 0x74, 0x68, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, - 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, - 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, - 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, - 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6c, - 0x69, 0x6d, 0x69, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, - 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x35, 0x0a, - 0x08, 0x6d, 0x6f, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x6d, 0x6f, 0x64, - 0x54, 0x69, 0x6d, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x69, 0x73, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x69, 0x73, 0x44, 0x69, 0x72, 0x22, 0x47, 0x0a, 0x09, 0x47, - 0x72, 0x65, 0x70, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, - 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, - 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x74, 0x65, 0x78, 0x74, 0x22, 0xd4, 0x02, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, - 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, - 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, - 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, - 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, - 0x70, 0x61, 0x74, 0x68, 0x73, 0x12, 0x38, 0x0a, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, - 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x65, - 0x70, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, - 0x29, 0x0a, 0x10, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x75, 0x74, 0x68, 0x6f, - 0x72, 0x69, 0x74, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2f, 0x0a, 0x13, 0x61, 0x75, - 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, - 0x74, 0x79, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x95, 0x01, 0x0a, 0x13, - 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x25, 0x0a, 0x0e, - 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x69, 0x6c, - 0x6c, 0x69, 0x73, 0x22, 0xb2, 0x01, 0x0a, 0x14, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, - 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x37, 0x0a, 0x05, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x63, - 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x12, 0x42, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, - 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x8b, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, - 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, + 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, + 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, + 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, + 0x22, 0xc5, 0x02, 0x0a, 0x19, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, + 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, + 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x70, + 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, 0x63, + 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, + 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, + 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, + 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, - 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x16, - 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, - 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0xa6, 0x02, 0x0a, 0x15, 0x43, 0x6f, 0x6d, 0x6d, 0x61, - 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, - 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, + 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, + 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x49, 0x64, 0x22, 0xf2, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, + 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x3a, 0x0a, 0x19, 0x65, 0x78, 0x70, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x5f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x17, 0x65, 0x78, 0x70, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x47, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x46, 0x0a, 0x19, 0x52, 0x65, 0x76, 0x6f, 0x6b, + 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, + 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0xe4, 0x02, 0x0a, 0x19, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, + 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, + 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x3b, 0x0a, 0x17, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, 0x52, 0x15, 0x65, 0x78, 0x70, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x88, 0x01, 0x01, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, + 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, + 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, + 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xa2, 0x02, 0x0a, 0x0b, 0x46, 0x69, 0x6c, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x40, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, + 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x20, 0x0a, 0x0b, 0x64, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, + 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x18, 0x0a, 0x07, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x08, + 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, + 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x6d, 0x6f, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x69, + 0x73, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x69, 0x73, 0x44, + 0x69, 0x72, 0x22, 0x47, 0x0a, 0x09, 0x47, 0x72, 0x65, 0x70, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, + 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, 0x22, 0xd4, 0x02, 0x0a, 0x0c, + 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x04, 0x69, 0x6e, + 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, + 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, + 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, - 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x65, 0x78, 0x69, - 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x12, 0x16, 0x0a, - 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, - 0x74, 0x64, 0x65, 0x72, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x6f, 0x66, - 0x66, 0x73, 0x65, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6e, 0x65, 0x78, 0x74, - 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, - 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, - 0x61, 0x74, 0x65, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, - 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, - 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, - 0x3f, 0x0a, 0x0b, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x12, - 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, - 0x64, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x74, 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x74, 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, - 0x2a, 0xd8, 0x02, 0x0a, 0x0d, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, - 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, - 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, - 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x10, 0x01, 0x12, 0x24, 0x0a, 0x20, 0x46, - 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, - 0x53, 0x4f, 0x4c, 0x56, 0x45, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x4f, 0x52, 0x49, 0x54, 0x59, 0x10, - 0x02, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, - 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, - 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x43, 0x52, 0x45, - 0x41, 0x54, 0x45, 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, - 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, - 0x05, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, - 0x49, 0x4f, 0x4e, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x06, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, - 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4d, - 0x4f, 0x56, 0x45, 0x10, 0x07, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, - 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x08, - 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, - 0x4f, 0x4e, 0x5f, 0x43, 0x4f, 0x50, 0x59, 0x10, 0x09, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, - 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x47, 0x4c, 0x4f, 0x42, - 0x10, 0x0a, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, - 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x47, 0x52, 0x45, 0x50, 0x10, 0x0b, 0x2a, 0xbd, 0x01, 0x0a, 0x0c, - 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, - 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, - 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x43, - 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4e, - 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, - 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x45, 0x44, 0x45, - 0x44, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, - 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x03, 0x12, 0x1b, 0x0a, - 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x43, - 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, - 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x45, 0x4e, 0x43, - 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x05, 0x32, 0xe4, 0x12, 0x0a, 0x18, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, - 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x2b, 0x2e, 0x6d, 0x65, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x65, + 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x12, 0x38, 0x0a, 0x07, + 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x65, 0x70, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x6d, + 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x69, 0x74, 0x79, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x12, 0x2f, 0x0a, 0x13, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x77, + 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, + 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x22, 0x95, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, - 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, - 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x11, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, - 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, + 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x6d, + 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x74, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, 0xb2, 0x01, 0x0a, 0x14, 0x43, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x49, 0x64, 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x42, 0x0a, 0x06, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, - 0x31, 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, - 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, - 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x11, 0x41, 0x74, - 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, - 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, - 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, + 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, + 0x8b, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0xa6, 0x02, + 0x0a, 0x15, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, + 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, + 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, + 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, + 0x64, 0x6f, 0x75, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, 0x12, 0x1f, 0x0a, 0x0b, + 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0a, 0x6e, 0x65, 0x78, 0x74, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, + 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x74, + 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x52, + 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, 0x3f, 0x0a, 0x0b, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x44, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x74, + 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, + 0x74, 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x2a, 0xd8, 0x02, 0x0a, 0x0d, 0x46, 0x69, 0x6c, 0x65, + 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x49, 0x4c, + 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, + 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x41, 0x44, + 0x10, 0x01, 0x12, 0x24, 0x0a, 0x20, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, + 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x53, 0x4f, 0x4c, 0x56, 0x45, 0x5f, 0x41, 0x55, 0x54, + 0x48, 0x4f, 0x52, 0x49, 0x54, 0x59, 0x10, 0x02, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, + 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x10, + 0x03, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, + 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, + 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, 0x05, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, + 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, + 0x06, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x10, 0x07, 0x12, 0x19, 0x0a, 0x15, + 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, + 0x45, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x08, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, + 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x43, 0x4f, 0x50, 0x59, 0x10, 0x09, + 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, + 0x4f, 0x4e, 0x5f, 0x47, 0x4c, 0x4f, 0x42, 0x10, 0x0a, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, + 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x47, 0x52, 0x45, 0x50, + 0x10, 0x0b, 0x2a, 0xbd, 0x01, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x1b, 0x0a, + 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, + 0x55, 0x43, 0x43, 0x45, 0x45, 0x44, 0x45, 0x44, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, + 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, + 0x45, 0x44, 0x10, 0x03, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, + 0x04, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x45, 0x5f, 0x46, 0x45, 0x4e, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, + 0x10, 0x05, 0x32, 0xe4, 0x12, 0x0a, 0x18, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, + 0x6c, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, + 0x6c, 0x65, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, + 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, + 0x11, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x72, 0x0a, 0x11, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, + 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, 0x61, + 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5b, 0x0a, 0x0a, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, + 0x52, 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, + 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x57, 0x0a, 0x08, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, 0x12, 0x24, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, - 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5b, - 0x0a, 0x0a, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, - 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x76, 0x31, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6c, - 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x08, 0x52, - 0x65, 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, 0x12, 0x24, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, - 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, - 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, - 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, - 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x12, 0x58, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x66, 0x65, + 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0a, 0x52, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x58, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2d, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0e, - 0x41, 0x62, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2d, - 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x73, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, - 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x73, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, - 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5f, 0x0a, 0x16, 0x50, 0x72, - 0x65, 0x70, 0x61, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5f, 0x0a, 0x16, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, - 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5e, 0x0a, 0x15, - 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, - 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x7b, 0x0a, 0x14, - 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x73, 0x12, 0x30, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x10, 0x52, 0x65, 0x6c, - 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2c, 0x2e, - 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, + 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x5f, 0x0a, 0x16, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x12, 0x5f, 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x12, 0x5a, 0x0a, 0x11, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, - 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, - 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, - 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x56, 0x0a, 0x0f, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x6f, 0x72, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5c, 0x0a, 0x12, 0x52, 0x65, 0x63, 0x6f, 0x76, - 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, + 0x70, 0x74, 0x79, 0x12, 0x5e, 0x0a, 0x15, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x7b, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x30, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, - 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x68, 0x0a, 0x18, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, - 0x65, 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, - 0x74, 0x12, 0x34, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, - 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x5c, 0x0a, 0x12, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, - 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x67, 0x72, - 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x72, 0x0a, - 0x11, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, - 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, - 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x4c, 0x0a, 0x05, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x63, - 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, - 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, + 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x58, 0x0a, 0x10, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, + 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5a, 0x0a, 0x11, 0x52, 0x65, + 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x56, 0x0a, 0x0f, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5c, + 0x0a, 0x12, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x76, + 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x68, 0x0a, 0x18, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5c, 0x0a, 0x12, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, + 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x63, 0x0a, 0x0c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, - 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, - 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x12, 0x72, 0x0a, 0x11, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, - 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, - 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, - 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, + 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x05, 0x46, 0x69, 0x6c, 0x65, + 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x0c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x29, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x42, 0xe4, 0x01, 0x0a, 0x17, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, - 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x0e, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, - 0x5a, 0x4b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, 0x61, - 0x63, 0x6b, 0x6c, 0x6f, 0x6b, 0x2f, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, - 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, - 0x31, 0x3b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x76, 0x31, 0xa2, 0x02, 0x03, - 0x4d, 0x45, 0x58, 0xaa, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x45, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, - 0x74, 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0xe2, - 0x02, 0x1f, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0xea, 0x02, 0x15, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x3a, 0x3a, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x43, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0xe4, 0x01, 0x0a, 0x17, 0x63, 0x6f, + 0x6d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x4b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x6c, 0x6f, 0x6b, 0x2f, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x67, 0x65, + 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x3b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, 0x45, 0x58, 0xaa, 0x02, 0x13, 0x4d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x56, 0x31, + 0xca, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x1f, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x15, 0x4d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x3a, 0x3a, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x3a, 0x56, 0x31, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, }) var ( @@ -3551,87 +3573,88 @@ var file_mecatl_execution_v1_execution_proto_depIdxs = []int32{ 3, // 19: mecatl.execution.v1.ReserveSuccessorRequest.owner:type_name -> mecatl.execution.v1.Owner 2, // 20: mecatl.execution.v1.ReferenceReservationResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef 3, // 21: mecatl.execution.v1.ListReferenceIntentsRequest.owner:type_name -> mecatl.execution.v1.Owner - 2, // 22: mecatl.execution.v1.ReferenceIntent.environment:type_name -> mecatl.execution.v1.EnvironmentRef - 38, // 23: mecatl.execution.v1.ReferenceIntent.created_at:type_name -> google.protobuf.Timestamp - 3, // 24: mecatl.execution.v1.ReferenceIntent.owner:type_name -> mecatl.execution.v1.Owner - 19, // 25: mecatl.execution.v1.ListReferenceIntentsResponse.intents:type_name -> mecatl.execution.v1.ReferenceIntent - 4, // 26: mecatl.execution.v1.ReleaseReferenceRequest.context:type_name -> mecatl.execution.v1.RequestContext - 2, // 27: mecatl.execution.v1.RetireEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef - 3, // 28: mecatl.execution.v1.RetireEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner - 2, // 29: mecatl.execution.v1.ReplaceExecutorRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef - 3, // 30: mecatl.execution.v1.ReplaceExecutorRequest.owner:type_name -> mecatl.execution.v1.Owner - 2, // 31: mecatl.execution.v1.RecoverEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef - 3, // 32: mecatl.execution.v1.RecoverEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner - 2, // 33: mecatl.execution.v1.DeleteRetiredEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef - 3, // 34: mecatl.execution.v1.DeleteRetiredEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner - 2, // 35: mecatl.execution.v1.RevokeEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef - 3, // 36: mecatl.execution.v1.RevokeEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner - 2, // 37: mecatl.execution.v1.MigrateEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef - 3, // 38: mecatl.execution.v1.MigrateEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner - 4, // 39: mecatl.execution.v1.FileRequest.context:type_name -> mecatl.execution.v1.RequestContext - 0, // 40: mecatl.execution.v1.FileRequest.operation:type_name -> mecatl.execution.v1.FileOperation - 38, // 41: mecatl.execution.v1.FileInfo.mod_time:type_name -> google.protobuf.Timestamp - 30, // 42: mecatl.execution.v1.FileResponse.info:type_name -> mecatl.execution.v1.FileInfo - 30, // 43: mecatl.execution.v1.FileResponse.entries:type_name -> mecatl.execution.v1.FileInfo - 31, // 44: mecatl.execution.v1.FileResponse.matches:type_name -> mecatl.execution.v1.GrepMatch - 4, // 45: mecatl.execution.v1.CommandStartRequest.context:type_name -> mecatl.execution.v1.RequestContext - 1, // 46: mecatl.execution.v1.CommandStartResponse.state:type_name -> mecatl.execution.v1.CommandState - 36, // 47: mecatl.execution.v1.CommandStartResponse.result:type_name -> mecatl.execution.v1.CommandStatusResponse - 4, // 48: mecatl.execution.v1.CommandQueryRequest.context:type_name -> mecatl.execution.v1.RequestContext - 1, // 49: mecatl.execution.v1.CommandStatusResponse.state:type_name -> mecatl.execution.v1.CommandState - 5, // 50: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:input_type -> mecatl.execution.v1.ValidateProfileRequest - 7, // 51: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:input_type -> mecatl.execution.v1.EnsureEnvironmentRequest - 9, // 52: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:input_type -> mecatl.execution.v1.AttachEnvironmentRequest - 11, // 53: mecatl.execution.v1.ExecutionProviderService.AcquireRun:input_type -> mecatl.execution.v1.AcquireRunRequest - 12, // 54: mecatl.execution.v1.ExecutionProviderService.RenewRun:input_type -> mecatl.execution.v1.RenewRunRequest - 13, // 55: mecatl.execution.v1.ExecutionProviderService.ReleaseRun:input_type -> mecatl.execution.v1.ReleaseRunRequest - 15, // 56: mecatl.execution.v1.ExecutionProviderService.CommitReference:input_type -> mecatl.execution.v1.ReferenceMutationRequest - 15, // 57: mecatl.execution.v1.ExecutionProviderService.AbortReference:input_type -> mecatl.execution.v1.ReferenceMutationRequest - 16, // 58: mecatl.execution.v1.ExecutionProviderService.ReserveSuccessor:input_type -> mecatl.execution.v1.ReserveSuccessorRequest - 15, // 59: mecatl.execution.v1.ExecutionProviderService.PrepareReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest - 15, // 60: mecatl.execution.v1.ExecutionProviderService.ConfirmReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest - 15, // 61: mecatl.execution.v1.ExecutionProviderService.CancelReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest - 18, // 62: mecatl.execution.v1.ExecutionProviderService.ListReferenceIntents:input_type -> mecatl.execution.v1.ListReferenceIntentsRequest - 21, // 63: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:input_type -> mecatl.execution.v1.ReleaseReferenceRequest - 22, // 64: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:input_type -> mecatl.execution.v1.RetireEnvironmentRequest - 23, // 65: mecatl.execution.v1.ExecutionProviderService.ReplaceExecutor:input_type -> mecatl.execution.v1.ReplaceExecutorRequest - 24, // 66: mecatl.execution.v1.ExecutionProviderService.RecoverEnvironment:input_type -> mecatl.execution.v1.RecoverEnvironmentRequest - 25, // 67: mecatl.execution.v1.ExecutionProviderService.DeleteRetiredEnvironment:input_type -> mecatl.execution.v1.DeleteRetiredEnvironmentRequest - 28, // 68: mecatl.execution.v1.ExecutionProviderService.MigrateEnvironment:input_type -> mecatl.execution.v1.MigrateEnvironmentRequest - 26, // 69: mecatl.execution.v1.ExecutionProviderService.RevokeEnvironment:input_type -> mecatl.execution.v1.RevokeEnvironmentRequest - 29, // 70: mecatl.execution.v1.ExecutionProviderService.Files:input_type -> mecatl.execution.v1.FileRequest - 33, // 71: mecatl.execution.v1.ExecutionProviderService.StartCommand:input_type -> mecatl.execution.v1.CommandStartRequest - 35, // 72: mecatl.execution.v1.ExecutionProviderService.CommandStatus:input_type -> mecatl.execution.v1.CommandQueryRequest - 35, // 73: mecatl.execution.v1.ExecutionProviderService.CancelCommand:input_type -> mecatl.execution.v1.CommandQueryRequest - 6, // 74: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:output_type -> mecatl.execution.v1.ValidateProfileResponse - 8, // 75: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:output_type -> mecatl.execution.v1.EnsureEnvironmentResponse - 10, // 76: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:output_type -> mecatl.execution.v1.AttachEnvironmentResponse - 14, // 77: mecatl.execution.v1.ExecutionProviderService.AcquireRun:output_type -> mecatl.execution.v1.RunClaimResponse - 14, // 78: mecatl.execution.v1.ExecutionProviderService.RenewRun:output_type -> mecatl.execution.v1.RunClaimResponse - 39, // 79: mecatl.execution.v1.ExecutionProviderService.ReleaseRun:output_type -> google.protobuf.Empty - 39, // 80: mecatl.execution.v1.ExecutionProviderService.CommitReference:output_type -> google.protobuf.Empty - 39, // 81: mecatl.execution.v1.ExecutionProviderService.AbortReference:output_type -> google.protobuf.Empty - 17, // 82: mecatl.execution.v1.ExecutionProviderService.ReserveSuccessor:output_type -> mecatl.execution.v1.ReferenceReservationResponse - 39, // 83: mecatl.execution.v1.ExecutionProviderService.PrepareReferenceDelete:output_type -> google.protobuf.Empty - 39, // 84: mecatl.execution.v1.ExecutionProviderService.ConfirmReferenceDelete:output_type -> google.protobuf.Empty - 39, // 85: mecatl.execution.v1.ExecutionProviderService.CancelReferenceDelete:output_type -> google.protobuf.Empty - 20, // 86: mecatl.execution.v1.ExecutionProviderService.ListReferenceIntents:output_type -> mecatl.execution.v1.ListReferenceIntentsResponse - 39, // 87: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:output_type -> google.protobuf.Empty - 39, // 88: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:output_type -> google.protobuf.Empty - 39, // 89: mecatl.execution.v1.ExecutionProviderService.ReplaceExecutor:output_type -> google.protobuf.Empty - 39, // 90: mecatl.execution.v1.ExecutionProviderService.RecoverEnvironment:output_type -> google.protobuf.Empty - 39, // 91: mecatl.execution.v1.ExecutionProviderService.DeleteRetiredEnvironment:output_type -> google.protobuf.Empty - 39, // 92: mecatl.execution.v1.ExecutionProviderService.MigrateEnvironment:output_type -> google.protobuf.Empty - 27, // 93: mecatl.execution.v1.ExecutionProviderService.RevokeEnvironment:output_type -> mecatl.execution.v1.RevokeEnvironmentResponse - 32, // 94: mecatl.execution.v1.ExecutionProviderService.Files:output_type -> mecatl.execution.v1.FileResponse - 34, // 95: mecatl.execution.v1.ExecutionProviderService.StartCommand:output_type -> mecatl.execution.v1.CommandStartResponse - 36, // 96: mecatl.execution.v1.ExecutionProviderService.CommandStatus:output_type -> mecatl.execution.v1.CommandStatusResponse - 36, // 97: mecatl.execution.v1.ExecutionProviderService.CancelCommand:output_type -> mecatl.execution.v1.CommandStatusResponse - 74, // [74:98] is the sub-list for method output_type - 50, // [50:74] is the sub-list for method input_type - 50, // [50:50] is the sub-list for extension type_name - 50, // [50:50] is the sub-list for extension extendee - 0, // [0:50] is the sub-list for field type_name + 2, // 22: mecatl.execution.v1.ListReferenceIntentsRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 2, // 23: mecatl.execution.v1.ReferenceIntent.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 38, // 24: mecatl.execution.v1.ReferenceIntent.created_at:type_name -> google.protobuf.Timestamp + 3, // 25: mecatl.execution.v1.ReferenceIntent.owner:type_name -> mecatl.execution.v1.Owner + 19, // 26: mecatl.execution.v1.ListReferenceIntentsResponse.intents:type_name -> mecatl.execution.v1.ReferenceIntent + 4, // 27: mecatl.execution.v1.ReleaseReferenceRequest.context:type_name -> mecatl.execution.v1.RequestContext + 2, // 28: mecatl.execution.v1.RetireEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 29: mecatl.execution.v1.RetireEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 30: mecatl.execution.v1.ReplaceExecutorRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 31: mecatl.execution.v1.ReplaceExecutorRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 32: mecatl.execution.v1.RecoverEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 33: mecatl.execution.v1.RecoverEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 34: mecatl.execution.v1.DeleteRetiredEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 35: mecatl.execution.v1.DeleteRetiredEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 36: mecatl.execution.v1.RevokeEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 37: mecatl.execution.v1.RevokeEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 38: mecatl.execution.v1.MigrateEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 39: mecatl.execution.v1.MigrateEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 4, // 40: mecatl.execution.v1.FileRequest.context:type_name -> mecatl.execution.v1.RequestContext + 0, // 41: mecatl.execution.v1.FileRequest.operation:type_name -> mecatl.execution.v1.FileOperation + 38, // 42: mecatl.execution.v1.FileInfo.mod_time:type_name -> google.protobuf.Timestamp + 30, // 43: mecatl.execution.v1.FileResponse.info:type_name -> mecatl.execution.v1.FileInfo + 30, // 44: mecatl.execution.v1.FileResponse.entries:type_name -> mecatl.execution.v1.FileInfo + 31, // 45: mecatl.execution.v1.FileResponse.matches:type_name -> mecatl.execution.v1.GrepMatch + 4, // 46: mecatl.execution.v1.CommandStartRequest.context:type_name -> mecatl.execution.v1.RequestContext + 1, // 47: mecatl.execution.v1.CommandStartResponse.state:type_name -> mecatl.execution.v1.CommandState + 36, // 48: mecatl.execution.v1.CommandStartResponse.result:type_name -> mecatl.execution.v1.CommandStatusResponse + 4, // 49: mecatl.execution.v1.CommandQueryRequest.context:type_name -> mecatl.execution.v1.RequestContext + 1, // 50: mecatl.execution.v1.CommandStatusResponse.state:type_name -> mecatl.execution.v1.CommandState + 5, // 51: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:input_type -> mecatl.execution.v1.ValidateProfileRequest + 7, // 52: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:input_type -> mecatl.execution.v1.EnsureEnvironmentRequest + 9, // 53: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:input_type -> mecatl.execution.v1.AttachEnvironmentRequest + 11, // 54: mecatl.execution.v1.ExecutionProviderService.AcquireRun:input_type -> mecatl.execution.v1.AcquireRunRequest + 12, // 55: mecatl.execution.v1.ExecutionProviderService.RenewRun:input_type -> mecatl.execution.v1.RenewRunRequest + 13, // 56: mecatl.execution.v1.ExecutionProviderService.ReleaseRun:input_type -> mecatl.execution.v1.ReleaseRunRequest + 15, // 57: mecatl.execution.v1.ExecutionProviderService.CommitReference:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 58: mecatl.execution.v1.ExecutionProviderService.AbortReference:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 16, // 59: mecatl.execution.v1.ExecutionProviderService.ReserveSuccessor:input_type -> mecatl.execution.v1.ReserveSuccessorRequest + 15, // 60: mecatl.execution.v1.ExecutionProviderService.PrepareReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 61: mecatl.execution.v1.ExecutionProviderService.ConfirmReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 62: mecatl.execution.v1.ExecutionProviderService.CancelReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 18, // 63: mecatl.execution.v1.ExecutionProviderService.ListReferenceIntents:input_type -> mecatl.execution.v1.ListReferenceIntentsRequest + 21, // 64: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:input_type -> mecatl.execution.v1.ReleaseReferenceRequest + 22, // 65: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:input_type -> mecatl.execution.v1.RetireEnvironmentRequest + 23, // 66: mecatl.execution.v1.ExecutionProviderService.ReplaceExecutor:input_type -> mecatl.execution.v1.ReplaceExecutorRequest + 24, // 67: mecatl.execution.v1.ExecutionProviderService.RecoverEnvironment:input_type -> mecatl.execution.v1.RecoverEnvironmentRequest + 25, // 68: mecatl.execution.v1.ExecutionProviderService.DeleteRetiredEnvironment:input_type -> mecatl.execution.v1.DeleteRetiredEnvironmentRequest + 28, // 69: mecatl.execution.v1.ExecutionProviderService.MigrateEnvironment:input_type -> mecatl.execution.v1.MigrateEnvironmentRequest + 26, // 70: mecatl.execution.v1.ExecutionProviderService.RevokeEnvironment:input_type -> mecatl.execution.v1.RevokeEnvironmentRequest + 29, // 71: mecatl.execution.v1.ExecutionProviderService.Files:input_type -> mecatl.execution.v1.FileRequest + 33, // 72: mecatl.execution.v1.ExecutionProviderService.StartCommand:input_type -> mecatl.execution.v1.CommandStartRequest + 35, // 73: mecatl.execution.v1.ExecutionProviderService.CommandStatus:input_type -> mecatl.execution.v1.CommandQueryRequest + 35, // 74: mecatl.execution.v1.ExecutionProviderService.CancelCommand:input_type -> mecatl.execution.v1.CommandQueryRequest + 6, // 75: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:output_type -> mecatl.execution.v1.ValidateProfileResponse + 8, // 76: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:output_type -> mecatl.execution.v1.EnsureEnvironmentResponse + 10, // 77: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:output_type -> mecatl.execution.v1.AttachEnvironmentResponse + 14, // 78: mecatl.execution.v1.ExecutionProviderService.AcquireRun:output_type -> mecatl.execution.v1.RunClaimResponse + 14, // 79: mecatl.execution.v1.ExecutionProviderService.RenewRun:output_type -> mecatl.execution.v1.RunClaimResponse + 39, // 80: mecatl.execution.v1.ExecutionProviderService.ReleaseRun:output_type -> google.protobuf.Empty + 39, // 81: mecatl.execution.v1.ExecutionProviderService.CommitReference:output_type -> google.protobuf.Empty + 39, // 82: mecatl.execution.v1.ExecutionProviderService.AbortReference:output_type -> google.protobuf.Empty + 17, // 83: mecatl.execution.v1.ExecutionProviderService.ReserveSuccessor:output_type -> mecatl.execution.v1.ReferenceReservationResponse + 39, // 84: mecatl.execution.v1.ExecutionProviderService.PrepareReferenceDelete:output_type -> google.protobuf.Empty + 39, // 85: mecatl.execution.v1.ExecutionProviderService.ConfirmReferenceDelete:output_type -> google.protobuf.Empty + 39, // 86: mecatl.execution.v1.ExecutionProviderService.CancelReferenceDelete:output_type -> google.protobuf.Empty + 20, // 87: mecatl.execution.v1.ExecutionProviderService.ListReferenceIntents:output_type -> mecatl.execution.v1.ListReferenceIntentsResponse + 39, // 88: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:output_type -> google.protobuf.Empty + 39, // 89: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:output_type -> google.protobuf.Empty + 39, // 90: mecatl.execution.v1.ExecutionProviderService.ReplaceExecutor:output_type -> google.protobuf.Empty + 39, // 91: mecatl.execution.v1.ExecutionProviderService.RecoverEnvironment:output_type -> google.protobuf.Empty + 39, // 92: mecatl.execution.v1.ExecutionProviderService.DeleteRetiredEnvironment:output_type -> google.protobuf.Empty + 39, // 93: mecatl.execution.v1.ExecutionProviderService.MigrateEnvironment:output_type -> google.protobuf.Empty + 27, // 94: mecatl.execution.v1.ExecutionProviderService.RevokeEnvironment:output_type -> mecatl.execution.v1.RevokeEnvironmentResponse + 32, // 95: mecatl.execution.v1.ExecutionProviderService.Files:output_type -> mecatl.execution.v1.FileResponse + 34, // 96: mecatl.execution.v1.ExecutionProviderService.StartCommand:output_type -> mecatl.execution.v1.CommandStartResponse + 36, // 97: mecatl.execution.v1.ExecutionProviderService.CommandStatus:output_type -> mecatl.execution.v1.CommandStatusResponse + 36, // 98: mecatl.execution.v1.ExecutionProviderService.CancelCommand:output_type -> mecatl.execution.v1.CommandStatusResponse + 75, // [75:99] is the sub-list for method output_type + 51, // [51:75] is the sub-list for method input_type + 51, // [51:51] is the sub-list for extension type_name + 51, // [51:51] is the sub-list for extension extendee + 0, // [0:51] is the sub-list for field type_name } func init() { file_mecatl_execution_v1_execution_proto_init() } diff --git a/contracts/proto/mecatl/execution/v1/execution.proto b/contracts/proto/mecatl/execution/v1/execution.proto index 36ca47912a..9b3b369092 100644 --- a/contracts/proto/mecatl/execution/v1/execution.proto +++ b/contracts/proto/mecatl/execution/v1/execution.proto @@ -132,7 +132,12 @@ message ReserveSuccessorRequest { string operation_id = 5; } message ReferenceReservationResponse { EnvironmentRef environment = 1; } -message ListReferenceIntentsRequest { Owner owner = 1; int32 limit = 2; } +message ListReferenceIntentsRequest { + Owner owner = 1; + int32 limit = 2; + EnvironmentRef environment = 3; + string binding_id = 4; +} message ReferenceIntent { EnvironmentRef environment = 1; string binding_id = 2; diff --git a/deploy/helm/mecatl-execution/crds/executionenvironment.yaml b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml index 7c3433a9f3..ef87a94910 100644 --- a/deploy/helm/mecatl-execution/crds/executionenvironment.yaml +++ b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml @@ -72,7 +72,6 @@ spec: storageSize: {type: string, minLength: 1} resources: type: object - additionalProperties: false required: [cpuRequest, memoryRequest, cpuLimit, memoryLimit] properties: cpuRequest: {type: string, minLength: 1} @@ -93,7 +92,6 @@ spec: maxItems: 32 items: type: object - additionalProperties: false required: [operationID, fingerprint, expectedGrantGeneration, grantGeneration] properties: operationID: {type: string, minLength: 1, maxLength: 1024} @@ -137,7 +135,16 @@ spec: epoch: {type: integer, format: int64, minimum: 1} grantGeneration: {type: integer, format: int64, minimum: 1} expiresAt: {type: string, format: date-time} - lastRenewOperationID: {type: string, maxLength: 1024} + renewReceipts: + type: array + maxItems: 32 + items: + type: object + required: [operationID, fingerprint, expiresAt] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + fingerprint: {type: string, pattern: "^[0-9a-f]{64}$"} + expiresAt: {type: string, format: date-time} activeOperation: type: object required: [id, operation, startedAt, claimID, runID, epoch, holderID, renewedAt, expiresAt] @@ -157,7 +164,7 @@ spec: properties: id: {type: string, minLength: 1, maxLength: 1024} type: {type: string, enum: [ReplaceExecutor, RetireEnvironment, DeleteRetiredEnvironment]} - phase: {type: string, enum: [Quiescing, WaitingForTermination, RemovingPodFinalizer, WaitingForPodDeletion, CreatingReplacement, DeletingPVC]} + phase: {type: string, enum: [Quiescing, WaitingForTermination, RemovingPodFinalizer, WaitingForPodDeletion, CreatingReplacement, DeletingPVC, ReleasingSlot]} expectedEpoch: {type: integer, format: int64, minimum: 1} expectedPodUID: {type: string, maxLength: 128} expectedPVCUID: {type: string, minLength: 1, maxLength: 128} @@ -172,6 +179,16 @@ spec: epoch: {type: integer, format: int64, minimum: 1} podPhase: {type: string, enum: [Succeeded, Failed]} observedAt: {type: string, format: date-time} + lastReplacement: + type: object + required: [operationID, previousPodUID, replacementPodUID, pvcUID, previousEpoch, replacementEpoch] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + previousPodUID: {type: string, minLength: 1, maxLength: 128} + replacementPodUID: {type: string, minLength: 1, maxLength: 128} + pvcUID: {type: string, minLength: 1, maxLength: 128} + previousEpoch: {type: integer, format: int64, minimum: 1} + replacementEpoch: {type: integer, format: int64, minimum: 2} migrationOperation: type: object required: [id, fromSchema, expectedPodUID, expectedPVCUID] diff --git a/deploy/mecatl-execution-kind/run.sh b/deploy/mecatl-execution-kind/run.sh index cc52e3b305..a805005e64 100755 --- a/deploy/mecatl-execution-kind/run.sh +++ b/deploy/mecatl-execution-kind/run.sh @@ -224,6 +224,38 @@ EOF kube -n execution-qualification wait --for=condition=Ready pod/network-fixture pod/network-intruder --timeout=2m fi +if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then + legacy_profiles="$state/legacy-profiles.yaml" + cat >"$legacy_profiles" < /workspace/migration-sentinel' + # Helm does not upgrade an existing CRD. Upgrade explicitly only after the + # API server persisted the legacy string references. + kube apply -f "$root/deploy/helm/mecatl-execution/crds/executionenvironment.yaml" + kube wait --for=condition=Established crd/executionenvironments.execution.mecatl.dev --timeout=60s +fi + kube -n execution-qualification create secret generic execution-security \ --from-file=grant-k1.pem="$state/pki/grant-key.pem" --from-file=tls.crt="$state/pki/provider.crt" \ --from-file=tls.key="$state/pki/provider.key" --from-file=clients.pem="$state/pki/ca.crt" --dry-run=client -o yaml | kube apply -f - @@ -241,8 +273,9 @@ kube -n execution-qualification rollout status deployment/oidc-issuer --timeout= runtime_values= if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then api_endpoint=$(kube -n default get endpoints kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}') + api_endpoint_port=$(kube -n default get endpoints kubernetes -o jsonpath='{.subsets[0].ports[0].port}') fixture_endpoint=$(kube -n execution-qualification get pod network-fixture -o jsonpath='{.status.podIP}') - case "$api_endpoint:$fixture_endpoint" in + case "$api_endpoint:$fixture_endpoint:$api_endpoint_port" in *[!0-9.:]*) echo "invalid discovered network-policy endpoint" >&2; exit 1 ;; :*|*:) echo "missing discovered network-policy endpoint" >&2; exit 1 ;; esac @@ -256,6 +289,9 @@ provider: dnsCIDRs: - 10.96.0.10/32 networkPolicy: + apiServerPorts: + - 443 + - $api_endpoint_port workloadProfiles: go: egress: @@ -286,7 +322,7 @@ helm_kube "$@" \ kube -n execution-qualification create configmap execution-mock --from-file=mock-script.json="$root/deploy/mecatl-execution-kind/mock-script.json" --dry-run=client -o yaml | kube apply -f - helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" \ --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ - --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" \ + --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" --set-string image.tag= \ --wait --timeout=5m kube -n execution-qualification rollout restart deployment/mecak8s kube -n execution-qualification rollout status deployment/mecak8s --timeout=240s diff --git a/e2e/k8s_execution/aa_production_migration_test.go b/e2e/k8s_execution/aa_production_migration_test.go new file mode 100644 index 0000000000..86ccae5a5f --- /dev/null +++ b/e2e/k8s_execution/aa_production_migration_test.go @@ -0,0 +1,9 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import "testing" + +func TestKindExecutionProductionCompatiblePrototypeMigration(t *testing.T) { + runKindExecutionProductionCompatiblePrototypeMigration(t) +} diff --git a/e2e/k8s_execution/fixture/legacyfixture/crd.yaml b/e2e/k8s_execution/fixture/legacyfixture/crd.yaml new file mode 100644 index 0000000000..a8fe3be98e --- /dev/null +++ b/e2e/k8s_execution/fixture/legacyfixture/crd.yaml @@ -0,0 +1,34 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: executionenvironments.execution.mecatl.dev +spec: + group: execution.mecatl.dev + scope: Namespaced + names: + plural: executionenvironments + singular: executionenvironment + kind: ExecutionEnvironment + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + properties: + references: + type: array + maxItems: 64 + items: + type: string diff --git a/e2e/k8s_execution/fixture/legacyfixture/main.go b/e2e/k8s_execution/fixture/legacyfixture/main.go new file mode 100644 index 0000000000..45833ecf9c --- /dev/null +++ b/e2e/k8s_execution/fixture/legacyfixture/main.go @@ -0,0 +1,52 @@ +//go:build kind_execution_e2e + +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" +) + +func main() { + if len(os.Args) != 4 { + fmt.Fprintln(os.Stderr, "usage: legacyfixture NAMESPACE PROFILES OUTPUT") + os.Exit(2) + } + config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) + if err != nil { + fail(err) + } + d, err := dynamic.NewForConfig(config) + if err != nil { + fail(err) + } + kube, err := kubernetes.NewForConfig(config) + if err != nil { + fail(err) + } + seed, err := executioncontroller.SeedLegacyMigrationFixture(context.Background(), d, kube, os.Args[1], os.Args[2]) + if err != nil { + fail(err) + } + encoded, err := json.Marshal(seed) + if err != nil { + fail(err) + } + encoded = append(encoded, '\n') + if err := os.WriteFile(os.Args[3], encoded, 0o600); err != nil { + fail(err) + } +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/e2e/k8s_execution/fixture/netprobe/main.go b/e2e/k8s_execution/fixture/netprobe/main.go index 4fa4862a80..fae09de4cb 100644 --- a/e2e/k8s_execution/fixture/netprobe/main.go +++ b/e2e/k8s_execution/fixture/netprobe/main.go @@ -30,7 +30,7 @@ func main() { conn, err := net.DialTimeout("tcp", *target, *timeout) if err != nil { fmt.Fprintln(os.Stderr, "connect denied or unavailable") - os.Exit(1) + os.Exit(42) } _ = conn.Close() fmt.Println("connected") diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go index e6a3fa5a7b..08b4beb6a5 100644 --- a/e2e/k8s_execution/production_qualification_test.go +++ b/e2e/k8s_execution/production_qualification_test.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net/http" "os" @@ -45,6 +46,31 @@ func productionClient(t *testing.T, ctx context.Context, state, kubeconfig strin return client, f } +func productionReplicaClients(t *testing.T, ctx context.Context, state, kubeconfig string) ([2]*executionclient.Client, [2]string) { + t.Helper() + var pods corev1.PodList + raw := runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "-o", "json") + if err := json.Unmarshal(raw, &pods); err != nil || len(pods.Items) != 2 { + t.Fatalf("decode two provider replicas: pods=%d err=%v", len(pods.Items), err) + } + if pods.Items[0].UID == "" || pods.Items[1].UID == "" || pods.Items[0].UID == pods.Items[1].UID { + t.Fatal("provider replica Pod UIDs are missing or identical") + } + var clients [2]*executionclient.Client + var podUIDs [2]string + for i := range clients { + forward := portForward(t, ctx, kubeconfig, "pod/"+pods.Items[i].Name, 8443) + client, err := executionclient.New(forward.addr, loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local")) + if err != nil { + t.Fatal(err) + } + clients[i] = client + podUIDs[i] = string(pods.Items[i].UID) + t.Cleanup(func() { client.Close() }) + } + return clients, podUIDs +} + func createProductionEnvironment(t *testing.T, ctx context.Context, c *executionclient.Client, suffix string) (executionenv.Owner, string, executionenv.AttachEnvironmentResponse) { t.Helper() owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "production-" + suffix} @@ -54,7 +80,7 @@ func createProductionEnvironment(t *testing.T, ctx context.Context, c *execution if err != nil { t.Fatalf("ensure production environment: %v", err) } - if err := c.CommitReference(ctx, executionenv.ReferenceRequest{Environment: ensured.Environment, Owner: owner, BindingID: binding, OperationID: "commit-" + binding}); err != nil { + if err := c.CommitReference(ctx, executionenv.ReferenceRequest{Environment: ensured.Environment, Owner: owner, BindingID: binding, OperationID: op}); err != nil { t.Fatalf("commit production reference: %v", err) } return owner, binding, waitReady(t, ctx, c, owner, binding, ensured.Environment) @@ -73,6 +99,10 @@ func TestKindExecutionProductionNetworkPolicyEnforced(t *testing.T) { if _, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "network_probe.go", Data: probeSource}); err != nil { t.Fatalf("install network probe in owned workspace: %v", err) } + built, err := client.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go build -o network-probe network_probe.go", TimeoutMillis: 30000}) + if err != nil || built.State != executionenv.CommandSucceeded || built.Result.ExitCode != 0 { + t.Fatalf("build network probe: state=%q exit=%d err=%v", built.State, built.Result.ExitCode, err) + } fixtureIP := kubeValue(t, ctx, kubeconfig, "get", "pod/network-fixture", "-n", namespace, "-o", "jsonpath={.status.podIP}") if out := runOwnedProbe(t, ctx, client, rc, fixtureIP+":8080"); out.State != executionenv.CommandSucceeded || out.Result.ExitCode != 0 { t.Fatal("profile-specific package endpoint was not reachable") @@ -83,14 +113,16 @@ func TestKindExecutionProductionNetworkPolicyEnforced(t *testing.T) { "provider": providerIP + ":8443", "api": "10.96.0.1:443", "peer": peerIP + ":8080", "dns": "10.96.0.10:53", "metadata": "169.254.169.254:80", "internet": "1.1.1.1:443", } { - if out := runOwnedProbe(t, ctx, client, rc, endpoint); out.State == executionenv.CommandSucceeded { - t.Fatalf("default-denied executor reached %s", name) + out := runOwnedProbe(t, ctx, client, rc, endpoint) + if out.State != executionenv.CommandFailed || out.Result.State != executionenv.CommandFailed || out.Result.ExitCode != 42 { + t.Fatalf("default-denied executor probe %s state=%q result_state=%q exit=%d, want failed/failed/42", name, out.State, out.Result.State, out.Result.ExitCode) } } out, err := command(ctx, kubeconfig, "exec", "-n", namespace, "pod/network-intruder", "--", "/ko-app/netprobe", "-target", providerIP+":8443", "-timeout", "2s").CombinedOutput() - if err == nil { - t.Fatalf("unrelated pod reached provider; output bytes=%d", len(out)) + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 42 { + t.Fatalf("unrelated pod denial exit proof=%q err=%v, want 42", strings.TrimSpace(string(out)), err) } if got := resourceCount(t, ctx, kubeconfig, "pods -l app.kubernetes.io/name=mecatl-execution"); got != 2 { t.Fatalf("ready provider replicas=%d, want 2", got) @@ -99,7 +131,7 @@ func TestKindExecutionProductionNetworkPolicyEnforced(t *testing.T) { func runOwnedProbe(t *testing.T, ctx context.Context, c *executionclient.Client, rc executionenv.RequestContext, endpoint string) executionenv.CommandStartResponse { t.Helper() - out, err := c.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go run network_probe.go " + endpoint, TimeoutMillis: 15000}) + out, err := c.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "./network-probe " + endpoint, TimeoutMillis: 15000}) if err != nil { t.Fatalf("network probe RPC failed: %v", err) } @@ -147,7 +179,11 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { defer cancel() oldClient, forward := productionClient(t, ctx, state, kubeconfig) owner, binding, attached := createProductionEnvironment(t, ctx, oldClient, "rotation") - oldRun, _ := acquireRun(t, ctx, oldClient, owner, binding, attached, "rotation-old-grant") + oldClaim, err := oldClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-old-grant", OperationID: "acquire-rotation-old-grant", TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + oldRun := executionenv.RequestContext{Environment: oldClaim.Environment, Owner: owner, BindingID: binding, RunID: oldClaim.RunID, ClaimID: oldClaim.ClaimID, Epoch: oldClaim.Epoch, GrantGeneration: oldClaim.GrantGeneration, Grant: oldClaim.Grant} if _, err := oldClient.File(ctx, executionenv.FileRequest{Context: oldRun, Operation: executionenv.OpFileCreate, Path: "rotation-sentinel.txt", Data: []byte("old-authority\n")}); err != nil { t.Fatal(err) } @@ -159,6 +195,12 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { if out, err := generator.CombinedOutput(); err != nil { t.Fatalf("generate synthetic rotation material: %v: %s", err, out) } + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cleanupCancel() + applySecurityCandidate(t, cleanupCtx, kubeconfig, rotationDir, filepath.Join(rotationDir, "restore-fixture-clients.json"), "bridge") + waitProviderReadyReplicas(t, cleanupCtx, kubeconfig, 2) + }) // A changed policy at the already-authoritative generation is rejected by // both replicas. The old snapshot cannot continue authorizing RPCs while the @@ -170,9 +212,7 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { } applySecurityCandidate(t, ctx, kubeconfig, filepath.Join(state, "pki"), filepath.Join(state, "pki", "manifest.json"), "initial") waitProviderReadyReplicas(t, ctx, kubeconfig, 2) - if got, err := oldClient.File(ctx, executionenv.FileRequest{Context: oldRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err != nil || string(got.Data) != "old-authority\n" { - t.Fatal("valid authority recovery lost or rewrote executor data") - } + waitFileContent(t, ctx, oldClient, oldRun, "rotation-sentinel.txt", "old-authority\n", "valid authority recovery") // Generation 2 bridges client trust before changing the server certificate. // The old connection remains usable, while a new-CA client can establish its @@ -232,7 +272,13 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { } runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=240s") - if _, err := freshClient.File(ctx, executionenv.FileRequest{Context: newRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err == nil { + postRestartForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + postRestartClient, err := executionclient.New(postRestartForward.addr, newTLS) + if err != nil { + t.Fatal(err) + } + defer postRestartClient.Close() + if _, err := postRestartClient.File(ctx, executionenv.FileRequest{Context: newRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err == nil { t.Fatal("revoked generation became usable after provider restart") } // Restore fixture client compatibility through a higher generation; this is @@ -241,6 +287,21 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { waitProviderReadyReplicas(t, ctx, kubeconfig, 2) } +func waitFileContent(t *testing.T, ctx context.Context, client *executionclient.Client, rc executionenv.RequestContext, path, want, proof string) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + got, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: path}) + if err == nil && string(got.Data) == want { + return + } + lastErr = err + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("%s did not authorize preserved data: %v", proof, lastErr) +} + func applySecurityCandidate(t *testing.T, ctx context.Context, kubeconfig, materialDir, manifest, mode string) { t.Helper() pki := filepath.Join(os.Getenv("MECATL_EXECUTION_QUAL_STATE"), "pki") @@ -297,19 +358,43 @@ func waitProviderReadyReplicas(t *testing.T, ctx context.Context, kubeconfig str func TestKindExecutionProductionReplicaLifecycle(t *testing.T) { state, kubeconfig, ctx, cancel := requireProduction(t) defer cancel() - client, _ := productionClient(t, ctx, state, kubeconfig) + replicas, replicaUIDs := productionReplicaClients(t, ctx, state, kubeconfig) + client := replicas[0] owner, binding, attached := createProductionEnvironment(t, ctx, client, "lifecycle") first := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) if first.SpecSchema != 2 || first.StatusSchema != 2 || first.PodUID == "" || first.PVCUID == "" { t.Fatal("schema-v2 exact runtime identities were not persisted") } - claim1, err := client.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "run-a", OperationID: "acquire-a", TTL: time.Minute}) - if err != nil { - t.Fatal(err) + type acquireResult struct { + claim executionenv.RunClaim + err error } - if _, err := client.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "run-b", OperationID: "acquire-b", TTL: time.Minute}); err == nil || !isRemoteCode(err, executionenv.CodeConflict) { - t.Fatalf("second replica run acquisition was not excluded: %v", err) + startAcquire := make(chan struct{}) + acquired := make(chan acquireResult, 2) + for i := range replicas { + i := i + go func() { + <-startAcquire + claim, err := replicas[i].AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: fmt.Sprintf("run-%d", i), OperationID: fmt.Sprintf("acquire-%d", i), TTL: time.Minute}) + acquired <- acquireResult{claim: claim, err: err} + }() + } + close(startAcquire) + var claim1 executionenv.RunClaim + conflicts := 0 + for range replicas { + result := <-acquired + if result.err == nil { + claim1 = result.claim + } else if isRemoteCode(result.err, executionenv.CodeConflict) { + conflicts++ + } else { + t.Fatalf("cross-replica run acquisition returned unexpected error: %v", result.err) + } + } + if claim1.ClaimID == "" || conflicts != 1 { + t.Fatalf("cross-replica run acquisition pod_uids=%v winner=%t conflicts=%d, want one each", replicaUIDs, claim1.ClaimID != "", conflicts) } statusWithClaim := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) if statusWithClaim.ActiveGrantGeneration != claim1.GrantGeneration { @@ -351,9 +436,33 @@ func TestKindExecutionProductionReplicaLifecycle(t *testing.T) { } release2() - newGeneration, err := client.RevokeEnvironment(ctx, attached.Environment, owner, reattached.GrantGeneration, "revoke-lifecycle") - if err != nil || newGeneration <= reattached.GrantGeneration { - t.Fatalf("revoke failed: generation=%d err=%v", newGeneration, err) + revokeResults := make(chan struct { + generation uint64 + err error + }, 2) + startRevoke := make(chan struct{}) + for i := range replicas { + i := i + go func() { + <-startRevoke + generation, err := replicas[i].RevokeEnvironment(ctx, attached.Environment, owner, reattached.GrantGeneration, "revoke-lifecycle") + revokeResults <- struct { + generation uint64 + err error + }{generation: generation, err: err} + }() + } + close(startRevoke) + var newGeneration uint64 + for range replicas { + result := <-revokeResults + if result.err != nil || result.generation <= reattached.GrantGeneration { + t.Fatalf("cross-replica revoke failed: generation=%d err=%v", result.generation, result.err) + } + if newGeneration != 0 && result.generation != newGeneration { + t.Fatalf("cross-replica revoke replay diverged: got=%d want=%d", result.generation, newGeneration) + } + newGeneration = result.generation } if _, err := client.File(ctx, executionenv.FileRequest{Context: rc2, Operation: executionenv.OpFileRead, Path: "lifecycle-proof.txt"}); err == nil { t.Fatal("revoked grant remained usable") @@ -482,23 +591,25 @@ func kubeValue(t *testing.T, ctx context.Context, kubeconfig string, args ...str func TestKindExecutionProductionHolderLossFencesActiveOperation(t *testing.T) { state, kubeconfig, ctx, cancel := requireProduction(t) defer cancel() - serviceClient, _ := productionClient(t, ctx, state, kubeconfig) - owner, binding, attached := createProductionEnvironment(t, ctx, serviceClient, "holder-loss") - rc, _ := acquireRun(t, ctx, serviceClient, owner, binding, attached, "holder-loss-run") + replicas, replicaUIDs := productionReplicaClients(t, ctx, state, kubeconfig) + holderClient, survivingClient := replicas[0], replicas[1] + owner, binding, attached := createProductionEnvironment(t, ctx, holderClient, "holder-loss") + rc, _ := acquireRun(t, ctx, holderClient, owner, binding, attached, "holder-loss-run") initial := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) var providers corev1.PodList - if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "-o", "json"), &providers); err != nil || len(providers.Items) < 2 { + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "-o", "json"), &providers); err != nil || len(providers.Items) != 2 { t.Fatal("holder-loss proof requires two provider replicas") } - holderPod := providers.Items[0].Name - holderForward := portForward(t, ctx, kubeconfig, "pod/"+holderPod, 8443) - holderTLS := loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") - holderClient, err := executionclient.New(holderForward.addr, holderTLS) - if err != nil { - t.Fatal(err) + holderPod := "" + for i := range providers.Items { + if string(providers.Items[i].UID) == replicaUIDs[0] { + holderPod = providers.Items[i].Name + } + } + if holderPod == "" { + t.Fatalf("holder replica UID %s no longer identifies a provider Pod", replicaUIDs[0]) } - defer holderClient.Close() commandDone := make(chan error, 1) go func() { @@ -509,10 +620,10 @@ func TestKindExecutionProductionHolderLossFencesActiveOperation(t *testing.T) { runKubectl(t, ctx, kubeconfig, "delete", "pod/"+holderPod, "-n", namespace, "--wait=true", "--timeout=90s") runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=180s") - if _, err := serviceClient.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "second-writer", Data: []byte("must-not-run")}); err == nil { + if _, err := survivingClient.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "second-writer", Data: []byte("must-not-run")}); err == nil { t.Fatal("surviving replica admitted an overlapping writer after holder loss") } - if _, err := serviceClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "overlap-run", OperationID: "overlap-run-acquire", TTL: time.Minute}); err == nil { + if _, err := survivingClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "overlap-run", OperationID: "overlap-run-acquire", TTL: time.Minute}); err == nil { t.Fatal("surviving replica admitted a new run while old operation ownership was unresolved") } fenced := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { @@ -530,22 +641,22 @@ func TestKindExecutionProductionHolderLossFencesActiveOperation(t *testing.T) { waitExecutorTerminal(t, ctx, kubeconfig, attached.Environment.ID) recovery := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: initial.Epoch, ExpectedPodUID: initial.PodUID, ExpectedPVCUID: initial.PVCUID, OperationID: "recover-holder-loss"} - if err := serviceClient.RecoverEnvironment(ctx, recovery); err != nil { + if err := survivingClient.RecoverEnvironment(ctx, recovery); err != nil { t.Fatalf("recover exact terminal executor: %v", err) } - if err := serviceClient.ReplaceExecutor(ctx, executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: initial.Epoch, ExpectedPodUID: initial.PodUID, ExpectedPVCUID: initial.PVCUID, OperationID: "replace-holder-loss"}); err != nil { + if err := survivingClient.ReplaceExecutor(ctx, executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: initial.Epoch, ExpectedPodUID: initial.PodUID, ExpectedPVCUID: initial.PVCUID, OperationID: "replace-holder-loss"}); err != nil { t.Fatalf("replace recovered executor: %v", err) } replaced := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Ready && s.PodUID != "" && s.PodUID != initial.PodUID }) if replaced.PVCUID != initial.PVCUID { t.Fatal("holder-loss recovery replaced the retained workspace") } - reattached := waitReady(t, ctx, serviceClient, owner, binding, attached.Environment) - verify, release := acquireRun(t, ctx, serviceClient, owner, binding, reattached, "holder-loss-verify") - if _, err := serviceClient.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "holder-loss-nonce"}); err != nil { + reattached := waitReady(t, ctx, survivingClient, owner, binding, attached.Environment) + verify, release := acquireRun(t, ctx, survivingClient, owner, binding, reattached, "holder-loss-verify") + if _, err := survivingClient.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "holder-loss-nonce"}); err != nil { t.Fatalf("post-recovery sentinel unavailable: %v", err) } - if _, err := serviceClient.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "second-writer"}); err == nil { + if _, err := survivingClient.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "second-writer"}); err == nil { t.Fatal("denied overlapping writer nevertheless reached the executor") } release() @@ -571,7 +682,8 @@ func waitExecutorTerminal(t *testing.T, ctx context.Context, kubeconfig, environ func TestKindExecutionProductionQuotaSaturation(t *testing.T) { state, kubeconfig, ctx, cancel := requireProduction(t) defer cancel() - client, _ := productionClient(t, ctx, state, kubeconfig) + replicas, replicaUIDs := productionReplicaClients(t, ctx, state, kubeconfig) + client := replicas[0] // First prove the provider's profile-wide CAS authority under concurrent // requests. Exactly one distinct allocation may consume quota-cas's sole slot. @@ -589,7 +701,7 @@ func TestKindExecutionProductionQuotaSaturation(t *testing.T) { <-start owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: fmt.Sprintf("quota-cas-%d", i)} binding := fmt.Sprintf("quota-cas-%d-%d", i, time.Now().UnixNano()) - out, err := client.Ensure(ctx, binding, "quota-cas", owner, "ensure-"+binding) + out, err := replicas[i].Ensure(ctx, binding, "quota-cas", owner, "ensure-"+binding) results <- ensureResult{owner: owner, binding: binding, out: out, err: err} }() } @@ -608,9 +720,9 @@ func TestKindExecutionProductionQuotaSaturation(t *testing.T) { } } if accepted == nil || denied != 1 { - t.Fatalf("quota CAS accepted=%v resource_exhausted=%d, want one each", accepted != nil, denied) + t.Fatalf("quota CAS pod_uids=%v accepted=%v resource_exhausted=%d, want one each", replicaUIDs, accepted != nil, denied) } - if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: accepted.out.Environment, Owner: accepted.owner, BindingID: accepted.binding, OperationID: "commit-" + accepted.binding}); err != nil { + if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: accepted.out.Environment, Owner: accepted.owner, BindingID: accepted.binding, OperationID: "ensure-" + accepted.binding}); err != nil { t.Fatal(err) } waitReady(t, ctx, client, accepted.owner, accepted.binding, accepted.out.Environment) @@ -660,7 +772,7 @@ func TestKindExecutionProductionQuotaSaturation(t *testing.T) { all := append(allocations, *accepted) for _, allocation := range all { - if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: allocation.out.Environment, Owner: allocation.owner, BindingID: allocation.binding, OperationID: "commit-" + allocation.binding}); err != nil { + if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: allocation.out.Environment, Owner: allocation.owner, BindingID: allocation.binding, OperationID: "ensure-" + allocation.binding}); err != nil { t.Fatal(err) } waitReady(t, ctx, client, allocation.owner, allocation.binding, allocation.out.Environment) @@ -703,6 +815,7 @@ func TestKindExecutionProductionPendingDeleteOutageRecovery(t *testing.T) { } runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=240s") + client, _ = productionClient(t, ctx, state, kubeconfig) intents, err := client.ListReferenceIntents(ctx, owner) if err != nil { t.Fatal(err) @@ -748,49 +861,88 @@ func TestKindExecutionProductionPendingDeleteOutageRecovery(t *testing.T) { } } -func TestKindExecutionProductionCompatiblePrototypeMigration(t *testing.T) { +type legacyMigrationFixture struct { + Environment executionenv.EnvironmentRef `json:"environment"` + Owner executionenv.Owner `json:"owner"` + Binding string `json:"binding"` + PodUID string `json:"pod_uid"` + PVCUID string `json:"pvc_uid"` + Malformed executionenv.EnvironmentRef `json:"malformed_environment"` + MalformedPodUID string `json:"malformed_pod_uid"` + MalformedPVCUID string `json:"malformed_pvc_uid"` + Insecure executionenv.EnvironmentRef `json:"insecure_environment"` + InsecurePodUID string `json:"insecure_pod_uid"` + InsecurePVCUID string `json:"insecure_pvc_uid"` +} + +func runKindExecutionProductionCompatiblePrototypeMigration(t *testing.T) { state, kubeconfig, ctx, cancel := requireProduction(t) defer cancel() client, _ := productionClient(t, ctx, state, kubeconfig) - owner, binding, attached := createProductionEnvironment(t, ctx, client, "migration") - before := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) - rc, release := acquireRun(t, ctx, client, owner, binding, attached, "migration-seed") - if _, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "migration-sentinel", Data: []byte("prototype-data\n")}); err != nil { - t.Fatal(err) + var fixture legacyMigrationFixture + raw, err := os.ReadFile(filepath.Join(state, "legacy-migration.json")) + if err != nil || json.Unmarshal(raw, &fixture) != nil { + t.Fatalf("load pre-upgrade legacy API fixture: %v", err) + } + if fixture.Environment.ID == "" || fixture.Environment.Revision == "" || fixture.PodUID == "" || fixture.PVCUID == "" || fixture.Binding == "" { + t.Fatal("pre-upgrade legacy API fixture identities are incomplete") } - release() - // The supported prototype profile is deliberately narrow: the stored CR, - // Pod, PVC, finalizer, ownership labels, and immutable security shape already - // match production; only schema markers and legacy reference encoding predate - // v2. Foreign or legacy-insecure Pods are never adopted by migration. - runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+attached.Environment.ID, "-n", namespace, "--type=merge", "-p", `{"spec":{"schemaVersion":1}}`) - runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+attached.Environment.ID, "-n", namespace, "--subresource=status", "--type=merge", "-p", `{"status":{"schemaVersion":1}}`) - waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return !s.Ready && s.ReadyReason == "IncompatibleSchema" }) - if _, err := client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); err == nil { + var stored map[string]any + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "executionenvironment", fixture.Environment.ID, "-n", namespace, "-o", "json"), &stored); err != nil { + t.Fatal(err) + } + status, _ := stored["status"].(map[string]any) + references, _ := status["references"].([]any) + if len(references) != 1 || references[0] != fixture.Binding { + t.Fatalf("legacy string references were not persisted through the real API upgrade: %#v", references) + } + before := readExecutionStatus(t, ctx, kubeconfig, fixture.Environment.ID) + if before.SpecSchema != 1 || before.StatusSchema != 1 || before.PodUID != fixture.PodUID || before.PVCUID != fixture.PVCUID { + t.Fatal("pre-upgrade fixture did not preserve schema-1 exact runtime identity") + } + if _, err := client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: fixture.Environment, Owner: fixture.Owner, BindingID: fixture.Binding}, Purpose: executionenv.PurposeSession}); err == nil { t.Fatal("prototype schema attached before explicit migration") } - if err := client.MigrateEnvironment(ctx, attached.Environment, owner, 1, "foreign-pod-uid", before.PVCUID, "migration-wrong-uid"); err == nil { + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, "foreign-pod-uid", before.PVCUID, "migration-wrong-uid"); err == nil { t.Fatal("migration adopted a foreign Pod UID") } - if err := client.MigrateEnvironment(ctx, attached.Environment, owner, 1, "", before.PVCUID, "migration-missing-uid"); err == nil { + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, "", before.PVCUID, "migration-missing-uid"); err == nil { t.Fatal("migration accepted a missing runtime UID") } - if err := client.MigrateEnvironment(ctx, attached.Environment, owner, 2, before.PodUID, before.PVCUID, "migration-unknown-version"); err == nil { + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 2, before.PodUID, before.PVCUID, "migration-unknown-version"); err == nil { t.Fatal("migration accepted an unrecognized source schema") } - if err := client.MigrateEnvironment(ctx, attached.Environment, owner, 1, before.PodUID, before.PVCUID, "migration-compatible-v1"); err != nil { + if err := client.MigrateEnvironment(ctx, fixture.Malformed, fixture.Owner, 1, fixture.MalformedPodUID, fixture.MalformedPVCUID, "migration-malformed-shape"); err == nil || !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("migration accepted a legacy reference shape outside the recognized prototype: %v", err) + } + if err := client.MigrateEnvironment(ctx, fixture.Insecure, fixture.Owner, 1, fixture.InsecurePodUID, fixture.InsecurePVCUID, "migration-insecure-runtime"); err == nil || !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("migration accepted an insecure legacy executor: %v", err) + } + if got := readExecutionStatus(t, ctx, kubeconfig, fixture.Insecure.ID); got.SpecSchema != 1 || got.StatusSchema != 1 { + t.Fatal("rejected insecure legacy executor was rewritten") + } + // Normalize the deliberately rejected test-owned object through the API so it + // cannot poison later client-scoped reference reconciliation in this retained cluster. + normalized := fmt.Sprintf(`[{"bindingID":"legacy-malformed-binding","state":"Published","operationID":"migration-fixture-normalize","createdAt":%q}]`, time.Now().UTC().Format(time.RFC3339Nano)) + runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Malformed.ID, "-n", namespace, "--subresource=status", "--type=merge", "-p", `{"status":{"references":`+normalized+`}}`) + insecureNormalized := fmt.Sprintf(`[{"bindingID":"legacy-insecure-binding","state":"Published","operationID":"migration-insecure-fixture-normalize","createdAt":%q}]`, time.Now().UTC().Format(time.RFC3339Nano)) + runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Insecure.ID, "-n", namespace, "--subresource=status", "--type=merge", "-p", `{"status":{"references":`+insecureNormalized+`}}`) + if err := client.MigrateEnvironment(ctx, fixture.Malformed, fixture.Owner, 1, fixture.MalformedPodUID, fixture.MalformedPVCUID, "migration-normalized-shape"); err != nil { + t.Fatalf("normalize rejected migration fixture: %v", err) + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, before.PodUID, before.PVCUID, "migration-compatible-v1"); err != nil { t.Fatalf("migrate compatible prototype: %v", err) } - after := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Ready && s.SpecSchema == 2 && s.StatusSchema == 2 }) + after := waitExecutionStatus(t, ctx, kubeconfig, fixture.Environment.ID, func(s executionStatus) bool { return s.Ready && s.SpecSchema == 2 && s.StatusSchema == 2 }) if after.PodUID != before.PodUID || after.PVCUID != before.PVCUID || after.Epoch != before.Epoch+1 { t.Fatal("migration changed runtime identity or failed to advance only the fence epoch") } - if got := executionReferences(t, ctx, kubeconfig, attached.Environment.ID); got[binding] != string(executionenv.ReferencePublished) { - t.Fatal("migration changed the published source reference") + if got := executionReferences(t, ctx, kubeconfig, fixture.Environment.ID); got[fixture.Binding] != string(executionenv.ReferencePublished) { + t.Fatal("migration did not convert the persisted legacy string reference") } - reattached := waitReady(t, ctx, client, owner, binding, attached.Environment) - verify, verifyRelease := acquireRun(t, ctx, client, owner, binding, reattached, "migration-verify") + reattached := waitReady(t, ctx, client, fixture.Owner, fixture.Binding, fixture.Environment) + verify, verifyRelease := acquireRun(t, ctx, client, fixture.Owner, fixture.Binding, reattached, "migration-verify") got, err := client.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "migration-sentinel"}) if err != nil || string(got.Data) != "prototype-data\n" { t.Fatal("migration lost prototype workspace data") @@ -813,10 +965,7 @@ func TestKindExecutionProductionClearForkLifecycle(t *testing.T) { source := createSession(t, ctx, agent.addr, alice) assertMockJourney(t, prompt(t, ctx, agent.addr, source, alice, "run the scripted remote qualification")) owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "alice"} - allocation, err := provider.Ensure(ctx, source, "go", owner, "clear-fork-lookup-"+source) - if err != nil { - t.Fatalf("resolve source placement: %v", err) - } + allocation := executionenv.EnsureEnvironmentResponse{Environment: environmentForBinding(t, ctx, kubeconfig, source)} attached := waitReady(t, ctx, provider, owner, source, allocation.Environment) before := readExecutionStatus(t, ctx, kubeconfig, allocation.Environment.ID) @@ -869,6 +1018,37 @@ func TestKindExecutionProductionClearForkLifecycle(t *testing.T) { } } +func environmentForBinding(t *testing.T, ctx context.Context, kubeconfig, binding string) executionenv.EnvironmentRef { + t.Helper() + var list struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Revision string `json:"revision"` + } `json:"spec"` + Status struct { + References []struct { + BindingID string `json:"bindingID"` + } `json:"references"` + } `json:"status"` + } `json:"items"` + } + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "executionenvironments", "-n", namespace, "-o", "json"), &list); err != nil { + t.Fatal(err) + } + for _, item := range list.Items { + for _, ref := range item.Status.References { + if ref.BindingID == binding && item.Metadata.Name != "" && item.Spec.Revision != "" { + return executionenv.EnvironmentRef{ID: item.Metadata.Name, Revision: item.Spec.Revision} + } + } + } + t.Fatalf("no exact execution environment owns binding %s", binding) + return executionenv.EnvironmentRef{} +} + func successorSession(t *testing.T, ctx context.Context, addr, source, token, operation string) string { t.Helper() status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+source+"/"+operation, token, []byte(`{}`)) diff --git a/e2e/k8s_execution/qualification_test.go b/e2e/k8s_execution/qualification_test.go index 24b698f16c..895f08ff80 100644 --- a/e2e/k8s_execution/qualification_test.go +++ b/e2e/k8s_execution/qualification_test.go @@ -36,6 +36,9 @@ func TestKindExecutionQualification(t *testing.T) { pki := filepath.Join(state, "pki") ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) defer cancel() + applyMockScript(t, ctx, kubeconfig, "mock-script.json") + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecak8s", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecak8s", "-n", namespace, "--timeout=240s") baseline := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev") providerForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) @@ -59,7 +62,7 @@ func TestKindExecutionQualification(t *testing.T) { if err != nil { t.Fatalf("ensure: %v", err) } - if err := providerClient.CommitReference(ctx, executionenv.ReferenceRequest{Environment: first.Environment, Owner: owner, BindingID: binding, OperationID: "commit-" + operationID}); err != nil { + if err := providerClient.CommitReference(ctx, executionenv.ReferenceRequest{Environment: first.Environment, Owner: owner, BindingID: binding, OperationID: operationID}); err != nil { t.Fatalf("commit reference: %v", err) } if first.Environment.ID == "" || first.Environment.Revision == "" { @@ -495,7 +498,11 @@ func applyReattachScript(t *testing.T, ctx context.Context, kubeconfig, state st t.Helper() root := filepath.Clean(filepath.Join(state, "..", "..", "..")) _ = root - source := filepath.Join(repoRoot(t), "deploy", "mecatl-execution-kind", "mock-script-reattach.json") + applyMockScript(t, ctx, kubeconfig, "mock-script-reattach.json") +} +func applyMockScript(t *testing.T, ctx context.Context, kubeconfig, filename string) { + t.Helper() + source := filepath.Join(repoRoot(t), "deploy", "mecatl-execution-kind", filename) cmd := command(ctx, kubeconfig, "create", "configmap", "execution-mock", "-n", namespace, "--from-file=mock-script.json="+source, "--dry-run=client", "-o", "yaml") rendered, err := cmd.Output() if err != nil { diff --git a/go.work.sum b/go.work.sum index c2e22bc597..1b5eafdf7c 100644 --- a/go.work.sum +++ b/go.work.sum @@ -478,6 +478,7 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/certificate-transparency-go v1.3.3 h1:hq/rSxztSkXN2tx/3jQqF6Xc0O565UQPdHrOWvZwybo= github.com/google/certificate-transparency-go v1.3.3/go.mod h1:iR17ZgSaXRzSa5qvjFl8TnVD5h8ky2JMVio+dzoKMgA= diff --git a/internal/adapter/executionclient/client.go b/internal/adapter/executionclient/client.go index e697f64054..7dfca295d4 100644 --- a/internal/adapter/executionclient/client.go +++ b/internal/adapter/executionclient/client.go @@ -184,6 +184,17 @@ func (c *Client) ListReferenceIntents(ctx context.Context, owner executionenv.Ow return c.listReferenceIntents(ctx, &executionv1.ListReferenceIntentsRequest{Owner: ownerToProto(owner), Limit: 64}) } +func (c *Client) FindReferenceIntent(ctx context.Context, owner executionenv.Owner, environment executionenv.EnvironmentRef, binding string) (executionenv.ReferenceIntent, error) { + intents, err := c.listReferenceIntents(ctx, &executionv1.ListReferenceIntentsRequest{Owner: ownerToProto(owner), Limit: 1, Environment: refToProto(environment), BindingId: binding}) + if err != nil { + return executionenv.ReferenceIntent{}, err + } + if len(intents) != 1 { + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} + } + return intents[0], nil +} + func (c *Client) ListAllReferenceIntents(ctx context.Context) ([]executionenv.ReferenceIntent, error) { return c.listReferenceIntents(ctx, &executionv1.ListReferenceIntentsRequest{Limit: 64}) } @@ -518,14 +529,17 @@ func (p *Provider) PrepareReferenceDelete(ctx context.Context, req server.Placem return nil, err } op := "" - intents, err := p.client.ListReferenceIntents(ctx, owner) - if err != nil { - return nil, mapPlacementError(err) - } - for _, intent := range intents { - if intent.Environment.ID == req.Ref.ID && intent.Environment.Revision == req.Ref.Revision && intent.BindingID == string(req.SourceBindingID) && intent.State == executionenv.ReferencePendingDelete { - op = intent.OperationID - break + exact := executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision} + intent, err := p.client.FindReferenceIntent(ctx, owner, exact, string(req.SourceBindingID)) + if err == nil { + if intent.State != executionenv.ReferencePendingDelete || intent.Environment != exact || intent.BindingID != string(req.SourceBindingID) { + return nil, server.ErrPlacementChanged + } + op = intent.OperationID + } else { + var remote *executionenv.Error + if !errors.As(err, &remote) || remote.Code != executionenv.CodeNotFound { + return nil, mapPlacementError(err) } } if op == "" { @@ -582,6 +596,11 @@ type runHandle struct { } func (h *runHandle) Environment() tool.Environment { return h.environment } +func (h *runHandle) RenewalDeadline() time.Time { + h.mu.Lock() + defer h.mu.Unlock() + return h.credentials.expiry() +} func (h *runHandle) Renew(ctx context.Context) error { h.mu.Lock() defer h.mu.Unlock() @@ -902,6 +921,12 @@ func (g *grantContext) refresh(ctx context.Context) (executionenv.RequestContext return g.current(ctx) } +func (g *grantContext) expiry() time.Time { + g.mu.Lock() + defer g.mu.Unlock() + return g.expiresAt +} + func (g *grantContext) replace(claim executionenv.RunClaim) { g.mu.Lock() defer g.mu.Unlock() diff --git a/internal/adapter/executionclient/client_test.go b/internal/adapter/executionclient/client_test.go index f4cf4c335e..b4054bea7f 100644 --- a/internal/adapter/executionclient/client_test.go +++ b/internal/adapter/executionclient/client_test.go @@ -112,6 +112,9 @@ func (*integrationBackend) CancelReferenceDelete(context.Context, executionenv.E func (*integrationBackend) ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) { return nil, nil } +func (*integrationBackend) FindReferenceIntent(context.Context, executionenv.EnvironmentRef, string, string, string) (executionenv.ReferenceIntent, error) { + return executionenv.ReferenceIntent{}, nil +} func certificate(t *testing.T, parent *x509.Certificate, parentKey *ecdsa.PrivateKey, serverName string, client bool) (tls.Certificate, *x509.Certificate, *ecdsa.PrivateKey) { t.Helper() diff --git a/internal/adapter/executioncontroller/admin_lifecycle.go b/internal/adapter/executioncontroller/admin_lifecycle.go index b551e4b975..ec0af39ce2 100644 --- a/internal/adapter/executioncontroller/admin_lifecycle.go +++ b/internal/adapter/executioncontroller/admin_lifecycle.go @@ -34,14 +34,26 @@ func (s *Store) RetireExact(ctx context.Context, q adminLifecycleRequest) error return s.startLifecycle(ctx, q, "RetireEnvironment", true) } -func (s *Store) startLifecycle(ctx context.Context, q adminLifecycleRequest, kind string, requireNoRefs bool) error { +func (s *Store) startLifecycle(ctx context.Context, q adminLifecycleRequest, kind string, requireNoRefs bool) error { //nolint:gocyclo // Exact admission keeps all identity and receipt checks together. if q.ExpectedEpoch == 0 || q.ExpectedEpoch > math.MaxInt64 || q.ExpectedPodUID == "" || q.ExpectedPVCUID == "" || q.OperationID == "" { return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "exact lifecycle identity is required"} } + if err := s.persistObservedTerminalProof(ctx, q); err != nil { + return err + } return s.retryUpdateStatus(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if kind == "ReplaceExecutor" && replacementReceiptMatches(o, q) { + return nil + } if err := exactAdminSubject(o, q); err != nil { return err } + if kind == "RetireEnvironment" && conditionTrue(o, "Retired") { + if exactTerminationProofOperationMatches(o, q) { + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is already retired by another operation"} + } if existing, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation"); found { if text(existing, "id") == q.OperationID && text(existing, "type") == kind && text(existing, "expectedPodUID") == q.ExpectedPodUID && text(existing, "expectedPVCUID") == q.ExpectedPVCUID && intNested(existing, "expectedEpoch") == int64(q.ExpectedEpoch) { //nolint:gosec // validated above. return nil @@ -71,12 +83,63 @@ func lifecycleAdmissionBlocked(o *unstructured.Unstructured, q adminLifecycleReq !conditionTrue(o, "Ready") && !exactTerminationProofMatches(o, q) } +func replacementReceiptMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + return textNested(o.Object, "status", "lastReplacement", operationIDField) == q.OperationID && + textNested(o.Object, "status", "lastReplacement", "previousPodUID") == q.ExpectedPodUID && + textNested(o.Object, "status", "lastReplacement", "pvcUID") == q.ExpectedPVCUID && + intNested(o.Object, "status", "lastReplacement", "previousEpoch") == int64(q.ExpectedEpoch) //nolint:gosec // q.ExpectedEpoch is bounded by startLifecycle. +} + func exactTerminationProofMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { return textNested(o.Object, "status", "terminationProof", "podUID") == q.ExpectedPodUID && textNested(o.Object, "status", "terminationProof", "pvcUID") == q.ExpectedPVCUID && intNested(o.Object, "status", "terminationProof", "epoch") == int64(q.ExpectedEpoch) //nolint:gosec // q.ExpectedEpoch is bounded above. } +func exactTerminationProofOperationMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + return exactTerminationProofMatches(o, q) && textNested(o.Object, "status", "terminationProof", operationIDField) == q.OperationID +} + +func (s *Store) persistObservedTerminalProof(ctx context.Context, q adminLifecycleRequest) error { //nolint:gocyclo // Exact terminal proof deliberately fails closed at every observable mismatch. + o, err := s.resources.Get(ctx, q.Environment.ID, metav1.GetOptions{}) + if err != nil { + return err + } + if replacementReceiptMatches(o, q) { + return nil + } + if err := exactAdminSubject(o, q); err != nil { + return err + } + if conditionTrue(o, "Ready") || conditionTrue(o, "Retired") || exactTerminationProofOperationMatches(o, q) { + return nil + } + if textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy || s.kube == nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not healthy and idle"} + } + podName, pvcName := textNested(o.Object, "status", "pod", "name"), textNested(o.Object, "status", "pvc", "name") + pod, err := s.kube.CoreV1().Pods(s.namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil || string(pod.UID) != q.ExpectedPodUID || !podTerminal(pod) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "exact terminal executor proof is unavailable"} + } + pvc, err := s.kube.CoreV1().PersistentVolumeClaims(s.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil || string(pvc.UID) != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "exact workspace identity is unavailable"} + } + return s.retryUpdateStatus(ctx, q.Environment.ID, func(current *unstructured.Unstructured) error { + if err := exactAdminSubject(current, q); err != nil { + return err + } + if conditionTrue(current, "Ready") || exactTerminationProofOperationMatches(current, q) { + return nil + } + if textNested(current.Object, "status", "activeRun", "claimID") != "" || textNested(current.Object, "status", "activeOperation", "id") != "" || textNested(current.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not healthy and idle"} + } + return unstructured.SetNestedMap(current.Object, terminationProof(q, pod), "status", "terminationProof") + }) +} + func exactAdminSubject(o *unstructured.Unstructured, q adminLifecycleRequest) error { if textNested(o.Object, "spec", "ownerHash") != q.OwnerHash || textNested(o.Object, "spec", "clientHash") != hashText(q.Client) || textNested(o.Object, "spec", "revision") != q.Environment.Revision { return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} @@ -97,6 +160,9 @@ func (s *Store) RecoverEnvironment(ctx context.Context, q adminLifecycleRequest) if err := exactAdminSubject(o, q); err != nil { return err } + if exactTerminationProofOperationMatches(o, q) { + return nil + } if textNested(o.Object, "status", "fenceState") != "FenceUnknown" { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not fenced"} } @@ -105,6 +171,9 @@ func (s *Store) RecoverEnvironment(ctx context.Context, q adminLifecycleRequest) }); err != nil { return err } + if podName == "" { + return nil + } pod, err := s.kube.CoreV1().Pods(s.namespace).Get(ctx, podName, metav1.GetOptions{}) if err != nil || string(pod.UID) != q.ExpectedPodUID || !podTerminal(pod) { return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "terminal executor proof unavailable; use the external-fencing operator runbook"} @@ -117,6 +186,9 @@ func (s *Store) RecoverEnvironment(ctx context.Context, q adminLifecycleRequest) if err := exactAdminSubject(o, q); err != nil { return err } + if exactTerminationProofOperationMatches(o, q) { + return nil + } proof := terminationProof(q, pod) if err := unstructured.SetNestedMap(o.Object, proof, "status", "terminationProof"); err != nil { return err @@ -281,6 +353,10 @@ func (s *Store) MigrateEnvironment(ctx context.Context, q adminLifecycleRequest) } func (s *Store) verifyRuntimeUIDs(ctx context.Context, o *unstructured.Unstructured, q adminLifecycleRequest) error { + profile, ok := s.profiles.get(textNested(o.Object, "spec", "profile")) + if !ok || profile.Digest != textNested(o.Object, "spec", "profileDigest") { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype profile is unavailable or changed"} + } pod, err := s.kube.CoreV1().Pods(s.namespace).Get(ctx, textNested(o.Object, "status", "pod", "name"), metav1.GetOptions{}) if err != nil || string(pod.UID) != q.ExpectedPodUID { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype executor identity is not observable"} @@ -289,6 +365,12 @@ func (s *Store) verifyRuntimeUIDs(ctx context.Context, o *unstructured.Unstructu if err != nil || string(pvc.UID) != q.ExpectedPVCUID { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype workspace identity is not observable"} } + if err := validatePVC(o, profile, pvc); err != nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype workspace shape is incompatible"} + } + if err := validatePod(o, profile, pvc.Name, pod); err != nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype executor shape is insecure or incompatible"} + } return nil } diff --git a/internal/adapter/executioncontroller/controller.go b/internal/adapter/executioncontroller/controller.go index 2bee571f6a..919d75e2a1 100644 --- a/internal/adapter/executioncontroller/controller.go +++ b/internal/adapter/executioncontroller/controller.go @@ -324,6 +324,9 @@ func validatePVC(env *unstructured.Unstructured, p resolvedProfile, pvc *corev1. } func validatePod(env *unstructured.Unstructured, p resolvedProfile, pvcName string, pod *corev1.Pod) error { //nolint:gocyclo // Every security-sensitive immutable field is checked explicitly. + if pod.DeletionTimestamp != nil { + return errors.New("pod is terminating") + } storedUID := textNested(env.Object, "status", "pod", "uid") if pod.Labels["execution.mecatl.dev/environment"] != env.GetName() || pod.Labels["execution.mecatl.dev/profile"] != hashText(textNested(env.Object, "spec", "profile"))[:16] || (storedUID != "" && storedUID != string(pod.UID)) || len(pod.OwnerReferences) != 1 { return errors.New("pod ownership identity mismatch") @@ -433,6 +436,9 @@ func setConditionObject(o *unstructured.Unstructured, name string, status bool, _ = unstructured.SetNestedSlice(o.Object, next, "status", "conditions") } func podReady(p *corev1.Pod) bool { + if p == nil || p.DeletionTimestamp != nil { + return false + } for _, c := range p.Status.Conditions { if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { return true diff --git a/internal/adapter/executioncontroller/controller_lifecycle.go b/internal/adapter/executioncontroller/controller_lifecycle.go index 5d5511cace..424623d5ea 100644 --- a/internal/adapter/executioncontroller/controller_lifecycle.go +++ b/internal/adapter/executioncontroller/controller_lifecycle.go @@ -24,11 +24,16 @@ func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.U if operationID == "" || pvcUID == "" { return r.setFenceUnknown(ctx, env, "lifecycle operation identity is malformed") } + proofMatches := textNested(env.Object, "status", "terminationProof", operationIDField) == operationID && textNested(env.Object, "status", "terminationProof", "podUID") == podUID && textNested(env.Object, "status", "terminationProof", "pvcUID") == pvcUID && intNested(env.Object, "status", "terminationProof", "epoch") == intNested(op, "expectedEpoch") + podIdentityMatches := textNested(env.Object, "status", "pod", "uid") == podUID + if phase == "CreatingReplacement" { + podIdentityMatches = textNested(env.Object, "status", "pod", "uid") == "" && proofMatches + } if kind == "DeleteRetiredEnvironment" { return r.reconcileRetainedDelete(ctx, env, op, pvcName, pvcUID) } epoch := intNested(op, "expectedEpoch") - if podUID == "" || epoch <= 0 || textNested(env.Object, "status", "pod", "uid") != podUID || textNested(env.Object, "status", "pvc", "uid") != pvcUID || intNested(env.Object, "status", "epoch") != epoch { + if podUID == "" || epoch <= 0 || !podIdentityMatches || textNested(env.Object, "status", "pvc", "uid") != pvcUID || intNested(env.Object, "status", "epoch") != epoch { return r.setFenceUnknown(ctx, env, "lifecycle operation no longer matches the exact executor and workspace") } pvc, err := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Get(ctx, pvcName, metav1.GetOptions{}) @@ -38,7 +43,6 @@ func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.U } return r.setFenceUnknown(ctx, env, "authoritative workspace identity is unavailable") } - proofMatches := textNested(env.Object, "status", "terminationProof", operationIDField) == operationID && textNested(env.Object, "status", "terminationProof", "podUID") == podUID && textNested(env.Object, "status", "terminationProof", "pvcUID") == pvcUID && intNested(env.Object, "status", "terminationProof", "epoch") == epoch switch phase { case "Quiescing": pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) @@ -188,6 +192,7 @@ func (r *Reconciler) finishReplacement(ctx context.Context, env *unstructured.Un } _ = unstructured.SetNestedField(o.Object, epoch+1, "status", "epoch") _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pod.Name, "uid": string(pod.UID)}, "status", "pod") + _ = unstructured.SetNestedMap(o.Object, map[string]any{operationIDField: operationID, "previousPodUID": text(op, "expectedPodUID"), "replacementPodUID": string(pod.UID), "pvcUID": text(op, "expectedPVCUID"), "previousEpoch": intNested(op, "expectedEpoch"), "replacementEpoch": epoch + 1}, "status", "lastReplacement") unstructured.RemoveNestedField(o.Object, "status", "lifecycleOperation") setConditionObject(o, "Ready", true, "ReplacementReady", "replacement executor is ready on the retained workspace") return nil @@ -195,11 +200,15 @@ func (r *Reconciler) finishReplacement(ctx context.Context, env *unstructured.Un } func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructured.Unstructured, op map[string]any, pvcName, pvcUID string) error { - if text(op, "phase") != "DeletingPVC" || !conditionTrue(env, "Retired") || !conditionTrue(env, "ExecutorTerminated") { + phase := text(op, "phase") + if (phase != "DeletingPVC" && phase != "ReleasingSlot") || !conditionTrue(env, "Retired") || !conditionTrue(env, "ExecutorTerminated") { return r.setFenceUnknown(ctx, env, "retained deletion state is invalid") } pvc, err := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Get(ctx, pvcName, metav1.GetOptions{}) if err == nil { + if phase != "DeletingPVC" { + return r.setFenceUnknown(ctx, env, "retained PVC reappeared after deallocation began") + } if string(pvc.UID) != pvcUID { return r.setFenceUnknown(ctx, env, "retained PVC UID changed") } @@ -212,11 +221,17 @@ func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructu if !apierrors.IsNotFound(err) { return err } + if phase == "DeletingPVC" { + return r.setLifecyclePhase(ctx, env, text(op, "id"), "ReleasingSlot") + } res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) current, err := res.Get(ctx, env.GetName(), metav1.GetOptions{}) if err != nil { return err } + if err := releaseProfileSlot(ctx, r.kube, r.namespace, textNested(current.Object, "spec", "profile"), current.GetName()); err != nil { + return err + } current.SetFinalizers(withoutString(current.GetFinalizers(), environmentFinalizer)) updated, err := res.Update(ctx, current, metav1.UpdateOptions{}) if err != nil { @@ -226,7 +241,7 @@ func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructu if err := res.Delete(ctx, updated.GetName(), metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { return err } - return releaseProfileSlot(ctx, r.kube, r.namespace, textNested(updated.Object, "spec", "profile"), updated.GetName()) + return nil } func withoutString(values []string, remove string) []string { diff --git a/internal/adapter/executioncontroller/controller_test.go b/internal/adapter/executioncontroller/controller_test.go index 7d90454547..098270d2ae 100644 --- a/internal/adapter/executioncontroller/controller_test.go +++ b/internal/adapter/executioncontroller/controller_test.go @@ -84,6 +84,40 @@ func TestReconcileCreatesTokenlessNonRootPodAndRetainedPVC(t *testing.T) { t.Fatal("tmp volume is not profile-bounded") } } +func TestTerminatingPodIsUnavailableDuringReconcile(t *testing.T) { + ctx := t.Context() + env := testEnvironment() + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + pod, err := k.CoreV1().Pods("ns").Get(ctx, "executor-test", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + now := metav1.Now() + pod.DeletionTimestamp = &now + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := k.CoreV1().Pods("ns").Update(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if conditionTrue(got, "Ready") { + t.Fatalf("terminating pod remained ready: %v", got.Object["status"]) + } +} + func TestRepeatedReconcileDoesNotRewriteUnchangedStatus(t *testing.T) { ctx := context.Background() env := testEnvironment() diff --git a/internal/adapter/executioncontroller/crd_schema_test.go b/internal/adapter/executioncontroller/crd_schema_test.go index ef5a834992..1ffa9b0fd4 100644 --- a/internal/adapter/executioncontroller/crd_schema_test.go +++ b/internal/adapter/executioncontroller/crd_schema_test.go @@ -6,13 +6,15 @@ import ( apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsvalidation "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation" "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/yaml" ) -func TestExecutionEnvironmentCRDPreservesControllerStatus(t *testing.T) { +func loadExecutionEnvironmentCRD(t *testing.T) (*apiextensionsv1.CustomResourceDefinition, *apiextensions.CustomResourceDefinition) { + t.Helper() raw, err := os.ReadFile("../../../deploy/helm/mecatl-execution/crds/executionenvironment.yaml") if err != nil { t.Fatal(err) @@ -21,6 +23,23 @@ func TestExecutionEnvironmentCRDPreservesControllerStatus(t *testing.T) { if err := yaml.Unmarshal(raw, &crd); err != nil { t.Fatal(err) } + apiextensionsv1.SetDefaults_CustomResourceDefinition(&crd) + var internal apiextensions.CustomResourceDefinition + if err := apiextensionsv1.Convert_v1_CustomResourceDefinition_To_apiextensions_CustomResourceDefinition(&crd, &internal, nil); err != nil { + t.Fatal(err) + } + return &crd, &internal +} + +func TestExecutionEnvironmentCRDIsAcceptedByKubernetesValidation(t *testing.T) { + _, internal := loadExecutionEnvironmentCRD(t) + if errs := apiextensionsvalidation.ValidateCustomResourceDefinition(t.Context(), internal); len(errs) != 0 { + t.Fatalf("CRD is not installable: %v", errs.ToAggregate()) + } +} + +func TestExecutionEnvironmentCRDPreservesControllerStatus(t *testing.T) { + crd, _ := loadExecutionEnvironmentCRD(t) var internal apiextensions.JSONSchemaProps if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(crd.Spec.Versions[0].Schema.OpenAPIV3Schema, &internal, nil); err != nil { t.Fatal(err) @@ -33,7 +52,8 @@ func TestExecutionEnvironmentCRDPreservesControllerStatus(t *testing.T) { "schemaVersion": int64(2), "observedGeneration": int64(9), "epoch": int64(3), "grantGeneration": int64(4), "fenceState": "Healthy", "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "op", "sourceBindingID": "source", "createdAt": "2026-09-17T00:00:00Z"}}, "pvc": map[string]any{"name": "pvc", "uid": "pvc-uid"}, "pod": map[string]any{"name": "pod", "uid": "pod-uid"}, - "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "ownerHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "clientHash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "epoch": int64(3), "grantGeneration": int64(4), "expiresAt": "2026-09-17T00:01:00Z", "lastRenewOperationID": "renew"}, + "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "ownerHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "clientHash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "epoch": int64(3), "grantGeneration": int64(4), "expiresAt": "2026-09-17T00:01:00Z"}, + "renewReceipts": []any{map[string]any{"operationID": "renew", "fingerprint": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "expiresAt": "2026-09-17T00:01:00Z"}}, "activeOperation": map[string]any{"id": "file", "operation": "file.read", "startedAt": "2026-09-17T00:00:00Z", "claimID": "claim", "runID": "run", "epoch": int64(3), "holderID": "holder", "renewedAt": "2026-09-17T00:00:01Z", "expiresAt": "2026-09-17T00:01:00Z"}, "lifecycleOperation": map[string]any{"id": "life", "type": "ReplaceExecutor", "phase": "Quiescing", "expectedEpoch": int64(3), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid", "createdAt": "2026-09-17T00:00:00Z"}, "terminationProof": map[string]any{"operationID": "life", "podUID": "pod-uid", "pvcUID": "pvc-uid", "epoch": int64(3), "podPhase": "Failed", "observedAt": "2026-09-17T00:00:00Z"}, @@ -41,7 +61,7 @@ func TestExecutionEnvironmentCRDPreservesControllerStatus(t *testing.T) { "conditions": []any{map[string]any{"type": "Ready", "status": "True", "reason": "Reconciled", "message": "ready", "observedGeneration": int64(9), "lastTransitionTime": "2026-09-17T00:00:00Z"}}, }} pruning.Prune(obj, structural, true) - for _, path := range [][]string{{"status", "schemaVersion"}, {"status", "observedGeneration"}, {"status", "epoch"}, {"status", "grantGeneration"}, {"status", "references"}, {"status", "pvc"}, {"status", "pod"}, {"status", "activeRun", "grantGeneration"}, {"status", "activeOperation"}, {"status", "lifecycleOperation"}, {"status", "terminationProof"}, {"status", "migrationOperation"}, {"status", "lastMigrationOperationID"}, {"status", "conditions"}} { + for _, path := range [][]string{{"status", "schemaVersion"}, {"status", "observedGeneration"}, {"status", "epoch"}, {"status", "grantGeneration"}, {"status", "references"}, {"status", "pvc"}, {"status", "pod"}, {"status", "activeRun", "grantGeneration"}, {"status", "renewReceipts"}, {"status", "activeOperation"}, {"status", "lifecycleOperation"}, {"status", "terminationProof"}, {"status", "migrationOperation"}, {"status", "lastMigrationOperationID"}, {"status", "conditions"}} { if _, found, err := unstructured.NestedFieldNoCopy(obj, path...); err != nil || !found { t.Fatalf("CRD pruning removed controller field %v (found=%t err=%v)", path, found, err) } diff --git a/internal/adapter/executioncontroller/handler.go b/internal/adapter/executioncontroller/handler.go index 4205472036..8e3eee2afe 100644 --- a/internal/adapter/executioncontroller/handler.go +++ b/internal/adapter/executioncontroller/handler.go @@ -91,6 +91,7 @@ type lifecycleBackend interface { ConfirmReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error CancelReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) + FindReferenceIntent(context.Context, executionenv.EnvironmentRef, string, string, string) (executionenv.ReferenceIntent, error) } type ownerIntentBackend interface { @@ -407,7 +408,20 @@ func (h *Handler) ListReferenceIntents(ctx context.Context, q *executionv1.ListR return nil, lifecycleErr } var intents []executionenv.ReferenceIntent - if q.Owner != nil { + exactRef := refFromProto(q.GetEnvironment()) + exactLookup := q.GetEnvironment() != nil || q.GetBindingId() != "" + if exactLookup { + owner, ok := ownerFromProto(q.GetOwner()) + if !ok || !validRef(exactRef) || !validBinding(q.GetBindingId()) { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + intent, findErr := lifecycle.FindReferenceIntent(ctx, exactRef, c.id, ownerHash(owner), q.GetBindingId()) + if findErr != nil { + return nil, backendError(findErr) + } + intent.Owner = owner + intents = []executionenv.ReferenceIntent{intent} + } else if q.Owner != nil { owner, ok := ownerFromProto(q.Owner) if !ok { return nil, wireError(executionenv.CodeInvalidArgument, false) diff --git a/internal/adapter/executioncontroller/handler_test.go b/internal/adapter/executioncontroller/handler_test.go index 7a67cc1e59..c3b662097b 100644 --- a/internal/adapter/executioncontroller/handler_test.go +++ b/internal/adapter/executioncontroller/handler_test.go @@ -98,6 +98,9 @@ func (*fakeBackend) CancelReferenceDelete(context.Context, executionenv.Environm func (*fakeBackend) ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) { return nil, nil } +func (*fakeBackend) FindReferenceIntent(context.Context, executionenv.EnvironmentRef, string, string, string) (executionenv.ReferenceIntent, error) { + return executionenv.ReferenceIntent{}, nil +} type adminFakeBackend struct { *fakeBackend diff --git a/internal/adapter/executioncontroller/legacy_fixture_kind.go b/internal/adapter/executioncontroller/legacy_fixture_kind.go new file mode 100644 index 0000000000..c2f7a1260f --- /dev/null +++ b/internal/adapter/executioncontroller/legacy_fixture_kind.go @@ -0,0 +1,158 @@ +//go:build kind_execution_e2e + +package executioncontroller + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// LegacyMigrationSeed identifies one pre-upgrade API-server fixture. +type LegacyMigrationSeed struct { + Environment executionenv.EnvironmentRef `json:"environment"` + Owner executionenv.Owner `json:"owner"` + Binding string `json:"binding"` + PodUID string `json:"pod_uid"` + PVCUID string `json:"pvc_uid"` + Malformed executionenv.EnvironmentRef `json:"malformed_environment"` + MalformedPodUID string `json:"malformed_pod_uid"` + MalformedPVCUID string `json:"malformed_pvc_uid"` + Insecure executionenv.EnvironmentRef `json:"insecure_environment"` + InsecurePodUID string `json:"insecure_pod_uid"` + InsecurePVCUID string `json:"insecure_pvc_uid"` +} + +// SeedLegacyMigrationFixture creates the narrow security-compatible prototype +// accepted by MigrateEnvironment plus malformed and insecure negative controls. +// It is excluded from regular builds. +func SeedLegacyMigrationFixture(ctx context.Context, d dynamic.Interface, kube kubernetes.Interface, namespace, profilesPath string) (LegacyMigrationSeed, error) { + profiles, err := LoadProfiles(profilesPath) + if err != nil { + return LegacyMigrationSeed{}, err + } + profile, ok := profiles.get("go") + if !ok { + return LegacyMigrationSeed{}, fmt.Errorf("go profile unavailable") + } + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "production-migration"} + client := "spiffe://mecatl.test/client/mecak8s" + recognized, err := seedOneLegacyEnvironment(ctx, d, kube, namespace, profile, owner, client, "legacy-migration", "legacy-migration-binding", []any{"legacy-migration-binding"}, false) + if err != nil { + return LegacyMigrationSeed{}, err + } + malformed, err := seedOneLegacyEnvironment(ctx, d, kube, namespace, profile, owner, client, "legacy-migration-malformed", "legacy-malformed-binding", []any{""}, false) + if err != nil { + return LegacyMigrationSeed{}, err + } + insecure, err := seedOneLegacyEnvironment(ctx, d, kube, namespace, profile, owner, client, "legacy-migration-insecure", "legacy-insecure-binding", []any{"legacy-insecure-binding"}, true) + if err != nil { + return LegacyMigrationSeed{}, err + } + return LegacyMigrationSeed{Environment: recognized.ref, Owner: owner, Binding: "legacy-migration-binding", PodUID: recognized.podUID, PVCUID: recognized.pvcUID, Malformed: malformed.ref, MalformedPodUID: malformed.podUID, MalformedPVCUID: malformed.pvcUID, Insecure: insecure.ref, InsecurePodUID: insecure.podUID, InsecurePVCUID: insecure.pvcUID}, nil +} + +type seededLegacyEnvironment struct { + ref executionenv.EnvironmentRef + podUID, pvcUID string +} + +func seedOneLegacyEnvironment(ctx context.Context, d dynamic.Interface, kube kubernetes.Interface, namespace string, profile resolvedProfile, owner executionenv.Owner, client, name, binding string, references []any, insecure bool) (seededLegacyEnvironment, error) { + revision, err := randomID() + if err != nil { + return seededLegacyEnvironment{}, err + } + spec := map[string]any{ + "schemaVersion": int64(1), "allocationID": name, "revision": revision, + "ownerHash": ownerHash(owner), "ownerIssuer": owner.Issuer, "ownerSubject": owner.Subject, + "clientHash": hashText(client), "bindingID": binding, "requestFingerprint": hashText("legacy-fixture-" + name), + "profile": "go", "profileDigest": profile.Digest, "image": profile.Spec.Image, + "storageClass": profile.Spec.StorageClass, "storageSize": profile.Spec.StorageSize, + "resources": map[string]any{"cpuRequest": profile.Spec.CPURequest, "memoryRequest": profile.Spec.MemoryRequest, "cpuLimit": profile.Spec.CPULimit, "memoryLimit": profile.Spec.MemoryLimit}, + "desired": "Active", + } + object := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", + "metadata": map[string]any{"name": name, "namespace": namespace, "finalizers": []any{environmentFinalizer}}, + "spec": spec, + }} + resources := d.Resource(ExecutionEnvironmentGVR).Namespace(namespace) + created, err := resources.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + created, err = resources.Create(ctx, object, metav1.CreateOptions{}) + } else if err == nil { + revision = textNested(created.Object, "spec", "revision") + podUID := textNested(created.Object, "status", "pod", "uid") + pvcUID := textNested(created.Object, "status", "pvc", "uid") + if revision != "" && podUID != "" && pvcUID != "" { + return seededLegacyEnvironment{ref: executionenv.EnvironmentRef{ID: name, Revision: revision}, podUID: podUID, pvcUID: pvcUID}, nil + } + } + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create legacy environment %s: %w", name, err) + } + r := NewReconciler(d, kube, namespace, &Profiles{byName: map[string]resolvedProfile{"go": profile}}) + pvcName, podName := resourceName("workspace", name), resourceName("executor", name) + pvc, err := r.ensurePVC(ctx, created, profile, pvcName) + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create legacy PVC %s: %w", name, err) + } + pod, err := r.ensurePod(ctx, created, profile, podName, pvcName) + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create legacy Pod %s: %w", name, err) + } + if insecure { + pod, err = replaceWithInsecureLegacyPod(ctx, kube, namespace, pod) + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create insecure legacy Pod %s: %w", name, err) + } + } + now := time.Now().UTC().Format(time.RFC3339Nano) + created.Object["status"] = map[string]any{ + "schemaVersion": int64(1), "observedGeneration": created.GetGeneration(), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": fenceHealthy, + "references": references, "pvc": map[string]any{"name": pvc.Name, "uid": string(pvc.UID)}, "pod": map[string]any{"name": pod.Name, "uid": string(pod.UID)}, + "conditions": []any{map[string]any{"type": "Ready", "status": "True", "reason": "LegacyFixtureReady", "message": "security-compatible prototype runtime created before CRD upgrade", "observedGeneration": created.GetGeneration(), "lastTransitionTime": now}}, + } + if _, err := resources.UpdateStatus(ctx, created, metav1.UpdateOptions{}); err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("persist legacy status %s: %w", name, err) + } + return seededLegacyEnvironment{ref: executionenv.EnvironmentRef{ID: name, Revision: revision}, podUID: string(pod.UID), pvcUID: string(pvc.UID)}, nil +} + +func replaceWithInsecureLegacyPod(ctx context.Context, kube kubernetes.Interface, namespace string, pod *corev1.Pod) (*corev1.Pod, error) { + pods := kube.CoreV1().Pods(namespace) + if _, err := pods.Patch(ctx, pod.Name, types.MergePatchType, []byte(`{"metadata":{"finalizers":[]}}`), metav1.PatchOptions{}); err != nil { + return nil, err + } + zero := int64(0) + if err := pods.Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &zero}); err != nil { + return nil, err + } + for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); { + if _, err := pods.Get(ctx, pod.Name, metav1.GetOptions{}); apierrors.IsNotFound(err) { + break + } else if err != nil { + return nil, err + } + time.Sleep(100 * time.Millisecond) + } + if _, err := pods.Get(ctx, pod.Name, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("secure fixture Pod deletion did not complete: %w", err) + } + candidate := pod.DeepCopy() + candidate.ObjectMeta = metav1.ObjectMeta{Name: pod.Name, Namespace: namespace, Labels: pod.Labels, Finalizers: pod.Finalizers, OwnerReferences: pod.OwnerReferences} + candidate.Status = corev1.PodStatus{} + automount := true + candidate.Spec.AutomountServiceAccountToken = &automount + return pods.Create(ctx, candidate, metav1.CreateOptions{}) +} diff --git a/internal/adapter/executioncontroller/production_lifecycle_test.go b/internal/adapter/executioncontroller/production_lifecycle_test.go index f3c3186469..627ffb84ae 100644 --- a/internal/adapter/executioncontroller/production_lifecycle_test.go +++ b/internal/adapter/executioncontroller/production_lifecycle_test.go @@ -59,17 +59,96 @@ func lifecycleAdminEnvironment(schema int64, refs []any) *unstructured.Unstructu } func terminalExecutor() *corev1.Pod { - return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "executor", Namespace: "ns", UID: types.UID("pod-uid"), Finalizers: []string{executorFinalizer}, Labels: map[string]string{"execution.mecatl.dev/environment": "env"}}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "executor"}}}, Status: corev1.PodStatus{Phase: corev1.PodFailed, ContainerStatuses: []corev1.ContainerStatus{{Name: "executor", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 137}}}}}} + profile, _ := testProfiles().get("go") + noPriv, nonroot, ro := false, true, true + uid := int64(65532) + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "executor", Namespace: "ns", UID: types.UID("pod-uid"), Finalizers: []string{executorFinalizer}, Labels: map[string]string{"execution.mecatl.dev/environment": "env", "execution.mecatl.dev/profile": hashText("go")[:16]}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "execution.mecatl.dev/v1alpha1", Kind: "ExecutionEnvironment", Name: "env", UID: types.UID("env-uid"), Controller: &nonroot}}}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noPriv, RuntimeClassName: &profile.Spec.RuntimeClassName, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &nonroot, RunAsUser: &uid, RunAsGroup: &uid, FSGroup: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "executor", Image: profile.Spec.Image, Command: []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &noPriv, ReadOnlyRootFilesystem: &ro, RunAsNonRoot: &nonroot, RunAsUser: &uid, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: profile.CPURequest, corev1.ResourceMemory: profile.MemoryRequest, corev1.ResourceEphemeralStorage: profile.EphemeralStorageRequest}, Limits: corev1.ResourceList{corev1.ResourceCPU: profile.CPULimit, corev1.ResourceMemory: profile.MemoryLimit, corev1.ResourceEphemeralStorage: profile.EphemeralStorageLimit}}, VolumeMounts: []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}}}, Volumes: []corev1.Volume{{Name: "workspace", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "workspace"}}}, {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &profile.TmpSizeLimit}}}}}, Status: corev1.PodStatus{Phase: corev1.PodFailed, ContainerStatuses: []corev1.ContainerStatus{{Name: "executor", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 137}}}}}} } func retainedPVC() *corev1.PersistentVolumeClaim { - return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "workspace", Namespace: "ns", UID: types.UID("pvc-uid"), Labels: map[string]string{"execution.mecatl.dev/environment": "env"}}} + profile, _ := testProfiles().get("go") + mode := corev1.PersistentVolumeFilesystem + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "workspace", Namespace: "ns", UID: types.UID("pvc-uid"), Labels: map[string]string{"execution.mecatl.dev/environment": "env", "execution.mecatl.dev/revision": "rev", "execution.mecatl.dev/allocation-uid": "env-uid"}}, Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &profile.Spec.StorageClass, AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, VolumeMode: &mode, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: profile.StorageSize}}}} } func adminRequestFixture() adminLifecycleRequest { return adminLifecycleRequest{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, OwnerHash: "owner", Client: "client", ExpectedEpoch: 4, ExpectedPodUID: "pod-uid", ExpectedPVCUID: "pvc-uid", OperationID: "admin-operation"} } +func TestHealthyTerminalPodCanStartLifecycleWithoutReadyCondition(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Ready", false, "PodTerminated", "executor exited naturally") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatal(err) + } + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatalf("retry was not idempotent: %v", err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "terminationProof", operationIDField) != q.OperationID || textNested(got.Object, "status", "lifecycleOperation", "id") != q.OperationID { + t.Fatalf("terminal proof and lifecycle operation were not persisted: %v", got.Object["status"]) + } +} + +func TestHealthyTerminalLifecycleRejectsIncompleteEvidence(t *testing.T) { + cases := map[string]func(*corev1.Pod){ + "missing pod": func(*corev1.Pod) {}, + "init running": func(p *corev1.Pod) { + p.Spec.InitContainers = []corev1.Container{{Name: "init"}} + p.Status.InitContainerStatuses = []corev1.ContainerStatus{{Name: "init", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}} + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Ready", false, "PodTerminated", "executor exited naturally") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + objects := []runtime.Object{retainedPVC()} + if name != "missing pod" { + pod := terminalExecutor() + mutate(pod) + objects = append(objects, pod) + } + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(objects...)) + if err := store.RetireExact(t.Context(), adminRequestFixture()); err == nil { + t.Fatal("incomplete terminal evidence admitted lifecycle") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "terminationProof", "podUID") != "" || textNested(got.Object, "status", "lifecycleOperation", "id") != "" { + t.Fatalf("rejected evidence changed lifecycle state: %v", got.Object["status"]) + } + }) + } +} + +func TestRetirementCompletionReceiptReplaysOnlyExactOperation(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, terminationProof(q, terminalExecutor()), "status", "terminationProof") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": q.OperationID, "type": "RetireEnvironment", "phase": "WaitingForPodDeletion", "expectedEpoch": int64(4), "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID, "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + r := NewReconciler(dynamicClient, kubefake.NewSimpleClientset(retainedPVC()), "ns", testProfiles()) + if err := r.finishRetirement(t.Context(), env, q.OperationID); err != nil { + t.Fatal(err) + } + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatalf("lost-reply retry failed: %v", err) + } + q.OperationID = "different-operation" + if err := store.RetireExact(t.Context(), q); err == nil { + t.Fatal("different operation silently succeeded against retired environment") + } + q = adminRequestFixture() + q.OwnerHash = "other-owner" + if err := store.RetireExact(t.Context(), q); err == nil { + t.Fatal("wrong subject replayed retirement receipt") + } +} + func TestOldSchemaFailsClosedAndExplicitMigrationConvertsReferences(t *testing.T) { env := lifecycleAdminEnvironment(0, []any{"session-a"}) dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) @@ -101,6 +180,24 @@ func TestOldSchemaFailsClosedAndExplicitMigrationConvertsReferences(t *testing.T } } +func TestInsecurePrototypeMigrationRejectedWithoutRewrite(t *testing.T) { + env := lifecycleAdminEnvironment(1, []any{"session-a"}) + pod := terminalExecutor() + automount := true + pod.Spec.AutomountServiceAccountToken = &automount + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(pod, retainedPVC())) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := store.MigrateEnvironment(t.Context(), q); err == nil { + t.Fatal("insecure prototype executor migrated") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if intNested(got.Object, "spec", "schemaVersion") != 1 || intNested(got.Object, "status", "schemaVersion") != 1 || intNested(got.Object, "status", "epoch") != 4 { + t.Fatalf("insecure prototype was rewritten: %v", got.Object) + } +} + func TestUnknownSchemaMigrationRejectedWithoutRewrite(t *testing.T) { env := lifecycleAdminEnvironment(7, []any{}) dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) @@ -183,6 +280,22 @@ func TestRecoverMissingPodRemainsFenceUnknown(t *testing.T) { } } +func TestCompletedReplacementReceiptReplaysExactOperation(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, map[string]any{operationIDField: q.OperationID, "previousPodUID": q.ExpectedPodUID, "replacementPodUID": "new-pod", "pvcUID": q.ExpectedPVCUID, "previousEpoch": int64(q.ExpectedEpoch), "replacementEpoch": int64(q.ExpectedEpoch + 1)}, "status", "lastReplacement") + _ = unstructured.SetNestedField(env.Object, int64(q.ExpectedEpoch+1), "status", "epoch") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"name": "executor", "uid": "new-pod"}, "status", "pod") + store := NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), "ns", testProfiles(), nil) + if err := store.ReplaceExecutor(t.Context(), q); err != nil { + t.Fatalf("exact completed replacement did not replay: %v", err) + } + q.OperationID = "different" + if err := store.ReplaceExecutor(t.Context(), q); err == nil { + t.Fatal("different operation replayed completed replacement") + } +} + func TestReplacementQuiescesAndPendingReferenceBlocksRetirement(t *testing.T) { published := map[string]any{"bindingID": "session", "state": "Published", "operationID": "publish", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} env := lifecycleAdminEnvironment(2, []any{published}) diff --git a/internal/adapter/executioncontroller/store.go b/internal/adapter/executioncontroller/store.go index fe22fd5638..4ae2b2b930 100644 --- a/internal/adapter/executioncontroller/store.go +++ b/internal/adapter/executioncontroller/store.go @@ -20,6 +20,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" "github.com/stacklok/mecatl/internal/executionenv" ) @@ -117,15 +118,15 @@ func (s *Store) ensurePending(ctx context.Context, client, owner string, atteste if !apierrors.IsNotFound(err) { return Allocation{}, fmt.Errorf("get execution environment: %w", err) } + revision, err := randomID() + if err != nil { + return Allocation{}, err + } if s.kube != nil { if err := s.reserveProfileSlot(ctx, profile, name, p.Spec.MaxEnvironments); err != nil { return Allocation{}, err } } - revision, err := randomID() - if err != nil { - return Allocation{}, err - } spec := map[string]any{"schemaVersion": currentSchemaVersion, "allocationID": name, "revision": revision, "ownerHash": owner, "clientHash": hashText(client), "bindingID": binding, "requestFingerprint": fp, "profile": profile, "profileDigest": p.Digest, "image": p.Spec.Image, "storageClass": p.Spec.StorageClass, "storageSize": p.Spec.StorageSize, "resources": map[string]any{"cpuRequest": p.Spec.CPURequest, "memoryRequest": p.Spec.MemoryRequest, "cpuLimit": p.Spec.CPULimit, "memoryLimit": p.Spec.MemoryLimit}, "desired": "Active"} if attested.Issuer != "" && attested.Subject != "" { spec["ownerIssuer"] = attested.Issuer @@ -137,6 +138,11 @@ func (s *Store) ensurePending(ctx context.Context, client, owner string, atteste created, err = s.resources.Get(ctx, name, metav1.GetOptions{}) } if err != nil { + if s.kube != nil && definitiveCreateRejection(err) { + if releaseErr := releaseProfileSlot(ctx, s.kube, s.namespace, profile, name); releaseErr != nil { + return Allocation{}, errors.Join(fmt.Errorf("create execution environment: %w", err), fmt.Errorf("release profile allocation: %w", releaseErr)) + } + } return Allocation{}, fmt.Errorf("create execution environment: %w", err) } if err := s.initializeStatus(ctx, created, binding, operationID); err != nil { @@ -149,6 +155,10 @@ func (s *Store) ensurePending(ctx context.Context, client, owner string, atteste return allocationFrom(created, client, owner, binding, profile, fp) } +func definitiveCreateRejection(err error) bool { + return apierrors.IsInvalid(err) || apierrors.IsForbidden(err) || apierrors.IsUnauthorized(err) || apierrors.IsBadRequest(err) || apierrors.IsMethodNotSupported(err) || apierrors.IsRequestEntityTooLargeError(err) +} + const profileAllocationConfigMap = "mecatl-execution-profile-allocations" func profileAllocationKey(profile string) string { @@ -156,7 +166,7 @@ func profileAllocationKey(profile string) string { return "profile-" + hex.EncodeToString(keySum[:16]) + ".json" } -func (s *Store) reserveProfileSlot(ctx context.Context, profile, allocationID string, limit int) error { +func (s *Store) reserveProfileSlot(ctx context.Context, profile, allocationID string, limit int) error { //nolint:gocyclo // Durable capacity CAS and CR reconciliation are intentionally one transaction loop. cms := s.kube.CoreV1().ConfigMaps(s.namespace) key := profileAllocationKey(profile) for range 8 { @@ -182,8 +192,10 @@ func (s *Store) reserveProfileSlot(ctx context.Context, profile, allocationID st return fmt.Errorf("list profile allocations: %w", err) } for i := range all.Items { - if textNested(all.Items[i].Object, "spec", "profile") == profile && textNested(all.Items[i].Object, "spec", "desired") != "Retired" && !slices.Contains(slots, all.Items[i].GetName()) { - slots = append(slots, all.Items[i].GetName()) + item := &all.Items[i] + deallocating := conditionTrue(item, "Retired") && conditionTrue(item, "ExecutorTerminated") && textNested(item.Object, "status", "lifecycleOperation", "type") == "DeleteRetiredEnvironment" && textNested(item.Object, "status", "lifecycleOperation", "phase") == "ReleasingSlot" + if textNested(item.Object, "spec", "profile") == profile && !deallocating && !slices.Contains(slots, item.GetName()) { + slots = append(slots, item.GetName()) } } if len(slots) >= limit { @@ -510,7 +522,7 @@ func (s *Store) retryUpdateStatus(ctx context.Context, name string, mutate func( } func (s *Store) retryUpdateStatusRaw(ctx context.Context, name string, mutate func(*unstructured.Unstructured) error) error { - for i := 0; i < 5; i++ { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { cur, err := s.resources.Get(ctx, name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} @@ -521,13 +533,13 @@ func (s *Store) retryUpdateStatusRaw(ctx context.Context, name string, mutate fu if err := mutate(cur); err != nil { return err } - if _, err = s.resources.UpdateStatus(ctx, cur, metav1.UpdateOptions{}); err == nil { - return nil - } else if !apierrors.IsConflict(err) { - return err - } + _, err = s.resources.UpdateStatus(ctx, cur, metav1.UpdateOptions{}) + return err + }) + if apierrors.IsConflict(err) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment changed concurrently", Retryable: true} } - return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment changed concurrently", Retryable: true} + return err } func requireCurrentSchema(o *unstructured.Unstructured) error { if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion || intNested(o.Object, "status", "schemaVersion") != currentSchemaVersion { diff --git a/internal/adapter/executioncontroller/store_concurrency_test.go b/internal/adapter/executioncontroller/store_concurrency_test.go index e8c1409359..75b43fa987 100644 --- a/internal/adapter/executioncontroller/store_concurrency_test.go +++ b/internal/adapter/executioncontroller/store_concurrency_test.go @@ -22,6 +22,86 @@ import ( "github.com/stacklok/mecatl/internal/executionenv" ) +func profileSlots(t *testing.T, kube *kubefake.Clientset) []string { + t.Helper() + cm, err := kube.CoreV1().ConfigMaps("ns").Get(t.Context(), profileAllocationConfigMap, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + var slots []string + if err := executionenv.DecodeStrict([]byte(cm.Data[profileAllocationKey("go")]), &slots); err != nil { + t.Fatal(err) + } + return slots +} + +func TestDefinitiveEnvironmentCreateFailureReleasesProfileSlot(t *testing.T) { + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + dynamicClient.PrependReactor("create", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: ExecutionEnvironmentGVR.Group, Resource: ExecutionEnvironmentGVR.Resource}, "env", errors.New("policy")) + }) + kube := kubefake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{}}) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kube) + if _, err := store.EnsurePending(t.Context(), "client", "owner", "binding", "go", "fp", "op"); err == nil { + t.Fatal("definitive create rejection succeeded") + } + if slots := profileSlots(t, kube); len(slots) != 0 { + t.Fatalf("definitive create rejection leaked slots: %v", slots) + } +} + +func TestAmbiguousEnvironmentCreateFailureRetainsProfileSlot(t *testing.T) { + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + dynamicClient.PrependReactor("create", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("transport outcome unknown") + }) + kube := kubefake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{}}) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kube) + if _, err := store.EnsurePending(t.Context(), "client", "owner", "binding", "go", "fp", "op"); err == nil { + t.Fatal("ambiguous create failure succeeded") + } + if slots := profileSlots(t, kube); len(slots) != 1 { + t.Fatalf("ambiguous create outcome did not retain conservative capacity: %v", slots) + } +} + +func TestRetainedDeleteWaitsForSlotReleaseBeforeRemovingCRFinalizer(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "delete", "type": "DeleteRetiredEnvironment", "phase": "DeletingPVC", "expectedPVCUID": "pvc-uid", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + allocationID := env.GetName() + raw := `["` + allocationID + `"]` + kube := kubefake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{profileAllocationKey("go"): raw}}) + var fail atomic.Bool + fail.Store(true) + kube.PrependReactor("get", "configmaps", func(k8stesting.Action) (bool, runtime.Object, error) { + if fail.Load() { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Resource: "configmaps"}, profileAllocationConfigMap, errors.New("outage")) + } + return false, nil, nil + }) + r := NewReconciler(dynamicClient, kube, "ns", testProfiles()) + if err := r.Reconcile(t.Context(), env.GetName()); err != nil { + t.Fatalf("failed to persist deallocation phase: %v", err) + } + if err := r.Reconcile(t.Context(), env.GetName()); err == nil { + t.Fatal("slot authority outage did not stop deletion") + } + got, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), env.GetName(), metav1.GetOptions{}) + if err != nil || !contains(got.GetFinalizers(), environmentFinalizer) { + t.Fatalf("CR was not retained for retry: finalizers=%v err=%v", got.GetFinalizers(), err) + } + fail.Store(false) + if err := r.Reconcile(t.Context(), env.GetName()); err != nil { + t.Fatal(err) + } + if slots := profileSlots(t, kube); len(slots) != 0 { + t.Fatalf("retry did not release slot: %v", slots) + } +} + func TestConcurrentProfileReservationsEnforceHardLimit(t *testing.T) { dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) ledger := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns", ResourceVersion: "1"}, Data: map[string]string{}} diff --git a/internal/adapter/executioncontroller/store_lifecycle.go b/internal/adapter/executioncontroller/store_lifecycle.go index 537edb78f3..a985c4f2d7 100644 --- a/internal/adapter/executioncontroller/store_lifecycle.go +++ b/internal/adapter/executioncontroller/store_lifecycle.go @@ -6,6 +6,7 @@ import ( "sort" "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -192,6 +193,29 @@ func (s *Store) ReserveSuccessor(ctx context.Context, ref executionenv.Environme }) } +func (s *Store) FindReferenceIntent(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) (executionenv.ReferenceIntent, error) { + o, err := s.resources.Get(ctx, ref.ID, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} + } + if err != nil { + return executionenv.ReferenceIntent{}, err + } + if textNested(o.Object, "spec", "revision") != ref.Revision || textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) { + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} + } + refs, err := referenceRecords(o) + if err != nil { + return executionenv.ReferenceIntent{}, err + } + for _, r := range refs { + if r.BindingID == binding && r.State != executionenv.ReferencePublished { + return executionenv.ReferenceIntent{Environment: ref, BindingID: r.BindingID, State: r.State, OperationID: r.OperationID, SourceBindingID: r.SourceBindingID, CreatedAt: r.CreatedAt}, nil + } + } + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} +} + func (s *Store) ListReferenceIntentsForClient(ctx context.Context, client string, limit int) ([]executionenv.ReferenceIntent, error) { if limit <= 0 || limit > maxReferences { limit = maxReferences diff --git a/internal/adapter/executioncontroller/store_lifecycle_test.go b/internal/adapter/executioncontroller/store_lifecycle_test.go index 1f1ba14770..bbb1bc9fd2 100644 --- a/internal/adapter/executioncontroller/store_lifecycle_test.go +++ b/internal/adapter/executioncontroller/store_lifecycle_test.go @@ -3,6 +3,7 @@ package executioncontroller import ( "context" "errors" + "fmt" "reflect" "testing" "time" @@ -26,6 +27,23 @@ func lifecycleEnvironment(now time.Time) *unstructured.Unstructured { }} } +func TestEnsurePendingOwnedPersistsOwnerAttestation(t *testing.T) { + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + store := NewStore(client, "ns", testProfiles(), nil) + owner := executionenv.Owner{Issuer: "https://issuer.example", Subject: "alice"} + allocation, err := store.EnsurePendingOwned(t.Context(), "client", ownerHash(owner), owner, "binding", "go", "fingerprint", "ensure-operation") + if err != nil { + t.Fatal(err) + } + env, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), allocation.Environment.ID, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(env.Object, "spec", "ownerIssuer") != owner.Issuer || textNested(env.Object, "spec", "ownerSubject") != owner.Subject { + t.Fatalf("owner attestation was not persisted: %v", env.Object["spec"]) + } +} + func TestRunClaimIsEnvironmentWideAndExact(t *testing.T) { now := time.Now().UTC() client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), lifecycleEnvironment(now)) @@ -76,6 +94,37 @@ func TestExpiredClaimWithActiveOperationCannotBeReplaced(t *testing.T) { } } +func TestExactReferenceIntentLookupIsNotTruncatedByGlobalLimit(t *testing.T) { + now := time.Now().UTC() + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + objects := make([]runtime.Object, 0, 67) + for i := 0; i < 66; i++ { + env := lifecycleEnvironment(now) + env.SetName(fmt.Sprintf("env-%02d", i)) + _ = unstructured.SetNestedField(env.Object, fmt.Sprintf("rev-%02d", i), "spec", "revision") + _ = unstructured.SetNestedField(env.Object, ownerHash(owner), "spec", "ownerHash") + _ = unstructured.SetNestedField(env.Object, owner.Issuer, "spec", "ownerIssuer") + _ = unstructured.SetNestedField(env.Object, owner.Subject, "spec", "ownerSubject") + if err := setReferenceRecords(env, []referenceRecord{{BindingID: fmt.Sprintf("binding-%02d", i), State: executionenv.ReferencePendingDelete, OperationID: fmt.Sprintf("delete-%02d", i), CreatedAt: now}}); err != nil { + t.Fatal(err) + } + objects = append(objects, env) + } + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), objects...) + store := NewStore(client, "ns", testProfiles(), nil) + target := executionenv.EnvironmentRef{ID: "env-65", Revision: "rev-65"} + intent, err := store.FindReferenceIntent(t.Context(), target, "client", ownerHash(owner), "binding-65") + if err != nil || intent.OperationID != "delete-65" { + t.Fatalf("exact target beyond global list limit: intent=%+v err=%v", intent, err) + } + if _, err := store.FindReferenceIntent(t.Context(), target, "other-client", ownerHash(owner), "binding-65"); err == nil { + t.Fatal("other client enumerated exact intent") + } + if _, err := store.FindReferenceIntent(t.Context(), target, "client", ownerHash(executionenv.Owner{Issuer: "issuer", Subject: "mallory"}), "binding-65"); err == nil { + t.Fatal("other owner enumerated exact intent") + } +} + func TestClientScopedIntentReconciliationRejectsEachOwnerMutationWithoutSideEffects(t *testing.T) { now := time.Now().UTC() owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} diff --git a/internal/adapter/executioncontroller/store_run.go b/internal/adapter/executioncontroller/store_run.go index bcc1521619..beb2cee30d 100644 --- a/internal/adapter/executioncontroller/store_run.go +++ b/internal/adapter/executioncontroller/store_run.go @@ -3,6 +3,7 @@ package executioncontroller import ( "context" + "fmt" "math" "time" @@ -130,24 +131,73 @@ func (s *Store) AcquireRun(ctx context.Context, ref executionenv.EnvironmentRef, return out, err } +func renewFingerprint(ref executionenv.EnvironmentRef, client, owner string, req executionenv.RunClaimRequest) string { + return fingerprint(ref.ID, ref.Revision, client, owner, req.BindingID, req.RunID, req.ClaimID, fmt.Sprint(req.Epoch), fmt.Sprint(req.GrantGeneration), req.TTL.String()) +} + +func replayRenewReceipt(o *unstructured.Unstructured, operationID, requestFingerprint string, now time.Time, claim executionenv.RunClaim) (executionenv.RunClaim, bool, error) { + receipts, _, err := unstructured.NestedSlice(o.Object, "status", "renewReceipts") + if err != nil { + return executionenv.RunClaim{}, false, &executionenv.Error{Code: executionenv.CodeConflict, Message: "run renewal receipts are invalid"} + } + for _, raw := range receipts { + receipt, ok := raw.(map[string]any) + if !ok || text(receipt, operationIDField) != operationID { + continue + } + if text(receipt, "fingerprint") != requestFingerprint { + return executionenv.RunClaim{}, true, &executionenv.Error{Code: executionenv.CodeConflict, Message: "run renewal operation conflicts"} + } + expires, parseErr := time.Parse(time.RFC3339Nano, text(receipt, "expiresAt")) + if parseErr != nil || !now.Before(expires) { + return executionenv.RunClaim{}, true, &executionenv.Error{Code: executionenv.CodeConflict, Message: "run renewal receipt is expired"} + } + claim.ExpiresAt = expires + return claim, true, nil + } + return executionenv.RunClaim{}, false, nil +} + +func appendRenewReceipt(o *unstructured.Unstructured, operationID, requestFingerprint string, expires time.Time) error { + receipts, _, err := unstructured.NestedSlice(o.Object, "status", "renewReceipts") + if err != nil { + return err + } + receipts = append(receipts, map[string]any{operationIDField: operationID, "fingerprint": requestFingerprint, "expiresAt": expires.Format(time.RFC3339Nano)}) + if len(receipts) > 32 { + receipts = receipts[len(receipts)-32:] + } + return unstructured.SetNestedSlice(o.Object, receipts, "status", "renewReceipts") +} + func (s *Store) RenewRun(ctx context.Context, ref executionenv.EnvironmentRef, client, owner string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { ttl, err := boundedRunTTL(req.TTL) if err != nil { return executionenv.RunClaim{}, err } + requestFingerprint := renewFingerprint(ref, client, owner, req) var out executionenv.RunClaim err = s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { cur, _, expiry, ok := activeRunFrom(o) - if !ok || !s.now().Before(expiry) || textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || cur.Environment != ref || cur.BindingID != req.BindingID || cur.RunID != req.RunID || cur.ClaimID != req.ClaimID || cur.Epoch != req.Epoch || cur.GrantGeneration != req.GrantGeneration || !generationMatches(o, req.GrantGeneration) { + if !ok || textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || cur.Environment != ref || cur.BindingID != req.BindingID || cur.RunID != req.RunID || cur.ClaimID != req.ClaimID || cur.Epoch != req.Epoch || cur.GrantGeneration != req.GrantGeneration || !generationMatches(o, req.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim mismatch"} + } + if replay, found, replayErr := replayRenewReceipt(o, req.OperationID, requestFingerprint, s.now(), cur); found { + out = replay + return replayErr + } + if !s.now().Before(expiry) { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim mismatch"} } expires := s.now().Add(ttl) m, _, _ := unstructured.NestedMap(o.Object, "status", "activeRun") m["expiresAt"] = expires.Format(time.RFC3339Nano) - m["lastRenewOperationID"] = req.OperationID if setErr := unstructured.SetNestedMap(o.Object, m, "status", "activeRun"); setErr != nil { return setErr } + if setErr := appendRenewReceipt(o, req.OperationID, requestFingerprint, expires); setErr != nil { + return setErr + } cur.ExpiresAt = expires out = cur return nil diff --git a/internal/adapter/executioncontroller/store_test.go b/internal/adapter/executioncontroller/store_test.go index db0a0171c4..c7f227db55 100644 --- a/internal/adapter/executioncontroller/store_test.go +++ b/internal/adapter/executioncontroller/store_test.go @@ -33,6 +33,35 @@ func TestRenewRunRejectsExpiredClaimAtBoundary(t *testing.T) { } } +func TestRenewRunReplaysExactOperationWithoutExtendingLease(t *testing.T) { + now := time.Date(2026, 9, 17, 4, 0, 0, 0, time.UTC) + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(7), "grantGeneration": int64(3), "fenceState": fenceHealthy, "references": []any{}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "epoch": int64(7), "grantGeneration": int64(3), "expiresAt": now.Add(time.Minute).Format(time.RFC3339Nano)}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + clock := now + store.now = func() time.Time { return clock } + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + req := executionenv.RunClaimRequest{BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: executionenv.MinRunTTL} + first, err := store.RenewRun(t.Context(), ref, "client", "owner", req) + if err != nil { + t.Fatal(err) + } + clock = clock.Add(10 * time.Second) + replay, err := store.RenewRun(t.Context(), ref, "client", "owner", req) + if err != nil || !replay.ExpiresAt.Equal(first.ExpiresAt) { + t.Fatalf("replay extended or rejected lease: first=%v replay=%v err=%v", first.ExpiresAt, replay.ExpiresAt, err) + } + req.TTL = time.Minute + if _, err := store.RenewRun(t.Context(), ref, "client", "owner", req); err == nil { + t.Fatal("same operation accepted conflicting inputs") + } + clock = first.ExpiresAt + req.TTL = executionenv.MinRunTTL + if _, err := store.RenewRun(t.Context(), ref, "client", "owner", req); err == nil { + t.Fatal("expired renewal receipt resurrected claim") + } +} + func TestStoreEnsureUsesStableLookupAndRejectsFingerprintDrift(t *testing.T) { client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) profiles := &Profiles{byName: map[string]resolvedProfile{"go": {Digest: "sha256:profile", Spec: ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100}}, "other": {Digest: "sha256:other", Spec: ProfileSpec{Image: "example@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100}}}} diff --git a/internal/adapter/server/execution_renewal_test.go b/internal/adapter/server/execution_renewal_test.go new file mode 100644 index 0000000000..06cc3d5ce1 --- /dev/null +++ b/internal/adapter/server/execution_renewal_test.go @@ -0,0 +1,33 @@ +package server + +import ( + "context" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/tool" +) + +type deadlineExecutionHandle struct { + deadline time.Time +} + +func (*deadlineExecutionHandle) Environment() tool.Environment { return tool.Environment{} } +func (*deadlineExecutionHandle) Renew(context.Context) error { return nil } +func (*deadlineExecutionHandle) Release(context.Context) error { return nil } +func (h *deadlineExecutionHandle) RenewalDeadline() time.Time { return h.deadline } + +var _ ExecutionRunHandle = (*deadlineExecutionHandle)(nil) + +func TestExecutionRenewDelayTracksShortGrantExpiry(t *testing.T) { + now := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC) + handle := &deadlineExecutionHandle{deadline: now.Add(5 * time.Second)} + delay, usable := executionRenewDelay(handle, now) + if !usable || delay <= 0 || delay >= 5*time.Second || delay < 100*time.Millisecond { + t.Fatalf("five-second grant schedule delay=%v usable=%t", delay, usable) + } + handle.deadline = now.Add(250 * time.Millisecond) + if _, usable := executionRenewDelay(handle, now); usable { + t.Fatal("unusable grant remainder would start execution renewal") + } +} diff --git a/internal/adapter/server/service.go b/internal/adapter/server/service.go index 68fa873f40..30e682c624 100644 --- a/internal/adapter/server/service.go +++ b/internal/adapter/server/service.go @@ -8093,6 +8093,29 @@ func (s *Service) acquireExecution(ctx context.Context, st *runState, sess *sess return env, nil } +type executionRenewalDeadline interface { + RenewalDeadline() time.Time +} + +func executionRenewDelay(handle ExecutionRunHandle, now time.Time) (time.Duration, bool) { + deadlineHandle, ok := handle.(executionRenewalDeadline) + if !ok { + return 20 * time.Second, true + } + remaining := deadlineHandle.RenewalDeadline().Sub(now) + if remaining <= 300*time.Millisecond { + return 0, false + } + delay := remaining / 3 + if delay > 20*time.Second { + delay = 20 * time.Second + } + if delay < 100*time.Millisecond { + delay = 100 * time.Millisecond + } + return delay, delay < remaining +} + func (s *Service) startExecutionRenewal(_ session.SessionID, st *runState, run *agent.Run) { st.executionMu.Lock() if st.execution == nil || st.executionClosed { @@ -8107,14 +8130,30 @@ func (s *Service) startExecutionRenewal(_ session.SessionID, st *runState, run * st.executionMu.Unlock() go func() { defer close(done) - ticker := time.NewTicker(20 * time.Second) - defer ticker.Stop() for { + delay, usable := executionRenewDelay(handle, time.Now()) + if !usable { + s.logDiscoveryError(context.Background(), "execution ownership renewal failed", errors.New("execution grant has insufficient time remaining")) + run.Cancel() + return + } + timer := time.NewTimer(delay) select { case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } return - case <-ticker.C: - renewCtx, stop := context.WithTimeout(ctx, 10*time.Second) + case <-timer.C: + remaining := 10 * time.Second + if deadlineHandle, ok := handle.(executionRenewalDeadline); ok { + remaining = time.Until(deadlineHandle.RenewalDeadline()) / 2 + if remaining <= 0 { + run.Cancel() + return + } + } + renewCtx, stop := context.WithTimeout(ctx, remaining) err := handle.Renew(renewCtx) stop() if err != nil { diff --git a/user-docs/building/deployment/mecak8s.md b/user-docs/building/deployment/mecak8s.md index 1f76649e62..a3df07d4da 100644 --- a/user-docs/building/deployment/mecak8s.md +++ b/user-docs/building/deployment/mecak8s.md @@ -306,6 +306,12 @@ Shell, but do not ingest project instructions, rules, skills, or source from the remote PVC. Schedules, SkillDraft, background Shell, and delegated filesystem execution are outside this draft. +Run claims and signed grants renew before the issued grant expiry, including when +an operator configures a short grant TTL. Renewal operation receipts retain the +newest 32 identities. Retrying an exact retained identity preserves the claim and +original expiry (the replacement signature may carry a new nonce); conflicting or +expired receipt replay is denied and never extends or resurrects the claim. + The provider retains one PVC per logical environment. Session deletion and chart uninstall do not delete committed workspace data. Retirement requires no live references and a terminal executor; even then, this candidate retains the PVC. @@ -315,6 +321,17 @@ accepts only a still-observable Pod in `Succeeded` or `Failed` phase with every container terminated and the exact PVC still present. If that proof is missing, use your platform's external fencing runbook; there is no acknowledgement flag. +The built-in recovery RPC requires the same exact identity and an independently +stable operation ID. It does not read a Secret and has no force or +acknowledgement field: + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedExecutionEpoch":"","expectedPodUid":"","expectedPvcUid":"","operationId":"recover-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/RecoverEnvironment +``` + #### Run an administrative lifecycle operation Administrative RPCs are client-scoped. The administrator certificate must present From c334f94e573a2f0dcf01437a2ec1e8af8460ca3f Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 17 Sep 2026 10:57:01 +0200 Subject: [PATCH 08/25] test: preserve valid identity across production rotation proofs Restore forward-generation trust for new connections, reacquire expired test claims safely, use the post-release epoch, and surface holder start failures without weakening lifecycle checks. Co-Authored-By: Mecatl --- e2e/k8s_execution/fixture/rotation/main.go | 5 +- .../fixture/rotation/main_test.go | 107 ++++++++++++++++++ .../live_qualification_unit_test.go | 18 +++ .../production_qualification_test.go | 93 ++++++++++++--- e2e/k8s_execution/qualification_test.go | 26 ++++- 5 files changed, 227 insertions(+), 22 deletions(-) create mode 100644 e2e/k8s_execution/fixture/rotation/main_test.go diff --git a/e2e/k8s_execution/fixture/rotation/main.go b/e2e/k8s_execution/fixture/rotation/main.go index 594baea45f..680163ad99 100644 --- a/e2e/k8s_execution/fixture/rotation/main.go +++ b/e2e/k8s_execution/fixture/rotation/main.go @@ -25,7 +25,10 @@ func main() { if len(os.Args) != 3 { panic("usage: rotation INITIAL_PKI OUTPUT_DIRECTORY") } - initial, out := os.Args[1], os.Args[2] + generate(os.Args[1], os.Args[2]) +} + +func generate(initial, out string) { must(os.MkdirAll(out, 0o700)) now := time.Now().UTC() oldCA := read(filepath.Join(initial, "ca.crt")) diff --git a/e2e/k8s_execution/fixture/rotation/main_test.go b/e2e/k8s_execution/fixture/rotation/main_test.go new file mode 100644 index 0000000000..e148c79b4e --- /dev/null +++ b/e2e/k8s_execution/fixture/rotation/main_test.go @@ -0,0 +1,107 @@ +//go:build kind_execution_e2e + +package main + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "path/filepath" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +func TestForwardRestoreTrustBundleSupportsFixtureClient(t *testing.T) { + now := time.Now().UTC() + initial := filepath.Join(t.TempDir(), "initial") + out := filepath.Join(t.TempDir(), "rotation") + if err := os.MkdirAll(initial, 0o700); err != nil { + t.Fatal(err) + } + + oldCAKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + oldCA := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "fixture CA"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign} + oldCADER, err := x509.CreateCertificate(rand.Reader, oldCA, oldCA, &oldCAKey.PublicKey, oldCAKey) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(initial, "ca.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: oldCADER}), 0o600); err != nil { + t.Fatal(err) + } + _, grant, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + grantDER, err := x509.MarshalPKCS8PrivateKey(grant) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(initial, "grant-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: grantDER}), 0o600); err != nil { + t.Fatal(err) + } + generate(initial, out) + + clientKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + clientTemplate := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "fixture client"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, oldCA, &clientKey.PublicKey, oldCAKey) + if err != nil { + t.Fatal(err) + } + clientCert := tls.Certificate{Certificate: [][]byte{clientDER}, PrivateKey: clientKey} + serverCert, err := tls.LoadX509KeyPair(filepath.Join(out, "provider-new.crt"), filepath.Join(out, "provider-new.key")) + if err != nil { + t.Fatal(err) + } + bundle, err := os.ReadFile(filepath.Join(out, "client-roots.pem")) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(bundle) { + t.Fatal("generated forward trust bundle contained no certificates") + } + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{serverCert}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: roots}))) + healthpb.RegisterHealthServer(server, health.NewServer()) + serveDone := make(chan error, 1) + go func() { serveDone <- server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + <-serveDone + }) + + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "mecatl-execution.execution-qualification.svc.cluster.local", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + if _, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}); err != nil { + t.Fatal("forward-restored TLS gRPC handshake failed") + } +} diff --git a/e2e/k8s_execution/live_qualification_unit_test.go b/e2e/k8s_execution/live_qualification_unit_test.go index a73a99d4bf..944d80e31d 100644 --- a/e2e/k8s_execution/live_qualification_unit_test.go +++ b/e2e/k8s_execution/live_qualification_unit_test.go @@ -8,8 +8,26 @@ import ( corev1 "k8s.io/api/core/v1" mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" + "github.com/stacklok/mecatl/internal/executionenv" ) +func TestReplacementProofRequestsUsePostClaimEpochForPositiveRequest(t *testing.T) { + ref := executionenv.EnvironmentRef{ID: "fixture", Revision: "revision"} + owner := executionenv.Owner{Issuer: "issuer", Subject: "subject"} + staleStatus := executionStatus{Epoch: 4, PodUID: "pod", PVCUID: "pvc"} + current := executionStatus{Epoch: 6, PodUID: "pod", PVCUID: "pvc"} + stale, wrong, exact := replacementProofRequests(ref, owner, staleStatus, current) + if stale.ExpectedEpoch != staleStatus.Epoch || stale.ExpectedPodUID != staleStatus.PodUID { + t.Fatal("stale negative request did not preserve the pre-claim authority") + } + if wrong.ExpectedEpoch != current.Epoch || wrong.ExpectedPodUID == current.PodUID { + t.Fatal("wrong-UID negative request did not use current authority") + } + if exact.ExpectedEpoch != current.Epoch || exact.ExpectedPodUID != current.PodUID || exact.ExpectedPVCUID != current.PVCUID { + t.Fatal("positive exact request did not use post-release authoritative state") + } +} + func TestModelRanExactShellRequiresDecodedCommandAndMatchingCallID(t *testing.T) { call := func(id, args string) *mecatlv1.Event { return &mecatlv1.Event{ToolCall: &mecatlv1.ToolCall{Id: id, Name: "Shell", Args: args}} diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go index 08b4beb6a5..57f2989197 100644 --- a/e2e/k8s_execution/production_qualification_test.go +++ b/e2e/k8s_execution/production_qualification_test.go @@ -198,8 +198,7 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { t.Cleanup(func() { cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cleanupCancel() - applySecurityCandidate(t, cleanupCtx, kubeconfig, rotationDir, filepath.Join(rotationDir, "restore-fixture-clients.json"), "bridge") - waitProviderReadyReplicas(t, cleanupCtx, kubeconfig, 2) + restoreFixtureSecurity(t, cleanupCtx, kubeconfig, rotationDir) }) // A changed policy at the already-authoritative generation is rejected by @@ -212,7 +211,15 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { } applySecurityCandidate(t, ctx, kubeconfig, filepath.Join(state, "pki"), filepath.Join(state, "pki", "manifest.json"), "initial") waitProviderReadyReplicas(t, ctx, kubeconfig, 2) - waitFileContent(t, ctx, oldClient, oldRun, "rotation-sentinel.txt", "old-authority\n", "valid authority recovery") + recovered, err := oldClient.RenewRun(ctx, executionenv.RunClaimRequest{Environment: oldRun.Environment, Owner: owner, BindingID: binding, RunID: oldRun.RunID, ClaimID: oldRun.ClaimID, Epoch: oldRun.Epoch, GrantGeneration: oldRun.GrantGeneration, OperationID: "renew-rotation-recovered-grant", TTL: time.Minute}) + if isRemoteCode(err, executionenv.CodeConflict) { + recovered, err = oldClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-recovered-grant", OperationID: "acquire-rotation-recovered-grant", TTL: time.Minute}) + } + if err != nil { + t.Fatalf("establish fresh claim after authority recovery: code=%s", remoteErrorCode(err)) + } + recoveredRun := executionenv.RequestContext{Environment: recovered.Environment, Owner: owner, BindingID: binding, RunID: recovered.RunID, ClaimID: recovered.ClaimID, Epoch: recovered.Epoch, GrantGeneration: recovered.GrantGeneration, Grant: recovered.Grant} + waitFileContent(t, ctx, oldClient, recoveredRun, "rotation-sentinel.txt", "old-authority\n", "valid authority recovery") // Generation 2 bridges client trust before changing the server certificate. // The old connection remains usable, while a new-CA client can establish its @@ -230,9 +237,9 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { } defer newClient.Close() newAttached := waitReady(t, ctx, newClient, owner, binding, attached.Environment) - renewed, err := newClient.RenewRun(ctx, executionenv.RunClaimRequest{Environment: oldRun.Environment, Owner: owner, BindingID: binding, RunID: oldRun.RunID, ClaimID: oldRun.ClaimID, Epoch: oldRun.Epoch, GrantGeneration: oldRun.GrantGeneration, OperationID: "rotation-renew-k2", TTL: time.Minute}) + renewed, err := newClient.RenewRun(ctx, executionenv.RunClaimRequest{Environment: recoveredRun.Environment, Owner: owner, BindingID: binding, RunID: recoveredRun.RunID, ClaimID: recoveredRun.ClaimID, Epoch: recoveredRun.Epoch, GrantGeneration: recoveredRun.GrantGeneration, OperationID: "rotation-renew-k2", TTL: time.Minute}) if err != nil { - t.Fatalf("renew old claim under bridge authority: %v", err) + t.Fatalf("renew recovered claim under bridge authority: code=%s", remoteErrorCode(err)) } newRun := executionenv.RequestContext{Environment: renewed.Environment, Owner: owner, BindingID: binding, RunID: renewed.RunID, ClaimID: renewed.ClaimID, Epoch: renewed.Epoch, GrantGeneration: renewed.GrantGeneration, Grant: renewed.Grant} @@ -244,7 +251,7 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { if _, err := oldClient.File(ctx, executionenv.FileRequest{Context: oldRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err == nil { t.Fatal("established client signed by the removed CA remained authorized") } - if _, err := newClient.File(ctx, executionenv.FileRequest{Context: oldRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err == nil { + if _, err := newClient.File(ctx, executionenv.FileRequest{Context: recoveredRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err == nil { t.Fatal("grant signed by revoked k1 remained authorized through the new client") } if got, err := newClient.File(ctx, executionenv.FileRequest{Context: newRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err != nil || string(got.Data) != "old-authority\n" { @@ -283,8 +290,7 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { } // Restore fixture client compatibility through a higher generation; this is // another forward rotation, never a high-water-mark rollback. - applySecurityCandidate(t, ctx, kubeconfig, rotationDir, filepath.Join(rotationDir, "restore-fixture-clients.json"), "bridge") - waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + restoreFixtureSecurity(t, ctx, kubeconfig, rotationDir) } func waitFileContent(t *testing.T, ctx context.Context, client *executionclient.Client, rc executionenv.RequestContext, path, want, proof string) { @@ -311,8 +317,13 @@ func applySecurityCandidate(t *testing.T, ctx context.Context, kubeconfig, mater secretArgs = append(secretArgs, "--from-file=tls.crt="+filepath.Join(pki, "provider.crt"), "--from-file=tls.key="+filepath.Join(pki, "provider.key"), "--from-file=clients.pem="+filepath.Join(pki, "ca.crt")) case "bridge": secretArgs = append(secretArgs, "--from-file=grant-k2.pem="+filepath.Join(materialDir, "grant-k2.pem"), "--from-file=tls.crt="+filepath.Join(pki, "provider.crt"), "--from-file=tls.key="+filepath.Join(pki, "provider.key"), "--from-file=clients.pem="+filepath.Join(materialDir, "bridge-clients.pem")) - case "final": - secretArgs = append(secretArgs, "--from-file=grant-k2.pem="+filepath.Join(materialDir, "grant-k2.pem"), "--from-file=tls.crt="+filepath.Join(materialDir, "provider-new.crt"), "--from-file=tls.key="+filepath.Join(materialDir, "provider-new.key"), "--from-file=clients.pem="+filepath.Join(materialDir, "final-clients.pem")) + case "final", "restore": + secretArgs = append(secretArgs, "--from-file=grant-k2.pem="+filepath.Join(materialDir, "grant-k2.pem"), "--from-file=tls.crt="+filepath.Join(materialDir, "provider-new.crt"), "--from-file=tls.key="+filepath.Join(materialDir, "provider-new.key")) + if mode == "final" { + secretArgs = append(secretArgs, "--from-file=clients.pem="+filepath.Join(materialDir, "final-clients.pem")) + } else { + secretArgs = append(secretArgs, "--from-file=clients.pem="+filepath.Join(materialDir, "bridge-clients.pem")) + } default: t.Fatalf("unknown security candidate mode %q", mode) } @@ -322,6 +333,26 @@ func applySecurityCandidate(t *testing.T, ctx context.Context, kubeconfig, mater applyKubectlInput(t, ctx, kubeconfig, config) } +func restoreFixtureSecurity(t *testing.T, ctx context.Context, kubeconfig, rotationDir string) { + t.Helper() + applySecurityCandidate(t, ctx, kubeconfig, rotationDir, filepath.Join(rotationDir, "restore-fixture-clients.json"), "restore") + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + + pki := filepath.Join(os.Getenv("MECATL_EXECUTION_QUAL_STATE"), "pki") + roots, err := os.ReadFile(filepath.Join(rotationDir, "client-roots.pem")) + if err != nil { + t.Fatal("read synthetic provider trust bundle") + } + if err := os.WriteFile(filepath.Join(pki, "provider-roots.pem"), roots, 0o600); err != nil { + t.Fatal("publish synthetic provider trust bundle") + } + clientSecret := runKubectl(t, ctx, kubeconfig, "create", "secret", "generic", "execution-client-tls", "-n", namespace, + "--from-file=ca.crt="+filepath.Join(rotationDir, "client-roots.pem"), "--from-file=tls.crt="+filepath.Join(pki, "mecak8s.crt"), "--from-file=tls.key="+filepath.Join(pki, "mecak8s.key"), "--dry-run=client", "-o", "yaml") + applyKubectlInput(t, ctx, kubeconfig, clientSecret) + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecak8s", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecak8s", "-n", namespace, "--timeout=240s") +} + func applyKubectlInput(t *testing.T, ctx context.Context, kubeconfig string, input []byte) { t.Helper() cmd := command(ctx, kubeconfig, "apply", "-f", "-") @@ -411,16 +442,17 @@ func TestKindExecutionProductionReplicaLifecycle(t *testing.T) { t.Fatal(err) } - wrong := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: first.Epoch, ExpectedPodUID: "wrong-pod-uid", ExpectedPVCUID: first.PVCUID, OperationID: "replace-wrong"} + current := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + stale, wrong, exact := replacementProofRequests(attached.Environment, owner, first, current) + if err := client.ReplaceExecutor(ctx, stale); err == nil { + t.Fatal("stale-epoch replacement unexpectedly succeeded") + } if err := client.ReplaceExecutor(ctx, wrong); err == nil { t.Fatal("wrong Pod UID replacement unexpectedly succeeded") } if got := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID).PodUID; got != first.PodUID { t.Fatal("wrong-UID replacement changed executor") } - exact := wrong - exact.ExpectedPodUID = first.PodUID - exact.OperationID = "replace-exact" if err := client.ReplaceExecutor(ctx, exact); err != nil { t.Fatalf("replace exact executor: %v", err) } @@ -508,6 +540,15 @@ type executionStatus struct { ReadyReason string } +func replacementProofRequests(ref executionenv.EnvironmentRef, owner executionenv.Owner, staleStatus, current executionStatus) (executionenv.RetireEnvironmentRequest, executionenv.RetireEnvironmentRequest, executionenv.RetireEnvironmentRequest) { + stale := executionenv.RetireEnvironmentRequest{Environment: ref, Owner: owner, ExpectedEpoch: staleStatus.Epoch, ExpectedPodUID: staleStatus.PodUID, ExpectedPVCUID: staleStatus.PVCUID, OperationID: "replace-stale-epoch"} + wrong := executionenv.RetireEnvironmentRequest{Environment: ref, Owner: owner, ExpectedEpoch: current.Epoch, ExpectedPodUID: "wrong-pod-uid", ExpectedPVCUID: current.PVCUID, OperationID: "replace-wrong"} + exact := wrong + exact.ExpectedPodUID = current.PodUID + exact.OperationID = "replace-exact" + return stale, wrong, exact +} + func readExecutionStatus(t *testing.T, ctx context.Context, kubeconfig, name string) executionStatus { t.Helper() var object map[string]any @@ -570,6 +611,28 @@ func waitExecutionStatus(t *testing.T, ctx context.Context, kubeconfig, name str return executionStatus{} } +func waitActiveOperationOrCommandError(t *testing.T, ctx context.Context, kubeconfig, name string, commandDone <-chan error) { + t.Helper() + deadline := time.NewTimer(4 * time.Minute) + defer deadline.Stop() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case err := <-commandDone: + t.Fatalf("command ended before active-operation publication: code=%s", remoteErrorCode(err)) + case <-ticker.C: + if readExecutionStatus(t, ctx, kubeconfig, name).ActiveOperationID != "" { + return + } + case <-deadline.C: + t.Fatal("active-operation publication timed out while command remained in flight") + case <-ctx.Done(): + t.Fatal("active-operation publication context ended") + } + } +} + func waitResourceAbsent(t *testing.T, ctx context.Context, kubeconfig, resource, name string) { t.Helper() deadline := time.Now().Add(3 * time.Minute) @@ -616,7 +679,7 @@ func TestKindExecutionProductionHolderLossFencesActiveOperation(t *testing.T) { _, commandErr := holderClient.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: `nohup sh -c 'sleep 45; kill 1' >/dev/null 2>&1 & i=0; trap '' HUP TERM; while [ $i -lt 90 ]; do i=$((i+1)); printf '%s\n' "$i" > holder-loss-nonce; sleep 1; done`, TimeoutMillis: 110000}) commandDone <- commandErr }() - waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.ActiveOperationID != "" }) + waitActiveOperationOrCommandError(t, ctx, kubeconfig, attached.Environment.ID, commandDone) runKubectl(t, ctx, kubeconfig, "delete", "pod/"+holderPod, "-n", namespace, "--wait=true", "--timeout=90s") runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=180s") diff --git a/e2e/k8s_execution/qualification_test.go b/e2e/k8s_execution/qualification_test.go index 895f08ff80..8cd7b4a4d2 100644 --- a/e2e/k8s_execution/qualification_test.go +++ b/e2e/k8s_execution/qualification_test.go @@ -144,11 +144,8 @@ func TestKindExecutionQualification(t *testing.T) { t.Fatal(err) } assertMockJourney(t, body) - lookup, err := providerClient.Ensure(ctx, sessionID, "go", owner, "independent-lookup-"+sessionID) - if err != nil { - t.Fatalf("resolve harness environment: %v", err) - } - attached := waitReady(t, ctx, providerClient, owner, sessionID, lookup.Environment) + lookup := environmentForBinding(t, ctx, kubeconfig, sessionID) + attached := waitReady(t, ctx, providerClient, owner, sessionID, lookup) rc, releaseVerification := acquireRun(t, ctx, providerClient, owner, sessionID, attached, fmt.Sprintf("independent-verify-%d", time.Now().UnixNano())) proof, err := providerClient.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: "proof.txt"}) if err != nil || string(proof.Data) != "beta\n" { @@ -322,9 +319,26 @@ func isRemoteCode(err error, code executionenv.ErrorCode) bool { return errors.As(err, &remote) && remote.Code == code } +func remoteErrorCode(err error) string { + if err == nil { + return "none" + } + var remote *executionenv.Error + if errors.As(err, &remote) && remote.Code.Valid() { + return string(remote.Code) + } + return "transport" +} + func loadTLS(t *testing.T, dir, name, serverName string) *tls.Config { t.Helper() - cfg, err := executionclient.LoadTLSConfig(executionclient.TLSFiles{CA: filepath.Join(dir, "ca.crt"), Cert: filepath.Join(dir, name+".crt"), Key: filepath.Join(dir, name+".key")}) + ca := filepath.Join(dir, "provider-roots.pem") + if _, err := os.Stat(ca); errors.Is(err, os.ErrNotExist) { + ca = filepath.Join(dir, "ca.crt") + } else if err != nil { + t.Fatal("inspect synthetic provider trust bundle") + } + cfg, err := executionclient.LoadTLSConfig(executionclient.TLSFiles{CA: ca, Cert: filepath.Join(dir, name+".crt"), Key: filepath.Join(dir, name+".key")}) if err != nil { t.Fatal(err) } From 7c9c523e005e5dd1539d0cf96fcf4cf7b8f0a1da Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 17 Sep 2026 11:45:09 +0200 Subject: [PATCH 09/25] fix: preserve authority through rotation and reference recovery Restore owner attestation in intent responses, verify presented intermediate chains against current trust, and tie readiness to authoritative state. Qualify phase-scoped claims and recreate endpoint connections after rollouts. Co-Authored-By: Mecatl --- cmd/mecatl-execution-provider/main.go | 4 +- .../production_qualification_test.go | 31 ++++- e2e/k8s_execution/qualification_test.go | 68 +++++++++++ .../adapter/executioncontroller/handler.go | 8 +- .../executioncontroller/handler_test.go | 20 +++- .../adapter/executioncontroller/security.go | 14 ++- .../executioncontroller/security_limit.go | 8 +- .../executioncontroller/security_test.go | 108 ++++++++++++++++++ 8 files changed, 245 insertions(+), 16 deletions(-) diff --git a/cmd/mecatl-execution-provider/main.go b/cmd/mecatl-execution-provider/main.go index afad7568f6..87a772fa61 100644 --- a/cmd/mecatl-execution-provider/main.go +++ b/cmd/mecatl-execution-provider/main.go @@ -110,7 +110,9 @@ func run() error { //nolint:gocyclo // Startup validation and owned-resource shu http.Error(w, "not ready: "+reconciler.ReadinessReason(), http.StatusServiceUnavailable) return } - if !security.CheckReady(r.Context()) { + readyCtx, stop := context.WithTimeout(r.Context(), 2*time.Second) + defer stop() + if !security.CheckReady(readyCtx) { http.Error(w, "not ready: security-authority-or-expiry", http.StatusServiceUnavailable) return } diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go index 57f2989197..73efbb8e56 100644 --- a/e2e/k8s_execution/production_qualification_test.go +++ b/e2e/k8s_execution/production_qualification_test.go @@ -38,10 +38,12 @@ func requireProduction(t *testing.T) (string, string, context.Context, context.C func productionClient(t *testing.T, ctx context.Context, state, kubeconfig string) (*executionclient.Client, *forward) { t.Helper() f := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) - client, err := executionclient.New(f.addr, loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local")) + tlsConfig := loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + client, err := executionclient.New(f.addr, tlsConfig) if err != nil { t.Fatal(err) } + waitProviderRPCReady(t, ctx, client, f.addr, tlsConfig) t.Cleanup(func() { client.Close() }) return client, f } @@ -220,6 +222,9 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { } recoveredRun := executionenv.RequestContext{Environment: recovered.Environment, Owner: owner, BindingID: binding, RunID: recovered.RunID, ClaimID: recovered.ClaimID, Epoch: recovered.Epoch, GrantGeneration: recovered.GrantGeneration, Grant: recovered.Grant} waitFileContent(t, ctx, oldClient, recoveredRun, "rotation-sentinel.txt", "old-authority\n", "valid authority recovery") + if err := oldClient.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: recovered.Environment, Owner: owner, BindingID: binding, RunID: recovered.RunID, ClaimID: recovered.ClaimID, Epoch: recovered.Epoch, GrantGeneration: recovered.GrantGeneration, OperationID: "rotation-release-recovered"}); err != nil { + t.Fatalf("release recovery-phase claim: code=%s", remoteErrorCode(err)) + } // Generation 2 bridges client trust before changing the server certificate. // The old connection remains usable, while a new-CA client can establish its @@ -237,11 +242,20 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { } defer newClient.Close() newAttached := waitReady(t, ctx, newClient, owner, binding, attached.Environment) - renewed, err := newClient.RenewRun(ctx, executionenv.RunClaimRequest{Environment: recoveredRun.Environment, Owner: owner, BindingID: binding, RunID: recoveredRun.RunID, ClaimID: recoveredRun.ClaimID, Epoch: recoveredRun.Epoch, GrantGeneration: recoveredRun.GrantGeneration, OperationID: "rotation-renew-k2", TTL: time.Minute}) + if _, err := newClient.RenewRun(ctx, executionenv.RunClaimRequest{Environment: recoveredRun.Environment, Owner: owner, BindingID: binding, RunID: recoveredRun.RunID, ClaimID: recoveredRun.ClaimID, Epoch: recoveredRun.Epoch, GrantGeneration: recoveredRun.GrantGeneration, OperationID: "rotation-renew-released-phase-claim", TTL: time.Minute}); !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("released recovery-phase claim renewal code=%s, want conflict", remoteErrorCode(err)) + } + renewed, err := newClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: newAttached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-k2-grant", OperationID: "rotation-acquire-k2", TTL: time.Minute}) if err != nil { - t.Fatalf("renew recovered claim under bridge authority: code=%s", remoteErrorCode(err)) + t.Fatalf("acquire bridge-authority claim: code=%s", remoteErrorCode(err)) } newRun := executionenv.RequestContext{Environment: renewed.Environment, Owner: owner, BindingID: binding, RunID: renewed.RunID, ClaimID: renewed.ClaimID, Epoch: renewed.Epoch, GrantGeneration: renewed.GrantGeneration, Grant: renewed.Grant} + if got, err := newClient.File(ctx, executionenv.FileRequest{Context: newRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err != nil || string(got.Data) != "old-authority\n" { + t.Fatalf("bridge authority did not authorize preserved data: %v", err) + } + if err := newClient.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: renewed.Environment, Owner: owner, BindingID: binding, RunID: renewed.RunID, ClaimID: renewed.ClaimID, Epoch: renewed.Epoch, GrantGeneration: renewed.GrantGeneration, OperationID: "rotation-release-k2-bridge"}); err != nil { + t.Fatalf("release bridge-phase claim: code=%s", remoteErrorCode(err)) + } // Generation 3 switches server TLS, removes the old client CA, and revokes // k1. Authorization is checked on every RPC, so the established old-client @@ -254,7 +268,13 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { if _, err := newClient.File(ctx, executionenv.FileRequest{Context: recoveredRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err == nil { t.Fatal("grant signed by revoked k1 remained authorized through the new client") } - if got, err := newClient.File(ctx, executionenv.FileRequest{Context: newRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err != nil || string(got.Data) != "old-authority\n" { + finalAttached := waitReady(t, ctx, newClient, owner, binding, attached.Environment) + finalClaim, err := newClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: finalAttached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-final-grant", OperationID: "rotation-acquire-final", TTL: time.Minute}) + if err != nil { + t.Fatalf("acquire final-authority claim: code=%s", remoteErrorCode(err)) + } + finalRun := executionenv.RequestContext{Environment: finalClaim.Environment, Owner: owner, BindingID: binding, RunID: finalClaim.RunID, ClaimID: finalClaim.ClaimID, Epoch: finalClaim.Epoch, GrantGeneration: finalClaim.GrantGeneration, Grant: finalClaim.Grant} + if got, err := newClient.File(ctx, executionenv.FileRequest{Context: finalRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err != nil || string(got.Data) != "old-authority\n" { t.Fatalf("new authority did not authorize preserved data: %v", err) } freshClient, err := executionclient.New(forward.addr, newTLS) @@ -266,7 +286,7 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { t.Fatalf("fresh new-CA client failed after TLS switch: %v", err) } - if err := newClient.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: renewed.Environment, Owner: owner, BindingID: binding, RunID: renewed.RunID, ClaimID: renewed.ClaimID, Epoch: renewed.Epoch, GrantGeneration: renewed.GrantGeneration, OperationID: "rotation-release-k2"}); err != nil { + if err := newClient.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: finalClaim.Environment, Owner: owner, BindingID: binding, RunID: finalClaim.RunID, ClaimID: finalClaim.ClaimID, Epoch: finalClaim.Epoch, GrantGeneration: finalClaim.GrantGeneration, OperationID: "rotation-release-final"}); err != nil { t.Fatal(err) } generation, err := newClient.RevokeEnvironment(ctx, attached.Environment, owner, newAttached.GrantGeneration, "rotation-revoke-replay") @@ -501,6 +521,7 @@ func TestKindExecutionProductionReplicaLifecycle(t *testing.T) { } runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=240s") + client, _ = productionClient(t, ctx, state, kubeconfig) if refreshed := waitReady(t, ctx, client, owner, binding, attached.Environment); refreshed.GrantGeneration != newGeneration { t.Fatal("revocation generation was not durable across provider restart") } diff --git a/e2e/k8s_execution/qualification_test.go b/e2e/k8s_execution/qualification_test.go index 8cd7b4a4d2..b2c3821367 100644 --- a/e2e/k8s_execution/qualification_test.go +++ b/e2e/k8s_execution/qualification_test.go @@ -17,6 +17,7 @@ import ( "os/exec" "path/filepath" "strings" + "syscall" "testing" "time" @@ -124,6 +125,14 @@ func TestKindExecutionQualification(t *testing.T) { if _, err := providerClient.ValidateProfile(ctx, "go"); err == nil { t.Fatal("provider endpoint loss unexpectedly fell back") } + providerForward = portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + providerTLS = loadTLS(t, pki, "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + providerClient, err = executionclient.New(providerForward.addr, providerTLS) + if err != nil { + t.Fatal(err) + } + waitProviderRPCReady(t, ctx, providerClient, providerForward.addr, providerTLS) + defer providerClient.Close() tokenForward := portForward(t, ctx, kubeconfig, "service/oidc-issuer", 8443) alice := fixtureToken(t, ctx, tokenForward.addr, pki, "alice") @@ -164,6 +173,17 @@ func TestKindExecutionQualification(t *testing.T) { applyReattachScript(t, ctx, kubeconfig, state) runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=180s") + providerForward = portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + providerTLS = loadTLS(t, pki, "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + providerClient, err = executionclient.New(providerForward.addr, providerTLS) + if err != nil { + t.Fatal(err) + } + waitProviderRPCReady(t, ctx, providerClient, providerForward.addr, providerTLS) + defer providerClient.Close() + if refreshed := waitReady(t, ctx, providerClient, owner, sessionID, lookup); refreshed.Environment != lookup { + t.Fatal("provider did not reattach the exact environment after restart") + } runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecak8s", "-n", namespace) runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecak8s", "-n", namespace, "--timeout=240s") agentForward = portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) @@ -330,6 +350,54 @@ func remoteErrorCode(err error) string { return "transport" } +func waitProviderRPCReady(t *testing.T, ctx context.Context, c *executionclient.Client, addr string, cfg *tls.Config) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + attemptCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + _, lastErr = c.ValidateProfile(attemptCtx, "go") + cancel() + if lastErr == nil { + return + } + time.Sleep(250 * time.Millisecond) + } + t.Fatalf("provider RPC did not become ready: code=%s tls=%s", remoteErrorCode(lastErr), classifyTLSFailure(addr, cfg)) +} + +func classifyTLSFailure(addr string, cfg *tls.Config) string { + dialer := &net.Dialer{Timeout: 2 * time.Second} + conn, err := tls.DialWithDialer(dialer, "tcp", addr, cfg.Clone()) + if err == nil { + _ = conn.Close() + return "ok" + } + var unknown x509.UnknownAuthorityError + if errors.As(err, &unknown) { + return "unknown-ca" + } + var invalid x509.CertificateInvalidError + if errors.As(err, &invalid) { + if invalid.Reason == x509.Expired { + return "expired" + } + return "bad-certificate" + } + var hostname x509.HostnameError + if errors.As(err, &hostname) { + return "bad-certificate" + } + if errors.Is(err, syscall.ECONNREFUSED) { + return "dial-refused" + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return "dial-timeout" + } + return "handshake-failed" +} + func loadTLS(t *testing.T, dir, name, serverName string) *tls.Config { t.Helper() ca := filepath.Join(dir, "provider-roots.pem") diff --git a/internal/adapter/executioncontroller/handler.go b/internal/adapter/executioncontroller/handler.go index 8e3eee2afe..13356c9ba3 100644 --- a/internal/adapter/executioncontroller/handler.go +++ b/internal/adapter/executioncontroller/handler.go @@ -133,15 +133,16 @@ func (h *Handler) client(ctx context.Context) (authenticatedClient, error) { if h.backend == nil || (h.cfg.Ready != nil && !h.cfg.Ready()) { return authenticatedClient{}, wireError(executionenv.CodeNotReady, true) } - leaf, handshakeVerified, err := clientCertificate(ctx) + chain, handshakeVerified, err := clientCertificates(ctx) if err != nil || h.cfg.Security == nil && !handshakeVerified { return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) } + leaf := chain[0] if h.cfg.Security != nil { if !h.cfg.Security.Ready() { return authenticatedClient{}, wireError(executionenv.CodeNotReady, true) } - id, policy, err := h.cfg.Security.authorize(ctx, leaf) + id, policy, err := h.cfg.Security.authorize(ctx, chain) if err != nil { return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) } @@ -427,6 +428,9 @@ func (h *Handler) ListReferenceIntents(ctx context.Context, q *executionv1.ListR return nil, wireError(executionenv.CodeInvalidArgument, false) } intents, err = lifecycle.ListReferenceIntents(ctx, c.id, ownerHash(owner), int(q.Limit)) + for i := range intents { + intents[i].Owner = owner + } } else if owned, ok := h.backend.(ownerIntentBackend); ok { intents, err = owned.ListReferenceIntentsForClient(ctx, c.id, int(q.Limit)) } else { diff --git a/internal/adapter/executioncontroller/handler_test.go b/internal/adapter/executioncontroller/handler_test.go index c3b662097b..089ac615b1 100644 --- a/internal/adapter/executioncontroller/handler_test.go +++ b/internal/adapter/executioncontroller/handler_test.go @@ -25,6 +25,7 @@ type fakeBackend struct { ensureCalls, created int allocations map[string]Allocation commandState executionenv.CommandState + intents []executionenv.ReferenceIntent } func newFakeBackend() *fakeBackend { return &fakeBackend{allocations: map[string]Allocation{}} } @@ -95,8 +96,8 @@ func (*fakeBackend) ConfirmReferenceDelete(context.Context, executionenv.Environ func (*fakeBackend) CancelReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { return nil } -func (*fakeBackend) ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) { - return nil, nil +func (f *fakeBackend) ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) { + return f.intents, nil } func (*fakeBackend) FindReferenceIntent(context.Context, executionenv.EnvironmentRef, string, string, string) (executionenv.ReferenceIntent, error) { return executionenv.ReferenceIntent{}, nil @@ -134,6 +135,21 @@ func structTLSState(cert *x509.Certificate) (s tls.ConnectionState) { return s } +func TestListReferenceIntentsProjectsAttestedOwner(t *testing.T) { + backend := newFakeBackend() + backend.intents = []executionenv.ReferenceIntent{{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", State: executionenv.ReferencePendingDelete, OperationID: "delete", CreatedAt: time.Now().UTC()}} + id := "spiffe://cluster/ns/mecak8s" + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{id: {MayAttestOwner: true}}}, backend) + owner := &executionv1.Owner{Issuer: "issuer", Subject: "alice"} + out, err := h.ListReferenceIntents(authenticatedContext(id), &executionv1.ListReferenceIntentsRequest{Owner: owner, Limit: 64}) + if err != nil { + t.Fatal(err) + } + if len(out.GetIntents()) != 1 || out.GetIntents()[0].GetOwner().GetIssuer() != owner.GetIssuer() || out.GetIntents()[0].GetOwner().GetSubject() != owner.GetSubject() { + t.Fatal("owner-scoped reference intent omitted its attested owner") + } +} + func TestMigrateEnvironmentRequiresExpectedSchemaPresence(t *testing.T) { backend := &adminFakeBackend{fakeBackend: newFakeBackend()} id := "spiffe://cluster/ns/admin" diff --git a/internal/adapter/executioncontroller/security.go b/internal/adapter/executioncontroller/security.go index d8f4e69272..601856b12d 100644 --- a/internal/adapter/executioncontroller/security.go +++ b/internal/adapter/executioncontroller/security.go @@ -177,13 +177,23 @@ func (m *SecurityManager) TLSConfig() *tls.Config { }} } -func (m *SecurityManager) authorize(ctx context.Context, leaf *x509.Certificate) (string, ClientPolicy, error) { +func (m *SecurityManager) authorize(ctx context.Context, chain []*x509.Certificate) (string, ClientPolicy, error) { + if len(chain) == 0 || chain[0] == nil { + return "", ClientPolicy{}, errors.New("client certificate is unavailable") + } now := m.now() s, err := m.authoritativeAt(ctx, now) if err != nil { return "", ClientPolicy{}, err } - opts := x509.VerifyOptions{Roots: s.clientCAs, Intermediates: x509.NewCertPool(), CurrentTime: now, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + intermediates := x509.NewCertPool() + for _, cert := range chain[1:] { + if cert != nil { + intermediates.AddCert(cert) + } + } + opts := x509.VerifyOptions{Roots: s.clientCAs, Intermediates: intermediates, CurrentTime: now, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + leaf := chain[0] if _, err := leaf.Verify(opts); err != nil { return "", ClientPolicy{}, err } diff --git a/internal/adapter/executioncontroller/security_limit.go b/internal/adapter/executioncontroller/security_limit.go index 301475ef5f..d99b96985e 100644 --- a/internal/adapter/executioncontroller/security_limit.go +++ b/internal/adapter/executioncontroller/security_limit.go @@ -33,11 +33,11 @@ func NewRPCLimiter(security *SecurityManager, globalMax, clientMax int) (*RPCLim // UnaryInterceptor authenticates against current policy before reserving capacity. func (l *RPCLimiter) UnaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { - client, _, err := clientCertificate(ctx) + chain, _, err := clientCertificates(ctx) if err != nil { return nil, status.Error(codes.Unauthenticated, "execution provider request failed") } - id, _, err := l.security.authorize(ctx, client) + id, _, err := l.security.authorize(ctx, chain) if err != nil { return nil, status.Error(codes.Unauthenticated, "execution provider request failed") } @@ -48,7 +48,7 @@ func (l *RPCLimiter) UnaryInterceptor(ctx context.Context, req any, _ *grpc.Unar return handler(ctx, req) } -func clientCertificate(ctx context.Context) (*x509.Certificate, bool, error) { +func clientCertificates(ctx context.Context) ([]*x509.Certificate, bool, error) { p, ok := peer.FromContext(ctx) if !ok { return nil, false, errors.New("peer is unavailable") @@ -57,7 +57,7 @@ func clientCertificate(ctx context.Context) (*x509.Certificate, bool, error) { if !ok || len(tlsInfo.State.PeerCertificates) == 0 { return nil, false, errors.New("client certificate is unavailable") } - return tlsInfo.State.PeerCertificates[0], len(tlsInfo.State.VerifiedChains) != 0, nil + return tlsInfo.State.PeerCertificates, len(tlsInfo.State.VerifiedChains) != 0, nil } func (l *RPCLimiter) acquire(id string) bool { diff --git a/internal/adapter/executioncontroller/security_test.go b/internal/adapter/executioncontroller/security_test.go index e0e22321fb..646c445545 100644 --- a/internal/adapter/executioncontroller/security_test.go +++ b/internal/adapter/executioncontroller/security_test.go @@ -129,6 +129,16 @@ func TestSecurityManagerReloadRollbackAndRecovery(t *testing.T) { if string(firstCertificate) == string(manager.current.Load().tlsConfig.Certificates[0].Certificate[0]) { t.Fatal("TLS identity did not rotate atomically") } + manifest.Generation = 4 + writeManifest(t, manifestPath, manifest) + peer := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + peer.now = manager.now + if err := peer.Reload(t.Context()); err != nil { + t.Fatalf("peer authority advance: %v", err) + } + if manager.CheckReady(t.Context()) || manager.Ready() { + t.Fatal("readiness retained a snapshot superseded by a peer") + } manifest.Generation = 1 writeManifest(t, manifestPath, manifest) if err := manager.Reload(t.Context()); err == nil || manager.Ready() { @@ -255,6 +265,104 @@ func TestSecurityManagerGuardsEveryRPCOnExistingConnection(t *testing.T) { } } +func TestSecurityManagerAuthorizesPresentedIntermediateAndRevokesRootOnExistingConnection(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + caPEM, serverPEM, serverKeyPEM, clientCert := rpcSecurityIntermediatePKI(t, now, "spiffe://example/client") + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + store := NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), "ns", testProfiles(), nil) + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig()))) + executionv1.RegisterExecutionProviderServiceServer(server, NewHandler(HandlerConfig{Security: manager}, store)) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(caPEM) + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "provider.test", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + client := executionv1.NewExecutionProviderServiceClient(conn) + call := func() error { + _, err := client.ValidateProfile(t.Context(), &executionv1.ValidateProfileRequest{Profile: "go"}) + return err + } + if err := call(); err != nil { + t.Fatalf("intermediate-signed client RPC: %v", err) + } + removedRoot, _, _, _ := rpcSecurityPKI(t, now, "spiffe://example/other") + if err := os.WriteFile(filepath.Join(dir, "clients.pem"), removedRoot, 0o600); err != nil { + t.Fatal(err) + } + manifest.Generation = 2 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if err := call(); status.Code(err) != codes.Unauthenticated { + t.Fatalf("existing connection survived removal of intermediate issuer root: %v", err) + } +} + +func rpcSecurityIntermediatePKI(t *testing.T, now time.Time, clientURI string) ([]byte, []byte, []byte, tls.Certificate) { + t.Helper() + rootPub, rootKey, _ := ed25519.GenerateKey(rand.Reader) + root := &x509.Certificate{SerialNumber: big.NewInt(201), Subject: pkix.Name{CommonName: "root"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + rootDER, err := x509.CreateCertificate(rand.Reader, root, root, rootPub, rootKey) + if err != nil { + t.Fatal(err) + } + intermediatePub, intermediateKey, _ := ed25519.GenerateKey(rand.Reader) + intermediate := &x509.Certificate{SerialNumber: big.NewInt(202), Subject: pkix.Name{CommonName: "intermediate"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + intermediateDER, err := x509.CreateCertificate(rand.Reader, intermediate, root, intermediatePub, rootKey) + if err != nil { + t.Fatal(err) + } + issue := func(serial int64, parent *x509.Certificate, parentKey ed25519.PrivateKey, uriText string, usages []x509.ExtKeyUsage, dns []string) ([]byte, ed25519.PrivateKey) { + pub, key, _ := ed25519.GenerateKey(rand.Reader) + tmpl := &x509.Certificate{SerialNumber: big.NewInt(serial), NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: usages, DNSNames: dns} + if uriText != "" { + u, _ := url.Parse(uriText) + tmpl.URIs = []*url.URL{u} + } + der, issueErr := x509.CreateCertificate(rand.Reader, tmpl, parent, pub, parentKey) + if issueErr != nil { + t.Fatal(issueErr) + } + return der, key + } + serverDER, serverKey := issue(203, root, rootKey, "spiffe://example/provider", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, []string{"provider.test"}) + clientDER, clientKey := issue(204, intermediate, intermediateKey, clientURI, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, nil) + encodeKey := func(key ed25519.PrivateKey) []byte { + raw, marshalErr := x509.MarshalPKCS8PrivateKey(key) + if marshalErr != nil { + t.Fatal(marshalErr) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: raw}) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: rootDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), encodeKey(serverKey), tls.Certificate{Certificate: [][]byte{clientDER, intermediateDER}, PrivateKey: clientKey} +} + func rpcSecurityPKI(t *testing.T, now time.Time, clientURI string) ([]byte, []byte, []byte, tls.Certificate) { t.Helper() caPub, caKey, _ := ed25519.GenerateKey(rand.Reader) From 3a4509690d744c264da99d7fc4ff9cb3dd0a11d2 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Mon, 21 Sep 2026 12:04:47 +0300 Subject: [PATCH 10/25] fix: requeue active execution lease expiry Reconcile operation-holder expiry without waiting for another Kubernetes event and improve bounded qualification diagnostics. Co-Authored-By: Mecatl --- .../production_qualification_test.go | 12 +- e2e/k8s_execution/qualification_test.go | 5 +- .../adapter/executioncontroller/controller.go | 4 +- .../executioncontroller/controller_test.go | 110 ++++++++++++++++++ internal/app/build.go | 4 +- 5 files changed, 127 insertions(+), 8 deletions(-) diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go index 73efbb8e56..0707d21e1a 100644 --- a/e2e/k8s_execution/production_qualification_test.go +++ b/e2e/k8s_execution/production_qualification_test.go @@ -621,14 +621,15 @@ func readExecutionStatus(t *testing.T, ctx context.Context, kubeconfig, name str func waitExecutionStatus(t *testing.T, ctx context.Context, kubeconfig, name string, accept func(executionStatus) bool) executionStatus { t.Helper() deadline := time.Now().Add(4 * time.Minute) + var last executionStatus for time.Now().Before(deadline) { - s := readExecutionStatus(t, ctx, kubeconfig, name) - if accept(s) { - return s + last = readExecutionStatus(t, ctx, kubeconfig, name) + if accept(last) { + return last } time.Sleep(time.Second) } - t.Fatal("execution status did not reach required lifecycle state") + t.Fatalf("execution status did not reach required lifecycle state: environment=%q ready=%t ready_reason=%q fence_state=%q active_operation=%t", name, last.Ready, last.ReadyReason, last.FenceState, last.ActiveOperationID != "") return executionStatus{} } @@ -911,7 +912,8 @@ func TestKindExecutionProductionPendingDeleteOutageRecovery(t *testing.T) { } } if !found { - t.Fatal("pending-delete intent was not durable across provider outage") + exact, exactErr := client.FindReferenceIntent(ctx, owner, attached.Environment, binding) + t.Fatalf("pending-delete intent was not durable across provider outage: listed=%d exact_code=%s exact_state=%q exact_operation_match=%t", len(intents), remoteErrorCode(exactErr), exact.State, exact.OperationID == request.OperationID) } intruderTLS := loadTLS(t, filepath.Join(state, "pki"), "intruder", "mecatl-execution.execution-qualification.svc.cluster.local") forward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) diff --git a/e2e/k8s_execution/qualification_test.go b/e2e/k8s_execution/qualification_test.go index b2c3821367..53773e6090 100644 --- a/e2e/k8s_execution/qualification_test.go +++ b/e2e/k8s_execution/qualification_test.go @@ -324,14 +324,17 @@ func acquireRun(t *testing.T, ctx context.Context, c *executionclient.Client, ow func waitReady(t *testing.T, ctx context.Context, c *executionclient.Client, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) executionenv.AttachEnvironmentResponse { t.Helper() deadline := time.Now().Add(3 * time.Minute) + lastCode := "none" + lastReady := false for time.Now().Before(deadline) { out, err := c.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}) + lastCode, lastReady = remoteErrorCode(err), out.Ready if err == nil && out.Ready { return out } time.Sleep(500 * time.Millisecond) } - t.Fatal("execution environment did not become ready") + t.Fatalf("execution environment %q did not become ready: last_code=%s last_ready=%t", ref.ID, lastCode, lastReady) return executionenv.AttachEnvironmentResponse{} } func isRemoteCode(err error, code executionenv.ErrorCode) bool { diff --git a/internal/adapter/executioncontroller/controller.go b/internal/adapter/executioncontroller/controller.go index 919d75e2a1..0cbe838171 100644 --- a/internal/adapter/executioncontroller/controller.go +++ b/internal/adapter/executioncontroller/controller.go @@ -208,9 +208,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, name string) error { } if expires := textNested(env.Object, "status", "activeOperation", "expiresAt"); expires != "" { deadline, parseErr := time.Parse(time.RFC3339Nano, expires) - if parseErr != nil || !r.now().Before(deadline) { + now := r.now() + if parseErr != nil || !now.Before(deadline) { return r.setFenceUnknown(ctx, env, "operation holder lease expired; operation identity retained for recovery") } + r.queue.AddAfter(name, deadline.Sub(now)) } profileName := textNested(env.Object, "spec", "profile") p, ok := r.profiles.get(profileName) diff --git a/internal/adapter/executioncontroller/controller_test.go b/internal/adapter/executioncontroller/controller_test.go index 098270d2ae..32550a81f6 100644 --- a/internal/adapter/executioncontroller/controller_test.go +++ b/internal/adapter/executioncontroller/controller_test.go @@ -16,6 +16,8 @@ import ( dynamicfake "k8s.io/client-go/dynamic/fake" kubefake "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/util/workqueue" + clocktesting "k8s.io/utils/clock/testing" ) func TestReconcileRefusesForeignExistingPVCWithoutPersistingUID(t *testing.T) { @@ -177,6 +179,114 @@ func TestRepeatedReconcileDoesNotRewriteUnchangedStatus(t *testing.T) { } } +func TestActiveOperationExpiryRequeuesWithoutAnotherEvent(t *testing.T) { + ctx := t.Context() + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kubeClient := kubefake.NewSimpleClientset() + r := NewReconciler(dynamicClient, kubeClient, "ns", testProfiles()) + initialQueue := r.queue + t.Cleanup(initialQueue.ShutDown) + + // Establish a fully reconciled, ready environment so only operation expiry can + // cause the transition below. + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + pod, err := kubeClient.CoreV1().Pods("ns").Get(ctx, "executor-test", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := kubeClient.CoreV1().Pods("ns").UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + initialQueue.ShutDown() + + now := time.Date(2026, 9, 21, 12, 0, 0, 0, time.UTC) + deadline := now.Add(100 * time.Millisecond) + current, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedMap(current.Object, map[string]any{ + "id": "operation", "expiresAt": deadline.Format(time.RFC3339Nano), + }, "status", "activeOperation"); err != nil { + t.Fatal(err) + } + if _, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(ctx, current, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + + fakeClock := clocktesting.NewFakeClock(now) + delayingQueue := workqueue.NewTypedDelayingQueueWithConfig[string](workqueue.TypedDelayingQueueConfig[string]{Clock: fakeClock}) + replacementQueue := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[string](), + workqueue.TypedRateLimitingQueueConfig[string]{DelayingQueue: delayingQueue}, + ) + t.Cleanup(replacementQueue.ShutDown) + r.queue = replacementQueue + r.now = fakeClock.Now + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + beforeExpiry, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(beforeExpiry, "Ready") || textNested(beforeExpiry.Object, "status", "fenceState") != fenceHealthy { + t.Fatalf("future operation lease disturbed ready state: %v", beforeExpiry.Object["status"]) + } + + waitUntil := time.Now().Add(time.Second) + for fakeClock.Waiters() < 2 && time.Now().Before(waitUntil) { + time.Sleep(time.Millisecond) + } + if fakeClock.Waiters() < 2 { + t.Fatal("reconcile did not register the operation-expiry deadline with the delayed queue") + } + + done := make(chan struct{}) + go func() { + r.worker(ctx) + close(done) + }() + t.Cleanup(func() { + replacementQueue.ShutDown() + <-done + }) + + fakeClock.Step(99 * time.Millisecond) + preDeadline, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(preDeadline, "Ready") || textNested(preDeadline.Object, "status", "fenceState") != fenceHealthy { + t.Fatalf("operation fenced before its deadline: %v", preDeadline.Object["status"]) + } + + fakeClock.Step(time.Millisecond) + workerDeadline := time.Now().Add(2 * time.Second) + for time.Now().Before(workerDeadline) { + got, getErr := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if getErr != nil { + t.Fatal(getErr) + } + if textNested(got.Object, "status", "fenceState") == "FenceUnknown" { + if textNested(got.Object, "status", "activeOperation", "id") != "operation" || conditionTrue(got, "Ready") { + t.Fatalf("expiry did not retain unresolved operation identity and clear readiness: %v", got.Object["status"]) + } + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("worker did not reconcile the active-operation deadline without another Kubernetes event") +} + func TestRuntimeAndRelevantStatusUpdatesEnqueue(t *testing.T) { r := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kubefake.NewSimpleClientset(), "ns", testProfiles()) r.enqueueRuntime(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"execution.mecatl.dev/environment": "exec-pod"}}}) diff --git a/internal/app/build.go b/internal/app/build.go index 108e73e8d6..b85a88010c 100644 --- a/internal/app/build.go +++ b/internal/app/build.go @@ -8534,7 +8534,9 @@ const ( "the terminal client, which starts an embedded server by default or attaches to a remote one via " + "`mecatui connect ADDRESS`; mecated the general-purpose gRPC and HTTP/SSE server; mecak8s the " + "Kubernetes-native server keeping session state in Redis; mecatequi a one-shot CI task returning " + - "a patch." + "a patch; mecatl-execution-provider the authenticated Kubernetes control plane that owns persistent " + + "execution-environment lifecycle; mecatl-executor the credential-free workload helper that performs " + + "one confined filesystem or command operation inside an executor Pod." // selfKnowledgePostureAxes is the load-bearing content clause: the three safety // axes, kept distinct. Conflating the per-session permission mode with the From deaf1c3d1dfe7ea92afc8fe826f1bc613080f219 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Mon, 21 Sep 2026 13:33:03 +0300 Subject: [PATCH 11/25] fix: harden Kubernetes execution authority lifecycle Co-Authored-By: Mecatl --- .../production_qualification_test.go | 119 +++++++++- .../adapter/executioncontroller/controller.go | 6 + .../controller_lifecycle.go | 105 ++++++--- .../adapter/executioncontroller/handler.go | 12 +- .../executioncontroller/handler_test.go | 14 ++ .../production_lifecycle_test.go | 116 ++++++++++ .../adapter/executioncontroller/security.go | 88 ++++++-- .../executioncontroller/security_limit.go | 2 +- .../executioncontroller/security_test.go | 211 +++++++++++++++++- 9 files changed, 609 insertions(+), 64 deletions(-) diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go index 0707d21e1a..59abfffb7c 100644 --- a/e2e/k8s_execution/production_qualification_test.go +++ b/e2e/k8s_execution/production_qualification_test.go @@ -269,9 +269,10 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { t.Fatal("grant signed by revoked k1 remained authorized through the new client") } finalAttached := waitReady(t, ctx, newClient, owner, binding, attached.Environment) - finalClaim, err := newClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: finalAttached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-final-grant", OperationID: "rotation-acquire-final", TTL: time.Minute}) + finalRequest := executionenv.RunClaimRequest{Environment: finalAttached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-final-grant", OperationID: "rotation-acquire-final", TTL: time.Minute} + finalClaim, err := acquireCurrentAuthorityRun(ctx, finalRequest, newClient.AcquireRun, 500*time.Millisecond) if err != nil { - t.Fatalf("acquire final-authority claim: code=%s", remoteErrorCode(err)) + t.Fatalf("acquire final-authority claim: code=%s err=%v", remoteErrorCode(err), err) } finalRun := executionenv.RequestContext{Environment: finalClaim.Environment, Owner: owner, BindingID: binding, RunID: finalClaim.RunID, ClaimID: finalClaim.ClaimID, Epoch: finalClaim.Epoch, GrantGeneration: finalClaim.GrantGeneration, Grant: finalClaim.Grant} if got, err := newClient.File(ctx, executionenv.FileRequest{Context: finalRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err != nil || string(got.Data) != "old-authority\n" { @@ -313,6 +314,120 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { restoreFixtureSecurity(t, ctx, kubeconfig, rotationDir) } +func acquireCurrentAuthorityRun(ctx context.Context, req executionenv.RunClaimRequest, acquire func(context.Context, executionenv.RunClaimRequest) (executionenv.RunClaim, error), retryDelay time.Duration) (executionenv.RunClaim, error) { + const maxAttempts = 8 + + var lastErr error + for attempt := 1; attempt <= maxAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return executionenv.RunClaim{}, err + } + claim, err := acquire(ctx, req) + if err == nil { + return claim, nil + } + if !isRetryableAuthorityConvergence(err) { + return executionenv.RunClaim{}, err + } + lastErr = err + if attempt == maxAttempts { + break + } + if retryDelay <= 0 { + continue + } + timer := time.NewTimer(retryDelay) + select { + case <-ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return executionenv.RunClaim{}, ctx.Err() + case <-timer.C: + } + } + return executionenv.RunClaim{}, fmt.Errorf("final-authority acquire did not converge after %d attempts: %w", maxAttempts, lastErr) +} + +func isRetryableAuthorityConvergence(err error) bool { + var remote *executionenv.Error + return errors.As(err, &remote) && remote.Code == executionenv.CodeNotReady && remote.Retryable +} + +func TestAcquireCurrentAuthorityRunRetriesOnlyRetryableNotReady(t *testing.T) { + req := executionenv.RunClaimRequest{Environment: executionenv.EnvironmentRef{ID: "environment", Revision: "revision"}, Owner: executionenv.Owner{Issuer: "issuer", Subject: "subject"}, BindingID: "binding", RunID: "run", OperationID: "same-operation", TTL: time.Minute} + success := executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: "claim"} + + for _, tc := range []struct { + name string + errs []error + wantCalls int + wantErr bool + }{ + {name: "retryable not ready", errs: []error{&executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true}}, wantCalls: 2}, + {name: "not ready without retryability", errs: []error{&executionenv.Error{Code: executionenv.CodeNotReady}}, wantCalls: 1, wantErr: true}, + {name: "retryable unauthenticated", errs: []error{&executionenv.Error{Code: executionenv.CodeUnauthenticated, Retryable: true}}, wantCalls: 1, wantErr: true}, + {name: "permission denied", errs: []error{&executionenv.Error{Code: executionenv.CodePermissionDenied}}, wantCalls: 1, wantErr: true}, + {name: "internal", errs: []error{&executionenv.Error{Code: executionenv.CodeInternal}}, wantCalls: 1, wantErr: true}, + {name: "malformed error", errs: []error{errors.New("unexpected failure")}, wantCalls: 1, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + claim, err := acquireCurrentAuthorityRun(t.Context(), req, func(_ context.Context, got executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + calls++ + if got != req { + t.Fatalf("retry request changed: got %+v want %+v", got, req) + } + if calls <= len(tc.errs) { + return executionenv.RunClaim{}, tc.errs[calls-1] + } + return success, nil + }, 0) + if calls != tc.wantCalls { + t.Fatalf("calls=%d want=%d", calls, tc.wantCalls) + } + if tc.wantErr { + if err == nil { + t.Fatal("expected terminal error") + } + return + } + if err != nil || claim != success { + t.Fatalf("claim=%+v err=%v, want %+v", claim, err, success) + } + }) + } +} + +func TestAcquireCurrentAuthorityRunStopsAtBoundAndHonorsContext(t *testing.T) { + req := executionenv.RunClaimRequest{OperationID: "same-operation"} + t.Run("bound", func(t *testing.T) { + calls := 0 + _, err := acquireCurrentAuthorityRun(t.Context(), req, func(context.Context, executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + calls++ + return executionenv.RunClaim{}, &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true} + }, 0) + if calls != 8 || !isRemoteCode(err, executionenv.CodeNotReady) { + t.Fatalf("calls=%d code=%s, want 8/not_ready", calls, remoteErrorCode(err)) + } + }) + t.Run("cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + calls := 0 + _, err := acquireCurrentAuthorityRun(ctx, req, func(context.Context, executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + calls++ + return executionenv.RunClaim{}, nil + }, 0) + if calls != 0 || !errors.Is(err, context.Canceled) { + t.Fatalf("calls=%d err=%v, want 0/context canceled", calls, err) + } + }) +} + func waitFileContent(t *testing.T, ctx context.Context, client *executionclient.Client, rc executionenv.RequestContext, path, want, proof string) { t.Helper() deadline := time.Now().Add(30 * time.Second) diff --git a/internal/adapter/executioncontroller/controller.go b/internal/adapter/executioncontroller/controller.go index 0cbe838171..a9e26fe5f0 100644 --- a/internal/adapter/executioncontroller/controller.go +++ b/internal/adapter/executioncontroller/controller.go @@ -365,6 +365,9 @@ func (r *Reconciler) reconcileDeletion(ctx context.Context, _ dynamic.ResourceIn } func (r *Reconciler) updateRuntimeStatus(ctx context.Context, env *unstructured.Unstructured, pvc *corev1.PersistentVolumeClaim, pod *corev1.Pod, ready bool) error { return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pvc.Name, "uid": string(pvc.UID)}, "status", "pvc") _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pod.Name, "uid": string(pod.UID)}, "status", "pod") _ = unstructured.SetNestedField(o.Object, o.GetGeneration(), "status", "observedGeneration") @@ -374,6 +377,9 @@ func (r *Reconciler) updateRuntimeStatus(ctx context.Context, env *unstructured. } func (r *Reconciler) setFenceUnknown(ctx context.Context, env *unstructured.Unstructured, msg string) error { return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } _ = unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") setConditionObject(o, "Ready", false, "FenceUnknown", msg) return nil diff --git a/internal/adapter/executioncontroller/controller_lifecycle.go b/internal/adapter/executioncontroller/controller_lifecycle.go index 424623d5ea..503bb6170c 100644 --- a/internal/adapter/executioncontroller/controller_lifecycle.go +++ b/internal/adapter/executioncontroller/controller_lifecycle.go @@ -3,6 +3,7 @@ package executioncontroller import ( "context" "errors" + "reflect" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -22,7 +23,7 @@ func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.U podUID, pvcUID := text(op, "expectedPodUID"), text(op, "expectedPVCUID") podName, pvcName := textNested(env.Object, "status", "pod", "name"), textNested(env.Object, "status", "pvc", "name") if operationID == "" || pvcUID == "" { - return r.setFenceUnknown(ctx, env, "lifecycle operation identity is malformed") + return r.setLifecycleFenceUnknown(ctx, env, "lifecycle operation identity is malformed") } proofMatches := textNested(env.Object, "status", "terminationProof", operationIDField) == operationID && textNested(env.Object, "status", "terminationProof", "podUID") == podUID && textNested(env.Object, "status", "terminationProof", "pvcUID") == pvcUID && intNested(env.Object, "status", "terminationProof", "epoch") == intNested(op, "expectedEpoch") podIdentityMatches := textNested(env.Object, "status", "pod", "uid") == podUID @@ -34,14 +35,14 @@ func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.U } epoch := intNested(op, "expectedEpoch") if podUID == "" || epoch <= 0 || !podIdentityMatches || textNested(env.Object, "status", "pvc", "uid") != pvcUID || intNested(env.Object, "status", "epoch") != epoch { - return r.setFenceUnknown(ctx, env, "lifecycle operation no longer matches the exact executor and workspace") + return r.setLifecycleFenceUnknown(ctx, env, "lifecycle operation no longer matches the exact executor and workspace") } pvc, err := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Get(ctx, pvcName, metav1.GetOptions{}) if err != nil || string(pvc.UID) != pvcUID { if err != nil && !apierrors.IsNotFound(err) { return err } - return r.setFenceUnknown(ctx, env, "authoritative workspace identity is unavailable") + return r.setLifecycleFenceUnknown(ctx, env, "authoritative workspace identity is unavailable") } switch phase { case "Quiescing": @@ -50,33 +51,33 @@ func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.U if getErr != nil && !apierrors.IsNotFound(getErr) { return getErr } - return r.setFenceUnknown(ctx, env, "executor disappeared before durable terminal proof") + return r.setLifecycleFenceUnknown(ctx, env, "executor disappeared before durable terminal proof") } uid := types.UID(podUID) grace := int64(30) if deleteErr := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, podName, metav1.DeleteOptions{GracePeriodSeconds: &grace, Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { return deleteErr } - return r.setLifecyclePhase(ctx, env, operationID, "WaitingForTermination") + return r.setLifecyclePhase(ctx, env, operationID, "Quiescing", "WaitingForTermination") case "WaitingForTermination": pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) if apierrors.IsNotFound(getErr) { - return r.setFenceUnknown(ctx, env, "executor disappeared before durable terminal proof") + return r.setLifecycleFenceUnknown(ctx, env, "executor disappeared before durable terminal proof") } if getErr != nil { return getErr } if string(pod.UID) != podUID { - return r.setFenceUnknown(ctx, env, "executor UID changed during quiescing") + return r.setLifecycleFenceUnknown(ctx, env, "executor UID changed during quiescing") } if !podTerminal(pod) { - return r.setCondition(ctx, env, "Ready", false, "AwaitingTerminalExecutor", "waiting for kubelet terminal phase and terminated container states") + return r.setLifecycleCondition(ctx, env, "Ready", false, "AwaitingTerminalExecutor", "waiting for kubelet terminal phase and terminated container states") } q := adminLifecycleRequest{ExpectedEpoch: uint64(epoch), ExpectedPodUID: podUID, ExpectedPVCUID: pvcUID, OperationID: operationID} //nolint:gosec // positivity checked above. return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { current, ok, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") - if !ok || text(current, "id") != operationID || text(current, "phase") != phase { - return &executionenv.Error{Code: executionenv.CodeConflict, Message: "lifecycle operation changed concurrently"} + if !ok || text(current, "id") != operationID || text(current, "phase") != phase || !runtimeObservationMatches(o, env) { + return lifecycleConflict() } if err := unstructured.SetNestedMap(o.Object, terminationProof(q, pod), "status", "terminationProof"); err != nil { return err @@ -86,12 +87,12 @@ func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.U }) case "RemovingPodFinalizer": if !proofMatches { - return r.setFenceUnknown(ctx, env, "durable terminal proof is missing or mismatched") + return r.setLifecycleFenceUnknown(ctx, env, "durable terminal proof is missing or mismatched") } pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) if getErr == nil { if string(pod.UID) != podUID || !podTerminal(pod) { - return r.setFenceUnknown(ctx, env, "terminal executor identity changed before finalizer removal") + return r.setLifecycleFenceUnknown(ctx, env, "terminal executor identity changed before finalizer removal") } pod = pod.DeepCopy() pod.Finalizers = withoutString(pod.Finalizers, executorFinalizer) @@ -101,15 +102,15 @@ func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.U } else if !apierrors.IsNotFound(getErr) { return getErr } - return r.setLifecyclePhase(ctx, env, operationID, "WaitingForPodDeletion") + return r.setLifecyclePhase(ctx, env, operationID, "RemovingPodFinalizer", "WaitingForPodDeletion") case "WaitingForPodDeletion": if !proofMatches { - return r.setFenceUnknown(ctx, env, "durable terminal proof is missing or mismatched") + return r.setLifecycleFenceUnknown(ctx, env, "durable terminal proof is missing or mismatched") } pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) if getErr == nil { if string(pod.UID) != podUID { - return r.setFenceUnknown(ctx, env, "unexpected executor occupies the retained name") + return r.setLifecycleFenceUnknown(ctx, env, "unexpected executor occupies the retained name") } uid := types.UID(podUID) if deleteErr := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, podName, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { @@ -127,17 +128,17 @@ func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.U case "CreatingReplacement": return r.finishReplacement(ctx, env, operationID) default: - return r.setFenceUnknown(ctx, env, "unknown lifecycle phase") + return r.setLifecycleFenceUnknown(ctx, env, "unknown lifecycle phase") } } -func (r *Reconciler) setLifecyclePhase(ctx context.Context, env *unstructured.Unstructured, operationID, phase string) error { +func (r *Reconciler) setLifecyclePhase(ctx context.Context, env *unstructured.Unstructured, operationID, expectedPhase, nextPhase string) error { return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { op, found, err := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") - if err != nil || !found || text(op, "id") != operationID { - return &executionenv.Error{Code: executionenv.CodeConflict, Message: "lifecycle operation changed concurrently"} + if err != nil || !found || text(op, "id") != operationID || text(op, "phase") != expectedPhase || !runtimeObservationMatches(o, env) { + return lifecycleConflict() } - op["phase"] = phase + op["phase"] = nextPhase return unstructured.SetNestedMap(o.Object, op, "status", "lifecycleOperation") }) } @@ -145,8 +146,8 @@ func (r *Reconciler) setLifecyclePhase(ctx context.Context, env *unstructured.Un func (r *Reconciler) finishRetirement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") - if !found || text(op, "id") != operationID { - return errors.New("lifecycle operation changed concurrently") + if !found || text(op, "id") != operationID || text(op, "type") != "RetireEnvironment" || text(op, "phase") != "WaitingForPodDeletion" || !runtimeObservationMatches(o, env) { + return lifecycleConflict() } unstructured.RemoveNestedField(o.Object, "status", "lifecycleOperation") setConditionObject(o, "ExecutorTerminated", true, "TerminalPodProof", "terminal executor proof is durable") @@ -159,8 +160,8 @@ func (r *Reconciler) finishRetirement(ctx context.Context, env *unstructured.Uns func (r *Reconciler) prepareReplacement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") - if !found || text(op, "id") != operationID || text(op, "phase") != "WaitingForPodDeletion" { - return errors.New("lifecycle operation changed concurrently") + if !found || text(op, "id") != operationID || text(op, "type") != "ReplaceExecutor" || text(op, "phase") != "WaitingForPodDeletion" || !runtimeObservationMatches(o, env) { + return lifecycleConflict() } unstructured.RemoveNestedField(o.Object, "status", "pod") op["phase"] = "CreatingReplacement" @@ -171,7 +172,7 @@ func (r *Reconciler) prepareReplacement(ctx context.Context, env *unstructured.U func (r *Reconciler) finishReplacement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { profile, ok := r.profiles.get(textNested(env.Object, "spec", "profile")) if !ok || profile.Digest != textNested(env.Object, "spec", "profileDigest") { - return r.setFenceUnknown(ctx, env, "replacement profile is unavailable") + return r.setLifecycleFenceUnknown(ctx, env, "replacement profile is unavailable") } pvcName := textNested(env.Object, "status", "pvc", "name") pod, err := r.ensurePod(ctx, env, profile, resourceName("executor", env.GetName()), pvcName) @@ -179,12 +180,12 @@ func (r *Reconciler) finishReplacement(ctx context.Context, env *unstructured.Un return err } if !podReady(pod) { - return r.setCondition(ctx, env, "Ready", false, "ReplacementStarting", "waiting for replacement executor readiness") + return r.setLifecycleCondition(ctx, env, "Ready", false, "ReplacementStarting", "waiting for replacement executor readiness") } return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") - if !found || text(op, "id") != operationID || text(op, "phase") != "CreatingReplacement" { - return errors.New("lifecycle operation changed concurrently") + if !found || text(op, "id") != operationID || text(op, "type") != "ReplaceExecutor" || text(op, "phase") != "CreatingReplacement" || !runtimeObservationMatches(o, env) { + return lifecycleConflict() } epoch := intNested(o.Object, "status", "epoch") if epoch <= 0 { @@ -199,18 +200,18 @@ func (r *Reconciler) finishReplacement(ctx context.Context, env *unstructured.Un }) } -func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructured.Unstructured, op map[string]any, pvcName, pvcUID string) error { +func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructured.Unstructured, op map[string]any, pvcName, pvcUID string) error { //nolint:gocyclo // Exact retained-deletion phases deliberately fail closed at each identity check. phase := text(op, "phase") if (phase != "DeletingPVC" && phase != "ReleasingSlot") || !conditionTrue(env, "Retired") || !conditionTrue(env, "ExecutorTerminated") { - return r.setFenceUnknown(ctx, env, "retained deletion state is invalid") + return r.setLifecycleFenceUnknown(ctx, env, "retained deletion state is invalid") } pvc, err := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Get(ctx, pvcName, metav1.GetOptions{}) if err == nil { if phase != "DeletingPVC" { - return r.setFenceUnknown(ctx, env, "retained PVC reappeared after deallocation began") + return r.setLifecycleFenceUnknown(ctx, env, "retained PVC reappeared after deallocation began") } if string(pvc.UID) != pvcUID { - return r.setFenceUnknown(ctx, env, "retained PVC UID changed") + return r.setLifecycleFenceUnknown(ctx, env, "retained PVC UID changed") } uid := types.UID(pvcUID) if deleteErr := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Delete(ctx, pvcName, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { @@ -222,13 +223,17 @@ func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructu return err } if phase == "DeletingPVC" { - return r.setLifecyclePhase(ctx, env, text(op, "id"), "ReleasingSlot") + return r.setLifecyclePhase(ctx, env, text(op, "id"), "DeletingPVC", "ReleasingSlot") } res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) current, err := res.Get(ctx, env.GetName(), metav1.GetOptions{}) if err != nil { return err } + currentOp, found, _ := unstructured.NestedMap(current.Object, "status", "lifecycleOperation") + if !found || text(currentOp, "id") != text(op, "id") || text(currentOp, "type") != text(op, "type") || text(currentOp, "phase") != "ReleasingSlot" || !runtimeObservationMatches(current, env) { + return lifecycleConflict() + } if err := releaseProfileSlot(ctx, r.kube, r.namespace, textNested(current.Object, "spec", "profile"), current.GetName()); err != nil { return err } @@ -244,6 +249,40 @@ func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructu return nil } +func lifecycleConflict() error { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "lifecycle operation changed concurrently"} +} + +func runtimeObservationMatches(current, observed *unstructured.Unstructured) bool { + currentOp, currentHasOp, _ := unstructured.NestedMap(current.Object, "status", "lifecycleOperation") + observedOp, observedHasOp, _ := unstructured.NestedMap(observed.Object, "status", "lifecycleOperation") + return currentHasOp == observedHasOp && reflect.DeepEqual(currentOp, observedOp) && + intNested(current.Object, "status", "epoch") == intNested(observed.Object, "status", "epoch") && + textNested(current.Object, "status", "pod", "uid") == textNested(observed.Object, "status", "pod", "uid") && + textNested(current.Object, "status", "pvc", "uid") == textNested(observed.Object, "status", "pvc", "uid") +} + +func (r *Reconciler) setLifecycleCondition(ctx context.Context, env *unstructured.Unstructured, name string, status bool, reason, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + setConditionObject(o, name, status, reason, msg) + return nil + }) +} + +func (r *Reconciler) setLifecycleFenceUnknown(ctx context.Context, env *unstructured.Unstructured, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + _ = unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") + setConditionObject(o, "Ready", false, "FenceUnknown", msg) + return nil + }) +} + func withoutString(values []string, remove string) []string { out := make([]string, 0, len(values)) for _, value := range values { diff --git a/internal/adapter/executioncontroller/handler.go b/internal/adapter/executioncontroller/handler.go index 13356c9ba3..f6f8d7e7ca 100644 --- a/internal/adapter/executioncontroller/handler.go +++ b/internal/adapter/executioncontroller/handler.go @@ -144,7 +144,7 @@ func (h *Handler) client(ctx context.Context) (authenticatedClient, error) { } id, policy, err := h.cfg.Security.authorize(ctx, chain) if err != nil { - return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) + return authenticatedClient{}, securityAuthorizationError(err) } return authenticatedClient{id: id, policy: policy}, nil } @@ -315,6 +315,9 @@ func (h *Handler) runClaimRequest(ctx context.Context, pRef *executionv1.Environ func (h *Handler) runClaimResponse(ctx context.Context, claim executionenv.RunClaim, client, owner string) (*executionv1.RunClaimResponse, error) { grant, expiry, err := h.signClaim(ctx, claim, client, owner) if err != nil { + if errors.Is(err, errAuthorityUnavailable) { + return nil, securityAuthorizationError(err) + } return nil, wireError(executionenv.CodeInternal, false) } return &executionv1.RunClaimResponse{Environment: refToProto(claim.Environment), BindingId: valid(claim.BindingID), RunId: valid(claim.RunID), ClaimId: valid(claim.ClaimID), Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: valid(grant), ExpiresAt: timestamppb.New(expiry)}, nil @@ -833,6 +836,13 @@ func fingerprint(fields ...string) string { return hex.EncodeToString(h.Sum(nil)) } +func securityAuthorizationError(err error) error { + if errors.Is(err, errAuthorityUnavailable) { + return wireError(executionenv.CodeNotReady, true) + } + return wireError(executionenv.CodeUnauthenticated, false) +} + func backendError(err error) error { var e *executionenv.Error if errors.As(err, &e) && e.Code.Valid() { diff --git a/internal/adapter/executioncontroller/handler_test.go b/internal/adapter/executioncontroller/handler_test.go index 089ac615b1..b7bc9311e2 100644 --- a/internal/adapter/executioncontroller/handler_test.go +++ b/internal/adapter/executioncontroller/handler_test.go @@ -333,6 +333,20 @@ func TestCanonicalClientIdentityRejectsCNAndAmbiguousURIs(t *testing.T) { } } +func TestRunClaimResponseKeepsNonAuthoritySigningFailureInternal(t *testing.T) { + h := NewHandler(HandlerConfig{Signer: GrantSigner{KeyID: "k1", Issuer: "issuer", Audience: "audience", Lifetime: time.Minute}}, newFakeBackend()) + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1} + _, err := h.runClaimResponse(t.Context(), claim, "spiffe://example/client", "owner") + st := status.Convert(err) + if st.Code() != codes.Internal { + t.Fatalf("code=%v", st.Code()) + } + details := st.Details() + if len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeInternal) || details[0].(*executionv1.ErrorDetail).Retryable { + t.Fatalf("details=%v", details) + } +} + func TestWireErrorDoesNotExposeBackendMessage(t *testing.T) { err := backendError(&executionenv.Error{Code: executionenv.CodeInternal, Message: "kubectl exec --token=secret"}) if got := status.Convert(err).Message(); got != "execution provider request failed" { diff --git a/internal/adapter/executioncontroller/production_lifecycle_test.go b/internal/adapter/executioncontroller/production_lifecycle_test.go index 627ffb84ae..06c72295c9 100644 --- a/internal/adapter/executioncontroller/production_lifecycle_test.go +++ b/internal/adapter/executioncontroller/production_lifecycle_test.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/types" dynamicfake "k8s.io/client-go/dynamic/fake" kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" "github.com/stacklok/mecatl/internal/executionenv" ) @@ -354,6 +355,121 @@ func TestReplacementPersistsTerminalProofBeforeRemovingPodFinalizer(t *testing.T } } +func replacementExecutor(ready bool) *corev1.Pod { + pod := terminalExecutor() + pod.UID = types.UID("new-pod") + pod.Status = corev1.PodStatus{Phase: corev1.PodRunning} + if ready { + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + } + return pod +} + +func replacementLifecycleEnvironment() *unstructured.Unstructured { + env := lifecycleAdminEnvironment(2, []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, terminationProof(q, terminalExecutor()), "status", "terminationProof") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": q.OperationID, "type": "ReplaceExecutor", "phase": "CreatingReplacement", "expectedEpoch": int64(4), "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID, "createdAt": "2026-09-21T00:00:00Z"}, "status", "lifecycleOperation") + unstructured.RemoveNestedField(env.Object, "status", "pod") + setConditionObject(env, "Ready", false, "ReplacementStarting", "waiting") + return env +} + +func TestStaleReplacementObservationCannotOverwriteCompletion(t *testing.T) { + stale := replacementLifecycleEnvironment() + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), stale.DeepCopy()) + kube := kubefake.NewSimpleClientset(replacementExecutor(false), retainedPVC()) + observed, release := make(chan struct{}), make(chan struct{}) + kube.PrependReactor("get", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + close(observed) + <-release + return false, nil, nil + }) + r := NewReconciler(dynamicClient, kube, "ns", testProfiles()) + done := make(chan error, 1) + go func() { done <- r.finishReplacement(t.Context(), stale, adminRequestFixture().OperationID) }() + <-observed + + current, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + _ = unstructured.SetNestedField(current.Object, int64(5), "status", "epoch") + _ = unstructured.SetNestedMap(current.Object, map[string]any{"name": "executor", "uid": "new-pod"}, "status", "pod") + _ = unstructured.SetNestedMap(current.Object, map[string]any{operationIDField: adminRequestFixture().OperationID, "previousPodUID": "pod-uid", "replacementPodUID": "new-pod", "pvcUID": "pvc-uid", "previousEpoch": int64(4), "replacementEpoch": int64(5)}, "status", "lastReplacement") + unstructured.RemoveNestedField(current.Object, "status", "lifecycleOperation") + setConditionObject(current, "Ready", true, "ReplacementReady", "replacement executor is ready on the retained workspace") + if _, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), current, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + close(release) + var controlled *executionenv.Error + if err := <-done; !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("stale replacement result=%v", err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if !conditionTrue(got, "Ready") || intNested(got.Object, "status", "epoch") != 5 || textNested(got.Object, "status", "pod", "uid") != "new-pod" { + t.Fatalf("stale unready write damaged completion: %v", got.Object["status"]) + } + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + if allocation, err := store.Attach(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "binding"); err != nil || !allocation.Ready || allocation.Epoch != 5 { + t.Fatalf("attach after completed replacement: allocation=%+v err=%v", allocation, err) + } +} + +func TestLifecycleStatusWritersRejectStaleObservations(t *testing.T) { + t.Run("phase regression", func(t *testing.T) { + stale := replacementLifecycleEnvironment() + _ = unstructured.SetNestedField(stale.Object, "Quiescing", "status", "lifecycleOperation", "phase") + current := stale.DeepCopy() + _ = unstructured.SetNestedField(current.Object, "RemovingPodFinalizer", "status", "lifecycleOperation", "phase") + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), current) + r := NewReconciler(client, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.setLifecyclePhase(t.Context(), stale, adminRequestFixture().OperationID, "Quiescing", "WaitingForTermination"); err == nil { + t.Fatal("stale phase transition succeeded") + } + got, _ := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if phase := textNested(got.Object, "status", "lifecycleOperation", "phase"); phase != "RemovingPodFinalizer" { + t.Fatalf("phase regressed to %q", phase) + } + }) + + t.Run("waiting and fence writes", func(t *testing.T) { + stale := replacementLifecycleEnvironment() + _ = unstructured.SetNestedField(stale.Object, "WaitingForTermination", "status", "lifecycleOperation", "phase") + current := stale.DeepCopy() + _ = unstructured.SetNestedField(current.Object, "RemovingPodFinalizer", "status", "lifecycleOperation", "phase") + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), current) + r := NewReconciler(client, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.setLifecycleCondition(t.Context(), stale, "Ready", false, "AwaitingTerminalExecutor", "waiting"); err == nil { + t.Fatal("stale waiting condition succeeded") + } + if err := r.setLifecycleFenceUnknown(t.Context(), stale, "stale failure"); err == nil { + t.Fatal("stale fence write succeeded") + } + got, _ := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != fenceHealthy || textNested(got.Object, "status", "lifecycleOperation", "phase") != "RemovingPodFinalizer" { + t.Fatalf("stale writer changed lifecycle status: %v", got.Object["status"]) + } + }) + + t.Run("ordinary reconcile after admission", func(t *testing.T) { + stale := lifecycleAdminEnvironment(2, []any{}) + current := stale.DeepCopy() + _ = unstructured.SetNestedMap(current.Object, map[string]any{"id": "replace", "type": "ReplaceExecutor", "phase": "Quiescing", "expectedEpoch": int64(4), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid", "createdAt": "2026-09-21T00:00:00Z"}, "status", "lifecycleOperation") + setConditionObject(current, "Ready", false, "Quiescing", "replacement admitted") + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), current) + r := NewReconciler(client, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.updateRuntimeStatus(t.Context(), stale, retainedPVC(), terminalExecutor(), true); err == nil { + t.Fatal("ordinary reconcile overwrote lifecycle admission") + } + got, _ := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if conditionTrue(got, "Ready") || textNested(got.Object, "status", "lifecycleOperation", "id") != "replace" { + t.Fatalf("ordinary reconcile damaged lifecycle admission: %v", got.Object["status"]) + } + }) +} + func TestWrongPVCUIDCannotStartRetainedDeletion(t *testing.T) { env := lifecycleAdminEnvironment(2, []any{}) setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") diff --git a/internal/adapter/executioncontroller/security.go b/internal/adapter/executioncontroller/security.go index 601856b12d..aa457bfccf 100644 --- a/internal/adapter/executioncontroller/security.go +++ b/internal/adapter/executioncontroller/security.go @@ -102,14 +102,17 @@ type securityLedger struct { Fingerprints map[string]string `json:"fingerprints"` } +type securityState struct { + snapshot *securitySnapshot +} + // SecurityManager reloads one immutable, versioned security snapshot from // projected files and binds its generation to a durable ConfigMap high-water mark. type SecurityManager struct { manifestPath, keyDirectory, namespace, configMap string kube kubernetes.Interface now func() time.Time - current atomic.Pointer[securitySnapshot] - ready atomic.Bool + state atomic.Pointer[securityState] } // NewSecurityManager constructs a fail-closed security material reloader. @@ -119,9 +122,8 @@ func NewSecurityManager(manifestPath, keyDirectory, namespace, configMap string, // Ready reports whether the current snapshot is authoritative and unexpired. func (m *SecurityManager) Ready() bool { - now := m.now() - s := m.current.Load() - return m.ready.Load() && snapshotValidAt(s, now) + state := m.state.Load() + return state != nil && snapshotValidAt(state.snapshot, m.now()) } // CheckReady verifies that the loaded snapshot still matches durable authority. @@ -132,18 +134,29 @@ func (m *SecurityManager) CheckReady(ctx context.Context) bool { // Reload atomically validates and publishes a complete security snapshot. func (m *SecurityManager) Reload(ctx context.Context) error { + observed := m.state.Load() candidate, err := m.load() if err != nil { - m.ready.Store(false) + m.invalidateObservedUnlessReplaced(ctx, observed) return err } if err := m.publishGeneration(ctx, candidate); err != nil { - m.ready.Store(false) + m.invalidateObservedUnlessReplaced(ctx, observed) return err } - m.current.Store(candidate) - m.ready.Store(true) - return nil + if err := m.verifyAuthority(ctx, candidate); err != nil { + m.invalidateObservedUnlessReplaced(ctx, observed) + return err + } + for { + current := m.state.Load() + if current != nil && current.snapshot.generation > candidate.generation { + return errors.New("security manifest generation rollback rejected") + } + if m.state.CompareAndSwap(current, &securityState{snapshot: candidate}) { + return nil + } + } } // Run periodically reloads security material until ctx is cancelled. @@ -169,11 +182,11 @@ func (m *SecurityManager) Run(ctx context.Context, interval time.Duration) { // TLSConfig returns a dynamic TLS configuration backed by the current snapshot. func (m *SecurityManager) TLSConfig() *tls.Config { return &tls.Config{MinVersion: tls.VersionTLS13, ClientAuth: tls.RequireAnyClientCert, GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { - s := m.current.Load() - if !m.Ready() { + state := m.state.Load() + if state == nil || !snapshotValidAt(state.snapshot, m.now()) { return nil, errors.New("security material is not ready") } - return s.tlsConfig.Clone(), nil + return state.snapshot.tlsConfig.Clone(), nil }} } @@ -216,22 +229,53 @@ func (m *SecurityManager) authoritative(ctx context.Context) (*securitySnapshot, return m.authoritativeAt(ctx, m.now()) } +var errAuthorityUnavailable = errors.New("security authority is unavailable") + +func authorityUnavailable(err error) error { + if err == nil { + return errAuthorityUnavailable + } + return fmt.Errorf("%w: %v", errAuthorityUnavailable, err) +} + func (m *SecurityManager) authoritativeAt(ctx context.Context, now time.Time) (*securitySnapshot, error) { - s := m.current.Load() - if !m.ready.Load() || !snapshotValidAt(s, now) || m.kube == nil { - return nil, errors.New("security material is not ready") + state := m.state.Load() + if state == nil || !snapshotValidAt(state.snapshot, now) || m.kube == nil { + return nil, authorityUnavailable(nil) + } + if err := m.verifyAuthority(ctx, state.snapshot); err != nil { + m.state.CompareAndSwap(state, nil) + return nil, authorityUnavailable(err) + } + return state.snapshot, nil +} + +func (m *SecurityManager) verifyAuthority(ctx context.Context, snapshot *securitySnapshot) error { + if snapshot == nil || m.kube == nil { + return errors.New("security material is not ready") } cm, err := m.kube.CoreV1().ConfigMaps(m.namespace).Get(ctx, m.configMap, metav1.GetOptions{}) if err != nil { - m.ready.Store(false) - return nil, err + return err } var ledger securityLedger - if err := executionenv.DecodeStrict([]byte(cm.Data[securityStateDataKey]), &ledger); err != nil || ledger.Generation != s.generation || ledger.Digest != s.digest || !maps.Equal(ledger.Fingerprints, s.fingerprints) { - m.ready.Store(false) - return nil, errors.New("loaded security generation is not authoritative") + if err := executionenv.DecodeStrict([]byte(cm.Data[securityStateDataKey]), &ledger); err != nil { + return err + } + if ledger.Generation != snapshot.generation || ledger.Digest != snapshot.digest || !maps.Equal(ledger.Fingerprints, snapshot.fingerprints) { + return errors.New("loaded security generation is not authoritative") + } + return nil +} + +func (m *SecurityManager) invalidateObservedUnlessReplaced(ctx context.Context, observed *securityState) { + if m.state.CompareAndSwap(observed, nil) { + return + } + current := m.state.Load() + if current != nil && m.verifyAuthority(ctx, current.snapshot) != nil { + m.state.CompareAndSwap(current, nil) } - return s, nil } func snapshotValidAt(s *securitySnapshot, now time.Time) bool { diff --git a/internal/adapter/executioncontroller/security_limit.go b/internal/adapter/executioncontroller/security_limit.go index d99b96985e..d172248c50 100644 --- a/internal/adapter/executioncontroller/security_limit.go +++ b/internal/adapter/executioncontroller/security_limit.go @@ -39,7 +39,7 @@ func (l *RPCLimiter) UnaryInterceptor(ctx context.Context, req any, _ *grpc.Unar } id, _, err := l.security.authorize(ctx, chain) if err != nil { - return nil, status.Error(codes.Unauthenticated, "execution provider request failed") + return nil, securityAuthorizationError(err) } if !l.acquire(id) { return nil, status.Error(codes.ResourceExhausted, "execution provider is at its concurrency limit") diff --git a/internal/adapter/executioncontroller/security_test.go b/internal/adapter/executioncontroller/security_test.go index 646c445545..9de09f5009 100644 --- a/internal/adapter/executioncontroller/security_test.go +++ b/internal/adapter/executioncontroller/security_test.go @@ -1,6 +1,7 @@ package executioncontroller import ( + "context" "crypto/ed25519" "crypto/rand" "crypto/sha256" @@ -10,6 +11,7 @@ import ( "encoding/hex" "encoding/json" "encoding/pem" + "errors" "math/big" "net" "net/url" @@ -17,23 +19,73 @@ import ( "path/filepath" "slices" "strings" + "sync" + "sync/atomic" "testing" "time" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + grpcpeer "google.golang.org/grpc/peer" "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" "github.com/stacklok/mecatl/internal/executionenv" ) +type durableClaimBackend struct { + *fakeBackend + mu sync.Mutex + acquired, renewed map[string]executionenv.RunClaim + acquireCalls, renewalCalls int +} + +func newDurableClaimBackend() *durableClaimBackend { + return &durableClaimBackend{fakeBackend: newFakeBackend(), acquired: make(map[string]executionenv.RunClaim), renewed: make(map[string]executionenv.RunClaim)} +} + +func (b *durableClaimBackend) AcquireRun(_ context.Context, ref executionenv.EnvironmentRef, _ string, _ string, binding, run, operation string, ttl time.Duration) (executionenv.RunClaim, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.acquireCalls++ + if claim, ok := b.acquired[operation]; ok { + return claim, nil + } + claim := executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: run, ClaimID: "claim-acquire", Epoch: 2, GrantGeneration: 1, ExpiresAt: time.Now().Add(ttl)} + b.acquired[operation] = claim + return claim, nil +} + +func (b *durableClaimBackend) RenewRun(_ context.Context, _ executionenv.EnvironmentRef, _ string, _ string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.renewalCalls++ + if claim, ok := b.renewed[req.OperationID]; ok { + return claim, nil + } + claim := executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: req.ClaimID, Epoch: req.Epoch, GrantGeneration: req.GrantGeneration + 1, ExpiresAt: time.Now().Add(req.TTL)} + b.renewed[req.OperationID] = claim + return claim, nil +} + +func (b *durableClaimBackend) claim(operation string, renew bool) (executionenv.RunClaim, int, bool) { + b.mu.Lock() + defer b.mu.Unlock() + if renew { + claim, ok := b.renewed[operation] + return claim, b.renewalCalls, ok + } + claim, ok := b.acquired[operation] + return claim, b.acquireCalls, ok +} + func TestSecurityManagerReloadRollbackAndRecovery(t *testing.T) { dir := t.TempDir() data := filepath.Join(dir, "..data") @@ -91,7 +143,7 @@ func TestSecurityManagerReloadRollbackAndRecovery(t *testing.T) { if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { t.Fatalf("recovery reload: ready=%v err=%v", manager.Ready(), err) } - firstCertificate := append([]byte(nil), manager.current.Load().tlsConfig.Certificates[0].Certificate[0]...) + firstCertificate := append([]byte(nil), manager.state.Load().snapshot.tlsConfig.Certificates[0].Certificate[0]...) cm, err := kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) if err != nil { t.Fatal(err) @@ -126,7 +178,7 @@ func TestSecurityManagerReloadRollbackAndRecovery(t *testing.T) { if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { t.Fatalf("TLS/CA correction did not recover: ready=%v err=%v", manager.Ready(), err) } - if string(firstCertificate) == string(manager.current.Load().tlsConfig.Certificates[0].Certificate[0]) { + if string(firstCertificate) == string(manager.state.Load().snapshot.tlsConfig.Certificates[0].Certificate[0]) { t.Fatal("TLS identity did not rotate atomically") } manifest.Generation = 4 @@ -146,6 +198,49 @@ func TestSecurityManagerReloadRollbackAndRecovery(t *testing.T) { } } +func TestSecurityManagerStaleAuthorityFailureCannotPoisonNewerState(t *testing.T) { + now := time.Now().UTC() + window := keyValidity{activateAt: now.Add(-time.Minute), verifyUntil: now.Add(time.Hour)} + oldSnapshot := &securitySnapshot{generation: 1, digest: "old", activeWindow: window, validUntil: now.Add(time.Hour), fingerprints: map[string]string{"k1:1": "old"}} + newSnapshot := &securitySnapshot{generation: 2, digest: "new", activeWindow: window, validUntil: now.Add(time.Hour), fingerprints: map[string]string{"k1:1": "old", "k1:2": "new"}} + raw, err := json.Marshal(securityLedger{Generation: newSnapshot.generation, Digest: newSnapshot.digest, Fingerprints: newSnapshot.fingerprints}) + if err != nil { + t.Fatal(err) + } + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{securityStateDataKey: string(raw)}}) + started, release := make(chan struct{}), make(chan struct{}) + var startOnce sync.Once + kube.PrependReactor("get", "configmaps", func(_ k8stesting.Action) (bool, runtime.Object, error) { + startOnce.Do(func() { close(started) }) + <-release + return false, nil, nil + }) + manager := NewSecurityManager("", "", "ns", "authority", kube) + manager.now = func() time.Time { return now } + oldState := &securityState{snapshot: oldSnapshot} + manager.state.Store(oldState) + done := make(chan error, 1) + go func() { + _, authErr := manager.authoritative(t.Context()) + done <- authErr + }() + <-started + newState := &securityState{snapshot: newSnapshot} + manager.state.Store(newState) + close(release) + if err := <-done; !errors.Is(err, errAuthorityUnavailable) { + t.Fatalf("stale authority check error=%v", err) + } + if manager.state.Load() != newState || !manager.Ready() { + t.Fatal("stale authority check invalidated the newer published state") + } + + manager.invalidateObservedUnlessReplaced(t.Context(), oldState) + if manager.state.Load() != newState || !manager.Ready() { + t.Fatal("stale reload failure invalidated the newer authoritative state") + } +} + func TestSecurityLedgerRejectsKeyVersionRollbackAcrossRestart(t *testing.T) { ledger := securityLedger{Generation: 2, Fingerprints: map[string]string{"key:2": "old"}} candidate := &securitySnapshot{generation: 3, fingerprints: map[string]string{"key:1": "new"}} @@ -181,7 +276,11 @@ func TestSecurityManagerGuardsEveryRPCOnExistingConnection(t *testing.T) { t.Fatal(err) } store := NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), "ns", testProfiles(), nil) - server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig()))) + limiter, err := NewRPCLimiter(manager, 4, 2) + if err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig())), grpc.UnaryInterceptor(limiter.UnaryInterceptor)) executionv1.RegisterExecutionProviderServiceServer(server, NewHandler(HandlerConfig{Security: manager}, store)) listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -219,8 +318,10 @@ func TestSecurityManagerGuardsEveryRPCOnExistingConnection(t *testing.T) { if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { t.Fatal(err) } - if err := call(); status.Code(err) != codes.Unauthenticated || manager.Ready() { + if err := call(); status.Code(err) != codes.Unavailable || manager.Ready() { t.Fatalf("old snapshot survived durable authority drift: ready=%v err=%v", manager.Ready(), err) + } else if details := status.Convert(err).Details(); len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeNotReady) || !details[0].(*executionv1.ErrorDetail).Retryable { + t.Fatalf("authority lag was not a structured retryable not_ready: %v", details) } cm, _ = kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) cm.Data[securityStateDataKey] = authoritativeState @@ -265,6 +366,102 @@ func TestSecurityManagerGuardsEveryRPCOnExistingConnection(t *testing.T) { } } +func TestRunClaimSigningAuthorityOutageIsRetryableAfterDurableMutation(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + caPEM, serverPEM, serverKeyPEM, clientCert := rpcSecurityPKI(t, now, "spiffe://example/client") + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + var reads, failAt atomic.Int64 + kube.PrependReactor("get", "configmaps", func(_ k8stesting.Action) (bool, runtime.Object, error) { + read := reads.Add(1) + if target := failAt.Load(); target != 0 && read == target { + return true, nil, errors.New("injected authority read outage") + } + return false, nil, nil + }) + armSigningOutage := func() { + reads.Store(0) + failAt.Store(2) // Authorization reads first; response signing reads second. + } + recoverAuthority := func() { + failAt.Store(0) + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("reload authority: %v", err) + } + } + assertNotReady := func(err error) { + t.Helper() + st := status.Convert(err) + details := st.Details() + if st.Code() != codes.Unavailable || len(details) != 1 { + t.Fatalf("code=%v details=%v", st.Code(), details) + } + detail, ok := details[0].(*executionv1.ErrorDetail) + if !ok || detail.Code != string(executionenv.CodeNotReady) || !detail.Retryable { + t.Fatalf("detail=%v", details) + } + } + leaf, err := x509.ParseCertificate(clientCert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + ctx := grpcpeer.NewContext(t.Context(), &grpcpeer.Peer{AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}, VerifiedChains: [][]*x509.Certificate{{leaf}}}}}) + backend := newDurableClaimBackend() + h := NewHandler(HandlerConfig{Security: manager}, backend) + owner := &executionv1.Owner{Issuer: "issuer", Subject: "alice"} + ref := &executionv1.EnvironmentRef{Id: "env", Revision: "rev"} + acquire := &executionv1.AcquireRunRequest{Environment: ref, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire-op", TtlMillis: time.Minute.Milliseconds()} + + armSigningOutage() + _, err = h.AcquireRun(ctx, acquire) + assertNotReady(err) + storedAcquire, calls, ok := backend.claim(acquire.OperationId, false) + if !ok || calls != 1 { + t.Fatalf("acquire mutation missing or retried automatically: stored=%t calls=%d", ok, calls) + } + recoverAuthority() + acquired, err := h.AcquireRun(ctx, acquire) + if err != nil { + t.Fatalf("explicit acquire retry: %v", err) + } + if callsClaim, retryCalls, ok := backend.claim(acquire.OperationId, false); !ok || retryCalls != 2 || callsClaim.ClaimID != storedAcquire.ClaimID || acquired.ClaimId != storedAcquire.ClaimID { + t.Fatalf("acquire retry did not converge: response=%q stored=%+v calls=%d", acquired.GetClaimId(), callsClaim, retryCalls) + } + + renew := &executionv1.RenewRunRequest{Environment: acquired.Environment, Owner: owner, BindingId: acquired.BindingId, RunId: acquired.RunId, ClaimId: acquired.ClaimId, Epoch: acquired.Epoch, GrantGeneration: acquired.GrantGeneration, OperationId: "renew-op", TtlMillis: time.Minute.Milliseconds()} + armSigningOutage() + _, err = h.RenewRun(ctx, renew) + assertNotReady(err) + storedRenew, calls, ok := backend.claim(renew.OperationId, true) + if !ok || calls != 1 { + t.Fatalf("renew mutation missing or retried automatically: stored=%t calls=%d", ok, calls) + } + recoverAuthority() + renewed, err := h.RenewRun(ctx, renew) + if err != nil { + t.Fatalf("explicit renew retry: %v", err) + } + if callsClaim, retryCalls, ok := backend.claim(renew.OperationId, true); !ok || retryCalls != 2 || callsClaim.ClaimID != storedRenew.ClaimID || callsClaim.GrantGeneration != storedRenew.GrantGeneration || renewed.ClaimId != storedRenew.ClaimID || renewed.GrantGeneration != storedRenew.GrantGeneration { + t.Fatalf("renew retry did not converge: response=%q/%d stored=%+v calls=%d", renewed.GetClaimId(), renewed.GetGrantGeneration(), callsClaim, retryCalls) + } +} + func TestSecurityManagerAuthorizesPresentedIntermediateAndRevokesRootOnExistingConnection(t *testing.T) { dir := t.TempDir() now := time.Now().UTC() @@ -287,7 +484,11 @@ func TestSecurityManagerAuthorizesPresentedIntermediateAndRevokesRootOnExistingC t.Fatal(err) } store := NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), "ns", testProfiles(), nil) - server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig()))) + limiter, err := NewRPCLimiter(manager, 4, 2) + if err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig())), grpc.UnaryInterceptor(limiter.UnaryInterceptor)) executionv1.RegisterExecutionProviderServiceServer(server, NewHandler(HandlerConfig{Security: manager}, store)) listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { From 6b851d0c232f83c8309b478d6584f97500cfe627 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Mon, 21 Sep 2026 20:45:55 +0300 Subject: [PATCH 12/25] fix: close Kubernetes execution lifecycle safety gaps Co-Authored-By: OpenAI --- deploy/helm/mecatl-execution/lifetime_test.go | 272 +++++++++++ .../mecatl-execution/templates/_helpers.tpl | 23 + .../templates/lifetime-check.yaml | 56 +++ .../templates/network-policy.yaml | 4 +- .../mecatl-execution/templates/provider.yaml | 17 +- deploy/mecatl-execution-kind/run.sh | 23 +- docs/adr/0027-cloud-native.md | 6 +- docs/design/IMPLEMENTATION-NOTES.md | 16 +- docs/design/PRODUCTION-READINESS.md | 1 + .../production_qualification_test.go | 1 + .../zz_production_helm_lifetime_test.go | 379 +++++++++++++++ .../executioncontroller/admin_lifecycle.go | 88 +++- .../admin_scope_security_test.go | 99 ++++ .../executioncontroller/admin_scope_test.go | 447 ++++++++++++++++++ .../adapter/executioncontroller/controller.go | 10 +- .../executioncontroller/controller_test.go | 8 +- .../adapter/executioncontroller/handler.go | 14 +- .../executioncontroller/handler_test.go | 2 +- .../provisioning_retry_test.go | 135 ++++++ .../adapter/executioncontroller/security.go | 28 +- .../store_concurrency_test.go | 2 +- .../executioncontroller/store_revoke.go | 8 +- .../executioncontroller/store_revoke_test.go | 8 +- user-docs/building/deployment/mecak8s.md | 122 ++++- user-docs/features/execution-environments.md | 31 +- 25 files changed, 1712 insertions(+), 88 deletions(-) create mode 100644 deploy/helm/mecatl-execution/lifetime_test.go create mode 100644 deploy/helm/mecatl-execution/templates/lifetime-check.yaml create mode 100644 e2e/k8s_execution/zz_production_helm_lifetime_test.go create mode 100644 internal/adapter/executioncontroller/admin_scope_security_test.go create mode 100644 internal/adapter/executioncontroller/admin_scope_test.go create mode 100644 internal/adapter/executioncontroller/provisioning_retry_test.go diff --git a/deploy/helm/mecatl-execution/lifetime_test.go b/deploy/helm/mecatl-execution/lifetime_test.go new file mode 100644 index 0000000000..b2d0adca33 --- /dev/null +++ b/deploy/helm/mecatl-execution/lifetime_test.go @@ -0,0 +1,272 @@ +package chart_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/yaml" +) + +// Exercise Helm's actual lookup/render path, with a loopback API containing only +// synthetic nonsecret objects. No ambient kubeconfig or cluster is consulted. +func TestChartRetainedLifetime(t *testing.T) { + fresh, err := renderLifetime(t, nil) + if err != nil { + t.Fatal(err) + } + retained := map[string]map[string]any{} + for _, obj := range fresh { + u := &unstructured.Unstructured{Object: obj} + if u.GetAnnotations()["helm.sh/resource-policy"] == "keep" { + retained[u.GetKind()+"/"+u.GetName()] = obj + } + } + if len(retained) != 6 { + t.Fatalf("retained resources=%d, want four ConfigMaps and two workload policies", len(retained)) + } + authority := "ConfigMap/test-mecatl-execution-security-authority" + capacity := "ConfigMap/mecatl-execution-profile-allocations" + retained[authority]["data"] = map[string]any{"state.json": `{"generation":7,"digest":"synthetic","fingerprints":{"k1:1":"synthetic"}}`} + retained[capacity]["data"] = map[string]any{"profile-go": `["allocation-uid"]`} + retained["ExecutionEnvironment/exec-test"] = map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "exec-test", "namespace": "ns"}} + for _, upgrade := range []bool{false, true} { + t.Run(fmt.Sprintf("reuse-upgrade-%t", upgrade), func(t *testing.T) { + var args []string + if upgrade { + args = append(args, "--is-upgrade") + } + rendered, err := renderLifetime(t, retained, args...) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{authority, capacity} { + if !reflect.DeepEqual(rendered[key]["data"], retained[key]["data"]) { + t.Fatalf("%s data was reset", key) + } + } + for key, obj := range retained { + if strings.HasPrefix(key, "NetworkPolicy/") && !reflect.DeepEqual(rendered[key]["spec"], obj["spec"]) { + t.Fatalf("%s changed confinement", key) + } + } + }) + } + tests := []struct { + name string + mutate func(map[string]map[string]any) + args []string + want string + }{ + {name: "active provider deployment", mutate: func(m map[string]map[string]any) { + m["Deployment/test-mecatl-execution"] = map[string]any{"apiVersion": "apps/v1", "kind": "Deployment", "metadata": map[string]any{"name": "test-mecatl-execution", "namespace": "ns"}, "spec": map[string]any{"replicas": 2}} + }, want: "quiesce the execution provider"}, + {name: "provider pod still terminating", mutate: func(m map[string]map[string]any) { + m["Pod/provider"] = map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": map[string]any{"name": "provider", "namespace": "ns", "deletionTimestamp": "2026-09-21T00:00:00Z", "labels": map[string]any{"app.kubernetes.io/name": "test-mecatl-execution"}}} + }, want: "wait for all execution provider Pods"}, + {name: "reserved capacity without CR", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, authority) + }, want: "bootstrap is forbidden"}, + {name: "surviving pod without CR", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, authority) + delete(m, capacity) + m["Pod/executor"] = map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": map[string]any{"name": "executor", "namespace": "ns", "labels": map[string]any{"execution.mecatl.dev/environment": "exec-test"}}} + }, want: "bootstrap is forbidden"}, + {name: "surviving PVC without CR", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, authority) + delete(m, capacity) + m["PersistentVolumeClaim/workspace"] = map[string]any{"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": map[string]any{"name": "workspace", "namespace": "ns", "labels": map[string]any{"execution.mecatl.dev/environment": "exec-test"}}} + }, want: "bootstrap is forbidden"}, + {name: "missing lifetime configuration", mutate: func(m map[string]map[string]any) { + delete(m["ConfigMap/test-mecatl-execution-profiles"]["data"].(map[string]any), "lifetime.json") + }, want: "configuration is incompatible"}, + {name: "missing authority", mutate: func(m map[string]map[string]any) { delete(m, authority) }, want: "bootstrap is forbidden"}, + {name: "empty authority", mutate: func(m map[string]map[string]any) { m[authority]["data"] = map[string]any{} }, want: "bootstrap is forbidden"}, + {name: "missing state", mutate: func(m map[string]map[string]any) { m[authority]["data"] = map[string]any{"other": "value"} }, want: "state.json"}, + {name: "missing capacity", mutate: func(m map[string]map[string]any) { delete(m, capacity) }, want: "bootstrap is forbidden"}, + {name: "missing deny", mutate: func(m map[string]map[string]any) { + delete(m, "NetworkPolicy/test-mecatl-execution-workload-default-deny") + }, want: "existing workload NetworkPolicies"}, + {name: "foreign release", mutate: func(m map[string]map[string]any) { + u := &unstructured.Unstructured{Object: m[capacity]} + a := u.GetAnnotations() + a["meta.helm.sh/release-name"] = "foreign" + u.SetAnnotations(a) + }, want: "foreign or ambiguous"}, + {name: "foreign namespace", mutate: func(m map[string]map[string]any) { + u := &unstructured.Unstructured{Object: m[authority]} + a := u.GetAnnotations() + a["meta.helm.sh/release-namespace"] = "foreign" + u.SetAnnotations(a) + }, want: "foreign or ambiguous"}, + {name: "unmanaged", mutate: func(m map[string]map[string]any) { + u := &unstructured.Unstructured{Object: m[authority]} + u.SetLabels(nil) + }, want: "foreign or ambiguous"}, + {name: "removed profile policy", args: []string{"--set-json", "networkPolicy.workloadProfiles={}"}, want: "configuration is incompatible"}, + {name: "changed egress", args: []string{"--set", "networkPolicy.workloadProfiles.go.egress[0].cidr=10.3.0.0/16"}, want: "configuration is incompatible"}, + {name: "changed profile", args: []string{"--set", "profiles.go.maxEnvironments=30"}, want: "configuration is incompatible"}, + {name: "changed fullname", args: []string{"--set", "fullnameOverride=other"}, want: "bootstrap is forbidden"}, + {name: "missing profiles with orphan policies", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, "ConfigMap/test-mecatl-execution-profiles") + delete(m, capacity) + }, want: "original release profiles"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := json.Marshal(retained) + if err != nil { + t.Fatal(err) + } + var objects map[string]map[string]any + if err := json.Unmarshal(data, &objects); err != nil { + t.Fatal(err) + } + if tt.mutate != nil { + tt.mutate(objects) + } + _, err = renderLifetime(t, objects, tt.args...) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("render error=%v, want %q", err, tt.want) + } + }) + } +} + +func renderLifetime(t *testing.T, objects map[string]map[string]any, extra ...string) (map[string]map[string]any, error) { + t.Helper() + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is required for real chart rendering") + } + resources := map[string][]map[string]any{} + for gv, kinds := range map[string][]string{ + "v1": {"ConfigMap", "Pod", "PersistentVolumeClaim", "ResourceQuota", "LimitRange", "ServiceAccount", "Service"}, + "execution.mecatl.dev/v1alpha1": {"ExecutionEnvironment"}, + "networking.k8s.io/v1": {"NetworkPolicy"}, + "apps/v1": {"Deployment"}, + "policy/v1": {"PodDisruptionBudget"}, + "rbac.authorization.k8s.io/v1": {"Role", "RoleBinding", "ClusterRole", "ClusterRoleBinding"}, + } { + for _, kind := range kinds { + plural := strings.ToLower(kind) + "s" + if kind == "NetworkPolicy" { + plural = "networkpolicies" + } + resources[gv] = append(resources[gv], map[string]any{"name": plural, "kind": kind, "namespaced": !strings.HasPrefix(kind, "Cluster")}) + } + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var response any + switch r.URL.Path { + case "/version": + response = map[string]any{"major": "1", "minor": "35", "gitVersion": "v1.35.0"} + case "/api": + response = map[string]any{"kind": "APIVersions", "apiVersion": "v1", "versions": []string{"v1"}} + case "/apis": + groups := []any{} + for _, group := range []string{"execution.mecatl.dev", "networking.k8s.io", "apps", "policy", "rbac.authorization.k8s.io"} { + version := "v1" + if group == "execution.mecatl.dev" { + version = "v1alpha1" + } + gv := map[string]any{"groupVersion": group + "/" + version, "version": version} + groups = append(groups, map[string]any{"name": group, "versions": []any{gv}, "preferredVersion": gv}) + } + response = map[string]any{"kind": "APIGroupList", "apiVersion": "v1", "groups": groups} + default: + for gv, rs := range resources { + prefix := "/apis/" + gv + if gv == "v1" { + prefix = "/api/v1" + } + if r.URL.Path == prefix { + response = map[string]any{"kind": "APIResourceList", "apiVersion": "v1", "groupVersion": gv, "resources": rs} + break + } + for _, resource := range rs { + base := prefix + "/namespaces/ns/" + resource["name"].(string) + if r.URL.Path == base { + items := []any{} + for _, obj := range objects { + if obj["kind"] == resource["kind"] { + items = append(items, obj) + } + } + response = map[string]any{"kind": resource["kind"].(string) + "List", "apiVersion": gv, "items": items} + } else if strings.HasPrefix(r.URL.Path, base+"/") { + if obj := objects[resource["kind"].(string)+"/"+strings.TrimPrefix(r.URL.Path, base+"/")]; obj != nil { + response = obj + } + } + } + } + } + if response == nil { + w.WriteHeader(http.StatusNotFound) + response = map[string]any{"kind": "Status", "apiVersion": "v1", "status": "Failure", "reason": "NotFound", "code": 404} + } + if err := json.NewEncoder(w).Encode(response); err != nil { + t.Error(err) + } + })) + defer server.Close() + // Repo-local scratch: synthetic kubeconfig has no credentials. + if err := os.MkdirAll("../../../.scratch", 0o700); err != nil { + t.Fatal(err) + } + dir, err := os.MkdirTemp("../../../.scratch", "chart-lifetime-") + if err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, "kubeconfig") + body := fmt.Sprintf("apiVersion: v1\nkind: Config\nclusters:\n- name: offline\n cluster:\n server: %s\ncontexts:\n- name: offline\n context:\n cluster: offline\n namespace: ns\ncurrent-context: offline\n", server.URL) + if err := os.WriteFile(config, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + args := []string{"template", "test", ".", "--dry-run=server", "--disable-openapi-validation", "--kubeconfig", config, "--kube-context", "offline", "--namespace", "ns", "-f", "../../mecatl-execution-kind/execution-values.yaml", "--set-string", "provider.image=example.invalid/provider@sha256:" + strings.Repeat("a", 64), "--set", "provider.securitySecretName=synthetic", "--set-literal", "provider.securityManifest={}", "--set-json", `networkPolicy.workloadProfiles={"go":{"egress":[{"cidr":"10.2.0.0/16","ports":[{"port":8080}]}]}}`} + cmd := exec.CommandContext(ctx, "helm", append(args, extra...)...) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + dir, "KUBECONFIG=" + config} + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("helm rendering: %w: %s", err, stderr.String()) + } + result := map[string]map[string]any{} + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(out), 4096) + for { + var obj map[string]any + err := decoder.Decode(&obj) + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if len(obj) == 0 { + continue + } + u := &unstructured.Unstructured{Object: obj} + result[u.GetKind()+"/"+u.GetName()] = obj + } + return result, nil +} diff --git a/deploy/helm/mecatl-execution/templates/_helpers.tpl b/deploy/helm/mecatl-execution/templates/_helpers.tpl index 66568c818a..bee4eefeb5 100644 --- a/deploy/helm/mecatl-execution/templates/_helpers.tpl +++ b/deploy/helm/mecatl-execution/templates/_helpers.tpl @@ -1,3 +1,26 @@ {{- define "mecatl-execution.fullname" -}} {{- default (printf "%s-mecatl-execution" .Release.Name) .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* Retained resources are adoptable only by their original Helm identity. */}} +{{- define "mecatl-execution.retainedMetadata" -}} +{{- $live := lookup .apiVersion .kind .root.Release.Namespace .name -}} +{{- if $live -}} +{{- if or $live.metadata.deletionTimestamp (ne (dig "app.kubernetes.io/managed-by" "" ($live.metadata.labels | default dict)) "Helm") (ne (dig "meta.helm.sh/release-name" "" ($live.metadata.annotations | default dict)) .root.Release.Name) (ne (dig "meta.helm.sh/release-namespace" "" ($live.metadata.annotations | default dict)) .root.Release.Namespace) -}} +{{- fail (printf "retained %s %s has foreign or ambiguous Helm ownership" .kind .name) -}} +{{- end -}} +{{- end -}} +name: {{ .name }} +namespace: {{ .root.Release.Namespace }} +labels: + app.kubernetes.io/managed-by: Helm +annotations: + helm.sh/resource-policy: keep + meta.helm.sh/release-name: {{ .root.Release.Name | quote }} + meta.helm.sh/release-namespace: {{ .root.Release.Namespace | quote }} +{{- end -}} + +{{/* Frozen workload identity prevents orphan allow policies from widening egress. */}} +{{- define "mecatl-execution.lifetimeConfig" -}} +{{- dict "profiles" .Values.profiles "networkPolicy" .Values.networkPolicy "securitySecretName" .Values.provider.securitySecretName | toJson -}} +{{- end -}} diff --git a/deploy/helm/mecatl-execution/templates/lifetime-check.yaml b/deploy/helm/mecatl-execution/templates/lifetime-check.yaml new file mode 100644 index 0000000000..b3d53405e5 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/lifetime-check.yaml @@ -0,0 +1,56 @@ +{{- $root := . -}} +{{- $name := include "mecatl-execution.fullname" . -}} +{{- $provider := lookup "apps/v1" "Deployment" .Release.Namespace $name -}} +{{- if gt (int (dig "spec" "replicas" 0 $provider)) 0 -}} +{{- fail "quiesce the execution provider (scale to zero and wait for its Pods to disappear) before upgrading" -}} +{{- end -}} +{{- $profiles := lookup "v1" "ConfigMap" .Release.Namespace (printf "%s-profiles" $name) -}} +{{- if $profiles -}} +{{- if ne (dig "lifetime.json" "" ($profiles.data | default dict)) (include "mecatl-execution.lifetimeConfig" .) -}} +{{- fail "retained execution profiles/network configuration is incompatible; preserve the original lifetime configuration" -}} +{{- end -}} +{{- end -}} +{{/* A reserved slot can precede CR publication; no CR is not bootstrap proof. */}} +{{- $capacity := lookup "v1" "ConfigMap" .Release.Namespace "mecatl-execution-profile-allocations" -}} +{{- $retained := not (empty $capacity.data) -}} +{{- range (lookup "execution.mecatl.dev/v1alpha1" "ExecutionEnvironment" .Release.Namespace "").items -}} +{{- $retained = true -}} +{{- end -}} +{{- range $kind := list "Pod" "PersistentVolumeClaim" -}} +{{- range (lookup "v1" $kind $root.Release.Namespace "").items -}} +{{- if and (eq $kind "Pod") (eq (dig "app.kubernetes.io/name" "" (.metadata.labels | default dict)) $name) -}} +{{- fail "wait for all execution provider Pods to disappear before upgrading" -}} +{{- end -}} +{{- if hasKey (.metadata.labels | default dict) "execution.mecatl.dev/environment" -}} +{{- $retained = true -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- if $retained -}} +{{- range $cm := list (printf "%s-profiles" $name) (printf "%s-security-manifest" $name) (printf "%s-security-authority" $name) "mecatl-execution-profile-allocations" -}} +{{- $live := lookup "v1" "ConfigMap" $root.Release.Namespace $cm -}} +{{- if not ($live.data | default dict) -}} +{{- fail (printf "retained execution allocations require nonempty %s; authority/capacity bootstrap is forbidden" $cm) -}} +{{- end -}} +{{- end -}} +{{- $authority := lookup "v1" "ConfigMap" .Release.Namespace (printf "%s-security-authority" $name) -}} +{{- if not (dig "state.json" "" ($authority.data | default dict)) -}} +{{- fail "retained execution allocations require security authority state.json; bootstrap is forbidden" -}} +{{- end -}} +{{- $policies := list (printf "%s-workload-default-deny" $name) -}} +{{- range $profile, $_ := .Values.networkPolicy.workloadProfiles -}} +{{- $policies = append $policies (printf "%s-profile-%s" $name (sha256sum $profile | trunc 16)) -}} +{{- end -}} +{{- range $policy := $policies -}} +{{- if not (lookup "networking.k8s.io/v1" "NetworkPolicy" $root.Release.Namespace $policy) -}} +{{- fail "retained execution allocations require their existing workload NetworkPolicies" -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- range (lookup "networking.k8s.io/v1" "NetworkPolicy" .Release.Namespace "").items -}} +{{- if or (hasKey (.spec.podSelector.matchLabels | default dict) "execution.mecatl.dev/profile") (hasSuffix "-workload-default-deny" .metadata.name) -}} +{{- if not $profiles -}} +{{- fail "retained workload policies require the original release profiles ConfigMap" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/mecatl-execution/templates/network-policy.yaml b/deploy/helm/mecatl-execution/templates/network-policy.yaml index 62b3f04578..43f77f268b 100644 --- a/deploy/helm/mecatl-execution/templates/network-policy.yaml +++ b/deploy/helm/mecatl-execution/templates/network-policy.yaml @@ -39,7 +39,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: {{ include "mecatl-execution.fullname" . }}-workload-default-deny + {{- include "mecatl-execution.retainedMetadata" (dict "root" . "apiVersion" "networking.k8s.io/v1" "kind" "NetworkPolicy" "name" (printf "%s-workload-default-deny" (include "mecatl-execution.fullname" .))) | nindent 2 }} spec: podSelector: matchExpressions: @@ -52,7 +52,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: {{ include "mecatl-execution.fullname" $ }}-profile-{{ sha256sum $name | trunc 16 }} + {{- include "mecatl-execution.retainedMetadata" (dict "root" $ "apiVersion" "networking.k8s.io/v1" "kind" "NetworkPolicy" "name" (printf "%s-profile-%s" (include "mecatl-execution.fullname" $) (sha256sum $name | trunc 16))) | nindent 2 }} spec: podSelector: matchLabels: diff --git a/deploy/helm/mecatl-execution/templates/provider.yaml b/deploy/helm/mecatl-execution/templates/provider.yaml index 7d2f0d058b..f17f92ee45 100644 --- a/deploy/helm/mecatl-execution/templates/provider.yaml +++ b/deploy/helm/mecatl-execution/templates/provider.yaml @@ -1,8 +1,9 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "mecatl-execution.fullname" . }}-profiles + {{- include "mecatl-execution.retainedMetadata" (dict "root" . "apiVersion" "v1" "kind" "ConfigMap" "name" (printf "%s-profiles" (include "mecatl-execution.fullname" .))) | nindent 2 }} data: + lifetime.json: {{ include "mecatl-execution.lifetimeConfig" . | quote }} profiles.yaml: | profiles: {{ toYaml .Values.profiles | indent 6 }} @@ -10,23 +11,19 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "mecatl-execution.fullname" . }}-security-manifest + {{- include "mecatl-execution.retainedMetadata" (dict "root" . "apiVersion" "v1" "kind" "ConfigMap" "name" (printf "%s-security-manifest" (include "mecatl-execution.fullname" .))) | nindent 2 }} data: manifest.json: | {{ required "provider.securityManifest is required" .Values.provider.securityManifest | indent 4 }} --- +{{- range $name := list (printf "%s-security-authority" (include "mecatl-execution.fullname" .)) "mecatl-execution-profile-allocations" }} apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "mecatl-execution.fullname" . }}-security-authority -data: {} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: mecatl-execution-profile-allocations -data: {} + {{- include "mecatl-execution.retainedMetadata" (dict "root" $ "apiVersion" "v1" "kind" "ConfigMap" "name" $name) | nindent 2 }} +data: {{ (lookup "v1" "ConfigMap" $.Release.Namespace $name).data | default dict | toJson }} --- +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deploy/mecatl-execution-kind/run.sh b/deploy/mecatl-execution-kind/run.sh index a805005e64..b8215a2f65 100755 --- a/deploy/mecatl-execution-kind/run.sh +++ b/deploy/mecatl-execution-kind/run.sh @@ -247,13 +247,6 @@ profiles: EOF kube apply -f "$root/e2e/k8s_execution/fixture/legacyfixture/crd.yaml" kube wait --for=condition=Established crd/executionenvironments.execution.mecatl.dev --timeout=60s - dev env KUBECONFIG="$kubeconfig" go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/legacyfixture execution-qualification "$legacy_profiles" "$state/legacy-migration.json" - kube -n execution-qualification wait --for=condition=Ready pod/executor-legacy-migration pod/executor-legacy-migration-malformed pod/executor-legacy-migration-insecure --timeout=3m - kube -n execution-qualification exec pod/executor-legacy-migration -- /bin/sh -c 'printf "prototype-data\n" > /workspace/migration-sentinel' - # Helm does not upgrade an existing CRD. Upgrade explicitly only after the - # API server persisted the legacy string references. - kube apply -f "$root/deploy/helm/mecatl-execution/crds/executionenvironment.yaml" - kube wait --for=condition=Established crd/executionenvironments.execution.mecatl.dev --timeout=60s fi kube -n execution-qualification create secret generic execution-security \ @@ -319,6 +312,22 @@ helm_kube "$@" \ --set-string profiles.quota-kube.image="$workload_image" \ --wait --timeout=4m +if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then + # Establish the chart-owned authority before creating retained allocations. + # Seed the prototype under the old CRD only while every provider is quiesced. + kube -n execution-qualification scale deployment/mecatl-execution --replicas=0 + kube -n execution-qualification wait --for=delete pod -l app.kubernetes.io/name=mecatl-execution --timeout=2m + dev env KUBECONFIG="$kubeconfig" go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/legacyfixture execution-qualification "$legacy_profiles" "$state/legacy-migration.json" + kube -n execution-qualification wait --for=condition=Ready pod/executor-legacy-migration pod/executor-legacy-migration-malformed pod/executor-legacy-migration-insecure --timeout=3m + kube -n execution-qualification exec pod/executor-legacy-migration -- /bin/sh -c 'printf "prototype-data\n" > /workspace/migration-sentinel' + # Helm does not upgrade existing CRDs. Preserve the stored legacy references + # until the explicit CRD upgrade; the first production test then migrates them. + kube apply -f "$root/deploy/helm/mecatl-execution/crds/executionenvironment.yaml" + kube wait --for=condition=Established crd/executionenvironments.execution.mecatl.dev --timeout=60s + kube -n execution-qualification scale deployment/mecatl-execution --replicas=2 + kube -n execution-qualification rollout status deployment/mecatl-execution --timeout=4m +fi + kube -n execution-qualification create configmap execution-mock --from-file=mock-script.json="$root/deploy/mecatl-execution-kind/mock-script.json" --dry-run=client -o yaml | kube apply -f - helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" \ --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ diff --git a/docs/adr/0027-cloud-native.md b/docs/adr/0027-cloud-native.md index 37148c7c03..764abe0465 100644 --- a/docs/adr/0027-cloud-native.md +++ b/docs/adr/0027-cloud-native.md @@ -797,9 +797,10 @@ durable artifact survives and is reloaded), or **lost** (gone, possibly leaking) | 80 | Native-execution provider gRPC connection and bounded RPC limiter identity counters | `executionclient.Provider` / execution-provider process | process / active RPC | provider close releases the connection; limiter reservations release at RPC return and idle per-client entries are removed | reconstructible from operator endpoint and mTLS files; limiter counters reset by design | `internal/adapter/executionclient/client.go` (`Client`, `Provider`, `Close`); `internal/adapter/executioncontroller/security_limit.go` (`RPCLimiter`) | | 81 | Native-execution active run and operation renewal handles | `executionclient.Provider` / `executioncontroller.Store` | run / operation | context cancellation joins renewal, while terminal receipts or explicit release clear only the matching durable holder; uncertain expiry is retained and fenced | persisted in `ExecutionEnvironment.status.activeRun` and `status.activeOperation`; restart observes rather than adopts an unknown holder | `internal/adapter/executionclient/client.go` (`Provider`); `internal/adapter/executioncontroller/store.go` (`Store`) | | 82 | Native-execution controller informers, queue, and worker | execution-provider `Reconciler` | process | provider context stops informers and worker; `Run` shuts down the queue | reconstructible from Kubernetes objects after profile-resource preflight and cache synchronization | `internal/adapter/executioncontroller/controller.go` (`Reconciler`, `Initialize`, `Run`) | -| 83 | Security authority immutable snapshot, reload ticker, and durable high-water ConfigMap | execution-provider `SecurityManager` | process snapshot / cluster-durable authority | context stops reload; candidates publish atomically, invalid candidates clear readiness, and no secret bytes enter the ConfigMap | snapshot reconstructible from projected files; generation, canonical digest, and public fingerprints persist in the authority ConfigMap | `internal/adapter/executioncontroller/security.go` (`SecurityManager`, `Reload`, `Run`) | -| 84 | Profile-allocation slot CAS ledger | execution-provider `Store` | namespace, per profile | exact final CR deletion releases a slot; conflicts retry with resource-version CAS and invalid state fails closed | persisted in the `mecatl-execution-profile-allocations` ConfigMap and reconciled against retained CRs | `internal/adapter/executioncontroller/store.go` (`reserveProfileSlot`, `releaseProfileSlot`) | +| 83 | Security authority immutable snapshot, reload ticker, and durable high-water ConfigMap | execution-provider `SecurityManager` | process snapshot / cluster-durable authority | context stops reload; candidates publish atomically, invalid candidates clear readiness, and no secret bytes enter the ConfigMap | snapshot reconstructible from projected files; generation, canonical digest, and public fingerprints persist in the authority ConfigMap, retained across default Helm uninstall and reused byte-for-byte by a quiesced same-identity reinstall | `internal/adapter/executioncontroller/security.go` (`SecurityManager`, `Reload`, `Run`); `deploy/helm/mecatl-execution/templates/provider.yaml` | +| 84 | Profile-allocation slot CAS ledger | execution-provider `Store` | namespace, per profile | exact final CR deletion releases a slot; conflicts retry with resource-version CAS and invalid state fails closed | persisted in the `mecatl-execution-profile-allocations` ConfigMap and reconciled against retained CRs; default Helm uninstall retains the ledger and same-identity reinstall preserves its data | `internal/adapter/executioncontroller/store.go` (`reserveProfileSlot`, `releaseProfileSlot`); `deploy/helm/mecatl-execution/templates/provider.yaml` | | 85 | Bounded revocation receipt ledger and lifecycle/migration proof state | `ExecutionEnvironment` status | environment | at most 32 revocation receipts are retained; lifecycle and migration records clear only after exact terminal/UID proof and successful phase completion | persisted in CR status; retries use operation ID and fingerprint, while evicted receipt IDs remain rejected | `internal/adapter/executioncontroller/store_revoke.go` (`RevokeEnvironment`); `internal/adapter/executioncontroller/admin_lifecycle.go`; `internal/adapter/executioncontroller/controller_lifecycle.go` | +| 86 | Native-execution workload NetworkPolicies and retained profile/security configuration | Helm release, namespace-scoped | retained allocation lifetime | default uninstall retains default-deny/profile policies and both configuration ConfigMaps; no deletion hook | persisted; `profiles` ConfigMap `lifetime.json` pins profiles, network configuration, and security Secret name. Same-release/namespace ownership checks permit adoption; incompatible configuration or missing ledgers with retained allocations fails closed. Operator-owned Secret material is retained outside Helm | `deploy/helm/mecatl-execution/templates/network-policy.yaml`; `deploy/helm/mecatl-execution/templates/provider.yaml`; `deploy/helm/mecatl-execution/templates/lifetime-check.yaml` | | 58 | Title coordinator (two workers, 64-item queue, dedupe set, and retry timers) | `server.Service` / `app.Build` | process | `Service.Close` cancels and joins workers before session dependencies close; queue admission is non-blocking and bounded | reset-by-design; durable session attempt records make a crash-interrupted claim terminal rather than rebilling | `internal/adapter/server/title_coordinator.go` (`titleCoordinator`) | | 59 | Provider OIDC bearer sources and runtime registry | `internal/cliconfig.NativeEndpointLoader` / `NativeEndpointRuntime` | process, one per configured OIDC provider | `NativeEndpointLoader.Close` closes every loader-owned runtime; each runtime closes its serving Store handles | reconstructible from explicit operator provider configuration and an exact protected record; access tokens reset and are refreshed from that record | `internal/cliconfig/native_endpoint.go` (`NativeEndpointLoader`, `NativeEndpointRuntime`, `Source`, `Close`) | | 60 | Caller OIDC JWKS validator cache | `authn/oidc.Validator` | process | validator `Close` stops its refresh work during command-root shutdown | reset/refetch: a restart fetches current signing keys; this authenticates callers only and is separate from provider issuer/gateway trust | `authn/oidc/oidc.go` (`Validator`, `Close`) | @@ -1202,6 +1203,7 @@ what is persisted), **reset-by-design** (documented, acceptable), | 49 | Native-execution run/operation ownership and renewal | `ExecutionEnvironment.status.activeRun` and `status.activeOperation` | renewal goroutines and handles disappear; holder, claim/run IDs, epoch, generation, expiry, and operation fingerprint remain | **persist-in-CR and fail closed**: expiry becomes `FenceUnknown`; another replica never clears unknown ownership by timeout alone | native execution | | 50 | Security authority and grant revocation | projected security files, authority high-water ConfigMap, and CR `grantGeneration` plus bounded revocation receipts | immutable in-memory snapshot and limiter counts reset; durable generation/digest/fingerprints, environment generation, and receipts remain | **persist authority metadata and derive snapshot**: reload all secret bytes atomically; drift, rollback, expiry, or unavailable ledger keeps readiness false | native execution | | 51 | Native-execution allocation, lifecycle proof, and schema migration | profile-allocation ConfigMap plus `ExecutionEnvironment` spec/status, including runtime UIDs, termination proof, lifecycle operation, and migration operation | controller queues reset; retained CR/PVC and every operation phase remain observable | **persist-in-Kubernetes**: cache sync and exact UID/CAS checks resume convergence; malformed or unknown schema is retained for explicit administration | native execution | +| 52 | Native-execution Helm lifetime configuration, isolation, and ledgers | retained workload NetworkPolicies, profile/security-manifest ConfigMaps, authority high-water ConfigMap, and capacity ConfigMap | provider absence leaves enforcement and data in Kubernetes; Helm release history may disappear but original ownership metadata remains | **persist-in-Kubernetes and explicitly adopt**: same release/namespace and unchanged lifetime configuration only; live lookup renders actual ledger data while writers are quiesced. Missing history with retained allocations, foreign ownership, and incompatible configuration fail closed; no bootstrap reset. CRDs upgrade manually before compatible provider/schema migration | native execution N2; List 1 rows 83, 84, 86 | | 36 | Title coordinator queue/dedupe and in-flight title call | `server.titleCoordinator` | process | `Service.Close` cancels and joins; pending queued work is discarded | reset-by-design: durable incomplete attempts are reloaded as `interrupted` and exhausted, so no uncertain provider call is retried | title coordinator (SHIPPED) | | 37 | Native endpoint access and refresh tokens | protected `mecatl/provider-oidc/v1` record; transient access material in `nativeBearerSource` | access tokens reset; the encrypted exact-identity record reloads refresh state only | persist-in-record / derive: no OAuth material enters a session snapshot, event, diagnostic, or RPC; an unavailable, corrupt, or identity-drifted record is not-enrolled | ADR 0329 | | 38 | Native endpoint issuer/gateway clients, keyring/store handles, and transaction locks | `NativeEndpointRuntime` and `TransactionLocker` | handles and locks are closed/released; a restart has no live runtime state | explicitly reattach/reset: Build or local lifecycle reopens them only from explicit configuration; issuer/gateway trust stays separate and the next lifecycle operation reacquires its lock | ADR 0329 | diff --git a/docs/design/IMPLEMENTATION-NOTES.md b/docs/design/IMPLEMENTATION-NOTES.md index 7dfb4c07a2..5784251218 100644 --- a/docs/design/IMPLEMENTATION-NOTES.md +++ b/docs/design/IMPLEMENTATION-NOTES.md @@ -6551,7 +6551,11 @@ uncertain completion `FenceUnknown`. `internal/adapter/executioncontroller/controller.go` (`Reconciler`) watches the namespaced `ExecutionEnvironment` CR and creates a retained PVC plus a -single executor Pod from an operator-only, digest-pinned profile. The Pod +single executor Pod from an operator-only, digest-pinned profile. Provisioning +errors return alongside any condition-write error so the existing rate-limited +workqueue retries without a finite budget or another Kubernetes event. Repeated +reconciliation never replaces a missing authoritative PVC/Pod or adopts a foreign +resource. The Pod has no service-account token and contains the fixed helper built by `build/execution-workload/Dockerfile`. `internal/executionexecutor/executor.go` (`Executor.Execute`) dispatches bounded file operations and foreground Shell in @@ -6580,6 +6584,16 @@ The controller validates the configured RuntimeClass and StorageClass names before readiness, then synchronizes informer caches without clearing peer operations. Security reload publishes one immutable TLS/client-policy/grant-key snapshot only after matching the durable authority ConfigMap high-water record. +`internal/adapter/executioncontroller/admin_lifecycle.go` (`adminSubject`) shares +creator-scope, exact owner, and revision admission across all six administrative +RPC families, including `internal/adapter/executioncontroller/store_revoke.go`. +The authenticated request carries `ClientPolicy.AdministratorFor` from the same +security snapshot that verified its peer; it does not re-read policy or substitute +the administrator for the immutable creator. Empty scope retains self-administration. +Sorted creator scopes participate in the authority digest; removal denies subsequent +requests and receipt replay on existing connections, without cancelling already +admitted safe lifecycle reconciliation. Data-plane and owner-attestation checks +are unchanged. Unproven executor loss remains `FenceUnknown`; only exact built-in terminal proof or an external platform fencing procedure can recover it. Remote project sources, background commands, schedules, and delegated filesystem execution diff --git a/docs/design/PRODUCTION-READINESS.md b/docs/design/PRODUCTION-READINESS.md index 681d8d3eb7..23c2614071 100644 --- a/docs/design/PRODUCTION-READINESS.md +++ b/docs/design/PRODUCTION-READINESS.md @@ -14,6 +14,7 @@ record; current behaviour is in the linked [architecture](../architecture.md) do | Subsystem | Status | Design record | Architecture | |---|---|---|---| +| Native Kubernetes execution provider | 🔨 Human-authorized draft development in existing [PR #1614](https://github.com/stacklok/mecatl/pull/1614), not shipped or approved. N2 candidate retains workload isolation, authority/capacity ledgers, and configuration through uninstall; ownership-checked lookup preserves data for same-release/namespace adoption and rejects incompatible profiles or missing history. Offline real-Helm rendering coverage and a production Kind install/upgrade/uninstall/reinstall test are authored. ⛔ Final deterministic CI, enforcing-CNI lifecycle execution, native real-provider coding smoke, API/schema/security reviews, and final acceptance evidence remain pending. Generic live Kubernetes proof is not native execution qualification. | [0349](../adr/0349-native-kubernetes-execution.md) · [acceptance contract](../acceptance/native-kubernetes-execution.md) | [native execution mechanics](./IMPLEMENTATION-NOTES.md#native-kubernetes-execution-provider) | | Multi-provider / multi-model | ✅ P0+P1+live listing & metadata · ✅ same-provider history carryover (issue #20) · ✅ experimental manual `openai-codex` subscription token (ADR 0215) · ✅ ToolHive OpenAI Responses and native Anthropic protocol providers over one gateway identity (ADR 0334) · ✅ internal opaque credential-store substrate consumed by MCP profiles with local encrypted-file Store and explicit read-only environment Reader (issues #519/#542) · ⛔ disk cache (P2) · ⛔ key acquisition/per-client routing/remote stores or Kubernetes Secret `resourceVersion` CAS backend (P3) | [MULTI-PROVIDER.md](../adr/0016-multi-provider.md) · [0215](../adr/0215-openai-subscription-manual-token.md) · [0218](../adr/0218-credential-store.md) · [0221](../adr/0221-read-only-credential-source.md) · [0334](../adr/0334-toolhive-protocol-specific-providers.md) | [providers](../architecture/providers.md) · [credential store](../architecture.md#internal-credential-store) | | Remote mecatui OIDC login, refresh, recovery, and logout | ✅ fixed-callback PKCE enrollment, actual-record-only root-scoped keyring migration, one refresh/enroll/logout target transaction with ambiguous-commit compensation, target-bound encrypted credentials, token-demand-gated proactive refresh, exact structured-rejection cleanup, ownership-checked same-target recovery, CAS-safe local-first logout, one-budget best-effort RFC 7009 revocation, and target-aware TLS defaults with saved-auth verified-TLS enforcement shipped; ✅ offline rotation/rejection/crash-residual/logout/TLS-policy coverage · ⚠️ no cross-store journal: a crash may require login or leave an unenumerable credential-only orphan · ⚠️ legacy zero-padded-port credential identities require one login · ⛔ live Kind qualification remains environment-dependent | [0277](../adr/0277-remote-mecatui-oidc.md) · [0287](../adr/0287-target-aware-mecatui-tls.md) · [0218](../adr/0218-credential-store.md) | [remote login](../tui.md#oidc-connected-server) | | Direct MCP OAuth dynamic client registration | 🔨 Core trusted-profile validation, durable CAS registration, generation-bound no-refresh grant, explicit reset/retry recovery, restart reuse, expiry-to-login-required, protocol/security fixtures, seven named acceptance proofs, and cloud-native resource re-audit complete · ⛔ repeat panel review, implementation PR/merge, and live local-mecatui qualification remain · ⛔ complete resource-bound public-client refresh ([issue #1355](https://github.com/stacklok/mecatl/issues/1355)) | [0325](../adr/0325-direct-mcp-dcr.md) | [credential store](../architecture.md#internal-credential-store) | diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go index 59abfffb7c..89cd787267 100644 --- a/e2e/k8s_execution/production_qualification_test.go +++ b/e2e/k8s_execution/production_qualification_test.go @@ -969,6 +969,7 @@ func TestKindExecutionProductionQuotaSaturation(t *testing.T) { t.Fatal("PVC quota admitted more than one additional workspace") } runKubectl(t, ctx, kubeconfig, "patch", "resourcequota/mecatl-execution", "-n", namespace, "--type=merge", "-p", `{"spec":{"hard":{"persistentvolumeclaims":"100"}}}`) + waitExecutionStatus(t, ctx, kubeconfig, blocked.out.Environment.ID, func(s executionStatus) bool { return s.Ready }) all := append(allocations, *accepted) for _, allocation := range all { diff --git a/e2e/k8s_execution/zz_production_helm_lifetime_test.go b/e2e/k8s_execution/zz_production_helm_lifetime_test.go new file mode 100644 index 0000000000..63dc51e4f5 --- /dev/null +++ b/e2e/k8s_execution/zz_production_helm_lifetime_test.go @@ -0,0 +1,379 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// Last in the serial production suite: rotation and migration share this +// throwaway release. Always restore its current authority, never the initial one. +func TestKindExecutionProductionHelmLifetime(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + requireOwnedHelmFixture(t, state, kubeconfig) + client, _ := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, client, "helm-lifetime") + rc, release := acquireRun(t, ctx, client, owner, binding, attached, "helm-lifetime-write") + probe := []byte("package main\nimport (\"net\";\"os\";\"time\")\nfunc main(){c,e:=net.DialTimeout(\"tcp\",os.Args[1],2*time.Second);if e!=nil{os.Exit(42)};c.Close()}\n") + for path, data := range map[string][]byte{"helm-sentinel": []byte("retained-helm-data\n"), "helm-probe.go": probe} { + if _, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: path, Data: data}); err != nil { + t.Fatal("create lifetime workspace data:", remoteErrorCode(err)) + } + } + built, err := client.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go build -o helm-probe helm-probe.go", TimeoutMillis: 30000}) + if err != nil || built.State != executionenv.CommandSucceeded || built.Result.ExitCode != 0 { + t.Fatal("build lifetime probe failed") + } + release() + // Occupy the single-slot profile; its reservation must still constrain Ensure + // after adoption, rather than only matching a ConfigMap annotation. + quotaBinding := fmt.Sprintf("helm-quota-%d", time.Now().UnixNano()) + quota, err := client.Ensure(ctx, quotaBinding, "quota-cas", owner, "ensure-"+quotaBinding) + if err != nil { + t.Fatal("reserve lifetime capacity:", remoteErrorCode(err)) + } + if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: quota.Environment, Owner: owner, BindingID: quotaBinding, OperationID: "ensure-" + quotaBinding}); err != nil { + t.Fatal("commit lifetime capacity:", remoteErrorCode(err)) + } + waitReady(t, ctx, client, owner, quotaBinding, quota.Environment) + + before := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + envUID := kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") + pod := kubeValue(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[0].metadata.name}") + if before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") { + t.Fatal("runtime status does not name the actual PVC/Pod UIDs") + } + fixtureIP := kubeValue(t, ctx, kubeconfig, "get", "pod/network-fixture", "-n", namespace, "-o", "jsonpath={.status.podIP}") + peerIP := kubeValue(t, ctx, kubeconfig, "get", "pod/network-intruder", "-n", namespace, "-o", "jsonpath={.status.podIP}") + + // Only nonsecret chart values and manifests are captured. No Helm manifest or + // Secret dump is written to artifacts; command errors report status only. + raw, err := helmLifetime(ctx, kubeconfig, "get", "values", "mecatl-execution", "-o", "json") + if err != nil { + t.Fatal("read owned release values:", err) + } + var values map[string]any + if err := json.Unmarshal(raw, &values); err != nil { + t.Fatal("decode release values") + } + current := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-manifest") + oldManifest := current.Data["manifest.json"] + var manifest map[string]any + if err := json.Unmarshal([]byte(oldManifest), &manifest); err != nil { + t.Fatal("decode nonsecret authority manifest") + } + generation, ok := manifest["generation"].(float64) + if !ok || generation < 1 { + t.Fatal("missing authority generation") + } + manifest["generation"] = generation + 1 + newManifest, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + provider, ok := values["provider"].(map[string]any) + if !ok { + t.Fatal("missing provider values") + } + provider["securityManifest"] = string(newManifest) + dir, err := os.MkdirTemp(state, "helm-lifetime-") + if err != nil { + t.Fatal(err) + } + valuesPath := filepath.Join(dir, "values.json") + raw, err = json.Marshal(values) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(valuesPath, raw, 0o600); err != nil { + t.Fatal(err) + } + chart := filepath.Join(repoRoot(t), "deploy/helm/mecatl-execution") + restore := func(ctx context.Context) error { + if err := quiesceLifetimeProvider(ctx, kubeconfig); err != nil { + return err + } + _, err := helmLifetime(ctx, kubeconfig, "upgrade", "--install", "mecatl-execution", chart, "-f", valuesPath, "--wait", "--timeout=4m") + return err + } + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 7*time.Minute) + defer cleanupCancel() + requireOwnedHelmFixture(t, state, kubeconfig) + if err := restore(cleanupCtx); err != nil { + t.Error("restore owned release/current authority failed:", err) + } + }) + capacityBefore := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-profile-allocations") + authorityBefore := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-authority") + var priorHistory struct { + Fingerprints map[string]string `json:"fingerprints"` + } + if err := json.Unmarshal([]byte(authorityBefore.Data["state.json"]), &priorHistory); err != nil || len(priorHistory.Fingerprints) == 0 { + t.Fatal("missing pre-upgrade authority history") + } + // Quiesce all writers before lookup snapshots are rendered into the upgrade. + if err := restore(ctx); err != nil { + t.Fatal("compatible Helm upgrade failed:", err) + } + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + ledgers := map[string]corev1.ConfigMap{} + for _, name := range []string{"mecatl-execution-security-authority", "mecatl-execution-profile-allocations", "mecatl-execution-profiles", "mecatl-execution-security-manifest"} { + ledgers[name] = lifetimeConfigMap(ctx, t, kubeconfig, name) + } + var highWater struct { + Generation uint64 `json:"generation"` + Fingerprints map[string]string `json:"fingerprints"` + } + if err := json.Unmarshal([]byte(ledgers["mecatl-execution-security-authority"].Data["state.json"]), &highWater); err != nil || highWater.Generation != uint64(generation+1) || !reflect.DeepEqual(highWater.Fingerprints, priorHistory.Fingerprints) { + t.Fatal("upgrade did not preserve key history and publish higher authority") + } + if capacityAfter := ledgers[capacityBefore.Name]; capacityAfter.UID != capacityBefore.UID || !reflect.DeepEqual(capacityAfter.Data, capacityBefore.Data) { + t.Fatal("compatible upgrade changed capacity reservations") + } + policies := lifetimePolicies(ctx, t, kubeconfig) + if len(policies) < 2 { + t.Fatal("workload policies missing before uninstall") + } + if _, err := helmLifetime(ctx, kubeconfig, "uninstall", "mecatl-execution", "--wait", "--timeout=2m"); err != nil { + t.Fatal("uninstall owned release failed:", err) + } + waitResourceAbsent(t, ctx, kubeconfig, "deployment", "mecatl-execution") + assertLifetimeRetained(ctx, t, kubeconfig, ledgers, policies) + after := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + if after.PVCUID != before.PVCUID || after.PodUID != before.PodUID || before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || envUID != kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") { + t.Fatal("uninstall replaced retained runtime identity") + } + // Bypass the absent provider only to observe the *same executor*. Positive + // reachability prevents an all-network-down result masquerading as isolation. + lifetimeProbe(ctx, t, kubeconfig, pod, fixtureIP+":8080", 0) + lifetimeProbe(ctx, t, kubeconfig, pod, peerIP+":8080", 42) + lifetimeProbe(ctx, t, kubeconfig, pod, "10.96.0.1:443", 42) + lifetimeProbe(ctx, t, kubeconfig, pod, "1.1.1.1:443", 42) + data := runKubectl(t, ctx, kubeconfig, "exec", "-n", namespace, pod, "--", "cat", "/workspace/helm-sentinel") + if string(data) != "retained-helm-data\n" { + t.Fatal("retained workspace data changed during provider absence") + } + // An incompatible profile is refused before a new release can mutate anything. + if _, err := helmLifetime(ctx, kubeconfig, "install", "mecatl-execution", chart, "-f", valuesPath, "--set", "profiles.go.maxEnvironments=30", "--dry-run=server"); err == nil { + t.Fatal("incompatible reinstall was accepted") + } + // Refusal must happen without adopting another release or bootstrapping a + // new authority name over this namespace's retained allocations. + for _, args := range [][]string{ + {"install", "foreign-execution", chart, "-f", valuesPath, "--dry-run=server"}, + {"install", "mecatl-execution", chart, "-f", valuesPath, "--set", "fullnameOverride=missing-history", "--dry-run=server"}, + } { + if _, err := helmLifetime(ctx, kubeconfig, args...); err == nil { + t.Fatal("foreign/missing-history reinstall was accepted") + } + } + assertLifetimeRetained(ctx, t, kubeconfig, ledgers, policies) + if _, err := helmLifetime(ctx, kubeconfig, "install", "mecatl-execution", chart, "-f", valuesPath, "--wait", "--timeout=4m"); err != nil { + t.Fatal("same-identity Helm reinstall/adoption failed:", err) + } + assertLifetimeRetained(ctx, t, kubeconfig, ledgers, policies) + reattachedClient, _ := productionClient(t, ctx, state, kubeconfig) + reattached := waitReady(t, ctx, reattachedClient, owner, binding, attached.Environment) + if reattached.Environment != attached.Environment { + t.Fatal("reattach changed exact environment revision") + } + after = readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + if after.PVCUID != before.PVCUID || after.PodUID != before.PodUID || before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || envUID != kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") { + t.Fatal("adoption replaced retained runtime identity") + } + rc, release = acquireRun(t, ctx, reattachedClient, owner, binding, reattached, "helm-lifetime-read") + waitFileContent(t, ctx, reattachedClient, rc, "helm-sentinel", "retained-helm-data\n", "same-release Helm adoption") + release() + if _, err := reattachedClient.Ensure(ctx, quotaBinding+"-excess", "quota-cas", owner, "ensure-"+quotaBinding+"-excess"); !isRemoteCode(err, executionenv.CodeResourceExhausted) { + t.Fatal("retained capacity did not reject excess allocation:", remoteErrorCode(err)) + } + + // The old manifest still has valid key material, windows, and clients. Only + // its generation is older. An existing connection must fail after reload. + patch, err := json.Marshal(map[string]any{"data": map[string]string{"manifest.json": oldManifest}}) + if err != nil { + t.Fatal(err) + } + runKubectl(t, ctx, kubeconfig, "patch", "configmap/mecatl-execution-security-manifest", "-n", namespace, "--type=merge", "-p", string(patch)) + waitProviderReadyReplicas(t, ctx, kubeconfig, 0) + if _, err := reattachedClient.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); !isRemoteCode(err, executionenv.CodeNotReady) { + t.Fatal("older still-valid authority did not fail closed with not_ready after reinstall:", remoteErrorCode(err)) + } + authority := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-authority") + if !reflect.DeepEqual(authority.Data, ledgers[authority.Name].Data) { + t.Fatal("stale candidate reset authority history") + } + if err := restore(ctx); err != nil { + t.Fatal("restore current authority:", err) + } + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + restoredClient, _ := productionClient(t, ctx, state, kubeconfig) + waitReady(t, ctx, restoredClient, owner, binding, attached.Environment) + // Retain the test's data by default, as the chart does; no forced cleanup or + // finalizer removal. The explicit test-cluster owner controls final disposal. +} + +func requireOwnedHelmFixture(t *testing.T, state, kubeconfig string) { + t.Helper() + ownership, err := os.ReadFile(filepath.Join(state, "ownership")) + if err != nil { + t.Fatal("lifetime test requires fixture ownership record") + } + fields := map[string]string{} + for line := range strings.SplitSeq(string(ownership), "\n") { + key, value, ok := strings.Cut(line, "=") + if ok { + fields[key] = value + } + } + cluster := fields["cluster"] + if !strings.HasPrefix(cluster, "mecatl-execution-qual-") || fields["context"] != "kind-"+cluster || fields["context"] != os.Getenv("MECATL_KUBE_CONTEXT") || fields["kubeconfig"] != kubeconfig || fields["namespace"] != namespace || fields["profile"] != "production" { + t.Fatal("lifetime test refuses non-owned cluster/context/namespace") + } +} + +func lifetimeConfigMap(ctx context.Context, t *testing.T, kubeconfig, name string) corev1.ConfigMap { + t.Helper() + var cm corev1.ConfigMap + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "configmap", name, "-n", namespace, "-o", "json"), &cm); err != nil { + t.Fatal("decode nonsecret lifetime ConfigMap") + } + if cm.UID == "" || cm.Labels["app.kubernetes.io/managed-by"] != "Helm" || cm.Annotations["meta.helm.sh/release-name"] != "mecatl-execution" || cm.Annotations["meta.helm.sh/release-namespace"] != namespace { + t.Fatal("refuse foreign lifetime ConfigMap", name) + } + return cm +} + +func lifetimePolicies(ctx context.Context, t *testing.T, kubeconfig string) map[string]networkingv1.NetworkPolicy { + t.Helper() + var list networkingv1.NetworkPolicyList + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "networkpolicy", "-n", namespace, "-o", "json"), &list); err != nil { + t.Fatal("decode lifetime policies") + } + result := map[string]networkingv1.NetworkPolicy{} + for _, p := range list.Items { + if p.Name == "mecatl-execution-workload-default-deny" || strings.HasPrefix(p.Name, "mecatl-execution-profile-") { + result[p.Name] = p + } + } + return result +} + +func assertLifetimeRetained(ctx context.Context, t *testing.T, kubeconfig string, cms map[string]corev1.ConfigMap, policies map[string]networkingv1.NetworkPolicy) { + t.Helper() + for name, before := range cms { + after := lifetimeConfigMap(ctx, t, kubeconfig, name) + if before.UID != after.UID || !reflect.DeepEqual(before.Data, after.Data) { + t.Fatal("retained ConfigMap identity/data changed", name) + } + } + live := lifetimePolicies(ctx, t, kubeconfig) + if len(live) != len(policies) { + t.Fatal("retained policy set changed") + } + for name, before := range policies { + after := live[name] + if before.UID != after.UID || !reflect.DeepEqual(before.Spec, after.Spec) { + t.Fatal("retained policy identity/confinement changed", name) + } + } +} + +func lifetimeProbe(ctx context.Context, t *testing.T, kubeconfig, pod, endpoint string, want int) { + t.Helper() + cmd := command(ctx, kubeconfig, "exec", "-n", namespace, pod, "--", "/workspace/helm-probe", endpoint) + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + err := cmd.Run() + if want == 0 && err == nil { + return + } + var exit *exec.ExitError + if want != 0 && errors.As(err, &exit) && exit.ExitCode() == want { + return + } + t.Fatalf("retained executor network probe did not produce exit %d", want) +} + +func quiesceLifetimeProvider(ctx context.Context, kubeconfig string) error { + cmd := command(ctx, kubeconfig, "get", "deployment/mecatl-execution", "-n", namespace, "--ignore-not-found", "-o", "json") + var out lifetimeOutput + cmd.Stdout = &out + cmd.Stderr = io.Discard + if err := cmd.Run(); err != nil { + return fmt.Errorf("read owned provider deployment: %w", err) + } + if out.Len() == 0 { + return nil + } + var deployment struct { + Metadata struct{ Labels, Annotations map[string]string } + } + if err := json.Unmarshal(out.Bytes(), &deployment); err != nil { + return errors.New("decode owned provider deployment") + } + if deployment.Metadata.Labels["app.kubernetes.io/managed-by"] != "Helm" || deployment.Metadata.Annotations["meta.helm.sh/release-name"] != "mecatl-execution" || deployment.Metadata.Annotations["meta.helm.sh/release-namespace"] != namespace { + return errors.New("refuse to quiesce a foreign provider deployment") + } + if err := command(ctx, kubeconfig, "scale", "deployment/mecatl-execution", "-n", namespace, "--replicas=0").Run(); err != nil { + return fmt.Errorf("quiesce owned provider: %w", err) + } + if err := command(ctx, kubeconfig, "wait", "--for=delete", "pod", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "--timeout=2m").Run(); err != nil { + return fmt.Errorf("wait for provider quiescence: %w", err) + } + return nil +} + +func helmLifetime(ctx context.Context, kubeconfig string, args ...string) ([]byte, error) { + full := append([]string{"--kubeconfig", kubeconfig, "--kube-context", os.Getenv("MECATL_KUBE_CONTEXT"), "--namespace", namespace}, args...) + cmd := exec.CommandContext(ctx, "helm", full...) + cmd.Env = cleanEnv() + var out lifetimeOutput + cmd.Stdout = &out + cmd.Stderr = io.Discard + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("Helm %s failed: %w (output suppressed)", args[0], err) + } + return out.Bytes(), nil +} + +func TestLifetimeOutputRejectsOversizedArtifacts(t *testing.T) { + var out lifetimeOutput + if _, err := io.Copy(&out, strings.NewReader(strings.Repeat("x", 1<<20))); err != nil || out.Len() != 1<<20 { + t.Fatal("bounded output did not accept its exact limit") + } + if _, err := io.Copy(&out, strings.NewReader("overflow")); err == nil || out.Len() != 1<<20 { + t.Fatal("output exceeded its bound through io.Copy") + } +} + +type lifetimeOutput struct{ buffer bytes.Buffer } + +func (b *lifetimeOutput) Len() int { return b.buffer.Len() } +func (b *lifetimeOutput) Bytes() []byte { return b.buffer.Bytes() } + +func (b *lifetimeOutput) Write(p []byte) (int, error) { + if b.Len()+len(p) > 1<<20 { + return 0, errors.New("lifetime output exceeds 1 MiB") + } + return b.buffer.Write(p) +} diff --git a/internal/adapter/executioncontroller/admin_lifecycle.go b/internal/adapter/executioncontroller/admin_lifecycle.go index ec0af39ce2..5ed26a8621 100644 --- a/internal/adapter/executioncontroller/admin_lifecycle.go +++ b/internal/adapter/executioncontroller/admin_lifecycle.go @@ -14,14 +14,15 @@ import ( ) type adminLifecycleRequest struct { - Environment executionenv.EnvironmentRef - OwnerHash string - Client string - ExpectedEpoch uint64 - ExpectedPodUID string - ExpectedPVCUID string - OperationID string - ExpectedSchema int64 + Environment executionenv.EnvironmentRef + OwnerHash string + Client string + AdministratorFor []string + ExpectedEpoch uint64 + ExpectedPodUID string + ExpectedPVCUID string + OperationID string + ExpectedSchema int64 } // ReplaceExecutor starts an exact, crash-recoverable executor replacement. @@ -41,7 +42,7 @@ func (s *Store) startLifecycle(ctx context.Context, q adminLifecycleRequest, kin if err := s.persistObservedTerminalProof(ctx, q); err != nil { return err } - return s.retryUpdateStatus(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + return s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { if kind == "ReplaceExecutor" && replacementReceiptMatches(o, q) { return nil } @@ -102,9 +103,18 @@ func exactTerminationProofOperationMatches(o *unstructured.Unstructured, q admin func (s *Store) persistObservedTerminalProof(ctx context.Context, q adminLifecycleRequest) error { //nolint:gocyclo // Exact terminal proof deliberately fails closed at every observable mismatch. o, err := s.resources.Get(ctx, q.Environment.ID, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } if err != nil { return err } + if err := adminSubject(o, q); err != nil { + return err + } + if err := requireCurrentSchema(o); err != nil { + return err + } if replacementReceiptMatches(o, q) { return nil } @@ -126,7 +136,7 @@ func (s *Store) persistObservedTerminalProof(ctx context.Context, q adminLifecyc if err != nil || string(pvc.UID) != q.ExpectedPVCUID { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "exact workspace identity is unavailable"} } - return s.retryUpdateStatus(ctx, q.Environment.ID, func(current *unstructured.Unstructured) error { + return s.retryAdminStatus(ctx, q, func(current *unstructured.Unstructured) error { if err := exactAdminSubject(current, q); err != nil { return err } @@ -140,10 +150,36 @@ func (s *Store) persistObservedTerminalProof(ctx context.Context, q adminLifecyc }) } -func exactAdminSubject(o *unstructured.Unstructured, q adminLifecycleRequest) error { - if textNested(o.Object, "spec", "ownerHash") != q.OwnerHash || textNested(o.Object, "spec", "clientHash") != hashText(q.Client) || textNested(o.Object, "spec", "revision") != q.Environment.Revision { +// adminSubject uses only the policy captured by authentication for this request. +// The actor remains distinct from the immutable creator, including on receipt replay. +func adminSubject(o *unstructured.Unstructured, q adminLifecycleRequest) error { + creator := textNested(o.Object, "spec", "clientHash") + allowed := creator == hashText(q.Client) + for _, uri := range q.AdministratorFor { + allowed = allowed || creator == hashText(uri) + } + if !allowed || textNested(o.Object, "spec", "ownerHash") != q.OwnerHash || textNested(o.Object, "spec", "revision") != q.Environment.Revision { return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} } + return nil +} + +func (s *Store) retryAdminStatus(ctx context.Context, q adminLifecycleRequest, mutate func(*unstructured.Unstructured) error) error { + return s.retryUpdateStatusRaw(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := adminSubject(o, q); err != nil { + return err + } + if err := requireCurrentSchema(o); err != nil { + return err + } + return mutate(o) + }) +} + +func exactAdminSubject(o *unstructured.Unstructured, q adminLifecycleRequest) error { + if err := adminSubject(o, q); err != nil { + return err + } if !epochMatches(o, q.ExpectedEpoch) || textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment execution identity changed"} } @@ -156,7 +192,7 @@ func (s *Store) RecoverEnvironment(ctx context.Context, q adminLifecycleRequest) return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "built-in recovery verifier is unavailable"} } podName, pvcName := "", "" - if err := s.retryUpdateStatus(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { if err := exactAdminSubject(o, q); err != nil { return err } @@ -182,7 +218,7 @@ func (s *Store) RecoverEnvironment(ctx context.Context, q adminLifecycleRequest) if err != nil || string(pvc.UID) != q.ExpectedPVCUID { return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "workspace identity cannot be verified"} } - return s.retryUpdateStatus(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + return s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { if err := exactAdminSubject(o, q); err != nil { return err } @@ -254,10 +290,7 @@ func (s *Store) DeleteRetiredEnvironment(ctx context.Context, q adminLifecycleRe if q.ExpectedPVCUID == "" || q.OperationID == "" { return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "exact retained workspace identity is required"} } - return s.retryUpdateStatus(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { - if textNested(o.Object, "spec", "ownerHash") != q.OwnerHash || textNested(o.Object, "spec", "clientHash") != hashText(q.Client) || textNested(o.Object, "spec", "revision") != q.Environment.Revision { - return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} - } + return s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { if textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "retained workspace identity changed"} } @@ -281,8 +314,11 @@ func (s *Store) MigrateEnvironment(ctx context.Context, q adminLifecycleRequest) return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "recognized prototype schema and exact runtime identities are required"} } if err := s.retryUpdateStatusRaw(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { - if textNested(o.Object, "spec", "ownerHash") != q.OwnerHash || textNested(o.Object, "spec", "clientHash") != hashText(q.Client) || textNested(o.Object, "spec", "revision") != q.Environment.Revision { - return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + if err := adminSubject(o, q); err != nil { + return err + } + if textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype runtime identity mismatch"} } if intNested(o.Object, "spec", "schemaVersion") == currentSchemaVersion && intNested(o.Object, "status", "schemaVersion") == currentSchemaVersion && textNested(o.Object, "status", "lastMigrationOperationID") == q.OperationID { return nil @@ -296,9 +332,6 @@ func (s *Store) MigrateEnvironment(ctx context.Context, q adminLifecycleRequest) if intNested(o.Object, "spec", "schemaVersion") != q.ExpectedSchema || intNested(o.Object, "status", "schemaVersion") != q.ExpectedSchema || textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "lifecycleOperation", "id") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype environment is not eligible for migration"} } - if textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { - return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype runtime identity mismatch"} - } if err := s.verifyRuntimeUIDs(ctx, o, q); err != nil { return err } @@ -315,9 +348,15 @@ func (s *Store) MigrateEnvironment(ctx context.Context, q adminLifecycleRequest) } for attempt := 0; attempt < 5; attempt++ { o, err := s.resources.Get(ctx, q.Environment.ID, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } if err != nil { return err } + if err := adminSubject(o, q); err != nil { + return err + } if intNested(o.Object, "spec", "schemaVersion") == currentSchemaVersion { break } @@ -334,6 +373,9 @@ func (s *Store) MigrateEnvironment(ctx context.Context, q adminLifecycleRequest) } } return s.retryUpdateStatusRaw(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := adminSubject(o, q); err != nil { + return err + } if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion || textNested(o.Object, "status", "migrationOperation", "id") != q.OperationID || intNested(o.Object, "status", "schemaVersion") != q.ExpectedSchema { if intNested(o.Object, "status", "schemaVersion") == currentSchemaVersion && textNested(o.Object, "status", "lastMigrationOperationID") == q.OperationID { return nil diff --git a/internal/adapter/executioncontroller/admin_scope_security_test.go b/internal/adapter/executioncontroller/admin_scope_security_test.go new file mode 100644 index 0000000000..9c3f02af5e --- /dev/null +++ b/internal/adapter/executioncontroller/admin_scope_security_test.go @@ -0,0 +1,99 @@ +package executioncontroller + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + "time" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func scopedManifestClient(t *testing.T, uri string, admin bool, scope []string) securityClientManifest { + t.Helper() + raw, err := json.Marshal(map[string]any{"uri": uri, "administrator": admin, "administratorFor": scope}) + if err != nil { + t.Fatal(err) + } + var entry securityClientManifest + if err := json.Unmarshal(raw, &entry); err != nil { + t.Fatal(err) + } + return entry +} + +func TestAdministratorScopeManifestValidation(t *testing.T) { + for _, tc := range []struct { + name, raw string + valid bool + }{ + {"absent", `{"uri":"spiffe://example/admin","administrator":true}`, true}, + {"empty", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":[]}`, true}, + {"scoped", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator"]}`, true}, + {"not admin", `{"uri":"spiffe://example/admin","administratorFor":["spiffe://example/creator"]}`, false}, + {"duplicate", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator","spiffe://example/creator"]}`, false}, + {"wildcard", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/*"]}`, false}, + {"escaped wildcard", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/%2A"]}`, false}, + {"case", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://EXAMPLE/creator"]}`, false}, + {"query", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator?q=1"]}`, false}, + {"fragment", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator#x"]}`, false}, + {"userinfo", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://user@example/creator"]}`, false}, + {"malformed", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["creator"]}`, false}, + {"unknown", `{"uri":"spiffe://example/admin","administrator":true,"adminFor":[]}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var entry securityClientManifest + err := executionenv.DecodeStrict([]byte(tc.raw), &entry) + if err == nil { + _, err = clientPolicies([]securityClientManifest{entry}) + } + if (err == nil) != tc.valid { + t.Fatalf("valid=%v err=%v", tc.valid, err) + } + }) + } + for _, count := range []int{256, 257} { + scope := make([]string, count) + for i := range scope { + scope[i] = fmt.Sprintf("spiffe://example/creator-%d", i) + } + entry := scopedManifestClient(t, "spiffe://example/admin", true, scope) + if _, err := clientPolicies([]securityClientManifest{entry}); (err == nil) != (count == 256) { + t.Fatalf("scope count=%d err=%v", count, err) + } + } +} + +func TestAdministratorScopeDigestIsNormalizedAndAuthorityBound(t *testing.T) { + a, b := "spiffe://example/a", "spiffe://example/b" + mf := securityManifest{Clients: []securityClientManifest{scopedManifestClient(t, "spiffe://example/admin", true, []string{b, a})}} + before, _ := json.Marshal(mf) + digest := func(m securityManifest) string { + t.Helper() + d, err := authorityDigest(m, time.Minute, time.Second, nil, nil, "server") + if err != nil { + t.Fatal(err) + } + return d + } + original := digest(mf) + after, _ := json.Marshal(mf) + if !reflect.DeepEqual(before, after) { + t.Fatal("digest mutated source manifest") + } + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, []string{a, b}) + if digest(mf) != original { + t.Fatal("scope ordering changed authority digest") + } + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, []string{a}) + if digest(mf) == original { + t.Fatal("scope removal did not change authority digest") + } + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, nil) + empty := digest(mf) + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, []string{}) + if digest(mf) != empty { + t.Fatal("empty and absent scope differ") + } +} diff --git a/internal/adapter/executioncontroller/admin_scope_test.go b/internal/adapter/executioncontroller/admin_scope_test.go new file mode 100644 index 0000000000..2f9f20255b --- /dev/null +++ b/internal/adapter/executioncontroller/admin_scope_test.go @@ -0,0 +1,447 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "net" + "os" + "path/filepath" + "reflect" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const scopeCreator = "spiffe://example/creator" +const scopeAdmin = "spiffe://example/operations" + +var scopeOwner = executionenv.Owner{Issuer: "https://issuer.example", Subject: "owner"} + +type adminScopeFixture struct { + client executionv1.ExecutionProviderServiceClient + dynamic *dynamicfake.FakeDynamicClient + manager *SecurityManager + manifest securityManifest + path string +} + +func newAdminScopeFixture(t *testing.T, route, actor string) *adminScopeFixture { + t.Helper() + env := lifecycleAdminEnvironment(2, []any{}) + if route == "data" { + env = runFixtureEnvironment(4, 4) + } + if route == "migrate" { + _ = unstructured.SetNestedField(env.Object, int64(1), "spec", "schemaVersion") + _ = unstructured.SetNestedField(env.Object, int64(1), "status", "schemaVersion") + } + if route == "recover" { + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + } + if route == "delete" { + setConditionObject(env, "Retired", true, "Retired", "retired") + setConditionObject(env, "ExecutorTerminated", true, "Terminated", "terminated") + } + _ = unstructured.SetNestedField(env.Object, hashText(scopeCreator), "spec", "clientHash") + _ = unstructured.SetNestedField(env.Object, ownerHash(scopeOwner), "spec", "ownerHash") + _ = unstructured.SetNestedField(env.Object, int64(4), "status", "grantGeneration") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC()) + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + dir, now := t.TempDir(), time.Now().UTC() + ca, cert, key, clientCert := rpcSecurityPKI(t, now, actor) + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + writePKCS8(t, filepath.Join(dir, "grant.pem"), priv) + for name, data := range map[string][]byte{"server.crt": cert, "server.key": key, "clients.pem": ca} { + if err := os.WriteFile(filepath.Join(dir, name), data, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(pub) + mf := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: actor}}} + path := filepath.Join(dir, "manifest.json") + writeManifest(t, path, mf) + manager := NewSecurityManager(path, dir, "ns", "authority", k) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig()))) + executionv1.RegisterExecutionProviderServiceServer(server, NewHandler(HandlerConfig{Security: manager}, store)) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(ca) + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "provider.test", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return &adminScopeFixture{client: executionv1.NewExecutionProviderServiceClient(conn), dynamic: d, manager: manager, manifest: mf, path: path} +} + +func (f *adminScopeFixture) policy(t *testing.T, admin bool, scope []string) { + t.Helper() + f.manifest.Generation++ + f.manifest.Clients[0] = scopedManifestClient(t, f.manifest.Clients[0].URI, admin, scope) + writeManifest(t, f.path, f.manifest) + if err := f.manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } +} + +func callScopedAdmin(ctx context.Context, c executionv1.ExecutionProviderServiceClient, route string, q adminLifecycleRequest, owner executionenv.Owner) error { + ref := &executionv1.EnvironmentRef{Id: q.Environment.ID, Revision: q.Environment.Revision} + o := &executionv1.Owner{Issuer: owner.Issuer, Subject: owner.Subject} + var err error + switch route { + case "retire": + _, err = c.RetireEnvironment(ctx, &executionv1.RetireEnvironmentRequest{Environment: ref, Owner: o, ExpectedExecutionEpoch: q.ExpectedEpoch, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "replace": + _, err = c.ReplaceExecutor(ctx, &executionv1.ReplaceExecutorRequest{Environment: ref, Owner: o, ExpectedExecutionEpoch: q.ExpectedEpoch, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "recover": + _, err = c.RecoverEnvironment(ctx, &executionv1.RecoverEnvironmentRequest{Environment: ref, Owner: o, ExpectedExecutionEpoch: q.ExpectedEpoch, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "delete": + _, err = c.DeleteRetiredEnvironment(ctx, &executionv1.DeleteRetiredEnvironmentRequest{Environment: ref, Owner: o, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "migrate": + schema := uint32(q.ExpectedSchema) + _, err = c.MigrateEnvironment(ctx, &executionv1.MigrateEnvironmentRequest{Environment: ref, Owner: o, ExpectedSchemaVersion: &schema, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "revoke": + _, err = c.RevokeEnvironment(ctx, &executionv1.RevokeEnvironmentRequest{Environment: ref, Owner: o, ExpectedGrantGeneration: q.ExpectedEpoch, OperationId: q.OperationID}) + default: + panic("unknown test route") + } + return err +} + +func TestScopedAdminAllRoutesOverMTLS(t *testing.T) { + for _, route := range []string{"retire", "replace", "recover", "delete", "migrate", "revoke"} { + t.Run(route, func(t *testing.T) { + f := newAdminScopeFixture(t, route, scopeAdmin) + q := adminRequestFixture() + q.ExpectedSchema = 1 + call := func(q adminLifecycleRequest, owner executionenv.Owner) error { + return callScopedAdmin(t.Context(), f.client, route, q, owner) + } + if err := call(q, scopeOwner); status.Code(err) != codes.PermissionDenied { + t.Fatalf("nonadmin: %v", err) + } + for _, scope := range [][]string{nil, {"spiffe://example/unrelated"}} { + f.policy(t, true, scope) + denied := call(q, scopeOwner) + missing := q + missing.Environment.ID = "absent" + absent := call(missing, scopeOwner) + if status.Code(denied) != codes.NotFound || status.Convert(denied).Message() != status.Convert(absent).Message() || status.Code(absent) != codes.NotFound { + t.Fatalf("scope/missing disclosure: denied=%v absent=%v", denied, absent) + } + } + // Creator login is deliberately absent from the allowlist. + f.policy(t, true, []string{scopeCreator}) + wrongOwner := scopeOwner + wrongOwner.Subject = "another-owner" + if err := call(q, wrongOwner); status.Code(err) != codes.NotFound { + t.Fatalf("owner mismatch: %v", err) + } + stale := q + stale.Environment.Revision = "stale" + if err := call(stale, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("revision mismatch: %v", err) + } + stale = q + if route == "revoke" { + stale.ExpectedEpoch++ + } else { + stale.ExpectedPVCUID = "stale" + } + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("stale runtime identity: %v", err) + } + if route == "retire" || route == "replace" || route == "recover" { + stale = q + stale.ExpectedEpoch++ + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("stale epoch: %v", err) + } + } + if route != "delete" && route != "revoke" { + stale = q + stale.ExpectedPodUID = "stale" + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("stale Pod UID: %v", err) + } + } + if route == "migrate" { + stale = q + stale.ExpectedSchema = 0 + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("wrong schema: %v", err) + } + } + before, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := call(q, scopeOwner); err != nil { + t.Fatalf("scoped admin rejected: %v", err) + } + if err := call(q, scopeOwner); err != nil { + t.Fatalf("exact replay: %v", err) + } + after, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if reflect.DeepEqual(before.Object["status"], after.Object["status"]) { + t.Fatal("successful administration did not persist its transition") + } + if route == "revoke" { + receipts, _, err := unstructured.NestedSlice(after.Object, "status", "revocationReceipts") + if err != nil || len(receipts) != 1 || intNested(after.Object, "status", "grantGeneration") != 5 { + t.Fatalf("revocation replay was not idempotent: %v", after.Object["status"]) + } + if text(receipts[0].(map[string]any), "fingerprint") != fingerprint("env", "rev", scopeAdmin, ownerHash(scopeOwner), "4") { + t.Fatal("revocation receipt replaced the authenticated actor with the creator") + } + } + for _, field := range []string{"clientHash", "ownerHash", "revision"} { + if textNested(before.Object, "spec", field) != textNested(after.Object, "spec", field) { + t.Fatalf("admin rewrote immutable %s", field) + } + } + // Even receipt replay is a new admission on the established connection. + f.policy(t, true, nil) + if err := call(q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("scope removal did not revoke replay: %v", err) + } + }) + t.Run(route+"/creator", func(t *testing.T) { + f := newAdminScopeFixture(t, route, scopeCreator) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := callScopedAdmin(t.Context(), f.client, route, q, scopeOwner); status.Code(err) != codes.PermissionDenied { + t.Fatalf("normal creator is admin: %v", err) + } + f.policy(t, true, nil) + if err := callScopedAdmin(t.Context(), f.client, route, q, scopeOwner); err != nil { + t.Fatalf("legacy self-admin: %v", err) + } + }) + } +} + +func TestScopedAdminCASRetryRechecksSubject(t *testing.T) { + for _, route := range []string{"retire", "replace", "recover", "delete", "migrate", "revoke"} { + t.Run(route, func(t *testing.T) { + f := newAdminScopeFixture(t, route, scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + var conflicted atomic.Bool + f.dynamic.PrependReactor("update", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + if conflicted.Swap(true) { + return false, nil, nil + } + current, err := f.dynamic.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if err != nil { + return true, nil, err + } + o := current.(*unstructured.Unstructured) + _ = unstructured.SetNestedField(o.Object, hashText("spiffe://example/other"), "spec", "clientHash") + if err := f.dynamic.Tracker().Update(ExecutionEnvironmentGVR, o, "ns"); err != nil { + return true, nil, err + } + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", nil) + }) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := callScopedAdmin(t.Context(), f.client, route, q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("CAS retry escaped scope: %v", err) + } + if !conflicted.Load() { + t.Fatal("did not exercise CAS retry") + } + }) + } +} + +func TestScopedAdminUsesAdmittedPolicySnapshot(t *testing.T) { + f := newAdminScopeFixture(t, "revoke", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + var changed atomic.Bool + f.manifest.Generation++ + f.manifest.Clients[0] = scopedManifestClient(t, scopeAdmin, true, nil) + writeManifest(t, f.path, f.manifest) + f.dynamic.PrependReactor("get", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + if !changed.Swap(true) { + // Authentication has already admitted this request; change the live + // snapshot before the store makes its subject decision. + if err := f.manager.Reload(t.Context()); err != nil { + return true, nil, err + } + } + return false, nil, nil + }) + q := adminRequestFixture() + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); err != nil { + t.Fatalf("admitted snapshot was replaced mid-request: %v", err) + } + if !changed.Load() { + t.Fatal("policy did not change during admission") + } + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("next request reused old scope: %v", err) + } +} + +func TestScopedAdminCompletedReplacementReceiptReauthorizes(t *testing.T) { + f := newAdminScopeFixture(t, "replace", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + q := adminRequestFixture() + env, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + _ = unstructured.SetNestedMap(env.Object, map[string]any{"operationID": q.OperationID, "previousPodUID": q.ExpectedPodUID, "replacementPodUID": "new-pod", "pvcUID": q.ExpectedPVCUID, "previousEpoch": int64(4), "replacementEpoch": int64(5)}, "status", "lastReplacement") + _ = unstructured.SetNestedField(env.Object, int64(5), "status", "epoch") + _ = unstructured.SetNestedField(env.Object, "new-pod", "status", "pod", "uid") + if _, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), env, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := callScopedAdmin(t.Context(), f.client, "replace", q, scopeOwner); err != nil { + t.Fatal(err) + } + wrongOwner := scopeOwner + wrongOwner.Subject = "other" + if err := callScopedAdmin(t.Context(), f.client, "replace", q, wrongOwner); status.Code(err) != codes.NotFound { + t.Fatalf("receipt bypassed owner check: %v", err) + } + f.policy(t, true, nil) + if err := callScopedAdmin(t.Context(), f.client, "replace", q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("receipt bypassed removed scope: %v", err) + } +} + +func TestScopedAdminMigrationReceiptRetainsUIDPreconditions(t *testing.T) { + f := newAdminScopeFixture(t, "migrate", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); err != nil { + t.Fatal(err) + } + q.ExpectedPVCUID = "stale" + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("migration receipt bypassed UID: %v", err) + } +} + +func TestScopedAdminEqualGenerationDriftFailsClosed(t *testing.T) { + f := newAdminScopeFixture(t, "revoke", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + q := adminRequestFixture() + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); err != nil { + t.Fatal(err) + } + f.manifest.Clients[0] = scopedManifestClient(t, scopeAdmin, true, nil) + writeManifest(t, f.path, f.manifest) + if err := f.manager.Reload(t.Context()); err == nil { + t.Fatal("equal-generation scope drift admitted") + } + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); status.Code(err) != codes.Unavailable { + t.Fatalf("stale snapshot admitted RPC: %v", err) + } +} + +func TestScopedAdminDoesNotGrantAttestationOrDataPlane(t *testing.T) { + f := newAdminScopeFixture(t, "data", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + ref := &executionv1.EnvironmentRef{Id: "env", Revision: "rev"} + owner := &executionv1.Owner{Issuer: scopeOwner.Issuer, Subject: scopeOwner.Subject} + request := &executionv1.RequestContext{Environment: ref, Owner: owner, BindingId: "binding", Epoch: 4, RunId: "run", ClaimId: "claim", GrantGeneration: 4} + // A correctly signed actor-bound grant still cannot cross creator ownership. + h := NewHandler(HandlerConfig{Security: f.manager}, nil) + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 4, GrantGeneration: 4} + grant, _, err := h.signClaim(t.Context(), claim, scopeAdmin, ownerHash(scopeOwner)) + if err != nil { + t.Fatal(err) + } + request.Grant = grant + calls := map[string]func() error{ + "ensure attestation": func() error { + _, err := f.client.EnsureEnvironment(t.Context(), &executionv1.EnsureEnvironmentRequest{BindingId: "new", Profile: "go", Owner: owner, OperationId: "ensure"}) + return err + }, + "attach": func() error { + _, err := f.client.AttachEnvironment(t.Context(), &executionv1.AttachEnvironmentRequest{Context: request, Purpose: "session"}) + return err + }, + "run": func() error { + _, err := f.client.AcquireRun(t.Context(), &executionv1.AcquireRunRequest{Environment: ref, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire", TtlMillis: 60000}) + return err + }, + "reference": func() error { + _, err := f.client.CommitReference(t.Context(), &executionv1.ReferenceMutationRequest{Environment: ref, Owner: owner, BindingId: "binding", OperationId: "commit"}) + return err + }, + "reference discovery": func() error { + _, err := f.client.ListReferenceIntents(t.Context(), &executionv1.ListReferenceIntentsRequest{Owner: owner}) + return err + }, + "files": func() error { + _, err := f.client.Files(t.Context(), &executionv1.FileRequest{Context: request, Operation: executionv1.FileOperation_FILE_OPERATION_READ, Path: "file"}) + return err + }, + "shell": func() error { + _, err := f.client.StartCommand(t.Context(), &executionv1.CommandStartRequest{Context: request, Command: "true"}) + return err + }, + } + before, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + for name, call := range calls { + want := codes.PermissionDenied + switch name { + case "run", "reference", "reference discovery": + want = codes.InvalidArgument + case "files", "shell": + want = codes.NotFound + } + if err := call(); status.Code(err) != want { + t.Errorf("%s: want %v, got %v", name, want, err) + } + } + after, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before.Object, after.Object) { + t.Fatal("denied data-plane calls changed environment") + } +} diff --git a/internal/adapter/executioncontroller/controller.go b/internal/adapter/executioncontroller/controller.go index a9e26fe5f0..2e1161f613 100644 --- a/internal/adapter/executioncontroller/controller.go +++ b/internal/adapter/executioncontroller/controller.go @@ -243,11 +243,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, name string) error { podName := resourceName("executor", name) pvc, err := r.ensurePVC(ctx, env, p, pvcName) if err != nil { - return r.setCondition(ctx, env, "Ready", false, "PVCUnavailable", "workspace PVC unavailable") + return errors.Join(err, r.setCondition(ctx, env, "Ready", false, "PVCUnavailable", "workspace PVC unavailable")) } pod, err := r.ensurePod(ctx, env, p, podName, pvcName) if err != nil { - return r.setCondition(ctx, env, "Ready", false, "ExecutorUnavailable", "executor Pod unavailable") + return errors.Join(err, r.setCondition(ctx, env, "Ready", false, "ExecutorUnavailable", "executor Pod unavailable")) } ready := podReady(pod) if err := r.updateRuntimeStatus(ctx, env, pvc, pod, ready); err != nil { @@ -291,8 +291,10 @@ func (r *Reconciler) ensurePod(ctx context.Context, env *unstructured.Unstructur return nil, err } if textNested(env.Object, "status", "pod", "uid") != "" { - _ = r.setFenceUnknown(ctx, env, "previous executor disappearance is not externally fenced") - return nil, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "previous executor termination unconfirmed"} + return nil, errors.Join( + &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "previous executor termination unconfirmed"}, + r.setFenceUnknown(ctx, env, "previous executor disappearance is not externally fenced"), + ) } cpuReq := p.CPURequest memReq := p.MemoryRequest diff --git a/internal/adapter/executioncontroller/controller_test.go b/internal/adapter/executioncontroller/controller_test.go index 32550a81f6..fe80cb70dd 100644 --- a/internal/adapter/executioncontroller/controller_test.go +++ b/internal/adapter/executioncontroller/controller_test.go @@ -28,8 +28,8 @@ func TestReconcileRefusesForeignExistingPVCWithoutPersistingUID(t *testing.T) { d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) k := kubefake.NewSimpleClientset(foreign) r := NewReconciler(d, k, "ns", testProfiles()) - if err := r.Reconcile(ctx, env.GetName()); err != nil { - t.Fatal(err) + if err := r.Reconcile(ctx, env.GetName()); err == nil { + t.Fatal("foreign PVC ownership mismatch was not returned") } got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) if err != nil { @@ -108,8 +108,8 @@ func TestTerminatingPodIsUnavailableDuringReconcile(t *testing.T) { if _, err := k.CoreV1().Pods("ns").Update(ctx, pod, metav1.UpdateOptions{}); err != nil { t.Fatal(err) } - if err := r.Reconcile(ctx, env.GetName()); err != nil { - t.Fatal(err) + if err := r.Reconcile(ctx, env.GetName()); err == nil { + t.Fatal("terminating Pod was not rejected") } got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) if err != nil { diff --git a/internal/adapter/executioncontroller/handler.go b/internal/adapter/executioncontroller/handler.go index f6f8d7e7ca..2b9e635bfe 100644 --- a/internal/adapter/executioncontroller/handler.go +++ b/internal/adapter/executioncontroller/handler.go @@ -28,7 +28,10 @@ import ( ) // ClientPolicy defines capabilities assigned to an authenticated provider client. -type ClientPolicy struct{ MayAttestOwner, Administrator bool } +type ClientPolicy struct { + MayAttestOwner, Administrator bool + AdministratorFor []string +} // GrantSigner configures short-lived environment grant issuance. type GrantSigner struct { @@ -105,7 +108,7 @@ type adminLifecycleBackend interface { RecoverEnvironment(context.Context, adminLifecycleRequest) error DeleteRetiredEnvironment(context.Context, adminLifecycleRequest) error MigrateEnvironment(context.Context, adminLifecycleRequest) error - RevokeEnvironment(context.Context, executionenv.EnvironmentRef, string, string, uint64, string) (uint64, error) + RevokeEnvironment(context.Context, adminLifecycleRequest, uint64) (uint64, error) } // Handler is the authenticated private execution-provider gRPC service. @@ -509,7 +512,7 @@ func (h *Handler) DeleteRetiredEnvironment(ctx context.Context, q *executionv1.D if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) || !validIdentity(q.GetExpectedPvcUid()) || !validOperationID(q.GetOperationId()) { return nil, wireError(executionenv.CodePermissionDenied, false) } - req := adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, ExpectedPVCUID: q.GetExpectedPvcUid(), OperationID: q.GetOperationId()} + req := adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, AdministratorFor: c.policy.AdministratorFor, ExpectedPVCUID: q.GetExpectedPvcUid(), OperationID: q.GetOperationId()} if err := h.adminLifecycleBackend.DeleteRetiredEnvironment(ctx, req); err != nil { return nil, backendError(err) } @@ -545,7 +548,8 @@ func (h *Handler) RevokeEnvironment(ctx context.Context, q *executionv1.RevokeEn if q.GetExpectedGrantGeneration() == 0 || q.GetExpectedGrantGeneration() > math.MaxInt64 || !validOperationID(q.GetOperationId()) { return nil, wireError(executionenv.CodeInvalidArgument, false) } - generation, err := h.adminLifecycleBackend.RevokeEnvironment(ctx, ref, c.id, ownerHash(owner), q.GetExpectedGrantGeneration(), q.GetOperationId()) + req := adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, AdministratorFor: c.policy.AdministratorFor, OperationID: q.GetOperationId()} + generation, err := h.adminLifecycleBackend.RevokeEnvironment(ctx, req, q.GetExpectedGrantGeneration()) if err != nil { return nil, backendError(err) } @@ -562,7 +566,7 @@ func (h *Handler) adminRequest(ctx context.Context, pRef *executionv1.Environmen if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) || epoch == 0 || epoch > math.MaxInt64 || !validIdentity(podUID) || !validIdentity(pvcUID) || !validOperationID(operationID) { return adminLifecycleRequest{}, wireError(executionenv.CodePermissionDenied, false) } - return adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, ExpectedEpoch: epoch, ExpectedPodUID: podUID, ExpectedPVCUID: pvcUID, OperationID: operationID}, nil + return adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, AdministratorFor: c.policy.AdministratorFor, ExpectedEpoch: epoch, ExpectedPodUID: podUID, ExpectedPVCUID: pvcUID, OperationID: operationID}, nil } // Files executes one bounded and authorized filesystem operation. diff --git a/internal/adapter/executioncontroller/handler_test.go b/internal/adapter/executioncontroller/handler_test.go index b7bc9311e2..314292c856 100644 --- a/internal/adapter/executioncontroller/handler_test.go +++ b/internal/adapter/executioncontroller/handler_test.go @@ -120,7 +120,7 @@ func (b *adminFakeBackend) MigrateEnvironment(_ context.Context, req adminLifecy b.migratedSchema = req.ExpectedSchema return nil } -func (*adminFakeBackend) RevokeEnvironment(context.Context, executionenv.EnvironmentRef, string, string, uint64, string) (uint64, error) { +func (*adminFakeBackend) RevokeEnvironment(context.Context, adminLifecycleRequest, uint64) (uint64, error) { return 2, nil } diff --git a/internal/adapter/executioncontroller/provisioning_retry_test.go b/internal/adapter/executioncontroller/provisioning_retry_test.go new file mode 100644 index 0000000000..d69a7f1476 --- /dev/null +++ b/internal/adapter/executioncontroller/provisioning_retry_test.go @@ -0,0 +1,135 @@ +package executioncontroller + +import ( + "errors" + "sync/atomic" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/util/workqueue" +) + +func TestProvisioningWorkerConvergesWithoutAnotherEvent(t *testing.T) { + for _, resource := range []string{"persistentvolumeclaims", "pods"} { + t.Run(resource, func(t *testing.T) { + ctx := t.Context() + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + var attempts, pvcCreates, podCreates atomic.Int32 + var restored atomic.Bool + k.PrependReactor("create", "*", func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.GetResource().Resource == resource { + attempts.Add(1) + if !restored.Load() { + return true, nil, errors.New("transient quota saturation") + } + } + switch obj := action.(k8stesting.CreateAction).GetObject().(type) { + case *corev1.PersistentVolumeClaim: + pvcCreates.Add(1) + obj.UID = "pvc-stable" + case *corev1.Pod: + podCreates.Add(1) + obj.UID = "pod-stable" + obj.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + } + return false, nil, nil + }) + r := NewReconciler(d, k, "ns", testProfiles()) + r.queue.ShutDown() + r.queue = workqueue.NewTypedRateLimitingQueue(workqueue.NewTypedItemExponentialFailureRateLimiter[string](10*time.Millisecond, 10*time.Millisecond)) + done := make(chan struct{}) + go func() { r.worker(ctx); close(done) }() + t.Cleanup(func() { r.queue.ShutDown(); <-done }) + start := time.Now() + r.queue.Add(env.GetName()) + // Exceed a customary finite retry budget before restoring admission. + waitProvisioning(t, func() bool { return attempts.Load() >= 24 }) + if time.Since(start) < 200*time.Millisecond { + t.Fatal("provisioning retries hot-looped instead of rate limiting") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil || conditionTrue(got, "Ready") || conditionTransition(got, "Ready") == "" { + t.Fatalf("failed provisioning did not retain Ready=false: env=%v err=%v", got, err) + } + if writes := statusUpdateCount(d.Actions()); writes != 1 { + t.Fatalf("unchanged failure rewrote status: writes=%d", writes) + } + // No CR, runtime, or queue event: only the failed external prerequisite changes. + restored.Store(true) + waitProvisioning(t, func() bool { + got, err = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + return err == nil && conditionTrue(got, "Ready") && r.queue.NumRequeues(env.GetName()) == 0 + }) + if textNested(got.Object, "status", "pvc", "uid") != "pvc-stable" || textNested(got.Object, "status", "pod", "uid") != "pod-stable" || pvcCreates.Load() != 1 || podCreates.Load() != 1 { + t.Fatalf("duplicate or changed runtime identity: pvc=%d pod=%d status=%v", pvcCreates.Load(), podCreates.Load(), got.Object["status"]) + } + }) + } +} + +func waitProvisioning(t *testing.T, ready func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if ready() { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("worker did not converge without another event") +} + +func TestProvisioningPreservesCreateAndConditionErrors(t *testing.T) { + for _, resource := range []string{"persistentvolumeclaims", "pods"} { + t.Run(resource, func(t *testing.T) { + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + createErr, statusErr := errors.New("create unavailable"), errors.New("status unavailable") + k.PrependReactor("create", resource, func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, createErr }) + d.PrependReactor("update", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, statusErr }) + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + if err := r.Reconcile(t.Context(), env.GetName()); !errors.Is(err, createErr) || !errors.Is(err, statusErr) { + t.Fatalf("lost provisioning/status error: %v", err) + } + }) + } +} + +func TestProvisioningRetriesNeverReplaceAuthoritativeResources(t *testing.T) { + for _, resource := range []string{"pvc", "pod"} { + t.Run(resource, func(t *testing.T) { + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + if err := unstructured.SetNestedField(env.Object, "missing-authority", "status", resource, "uid"); err != nil { + t.Fatal(err) + } + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + for range 3 { + if err := r.Reconcile(t.Context(), env.GetName()); err == nil { + t.Fatal("missing authoritative resource was not rejected") + } + } + for _, action := range k.Actions() { + if action.GetVerb() == "create" && (resource == "pvc" || action.GetResource().Resource == "pods") { + t.Fatalf("replaced missing authority: %v", action) + } + } + }) + } +} diff --git a/internal/adapter/executioncontroller/security.go b/internal/adapter/executioncontroller/security.go index aa457bfccf..11021843de 100644 --- a/internal/adapter/executioncontroller/security.go +++ b/internal/adapter/executioncontroller/security.go @@ -52,9 +52,10 @@ type securityKeyManifest struct { } type securityClientManifest struct { - URI string `json:"uri"` - MayAttestOwner bool `json:"mayAttestOwner"` - Administrator bool `json:"administrator"` + URI string `json:"uri"` + MayAttestOwner bool `json:"mayAttestOwner"` + Administrator bool `json:"administrator"` + AdministratorFor []string `json:"administratorFor,omitempty"` } type securityTLSManifest struct { @@ -493,7 +494,22 @@ func clientPolicies(entries []securityClientManifest) (map[string]ClientPolicy, if _, exists := clients[entry.URI]; exists { return nil, errors.New("client URI is duplicated") } - clients[entry.URI] = ClientPolicy{MayAttestOwner: entry.MayAttestOwner, Administrator: entry.Administrator} + if len(entry.AdministratorFor) > maxSecurityClients || len(entry.AdministratorFor) > 0 && !entry.Administrator { + return nil, errors.New("administrator scope requires administrator authority and at most 256 creators") + } + scope := slices.Clone(entry.AdministratorFor) + slices.Sort(scope) + for i, creator := range scope { + uri, err := url.Parse(creator) + if err != nil || strings.ContainsAny(creator, "*?") || strings.ContainsAny(uri.Path, "*?") { + return nil, errors.New("administrator creator URI is invalid") + } + canonical, err := canonicalClientIdentity(&x509.Certificate{URIs: []*url.URL{uri}}) + if err != nil || canonical != creator || i > 0 && scope[i-1] == creator { + return nil, errors.New("administrator creator URI is not canonical or is duplicated") + } + } + clients[entry.URI] = ClientPolicy{MayAttestOwner: entry.MayAttestOwner, Administrator: entry.Administrator, AdministratorFor: scope} } return clients, nil } @@ -502,6 +518,10 @@ func authorityDigest(mf securityManifest, ttl, skew time.Duration, keyFingerprin keys := slices.Clone(mf.Keys) slices.SortFunc(keys, func(a, b securityKeyManifest) int { return strings.Compare(a.ID, b.ID) }) clients := slices.Clone(mf.Clients) + for i := range clients { + clients[i].AdministratorFor = slices.Clone(clients[i].AdministratorFor) + slices.Sort(clients[i].AdministratorFor) + } slices.SortFunc(clients, func(a, b securityClientManifest) int { return strings.Compare(a.URI, b.URI) }) canonical := struct { Version int diff --git a/internal/adapter/executioncontroller/store_concurrency_test.go b/internal/adapter/executioncontroller/store_concurrency_test.go index 75b43fa987..65c577f1bb 100644 --- a/internal/adapter/executioncontroller/store_concurrency_test.go +++ b/internal/adapter/executioncontroller/store_concurrency_test.go @@ -214,7 +214,7 @@ func TestAcquireRunAndRevokeUseResourceVersionCAS(t *testing.T) { results <- err }() go func() { - _, err := second.RevokeEnvironment(context.Background(), ref, "client", "owner", 1, "revoke") + _, err := second.RevokeEnvironment(context.Background(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke"}, 1) results <- err }() for range 2 { diff --git a/internal/adapter/executioncontroller/store_revoke.go b/internal/adapter/executioncontroller/store_revoke.go index 12ffe95897..88e73db467 100644 --- a/internal/adapter/executioncontroller/store_revoke.go +++ b/internal/adapter/executioncontroller/store_revoke.go @@ -13,16 +13,14 @@ import ( // RevokeEnvironment atomically invalidates every issued grant without claiming // that an already-dispatched executor operation has stopped. -func (s *Store) RevokeEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, client, owner string, expected uint64, operationID string) (uint64, error) { +func (s *Store) RevokeEnvironment(ctx context.Context, q adminLifecycleRequest, expected uint64) (uint64, error) { + ref, client, owner, operationID := q.Environment, q.Client, q.OwnerHash, q.OperationID if expected == 0 || expected >= math.MaxInt64 || operationID == "" || len(operationID) > executionenv.MaxIdentityBytes { return 0, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "grant generation or operation identity is invalid"} } requestFingerprint := fingerprint(ref.ID, ref.Revision, client, owner, fmt.Sprint(expected)) var next uint64 - err := s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { - if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { - return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} - } + err := s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { receipts, found, err := unstructured.NestedSlice(o.Object, "status", "revocationReceipts") if err != nil { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "revocation receipt ledger is invalid"} diff --git a/internal/adapter/executioncontroller/store_revoke_test.go b/internal/adapter/executioncontroller/store_revoke_test.go index 5a68dfd8c9..b83500af38 100644 --- a/internal/adapter/executioncontroller/store_revoke_test.go +++ b/internal/adapter/executioncontroller/store_revoke_test.go @@ -22,7 +22,7 @@ func TestRevokeEnvironmentFencesOldClaimWithoutChangingExecutionEpoch(t *testing client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) store := NewStore(client, "ns", testProfiles(), &terminalErrorExecutor{}) ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} - next, err := store.RevokeEnvironment(t.Context(), ref, "client", "owner", 3, "revoke-1") + next, err := store.RevokeEnvironment(t.Context(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke-1"}, 3) if err != nil || next != 4 { t.Fatalf("revoke=(%d,%v)", next, err) } @@ -42,11 +42,11 @@ func TestRevokeEnvironmentFencesOldClaimWithoutChangingExecutionEpoch(t *testing if _, err := store.RenewRun(t.Context(), ref, "client", "owner", executionenv.RunClaimRequest{Environment: ref, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: time.Minute}); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { t.Fatalf("old renewal error=%v", err) } - replayed, err := store.RevokeEnvironment(t.Context(), ref, "client", "owner", 3, "revoke-1") + replayed, err := store.RevokeEnvironment(t.Context(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke-1"}, 3) if err != nil || replayed != 4 { t.Fatalf("replayed revoke=(%d,%v)", replayed, err) } - if _, err := store.RevokeEnvironment(t.Context(), ref, "client", "owner", 3, "revoke-2"); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + if _, err := store.RevokeEnvironment(t.Context(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke-2"}, 3); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { t.Fatalf("stale CAS error=%v", err) } } @@ -55,7 +55,7 @@ func TestRevokeEnvironmentOverflowFailsClosed(t *testing.T) { env := runFixtureEnvironment(1, math.MaxInt64) client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) store := NewStore(client, "ns", testProfiles(), nil) - if _, err := store.RevokeEnvironment(context.Background(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", math.MaxInt64, "overflow"); err == nil { + if _, err := store.RevokeEnvironment(context.Background(), adminLifecycleRequest{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, Client: "client", OwnerHash: "owner", OperationID: "overflow"}, math.MaxInt64); err == nil { t.Fatal("overflow accepted") } } diff --git a/user-docs/building/deployment/mecak8s.md b/user-docs/building/deployment/mecak8s.md index a3df07d4da..5c23e619af 100644 --- a/user-docs/building/deployment/mecak8s.md +++ b/user-docs/building/deployment/mecak8s.md @@ -183,6 +183,7 @@ The following profile shows every required chart key. Save it as `execution-values.yaml` and replace each placeholder: ```yaml +fullnameOverride: mecatl-execution provider: image: @sha256: imagePullPolicy: IfNotPresent @@ -218,7 +219,8 @@ provider: }, { "uri": "spiffe://cluster.example.com/ns/mecatl/sa/execution-admin", "mayAttestOwner": false, - "administrator": true + "administrator": true, + "administratorFor": ["spiffe://cluster.example.com/ns/mecatl/sa/mecak8s"] }] } clientIngressSelectors: @@ -274,10 +276,11 @@ and verification dates with a current, reviewed rotation window. Increase `generation` for every authority change, including a CA, client policy, issuer/audience, key state, key window, or TLS identity change. -Install the provider chart separately from `mecak8s`: +Install the provider chart separately from `mecak8s`, before allocating any +execution environments. Use a namespace dedicated to this provider release: ```sh -helm upgrade --install mecatl-execution ./deploy/helm/mecatl-execution \ +helm install mecatl-execution ./deploy/helm/mecatl-execution \ --namespace \ --values execution-values.yaml kubectl rollout status deployment/mecatl-execution --namespace @@ -306,6 +309,14 @@ Shell, but do not ingest project instructions, rules, skills, or source from the remote PVC. Schedules, SkillDraft, background Shell, and delegated filesystem execution are outside this draft. +If allocation reports `Ready=false` with `PVCUnavailable` or +`ExecutorUnavailable`, inspect the namespace's quota and Kubernetes admission +failures. Restore the failed prerequisite and wait for `Ready=True`. The +controller keeps retrying through its rate-limited queue while the allocation +exists; no reference edit, restart, or manual reconciliation is needed. Missing +authoritative PVCs or Pods and ownership mismatches remain fail-closed and are +never repaired by creating a replacement. + Run claims and signed grants renew before the issued grant expiry, including when an operator configures a short grant TTL. Renewal operation receipts retain the newest 32 identities. Retrying an exact retained identity preserves the claim and @@ -332,14 +343,107 @@ grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/RecoverEnvironment ``` +#### Upgrade, uninstall, and reinstall the execution provider + +The supported lifecycle keeps the **same Helm release name, namespace, resource +names, profiles, network policy configuration, and security Secret name**. Keep +`execution-values.yaml` and the current nonsecret authority manifest in your +operator configuration store. Retain the operator-owned Secret and its key +history independently; the chart neither owns nor reads Secret contents. + +For a compatible provider upgrade: + +1. Stop new client traffic and finish or explicitly fence active work. Stop all + provider replicas before Helm reads the ledgers: + + ```sh + kubectl --namespace scale deployment/mecatl-execution --replicas=0 + kubectl --namespace wait --for=delete pod \ + --selector app.kubernetes.io/name=mecatl-execution --timeout=2m + ``` + +2. Apply the reviewed CRD schema **before** starting the upgraded provider. Helm + does not upgrade existing CRDs: + + ```sh + kubectl apply -f deploy/helm/mecatl-execution/crds/executionenvironment.yaml + kubectl wait --for=condition=Established \ + crd/executionenvironments.execution.mecatl.dev --timeout=60s + ``` + +3. Upgrade with the preserved lifetime configuration and current authority + manifest. Helm reads the existing ConfigMaps and includes their actual ledger + data, rather than empty bootstrap data, in the new release: + + ```sh + helm upgrade mecatl-execution ./deploy/helm/mecatl-execution \ + --namespace --values execution-values.yaml --wait --timeout=4m + ``` + +4. Complete any supported, explicit environment-schema migration while client + traffic remains quiesced. Verify readiness, exact environment/PVC UIDs, data, + and network confinement before resuming traffic. Unknown schema versions and + mixed-version provider operation are unsupported. + +Default uninstall removes the provider but keeps runtime CRs, PVCs, surviving +executors, workload default-deny and profile NetworkPolicies, both authority and +capacity ledgers, and the profile/security-manifest ConfigMaps. Uninstall is not +executor termination proof or storage disposal: + +```sh +helm uninstall mecatl-execution --namespace +``` + +Reinstall with the same identity and preserved values: + +```sh +helm install mecatl-execution ./deploy/helm/mecatl-execution \ + --namespace --values execution-values.yaml --wait --timeout=4m +``` + +Helm adopts retained resources only when their managed-by label and release-name +and release-namespace annotations match. The chart rejects missing or empty +ledgers while allocations or capacity reservations survive, and rejects changes +to its retained lifetime configuration. Profile removal or egress edits are deliberately outside this +upgrade path: retained allow policies are additive, so leaving an obsolete policy +could widen access. The provider rejects an older authority manifest against the +retained high-water generation and key history; never reset that ledger to make +readiness pass. + +Use live Helm install/upgrade for this lifecycle. Offline `helm template` cannot +perform ownership or history lookups and is not an adoption mechanism. Keep +provider writers stopped for upgrades; lookup plus apply is not a cross-resource +transaction. Rendering rejects a nonzero existing provider Deployment or any +remaining provider Pod, including a terminating Pod. Installations without the retained lifetime configuration/history +cannot be automatically adopted. Changed release/namespace adoption, chart +rollback, `--take-ownership`, CRD or namespace deletion with retained resources, +and force-finalizer cleanup are unsupported. Missing history requires recovery +from trusted retained state, not a fresh generation-1 authority. + #### Run an administrative lifecycle operation -Administrative RPCs are client-scoped. The administrator certificate must present -the same canonical URI identity that originated the environment and the request -must carry the exact owner. You can issue a separate short-lived administrator -certificate with that URI and mark the URI `administrator: true` during the -maintenance window. A namespace-wide administrator policy is not part of this -contract. +Administrative RPCs require `administrator: true`. For a distinct operations +identity, set its `administratorFor` list to the canonical URI of each client +that created the environments it may administer, as in the values example above. +An absent or empty list permits only self-administration. Keep +`mayAttestOwner: false` for an operations-only identity; administrative scope +confers no filesystem, Shell, attach, run, or reference access. See the +[scope constraints](../../features/execution-environments.md#production-security-material). + +Before first publishing `administratorFor`, quiesce client traffic and upgrade +all provider replicas to a version that understands the field. Older strict +manifest decoders reject it; mixed-version rolling operation is unsupported. +Preserve the authority high-water ConfigMap throughout the upgrade. Publish the +reviewed manifest with a higher `generation`, verify provider readiness, then +resume client traffic. + +Every administrative request still requires the original exact owner and +revision plus the operation's epoch, UID, schema, or generation preconditions. +The scope names creators, not owners; it can include a creator whose login has +been removed. After maintenance, remove its scope entry and increase +`generation` again. Subsequent requests, including receipt retries on established +connections, are denied; already admitted lifecycle operations may finish safe +reconciliation. First capture the private identity while the environment still has a reference: diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index 1c374de050..1fc9b365a2 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -149,7 +149,8 @@ this shape: { "uri": "spiffe://cluster.example.com/ns/mecatl/sa/execution-admin", "mayAttestOwner": false, - "administrator": true + "administrator": true, + "administratorFor": ["spiffe://cluster.example.com/ns/mecatl/sa/mecak8s"] } ] } @@ -158,6 +159,22 @@ this shape: The all-zero fingerprint and 2027 dates are non-secret example values. Replace them with the public-key fingerprint and a reviewed active window before use. +`administratorFor` permits administration of environments created by the listed +client URIs, alongside the administrator's own environments. A nonempty list +requires `administrator: true` and accepts at most 256 unique canonical URIs: +nonempty scheme and host, lowercase scheme and host, and no userinfo, query, +fragment, or wildcard. Matching is exact, with no namespace or prefix grant. +An absent or empty list preserves self-administration only. The creator need +not remain in the client allowlist. + +The scope applies to `RetireEnvironment`, `ReplaceExecutor`, `RecoverEnvironment`, +`DeleteRetiredEnvironment`, `MigrateEnvironment`, and `RevokeEnvironment`. It +preserves each operation's exact owner and identity checks and grants no +`mayAttestOwner`, attach, file, command, run, or reference authority. Scope order +does not change the authority digest; adding or removing a creator does. Follow +the [administrative runbook](../building/deployment/mecak8s.md#run-an-administrative-lifecycle-operation) +for quiesced upgrades and scope removal. + Use only basename file names. The projected Secret keys in this example are `grant-k1.pem`, `tls.crt`, `tls.key`, and `clients.pem`; an external secret manager owns their bytes. Kubernetes projected-volume `..data` symlinks are supported, @@ -198,11 +215,13 @@ name alone as a hostile-workload isolation guarantee. CPU, memory, ephemeral storage, and `/tmp` are explicitly bounded per profile. Each profile also requires `maxEnvironments`; the namespace quota is the final global bound. -For an existing single-key installation, first create the authority ConfigMap and -a generation-1 manifest that names the existing TLS and grant files, then upgrade -the chart. Confirm `/ready` before rotating. Do not reuse the old key ID with new -key bytes. Schema-2 environment migration is independent of keyring migration: -complete both before enabling new sessions. +Preserve the authority ConfigMap and current manifest across upgrades and +same-release reinstalls. A missing ledger with retained allocations is not a new +installation: do not bootstrap it at generation 1. See the +[provider lifecycle procedure](/building/deployment/mecak8s.md#upgrade-uninstall-and-reinstall-the-execution-provider) +for ownership checks, retained network protection, and quiesced CRD/provider +upgrade order. Schema-2 environment migration remains an explicit operation; +complete it before enabling new sessions. ## Live qualification (experimental) From e664ee14cda1900a9ea2028dbc65f037c5f85a57 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Mon, 21 Sep 2026 21:26:52 +0300 Subject: [PATCH 13/25] docs: update native execution tracker for renumbered draft ADR Keep the implementation-only tracker link aligned with the integrated plan; preserve draft status and runtime behavior. Co-Authored-By: OpenAI --- docs/design/PRODUCTION-READINESS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/PRODUCTION-READINESS.md b/docs/design/PRODUCTION-READINESS.md index 23c2614071..509b27a100 100644 --- a/docs/design/PRODUCTION-READINESS.md +++ b/docs/design/PRODUCTION-READINESS.md @@ -14,7 +14,7 @@ record; current behaviour is in the linked [architecture](../architecture.md) do | Subsystem | Status | Design record | Architecture | |---|---|---|---| -| Native Kubernetes execution provider | 🔨 Human-authorized draft development in existing [PR #1614](https://github.com/stacklok/mecatl/pull/1614), not shipped or approved. N2 candidate retains workload isolation, authority/capacity ledgers, and configuration through uninstall; ownership-checked lookup preserves data for same-release/namespace adoption and rejects incompatible profiles or missing history. Offline real-Helm rendering coverage and a production Kind install/upgrade/uninstall/reinstall test are authored. ⛔ Final deterministic CI, enforcing-CNI lifecycle execution, native real-provider coding smoke, API/schema/security reviews, and final acceptance evidence remain pending. Generic live Kubernetes proof is not native execution qualification. | [0349](../adr/0349-native-kubernetes-execution.md) · [acceptance contract](../acceptance/native-kubernetes-execution.md) | [native execution mechanics](./IMPLEMENTATION-NOTES.md#native-kubernetes-execution-provider) | +| Native Kubernetes execution provider | 🔨 Human-authorized draft development in existing [PR #1614](https://github.com/stacklok/mecatl/pull/1614), not shipped or approved. N2 candidate retains workload isolation, authority/capacity ledgers, and configuration through uninstall; ownership-checked lookup preserves data for same-release/namespace adoption and rejects incompatible profiles or missing history. Offline real-Helm rendering coverage and a production Kind install/upgrade/uninstall/reinstall test are authored. ⛔ Final deterministic CI, enforcing-CNI lifecycle execution, native real-provider coding smoke, API/schema/security reviews, and final acceptance evidence remain pending. Generic live Kubernetes proof is not native execution qualification. | [0350](../adr/0350-native-kubernetes-execution.md) · [acceptance contract](../acceptance/native-kubernetes-execution.md) | [native execution mechanics](./IMPLEMENTATION-NOTES.md#native-kubernetes-execution-provider) | | Multi-provider / multi-model | ✅ P0+P1+live listing & metadata · ✅ same-provider history carryover (issue #20) · ✅ experimental manual `openai-codex` subscription token (ADR 0215) · ✅ ToolHive OpenAI Responses and native Anthropic protocol providers over one gateway identity (ADR 0334) · ✅ internal opaque credential-store substrate consumed by MCP profiles with local encrypted-file Store and explicit read-only environment Reader (issues #519/#542) · ⛔ disk cache (P2) · ⛔ key acquisition/per-client routing/remote stores or Kubernetes Secret `resourceVersion` CAS backend (P3) | [MULTI-PROVIDER.md](../adr/0016-multi-provider.md) · [0215](../adr/0215-openai-subscription-manual-token.md) · [0218](../adr/0218-credential-store.md) · [0221](../adr/0221-read-only-credential-source.md) · [0334](../adr/0334-toolhive-protocol-specific-providers.md) | [providers](../architecture/providers.md) · [credential store](../architecture.md#internal-credential-store) | | Remote mecatui OIDC login, refresh, recovery, and logout | ✅ fixed-callback PKCE enrollment, actual-record-only root-scoped keyring migration, one refresh/enroll/logout target transaction with ambiguous-commit compensation, target-bound encrypted credentials, token-demand-gated proactive refresh, exact structured-rejection cleanup, ownership-checked same-target recovery, CAS-safe local-first logout, one-budget best-effort RFC 7009 revocation, and target-aware TLS defaults with saved-auth verified-TLS enforcement shipped; ✅ offline rotation/rejection/crash-residual/logout/TLS-policy coverage · ⚠️ no cross-store journal: a crash may require login or leave an unenumerable credential-only orphan · ⚠️ legacy zero-padded-port credential identities require one login · ⛔ live Kind qualification remains environment-dependent | [0277](../adr/0277-remote-mecatui-oidc.md) · [0287](../adr/0287-target-aware-mecatui-tls.md) · [0218](../adr/0218-credential-store.md) | [remote login](../tui.md#oidc-connected-server) | | Direct MCP OAuth dynamic client registration | 🔨 Core trusted-profile validation, durable CAS registration, generation-bound no-refresh grant, explicit reset/retry recovery, restart reuse, expiry-to-login-required, protocol/security fixtures, seven named acceptance proofs, and cloud-native resource re-audit complete · ⛔ repeat panel review, implementation PR/merge, and live local-mecatui qualification remain · ⛔ complete resource-bound public-client refresh ([issue #1355](https://github.com/stacklok/mecatl/issues/1355)) | [0325](../adr/0325-direct-mcp-dcr.md) | [credential store](../architecture.md#internal-credential-store) | From 9bb4d89b8aa497c9d2cd6aa4a94ab9262a913364 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Mon, 21 Sep 2026 22:05:23 +0300 Subject: [PATCH 14/25] fix: enforce exact migration replay and qualification coverage Co-Authored-By: OpenAI --- .../crds/executionenvironment.yaml | 1 + deploy/helm/mecatl-execution/lifetime_test.go | 14 +-- .../templates/lifetime-check.yaml | 3 + e2e/k8s_execution/ab_production_admin_test.go | 70 +++++++++++ e2e/k8s_execution/fixture/pki/main.go | 5 + e2e/k8s_execution/fixture/rotation/main.go | 9 +- .../production_qualification_test.go | 15 +++ .../zz_production_helm_lifetime_test.go | 77 +++++++++++- .../executioncontroller/admin_lifecycle.go | 25 +++- .../admin_scope_security_test.go | 10 +- .../executioncontroller/admin_scope_test.go | 114 ++++++++++++++++++ .../executioncontroller/crd_schema_test.go | 25 +++- .../production_lifecycle_test.go | 11 ++ user-docs/building/deployment/mecak8s.md | 42 +++++-- user-docs/features/execution-environments.md | 7 +- 15 files changed, 390 insertions(+), 38 deletions(-) create mode 100644 e2e/k8s_execution/ab_production_admin_test.go diff --git a/deploy/helm/mecatl-execution/crds/executionenvironment.yaml b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml index ef87a94910..f31fc8340c 100644 --- a/deploy/helm/mecatl-execution/crds/executionenvironment.yaml +++ b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml @@ -198,6 +198,7 @@ spec: expectedPodUID: {type: string, minLength: 1, maxLength: 128} expectedPVCUID: {type: string, minLength: 1, maxLength: 128} lastMigrationOperationID: {type: string, maxLength: 1024} + lastMigrationFromSchema: {type: integer, format: int64, enum: [0, 1]} conditions: type: array items: diff --git a/deploy/helm/mecatl-execution/lifetime_test.go b/deploy/helm/mecatl-execution/lifetime_test.go index b2d0adca33..fa4d0b2c5f 100644 --- a/deploy/helm/mecatl-execution/lifetime_test.go +++ b/deploy/helm/mecatl-execution/lifetime_test.go @@ -94,7 +94,7 @@ func TestChartRetainedLifetime(t *testing.T) { }, want: "bootstrap is forbidden"}, {name: "missing lifetime configuration", mutate: func(m map[string]map[string]any) { delete(m["ConfigMap/test-mecatl-execution-profiles"]["data"].(map[string]any), "lifetime.json") - }, want: "configuration is incompatible"}, + }, want: "missing lifetime.json history"}, {name: "missing authority", mutate: func(m map[string]map[string]any) { delete(m, authority) }, want: "bootstrap is forbidden"}, {name: "empty authority", mutate: func(m map[string]map[string]any) { m[authority]["data"] = map[string]any{} }, want: "bootstrap is forbidden"}, {name: "missing state", mutate: func(m map[string]map[string]any) { m[authority]["data"] = map[string]any{"other": "value"} }, want: "state.json"}, @@ -130,18 +130,14 @@ func TestChartRetainedLifetime(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - data, err := json.Marshal(retained) - if err != nil { - t.Fatal(err) - } - var objects map[string]map[string]any - if err := json.Unmarshal(data, &objects); err != nil { - t.Fatal(err) + objects := make(map[string]map[string]any, len(retained)) + for key, obj := range retained { + objects[key] = (&unstructured.Unstructured{Object: obj}).DeepCopy().Object } if tt.mutate != nil { tt.mutate(objects) } - _, err = renderLifetime(t, objects, tt.args...) + _, err := renderLifetime(t, objects, tt.args...) if err == nil || !strings.Contains(err.Error(), tt.want) { t.Fatalf("render error=%v, want %q", err, tt.want) } diff --git a/deploy/helm/mecatl-execution/templates/lifetime-check.yaml b/deploy/helm/mecatl-execution/templates/lifetime-check.yaml index b3d53405e5..592a7cdff3 100644 --- a/deploy/helm/mecatl-execution/templates/lifetime-check.yaml +++ b/deploy/helm/mecatl-execution/templates/lifetime-check.yaml @@ -6,6 +6,9 @@ {{- end -}} {{- $profiles := lookup "v1" "ConfigMap" .Release.Namespace (printf "%s-profiles" $name) -}} {{- if $profiles -}} +{{- if not (dig "lifetime.json" "" ($profiles.data | default dict)) -}} +{{- fail "retained execution profiles are missing lifetime.json history; automatic adoption is unsupported, recover trusted retained history before upgrading" -}} +{{- end -}} {{- if ne (dig "lifetime.json" "" ($profiles.data | default dict)) (include "mecatl-execution.lifetimeConfig" .) -}} {{- fail "retained execution profiles/network configuration is incompatible; preserve the original lifetime configuration" -}} {{- end -}} diff --git a/e2e/k8s_execution/ab_production_admin_test.go b/e2e/k8s_execution/ab_production_admin_test.go new file mode 100644 index 0000000000..3932ec14fe --- /dev/null +++ b/e2e/k8s_execution/ab_production_admin_test.go @@ -0,0 +1,70 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestKindExecutionProductionScopedAdministrator(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + qualifyDistinctAdministrator(t, ctx, state, kubeconfig, "initial") +} + +func qualifyDistinctAdministrator(t *testing.T, ctx context.Context, state, kubeconfig, phase string) { + t.Helper() + creator, forward := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, creator, "admin-"+phase) + clients := map[string]*executionclient.Client{} + for _, name := range []string{"operations", "wrong-scope", "intruder"} { + client, err := executionclient.New(forward.addr, loadTLS(t, filepath.Join(state, "pki"), name, "mecatl-execution.execution-qualification.svc.cluster.local")) + if err != nil { + t.Fatal("create synthetic admin client") + } + t.Cleanup(func() { client.Close() }) + clients[name] = client + } + rc, release := acquireRun(t, ctx, creator, owner, binding, attached, "admin-data-"+phase) + if _, err := creator.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "admin-sentinel", Data: []byte("creator-only\n")}); err != nil { + t.Fatal("creator data-plane positive:", remoteErrorCode(err)) + } + admin := clients["operations"] + if _, err := admin.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); !isRemoteCode(err, executionenv.CodeNotFound) { + t.Fatal("scoped administrator acquired creator attach privilege:", remoteErrorCode(err)) + } + if _, err := admin.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: "admin-sentinel"}); !isRemoteCode(err, executionenv.CodePermissionDenied) { + t.Fatal("scoped administrator used creator file grant:", remoteErrorCode(err)) + } + if _, err := admin.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "true"}); !isRemoteCode(err, executionenv.CodePermissionDenied) { + t.Fatal("scoped administrator used creator command grant:", remoteErrorCode(err)) + } + release() + before := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + request := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: before.Epoch, ExpectedPodUID: before.PodUID, ExpectedPVCUID: before.PVCUID, OperationID: "admin-replace-" + phase} + for name, code := range map[string]executionenv.ErrorCode{"wrong-scope": executionenv.CodeNotFound, "intruder": executionenv.CodePermissionDenied} { + if err := clients[name].ReplaceExecutor(ctx, request); !isRemoteCode(err, code) { + t.Fatalf("%s admin refusal: %s", name, remoteErrorCode(err)) + } + } + if after := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID); after != before { + t.Fatal("denied admin calls changed runtime state") + } + if err := admin.ReplaceExecutor(ctx, request); err != nil { + t.Fatal("distinct scoped administrator replacement:", remoteErrorCode(err)) + } + after := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Ready && s.PodUID != before.PodUID }) + if after.PVCUID != before.PVCUID || after.Epoch <= before.Epoch { + t.Fatal("scoped replacement failed exact PVC/epoch reconciliation") + } + reattached := waitReady(t, ctx, creator, owner, binding, attached.Environment) + rc, release = acquireRun(t, ctx, creator, owner, binding, reattached, "admin-verify-"+phase) + waitFileContent(t, ctx, creator, rc, "admin-sentinel", "creator-only\n", "scoped replacement") + release() + retireSyntheticEnvironment(t, ctx, creator, kubeconfig, owner, binding, attached.Environment) +} diff --git a/e2e/k8s_execution/fixture/pki/main.go b/e2e/k8s_execution/fixture/pki/main.go index d6121c138b..3a2c2205e7 100644 --- a/e2e/k8s_execution/fixture/pki/main.go +++ b/e2e/k8s_execution/fixture/pki/main.go @@ -38,6 +38,9 @@ func main() { must(issue(dir, "provider", ca, caKey, []string{"mecatl-execution", "mecatl-execution.execution-qualification.svc", "mecatl-execution.execution-qualification.svc.cluster.local"}, "", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth})) must(issue(dir, "mecak8s", ca, caKey, nil, "spiffe://mecatl.test/client/mecak8s", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})) must(issue(dir, "intruder", ca, caKey, nil, "spiffe://mecatl.test/client/intruder", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})) + for _, name := range []string{"operations", "wrong-scope"} { + must(issue(dir, name, ca, caKey, nil, "spiffe://mecatl.test/client/"+name, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})) + } grantPub, grantKey, err := ed25519.GenerateKey(rand.Reader) must(err) grantDER, err := x509.MarshalPKCS8PrivateKey(grantKey) @@ -51,6 +54,8 @@ func main() { "clients": []any{ map[string]any{"uri": "spiffe://mecatl.test/client/mecak8s", "mayAttestOwner": true, "administrator": true}, map[string]any{"uri": "spiffe://mecatl.test/client/intruder", "mayAttestOwner": true, "administrator": false}, + map[string]any{"uri": "spiffe://mecatl.test/client/operations", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/mecak8s"}}, + map[string]any{"uri": "spiffe://mecatl.test/client/wrong-scope", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/intruder"}}, }, }) must(err) diff --git a/e2e/k8s_execution/fixture/rotation/main.go b/e2e/k8s_execution/fixture/rotation/main.go index 680163ad99..c00373feb0 100644 --- a/e2e/k8s_execution/fixture/rotation/main.go +++ b/e2e/k8s_execution/fixture/rotation/main.go @@ -82,8 +82,13 @@ func manifest(generation uint64, audience, active string, keys []keyEntry, cert, } return map[string]any{ "version": 1, "generation": generation, "issuer": "https://mecatl.execution.test", "audience": audience, "activeKeyID": active, "grantTTL": "1m", "clockSkew": "5s", "keys": entries, - "tls": map[string]any{"certificateFile": cert, "privateKeyFile": privateKey, "clientCAFile": clients}, - "clients": []any{map[string]any{"uri": "spiffe://mecatl.test/client/mecak8s", "mayAttestOwner": true, "administrator": true}, map[string]any{"uri": "spiffe://mecatl.test/client/intruder", "mayAttestOwner": true, "administrator": false}}, + "tls": map[string]any{"certificateFile": cert, "privateKeyFile": privateKey, "clientCAFile": clients}, + "clients": []any{ + map[string]any{"uri": "spiffe://mecatl.test/client/mecak8s", "mayAttestOwner": true, "administrator": true}, + map[string]any{"uri": "spiffe://mecatl.test/client/intruder", "mayAttestOwner": true, "administrator": false}, + map[string]any{"uri": "spiffe://mecatl.test/client/operations", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/mecak8s"}}, + map[string]any{"uri": "spiffe://mecatl.test/client/wrong-scope", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/intruder"}}, + }, } } diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go index 89cd787267..28d2b91785 100644 --- a/e2e/k8s_execution/production_qualification_test.go +++ b/e2e/k8s_execution/production_qualification_test.go @@ -312,6 +312,7 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { // Restore fixture client compatibility through a higher generation; this is // another forward rotation, never a high-water-mark rollback. restoreFixtureSecurity(t, ctx, kubeconfig, rotationDir) + qualifyDistinctAdministrator(t, ctx, state, kubeconfig, "rotated") } func acquireCurrentAuthorityRun(ctx context.Context, req executionenv.RunClaimRequest, acquire func(context.Context, executionenv.RunClaimRequest) (executionenv.RunClaim, error), retryDelay time.Duration) (executionenv.RunClaim, error) { @@ -1136,6 +1137,20 @@ func runKindExecutionProductionCompatiblePrototypeMigration(t *testing.T) { if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, before.PodUID, before.PVCUID, "migration-compatible-v1"); err != nil { t.Fatalf("migrate compatible prototype: %v", err) } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, before.PodUID, before.PVCUID, "migration-compatible-v1"); err != nil { + t.Fatalf("exact migration replay: %v", err) + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 0, before.PodUID, before.PVCUID, "migration-compatible-v1"); !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatal("migration receipt accepted changed source schema:", remoteErrorCode(err)) + } + if got := kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", fixture.Environment.ID, "-n", namespace, "-o", "jsonpath={.status.lastMigrationFromSchema}"); got != "1" { + t.Fatal("API server pruned the exact migration receipt") + } + runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Environment.ID, "-n", namespace, "--subresource=status", "--type=merge", "--dry-run=server", "-p", `{"status":{"lastMigrationFromSchema":0}}`) + invalid := command(ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Environment.ID, "-n", namespace, "--subresource=status", "--type=merge", "--dry-run=server", "-p", `{"status":{"lastMigrationFromSchema":2}}`) + if out, err := invalid.CombinedOutput(); err == nil || !bytes.Contains(out, []byte("lastMigrationFromSchema")) || !bytes.Contains(out, []byte("Unsupported value")) { + t.Fatal("API server did not reject an unsupported migration receipt schema") + } after := waitExecutionStatus(t, ctx, kubeconfig, fixture.Environment.ID, func(s executionStatus) bool { return s.Ready && s.SpecSchema == 2 && s.StatusSchema == 2 }) if after.PodUID != before.PodUID || after.PVCUID != before.PVCUID || after.Epoch != before.Epoch+1 { t.Fatal("migration changed runtime identity or failed to advance only the fence epoch") diff --git a/e2e/k8s_execution/zz_production_helm_lifetime_test.go b/e2e/k8s_execution/zz_production_helm_lifetime_test.go index 63dc51e4f5..266f8937cd 100644 --- a/e2e/k8s_execution/zz_production_helm_lifetime_test.go +++ b/e2e/k8s_execution/zz_production_helm_lifetime_test.go @@ -5,7 +5,11 @@ package k8s_execution_test import ( "bytes" "context" + "crypto/ed25519" + "crypto/x509" + "encoding/base64" "encoding/json" + "encoding/pem" "errors" "fmt" "io" @@ -20,6 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + "github.com/stacklok/mecatl/internal/adapter/executionclient" "github.com/stacklok/mecatl/internal/executionenv" ) @@ -42,6 +47,7 @@ func TestKindExecutionProductionHelmLifetime(t *testing.T) { if err != nil || built.State != executionenv.CommandSucceeded || built.Result.ExitCode != 0 { t.Fatal("build lifetime probe failed") } + assertRetiredFixtureKeyDenied(t, ctx, client, state, rc) release() // Occupy the single-slot profile; its reservation must still constrain Ensure // after adoption, rather than only matching a ConfigMap annotation. @@ -130,6 +136,17 @@ func TestKindExecutionProductionHelmLifetime(t *testing.T) { if err := json.Unmarshal([]byte(authorityBefore.Data["state.json"]), &priorHistory); err != nil || len(priorHistory.Fingerprints) == 0 { t.Fatal("missing pre-upgrade authority history") } + policiesBefore := lifetimePolicies(ctx, t, kubeconfig) + liveLedgers := map[string]corev1.ConfigMap{capacityBefore.Name: capacityBefore, authorityBefore.Name: authorityBefore, current.Name: current} + liveLedgers["mecatl-execution-profiles"] = lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-profiles") + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + if _, err := helmLifetime(ctx, kubeconfig, "upgrade", "mecatl-execution", chart, "-f", valuesPath, "--wait", "--timeout=4m"); !errors.Is(err, errLifetimeNotQuiesced) { + t.Fatal("live same-release upgrade did not reject provider writers with the quiescence error") + } + assertLifetimeRetained(ctx, t, kubeconfig, liveLedgers, policiesBefore) + if status := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID); status != before || envUID != kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") { + t.Fatal("rejected live upgrade changed runtime identity or status") + } // Quiesce all writers before lookup snapshots are rendered into the upgrade. if err := restore(ctx); err != nil { t.Fatal("compatible Helm upgrade failed:", err) @@ -202,6 +219,7 @@ func TestKindExecutionProductionHelmLifetime(t *testing.T) { } rc, release = acquireRun(t, ctx, reattachedClient, owner, binding, reattached, "helm-lifetime-read") waitFileContent(t, ctx, reattachedClient, rc, "helm-sentinel", "retained-helm-data\n", "same-release Helm adoption") + assertRetiredFixtureKeyDenied(t, ctx, reattachedClient, state, rc) release() if _, err := reattachedClient.Ensure(ctx, quotaBinding+"-excess", "quota-cas", owner, "ensure-"+quotaBinding+"-excess"); !isRemoteCode(err, executionenv.CodeResourceExhausted) { t.Fatal("retained capacity did not reject excess allocation:", remoteErrorCode(err)) @@ -232,6 +250,56 @@ func TestKindExecutionProductionHelmLifetime(t *testing.T) { // finalizer removal. The explicit test-cluster owner controls final disposal. } +// Quiesced upgrades can outlive a grant/claim. Re-sign the CURRENT claim with +// the fixture's retired k1, before and after adoption: expiry, released claims, +// and stale epochs cannot masquerade as durable authority rejection. +func assertRetiredFixtureKeyDenied(t *testing.T, ctx context.Context, client *executionclient.Client, state string, rc executionenv.RequestContext) { + t.Helper() + parts := strings.Split(rc.Grant, ".") + if len(parts) != 3 { + t.Fatal("invalid current fixture grant") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + t.Fatal("decode current fixture grant") + } + var claims executionenv.GrantClaims + if err := json.Unmarshal(payload, &claims); err != nil { + t.Fatal("decode current fixture claims") + } + material, err := os.ReadFile(filepath.Join(state, "pki", "grant-key.pem")) + if err != nil { + t.Fatal("read test-owned retired signing key") + } + block, _ := pem.Decode(material) + if block == nil { + t.Fatal("decode test-owned signing key") + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + t.Fatal("parse test-owned signing key") + } + private, ok := key.(ed25519.PrivateKey) + if !ok { + t.Fatal("fixture signing key is not Ed25519") + } + claims.KeyID = "k1" + old := rc + old.Grant, err = executionenv.SignGrant(private, claims) + if err != nil { + t.Fatal("sign retired-authority fixture grant") + } + verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": private.Public().(ed25519.PublicKey)}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: time.Minute} + if _, err := verifier.Verify(old.Grant, executionenv.GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: rc.GrantGeneration, Operation: executionenv.OpFileRead}); err != nil { + t.Fatal("retired-authority control is not cryptographically valid and current") + } + waitFileContent(t, ctx, client, rc, "helm-sentinel", "retained-helm-data\n", "current authority before retired-key probe") + if _, err := client.File(ctx, executionenv.FileRequest{Context: old, Operation: executionenv.OpFileRead, Path: "helm-sentinel"}); !isRemoteCode(err, executionenv.CodePermissionDenied) { + t.Fatal("retired signing authority did not remain denied:", remoteErrorCode(err)) + } + waitFileContent(t, ctx, client, rc, "helm-sentinel", "retained-helm-data\n", "current authority after retired-key probe") +} + func requireOwnedHelmFixture(t *testing.T, state, kubeconfig string) { t.Helper() ownership, err := os.ReadFile(filepath.Join(state, "ownership")) @@ -343,14 +411,19 @@ func quiesceLifetimeProvider(ctx context.Context, kubeconfig string) error { return nil } +var errLifetimeNotQuiesced = errors.New("execution provider must be quiesced") + func helmLifetime(ctx context.Context, kubeconfig string, args ...string) ([]byte, error) { full := append([]string{"--kubeconfig", kubeconfig, "--kube-context", os.Getenv("MECATL_KUBE_CONTEXT"), "--namespace", namespace}, args...) cmd := exec.CommandContext(ctx, "helm", full...) cmd.Env = cleanEnv() - var out lifetimeOutput + var out, stderr lifetimeOutput cmd.Stdout = &out - cmd.Stderr = io.Discard + cmd.Stderr = &stderr if err := cmd.Run(); err != nil { + if bytes.Contains(stderr.Bytes(), []byte("quiesce the execution provider (scale to zero and wait for its Pods to disappear) before upgrading")) { + return nil, errLifetimeNotQuiesced + } return nil, fmt.Errorf("Helm %s failed: %w (output suppressed)", args[0], err) } return out.Bytes(), nil diff --git a/internal/adapter/executioncontroller/admin_lifecycle.go b/internal/adapter/executioncontroller/admin_lifecycle.go index 5ed26a8621..4c41e28f2e 100644 --- a/internal/adapter/executioncontroller/admin_lifecycle.go +++ b/internal/adapter/executioncontroller/admin_lifecycle.go @@ -320,11 +320,12 @@ func (s *Store) MigrateEnvironment(ctx context.Context, q adminLifecycleRequest) if textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype runtime identity mismatch"} } - if intNested(o.Object, "spec", "schemaVersion") == currentSchemaVersion && intNested(o.Object, "status", "schemaVersion") == currentSchemaVersion && textNested(o.Object, "status", "lastMigrationOperationID") == q.OperationID { + if completedMigrationMatches(o, q) { return nil } if migrationID := textNested(o.Object, "status", "migrationOperation", "id"); migrationID != "" { - if migrationID != q.OperationID || intNested(o.Object, "status", "migrationOperation", "fromSchema") != q.ExpectedSchema { + fromSchema, found, err := unstructured.NestedInt64(o.Object, "status", "migrationOperation", "fromSchema") + if migrationID != q.OperationID || err != nil || !found || fromSchema != q.ExpectedSchema { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another schema migration is active"} } return nil @@ -376,10 +377,11 @@ func (s *Store) MigrateEnvironment(ctx context.Context, q adminLifecycleRequest) if err := adminSubject(o, q); err != nil { return err } - if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion || textNested(o.Object, "status", "migrationOperation", "id") != q.OperationID || intNested(o.Object, "status", "schemaVersion") != q.ExpectedSchema { - if intNested(o.Object, "status", "schemaVersion") == currentSchemaVersion && textNested(o.Object, "status", "lastMigrationOperationID") == q.OperationID { - return nil - } + if completedMigrationMatches(o, q) { + return nil + } + fromSchema, found, schemaErr := unstructured.NestedInt64(o.Object, "status", "migrationOperation", "fromSchema") + if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion || textNested(o.Object, "status", "migrationOperation", "id") != q.OperationID || intNested(o.Object, "status", "schemaVersion") != q.ExpectedSchema || schemaErr != nil || !found || fromSchema != q.ExpectedSchema || textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "schema migration phase mismatch"} } epoch, ok := epochValue(o) @@ -389,11 +391,22 @@ func (s *Store) MigrateEnvironment(ctx context.Context, q adminLifecycleRequest) _ = unstructured.SetNestedField(o.Object, int64(epoch+1), "status", "epoch") //nolint:gosec // checked above. _ = unstructured.SetNestedField(o.Object, currentSchemaVersion, "status", "schemaVersion") _ = unstructured.SetNestedField(o.Object, q.OperationID, "status", "lastMigrationOperationID") + _ = unstructured.SetNestedField(o.Object, q.ExpectedSchema, "status", "lastMigrationFromSchema") unstructured.RemoveNestedField(o.Object, "status", "migrationOperation") return nil }) } +func completedMigrationMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + fromSchema, found, err := unstructured.NestedInt64(o.Object, "status", "lastMigrationFromSchema") + return err == nil && found && fromSchema == q.ExpectedSchema && + intNested(o.Object, "spec", "schemaVersion") == currentSchemaVersion && + intNested(o.Object, "status", "schemaVersion") == currentSchemaVersion && + textNested(o.Object, "status", "lastMigrationOperationID") == q.OperationID && + textNested(o.Object, "status", "pod", "uid") == q.ExpectedPodUID && + textNested(o.Object, "status", "pvc", "uid") == q.ExpectedPVCUID +} + func (s *Store) verifyRuntimeUIDs(ctx context.Context, o *unstructured.Unstructured, q adminLifecycleRequest) error { profile, ok := s.profiles.get(textNested(o.Object, "spec", "profile")) if !ok || profile.Digest != textNested(o.Object, "spec", "profileDigest") { diff --git a/internal/adapter/executioncontroller/admin_scope_security_test.go b/internal/adapter/executioncontroller/admin_scope_security_test.go index 9c3f02af5e..8e7992dce2 100644 --- a/internal/adapter/executioncontroller/admin_scope_security_test.go +++ b/internal/adapter/executioncontroller/admin_scope_security_test.go @@ -12,15 +12,7 @@ import ( func scopedManifestClient(t *testing.T, uri string, admin bool, scope []string) securityClientManifest { t.Helper() - raw, err := json.Marshal(map[string]any{"uri": uri, "administrator": admin, "administratorFor": scope}) - if err != nil { - t.Fatal(err) - } - var entry securityClientManifest - if err := json.Unmarshal(raw, &entry); err != nil { - t.Fatal(err) - } - return entry + return securityClientManifest{URI: uri, Administrator: admin, AdministratorFor: scope} } func TestAdministratorScopeManifestValidation(t *testing.T) { diff --git a/internal/adapter/executioncontroller/admin_scope_test.go b/internal/adapter/executioncontroller/admin_scope_test.go index 2f9f20255b..3bbf5fb4d3 100644 --- a/internal/adapter/executioncontroller/admin_scope_test.go +++ b/internal/adapter/executioncontroller/admin_scope_test.go @@ -354,12 +354,58 @@ func TestScopedAdminMigrationReceiptRetainsUIDPreconditions(t *testing.T) { if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); err != nil { t.Fatal(err) } + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); err != nil { + t.Fatalf("exact migration replay failed: %v", err) + } + q.ExpectedSchema = 0 + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("migration receipt bypassed source schema: %v", err) + } + q.ExpectedSchema = 1 q.ExpectedPVCUID = "stale" if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); status.Code(err) != codes.Aborted { t.Fatalf("migration receipt bypassed UID: %v", err) } } +func TestMigrationCompletedCASReplayRequiresExactSourceSchema(t *testing.T) { + for _, receipt := range []string{"exact", "changed", "absent"} { + t.Run(receipt, func(t *testing.T) { + f := newAdminScopeFixture(t, "migrate", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + var raced bool + f.dynamic.PrependReactor("update", "executionenvironments", func(action k8stesting.Action) (bool, runtime.Object, error) { + u := action.(k8stesting.UpdateAction).GetObject().(*unstructured.Unstructured) + if raced || textNested(u.Object, "status", "lastMigrationOperationID") == "" { + return false, nil, nil + } + raced = true + completed := u.DeepCopy() + switch receipt { + case "changed": + _ = unstructured.SetNestedField(completed.Object, int64(0), "status", "lastMigrationFromSchema") + case "absent": + unstructured.RemoveNestedField(completed.Object, "status", "lastMigrationFromSchema") + } + if err := f.dynamic.Tracker().Update(ExecutionEnvironmentGVR, completed, "ns"); err != nil { + t.Fatal(err) + } + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", nil) + }) + q := adminRequestFixture() + q.ExpectedSchema = 1 + err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner) + want := codes.Aborted + if receipt == "exact" { + want = codes.OK + } + if !raced || status.Code(err) != want { + t.Fatalf("CAS replay raced=%t code=%v, want %v", raced, status.Code(err), want) + } + }) + } +} + func TestScopedAdminEqualGenerationDriftFailsClosed(t *testing.T) { f := newAdminScopeFixture(t, "revoke", scopeAdmin) f.policy(t, true, []string{scopeCreator}) @@ -377,6 +423,74 @@ func TestScopedAdminEqualGenerationDriftFailsClosed(t *testing.T) { } } +func TestScopedAdminWithOwnerAttestationCannotUseAnotherCreatorsDataPlane(t *testing.T) { + f := newAdminScopeFixture(t, "data", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + f.manifest.Generation++ + f.manifest.Clients[0].MayAttestOwner = true + writeManifest(t, f.path, f.manifest) + if err := f.manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + ref := &executionv1.EnvironmentRef{Id: "env", Revision: "rev"} + owner := &executionv1.Owner{Issuer: scopeOwner.Issuer, Subject: scopeOwner.Subject} + before, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + refs, err := referenceRecords(before) + if err != nil { + t.Fatal(err) + } + refs = append(refs, referenceRecord{BindingID: "pending", State: executionenv.ReferencePendingCreate, OperationID: "pending-op", CreatedAt: time.Now().UTC()}) + if err := setReferenceRecords(before, refs); err != nil { + t.Fatal(err) + } + if _, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), before, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + for name, call := range map[string]func() error{ + "attach": func() error { + _, err := f.client.AttachEnvironment(t.Context(), &executionv1.AttachEnvironmentRequest{Context: &executionv1.RequestContext{Environment: ref, Owner: owner, BindingId: "binding"}, Purpose: "session"}) + return err + }, + "acquire": func() error { + _, err := f.client.AcquireRun(t.Context(), &executionv1.AcquireRunRequest{Environment: ref, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire", TtlMillis: 60000}) + return err + }, + "reference": func() error { + _, err := f.client.CommitReference(t.Context(), &executionv1.ReferenceMutationRequest{Environment: ref, Owner: owner, BindingId: "pending", OperationId: "pending-op"}) + return err + }, + "exact discovery": func() error { + _, err := f.client.ListReferenceIntents(t.Context(), &executionv1.ListReferenceIntentsRequest{Environment: ref, Owner: owner, BindingId: "pending"}) + return err + }, + } { + if err := call(); status.Code(err) != codes.NotFound { + t.Errorf("%s: expected creator-bound not found, got %v", name, err) + } + } + for _, query := range []*executionv1.ListReferenceIntentsRequest{{Owner: owner}, {}} { + list, err := f.client.ListReferenceIntents(t.Context(), query) + if err != nil || len(list.GetIntents()) != 0 { + t.Fatal("discovery exposed another creator's pending reference") + } + } + after, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(before.Object, after.Object) { + t.Fatal("creator-bound denials changed foreign environment") + } + own, err := f.client.EnsureEnvironment(t.Context(), &executionv1.EnsureEnvironmentRequest{BindingId: "own", Profile: "go", Owner: owner, OperationId: "own-create"}) + if err != nil || own.GetEnvironment().GetId() == "env" { + t.Fatalf("explicit owner attestation did not permit own allocation: %v", err) + } + list, err := f.client.ListReferenceIntents(t.Context(), &executionv1.ListReferenceIntentsRequest{Owner: owner}) + if err != nil || len(list.GetIntents()) != 1 || list.Intents[0].GetEnvironment().GetId() != own.GetEnvironment().GetId() { + t.Fatal("discovery did not isolate the actor's own pending allocation") + } +} + func TestScopedAdminDoesNotGrantAttestationOrDataPlane(t *testing.T) { f := newAdminScopeFixture(t, "data", scopeAdmin) f.policy(t, true, []string{scopeCreator}) diff --git a/internal/adapter/executioncontroller/crd_schema_test.go b/internal/adapter/executioncontroller/crd_schema_test.go index 1ffa9b0fd4..4ccfed4d8f 100644 --- a/internal/adapter/executioncontroller/crd_schema_test.go +++ b/internal/adapter/executioncontroller/crd_schema_test.go @@ -9,6 +9,7 @@ import ( apiextensionsvalidation "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation" "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" + apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/yaml" ) @@ -38,6 +39,26 @@ func TestExecutionEnvironmentCRDIsAcceptedByKubernetesValidation(t *testing.T) { } } +func TestMigrationReceiptSchemaAdmission(t *testing.T) { + crd, _ := loadExecutionEnvironmentCRD(t) + field := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"].Properties["lastMigrationFromSchema"] + var internal apiextensions.JSONSchemaProps + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(&field, &internal, nil); err != nil { + t.Fatal(err) + } + validator, _, err := apiservervalidation.NewSchemaValidator(&internal) + if err != nil { + t.Fatal(err) + } + for _, value := range []any{int64(0), int64(1), int64(-1), int64(2), "0"} { + errs := apiservervalidation.ValidateCustomResource(nil, value, validator) + valid := value == int64(0) || value == int64(1) + if (len(errs) == 0) != valid { + t.Fatalf("receipt source schema %v: valid=%t errors=%v", value, valid, errs) + } + } +} + func TestExecutionEnvironmentCRDPreservesControllerStatus(t *testing.T) { crd, _ := loadExecutionEnvironmentCRD(t) var internal apiextensions.JSONSchemaProps @@ -57,11 +78,11 @@ func TestExecutionEnvironmentCRDPreservesControllerStatus(t *testing.T) { "activeOperation": map[string]any{"id": "file", "operation": "file.read", "startedAt": "2026-09-17T00:00:00Z", "claimID": "claim", "runID": "run", "epoch": int64(3), "holderID": "holder", "renewedAt": "2026-09-17T00:00:01Z", "expiresAt": "2026-09-17T00:01:00Z"}, "lifecycleOperation": map[string]any{"id": "life", "type": "ReplaceExecutor", "phase": "Quiescing", "expectedEpoch": int64(3), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid", "createdAt": "2026-09-17T00:00:00Z"}, "terminationProof": map[string]any{"operationID": "life", "podUID": "pod-uid", "pvcUID": "pvc-uid", "epoch": int64(3), "podPhase": "Failed", "observedAt": "2026-09-17T00:00:00Z"}, - "migrationOperation": map[string]any{"id": "migration", "fromSchema": int64(1), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid"}, "lastMigrationOperationID": "migration", + "migrationOperation": map[string]any{"id": "migration", "fromSchema": int64(1), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid"}, "lastMigrationOperationID": "migration", "lastMigrationFromSchema": int64(0), "conditions": []any{map[string]any{"type": "Ready", "status": "True", "reason": "Reconciled", "message": "ready", "observedGeneration": int64(9), "lastTransitionTime": "2026-09-17T00:00:00Z"}}, }} pruning.Prune(obj, structural, true) - for _, path := range [][]string{{"status", "schemaVersion"}, {"status", "observedGeneration"}, {"status", "epoch"}, {"status", "grantGeneration"}, {"status", "references"}, {"status", "pvc"}, {"status", "pod"}, {"status", "activeRun", "grantGeneration"}, {"status", "renewReceipts"}, {"status", "activeOperation"}, {"status", "lifecycleOperation"}, {"status", "terminationProof"}, {"status", "migrationOperation"}, {"status", "lastMigrationOperationID"}, {"status", "conditions"}} { + for _, path := range [][]string{{"status", "schemaVersion"}, {"status", "observedGeneration"}, {"status", "epoch"}, {"status", "grantGeneration"}, {"status", "references"}, {"status", "pvc"}, {"status", "pod"}, {"status", "activeRun", "grantGeneration"}, {"status", "renewReceipts"}, {"status", "activeOperation"}, {"status", "lifecycleOperation"}, {"status", "terminationProof"}, {"status", "migrationOperation"}, {"status", "lastMigrationOperationID"}, {"status", "lastMigrationFromSchema"}, {"status", "conditions"}} { if _, found, err := unstructured.NestedFieldNoCopy(obj, path...); err != nil || !found { t.Fatalf("CRD pruning removed controller field %v (found=%t err=%v)", path, found, err) } diff --git a/internal/adapter/executioncontroller/production_lifecycle_test.go b/internal/adapter/executioncontroller/production_lifecycle_test.go index 06c72295c9..922df1dc5e 100644 --- a/internal/adapter/executioncontroller/production_lifecycle_test.go +++ b/internal/adapter/executioncontroller/production_lifecycle_test.go @@ -179,6 +179,17 @@ func TestOldSchemaFailsClosedAndExplicitMigrationConvertsReferences(t *testing.T if err != nil || len(refs) != 1 || refs[0].BindingID != "session-a" || refs[0].State != executionenv.ReferencePublished { t.Fatalf("migrated refs=%+v err=%v", refs, err) } + fromSchema, found, err := unstructured.NestedInt64(got.Object, "status", "lastMigrationFromSchema") + if err != nil || !found || fromSchema != 0 { + t.Fatal("schema-zero migration lost exact receipt presence") + } + unstructured.RemoveNestedField(got.Object, "status", "lastMigrationFromSchema") + if _, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), got, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := store.MigrateEnvironment(t.Context(), q); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("old receipt without source schema replayed: %v", err) + } } func TestInsecurePrototypeMigrationRejectedWithoutRewrite(t *testing.T) { diff --git a/user-docs/building/deployment/mecak8s.md b/user-docs/building/deployment/mecak8s.md index 5c23e619af..202e3d64d8 100644 --- a/user-docs/building/deployment/mecak8s.md +++ b/user-docs/building/deployment/mecak8s.md @@ -153,6 +153,12 @@ shared placement, no-FS, child-environment, and reattachment model. ### Kubernetes execution provider +**DRAFT candidate, not released or approved for production.** These instructions +apply to the optional native provider, not to the maturity of `mecak8s` itself. +Use an operator-controlled cluster. This slice supports foreground commands, +has no force-takeover recovery, and does not promise hostile multi-tenant +isolation. + The optional execution provider runs as a separate service and controller with its own `mecatl-execution` chart. `mecak8s` remains a client and receives no Pod, PVC, custom-resource, or controller management permissions. @@ -270,8 +276,9 @@ resourceGovernance: limitsEphemeralStorage: 80Gi ``` -Replace the all-zero `publicKeySHA256` with the SHA-256 fingerprint of the -Ed25519 public key corresponding to `grant-k1.pem`. Replace the sample activation +Replace the all-zero `publicKeySHA256` with the lowercase hexadecimal SHA-256 +hash of the **raw 32-byte Ed25519 public key** corresponding to `grant-k1.pem`, +not the PEM text or DER encoding. Replace the sample activation and verification dates with a current, reviewed rotation window. Increase `generation` for every authority change, including a CA, client policy, issuer/audience, key state, key window, or TLS identity change. @@ -351,6 +358,13 @@ names, profiles, network policy configuration, and security Secret name**. Keep operator configuration store. Retain the operator-owned Secret and its key history independently; the chart neither owns nor reads Secret contents. +**Before quiescing:** verify that the retained profiles ConfigMap has a nonempty +`data["lifetime.json"]` and that the authority/capacity ledgers and original +release ownership are intact. Pre-retention installations without this history +cannot be automatically adopted. Stop this procedure and recover trusted retained +history; do not synthesize it from proposed values or reset authority. The chart +reports missing history separately from an incompatible lifetime configuration. + For a compatible provider upgrade: 1. Stop new client traffic and finish or explicitly fence active work. Stop all @@ -414,11 +428,17 @@ Use live Helm install/upgrade for this lifecycle. Offline `helm template` cannot perform ownership or history lookups and is not an adoption mechanism. Keep provider writers stopped for upgrades; lookup plus apply is not a cross-resource transaction. Rendering rejects a nonzero existing provider Deployment or any -remaining provider Pod, including a terminating Pod. Installations without the retained lifetime configuration/history -cannot be automatically adopted. Changed release/namespace adoption, chart -rollback, `--take-ownership`, CRD or namespace deletion with retained resources, -and force-finalizer cleanup are unsupported. Missing history requires recovery -from trusted retained state, not a fresh generation-1 authority. +remaining provider Pod, including a terminating Pod. Changed release/namespace +adoption, chart rollback, `--take-ownership`, CRD or namespace deletion with +retained resources, and force-finalizer cleanup are unsupported. + +Final infrastructure decommission is not automated by this candidate. Supported +`RetireEnvironment` and `DeleteRetiredEnvironment` operations can terminate and +dispose of eligible individual environments; they do not remove the retained +NetworkPolicies, configuration, ledgers, or CRD. There is no supported final +infrastructure-cleanup procedure here. Manual destructive cleanup is outside the +supported lifecycle. Keep retention defaults and authority history, and never +reset ledgers or remove finalizers to bypass a failed safety check. #### Run an administrative lifecycle operation @@ -445,6 +465,14 @@ been removed. After maintenance, remove its scope entry and increase connections, are denied; already admitted lifecycle operations may finish safe reconciliation. +An administrative `not_found` response deliberately does not distinguish a +missing environment from a wrong owner, revision, or creator scope. Check the +exact environment ID/revision and original owner against your authorized +operations record, then check that `administratorFor` names the original +creator's exact canonical client URI. If the scope is wrong, publish a reviewed +manifest at a higher `generation` and verify readiness before retrying. The RPC +will not disclose another creator's data to diagnose a scope mismatch. + First capture the private identity while the environment still has a reference: ```sh diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index 1fc9b365a2..216dff4e16 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -80,6 +80,10 @@ using a local workspace. ## Native Kubernetes lifecycle and retention +**DRAFT candidate, not released or approved for production.** The native provider +is intended for an operator-controlled cluster and foreground commands only. +It has no force-takeover recovery and is not a hostile multi-tenancy boundary. + The optional Kubernetes execution provider stores environment ownership in a namespaced `ExecutionEnvironment`. Provider replicas coordinate through Kubernetes resource-version compare-and-swap; a replica restart does not clear @@ -157,7 +161,8 @@ this shape: ``` The all-zero fingerprint and 2027 dates are non-secret example values. Replace -them with the public-key fingerprint and a reviewed active window before use. +the fingerprint with the lowercase hexadecimal SHA-256 hash of the raw 32-byte +Ed25519 public key, not PEM text or DER encoding, and use a reviewed active window. `administratorFor` permits administration of environments created by the listed client URIs, alongside the administrator's own environments. A nonempty list From 643ed843b7ae0160c10aca5a86fd3d8d60a1e9ea Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Mon, 21 Sep 2026 22:21:10 +0300 Subject: [PATCH 15/25] fix(execution): expire migration receipts on executor replacement Co-Authored-By: OpenAI --- .../executioncontroller/admin_scope_test.go | 5 +- .../controller_lifecycle.go | 3 + .../production_lifecycle_test.go | 74 +++++++++++++++++++ user-docs/features/execution-environments.md | 4 +- 4 files changed, 84 insertions(+), 2 deletions(-) diff --git a/internal/adapter/executioncontroller/admin_scope_test.go b/internal/adapter/executioncontroller/admin_scope_test.go index 3bbf5fb4d3..b12fe48d09 100644 --- a/internal/adapter/executioncontroller/admin_scope_test.go +++ b/internal/adapter/executioncontroller/admin_scope_test.go @@ -369,7 +369,7 @@ func TestScopedAdminMigrationReceiptRetainsUIDPreconditions(t *testing.T) { } func TestMigrationCompletedCASReplayRequiresExactSourceSchema(t *testing.T) { - for _, receipt := range []string{"exact", "changed", "absent"} { + for _, receipt := range []string{"exact", "changed", "absent", "expired"} { t.Run(receipt, func(t *testing.T) { f := newAdminScopeFixture(t, "migrate", scopeAdmin) f.policy(t, true, []string{scopeCreator}) @@ -386,6 +386,9 @@ func TestMigrationCompletedCASReplayRequiresExactSourceSchema(t *testing.T) { _ = unstructured.SetNestedField(completed.Object, int64(0), "status", "lastMigrationFromSchema") case "absent": unstructured.RemoveNestedField(completed.Object, "status", "lastMigrationFromSchema") + case "expired": + unstructured.RemoveNestedField(completed.Object, "status", "lastMigrationOperationID") + unstructured.RemoveNestedField(completed.Object, "status", "lastMigrationFromSchema") } if err := f.dynamic.Tracker().Update(ExecutionEnvironmentGVR, completed, "ns"); err != nil { t.Fatal(err) diff --git a/internal/adapter/executioncontroller/controller_lifecycle.go b/internal/adapter/executioncontroller/controller_lifecycle.go index 503bb6170c..5f762fd36c 100644 --- a/internal/adapter/executioncontroller/controller_lifecycle.go +++ b/internal/adapter/executioncontroller/controller_lifecycle.go @@ -193,6 +193,9 @@ func (r *Reconciler) finishReplacement(ctx context.Context, env *unstructured.Un } _ = unstructured.SetNestedField(o.Object, epoch+1, "status", "epoch") _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pod.Name, "uid": string(pod.UID)}, "status", "pod") + // Completed migration receipts are bound to the runtime being replaced. + unstructured.RemoveNestedField(o.Object, "status", "lastMigrationOperationID") + unstructured.RemoveNestedField(o.Object, "status", "lastMigrationFromSchema") _ = unstructured.SetNestedMap(o.Object, map[string]any{operationIDField: operationID, "previousPodUID": text(op, "expectedPodUID"), "replacementPodUID": string(pod.UID), "pvcUID": text(op, "expectedPVCUID"), "previousEpoch": intNested(op, "expectedEpoch"), "replacementEpoch": epoch + 1}, "status", "lastReplacement") unstructured.RemoveNestedField(o.Object, "status", "lifecycleOperation") setConditionObject(o, "Ready", true, "ReplacementReady", "replacement executor is ready on the retained workspace") diff --git a/internal/adapter/executioncontroller/production_lifecycle_test.go b/internal/adapter/executioncontroller/production_lifecycle_test.go index 922df1dc5e..c62a9910c2 100644 --- a/internal/adapter/executioncontroller/production_lifecycle_test.go +++ b/internal/adapter/executioncontroller/production_lifecycle_test.go @@ -192,6 +192,80 @@ func TestOldSchemaFailsClosedAndExplicitMigrationConvertsReferences(t *testing.T } } +func TestMigrationReceiptExpiresAfterReconciledReplacement(t *testing.T) { + for name, schema := range map[string]int64{"legacy": 0, "prototype": 1} { + t.Run(name, func(t *testing.T) { + env := lifecycleAdminEnvironment(schema, []any{"session-a"}) + pod, pvc := terminalExecutor(), retainedPVC() + pod.Name, pvc.Name = resourceName("executor", "env"), resourceName("workspace", "env") + pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + _ = unstructured.SetNestedField(env.Object, pod.Name, "status", "pod", "name") + _ = unstructured.SetNestedField(env.Object, pvc.Name, "status", "pvc", "name") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(pod, pvc) + // The fake API must retain a finalizer-protected Pod until the real + // reconciler has recorded terminal proof and removed its finalizer. + k.PrependReactor("delete", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + obj, err := k.Tracker().Get(corev1.SchemeGroupVersion.WithResource("pods"), "ns", action.(k8stesting.DeleteAction).GetName()) + if err != nil { + return true, nil, err + } + return len(obj.(*corev1.Pod).Finalizers) != 0, nil, nil + }) + k.PrependReactor("create", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + created := action.(k8stesting.CreateAction).GetObject().(*corev1.Pod) + created.UID = "replacement-pod" + created.Status = replacementExecutor(true).Status + return false, nil, nil + }) + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + q := adminRequestFixture() + q.ExpectedSchema = schema + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatalf("exact receipt replay before replacement: %v", err) + } + replace := q + replace.OperationID = "replace-after-migration" + replace.ExpectedEpoch++ + if err := store.ReplaceExecutor(t.Context(), replace); err != nil { + t.Fatal(err) + } + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + // Drive every replacement phase, then an ordinary ready reconcile. + for range 8 { + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(got, "Ready") || textNested(got.Object, "status", "pod", "uid") != "replacement-pod" || textNested(got.Object, "status", "pvc", "uid") != q.ExpectedPVCUID || intNested(got.Object, "status", "epoch") != 6 { + t.Fatalf("replacement did not complete normally: %v", got.Object["status"]) + } + changed := q + changed.ExpectedPodUID = "replacement-pod" + for _, replay := range []adminLifecycleRequest{changed, q} { + var controlled *executionenv.Error + if err := store.MigrateEnvironment(t.Context(), replay); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("expired migration replay with Pod %q: %v", replay.ExpectedPodUID, err) + } + } + if textNested(got.Object, "status", "lastMigrationOperationID") != "" { + t.Fatal("replacement retained migration operation receipt") + } + if _, found, err := unstructured.NestedInt64(got.Object, "status", "lastMigrationFromSchema"); err != nil || found { + t.Fatal("replacement retained migration source schema receipt") + } + }) + } +} + func TestInsecurePrototypeMigrationRejectedWithoutRewrite(t *testing.T) { env := lifecycleAdminEnvironment(1, []any{"session-a"}) pod := terminalExecutor() diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index 216dff4e16..9ac7c5b3e8 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -110,7 +110,9 @@ Persisted prototype resources use an explicit administrator migration. Normal operations reject schema versions other than 2. `MigrateEnvironment` accepts only recognized versions 0 and 1, verifies the exact live Pod and PVC identities, and requires a healthy, idle environment. Unknown or malformed state is retained -unchanged rather than reset. +unchanged rather than reset. Completed migration receipts expire when executor +replacement publishes a new Pod identity. After replacement, retrying the old +migration operation returns a conflict with either the original or replacement UID. ### Production security material From 0df361480b2b9c1b9a02ca5c7f55362d0e257966 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Mon, 21 Sep 2026 23:08:09 +0300 Subject: [PATCH 16/25] fix(execution): wait for quota accounting and enforce remote conformance Co-Authored-By: OpenAI --- cmd/mecak8s/execution_startup_test.go | 70 ++++++++++ docs/design/IMPLEMENTATION-NOTES.md | 7 +- e2e/k8s_execution/file_tools_test.go | 61 +++++++++ e2e/k8s_execution/qualification_test.go | 1 + internal/adapter/executionclient/client.go | 9 +- .../executionclient/files_conformance_test.go | 124 ++++++++++++++++++ .../service_integration_test.go | 13 +- .../disabled_composition_test.go | 95 ++++++++++++++ .../file_authorization_test.go | 101 ++++++++++++++ .../adapter/executioncontroller/handler.go | 2 +- .../legacy_fixture_kind.go | 21 +++ .../legacy_fixture_kind_test.go | 83 ++++++++++++ internal/app/build.go | 20 ++- internal/app/project_ingestion.go | 4 +- internal/app/remote_execution_test.go | 90 +++++++++++-- internal/executionenv/errors.go | 3 +- .../executionexecutor/confinement_test.go | 76 +++++++++++ internal/executionexecutor/executor.go | 2 +- user-docs/features/execution-environments.md | 10 ++ 19 files changed, 768 insertions(+), 24 deletions(-) create mode 100644 cmd/mecak8s/execution_startup_test.go create mode 100644 e2e/k8s_execution/file_tools_test.go create mode 100644 internal/adapter/executionclient/files_conformance_test.go create mode 100644 internal/adapter/executioncontroller/disabled_composition_test.go create mode 100644 internal/adapter/executioncontroller/file_authorization_test.go create mode 100644 internal/adapter/executioncontroller/legacy_fixture_kind_test.go create mode 100644 internal/executionexecutor/confinement_test.go diff --git a/cmd/mecak8s/execution_startup_test.go b/cmd/mecak8s/execution_startup_test.go new file mode 100644 index 0000000000..06c2812538 --- /dev/null +++ b/cmd/mecak8s/execution_startup_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" +) + +func TestDisabledExecutionStartupDoesNotContactExecutionOrKubernetes(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "config")) + t.Setenv("XDG_STATE_HOME", filepath.Join(home, "state")) + var requests atomic.Int32 + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer endpoint.Close() + kubeconfig := filepath.Join(home, "kubeconfig") + contents := "apiVersion: v1\nkind: Config\nclusters:\n- name: poison\n cluster:\n server: " + endpoint.URL + "\ncontexts:\n- name: poison\n context:\n cluster: poison\n user: offline\ncurrent-context: poison\nusers:\n- name: offline\n user: {}\n" + if err := os.WriteFile(kubeconfig, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("KUBECONFIG", kubeconfig) + oldArgs, oldLogger := os.Args, slog.Default() + t.Cleanup(func() { os.Args = oldArgs; slog.SetDefault(oldLogger) }) + // Exercise run(), including actual app.Build, stopping at the first listener. + // Poison TLS paths would fail before serve if the disabled client were built. + os.Args = []string{"mecak8s", "--mock", "--posture=strict", "--no-soul", "--no-user-model", "--no-scheduler", "--session-lease-k8s-namespace=", "--grpc-addr=invalid-address", "--execution-enabled=false", "--execution-endpoint=" + strings.TrimPrefix(endpoint.URL, "http://"), "--execution-profile=unused", "--execution-tls-ca=" + filepath.Join(home, "absent-ca"), "--execution-tls-cert=" + filepath.Join(home, "absent-cert"), "--execution-tls-key=" + filepath.Join(home, "absent-key")} + err := run() + if err == nil || !strings.Contains(err.Error(), "invalid-address") { + t.Fatalf("did not reach listener after composition: %v", err) + } + if requests.Load() != 0 { + t.Fatalf("disabled startup made %d API requests", requests.Load()) + } +} + +func TestExecutionPreflightRejectsIncompleteAndIncompatibleCLI(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + required := []string{"--execution-endpoint=127.0.0.1:1", "--execution-profile=go", "--execution-tls-ca=unused", "--execution-tls-cert=unused", "--execution-tls-key=unused"} + base := []string{"--mock", "--execution-enabled", "--oidc-issuer=https://issuer.example", "--oidc-audience=mecatl"} + if _, err := parseFlags(append(append([]string{}, base...), required...)); err != nil { + t.Fatalf("complete configuration rejected: %v", err) + } + for missing := range required { + args := append([]string{}, base...) + for i, flag := range required { + if i != missing { + args = append(args, flag) + } + } + if _, err := parseFlags(args); err == nil || !strings.Contains(err.Error(), "--execution-enabled requires") { + t.Fatalf("missing %s: %v", required[missing], err) + } + } + for _, extra := range []string{"--workspace=unused", "--redis-filesystem", "--enable-parallel", "--enable-teams"} { + args := append(append(append([]string{}, base...), required...), extra) + if _, err := parseFlags(args); err == nil { + t.Fatalf("incompatible %s accepted", extra) + } + } +} diff --git a/docs/design/IMPLEMENTATION-NOTES.md b/docs/design/IMPLEMENTATION-NOTES.md index dc0147f0ba..5f56c0bdbd 100644 --- a/docs/design/IMPLEMENTATION-NOTES.md +++ b/docs/design/IMPLEMENTATION-NOTES.md @@ -6566,7 +6566,12 @@ has no service-account token and contains the fixed helper built by fail closed before the external authority evaluator runs; there is no lexical client-side substitute. -Composition selects this path only when `Config.RemoteExecution` is set. The +Composition selects this path only when `Config.RemoteExecution` is set. +`internal/app/project_ingestion.go` (`projectIngestionAdmitted`) rejects host-project +steering for remote deployments even when that local root is trusted. The remote +instruction variant retains user-tier rules; command listing never opens a local +workspace and the remote engine uses the no-op command expander. Operator-global +skills remain available. The remote catalog excludes local project ingestion, schedules, SkillDraft, Parallel, and Team, while explicit `profile:"no-fs"` still selects the existing no-FS placement. The `mecatl-execution` and `mecak8s` charts remain independent. Run claims, diff --git a/e2e/k8s_execution/file_tools_test.go b/e2e/k8s_execution/file_tools_test.go new file mode 100644 index 0000000000..65b88f56cb --- /dev/null +++ b/e2e/k8s_execution/file_tools_test.go @@ -0,0 +1,61 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/stacklok/mecatl/engine/adapter/fstools" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func qualifyRemoteFileTools(t *testing.T, ctx context.Context, client *executionclient.Client, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) { + t.Helper() + provider, err := executionclient.NewProvider(client, "go") + if err != nil { + t.Fatal(err) + } + handle, err := provider.AcquireRun(ctx, server.ExecutionRunRequest{Ref: session.EnvironmentRef{Kind: "kubernetes", ID: ref.ID, Revision: ref.Revision}, Principal: &session.Principal{Issuer: owner.Issuer, Subject: owner.Subject, GrantType: session.GrantTypeUser}, BindingID: session.SessionID(binding), RunID: "file-tool-matrix"}) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := handle.Release(ctx); err != nil { + t.Error(err) + } + }() + env := handle.Environment() + call := func(body tool.Tool, args string, wantError bool, contains string) { + t.Helper() + result, err := body.Execute(ctx, session.ToolCall{ID: "matrix", Name: body.Spec().Name, Args: json.RawMessage(args)}, env) + if err != nil || result.IsError != wantError || !strings.Contains(result.Content, contains) { + t.Fatalf("%s: result=%+v err=%v", body.Spec().Name, result, err) + } + } + if _, err := env.Workspace().CreateFile(ctx, "matrix/source.go", []byte("package matrix\n// before\n")); err != nil { + t.Fatal(err) + } + call(fstools.EditTool{}, `{"path":"matrix/source.go","old_string":"before","new_string":"after"}`, true, "") + call(fstools.WriteTool{}, `{"path":"matrix/source.go","content":"clobber"}`, true, "") + call(fstools.CopyTool{}, `{"source":"matrix/source.go","destination":"matrix/copied.go"}`, false, "") + call(fstools.MoveTool{}, `{"source":"matrix/copied.go","destination":"matrix/moved.go"}`, false, "") + call(fstools.RemoveTool{}, `{"path":"matrix/moved.go"}`, false, "") + call(fstools.ReadTool{}, `{"path":"matrix/source.go"}`, false, "before") + call(fstools.EditTool{}, `{"path":"matrix/source.go","old_string":"before","new_string":"after"}`, false, "") + call(fstools.WriteTool{}, `{"path":"matrix/source.go","content":"package matrix\n// final\n"}`, false, "") + call(fstools.WriteTool{}, `{"path":"matrix/new.go","content":"package matrix\n"}`, false, "") + call(fstools.ListDirTool{}, `{"path":"matrix"}`, false, "source.go") + call(fstools.GrepTool{}, `{"path":"matrix/*.go","pattern":"final"}`, false, "final") + call(fstools.GlobTool{}, `{"pattern":"matrix/*.go"}`, false, "source.go") + data, err := env.Workspace().Read(ctx, "matrix/source.go") + if err != nil || string(data) != "package matrix\n// final\n" { + t.Fatalf("persisted matrix=%q err=%v", data, err) + } +} diff --git a/e2e/k8s_execution/qualification_test.go b/e2e/k8s_execution/qualification_test.go index 53773e6090..6fc28b8f39 100644 --- a/e2e/k8s_execution/qualification_test.go +++ b/e2e/k8s_execution/qualification_test.go @@ -165,6 +165,7 @@ func TestKindExecutionQualification(t *testing.T) { t.Fatalf("independent go test proof failed: err=%v state=%s", err, verification.State) } releaseVerification() + qualifyRemoteFileTools(t, ctx, providerClient, owner, sessionID, lookup) if status := getSession(t, ctx, agentForward.addr, sessionID, bob); status != http.StatusNotFound { t.Fatalf("different OIDC owner read status = %d, want 404", status) } diff --git a/internal/adapter/executionclient/client.go b/internal/adapter/executionclient/client.go index 7dfca295d4..9165a4be8e 100644 --- a/internal/adapter/executionclient/client.go +++ b/internal/adapter/executionclient/client.go @@ -350,7 +350,7 @@ func grpcCodeForError(code executionenv.ErrorCode) codes.Code { return codes.NotFound case executionenv.CodeAlreadyExists: return codes.AlreadyExists - case executionenv.CodeConflict, executionenv.CodeVersionMismatch: + case executionenv.CodeConflict, executionenv.CodeVersionMismatch, executionenv.CodeDirectoryNotEmpty: return codes.Aborted case executionenv.CodeNotReady: return codes.Unavailable @@ -870,7 +870,7 @@ func mapPlacementError(err error) error { switch remote.Code { case executionenv.CodeNotFound, executionenv.CodePermissionDenied, executionenv.CodeUnauthenticated: return server.ErrPlacementNotFound - case executionenv.CodeConflict, executionenv.CodeVersionMismatch: + case executionenv.CodeConflict, executionenv.CodeVersionMismatch, executionenv.CodeDirectoryNotEmpty: return server.ErrPlacementChanged case executionenv.CodeInvalidArgument: return server.ErrInvalidPlacementSelection @@ -1030,6 +1030,9 @@ func (w *workspace) CreateFile(ctx context.Context, path string, data []byte) (t func (w *workspace) ReplaceFile(ctx context.Context, path string, old tool.FileVersion, data []byte) (tool.FileVersion, error) { v, e := tool.EncodeFileVersion(old) if e != nil { + if _, statErr := w.Stat(ctx, path); statErr != nil { + return tool.FileVersion{}, statErr + } return tool.FileVersion{}, e } r, e := w.call(ctx, executionenv.OpFileReplace, path, "", "", data, v) @@ -1095,6 +1098,8 @@ func mapFileError(path string, err error) error { return fmt.Errorf("%s: %w", path, fs.ErrExist) case executionenv.CodeVersionMismatch: return &tool.VersionMismatchError{Path: path} + case executionenv.CodeDirectoryNotEmpty: + return fmt.Errorf("%s: %w", path, tool.ErrDirectoryNotEmpty) } } return err diff --git a/internal/adapter/executionclient/files_conformance_test.go b/internal/adapter/executionclient/files_conformance_test.go new file mode 100644 index 0000000000..ebf9985ef8 --- /dev/null +++ b/internal/adapter/executionclient/files_conformance_test.go @@ -0,0 +1,124 @@ +package executionclient + +import ( + "context" + "encoding/json" + "errors" + "io/fs" + "strings" + "testing" + + "github.com/stacklok/mecatl/engine/adapter/fsconformance" + "github.com/stacklok/mecatl/engine/adapter/fstools" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" + "github.com/stacklok/mecatl/internal/executionexecutor" +) + +type executorBackend struct { + integrationBackend + executor *executionexecutor.Executor +} + +func (b *executorBackend) File(ctx context.Context, _, _ string, q executionenv.FileRequest) (executionenv.FileResponse, error) { + r, err := b.executor.Execute(ctx, executionenv.ExecutorRequest{Operation: q.Operation, Path: q.Path, Destination: q.Destination, Pattern: q.Pattern, Data: q.Data, Version: q.Version, Limit: q.Limit}) + return r.FileResponse, err +} + +func remoteExecutorEnvironment(t *testing.T) tool.Environment { + t.Helper() + x, err := executionexecutor.New(t.TempDir(), executionexecutor.Limits{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = x.Close() }) + fx := startFixture(t, &executorBackend{executor: x}, nil) + t.Cleanup(fx.stop) + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + t.Cleanup(client.Close) + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + binding, err := provider.Bind(t.Context(), server.PlacementBindRequest{Selector: server.DefaultPlacement(), Principal: principal, BindingID: "files"}) + if err != nil { + t.Fatal(err) + } + if err := binding.Commit(t.Context()); err != nil { + t.Fatal(err) + } + handle, err := provider.AcquireRun(t.Context(), server.ExecutionRunRequest{Ref: binding.Ref, Principal: principal, BindingID: "files", RunID: "run-files"}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = handle.Release(context.Background()) }) + return handle.Environment() +} + +func TestRemoteWorkspaceConformanceThroughSignedGRPC(t *testing.T) { + factory := func(t *testing.T) tool.Workspace { return remoteExecutorEnvironment(t).Workspace() } + fsconformance.Run(t, factory) + fsconformance.RunNamespace(t, factory) +} + +func TestRemoteFileToolsPreserveLedgerAndNamespaceContracts(t *testing.T) { + env := remoteExecutorEnvironment(t) + ctx := t.Context() + call := func(body tool.Tool, args string, wantError bool, contains string) { + t.Helper() + result, err := body.Execute(ctx, session.ToolCall{ID: "file", Name: body.Spec().Name, Args: json.RawMessage(args)}, env) + if err != nil || result.IsError != wantError || !strings.Contains(result.Content, contains) { + t.Fatalf("%s: result=%+v error=%v", body.Spec().Name, result, err) + } + } + if _, err := env.Workspace().CreateFile(ctx, "main.go", []byte("package main\n// before\n")); err != nil { + t.Fatal(err) + } + call(fstools.EditTool{}, `{"path":"main.go","old_string":"before","new_string":"after"}`, true, "") + call(fstools.WriteTool{}, `{"path":"main.go","content":"clobber"}`, true, "") + call(fstools.ReadTool{}, `{"path":"main.go"}`, false, "before") + call(fstools.EditTool{}, `{"path":"main.go","old_string":"before","new_string":"after"}`, false, "") + call(fstools.ReadTool{}, `{"path":"main.go"}`, false, "after") + // A mutation outside the read ledger invalidates both Edit and existing Write. + _, version, err := env.Workspace().ReadVersion(ctx, "main.go") + if err != nil { + t.Fatal(err) + } + if _, err := env.Workspace().ReplaceFile(ctx, "main.go", version, []byte("package main\n// concurrent\n")); err != nil { + t.Fatal(err) + } + call(fstools.EditTool{}, `{"path":"main.go","old_string":"concurrent","new_string":"lost"}`, true, "") + call(fstools.WriteTool{}, `{"path":"main.go","content":"lost"}`, true, "") + call(fstools.ReadTool{}, `{"path":"main.go"}`, false, "concurrent") + call(fstools.WriteTool{}, `{"path":"main.go","content":"package main\n// final\n"}`, false, "") + call(fstools.ListDirTool{}, `{"path":"."}`, false, "main.go") + call(fstools.GrepTool{}, `{"path":"*.go","pattern":"final"}`, false, "final") + call(fstools.GlobTool{}, `{"pattern":"**/*.go"}`, false, "main.go") + call(fstools.WriteTool{}, `{"path":"fresh.go","content":"package main\n"}`, false, "") + // Seed without Write's automatic ledger update: namespace operations need no ledger. + if _, err := env.Workspace().CreateFile(ctx, "unread.go", []byte("package unread\n")); err != nil { + t.Fatal(err) + } + call(fstools.CopyTool{}, `{"source":"unread.go","destination":"copied.go"}`, false, "") + call(fstools.MoveTool{}, `{"source":"copied.go","destination":"moved.go"}`, false, "") + data, err := env.Workspace().Read(ctx, "moved.go") + if err != nil || string(data) != "package unread\n" { + t.Fatalf("moved=%q error=%v", data, err) + } + call(fstools.CopyTool{}, `{"source":"unread.go","destination":"main.go"}`, true, "") + call(fstools.MoveTool{}, `{"source":"moved.go","destination":"main.go"}`, true, "") + call(fstools.RemoveTool{}, `{"path":"moved.go"}`, false, "") + if _, err := env.Workspace().Stat(ctx, "moved.go"); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("removed file: %v", err) + } + data, err = env.Workspace().Read(ctx, "main.go") + if err != nil || string(data) != "package main\n// final\n" { + t.Fatalf("final=%q error=%v", data, err) + } +} diff --git a/internal/adapter/executionclient/service_integration_test.go b/internal/adapter/executionclient/service_integration_test.go index 9a8381bccc..84bb26e343 100644 --- a/internal/adapter/executionclient/service_integration_test.go +++ b/internal/adapter/executionclient/service_integration_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "sync/atomic" "testing" "time" @@ -87,9 +88,10 @@ func TestServiceUsesRealMTLSProviderStoreAndReleasesOnlyAfterDrain(t *testing.T) } catalog := tool.NewCatalog() catalog.MustRegister(adaptertools.ReadTool{}) - catalog.MustRegister(adaptertools.NewShellTool()) + catalog.MustRegister(agent.NewShellTool()) llm := mockllm.New( mockllm.ToolCallTurn(session.NewToolCall("read", "Read", json.RawMessage(`{"path":"main.go"}`))), + mockllm.ToolCallTurn(session.NewToolCall("background", "Shell", json.RawMessage(`{"command":"touch background-leak","background":true}`))), mockllm.ToolCallTurn(session.NewToolCall("shell", "Shell", json.RawMessage(`{"command":"true"}`))), mockllm.TextTurn("done"), mockllm.TextTurn("continued"), @@ -114,7 +116,14 @@ func TestServiceUsesRealMTLSProviderStoreAndReleasesOnlyAfterDrain(t *testing.T) if err != nil { t.Fatal(err) } - for range run.Events() { + backgroundRejected := false + for event := range run.Events() { + if result := event.ToolResult; result != nil && result.CallID == "background" { + backgroundRejected = result.IsError && strings.Contains(result.Content, "not supported by this command runner") + } + } + if !backgroundRejected { + t.Fatal("remote background Shell did not return the named capability error") } if executor.calls.Load() != 3 { t.Fatalf("executor calls=%d, want authority+read+shell", executor.calls.Load()) diff --git a/internal/adapter/executioncontroller/disabled_composition_test.go b/internal/adapter/executioncontroller/disabled_composition_test.go new file mode 100644 index 0000000000..3de7bfe3da --- /dev/null +++ b/internal/adapter/executioncontroller/disabled_composition_test.go @@ -0,0 +1,95 @@ +package executioncontroller + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/internal/app" +) + +func TestDisabledCompositionPreservesAllocationsAndIndependentReconciliation(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + t.Setenv("XDG_STATE_HOME", t.TempDir()) + ctx := t.Context() + env := testEnvironment() + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewClientset() + provider := NewReconciler(d, k, "ns", testProfiles()) + // First persist the finalizer, then provision the runtime. + if err := provider.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := provider.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + resources := d.Resource(ExecutionEnvironmentGVR).Namespace("ns") + before, err := resources.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + pods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pods.Items) != 1 { + t.Fatalf("pods=%v err=%v", pods, err) + } + pvcs, err := k.CoreV1().PersistentVolumeClaims("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pvcs.Items) != 1 { + t.Fatalf("pvcs=%v err=%v", pvcs, err) + } + d.ClearActions() + k.ClearActions() + // No execution integration is passed to the real composition root. Existing + // runtime objects belong to the independent provider, not the harness lifetime. + built, err := app.Build(ctx, app.Config{UseMock: true, NoSoul: true, NoUserModel: true}) + if err != nil { + t.Fatal(err) + } + sess, err := built.Service.CreateSession(ctx, session.ModeDefault, session.Limits{}) + if err != nil { + built.Close() + t.Fatal(err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "offline") + if err != nil { + built.Close() + t.Fatal(err) + } + for range run.Events() { + } + built.Service.FinishRun(sess.ID, run) + built.Close() + if len(d.Actions()) != 0 || len(k.Actions()) != 0 { + t.Fatal("disabled composition touched provider resources") + } + after, err := resources.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(before, after) { + t.Fatalf("allocation changed: %v", err) + } + afterPods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || !reflect.DeepEqual(pods.Items, afterPods.Items) { + t.Fatalf("executor changed: %v", err) + } + afterPVCs, err := k.CoreV1().PersistentVolumeClaims("ns").List(ctx, metav1.ListOptions{}) + if err != nil || !reflect.DeepEqual(pvcs.Items, afterPVCs.Items) { + t.Fatalf("storage changed: %v", err) + } + pod := pods.Items[0].DeepCopy() + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := k.CoreV1().Pods("ns").UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := provider.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + ready, err := resources.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil || !conditionTrue(ready, "Ready") { + t.Fatalf("independent provider failed to converge: %v", err) + } +} diff --git a/internal/adapter/executioncontroller/file_authorization_test.go b/internal/adapter/executioncontroller/file_authorization_test.go new file mode 100644 index 0000000000..d7558b6048 --- /dev/null +++ b/internal/adapter/executioncontroller/file_authorization_test.go @@ -0,0 +1,101 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type countedFileBackend struct { + *fakeBackend + calls int +} + +func (b *countedFileBackend) File(context.Context, string, string, executionenv.FileRequest) (executionenv.FileResponse, error) { + b.calls++ + return executionenv.FileResponse{}, nil +} + +func TestEveryFileOperationRequiresExactCurrentGrant(t *testing.T) { + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + const client = "spiffe://cluster/ns/mecak8s" + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + for value, name := range executionv1.FileOperation_name { + op, ok := operationFromProto(executionv1.FileOperation(value)) + if !ok { + continue + } + t.Run(name, func(t *testing.T) { + for _, failure := range []string{"client", "owner", "environment", "revision", "epoch", "operation", "expired", "revoked"} { + t.Run(failure, func(t *testing.T) { + backend := &countedFileBackend{fakeBackend: newFakeBackend()} + verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"key": public}, Issuer: "provider", Audience: "executor", MaxLifetime: time.Minute, Now: func() time.Time { return now }} + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{client: {MayAttestOwner: true}}, Verifier: verifier}, backend) + claims := executionenv.GrantClaims{KeyID: "key", Issuer: "provider", Audience: "executor", Client: client, OwnerHash: ownerHash(owner), BindingID: "binding", RunID: "run", ClaimID: "claim", Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, Epoch: 1, GrantGeneration: 1, Operations: []executionenv.Operation{op}, NotBefore: now.Add(-time.Second), ExpiresAt: now.Add(30 * time.Second), Nonce: "nonce"} + sign := func() string { + token, err := executionenv.SignGrant(private, claims) + if err != nil { + t.Fatal(err) + } + return token + } + req := &executionv1.FileRequest{Context: &executionv1.RequestContext{Environment: refToProto(claims.Environment), Owner: &executionv1.Owner{Issuer: owner.Issuer, Subject: owner.Subject}, BindingId: "binding", RunId: "run", ClaimId: "claim", Epoch: 1, GrantGeneration: 1, Grant: sign()}, Operation: executionv1.FileOperation(value), Path: "file"} + switch op { + case executionenv.OpFileReplace: + req.Version = []byte("version") + case executionenv.OpFileCopy, executionenv.OpFileRename: + req.Destination = "destination" + case executionenv.OpFileGlob: + req.Path = "" + req.Pattern = "*" + case executionenv.OpFileGrep: + req.Pattern = "text" + } + ctx := authenticatedContext(client) + if _, err := h.Files(ctx, req); err != nil || backend.calls != 1 { + t.Fatalf("positive control: calls=%d err=%v", backend.calls, err) + } + negative := proto.Clone(req).(*executionv1.FileRequest) + want := codes.PermissionDenied + switch failure { + case "client": + claims.Client = "spiffe://cluster/ns/other" + case "owner": + claims.OwnerHash = "other" + case "environment": + claims.Environment.ID = "other" + case "revision": + claims.Environment.Revision = "other" + case "epoch": + claims.Epoch++ + case "operation": + claims.Operations = []executionenv.Operation{executionenv.OpCommandStart} + case "expired": + claims.NotBefore = now.Add(-time.Minute) + claims.ExpiresAt = now.Add(-time.Second) + want = codes.Unauthenticated + case "revoked": + h.cfg.Verifier.RevokedNonces = map[string]struct{}{"nonce": {}} + } + negative.Context.Grant = sign() + if _, err := h.Files(ctx, negative); status.Code(err) != want || backend.calls != 1 { + t.Fatalf("negative control: calls=%d code=%v err=%v", backend.calls, status.Code(err), err) + } + }) + } + }) + } +} diff --git a/internal/adapter/executioncontroller/handler.go b/internal/adapter/executioncontroller/handler.go index 2b9e635bfe..a929fcfdd1 100644 --- a/internal/adapter/executioncontroller/handler.go +++ b/internal/adapter/executioncontroller/handler.go @@ -870,7 +870,7 @@ func wireError(code executionenv.ErrorCode, retry bool) error { grpcCode = codes.NotFound case executionenv.CodeAlreadyExists: grpcCode = codes.AlreadyExists - case executionenv.CodeConflict, executionenv.CodeVersionMismatch: + case executionenv.CodeConflict, executionenv.CodeVersionMismatch, executionenv.CodeDirectoryNotEmpty: grpcCode = codes.Aborted case executionenv.CodeNotReady: grpcCode = codes.Unavailable diff --git a/internal/adapter/executioncontroller/legacy_fixture_kind.go b/internal/adapter/executioncontroller/legacy_fixture_kind.go index c2f7a1260f..664a59f6a1 100644 --- a/internal/adapter/executioncontroller/legacy_fixture_kind.go +++ b/internal/adapter/executioncontroller/legacy_fixture_kind.go @@ -12,6 +12,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" @@ -68,6 +69,26 @@ type seededLegacyEnvironment struct { } func seedOneLegacyEnvironment(ctx context.Context, d dynamic.Interface, kube kubernetes.Interface, namespace string, profile resolvedProfile, owner executionenv.Owner, client, name, binding string, references []any, insecure bool) (seededLegacyEnvironment, error) { + // Deployment readiness does not imply that quota admission has initialized + // accounting, especially for the freshly installed custom resource. + if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + quota, err := kube.CoreV1().ResourceQuotas(namespace).Get(ctx, "mecatl-execution", metav1.GetOptions{}) + if err != nil { + return false, err + } + for name, configured := range quota.Spec.Hard { + hard, ok := quota.Status.Hard[name] + if !ok || hard.Cmp(configured) != 0 { + return false, nil + } + if _, ok := quota.Status.Used[name]; !ok { + return false, nil + } + } + return true, nil + }); err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("wait for legacy fixture quota accounting: %w", err) + } revision, err := randomID() if err != nil { return seededLegacyEnvironment{}, err diff --git a/internal/adapter/executioncontroller/legacy_fixture_kind_test.go b/internal/adapter/executioncontroller/legacy_fixture_kind_test.go new file mode 100644 index 0000000000..5fcce3f0c8 --- /dev/null +++ b/internal/adapter/executioncontroller/legacy_fixture_kind_test.go @@ -0,0 +1,83 @@ +//go:build kind_execution_e2e + +package executioncontroller + +import ( + "context" + "errors" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestLegacyFixtureWaitsForQuotaAccountingBeforeCreate(t *testing.T) { + for _, mode := range []string{"delayed", "timeout", "cancel", "forbidden"} { + t.Run(mode, func(t *testing.T) { + quota := &corev1.ResourceQuota{ObjectMeta: metav1.ObjectMeta{Name: "mecatl-execution", Namespace: "test"}, Spec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{corev1.ResourceName("count/executionenvironments.execution.mecatl.dev"): resource.MustParse("10"), corev1.ResourcePods: resource.MustParse("10")}}} + kube := kubefake.NewClientset() + reads, creates := 0, 0 + ctx, cancel := context.WithTimeout(context.Background(), 350*time.Millisecond) + defer cancel() + kube.PrependReactor("get", "resourcequotas", func(ktesting.Action) (bool, runtime.Object, error) { + reads++ + if mode == "forbidden" { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Resource: "resourcequotas"}, quota.Name, errors.New("denied")) + } + // Partially initialized status must not pass: all configured resources matter. + quota.Status.Hard = quota.Spec.Hard.DeepCopy() + quota.Status.Used = corev1.ResourceList{corev1.ResourcePods: resource.MustParse("0")} + if mode == "delayed" && reads >= 2 { + quota.Status.Used = quota.Spec.Hard.DeepCopy() + } + if mode == "cancel" { + cancel() + } + return true, quota.DeepCopy(), nil + }) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + stop := errors.New("create reached") + d.PrependReactor("create", "executionenvironments", func(ktesting.Action) (bool, runtime.Object, error) { + creates++ + if reads < 2 || mode != "delayed" { + t.Error("create before quota accounting initialized") + } + return true, nil, stop + }) + _, err := seedOneLegacyEnvironment(ctx, d, kube, "test", resolvedProfile{}, executionenv.Owner{}, "client", "legacy", "binding", nil, false) + if mode == "delayed" { + if !errors.Is(err, stop) || creates != 1 { + t.Fatalf("creates=%d reads=%d error=%v", creates, reads, err) + } + return + } + if err == nil || creates != 0 { + t.Fatalf("creates=%d error=%v", creates, err) + } + switch mode { + case "timeout": + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) + } + case "cancel": + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + case "forbidden": + if !apierrors.IsForbidden(err) || reads != 1 { + t.Fatalf("reads=%d error=%v", reads, err) + } + } + }) + } +} diff --git a/internal/app/build.go b/internal/app/build.go index b85a88010c..1969796cc4 100644 --- a/internal/app/build.go +++ b/internal/app/build.go @@ -3255,7 +3255,7 @@ func sessionEngineFactoryWithTools( learningCfg.automaticAdmissionLedger = assets.automaticAdmissionLedger learningCfg.learningSourceStore = store deps := engineDepsForProvider(cfg, resolvedProvider, resolvedModel, windowFn, store, sessionPolicy, hooks, mcpProvider, sessionInstructions) - deps.PromptConfig = applyRemoteExecutionPosture(deps.PromptConfig, remote) + deps = applyRemoteExecutionPosture(deps, remote) attachOperatorProfile(&deps, assets.userModelStore) deps.LearningMode = learningCfg.LearningMode deps.LearningObserver = bindMaterializationLifecycle(buildReflectionObserver(learningCfg, resolvedProvider, learningCfg.Model, assets.userModelStore, assets.memStore, assets.reflectionRepository, assets.reflectionCoordinator, assets.learningAdmission, buildProcedureProcessor(learningCfg, assets)), assets.reflectionLifecycle) @@ -4132,9 +4132,14 @@ func buildEngine(ctx context.Context, cfg Config, reg *providerRegistry, provide // the tier-0 project MemoryIndexAssembler. Operator facts no longer ride a // turn-0 user fragment; the same user store is loaded per request into the // volatile system-prompt suffix below. + remoteRules := prompt.RulesSource(nil) + if cfg.RemoteExecution { + // Project admission is off for this deployment; retain user-tier rules. + remoteRules = rulesSrc + } instructions := sessionInstructionSet{ standard: buildInstructionAssembler(rulesSrc, soulSrc, memStore, userModelStore, !projectIngestionAdmitted(cfg)), - remote: buildInstructionAssembler(nil, soulSrc, memStore, userModelStore, true), + remote: buildInstructionAssembler(remoteRules, soulSrc, memStore, userModelStore, true), } // Guardrails decorate the ordinary hook chain. Completion learning has its own @@ -4689,6 +4694,9 @@ func buildCommandExpander(cfg Config, mcpProvider mcp.Provider) prompt.CommandEx // and exactly reattaches the owned session, this lister opens a fresh osfs Workspace // at that provider-verified private root so discovery reflects current command files. func buildCommandLister(cfg Config, mcpProvider mcp.Provider) server.CommandLister { + if cfg.RemoteExecution { + return nil + } exp := buildCommandExpander(cfg, mcpProvider) lister, ok := exp.(prompt.CommandLister) if !ok { @@ -8238,16 +8246,18 @@ const workspaceRootForPrompt = "/workspace" const remoteExecutionPostureNote = "This session uses a persistent remote Kubernetes workspace. Use the filesystem tools and foreground Shell for work in /workspace. Local project instructions, rules, project-scoped skills, commands, schedules, SkillDraft, Parallel, Team, and Subagent delegation are unavailable; operator-global skills, MCP, memory, and web tools remain available. Never assume harness-local files are part of this workspace." -func applyRemoteExecutionPosture(pc prompt.Config, enabled bool) prompt.Config { +func applyRemoteExecutionPosture(deps agent.Deps, enabled bool) agent.Deps { if !enabled { - return pc + return deps } + deps.CommandExpander = prompt.NoopExpander{} + pc := &deps.PromptConfig pc.Env.Cwd, pc.Env.Shell, pc.Env.GitStatus = workspaceRootForPrompt, "remote foreground shell", "" if pc.Role == "" { pc.Role = prompt.DefaultRole() } pc.Role += "\n\n" + remoteExecutionPostureNote - return pc + return deps } func applyDebugSessionPosture(pc prompt.Config, target session.SessionID, selectedServers []string) prompt.Config { diff --git a/internal/app/project_ingestion.go b/internal/app/project_ingestion.go index 19ca62f0ee..ef365fc40a 100644 --- a/internal/app/project_ingestion.go +++ b/internal/app/project_ingestion.go @@ -14,7 +14,9 @@ package app // shell also reads cfg.TrustProject directly because both decisions intentionally // express the same operator vouch for the workspace and its .git. func projectIngestionAdmitted(cfg Config) bool { - return cfg.TrustProject + // Local project trust cannot authorize source ingestion for a remote + // deployment; its explicit no-FS attenuation has no project source either. + return cfg.TrustProject && !cfg.RemoteExecution } // projectIngestionAdmittedForRoot binds the single project-ingestion decision to diff --git a/internal/app/remote_execution_test.go b/internal/app/remote_execution_test.go index aeb39b2c2f..76887e14b8 100644 --- a/internal/app/remote_execution_test.go +++ b/internal/app/remote_execution_test.go @@ -3,6 +3,7 @@ package app import ( "context" "encoding/json" + "errors" "os" "path/filepath" "strings" @@ -19,11 +20,12 @@ import ( ) type remoteFactoryPlacement struct { - env tool.Environment - binds *int + env tool.Environment + binds *int + validateErr error } -func (remoteFactoryPlacement) ValidatePlacement(context.Context) error { return nil } +func (p remoteFactoryPlacement) ValidatePlacement(context.Context) error { return p.validateErr } func (p remoteFactoryPlacement) Bind(_ context.Context, req server.PlacementBindRequest) (server.PlacementBinding, error) { if p.binds != nil { *p.binds++ @@ -40,7 +42,25 @@ func (p remoteFactoryPlacement) Reattach(_ context.Context, req server.Placement return server.PlacementBinding{Environment: p.env, Ref: p.env.Ref()}, nil } +func TestRemoteExecutionPreflightNeverAllocates(t *testing.T) { + fakeRulesEnv(t, t.TempDir(), t.TempDir()) + for _, validation := range []error{nil, server.ErrInvalidPlacementBinding, server.ErrPlacementUnavailable} { + binds := 0 + built, err := buildIsolated(t, t.Context(), Config{UseMock: true, NoSoul: true, RemoteExecution: true, PlacementScope: "remote", PlacementProvider: remoteFactoryPlacement{binds: &binds, validateErr: validation}}) + if built != nil { + built.Close() + } + if validation == nil && err != nil || validation != nil && !errors.Is(err, validation) { + t.Fatalf("validation=%v startup error=%v", validation, err) + } + if binds != 0 { + t.Fatalf("preflight allocated %d environments", binds) + } + } +} + func TestRemoteDeploymentNoFSUsesLocalAttenuationWithoutProviderCall(t *testing.T) { + fakeRulesEnv(t, t.TempDir(), t.TempDir()) binds := 0 remoteWS := memfs.NewWorkspace("/workspace") remoteRef := session.EnvironmentRef{Kind: "kubernetes", ID: "env-1", Revision: "rev-1"} @@ -81,7 +101,15 @@ func TestRemoteDeploymentNoFSUsesLocalAttenuationWithoutProviderCall(t *testing. } func TestRemoteExecutionRealFactoryCarriesPostureAndAttenuatedCatalog(t *testing.T) { - ws := memfs.NewWorkspace("/workspace") + xdg := t.TempDir() + fakeRulesEnv(t, xdg, t.TempDir()) + writeUserRule(t, xdg, "operator-rule", "OPERATOR_RULE_MARKER\n") + writeSkill(t, filepath.Join(xdg, "mecatl/skills"), "operator-skill", "Operator skill", "OPERATOR_SKILL_BODY") + localProject := t.TempDir() + ws := memfs.NewWorkspace(localProject) + if _, err := ws.CreateFile(context.Background(), "AGENTS.md", []byte("REMOTE_PROJECT_MARKER_DO_NOT_LOAD")); err != nil { + t.Fatal(err) + } if _, err := ws.CreateFile(context.Background(), "main.go", []byte("package main\n")); err != nil { t.Fatal(err) } @@ -90,16 +118,32 @@ func TestRemoteExecutionRealFactoryCarriesPostureAndAttenuatedCatalog(t *testing var mu sync.Mutex var captured port.LLMRequest provider := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { mu.Lock(); captured = req; mu.Unlock() })}, - mockllm.ToolCallTurn(session.ToolCall{ID: "unknown", Name: "NoSuchTool", Args: json.RawMessage(`{}`)}), + mockllm.ToolCallTurn( + session.ToolCall{ID: "Subagent", Name: "Subagent", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "Parallel", Name: "Parallel", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "Team", Name: "Team", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "SkillDraft", Name: "SkillDraft", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "Schedule", Name: "Schedule", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "project-skill", Name: "Skill", Args: json.RawMessage(`{"name":"project-only"}`)}, + session.ToolCall{ID: "operator-skill", Name: "Skill", Args: json.RawMessage(`{"name":"operator-skill"}`)}, + ), mockllm.ToolCallTurn(session.ToolCall{ID: "read", Name: "Read", Args: json.RawMessage(`{"path":"main.go"}`)}), mockllm.TextTurn(""), mockllm.TextTurn("done"), ) - localProject := t.TempDir() if err := os.WriteFile(filepath.Join(localProject, "AGENTS.md"), []byte("LOCAL_PROJECT_MARKER_DO_NOT_LOAD\n"), 0o600); err != nil { t.Fatal(err) } - built, err := buildIsolated(t, context.Background(), Config{MockProvider: provider, PlacementProvider: remoteFactoryPlacement{env: env}, PlacementScope: "remote", RemoteExecution: true, Workspace: localProject, TrustProject: true, NoSoul: true, SchedulerEnabled: false}) + writeProjectRule(t, localProject, "project-rule", "LOCAL_RULE_MARKER_DO_NOT_LOAD\n") + writeSkill(t, filepath.Join(localProject, ".mecatl/skills"), "project-only", "LOCAL_SKILL_MARKER_DO_NOT_LOAD", "LOCAL_SKILL_BODY_DO_NOT_LOAD") + commandDir := filepath.Join(localProject, ".claude", "commands") + if err := os.MkdirAll(commandDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(commandDir, "project-only.md"), []byte("LOCAL_COMMAND_MARKER_DO_NOT_LOAD"), 0o600); err != nil { + t.Fatal(err) + } + built, err := buildIsolated(t, context.Background(), Config{MockProvider: provider, PlacementProvider: remoteFactoryPlacement{env: env}, PlacementScope: "remote", RemoteExecution: true, Workspace: localProject, TrustProject: true, AllowAllTools: true, EnableCommands: true, SkillsConventional: true, NoSoul: true, SchedulerEnabled: false}) if err != nil { t.Fatal(err) } @@ -110,7 +154,13 @@ func TestRemoteExecutionRealFactoryCarriesPostureAndAttenuatedCatalog(t *testing if err != nil { t.Fatal(err) } - run, err := built.Service.StartRun(ctx, sess.ID, "read the remote fixture") + if commands, err := built.Service.ListCommandsForSession(ctx, sess.ID); err != nil || len(commands) != 0 { + t.Fatalf("remote command discovery fell back locally: %v, %v", commands, err) + } + if _, err := built.Service.ListWorktreesForSession(ctx, sess.ID); !errors.Is(err, server.ErrPlacementUnavailable) { + t.Fatalf("remote worktree discovery did not reject unsupported placement: %v", err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "/project-only") if err != nil { t.Fatal(err) } @@ -122,8 +172,28 @@ func TestRemoteExecutionRealFactoryCarriesPostureAndAttenuatedCatalog(t *testing if !strings.Contains(captured.System.StablePrefix, remoteExecutionPostureNote) { t.Fatal("real per-session factory omitted remote execution posture") } - if strings.Contains(captured.System.Render(), "LOCAL_PROJECT_MARKER_DO_NOT_LOAD") { - t.Fatal("remote session ingested host-local project instructions") + requestJSON, err := json.Marshal(captured) + if err != nil { + t.Fatal(err) + } + for _, marker := range []string{"LOCAL_PROJECT_MARKER", "REMOTE_PROJECT_MARKER", "LOCAL_RULE_MARKER", "LOCAL_SKILL_MARKER", "LOCAL_SKILL_BODY", "LOCAL_COMMAND_MARKER"} { + if strings.Contains(string(requestJSON), marker) { + t.Errorf("remote request ingested forbidden project source %s", marker) + } + } + if !strings.Contains(string(requestJSON), "OPERATOR_RULE_MARKER") || !strings.Contains(string(requestJSON), "OPERATOR_SKILL_BODY") { + t.Fatal("remote request dropped operator-global rules or skills") + } + failed := map[string]bool{} + for _, message := range captured.Messages { + if result := message.ToolResult; result != nil { + failed[string(result.CallID)] = result.IsError + } + } + for _, id := range []string{"Subagent", "Parallel", "Team", "SkillDraft", "Schedule", "project-skill"} { + if !failed[id] { + t.Errorf("unsupported invocation %s did not return a model-visible error", id) + } } names := map[string]bool{} for _, spec := range captured.Tools { diff --git a/internal/executionenv/errors.go b/internal/executionenv/errors.go index e5ab5847c7..6a65285a82 100644 --- a/internal/executionenv/errors.go +++ b/internal/executionenv/errors.go @@ -20,6 +20,7 @@ const ( CodeAlreadyExists ErrorCode = "already_exists" CodeConflict ErrorCode = "conflict" CodeVersionMismatch ErrorCode = "version_mismatch" + CodeDirectoryNotEmpty ErrorCode = "directory_not_empty" CodeNotReady ErrorCode = "not_ready" CodeFenceUnknown ErrorCode = "fence_unknown" CodeResourceExhausted ErrorCode = "resource_exhausted" @@ -30,7 +31,7 @@ const ( func (c ErrorCode) Valid() bool { switch c { case CodeInvalidArgument, CodeUnauthenticated, CodePermissionDenied, CodeNotFound, - CodeAlreadyExists, CodeConflict, CodeVersionMismatch, CodeNotReady, CodeFenceUnknown, + CodeAlreadyExists, CodeConflict, CodeVersionMismatch, CodeDirectoryNotEmpty, CodeNotReady, CodeFenceUnknown, CodeResourceExhausted, CodeInternal: return true default: diff --git a/internal/executionexecutor/confinement_test.go b/internal/executionexecutor/confinement_test.go new file mode 100644 index 0000000000..e7db9a73b5 --- /dev/null +++ b/internal/executionexecutor/confinement_test.go @@ -0,0 +1,76 @@ +package executionexecutor + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestFileOperationsNeverTraverseExternalSymlink(t *testing.T) { + root, outside := t.TempDir(), t.TempDir() + sentinel := filepath.Join(outside, "file") + if err := os.WriteFile(sentinel, []byte("outside-secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil { + t.Fatal(err) + } + x, err := New(root, Limits{}) + if err != nil { + t.Fatal(err) + } + defer x.Close() + for _, op := range []executionenv.Operation{executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileCreate, executionenv.OpFileReplace, executionenv.OpFileList, executionenv.OpFileRemove, executionenv.OpFileRename, executionenv.OpFileCopy} { + t.Run(string(op), func(t *testing.T) { + req := executionenv.ExecutorRequest{Operation: op, Path: "escape/file", Destination: "destination", Version: "version"} + if op == executionenv.OpFileList { + req.Path = "escape" + } + if op == executionenv.OpFileCreate { + req.Path = "escape/new" + } + if _, err := x.Execute(t.Context(), req); err == nil { + t.Fatal("external symlink traversal accepted") + } + if op == executionenv.OpFileCopy || op == executionenv.OpFileRename { + if _, err := x.Execute(t.Context(), executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: string(op), Data: []byte("local")}); err != nil { + t.Fatal(err) + } + req.Path = string(op) + req.Destination = "escape/new" + if _, err := x.Execute(t.Context(), req); err == nil { + t.Fatal("external destination accepted") + } + } + }) + } + for _, op := range []executionenv.Operation{executionenv.OpFileGlob, executionenv.OpFileGrep} { + req := executionenv.ExecutorRequest{Operation: op, Pattern: "**/*", Limit: 100} + if op == executionenv.OpFileGrep { + req.Path = "**/*" + req.Pattern = "outside-secret" + } + result, err := x.Execute(t.Context(), req) + if err != nil { + continue + } // Honest confinement errors are also safe. + for _, path := range result.Paths { + if strings.HasPrefix(path, "escape/") { + t.Fatalf("glob escaped: %q", path) + } + } + if len(result.Matches) != 0 { + t.Fatalf("grep leaked external content: %+v", result.Matches) + } + } + data, err := os.ReadFile(sentinel) + if err != nil || string(data) != "outside-secret" { + t.Fatalf("external file mutated: %q %v", data, err) + } + if _, err := os.Stat(filepath.Join(outside, "new")); !os.IsNotExist(err) { + t.Fatalf("external file created: %v", err) + } +} diff --git a/internal/executionexecutor/executor.go b/internal/executionexecutor/executor.go index f6e13a7aa0..c429d41310 100644 --- a/internal/executionexecutor/executor.go +++ b/internal/executionexecutor/executor.go @@ -239,7 +239,7 @@ func mapErr(err error) error { case errors.Is(err, fs.ErrExist): return coded(executionenv.CodeAlreadyExists, "destination already exists") case errors.Is(err, tool.ErrDirectoryNotEmpty): - return coded(executionenv.CodeConflict, "directory is not empty") + return coded(executionenv.CodeDirectoryNotEmpty, "directory is not empty") case errors.Is(err, osfs.ErrPathEscape): return coded(executionenv.CodePermissionDenied, "path is outside workspace") default: diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index 9ac7c5b3e8..6e8bf8391c 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -84,6 +84,16 @@ using a local workspace. is intended for an operator-controlled cluster and foreground commands only. It has no force-takeover recovery and is not a hostile multi-tenancy boundary. +Remote deployments retain operator-global rules and skills. Local project trust +cannot admit host-local AGENTS.md, rules, skills, commands, or Git context into a +remote session. Command discovery and slash-command expansion are unavailable +for remote sessions. Explicit `no-fs` sessions keep their file-less catalog and +allocate no execution environment. + +The remote filesystem preserves the existing Read/Edit/Write version checks and +non-clobbering Copy/Move behavior. Copy, Move, and Remove do not require prior +content reads; Remove remains non-recursive. + The optional Kubernetes execution provider stores environment ownership in a namespaced `ExecutionEnvironment`. Provider replicas coordinate through Kubernetes resource-version compare-and-swap; a replica restart does not clear From 773a203df5de9b27612d190a774a857590702cbd Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Mon, 21 Sep 2026 23:39:41 +0300 Subject: [PATCH 17/25] fix(execution): preserve remote operator permissions and close offline proofs Co-Authored-By: OpenAI --- .../adapter/executionclient/client_test.go | 4 +- .../service_integration_test.go | 64 +++++++++- internal/app/build.go | 3 +- internal/app/remote_execution_test.go | 115 +++++++++++++++++- internal/executionexecutor/executor_test.go | 65 +++++++++- user-docs/features/execution-environments.md | 5 +- 6 files changed, 248 insertions(+), 8 deletions(-) diff --git a/internal/adapter/executionclient/client_test.go b/internal/adapter/executionclient/client_test.go index b4054bea7f..59f86ddcf5 100644 --- a/internal/adapter/executionclient/client_test.go +++ b/internal/adapter/executionclient/client_test.go @@ -70,7 +70,7 @@ func (b *integrationBackend) File(_ context.Context, _, _ string, q executionenv return executionenv.FileResponse{Data: []byte("from-grpc"), Version: string([]byte{0xff, 0, 1})}, nil } func (*integrationBackend) StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { - return executionenv.CommandStartResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Result: executionenv.CommandStatusResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Stdout: []byte("ok\n")}}, nil + return executionenv.CommandStartResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Result: executionenv.CommandStatusResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Stdout: []byte("ok\n\xff"), Stderr: []byte("err\xe2")}}, nil } func (*integrationBackend) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeInternal, Message: "unimplemented"} @@ -217,7 +217,7 @@ func TestProviderThroughRealGRPCSignedHandlerRefreshesAndReattachesExactly(t *te t.Fatalf("opaque version=%q err=%v", encoded, err) } result, err := runEnv.CommandRunner().Run(context.Background(), "go test ./...") - if err != nil || result.Stdout != "ok\n" { + if err != nil || result.Stdout != "ok\n�" || result.Stderr != "err�" { t.Fatalf("result=%+v err=%v", result, err) } if backend.ensureCalls != 1 || backend.attachCalls < 2 || backend.fileCalls != 2 { diff --git a/internal/adapter/executionclient/service_integration_test.go b/internal/adapter/executionclient/service_integration_test.go index 84bb26e343..3e2ba1b0ce 100644 --- a/internal/adapter/executionclient/service_integration_test.go +++ b/internal/adapter/executionclient/service_integration_test.go @@ -5,12 +5,15 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "os" "path/filepath" + "reflect" "strings" "sync/atomic" "testing" "time" + "unicode/utf8" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -22,6 +25,7 @@ import ( "github.com/stacklok/mecatl/engine/adapter/mockllm" "github.com/stacklok/mecatl/engine/adapter/permpolicy" "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/port" "github.com/stacklok/mecatl/engine/session" "github.com/stacklok/mecatl/engine/tool" "github.com/stacklok/mecatl/internal/adapter/executioncontroller" @@ -40,7 +44,7 @@ func (e *recordingExecutor) Execute(_ context.Context, _ string, req executionen case executionenv.OpFileRead: return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Data: []byte("package main\n"), Version: "v1"}}, nil case executionenv.OpCommandStart: - return executionenv.ExecutorResponse{Command: &executionenv.CommandStatusResponse{CommandID: req.CommandID, State: executionenv.CommandSucceeded, Stdout: []byte("ok\n"), TerminalReceipt: "complete"}}, nil + return executionenv.ExecutorResponse{Command: &executionenv.CommandStatusResponse{CommandID: req.CommandID, State: executionenv.CommandSucceeded, Stdout: []byte("out\xff"), Stderr: []byte("err\xe2"), TerminalReceipt: "complete"}}, nil default: return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "unsupported test operation"} } @@ -89,7 +93,14 @@ func TestServiceUsesRealMTLSProviderStoreAndReleasesOnlyAfterDrain(t *testing.T) catalog := tool.NewCatalog() catalog.MustRegister(adaptertools.ReadTool{}) catalog.MustRegister(agent.NewShellTool()) - llm := mockllm.New( + var modelShellResult string + llm := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { + for _, message := range req.Messages { + if result := message.ToolResult; result != nil && result.CallID == "shell" { + modelShellResult = result.Content + } + } + })}, mockllm.ToolCallTurn(session.NewToolCall("read", "Read", json.RawMessage(`{"path":"main.go"}`))), mockllm.ToolCallTurn(session.NewToolCall("background", "Shell", json.RawMessage(`{"command":"touch background-leak","background":true}`))), mockllm.ToolCallTurn(session.NewToolCall("shell", "Shell", json.RawMessage(`{"command":"true"}`))), @@ -117,7 +128,11 @@ func TestServiceUsesRealMTLSProviderStoreAndReleasesOnlyAfterDrain(t *testing.T) t.Fatal(err) } backgroundRejected := false + var streamedShellResult string for event := range run.Events() { + if result := event.ToolResult; result != nil && result.CallID == "shell" { + streamedShellResult = result.Content + } if result := event.ToolResult; result != nil && result.CallID == "background" { backgroundRejected = result.IsError && strings.Contains(result.Content, "not supported by this command runner") } @@ -125,6 +140,9 @@ func TestServiceUsesRealMTLSProviderStoreAndReleasesOnlyAfterDrain(t *testing.T) if !backgroundRejected { t.Fatal("remote background Shell did not return the named capability error") } + if !utf8.ValidString(streamedShellResult) || !strings.Contains(streamedShellResult, "out�") || !strings.Contains(streamedShellResult, "err�") || modelShellResult != streamedShellResult { + t.Fatalf("private protobuf bytes were not repaired identically for client/model: stream=%q model=%q", streamedShellResult, modelShellResult) + } if executor.calls.Load() != 3 { t.Fatalf("executor calls=%d, want authority+read+shell", executor.calls.Load()) } @@ -145,5 +163,47 @@ func TestServiceUsesRealMTLSProviderStoreAndReleasesOnlyAfterDrain(t *testing.T) } for range continued.Events() { } + svc.Persist(ctx, sess.ID) svc.FinishRun(sess.ID, continued) + + stored, err = dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, "env-real", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + before, err := sessions.Load(ctx, sess.ID) + if err != nil || len(before.Conversation.Messages) == 0 { + t.Fatalf("source history missing: %v", err) + } + placement := server.SuccessorPlacement{Selector: "unsupported-worktree", SelectorPresent: true} + // A well-formed selector and binding ID reach the native provider's unsupported + // selection gate, not the Service's malformed-request validation. + if _, err := provider.Bind(ctx, server.PlacementBindRequest{Selector: server.SelectWorktree(sess.ID, sess.EnvironmentRef, placement.Selector), BindingID: "unused-destination", Principal: principal}); !errors.Is(err, server.ErrInvalidPlacementSelection) { + t.Fatalf("native selector gate: %v", err) + } + for _, operation := range []string{"clear", "fork"} { + t.Run(operation+"-unsupported-selector", func(t *testing.T) { + var destination session.SessionID + var err error + if operation == "clear" { + destination, err = svc.ClearSessionSuccessor(ctx, sess.ID, placement) + } else { + destination, err = svc.ForkSessionSuccessor(ctx, server.ForkSuccessorRequest{Source: sess.ID, Placement: placement}) + } + if !errors.Is(err, server.ErrInvalidPlacementSelection) || destination != "" { + t.Fatalf("destination=%q error=%v", destination, err) + } + after, err := sessions.Load(ctx, sess.ID) + if err != nil || after.EnvironmentRef != before.EnvironmentRef || !reflect.DeepEqual(after.Conversation, before.Conversation) { + t.Fatalf("unsupported successor changed source ref/history: %v", err) + } + all, err := svc.ListSessions(ctx) + if err != nil || len(all) != 1 || all[0].SessionID != string(sess.ID) { + t.Fatalf("unsupported successor published a destination: %+v, %v", all, err) + } + current, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, "env-real", metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(current.Object, stored.Object) { + t.Fatalf("unsupported successor mutated provider references: %v", err) + } + }) + } } diff --git a/internal/app/build.go b/internal/app/build.go index 1969796cc4..96dba602a9 100644 --- a/internal/app/build.go +++ b/internal/app/build.go @@ -4176,8 +4176,9 @@ func buildEngine(ctx context.Context, cfg Config, reg *providerRegistry, provide // and the un-knobbed auto stay byte-identical. sharedPolicy := sessionPermissionPolicies{ standard: escapePolicyForConfig(cfg, reg, provider, policy), + // Pin no workspace: retain operator permissions without project discovery. remote: escapePolicyForConfig(cfg, reg, provider, - permpolicy.NewPolicyWithResolver(mainRules(cfg), learned, nil, mainEvaluatorOptions(cfg)...)), + permpolicy.NewPolicyWithResolver(mainRules(cfg), learned, childPermResolverFor(cfg, ""), mainEvaluatorOptions(cfg)...)), } var learningAdmission *learningAdmission if cfg.operatorLearningMode != learning.Off { diff --git a/internal/app/remote_execution_test.go b/internal/app/remote_execution_test.go index 76887e14b8..b4a85d4ce1 100644 --- a/internal/app/remote_execution_test.go +++ b/internal/app/remote_execution_test.go @@ -100,12 +100,122 @@ func TestRemoteDeploymentNoFSUsesLocalAttenuationWithoutProviderCall(t *testing. } } +type remotePermissionRunner struct { + *memfs.CommandRunner + root string +} + +func (r remotePermissionRunner) BoundWorkspaceRoot() string { return r.root } + +func TestRemoteExecutionPreservesOperatorPermissionsUnderAuto(t *testing.T) { + for _, source := range []string{"global", "explicit"} { + t.Run(source, func(t *testing.T) { + xdg := t.TempDir() + fakeRulesEnv(t, xdg, t.TempDir()) + t.Setenv("XDG_STATE_HOME", t.TempDir()) + operatorPath := filepath.Join(xdg, "mecatl", "settings.yaml") + if source == "explicit" { + operatorPath = filepath.Join(t.TempDir(), "operator.yaml") + } + writeProjectFile(t, operatorPath, "permissions:\n deny:\n - 'Shell(echo denied)'\n ask:\n - 'Shell(echo asked)'\n") + project := t.TempDir() + projectRules := "permissions:\n deny:\n - 'Shell(echo allowed)'\n ask:\n - 'Shell(echo project-ask)'\n allow:\n - 'Shell(echo denied)'\n - 'Shell(echo asked)'\n" + mkdirProjectSettings(t, project, projectRules) + ws := memfs.NewWorkspace(project) + if _, err := ws.CreateFile(t.Context(), ".mecatl/settings.yaml", []byte(projectRules)); err != nil { + t.Fatal(err) + } + runner := memfs.NewCommandRunner() + runner.SetResult(&tool.CommandResult{Stdout: "EXECUTED"}, nil) + ref := session.EnvironmentRef{Kind: "kubernetes", ID: "permissions", Revision: "v1"} + env := tool.MustEnvironment(ref, ws, memledger.New(), remotePermissionRunner{runner, project}) + llm := mockllm.New( + mockllm.ToolCallTurn( + session.NewToolCall("denied", "Shell", json.RawMessage(`{"command":"echo denied"}`)), + session.NewToolCall("asked", "Shell", json.RawMessage(`{"command":"echo asked"}`)), + session.NewToolCall("allowed", "Shell", json.RawMessage(`{"command":"echo allowed"}`)), + session.NewToolCall("project-ask", "Shell", json.RawMessage(`{"command":"echo project-ask"}`)), + ), mockllm.TextTurn("done")) + cfg := Config{MockProvider: llm, Workspace: project, Posture: PostureAuto, PermissionsConventional: true, NoSoul: true, RemoteExecution: true, PlacementScope: "remote", PlacementProvider: remoteFactoryPlacement{env: env}} + if source == "explicit" { + cfg.PermissionConfigs = []string{operatorPath} + } + built, err := buildIsolated(t, t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + defer built.Close() + ctx := session.WithPrincipal(t.Context(), &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser}) + sess, err := built.Service.CreateSession(ctx, session.ModeDefault, session.Limits{}) + if err != nil { + t.Fatal(err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "check permissions") + if err != nil { + t.Fatal(err) + } + asks := 0 + results := map[session.ToolCallID]session.ToolResult{} + for ev := range run.Events() { + if ev.Ask != nil { + asks++ + if ev.Ask.Call != "asked" || !ev.Ask.ConfiguredAsk { + t.Errorf("unexpected permission ask: %+v", ev.Ask) + } + if _, err := built.Service.ApproveRun(ctx, sess.ID, ev.Ask.AskID, session.VerdictDeny, ""); err != nil { + t.Fatal(err) + } + } + if ev.ToolResult != nil { + results[ev.ToolResult.CallID] = *ev.ToolResult + } + } + built.Service.FinishRun(sess.ID, run) + if asks != 1 { + t.Errorf("configured asks = %d, want 1", asks) + } + for _, id := range []session.ToolCallID{"denied", "asked", "allowed", "project-ask"} { + result, ok := results[id] + blocked := id == "denied" || id == "asked" + if !ok || result.IsError != blocked || strings.Contains(result.Content, "EXECUTED") == blocked { + t.Errorf("%s result = %+v, present=%t, want blocked=%t", id, result, ok, blocked) + } + } + }) + } +} + func TestRemoteExecutionRealFactoryCarriesPostureAndAttenuatedCatalog(t *testing.T) { xdg := t.TempDir() fakeRulesEnv(t, xdg, t.TempDir()) + t.Setenv("XDG_STATE_HOME", t.TempDir()) writeUserRule(t, xdg, "operator-rule", "OPERATOR_RULE_MARKER\n") writeSkill(t, filepath.Join(xdg, "mecatl/skills"), "operator-skill", "Operator skill", "OPERATOR_SKILL_BODY") localProject := t.TempDir() + initTestRepo(t, localProject) + localSnapshotSeen := false + localLLM := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { + localSnapshotSeen = strings.Contains(req.System.Render(), "initial commit") && strings.Contains(req.System.Render(), "") + })}, mockllm.TextTurn("local done")) + local, err := buildIsolated(t, t.Context(), Config{MockProvider: localLLM, Workspace: localProject, TrustProject: true, NoSoul: true, Shell: "/bin/sh"}) + if err != nil { + t.Fatal(err) + } + defer local.Close() + localSession, err := local.Service.CreateSession(t.Context(), session.ModeDefault, session.Limits{}) + if err != nil { + t.Fatal(err) + } + localRun, err := local.Service.StartRun(t.Context(), localSession.ID, "inspect local") + if err != nil { + t.Fatal(err) + } + for range localRun.Events() { + } + local.Service.FinishRun(localSession.ID, localRun) + if !localSnapshotSeen { + t.Fatal("local factory omitted the test repository's Git snapshot") + } ws := memfs.NewWorkspace(localProject) if _, err := ws.CreateFile(context.Background(), "AGENTS.md", []byte("REMOTE_PROJECT_MARKER_DO_NOT_LOAD")); err != nil { t.Fatal(err) @@ -172,11 +282,14 @@ func TestRemoteExecutionRealFactoryCarriesPostureAndAttenuatedCatalog(t *testing if !strings.Contains(captured.System.StablePrefix, remoteExecutionPostureNote) { t.Fatal("real per-session factory omitted remote execution posture") } + if strings.Contains(captured.System.Render(), "") { + t.Fatal("remote factory ingested the local Git snapshot") + } requestJSON, err := json.Marshal(captured) if err != nil { t.Fatal(err) } - for _, marker := range []string{"LOCAL_PROJECT_MARKER", "REMOTE_PROJECT_MARKER", "LOCAL_RULE_MARKER", "LOCAL_SKILL_MARKER", "LOCAL_SKILL_BODY", "LOCAL_COMMAND_MARKER"} { + for _, marker := range []string{"LOCAL_PROJECT_MARKER", "REMOTE_PROJECT_MARKER", "LOCAL_RULE_MARKER", "LOCAL_SKILL_MARKER", "LOCAL_SKILL_BODY", "LOCAL_COMMAND_MARKER", "initial commit"} { if strings.Contains(string(requestJSON), marker) { t.Errorf("remote request ingested forbidden project source %s", marker) } diff --git a/internal/executionexecutor/executor_test.go b/internal/executionexecutor/executor_test.go index 9d68f3765c..90157af3e3 100644 --- a/internal/executionexecutor/executor_test.go +++ b/internal/executionexecutor/executor_test.go @@ -8,9 +8,15 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" "time" + "unicode/utf8" + "github.com/stacklok/mecatl/engine/adapter/fstools" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" "github.com/stacklok/mecatl/internal/executionenv" ) @@ -109,10 +115,67 @@ func TestSuccessfulDetachedChildIsKilledBeforeTerminalReceipt(t *testing.T) { } } +func TestNativeShellCapsCombinedOutputAndRepairsUTF8(t *testing.T) { + for _, tc := range []struct { + name, command string + wantStdout, wantStderr int + }{ + {"stdout", `i=0; while [ "$i" -lt 2048 ]; do printf x; i=$((i+1)); done`, 1024, 0}, + {"stderr", `i=0; while [ "$i" -lt 2048 ]; do printf x >&2; i=$((i+1)); done`, 0, 1024}, + {"split-malformed", `printf '\377'; printf '\376' >&2; i=0; while [ "$i" -lt 700 ]; do printf x; printf y >&2; i=$((i+1)); done`, 703, 321}, + } { + t.Run(tc.name, func(t *testing.T) { + response := runExecutorHelper(t, t.TempDir(), executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "bounded", Command: tc.command}) + result := response.Command + if result == nil || result.State != executionenv.CommandSucceeded || result.TerminalReceipt == "" { + t.Fatalf("native command failed: %+v", response) + } + if !result.Truncated || len(result.Stdout) != tc.wantStdout || len(result.Stderr) != tc.wantStderr || len(result.Stdout)+len(result.Stderr) > 1024 { + t.Fatalf("stdout=%d stderr=%d truncated=%t", len(result.Stdout), len(result.Stderr), result.Truncated) + } + if !utf8.Valid(result.Stdout) || !utf8.Valid(result.Stderr) { + t.Fatal("native command result contains malformed UTF-8") + } + if tc.name == "split-malformed" && (!strings.HasPrefix(string(result.Stdout), "�") || !strings.HasPrefix(string(result.Stderr), "�")) { + t.Fatal("malformed stream bytes were not repaired") + } + }) + } +} + +func TestNativeShellWriteInvalidatesRecordedRead(t *testing.T) { + root := t.TempDir() + x, err := New(root, Limits{}) + if err != nil { + t.Fatal(err) + } + defer x.Close() + env := tool.MustEnvironment(session.EnvironmentRef{Kind: "kubernetes", ID: "shell-cas", Revision: "v1"}, x.workspace, memledger.New(), nil) + if _, err := x.workspace.CreateFile(t.Context(), "file.txt", []byte("before")); err != nil { + t.Fatal(err) + } + read, err := (fstools.ReadTool{}).Execute(t.Context(), session.NewToolCall("read", "Read", json.RawMessage(`{"path":"file.txt"}`)), env) + if err != nil || read.IsError { + t.Fatalf("record read: %+v, %v", read, err) + } + response := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "write", Command: "printf shell-change > file.txt"}) + if response.Command == nil || response.Command.State != executionenv.CommandSucceeded { + t.Fatalf("Shell write failed: %+v", response) + } + edit, err := (fstools.EditTool{}).Execute(t.Context(), session.NewToolCall("edit", "Edit", json.RawMessage(`{"path":"file.txt","old_string":"shell-change","new_string":"lost"}`)), env) + if err != nil || !edit.IsError || !strings.Contains(edit.Content, "changed since") { + t.Fatalf("Edit must reject the stale recorded version, not an exact-match failure: %+v, %v", edit, err) + } + data, err := x.workspace.Read(t.Context(), "file.txt") + if err != nil || string(data) != "shell-change" { + t.Fatalf("Shell change overwritten: %q, %v", data, err) + } +} + func runExecutorHelper(t *testing.T, root string, request executionenv.ExecutorRequest) executionenv.ExecutorResponse { t.Helper() cmd := exec.Command(os.Args[0], "-test.run=^TestExecutorCommandHelper$") - cmd.Env = append(os.Environ(), "MECATL_EXECUTOR_TEST_HELPER=1", "MECATL_EXECUTOR_TEST_ROOT="+root) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + root, "MECATL_EXECUTOR_TEST_HELPER=1", "MECATL_EXECUTOR_TEST_ROOT=" + root} stdin, err := cmd.StdinPipe() if err != nil { t.Fatal(err) diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index 6e8bf8391c..70708d82b9 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -84,7 +84,10 @@ using a local workspace. is intended for an operator-controlled cluster and foreground commands only. It has no force-takeover recovery and is not a hostile multi-tenancy boundary. -Remote deployments retain operator-global rules and skills. Local project trust +Remote deployments retain operator-global prompt rules and skills. Permission +rules from enabled user-global settings and explicit permission files also apply, +including configured Deny and Ask rules under `auto` posture. Project permission +files are excluded. Local project trust cannot admit host-local AGENTS.md, rules, skills, commands, or Git context into a remote session. Command discovery and slash-command expansion are unavailable for remote sessions. Explicit `no-fs` sessions keep their file-less catalog and From 96e16e926f1edb2c34ee40a07f55523ee0a7f7f2 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Mon, 21 Sep 2026 23:56:39 +0300 Subject: [PATCH 18/25] fix(execution): track repaired output clipping and reproduce rotation skew Co-Authored-By: OpenAI --- .../security_rotation_test.go | 176 ++++++++++++++++++ internal/executionexecutor/executor.go | 18 +- internal/executionexecutor/executor_test.go | 3 + 3 files changed, 187 insertions(+), 10 deletions(-) create mode 100644 internal/adapter/executioncontroller/security_rotation_test.go diff --git a/internal/adapter/executioncontroller/security_rotation_test.go b/internal/adapter/executioncontroller/security_rotation_test.go new file mode 100644 index 0000000000..8595844b4c --- /dev/null +++ b/internal/adapter/executioncontroller/security_rotation_test.go @@ -0,0 +1,176 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "os" + "path/filepath" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +// These are synthetic fixture names, including the former mutable Secret keys. +func rotationSecurityFixture(t *testing.T) (*SecurityManager, securityManifest, context.Context, map[string][]byte) { + t.Helper() + dir := t.TempDir() + now := time.Now().UTC() + oldCA, oldCert, oldKey := syntheticPKI(t, now) + newCA, newCert, newKey, client := rpcSecurityPKI(t, now, "spiffe://example/client") + writeRotationMaterial(t, dir, map[string][]byte{"tls.crt": oldCert, "tls.key": oldKey, "clients.pem": append(oldCA, newCA...)}) + manifest := securityManifest{Version: 1, Generation: 2, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k2", GrantTTLText: "1m", ClockSkewText: "5s", TLS: securityTLSManifest{CertificateFile: "tls.crt", PrivateKeyFile: "tls.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + for i, id := range []string{"k1", "k2"} { + pub, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + name := "grant-" + id + ".pem" + writePKCS8(t, filepath.Join(dir, name), private) + fp := sha256.Sum256(pub) + manifest.Keys = append(manifest.Keys, securityKeyManifest{ID: id, Version: uint64(i + 1), File: name, PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}) + } + path := filepath.Join(dir, "manifest.json") + writeManifest(t, path, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(path, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + leaf, err := x509.ParseCertificate(client.Certificate[0]) + if err != nil { + t.Fatal(err) + } + ctx := peer.NewContext(t.Context(), &peer.Peer{AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}, VerifiedChains: [][]*x509.Certificate{{leaf}}}}}) + return manager, manifest, ctx, map[string][]byte{"tls.crt": newCert, "tls.key": newKey, "clients.pem": newCA} +} + +func writeRotationMaterial(t *testing.T, dir string, files map[string][]byte) { + t.Helper() + for name, contents := range files { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } +} + +func TestSecurityRotationPinnedReplicaReadConvergesBeforeDispatch(t *testing.T) { + manager, manifest, ctx, final := rotationSecurityFixture(t) + backend := &countedFileBackend{fakeBackend: newFakeBackend()} + h := NewHandler(HandlerConfig{Security: manager}, backend) + owner := &executionv1.Owner{Issuer: "issuer", Subject: "alice"} + claim, err := h.AcquireRun(ctx, &executionv1.AcquireRunRequest{Environment: &executionv1.EnvironmentRef{Id: "env", Revision: "rev"}, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire", TtlMillis: time.Minute.Milliseconds()}) + if err != nil { + t.Fatal("bridge k2 acquire failed") + } + request := &executionv1.FileRequest{Context: &executionv1.RequestContext{Environment: claim.Environment, Owner: owner, BindingId: claim.BindingId, RunId: claim.RunId, ClaimId: claim.ClaimId, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant}, Operation: executionv1.FileOperation_FILE_OPERATION_READ, Path: "sentinel"} + if _, err := h.Files(ctx, request); err != nil || backend.calls != 1 { + t.Fatal("bridge claim positive control failed") + } + manifest.Generation = 3 + manifest.Keys[0].State = "revoked" + manifest.TLS = securityTLSManifest{CertificateFile: "provider-new.crt", PrivateKeyFile: "provider-new.key", ClientCAFile: "final-clients.pem"} + writeRotationMaterial(t, manager.keyDirectory, map[string][]byte{"provider-new.crt": final["tls.crt"], "provider-new.key": final["tls.key"], "final-clients.pem": final["clients.pem"]}) + writeManifest(t, manager.manifestPath, manifest) + other := NewSecurityManager(manager.manifestPath, manager.keyDirectory, "ns", "authority", manager.kube) + other.now = manager.now + if err := other.Reload(t.Context()); err != nil { + t.Fatal(err) + } + backend.calls = 0 + _, err = h.Files(ctx, request) + st := status.Convert(err) + details := st.Details() + if st.Code() != codes.Unavailable || len(details) != 1 || backend.calls != 0 { + t.Fatalf("lagging replica: code=%s dispatches=%d", st.Code(), backend.calls) + } + detail, ok := details[0].(*executionv1.ErrorDetail) + if !ok || detail.Code != string(executionenv.CodeNotReady) || !detail.Retryable { + t.Fatal("lag was not structured retryable not_ready") + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := h.Files(ctx, request); err != nil || backend.calls != 1 { + t.Fatalf("same current claim after reload: code=%s dispatches=%d", status.Code(err), backend.calls) + } +} + +func TestSecurityRotationMutableNamesPoisonSameGeneration(t *testing.T) { + manager, manifest, _, final := rotationSecurityFixture(t) + manifest.Generation = 3 + manifest.Keys[0].State = "revoked" + // ConfigMap projection arrives before the Secret: all old names still exist. + writeManifest(t, manager.manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("mixed bundle did not reproduce publication: %v", err) + } + mixed := manager.state.Load().snapshot.digest + writeRotationMaterial(t, manager.keyDirectory, final) + complete, err := manager.load() + if err != nil || complete.digest == mixed { + t.Fatal("complete bundle did not differ from published mixed bundle") + } + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("same-generation material drift was not rejected") + } + if err := manager.Reload(t.Context()); err == nil { + t.Fatal("retry unexpectedly repaired the poisoned ledger") + } + // Recovery requires a forward generation, not a retry or ledger reset. + manifest.Generation++ + writeManifest(t, manager.manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } +} + +func TestSecurityRotationImmutableNamesHandleProjectionSkew(t *testing.T) { + for _, manifestFirst := range []bool{false, true} { + name := "material-first" + if manifestFirst { + name = "manifest-first" + } + t.Run(name, func(t *testing.T) { + manager, manifest, _, final := rotationSecurityFixture(t) + before := manager.state.Load().snapshot.digest + manifest.Generation = 3 + manifest.Keys[0].State = "revoked" + manifest.TLS = securityTLSManifest{CertificateFile: "provider-new.crt", PrivateKeyFile: "provider-new.key", ClientCAFile: "final-clients.pem"} + if manifestFirst { + writeManifest(t, manager.manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("missing immutable material did not fail closed") + } + } + writeRotationMaterial(t, manager.keyDirectory, map[string][]byte{"provider-new.crt": final["tls.crt"], "provider-new.key": final["tls.key"], "final-clients.pem": final["clients.pem"]}) + if !manifestFirst { + if err := manager.Reload(t.Context()); err != nil || manager.state.Load().snapshot.digest != before { + t.Fatal("staging new names changed the old manifest's authority") + } + writeManifest(t, manager.manifestPath, manifest) + } + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("complete immutable bundle did not converge: %v", err) + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal("same-generation immutable reload drifted") + } + }) + } +} diff --git a/internal/executionexecutor/executor.go b/internal/executionexecutor/executor.go index c429d41310..1c85956a53 100644 --- a/internal/executionexecutor/executor.go +++ b/internal/executionexecutor/executor.go @@ -202,25 +202,23 @@ func (e *Executor) resultLimit(requested int) int { return e.limits.MaxEntries } func capStreams(stdout, stderr []byte, n int) ([]byte, []byte, bool) { - original := len(stdout) + len(stderr) - stdout = capOutput(stdout, n) + var stdoutClipped, stderrClipped bool + stdout, stdoutClipped = capOutput(stdout, n) remaining := n - len(stdout) - if remaining < 0 { - remaining = 0 - } - stderr = capOutput(stderr, remaining) - return stdout, stderr, original > len(stdout)+len(stderr) + stderr, stderrClipped = capOutput(stderr, remaining) + return stdout, stderr, stdoutClipped || stderrClipped } -func capOutput(b []byte, n int) []byte { +func capOutput(b []byte, n int) ([]byte, bool) { b = []byte(strings.ToValidUTF8(string(b), "�")) - if len(b) > n { + clipped := len(b) > n + if clipped { b = b[:n] for !utf8.Valid(b) { b = b[:len(b)-1] } } - return b + return b, clipped } func bad(msg string) error { return coded(executionenv.CodeInvalidArgument, msg) } func coded(code executionenv.ErrorCode, msg string) error { diff --git a/internal/executionexecutor/executor_test.go b/internal/executionexecutor/executor_test.go index 90157af3e3..e99f7caa3d 100644 --- a/internal/executionexecutor/executor_test.go +++ b/internal/executionexecutor/executor_test.go @@ -120,6 +120,9 @@ func TestNativeShellCapsCombinedOutputAndRepairsUTF8(t *testing.T) { name, command string wantStdout, wantStderr int }{ + {"stdout-exact-cap-repair", `i=0; while [ "$i" -lt 1020 ]; do printf x; i=$((i+1)); done; printf '\377abc'`, 1024, 0}, + {"stderr-exact-cap-repair", `i=0; while [ "$i" -lt 1020 ]; do printf x >&2; i=$((i+1)); done; printf '\377abc' >&2`, 0, 1024}, + {"combined-exact-cap-repair", `printf xxxx; i=0; while [ "$i" -lt 1016 ]; do printf x >&2; i=$((i+1)); done; printf '\377abc' >&2`, 4, 1020}, {"stdout", `i=0; while [ "$i" -lt 2048 ]; do printf x; i=$((i+1)); done`, 1024, 0}, {"stderr", `i=0; while [ "$i" -lt 2048 ]; do printf x >&2; i=$((i+1)); done`, 0, 1024}, {"split-malformed", `printf '\377'; printf '\376' >&2; i=0; while [ "$i" -lt 700 ]; do printf x; printf y >&2; i=$((i+1)); done`, 703, 321}, From faacfcde71d06efb30ff0a4644bb124200411237 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Tue, 22 Sep 2026 00:06:56 +0300 Subject: [PATCH 19/25] fix(execution): publish immutable rotation bundles and verify read authority Co-Authored-By: OpenAI --- e2e/k8s_execution/fixture/rotation/main.go | 6 +- .../fixture/rotation/main_test.go | 20 ++ .../production_qualification_test.go | 182 +++++++++++++++--- .../zz_production_helm_lifetime_test.go | 15 +- internal/executionexecutor/executor_test.go | 9 + user-docs/building/deployment/mecak8s.md | 34 ++++ user-docs/features/execution-environments.md | 9 +- 7 files changed, 231 insertions(+), 44 deletions(-) diff --git a/e2e/k8s_execution/fixture/rotation/main.go b/e2e/k8s_execution/fixture/rotation/main.go index c00373feb0..3eb0c748fe 100644 --- a/e2e/k8s_execution/fixture/rotation/main.go +++ b/e2e/k8s_execution/fixture/rotation/main.go @@ -58,10 +58,10 @@ func generate(initial, out string) { key("k1", 1, "grant-k1.pem", oldGrant.Public().(ed25519.PublicKey), "active", now.Add(5*time.Hour)), key("k2", 2, "grant-k2.pem", pub2, "active", now.Add(5*time.Hour)), } - writeManifest(filepath.Join(out, "bridge.json"), manifest(2, "mecatl-execution", "k2", keys, "tls.crt", "tls.key", "clients.pem")) + writeManifest(filepath.Join(out, "bridge.json"), manifest(2, "mecatl-execution", "k2", keys, "tls.crt", "tls.key", "bridge-clients.pem")) keys[0].State = "revoked" - writeManifest(filepath.Join(out, "final.json"), manifest(3, "mecatl-execution", "k2", keys, "tls.crt", "tls.key", "clients.pem")) - writeManifest(filepath.Join(out, "restore-fixture-clients.json"), manifest(4, "mecatl-execution", "k2", keys, "tls.crt", "tls.key", "clients.pem")) + writeManifest(filepath.Join(out, "final.json"), manifest(3, "mecatl-execution", "k2", keys, "provider-new.crt", "provider-new.key", "final-clients.pem")) + writeManifest(filepath.Join(out, "restore-fixture-clients.json"), manifest(4, "mecatl-execution", "k2", keys, "provider-new.crt", "provider-new.key", "bridge-clients.pem")) } type keyEntry struct { diff --git a/e2e/k8s_execution/fixture/rotation/main_test.go b/e2e/k8s_execution/fixture/rotation/main_test.go index e148c79b4e..bbbd6a5f9b 100644 --- a/e2e/k8s_execution/fixture/rotation/main_test.go +++ b/e2e/k8s_execution/fixture/rotation/main_test.go @@ -10,6 +10,7 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/json" "encoding/pem" "math/big" "net" @@ -56,6 +57,25 @@ func TestForwardRestoreTrustBundleSupportsFixtureClient(t *testing.T) { t.Fatal(err) } generate(initial, out) + for _, tc := range []struct { + file, cert, key, ca string + }{ + {"bridge.json", "tls.crt", "tls.key", "bridge-clients.pem"}, + {"final.json", "provider-new.crt", "provider-new.key", "final-clients.pem"}, + {"restore-fixture-clients.json", "provider-new.crt", "provider-new.key", "bridge-clients.pem"}, + } { + var manifest struct { + TLS struct { + CertificateFile, PrivateKeyFile, ClientCAFile string + } + } + if err := json.Unmarshal(read(filepath.Join(out, tc.file)), &manifest); err != nil { + t.Fatal(err) + } + if manifest.TLS.CertificateFile != tc.cert || manifest.TLS.PrivateKeyFile != tc.key || manifest.TLS.ClientCAFile != tc.ca { + t.Errorf("%s must reference immutable material names", tc.file) + } + } clientKey, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go index 28d2b91785..a259fea642 100644 --- a/e2e/k8s_execution/production_qualification_test.go +++ b/e2e/k8s_execution/production_qualification_test.go @@ -230,6 +230,7 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { // The old connection remains usable, while a new-CA client can establish its // own connection and receives grants from k2. applySecurityCandidate(t, ctx, kubeconfig, rotationDir, filepath.Join(rotationDir, "bridge.json"), "bridge") + waitSecurityGeneration(ctx, t, kubeconfig, 2) waitProviderReadyReplicas(t, ctx, kubeconfig, 2) newTLS, err := executionclient.LoadTLSConfig(executionclient.TLSFiles{CA: filepath.Join(rotationDir, "client-roots.pem"), Cert: filepath.Join(rotationDir, "mecak8s-new.crt"), Key: filepath.Join(rotationDir, "mecak8s-new.key")}) if err != nil { @@ -250,9 +251,7 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { t.Fatalf("acquire bridge-authority claim: code=%s", remoteErrorCode(err)) } newRun := executionenv.RequestContext{Environment: renewed.Environment, Owner: owner, BindingID: binding, RunID: renewed.RunID, ClaimID: renewed.ClaimID, Epoch: renewed.Epoch, GrantGeneration: renewed.GrantGeneration, Grant: renewed.Grant} - if got, err := newClient.File(ctx, executionenv.FileRequest{Context: newRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err != nil || string(got.Data) != "old-authority\n" { - t.Fatalf("bridge authority did not authorize preserved data: %v", err) - } + waitFileContent(t, ctx, newClient, newRun, "rotation-sentinel.txt", "old-authority\n", "bridge authority") if err := newClient.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: renewed.Environment, Owner: owner, BindingID: binding, RunID: renewed.RunID, ClaimID: renewed.ClaimID, Epoch: renewed.Epoch, GrantGeneration: renewed.GrantGeneration, OperationID: "rotation-release-k2-bridge"}); err != nil { t.Fatalf("release bridge-phase claim: code=%s", remoteErrorCode(err)) } @@ -261,23 +260,24 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { // k1. Authorization is checked on every RPC, so the established old-client // HTTP/2 connection is denied rather than passing until reconnect. applySecurityCandidate(t, ctx, kubeconfig, rotationDir, filepath.Join(rotationDir, "final.json"), "final") + waitSecurityGeneration(ctx, t, kubeconfig, 3) waitProviderReadyReplicas(t, ctx, kubeconfig, 2) - if _, err := oldClient.File(ctx, executionenv.FileRequest{Context: oldRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err == nil { - t.Fatal("established client signed by the removed CA remained authorized") - } - if _, err := newClient.File(ctx, executionenv.FileRequest{Context: recoveredRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err == nil { - t.Fatal("grant signed by revoked k1 remained authorized through the new client") - } finalAttached := waitReady(t, ctx, newClient, owner, binding, attached.Environment) finalRequest := executionenv.RunClaimRequest{Environment: finalAttached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-final-grant", OperationID: "rotation-acquire-final", TTL: time.Minute} finalClaim, err := acquireCurrentAuthorityRun(ctx, finalRequest, newClient.AcquireRun, 500*time.Millisecond) if err != nil { - t.Fatalf("acquire final-authority claim: code=%s err=%v", remoteErrorCode(err), err) + var remote *executionenv.Error + t.Fatalf("acquire final-authority claim: errorCode=%s retryable=%t", remoteErrorCode(err), errors.As(err, &remote) && remote.Retryable) } finalRun := executionenv.RequestContext{Environment: finalClaim.Environment, Owner: owner, BindingID: binding, RunID: finalClaim.RunID, ClaimID: finalClaim.ClaimID, Epoch: finalClaim.Epoch, GrantGeneration: finalClaim.GrantGeneration, Grant: finalClaim.Grant} - if got, err := newClient.File(ctx, executionenv.FileRequest{Context: finalRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err != nil || string(got.Data) != "old-authority\n" { - t.Fatalf("new authority did not authorize preserved data: %v", err) - } + waitFileContent(t, ctx, newClient, finalRun, "rotation-sentinel.txt", "old-authority\n", "final authority") + // Both fixture certificates attest the same URI. Only the CA differs; the + // current claim below has just succeeded and has not been released. + if _, err := oldClient.File(ctx, executionenv.FileRequest{Context: finalRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); !isRemoteCode(err, executionenv.CodeUnauthenticated) { + var remote *executionenv.Error + t.Fatalf("removed client CA rejection: errorCode=%s retryable=%t", remoteErrorCode(err), errors.As(err, &remote) && remote.Retryable) + } + assertRetiredFixtureKeyDenied(ctx, t, newClient, state, finalRun, "rotation-sentinel.txt", "old-authority\n") freshClient, err := executionclient.New(forward.addr, newTLS) if err != nil { t.Fatal(err) @@ -431,34 +431,126 @@ func TestAcquireCurrentAuthorityRunStopsAtBoundAndHonorsContext(t *testing.T) { func waitFileContent(t *testing.T, ctx context.Context, client *executionclient.Client, rc executionenv.RequestContext, path, want, proof string) { t.Helper() - deadline := time.Now().Add(30 * time.Second) - var lastErr error - for time.Now().Before(deadline) { - got, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: path}) - if err == nil && string(got.Data) == want { - return + if err := readCurrentAuthorityContent(ctx, func(ctx context.Context) (executionenv.FileResponse, error) { + return client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: path}) + }, want, 500*time.Millisecond); err != nil { + var remote *executionenv.Error + retryable := errors.As(err, &remote) && remote.Retryable + t.Fatalf("%s did not authorize preserved data: errorCode=%s retryable=%t", proof, remoteErrorCode(err), retryable) + } +} + +func readCurrentAuthorityContent(ctx context.Context, read func(context.Context) (executionenv.FileResponse, error), want string, delay time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + const maxAttempts = 60 + for attempt := 0; ; attempt++ { + if err := ctx.Err(); err != nil { + return err + } + got, err := read(ctx) + if err == nil { + if string(got.Data) != want { + return errors.New("authority read returned unexpected content") + } + return nil + } + if !isRetryableAuthorityConvergence(err) || attempt == maxAttempts-1 { + return err + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} + +func TestReadCurrentAuthorityContentRetriesOnlyPreDispatchLag(t *testing.T) { + for _, tc := range []struct { + name string + err error + content string + wantCalls int + wantErr bool + }{ + {"success", nil, "sentinel", 1, false}, + {"authority lag", &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true}, "", 2, false}, + {"wrong content", nil, "wrong", 1, true}, + {"store not ready", &executionenv.Error{Code: executionenv.CodeNotReady}, "", 1, true}, + {"expired client", &executionenv.Error{Code: executionenv.CodeUnauthenticated, Retryable: true}, "", 1, true}, + {"revoked key", &executionenv.Error{Code: executionenv.CodePermissionDenied}, "", 1, true}, + {"unknown error", errors.New("not_ready"), "", 1, true}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + err := readCurrentAuthorityContent(t.Context(), func(context.Context) (executionenv.FileResponse, error) { + calls++ + if calls == 1 { + return executionenv.FileResponse{Data: []byte(tc.content)}, tc.err + } + return executionenv.FileResponse{Data: []byte("sentinel")}, nil + }, "sentinel", 0) + if calls != tc.wantCalls || (err != nil) != tc.wantErr { + t.Fatalf("calls=%d error=%t", calls, err != nil) + } + }) + } +} + +func TestReadCurrentAuthorityContentBoundAndCancellation(t *testing.T) { + t.Run("persistent lag", func(t *testing.T) { + calls := 0 + err := readCurrentAuthorityContent(t.Context(), func(context.Context) (executionenv.FileResponse, error) { + calls++ + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true} + }, "sentinel", 0) + if calls != 60 || !isRetryableAuthorityConvergence(err) { + t.Fatalf("calls=%d errorCode=%s", calls, remoteErrorCode(err)) + } + }) + for _, cancelInCall := range []bool{false, true} { + ctx, cancel := context.WithCancel(t.Context()) + if !cancelInCall { + cancel() + } + calls := 0 + err := readCurrentAuthorityContent(ctx, func(ctx context.Context) (executionenv.FileResponse, error) { + calls++ + if _, ok := ctx.Deadline(); !ok { + t.Fatal("read has no bounded deadline") + } + cancel() + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true} + }, "sentinel", time.Hour) + cancel() + wantCalls := 0 + if cancelInCall { + wantCalls = 1 + } + if !errors.Is(err, context.Canceled) || calls != wantCalls { + t.Fatalf("calls=%d cancelled=%t", calls, errors.Is(err, context.Canceled)) } - lastErr = err - time.Sleep(500 * time.Millisecond) } - t.Fatalf("%s did not authorize preserved data: %v", proof, lastErr) } func applySecurityCandidate(t *testing.T, ctx context.Context, kubeconfig, materialDir, manifest, mode string) { t.Helper() pki := filepath.Join(os.Getenv("MECATL_EXECUTION_QUAL_STATE"), "pki") - secretArgs := []string{"create", "secret", "generic", "execution-security", "-n", namespace, "--from-file=grant-k1.pem=" + filepath.Join(pki, "grant-key.pem")} + // Retain every previously referenced name with its original bytes. Secret + // and manifest projections may arrive in either order at each replica. + secretArgs := []string{"create", "secret", "generic", "execution-security", "-n", namespace, + "--from-file=grant-k1.pem=" + filepath.Join(pki, "grant-key.pem"), + "--from-file=tls.crt=" + filepath.Join(pki, "provider.crt"), + "--from-file=tls.key=" + filepath.Join(pki, "provider.key"), + "--from-file=clients.pem=" + filepath.Join(pki, "ca.crt")} switch mode { case "initial": - secretArgs = append(secretArgs, "--from-file=tls.crt="+filepath.Join(pki, "provider.crt"), "--from-file=tls.key="+filepath.Join(pki, "provider.key"), "--from-file=clients.pem="+filepath.Join(pki, "ca.crt")) - case "bridge": - secretArgs = append(secretArgs, "--from-file=grant-k2.pem="+filepath.Join(materialDir, "grant-k2.pem"), "--from-file=tls.crt="+filepath.Join(pki, "provider.crt"), "--from-file=tls.key="+filepath.Join(pki, "provider.key"), "--from-file=clients.pem="+filepath.Join(materialDir, "bridge-clients.pem")) - case "final", "restore": - secretArgs = append(secretArgs, "--from-file=grant-k2.pem="+filepath.Join(materialDir, "grant-k2.pem"), "--from-file=tls.crt="+filepath.Join(materialDir, "provider-new.crt"), "--from-file=tls.key="+filepath.Join(materialDir, "provider-new.key")) - if mode == "final" { - secretArgs = append(secretArgs, "--from-file=clients.pem="+filepath.Join(materialDir, "final-clients.pem")) - } else { - secretArgs = append(secretArgs, "--from-file=clients.pem="+filepath.Join(materialDir, "bridge-clients.pem")) + case "bridge", "final", "restore": + for _, name := range []string{"grant-k2.pem", "provider-new.crt", "provider-new.key", "bridge-clients.pem", "final-clients.pem"} { + secretArgs = append(secretArgs, "--from-file="+name+"="+filepath.Join(materialDir, name)) } default: t.Fatalf("unknown security candidate mode %q", mode) @@ -472,6 +564,7 @@ func applySecurityCandidate(t *testing.T, ctx context.Context, kubeconfig, mater func restoreFixtureSecurity(t *testing.T, ctx context.Context, kubeconfig, rotationDir string) { t.Helper() applySecurityCandidate(t, ctx, kubeconfig, rotationDir, filepath.Join(rotationDir, "restore-fixture-clients.json"), "restore") + waitSecurityGeneration(ctx, t, kubeconfig, 4) waitProviderReadyReplicas(t, ctx, kubeconfig, 2) pki := filepath.Join(os.Getenv("MECATL_EXECUTION_QUAL_STATE"), "pki") @@ -498,6 +591,31 @@ func applyKubectlInput(t *testing.T, ctx context.Context, kubeconfig string, inp } } +// PodReady can still describe the prior generation. Observe the nonsecret +// ledger first; the pinned replica's RPC barrier then proves local convergence. +func waitSecurityGeneration(ctx context.Context, t *testing.T, kubeconfig string, want uint64) { + t.Helper() + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + for { + cm := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-authority") + var ledger struct{ Generation uint64 } + if err := json.Unmarshal([]byte(cm.Data["state.json"]), &ledger); err != nil || ledger.Generation > want { + t.Fatal("unexpected authority ledger generation") + } + if ledger.Generation == want { + return + } + timer := time.NewTimer(500 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + t.Fatalf("authority ledger did not reach generation %d", want) + case <-timer.C: + } + } +} + func waitProviderReadyReplicas(t *testing.T, ctx context.Context, kubeconfig string, want int) { t.Helper() deadline := time.Now().Add(2 * time.Minute) diff --git a/e2e/k8s_execution/zz_production_helm_lifetime_test.go b/e2e/k8s_execution/zz_production_helm_lifetime_test.go index 266f8937cd..cda30de1ff 100644 --- a/e2e/k8s_execution/zz_production_helm_lifetime_test.go +++ b/e2e/k8s_execution/zz_production_helm_lifetime_test.go @@ -47,7 +47,7 @@ func TestKindExecutionProductionHelmLifetime(t *testing.T) { if err != nil || built.State != executionenv.CommandSucceeded || built.Result.ExitCode != 0 { t.Fatal("build lifetime probe failed") } - assertRetiredFixtureKeyDenied(t, ctx, client, state, rc) + assertRetiredFixtureKeyDenied(ctx, t, client, state, rc, "helm-sentinel", "retained-helm-data\n") release() // Occupy the single-slot profile; its reservation must still constrain Ensure // after adoption, rather than only matching a ConfigMap annotation. @@ -219,7 +219,7 @@ func TestKindExecutionProductionHelmLifetime(t *testing.T) { } rc, release = acquireRun(t, ctx, reattachedClient, owner, binding, reattached, "helm-lifetime-read") waitFileContent(t, ctx, reattachedClient, rc, "helm-sentinel", "retained-helm-data\n", "same-release Helm adoption") - assertRetiredFixtureKeyDenied(t, ctx, reattachedClient, state, rc) + assertRetiredFixtureKeyDenied(ctx, t, reattachedClient, state, rc, "helm-sentinel", "retained-helm-data\n") release() if _, err := reattachedClient.Ensure(ctx, quotaBinding+"-excess", "quota-cas", owner, "ensure-"+quotaBinding+"-excess"); !isRemoteCode(err, executionenv.CodeResourceExhausted) { t.Fatal("retained capacity did not reject excess allocation:", remoteErrorCode(err)) @@ -253,7 +253,7 @@ func TestKindExecutionProductionHelmLifetime(t *testing.T) { // Quiesced upgrades can outlive a grant/claim. Re-sign the CURRENT claim with // the fixture's retired k1, before and after adoption: expiry, released claims, // and stale epochs cannot masquerade as durable authority rejection. -func assertRetiredFixtureKeyDenied(t *testing.T, ctx context.Context, client *executionclient.Client, state string, rc executionenv.RequestContext) { +func assertRetiredFixtureKeyDenied(ctx context.Context, t *testing.T, client *executionclient.Client, state string, rc executionenv.RequestContext, path, want string) { t.Helper() parts := strings.Split(rc.Grant, ".") if len(parts) != 3 { @@ -293,11 +293,12 @@ func assertRetiredFixtureKeyDenied(t *testing.T, ctx context.Context, client *ex if _, err := verifier.Verify(old.Grant, executionenv.GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: rc.GrantGeneration, Operation: executionenv.OpFileRead}); err != nil { t.Fatal("retired-authority control is not cryptographically valid and current") } - waitFileContent(t, ctx, client, rc, "helm-sentinel", "retained-helm-data\n", "current authority before retired-key probe") - if _, err := client.File(ctx, executionenv.FileRequest{Context: old, Operation: executionenv.OpFileRead, Path: "helm-sentinel"}); !isRemoteCode(err, executionenv.CodePermissionDenied) { - t.Fatal("retired signing authority did not remain denied:", remoteErrorCode(err)) + waitFileContent(t, ctx, client, rc, path, want, "current authority before retired-key probe") + if _, err := client.File(ctx, executionenv.FileRequest{Context: old, Operation: executionenv.OpFileRead, Path: path}); !isRemoteCode(err, executionenv.CodePermissionDenied) { + var remote *executionenv.Error + t.Fatalf("retired signing authority rejection: errorCode=%s retryable=%t", remoteErrorCode(err), errors.As(err, &remote) && remote.Retryable) } - waitFileContent(t, ctx, client, rc, "helm-sentinel", "retained-helm-data\n", "current authority after retired-key probe") + waitFileContent(t, ctx, client, rc, path, want, "current authority after retired-key probe") } func requireOwnedHelmFixture(t *testing.T, state, kubeconfig string) { diff --git a/internal/executionexecutor/executor_test.go b/internal/executionexecutor/executor_test.go index e99f7caa3d..b4739a61af 100644 --- a/internal/executionexecutor/executor_test.go +++ b/internal/executionexecutor/executor_test.go @@ -139,6 +139,15 @@ func TestNativeShellCapsCombinedOutputAndRepairsUTF8(t *testing.T) { if !utf8.Valid(result.Stdout) || !utf8.Valid(result.Stderr) { t.Fatal("native command result contains malformed UTF-8") } + if strings.HasSuffix(tc.name, "exact-cap-repair") { + stream := result.Stdout + if tc.wantStderr != 0 { + stream = result.Stderr + } + if !strings.HasSuffix(string(stream), "�a") { + t.Fatal("repair-expansion clip did not retain the expected prefix") + } + } if tc.name == "split-malformed" && (!strings.HasPrefix(string(result.Stdout), "�") || !strings.HasPrefix(string(result.Stderr), "�")) { t.Fatal("malformed stream bytes were not repaired") } diff --git a/user-docs/building/deployment/mecak8s.md b/user-docs/building/deployment/mecak8s.md index 202e3d64d8..6ccef1b03c 100644 --- a/user-docs/building/deployment/mecak8s.md +++ b/user-docs/building/deployment/mecak8s.md @@ -350,6 +350,40 @@ grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/RecoverEnvironment ``` +#### Rotate execution-provider authority + +Give every changed material file a new, generation-specific basename. This applies +to signing keys, server certificates, server private keys, and client-CA bundles. +For example, a second bundle can use `grant-k2.pem`, `server-g2.crt`, +`server-g2.key`, and `clients-g2.pem`. Keep each name's bytes immutable. + +1. Add the new files to the operator-managed security Secret, retaining the files + referenced by the current manifest. Stage the material before changing + `provider.securityManifest`. +2. Publish a higher-generation manifest whose existing `file`, `certificateFile`, + `privateKeyFile`, and `clientCAFile` fields reference those names. For CA + rotation, first publish a separately named overlap bundle, move clients and + server trust, then publish another higher generation that removes old trust. +3. Verify that the authority ConfigMap has reached the intended generation and + use a current claim to read known workspace content. Pod readiness alone can + still reflect the previous generation. A replica whose snapshot lags the + ledger rejects requests before dispatch with structured `not_ready` and + `retryable=true`. Bound any read-only verification poll and stop on wrong + content, a nonretryable error, or another error code. +4. Retain overlap files until no live or in-flight manifest references them. + Retain verification keys through their grant windows before revoking them. + Never reuse a retired name for different bytes. + +Kubernetes projects Secret and ConfigMap updates independently. A manifest that +arrives before its new files fails closed until they arrive; staged files leave +an older manifest unchanged. Overwriting referenced TLS or CA files can instead +publish a mixed bundle's digest permanently at the new generation. If that has +happened, publish a complete, immutable bundle at a **higher** generation. +Repeated requests or a restart cannot repair same-generation digest drift; +preserve the authority ledger rather than resetting it. The provider confines +file access to the mounted security directory, but immutable publication remains +your secret-management procedure's responsibility. + #### Upgrade, uninstall, and reinstall the execution provider The supported lifecycle keeps the **same Helm release name, namespace, resource diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index 70708d82b9..b28c9c9437 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -198,8 +198,13 @@ for quiesced upgrades and scope removal. Use only basename file names. The projected Secret keys in this example are `grant-k1.pem`, `tls.crt`, `tls.key`, and `clients.pem`; an external secret manager owns their bytes. Kubernetes projected-volume `..data` symlinks are supported, -but paths escaping the mounted directory are rejected. Increase `generation` for -every authority change. Key IDs and `(id, version)` fingerprints cannot be +but paths escaping the mounted directory are rejected. Keep every referenced +filename immutable and use a new name for changed signing, TLS, or CA material. +Stage those files before publishing a higher-generation manifest and retain the +overlap files. Secret and ConfigMap projections are independent; see the +[authority rotation procedure](../building/deployment/mecak8s.md#rotate-execution-provider-authority) +for publication, verification, and recovery from mixed-material digest drift. +Increase `generation` for every authority change. Key IDs and `(id, version)` fingerprints cannot be reused; the provider persists a bounded high-water ledger in its authority ConfigMap. Keep retired keys as `verify-only` until all grants expire, then mark them `revoked`. Invalid, incomplete, rolled-back, or newly expired material makes From 19853dbd3e704d4c0939852b84faaa22ef9ef425 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Tue, 22 Sep 2026 00:31:24 +0300 Subject: [PATCH 20/25] fix(execution): wire bounded manual native qualification Stage the live credential only after same-run production qualification, validate owned cleanup, and allowlist bounded evidence. Prove durable revocation after restart with a current signed claim and exact generation receipt. Co-Authored-By: OpenAI --- .github/workflows/e2e-live.yml | 173 ++++++++++++++- deploy/mecatl-execution-kind/ci_test.go | 204 ++++++++++++++++++ deploy/mecatl-execution-kind/live.sh | 10 +- .../production_qualification_test.go | 23 +- .../zz_production_helm_lifetime_test.go | 72 +++++-- user-docs/features/execution-environments.md | 48 +++++ 6 files changed, 509 insertions(+), 21 deletions(-) create mode 100644 deploy/mecatl-execution-kind/ci_test.go diff --git a/.github/workflows/e2e-live.yml b/.github/workflows/e2e-live.yml index 9d1a019bfb..1355a6cb00 100644 --- a/.github/workflows/e2e-live.yml +++ b/.github/workflows/e2e-live.yml @@ -18,9 +18,9 @@ # even when labelled; see ci.yml's header for the pwn-request rationale. # - The job `if:` also requires github.repository == 'stacklok/mecatl', so a # fork's own scheduled/dispatched runs never reference the secret. -# - A guard step checks OPENROUTER_API_KEY availability and skips the live -# run cleanly (notice, green) when it is absent — the workflow never -# hard-fails just because the secret isn't configured. +# - Ordinary live jobs skip cleanly when OPENROUTER_API_KEY is absent. The +# manual-only native qualification instead fails closed on a missing key. +# It requires a successful same-run production qualification before staging. # # All third-party actions are SHA-pinned (with a # vX.Y.Z comment) so a # re-pointed tag from a compromised maintainer cannot change what runs here. @@ -31,6 +31,15 @@ on: # Nightly, off-peak UTC. Odd minute to avoid the top-of-hour thundering herd. - cron: '17 3 * * *' workflow_dispatch: + inputs: + native_execution: + description: Run only native execution production + OpenRouter qualification + type: boolean + default: false + expected_sha: + description: Optional exact reviewed commit SHA (native execution only) + type: string + default: '' pull_request: types: [opened, synchronize, reopened, labeled] @@ -58,6 +67,7 @@ jobs: # Gate 2: PRs run only when a maintainer opted in via the `e2e-live` label. if: >- github.repository == 'stacklok/mecatl' && + !(github.event_name == 'workflow_dispatch' && inputs.native_execution) && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'e2e-live')) # One live run at a time, globally: overlapping dispatches/labelled pushes @@ -177,6 +187,7 @@ jobs: # and PRs run only when a maintainer opted in via the `e2e-live` label. if: >- github.repository == 'stacklok/mecatl' && + !(github.event_name == 'workflow_dispatch' && inputs.native_execution) && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'e2e-live')) # Separate group from `e2e-live` so the two live jobs run in parallel; they @@ -258,3 +269,159 @@ jobs: path: k8s-e2e-live-output.log retention-days: 7 if-no-files-found: warn + + native-execution-live: + name: Native execution (production + OpenRouter) + if: >- + github.repository == 'stacklok/mecatl' && + github.event_name == 'workflow_dispatch' && inputs.native_execution + runs-on: ubuntu-24.04 + timeout-minutes: 100 + concurrency: + group: e2e-live-native-execution + cancel-in-progress: false + env: + NATIVE_CI_DIR: ${{ github.workspace }}/.scratch/native-live-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Verify reviewed commit and reserve private CI directory + id: prepare + env: + EXPECTED_SHA: ${{ inputs.expected_sha }} + EVENT_SHA: ${{ github.sha }} + run: | + actual=$(git rev-parse HEAD) + test "$actual" = "$EVENT_SHA" + if [ -n "$EXPECTED_SHA" ] && [ "$actual" != "$EXPECTED_SHA" ]; then + echo '::error::Reviewed SHA does not match dispatch SHA' + exit 1 + fi + printf '## Native execution qualification\n\nCommit: `%s`\n' "$actual" >> "$GITHUB_STEP_SUMMARY" + umask 077 + mkdir -p .scratch + mkdir -m 700 "$NATIVE_CI_DIR" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v3.18.4 + - name: Install pinned Kind and ko + run: | + go install sigs.k8s.io/kind@v0.33.0 + go install github.com/google/ko@v0.18.1 + command -v kubectl jq docker + # No provider secret here. run.sh installs pinned Calico and retains the + # owned cluster; the success dependency, not ownership.profile, admits live. + - name: Qualify production execution + id: production + env: + CONTAINER_ENGINE: docker + MECATL_EXECUTION_QUAL_CI: "0" + run: task e2e:k8s:execution:production + - name: Stage private provider credential + id: credential + if: success() && steps.production.outcome == 'success' + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + set +x + if [ -z "$OPENROUTER_API_KEY" ]; then + echo '::error::OPENROUTER_API_KEY is required for native qualification' + exit 1 + fi + umask 077 + set -C + printf '%s' "$OPENROUTER_API_KEY" > "$NATIVE_CI_DIR/provider-key" + unset OPENROUTER_API_KEY + - name: Qualify native OpenRouter coding + id: live + if: success() && steps.production.outcome == 'success' && steps.credential.outcome == 'success' + run: | + IFS= read -r state < .scratch/k8s-execution/current + MECATL_EXECUTION_QUAL_STATE="$state" \ + MECATL_EXECUTION_CREDENTIAL_FILE="$NATIVE_CI_DIR/provider-key" \ + ./deploy/mecatl-execution-kind/live.sh + # Credential removal is independent of cluster availability. Fail closed + # on ownership drift; runner disposal is a last resort, not cleanup proof. + - name: Clean up exact owned resources + id: cleanup + if: always() && steps.prepare.outcome == 'success' + env: + PRODUCTION_OUTCOME: ${{ steps.production.outcome }} + run: | + key="$NATIVE_CI_DIR/provider-key" + if [ -e "$key" ] || [ -L "$key" ]; then + test -f "$key" && test ! -L "$key" + rm -- "$key" + fi + if [ "$PRODUCTION_OUTCOME" = skipped ]; then exit 0; fi + pointer="$GITHUB_WORKSPACE/.scratch/k8s-execution/current" + test -f "$pointer" && test ! -L "$pointer" + IFS= read -r state < "$pointer" + test ! -L "$state" + state=$(cd -- "$state" && pwd -P) + case "$state" in "$GITHUB_WORKSPACE/.scratch/k8s-execution/"*) ;; *) exit 1 ;; esac + ownership="$state/ownership" + test -f "$ownership" && test ! -L "$ownership" + owned_value() { + awk -F= -v key="$1" '$1 == key {value=substr($0,length(key)+2); count++} END {if(count != 1 || value == "") exit 1; print value}' "$ownership" + } + cluster=$(owned_value cluster) + case "$cluster" in mecatl-execution-qual-*) ;; *) exit 1 ;; esac + case "$cluster" in *[!a-z0-9-]*) exit 1 ;; esac + test "${#cluster}" -le 63 + test "$(owned_value owner)" = "$USER" + test "$(owned_value runtime)" = docker + test "$(owned_value profile)" = production + test "$(owned_value namespace)" = execution-qualification + context="kind-$cluster" + test "$(owned_value context)" = "$context" + kubeconfig="$state/kubeconfig" + test "$(owned_value kubeconfig)" = "$kubeconfig" + test -f "$kubeconfig" && test ! -L "$kubeconfig" + test "$(stat -c '%a' "$kubeconfig")" = 600 + test "$(kubectl --kubeconfig "$kubeconfig" --context "$context" config current-context)" = "$context" + test "$(docker inspect "${cluster}-control-plane" --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}')" = "$cluster" + kind delete cluster --name "$cluster" + clusters=$(kind get clusters) + if printf '%s\n' "$clusters" | grep -Fx -- "$cluster"; then exit 1; fi + # This file is emitted by the typed live test, not a transcript. Bound + # its upload independently and never collect the surrounding state. + summary="$state/live-summary.json" + if [ -e "$summary" ]; then + test -f "$summary" && test ! -L "$summary" + test "$(stat -c '%s' "$summary")" -le 16384 + cp -- "$summary" "$NATIVE_CI_DIR/live-summary.json" + fi + - name: Record sanitized qualification status + if: always() && steps.prepare.outcome == 'success' + env: + PRODUCTION_OUTCOME: ${{ steps.production.outcome }} + CREDENTIAL_OUTCOME: ${{ steps.credential.outcome }} + LIVE_OUTCOME: ${{ steps.live.outcome }} + CLEANUP_OUTCOME: ${{ steps.cleanup.outcome }} + run: | + printf 'production=%s\ncredential=%s\nlive=%s\ncleanup=%s\n' \ + "$PRODUCTION_OUTCOME" "$CREDENTIAL_OUTCOME" "$LIVE_OUTCOME" "$CLEANUP_OUTCOME" \ + > "$NATIVE_CI_DIR/qualification-status.txt" + cat "$NATIVE_CI_DIR/qualification-status.txt" >> "$GITHUB_STEP_SUMMARY" + - name: Upload only bounded native qualification evidence + if: always() && steps.prepare.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-execution-live-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ env.NATIVE_CI_DIR }}/live-summary.json + ${{ env.NATIVE_CI_DIR }}/qualification-status.txt + include-hidden-files: true + retention-days: 7 + if-no-files-found: error diff --git a/deploy/mecatl-execution-kind/ci_test.go b/deploy/mecatl-execution-kind/ci_test.go new file mode 100644 index 0000000000..69f44fd8f6 --- /dev/null +++ b/deploy/mecatl-execution-kind/ci_test.go @@ -0,0 +1,204 @@ +package executionkind_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/goccy/go-yaml" +) + +type workflowStep struct { + ID string `yaml:"id"` + Run string `yaml:"run"` + If string `yaml:"if"` + Env map[string]string `yaml:"env"` + With map[string]any `yaml:"with"` +} + +func nativeSteps(t *testing.T) map[string]workflowStep { + t.Helper() + data, err := os.ReadFile("../../.github/workflows/e2e-live.yml") + if err != nil { + t.Fatal(err) + } + var workflow struct { + Jobs map[string]struct { + If string `yaml:"if"` + Steps []workflowStep `yaml:"steps"` + Timeout int `yaml:"timeout-minutes"` + } `yaml:"jobs"` + } + if err := yaml.Unmarshal(data, &workflow); err != nil { + t.Fatal(err) + } + job := workflow.Jobs["native-execution-live"] + if strings.Join(strings.Fields(job.If), " ") != "github.repository == 'stacklok/mecatl' && github.event_name == 'workflow_dispatch' && inputs.native_execution" || job.Timeout != 100 { + t.Fatal("native job must remain explicitly dispatched, repo-gated, and bounded") + } + steps := make(map[string]workflowStep) + for _, step := range job.Steps { + if step.ID != "" { + steps[step.ID] = step + } + if strings.Contains(step.Run, "${{") { + t.Fatal("workflow expressions must cross into shell through env") + } + if step.With["cache"] == true { + t.Fatal("native credential job must not save a cache") + } + } + if steps["production"].Env["MECATL_EXECUTION_QUAL_CI"] != "0" || len(steps["credential"].Env) != 1 || steps["credential"].Env["OPENROUTER_API_KEY"] != "${{ secrets.OPENROUTER_API_KEY }}" { + t.Fatal("production retention or step-scoped credential wiring drifted") + } + if steps["credential"].If != "success() && steps.production.outcome == 'success'" || steps["live"].If != "success() && steps.production.outcome == 'success' && steps.credential.outcome == 'success'" || steps["cleanup"].If != "always() && steps.prepare.outcome == 'success'" { + t.Fatal("qualification must depend on actual production success and always clean up") + } + return steps +} + +func writeFixture(t *testing.T, path, body string, mode os.FileMode) { + t.Helper() + if err := os.WriteFile(path, []byte(body), mode); err != nil { + t.Fatal(err) + } +} + +func runStep(t *testing.T, root, script string, env ...string) ([]byte, error) { + t.Helper() + cmd := exec.CommandContext(t.Context(), "bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", script) + cmd.Dir = root + cmd.Env = append([]string{"PATH=" + os.Getenv("PATH"), "HOME=" + root}, env...) + return cmd.CombinedOutput() +} + +func TestNativeWorkflowCommitAndCredentialBoundary(t *testing.T) { + steps := nativeSteps(t) + for _, expected := range []string{"", "reviewed", "wrong", "$(touch injected)"} { + t.Run("sha="+expected, func(t *testing.T) { + root := t.TempDir() + bin := filepath.Join(root, "bin") + if err := os.Mkdir(bin, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(bin, "git"), "#!/bin/sh\nprintf 'reviewed\\n'\n", 0o700) + dir := filepath.Join(root, ".scratch", "ci") + out, err := runStep(t, root, steps["prepare"].Run, "PATH="+bin+":"+os.Getenv("PATH"), "NATIVE_CI_DIR="+dir, "EVENT_SHA=reviewed", "EXPECTED_SHA="+expected, "GITHUB_STEP_SUMMARY="+filepath.Join(root, "summary")) + wantOK := expected == "" || expected == "reviewed" + if (err == nil) != wantOK { + t.Fatalf("SHA gate: %v: %s", err, out) + } + if _, err := os.Stat(filepath.Join(root, "injected")); !os.IsNotExist(err) { + t.Fatal("expected SHA was executed") + } + if !wantOK { + return + } + info, err := os.Stat(dir) + if err != nil || info.Mode().Perm() != 0o700 { + t.Fatal("CI directory is not private") + } + if _, err := runStep(t, root, steps["credential"].Run, "NATIVE_CI_DIR="+dir, "OPENROUTER_API_KEY="); err == nil { + t.Fatal("missing credential skipped successfully") + } + const fake = "synthetic-offline-key" + out, err = runStep(t, root, steps["credential"].Run+"\ntest -z \"${OPENROUTER_API_KEY+x}\"", "NATIVE_CI_DIR="+dir, "OPENROUTER_API_KEY="+fake) + if err != nil || strings.Contains(string(out), fake) { + t.Fatal("credential staging failed or disclosed synthetic credential") + } + info, err = os.Stat(filepath.Join(dir, "provider-key")) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatal("credential file is not private") + } + if _, err := runStep(t, root, steps["credential"].Run, "NATIVE_CI_DIR="+dir, "OPENROUTER_API_KEY=replacement"); err == nil { + t.Fatal("credential staging overwrote existing file") + } + data, err := os.ReadFile(filepath.Join(dir, "provider-key")) + if err != nil || string(data) != fake { + t.Fatal("existing synthetic credential changed") + } + }) + } +} + +func TestNativeWorkflowCleanupOwnership(t *testing.T) { + steps := nativeSteps(t) + for _, fault := range []string{"", "owner", "context", "label", "duplicate", "symlink", "delete", "list"} { + t.Run("fault="+fault, func(t *testing.T) { + root := t.TempDir() + state := filepath.Join(root, ".scratch", "k8s-execution", "owned") + dir := filepath.Join(root, ".scratch", "ci") + bin := filepath.Join(root, "bin") + for _, path := range []string{state, dir, bin} { + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + } + const cluster = "mecatl-execution-qual-offline" + context := "kind-" + cluster + owner := "fixture" + if fault == "owner" { + owner = "someone-else" + } + if fault == "context" { + context = "ambient" + } + kubeconfig := filepath.Join(state, "kubeconfig") + writeFixture(t, kubeconfig, "synthetic", 0o600) + ownership := "cluster=" + cluster + "\ncontext=" + context + "\nowner=" + owner + "\nruntime=docker\nprofile=production\nnamespace=execution-qualification\nkubeconfig=" + kubeconfig + "\n" + if fault == "duplicate" { + ownership += "owner=fixture\n" + } + writeFixture(t, filepath.Join(state, "ownership"), ownership, 0o600) + pointer := filepath.Join(filepath.Dir(state), "current") + if fault == "symlink" { + if err := os.Symlink(kubeconfig, pointer); err != nil { + t.Fatal(err) + } + } else { + writeFixture(t, pointer, state+"\n", 0o600) + } + writeFixture(t, filepath.Join(dir, "provider-key"), "synthetic", 0o600) + writeFixture(t, filepath.Join(state, "live-summary.json"), "{}\n", 0o600) + writeFixture(t, filepath.Join(bin, "kubectl"), "#!/bin/sh\nprintf '%s\\n' 'kind-"+cluster+"'\n", 0o700) + writeFixture(t, filepath.Join(bin, "docker"), "#!/bin/sh\nif [ \"$FAULT\" = label ]; then exit 1; fi\nprintf '%s\\n' '"+cluster+"'\n", 0o700) + writeFixture(t, filepath.Join(bin, "kind"), "#!/bin/sh\nif [ \"$1\" = delete ]; then\n test \"$*\" = 'delete cluster --name "+cluster+"' || exit 1\n test \"$FAULT\" != delete || exit 1\n printf deleted > \"$MARKER\"\nelse\n test \"$FAULT\" != list || exit 1\nfi\n", 0o700) + marker := filepath.Join(root, "deleted") + out, err := runStep(t, root, steps["cleanup"].Run, "PATH="+bin+":"+os.Getenv("PATH"), "NATIVE_CI_DIR="+dir, "GITHUB_WORKSPACE="+root, "USER=fixture", "PRODUCTION_OUTCOME=success", "FAULT="+fault, "MARKER="+marker) + if (err == nil) != (fault == "") { + t.Fatalf("cleanup result: %v: %s", err, out) + } + if _, err := os.Stat(filepath.Join(dir, "provider-key")); !os.IsNotExist(err) { + t.Fatal("CI credential was not removed independently") + } + _, deleted := os.Stat(marker) + if (deleted == nil) != (fault == "" || fault == "list") { + t.Fatal("wrong cluster deletion decision") + } + }) + } +} + +func TestLiveImageDigestUsesRuntimeSpecificInspect(t *testing.T) { + data, err := os.ReadFile("live.sh") + if err != nil { + t.Fatal(err) + } + start := strings.Index(string(data), "if [ \"$runtime\" = podman ]; then\n podman image exists") + end := strings.Index(string(data), "\nworkload_tag=") + if start < 0 || end < start { + t.Fatal("image build preparation missing") + } + for _, runtime := range []string{"docker", "podman"} { + t.Run(runtime, func(t *testing.T) { + root := t.TempDir() + writeFixture(t, filepath.Join(root, runtime), "#!/bin/sh\ncase \"$*\" in\n 'image exists docker.io/library/golang:1.27'|'image inspect docker.io/library/golang:1.27') exit 0 ;;\n 'image inspect docker.io/library/golang:1.27 --format {{.Digest}}') test \"$RUNTIME\" = podman || exit 1; printf 'sha256:fake' ;;\n 'image inspect docker.io/library/golang:1.27 --format {{index .RepoDigests 0}}') test \"$RUNTIME\" = docker || exit 1; printf 'golang@sha256:fake' ;;\n *) exit 1 ;;\nesac\n", 0o700) + out, err := runStep(t, root, string(data[start:end])+"\ntest \"$go_image\" = docker.io/library/golang@sha256:fake", "PATH="+root+":"+os.Getenv("PATH"), "runtime="+runtime, "RUNTIME="+runtime) + if err != nil { + t.Fatalf("digest selection: %v: %s", err, out) + } + }) + } +} diff --git a/deploy/mecatl-execution-kind/live.sh b/deploy/mecatl-execution-kind/live.sh index 99a222f01c..00c092dcdd 100755 --- a/deploy/mecatl-execution-kind/live.sh +++ b/deploy/mecatl-execution-kind/live.sh @@ -98,8 +98,14 @@ load_image() { provider_tag=$(build_ko ./cmd/mecatl-execution-provider ko.local/mecatl-execution-provider) agent_tag=$(build_ko ./cmd/mecak8s ko.local/mecak8s) -$runtime image inspect docker.io/library/golang:1.27 >/dev/null 2>&1 || $runtime pull docker.io/library/golang:1.27 >/dev/null -go_digest=$($runtime image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') +if [ "$runtime" = podman ]; then + podman image exists docker.io/library/golang:1.27 || podman pull docker.io/library/golang:1.27 >/dev/null + go_digest=$(podman image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') +else + docker image inspect docker.io/library/golang:1.27 >/dev/null 2>&1 || docker pull docker.io/library/golang:1.27 >/dev/null + go_ref=$(docker image inspect docker.io/library/golang:1.27 --format '{{index .RepoDigests 0}}') + go_digest=${go_ref#*@} +fi go_image="docker.io/library/golang@${go_digest}" workload_tag=localhost/mecatl-execution-workload:e2e $runtime build --build-arg GO_IMAGE="$go_image" -f "$root/build/execution-workload/Dockerfile" -t "$workload_tag" "$root" >/dev/null diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go index a259fea642..02a9950820 100644 --- a/e2e/k8s_execution/production_qualification_test.go +++ b/e2e/k8s_execution/production_qualification_test.go @@ -306,9 +306,28 @@ func TestKindExecutionProductionSecurityRotation(t *testing.T) { t.Fatal(err) } defer postRestartClient.Close() - if _, err := postRestartClient.File(ctx, executionenv.FileRequest{Context: newRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err == nil { - t.Fatal("revoked generation became usable after provider restart") + postRestartAttached := waitReady(t, ctx, postRestartClient, owner, binding, attached.Environment) + if postRestartAttached.GrantGeneration != generation || generation != newAttached.GrantGeneration+1 { + t.Fatalf("persisted revocation generation=%d want=%d", postRestartAttached.GrantGeneration, generation) } + replayed, err = postRestartClient.RevokeEnvironment(ctx, attached.Environment, owner, newAttached.GrantGeneration, "rotation-revoke-replay") + if err != nil || replayed != generation { + t.Fatalf("post-restart revoke receipt generation=%d want=%d code=%s", replayed, generation, remoteErrorCode(err)) + } + currentRun, releaseCurrent := acquireRun(t, ctx, postRestartClient, owner, binding, postRestartAttached, "rotation-post-restart") + defer releaseCurrent() + if currentRun.GrantGeneration != generation { + t.Fatalf("post-restart claim generation=%d want=%d", currentRun.GrantGeneration, generation) + } + // Keep the live claim's identity and lifetime, signing with the trusted k2. + // Only the generation is stale: neither a released claim nor retired k1 can + // explain the denial. Successful reads bracket the probe on this connection. + stale := resignFixtureGrant(t, currentRun, filepath.Join(rotationDir, "grant-k2.pem"), "k2", newAttached.GrantGeneration) + waitFileContent(t, ctx, postRestartClient, currentRun, "rotation-sentinel.txt", "old-authority\n", "post-restart revocation positive control") + if _, err := postRestartClient.File(ctx, executionenv.FileRequest{Context: stale, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("revoked claim generation rejection: code=%s, want conflict", remoteErrorCode(err)) + } + waitFileContent(t, ctx, postRestartClient, currentRun, "rotation-sentinel.txt", "old-authority\n", "post-restart revocation positive control after probe") // Restore fixture client compatibility through a higher generation; this is // another forward rotation, never a high-water-mark rollback. restoreFixtureSecurity(t, ctx, kubeconfig, rotationDir) diff --git a/e2e/k8s_execution/zz_production_helm_lifetime_test.go b/e2e/k8s_execution/zz_production_helm_lifetime_test.go index cda30de1ff..425dd87cdc 100644 --- a/e2e/k8s_execution/zz_production_helm_lifetime_test.go +++ b/e2e/k8s_execution/zz_production_helm_lifetime_test.go @@ -254,6 +254,17 @@ func TestKindExecutionProductionHelmLifetime(t *testing.T) { // the fixture's retired k1, before and after adoption: expiry, released claims, // and stale epochs cannot masquerade as durable authority rejection. func assertRetiredFixtureKeyDenied(ctx context.Context, t *testing.T, client *executionclient.Client, state string, rc executionenv.RequestContext, path, want string) { + t.Helper() + old := resignFixtureGrant(t, rc, filepath.Join(state, "pki", "grant-key.pem"), "k1", rc.GrantGeneration) + waitFileContent(t, ctx, client, rc, path, want, "current authority before retired-key probe") + if _, err := client.File(ctx, executionenv.FileRequest{Context: old, Operation: executionenv.OpFileRead, Path: path}); !isRemoteCode(err, executionenv.CodePermissionDenied) { + var remote *executionenv.Error + t.Fatalf("retired signing authority rejection: errorCode=%s retryable=%t", remoteErrorCode(err), errors.As(err, &remote) && remote.Retryable) + } + waitFileContent(t, ctx, client, rc, path, want, "current authority after retired-key probe") +} + +func resignFixtureGrant(t *testing.T, rc executionenv.RequestContext, keyPath, keyID string, generation uint64) executionenv.RequestContext { t.Helper() parts := strings.Split(rc.Grant, ".") if len(parts) != 3 { @@ -267,9 +278,9 @@ func assertRetiredFixtureKeyDenied(ctx context.Context, t *testing.T, client *ex if err := json.Unmarshal(payload, &claims); err != nil { t.Fatal("decode current fixture claims") } - material, err := os.ReadFile(filepath.Join(state, "pki", "grant-key.pem")) + material, err := os.ReadFile(keyPath) if err != nil { - t.Fatal("read test-owned retired signing key") + t.Fatal("read test-owned signing key") } block, _ := pem.Decode(material) if block == nil { @@ -283,22 +294,55 @@ func assertRetiredFixtureKeyDenied(ctx context.Context, t *testing.T, client *ex if !ok { t.Fatal("fixture signing key is not Ed25519") } - claims.KeyID = "k1" - old := rc - old.Grant, err = executionenv.SignGrant(private, claims) + claims.KeyID = keyID + claims.GrantGeneration = generation + probe := rc + probe.GrantGeneration = generation + probe.Grant, err = executionenv.SignGrant(private, claims) if err != nil { - t.Fatal("sign retired-authority fixture grant") + t.Fatal("sign fixture grant") } - verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": private.Public().(ed25519.PublicKey)}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: time.Minute} - if _, err := verifier.Verify(old.Grant, executionenv.GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: rc.GrantGeneration, Operation: executionenv.OpFileRead}); err != nil { - t.Fatal("retired-authority control is not cryptographically valid and current") + verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{keyID: private.Public().(ed25519.PublicKey)}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: time.Minute} + if _, err := verifier.Verify(probe.Grant, executionenv.GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: generation, Operation: executionenv.OpFileRead}); err != nil { + t.Fatal("fixture probe is not cryptographically valid and current") } - waitFileContent(t, ctx, client, rc, path, want, "current authority before retired-key probe") - if _, err := client.File(ctx, executionenv.FileRequest{Context: old, Operation: executionenv.OpFileRead, Path: path}); !isRemoteCode(err, executionenv.CodePermissionDenied) { - var remote *executionenv.Error - t.Fatalf("retired signing authority rejection: errorCode=%s retryable=%t", remoteErrorCode(err), errors.As(err, &remote) && remote.Retryable) + return probe +} + +func TestResignFixtureGrantChangesOnlyRequestedAuthority(t *testing.T) { + key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, ed25519.SeedSize)) + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal("marshal synthetic signing key") + } + keyPath := filepath.Join(t.TempDir(), "synthetic.pem") + if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o600); err != nil { + t.Fatal("write synthetic signing key") + } + now := time.Now().UTC().Truncate(time.Second) + claims := executionenv.GrantClaims{KeyID: "k2", Issuer: "fixture", Audience: "fixture", Client: "fixture-client", OwnerHash: "fixture-owner", BindingID: "binding", RunID: "run", ClaimID: "claim", Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, Epoch: 3, GrantGeneration: 2, Operations: []executionenv.Operation{executionenv.OpFileRead}, NotBefore: now.Add(-time.Second), ExpiresAt: now.Add(30 * time.Second), Nonce: "fixture-nonce"} + grant, err := executionenv.SignGrant(key, claims) + if err != nil { + t.Fatal("sign synthetic control") + } + rc := executionenv.RequestContext{Environment: claims.Environment, Owner: executionenv.Owner{Issuer: "issuer", Subject: "subject"}, BindingID: claims.BindingID, RunID: claims.RunID, ClaimID: claims.ClaimID, Epoch: claims.Epoch, GrantGeneration: claims.GrantGeneration, Grant: grant} + for _, probe := range []struct { + keyID string + generation uint64 + }{{"k2", 1}, {"k1", 2}} { + resigned := resignFixtureGrant(t, rc, keyPath, probe.keyID, probe.generation) + verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{probe.keyID: key.Public().(ed25519.PublicKey)}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: time.Minute} + got, err := verifier.Verify(resigned.Grant, executionenv.GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: probe.generation, Operation: executionenv.OpFileRead}) + want := claims + want.KeyID, want.GrantGeneration = probe.keyID, probe.generation + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatal("probe changed claims beyond key ID and generation") + } + resigned.Grant, resigned.GrantGeneration = rc.Grant, rc.GrantGeneration + if resigned != rc { + t.Fatal("probe changed request identity") + } } - waitFileContent(t, ctx, client, rc, path, want, "current authority after retired-key probe") } func requireOwnedHelmFixture(t *testing.T, state, kubeconfig string) { diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index b28c9c9437..ba20dbaf21 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -272,6 +272,54 @@ successful `go test` independently through the typed gRPC execution service. Finally it restores the mock deployment and removes only its run-scoped Secret; the owned cluster and execution workspaces remain for inspection. +### Run the native qualification in GitHub Actions + +After independent review of the exact candidate, a maintainer can dispatch the +existing `e2e-live.yml` workflow in `stacklok/mecatl`: + +```sh +gh workflow run e2e-live.yml --repo stacklok/mecatl --ref \ + -f native_execution=true -f expected_sha= +``` + +`native_execution` defaults to `false`. Setting it to `true` runs only the native +job, avoiding duplicate paid inference in the ordinary live jobs. Schedules and +labelled PR runs retain their ordinary behavior and never select this job. +`expected_sha` is optional, but supply it for reviewed qualification: checkout +uses the dispatch event SHA, verifies exact equality before credential staging, +and records that SHA in the job summary. A moved branch fails the check. + +The trust gates are maintainer dispatch, the `stacklok/mecatl` repository check, +and access to the existing `OPENROUTER_API_KEY` repository secret. There is no +GitHub environment-approval gate. Review the workflow and all executed source +before dispatch; the SHA check establishes identity, not code safety. Checkout +retains no Git credentials, the token has read-only contents access, and the +native job uses no build cache. + +The same runner first completes the production Kind+Calico qualification without +a provider key or automatic cluster deletion. Only a successful production step +allows credential staging; a production ownership record alone is insufficient. +The workflow stages the key in a new 0700 CI directory and 0600 file, then removes +it from the environment before `live.sh` validates the retained owned state, +reruns mock qualification, and invokes the trusted credential loader. A missing +secret fails qualification rather than skipping successfully. The 100-minute job +bound includes builds, the 45-minute production test bound, 15-minute mock rerun, +10-minute live smoke, and cleanup; token limits are not a hard dollar cap. + +The live script restores mock configuration and deletes its Secret by recorded +UID using an independent cleanup context. Failures fail qualification. An always-run +workflow step removes only the CI-created credential file and deletes only the +recorded cluster after validating its owner, name, namespace, profile, container +label, private kubeconfig, and context. Ownership drift fails cleanup without +selecting another cluster. Runner disposal is a last resort, not evidence that +cleanup succeeded. + +Artifacts are retained for seven days: only a size-bounded `live-summary.json` +when available and a sanitized `qualification-status.txt`. PKI, kubeconfigs, +Secret receipts, Helm values, full scratch directories, and transcripts are never +uploaded by the native job. Check both live and cleanup outcomes before treating +the recorded commit as qualified. + ## Limitations - No-FS sessions cannot use local file tools, shell commands, workspace forks, From 2e1aa6bbfc59bcc8f87211483ddc0b63cdd60669 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Tue, 22 Sep 2026 00:48:56 +0300 Subject: [PATCH 21/25] fix(ci): make retained native qualification safe to rerun Verify image alias targets before reuse, bound qualification stages with cleanup headroom, and bind downstream ownership to production step outputs. Cover retained loads, conflicts, partial setup, and signals with offline fake commands. Co-Authored-By: OpenAI --- .github/workflows/e2e-live.yml | 34 +++- deploy/mecatl-execution-kind/ci_test.go | 67 +++++-- deploy/mecatl-execution-kind/images.sh | 18 ++ deploy/mecatl-execution-kind/live.sh | 13 +- deploy/mecatl-execution-kind/repair_test.go | 198 +++++++++++++++++++ deploy/mecatl-execution-kind/run.sh | 29 ++- user-docs/features/execution-environments.md | 30 ++- 7 files changed, 334 insertions(+), 55 deletions(-) create mode 100644 deploy/mecatl-execution-kind/images.sh create mode 100644 deploy/mecatl-execution-kind/repair_test.go diff --git a/.github/workflows/e2e-live.yml b/.github/workflows/e2e-live.yml index 1355a6cb00..db40e05b0b 100644 --- a/.github/workflows/e2e-live.yml +++ b/.github/workflows/e2e-live.yml @@ -283,11 +283,15 @@ jobs: env: NATIVE_CI_DIR: ${{ github.workspace }}/.scratch/native-live-${{ github.run_id }}-${{ github.run_attempt }} steps: + # Step ceilings sum to 98m: setup 9, production 50, credential 1, + # live 25, cleanup 10, reporting 3. The job retains 2m overhead. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 1 with: ref: ${{ github.sha }} persist-credentials: false - name: Verify reviewed commit and reserve private CI directory + timeout-minutes: 1 id: prepare env: EXPECTED_SHA: ${{ inputs.expected_sha }} @@ -304,17 +308,21 @@ jobs: mkdir -p .scratch mkdir -m 700 "$NATIVE_CI_DIR" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + timeout-minutes: 2 with: go-version-file: go.mod cache: false - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 + timeout-minutes: 1 with: version: 3.x repo-token: ${{ secrets.GITHUB_TOKEN }} - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + timeout-minutes: 1 with: version: v3.18.4 - name: Install pinned Kind and ko + timeout-minutes: 3 run: | go install sigs.k8s.io/kind@v0.33.0 go install github.com/google/ko@v0.18.1 @@ -322,12 +330,14 @@ jobs: # No provider secret here. run.sh installs pinned Calico and retains the # owned cluster; the success dependency, not ownership.profile, admits live. - name: Qualify production execution + timeout-minutes: 50 id: production env: CONTAINER_ENGINE: docker MECATL_EXECUTION_QUAL_CI: "0" - run: task e2e:k8s:execution:production + run: MECATL_EXECUTION_QUAL_OUTPUT="$GITHUB_OUTPUT" timeout --kill-after=15s 49m task e2e:k8s:execution:production - name: Stage private provider credential + timeout-minutes: 1 id: credential if: success() && steps.production.outcome == 'success' env: @@ -343,20 +353,27 @@ jobs: printf '%s' "$OPENROUTER_API_KEY" > "$NATIVE_CI_DIR/provider-key" unset OPENROUTER_API_KEY - name: Qualify native OpenRouter coding + timeout-minutes: 25 id: live if: success() && steps.production.outcome == 'success' && steps.credential.outcome == 'success' + env: + MECATL_EXECUTION_QUAL_STATE: ${{ steps.production.outputs.state }} + NATIVE_CLUSTER: ${{ steps.production.outputs.cluster }} run: | - IFS= read -r state < .scratch/k8s-execution/current - MECATL_EXECUTION_QUAL_STATE="$state" \ + test -n "$NATIVE_CLUSTER" + grep -Fx -- "cluster=$NATIVE_CLUSTER" "$MECATL_EXECUTION_QUAL_STATE/ownership" >/dev/null MECATL_EXECUTION_CREDENTIAL_FILE="$NATIVE_CI_DIR/provider-key" \ - ./deploy/mecatl-execution-kind/live.sh + timeout --kill-after=15s 24m ./deploy/mecatl-execution-kind/live.sh # Credential removal is independent of cluster availability. Fail closed # on ownership drift; runner disposal is a last resort, not cleanup proof. - name: Clean up exact owned resources + timeout-minutes: 10 id: cleanup if: always() && steps.prepare.outcome == 'success' env: PRODUCTION_OUTCOME: ${{ steps.production.outcome }} + MECATL_EXECUTION_QUAL_STATE: ${{ steps.production.outputs.state }} + NATIVE_CLUSTER: ${{ steps.production.outputs.cluster }} run: | key="$NATIVE_CI_DIR/provider-key" if [ -e "$key" ] || [ -L "$key" ]; then @@ -364,9 +381,9 @@ jobs: rm -- "$key" fi if [ "$PRODUCTION_OUTCOME" = skipped ]; then exit 0; fi - pointer="$GITHUB_WORKSPACE/.scratch/k8s-execution/current" - test -f "$pointer" && test ! -L "$pointer" - IFS= read -r state < "$pointer" + trap 'echo "::error::Owned cleanup incomplete; hosted-runner disposal is the fallback, not cleanup proof"' ERR + state="$MECATL_EXECUTION_QUAL_STATE" + test -n "$state" && test -n "$NATIVE_CLUSTER" test ! -L "$state" state=$(cd -- "$state" && pwd -P) case "$state" in "$GITHUB_WORKSPACE/.scratch/k8s-execution/"*) ;; *) exit 1 ;; esac @@ -376,6 +393,7 @@ jobs: awk -F= -v key="$1" '$1 == key {value=substr($0,length(key)+2); count++} END {if(count != 1 || value == "") exit 1; print value}' "$ownership" } cluster=$(owned_value cluster) + test "$cluster" = "$NATIVE_CLUSTER" case "$cluster" in mecatl-execution-qual-*) ;; *) exit 1 ;; esac case "$cluster" in *[!a-z0-9-]*) exit 1 ;; esac test "${#cluster}" -le 63 @@ -403,6 +421,7 @@ jobs: cp -- "$summary" "$NATIVE_CI_DIR/live-summary.json" fi - name: Record sanitized qualification status + timeout-minutes: 1 if: always() && steps.prepare.outcome == 'success' env: PRODUCTION_OUTCOME: ${{ steps.production.outcome }} @@ -415,6 +434,7 @@ jobs: > "$NATIVE_CI_DIR/qualification-status.txt" cat "$NATIVE_CI_DIR/qualification-status.txt" >> "$GITHUB_STEP_SUMMARY" - name: Upload only bounded native qualification evidence + timeout-minutes: 2 if: always() && steps.prepare.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/deploy/mecatl-execution-kind/ci_test.go b/deploy/mecatl-execution-kind/ci_test.go index 69f44fd8f6..5911e477b1 100644 --- a/deploy/mecatl-execution-kind/ci_test.go +++ b/deploy/mecatl-execution-kind/ci_test.go @@ -11,11 +11,12 @@ import ( ) type workflowStep struct { - ID string `yaml:"id"` - Run string `yaml:"run"` - If string `yaml:"if"` - Env map[string]string `yaml:"env"` - With map[string]any `yaml:"with"` + ID string `yaml:"id"` + Timeout int `yaml:"timeout-minutes"` + Run string `yaml:"run"` + If string `yaml:"if"` + Env map[string]string `yaml:"env"` + With map[string]any `yaml:"with"` } func nativeSteps(t *testing.T) map[string]workflowStep { @@ -39,7 +40,12 @@ func nativeSteps(t *testing.T) map[string]workflowStep { t.Fatal("native job must remain explicitly dispatched, repo-gated, and bounded") } steps := make(map[string]workflowStep) + total := 0 for _, step := range job.Steps { + if step.Timeout <= 0 { + t.Fatal("every native step needs an explicit ceiling") + } + total += step.Timeout if step.ID != "" { steps[step.ID] = step } @@ -50,6 +56,15 @@ func nativeSteps(t *testing.T) map[string]workflowStep { t.Fatal("native credential job must not save a cache") } } + if total > job.Timeout-2 || steps["cleanup"].Timeout < 10 { + t.Fatal("stage ceilings must reserve cleanup and job overhead") + } + for _, id := range []string{"live", "cleanup"} { + step := steps[id] + if step.Env["MECATL_EXECUTION_QUAL_STATE"] != "${{ steps.production.outputs.state }}" || step.Env["NATIVE_CLUSTER"] != "${{ steps.production.outputs.cluster }}" || strings.Contains(step.Run, "/current") { + t.Fatal("downstream ownership must use production outputs, never current") + } + } if steps["production"].Env["MECATL_EXECUTION_QUAL_CI"] != "0" || len(steps["credential"].Env) != 1 || steps["credential"].Env["OPENROUTER_API_KEY"] != "${{ secrets.OPENROUTER_API_KEY }}" { t.Fatal("production retention or step-scoped credential wiring drifted") } @@ -125,7 +140,7 @@ func TestNativeWorkflowCommitAndCredentialBoundary(t *testing.T) { func TestNativeWorkflowCleanupOwnership(t *testing.T) { steps := nativeSteps(t) - for _, fault := range []string{"", "owner", "context", "label", "duplicate", "symlink", "delete", "list"} { + for _, fault := range []string{"", "owner", "context", "label", "duplicate", "symlink", "delete", "list", "switched-pointer", "captured-cluster", "missing-kubeconfig", "missing-ownership", "missing-output", "partial-with-kubeconfig"} { t.Run("fault="+fault, func(t *testing.T) { root := t.TempDir() state := filepath.Join(root, ".scratch", "k8s-execution", "owned") @@ -146,19 +161,38 @@ func TestNativeWorkflowCleanupOwnership(t *testing.T) { context = "ambient" } kubeconfig := filepath.Join(state, "kubeconfig") - writeFixture(t, kubeconfig, "synthetic", 0o600) + if fault != "missing-kubeconfig" { + writeFixture(t, kubeconfig, "synthetic", 0o600) + } ownership := "cluster=" + cluster + "\ncontext=" + context + "\nowner=" + owner + "\nruntime=docker\nprofile=production\nnamespace=execution-qualification\nkubeconfig=" + kubeconfig + "\n" if fault == "duplicate" { ownership += "owner=fixture\n" } - writeFixture(t, filepath.Join(state, "ownership"), ownership, 0o600) + if fault != "missing-ownership" { + writeFixture(t, filepath.Join(state, "ownership"), ownership, 0o600) + } pointer := filepath.Join(filepath.Dir(state), "current") + writeFixture(t, pointer, state+"\n", 0o600) + if fault == "switched-pointer" { + foreign := filepath.Join(filepath.Dir(state), "foreign") + if err := os.Mkdir(foreign, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(foreign, "ownership"), strings.ReplaceAll(ownership, cluster, "mecatl-execution-qual-foreign"), 0o600) + writeFixture(t, pointer, foreign+"\n", 0o600) + } + capturedState, capturedCluster := state, cluster if fault == "symlink" { - if err := os.Symlink(kubeconfig, pointer); err != nil { + capturedState = filepath.Join(filepath.Dir(state), "alias") + if err := os.Symlink(state, capturedState); err != nil { t.Fatal(err) } - } else { - writeFixture(t, pointer, state+"\n", 0o600) + } + if fault == "captured-cluster" { + capturedCluster = "mecatl-execution-qual-other" + } + if fault == "missing-output" { + capturedState, capturedCluster = "", "" } writeFixture(t, filepath.Join(dir, "provider-key"), "synthetic", 0o600) writeFixture(t, filepath.Join(state, "live-summary.json"), "{}\n", 0o600) @@ -166,15 +200,20 @@ func TestNativeWorkflowCleanupOwnership(t *testing.T) { writeFixture(t, filepath.Join(bin, "docker"), "#!/bin/sh\nif [ \"$FAULT\" = label ]; then exit 1; fi\nprintf '%s\\n' '"+cluster+"'\n", 0o700) writeFixture(t, filepath.Join(bin, "kind"), "#!/bin/sh\nif [ \"$1\" = delete ]; then\n test \"$*\" = 'delete cluster --name "+cluster+"' || exit 1\n test \"$FAULT\" != delete || exit 1\n printf deleted > \"$MARKER\"\nelse\n test \"$FAULT\" != list || exit 1\nfi\n", 0o700) marker := filepath.Join(root, "deleted") - out, err := runStep(t, root, steps["cleanup"].Run, "PATH="+bin+":"+os.Getenv("PATH"), "NATIVE_CI_DIR="+dir, "GITHUB_WORKSPACE="+root, "USER=fixture", "PRODUCTION_OUTCOME=success", "FAULT="+fault, "MARKER="+marker) - if (err == nil) != (fault == "") { + outcome := "success" + if strings.HasPrefix(fault, "missing-") || fault == "partial-with-kubeconfig" { + outcome = "failure" + } + out, err := runStep(t, root, steps["cleanup"].Run, "PATH="+bin+":"+os.Getenv("PATH"), "NATIVE_CI_DIR="+dir, "GITHUB_WORKSPACE="+root, "USER=fixture", "PRODUCTION_OUTCOME="+outcome, "MECATL_EXECUTION_QUAL_STATE="+capturedState, "NATIVE_CLUSTER="+capturedCluster, "FAULT="+fault, "MARKER="+marker) + wantOK := fault == "" || fault == "switched-pointer" || fault == "partial-with-kubeconfig" + if (err == nil) != wantOK { t.Fatalf("cleanup result: %v: %s", err, out) } if _, err := os.Stat(filepath.Join(dir, "provider-key")); !os.IsNotExist(err) { t.Fatal("CI credential was not removed independently") } _, deleted := os.Stat(marker) - if (deleted == nil) != (fault == "" || fault == "list") { + if (deleted == nil) != (wantOK || fault == "list") { t.Fatal("wrong cluster deletion decision") } }) diff --git a/deploy/mecatl-execution-kind/images.sh b/deploy/mecatl-execution-kind/images.sh new file mode 100644 index 0000000000..7f9859f582 --- /dev/null +++ b/deploy/mecatl-execution-kind/images.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# The alias name is not proof of its target: compare containerd's canonical +# descriptor before reusing it. Never overwrite a conflicting retained alias. +pin_loaded() { + tagged=$1 + images=$("$runtime" exec "${cluster}-control-plane" ctr -n k8s.io images ls) || return 1 + digest=$(printf '%s\n' "$images" | awk -v ref="$tagged" '$1 == ref {print $3; exit}') + printf '%s\n' "$digest" | grep -Eq '^sha256:[0-9a-f]{64}$' || { echo "loaded image digest unavailable or invalid" >&2; return 1; } + pinned="${tagged%:*}@${digest}" + existing=$(printf '%s\n' "$images" | awk -v ref="$pinned" '$1 == ref {print $3; exit}') + if [ -n "$existing" ]; then + [ "$existing" = "$digest" ] || { echo "loaded image alias conflicts with canonical digest" >&2; return 1; } + else + "$runtime" exec "${cluster}-control-plane" ctr -n k8s.io images tag "$tagged" "$pinned" >/dev/null || return 1 + fi + printf '%s\n' "$pinned" +} diff --git a/deploy/mecatl-execution-kind/live.sh b/deploy/mecatl-execution-kind/live.sh index 00c092dcdd..78c262cbfa 100755 --- a/deploy/mecatl-execution-kind/live.sh +++ b/deploy/mecatl-execution-kind/live.sh @@ -80,14 +80,7 @@ kube_jq() { } [ "$(kube config current-context)" = "$context" ] || { echo "owned kube context mismatch" >&2; exit 1; } build_ko() { package=$1 repo=$2; dev env KIND_EXPERIMENTAL_PROVIDER="${KIND_EXPERIMENTAL_PROVIDER:-}" KO_DOCKER_REPO="$repo" ko build --local --bare "$package" | tail -n 1; } -pin_loaded() { - tagged=$1 - digest=$($runtime exec "${cluster}-control-plane" ctr -n k8s.io images ls | awk -v ref="$tagged" '$1 == ref {print $3; exit}') - [ -n "$digest" ] || { echo "loaded image digest unavailable" >&2; exit 1; } - pinned="${tagged%:*}@${digest}" - $runtime exec "${cluster}-control-plane" ctr -n k8s.io images tag "$tagged" "$pinned" >/dev/null - printf '%s\n' "$pinned" -} +. "$root/deploy/mecatl-execution-kind/images.sh" load_image() { tagged=$1 archive="$state/images/live-$(printf '%s' "$tagged" | sha256sum | cut -c1-16).tar" @@ -169,7 +162,9 @@ restore() { fi exit "$status" } -trap restore EXIT INT TERM +trap restore EXIT +trap 'exit 130' INT +trap 'exit 143' TERM dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_CREDENTIAL_FILE="$MECATL_EXECUTION_CREDENTIAL_FILE" \ go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/credentialloader stage "$kubeconfig" "$context" "$secret" "$receipt" diff --git a/deploy/mecatl-execution-kind/repair_test.go b/deploy/mecatl-execution-kind/repair_test.go new file mode 100644 index 0000000000..486b797a6b --- /dev/null +++ b/deploy/mecatl-execution-kind/repair_test.go @@ -0,0 +1,198 @@ +package executionkind_test + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func scriptRange(t *testing.T, file, start, end string) string { + t.Helper() + data, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + body := string(data) + i, j := strings.Index(body, start), strings.Index(body, end) + if i < 0 || j <= i { + t.Fatalf("missing script range in %s", file) + } + return body[i:j] +} + +func TestProductionThenLiveImageAlias(t *testing.T) { + production := scriptRange(t, "run.sh", "for item in", "printf 'provider=%s") + live := scriptRange(t, "live.sh", ". \"$root/deploy/mecatl-execution-kind/images.sh\"", "\nprovider_tag=$(build_ko") + helper, err := os.ReadFile("images.sh") + if err != nil { + t.Fatal(err) + } + for _, runtime := range []string{"docker", "podman"} { + for _, conflict := range []bool{false, true} { + t.Run(runtime+"/conflict="+map[bool]string{false: "false", true: "true"}[conflict], func(t *testing.T) { + root := t.TempDir() + for _, dir := range []string{"bin", "images", "deploy/mecatl-execution-kind"} { + if err := os.MkdirAll(filepath.Join(root, dir), 0o700); err != nil { + t.Fatal(err) + } + } + writeFixture(t, filepath.Join(root, "deploy/mecatl-execution-kind/images.sh"), string(helper), 0o600) + digest := "sha256:" + strings.Repeat("a", 64) + tag := "ko.local/provider:head" + alias := "ko.local/provider@" + digest + db := filepath.Join(root, "db") + var sourceRows strings.Builder + for _, name := range []string{"provider", "agent", "oidc", "netprobe", "workload"} { + sourceRows.WriteString("ko.local/" + name + ":head type " + digest + "\n") + } + writeFixture(t, db, sourceRows.String(), 0o600) + writeFixture(t, filepath.Join(root, "bin", runtime), `#!/bin/sh +set -eu +case "$1" in +save) : > "$4" ;; +exec) + test "$2" = owned-control-plane + shift 5 + test "$1" = images + case "$2" in + ls) cat "$DB" ;; + tag) + test "$#" = 4 + if awk -v ref="$4" '$1 == ref {found=1} END {exit !found}' "$DB"; then + echo 'already exists' >&2; exit 1 + fi + digest=$(awk -v ref="$3" '$1 == ref {print $3}' "$DB") + printf '%s type %s\n' "$4" "$digest" >> "$DB" + printf 'tag\n' >> "$TAGS" + ;; + *) exit 1 ;; + esac ;; +*) exit 1 ;; +esac +`, 0o700) + writeFixture(t, filepath.Join(root, "bin/kind"), "#!/bin/sh\nset -eu\ntest \"$1 $2\" = 'load image-archive'\ntest -f \"$3\"\ntest \"$4 $5\" = '--name owned'\n", 0o700) + env := []string{"PATH=" + filepath.Join(root, "bin") + ":" + os.Getenv("PATH"), "root=" + root, "state=" + root, "runtime=" + runtime, "cluster=owned", "DB=" + db, "TAGS=" + filepath.Join(root, "tags")} + for _, name := range []string{"provider", "agent", "oidc", "netprobe", "workload"} { + env = append(env, name+"_tag=ko.local/"+name+":head") + } + out, err := runStep(t, root, production, env...) + if err != nil { + t.Fatalf("fresh production load: %v: %s", err, out) + } + if conflict { + writeFixture(t, db, tag+" type "+digest+"\n"+alias+" type sha256:"+strings.Repeat("b", 64)+"\n", 0o600) + } + out, err = runStep(t, root, live+"\nload_image \"$provider_tag\"", env...) + if (err == nil) == conflict { + t.Fatalf("retained live load: %v: %s", err, out) + } + if !conflict && strings.TrimSpace(string(out)) != alias { + t.Fatalf("wrong alias: %s", out) + } + tags, err := os.ReadFile(filepath.Join(root, "tags")) + if err != nil || string(tags) != strings.Repeat("tag\n", 5) { + t.Fatalf("must create each alias once, never overwrite/re-tag: %q, %v", tags, err) + } + }) + } + } +} + +func TestLiveSignalPreservesFailureAndAttemptsCleanup(t *testing.T) { + restore := scriptRange(t, "live.sh", "restore() {", "\ndev env -i HOME=\"$HOME\" PATH=\"$PATH\" KUBECONFIG=\"$kubeconfig\" MECATL_KUBE_CONTEXT=\"$context\" MECATL_EXECUTION_CREDENTIAL_FILE=") + for _, signal := range []struct { + name string + code string + }{{"INT", "130"}, {"TERM", "143"}} { + t.Run(signal.name, func(t *testing.T) { + root := t.TempDir() + writeFixture(t, filepath.Join(root, "receipt"), "synthetic", 0o600) + writeFixture(t, filepath.Join(root, "signal.sh"), "#!/bin/sh\nset -eu\n"+` +helm_kube() { printf 'restore\n' >> "$MARKER"; } +dev() { printf 'delete\n' >> "$MARKER"; } +restore_needed=1 +`+restore+"\nkill -"+signal.name+" $$\nexit 99\n", 0o700) + out, err := runStep(t, root, "if sh ./signal.sh; then exit 1; else test \"$?\" -eq "+signal.code+"; fi", "root="+root, "MARKER="+filepath.Join(root, "cleanup"), "receipt="+filepath.Join(root, "receipt"), "agent_image=synthetic@sha256:fake", "kubeconfig=synthetic", "context=synthetic", "secret=synthetic") + if err != nil { + t.Fatalf("signal was reported as success: %v: %s", err, out) + } + calls, err := os.ReadFile(filepath.Join(root, "cleanup")) + if err != nil || string(calls) != "restore\ndelete\n" { + t.Fatalf("cleanup must run once after signal: %q: %v", calls, err) + } + }) + } +} + +func TestProductionPublishesOwnershipBeforePartialCreation(t *testing.T) { + data, err := os.ReadFile("run.sh") + if err != nil { + t.Fatal(err) + } + for _, fault := range []string{"partial", "discovery", "collision"} { + t.Run(fault, func(t *testing.T) { + root := t.TempDir() + scripts := filepath.Join(root, "deploy/mecatl-execution-kind") + bin := filepath.Join(root, "bin") + for _, dir := range []string{scripts, bin} { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + } + writeFixture(t, filepath.Join(scripts, "run.sh"), string(data), 0o700) + writeFixture(t, filepath.Join(bin, "docker"), "#!/bin/sh\nexit 99\n", 0o700) + writeFixture(t, filepath.Join(root, "fault"), fault+"\n", 0o600) + writeFixture(t, filepath.Join(bin, "kind"), `#!/bin/sh +set -eu +IFS= read -r fault < "$HOME/fault" +case "$1" in +get) + case "$fault" in + discovery) exit 1 ;; + collision) awk -F= '$1 == "cluster" {print $2}' .scratch/k8s-execution/*/ownership ;; + esac ;; +create) + test -s "$MECATL_EXECUTION_QUAL_OUTPUT" + printf 'creation attempted\n' > "$HOME/attempted" + exit 1 ;; +*) exit 99 ;; +esac +`, 0o700) + output := filepath.Join(root, "output") + marker := filepath.Join(root, "attempted") + out, err := runStep(t, root, "./deploy/mecatl-execution-kind/run.sh", "PATH="+bin+":"+os.Getenv("PATH"), "CONTAINER_ENGINE=docker", "MECATL_EXECUTION_QUAL_PROFILE=production", "MECATL_EXECUTION_QUAL_OUTPUT="+output) + if err == nil { + t.Fatalf("partial production unexpectedly succeeded: %s", out) + } + outputs, err := os.ReadFile(output) + if fault != "partial" { + if !os.IsNotExist(err) { + t.Fatal("ownership published before collision/discovery gate") + } + if _, err := os.Stat(marker); !os.IsNotExist(err) { + t.Fatal("creation attempted after failed preflight") + } + return + } + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(outputs)), "\n") + if len(lines) != 2 || !strings.HasPrefix(lines[0], "state=") || !strings.HasPrefix(lines[1], "cluster=mecatl-execution-qual-") { + t.Fatalf("invalid ownership outputs: %q", outputs) + } + state := strings.TrimPrefix(lines[0], "state=") + ownership, err := os.ReadFile(filepath.Join(state, "ownership")) + if err != nil || !strings.Contains(string(ownership), lines[1]+"\n") { + t.Fatal("output does not bind the initial ownership record") + } + if _, err := os.Stat(marker); err != nil { + t.Fatal("fixture did not reach partial cluster creation") + } + if _, err := os.Stat(filepath.Join(state, "kubeconfig")); !os.IsNotExist(err) { + t.Fatal("partial fixture unexpectedly has a kubeconfig") + } + }) + } +} diff --git a/deploy/mecatl-execution-kind/run.sh b/deploy/mecatl-execution-kind/run.sh index b8215a2f65..d85dd74e52 100755 --- a/deploy/mecatl-execution-kind/run.sh +++ b/deploy/mecatl-execution-kind/run.sh @@ -9,6 +9,7 @@ if [ "${MECATL_EXECUTION_QUAL_CLEAN_ENV:-}" != 1 ]; then DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-}" CONTAINER_HOST="${CONTAINER_HOST:-}" DOCKER_HOST="${DOCKER_HOST:-}" \ CONTAINER_ENGINE="${CONTAINER_ENGINE:-}" MECATL_EXECUTION_QUAL_CI="${MECATL_EXECUTION_QUAL_CI:-}" MECATL_EXECUTION_QUAL_PROFILE="${MECATL_EXECUTION_QUAL_PROFILE:-}" \ TOOLBOX_PATH="${TOOLBOX_PATH:-}" MECATL_EXECUTION_DEV_TOOLBOX="${MECATL_EXECUTION_DEV_TOOLBOX:-}" MECATL_EXECUTION_K8S_TOOLBOX="${MECATL_EXECUTION_K8S_TOOLBOX:-}" \ + MECATL_EXECUTION_QUAL_OUTPUT="${MECATL_EXECUTION_QUAL_OUTPUT:-}" \ MECATL_EXECUTION_QUAL_CLEAN_ENV=1 "$0" "$@" fi @@ -32,17 +33,23 @@ printf 'cluster=%s\ncontext=%s\nkubeconfig=%s\nnamespace=execution-qualification "$cluster" "$context" "$kubeconfig" "${USER:-user}" "$runtime" "${MECATL_EXECUTION_QUAL_PROFILE:-development}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$state/ownership" # Refuse collisions; never delete or reuse any cluster. -if kind get clusters 2>/dev/null | grep -Fx "$cluster" >/dev/null; then - echo "qualification cluster already exists: $cluster" >&2 - exit 1 -fi -printf '%s\n' "$state" >"$root/.scratch/k8s-execution/current" - if [ "$runtime" = podman ]; then export KIND_EXPERIMENTAL_PROVIDER=podman else unset KIND_EXPERIMENTAL_PROVIDER fi +clusters=$(kind get clusters) || { echo "qualification cluster discovery failed" >&2; exit 1; } +if printf '%s\n' "$clusters" | grep -Fx "$cluster" >/dev/null; then + echo "qualification cluster already exists: $cluster" >&2 + exit 1 +fi +# Publish the exact owned identity before creation can partially fail. Downstream +# CI steps consume these outputs, never the mutable local convenience pointer. +if [ -n "${MECATL_EXECUTION_QUAL_OUTPUT:-}" ]; then + printf 'state=%s\ncluster=%s\n' "$state" "$cluster" >>"$MECATL_EXECUTION_QUAL_OUTPUT" +fi +printf '%s\n' "$state" >"$root/.scratch/k8s-execution/current" + if [ "${MECATL_EXECUTION_QUAL_CI:-}" = 1 ]; then cleanup_cluster() { status=$? @@ -157,15 +164,7 @@ for item in "$provider_tag" "$agent_tag" "$oidc_tag" "$netprobe_tag" "$workload_ "$runtime" save "$item" -o "$archive" >/dev/null kind load image-archive "$archive" --name "$cluster" done -node="${cluster}-control-plane" -pin_loaded() { - tagged=$1 - digest=$("$runtime" exec "$node" ctr -n k8s.io images ls | awk -v ref="$tagged" '$1 == ref {print $3; exit}') - [ -n "$digest" ] || { echo "loaded image digest unavailable for $tagged" >&2; exit 1; } - pinned="${tagged%:*}@${digest}" - "$runtime" exec "$node" ctr -n k8s.io images tag "$tagged" "$pinned" >/dev/null - printf '%s\n' "$pinned" -} +. "$root/deploy/mecatl-execution-kind/images.sh" provider_image=$(pin_loaded "$provider_tag") agent_image=$(pin_loaded "$agent_tag") oidc_image=$(pin_loaded "$oidc_tag") diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index ba20dbaf21..9edafed120 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -303,16 +303,26 @@ The workflow stages the key in a new 0700 CI directory and 0600 file, then remov it from the environment before `live.sh` validates the retained owned state, reruns mock qualification, and invokes the trusted credential loader. A missing secret fails qualification rather than skipping successfully. The 100-minute job -bound includes builds, the 45-minute production test bound, 15-minute mock rerun, -10-minute live smoke, and cleanup; token limits are not a hard dollar cap. - -The live script restores mock configuration and deletes its Secret by recorded -UID using an independent cleanup context. Failures fail qualification. An always-run -workflow step removes only the CI-created credential file and deletes only the -recorded cluster after validating its owner, name, namespace, profile, container -label, private kubeconfig, and context. Ownership drift fails cleanup without -selecting another cluster. Runner disposal is a last resort, not evidence that -cleanup succeeded. +has explicit step ceilings: setup 9 minutes, production 50, credential staging 1, +live 25, cleanup 10, and status/artifact reporting 3, leaving 2 minutes of overhead. +Production and live subprocess groups receive TERM after 49 and 24 minutes, +respectively, then KILL after a 15-second grace. These stage ceilings are +intentionally smaller than the sum of all subordinate build, rollout, and test +bounds. Hitting one fails qualification; it does not prove completion. Per-agent +token limits remain unchanged and are not a hard dollar cap. + +The live script attempts to restore mock configuration and delete its Secret by +recorded UID using an independent cleanup context. A signal or stage deadline can +interrupt that attempt. An always-run workflow step has a separate 10-minute +ceiling to remove the CI-created credential file and delete the owned cluster. +Production publishes its state path and cluster identity after collision checks +and before cluster creation; live and cleanup use those fixed step outputs, not +the mutable local `current` pointer. Cleanup validates the recorded owner, name, +namespace, profile, container label, private kubeconfig, and context against that +identity. Ownership drift fails cleanup without selecting another cluster. +Failure before kubeconfig creation also fails cleanup without deleting a cluster. +Cleanup is bounded best effort: hard cancellation or runner loss can prevent it. +Hosted-runner disposal is the fallback, not evidence that cleanup succeeded. Artifacts are retained for seven days: only a size-bounded `live-summary.json` when available and a sanitized `qualification-status.txt`. PKI, kubeconfigs, From 0e8c8a5ca105766ea3493fa3b80c9c547baa9009 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Tue, 22 Sep 2026 01:32:32 +0300 Subject: [PATCH 22/25] fix(ci): avoid ShellCheck warning in qualification summary Co-Authored-By: OpenAI --- .github/workflows/e2e-live.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-live.yml b/.github/workflows/e2e-live.yml index db40e05b0b..f4a39ae683 100644 --- a/.github/workflows/e2e-live.yml +++ b/.github/workflows/e2e-live.yml @@ -303,7 +303,7 @@ jobs: echo '::error::Reviewed SHA does not match dispatch SHA' exit 1 fi - printf '## Native execution qualification\n\nCommit: `%s`\n' "$actual" >> "$GITHUB_STEP_SUMMARY" + printf '## Native execution qualification\n\nCommit: %s\n' "$actual" >> "$GITHUB_STEP_SUMMARY" umask 077 mkdir -p .scratch mkdir -m 700 "$NATIVE_CI_DIR" From 53ab454531989af1eee32a2b25bf5eb441b07a7b Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Tue, 22 Sep 2026 01:54:48 +0300 Subject: [PATCH 23/25] fix(execution): repair retained deletion and qualification evidence Keep admitted retained deletion driveable across peer reconciliation and deletion timestamps. Replace the holder-loss descendant timer with exact-owned kubelet termination, respect quota resync in the fixture, and collect bounded sanitized failure evidence before cleanup. Co-Authored-By: OpenAI --- .github/workflows/e2e-live.yml | 8 + deploy/mecatl-execution-kind/ci_test.go | 14 +- .../mecatl-execution-kind/collect-failure.sh | 22 +++ .../mecatl-execution-kind/failure-evidence.jq | 32 ++++ .../failure_evidence_test.go | 93 ++++++++++ deploy/mecatl-execution-kind/run.sh | 6 +- .../acceptance/native-kubernetes-execution.md | 61 ++++++- e2e/k8s_execution/holder_loss_helpers_test.go | 61 +++++++ e2e/k8s_execution/holder_loss_test.go | 68 ++++++++ .../production_qualification_test.go | 51 ++++-- .../executioncontroller/admin_lifecycle.go | 4 +- .../adapter/executioncontroller/controller.go | 5 + .../controller_lifecycle.go | 13 +- .../legacy_fixture_kind.go | 36 +++- .../legacy_fixture_kind_test.go | 30 +++- .../retained_delete_race_test.go | 159 ++++++++++++++++++ user-docs/features/execution-environments.md | 16 +- 17 files changed, 635 insertions(+), 44 deletions(-) create mode 100644 deploy/mecatl-execution-kind/collect-failure.sh create mode 100644 deploy/mecatl-execution-kind/failure-evidence.jq create mode 100644 deploy/mecatl-execution-kind/failure_evidence_test.go create mode 100644 e2e/k8s_execution/holder_loss_helpers_test.go create mode 100644 e2e/k8s_execution/holder_loss_test.go create mode 100644 internal/adapter/executioncontroller/retained_delete_race_test.go diff --git a/.github/workflows/e2e-live.yml b/.github/workflows/e2e-live.yml index f4a39ae683..70a4303ac5 100644 --- a/.github/workflows/e2e-live.yml +++ b/.github/workflows/e2e-live.yml @@ -409,6 +409,13 @@ jobs: test "$(stat -c '%a' "$kubeconfig")" = 600 test "$(kubectl --kubeconfig "$kubeconfig" --context "$context" config current-context)" = "$context" test "$(docker inspect "${cluster}-control-plane" --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}')" = "$cluster" + if [ "$PRODUCTION_OUTCOME" != success ]; then + # Collect before deletion, independently of whether cleanup succeeds. + # The helper emits closed-vocabulary JSON only, capped at 1 MiB. + timeout --kill-after=5s 60s sh ./deploy/mecatl-execution-kind/collect-failure.sh \ + "$kubeconfig" "$context" "$NATIVE_CI_DIR/production-diagnostics.jsonl" \ + || echo '::warning::Bounded production diagnostics incomplete' + fi kind delete cluster --name "$cluster" clusters=$(kind get clusters) if printf '%s\n' "$clusters" | grep -Fx -- "$cluster"; then exit 1; fi @@ -442,6 +449,7 @@ jobs: path: | ${{ env.NATIVE_CI_DIR }}/live-summary.json ${{ env.NATIVE_CI_DIR }}/qualification-status.txt + ${{ env.NATIVE_CI_DIR }}/production-diagnostics.jsonl include-hidden-files: true retention-days: 7 if-no-files-found: error diff --git a/deploy/mecatl-execution-kind/ci_test.go b/deploy/mecatl-execution-kind/ci_test.go index 5911e477b1..621d437f6c 100644 --- a/deploy/mecatl-execution-kind/ci_test.go +++ b/deploy/mecatl-execution-kind/ci_test.go @@ -200,8 +200,15 @@ func TestNativeWorkflowCleanupOwnership(t *testing.T) { writeFixture(t, filepath.Join(bin, "docker"), "#!/bin/sh\nif [ \"$FAULT\" = label ]; then exit 1; fi\nprintf '%s\\n' '"+cluster+"'\n", 0o700) writeFixture(t, filepath.Join(bin, "kind"), "#!/bin/sh\nif [ \"$1\" = delete ]; then\n test \"$*\" = 'delete cluster --name "+cluster+"' || exit 1\n test \"$FAULT\" != delete || exit 1\n printf deleted > \"$MARKER\"\nelse\n test \"$FAULT\" != list || exit 1\nfi\n", 0o700) marker := filepath.Join(root, "deleted") + if fault == "delete" || fault == "list" || fault == "partial-with-kubeconfig" { + scripts := filepath.Join(root, "deploy/mecatl-execution-kind") + if err := os.MkdirAll(scripts, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(scripts, "collect-failure.sh"), "#!/bin/sh\ntest ! -e \"$MARKER\" || exit 1\nprintf '{\"kind\":\"collection\"}\\n' > \"$3\"\n", 0o700) + } outcome := "success" - if strings.HasPrefix(fault, "missing-") || fault == "partial-with-kubeconfig" { + if strings.HasPrefix(fault, "missing-") || fault == "partial-with-kubeconfig" || fault == "delete" || fault == "list" { outcome = "failure" } out, err := runStep(t, root, steps["cleanup"].Run, "PATH="+bin+":"+os.Getenv("PATH"), "NATIVE_CI_DIR="+dir, "GITHUB_WORKSPACE="+root, "USER=fixture", "PRODUCTION_OUTCOME="+outcome, "MECATL_EXECUTION_QUAL_STATE="+capturedState, "NATIVE_CLUSTER="+capturedCluster, "FAULT="+fault, "MARKER="+marker) @@ -212,6 +219,11 @@ func TestNativeWorkflowCleanupOwnership(t *testing.T) { if _, err := os.Stat(filepath.Join(dir, "provider-key")); !os.IsNotExist(err) { t.Fatal("CI credential was not removed independently") } + if fault == "delete" || fault == "list" || fault == "partial-with-kubeconfig" { + if data, err := os.ReadFile(filepath.Join(dir, "production-diagnostics.jsonl")); err != nil || !strings.Contains(string(data), "collection") { + t.Fatal("failure evidence must be captured before cleanup, even when cleanup fails") + } + } _, deleted := os.Stat(marker) if (deleted == nil) != (wantOK || fault == "list") { t.Fatal("wrong cluster deletion decision") diff --git a/deploy/mecatl-execution-kind/collect-failure.sh b/deploy/mecatl-execution-kind/collect-failure.sh new file mode 100644 index 0000000000..7e3b93d0f3 --- /dev/null +++ b/deploy/mecatl-execution-kind/collect-failure.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu +# Called only with the owned fixture's explicit kubeconfig/context. Each request +# is bounded; the caller also bounds the whole collector (including jq). +[ "$#" -eq 3 ] || exit 2 +kubeconfig=$1 +context=$2 +output=$3 +filter=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/failure-evidence.jq +umask 077 +set -C +{ + for resource in executionenvironments pods persistentvolumeclaims resourcequotas events; do + if ! kubectl --kubeconfig "$kubeconfig" --context "$context" --request-timeout=10s get "$resource" -n execution-qualification -o json 2>/dev/null; then + printf '{"unavailable":true}\n' + fi + done +} | jq -cs -f "$filter" 2>/dev/null | head -c 1048576 > "$output" +# Never substitute raw stderr or a manifest when a response cannot be projected. +if [ ! -s "$output" ]; then + printf '{"kind":"collection","unavailable":true}\n' >> "$output" +fi diff --git a/deploy/mecatl-execution-kind/failure-evidence.jq b/deploy/mecatl-execution-kind/failure-evidence.jq new file mode 100644 index 0000000000..8287d31cf1 --- /dev/null +++ b/deploy/mecatl-execution-kind/failure-evidence.jq @@ -0,0 +1,32 @@ +# Only closed vocabulary, booleans and counts may leave the fixture. Names, +# messages, UIDs, spec data, lease identities and timestamps stay in memory. +def known($allowed): . as $v | if ($allowed | index($v)) != null then $v else "other" end; +def finalizers: + [.metadata.finalizers[]? | select(. == "execution.mecatl.dev/retain-workspace" or . == "execution.mecatl.dev/verify-termination" or . == "kubernetes.io/pvc-protection")]; +def quota_keys: + ["pods", "persistentvolumeclaims", "count/executionenvironments.execution.mecatl.dev", "requests.cpu", "requests.memory", "requests.storage", "requests.ephemeral-storage", "limits.cpu", "limits.memory", "limits.ephemeral-storage"]; +[.[] | .items[]?] as $items | +($items | map(select(.kind == "Pod"))) as $pods | +($items | map(select(.kind == "PersistentVolumeClaim"))) as $pvcs | +($items | map(select(.kind == "ExecutionEnvironment"))) as $envs | +{kind:"collection", unavailable:([.[] | select(.unavailable == true)] | length), environments:($envs|length), pods:($pods|length), pvcs:($pvcs|length)}, +($envs[:32][] | . as $env | + ([$pods[] | select(.metadata.name == $env.status.pod.name)][0]) as $pod | + ([$pvcs[] | select(.metadata.name == $env.status.pvc.name)][0]) as $pvc | + {kind:"environment", deleting:(.metadata.deletionTimestamp != null), finalizers:finalizers, + lifecycle:(.status.lifecycleOperation.type | known(["DeleteRetiredEnvironment","RetireEnvironment","ReplaceExecutor"])), + phase:(.status.lifecycleOperation.phase | known(["DeletingPVC","ReleasingSlot","Quiescing","WaitingForTermination","RemovingPodFinalizer","WaitingForPodDeletion","CreatingReplacement"])), + pod_present:($pod != null), pod_uid_matches:($pod != null and .status.pod.uid != null and .status.pod.uid == $pod.metadata.uid), + pvc_present:($pvc != null), pvc_uid_matches:($pvc != null and .status.pvc.uid != null and .status.pvc.uid == $pvc.metadata.uid), + operation_pvc_uid_matches:(.status.lifecycleOperation.expectedPVCUID != null and .status.lifecycleOperation.expectedPVCUID == .status.pvc.uid), + operation_pod_uid_matches:(.status.lifecycleOperation.expectedPodUID != null and .status.lifecycleOperation.expectedPodUID == .status.pod.uid), + lease_present:(.status.activeOperation.expiresAt != null), + lease_expired:((.status.activeOperation.expiresAt // "" | sub("\\.[0-9]+Z$"; "Z") | try fromdateiso8601 catch null) as $expiry | if $expiry == null then null else $expiry <= now end), + conditions:[.status.conditions[:16][]? | {type:(.type | known(["Ready","Retired","ExecutorTerminated","DeletionBlocked"])), status:(.status | known(["True","False","Unknown"])), reason:(.reason | known(["FenceUnknown","ExactLifecycleRequired","WorkspaceRetained","TerminalPodProof","AwaitingTerminalExecutor","Retained","ReplacementStarting","ReplacementReady","PVCUnavailable","ExecutorUnavailable","Reconciled","InvalidProfile","IncompatibleSchema"]))}]}), +($pods[:64][] | {kind:"pod", deleting:(.metadata.deletionTimestamp != null), finalizers:finalizers, + phase:(.status.phase | known(["Pending","Running","Succeeded","Failed","Unknown"])), + containers:([.status.containerStatuses[]?, .status.initContainerStatuses[]?, .status.ephemeralContainerStatuses[]?][:16] | map({terminated:(.state.terminated != null), reason:((.state.terminated.reason // .state.waiting.reason) | known(["Completed","Error","OOMKilled","ContainerStatusUnknown","CrashLoopBackOff","ImagePullBackOff","ErrImagePull","ContainerCreating"]))}))}), +($pvcs[:64][] | {kind:"pvc", deleting:(.metadata.deletionTimestamp != null), finalizers:finalizers}), +($items[] | select(.kind == "ResourceQuota") | . as $q | + {kind:"quota", missing_or_mismatched_keys:[quota_keys[] | . as $key | select($q.spec.hard[$key] != null and ($q.status.hard[$key] != $q.spec.hard[$key] or $q.status.used[$key] == null))]}), +($items | map(select(.kind == "Event") | .reason | known(["FailedScheduling","FailedMount","FailedAttachVolume","FailedBinding","ProvisioningFailed","FailedCreate","Failed","BackOff","Killing","Scheduled","Pulled","Created","Started"])) | group_by(.)[] | {kind:"event", reason:.[0], count:length}) diff --git a/deploy/mecatl-execution-kind/failure_evidence_test.go b/deploy/mecatl-execution-kind/failure_evidence_test.go new file mode 100644 index 0000000000..7bf420586d --- /dev/null +++ b/deploy/mecatl-execution-kind/failure_evidence_test.go @@ -0,0 +1,93 @@ +package executionkind_test + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestFailureEvidenceRejectsProducerText(t *testing.T) { + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("jq unavailable") + } + // Even a syntactically valid metadata name or reason is not an output grant. + const hostile = "PRIVATE-URL-TOKEN-COMMAND" + fixture := `{"items":[{"kind":"ExecutionEnvironment","metadata":{"name":"` + hostile + `","deletionTimestamp":"private","finalizers":["` + hostile + `","execution.mecatl.dev/retain-workspace"]},"status":{"pod":{"name":"p","uid":"expected"},"pvc":{"name":"v","uid":"expected"},"activeOperation":{"id":"` + hostile + `","expiresAt":"2000-01-01T00:00:00.123Z"},"lifecycleOperation":{"type":"DeleteRetiredEnvironment","phase":"ReleasingSlot"},"conditions":[{"type":"Ready","status":"False","reason":"FenceUnknown","message":"` + hostile + `"}]}},{"kind":"Pod","metadata":{"name":"p","uid":"foreign","finalizers":["execution.mecatl.dev/verify-termination"]},"status":{"phase":"Failed","containerStatuses":[{"name":"` + hostile + `","state":{"terminated":{"reason":"Error","message":"` + hostile + `"}}}]}},{"kind":"PersistentVolumeClaim","metadata":{"name":"v","uid":"expected","finalizers":["kubernetes.io/pvc-protection"]}},{"kind":"ResourceQuota","spec":{"hard":{"pods":"10","requests.cpu":"10","` + hostile + `":"10"}},"status":{"hard":{"pods":"10","requests.cpu":"20"},"used":{"requests.cpu":"0"}}},{"kind":"Event","reason":"` + hostile + `","message":"` + hostile + `"}]}` + cmd := exec.CommandContext(t.Context(), "jq", "-cs", "-f", "failure-evidence.jq") + cmd.Stdin = strings.NewReader(fixture) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("filter: %v: %s", err, out) + } + if strings.Contains(string(out), hostile) || len(out) > 16384 { + t.Fatal("untrusted evidence escaped") + } + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + var v map[string]any + if err := json.Unmarshal([]byte(line), &v); err != nil { + t.Fatal(err) + } + if v["kind"] == "environment" { + if v["pod_uid_matches"] != false || v["pvc_uid_matches"] != true || v["lease_expired"] != true || v["deleting"] != true || len(v["finalizers"].([]any)) != 1 { + t.Fatalf("lost diagnostic facts: %s", line) + } + } + if v["kind"] == "quota" && len(v["missing_or_mismatched_keys"].([]any)) != 2 { + t.Fatalf("lost quota mismatch: %s", line) + } + } +} + +func TestFailureCollectorBoundsAndContext(t *testing.T) { + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("jq unavailable") + } + root := t.TempDir() + bin := filepath.Join(root, "bin") + if err := os.Mkdir(bin, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(bin, "kubectl"), `#!/bin/sh +set -eu +test "$1 $2 $3 $4 $5" = '--kubeconfig synthetic --context kind-owned --request-timeout=10s' +if [ -n "${EVIDENCE_FIXTURE:-}" ]; then cat "$EVIDENCE_FIXTURE"; exit; fi +case "$7" in +pods) printf 'PRIVATE-ERROR' >&2; exit 1 ;; +*) printf '{"items":[{"kind":"Event","reason":"FailedMount","message":"PRIVATE-DATA"}]}' ;; +esac +`, 0o700) + outPath := filepath.Join(root, "evidence") + script, err := filepath.Abs("collect-failure.sh") + if err != nil { + t.Fatal(err) + } + out, err := runStep(t, root, "sh \"$COLLECTOR\" synthetic kind-owned \"$OUT\"", "COLLECTOR="+script, "OUT="+outPath, "PATH="+bin+":"+os.Getenv("PATH")) + if err != nil { + t.Fatalf("collector failed: %v: %s", err, out) + } + data, err := os.ReadFile(outPath) + if err != nil || len(data) > 16384 || len(data) == 0 || strings.Contains(string(data), "PRIVATE") || strings.Contains(string(out), "PRIVATE") { + t.Fatalf("unsafe/missing evidence: %v", err) + } + if !strings.Contains(string(data), `"unavailable":1`) { + t.Fatal("API failure not recorded") + } + // A large fleet must retain late quota evidence, not exhaust a summary-sized + // cap on environment/Pod rows before reaching the quota and event sections. + container := `{"state":{"terminated":{"reason":"Completed"}}}` + pod := `{"kind":"Pod","status":{"phase":"Failed","containerStatuses":[` + strings.TrimSuffix(strings.Repeat(container+",", 16), ",") + `]}}` + fixture := filepath.Join(root, "fleet.json") + writeFixture(t, fixture, `{"items":[`+strings.Repeat(pod+",", 64)+`{"kind":"ResourceQuota","spec":{"hard":{"pods":"1"}}}]}`, 0o600) + outPath = filepath.Join(root, "fleet-evidence") + out, err = runStep(t, root, "sh \"$COLLECTOR\" synthetic kind-owned \"$OUT\"", "COLLECTOR="+script, "OUT="+outPath, "EVIDENCE_FIXTURE="+fixture, "PATH="+bin+":"+os.Getenv("PATH")) + if err != nil { + t.Fatalf("fleet collector: %v: %s", err, out) + } + data, err = os.ReadFile(outPath) + if err != nil || len(data) > 1<<20 || !strings.Contains(string(data), `"kind":"quota"`) { + t.Fatal("fleet evidence lost quota or exceeded artifact cap") + } +} diff --git a/deploy/mecatl-execution-kind/run.sh b/deploy/mecatl-execution-kind/run.sh index d85dd74e52..0e69768341 100755 --- a/deploy/mecatl-execution-kind/run.sh +++ b/deploy/mecatl-execution-kind/run.sh @@ -55,10 +55,8 @@ if [ "${MECATL_EXECUTION_QUAL_CI:-}" = 1 ]; then status=$? artifact="$state/production-failure-artifact.txt" if [ "$status" -ne 0 ]; then - kubectl --kubeconfig "$kubeconfig" --context "$context" get events -n execution-qualification \ - -o 'custom-columns=OBJECT:.involvedObject.name,REASON:.reason' --no-headers >"$artifact" 2>/dev/null || printf 'qualification_setup_failed\n' >"$artifact" - head -c 1048576 "$artifact" >"$artifact.bounded" - mv "$artifact.bounded" "$artifact" + timeout --kill-after=5s 60s sh "$root/deploy/mecatl-execution-kind/collect-failure.sh" \ + "$kubeconfig" "$context" "$artifact" || printf 'qualification_diagnostics_incomplete\n' >&2 fi kind delete cluster --name "$cluster" return "$status" diff --git a/docs/acceptance/native-kubernetes-execution.md b/docs/acceptance/native-kubernetes-execution.md index 44bdffbd4e..e21c8dda06 100644 --- a/docs/acceptance/native-kubernetes-execution.md +++ b/docs/acceptance/native-kubernetes-execution.md @@ -471,19 +471,20 @@ at `9bb4d89b8aa497c9d2cd6aa4a94ab9262a913364` failed **before qualification test seeding `legacy-migration`: ResourceQuota accounting for `count/executionenvironments.execution.mecatl.dev` was still unknown. Provider Deployment readiness was not quota readiness. This supersedes a test-race diagnosis for that job. -The implementation adds a two-minute, context-aware quota-accounting wait with delayed-status, +The implementation adds a six-minute, context-aware quota-accounting wait (the +five-minute Kubernetes default usage resync plus margin) with delayed-status, timeout, cancellation, and forbidden-response controls in `TestLegacyFixtureWaitsForQuotaAccountingBeforeCreate`. It does not disable quota or retry arbitrary forbidden creates. Those offline controls and tagged compilation are not a rerun of the production job. -**Latest reported production context:** the operator supplied run +**Earlier reported production context:** the operator supplied run [35650128625, job 106500190798](https://github.com/stacklok/mecatl/actions/runs/35650128625/job/106500190798) at `f7919f6c`: 10/11 production tests passed, including scoped administration, Helm lifecycle/quota, and general file tools. Security rotation failed after about 250 seconds on the final-authority Files READ immediately after successful AcquireRun (`production_qualification_test.go`, former line 279). This report was not independently re-fetched during the offline repair. It supersedes the earlier -setup failure as the current reported runtime blocker, not as a final-candidate success. +setup failure for that reported run, not as a final-candidate success. **Deterministic repair evidence:** implementation `96e16e926` adds `internal/adapter/executioncontroller/security_rotation_test.go`: @@ -513,6 +514,54 @@ offline demo pass. Tagged whole-fixture lint exposes unrelated pre-existing find gate against baseline `2d32864cee8378939cb55137e1ab369e70a18349` passes with zero issues. These results do not replace final runtime qualification; no live inference or cluster run is claimed. +**Current repair evidence (offline, after `0e8c8a5ca`):** the operator reports all +11 native production tests passing at `7e498bd73`, followed by production failures +at `2e1aa6bbf` before live qualification. Manual run +[35659529618](https://github.com/stacklok/mecatl/actions/runs/35659529618) failed before +provider credential staging; no provider credentials were used. These reports +were not independently re-fetched during this repair. The production timeouts +remain unresolved incidents, not evidence that either source defect below caused them. + +- `TestRetainedDeletePeerBetweenFinalizerUpdateAndDelete` deterministically failed + when a second reconciler re-added the environment finalizer between Update and + Delete, stranding the authorized operation behind the generic deletion branch. + It also failed for an already-terminating authorized CR. Both pass after routing + only admitted retained deletion ahead of generic finalization. The test checks + one successful CR/PVC deletion and released capacity. Foreign-CR identity and + unapproved direct deletion have separate fail-closed controls; Delete also pins + resourceVersion and UID. No CRD, protobuf, or authorization contract changed. +- The holder-loss fixture's `nohup` timer was a descendant of the executor command. + Disconnect cancellation kills the process group and reaps descendants, so that + timer could legitimately disappear without terminating the Pod. The fixture now + proves FenceUnknown and overlap denial first, then uses exact-owned-Pod UID-guarded + Kubernetes deletion with normal grace. It waits for Succeeded/Failed and every + declared container's terminated state; absence is never proof. Offline tests + reject foreign UID/owner and incomplete terminal evidence. This causal mechanism + is source-proven; the historical timeout's cause is not. +- The legacy quota fixture waits up to six minutes, respects cancellation, fails + immediately on forbidden responses, and reports only allowlisted missing/mismatched + quota keys and elapsed time on deadline. Tiny-context offline tests cover readiness, + deadline, and cancellation without a six-minute test or blanket create retries. +- The manual workflow collects bounded, closed-vocabulary diagnostics before owned + cleanup when production fails. Offline fixture tests verify redaction, UID-match + booleans, known finalizers, quota key names, and collection even when deletion fails. + The CI-owned path uses the same collector. No raw manifests, commands, grants, + credentials, private URLs, or PKI enter its artifact allowlist. + +The retained-deletion, terminal-proof, quota-diagnostic, and pre-cleanup evidence +regressions were observed red then green offline. Final amended-candidate production +and native live runs remain **PENDING**; no cluster, live model, dispatch, or push +was performed for this repair. `0e8c8a5ca` removed SC2016, but earlier local +actionlint ran without ShellCheck and is not a ShellCheck pass. The plan baseline +remains `7d9e37a128b17107e2d83467c7767146e9c5efb8` in the normal-merge ancestry. +Current offline gates pass: targeted controller/chart/contract and tagged fixture +race tests, `task lint`, tagged changed-line lint, one full `task test` attempt +(including standalone modules), `task docs` plus final docs/checker checks, +`task site:build`, draft/report-only native-plan trace (32 ACs, zero missing), +and the offline demo. Local actionlint passed with ShellCheck explicitly disabled; +ShellCheck was unavailable on the host and in the existing dev toolbox. CI remains +the authority for ShellCheck and final runtime qualification. + [PR #1728](https://github.com/stacklok/mecatl/pull/1728), commit `6501b5924`, is already integrated in the implementation ancestry. Its generic live-compaction repair is not native-provider qualification. Final-candidate native-provider live and amended @@ -540,15 +589,15 @@ new-head CI, final live execution, or human contract/panel approval. | AC4.5 | `internal/adapter/executioncontroller/admin_scope_security_test.go`: `TestAdministratorScopeManifestValidation`, `TestAdministratorScopeDigestIsNormalizedAndAuthorityBound`; `internal/adapter/executioncontroller/admin_scope_test.go`: all-routes scope removal plus `TestScopedAdminEqualGenerationDriftFailsClosed`. Compatibility/upgrade human review PENDING. | | AC5.1 | `internal/adapter/executionclient/client_test.go`: exact reattachment test above; production `TestKindExecutionProductionReplicaLifecycle` and security rotation. | | AC5.2 | `internal/adapter/executioncontroller/production_lifecycle_test.go`: `TestReplacementPersistsTerminalProofBeforeRemovingPodFinalizer`; production replica lifecycle. | -| AC5.3 | `internal/adapter/executioncontroller/production_lifecycle_test.go`: `TestReplacementQuiescesAndPendingReferenceBlocksRetirement`. | +| AC5.3 | `internal/adapter/executioncontroller/production_lifecycle_test.go`: `TestReplacementQuiescesAndPendingReferenceBlocksRetirement`; `internal/adapter/executioncontroller/retained_delete_race_test.go`: `TestRetainedDeletePeerBetweenFinalizerUpdateAndDelete`, `TestRetainedDeleteDoesNotRemoveForeignCR`, `TestDirectCRDeleteStillBlocked` (offline race controls; final runtime evidence PENDING). | | AC5.4 | `internal/adapter/executionclient/reference_ambiguity_test.go`: `TestAmbiguousCommitIsRetainedAndNeverAbortedByCleanup`; production `TestKindExecutionProductionPendingDeleteOutageRecovery`. | | AC5.5, AC5.7 | `deploy/helm/mecatl-execution/chart_test.go`: `TestChartHasNoDeletionHook`; `deploy/helm/mecatl-execution/lifetime_test.go`: `TestChartRetainedLifetime` exercises lookup/adoption, missing history, foreign ownership, profile drift, and retained policies/ledgers offline. `e2e/k8s_execution/zz_production_helm_lifetime_test.go`: `TestKindExecutionProductionHelmLifetime` exercises actual upgrade/uninstall/reinstall and enforcement; final runtime evidence PENDING. | | AC5.6 | Deployment-documentation inspection and docs gates required on the final implementation; Kind is restart proof only. | | AC6.1–AC6.2 | `internal/app/remote_execution_test.go`: `TestRemoteExecutionRealFactoryCarriesPostureAndAttenuatedCatalog` includes a nonempty LOCAL Git snapshot control and REMOTE exclusion, poison-source and operator-global prompt controls, forbidden-tool errors and command/worktree discovery. `TestRemoteExecutionPreservesOperatorPermissionsUnderAuto` exercises global/explicit-file Deny and configured Ask, excluding project permissions, with allowed in-memory Shell controls. `TestRemoteDeploymentNoFSUsesLocalAttenuationWithoutProviderCall` retains the no-FS check. `internal/adapter/executionclient/service_integration_test.go`: `TestServiceUsesRealMTLSProviderStoreAndReleasesOnlyAfterDrain` rejects background Shell and actual Clear/Fork unsupported-selector calls with named errors, no destination publication, and unchanged source ref/history and provider references. | | AC6.3 | `internal/adapter/executioncontroller/store_lifecycle_test.go`: `TestReferenceTransactionsRetainUnknownAndNeverChangeSource`; production `TestKindExecutionProductionClearForkLifecycle`. | -| AC7.1 | `.github/workflows/k8s-e2e.yml` runs `task e2e:k8s:execution:production`; the historical reference job succeeded. The latest operator-reported job at `f7919f6c` passed 10/11 production tests and failed rotation's final read; it progressed past the earlier quota setup failure. Offline rotation evidence and read-only barrier controls are listed above. `internal/adapter/executioncontroller/legacy_fixture_kind_test.go` covers the bounded quota-accounting wait. Final amended-candidate run **PENDING**. | +| AC7.1 | `.github/workflows/k8s-e2e.yml` runs `task e2e:k8s:execution:production`; historical and operator-reported runs are distinguished above. The reported 11/11 pass at `7e498bd73` does not supersede later failures at `2e1aa6bbf`. `internal/adapter/executioncontroller/legacy_fixture_kind_test.go` covers the six-minute quota-accounting bound with tiny-context tests. `e2e/k8s_execution/holder_loss_test.go` covers the exact-owned terminal stimulus and complete kubelet evidence offline. Final amended-candidate run **PENDING**. | | AC7.2 | `e2e/k8s_execution/qualification_test.go`: `TestKindExecutionQualification`, run by the successful production profile. | -| AC7.3 | Production `TestKindExecutionProductionFailureArtifactBoundary`; workflow bounded uploads and ownership-scoped cleanup. | +| AC7.3 | Production `TestKindExecutionProductionFailureArtifactBoundary`; `deploy/mecatl-execution-kind/failure_evidence_test.go` tests closed-vocabulary projection and API-error redaction; `TestNativeWorkflowCleanupOwnership` proves evidence collection before failed cleanup. Workflow bounded uploads and ownership-scoped cleanup; final runtime evidence PENDING. | | AC7.4 | `e2e/k8s_execution/live_qualification_test.go`: `TestKindExecutionLiveQualification` exists; successful final native-provider OpenRouter evidence **PENDING**. | | AC7.5 | `e2e/k8s_execution/fixture/credentialloader/main_test.go`: `TestLoadCredentialAndRedaction`, `TestDeletePinsReceiptUIDAndLeavesReplacement`; these offline tests do not substitute for live execution. | diff --git a/e2e/k8s_execution/holder_loss_helpers_test.go b/e2e/k8s_execution/holder_loss_helpers_test.go new file mode 100644 index 0000000000..c1ff8aa4ed --- /dev/null +++ b/e2e/k8s_execution/holder_loss_helpers_test.go @@ -0,0 +1,61 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "errors" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" +) + +func terminateOwnedExecutor(ctx context.Context, kube kubernetes.Interface, environmentID, environmentUID, podUID string) error { + pods, err := kube.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "execution.mecatl.dev/environment=" + environmentID}) + if err != nil { + return err + } + if len(pods.Items) != 1 || podUID == "" || environmentUID == "" { + return errors.New("exact owned executor unavailable") + } + pod := pods.Items[0] + if string(pod.UID) != podUID || len(pod.OwnerReferences) != 1 { + return errors.New("executor identity changed") + } + owner := pod.OwnerReferences[0] + if owner.APIVersion != "execution.mecatl.dev/v1alpha1" || owner.Kind != "ExecutionEnvironment" || owner.Name != environmentID || string(owner.UID) != environmentUID || owner.Controller == nil || !*owner.Controller { + return errors.New("executor owner changed") + } + uid, grace := types.UID(podUID), int64(30) + return kube.CoreV1().Pods(namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &grace, Preconditions: &metav1.Preconditions{UID: &uid}}) +} + +func executorTerminalProof(pod *corev1.Pod, uid string) bool { + if pod == nil || string(pod.UID) != uid || uid == "" || (pod.Status.Phase != corev1.PodFailed && pod.Status.Phase != corev1.PodSucceeded) { + return false + } + names := make(map[string]bool) + for _, c := range pod.Spec.Containers { + names[c.Name] = false + } + for _, c := range pod.Spec.InitContainers { + names[c.Name] = false + } + for _, c := range pod.Spec.EphemeralContainers { + names[c.Name] = false + } + statuses := append(append(append([]corev1.ContainerStatus{}, pod.Status.ContainerStatuses...), pod.Status.InitContainerStatuses...), pod.Status.EphemeralContainerStatuses...) + if len(pod.Spec.Containers) == 0 || len(statuses) != len(names) { + return false + } + for _, s := range statuses { + done, found := names[s.Name] + if !found || done || s.State.Terminated == nil { + return false + } + names[s.Name] = true + } + return true +} diff --git a/e2e/k8s_execution/holder_loss_test.go b/e2e/k8s_execution/holder_loss_test.go new file mode 100644 index 0000000000..f4525d930f --- /dev/null +++ b/e2e/k8s_execution/holder_loss_test.go @@ -0,0 +1,68 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +func TestHolderLossTerminalStimulusPinsOwnedPod(t *testing.T) { + for _, changed := range []string{"", "uid", "owner", "absent"} { + t.Run(changed, func(t *testing.T) { + controlled := true + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "executor", Namespace: namespace, UID: "pod-uid", Labels: map[string]string{"execution.mecatl.dev/environment": "env"}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "execution.mecatl.dev/v1alpha1", Kind: "ExecutionEnvironment", Name: "env", UID: "env-uid", Controller: &controlled}}}} + if changed == "uid" { + pod.UID = "foreign" + } + if changed == "owner" { + pod.OwnerReferences[0].UID = "foreign" + } + k := kubefake.NewClientset(pod) + if changed == "absent" { + k = kubefake.NewClientset() + } + deletes := 0 + k.PrependReactor("delete", "pods", func(a ktesting.Action) (bool, runtime.Object, error) { + deletes++ + opts := a.(ktesting.DeleteAction).GetDeleteOptions() + if opts.GracePeriodSeconds == nil || *opts.GracePeriodSeconds != 30 || opts.Preconditions == nil || opts.Preconditions.UID == nil || *opts.Preconditions.UID != "pod-uid" { + t.Fatal("deletion must pin UID with normal grace") + } + return true, nil, nil + }) + err := terminateOwnedExecutor(context.Background(), k, "env", "env-uid", "pod-uid") + if (err == nil) != (changed == "") || deletes != map[bool]int{true: 1, false: 0}[changed == ""] { + t.Fatalf("deletes=%d err=%v", deletes, err) + } + for _, a := range k.Actions() { + if a.GetVerb() != "list" && a.GetVerb() != "delete" { + t.Fatal("stimulus must not patch finalizers") + } + } + }) + } +} + +func TestHolderLossTerminalProofRequiresExactCompleteEvidence(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{UID: "pod-uid"}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "executor"}}, InitContainers: []corev1.Container{{Name: "init"}}}, Status: corev1.PodStatus{Phase: corev1.PodFailed, ContainerStatuses: []corev1.ContainerStatus{{Name: "executor", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{}}}}, InitContainerStatuses: []corev1.ContainerStatus{{Name: "init", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{}}}}}} + if !executorTerminalProof(pod, "pod-uid") { + t.Fatal("terminal proof rejected") + } + for _, mutate := range []func(*corev1.Pod){func(p *corev1.Pod) { p.UID = "foreign" }, func(p *corev1.Pod) { p.Status.Phase = corev1.PodRunning }, func(p *corev1.Pod) { p.Status.InitContainerStatuses = nil }, func(p *corev1.Pod) { p.Status.ContainerStatuses[0].Name = "other" }, func(p *corev1.Pod) { p.Status.ContainerStatuses[0].State.Terminated = nil }} { + candidate := pod.DeepCopy() + mutate(candidate) + if executorTerminalProof(candidate, "pod-uid") { + t.Fatal("incomplete or foreign evidence accepted") + } + } + if executorTerminalProof(nil, "pod-uid") { + t.Fatal("absence is not terminal proof") + } +} diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go index 02a9950820..86f0e51ad8 100644 --- a/e2e/k8s_execution/production_qualification_test.go +++ b/e2e/k8s_execution/production_qualification_test.go @@ -17,6 +17,8 @@ import ( "time" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" "github.com/stacklok/mecatl/internal/adapter/executionclient" "github.com/stacklok/mecatl/internal/executionenv" @@ -951,7 +953,7 @@ func TestKindExecutionProductionHolderLossFencesActiveOperation(t *testing.T) { commandDone := make(chan error, 1) go func() { - _, commandErr := holderClient.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: `nohup sh -c 'sleep 45; kill 1' >/dev/null 2>&1 & i=0; trap '' HUP TERM; while [ $i -lt 90 ]; do i=$((i+1)); printf '%s\n' "$i" > holder-loss-nonce; sleep 1; done`, TimeoutMillis: 110000}) + _, commandErr := holderClient.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: `i=0; trap '' HUP TERM; while [ $i -lt 90 ]; do i=$((i+1)); printf '%s\n' "$i" > holder-loss-nonce; sleep 1; done`, TimeoutMillis: 110000}) commandDone <- commandErr }() waitActiveOperationOrCommandError(t, ctx, kubeconfig, attached.Environment.ID, commandDone) @@ -977,7 +979,27 @@ func TestKindExecutionProductionHolderLossFencesActiveOperation(t *testing.T) { // durable operation identity above, not RPC completion, is the proof. } - waitExecutorTerminal(t, ctx, kubeconfig, attached.Environment.ID) + // Exec disconnect cancellation kills the command process group and reaped + // descendants, so a child timer cannot reliably terminate PID 1. Only after + // proving unresolved ownership above, ask the kubelet to stop this exact Pod. + contextName := os.Getenv("MECATL_KUBE_CONTEXT") + if contextName == "" { + t.Fatal("explicit fixture Kubernetes context required") + } + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, &clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig() + if err != nil { + t.Fatal("load explicit fixture Kubernetes configuration") + } + config.Timeout = 15 * time.Second + kube, err := kubernetes.NewForConfig(config) + if err != nil { + t.Fatal("construct fixture Kubernetes client") + } + envUID := kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") + if err := terminateOwnedExecutor(ctx, kube, attached.Environment.ID, envUID, initial.PodUID); err != nil { + t.Fatal("exact owned executor termination request failed") + } + waitExecutorTerminal(ctx, t, kubeconfig, attached.Environment.ID, initial.PodUID) recovery := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: initial.Epoch, ExpectedPodUID: initial.PodUID, ExpectedPVCUID: initial.PVCUID, OperationID: "recover-holder-loss"} if err := survivingClient.RecoverEnvironment(ctx, recovery); err != nil { t.Fatalf("recover exact terminal executor: %v", err) @@ -1000,17 +1022,14 @@ func TestKindExecutionProductionHolderLossFencesActiveOperation(t *testing.T) { release() } -func waitExecutorTerminal(t *testing.T, ctx context.Context, kubeconfig, environmentID string) { +func waitExecutorTerminal(ctx context.Context, t *testing.T, kubeconfig, environmentID, podUID string) { t.Helper() deadline := time.Now().Add(2 * time.Minute) for time.Now().Before(deadline) { var pods corev1.PodList raw := runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment="+environmentID, "-o", "json") - if json.Unmarshal(raw, &pods) == nil && len(pods.Items) == 1 && (pods.Items[0].Status.Phase == corev1.PodFailed || pods.Items[0].Status.Phase == corev1.PodSucceeded) { - terminated := len(pods.Items[0].Status.ContainerStatuses) == 1 && pods.Items[0].Status.ContainerStatuses[0].State.Terminated != nil - if terminated { - return - } + if json.Unmarshal(raw, &pods) == nil && len(pods.Items) == 1 && executorTerminalProof(&pods.Items[0], podUID) { + return } time.Sleep(time.Second) } @@ -1462,7 +1481,18 @@ func executionReferences(t *testing.T, ctx context.Context, kubeconfig, environm func TestKindExecutionProductionFailureArtifactBoundary(t *testing.T) { state, kubeconfig, ctx, cancel := requireProduction(t) defer cancel() - artifact := runKubectl(t, ctx, kubeconfig, "get", "events", "-n", namespace, "-o", "custom-columns=OBJECT:.involvedObject.name,REASON:.reason", "--no-headers") + ctx, stop := context.WithTimeout(ctx, time.Minute) + defer stop() + output := filepath.Join(state, fmt.Sprintf("artifact-boundary-%d.jsonl", time.Now().UnixNano())) + cmd := exec.CommandContext(ctx, "sh", "../../deploy/mecatl-execution-kind/collect-failure.sh", kubeconfig, os.Getenv("MECATL_KUBE_CONTEXT"), output) + cmd.Env = cleanEnv() + if err := cmd.Run(); err != nil { + t.Fatal("bounded failure collector failed") + } + artifact, err := os.ReadFile(output) + if err != nil { + t.Fatal("sanitized failure artifact unavailable") + } if len(artifact) > 1<<20 { t.Fatal("sanitized failure artifact exceeds one MiB") } @@ -1471,7 +1501,4 @@ func TestKindExecutionProductionFailureArtifactBoundary(t *testing.T) { t.Fatalf("sanitized failure artifact contains forbidden class %q", forbidden) } } - if err := os.WriteFile(filepath.Join(state, "production-failure-artifact.txt"), artifact, 0o600); err != nil { - t.Fatal(err) - } } diff --git a/internal/adapter/executioncontroller/admin_lifecycle.go b/internal/adapter/executioncontroller/admin_lifecycle.go index 4c41e28f2e..346b3035da 100644 --- a/internal/adapter/executioncontroller/admin_lifecycle.go +++ b/internal/adapter/executioncontroller/admin_lifecycle.go @@ -295,7 +295,7 @@ func (s *Store) DeleteRetiredEnvironment(ctx context.Context, q adminLifecycleRe return &executionenv.Error{Code: executionenv.CodeConflict, Message: "retained workspace identity changed"} } if existing, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation"); found { - if text(existing, "id") == q.OperationID && text(existing, "type") == "DeleteRetiredEnvironment" { + if text(existing, "id") == q.OperationID && text(existing, "type") == deleteRetiredEnvironment { return nil } return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another lifecycle operation is active"} @@ -304,7 +304,7 @@ func (s *Store) DeleteRetiredEnvironment(ctx context.Context, q adminLifecycleRe if err != nil || len(refs) != 0 || textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || !conditionTrue(o, "Retired") || !conditionTrue(o, "ExecutorTerminated") { return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not safely retained and unreferenced"} } - return unstructured.SetNestedMap(o.Object, map[string]any{"id": q.OperationID, "type": "DeleteRetiredEnvironment", "phase": "DeletingPVC", "expectedPVCUID": q.ExpectedPVCUID, "createdAt": s.now().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + return unstructured.SetNestedMap(o.Object, map[string]any{"id": q.OperationID, "type": deleteRetiredEnvironment, "phase": "DeletingPVC", "expectedPVCUID": q.ExpectedPVCUID, "createdAt": s.now().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") }) } diff --git a/internal/adapter/executioncontroller/controller.go b/internal/adapter/executioncontroller/controller.go index 2e1161f613..f1c3f7deb3 100644 --- a/internal/adapter/executioncontroller/controller.go +++ b/internal/adapter/executioncontroller/controller.go @@ -219,6 +219,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, name string) error { if !ok || p.Digest != textNested(env.Object, "spec", "profileDigest") { return r.setCondition(ctx, env, "Ready", false, "InvalidProfile", "configured profile is unavailable or changed") } + // The admitted durable delete owns finalization, including after DELETE has + // set deletionTimestamp. A peer must not re-arm the generic finalizer. + if textNested(env.Object, "status", "lifecycleOperation", "type") == deleteRetiredEnvironment { + return r.reconcileLifecycle(ctx, env) + } if env.GetDeletionTimestamp() != nil { return r.reconcileDeletion(ctx, res, env) } diff --git a/internal/adapter/executioncontroller/controller_lifecycle.go b/internal/adapter/executioncontroller/controller_lifecycle.go index 5f762fd36c..644c49cadf 100644 --- a/internal/adapter/executioncontroller/controller_lifecycle.go +++ b/internal/adapter/executioncontroller/controller_lifecycle.go @@ -13,6 +13,8 @@ import ( "github.com/stacklok/mecatl/internal/executionenv" ) +const deleteRetiredEnvironment = "DeleteRetiredEnvironment" + func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.Unstructured) error { //nolint:gocyclo // Durable phases are deliberately explicit. op, found, err := unstructured.NestedMap(env.Object, "status", "lifecycleOperation") if err != nil || !found { @@ -30,7 +32,7 @@ func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.U if phase == "CreatingReplacement" { podIdentityMatches = textNested(env.Object, "status", "pod", "uid") == "" && proofMatches } - if kind == "DeleteRetiredEnvironment" { + if kind == deleteRetiredEnvironment { return r.reconcileRetainedDelete(ctx, env, op, pvcName, pvcUID) } epoch := intNested(op, "expectedEpoch") @@ -230,11 +232,14 @@ func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructu } res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) current, err := res.Get(ctx, env.GetName(), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } if err != nil { return err } currentOp, found, _ := unstructured.NestedMap(current.Object, "status", "lifecycleOperation") - if !found || text(currentOp, "id") != text(op, "id") || text(currentOp, "type") != text(op, "type") || text(currentOp, "phase") != "ReleasingSlot" || !runtimeObservationMatches(current, env) { + if current.GetUID() != env.GetUID() || !found || text(currentOp, "id") != text(op, "id") || text(currentOp, "type") != text(op, "type") || text(currentOp, "phase") != "ReleasingSlot" || !runtimeObservationMatches(current, env) { return lifecycleConflict() } if err := releaseProfileSlot(ctx, r.kube, r.namespace, textNested(current.Object, "spec", "profile"), current.GetName()); err != nil { @@ -245,8 +250,8 @@ func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructu if err != nil { return err } - uid := updated.GetUID() - if err := res.Delete(ctx, updated.GetName(), metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + uid, version := updated.GetUID(), updated.GetResourceVersion() + if err := res.Delete(ctx, updated.GetName(), metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}}); err != nil && !apierrors.IsNotFound(err) { return err } return nil diff --git a/internal/adapter/executioncontroller/legacy_fixture_kind.go b/internal/adapter/executioncontroller/legacy_fixture_kind.go index 664a59f6a1..e8a13cca55 100644 --- a/internal/adapter/executioncontroller/legacy_fixture_kind.go +++ b/internal/adapter/executioncontroller/legacy_fixture_kind.go @@ -4,7 +4,9 @@ package executioncontroller import ( "context" + "errors" "fmt" + "slices" "time" corev1 "k8s.io/api/core/v1" @@ -63,6 +65,10 @@ func SeedLegacyMigrationFixture(ctx context.Context, d dynamic.Interface, kube k return LegacyMigrationSeed{Environment: recognized.ref, Owner: owner, Binding: "legacy-migration-binding", PodUID: recognized.podUID, PVCUID: recognized.pvcUID, Malformed: malformed.ref, MalformedPodUID: malformed.podUID, MalformedPVCUID: malformed.pvcUID, Insecure: insecure.ref, InsecurePodUID: insecure.podUID, InsecurePVCUID: insecure.pvcUID}, nil } +// Kubernetes defaults resource-quota usage resync to five minutes. Discovery of +// a new CRD need not enqueue a quota that has not accounted for that resource. +const legacyQuotaWait = 6 * time.Minute + type seededLegacyEnvironment struct { ref executionenv.EnvironmentRef podUID, pvcUID string @@ -71,23 +77,35 @@ type seededLegacyEnvironment struct { func seedOneLegacyEnvironment(ctx context.Context, d dynamic.Interface, kube kubernetes.Interface, namespace string, profile resolvedProfile, owner executionenv.Owner, client, name, binding string, references []any, insecure bool) (seededLegacyEnvironment, error) { // Deployment readiness does not imply that quota admission has initialized // accounting, especially for the freshly installed custom resource. - if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + started := time.Now() + var missing []string + if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, legacyQuotaWait, true, func(ctx context.Context) (bool, error) { quota, err := kube.CoreV1().ResourceQuotas(namespace).Get(ctx, "mecatl-execution", metav1.GetOptions{}) if err != nil { return false, err } + missing = nil + ready := true for name, configured := range quota.Spec.Hard { - hard, ok := quota.Status.Hard[name] - if !ok || hard.Cmp(configured) != 0 { - return false, nil - } - if _, ok := quota.Status.Used[name]; !ok { - return false, nil + hard, hasHard := quota.Status.Hard[name] + _, hasUsed := quota.Status.Used[name] + if !hasHard || hard.Cmp(configured) != 0 || !hasUsed { + ready = false + switch name { + case "pods", "persistentvolumeclaims", "count/executionenvironments.execution.mecatl.dev", "requests.cpu", "requests.memory", "requests.storage", "requests.ephemeral-storage", "limits.cpu", "limits.memory", "limits.ephemeral-storage": + missing = append(missing, string(name)) + } } } - return true, nil + return ready, nil }); err != nil { - return seededLegacyEnvironment{}, fmt.Errorf("wait for legacy fixture quota accounting: %w", err) + if errors.Is(err, context.DeadlineExceeded) { + err = context.DeadlineExceeded + } else if errors.Is(err, context.Canceled) { + err = context.Canceled + } + slices.Sort(missing) + return seededLegacyEnvironment{}, fmt.Errorf("wait for legacy fixture quota accounting: missing_or_mismatched_keys=%v elapsed=%s: %w", missing, time.Since(started).Round(time.Millisecond), err) } revision, err := randomID() if err != nil { diff --git a/internal/adapter/executioncontroller/legacy_fixture_kind_test.go b/internal/adapter/executioncontroller/legacy_fixture_kind_test.go index 5fcce3f0c8..429b039b4a 100644 --- a/internal/adapter/executioncontroller/legacy_fixture_kind_test.go +++ b/internal/adapter/executioncontroller/legacy_fixture_kind_test.go @@ -5,6 +5,7 @@ package executioncontroller import ( "context" "errors" + "strings" "testing" "time" @@ -21,10 +22,28 @@ import ( "github.com/stacklok/mecatl/internal/executionenv" ) +func TestLegacyQuotaWaitCoversDefaultControllerResync(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + kube := kubefake.NewClientset() + kube.PrependReactor("get", "resourcequotas", func(ktesting.Action) (bool, runtime.Object, error) { + cancel() + return true, nil, context.Canceled + }) + if legacyQuotaWait < 6*time.Minute { + t.Fatal("fixture must cover the five-minute quota resync plus margin") + } + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + _, err := seedOneLegacyEnvironment(ctx, d, kube, "test", resolvedProfile{}, executionenv.Owner{}, "client", "legacy", "binding", nil, false) + if !errors.Is(err, context.Canceled) || len(d.Actions()) != 0 { + t.Fatal("cancellation must stop before creating fixtures") + } +} + func TestLegacyFixtureWaitsForQuotaAccountingBeforeCreate(t *testing.T) { for _, mode := range []string{"delayed", "timeout", "cancel", "forbidden"} { t.Run(mode, func(t *testing.T) { - quota := &corev1.ResourceQuota{ObjectMeta: metav1.ObjectMeta{Name: "mecatl-execution", Namespace: "test"}, Spec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{corev1.ResourceName("count/executionenvironments.execution.mecatl.dev"): resource.MustParse("10"), corev1.ResourcePods: resource.MustParse("10")}}} + quota := &corev1.ResourceQuota{ObjectMeta: metav1.ObjectMeta{Name: "mecatl-execution", Namespace: "test"}, Spec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{corev1.ResourceName("count/executionenvironments.execution.mecatl.dev"): resource.MustParse("10"), corev1.ResourcePods: resource.MustParse("10"), corev1.ResourceName("private-quota-key"): resource.MustParse("12345")}}} kube := kubefake.NewClientset() reads, creates := 0, 0 ctx, cancel := context.WithTimeout(context.Background(), 350*time.Millisecond) @@ -37,6 +56,9 @@ func TestLegacyFixtureWaitsForQuotaAccountingBeforeCreate(t *testing.T) { // Partially initialized status must not pass: all configured resources matter. quota.Status.Hard = quota.Spec.Hard.DeepCopy() quota.Status.Used = corev1.ResourceList{corev1.ResourcePods: resource.MustParse("0")} + if mode == "timeout" { + quota.Status.Hard[corev1.ResourcePods] = resource.MustParse("20") + } if mode == "delayed" && reads >= 2 { quota.Status.Used = quota.Spec.Hard.DeepCopy() } @@ -66,6 +88,12 @@ func TestLegacyFixtureWaitsForQuotaAccountingBeforeCreate(t *testing.T) { } switch mode { case "timeout": + if strings.Contains(err.Error(), "private-quota-key") || strings.Contains(err.Error(), "12345") || !strings.Contains(err.Error(), "pods") { + t.Fatalf("quota diagnostics lost mismatch or leaked data: %v", err) + } + if !strings.Contains(err.Error(), "count/executionenvironments.execution.mecatl.dev") || !strings.Contains(err.Error(), "elapsed=") { + t.Fatalf("missing bounded quota diagnostic: %v", err) + } if !errors.Is(err, context.DeadlineExceeded) { t.Fatal(err) } diff --git a/internal/adapter/executioncontroller/retained_delete_race_test.go b/internal/adapter/executioncontroller/retained_delete_race_test.go new file mode 100644 index 0000000000..90bc1978bf --- /dev/null +++ b/internal/adapter/executioncontroller/retained_delete_race_test.go @@ -0,0 +1,159 @@ +package executioncontroller + +import ( + "errors" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +func TestRetainedDeletePeerBetweenFinalizerUpdateAndDelete(t *testing.T) { + for _, terminating := range []bool{false, true} { + t.Run(map[bool]string{false: "peer-before-delete", true: "already-terminating"}[terminating], func(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(retainedPVC(), &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{profileAllocationKey("go"): `["env"]`}}) + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + if err := store.DeleteRetiredEnvironment(t.Context(), adminRequestFixture()); err != nil { + t.Fatal(err) + } + // Separate fake clients avoid the fake's per-client reactor lock, while both + // reconcilers observe the same API-server tracker. + peerClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + peerClient.PrependReactor("*", "*", ktesting.ObjectReaction(d.Tracker())) + peer := NewReconciler(peerClient, k, "ns", testProfiles()) + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(peer.queue.ShutDown) + t.Cleanup(r.queue.ShutDown) + pvcsDeleted, envsDeleted := 0, 0 + k.PrependReactor("delete", "persistentvolumeclaims", func(a ktesting.Action) (bool, runtime.Object, error) { + opts := a.(ktesting.DeleteAction).GetDeleteOptions() + if opts.Preconditions == nil || opts.Preconditions.UID == nil || *opts.Preconditions.UID != "pvc-uid" { + t.Fatal("PVC deletion lost exact UID") + } + pvcsDeleted++ + return false, nil, nil + }) + deletion := func(a ktesting.Action) (bool, runtime.Object, error) { + obj, err := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if err != nil { + return true, nil, err + } + cur := obj.(*unstructured.Unstructured) + opts := a.(ktesting.DeleteAction).GetDeleteOptions() + if opts.Preconditions == nil || opts.Preconditions.UID == nil || *opts.Preconditions.UID != cur.GetUID() { + t.Fatal("CR deletion lost exact UID") + } + if contains(cur.GetFinalizers(), environmentFinalizer) { + now := metav1.Now() + cur.SetDeletionTimestamp(&now) + return true, nil, d.Tracker().Update(ExecutionEnvironmentGVR, cur, "ns") + } + envsDeleted++ + return true, nil, d.Tracker().Delete(ExecutionEnvironmentGVR, "ns", "env") + } + peerClient.PrependReactor("delete", "executionenvironments", deletion) + interleaved := false + d.PrependReactor("delete", "executionenvironments", func(a ktesting.Action) (bool, runtime.Object, error) { + if !interleaved { + interleaved = true + if err := peer.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + } + return deletion(a) + }) + if terminating { + obj, _ := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + cur := obj.(*unstructured.Unstructured) + now := metav1.Now() + cur.SetDeletionTimestamp(&now) + if err := d.Tracker().Update(ExecutionEnvironmentGVR, cur, "ns"); err != nil { + t.Fatal(err) + } + } + for range 5 { + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + } + if _, err := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env"); !apierrors.IsNotFound(err) { + t.Fatalf("authorized deletion stranded: %v", err) + } + if pvcsDeleted != 1 || envsDeleted != 1 { + t.Fatalf("successful deletes PVC=%d CR=%d", pvcsDeleted, envsDeleted) + } + if len(profileSlots(t, k)) != 0 { + t.Fatal("capacity was not released") + } + }) + } +} + +func TestRetainedDeleteDoesNotRemoveForeignCR(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + if err := store.DeleteRetiredEnvironment(t.Context(), adminRequestFixture()); err != nil { + t.Fatal(err) + } + env, _ = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + _ = unstructured.SetNestedField(env.Object, "ReleasingSlot", "status", "lifecycleOperation", "phase") + foreign := env.DeepCopy() + foreign.SetUID("foreign") + if err := d.Tracker().Update(ExecutionEnvironmentGVR, foreign, "ns"); err != nil { + t.Fatal(err) + } + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + op, _, _ := unstructured.NestedMap(env.Object, "status", "lifecycleOperation") + if err := r.reconcileRetainedDelete(t.Context(), env, op, "workspace", "pvc-uid"); err == nil { + t.Fatal("foreign CR did not conflict") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil || got.GetUID() != "foreign" || !contains(got.GetFinalizers(), environmentFinalizer) { + t.Fatal("foreign CR touched") + } + for _, a := range k.Actions() { + if a.GetVerb() != "get" { + t.Fatal("foreign CR released capacity") + } + } +} + +func TestDirectCRDeleteStillBlocked(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + now := metav1.Now() + env.SetDeletionTimestamp(&now) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(retainedPVC(), terminalExecutor()) + k.PrependReactor("delete", "*", func(ktesting.Action) (bool, runtime.Object, error) { + t.Error("direct delete destroyed runtime") + return true, nil, apierrors.NewForbidden(schema.GroupResource{}, "", errors.New("unexpected")) + }) + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil || !contains(got.GetFinalizers(), environmentFinalizer) { + t.Fatal("direct deletion lost finalizer") + } + if !conditionTrue(got, "DeletionBlocked") { + t.Fatal("direct deletion not blocked") + } +} diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index 9edafed120..2500ebc1b7 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -324,11 +324,17 @@ Failure before kubeconfig creation also fails cleanup without deleting a cluster Cleanup is bounded best effort: hard cancellation or runner loss can prevent it. Hosted-runner disposal is the fallback, not evidence that cleanup succeeded. -Artifacts are retained for seven days: only a size-bounded `live-summary.json` -when available and a sanitized `qualification-status.txt`. PKI, kubeconfigs, -Secret receipts, Helm values, full scratch directories, and transcripts are never -uploaded by the native job. Check both live and cleanup outcomes before treating -the recorded commit as qualified. +Artifacts are retained for seven days: a size-bounded `live-summary.json` when +available, a sanitized `qualification-status.txt`, and `production-diagnostics.jsonl` +when production fails and owned-cluster validation succeeds. Cleanup collects the +production diagnostics before deleting the cluster, with a 60-second bound and +10-second API request deadlines. The 1 MiB diagnostic file contains only known +condition/reason classes, deletion and UID-match booleans, known finalizers, Pod +phases, container exit reasons, lease-expiry status, quota key names, and event +reason counts. It remains available if cluster deletion subsequently fails. +PKI, kubeconfigs, Secret receipts, Helm values, full scratch directories, raw +resource manifests, and transcripts are never uploaded by the native job. Check +both live and cleanup outcomes before treating the recorded commit as qualified. ## Limitations From 53fdb9203b91c4f3551c5217adf4590c04b01fa7 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Tue, 22 Sep 2026 02:13:18 +0300 Subject: [PATCH 24/25] fix(execution): close final fixture and cleanup review gaps Co-Authored-By: OpenAI --- deploy/mecatl-execution-kind/ci_test.go | 24 ++++-- .../acceptance/native-kubernetes-execution.md | 4 +- .../legacy_fixture_kind.go | 5 ++ .../legacy_fixture_kind_test.go | 21 ++++- .../retained_delete_race_test.go | 78 ++++++++++++++++--- 5 files changed, 111 insertions(+), 21 deletions(-) diff --git a/deploy/mecatl-execution-kind/ci_test.go b/deploy/mecatl-execution-kind/ci_test.go index 621d437f6c..db8c2dfc8f 100644 --- a/deploy/mecatl-execution-kind/ci_test.go +++ b/deploy/mecatl-execution-kind/ci_test.go @@ -140,8 +140,9 @@ func TestNativeWorkflowCommitAndCredentialBoundary(t *testing.T) { func TestNativeWorkflowCleanupOwnership(t *testing.T) { steps := nativeSteps(t) - for _, fault := range []string{"", "owner", "context", "label", "duplicate", "symlink", "delete", "list", "switched-pointer", "captured-cluster", "missing-kubeconfig", "missing-ownership", "missing-output", "partial-with-kubeconfig"} { + for _, fault := range []string{"", "owner", "context", "label", "duplicate", "symlink", "delete", "list", "switched-pointer", "captured-cluster", "missing-kubeconfig", "missing-ownership", "missing-output", "partial-with-kubeconfig", "collector-error", "collector-timeout", "collector-error-delete", "collector-timeout-delete"} { t.Run("fault="+fault, func(t *testing.T) { + collectorFails := strings.HasPrefix(fault, "collector-") root := t.TempDir() state := filepath.Join(root, ".scratch", "k8s-execution", "owned") dir := filepath.Join(root, ".scratch", "ci") @@ -198,21 +199,32 @@ func TestNativeWorkflowCleanupOwnership(t *testing.T) { writeFixture(t, filepath.Join(state, "live-summary.json"), "{}\n", 0o600) writeFixture(t, filepath.Join(bin, "kubectl"), "#!/bin/sh\nprintf '%s\\n' 'kind-"+cluster+"'\n", 0o700) writeFixture(t, filepath.Join(bin, "docker"), "#!/bin/sh\nif [ \"$FAULT\" = label ]; then exit 1; fi\nprintf '%s\\n' '"+cluster+"'\n", 0o700) - writeFixture(t, filepath.Join(bin, "kind"), "#!/bin/sh\nif [ \"$1\" = delete ]; then\n test \"$*\" = 'delete cluster --name "+cluster+"' || exit 1\n test \"$FAULT\" != delete || exit 1\n printf deleted > \"$MARKER\"\nelse\n test \"$FAULT\" != list || exit 1\nfi\n", 0o700) + writeFixture(t, filepath.Join(bin, "kind"), "#!/bin/sh\nif [ \"$1\" = delete ]; then\n test \"$*\" = 'delete cluster --name "+cluster+"' || exit 1\n printf '%s\\n' \"$*\" >> \"$MARKER.attempt\"\n case \"$FAULT\" in delete|collector-*-delete) exit 1 ;; esac\n printf deleted > \"$MARKER\"\nelse\n test \"$FAULT\" != list || exit 1\nfi\n", 0o700) marker := filepath.Join(root, "deleted") - if fault == "delete" || fault == "list" || fault == "partial-with-kubeconfig" { + if fault == "delete" || fault == "list" || fault == "partial-with-kubeconfig" || collectorFails { scripts := filepath.Join(root, "deploy/mecatl-execution-kind") if err := os.MkdirAll(scripts, 0o700); err != nil { t.Fatal(err) } - writeFixture(t, filepath.Join(scripts, "collect-failure.sh"), "#!/bin/sh\ntest ! -e \"$MARKER\" || exit 1\nprintf '{\"kind\":\"collection\"}\\n' > \"$3\"\n", 0o700) + writeFixture(t, filepath.Join(scripts, "collect-failure.sh"), "#!/bin/sh\ntest ! -e \"$MARKER.attempt\" || exit 1\nprintf collected > \"$MARKER.collection\"\ncase \"$FAULT\" in collector-error*) exit 17 ;; collector-timeout*) exit 124 ;; esac\nprintf '{\"kind\":\"collection\"}\\n' > \"$3\"\n", 0o700) } outcome := "success" - if strings.HasPrefix(fault, "missing-") || fault == "partial-with-kubeconfig" || fault == "delete" || fault == "list" { + if strings.HasPrefix(fault, "missing-") || fault == "partial-with-kubeconfig" || fault == "delete" || fault == "list" || collectorFails { outcome = "failure" } out, err := runStep(t, root, steps["cleanup"].Run, "PATH="+bin+":"+os.Getenv("PATH"), "NATIVE_CI_DIR="+dir, "GITHUB_WORKSPACE="+root, "USER=fixture", "PRODUCTION_OUTCOME="+outcome, "MECATL_EXECUTION_QUAL_STATE="+capturedState, "NATIVE_CLUSTER="+capturedCluster, "FAULT="+fault, "MARKER="+marker) - wantOK := fault == "" || fault == "switched-pointer" || fault == "partial-with-kubeconfig" + wantOK := fault == "" || fault == "switched-pointer" || fault == "partial-with-kubeconfig" || fault == "collector-error" || fault == "collector-timeout" + if collectorFails { + if !strings.Contains(string(out), "::warning::Bounded production diagnostics incomplete") { + t.Fatal("collector failure must warn") + } + if data, err := os.ReadFile(marker + ".collection"); err != nil || string(data) != "collected" { + t.Fatal("collector must run before deletion") + } + if data, err := os.ReadFile(marker + ".attempt"); err != nil || string(data) != "delete cluster --name "+cluster+"\n" { + t.Fatal("collector failure must not skip exact owned-cluster deletion") + } + } if (err == nil) != wantOK { t.Fatalf("cleanup result: %v: %s", err, out) } diff --git a/docs/acceptance/native-kubernetes-execution.md b/docs/acceptance/native-kubernetes-execution.md index e21c8dda06..b63cfdfc93 100644 --- a/docs/acceptance/native-kubernetes-execution.md +++ b/docs/acceptance/native-kubernetes-execution.md @@ -554,7 +554,7 @@ and native live runs remain **PENDING**; no cluster, live model, dispatch, or pu was performed for this repair. `0e8c8a5ca` removed SC2016, but earlier local actionlint ran without ShellCheck and is not a ShellCheck pass. The plan baseline remains `7d9e37a128b17107e2d83467c7767146e9c5efb8` in the normal-merge ancestry. -Current offline gates pass: targeted controller/chart/contract and tagged fixture +Reported offline gates at `53ab45453` passed: targeted controller/chart/contract and tagged fixture race tests, `task lint`, tagged changed-line lint, one full `task test` attempt (including standalone modules), `task docs` plus final docs/checker checks, `task site:build`, draft/report-only native-plan trace (32 ACs, zero missing), @@ -583,7 +583,7 @@ new-head CI, final live execution, or human contract/panel approval. | AC3.3 | `internal/adapter/executioncontroller/file_authorization_test.go`: every supported file operation has positive and negative signed-grant controls; `internal/executionexecutor/confinement_test.go`: physical symlink traversal/search checks. Existing security manager, protocol, durable revocation, and detached-control tests remain complementary. | | AC3.4 | `internal/executionexecutor/executor_test.go`: `TestNativeShellCapsCombinedOutputAndRepairsUTF8`, `TestNativeShellWriteInvalidatesRecordedRead` execute the native helper for combined output bounds, malformed bytes, and actual Shell-write→Edit version refusal. `internal/adapter/executionclient/client_test.go`: `TestProviderThroughRealGRPCSignedHandlerRefreshesAndReattachesExactly` checks runner text repair after private protobuf bytes; `internal/adapter/executionclient/service_integration_test.go`: `TestServiceUsesRealMTLSProviderStoreAndReleasesOnlyAfterDrain` checks identical valid UTF-8 in client events and model history. `internal/adapter/executioncontroller/store_test.go`: `TestCancelledStoreOperationStaysActiveUntilBackendStopsThenFences`; production `TestKindExecutionProductionHolderLossFencesActiveOperation` remains runtime qualification. | | AC4.1 | `internal/adapter/executioncontroller/store_lifecycle_test.go`: `TestClientScopedIntentListReturnsAttestedOwnerOnlyToOwningClient`; Kind isolation and pending-delete tests; scoped-admin non-escalation is exercised by the AC4.4 tests. | -| AC4.2 | `internal/adapter/executioncontroller/store_revoke_test.go`: `TestRevokeEnvironmentFencesOldClaimWithoutChangingExecutionEpoch`; `store_concurrency_test.go`: `TestAcquireRunAndRevokeUseResourceVersionCAS`; the three deterministic rotation tests listed above at repair commit `96e16e926`. Latest reported production rotation failed; final runtime success PENDING. | +| AC4.2 | `internal/adapter/executioncontroller/store_revoke_test.go`: `TestRevokeEnvironmentFencesOldClaimWithoutChangingExecutionEpoch`; `store_concurrency_test.go`: `TestAcquireRunAndRevokeUseResourceVersionCAS`; the three deterministic rotation tests listed above at repair commit `96e16e926`. The reported production rotation at `f7919f6c` failed; final runtime success PENDING. | | AC4.3 | `internal/adapter/executioncontroller/production_lifecycle_test.go`: `TestExpiredOperationLeaseFencesWithoutClearingIdentity`, `TestRecoverMissingPodRemainsFenceUnknown`, `TestRecoveredTerminalProofCanStartExactReplacement`; production holder loss. | | AC4.4 | `internal/adapter/executioncontroller/admin_scope_test.go`: `TestScopedAdminAllRoutesOverMTLS`, `TestScopedAdminCASRetryRechecksSubject`, `TestScopedAdminMigrationReceiptRetainsUIDPreconditions`, `TestMigrationCompletedCASReplayRequiresExactSourceSchema`, `TestMigrationReceiptExpiresAfterReconciledReplacement`, `TestScopedAdminDoesNotGrantAttestationOrDataPlane`, `TestScopedAdminWithOwnerAttestationCannotUseAnotherCreatorsDataPlane`. Replacement-expiry test is in `internal/adapter/executioncontroller/production_lifecycle_test.go` in the pinned implementation candidate. `e2e/k8s_execution/ab_production_admin_test.go`: `TestKindExecutionProductionScopedAdministrator` exists (replacement/non-escalation, not all six routes); final Kind evidence PENDING. | | AC4.5 | `internal/adapter/executioncontroller/admin_scope_security_test.go`: `TestAdministratorScopeManifestValidation`, `TestAdministratorScopeDigestIsNormalizedAndAuthorityBound`; `internal/adapter/executioncontroller/admin_scope_test.go`: all-routes scope removal plus `TestScopedAdminEqualGenerationDriftFailsClosed`. Compatibility/upgrade human review PENDING. | diff --git a/internal/adapter/executioncontroller/legacy_fixture_kind.go b/internal/adapter/executioncontroller/legacy_fixture_kind.go index e8a13cca55..5266a19c2c 100644 --- a/internal/adapter/executioncontroller/legacy_fixture_kind.go +++ b/internal/adapter/executioncontroller/legacy_fixture_kind.go @@ -103,7 +103,12 @@ func seedOneLegacyEnvironment(ctx context.Context, d dynamic.Interface, kube kub err = context.DeadlineExceeded } else if errors.Is(err, context.Canceled) { err = context.Canceled + } else if apierrors.IsForbidden(err) { + err = &apierrors.StatusError{ErrStatus: metav1.Status{Reason: metav1.StatusReasonForbidden, Code: 403, Message: "forbidden"}} + } else { + err = errors.New("api_error") } + // The fixture CLI prints this diagnostic; never wrap raw API response data. slices.Sort(missing) return seededLegacyEnvironment{}, fmt.Errorf("wait for legacy fixture quota accounting: missing_or_mismatched_keys=%v elapsed=%s: %w", missing, time.Since(started).Round(time.Millisecond), err) } diff --git a/internal/adapter/executioncontroller/legacy_fixture_kind_test.go b/internal/adapter/executioncontroller/legacy_fixture_kind_test.go index 429b039b4a..fc710b19b4 100644 --- a/internal/adapter/executioncontroller/legacy_fixture_kind_test.go +++ b/internal/adapter/executioncontroller/legacy_fixture_kind_test.go @@ -5,6 +5,7 @@ package executioncontroller import ( "context" "errors" + "fmt" "strings" "testing" "time" @@ -41,17 +42,21 @@ func TestLegacyQuotaWaitCoversDefaultControllerResync(t *testing.T) { } func TestLegacyFixtureWaitsForQuotaAccountingBeforeCreate(t *testing.T) { - for _, mode := range []string{"delayed", "timeout", "cancel", "forbidden"} { + for _, mode := range []string{"delayed", "timeout", "cancel", "forbidden", "api-error"} { t.Run(mode, func(t *testing.T) { quota := &corev1.ResourceQuota{ObjectMeta: metav1.ObjectMeta{Name: "mecatl-execution", Namespace: "test"}, Spec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{corev1.ResourceName("count/executionenvironments.execution.mecatl.dev"): resource.MustParse("10"), corev1.ResourcePods: resource.MustParse("10"), corev1.ResourceName("private-quota-key"): resource.MustParse("12345")}}} kube := kubefake.NewClientset() reads, creates := 0, 0 ctx, cancel := context.WithTimeout(context.Background(), 350*time.Millisecond) defer cancel() + const hostile = "https://sentinel.invalid/private?token=sk-fake-quota-secret" kube.PrependReactor("get", "resourcequotas", func(ktesting.Action) (bool, runtime.Object, error) { reads++ if mode == "forbidden" { - return true, nil, apierrors.NewForbidden(schema.GroupResource{Resource: "resourcequotas"}, quota.Name, errors.New("denied")) + return true, nil, apierrors.NewForbidden(schema.GroupResource{Resource: "resourcequotas"}, quota.Name, errors.New(hostile)) + } + if mode == "api-error" { + return true, nil, apierrors.NewInternalError(errors.New(hostile)) } // Partially initialized status must not pass: all configured resources matter. quota.Status.Hard = quota.Spec.Hard.DeepCopy() @@ -86,6 +91,18 @@ func TestLegacyFixtureWaitsForQuotaAccountingBeforeCreate(t *testing.T) { if err == nil || creates != 0 { t.Fatalf("creates=%d error=%v", creates, err) } + if mode == "forbidden" || mode == "api-error" { + var reported strings.Builder + fmt.Fprintln(&reported, err) // Same reporting boundary as legacyfixture.fail. + for _, secret := range []string{"https://sentinel.invalid", "sk-fake-quota-secret"} { + if strings.Contains(err.Error(), secret) || strings.Contains(reported.String(), secret) { + t.Fatal("quota error disclosed API response data") + } + } + if reads != 1 { + t.Fatalf("API error retried: reads=%d", reads) + } + } switch mode { case "timeout": if strings.Contains(err.Error(), "private-quota-key") || strings.Contains(err.Error(), "12345") || !strings.Contains(err.Error(), "pods") { diff --git a/internal/adapter/executioncontroller/retained_delete_race_test.go b/internal/adapter/executioncontroller/retained_delete_race_test.go index 90bc1978bf..3be3ce5011 100644 --- a/internal/adapter/executioncontroller/retained_delete_race_test.go +++ b/internal/adapter/executioncontroller/retained_delete_race_test.go @@ -2,6 +2,7 @@ package executioncontroller import ( "errors" + "strconv" "testing" corev1 "k8s.io/api/core/v1" @@ -16,12 +17,35 @@ import ( ) func TestRetainedDeletePeerBetweenFinalizerUpdateAndDelete(t *testing.T) { - for _, terminating := range []bool{false, true} { - t.Run(map[bool]string{false: "peer-before-delete", true: "already-terminating"}[terminating], func(t *testing.T) { + for _, mode := range []string{"peer-before-delete", "already-terminating", "stale-resource-version"} { + t.Run(mode, func(t *testing.T) { env := lifecycleAdminEnvironment(2, []any{}) + env.SetResourceVersion("1") setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + // The tracker does not implement API-server resource versions or CAS. + versionedUpdate := func(a ktesting.Action) (bool, runtime.Object, error) { + obj, err := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if err != nil { + return true, nil, err + } + cur := obj.(*unstructured.Unstructured) + next := a.(ktesting.UpdateAction).GetObject().(*unstructured.Unstructured).DeepCopy() + version, err := strconv.Atoi(cur.GetResourceVersion()) + if err != nil || version < 1 || next.GetResourceVersion() == "" { + t.Fatal("fixture lost API-server resource version") + } + if next.GetResourceVersion() != cur.GetResourceVersion() { + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", errors.New("stale update")) + } + if !contains(cur.GetFinalizers(), environmentFinalizer) && contains(next.GetFinalizers(), environmentFinalizer) { + t.Fatal("peer re-added finalizer during admitted retained deletion") + } + next.SetResourceVersion(strconv.Itoa(version + 1)) + return true, next, d.Tracker().Update(ExecutionEnvironmentGVR, next, "ns") + } + d.PrependReactor("update", "executionenvironments", versionedUpdate) k := kubefake.NewSimpleClientset(retainedPVC(), &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{profileAllocationKey("go"): `["env"]`}}) store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) if err := store.DeleteRetiredEnvironment(t.Context(), adminRequestFixture()); err != nil { @@ -31,6 +55,7 @@ func TestRetainedDeletePeerBetweenFinalizerUpdateAndDelete(t *testing.T) { // reconcilers observe the same API-server tracker. peerClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) peerClient.PrependReactor("*", "*", ktesting.ObjectReaction(d.Tracker())) + peerClient.PrependReactor("update", "executionenvironments", versionedUpdate) peer := NewReconciler(peerClient, k, "ns", testProfiles()) r := NewReconciler(d, k, "ns", testProfiles()) t.Cleanup(peer.queue.ShutDown) @@ -44,20 +69,26 @@ func TestRetainedDeletePeerBetweenFinalizerUpdateAndDelete(t *testing.T) { pvcsDeleted++ return false, nil, nil }) + staleDeletes := 0 deletion := func(a ktesting.Action) (bool, runtime.Object, error) { + opts := a.(ktesting.DeleteAction).GetDeleteOptions() + if opts.Preconditions == nil || opts.Preconditions.ResourceVersion == nil || *opts.Preconditions.ResourceVersion == "" { + t.Fatal("CR deletion lost nonempty resource version") + } obj, err := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") if err != nil { return true, nil, err } cur := obj.(*unstructured.Unstructured) - opts := a.(ktesting.DeleteAction).GetDeleteOptions() - if opts.Preconditions == nil || opts.Preconditions.UID == nil || *opts.Preconditions.UID != cur.GetUID() { + if opts.Preconditions.UID == nil || *opts.Preconditions.UID != cur.GetUID() { t.Fatal("CR deletion lost exact UID") } + if *opts.Preconditions.ResourceVersion != cur.GetResourceVersion() { + staleDeletes++ + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", errors.New("stale delete")) + } if contains(cur.GetFinalizers(), environmentFinalizer) { - now := metav1.Now() - cur.SetDeletionTimestamp(&now) - return true, nil, d.Tracker().Update(ExecutionEnvironmentGVR, cur, "ns") + t.Fatal("admitted retained deletion re-added finalizer") } envsDeleted++ return true, nil, d.Tracker().Delete(ExecutionEnvironmentGVR, "ns", "env") @@ -67,26 +98,51 @@ func TestRetainedDeletePeerBetweenFinalizerUpdateAndDelete(t *testing.T) { d.PrependReactor("delete", "executionenvironments", func(a ktesting.Action) (bool, runtime.Object, error) { if !interleaved { interleaved = true - if err := peer.Reconcile(t.Context(), "env"); err != nil { + if mode == "stale-resource-version" { + res := peerClient.Resource(ExecutionEnvironmentGVR).Namespace("ns") + cur, err := res.Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + cur.SetAnnotations(map[string]string{"peer-observation": "updated"}) + if _, err := res.Update(t.Context(), cur, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + } else if err := peer.Reconcile(t.Context(), "env"); err != nil { t.Fatal(err) } } return deletion(a) }) - if terminating { + if mode == "already-terminating" { obj, _ := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") cur := obj.(*unstructured.Unstructured) now := metav1.Now() cur.SetDeletionTimestamp(&now) - if err := d.Tracker().Update(ExecutionEnvironmentGVR, cur, "ns"); err != nil { + if _, err := peerClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Update(t.Context(), cur, metav1.UpdateOptions{}); err != nil { t.Fatal(err) } } + conflicts := 0 for range 5 { if err := r.Reconcile(t.Context(), "env"); err != nil { - t.Fatal(err) + if mode != "stale-resource-version" || !apierrors.IsConflict(err) { + t.Fatal(err) + } + conflicts++ + cur, getErr := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if getErr != nil || contains(cur.(*unstructured.Unstructured).GetFinalizers(), environmentFinalizer) || envsDeleted != 0 { + t.Fatal("stale deletion must preserve CR without re-adding finalizer") + } } } + wantConflicts := 0 + if mode == "stale-resource-version" { + wantConflicts = 1 + } + if !interleaved || staleDeletes != wantConflicts || conflicts != wantConflicts { + t.Fatalf("interleaved=%v stale deletes=%d returned conflicts=%d", interleaved, staleDeletes, conflicts) + } if _, err := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env"); !apierrors.IsNotFound(err) { t.Fatalf("authorized deletion stranded: %v", err) } From 19097d3674dc1dc6c1b8d0ac82169705da9b9deb Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Tue, 22 Sep 2026 02:48:19 +0300 Subject: [PATCH 25/25] fix(execution): clear image tag for live digest upgrades Co-Authored-By: OpenAI --- deploy/mecatl-execution-kind/live.sh | 6 +-- deploy/mecatl-execution-kind/repair_test.go | 41 +++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/deploy/mecatl-execution-kind/live.sh b/deploy/mecatl-execution-kind/live.sh index 78c262cbfa..f769c23ca3 100755 --- a/deploy/mecatl-execution-kind/live.sh +++ b/deploy/mecatl-execution-kind/live.sh @@ -119,7 +119,7 @@ kube -n local-path-storage get configmap local-path-config -o json \ kube -n execution-qualification create configmap execution-mock --from-file=mock-script.json="$root/deploy/mecatl-execution-kind/mock-script.json" --dry-run=client -o yaml | kube apply -f - kube -n execution-qualification rollout status deployment/mecatl-execution --timeout=240s helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ - --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" --wait --timeout=5m + --set-string image.repository="${agent_image%@*}" --set-string image.tag= --set-string image.digest="${agent_image#*@}" --wait --timeout=5m kube -n execution-qualification rollout restart deployment/mecak8s kube -n execution-qualification rollout status deployment/mecak8s --timeout=240s dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_QUAL_STATE="$state" \ @@ -136,7 +136,7 @@ restore() { secret_cleaned=0 if [ "$restore_needed" -eq 1 ]; then if helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ - --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" --wait --timeout=5m >/dev/null 2>&1; then + --set-string image.repository="${agent_image%@*}" --set-string image.tag= --set-string image.digest="${agent_image#*@}" --wait --timeout=5m >/dev/null 2>&1; then mock_restored=1 else echo "mock profile restoration failed; cluster retained for inspection" >&2 @@ -170,7 +170,7 @@ dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEX go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/credentialloader stage "$kubeconfig" "$context" "$secret" "$receipt" restore_needed=1 helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ - --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" \ + --set-string image.repository="${agent_image%@*}" --set-string image.tag= --set-string image.digest="${agent_image#*@}" \ --set mockProvider=false --set security.allowUnsafeRealProvider=true --set defaultProvider=openrouter --set-string model=anthropic/claude-haiku-4.5 --set maxRunTokens=32000 \ --set-json 'extraArgs=["--no-soul","--no-user-model","--permissions-conventional=false","--agents-conventional=false"]' \ --set extraVolumes=null --set extraVolumeMounts=null \ diff --git a/deploy/mecatl-execution-kind/repair_test.go b/deploy/mecatl-execution-kind/repair_test.go index 486b797a6b..cffde32129 100644 --- a/deploy/mecatl-execution-kind/repair_test.go +++ b/deploy/mecatl-execution-kind/repair_test.go @@ -99,6 +99,47 @@ esac } } +func TestLiveHelmDigestExclusiveThroughRestoration(t *testing.T) { + live := scriptRange(t, "live.sh", "helm_kube upgrade --install", "\necho \"live qualification passed") + root := t.TempDir() + marker := filepath.Join(root, "helm-calls") + digest := "sha256:" + strings.Repeat("a", 64) + out, err := runStep(t, root, ` +set -u +kube() { :; } +dev() { :; } +helm_kube() { + test "$1 $2 $3" = 'upgrade --install mecak8s' || return 1 + # Model a nonempty inherited tag; apply the actual CLI overrides in order. + tag=e2e digest= repository= profile=mock + while [ "$#" -gt 0 ]; do + case "$1" in + --set-string|--set) + shift + case "$1" in + image.tag=*) tag=${1#*=} ;; + image.digest=*) digest=${1#*=} ;; + image.repository=*) repository=${1#*=} ;; + mockProvider=false) profile=live ;; + esac ;; + esac + shift + done + test -z "$tag" && test "$repository@$digest" = "$agent_image" || return 1 + printf '%s\n' "$profile" >> "$MARKER" +} +`+live, "root="+root, "state="+root, "MARKER="+marker, + "agent_image=ko.local/mecak8s@"+digest, "kubeconfig=synthetic", "context=synthetic", + "cluster=owned", "MECATL_EXECUTION_CREDENTIAL_FILE=unused") + if err != nil { + t.Fatalf("digest-only Helm lifecycle failed: %v: %s", err, out) + } + calls, err := os.ReadFile(marker) + if err != nil || string(calls) != "mock\nlive\nmock\n" { + t.Fatalf("expected digest-only mock setup, live upgrade, and mock restoration: %q: %v", calls, err) + } +} + func TestLiveSignalPreservesFailureAndAttemptsCleanup(t *testing.T) { restore := scriptRange(t, "live.sh", "restore() {", "\ndev env -i HOME=\"$HOME\" PATH=\"$PATH\" KUBECONFIG=\"$kubeconfig\" MECATL_KUBE_CONTEXT=\"$context\" MECATL_EXECUTION_CREDENTIAL_FILE=") for _, signal := range []struct {