From 358d1a2ffe22ca61f09efb68f16e95054b1b956e Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Wed, 14 Oct 2026 10:08:47 +0200 Subject: [PATCH 1/9] docs(design): sync simulate one-pager with merged upjet diff server Signed-off-by: Christopher Haar --- design/one-pager-cli-simulate.md | 162 +++++++++++++++++-------------- 1 file changed, 89 insertions(+), 73 deletions(-) diff --git a/design/one-pager-cli-simulate.md b/design/one-pager-cli-simulate.md index 6f662497..7eed119c 100644 --- a/design/one-pager-cli-simulate.md +++ b/design/one-pager-cli-simulate.md @@ -39,20 +39,20 @@ external resource to update it, so applying such a change does not recreate anything, it wedges. The provider refuses the update, the MR sits unsynced, and a human has to intervene. Replacement, computed values, provider defaults, and custom diff logic are provider knowledge. A companion one-pager in -crossplane/upjet proposes exposing that knowledge: a `PlanService` protocol -that upjet providers serve from an `internal plan-server` subcommand, computing -the provider's own Terraform diff from a desired resource and its live state, -statelessly, with no cloud credentials and no cluster access. What that proposal -deliberately leaves to the client is orchestration: something has to figure out -which providers a change involves, run their plan servers, route resources to -them, and render the results. +crossplane/upjet proposed exposing that knowledge, and upjet has since merged +it: a `PlanService` protocol that upjet providers serve from an `internal +diff-server` subcommand, computing the provider's own Terraform diff from a +desired resource and its live state, statelessly, with no cloud credentials +and no cluster access. What that protocol deliberately leaves to the client is +orchestration: something has to figure out which providers a change involves, +run their diff servers, route resources to them, and render the results. The CLI is that client. It already does exactly this dance for functions: `crossplane composition render` pulls function packages, runs them as local Docker containers bound to ephemeral localhost ports, and drives them over gRPC. And with the project workflow now in the CLI (`crossplane project init|build|run`), the CLI knows a project's provider dependencies, so it -knows which plan servers to run without being told. +knows which diff servers to run without being told. ## Goals @@ -113,14 +113,14 @@ A Crossplane provider will do neither, so replaces get their own count instead of inflating actions that will not happen. For each input resource the CLI fetches the live MR from the cluster if one -exists (its `status.atProvider` and external-name annotation are the prior -state), sends desired and live to the routed plan server, and renders the -response. A resource with no live counterpart plans as a create "plan +exists, sends the desired resource and the whole live one to the routed diff +server (the live MR's `status.atProvider` and external-name annotation ride +along as the actual state), and renders the response. A resource with no live counterpart plans as a create "plan before apply". Which providers answer those plans comes from the current kubeconfig context by default: the CLI lists the providers installed on the target -cluster and runs their package images as plan servers, so the preview uses +cluster and runs their package images as diff servers, so the preview uses the provider versions the cluster actually runs. `--provider-images` is optional and overrides that discovery for the cases where the cluster is not the source of truth, most importantly the provider upgrade preview @@ -130,8 +130,8 @@ yet. ### Previewing a Provider Upgrade Provider upgrades fall out of the same command with no new machinery, -because the plan server *is* the provider version being asked about. To -preview an upgrade, run the new version's image as the plan server and +because the diff server *is* the provider version being asked about. To +preview an upgrade, run the new version's image as the diff server and hand it the fleet: ```console @@ -145,18 +145,19 @@ Plan: 0 to add, 3 to change, 0 to destroy, 1 requires replacement. 210 resources are unchanged under the new provider version. ``` -With `--all` the CLI enumerates instead of reading files: it asks the plan -server which API groups it serves (`GetInfo` again), lists every managed -resource of those kinds on the cluster, and sends each one with its own +With `--all` the CLI enumerates instead of reading files: it reads the +provider package's CRDs to learn which groups and kinds the new version +serves, lists every managed resource of those kinds on the cluster, and +sends each one with its own spec as desired state and its own live resource as prior state. The question each plan answers is "does the new provider version consider this resource up to date, exactly as it stands?". Anything that is not a no-op is a finding: a changed default, a schema migration, a new replacement -rule, a validation the next reconcile would trip over. Replaces and new -error diagnostics are the highest signal, because those are the resources -that wedge right after the upgrade. +rule, a validation the next reconcile would trip over. Replaces and +resources the new version refuses to plan are the highest signal, because +those are the resources that wedge right after the upgrade. -Because the plan server never refreshes, previewing the whole fleet costs +Because the diff server never refreshes, previewing the whole fleet costs zero cloud API calls and needs no cloud credentials: the CLI reads the cluster with the user's kubeconfig and everything else happens locally. That makes "plan the fleet before every provider bump" cheap enough to run @@ -184,7 +185,7 @@ The flow: each Composition composes. `--namespace` and `--name` scope the run. 2. **Prepare.** Build the project's embedded functions locally, exactly as `project run` does, so the simulation reflects the code in the working - tree. Resolve the project's provider dependencies and start their plan + tree. Resolve the project's provider dependencies and start their diff servers (below). 3. **Render.** For each XR, fetch its current composed resources from the cluster and re-render it with the *local* Composition and functions, @@ -204,7 +205,7 @@ The flow: detection so resources the new Composition no longer produces show as deletes. 5. **Plan the cloud layer.** For every rendered MR, merge in the live MR's - external-name and status, route it to a plan server by API group, and get + external-name and status, route it to a diff server by group and kind, and get the provider's verdict: no-op, update, or the replace nobody spotted in review. 6. **Report.** Per-XR diffs with per-resource actions, a @@ -280,10 +281,10 @@ type simulateCmd struct { Input []string `short:"f" help:"Project file (default), or changed Composition or XR files."` Namespace string `help:"Simulate only XRs in this namespace."` Name string `help:"Simulate only the XR with this name."` - ProviderImages []string `help:"Override the provider images to run as plan servers."` + ProviderImages []string `help:"Override the provider images to run as diff servers."` IncludePinned bool `help:"Also simulate XRs pinned to older composition revisions."` SkipPlan bool `help:"Skip the cloud-layer plan; show only the cluster-layer diff."` - KeepPlanServers bool `help:"Leave plan server containers running for reuse by later runs."` + KeepDiffServers bool `help:"Leave diff server containers running for reuse by later runs."` MaxIterations uint `default:"10" help:"Cap on requirement resolution passes per XR."` Timeout time.Duration `default:"2m" help:"Per XR render and plan timeout."` MaxConcurrency uint `default:"8" help:"XRs simulated concurrently."` @@ -291,12 +292,12 @@ type simulateCmd struct { } ``` -`crossplane resource simulate` shares the plan server, timeout, and output +`crossplane resource simulate` shares the diff server, timeout, and output flags, and takes its resources from `-f` files instead of project discovery. Both commands enter the tree with the CLI's `maturity:"alpha"` tag and graduate the way every other command does. -### Running Plan Servers Like Render Runs Functions +### Running Diff Servers Like Render Runs Functions The CLI treats provider packages the way render treats function packages: as images it can run locally and talk to over gRPC. @@ -305,37 +306,44 @@ The provider images come from the project's `dependsOn` for `project simulate`, from the providers installed on the current cluster context for `resource simulate`, or from an explicit `--provider-images` override of either. The CLI starts each provider -image with its `internal plan-server` entrypoint as a local Docker +image with its `internal diff-server` entrypoint as a local Docker container, following the same runtime conventions render established for -functions (pull policy, cleanup on exit). How the CLI talks to that -container, a gRPC connection on an ephemeral localhost port or a protobuf -exchange over the container's stdin and stdout, is part of the plan service -contract, so the upjet one-pager decides it and this document follows. The -exchange is the same either way: the CLI first asks each server `GetInfo`, -which returns the API groups the server can plan (`s3.aws.upbound.io`, -`kms.aws.upbound.io`, ...), and builds a routing table from the answer. -Routing by declared capability rather than image-name convention means -family providers coexist and a resource with no route is detected up -front, not by a failed call. +functions (pull policy, cleanup on exit), and talks to it over plaintext +gRPC on an ephemeral localhost port, as upjet's merged protocol defines +(the server can also bind a unix socket, and registers gRPC reflection, +which doubles as the readiness probe). What a server can plan is declared +before any container starts: a provider package that serves diffs carries +the `DiffServer` capability in its package metadata (`spec.capabilities`), +and the CRDs inside the package say exactly which groups and kinds that +provider serves. The CLI checks the capability first, starts containers +only for packages that have it, and builds its routing table from the +packages' CRDs, by group and kind. Routing by declared package contents +rather than image-name convention means family providers coexist and a +resource with no route is detected up front, not by a failed call. The +protocol is alpha, merged to upjet's main branch ahead of a release, and +simulate enters the CLI as alpha alongside it. That split is what makes the `--provider-images` override simple: the flag only changes where the image list comes from, never how routing works. The -CLI never needs to know up front which image serves which API group, not -from a registry, not from image names, not from cluster state. It starts -whatever images the list contains and lets each server declare its groups -through `GetInfo`. Override the list with a new provider version and the -routing table rebuilds itself from what that version actually serves. - -Because plan servers are stateless and credential-free, all state travels +CLI never hardcodes which image serves which group and kind, not from +image names, not from cluster state. It inspects whatever images the list +contains and lets each package declare its capability and its kinds +through its own metadata. Override the list with a new provider version +and the routing table rebuilds itself from what that version actually +ships. + +Because diff servers are stateless and credential-free, all state travels in the request. There is nothing to configure: no ProviderConfig, no cloud -credentials, no reach into the cluster from the container. The -CLI fetches live state with the user's own kubeconfig and RBAC; the plan -server only ever sees what the user could already read. +credentials, no reach into the cluster from the container. The CLI fetches +live state, and the Secrets or referenced objects a plan may need to +resolve, with the user's own kubeconfig and RBAC, and sends them along in +the request; the diff server only ever sees what the user could already +read. -Startup is the one cost that hurts across runs: a plan server takes a few +Startup is the one cost that hurts across runs: a diff server takes a few seconds to come up, and an edit-simulate-edit loop pays that on every run if containers are torn down at exit. So containers get deterministic -names derived from the provider image digest, and `--keep-plan-servers` +names derived from the provider image digest, and `--keep-diff-servers` leaves them running when the command exits: the next run finds a live server with a matching name and reuses it instead of starting a new one. Because the name carries the digest, a changed image never matches a @@ -347,12 +355,17 @@ but that belongs to render. ### From Response to Printed Diff -What a plan server returns is deliberately close to what the CLI prints. -Every response carries the action, the field changes with current and -desired values, replacement information, and provider diagnostics, and -every field path arrives as a CRD path -`spec.forProvider.deletionWindowInDays`. -The server also redacts sensitive values and marks computed ones. +What a diff server returns is deliberately close to what the CLI prints. +Every response carries the action and the field changes with current and +planned values, each change marked with whether it forces replacement and +whether it originates from the desired state or from the provider +(defaults and late-initialized values), and every field path arrives as a +CRD path `spec.forProvider.deletionWindowInDays`. The server also redacts +sensitive values, marks computed ones as known only after apply, and marks +values it cannot resolve, an unresolved reference or Secret, rather than +guessing. Failures are not part of the response: a server that cannot plan +a resource says so through its error status, and simulate turns that into +the approximate fallback below. Before sending, the CLI has to answer which live resource a rendered one corresponds to, and names are not enough: a re-render can generate fresh @@ -369,9 +382,9 @@ plan as a delete instead of silently disappearing from the preview. Printing then follows one rule: show users their own manifest, annotated. Each resource prints as YAML in CRD terms with changed fields marked inline (`~ field: "old" -> "new"`), sensitive values stay redacted, computed -values print as `(known after apply)`, and fields that force replacement -carry that warning on the line where the user's eye already is. Provider -diagnostics print under the resource they belong to. Actions map to the +values print as `(known after apply)`, unresolved references print as +`(unresolved reference)`, and fields that force replacement carry that +warning on the line where the user's eye already is. Actions map to the summary symbols (`[+]` create, `[~]` update, `[-/+]` replace, `[-]` delete, `[=]` no-op, `[?]` approximate fallback). @@ -381,25 +394,28 @@ the consumer, not the tool: `-o diff` prints the same result as a line-by-line unified diff (`- foo: bar` / `+ foo: baz`), the format crossplane-diff emits today, which renders directly in Markdown `diff` code blocks for CI comments. What only the plan layer knows (replacement, -known-after-apply, diagnostics) prints in the per-resource header and the +known-after-apply values) prints in the per-resource header and the summary in diff mode, since those have no inline line to attach to. The same data serializes as `-o json|yaml`, one typed result per resource, so a pull-request bot can gate on "any replace anywhere" without parsing console output. -### When There Is No Plan Server +### When There Is No Diff Server Not every provider is an upjet provider, and not every upjet provider will serve plans on day one. Simulate treats planning as a capability it detects, not a requirement: -* If a resource routes to a plan server, its diff is provider-computed and +* If a resource routes to a diff server, its diff is provider-computed and accurate: replacement detection, known-after-apply values, sensitive-value redaction. -* If it does not, the CLI falls back to a client-side comparison of desired - against live spec, rendered with a distinct marker (`[?]`) and a closing - warning that the result is approximate: no replacement detection, no - computed values. +* A routed server can still refuse an individual resource it cannot compute + a diff for; the server says so explicitly in its error status, and the + resource takes the fallback below instead of failing the run. +* If a resource routes nowhere, or its server refused it, the CLI falls + back to a client-side comparison of desired against live spec, rendered + with a distinct marker (`[?]`) and a closing warning that the result is + approximate: no replacement detection, no computed values. A mixed control plane gets a complete preview that is honest about which parts are provider-grade, and every provider that adopts the plan protocol @@ -453,11 +469,11 @@ standalone packaging forced them to build. and `--name` to scope the cluster side), and both gain the plan layer as their upgrade. That upgrade is strictly additive, so today's diff functionality is subsumed rather than replaced: planning is a capability - simulate detects per resource, and a resource with no plan server gets + simulate detects per resource, and a resource with no diff server gets the client-side diff crossplane-diff computes today ("When There Is No - Plan Server" above). Run simulate where no provider serves plans and the + Diff Server" above). Run simulate where no provider serves plans and the output is today's diff, unchanged. For users who want only the - cluster-layer diff, `--skip-plan` makes that explicit: no plan servers + cluster-layer diff, `--skip-plan` makes that explicit: no diff servers start, no containers run, and the command does exactly what diff does today. That keeps "diff without simulation" one flag away instead of a second command. Whether thin aliases remain for existing crossplane-diff @@ -483,11 +499,11 @@ of the control plane. Per XR: the same render work `crossplane render` does (plus requirement-resolution iterations, capped), one server-side dry-run per composed resource, and one plan computation per rendered MR, each of which is a pure in-process schema diff on the server, no cloud calls. -(Whether the protocol carries those computations one per request or -batched is a plan protocol question, decided in the upjet one-pager.) -Plan-server startup is the dominant fixed cost, a few seconds per distinct +The protocol carries one resource per unary request; a fleet preview is +many small requests, each cheap. +Diff-server startup is the dominant fixed cost, a few seconds per distinct provider image, paid once per run and torn down after, or paid once across -many runs with `--keep-plan-servers`. Nothing simulate does +many runs with `--keep-diff-servers`. Nothing simulate does writes to the cluster or the cloud: reads plus dry-runs on one side, a credential-free diff on the other. From 0d81d40676103dd55e380e752c52e3ae6cf2bdc8 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Wed, 14 Oct 2026 19:47:17 +0200 Subject: [PATCH 2/9] feat(resource): add resource simulate as alpha command Signed-off-by: Christopher Haar --- cmd/crossplane/resource/resource.go | 2 + cmd/crossplane/simulate/help/simulate.md | 48 +++ cmd/crossplane/simulate/simulate.go | 350 +++++++++++++++++++++ cmd/crossplane/simulate/simulate_test.go | 218 +++++++++++++ flake.lock | 6 +- go.mod | 11 +- go.sum | 14 +- internal/diffserver/manager.go | 378 +++++++++++++++++++++++ internal/diffserver/manager_test.go | 340 ++++++++++++++++++++ internal/docker/docker.go | 50 +++ internal/plan/fallback.go | 157 ++++++++++ internal/plan/fallback_test.go | 184 +++++++++++ internal/plan/output.go | 317 +++++++++++++++++++ internal/plan/output_test.go | 152 +++++++++ internal/plan/plan.go | 64 ++++ internal/plan/plan_test.go | 97 ++++++ internal/plan/proto.go | 126 ++++++++ internal/plan/proto_test.go | 233 ++++++++++++++ internal/plan/types.go | 125 ++++++++ internal/terminal/spinner.go | 37 ++- internal/xpkg/metadata.go | 17 + nix/vendor-hashes.nix | 2 +- 22 files changed, 2911 insertions(+), 17 deletions(-) create mode 100644 cmd/crossplane/simulate/help/simulate.md create mode 100644 cmd/crossplane/simulate/simulate.go create mode 100644 cmd/crossplane/simulate/simulate_test.go create mode 100644 internal/diffserver/manager.go create mode 100644 internal/diffserver/manager_test.go create mode 100644 internal/plan/fallback.go create mode 100644 internal/plan/fallback_test.go create mode 100644 internal/plan/output.go create mode 100644 internal/plan/output_test.go create mode 100644 internal/plan/plan.go create mode 100644 internal/plan/plan_test.go create mode 100644 internal/plan/proto.go create mode 100644 internal/plan/proto_test.go create mode 100644 internal/plan/types.go diff --git a/cmd/crossplane/resource/resource.go b/cmd/crossplane/resource/resource.go index c930152e..5186ad61 100644 --- a/cmd/crossplane/resource/resource.go +++ b/cmd/crossplane/resource/resource.go @@ -18,12 +18,14 @@ limitations under the License. package resource import ( + "github.com/crossplane/cli/v2/cmd/crossplane/simulate" "github.com/crossplane/cli/v2/cmd/crossplane/trace" "github.com/crossplane/cli/v2/cmd/crossplane/validate" ) // Cmd contains commands for working with Crossplane resources. type Cmd struct { + Simulate simulate.Cmd `cmd:"" help:"Preview what applying managed resources would do." maturity:"alpha"` Trace trace.Cmd `cmd:"" help:"Trace a Crossplane resource for troubleshooting."` Validate validate.Cmd `cmd:"" help:"Validate Crossplane resources."` } diff --git a/cmd/crossplane/simulate/help/simulate.md b/cmd/crossplane/simulate/help/simulate.md new file mode 100644 index 00000000..0f9f328b --- /dev/null +++ b/cmd/crossplane/simulate/help/simulate.md @@ -0,0 +1,48 @@ +This command previews what applying one or more managed resources would do, +without applying anything: which resources it would create, which fields it +would change, and, most importantly, which changes the provider could only +apply by destroying and recreating the external resource. + +For each input resource the CLI fetches the live managed resource from the +cluster, if one exists, and asks the provider itself for the verdict: it runs +the provider's package image locally as a diff server (a stateless, +credential-free gRPC service upjet-based providers serve) and sends it the +desired and live state. The provider computes its own diff, so the preview +includes replacement detection, values known only after apply, and redacted +sensitive values. A resource with no live counterpart plans as a create. + +Which providers run as diff servers comes from the current kubeconfig context +by default: the CLI lists the providers installed on the cluster and uses +their package images, so the preview matches the provider versions the +cluster actually runs. --provider-images overrides that discovery, for +example to preview how a provider version that is not installed yet would +treat existing resources. + +Not every provider serves diffs. A provider package declares the capability +in its metadata, and a resource whose provider serves no diff server (or +whose diff server cannot compute a diff for it) still appears in the preview +with an approximate client-side comparison, marked [?], rather than failing +the run or silently dropping out. + +Nothing simulate does writes to the cluster or the cloud: it reads the +cluster with your kubeconfig and RBAC, and the diff servers run locally with +no credentials and no cluster access. + +Diff server startup takes a few seconds per provider image. In an +edit-simulate-edit loop, --keep-diff-servers leaves the containers running so +the next run reuses them; they are named `crossplane-diff-server-DIGEST` and +are cleaned up by any later run without the flag, or manually with docker rm. + +Examples: + + # Preview what applying resources.yaml would do. + crossplane resource simulate -f resources.yaml + + # Preview against a specific provider version instead of the installed one. + crossplane resource simulate -f resources.yaml --provider-images xpkg.upbound.io/upbound/provider-aws-ec2:v2.9.0 + + # Keep diff servers running for faster subsequent runs. + crossplane resource simulate -f resources.yaml --keep-diff-servers + + # Emit machine-readable results for CI gating. + crossplane resource simulate -f resources.yaml -o json diff --git a/cmd/crossplane/simulate/simulate.go b/cmd/crossplane/simulate/simulate.go new file mode 100644 index 00000000..69ecbb26 --- /dev/null +++ b/cmd/crossplane/simulate/simulate.go @@ -0,0 +1,350 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package simulate contains the resource simulate command: preview what +// applying managed resources would do, to the cluster and to the cloud +// behind it, by asking the providers' own diff servers. +package simulate + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/alecthomas/kong" + diffv1alpha1 "github.com/crossplane/upjet/v2/proto/diff/v1alpha1" + "github.com/spf13/afero" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + "github.com/crossplane/crossplane-runtime/v2/pkg/logging" + + "github.com/crossplane/crossplane/apis/v2/pkg" + pkgv1 "github.com/crossplane/crossplane/apis/v2/pkg/v1" + + "github.com/crossplane/cli/v2/cmd/crossplane/common/load" + "github.com/crossplane/cli/v2/internal/diffserver" + "github.com/crossplane/cli/v2/internal/docker" + "github.com/crossplane/cli/v2/internal/kube" + "github.com/crossplane/cli/v2/internal/plan" + "github.com/crossplane/cli/v2/internal/terminal" + clixpkg "github.com/crossplane/cli/v2/internal/xpkg" + + _ "embed" +) + +//go:embed help/simulate.md +var helpDetail string + +// Cmd previews what applying managed resources would do. +type Cmd struct { + Filename string `help:"YAML file(s) or directory with managed resources, or '-' for stdin (comma-separated)." predictor:"yaml_file_or_directory" required:"" short:"f"` + + ProviderImages []string `help:"Override the provider images to run as diff servers. Defaults to the providers installed on the cluster." name:"provider-images"` + KeepDiffServers bool `help:"Leave diff server containers running for reuse by later runs." name:"keep-diff-servers"` + Context string `default:"" help:"Kubernetes context." name:"context" predictor:"context" short:"c"` + CacheDir string `default:"~/.crossplane/cache" help:"Cache directory for downloaded provider packages." name:"cache-dir" predictor:"directory"` + Timeout time.Duration `default:"2m" help:"Per-resource plan timeout."` + Output string `default:"plan" enum:"plan,json,yaml" help:"Output format. One of: plan, json, yaml." name:"output" short:"o"` + + Impersonation kube.ImpersonationFlags `embed:""` +} + +// Help returns the help message for the simulate command. +func (c *Cmd) Help() string { + return helpDetail +} + +// AfterApply expands the cache directory. +func (c *Cmd) AfterApply() error { + if strings.HasPrefix(c.CacheDir, "~/") { + homeDir, err := os.UserHomeDir() + if err != nil { + return errors.Wrap(err, "cannot determine home directory for cache dir") + } + c.CacheDir = filepath.Join(homeDir, c.CacheDir[2:]) + } + return nil +} + +// Run runs the simulate command. +func (c *Cmd) Run(k *kong.Context, logger logging.Logger, sp terminal.SpinnerPrinter) error { + ctx := context.Background() + + loader, err := load.NewLoader(c.Filename) + if err != nil { + return errors.Wrapf(err, "cannot load resources from %q", c.Filename) + } + resources, err := loader.Load() + if err != nil { + return errors.Wrapf(err, "cannot load resources from %q", c.Filename) + } + if len(resources) == 0 { + return errors.Errorf("no resources found in %q", c.Filename) + } + + if err := docker.Check(ctx); err != nil { + return errors.Wrap(err, "cannot connect to Docker, which simulate needs to run provider diff servers") + } + + kubeClient, rmapper, defaultNamespace, err := c.setupKubeClient(logger) + if err != nil { + return err + } + + images := c.ProviderImages + if len(images) == 0 { + if images, err = installedProviderImages(ctx, kubeClient); err != nil { + return errors.Wrap(err, "cannot discover providers installed on the cluster") + } + } + if len(images) == 0 { + _, _ = fmt.Fprintln(k.Stderr, "No providers found to run as diff servers; all results will be approximate.") + } + + pkgClient, err := clixpkg.NewClient(clixpkg.NewRemoteFetcher(), clixpkg.WithCacheDir(afero.NewOsFs(), c.CacheDir)) + if err != nil { + return errors.Wrap(err, "cannot create package client") + } + + mgr := diffserver.NewManager(pkgClient, logger, diffserver.WithKeepServers(c.KeepDiffServers)) + var skipped []diffserver.Skipped + if err := sp.WrapWithSuccessSpinner("Starting provider diff servers", func() error { + var err error + skipped, err = mgr.Start(ctx, images) + return err + }); err != nil { + return errors.Wrap(err, "cannot start diff servers") + } + defer func() { + if err := mgr.Stop(context.Background()); err != nil { + _, _ = fmt.Fprintf(k.Stderr, "Warning: cannot clean up diff servers: %s\n", err) + } + }() + for _, s := range skipped { + _, _ = fmt.Fprintf(k.Stderr, "Not running a diff server for %s: %s\n", s.Image, s.Reason) + } + + plans := make([]*plan.ResourcePlan, 0, len(resources)) + for _, desired := range resources { + plans = append(plans, c.simulateOne(ctx, kubeClient, rmapper, defaultNamespace, mgr, desired)) + } + + return plan.Render(k.Stdout, c.Output, plans) +} + +// router routes a resource's group and kind to a diff server client. It is +// the slice of diffserver.Manager the per-resource flow needs, extracted so +// tests can fake routing. +type router interface { + Route(gk schema.GroupKind) (diffv1alpha1.PlanServiceClient, bool) +} + +// simulateOne plans a single resource: fetch its live counterpart, route it +// to a diff server, and fall back to the approximate client-side diff when no +// server can plan it. Failures become per-resource errors so one resource +// never fails the whole run. +func (c *Cmd) simulateOne(ctx context.Context, kubeClient client.Client, rmapper meta.RESTMapper, defaultNamespace string, mgr router, desired *unstructured.Unstructured) *plan.ResourcePlan { + if desired.GetName() == "" { + p := plan.NewResourcePlan(desired) + p.Error = "resource has no name" + return p + } + + live, err := fetchLive(ctx, kubeClient, rmapper, defaultNamespace, desired) + if err != nil { + p := plan.NewResourcePlan(desired) + p.Error = fmt.Sprintf("cannot fetch live resource: %s", err) + return p + } + + gk := desired.GroupVersionKind().GroupKind() + planClient, ok := mgr.Route(gk) + if !ok { + return plan.Fallback(desired, live, fmt.Sprintf("no diff server for %s", gk)) + } + + pctx, cancel := context.WithTimeout(ctx, c.Timeout) + defer cancel() + + store := objectStore(ctx, kubeClient, rmapper, defaultNamespace, desired) + + p, err := plan.Plan(pctx, planClient, desired, live, store) + if err != nil { + if plan.IsNotSupported(err) { + return plan.Fallback(desired, live, "the provider cannot compute a diff for this resource") + } + p := plan.NewResourcePlan(desired) + p.Error = fmt.Sprintf("diff server cannot plan: %s", err) + return p + } + return p +} + +// fetchLive returns the live counterpart of the desired resource, or nil when +// it does not exist (including when its kind is not installed on the +// cluster), in which case the resource plans as a create. +func fetchLive(ctx context.Context, kubeClient client.Client, rmapper meta.RESTMapper, defaultNamespace string, desired *unstructured.Unstructured) (*unstructured.Unstructured, error) { + gvk := desired.GroupVersionKind() + + mapping, err := rmapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + // The kind is not served by the cluster: nothing can exist yet. The + // diff server can still plan the resource as a create. + return nil, nil //nolint:nilerr // Intentional: an unserved kind means no live counterpart, not a failure. + } + + key := client.ObjectKey{Name: desired.GetName()} + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + key.Namespace = desired.GetNamespace() + if key.Namespace == "" { + key.Namespace = defaultNamespace + } + } + + live := &unstructured.Unstructured{} + live.SetGroupVersionKind(gvk) + if err := kubeClient.Get(ctx, key, live); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return live, nil +} + +// objectStore gathers the cluster objects the diff server needs to resolve +// the desired resource: today the ProviderConfig it references. The diff +// server is offline and only sees what travels in the request, so anything it +// must resolve is fetched here, with the user's own kubeconfig and RBAC. The +// ProviderConfig's credentials Secret deliberately stays out: the server runs +// offline with placeholder credentials and never needs the real ones. +// Gathering is best effort: a missing object simply stays out of the store, +// and the server reports precisely what it could not resolve. +func objectStore(ctx context.Context, kubeClient client.Client, rmapper meta.RESTMapper, defaultNamespace string, desired *unstructured.Unstructured) []*unstructured.Unstructured { + name, _, _ := unstructured.NestedString(desired.Object, "spec", "providerConfigRef", "name") + if name == "" { + name = "default" + } + kind, _, _ := unstructured.NestedString(desired.Object, "spec", "providerConfigRef", "kind") + if kind == "" { + kind = "ProviderConfig" + } + + // The provider config lives in the provider family's group, one label + // above the resource's group: ec2.aws.m.upbound.io -> aws.m.upbound.io. + group := desired.GroupVersionKind().Group + i := strings.Index(group, ".") + if i < 0 { + return nil + } + group = group[i+1:] + + mapping, err := rmapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}) + if err != nil { + return nil + } + + pc := &unstructured.Unstructured{} + pc.SetGroupVersionKind(mapping.GroupVersionKind) + key := client.ObjectKey{Name: name} + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + key.Namespace = desired.GetNamespace() + if key.Namespace == "" { + key.Namespace = defaultNamespace + } + } + if err := kubeClient.Get(ctx, key, pc); err != nil { + return nil + } + return []*unstructured.Unstructured{pc} +} + +// installedProviderImages lists the package images of the providers installed +// on the cluster, preferring the resolved identifier over the possibly +// constraint-form spec.package. +func installedProviderImages(ctx context.Context, kubeClient client.Client) ([]string, error) { + providers := &pkgv1.ProviderList{} + if err := kubeClient.List(ctx, providers); err != nil { + return nil, err + } + images := make([]string, 0, len(providers.Items)) + for _, p := range providers.Items { + img := p.Status.CurrentIdentifier + if img == "" { + img = p.Spec.Package + } + if img != "" { + images = append(images, img) + } + } + return images, nil +} + +func (c *Cmd) setupKubeClient(logger logging.Logger) (client.Client, meta.RESTMapper, string, error) { + clientconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), + &clientcmd.ConfigOverrides{CurrentContext: c.Context}, + ) + + kubeconfig, err := clientconfig.ClientConfig() + if err != nil { + return nil, nil, "", errors.Wrap(err, "failed to get kubeconfig") + } + + c.Impersonation.Apply(kubeconfig) + + if kubeconfig.QPS == 0 { + kubeconfig.QPS = 20 + } + if kubeconfig.Burst == 0 { + kubeconfig.Burst = 30 + } + + logger.Debug("Found kubeconfig") + + cl, err := client.New(kubeconfig, client.Options{Scheme: scheme.Scheme}) + if err != nil { + return nil, nil, "", errors.Wrap(err, "cannot init kubeclient") + } + _ = pkg.AddToScheme(cl.Scheme()) + + discoveryClient, err := discovery.NewDiscoveryClientForConfig(kubeconfig) + if err != nil { + return nil, nil, "", errors.Wrap(err, "cannot get discovery client") + } + d := memory.NewMemCacheClient(discoveryClient) + rmapper := restmapper.NewShortcutExpander(restmapper.NewDeferredDiscoveryRESTMapper(d), d, nil) + + namespace, _, err := clientconfig.Namespace() + if err != nil { + return nil, nil, "", errors.Wrap(err, "failed to get namespace from kubeconfig") + } + + return cl, rmapper, namespace, nil +} diff --git a/cmd/crossplane/simulate/simulate_test.go b/cmd/crossplane/simulate/simulate_test.go new file mode 100644 index 00000000..bac37614 --- /dev/null +++ b/cmd/crossplane/simulate/simulate_test.go @@ -0,0 +1,218 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package simulate + +import ( + "context" + "testing" + "time" + + diffv1alpha1 "github.com/crossplane/upjet/v2/proto/diff/v1alpha1" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/crossplane/cli/v2/internal/plan" +) + +var vpcGVK = schema.GroupVersionKind{Group: "ec2.aws.upbound.io", Version: "v1beta1", Kind: "VPC"} + +func vpc(name, cidr string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "ec2.aws.upbound.io/v1beta1", + "kind": "VPC", + "metadata": map[string]any{"name": name}, + "spec": map[string]any{"forProvider": map[string]any{"cidrBlock": cidr}}, + }} +} + +// fakeRouter routes every resource to the given client, or nowhere. +type fakeRouter struct { + client diffv1alpha1.PlanServiceClient +} + +func (f *fakeRouter) Route(schema.GroupKind) (diffv1alpha1.PlanServiceClient, bool) { + if f.client == nil { + return nil, false + } + return f.client, true +} + +// fakePlanClient answers Plan in-process. +type fakePlanClient struct { + rsp *diffv1alpha1.PlanResponse + err error + got *diffv1alpha1.PlanRequest +} + +func (f *fakePlanClient) Plan(_ context.Context, in *diffv1alpha1.PlanRequest, _ ...grpc.CallOption) (*diffv1alpha1.PlanResponse, error) { + f.got = in + return f.rsp, f.err +} + +func notSupportedErr(t *testing.T) error { + t.Helper() + st, err := status.New(codes.FailedPrecondition, "cannot compute diff").WithDetails(&errdetails.PreconditionFailure{ + Violations: []*errdetails.PreconditionFailure_Violation{{Type: "DIFF_COMPUTATION_NOT_SUPPORTED"}}, + }) + if err != nil { + t.Fatalf("cannot build status: %v", err) + } + return st.Err() +} + +func TestSimulateOne(t *testing.T) { + ctx := context.Background() + + s := runtime.NewScheme() + s.AddKnownTypeWithName(vpcGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(vpcGVK.GroupVersion().WithKind("VPCList"), &unstructured.UnstructuredList{}) + + rm := meta.NewDefaultRESTMapper([]schema.GroupVersion{vpcGVK.GroupVersion()}) + rm.Add(vpcGVK, meta.RESTScopeRoot) + + cmd := &Cmd{Timeout: time.Second} + + t.Run("RoutedUpdate", func(t *testing.T) { + live := vpc("my-vpc", "10.0.0.0/16") + kube := fake.NewClientBuilder().WithScheme(s).WithObjects(live).Build() + pc := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_UPDATE}} + + got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, vpc("my-vpc", "10.1.0.0/16")) + + if got.Action != plan.ActionUpdate || got.Approximate || got.Error != "" { + t.Errorf("simulateOne(...): want a provider-computed update, got %+v", got) + } + if pc.got.GetActualResource() == nil { + t.Errorf("simulateOne(...): want the live resource sent as actual state") + } + }) + + t.Run("NoLiveCounterpartPlansCreate", func(t *testing.T) { + kube := fake.NewClientBuilder().WithScheme(s).Build() + pc := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_CREATE}} + + got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, vpc("new-vpc", "10.0.0.0/16")) + + if got.Action != plan.ActionCreate || got.Error != "" { + t.Errorf("simulateOne(...): want a create, got %+v", got) + } + if pc.got.GetActualResource() != nil { + t.Errorf("simulateOne(...): want no actual state for a missing live resource") + } + }) + + t.Run("NoRouteFallsBack", func(t *testing.T) { + live := vpc("my-vpc", "10.0.0.0/16") + kube := fake.NewClientBuilder().WithScheme(s).WithObjects(live).Build() + + got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{}, vpc("my-vpc", "10.1.0.0/16")) + + if !got.Approximate || got.Action != plan.ActionUpdate { + t.Errorf("simulateOne(...): want an approximate update fallback, got %+v", got) + } + }) + + t.Run("NotSupportedFallsBack", func(t *testing.T) { + live := vpc("my-vpc", "10.0.0.0/16") + kube := fake.NewClientBuilder().WithScheme(s).WithObjects(live).Build() + pc := &fakePlanClient{err: notSupportedErr(t)} + + got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, vpc("my-vpc", "10.1.0.0/16")) + + if !got.Approximate || got.Action != plan.ActionUpdate { + t.Errorf("simulateOne(...): want an approximate fallback on DIFF_COMPUTATION_NOT_SUPPORTED, got %+v", got) + } + }) + + t.Run("ServerErrorBecomesResourceError", func(t *testing.T) { + kube := fake.NewClientBuilder().WithScheme(s).Build() + pc := &fakePlanClient{err: status.Error(codes.Internal, "boom")} + + got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, vpc("my-vpc", "10.1.0.0/16")) + + if got.Error == "" { + t.Errorf("simulateOne(...): want a per-resource error, got %+v", got) + } + }) + + t.Run("SeedsProviderConfigIntoObjectStore", func(t *testing.T) { + pcGVK := schema.GroupVersionKind{Group: "aws.upbound.io", Version: "v1beta1", Kind: "ProviderConfig"} + secretGVK := schema.GroupVersionKind{Version: "v1", Kind: "Secret"} + + seedScheme := runtime.NewScheme() + seedScheme.AddKnownTypeWithName(vpcGVK, &unstructured.Unstructured{}) + seedScheme.AddKnownTypeWithName(vpcGVK.GroupVersion().WithKind("VPCList"), &unstructured.UnstructuredList{}) + seedScheme.AddKnownTypeWithName(pcGVK, &unstructured.Unstructured{}) + seedScheme.AddKnownTypeWithName(pcGVK.GroupVersion().WithKind("ProviderConfigList"), &unstructured.UnstructuredList{}) + seedScheme.AddKnownTypeWithName(secretGVK, &unstructured.Unstructured{}) + seedScheme.AddKnownTypeWithName(secretGVK.GroupVersion().WithKind("SecretList"), &unstructured.UnstructuredList{}) + + seedRM := meta.NewDefaultRESTMapper([]schema.GroupVersion{vpcGVK.GroupVersion(), pcGVK.GroupVersion()}) + seedRM.Add(vpcGVK, meta.RESTScopeRoot) + seedRM.Add(pcGVK, meta.RESTScopeRoot) + + pc := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "aws.upbound.io/v1beta1", + "kind": "ProviderConfig", + "metadata": map[string]any{"name": "default"}, + "spec": map[string]any{"credentials": map[string]any{ + "source": "Secret", + "secretRef": map[string]any{"name": "aws-creds", "namespace": "crossplane-system", "key": "creds"}, + }}, + }} + secret := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]any{"name": "aws-creds", "namespace": "crossplane-system"}, + }} + + kube := fake.NewClientBuilder().WithScheme(seedScheme).WithObjects(pc, secret).Build() + pc2 := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_CREATE}} + + got := cmd.simulateOne(ctx, kube, seedRM, "default", &fakeRouter{client: pc2}, vpc("my-vpc", "10.0.0.0/16")) + + if got.Error != "" { + t.Fatalf("simulateOne(...): unexpected error: %s", got.Error) + } + if n := len(pc2.got.GetKubernetesObjectStore()); n != 1 { + t.Errorf("simulateOne(...): want the ProviderConfig in the object store, got %d objects", n) + } + if k := pc2.got.GetKubernetesObjectStore()[0].GetFields()["kind"].GetStringValue(); k != "ProviderConfig" { + t.Errorf("simulateOne(...): want a ProviderConfig in the object store, got %q", k) + } + }) + + t.Run("UnservedKindPlansCreate", func(t *testing.T) { + // A kind the cluster does not serve: no RESTMapping, no live fetch. + kube := fake.NewClientBuilder().WithScheme(s).Build() + emptyRM := meta.NewDefaultRESTMapper(nil) + pc := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_CREATE}} + + got := cmd.simulateOne(ctx, kube, emptyRM, "default", &fakeRouter{client: pc}, vpc("my-vpc", "10.0.0.0/16")) + + if got.Action != plan.ActionCreate || got.Error != "" { + t.Errorf("simulateOne(...): want a create for an unserved kind, got %+v", got) + } + }) +} diff --git a/flake.lock b/flake.lock index 94721b95..bff701fb 100644 --- a/flake.lock +++ b/flake.lock @@ -18,11 +18,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1789370336, - "narHash": "sha256-6RSEDHIWQtesQKWSu5qRai8L2h4KgCgMEfJHstW99G4=", + "lastModified": 1791637130, + "narHash": "sha256-TuvwvSjJrpqyIYdjhQ7Wu+ADVfeOpO2Um21/V7TOubw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c7def046b9a883d46974757852106483d741586f", + "rev": "ca1659429c950a11a565b5764e37a1b704e17ec8", "type": "github" }, "original": { diff --git a/go.mod b/go.mod index db51dedd..fbd3db8b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/crossplane/cli/v2 -go 1.26.0 +go 1.26.8 require ( charm.land/bubbles/v2 v2.2.1 @@ -14,8 +14,9 @@ require ( github.com/charmbracelet/x/term v0.2.2 github.com/containerd/errdefs v1.0.0 github.com/crossplane/crossplane-runtime/v2 v2.4.0 - github.com/crossplane/crossplane/apis/v2 v2.4.0 + github.com/crossplane/crossplane/apis/v2 v2.4.1 github.com/crossplane/function-sdk-go v0.7.1 + github.com/crossplane/upjet/v2 v2.5.1-0.20261005082904-c3b7d449d0d4 github.com/docker/cli v29.8.1+incompatible github.com/emicklei/dot v1.11.0 github.com/getkin/kin-openapi v0.149.0 @@ -39,6 +40,7 @@ require ( golang.org/x/sync v0.22.0 golang.org/x/text v0.41.0 golang.org/x/tools v0.49.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 @@ -53,7 +55,7 @@ require ( k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4 k8s.io/metrics v0.35.1 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 - sigs.k8s.io/controller-runtime v0.23.1 + sigs.k8s.io/controller-runtime v0.23.3 sigs.k8s.io/kind v0.30.0 sigs.k8s.io/yaml v1.6.0 ) @@ -281,7 +283,7 @@ require ( github.com/xlab/treeprint v1.2.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - github.com/yuin/goldmark v1.7.17 // indirect + github.com/yuin/goldmark v1.8.4 // indirect github.com/yuin/goldmark-emoji v1.0.6 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect @@ -307,7 +309,6 @@ require ( golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index de06ec2d..f927e28b 100644 --- a/go.sum +++ b/go.sum @@ -228,10 +228,12 @@ github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/crossplane/crossplane-runtime/v2 v2.4.0 h1:m5iGVHbtEh9nZYz7w/wUWDsXVlpG2c8v5i0SZScwHh8= github.com/crossplane/crossplane-runtime/v2 v2.4.0/go.mod h1:Pf+hg5A/46QaGjKi7Pk+HdsrFOA/THvST9qM4El8Rzs= -github.com/crossplane/crossplane/apis/v2 v2.4.0 h1:QmZqZe+vvi9IrnEK8rW7UsCDQBqagK6HFV1mYEw/DkQ= -github.com/crossplane/crossplane/apis/v2 v2.4.0/go.mod h1:Ev+FjQqTTWJWoaXi63uKER215kDjTo8PghDCT0JapZs= +github.com/crossplane/crossplane/apis/v2 v2.4.1 h1:ye1vpLjSDicKQbKnMzPjzWNxwOdqYVlKDWSHt9VVz5g= +github.com/crossplane/crossplane/apis/v2 v2.4.1/go.mod h1:OBNx+tl5RQh31Hqo2Q1KZ5tlwZXUIetAvpKKozP8HhU= github.com/crossplane/function-sdk-go v0.7.1 h1:BEM2b7nAr8rJYc9tjwnoMfVVy1veCA/juD8APYCVLi0= github.com/crossplane/function-sdk-go v0.7.1/go.mod h1:AOD9l/tlKYP6giyT8ZVxrcTUCYliRDKO3aRySojsHVY= +github.com/crossplane/upjet/v2 v2.5.1-0.20261005082904-c3b7d449d0d4 h1:EBkXDtRqz4kx8lDgiUqstneHP20VNH1UWJD1TvbgBLM= +github.com/crossplane/upjet/v2 v2.5.1-0.20261005082904-c3b7d449d0d4/go.mod h1:NQQKxhgkTx7PFsy3V3EIZoeKKg024eQMZp4tyW/TW18= github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 h1:uX1JmpONuD549D73r6cgnxyUu18Zb7yHAy5AYU0Pm4Q= github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= @@ -818,8 +820,8 @@ github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.7.17 h1:p36OVWwRb246iHxA/U4p8OPEpOTESm4n+g+8t0EE5uA= -github.com/yuin/goldmark v1.7.17/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/goldmark v1.8.4 h1:oat/nd3U6NeQqFEL3xpEJq7d7c86NI+DbSNGAs4xnjA= +github.com/yuin/goldmark v1.8.4/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs= github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= @@ -1082,8 +1084,8 @@ pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.23.1 h1:TjJSM80Nf43Mg21+RCy3J70aj/W6KyvDtOlpKf+PupE= -sigs.k8s.io/controller-runtime v0.23.1/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= +sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80= +sigs.k8s.io/controller-runtime v0.23.3/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= sigs.k8s.io/controller-tools v0.20.0 h1:VWZF71pwSQ2lZZCt7hFGJsOfDc5dVG28/IysjjMWXL8= sigs.k8s.io/controller-tools v0.20.0/go.mod h1:b4qPmjGU3iZwqn34alUU5tILhNa9+VXK+J3QV0fT/uU= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/internal/diffserver/manager.go b/internal/diffserver/manager.go new file mode 100644 index 00000000..9d0278e2 --- /dev/null +++ b/internal/diffserver/manager.go @@ -0,0 +1,378 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package diffserver runs provider packages as local diff server containers +// and routes resources to them by group and kind, the way render runs +// function packages. Whether a package can serve diffs is declared in its +// metadata (the DiffServer capability), and which resources it serves comes +// from the CRDs it ships, so everything is known before a container starts. +package diffserver + +import ( + "context" + "fmt" + "net/netip" + "strings" + "sync" + "time" + + diffv1alpha1 "github.com/crossplane/upjet/v2/proto/diff/v1alpha1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + "github.com/crossplane/crossplane-runtime/v2/pkg/logging" + runtimexpkg "github.com/crossplane/crossplane-runtime/v2/pkg/xpkg" + + pkgmetav1 "github.com/crossplane/crossplane/apis/v2/pkg/meta/v1" + + "github.com/crossplane/cli/v2/internal/docker" + clixpkg "github.com/crossplane/cli/v2/internal/xpkg" +) + +const ( + // CapabilityDiffServer is the package capability a provider declares when + // its image serves the diff protocol via `internal diff-server`. + CapabilityDiffServer = "DiffServer" + + // containerPort is the port the diff server binds inside the container, + // upjet's default for `internal diff-server --address`. + containerPort = 9099 + + // containerPrefix names diff server containers. The full name carries the + // package digest, so a changed image never matches a stale container and + // leftovers from --keep-diff-servers are easy to find. + containerPrefix = "crossplane-diff-server-" + + readyPollInterval = 500 * time.Millisecond + defaultReadyTimeout = 60 * time.Second +) + +// Server is a running diff server for one provider package. +type Server struct { + Image string + Address string + ContainerID string + GroupKinds []schema.GroupKind + + conn *grpc.ClientConn + client diffv1alpha1.PlanServiceClient + reused bool +} + +// Skipped records a provider image that does not run a diff server this run, +// and why. Resources it would have served fall back to the approximate +// client-side diff. +type Skipped struct { + Image string + Reason string +} + +// containerRuntime is the slice of internal/docker the manager uses, +// extracted so tests can fake container lifecycle. +type containerRuntime interface { + StartContainer(ctx context.Context, name, image string, opts ...docker.StartContainerOption) (string, error) + StartContainerByID(ctx context.Context, id string) error + StopContainerByID(ctx context.Context, id string) error + GetRunningContainerIDByName(ctx context.Context, name string) (id string, found, running bool, err error) + GetHostPort(ctx context.Context, id string, port int) (string, error) +} + +// Manager starts diff server containers for provider packages and routes +// resources to them. +type Manager struct { + pkgs runtimexpkg.Client + rt containerRuntime + log logging.Logger + keep bool + readyTimeout time.Duration + + mu sync.Mutex + routes map[schema.GroupKind]*Server + servers []*Server +} + +// Option configures a Manager. +type Option func(*Manager) + +// WithKeepServers leaves diff server containers running when the manager +// stops, so later runs find and reuse them by name. +func WithKeepServers(keep bool) Option { + return func(m *Manager) { m.keep = keep } +} + +// WithContainerRuntime replaces the container runtime, for testing. +func WithContainerRuntime(rt containerRuntime) Option { + return func(m *Manager) { m.rt = rt } +} + +// WithReadyTimeout sets how long to wait for a started diff server to answer. +func WithReadyTimeout(d time.Duration) Option { + return func(m *Manager) { m.readyTimeout = d } +} + +// NewManager returns a Manager that fetches provider packages with pkgs and +// runs their images as local diff server containers. +func NewManager(pkgs runtimexpkg.Client, log logging.Logger, opts ...Option) *Manager { + m := &Manager{ + pkgs: pkgs, + rt: dockerRuntime{}, + log: log, + readyTimeout: defaultReadyTimeout, + routes: map[schema.GroupKind]*Server{}, + } + for _, opt := range opts { + opt(m) + } + return m +} + +// Start inspects each provider image and runs a diff server container for +// every package that declares the DiffServer capability, in parallel and best +// effort: an image that cannot serve (no capability, fetch or startup +// failure) is returned as skipped rather than failing the run, since its +// resources degrade to the approximate client-side diff. +func (m *Manager) Start(ctx context.Context, images []string) ([]Skipped, error) { + var ( + wg sync.WaitGroup + mu sync.Mutex + skipped []Skipped + ) + for _, img := range images { + wg.Add(1) + go func(img string) { + defer wg.Done() + s, skip := m.startOne(ctx, img) + mu.Lock() + defer mu.Unlock() + if skip != nil { + skipped = append(skipped, *skip) + return + } + m.register(s) + }(img) + } + wg.Wait() + return skipped, ctx.Err() +} + +func (m *Manager) register(s *Server) { + m.mu.Lock() + defer m.mu.Unlock() + m.servers = append(m.servers, s) + for _, gk := range s.GroupKinds { + if other, ok := m.routes[gk]; ok { + m.log.Info("Multiple diff servers serve the same kind; keeping the first", + "groupKind", gk.String(), "kept", other.Image, "ignored", s.Image) + continue + } + m.routes[gk] = s + } +} + +func (m *Manager) startOne(ctx context.Context, image string) (*Server, *Skipped) { + pkg, err := m.pkgs.Get(ctx, image) + if err != nil { + return nil, &Skipped{Image: image, Reason: fmt.Sprintf("cannot fetch package: %s", err)} + } + + meta := pkg.GetMeta() + if meta == nil || !pkgmetav1.CapabilitiesContainFuzzyMatch(meta.GetCapabilities(), CapabilityDiffServer) { + return nil, &Skipped{Image: image, Reason: "package does not declare the DiffServer capability"} + } + + gks := clixpkg.GroupKinds(pkg.Package) + if len(gks) == 0 { + return nil, &Skipped{Image: image, Reason: "package contains no CRDs to route"} + } + + id, reused, err := m.ensureContainer(ctx, containerName(pkg.Digest), image) + if err != nil { + return nil, &Skipped{Image: image, Reason: fmt.Sprintf("cannot start diff server container: %s", err)} + } + + server, err := m.connect(ctx, id, image) + if err != nil { + if !reused && !m.keep { + if serr := m.rt.StopContainerByID(ctx, id); serr != nil { + m.log.Info("Cannot clean up failed diff server container", "container", id, "err", serr) + } + } + return nil, &Skipped{Image: image, Reason: err.Error()} + } + + server.GroupKinds = gks + server.reused = reused + m.log.Debug("Diff server ready", "image", image, "address", server.Address, "reused", reused) + return server, nil +} + +// ensureContainer finds a live container with the deterministic name for this +// package digest and reuses it, or starts a new one. +func (m *Manager) ensureContainer(ctx context.Context, name, image string) (id string, reused bool, err error) { + id, found, running, err := m.rt.GetRunningContainerIDByName(ctx, name) + if err != nil { + return "", false, err + } + if found { + if !running { + if err := m.rt.StartContainerByID(ctx, id); err != nil { + return "", false, err + } + } + return id, true, nil + } + + id, err = m.rt.StartContainer(ctx, name, image, + // --debug precedes the subcommand: it is the provider's global flag. + docker.StartWithCommand([]string{"--debug", "internal", "diff-server", "--address", fmt.Sprintf(":%d", containerPort)}), + docker.StartWithPublishedPort(netip.MustParseAddr("127.0.0.1"), containerPort), + ) + return id, false, err +} + +func (m *Manager) connect(ctx context.Context, containerID, image string) (*Server, error) { + addr, err := m.rt.GetHostPort(ctx, containerID, containerPort) + if err != nil { + return nil, errors.Wrap(err, "cannot determine diff server address") + } + + conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, errors.Wrap(err, "cannot create gRPC client") + } + + client := diffv1alpha1.NewPlanServiceClient(conn) + if err := waitReady(ctx, client, m.readyTimeout); err != nil { + _ = conn.Close() + return nil, err + } + + return &Server{ + Image: image, + Address: addr, + ContainerID: containerID, + conn: conn, + client: client, + }, nil +} + +// waitReady polls the server with an empty Plan request until it answers. +// Docker publishes the host port before the server inside the container binds +// its own, so a successful TCP connect proves nothing; any gRPC status from +// the server itself (an empty request draws INVALID_ARGUMENT) means it is +// serving. +func waitReady(ctx context.Context, c diffv1alpha1.PlanServiceClient, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + var last error + for { + _, err := c.Plan(ctx, &diffv1alpha1.PlanRequest{}) + if err == nil { + return nil + } + if s, ok := status.FromError(err); ok && s.Code() != codes.Unavailable && s.Code() != codes.DeadlineExceeded && s.Code() != codes.Canceled { + return nil + } + last = err + + select { + case <-ctx.Done(): + return errors.Wrap(last, "timed out waiting for diff server to become ready") + case <-time.After(readyPollInterval): + } + } +} + +// Route returns the diff server client that serves the given group and kind, +// if any. +func (m *Manager) Route(gk schema.GroupKind) (diffv1alpha1.PlanServiceClient, bool) { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.routes[gk] + if !ok { + return nil, false + } + return s.client, true +} + +// Servers returns the running diff servers. +func (m *Manager) Servers() []*Server { + m.mu.Lock() + defer m.mu.Unlock() + return append([]*Server(nil), m.servers...) +} + +// Stop closes connections and tears down containers. With keep set the +// containers stay running so the next run finds them by name and skips +// startup. +func (m *Manager) Stop(ctx context.Context) error { + m.mu.Lock() + defer m.mu.Unlock() + + var errs []error + for _, s := range m.servers { + if s.conn != nil { + _ = s.conn.Close() + } + if m.keep { + continue + } + if err := m.rt.StopContainerByID(ctx, s.ContainerID); err != nil { + errs = append(errs, errors.Wrapf(err, "cannot stop diff server container %s", s.ContainerID)) + } + } + return errors.Join(errs...) +} + +func containerName(digest string) string { + d := strings.TrimPrefix(digest, "sha256:") + if len(d) > 12 { + d = d[:12] + } + return containerPrefix + d +} + +// dockerRuntime implements containerRuntime with the real Docker daemon. +type dockerRuntime struct{} + +func (dockerRuntime) StartContainer(ctx context.Context, name, image string, opts ...docker.StartContainerOption) (string, error) { + return docker.StartContainer(ctx, name, image, opts...) +} + +func (dockerRuntime) StartContainerByID(ctx context.Context, id string) error { + return docker.StartContainerByID(ctx, id) +} + +func (dockerRuntime) StopContainerByID(ctx context.Context, id string) error { + return docker.StopContainerByID(ctx, id) +} + +func (dockerRuntime) GetRunningContainerIDByName(ctx context.Context, name string) (string, bool, bool, error) { + c, found, err := docker.GetContainerByName(ctx, name, true) + if err != nil || !found { + return "", false, false, err + } + return c.ID, true, strings.EqualFold(string(c.State), "running"), nil +} + +func (dockerRuntime) GetHostPort(ctx context.Context, id string, port int) (string, error) { + return docker.GetHostPort(ctx, id, port) +} diff --git a/internal/diffserver/manager_test.go b/internal/diffserver/manager_test.go new file mode 100644 index 00000000..705bc4a8 --- /dev/null +++ b/internal/diffserver/manager_test.go @@ -0,0 +1,340 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package diffserver + +import ( + "context" + "io" + "net" + "strings" + "sync" + "testing" + "time" + + diffv1alpha1 "github.com/crossplane/upjet/v2/proto/diff/v1alpha1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/crossplane/crossplane-runtime/v2/pkg/logging" + runtimexpkg "github.com/crossplane/crossplane-runtime/v2/pkg/xpkg" + "github.com/crossplane/crossplane-runtime/v2/pkg/xpkg/parser" + + "github.com/crossplane/cli/v2/internal/docker" +) + +const ( + pkgWithCapability = `--- +apiVersion: meta.pkg.crossplane.io/v1 +kind: Provider +metadata: + name: provider-aws-ec2 +spec: + capabilities: + - SafeStart + - DiffServer +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpcs.ec2.aws.upbound.io +spec: + group: ec2.aws.upbound.io + names: + kind: VPC + plural: vpcs + scope: Cluster + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpcs.ec2.aws.m.upbound.io +spec: + group: ec2.aws.m.upbound.io + names: + kind: VPC + plural: vpcs + scope: Namespaced + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object +` + + pkgWithoutCapability = `--- +apiVersion: meta.pkg.crossplane.io/v1 +kind: Provider +metadata: + name: provider-aws-s3 +spec: + capabilities: + - SafeStart +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: buckets.s3.aws.upbound.io +spec: + group: s3.aws.upbound.io + names: + kind: Bucket + plural: buckets + scope: Cluster + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object +` +) + +func parsePackage(t *testing.T, yamlDoc, digest string) *runtimexpkg.Package { + t.Helper() + + metaScheme, err := runtimexpkg.BuildMetaScheme() + if err != nil { + t.Fatalf("cannot build meta scheme: %v", err) + } + objScheme, err := runtimexpkg.BuildObjectScheme() + if err != nil { + t.Fatalf("cannot build object scheme: %v", err) + } + + pkg, err := parser.New(metaScheme, objScheme).Parse(context.Background(), io.NopCloser(strings.NewReader(yamlDoc))) + if err != nil { + t.Fatalf("cannot parse test package: %v", err) + } + return &runtimexpkg.Package{Package: pkg, Digest: digest} +} + +// fakePkgClient implements the runtime xpkg.Client interface with canned +// packages. +type fakePkgClient struct { + pkgs map[string]*runtimexpkg.Package +} + +func (f *fakePkgClient) Get(_ context.Context, ref string, _ ...runtimexpkg.GetOption) (*runtimexpkg.Package, error) { + p, ok := f.pkgs[ref] + if !ok { + return nil, status.Error(codes.NotFound, "no such package") + } + return p, nil +} + +func (f *fakePkgClient) ListVersions(_ context.Context, _ string, _ ...runtimexpkg.GetOption) ([]string, error) { + return nil, nil +} + +// fakeRuntime fakes container lifecycle, pointing every container at addr. +type fakeRuntime struct { + mu sync.Mutex + addr string + existing map[string]fakeContainer // name -> container + started []string // names passed to StartContainer + stopped []string // ids passed to StopContainerByID +} + +type fakeContainer struct { + id string + running bool +} + +func (f *fakeRuntime) StartContainer(_ context.Context, name, _ string, _ ...docker.StartContainerOption) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.started = append(f.started, name) + return "new-" + name, nil +} + +func (f *fakeRuntime) StopContainerByID(_ context.Context, id string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.stopped = append(f.stopped, id) + return nil +} + +func (f *fakeRuntime) StartContainerByID(_ context.Context, _ string) error { return nil } + +func (f *fakeRuntime) GetRunningContainerIDByName(_ context.Context, name string) (string, bool, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + c, ok := f.existing[name] + if !ok { + return "", false, false, nil + } + return c.id, true, c.running, nil +} + +func (f *fakeRuntime) GetHostPort(_ context.Context, _ string, _ int) (string, error) { + return f.addr, nil +} + +// readyServer serves a real PlanService on a loopback listener that answers +// every request with INVALID_ARGUMENT, which is all readiness needs. +type readyServer struct { + diffv1alpha1.UnimplementedPlanServiceServer +} + +func (readyServer) Plan(context.Context, *diffv1alpha1.PlanRequest) (*diffv1alpha1.PlanResponse, error) { + return nil, status.Error(codes.InvalidArgument, "empty request") +} + +func servePlanService(t *testing.T) string { + t.Helper() + + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("cannot listen: %v", err) + } + s := grpc.NewServer() + diffv1alpha1.RegisterPlanServiceServer(s, readyServer{}) + go func() { _ = s.Serve(lis) }() + t.Cleanup(s.Stop) + return lis.Addr().String() +} + +func TestManagerStart(t *testing.T) { + ctx := context.Background() + log := logging.NewNopLogger() + + pkgs := &fakePkgClient{pkgs: map[string]*runtimexpkg.Package{ + "xpkg.example.org/provider-aws-ec2:v2.9.0": parsePackage(t, pkgWithCapability, "sha256:0123456789abcdef"), + "xpkg.example.org/provider-aws-s3:v2.9.0": parsePackage(t, pkgWithoutCapability, "sha256:fedcba9876543210"), + }} + + t.Run("NoCapabilitySkipsWithoutStarting", func(t *testing.T) { + rt := &fakeRuntime{} + m := NewManager(pkgs, log, WithContainerRuntime(rt)) + + skipped, err := m.Start(ctx, []string{"xpkg.example.org/provider-aws-s3:v2.9.0"}) + if err != nil { + t.Fatalf("Start(...): unexpected error: %v", err) + } + if len(skipped) != 1 || !strings.Contains(skipped[0].Reason, "DiffServer capability") { + t.Errorf("Start(...): want one capability skip, got %+v", skipped) + } + if len(rt.started) != 0 { + t.Errorf("Start(...): no container should start for a package without the capability") + } + }) + + t.Run("RoutesBuiltFromPackageCRDs", func(t *testing.T) { + rt := &fakeRuntime{addr: servePlanService(t)} + m := NewManager(pkgs, log, WithContainerRuntime(rt)) + + skipped, err := m.Start(ctx, []string{"xpkg.example.org/provider-aws-ec2:v2.9.0"}) + if err != nil { + t.Fatalf("Start(...): unexpected error: %v", err) + } + if len(skipped) != 0 { + t.Fatalf("Start(...): unexpected skips: %+v", skipped) + } + + for _, gk := range []schema.GroupKind{ + {Group: "ec2.aws.upbound.io", Kind: "VPC"}, + {Group: "ec2.aws.m.upbound.io", Kind: "VPC"}, + } { + if _, ok := m.Route(gk); !ok { + t.Errorf("Route(%s): want a route from the package CRDs", gk) + } + } + if _, ok := m.Route(schema.GroupKind{Group: "s3.aws.upbound.io", Kind: "Bucket"}); ok { + t.Errorf("Route(Bucket.s3): want no route for a kind the package does not ship") + } + + wantName := containerPrefix + "0123456789ab" + if len(rt.started) != 1 || rt.started[0] != wantName { + t.Errorf("Start(...): want container %q started, got %v", wantName, rt.started) + } + + if err := m.Stop(ctx); err != nil { + t.Fatalf("Stop(...): unexpected error: %v", err) + } + if len(rt.stopped) != 1 { + t.Errorf("Stop(...): want the container stopped, got %v", rt.stopped) + } + }) + + t.Run("ReusesRunningContainerByName", func(t *testing.T) { + rt := &fakeRuntime{ + addr: servePlanService(t), + existing: map[string]fakeContainer{ + containerPrefix + "0123456789ab": {id: "existing-id", running: true}, + }, + } + m := NewManager(pkgs, log, WithContainerRuntime(rt)) + + skipped, err := m.Start(ctx, []string{"xpkg.example.org/provider-aws-ec2:v2.9.0"}) + if err != nil || len(skipped) != 0 { + t.Fatalf("Start(...): unexpected error or skips: %v %+v", err, skipped) + } + if len(rt.started) != 0 { + t.Errorf("Start(...): want the existing container reused, got new containers %v", rt.started) + } + }) + + t.Run("KeepLeavesContainersRunning", func(t *testing.T) { + rt := &fakeRuntime{addr: servePlanService(t)} + m := NewManager(pkgs, log, WithContainerRuntime(rt), WithKeepServers(true)) + + if _, err := m.Start(ctx, []string{"xpkg.example.org/provider-aws-ec2:v2.9.0"}); err != nil { + t.Fatalf("Start(...): unexpected error: %v", err) + } + if err := m.Stop(ctx); err != nil { + t.Fatalf("Stop(...): unexpected error: %v", err) + } + if len(rt.stopped) != 0 { + t.Errorf("Stop(...): want containers left running with keep, got stops %v", rt.stopped) + } + }) + + t.Run("NotReadyStopsContainerAndSkips", func(t *testing.T) { + // A listener that accepts no gRPC: readiness never succeeds. + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("cannot listen: %v", err) + } + t.Cleanup(func() { _ = lis.Close() }) + + rt := &fakeRuntime{addr: lis.Addr().String()} + m := NewManager(pkgs, log, WithContainerRuntime(rt), WithReadyTimeout(700*time.Millisecond)) + + skipped, err := m.Start(ctx, []string{"xpkg.example.org/provider-aws-ec2:v2.9.0"}) + if err != nil { + t.Fatalf("Start(...): unexpected error: %v", err) + } + if len(skipped) != 1 || !strings.Contains(skipped[0].Reason, "ready") { + t.Errorf("Start(...): want a readiness skip, got %+v", skipped) + } + if len(rt.stopped) != 1 { + t.Errorf("Start(...): want the unready container cleaned up, got stops %v", rt.stopped) + } + }) +} diff --git a/internal/docker/docker.go b/internal/docker/docker.go index 9ca70b70..3331b4f8 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -27,6 +27,8 @@ import ( "fmt" "io" "maps" + "net" + "net/netip" "path" "path/filepath" "slices" @@ -268,6 +270,54 @@ func StartWithCopyFiles(tarball []byte, path string) StartContainerOption { } } +// StartWithPublishedPort publishes the container's TCP containerPort on an +// ephemeral port on hostIP; the engine allocates the host port. Use +// GetHostPort after the container starts to learn the allocated address. +func StartWithPublishedPort(hostIP netip.Addr, containerPort int) StartContainerOption { + return func(cfg *startContainerConfig) { + port := network.MustParsePort(fmt.Sprintf("%d/tcp", containerPort)) + if cfg.containerConfig.ExposedPorts == nil { + cfg.containerConfig.ExposedPorts = network.PortSet{} + } + cfg.containerConfig.ExposedPorts[port] = struct{}{} + if cfg.hostConfig == nil { + cfg.hostConfig = &container.HostConfig{} + } + if cfg.hostConfig.PortBindings == nil { + cfg.hostConfig.PortBindings = network.PortMap{} + } + cfg.hostConfig.PortBindings[port] = []network.PortBinding{{ + HostIP: hostIP, + HostPort: "0", // "0" => engine allocates an ephemeral port + }} + } +} + +// GetHostPort returns the host address (host:port) a container's published +// TCP containerPort is bound to. +func GetHostPort(ctx context.Context, containerID string, containerPort int) (string, error) { + cli, err := NewClient() + if err != nil { + return "", err + } + + inspect, err := cli.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{}) + if err != nil { + return "", errors.Wrap(err, "cannot inspect container") + } + + port := network.MustParsePort(fmt.Sprintf("%d/tcp", containerPort)) + bindings := inspect.Container.NetworkSettings.Ports[port] + if len(bindings) == 0 { + return "", errors.Errorf("container %q has no published binding for port %s", containerID, port.Port()) + } + binding := bindings[0] + if !binding.HostIP.IsValid() { + return "", errors.Errorf("container %q has a port binding for %s but no host address", containerID, port.Port()) + } + return net.JoinHostPort(binding.HostIP.String(), binding.HostPort), nil +} + // StartWithEnv adds environment variables that will be passed to the container. func StartWithEnv(env ...string) StartContainerOption { return func(cfg *startContainerConfig) { diff --git a/internal/plan/fallback.go b/internal/plan/fallback.go new file mode 100644 index 00000000..f3baa652 --- /dev/null +++ b/internal/plan/fallback.go @@ -0,0 +1,157 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "bytes" + "encoding/json" + "fmt" + "maps" + "slices" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Fallback computes an approximate client-side plan for desired by comparing +// its spec against the live resource's spec. It is used when no diff server +// can plan the resource, and is honest about its limits: no replacement +// detection, no computed values, no provider defaults. live nil means the +// resource does not exist and plans as a create. +func Fallback(desired, live *unstructured.Unstructured, reason string) *ResourcePlan { + p := NewResourcePlan(desired) + p.Approximate = true + p.FallbackReason = reason + + desiredSpec, _, _ := unstructured.NestedMap(desired.Object, "spec") + + if live == nil { + p.Action = ActionCreate + p.Changes = createChanges("spec", desiredSpec) + return p + } + + liveSpec, _, _ := unstructured.NestedMap(live.Object, "spec") + p.Changes = diffMaps("spec", liveSpec, desiredSpec) + if len(p.Changes) == 0 { + p.Action = ActionNoOp + return p + } + p.Action = ActionUpdate + return p +} + +// createChanges flattens a desired map into planned-only field changes, so a +// create lists what it would set. +func createChanges(path string, m map[string]any) []FieldChange { + out := make([]FieldChange, 0, len(m)) + for _, k := range slices.Sorted(maps.Keys(m)) { + out = append(out, flattenValue(path+"."+k, m[k], false)...) + } + return out +} + +// flattenValue emits one planned-only (or actual-only, for removals) change +// per leaf under path. +func flattenValue(path string, v any, removed bool) []FieldChange { + switch t := v.(type) { + case map[string]any: + out := make([]FieldChange, 0, len(t)) + for _, k := range slices.Sorted(maps.Keys(t)) { + out = append(out, flattenValue(path+"."+k, t[k], removed)...) + } + return out + case []any: + out := make([]FieldChange, 0, len(t)) + for i, e := range t { + out = append(out, flattenValue(fmt.Sprintf("%s[%d]", path, i), e, removed)...) + } + return out + default: + fc := FieldChange{Path: path} + if removed { + fc.Actual = &Value{Value: v} + } else { + fc.Planned = &Value{Value: v} + } + return []FieldChange{fc} + } +} + +// diffMaps compares two maps recursively and returns the changes that turn +// actual into planned, with paths rooted at path. +func diffMaps(path string, actual, planned map[string]any) []FieldChange { + var out []FieldChange + keys := map[string]struct{}{} + for k := range actual { + keys[k] = struct{}{} + } + for k := range planned { + keys[k] = struct{}{} + } + for _, k := range slices.Sorted(maps.Keys(keys)) { + kp := path + "." + k + av, aok := actual[k] + pv, pok := planned[k] + switch { + case !aok: + out = append(out, flattenValue(kp, pv, false)...) + case !pok: + out = append(out, flattenValue(kp, av, true)...) + default: + out = append(out, diffValues(kp, av, pv)...) + } + } + return out +} + +func diffValues(path string, actual, planned any) []FieldChange { + am, aok := actual.(map[string]any) + pm, pok := planned.(map[string]any) + if aok && pok { + return diffMaps(path, am, pm) + } + + as, aok := actual.([]any) + ps, pok := planned.([]any) + if aok && pok && len(as) == len(ps) { + var out []FieldChange + for i := range as { + out = append(out, diffValues(fmt.Sprintf("%s[%d]", path, i), as[i], ps[i])...) + } + return out + } + + if equalValues(actual, planned) { + return nil + } + return []FieldChange{{ + Path: path, + Actual: &Value{Value: actual}, + Planned: &Value{Value: planned}, + }} +} + +// equalValues compares two leaf values by their JSON encoding. Both sides +// come from JSON or YAML decoding, but with skewed numeric types: YAML yields +// float64 where the API server returns int64. Encoding normalizes that (7 and +// 7.0 both encode as "7") and json.Marshal sorts map keys, so the comparison +// is deterministic for composite values too. +func equalValues(a, b any) bool { + ab, aerr := json.Marshal(a) + bb, berr := json.Marshal(b) + return aerr == nil && berr == nil && bytes.Equal(ab, bb) +} diff --git a/internal/plan/fallback_test.go b/internal/plan/fallback_test.go new file mode 100644 index 00000000..8382c680 --- /dev/null +++ b/internal/plan/fallback_test.go @@ -0,0 +1,184 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func mr(spec map[string]any) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "ec2.aws.upbound.io/v1beta1", + "kind": "VPC", + "metadata": map[string]any{"name": "my-vpc"}, + "spec": spec, + }} +} + +func TestFallback(t *testing.T) { + type args struct { + desired *unstructured.Unstructured + live *unstructured.Unstructured + } + type want struct { + action Action + changes []FieldChange + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Create": { + reason: "No live counterpart plans as a create listing the desired spec.", + args: args{ + desired: mr(map[string]any{"forProvider": map[string]any{ + "cidrBlock": "10.0.0.0/16", + "region": "us-west-1", + }}), + }, + want: want{ + action: ActionCreate, + changes: []FieldChange{ + {Path: "spec.forProvider.cidrBlock", Planned: &Value{Value: "10.0.0.0/16"}}, + {Path: "spec.forProvider.region", Planned: &Value{Value: "us-west-1"}}, + }, + }, + }, + "NoOp": { + reason: "An identical spec plans as a no-op, even when numeric types differ between YAML and the API server.", + args: args{ + desired: mr(map[string]any{"forProvider": map[string]any{"retention": int64(7)}}), + live: mr(map[string]any{"forProvider": map[string]any{"retention": float64(7)}}), + }, + want: want{action: ActionNoOp}, + }, + "Update": { + reason: "A changed nested field plans as an update with both sides.", + args: args{ + desired: mr(map[string]any{"forProvider": map[string]any{"cidrBlock": "10.1.0.0/16"}}), + live: mr(map[string]any{"forProvider": map[string]any{"cidrBlock": "10.0.0.0/16"}}), + }, + want: want{ + action: ActionUpdate, + changes: []FieldChange{{ + Path: "spec.forProvider.cidrBlock", + Actual: &Value{Value: "10.0.0.0/16"}, + Planned: &Value{Value: "10.1.0.0/16"}, + }}, + }, + }, + "AddAndRemove": { + reason: "Fields only on one side plan as additions and removals.", + args: args{ + desired: mr(map[string]any{"forProvider": map[string]any{"added": true}}), + live: mr(map[string]any{"forProvider": map[string]any{"removed": "gone"}}), + }, + want: want{ + action: ActionUpdate, + changes: []FieldChange{ + {Path: "spec.forProvider.added", Planned: &Value{Value: true}}, + {Path: "spec.forProvider.removed", Actual: &Value{Value: "gone"}}, + }, + }, + }, + "ListElementChange": { + reason: "Same-length lists diff element-wise with indexed paths.", + args: args{ + desired: mr(map[string]any{"forProvider": map[string]any{"zones": []any{"a", "c"}}}), + live: mr(map[string]any{"forProvider": map[string]any{"zones": []any{"a", "b"}}}), + }, + want: want{ + action: ActionUpdate, + changes: []FieldChange{{ + Path: "spec.forProvider.zones[1]", + Actual: &Value{Value: "b"}, + Planned: &Value{Value: "c"}, + }}, + }, + }, + "DeepNestedListOfMaps": { + reason: "Changes inside maps nested in lists diff leaf-wise with indexed paths; untouched siblings stay silent.", + args: args{ + desired: mr(map[string]any{"forProvider": map[string]any{ + "parentIdRef": map[string]any{"name": "nebius-eu-north", "namespace": "modelplane-system"}, + "template": map[string]any{ + "filesystems": []any{map[string]any{ + "attachMode": "READ_ONLY", + "existingFilesystem": map[string]any{"idSelector": map[string]any{"matchControllerRef": true}}, + "mountTag": "modelplane-cache", + }}, + }, + }}), + live: mr(map[string]any{"forProvider": map[string]any{ + "parentIdRef": map[string]any{"name": "nebius-eu-north", "namespace": "modelplane-system"}, + "template": map[string]any{ + "filesystems": []any{map[string]any{ + "attachMode": "READ_WRITE", + "existingFilesystem": map[string]any{"idSelector": map[string]any{"matchControllerRef": true}}, + "mountTag": "modelplane-cache", + }}, + }, + }}), + }, + want: want{ + action: ActionUpdate, + changes: []FieldChange{{ + Path: "spec.forProvider.template.filesystems[0].attachMode", + Actual: &Value{Value: "READ_WRITE"}, + Planned: &Value{Value: "READ_ONLY"}, + }}, + }, + }, + "ListLengthChange": { + reason: "Lists of different lengths diff as one whole-list change.", + args: args{ + desired: mr(map[string]any{"forProvider": map[string]any{"zones": []any{"a"}}}), + live: mr(map[string]any{"forProvider": map[string]any{"zones": []any{"a", "b"}}}), + }, + want: want{ + action: ActionUpdate, + changes: []FieldChange{{ + Path: "spec.forProvider.zones", + Actual: &Value{Value: []any{"a", "b"}}, + Planned: &Value{Value: []any{"a"}}, + }}, + }, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + got := Fallback(tc.args.desired, tc.args.live, "test") + + if !got.Approximate { + t.Errorf("\n%s\nFallback(...): want Approximate, got exact", tc.reason) + } + if got.Action != tc.want.action { + t.Errorf("\n%s\nFallback(...): want action %q, got %q", tc.reason, tc.want.action, got.Action) + } + if diff := cmp.Diff(tc.want.changes, got.Changes, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("\n%s\nFallback(...): -want changes, +got:\n%s", tc.reason, diff) + } + }) + } +} diff --git a/internal/plan/output.go b/internal/plan/output.go new file mode 100644 index 00000000..756a00ce --- /dev/null +++ b/internal/plan/output.go @@ -0,0 +1,317 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "encoding/json" + "fmt" + "io" + "maps" + "slices" + "sort" + "strings" + + "sigs.k8s.io/yaml" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" +) + +// The output formats simulate commands share. A unified diff format +// deliberately does not exist yet: crossplane-diff renders real +// full-manifest unified diffs today, and that renderer is what a future +// -o diff should be once its machinery moves into this repo, rather than a +// lower-fidelity imitation built from the plan layer's field changes. +const ( + FormatPlan = "plan" + FormatJSON = "json" + FormatYAML = "yaml" +) + +// Summary counts plans by what they would do. +type Summary struct { + Create int `json:"create"` + Update int `json:"update"` + Destroy int `json:"destroy"` + Replace int `json:"replace"` + NoOp int `json:"noop"` + Approximate int `json:"approximate"` + Errored int `json:"errored"` +} + +// Summarize counts the given plans. +func Summarize(plans []*ResourcePlan) Summary { + var s Summary + for _, p := range plans { + if p.Error != "" { + s.Errored++ + continue + } + if p.Approximate { + s.Approximate++ + } + switch p.Action { + case ActionCreate: + s.Create++ + case ActionUpdate: + s.Update++ + case ActionDelete: + s.Destroy++ + case ActionReplace: + s.Replace++ + case ActionNoOp: + s.NoOp++ + } + } + return s +} + +// ResultFormatVersion versions the serialized result schema so automation +// consuming it can detect incompatible changes. The schema is alpha; the +// version moves to 1.0 when it stabilizes. +const ResultFormatVersion = "0.1" + +// Result is the serialized form of a simulate run, for -o json|yaml. Errored +// is the primary gate for wrapping automation: when true, at least one +// resource could not be planned and the run's answer is incomplete. +type Result struct { + FormatVersion string `json:"formatVersion"` + Errored bool `json:"errored"` + Summary Summary `json:"summary"` + Plans []*ResourcePlan `json:"plans"` +} + +// NewResult assembles the serializable Result for the given plans. +func NewResult(plans []*ResourcePlan) Result { + s := Summarize(plans) + return Result{ + FormatVersion: ResultFormatVersion, + Errored: s.Errored > 0, + Summary: s, + Plans: plans, + } +} + +// Render writes the plans to w in the given format. +func Render(w io.Writer, format string, plans []*ResourcePlan) error { + switch format { + case FormatPlan: + return renderPlan(w, plans) + case FormatJSON: + out, err := json.MarshalIndent(NewResult(plans), "", " ") + if err != nil { + return errors.Wrap(err, "cannot marshal plans to JSON") + } + _, err = fmt.Fprintln(w, string(out)) + return err + case FormatYAML: + out, err := yaml.Marshal(NewResult(plans)) + if err != nil { + return errors.Wrap(err, "cannot marshal plans to YAML") + } + _, err = fmt.Fprint(w, string(out)) + return err + } + return errors.Errorf("unknown output format %q", format) +} + +func renderPlan(w io.Writer, plans []*ResourcePlan) error { + for _, p := range plans { + _, _ = fmt.Fprintln(w, header(p)) + if p.Error != "" { + _, _ = fmt.Fprintf(w, " cannot plan: %s\n", p.Error) + _, _ = fmt.Fprintln(w) + continue + } + if p.Approximate { + _, _ = fmt.Fprintf(w, " (approximate: %s)\n", p.FallbackReason) + } + renderChangeTree(w, p.Changes) + _, _ = fmt.Fprintln(w) + } + renderSummary(w, plans) + return nil +} + +// header returns the per-resource line, e.g. +// "[~] Key.kms.aws.upbound.io my-key (update)". +func header(p *ResourcePlan) string { + gvk := p.GroupVersionKind() + name := p.Name + if p.Namespace != "" { + name = p.Namespace + "/" + name + } + what := string(p.Action) + if p.Error != "" { + what = "error" + } + return fmt.Sprintf("%s %s.%s %s (%s)", symbol(p), gvk.Kind, gvk.Group, name, what) +} + +func symbol(p *ResourcePlan) string { + if p.Error != "" { + return "[!]" + } + if p.Approximate { + return "[?]" + } + switch p.Action { + case ActionCreate: + return "[+]" + case ActionUpdate: + return "[~]" + case ActionReplace: + return "[-/+]" + case ActionDelete: + return "[-]" + case ActionNoOp: + return "[=]" + } + return "[?]" +} + +// renderChangeTree prints changes as the user's manifest, annotated: nested +// field names as YAML-style keys, changed leaves marked inline. +func renderChangeTree(w io.Writer, changes []FieldChange) { + var prev []string + for _, c := range sortedChanges(changes) { + segs := strings.Split(c.Path, ".") + if len(segs) == 0 { + continue + } + parents, leaf := segs[:len(segs)-1], segs[len(segs)-1] + + // Print the parent keys that differ from the previous change's path. + common := 0 + for common < len(parents) && common < len(prev) && parents[common] == prev[common] { + common++ + } + for i := common; i < len(parents); i++ { + _, _ = fmt.Fprintf(w, " %s%s:\n", strings.Repeat(" ", i), parents[i]) + } + prev = parents + + indent := " " + strings.Repeat(" ", len(parents)) + note := "" + if c.RequiresReplace { + note = " # forces replacement" + } + switch { + case c.Actual != nil && c.Planned != nil: + _, _ = fmt.Fprintf(w, "%s~ %s: %s -> %s%s\n", indent, leaf, formatValue(c.Actual), formatValue(c.Planned), note) + case c.Planned != nil: + _, _ = fmt.Fprintf(w, "%s+ %s: %s%s\n", indent, leaf, formatValue(c.Planned), note) + case c.Actual != nil: + _, _ = fmt.Fprintf(w, "%s- %s: %s%s\n", indent, leaf, formatValue(c.Actual), note) + } + } +} + +func renderSummary(w io.Writer, plans []*ResourcePlan) { + s := Summarize(plans) + + line := fmt.Sprintf("Plan: %d to add, %d to change, %d to destroy", s.Create, s.Update, s.Destroy) + if s.Replace > 0 { + verb := "requires" + if s.Replace > 1 { + verb = "require" + } + line += fmt.Sprintf(", %d %s replacement", s.Replace, verb) + } + _, _ = fmt.Fprintln(w, line+".") + + if s.Replace > 0 { + fields := replaceFields(plans) + _, _ = fmt.Fprintln(w) + _, _ = fmt.Fprintf(w, "Warning: %s force replacement: %s.\n", pluralize(s.Replace, "resource has fields that", "resources have fields that"), strings.Join(fields, ", ")) + _, _ = fmt.Fprintln(w, "Crossplane providers never destroy and recreate to apply a change.") + _, _ = fmt.Fprintln(w, "Applying this leaves the resource unsynced until it is replaced manually.") + } + + if s.Approximate > 0 { + _, _ = fmt.Fprintln(w) + _, _ = fmt.Fprintf(w, "Warning: %s approximate, computed client-side without a provider diff\n", pluralize(s.Approximate, "result is", "results are")) + _, _ = fmt.Fprintln(w, "server: no replacement detection, no computed values.") + } + + if s.Errored > 0 { + _, _ = fmt.Fprintln(w) + _, _ = fmt.Fprintf(w, "Warning: %s could not be planned.\n", pluralize(s.Errored, "resource", "resources")) + } +} + +func pluralize(n int, singular, plural string) string { + if n == 1 { + return fmt.Sprintf("%d %s", n, singular) + } + return fmt.Sprintf("%d %s", n, plural) +} + +// replaceFields returns the deduplicated, sorted field names (last path +// segments) that force replacement across all plans. +func replaceFields(plans []*ResourcePlan) []string { + set := map[string]struct{}{} + for _, p := range plans { + for _, f := range lastSegments(p.ReplacePaths) { + set[f] = struct{}{} + } + } + return slices.Sorted(maps.Keys(set)) +} + +func lastSegments(paths []string) []string { + out := make([]string, 0, len(paths)) + for _, p := range paths { + segs := strings.Split(p, ".") + out = append(out, segs[len(segs)-1]) + } + return out +} + +func sortedChanges(changes []FieldChange) []FieldChange { + out := make([]FieldChange, len(changes)) + copy(out, changes) + sort.SliceStable(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} + +// formatValue renders one side of a change for humans. +func formatValue(v *Value) string { + switch { + case v == nil: + return "(none)" + case v.Sensitive: + return "(sensitive value)" + case v.Unknown: + return "(known after apply)" + case v.Unresolved: + return "(unresolved reference)" + } + switch t := v.Value.(type) { + case nil: + return "null" + case string: + return fmt.Sprintf("%q", t) + case map[string]any, []any: + out, err := json.Marshal(t) + if err != nil { + return fmt.Sprintf("%v", t) + } + return string(out) + default: + return fmt.Sprintf("%v", t) + } +} diff --git a/internal/plan/output_test.go b/internal/plan/output_test.go new file mode 100644 index 00000000..a5339bcb --- /dev/null +++ b/internal/plan/output_test.go @@ -0,0 +1,152 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func testPlans() []*ResourcePlan { + return []*ResourcePlan{ + { + APIVersion: "ec2.aws.upbound.io/v1beta1", Kind: "VPC", Name: "my-vpc", + Action: ActionCreate, + Changes: []FieldChange{ + {Path: "spec.forProvider.cidrBlock", Planned: &Value{Value: "10.0.0.0/16"}}, + {Path: "spec.forProvider.region", Planned: &Value{Value: "us-west-1"}}, + }, + }, + { + APIVersion: "rds.aws.upbound.io/v1beta1", Kind: "Instance", Name: "prod-db", + Action: ActionReplace, + Changes: []FieldChange{{ + Path: "spec.forProvider.engineVersion", + Actual: &Value{Value: "14.7"}, + Planned: &Value{Value: "15.4"}, + RequiresReplace: true, + }}, + ReplacePaths: []string{"spec.forProvider.engineVersion"}, + }, + { + APIVersion: "kms.aws.upbound.io/v1beta1", Kind: "Key", Name: "my-key", + Action: ActionNoOp, + }, + { + APIVersion: "s3.aws.upbound.io/v1beta2", Kind: "Bucket", Name: "my-bucket", + Action: ActionUpdate, + Approximate: true, FallbackReason: "no diff server for Bucket.s3.aws.upbound.io", + Changes: []FieldChange{{ + Path: "spec.forProvider.forceDestroy", + Actual: &Value{Value: false}, + Planned: &Value{Value: true}, + }}, + }, + { + APIVersion: "ec2.aws.upbound.io/v1beta1", Kind: "Subnet", Name: "my-subnet", + Error: "rpc error: code = Internal desc = boom", + }, + } +} + +func TestRenderPlan(t *testing.T) { + want := `[+] VPC.ec2.aws.upbound.io my-vpc (create) + spec: + forProvider: + + cidrBlock: "10.0.0.0/16" + + region: "us-west-1" + +[-/+] Instance.rds.aws.upbound.io prod-db (replace) + spec: + forProvider: + ~ engineVersion: "14.7" -> "15.4" # forces replacement + +[=] Key.kms.aws.upbound.io my-key (no-op) + +[?] Bucket.s3.aws.upbound.io my-bucket (update) + (approximate: no diff server for Bucket.s3.aws.upbound.io) + spec: + forProvider: + ~ forceDestroy: false -> true + +[!] Subnet.ec2.aws.upbound.io my-subnet (error) + cannot plan: rpc error: code = Internal desc = boom + +Plan: 1 to add, 1 to change, 0 to destroy, 1 requires replacement. + +Warning: 1 resource has fields that force replacement: engineVersion. +Crossplane providers never destroy and recreate to apply a change. +Applying this leaves the resource unsynced until it is replaced manually. + +Warning: 1 result is approximate, computed client-side without a provider diff +server: no replacement detection, no computed values. + +Warning: 1 resource could not be planned. +` + + var buf bytes.Buffer + if err := Render(&buf, FormatPlan, testPlans()); err != nil { + t.Fatalf("Render(plan): unexpected error: %v", err) + } + if diff := cmp.Diff(want, buf.String()); diff != "" { + t.Errorf("Render(plan): -want, +got:\n%s", diff) + } +} + +func TestRenderUnknownFormat(t *testing.T) { + var buf bytes.Buffer + if err := Render(&buf, "diff", testPlans()); err == nil { + t.Errorf("Render(diff): want an error for an unsupported format") + } +} + +func TestRenderJSON(t *testing.T) { + var buf bytes.Buffer + if err := Render(&buf, FormatJSON, testPlans()); err != nil { + t.Fatalf("Render(json): unexpected error: %v", err) + } + + got := Result{} + if err := json.Unmarshal(buf.Bytes(), &got); err != nil { + t.Fatalf("Render(json): output is not valid JSON: %v", err) + } + + want := Summary{Create: 1, Update: 1, Replace: 1, NoOp: 1, Approximate: 1, Errored: 1} + if diff := cmp.Diff(want, got.Summary); diff != "" { + t.Errorf("Render(json): -want summary, +got:\n%s", diff) + } + if len(got.Plans) != 5 { + t.Errorf("Render(json): want 5 plans, got %d", len(got.Plans)) + } + if got.FormatVersion != ResultFormatVersion { + t.Errorf("Render(json): want formatVersion %q, got %q", ResultFormatVersion, got.FormatVersion) + } + if !got.Errored { + t.Errorf("Render(json): want errored true with a failed plan in the set") + } +} + +func TestSummarize(t *testing.T) { + got := Summarize(testPlans()) + want := Summary{Create: 1, Update: 1, Replace: 1, NoOp: 1, Approximate: 1, Errored: 1} + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Summarize(...): -want, +got:\n%s", diff) + } +} diff --git a/internal/plan/plan.go b/internal/plan/plan.go new file mode 100644 index 00000000..bff4dce6 --- /dev/null +++ b/internal/plan/plan.go @@ -0,0 +1,64 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "context" + + diffv1alpha1 "github.com/crossplane/upjet/v2/proto/diff/v1alpha1" + "google.golang.org/protobuf/types/known/structpb" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" +) + +// Plan asks a provider diff server what applying desired would do. actual is +// the live resource the desired one corresponds to, carrying its +// status.atProvider and external-name annotation; nil means the resource does +// not exist yet and plans as a create. store seeds the server's in-memory +// Kubernetes client with objects (Secrets, referenced resources) it may need +// to resolve the desired state. +// +// gRPC errors are returned as-is so callers can branch on them; use +// IsNotSupported to detect the "this resource cannot be planned" case that +// should fall back to an approximate client-side diff. +func Plan(ctx context.Context, c diffv1alpha1.PlanServiceClient, desired, actual *unstructured.Unstructured, store []*unstructured.Unstructured) (*ResourcePlan, error) { + req := &diffv1alpha1.PlanRequest{} + + var err error + if req.DesiredResource, err = structpb.NewStruct(desired.Object); err != nil { + return nil, errors.Wrap(err, "cannot convert desired resource") + } + if actual != nil { + if req.ActualResource, err = structpb.NewStruct(actual.Object); err != nil { + return nil, errors.Wrap(err, "cannot convert actual resource") + } + } + for _, o := range store { + s, err := structpb.NewStruct(o.Object) + if err != nil { + return nil, errors.Wrapf(err, "cannot convert object store entry %s/%s", o.GetNamespace(), o.GetName()) + } + req.KubernetesObjectStore = append(req.KubernetesObjectStore, s) + } + + rsp, err := c.Plan(ctx, req) + if err != nil { + return nil, err + } + return FromResponse(desired, rsp) +} diff --git a/internal/plan/plan_test.go b/internal/plan/plan_test.go new file mode 100644 index 00000000..56355b13 --- /dev/null +++ b/internal/plan/plan_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "context" + "testing" + + diffv1alpha1 "github.com/crossplane/upjet/v2/proto/diff/v1alpha1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// fakePlanClient implements diffv1alpha1.PlanServiceClient in-process, +// capturing the request and returning a canned response or error. +type fakePlanClient struct { + rsp *diffv1alpha1.PlanResponse + err error + got *diffv1alpha1.PlanRequest +} + +func (f *fakePlanClient) Plan(_ context.Context, in *diffv1alpha1.PlanRequest, _ ...grpc.CallOption) (*diffv1alpha1.PlanResponse, error) { + f.got = in + return f.rsp, f.err +} + +func TestPlan(t *testing.T) { + desired := mr(map[string]any{"forProvider": map[string]any{"cidrBlock": "10.1.0.0/16"}}) + actual := mr(map[string]any{"forProvider": map[string]any{"cidrBlock": "10.0.0.0/16"}}) + secret := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]any{"name": "creds", "namespace": "default"}, + }} + + t.Run("RequestCarriesAllState", func(t *testing.T) { + c := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_UPDATE}} + + got, err := Plan(context.Background(), c, desired, actual, []*unstructured.Unstructured{secret}) + if err != nil { + t.Fatalf("Plan(...): unexpected error: %v", err) + } + if got.Action != ActionUpdate { + t.Errorf("Plan(...): want action %q, got %q", ActionUpdate, got.Action) + } + + if c.got.GetDesiredResource().GetFields()["kind"].GetStringValue() != "VPC" { + t.Errorf("Plan(...): desired resource not carried in request") + } + if c.got.GetActualResource() == nil { + t.Errorf("Plan(...): actual resource not carried in request") + } + if len(c.got.GetKubernetesObjectStore()) != 1 { + t.Errorf("Plan(...): want 1 object store entry, got %d", len(c.got.GetKubernetesObjectStore())) + } + }) + + t.Run("NoActualMeansCreate", func(t *testing.T) { + c := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_CREATE}} + + got, err := Plan(context.Background(), c, desired, nil, nil) + if err != nil { + t.Fatalf("Plan(...): unexpected error: %v", err) + } + if got.Action != ActionCreate { + t.Errorf("Plan(...): want action %q, got %q", ActionCreate, got.Action) + } + if c.got.GetActualResource() != nil { + t.Errorf("Plan(...): actual resource should be unset for a create") + } + }) + + t.Run("ServerErrorsPassThrough", func(t *testing.T) { + c := &fakePlanClient{err: status.Error(codes.NotFound, "not served here")} + + _, err := Plan(context.Background(), c, desired, nil, nil) + if status.Code(err) != codes.NotFound { + t.Errorf("Plan(...): want NotFound to pass through, got %v", err) + } + }) +} diff --git a/internal/plan/proto.go b/internal/plan/proto.go new file mode 100644 index 00000000..008343ea --- /dev/null +++ b/internal/plan/proto.go @@ -0,0 +1,126 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + diffv1alpha1 "github.com/crossplane/upjet/v2/proto/diff/v1alpha1" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" +) + +// violationDiffComputationNotSupported is the PreconditionFailure violation +// type a diff server attaches to a FAILED_PRECONDITION status when it serves +// a resource's API group but cannot compute a diff for it. It is defined in +// upjet's pkg/diffserver/internal/plan but not exported. +const violationDiffComputationNotSupported = "DIFF_COMPUTATION_NOT_SUPPORTED" + +// FromResponse converts a diff server's PlanResponse into the ResourcePlan +// for the given desired resource. +func FromResponse(desired *unstructured.Unstructured, rsp *diffv1alpha1.PlanResponse) (*ResourcePlan, error) { + p := NewResourcePlan(desired) + + switch rsp.GetAction() { + case diffv1alpha1.Action_ACTION_NO_OP: + p.Action = ActionNoOp + case diffv1alpha1.Action_ACTION_CREATE: + p.Action = ActionCreate + case diffv1alpha1.Action_ACTION_UPDATE: + p.Action = ActionUpdate + case diffv1alpha1.Action_ACTION_REPLACE: + p.Action = ActionReplace + case diffv1alpha1.Action_ACTION_UNSPECIFIED: + return nil, errors.New("diff server returned an unspecified action") + default: + return nil, errors.Errorf("diff server returned an unknown action %q", rsp.GetAction()) + } + + for _, c := range rsp.GetChanges() { + fc := FieldChange{ + Path: c.GetField(), + Actual: fromFieldValue(c.GetActual()), + Planned: fromFieldValue(c.GetPlanned()), + RequiresReplace: c.GetRequiresReplace(), + Origin: fromOrigin(c.GetOrigin()), + } + p.Changes = append(p.Changes, fc) + if fc.RequiresReplace { + p.ReplacePaths = append(p.ReplacePaths, fc.Path) + } + } + + return p, nil +} + +func fromFieldValue(fv *diffv1alpha1.FieldValue) *Value { + if fv == nil { + return nil + } + switch k := fv.GetKind().(type) { + case *diffv1alpha1.FieldValue_Value: + return &Value{Value: k.Value.AsInterface()} + case *diffv1alpha1.FieldValue_Absence: + switch k.Absence { + case diffv1alpha1.Absence_ABSENCE_UNKNOWN: + return &Value{Unknown: true} + case diffv1alpha1.Absence_ABSENCE_SENSITIVE: + return &Value{Sensitive: true} + case diffv1alpha1.Absence_ABSENCE_UNRESOLVED: + return &Value{Unresolved: true} + case diffv1alpha1.Absence_ABSENCE_UNSPECIFIED: + return nil + } + } + return nil +} + +func fromOrigin(o diffv1alpha1.Origin) Origin { + switch o { + case diffv1alpha1.Origin_ORIGIN_DESIRED_STATE: + return OriginDesired + case diffv1alpha1.Origin_ORIGIN_PROVIDER: + return OriginProvider + case diffv1alpha1.Origin_ORIGIN_UNSPECIFIED: + return "" + } + return "" +} + +// IsNotSupported reports whether err is a diff server saying it serves this +// resource's API group but cannot compute a diff for it. Callers fall back to +// the approximate client-side diff for such resources. +func IsNotSupported(err error) bool { + s, ok := status.FromError(err) + if !ok || s.Code() != codes.FailedPrecondition { + return false + } + for _, d := range s.Details() { + pf, ok := d.(*errdetails.PreconditionFailure) + if !ok { + continue + } + for _, v := range pf.GetViolations() { + if v.GetType() == violationDiffComputationNotSupported { + return true + } + } + } + return false +} diff --git a/internal/plan/proto_test.go b/internal/plan/proto_test.go new file mode 100644 index 00000000..f05d1fbb --- /dev/null +++ b/internal/plan/proto_test.go @@ -0,0 +1,233 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "testing" + + diffv1alpha1 "github.com/crossplane/upjet/v2/proto/diff/v1alpha1" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" +) + +func testDesired() *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "ec2.aws.upbound.io/v1beta1", + "kind": "VPC", + "metadata": map[string]any{"name": "my-vpc"}, + }} +} + +func value(v any) *diffv1alpha1.FieldValue { + pv, err := structpb.NewValue(v) + if err != nil { + panic(err) + } + return &diffv1alpha1.FieldValue{Kind: &diffv1alpha1.FieldValue_Value{Value: pv}} +} + +func absence(a diffv1alpha1.Absence) *diffv1alpha1.FieldValue { + return &diffv1alpha1.FieldValue{Kind: &diffv1alpha1.FieldValue_Absence{Absence: a}} +} + +func TestFromResponse(t *testing.T) { + type args struct { + rsp *diffv1alpha1.PlanResponse + } + type want struct { + plan *ResourcePlan + err error + } + + cases := map[string]struct { + reason string + args args + want want + }{ + "Update": { + reason: "An update with a concrete value change should decode action, values, and origin.", + args: args{rsp: &diffv1alpha1.PlanResponse{ + Action: diffv1alpha1.Action_ACTION_UPDATE, + Changes: []*diffv1alpha1.FieldChange{{ + Field: "spec.forProvider.cidrBlock", + Actual: value("10.0.0.0/16"), + Planned: value("10.1.0.0/16"), + Origin: diffv1alpha1.Origin_ORIGIN_DESIRED_STATE, + }}, + }}, + want: want{plan: &ResourcePlan{ + APIVersion: "ec2.aws.upbound.io/v1beta1", + Kind: "VPC", + Name: "my-vpc", + Action: ActionUpdate, + Changes: []FieldChange{{ + Path: "spec.forProvider.cidrBlock", + Actual: &Value{Value: "10.0.0.0/16"}, + Planned: &Value{Value: "10.1.0.0/16"}, + Origin: OriginDesired, + }}, + }}, + }, + "Replace": { + reason: "A change that requires replacement should be recorded in the change and in ReplacePaths.", + args: args{rsp: &diffv1alpha1.PlanResponse{ + Action: diffv1alpha1.Action_ACTION_REPLACE, + Changes: []*diffv1alpha1.FieldChange{{ + Field: "spec.forProvider.engineVersion", + Actual: value("14.7"), + Planned: value("15.4"), + RequiresReplace: true, + }}, + }}, + want: want{plan: &ResourcePlan{ + APIVersion: "ec2.aws.upbound.io/v1beta1", + Kind: "VPC", + Name: "my-vpc", + Action: ActionReplace, + Changes: []FieldChange{{ + Path: "spec.forProvider.engineVersion", + Actual: &Value{Value: "14.7"}, + Planned: &Value{Value: "15.4"}, + RequiresReplace: true, + }}, + ReplacePaths: []string{"spec.forProvider.engineVersion"}, + }}, + }, + "Absences": { + reason: "Sensitive, unknown, and unresolved values should decode to their markers.", + args: args{rsp: &diffv1alpha1.PlanResponse{ + Action: diffv1alpha1.Action_ACTION_CREATE, + Changes: []*diffv1alpha1.FieldChange{ + {Field: "spec.forProvider.password", Planned: absence(diffv1alpha1.Absence_ABSENCE_SENSITIVE)}, + {Field: "status.atProvider.id", Planned: absence(diffv1alpha1.Absence_ABSENCE_UNKNOWN), Origin: diffv1alpha1.Origin_ORIGIN_PROVIDER}, + {Field: "spec.forProvider.vpcId", Planned: absence(diffv1alpha1.Absence_ABSENCE_UNRESOLVED)}, + }, + }}, + want: want{plan: &ResourcePlan{ + APIVersion: "ec2.aws.upbound.io/v1beta1", + Kind: "VPC", + Name: "my-vpc", + Action: ActionCreate, + Changes: []FieldChange{ + {Path: "spec.forProvider.password", Planned: &Value{Sensitive: true}}, + {Path: "status.atProvider.id", Planned: &Value{Unknown: true}, Origin: OriginProvider}, + {Path: "spec.forProvider.vpcId", Planned: &Value{Unresolved: true}}, + }, + }}, + }, + "NoOp": { + reason: "A no-op should decode with no changes.", + args: args{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_NO_OP}}, + want: want{plan: &ResourcePlan{ + APIVersion: "ec2.aws.upbound.io/v1beta1", + Kind: "VPC", + Name: "my-vpc", + Action: ActionNoOp, + }}, + }, + "UnspecifiedAction": { + reason: "An unspecified action is a server bug and should error rather than guess.", + args: args{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_UNSPECIFIED}}, + want: want{err: errors.New("diff server returned an unspecified action")}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + got, err := FromResponse(testDesired(), tc.args.rsp) + + if tc.want.err != nil { + if err == nil || err.Error() != tc.want.err.Error() { + t.Errorf("\n%s\nFromResponse(...): want error %q, got %v", tc.reason, tc.want.err, err) + } + return + } + if err != nil { + t.Fatalf("\n%s\nFromResponse(...): unexpected error: %v", tc.reason, err) + } + if diff := cmp.Diff(tc.want.plan, got, cmpopts.IgnoreFields(ResourcePlan{}, "Desired")); diff != "" { + t.Errorf("\n%s\nFromResponse(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} + +func TestIsNotSupported(t *testing.T) { + notSupported, err := status.New(codes.FailedPrecondition, "cannot compute diff").WithDetails(&errdetails.PreconditionFailure{ + Violations: []*errdetails.PreconditionFailure_Violation{{ + Type: violationDiffComputationNotSupported, + Subject: "ec2.aws.upbound.io/v1beta1, Kind=VPC", + }}, + }) + if err != nil { + t.Fatalf("cannot build status: %v", err) + } + + otherPrecondition, err := status.New(codes.FailedPrecondition, "something else").WithDetails(&errdetails.PreconditionFailure{ + Violations: []*errdetails.PreconditionFailure_Violation{{Type: "SOMETHING_ELSE"}}, + }) + if err != nil { + t.Fatalf("cannot build status: %v", err) + } + + cases := map[string]struct { + reason string + err error + want bool + }{ + "NotSupported": { + reason: "FAILED_PRECONDITION with the DIFF_COMPUTATION_NOT_SUPPORTED violation is the fallback signal.", + err: notSupported.Err(), + want: true, + }, + "OtherViolation": { + reason: "FAILED_PRECONDITION with another violation type is not the fallback signal.", + err: otherPrecondition.Err(), + want: false, + }, + "PlainPrecondition": { + reason: "FAILED_PRECONDITION without details is not the fallback signal.", + err: status.Error(codes.FailedPrecondition, "nope"), + want: false, + }, + "OtherCode": { + reason: "Other gRPC codes are real failures.", + err: status.Error(codes.Internal, "boom"), + want: false, + }, + "NotAStatus": { + reason: "A non-gRPC error is a real failure.", + err: errors.New("boom"), + want: false, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + if got := IsNotSupported(tc.err); got != tc.want { + t.Errorf("\n%s\nIsNotSupported(...): want %t, got %t", tc.reason, tc.want, got) + } + }) + } +} diff --git a/internal/plan/types.go b/internal/plan/types.go new file mode 100644 index 00000000..29ab7fdd --- /dev/null +++ b/internal/plan/types.go @@ -0,0 +1,125 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package plan contains the shared per-resource plan model for simulate +// commands: the typed result, the client for provider diff servers, the +// client-side approximate fallback, and the output renderers. +package plan + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// Action is what applying a resource would do. +type Action string + +// The actions a plan can report. Replace is never something a Crossplane +// provider does to apply a change; it reports that the change could only be +// applied by destroying and recreating the external resource. Delete is +// reserved for composition-driven removals (project simulate); a diff server +// never returns it. +const ( + ActionNoOp Action = "no-op" + ActionCreate Action = "create" + ActionUpdate Action = "update" + ActionReplace Action = "replace" + ActionDelete Action = "delete" +) + +// Origin says where a planned change comes from. +type Origin string + +// A change originates either from the desired state the user wrote, or from +// the provider (a default, or a late-initialized value). +const ( + OriginDesired Origin = "desired" + OriginProvider Origin = "provider" +) + +// Value is one side of a field change. Exactly one of Value, Unknown, +// Sensitive, or Unresolved describes it. A nil *Value means the field does +// not exist on that side at all. +type Value struct { + // Value is the concrete value, when it is known and printable. + Value any `json:"value,omitempty"` + // Unknown means the value is known only after apply (computed). + Unknown bool `json:"unknown,omitempty"` + // Sensitive means the value exists but is redacted. + Sensitive bool `json:"sensitive,omitempty"` + // Unresolved means the value comes from a reference or Secret the diff + // server could not resolve. + Unresolved bool `json:"unresolved,omitempty"` +} + +// FieldChange is one changed field in a plan, in CRD terms. +type FieldChange struct { + // Path is the CRD field path, e.g. spec.forProvider.engineVersion. + Path string `json:"path"` + // Actual is the field's current value; nil when the field does not exist + // yet. + Actual *Value `json:"actual,omitempty"` + // Planned is the field's value after apply; nil when the change removes + // the field. + Planned *Value `json:"planned,omitempty"` + // RequiresReplace means this change can only be applied by destroying and + // recreating the external resource. + RequiresReplace bool `json:"requiresReplace,omitempty"` + // Origin says whether the change comes from the desired state or from the + // provider. Empty when unknown (e.g. the approximate fallback). + Origin Origin `json:"origin,omitempty"` +} + +// ResourcePlan is the plan for a single resource: what applying it would do. +type ResourcePlan struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Name string `json:"name"` + Namespace string `json:"namespace,omitempty"` + Action Action `json:"action,omitempty"` + Changes []FieldChange `json:"changes,omitempty"` + // ReplacePaths lists the field paths that force replacement. + ReplacePaths []string `json:"replacePaths,omitempty"` + // Approximate means this plan was computed client-side, without a provider + // diff server: no replacement detection, no computed values. + Approximate bool `json:"approximate,omitempty"` + // FallbackReason says why this plan is approximate. + FallbackReason string `json:"fallbackReason,omitempty"` + // Error is set when no plan could be computed for this resource at all. + Error string `json:"error,omitempty"` + + // Desired is the resource the plan is for. It is carried for rendering + // (the plan output shows the user's own manifest, annotated) and is not + // part of the serialized result. + Desired *unstructured.Unstructured `json:"-"` +} + +// GroupVersionKind returns the plan's resource type. +func (p *ResourcePlan) GroupVersionKind() schema.GroupVersionKind { + return schema.FromAPIVersionAndKind(p.APIVersion, p.Kind) +} + +// NewResourcePlan returns a ResourcePlan identifying the given desired +// resource, with no action decided yet. +func NewResourcePlan(desired *unstructured.Unstructured) *ResourcePlan { + return &ResourcePlan{ + APIVersion: desired.GetAPIVersion(), + Kind: desired.GetKind(), + Name: desired.GetName(), + Namespace: desired.GetNamespace(), + Desired: desired, + } +} diff --git a/internal/terminal/spinner.go b/internal/terminal/spinner.go index a3fcf54d..091ee446 100644 --- a/internal/terminal/spinner.go +++ b/internal/terminal/spinner.go @@ -28,10 +28,39 @@ import ( bspinner "charm.land/bubbles/v2/spinner" tea "charm.land/bubbletea/v2" "charm.land/lipgloss/v2" + "github.com/charmbracelet/x/term" "github.com/crossplane/cli/v2/internal/async" ) +// ttyInput returns the input to attach to a spinner program. The spinners +// never use key input, but bubbletea queries the terminal for capabilities +// (synchronized output and grapheme clustering) on startup, and the terminal +// writes its replies to stdin: without a reader in raw mode they echo as +// stray escape sequences after the spinner text. Attaching a TTY stdin lets +// bubbletea consume them. Piped stdin stays detached so spinners never eat +// piped data. +func ttyInput() io.Reader { + if term.IsTerminal(os.Stdin.Fd()) { + return os.Stdin + } + return nil +} + +// interruptKey handles ctrl+c for a spinner program with a TTY attached: raw +// mode swallows SIGINT, so the key arrives as input and must mirror what the +// signal handler does for external signals. +func interruptKey(p *tea.Program, msg tea.Msg) { + k, ok := msg.(tea.KeyPressMsg) + if !ok || k.String() != "ctrl+c" { + return + } + if p != nil { + _ = p.ReleaseTerminal() + } + os.Exit(130) +} + // SpinnerPrinter prints spinners to the console. type SpinnerPrinter interface { // NewSuccessSpinner returns a new success spinner. @@ -207,6 +236,8 @@ func (m *MultiSpinner) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.mu.Lock() defer m.mu.Unlock() + interruptKey(m.program, msg) + if _, ok := msg.(tickMsg); !ok { return m, nil } @@ -276,7 +307,7 @@ func (m *MultiSpinner) Fail(title string) { // Start starts the spinners. func (m *MultiSpinner) Start() { m.program = tea.NewProgram(m, - tea.WithInput(nil), + tea.WithInput(ttyInput()), tea.WithoutSignalHandler(), tea.WithOutput(m.out), ) @@ -348,6 +379,8 @@ func (ss *SuccessSpinner) Update(msg tea.Msg) (tea.Model, tea.Cmd) { ss.mu.Lock() defer ss.mu.Unlock() + interruptKey(ss.program, msg) + if _, ok := msg.(tickMsg); !ok { return ss, nil } @@ -415,7 +448,7 @@ func (ss *SuccessSpinner) Logf(format string, args ...any) { func (ss *SuccessSpinner) Start() { ss.program = tea.NewProgram(ss, tea.WithOutput(ss.out), - tea.WithInput(nil), + tea.WithInput(ttyInput()), tea.WithoutSignalHandler(), ) diff --git a/internal/xpkg/metadata.go b/internal/xpkg/metadata.go index 91d722ce..44ae2829 100644 --- a/internal/xpkg/metadata.go +++ b/internal/xpkg/metadata.go @@ -23,6 +23,7 @@ import ( apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/yaml" "github.com/crossplane/crossplane-runtime/v2/pkg/errors" @@ -121,3 +122,19 @@ func convertXRDv2ToV1(xrd *xrdv2.CompositeResourceDefinition) (*xrdv1.CompositeR func crdFilename(plural, group string) string { return fmt.Sprintf("%s.%s.yaml", plural, group) } + +// GroupKinds returns the group and kind of every CustomResourceDefinition in +// the package. XRDs and other objects are skipped: for routing resources to a +// provider's diff server only the provider's own CRDs matter. +func GroupKinds(pkg *parser.Package) []schema.GroupKind { + var gks []schema.GroupKind + for _, obj := range pkg.GetObjects() { + switch c := obj.(type) { + case *apiextv1.CustomResourceDefinition: + gks = append(gks, schema.GroupKind{Group: c.Spec.Group, Kind: c.Spec.Names.Kind}) + case *apiextv1beta1.CustomResourceDefinition: + gks = append(gks, schema.GroupKind{Group: c.Spec.Group, Kind: c.Spec.Names.Kind}) + } + } + return gks +} diff --git a/nix/vendor-hashes.nix b/nix/vendor-hashes.nix index 65849b3e..2e92b6f5 100644 --- a/nix/vendor-hashes.nix +++ b/nix/vendor-hashes.nix @@ -12,5 +12,5 @@ # hand.) { # Root module: github.com/crossplane/cli/v2 - root = "sha256-6mWzuyq9PqJniswsNdi5LJl4QLe7XcyWM+s+UHcDkiE="; + root = "sha256-MO9qTpEGofkSmktxC+ubi9m46TvDh570taHwKIQHTGA="; } From 10d45590b39af227125ead0ce29d314bff7e7eb6 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Fri, 16 Oct 2026 18:08:55 +0200 Subject: [PATCH 3/9] feat(docker): allow override like in function, use provider input as manifests to have annotations support, use imageconfigs to override Signed-off-by: Christopher Haar --- cmd/crossplane/simulate/help/simulate.md | 43 ++-- cmd/crossplane/simulate/simulate.go | 256 ++++++++++++++++++++--- cmd/crossplane/simulate/simulate_test.go | 104 +++++++++ design/one-pager-cli-simulate.md | 39 ++-- internal/diffserver/manager.go | 70 ++++++- internal/diffserver/manager_test.go | 10 +- internal/plan/tags.go | 98 +++++++++ internal/plan/tags_test.go | 130 ++++++++++++ 8 files changed, 672 insertions(+), 78 deletions(-) create mode 100644 internal/plan/tags.go create mode 100644 internal/plan/tags_test.go diff --git a/cmd/crossplane/simulate/help/simulate.md b/cmd/crossplane/simulate/help/simulate.md index 0f9f328b..ed11afa3 100644 --- a/cmd/crossplane/simulate/help/simulate.md +++ b/cmd/crossplane/simulate/help/simulate.md @@ -12,11 +12,15 @@ includes replacement detection, values known only after apply, and redacted sensitive values. A resource with no live counterpart plans as a create. Which providers run as diff servers comes from the current kubeconfig context -by default: the CLI lists the providers installed on the cluster and uses -their package images, so the preview matches the provider versions the -cluster actually runs. --provider-images overrides that discovery, for +by default: the CLI follows each resource's kind to the provider that +installed it on the cluster, so the preview matches the provider versions +actually serving those resources. --providers overrides that discovery with +a file of Provider manifests (like render takes Function manifests), for example to preview how a provider version that is not installed yet would -treat existing resources. +treat existing resources. The manifests also carry per-provider runtime +annotations (`simulate.crossplane.io/runtime-docker-publish-address` and +`simulate.crossplane.io/runtime-docker-target`) for Docker setups where the +published port is not reachable on the address the daemon reports. Not every provider serves diffs. A provider package declares the capability in its metadata, and a resource whose provider serves no diff server (or @@ -33,16 +37,29 @@ edit-simulate-edit loop, --keep-diff-servers leaves the containers running so the next run reuses them; they are named `crossplane-diff-server-DIGEST` and are cleaned up by any later run without the flag, or manually with docker rm. -Examples: +## Examples - # Preview what applying resources.yaml would do. - crossplane resource simulate -f resources.yaml +Preview what applying resources.yaml would do: - # Preview against a specific provider version instead of the installed one. - crossplane resource simulate -f resources.yaml --provider-images xpkg.upbound.io/upbound/provider-aws-ec2:v2.9.0 +```shell +crossplane resource simulate -f resources.yaml +``` - # Keep diff servers running for faster subsequent runs. - crossplane resource simulate -f resources.yaml --keep-diff-servers +Preview against specific provider versions instead of the installed ones, +from a checked-in file of Provider manifests: - # Emit machine-readable results for CI gating. - crossplane resource simulate -f resources.yaml -o json +```shell +crossplane resource simulate -f resources.yaml --providers providers.yaml +``` + +Keep diff servers running for faster subsequent runs: + +```shell +crossplane resource simulate -f resources.yaml --keep-diff-servers +``` + +Emit machine-readable results for CI gating: + +```shell +crossplane resource simulate -f resources.yaml -o json +``` diff --git a/cmd/crossplane/simulate/simulate.go b/cmd/crossplane/simulate/simulate.go index 69ecbb26..56472108 100644 --- a/cmd/crossplane/simulate/simulate.go +++ b/cmd/crossplane/simulate/simulate.go @@ -22,6 +22,7 @@ package simulate import ( "context" "fmt" + "net/netip" "os" "path/filepath" "strings" @@ -33,6 +34,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery" "k8s.io/client-go/discovery/cached/memory" @@ -46,12 +48,14 @@ import ( "github.com/crossplane/crossplane/apis/v2/pkg" pkgv1 "github.com/crossplane/crossplane/apis/v2/pkg/v1" + pkgv1beta1 "github.com/crossplane/crossplane/apis/v2/pkg/v1beta1" "github.com/crossplane/cli/v2/cmd/crossplane/common/load" "github.com/crossplane/cli/v2/internal/diffserver" "github.com/crossplane/cli/v2/internal/docker" "github.com/crossplane/cli/v2/internal/kube" "github.com/crossplane/cli/v2/internal/plan" + "github.com/crossplane/cli/v2/internal/project/projectfile" "github.com/crossplane/cli/v2/internal/terminal" clixpkg "github.com/crossplane/cli/v2/internal/xpkg" @@ -61,16 +65,30 @@ import ( //go:embed help/simulate.md var helpDetail string +// Per-provider runtime annotations on Provider manifests passed via +// --providers, mirroring render's per-function runtime-docker annotations. +const ( + // AnnotationKeyDockerPublishAddress sets the address Docker publishes + // this provider's diff server port on. Defaults to loopback only. + AnnotationKeyDockerPublishAddress = "simulate.crossplane.io/runtime-docker-publish-address" + // AnnotationKeyDockerTarget sets the host used to connect to this + // provider's diff server when it differs from the publish address, e.g. + // with a VM-based or remote Docker daemon. + AnnotationKeyDockerTarget = "simulate.crossplane.io/runtime-docker-target" +) + // Cmd previews what applying managed resources would do. type Cmd struct { Filename string `help:"YAML file(s) or directory with managed resources, or '-' for stdin (comma-separated)." predictor:"yaml_file_or_directory" required:"" short:"f"` - ProviderImages []string `help:"Override the provider images to run as diff servers. Defaults to the providers installed on the cluster." name:"provider-images"` - KeepDiffServers bool `help:"Leave diff server containers running for reuse by later runs." name:"keep-diff-servers"` - Context string `default:"" help:"Kubernetes context." name:"context" predictor:"context" short:"c"` - CacheDir string `default:"~/.crossplane/cache" help:"Cache directory for downloaded provider packages." name:"cache-dir" predictor:"directory"` - Timeout time.Duration `default:"2m" help:"Per-resource plan timeout."` - Output string `default:"plan" enum:"plan,json,yaml" help:"Output format. One of: plan, json, yaml." name:"output" short:"o"` + Providers string `help:"YAML file or directory with Provider manifests to run as diff servers, overriding discovery from the cluster. Supports per-provider runtime annotations." name:"providers" optional:"" predictor:"yaml_file_or_directory" type:"path"` + ImageConfigs string `help:"YAML file with ImageConfig manifests that rewrite provider image paths for local pulls. Defaults to the project's imageConfigs when a project file is present." name:"image-configs" optional:"" predictor:"yaml_file" type:"existingfile"` + ProjectFile string `default:"${project_file}" help:"Path to a project definition file to read imageConfigs from." name:"project-file"` + KeepDiffServers bool `help:"Leave diff server containers running for reuse by later runs." name:"keep-diff-servers"` + Context string `default:"" help:"Kubernetes context." name:"context" predictor:"context" short:"c"` + CacheDir string `default:"~/.crossplane/cache" help:"Cache directory for downloaded provider packages." name:"cache-dir" predictor:"directory"` + Timeout time.Duration `default:"2m" help:"Per-resource plan timeout."` + Output string `default:"plan" enum:"plan,json,yaml" help:"Output format. One of: plan, json, yaml." name:"output" short:"o"` Impersonation kube.ImpersonationFlags `embed:""` } @@ -117,17 +135,34 @@ func (c *Cmd) Run(k *kong.Context, logger logging.Logger, sp terminal.SpinnerPri return err } - images := c.ProviderImages - if len(images) == 0 { - if images, err = installedProviderImages(ctx, kubeClient); err != nil { - return errors.Wrap(err, "cannot discover providers installed on the cluster") + imageConfigs, err := c.imageConfigs() + if err != nil { + return err + } + + var inputs []diffserver.Input + if c.Providers != "" { + if inputs, err = loadProviderInputs(c.Providers); err != nil { + return errors.Wrapf(err, "cannot load provider manifests from %q", c.Providers) + } + } else { + // With local ImageConfigs the rewrites apply to original package + // references, so discovery must hand those out; without them the + // cluster's resolved reference is the exact artifact it runs. + images, err := relevantProviderImages(ctx, kubeClient, rmapper, resources, len(imageConfigs) > 0) + if err != nil { + return errors.Wrap(err, "cannot discover the providers serving the given resources") } + inputs = diffserver.Images(images) } - if len(images) == 0 { + if len(inputs) == 0 { _, _ = fmt.Fprintln(k.Stderr, "No providers found to run as diff servers; all results will be approximate.") } - pkgClient, err := clixpkg.NewClient(clixpkg.NewRemoteFetcher(), clixpkg.WithCacheDir(afero.NewOsFs(), c.CacheDir)) + pkgClient, err := clixpkg.NewClient(clixpkg.NewRemoteFetcher(), + clixpkg.WithCacheDir(afero.NewOsFs(), c.CacheDir), + clixpkg.WithImageConfigs(imageConfigs), + ) if err != nil { return errors.Wrap(err, "cannot create package client") } @@ -136,7 +171,7 @@ func (c *Cmd) Run(k *kong.Context, logger logging.Logger, sp terminal.SpinnerPri var skipped []diffserver.Skipped if err := sp.WrapWithSuccessSpinner("Starting provider diff servers", func() error { var err error - skipped, err = mgr.Start(ctx, images) + skipped, err = mgr.Start(ctx, inputs) return err }); err != nil { return errors.Wrap(err, "cannot start diff servers") @@ -158,6 +193,90 @@ func (c *Cmd) Run(k *kong.Context, logger logging.Logger, sp terminal.SpinnerPri return plan.Render(k.Stdout, c.Output, plans) } +// imageConfigs returns the ImageConfigs to rewrite provider image paths with: +// the --image-configs file when given, else the imageConfigs of the project +// the command happens to run in, if any. Running outside a project is the +// norm for this command, so a missing project file is not an error. +func (c *Cmd) imageConfigs() ([]pkgv1beta1.ImageConfig, error) { + if c.ImageConfigs != "" { + ics, err := loadImageConfigs(c.ImageConfigs) + return ics, errors.Wrapf(err, "cannot load image configs from %q", c.ImageConfigs) + } + + projFilePath, err := filepath.Abs(c.ProjectFile) + if err != nil { + return nil, nil //nolint:nilerr // Best effort: no resolvable project file means no project image configs. + } + if _, err := os.Stat(projFilePath); err != nil { + return nil, nil //nolint:nilerr // Best effort: no project file means no project image configs. + } + projFS := afero.NewBasePathFs(afero.NewOsFs(), filepath.Dir(projFilePath)) + proj, err := projectfile.Parse(projFS, filepath.Base(projFilePath)) + if err != nil { + return nil, errors.Wrapf(err, "cannot parse project file %q", c.ProjectFile) + } + return proj.Spec.ImageConfigs, nil +} + +// loadProviderInputs reads Provider manifests and turns them into diff server +// inputs: the package image from spec.package, and per-provider Docker +// overrides from the simulate.crossplane.io runtime annotations. +func loadProviderInputs(path string) ([]diffserver.Input, error) { + loader, err := load.NewLoader(path) + if err != nil { + return nil, err + } + objs, err := loader.Load() + if err != nil { + return nil, err + } + + inputs := make([]diffserver.Input, 0, len(objs)) + for _, o := range objs { + if o.GetKind() != "Provider" { + return nil, errors.Errorf("expected only Provider manifests, found %s", o.GetKind()) + } + in := diffserver.Input{} + if in.Image, _, _ = unstructured.NestedString(o.Object, "spec", "package"); in.Image == "" { + return nil, errors.Errorf("provider %q has no spec.package", o.GetName()) + } + annotations := o.GetAnnotations() + if v := annotations[AnnotationKeyDockerPublishAddress]; v != "" { + if in.PublishAddress, err = netip.ParseAddr(v); err != nil { + return nil, errors.Wrapf(err, "provider %q has an invalid %s annotation", o.GetName(), AnnotationKeyDockerPublishAddress) + } + } + in.Target = annotations[AnnotationKeyDockerTarget] + inputs = append(inputs, in) + } + return inputs, nil +} + +// loadImageConfigs reads ImageConfig manifests from a YAML file. +func loadImageConfigs(path string) ([]pkgv1beta1.ImageConfig, error) { + loader, err := load.NewLoader(path) + if err != nil { + return nil, err + } + objs, err := loader.Load() + if err != nil { + return nil, err + } + + ics := make([]pkgv1beta1.ImageConfig, 0, len(objs)) + for _, o := range objs { + if o.GetKind() != "ImageConfig" { + return nil, errors.Errorf("expected only ImageConfig manifests, found %s", o.GetKind()) + } + ic := pkgv1beta1.ImageConfig{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(o.Object, &ic); err != nil { + return nil, errors.Wrapf(err, "cannot parse ImageConfig %s", o.GetName()) + } + ics = append(ics, ic) + } + return ics, nil +} + // router routes a resource's group and kind to a diff server client. It is // the slice of diffserver.Manager the per-resource flow needs, extracted so // tests can fake routing. @@ -183,6 +302,11 @@ func (c *Cmd) simulateOne(ctx context.Context, kubeClient client.Client, rmapper return p } + // Crossplane's external tags survive every apply (the provider's + // initializer re-adds them), so a preview must never claim they would be + // removed. + plan.MergeExternalTags(desired, live) + gk := desired.GroupVersionKind().GroupKind() planClient, ok := mgr.Route(gk) if !ok { @@ -192,7 +316,7 @@ func (c *Cmd) simulateOne(ctx context.Context, kubeClient client.Client, rmapper pctx, cancel := context.WithTimeout(ctx, c.Timeout) defer cancel() - store := objectStore(ctx, kubeClient, rmapper, defaultNamespace, desired) + store := objectStore(ctx, kubeClient, rmapper, defaultNamespace, desired, live) p, err := plan.Plan(pctx, planClient, desired, live, store) if err != nil { @@ -246,18 +370,30 @@ func fetchLive(ctx context.Context, kubeClient client.Client, rmapper meta.RESTM // offline with placeholder credentials and never needs the real ones. // Gathering is best effort: a missing object simply stays out of the store, // and the server reports precisely what it could not resolve. -func objectStore(ctx context.Context, kubeClient client.Client, rmapper meta.RESTMapper, defaultNamespace string, desired *unstructured.Unstructured) []*unstructured.Unstructured { - name, _, _ := unstructured.NestedString(desired.Object, "spec", "providerConfigRef", "name") +func objectStore(ctx context.Context, kubeClient client.Client, rmapper meta.RESTMapper, defaultNamespace string, desired, live *unstructured.Unstructured) []*unstructured.Unstructured { + // Read the reference from the live resource when one exists: the API + // server has already applied the CRD's defaults there, so its kind and + // name are authoritative. The desired manifest may omit both. + src := desired + if live != nil { + src = live + } + name, _, _ := unstructured.NestedString(src.Object, "spec", "providerConfigRef", "name") if name == "" { name = "default" } - kind, _, _ := unstructured.NestedString(desired.Object, "spec", "providerConfigRef", "kind") + kind, _, _ := unstructured.NestedString(src.Object, "spec", "providerConfigRef", "kind") if kind == "" { + // Legacy cluster-scoped resources have no kind in the reference and + // always mean ProviderConfig. Namespaced resources default to + // ClusterProviderConfig, per the CRD default the API server applies. kind = "ProviderConfig" + gvk := desired.GroupVersionKind() + if m, err := rmapper.RESTMapping(gvk.GroupKind(), gvk.Version); err == nil && m.Scope.Name() == meta.RESTScopeNameNamespace { + kind = "ClusterProviderConfig" + } } - // The provider config lives in the provider family's group, one label - // above the resource's group: ec2.aws.m.upbound.io -> aws.m.upbound.io. group := desired.GroupVersionKind().Group i := strings.Index(group, ".") if i < 0 { @@ -285,21 +421,72 @@ func objectStore(ctx context.Context, kubeClient client.Client, rmapper meta.RES return []*unstructured.Unstructured{pc} } -// installedProviderImages lists the package images of the providers installed -// on the cluster, preferring the resolved identifier over the possibly -// constraint-form spec.package. -func installedProviderImages(ctx context.Context, kubeClient client.Client) ([]string, error) { - providers := &pkgv1.ProviderList{} - if err := kubeClient.List(ctx, providers); err != nil { - return nil, err - } - images := make([]string, 0, len(providers.Items)) - for _, p := range providers.Items { - img := p.Status.CurrentIdentifier - if img == "" { - img = p.Spec.Package +// relevantProviderImages returns the package images of exactly the providers +// that serve the given resources, so simulate never starts a diff server it +// will not route to. Discovery follows the ownership chain on the cluster: +// each resource's kind maps to a CRD, the CRD is owned by the +// ProviderRevision that installed it, and the revision carries the resolved +// package image. Kinds that resolve to no provider are skipped; their +// resources degrade to the approximate client-side diff. +func relevantProviderImages(ctx context.Context, kubeClient client.Client, rmapper meta.RESTMapper, resources []*unstructured.Unstructured, preferOriginal bool) ([]string, error) { + seenGK := map[schema.GroupKind]bool{} + seenImage := map[string]bool{} + var images []string + + for _, r := range resources { + gvk := r.GroupVersionKind() + gk := gvk.GroupKind() + if seenGK[gk] { + continue + } + seenGK[gk] = true + + mapping, err := rmapper.RESTMapping(gk, gvk.Version) + if err != nil { + continue // The kind is not installed; nothing serves it. + } + + crd := &unstructured.Unstructured{} + crd.SetGroupVersionKind(schema.GroupVersionKind{Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"}) + crdName := mapping.Resource.Resource + "." + gk.Group + if err := kubeClient.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return nil, errors.Wrapf(err, "cannot get CRD %s", crdName) + } + + revision := "" + for _, ref := range crd.GetOwnerReferences() { + if ref.Kind == "ProviderRevision" && strings.HasPrefix(ref.APIVersion, "pkg.crossplane.io/") { + revision = ref.Name + break + } + } + if revision == "" { + continue // Not provider-installed (e.g. a hand-applied CRD). + } + + prv := &pkgv1.ProviderRevision{} + if err := kubeClient.Get(ctx, client.ObjectKey{Name: revision}, prv); err != nil { + return nil, errors.Wrapf(err, "cannot get provider revision %s owning CRD %s", revision, crdName) + } + // Prefer the resolved package: after the cluster's ImageConfig + // rewriting, it is the identity of the artifact the cluster actually + // runs, which is what the preview promises to use. If that reference + // is not pullable from this machine (e.g. a cluster-only mirror), the + // manager reports the image as skipped, its resources degrade to the + // approximate diff, and --provider-images remains the override for + // picking a locally reachable image instead. With local ImageConfigs + // in play (preferOriginal) the original reference is handed out + // instead, because that is what local rewrite rules are written + // against. + img := prv.Status.ResolvedPackage + if preferOriginal || img == "" { + img = prv.Spec.Package } - if img != "" { + if img != "" && !seenImage[img] { + seenImage[img] = true images = append(images, img) } } @@ -319,6 +506,9 @@ func (c *Cmd) setupKubeClient(logger logging.Logger) (client.Client, meta.RESTMa c.Impersonation.Apply(kubeconfig) + // client-go defaults unset QPS/Burst to 5/10 with client-side throttling, + // which is painfully slow for the several reads simulate does per + // resource. Same approach as trace's setupKubeClient. if kubeconfig.QPS == 0 { kubeconfig.QPS = 20 } diff --git a/cmd/crossplane/simulate/simulate_test.go b/cmd/crossplane/simulate/simulate_test.go index bac37614..14d20584 100644 --- a/cmd/crossplane/simulate/simulate_test.go +++ b/cmd/crossplane/simulate/simulate_test.go @@ -18,10 +18,14 @@ package simulate import ( "context" + "net/netip" + "os" + "path/filepath" "testing" "time" diffv1alpha1 "github.com/crossplane/upjet/v2/proto/diff/v1alpha1" + "github.com/google/go-cmp/cmp" "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -32,6 +36,10 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/crossplane/crossplane/apis/v2/pkg" + pkgv1 "github.com/crossplane/crossplane/apis/v2/pkg/v1" + + "github.com/crossplane/cli/v2/internal/diffserver" "github.com/crossplane/cli/v2/internal/plan" ) @@ -81,6 +89,56 @@ func notSupportedErr(t *testing.T) error { return st.Err() } +func TestLoadProviderInputs(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "providers.yaml") + manifest := `--- +apiVersion: pkg.crossplane.io/v1 +kind: Provider +metadata: + name: provider-aws-ec2 + annotations: + simulate.crossplane.io/runtime-docker-publish-address: 0.0.0.0 + simulate.crossplane.io/runtime-docker-target: 192.168.64.2 +spec: + package: xpkg.upbound.io/upbound/provider-aws-ec2:v2.9.0 +--- +apiVersion: pkg.crossplane.io/v1 +kind: Provider +metadata: + name: provider-aws-s3 +spec: + package: xpkg.upbound.io/upbound/provider-aws-s3:v2.9.0 +` + if err := os.WriteFile(path, []byte(manifest), 0o600); err != nil { + t.Fatalf("cannot write manifest: %v", err) + } + + got, err := loadProviderInputs(path) + if err != nil { + t.Fatalf("loadProviderInputs(...): unexpected error: %v", err) + } + want := []diffserver.Input{ + { + Image: "xpkg.upbound.io/upbound/provider-aws-ec2:v2.9.0", + PublishAddress: netip.MustParseAddr("0.0.0.0"), + Target: "192.168.64.2", + }, + {Image: "xpkg.upbound.io/upbound/provider-aws-s3:v2.9.0"}, + } + if diff := cmp.Diff(want, got, cmp.Comparer(func(a, b netip.Addr) bool { return a == b })); diff != "" { + t.Errorf("loadProviderInputs(...): -want, +got:\n%s", diff) + } + + notAProvider := filepath.Join(dir, "other.yaml") + if err := os.WriteFile(notAProvider, []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: x\n"), 0o600); err != nil { + t.Fatalf("cannot write manifest: %v", err) + } + if _, err := loadProviderInputs(notAProvider); err == nil { + t.Errorf("loadProviderInputs(...): want an error for non-Provider manifests") + } +} + func TestSimulateOne(t *testing.T) { ctx := context.Background() @@ -203,6 +261,52 @@ func TestSimulateOne(t *testing.T) { } }) + t.Run("DiscoversOnlyRelevantProviders", func(t *testing.T) { + crdGVK := schema.GroupVersionKind{Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"} + + discScheme := runtime.NewScheme() + discScheme.AddKnownTypeWithName(crdGVK, &unstructured.Unstructured{}) + discScheme.AddKnownTypeWithName(crdGVK.GroupVersion().WithKind("CustomResourceDefinitionList"), &unstructured.UnstructuredList{}) + if err := pkg.AddToScheme(discScheme); err != nil { + t.Fatalf("cannot add pkg scheme: %v", err) + } + + crd := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": map[string]any{ + "name": "vpcs.ec2.aws.upbound.io", + "ownerReferences": []any{map[string]any{ + "apiVersion": "pkg.crossplane.io/v1", + "kind": "ProviderRevision", + "name": "provider-aws-ec2-abc123", + "uid": "00000000-0000-0000-0000-000000000000", + }}, + }, + }} + prv := &pkgv1.ProviderRevision{} + prv.SetName("provider-aws-ec2-abc123") + prv.Status.ResolvedPackage = "xpkg.upbound.io/upbound/provider-aws-ec2:v2.9.0" + + kube := fake.NewClientBuilder().WithScheme(discScheme).WithObjects(crd, prv).Build() + + // Two VPCs (same kind) plus a kind with no CRD on the cluster: exactly + // one provider image, once. + unserved := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "kms.aws.upbound.io/v1beta1", + "kind": "Key", + "metadata": map[string]any{"name": "k"}, + }} + got, err := relevantProviderImages(ctx, kube, rm, []*unstructured.Unstructured{vpc("a", "10.0.0.0/16"), vpc("b", "10.1.0.0/16"), unserved}, false) + if err != nil { + t.Fatalf("relevantProviderImages(...): unexpected error: %v", err) + } + want := []string{"xpkg.upbound.io/upbound/provider-aws-ec2:v2.9.0"} + if len(got) != 1 || got[0] != want[0] { + t.Errorf("relevantProviderImages(...): want %v, got %v", want, got) + } + }) + t.Run("UnservedKindPlansCreate", func(t *testing.T) { // A kind the cluster does not serve: no RESTMapping, no live fetch. kube := fake.NewClientBuilder().WithScheme(s).Build() diff --git a/design/one-pager-cli-simulate.md b/design/one-pager-cli-simulate.md index 7eed119c..776aeff4 100644 --- a/design/one-pager-cli-simulate.md +++ b/design/one-pager-cli-simulate.md @@ -119,13 +119,19 @@ along as the actual state), and renders the response. A resource with no live co before apply". Which providers answer those plans comes from the current kubeconfig -context by default: the CLI lists the providers installed on the target -cluster and runs their package images as diff servers, so the preview uses -the provider versions the cluster actually runs. `--provider-images` is -optional and overrides that discovery for the cases where the cluster is -not the source of truth, most importantly the provider upgrade preview -below, where the point is to plan with a version that is not installed -yet. +context by default: the CLI follows each resource's kind to its CRD and the +provider revision that installed it, and runs those providers' package +images as diff servers, so the preview uses exactly the provider versions +serving the resources being simulated. `--providers` is optional and +overrides that discovery with a file of Provider manifests, the way render +takes Function manifests, for the cases where the cluster is not the +source of truth, most importantly the provider upgrade preview below, +where the point is to plan with a version that is not installed yet. +Manifests rather than bare image references, so the override can be +checked in and shared, and so per-provider runtime configuration has a +carrier: the `simulate.crossplane.io/runtime-docker-*` annotations mirror +render's per-function runtime annotations for the Docker setups where the +published port is not reachable on the address the daemon reports. ### Previewing a Provider Upgrade @@ -135,7 +141,7 @@ preview an upgrade, run the new version's image as the diff server and hand it the fleet: ```console -$ crossplane resource simulate --all --provider-images xpkg.example.org/provider-aws-s3:v2.1.0 +$ crossplane resource simulate --all --providers provider-aws-s3-v2.1.0.yaml Found 214 managed resources in groups served by provider-aws-s3. [-/+] Bucket.s3.aws.upbound.io legacy-logs (replace) @@ -281,7 +287,7 @@ type simulateCmd struct { Input []string `short:"f" help:"Project file (default), or changed Composition or XR files."` Namespace string `help:"Simulate only XRs in this namespace."` Name string `help:"Simulate only the XR with this name."` - ProviderImages []string `help:"Override the provider images to run as diff servers."` + Providers string `help:"Provider manifests to run as diff servers, overriding cluster discovery."` IncludePinned bool `help:"Also simulate XRs pinned to older composition revisions."` SkipPlan bool `help:"Skip the cloud-layer plan; show only the cluster-layer diff."` KeepDiffServers bool `help:"Leave diff server containers running for reuse by later runs."` @@ -303,9 +309,10 @@ The CLI treats provider packages the way render treats function packages: as images it can run locally and talk to over gRPC. The provider images come from the project's `dependsOn` for `project -simulate`, from the providers installed on the current cluster context for -`resource simulate`, or from an explicit `--provider-images` override of -either. The CLI starts each provider +simulate`, from the providers serving the simulated resources on the +current cluster context for `resource simulate`, or from an explicit +`--providers` override of either, a file of Provider manifests, the way +render takes Function manifests. The CLI starts each provider image with its `internal diff-server` entrypoint as a local Docker container, following the same runtime conventions render established for functions (pull policy, cleanup on exit), and talks to it over plaintext @@ -323,11 +330,11 @@ resource with no route is detected up front, not by a failed call. The protocol is alpha, merged to upjet's main branch ahead of a release, and simulate enters the CLI as alpha alongside it. -That split is what makes the `--provider-images` override simple: the flag -only changes where the image list comes from, never how routing works. The +That split is what makes the `--providers` override simple: the flag only +changes where the provider list comes from, never how routing works. The CLI never hardcodes which image serves which group and kind, not from -image names, not from cluster state. It inspects whatever images the list -contains and lets each package declare its capability and its kinds +image names, not from cluster state. It inspects whatever packages the +manifests name and lets each package declare its capability and its kinds through its own metadata. Override the list with a new provider version and the routing table rebuilds itself from what that version actually ships. diff --git a/internal/diffserver/manager.go b/internal/diffserver/manager.go index 9d0278e2..df1e1aa1 100644 --- a/internal/diffserver/manager.go +++ b/internal/diffserver/manager.go @@ -24,6 +24,7 @@ package diffserver import ( "context" "fmt" + "net" "net/netip" "strings" "sync" @@ -84,6 +85,16 @@ type Skipped struct { Reason string } +// Input describes one provider package to run as a diff server. The zero +// values of PublishAddress and Target mean the manager's defaults apply; set +// them to override Docker addressing for this provider only, the way render's +// per-function runtime annotations do. +type Input struct { + Image string + PublishAddress netip.Addr + Target string +} + // containerRuntime is the slice of internal/docker the manager uses, // extracted so tests can fake container lifecycle. type containerRuntime interface { @@ -102,6 +113,7 @@ type Manager struct { log logging.Logger keep bool readyTimeout time.Duration + publishAddr netip.Addr mu sync.Mutex routes map[schema.GroupKind]*Server @@ -135,6 +147,7 @@ func NewManager(pkgs runtimexpkg.Client, log logging.Logger, opts ...Option) *Ma rt: dockerRuntime{}, log: log, readyTimeout: defaultReadyTimeout, + publishAddr: netip.MustParseAddr("127.0.0.1"), routes: map[schema.GroupKind]*Server{}, } for _, opt := range opts { @@ -148,17 +161,17 @@ func NewManager(pkgs runtimexpkg.Client, log logging.Logger, opts ...Option) *Ma // effort: an image that cannot serve (no capability, fetch or startup // failure) is returned as skipped rather than failing the run, since its // resources degrade to the approximate client-side diff. -func (m *Manager) Start(ctx context.Context, images []string) ([]Skipped, error) { +func (m *Manager) Start(ctx context.Context, inputs []Input) ([]Skipped, error) { var ( wg sync.WaitGroup mu sync.Mutex skipped []Skipped ) - for _, img := range images { + for _, in := range inputs { wg.Add(1) - go func(img string) { + go func(in Input) { defer wg.Done() - s, skip := m.startOne(ctx, img) + s, skip := m.startOne(ctx, in) mu.Lock() defer mu.Unlock() if skip != nil { @@ -166,12 +179,22 @@ func (m *Manager) Start(ctx context.Context, images []string) ([]Skipped, error) return } m.register(s) - }(img) + }(in) } wg.Wait() return skipped, ctx.Err() } +// Images wraps plain image references into Inputs with no per-provider +// overrides. +func Images(images []string) []Input { + ins := make([]Input, 0, len(images)) + for _, img := range images { + ins = append(ins, Input{Image: img}) + } + return ins +} + func (m *Manager) register(s *Server) { m.mu.Lock() defer m.mu.Unlock() @@ -186,7 +209,9 @@ func (m *Manager) register(s *Server) { } } -func (m *Manager) startOne(ctx context.Context, image string) (*Server, *Skipped) { +func (m *Manager) startOne(ctx context.Context, in Input) (*Server, *Skipped) { + image := in.Image + pkg, err := m.pkgs.Get(ctx, image) if err != nil { return nil, &Skipped{Image: image, Reason: fmt.Sprintf("cannot fetch package: %s", err)} @@ -202,12 +227,28 @@ func (m *Manager) startOne(ctx context.Context, image string) (*Server, *Skipped return nil, &Skipped{Image: image, Reason: "package contains no CRDs to route"} } - id, reused, err := m.ensureContainer(ctx, containerName(pkg.Digest), image) + // Run the container from the reference the package client resolved, so + // ImageConfig path rewriting applies to the container image exactly as it + // applied to the package fetch. + runImage := pkg.ResolvedRef() + if pkg.ResolvedSource == "" { + runImage = image + } + + // The manager's loopback default is overridable per provider: Input + // carries the publish and target addresses from the provider manifest's + // runtime annotations, the way render's per-function annotations work. + publishAddr := m.publishAddr + if in.PublishAddress.IsValid() { + publishAddr = in.PublishAddress + } + + id, reused, err := m.ensureContainer(ctx, containerName(pkg.Digest), runImage, publishAddr) if err != nil { return nil, &Skipped{Image: image, Reason: fmt.Sprintf("cannot start diff server container: %s", err)} } - server, err := m.connect(ctx, id, image) + server, err := m.connect(ctx, id, image, in.Target) if err != nil { if !reused && !m.keep { if serr := m.rt.StopContainerByID(ctx, id); serr != nil { @@ -225,7 +266,7 @@ func (m *Manager) startOne(ctx context.Context, image string) (*Server, *Skipped // ensureContainer finds a live container with the deterministic name for this // package digest and reuses it, or starts a new one. -func (m *Manager) ensureContainer(ctx context.Context, name, image string) (id string, reused bool, err error) { +func (m *Manager) ensureContainer(ctx context.Context, name, image string, publishAddr netip.Addr) (id string, reused bool, err error) { id, found, running, err := m.rt.GetRunningContainerIDByName(ctx, name) if err != nil { return "", false, err @@ -242,16 +283,23 @@ func (m *Manager) ensureContainer(ctx context.Context, name, image string) (id s id, err = m.rt.StartContainer(ctx, name, image, // --debug precedes the subcommand: it is the provider's global flag. docker.StartWithCommand([]string{"--debug", "internal", "diff-server", "--address", fmt.Sprintf(":%d", containerPort)}), - docker.StartWithPublishedPort(netip.MustParseAddr("127.0.0.1"), containerPort), + docker.StartWithPublishedPort(publishAddr, containerPort), ) return id, false, err } -func (m *Manager) connect(ctx context.Context, containerID, image string) (*Server, error) { +func (m *Manager) connect(ctx context.Context, containerID, image, target string) (*Server, error) { addr, err := m.rt.GetHostPort(ctx, containerID, containerPort) if err != nil { return nil, errors.Wrap(err, "cannot determine diff server address") } + if target != "" { + _, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, errors.Wrapf(err, "cannot parse diff server address %q", addr) + } + addr = net.JoinHostPort(target, port) + } conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { diff --git a/internal/diffserver/manager_test.go b/internal/diffserver/manager_test.go index 705bc4a8..41641863 100644 --- a/internal/diffserver/manager_test.go +++ b/internal/diffserver/manager_test.go @@ -233,7 +233,7 @@ func TestManagerStart(t *testing.T) { rt := &fakeRuntime{} m := NewManager(pkgs, log, WithContainerRuntime(rt)) - skipped, err := m.Start(ctx, []string{"xpkg.example.org/provider-aws-s3:v2.9.0"}) + skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-s3:v2.9.0"})) if err != nil { t.Fatalf("Start(...): unexpected error: %v", err) } @@ -249,7 +249,7 @@ func TestManagerStart(t *testing.T) { rt := &fakeRuntime{addr: servePlanService(t)} m := NewManager(pkgs, log, WithContainerRuntime(rt)) - skipped, err := m.Start(ctx, []string{"xpkg.example.org/provider-aws-ec2:v2.9.0"}) + skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"})) if err != nil { t.Fatalf("Start(...): unexpected error: %v", err) } @@ -291,7 +291,7 @@ func TestManagerStart(t *testing.T) { } m := NewManager(pkgs, log, WithContainerRuntime(rt)) - skipped, err := m.Start(ctx, []string{"xpkg.example.org/provider-aws-ec2:v2.9.0"}) + skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"})) if err != nil || len(skipped) != 0 { t.Fatalf("Start(...): unexpected error or skips: %v %+v", err, skipped) } @@ -304,7 +304,7 @@ func TestManagerStart(t *testing.T) { rt := &fakeRuntime{addr: servePlanService(t)} m := NewManager(pkgs, log, WithContainerRuntime(rt), WithKeepServers(true)) - if _, err := m.Start(ctx, []string{"xpkg.example.org/provider-aws-ec2:v2.9.0"}); err != nil { + if _, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"})); err != nil { t.Fatalf("Start(...): unexpected error: %v", err) } if err := m.Stop(ctx); err != nil { @@ -326,7 +326,7 @@ func TestManagerStart(t *testing.T) { rt := &fakeRuntime{addr: lis.Addr().String()} m := NewManager(pkgs, log, WithContainerRuntime(rt), WithReadyTimeout(700*time.Millisecond)) - skipped, err := m.Start(ctx, []string{"xpkg.example.org/provider-aws-ec2:v2.9.0"}) + skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"})) if err != nil { t.Fatalf("Start(...): unexpected error: %v", err) } diff --git a/internal/plan/tags.go b/internal/plan/tags.go new file mode 100644 index 00000000..8518aa40 --- /dev/null +++ b/internal/plan/tags.go @@ -0,0 +1,98 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "slices" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + xpresource "github.com/crossplane/crossplane-runtime/v2/pkg/resource" +) + +// externalTagKeys returns the tags Crossplane providers inject into every +// managed resource they own, identifying the managing kind, name, and provider +// config. Upjet providers write them into the resource's tags (or labels) +// field on the cluster through an initializer, before the first reconcile. +func externalTagKeys() []string { + return []string{ + xpresource.ExternalResourceTagKeyKind, + xpresource.ExternalResourceTagKeyName, + xpresource.ExternalResourceTagKeyProvider, + } +} + +// MergeExternalTags copies Crossplane's external tags from the live resource +// into the desired one, wherever the live spec carries all of them in one map +// (the initializer's signature). Applying a manifest without these tags never +// removes them, because the provider's initializer re-adds them before +// reconciling; without this merge every simulation of such a resource would +// report their removal. The diff server does not run initializers, so the +// merge happens here, on the client. +func MergeExternalTags(desired, live *unstructured.Unstructured) { + if desired == nil || live == nil { + return + } + liveSpec, ok, _ := unstructured.NestedMap(live.Object, "spec") + if !ok { + return + } + + var found [][]string + findExternalTagMaps(liveSpec, []string{"spec"}, &found) + for _, path := range found { + liveTags, ok, _ := unstructured.NestedMap(live.Object, path...) + if !ok { + continue + } + for _, key := range externalTagKeys() { + v, ok := liveTags[key].(string) + if !ok { + continue + } + // Best effort: a desired tree too malformed to set the field on + // will fail planning on its own terms. + _ = unstructured.SetNestedField(desired.Object, v, append(slices.Clone(path), key)...) + } + } +} + +// findExternalTagMaps collects the paths of all maps under node that contain +// every external tag key. +func findExternalTagMaps(node map[string]any, path []string, found *[][]string) { + for k, v := range node { + m, ok := v.(map[string]any) + if !ok { + continue + } + p := append(slices.Clone(path), k) + if hasAllExternalTagKeys(m) { + *found = append(*found, p) + continue + } + findExternalTagMaps(m, p, found) + } +} + +func hasAllExternalTagKeys(m map[string]any) bool { + for _, key := range externalTagKeys() { + if _, ok := m[key].(string); !ok { + return false + } + } + return true +} diff --git a/internal/plan/tags_test.go b/internal/plan/tags_test.go new file mode 100644 index 00000000..84130412 --- /dev/null +++ b/internal/plan/tags_test.go @@ -0,0 +1,130 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "maps" + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func externalTags(extra map[string]any) map[string]any { + m := map[string]any{ + "crossplane-kind": "bucket.s3.aws.upbound.io", + "crossplane-name": "my-bucket", + "crossplane-providerconfig": "default", + } + maps.Copy(m, extra) + return m +} + +func TestMergeExternalTags(t *testing.T) { + cases := map[string]struct { + reason string + desired map[string]any // spec of the desired resource + live map[string]any // spec of the live resource + want map[string]any // expected desired spec after the merge + }{ + "AddsTagsToUntaggedManifest": { + reason: "A manifest without tags gains exactly the external tags the initializer guarantees, not the user's other live tags.", + desired: map[string]any{"forProvider": map[string]any{ + "region": "eu-central-1", + }}, + live: map[string]any{"forProvider": map[string]any{ + "region": "eu-central-1", + "tags": externalTags(map[string]any{"team": "platform"}), + }}, + want: map[string]any{"forProvider": map[string]any{ + "region": "eu-central-1", + "tags": externalTags(nil), + }}, + }, + "KeepsUserTags": { + reason: "A manifest with its own tags keeps them and gains the external ones.", + desired: map[string]any{"forProvider": map[string]any{ + "tags": map[string]any{"env": "prod"}, + }}, + live: map[string]any{"forProvider": map[string]any{ + "tags": externalTags(nil), + }}, + want: map[string]any{"forProvider": map[string]any{ + "tags": externalTags(map[string]any{"env": "prod"}), + }}, + }, + "LiveValuesWin": { + reason: "The initializer overwrites external tag values, so the live values win over manifest ones.", + desired: map[string]any{"forProvider": map[string]any{ + "tags": map[string]any{"crossplane-name": "wrong"}, + }}, + live: map[string]any{"forProvider": map[string]any{ + "tags": externalTags(nil), + }}, + want: map[string]any{"forProvider": map[string]any{ + "tags": externalTags(nil), + }}, + }, + "IgnoresMapsWithoutFullSignature": { + reason: "A live map carrying only some external tag keys is not the initializer's work and stays untouched.", + desired: map[string]any{"forProvider": map[string]any{ + "region": "eu-central-1", + }}, + live: map[string]any{"forProvider": map[string]any{ + "tags": map[string]any{"crossplane-kind": "something"}, + }}, + want: map[string]any{"forProvider": map[string]any{ + "region": "eu-central-1", + }}, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + desired := mr(tc.desired) + MergeExternalTags(desired, mr(tc.live)) + + got, _, _ := unstructured.NestedMap(desired.Object, "spec") + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("\n%s\nMergeExternalTags(...): -want desired spec, +got:\n%s", tc.reason, diff) + } + }) + } + + t.Run("NoLiveResourceIsANoOp", func(t *testing.T) { + desired := mr(map[string]any{"forProvider": map[string]any{"region": "eu-central-1"}}) + MergeExternalTags(desired, nil) + if _, ok, _ := unstructured.NestedMap(desired.Object, "spec", "forProvider", "tags"); ok { + t.Errorf("MergeExternalTags(desired, nil): a create must not invent tags") + } + }) + + t.Run("FallbackReportsNoTagRemoval", func(t *testing.T) { + desired := mr(map[string]any{"forProvider": map[string]any{"region": "eu-central-1"}}) + live := mr(map[string]any{"forProvider": map[string]any{ + "region": "eu-central-1", + "tags": externalTags(nil), + }}) + + MergeExternalTags(desired, live) + got := Fallback(desired, live, "test") + + if got.Action != ActionNoOp || len(got.Changes) != 0 { + t.Errorf("Fallback after MergeExternalTags: want a clean no-op, got action %q with changes %+v", got.Action, got.Changes) + } + }) +} From e53395a0369d275bec08da4b5ea693fcb2b1caf6 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Fri, 16 Oct 2026 18:33:47 +0200 Subject: [PATCH 4/9] feat(plumbing): update the handling of diff servers like functions ux Signed-off-by: Christopher Haar --- cmd/crossplane/simulate/help/simulate.md | 14 ++- cmd/crossplane/simulate/simulate.go | 43 +++++--- design/one-pager-cli-simulate.md | 21 ++-- internal/diffserver/manager.go | 120 ++++++++++++++++++----- internal/diffserver/manager_test.go | 8 +- internal/docker/docker.go | 10 ++ internal/terminal/spinner.go | 19 ++-- 7 files changed, 168 insertions(+), 67 deletions(-) diff --git a/cmd/crossplane/simulate/help/simulate.md b/cmd/crossplane/simulate/help/simulate.md index ed11afa3..ebeb18be 100644 --- a/cmd/crossplane/simulate/help/simulate.md +++ b/cmd/crossplane/simulate/help/simulate.md @@ -33,9 +33,12 @@ cluster with your kubeconfig and RBAC, and the diff servers run locally with no credentials and no cluster access. Diff server startup takes a few seconds per provider image. In an -edit-simulate-edit loop, --keep-diff-servers leaves the containers running so -the next run reuses them; they are named `crossplane-diff-server-DIGEST` and -are cleaned up by any later run without the flag, or manually with docker rm. +edit-simulate-edit loop, annotate a provider manifest with +`simulate.crossplane.io/runtime-docker-cleanup: Orphan` (like render's +per-function cleanup annotation) to leave its container running so the next +run reuses it; containers are named `crossplane-diff-server-DIGEST`, carry +`simulate.crossplane.io/managed-by` and session labels, and are cleaned up by +any later run without the annotation, or manually with docker rm. ## Examples @@ -52,10 +55,11 @@ from a checked-in file of Provider manifests: crossplane resource simulate -f resources.yaml --providers providers.yaml ``` -Keep diff servers running for faster subsequent runs: +Keep diff servers running for faster subsequent runs, via a provider manifest +annotated with `simulate.crossplane.io/runtime-docker-cleanup: Orphan`: ```shell -crossplane resource simulate -f resources.yaml --keep-diff-servers +crossplane resource simulate -f resources.yaml --providers providers.yaml ``` Emit machine-readable results for CI gating: diff --git a/cmd/crossplane/simulate/simulate.go b/cmd/crossplane/simulate/simulate.go index 56472108..4aef57ca 100644 --- a/cmd/crossplane/simulate/simulate.go +++ b/cmd/crossplane/simulate/simulate.go @@ -75,20 +75,26 @@ const ( // provider's diff server when it differs from the publish address, e.g. // with a VM-based or remote Docker daemon. AnnotationKeyDockerTarget = "simulate.crossplane.io/runtime-docker-target" + + // AnnotationKeyDockerCleanup configures how a provider's diff server + // container is cleaned up once the run is done: Remove (the default) + // tears it down, Orphan leaves it running so later runs reuse it without + // paying startup. + AnnotationKeyDockerCleanup = "simulate.crossplane.io/runtime-docker-cleanup" ) // Cmd previews what applying managed resources would do. type Cmd struct { Filename string `help:"YAML file(s) or directory with managed resources, or '-' for stdin (comma-separated)." predictor:"yaml_file_or_directory" required:"" short:"f"` - Providers string `help:"YAML file or directory with Provider manifests to run as diff servers, overriding discovery from the cluster. Supports per-provider runtime annotations." name:"providers" optional:"" predictor:"yaml_file_or_directory" type:"path"` - ImageConfigs string `help:"YAML file with ImageConfig manifests that rewrite provider image paths for local pulls. Defaults to the project's imageConfigs when a project file is present." name:"image-configs" optional:"" predictor:"yaml_file" type:"existingfile"` - ProjectFile string `default:"${project_file}" help:"Path to a project definition file to read imageConfigs from." name:"project-file"` - KeepDiffServers bool `help:"Leave diff server containers running for reuse by later runs." name:"keep-diff-servers"` - Context string `default:"" help:"Kubernetes context." name:"context" predictor:"context" short:"c"` - CacheDir string `default:"~/.crossplane/cache" help:"Cache directory for downloaded provider packages." name:"cache-dir" predictor:"directory"` - Timeout time.Duration `default:"2m" help:"Per-resource plan timeout."` - Output string `default:"plan" enum:"plan,json,yaml" help:"Output format. One of: plan, json, yaml." name:"output" short:"o"` + Providers string `help:"YAML file or directory with Provider manifests to run as diff servers, overriding discovery from the cluster. Supports per-provider runtime annotations." name:"providers" optional:"" predictor:"yaml_file_or_directory" type:"path"` + ImageConfigs string `help:"YAML file with ImageConfig manifests that rewrite provider image paths for local pulls. Defaults to the project's imageConfigs when a project file is present." name:"image-configs" optional:"" predictor:"yaml_file" type:"existingfile"` + ProjectFile string `default:"${project_file}" help:"Path to a project definition file to read imageConfigs from." name:"project-file"` + SessionID string `help:"Session ID to apply as the simulate.crossplane.io/session label on diff server containers. Defaults to a random per-process ID." placeholder:"ID"` + Context string `default:"" help:"Kubernetes context." name:"context" predictor:"context" short:"c"` + CacheDir string `default:"~/.crossplane/cache" help:"Cache directory for downloaded provider packages." name:"cache-dir" predictor:"directory"` + Timeout time.Duration `default:"2m" help:"Per-resource plan timeout."` + Output string `default:"plan" enum:"plan,json,yaml" help:"Output format. One of: plan, json, yaml." name:"output" short:"o"` Impersonation kube.ImpersonationFlags `embed:""` } @@ -110,10 +116,10 @@ func (c *Cmd) AfterApply() error { return nil } -// Run runs the simulate command. -func (c *Cmd) Run(k *kong.Context, logger logging.Logger, sp terminal.SpinnerPrinter) error { - ctx := context.Background() - +// Run runs the simulate command. ctx is the signal-bound command context from +// main: cancellation on SIGINT/SIGTERM unwinds in-flight plans so the +// deferred diff server cleanup runs. +func (c *Cmd) Run(ctx context.Context, k *kong.Context, logger logging.Logger, sp terminal.SpinnerPrinter) error { loader, err := load.NewLoader(c.Filename) if err != nil { return errors.Wrapf(err, "cannot load resources from %q", c.Filename) @@ -167,7 +173,7 @@ func (c *Cmd) Run(k *kong.Context, logger logging.Logger, sp terminal.SpinnerPri return errors.Wrap(err, "cannot create package client") } - mgr := diffserver.NewManager(pkgClient, logger, diffserver.WithKeepServers(c.KeepDiffServers)) + mgr := diffserver.NewManager(pkgClient, logger, diffserver.WithSessionID(c.SessionID)) var skipped []diffserver.Skipped if err := sp.WrapWithSuccessSpinner("Starting provider diff servers", func() error { var err error @@ -176,8 +182,10 @@ func (c *Cmd) Run(k *kong.Context, logger logging.Logger, sp terminal.SpinnerPri }); err != nil { return errors.Wrap(err, "cannot start diff servers") } + // Cleanup runs even when ctx was cancelled by a signal, so an interrupted + // run still tears its containers down. defer func() { - if err := mgr.Stop(context.Background()); err != nil { + if err := mgr.Stop(context.WithoutCancel(ctx)); err != nil { _, _ = fmt.Fprintf(k.Stderr, "Warning: cannot clean up diff servers: %s\n", err) } }() @@ -247,6 +255,13 @@ func loadProviderInputs(path string) ([]diffserver.Input, error) { } } in.Target = annotations[AnnotationKeyDockerTarget] + switch cleanup := diffserver.Cleanup(annotations[AnnotationKeyDockerCleanup]); cleanup { + case diffserver.CleanupRemove, diffserver.CleanupOrphan, "": + in.Cleanup = cleanup + default: + return nil, errors.Errorf("provider %q has an invalid %s annotation %q: must be %s or %s", + o.GetName(), AnnotationKeyDockerCleanup, cleanup, diffserver.CleanupRemove, diffserver.CleanupOrphan) + } inputs = append(inputs, in) } return inputs, nil diff --git a/design/one-pager-cli-simulate.md b/design/one-pager-cli-simulate.md index 776aeff4..d19a8acb 100644 --- a/design/one-pager-cli-simulate.md +++ b/design/one-pager-cli-simulate.md @@ -290,7 +290,7 @@ type simulateCmd struct { Providers string `help:"Provider manifests to run as diff servers, overriding cluster discovery."` IncludePinned bool `help:"Also simulate XRs pinned to older composition revisions."` SkipPlan bool `help:"Skip the cloud-layer plan; show only the cluster-layer diff."` - KeepDiffServers bool `help:"Leave diff server containers running for reuse by later runs."` + SessionID string `help:"Session label for the containers this run creates."` MaxIterations uint `default:"10" help:"Cap on requirement resolution passes per XR."` Timeout time.Duration `default:"2m" help:"Per XR render and plan timeout."` MaxConcurrency uint `default:"8" help:"XRs simulated concurrently."` @@ -350,15 +350,16 @@ read. Startup is the one cost that hurts across runs: a diff server takes a few seconds to come up, and an edit-simulate-edit loop pays that on every run if containers are torn down at exit. So containers get deterministic -names derived from the provider image digest, and `--keep-diff-servers` -leaves them running when the command exits: the next run finds a live -server with a matching name and reuses it instead of starting a new one. -Because the name carries the digest, a changed image never matches a +names derived from the provider image digest, and a provider manifest +annotated `simulate.crossplane.io/runtime-docker-cleanup: Orphan`, the same +cleanup policy render's function containers take through their annotation, +leaves its container running when the command exits: the next run finds a +live server with a matching name and reuses it instead of starting a new +one. Because the name carries the digest, a changed image never matches a stale server, and leftovers are easy to find and remove since they all -follow one naming scheme. The default stays clean, start and tear down -per run; the flag is for the loops where those seconds add up. The same -naming convention is worth extending to render's function containers, -but that belongs to render. +follow one naming scheme and carry render-style managed-by and session +labels. The default stays clean, start and tear down per run; Orphan is +for the loops where those seconds add up. ### From Response to Printed Diff @@ -510,7 +511,7 @@ The protocol carries one resource per unary request; a fleet preview is many small requests, each cheap. Diff-server startup is the dominant fixed cost, a few seconds per distinct provider image, paid once per run and torn down after, or paid once across -many runs with `--keep-diff-servers`. Nothing simulate does +many runs with the Orphan cleanup policy. Nothing simulate does writes to the cluster or the cloud: reads plus dry-runs on one side, a credential-free diff on the other. diff --git a/internal/diffserver/manager.go b/internal/diffserver/manager.go index df1e1aa1..8f9b3b74 100644 --- a/internal/diffserver/manager.go +++ b/internal/diffserver/manager.go @@ -36,6 +36,7 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/rand" "github.com/crossplane/crossplane-runtime/v2/pkg/errors" "github.com/crossplane/crossplane-runtime/v2/pkg/logging" @@ -58,13 +59,68 @@ const ( // containerPrefix names diff server containers. The full name carries the // package digest, so a changed image never matches a stale container and - // leftovers from --keep-diff-servers are easy to find. + // orphaned leftovers are easy to find. containerPrefix = "crossplane-diff-server-" readyPollInterval = 500 * time.Millisecond defaultReadyTimeout = 60 * time.Second ) +// Cleanup is how a diff server container is cleaned up when the manager +// stops, following render's per-function cleanup policies. +type Cleanup string + +const ( + // CleanupRemove is the default: stop and remove the container. + CleanupRemove Cleanup = "Remove" + + // CleanupOrphan leaves the container running, so a later run finds it by + // its deterministic name and reuses it without paying startup. Render's + // function runtimes support a third policy, Stop; a stopped diff server + // serves nothing and restarting costs nearly as much as starting, so + // simulate omits it. + CleanupOrphan Cleanup = "Orphan" +) + +// Labels applied to the Docker containers simulate creates, mirroring +// render's, so leftovers (a crash, or an Orphan cleanup policy) can be +// identified and swept. +const ( + // LabelKeyManagedBy is applied to every diff server container. Its value + // is always LabelValueManagedBy. + LabelKeyManagedBy = "simulate.crossplane.io/managed-by" + + // LabelValueManagedBy is the value of LabelKeyManagedBy. + LabelValueManagedBy = "crossplane-cli" + + // LabelKeySession identifies the run that created a container. Orphaned + // containers outlive their session by design; the label records which run + // created them. + LabelKeySession = "simulate.crossplane.io/session" + + // LabelKeyCleanup records the effective cleanup policy a container was + // created with. + LabelKeyCleanup = "simulate.crossplane.io/cleanup" +) + +// sessionID returns an identifier unique to this process, matching render's +// per-process Docker session IDs. +// +//nolint:gochecknoglobals // Must be stable for the life of the process. +var sessionID = sync.OnceValue(func() string { return rand.String(12) }) + +// managedLabels returns the labels applied to every container the manager +// creates. An empty session falls back to the per-process session ID. +func managedLabels(session string) map[string]string { + if session == "" { + session = sessionID() + } + return map[string]string{ + LabelKeyManagedBy: LabelValueManagedBy, + LabelKeySession: session, + } +} + // Server is a running diff server for one provider package. type Server struct { Image string @@ -72,9 +128,10 @@ type Server struct { ContainerID string GroupKinds []schema.GroupKind - conn *grpc.ClientConn - client diffv1alpha1.PlanServiceClient - reused bool + conn *grpc.ClientConn + client diffv1alpha1.PlanServiceClient + reused bool + cleanup Cleanup } // Skipped records a provider image that does not run a diff server this run, @@ -86,13 +143,14 @@ type Skipped struct { } // Input describes one provider package to run as a diff server. The zero -// values of PublishAddress and Target mean the manager's defaults apply; set -// them to override Docker addressing for this provider only, the way render's -// per-function runtime annotations do. +// values mean the manager's defaults apply; set them to override Docker +// behavior for this provider only, the way render's per-function runtime +// annotations do. type Input struct { Image string PublishAddress netip.Addr Target string + Cleanup Cleanup } // containerRuntime is the slice of internal/docker the manager uses, @@ -111,9 +169,9 @@ type Manager struct { pkgs runtimexpkg.Client rt containerRuntime log logging.Logger - keep bool readyTimeout time.Duration publishAddr netip.Addr + session string mu sync.Mutex routes map[schema.GroupKind]*Server @@ -123,12 +181,6 @@ type Manager struct { // Option configures a Manager. type Option func(*Manager) -// WithKeepServers leaves diff server containers running when the manager -// stops, so later runs find and reuse them by name. -func WithKeepServers(keep bool) Option { - return func(m *Manager) { m.keep = keep } -} - // WithContainerRuntime replaces the container runtime, for testing. func WithContainerRuntime(rt containerRuntime) Option { return func(m *Manager) { m.rt = rt } @@ -139,6 +191,13 @@ func WithReadyTimeout(d time.Duration) Option { return func(m *Manager) { m.readyTimeout = d } } +// WithSessionID sets the session label value for the containers the manager +// creates, so callers orchestrating a larger run can scope cleanup to it. +// Empty means a random per-process ID, matching render's session labels. +func WithSessionID(session string) Option { + return func(m *Manager) { m.session = session } +} + // NewManager returns a Manager that fetches provider packages with pkgs and // runs their images as local diff server containers. func NewManager(pkgs runtimexpkg.Client, log logging.Logger, opts ...Option) *Manager { @@ -235,22 +294,31 @@ func (m *Manager) startOne(ctx context.Context, in Input) (*Server, *Skipped) { runImage = image } - // The manager's loopback default is overridable per provider: Input - // carries the publish and target addresses from the provider manifest's - // runtime annotations, the way render's per-function annotations work. + // The manager's defaults are overridable per provider: Input carries the + // publish and target addresses and the cleanup policy from the provider + // manifest's runtime annotations, the way render's per-function + // annotations work. publishAddr := m.publishAddr if in.PublishAddress.IsValid() { publishAddr = in.PublishAddress } + cleanup := in.Cleanup + if cleanup == "" { + cleanup = CleanupRemove + } + + labels := managedLabels(m.session) + labels[LabelKeyCleanup] = string(cleanup) - id, reused, err := m.ensureContainer(ctx, containerName(pkg.Digest), runImage, publishAddr) + id, reused, err := m.ensureContainer(ctx, containerName(pkg.Digest), runImage, publishAddr, labels) if err != nil { return nil, &Skipped{Image: image, Reason: fmt.Sprintf("cannot start diff server container: %s", err)} } server, err := m.connect(ctx, id, image, in.Target) if err != nil { - if !reused && !m.keep { + // An orphaned container is left behind even on failure, for debugging. + if !reused && cleanup != CleanupOrphan { if serr := m.rt.StopContainerByID(ctx, id); serr != nil { m.log.Info("Cannot clean up failed diff server container", "container", id, "err", serr) } @@ -260,13 +328,14 @@ func (m *Manager) startOne(ctx context.Context, in Input) (*Server, *Skipped) { server.GroupKinds = gks server.reused = reused - m.log.Debug("Diff server ready", "image", image, "address", server.Address, "reused", reused) + server.cleanup = cleanup + m.log.Debug("Diff server ready", "image", image, "address", server.Address, "reused", reused, "cleanup", cleanup) return server, nil } // ensureContainer finds a live container with the deterministic name for this // package digest and reuses it, or starts a new one. -func (m *Manager) ensureContainer(ctx context.Context, name, image string, publishAddr netip.Addr) (id string, reused bool, err error) { +func (m *Manager) ensureContainer(ctx context.Context, name, image string, publishAddr netip.Addr, labels map[string]string) (id string, reused bool, err error) { id, found, running, err := m.rt.GetRunningContainerIDByName(ctx, name) if err != nil { return "", false, err @@ -284,6 +353,7 @@ func (m *Manager) ensureContainer(ctx context.Context, name, image string, publi // --debug precedes the subcommand: it is the provider's global flag. docker.StartWithCommand([]string{"--debug", "internal", "diff-server", "--address", fmt.Sprintf(":%d", containerPort)}), docker.StartWithPublishedPort(publishAddr, containerPort), + docker.StartWithLabels(labels), ) return id, false, err } @@ -368,9 +438,9 @@ func (m *Manager) Servers() []*Server { return append([]*Server(nil), m.servers...) } -// Stop closes connections and tears down containers. With keep set the -// containers stay running so the next run finds them by name and skips -// startup. +// Stop closes connections and tears down containers according to each +// server's cleanup policy: Remove tears the container down, Orphan leaves it +// running so the next run finds it by name and skips startup. func (m *Manager) Stop(ctx context.Context) error { m.mu.Lock() defer m.mu.Unlock() @@ -380,7 +450,7 @@ func (m *Manager) Stop(ctx context.Context) error { if s.conn != nil { _ = s.conn.Close() } - if m.keep { + if s.cleanup == CleanupOrphan { continue } if err := m.rt.StopContainerByID(ctx, s.ContainerID); err != nil { diff --git a/internal/diffserver/manager_test.go b/internal/diffserver/manager_test.go index 41641863..b9ef42be 100644 --- a/internal/diffserver/manager_test.go +++ b/internal/diffserver/manager_test.go @@ -300,18 +300,18 @@ func TestManagerStart(t *testing.T) { } }) - t.Run("KeepLeavesContainersRunning", func(t *testing.T) { + t.Run("OrphanCleanupLeavesContainersRunning", func(t *testing.T) { rt := &fakeRuntime{addr: servePlanService(t)} - m := NewManager(pkgs, log, WithContainerRuntime(rt), WithKeepServers(true)) + m := NewManager(pkgs, log, WithContainerRuntime(rt)) - if _, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"})); err != nil { + if _, err := m.Start(ctx, []Input{{Image: "xpkg.example.org/provider-aws-ec2:v2.9.0", Cleanup: CleanupOrphan}}); err != nil { t.Fatalf("Start(...): unexpected error: %v", err) } if err := m.Stop(ctx); err != nil { t.Fatalf("Stop(...): unexpected error: %v", err) } if len(rt.stopped) != 0 { - t.Errorf("Stop(...): want containers left running with keep, got stops %v", rt.stopped) + t.Errorf("Stop(...): want containers left running with the Orphan cleanup policy, got stops %v", rt.stopped) } }) diff --git a/internal/docker/docker.go b/internal/docker/docker.go index 3331b4f8..e97e5894 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -270,6 +270,16 @@ func StartWithCopyFiles(tarball []byte, path string) StartContainerOption { } } +// StartWithLabels adds labels to the container. +func StartWithLabels(labels map[string]string) StartContainerOption { + return func(cfg *startContainerConfig) { + if cfg.containerConfig.Labels == nil { + cfg.containerConfig.Labels = map[string]string{} + } + maps.Copy(cfg.containerConfig.Labels, labels) + } +} + // StartWithPublishedPort publishes the container's TCP containerPort on an // ephemeral port on hostIP; the engine allocates the host port. Use // GetHostPort after the container starts to learn the allocated address. diff --git a/internal/terminal/spinner.go b/internal/terminal/spinner.go index 091ee446..959e4adc 100644 --- a/internal/terminal/spinner.go +++ b/internal/terminal/spinner.go @@ -23,6 +23,7 @@ import ( "os" "strings" "sync" + "syscall" "time" bspinner "charm.land/bubbles/v2/spinner" @@ -48,17 +49,17 @@ func ttyInput() io.Reader { } // interruptKey handles ctrl+c for a spinner program with a TTY attached: raw -// mode swallows SIGINT, so the key arrives as input and must mirror what the -// signal handler does for external signals. -func interruptKey(p *tea.Program, msg tea.Msg) { +// mode swallows the terminal's SIGINT, so the key arrives as input and must +// be re-raised as the signal it stands for. Main's signal.NotifyContext then +// cancels the command context, in-flight work unwinds, and deferred cleanup +// runs, exactly as for an external signal. Exiting here instead would skip +// that cleanup. +func interruptKey(msg tea.Msg) { k, ok := msg.(tea.KeyPressMsg) if !ok || k.String() != "ctrl+c" { return } - if p != nil { - _ = p.ReleaseTerminal() - } - os.Exit(130) + _ = syscall.Kill(os.Getpid(), syscall.SIGINT) } // SpinnerPrinter prints spinners to the console. @@ -236,7 +237,7 @@ func (m *MultiSpinner) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.mu.Lock() defer m.mu.Unlock() - interruptKey(m.program, msg) + interruptKey(msg) if _, ok := msg.(tickMsg); !ok { return m, nil @@ -379,7 +380,7 @@ func (ss *SuccessSpinner) Update(msg tea.Msg) (tea.Model, tea.Cmd) { ss.mu.Lock() defer ss.mu.Unlock() - interruptKey(ss.program, msg) + interruptKey(msg) if _, ok := msg.(tickMsg); !ok { return ss, nil From 52032593e435eb7331b40f4127f29cd30adcc2eb Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Fri, 16 Oct 2026 19:47:13 +0200 Subject: [PATCH 5/9] revert(terminal): drop spinner tty/interrupt changes from this PR Signed-off-by: Christopher Haar --- internal/terminal/spinner.go | 38 ++---------------------------------- 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/internal/terminal/spinner.go b/internal/terminal/spinner.go index 959e4adc..a3fcf54d 100644 --- a/internal/terminal/spinner.go +++ b/internal/terminal/spinner.go @@ -23,45 +23,15 @@ import ( "os" "strings" "sync" - "syscall" "time" bspinner "charm.land/bubbles/v2/spinner" tea "charm.land/bubbletea/v2" "charm.land/lipgloss/v2" - "github.com/charmbracelet/x/term" "github.com/crossplane/cli/v2/internal/async" ) -// ttyInput returns the input to attach to a spinner program. The spinners -// never use key input, but bubbletea queries the terminal for capabilities -// (synchronized output and grapheme clustering) on startup, and the terminal -// writes its replies to stdin: without a reader in raw mode they echo as -// stray escape sequences after the spinner text. Attaching a TTY stdin lets -// bubbletea consume them. Piped stdin stays detached so spinners never eat -// piped data. -func ttyInput() io.Reader { - if term.IsTerminal(os.Stdin.Fd()) { - return os.Stdin - } - return nil -} - -// interruptKey handles ctrl+c for a spinner program with a TTY attached: raw -// mode swallows the terminal's SIGINT, so the key arrives as input and must -// be re-raised as the signal it stands for. Main's signal.NotifyContext then -// cancels the command context, in-flight work unwinds, and deferred cleanup -// runs, exactly as for an external signal. Exiting here instead would skip -// that cleanup. -func interruptKey(msg tea.Msg) { - k, ok := msg.(tea.KeyPressMsg) - if !ok || k.String() != "ctrl+c" { - return - } - _ = syscall.Kill(os.Getpid(), syscall.SIGINT) -} - // SpinnerPrinter prints spinners to the console. type SpinnerPrinter interface { // NewSuccessSpinner returns a new success spinner. @@ -237,8 +207,6 @@ func (m *MultiSpinner) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.mu.Lock() defer m.mu.Unlock() - interruptKey(msg) - if _, ok := msg.(tickMsg); !ok { return m, nil } @@ -308,7 +276,7 @@ func (m *MultiSpinner) Fail(title string) { // Start starts the spinners. func (m *MultiSpinner) Start() { m.program = tea.NewProgram(m, - tea.WithInput(ttyInput()), + tea.WithInput(nil), tea.WithoutSignalHandler(), tea.WithOutput(m.out), ) @@ -380,8 +348,6 @@ func (ss *SuccessSpinner) Update(msg tea.Msg) (tea.Model, tea.Cmd) { ss.mu.Lock() defer ss.mu.Unlock() - interruptKey(msg) - if _, ok := msg.(tickMsg); !ok { return ss, nil } @@ -449,7 +415,7 @@ func (ss *SuccessSpinner) Logf(format string, args ...any) { func (ss *SuccessSpinner) Start() { ss.program = tea.NewProgram(ss, tea.WithOutput(ss.out), - tea.WithInput(ttyInput()), + tea.WithInput(nil), tea.WithoutSignalHandler(), ) From 387d119835196889d71a099db384446b5e603716 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Sat, 17 Oct 2026 08:01:31 +0200 Subject: [PATCH 6/9] fix(simulate): address review feedback on discovery, rendering and docs Signed-off-by: Christopher Haar --- cmd/crossplane/simulate/help/simulate.md | 18 ++-- cmd/crossplane/simulate/simulate.go | 116 +++++++++++++++++------ cmd/crossplane/simulate/simulate_test.go | 91 +++++++++++++++--- internal/plan/output.go | 60 ++++++++---- internal/plan/output_test.go | 19 ++-- internal/plan/plan.go | 14 +-- internal/plan/plan_test.go | 11 ++- 7 files changed, 239 insertions(+), 90 deletions(-) diff --git a/cmd/crossplane/simulate/help/simulate.md b/cmd/crossplane/simulate/help/simulate.md index ebeb18be..c0d5a91d 100644 --- a/cmd/crossplane/simulate/help/simulate.md +++ b/cmd/crossplane/simulate/help/simulate.md @@ -16,29 +16,29 @@ by default: the CLI follows each resource's kind to the provider that installed it on the cluster, so the preview matches the provider versions actually serving those resources. --providers overrides that discovery with a file of Provider manifests (like render takes Function manifests), for -example to preview how a provider version that is not installed yet would +example to preview how a provider version that isn't installed yet would treat existing resources. The manifests also carry per-provider runtime annotations (`simulate.crossplane.io/runtime-docker-publish-address` and `simulate.crossplane.io/runtime-docker-target`) for Docker setups where the -published port is not reachable on the address the daemon reports. +published port isn't reachable on the address the daemon reports. Not every provider serves diffs. A provider package declares the capability in its metadata, and a resource whose provider serves no diff server (or -whose diff server cannot compute a diff for it) still appears in the preview +whose diff server can't compute a diff for it) still appears in the preview with an approximate client-side comparison, marked [?], rather than failing -the run or silently dropping out. +the run or dropping out of the preview. Nothing simulate does writes to the cluster or the cloud: it reads the cluster with your kubeconfig and RBAC, and the diff servers run locally with no credentials and no cluster access. -Diff server startup takes a few seconds per provider image. In an +Diff server startup takes seconds per provider image. In an edit-simulate-edit loop, annotate a provider manifest with `simulate.crossplane.io/runtime-docker-cleanup: Orphan` (like render's per-function cleanup annotation) to leave its container running so the next -run reuses it; containers are named `crossplane-diff-server-DIGEST`, carry -`simulate.crossplane.io/managed-by` and session labels, and are cleaned up by -any later run without the annotation, or manually with docker rm. +run reuses it. The containers have names like `crossplane-diff-server-DIGEST` +and carry `simulate.crossplane.io/managed-by` and session labels; any later +run without the annotation cleans them up, or remove them with docker rm. ## Examples @@ -55,7 +55,7 @@ from a checked-in file of Provider manifests: crossplane resource simulate -f resources.yaml --providers providers.yaml ``` -Keep diff servers running for faster subsequent runs, via a provider manifest +Keep diff servers running so later runs start faster, via a provider manifest annotated with `simulate.crossplane.io/runtime-docker-cleanup: Orphan`: ```shell diff --git a/cmd/crossplane/simulate/simulate.go b/cmd/crossplane/simulate/simulate.go index 4aef57ca..a31b8749 100644 --- a/cmd/crossplane/simulate/simulate.go +++ b/cmd/crossplane/simulate/simulate.go @@ -31,6 +31,7 @@ import ( "github.com/alecthomas/kong" diffv1alpha1 "github.com/crossplane/upjet/v2/proto/diff/v1alpha1" "github.com/spf13/afero" + "google.golang.org/protobuf/types/known/structpb" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -85,11 +86,11 @@ const ( // Cmd previews what applying managed resources would do. type Cmd struct { - Filename string `help:"YAML file(s) or directory with managed resources, or '-' for stdin (comma-separated)." predictor:"yaml_file_or_directory" required:"" short:"f"` + Filename string `help:"YAML files or directory with managed resources, or '-' for stdin (comma-separated)." predictor:"yaml_file_or_directory" required:"" short:"f"` Providers string `help:"YAML file or directory with Provider manifests to run as diff servers, overriding discovery from the cluster. Supports per-provider runtime annotations." name:"providers" optional:"" predictor:"yaml_file_or_directory" type:"path"` ImageConfigs string `help:"YAML file with ImageConfig manifests that rewrite provider image paths for local pulls. Defaults to the project's imageConfigs when a project file is present." name:"image-configs" optional:"" predictor:"yaml_file" type:"existingfile"` - ProjectFile string `default:"${project_file}" help:"Path to a project definition file to read imageConfigs from." name:"project-file"` + ProjectFile string `help:"Path to a project definition file to read imageConfigs from. Defaults to ${project_file} when present." name:"project-file" optional:""` SessionID string `help:"Session ID to apply as the simulate.crossplane.io/session label on diff server containers. Defaults to a random per-process ID." placeholder:"ID"` Context string `default:"" help:"Kubernetes context." name:"context" predictor:"context" short:"c"` CacheDir string `default:"~/.crossplane/cache" help:"Cache directory for downloaded provider packages." name:"cache-dir" predictor:"directory"` @@ -132,6 +133,11 @@ func (c *Cmd) Run(ctx context.Context, k *kong.Context, logger logging.Logger, s return errors.Errorf("no resources found in %q", c.Filename) } + renderer, err := plan.NewRenderer(c.Output) + if err != nil { + return err + } + if err := docker.Check(ctx); err != nil { return errors.Wrap(err, "cannot connect to Docker, which simulate needs to run provider diff servers") } @@ -194,34 +200,43 @@ func (c *Cmd) Run(ctx context.Context, k *kong.Context, logger logging.Logger, s } plans := make([]*plan.ResourcePlan, 0, len(resources)) + stores := newObjectStoreCache() for _, desired := range resources { - plans = append(plans, c.simulateOne(ctx, kubeClient, rmapper, defaultNamespace, mgr, desired)) + plans = append(plans, c.simulateOne(ctx, kubeClient, rmapper, defaultNamespace, mgr, stores, desired)) } - return plan.Render(k.Stdout, c.Output, plans) + return renderer.Render(k.Stdout, plans) } // imageConfigs returns the ImageConfigs to rewrite provider image paths with: // the --image-configs file when given, else the imageConfigs of the project // the command happens to run in, if any. Running outside a project is the -// norm for this command, so a missing project file is not an error. +// norm for this command, so a missing project file is only an error when the +// user asked for one explicitly with --project-file. func (c *Cmd) imageConfigs() ([]pkgv1beta1.ImageConfig, error) { if c.ImageConfigs != "" { ics, err := loadImageConfigs(c.ImageConfigs) return ics, errors.Wrapf(err, "cannot load image configs from %q", c.ImageConfigs) } - projFilePath, err := filepath.Abs(c.ProjectFile) + name, explicit := c.ProjectFile, true + if name == "" { + name, explicit = clixpkg.ProjectFile, false + } + projFilePath, err := filepath.Abs(name) if err != nil { - return nil, nil //nolint:nilerr // Best effort: no resolvable project file means no project image configs. + return nil, errors.Wrapf(err, "cannot determine path for %q", name) } if _, err := os.Stat(projFilePath); err != nil { + if explicit { + return nil, errors.Wrapf(err, "cannot read project file %q", name) + } return nil, nil //nolint:nilerr // Best effort: no project file means no project image configs. } projFS := afero.NewBasePathFs(afero.NewOsFs(), filepath.Dir(projFilePath)) proj, err := projectfile.Parse(projFS, filepath.Base(projFilePath)) if err != nil { - return nil, errors.Wrapf(err, "cannot parse project file %q", c.ProjectFile) + return nil, errors.Wrapf(err, "cannot parse project file %q", name) } return proj.Spec.ImageConfigs, nil } @@ -303,7 +318,7 @@ type router interface { // to a diff server, and fall back to the approximate client-side diff when no // server can plan it. Failures become per-resource errors so one resource // never fails the whole run. -func (c *Cmd) simulateOne(ctx context.Context, kubeClient client.Client, rmapper meta.RESTMapper, defaultNamespace string, mgr router, desired *unstructured.Unstructured) *plan.ResourcePlan { +func (c *Cmd) simulateOne(ctx context.Context, kubeClient client.Client, rmapper meta.RESTMapper, defaultNamespace string, mgr router, stores *objectStoreCache, desired *unstructured.Unstructured) *plan.ResourcePlan { if desired.GetName() == "" { p := plan.NewResourcePlan(desired) p.Error = "resource has no name" @@ -331,7 +346,7 @@ func (c *Cmd) simulateOne(ctx context.Context, kubeClient client.Client, rmapper pctx, cancel := context.WithTimeout(ctx, c.Timeout) defer cancel() - store := objectStore(ctx, kubeClient, rmapper, defaultNamespace, desired, live) + store := stores.gather(ctx, kubeClient, rmapper, defaultNamespace, desired, live) p, err := plan.Plan(pctx, planClient, desired, live, store) if err != nil { @@ -377,7 +392,25 @@ func fetchLive(ctx context.Context, kubeClient client.Client, rmapper meta.RESTM return live, nil } -// objectStore gathers the cluster objects the diff server needs to resolve +// objectStoreCache caches gathered object stores by the resolved +// ProviderConfig identity, so a run over many resources referencing the same +// ProviderConfig fetches and converts it once rather than per resource. +// Missing objects are cached too: a reference the cluster cannot resolve is +// not refetched for every resource that carries it. +type objectStoreCache struct { + cache map[objectStoreKey][]*structpb.Struct +} + +type objectStoreKey struct { + gvk schema.GroupVersionKind + key client.ObjectKey +} + +func newObjectStoreCache() *objectStoreCache { + return &objectStoreCache{cache: map[objectStoreKey][]*structpb.Struct{}} +} + +// gather returns the cluster objects the diff server needs to resolve // the desired resource: today the ProviderConfig it references. The diff // server is offline and only sees what travels in the request, so anything it // must resolve is fetched here, with the user's own kubeconfig and RBAC. The @@ -385,7 +418,7 @@ func fetchLive(ctx context.Context, kubeClient client.Client, rmapper meta.RESTM // offline with placeholder credentials and never needs the real ones. // Gathering is best effort: a missing object simply stays out of the store, // and the server reports precisely what it could not resolve. -func objectStore(ctx context.Context, kubeClient client.Client, rmapper meta.RESTMapper, defaultNamespace string, desired, live *unstructured.Unstructured) []*unstructured.Unstructured { +func (c *objectStoreCache) gather(ctx context.Context, kubeClient client.Client, rmapper meta.RESTMapper, defaultNamespace string, desired, live *unstructured.Unstructured) []*structpb.Struct { // Read the reference from the live resource when one exists: the API // server has already applied the CRD's defaults there, so its kind and // name are authoritative. The desired manifest may omit both. @@ -409,20 +442,26 @@ func objectStore(ctx context.Context, kubeClient client.Client, rmapper meta.RES } } + // The ProviderConfig's group relates to the resource's group, but not + // uniformly: most providers serve it one level up (s3.aws.upbound.io → + // aws.upbound.io), others in the resource's own group + // (kubernetes.crossplane.io). Probe from the resource's group toward the + // root and take the first group that serves the kind. group := desired.GroupVersionKind().Group - i := strings.Index(group, ".") - if i < 0 { - return nil - } - group = group[i+1:] - - mapping, err := rmapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}) - if err != nil { - return nil + var mapping *meta.RESTMapping + for { + m, err := rmapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}) + if err == nil { + mapping = m + break + } + i := strings.Index(group, ".") + if i < 0 { + return nil + } + group = group[i+1:] } - pc := &unstructured.Unstructured{} - pc.SetGroupVersionKind(mapping.GroupVersionKind) key := client.ObjectKey{Name: name} if mapping.Scope.Name() == meta.RESTScopeNameNamespace { key.Namespace = desired.GetNamespace() @@ -430,19 +469,31 @@ func objectStore(ctx context.Context, kubeClient client.Client, rmapper meta.RES key.Namespace = defaultNamespace } } - if err := kubeClient.Get(ctx, key, pc); err != nil { - return nil + + ck := objectStoreKey{gvk: mapping.GroupVersionKind, key: key} + if store, ok := c.cache[ck]; ok { + return store + } + + var store []*structpb.Struct + pc := &unstructured.Unstructured{} + pc.SetGroupVersionKind(mapping.GroupVersionKind) + if err := kubeClient.Get(ctx, key, pc); err == nil { + if s, err := structpb.NewStruct(pc.Object); err == nil { + store = []*structpb.Struct{s} + } } - return []*unstructured.Unstructured{pc} + c.cache[ck] = store + return store } // relevantProviderImages returns the package images of exactly the providers // that serve the given resources, so simulate never starts a diff server it // will not route to. Discovery follows the ownership chain on the cluster: -// each resource's kind maps to a CRD, the CRD is owned by the -// ProviderRevision that installed it, and the revision carries the resolved -// package image. Kinds that resolve to no provider are skipped; their -// resources degrade to the approximate client-side diff. +// each resource's kind maps to a CRD, the CRD is controlled by the active +// ProviderRevision, and the revision carries the resolved package image. +// Kinds that resolve to no provider are skipped; their resources degrade to +// the approximate client-side diff. func relevantProviderImages(ctx context.Context, kubeClient client.Client, rmapper meta.RESTMapper, resources []*unstructured.Unstructured, preferOriginal bool) ([]string, error) { seenGK := map[schema.GroupKind]bool{} seenImage := map[string]bool{} @@ -471,9 +522,12 @@ func relevantProviderImages(ctx context.Context, kubeClient client.Client, rmapp return nil, errors.Wrapf(err, "cannot get CRD %s", crdName) } + // After an upgrade every revision of the provider still owns the CRD; + // only the active revision holds the controller reference, and that is + // the one serving the resources. revision := "" for _, ref := range crd.GetOwnerReferences() { - if ref.Kind == "ProviderRevision" && strings.HasPrefix(ref.APIVersion, "pkg.crossplane.io/") { + if ref.Kind == "ProviderRevision" && strings.HasPrefix(ref.APIVersion, "pkg.crossplane.io/") && ref.Controller != nil && *ref.Controller { revision = ref.Name break } diff --git a/cmd/crossplane/simulate/simulate_test.go b/cmd/crossplane/simulate/simulate_test.go index 14d20584..f0d0fe6b 100644 --- a/cmd/crossplane/simulate/simulate_test.go +++ b/cmd/crossplane/simulate/simulate_test.go @@ -34,7 +34,9 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/crossplane/crossplane/apis/v2/pkg" pkgv1 "github.com/crossplane/crossplane/apis/v2/pkg/v1" @@ -156,7 +158,7 @@ func TestSimulateOne(t *testing.T) { kube := fake.NewClientBuilder().WithScheme(s).WithObjects(live).Build() pc := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_UPDATE}} - got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, vpc("my-vpc", "10.1.0.0/16")) + got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, newObjectStoreCache(), vpc("my-vpc", "10.1.0.0/16")) if got.Action != plan.ActionUpdate || got.Approximate || got.Error != "" { t.Errorf("simulateOne(...): want a provider-computed update, got %+v", got) @@ -170,7 +172,7 @@ func TestSimulateOne(t *testing.T) { kube := fake.NewClientBuilder().WithScheme(s).Build() pc := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_CREATE}} - got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, vpc("new-vpc", "10.0.0.0/16")) + got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, newObjectStoreCache(), vpc("new-vpc", "10.0.0.0/16")) if got.Action != plan.ActionCreate || got.Error != "" { t.Errorf("simulateOne(...): want a create, got %+v", got) @@ -184,7 +186,7 @@ func TestSimulateOne(t *testing.T) { live := vpc("my-vpc", "10.0.0.0/16") kube := fake.NewClientBuilder().WithScheme(s).WithObjects(live).Build() - got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{}, vpc("my-vpc", "10.1.0.0/16")) + got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{}, newObjectStoreCache(), vpc("my-vpc", "10.1.0.0/16")) if !got.Approximate || got.Action != plan.ActionUpdate { t.Errorf("simulateOne(...): want an approximate update fallback, got %+v", got) @@ -196,7 +198,7 @@ func TestSimulateOne(t *testing.T) { kube := fake.NewClientBuilder().WithScheme(s).WithObjects(live).Build() pc := &fakePlanClient{err: notSupportedErr(t)} - got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, vpc("my-vpc", "10.1.0.0/16")) + got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, newObjectStoreCache(), vpc("my-vpc", "10.1.0.0/16")) if !got.Approximate || got.Action != plan.ActionUpdate { t.Errorf("simulateOne(...): want an approximate fallback on DIFF_COMPUTATION_NOT_SUPPORTED, got %+v", got) @@ -207,7 +209,7 @@ func TestSimulateOne(t *testing.T) { kube := fake.NewClientBuilder().WithScheme(s).Build() pc := &fakePlanClient{err: status.Error(codes.Internal, "boom")} - got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, vpc("my-vpc", "10.1.0.0/16")) + got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, newObjectStoreCache(), vpc("my-vpc", "10.1.0.0/16")) if got.Error == "" { t.Errorf("simulateOne(...): want a per-resource error, got %+v", got) @@ -248,7 +250,7 @@ func TestSimulateOne(t *testing.T) { kube := fake.NewClientBuilder().WithScheme(seedScheme).WithObjects(pc, secret).Build() pc2 := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_CREATE}} - got := cmd.simulateOne(ctx, kube, seedRM, "default", &fakeRouter{client: pc2}, vpc("my-vpc", "10.0.0.0/16")) + got := cmd.simulateOne(ctx, kube, seedRM, "default", &fakeRouter{client: pc2}, newObjectStoreCache(), vpc("my-vpc", "10.0.0.0/16")) if got.Error != "" { t.Fatalf("simulateOne(...): unexpected error: %s", got.Error) @@ -261,6 +263,57 @@ func TestSimulateOne(t *testing.T) { } }) + t.Run("FindsProviderConfigInResourceGroup", func(t *testing.T) { + // provider-kubernetes serves its ProviderConfig in the same group as + // its resources, not one level up like upjet-based providers. + objGVK := schema.GroupVersionKind{Group: "kubernetes.crossplane.io", Version: "v1alpha2", Kind: "Object"} + pcGVK := schema.GroupVersionKind{Group: "kubernetes.crossplane.io", Version: "v1alpha1", Kind: "ProviderConfig"} + + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(objGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(objGVK.GroupVersion().WithKind("ObjectList"), &unstructured.UnstructuredList{}) + scheme.AddKnownTypeWithName(pcGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(pcGVK.GroupVersion().WithKind("ProviderConfigList"), &unstructured.UnstructuredList{}) + + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{objGVK.GroupVersion(), pcGVK.GroupVersion()}) + mapper.Add(objGVK, meta.RESTScopeRoot) + mapper.Add(pcGVK, meta.RESTScopeRoot) + + pc := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "kubernetes.crossplane.io/v1alpha1", + "kind": "ProviderConfig", + "metadata": map[string]any{"name": "default"}, + }} + desired := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "kubernetes.crossplane.io/v1alpha2", + "kind": "Object", + "metadata": map[string]any{"name": "my-object"}, + }} + + gets := 0 + kube := fake.NewClientBuilder().WithScheme(scheme).WithObjects(pc). + WithInterceptorFuncs(interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + gets++ + return c.Get(ctx, key, obj, opts...) + }}).Build() + + stores := newObjectStoreCache() + got := stores.gather(ctx, kube, mapper, "default", desired, nil) + + if len(got) != 1 || got[0].GetFields()["kind"].GetStringValue() != "ProviderConfig" { + t.Errorf("gather(...): want the same-group ProviderConfig, got %v", got) + } + + // A second resource referencing the same ProviderConfig is served from + // the cache without another fetch. + if again := stores.gather(ctx, kube, mapper, "default", desired, nil); len(again) != 1 { + t.Errorf("gather(...): want the cached ProviderConfig, got %v", again) + } + if gets != 1 { + t.Errorf("gather(...): want 1 fetch for 2 gathers of the same ProviderConfig, got %d", gets) + } + }) + t.Run("DiscoversOnlyRelevantProviders", func(t *testing.T) { crdGVK := schema.GroupVersionKind{Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"} @@ -271,17 +324,29 @@ func TestSimulateOne(t *testing.T) { t.Fatalf("cannot add pkg scheme: %v", err) } + // After an upgrade the inactive old revision still owns the CRD; only + // the active revision is its controller, and discovery must follow + // the controller reference. crd := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", "metadata": map[string]any{ "name": "vpcs.ec2.aws.upbound.io", - "ownerReferences": []any{map[string]any{ - "apiVersion": "pkg.crossplane.io/v1", - "kind": "ProviderRevision", - "name": "provider-aws-ec2-abc123", - "uid": "00000000-0000-0000-0000-000000000000", - }}, + "ownerReferences": []any{ + map[string]any{ + "apiVersion": "pkg.crossplane.io/v1", + "kind": "ProviderRevision", + "name": "provider-aws-ec2-old999", + "uid": "00000000-0000-0000-0000-000000000001", + }, + map[string]any{ + "apiVersion": "pkg.crossplane.io/v1", + "kind": "ProviderRevision", + "name": "provider-aws-ec2-abc123", + "uid": "00000000-0000-0000-0000-000000000000", + "controller": true, + }, + }, }, }} prv := &pkgv1.ProviderRevision{} @@ -313,7 +378,7 @@ func TestSimulateOne(t *testing.T) { emptyRM := meta.NewDefaultRESTMapper(nil) pc := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_CREATE}} - got := cmd.simulateOne(ctx, kube, emptyRM, "default", &fakeRouter{client: pc}, vpc("my-vpc", "10.0.0.0/16")) + got := cmd.simulateOne(ctx, kube, emptyRM, "default", &fakeRouter{client: pc}, newObjectStoreCache(), vpc("my-vpc", "10.0.0.0/16")) if got.Action != plan.ActionCreate || got.Error != "" { t.Errorf("simulateOne(...): want a create for an unserved kind, got %+v", got) diff --git a/internal/plan/output.go b/internal/plan/output.go index 756a00ce..dc049f6d 100644 --- a/internal/plan/output.go +++ b/internal/plan/output.go @@ -105,30 +105,56 @@ func NewResult(plans []*ResourcePlan) Result { } } -// Render writes the plans to w in the given format. -func Render(w io.Writer, format string, plans []*ResourcePlan) error { +// A Renderer writes resource plans to w in one output format. +type Renderer interface { + Render(w io.Writer, plans []*ResourcePlan) error +} + +// NewRenderer returns the Renderer for the given output format. +func NewRenderer(format string) (Renderer, error) { switch format { case FormatPlan: - return renderPlan(w, plans) + return PlanRenderer{}, nil case FormatJSON: - out, err := json.MarshalIndent(NewResult(plans), "", " ") - if err != nil { - return errors.Wrap(err, "cannot marshal plans to JSON") - } - _, err = fmt.Fprintln(w, string(out)) - return err + return JSONRenderer{}, nil case FormatYAML: - out, err := yaml.Marshal(NewResult(plans)) - if err != nil { - return errors.Wrap(err, "cannot marshal plans to YAML") - } - _, err = fmt.Fprint(w, string(out)) - return err + return YAMLRenderer{}, nil } - return errors.Errorf("unknown output format %q", format) + return nil, errors.Errorf("unknown output format %q", format) } -func renderPlan(w io.Writer, plans []*ResourcePlan) error { +// JSONRenderer renders the serialized Result as indented JSON. +type JSONRenderer struct{} + +// Render writes the plans to w as JSON. +func (JSONRenderer) Render(w io.Writer, plans []*ResourcePlan) error { + out, err := json.MarshalIndent(NewResult(plans), "", " ") + if err != nil { + return errors.Wrap(err, "cannot marshal plans to JSON") + } + _, err = fmt.Fprintln(w, string(out)) + return err +} + +// YAMLRenderer renders the serialized Result as YAML. +type YAMLRenderer struct{} + +// Render writes the plans to w as YAML. +func (YAMLRenderer) Render(w io.Writer, plans []*ResourcePlan) error { + out, err := yaml.Marshal(NewResult(plans)) + if err != nil { + return errors.Wrap(err, "cannot marshal plans to YAML") + } + _, err = fmt.Fprint(w, string(out)) + return err +} + +// PlanRenderer renders the human-readable plan format: a header and change +// tree per resource, then a summary with warnings. +type PlanRenderer struct{} + +// Render writes the plans to w in the human-readable plan format. +func (PlanRenderer) Render(w io.Writer, plans []*ResourcePlan) error { for _, p := range plans { _, _ = fmt.Fprintln(w, header(p)) if p.Error != "" { diff --git a/internal/plan/output_test.go b/internal/plan/output_test.go index a5339bcb..a373c7ee 100644 --- a/internal/plan/output_test.go +++ b/internal/plan/output_test.go @@ -102,7 +102,11 @@ Warning: 1 resource could not be planned. ` var buf bytes.Buffer - if err := Render(&buf, FormatPlan, testPlans()); err != nil { + r, err := NewRenderer(FormatPlan) + if err != nil { + t.Fatalf("NewRenderer(plan): unexpected error: %v", err) + } + if err := r.Render(&buf, testPlans()); err != nil { t.Fatalf("Render(plan): unexpected error: %v", err) } if diff := cmp.Diff(want, buf.String()); diff != "" { @@ -110,16 +114,19 @@ Warning: 1 resource could not be planned. } } -func TestRenderUnknownFormat(t *testing.T) { - var buf bytes.Buffer - if err := Render(&buf, "diff", testPlans()); err == nil { - t.Errorf("Render(diff): want an error for an unsupported format") +func TestNewRendererUnknownFormat(t *testing.T) { + if _, err := NewRenderer("diff"); err == nil { + t.Errorf("NewRenderer(diff): want an error for an unsupported format") } } func TestRenderJSON(t *testing.T) { var buf bytes.Buffer - if err := Render(&buf, FormatJSON, testPlans()); err != nil { + r, err := NewRenderer(FormatJSON) + if err != nil { + t.Fatalf("NewRenderer(json): unexpected error: %v", err) + } + if err := r.Render(&buf, testPlans()); err != nil { t.Fatalf("Render(json): unexpected error: %v", err) } diff --git a/internal/plan/plan.go b/internal/plan/plan.go index bff4dce6..581665d3 100644 --- a/internal/plan/plan.go +++ b/internal/plan/plan.go @@ -31,13 +31,14 @@ import ( // status.atProvider and external-name annotation; nil means the resource does // not exist yet and plans as a create. store seeds the server's in-memory // Kubernetes client with objects (Secrets, referenced resources) it may need -// to resolve the desired state. +// to resolve the desired state; it is already in proto form so callers can +// convert a shared object once across many Plan calls. // // gRPC errors are returned as-is so callers can branch on them; use // IsNotSupported to detect the "this resource cannot be planned" case that // should fall back to an approximate client-side diff. -func Plan(ctx context.Context, c diffv1alpha1.PlanServiceClient, desired, actual *unstructured.Unstructured, store []*unstructured.Unstructured) (*ResourcePlan, error) { - req := &diffv1alpha1.PlanRequest{} +func Plan(ctx context.Context, c diffv1alpha1.PlanServiceClient, desired, actual *unstructured.Unstructured, store []*structpb.Struct) (*ResourcePlan, error) { + req := &diffv1alpha1.PlanRequest{KubernetesObjectStore: store} var err error if req.DesiredResource, err = structpb.NewStruct(desired.Object); err != nil { @@ -48,13 +49,6 @@ func Plan(ctx context.Context, c diffv1alpha1.PlanServiceClient, desired, actual return nil, errors.Wrap(err, "cannot convert actual resource") } } - for _, o := range store { - s, err := structpb.NewStruct(o.Object) - if err != nil { - return nil, errors.Wrapf(err, "cannot convert object store entry %s/%s", o.GetNamespace(), o.GetName()) - } - req.KubernetesObjectStore = append(req.KubernetesObjectStore, s) - } rsp, err := c.Plan(ctx, req) if err != nil { diff --git a/internal/plan/plan_test.go b/internal/plan/plan_test.go index 56355b13..3e2cf2c4 100644 --- a/internal/plan/plan_test.go +++ b/internal/plan/plan_test.go @@ -24,7 +24,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "google.golang.org/protobuf/types/known/structpb" ) // fakePlanClient implements diffv1alpha1.PlanServiceClient in-process, @@ -43,16 +43,19 @@ func (f *fakePlanClient) Plan(_ context.Context, in *diffv1alpha1.PlanRequest, _ func TestPlan(t *testing.T) { desired := mr(map[string]any{"forProvider": map[string]any{"cidrBlock": "10.1.0.0/16"}}) actual := mr(map[string]any{"forProvider": map[string]any{"cidrBlock": "10.0.0.0/16"}}) - secret := &unstructured.Unstructured{Object: map[string]any{ + secret, err := structpb.NewStruct(map[string]any{ "apiVersion": "v1", "kind": "Secret", "metadata": map[string]any{"name": "creds", "namespace": "default"}, - }} + }) + if err != nil { + t.Fatalf("cannot convert secret: %v", err) + } t.Run("RequestCarriesAllState", func(t *testing.T) { c := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_UPDATE}} - got, err := Plan(context.Background(), c, desired, actual, []*unstructured.Unstructured{secret}) + got, err := Plan(context.Background(), c, desired, actual, []*structpb.Struct{secret}) if err != nil { t.Fatalf("Plan(...): unexpected error: %v", err) } From 313e587e7bc71e3c2aaf4094e1f030f97d493010 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Sat, 17 Oct 2026 09:27:42 +0200 Subject: [PATCH 7/9] feat(refactor): containerruntime to reuse between functions and diff Signed-off-by: Christopher Haar --- cmd/crossplane/render/runtime_docker.go | 437 ++--------------- cmd/crossplane/render/runtime_docker_test.go | 98 +--- cmd/crossplane/simulate/simulate.go | 20 +- cmd/crossplane/simulate/simulate_test.go | 21 + internal/containerruntime/container.go | 454 ++++++++++++++++++ internal/containerruntime/containerruntime.go | 125 +++++ internal/containerruntime/fake/fake.go | 88 ++++ internal/diffserver/manager.go | 168 ++----- internal/diffserver/manager_test.go | 159 +++--- internal/docker/docker.go | 60 --- internal/plan/output.go | 24 +- 11 files changed, 922 insertions(+), 732 deletions(-) create mode 100644 internal/containerruntime/container.go create mode 100644 internal/containerruntime/containerruntime.go create mode 100644 internal/containerruntime/fake/fake.go diff --git a/cmd/crossplane/render/runtime_docker.go b/cmd/crossplane/render/runtime_docker.go index 466b70a0..17c79413 100644 --- a/cmd/crossplane/render/runtime_docker.go +++ b/cmd/crossplane/render/runtime_docker.go @@ -18,32 +18,25 @@ package render import ( "context" - "encoding/base64" - "fmt" - "io" - "net" - "net/netip" "strings" - "sync" - "time" - "github.com/containerd/errdefs" "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/moby/moby/api/types/container" - "github.com/moby/moby/api/types/network" - "github.com/moby/moby/client" - "k8s.io/apimachinery/pkg/util/rand" "github.com/crossplane/crossplane-runtime/v2/pkg/errors" "github.com/crossplane/crossplane-runtime/v2/pkg/logging" pkgv1 "github.com/crossplane/crossplane/apis/v2/pkg/v1" + + "github.com/crossplane/cli/v2/internal/containerruntime" ) // FunctionPort is the port that Composition Functions listen on inside their container. const FunctionPort = 9443 +// labelDomain prefixes the labels render applies to the Docker resources it +// creates. +const labelDomain = "render.crossplane.io" + // Annotations that can be used to configure the Docker runtime. const ( // AnnotationKeyRuntimeDockerCleanup configures how a Function's Docker @@ -96,88 +89,63 @@ const ( const ( // LabelKeyManagedBy is applied to every Docker container and network // created by render. Its value is always LabelValueManagedByCrossplane. - LabelKeyManagedBy = "render.crossplane.io/managed-by" + LabelKeyManagedBy = labelDomain + "/managed-by" // LabelValueManagedByCrossplane is the value of LabelKeyManagedBy. - LabelValueManagedByCrossplane = "crossplane-cli" + LabelValueManagedByCrossplane = containerruntime.LabelValueManagedBy // LabelKeySession is applied to every Docker container and network created // by render. Its value identifies the render session that created the // resource, so cleanup can be scoped to a single run. Callers may supply // the session ID (see EngineFlags.SessionID); otherwise a per-process ID is // generated. - LabelKeySession = "render.crossplane.io/session" + LabelKeySession = labelDomain + "/session" // LabelKeyCleanup is applied to Function containers and records the // effective DockerCleanup policy the container was created with. - LabelKeyCleanup = "render.crossplane.io/cleanup" + LabelKeyCleanup = labelDomain + "/cleanup" ) // sessionID returns an identifier unique to this process. It is the session ID // used for Docker resources when the caller doesn't supply one. // //nolint:gochecknoglobals // Must be stable for the life of the process. -var sessionID = sync.OnceValue(func() string { return rand.String(12) }) +var sessionID = containerruntime.SessionID // managedLabels returns the labels applied to every Docker resource render // creates. An empty session falls back to the per-process session ID. func managedLabels(session string) map[string]string { - if session == "" { - session = sessionID() - } - - return map[string]string{ - LabelKeyManagedBy: LabelValueManagedByCrossplane, - LabelKeySession: session, - } + return containerruntime.ManagedLabels(labelDomain, session) } // functionContainerLabels returns the labels applied to a Function container // created with the supplied session and cleanup policy. func functionContainerLabels(session string, cleanup DockerCleanup) map[string]string { - l := managedLabels(session) - l[LabelKeyCleanup] = string(cleanup) - return l + return containerruntime.ContainerLabels(labelDomain, session, cleanup) } // DockerCleanup specifies what Docker should do with a Function container after // it has been run. -type DockerCleanup string +type DockerCleanup = containerruntime.Cleanup // Supported AnnotationKeyRuntimeDockerCleanup values. const ( // AnnotationValueRuntimeDockerCleanupStop stops the container once - // rendering is done, waiting up to containerStopGracePeriod for it to exit - // on SIGTERM before Docker kills it. - AnnotationValueRuntimeDockerCleanupStop DockerCleanup = "Stop" + // rendering is done, waiting up to the grace period for it to exit on + // SIGTERM before Docker kills it. + AnnotationValueRuntimeDockerCleanupStop = containerruntime.CleanupStop // AnnotationValueRuntimeDockerCleanupRemove is the default. It stops the - // container once rendering is done, waiting up to containerStopGracePeriod - // for it to exit on SIGTERM, then force removes it. The container is - // removed even if the graceful stop fails. - AnnotationValueRuntimeDockerCleanupRemove DockerCleanup = "Remove" + // container once rendering is done, waiting up to the grace period for it + // to exit on SIGTERM, then force removes it. The container is removed + // even if the graceful stop fails. + AnnotationValueRuntimeDockerCleanupRemove = containerruntime.CleanupRemove // AnnotationValueRuntimeDockerCleanupOrphan leaves the container running // once rendering is done. - AnnotationValueRuntimeDockerCleanupOrphan DockerCleanup = "Orphan" - - AnnotationValueRuntimeDockerCleanupDefault = AnnotationValueRuntimeDockerCleanupRemove -) + AnnotationValueRuntimeDockerCleanupOrphan = containerruntime.CleanupOrphan -const ( - // containerStopGracePeriod is how long the Stop and Remove cleanup - // policies wait for a Function container to exit on SIGTERM before Docker - // kills it. - containerStopGracePeriod = 3 * time.Second - - // containerStopTimeout bounds the ContainerStop request: the grace period - // plus time for Docker to kill the container. - containerStopTimeout = containerStopGracePeriod + 2*time.Second - - // containerRemoveTimeout bounds the ContainerRemove request. It's separate - // from containerStopTimeout so a stop that hangs can't leave removal - // without time to run. - containerRemoveTimeout = 10 * time.Second + AnnotationValueRuntimeDockerCleanupDefault = containerruntime.CleanupDefault ) // AnnotationKeyRuntimeDockerPullPolicy can be added to a Function to control how its runtime @@ -186,21 +154,21 @@ const AnnotationKeyRuntimeDockerPullPolicy = "render.crossplane.io/runtime-docke // DockerPullPolicy can be added to a Function to control how its runtime image // is pulled by Docker. -type DockerPullPolicy string +type DockerPullPolicy = containerruntime.PullPolicy // Supported pull policies. const ( // AnnotationValueRuntimeDockerPullPolicyAlways always pulls the image. - AnnotationValueRuntimeDockerPullPolicyAlways DockerPullPolicy = "Always" + AnnotationValueRuntimeDockerPullPolicyAlways = containerruntime.PullAlways // AnnotationValueRuntimeDockerPullPolicyNever never pulls the image. - AnnotationValueRuntimeDockerPullPolicyNever DockerPullPolicy = "Never" + AnnotationValueRuntimeDockerPullPolicyNever = containerruntime.PullNever // AnnotationValueRuntimeDockerPullPolicyIfNotPresent pulls the image if // it's not present. - AnnotationValueRuntimeDockerPullPolicyIfNotPresent DockerPullPolicy = "IfNotPresent" + AnnotationValueRuntimeDockerPullPolicyIfNotPresent = containerruntime.PullIfNotPresent - AnnotationValueRuntimeDockerPullPolicyDefault DockerPullPolicy = AnnotationValueRuntimeDockerPullPolicyIfNotPresent + AnnotationValueRuntimeDockerPullPolicyDefault = containerruntime.PullDefault ) // RuntimeDocker uses a Docker daemon to run a Function. @@ -248,28 +216,9 @@ type RuntimeDocker struct { // it nil and Start builds a real client from the environment. Tests // substitute a fake to exercise container lifecycle and cleanup handling // without a real Docker daemon. - dockerClient containerClient + dockerClient containerruntime.Client } -// containerClient is the subset of the Docker client RuntimeDocker uses to -// pull images and manage a Function's container. -type containerClient interface { - pullClient - containerCleanupClient - ContainerInspect(ctx context.Context, containerID string, options client.ContainerInspectOptions) (client.ContainerInspectResult, error) - ContainerCreate(ctx context.Context, options client.ContainerCreateOptions) (client.ContainerCreateResult, error) - ContainerStart(ctx context.Context, containerID string, options client.ContainerStartOptions) (client.ContainerStartResult, error) -} - -// containerCleanupClient is the subset of the Docker client RuntimeDocker's -// stop function uses to clean up a Function's container. -type containerCleanupClient interface { - ContainerStop(ctx context.Context, containerID string, options client.ContainerStopOptions) (client.ContainerStopResult, error) - ContainerRemove(ctx context.Context, containerID string, options client.ContainerRemoveOptions) (client.ContainerRemoveResult, error) -} - -var _ containerClient = (*client.Client)(nil) - // GetDockerPullPolicy extracts PullPolicy configuration from the supplied // Function. func GetDockerPullPolicy(fn pkgv1.Function) (DockerPullPolicy, error) { @@ -361,315 +310,33 @@ func GetRuntimeDocker(fn pkgv1.Function, log logging.Logger) (*RuntimeDocker, er var _ Runtime = &RuntimeDocker{} -func (r *RuntimeDocker) findContainer(ctx context.Context, cli containerClient) (string, error) { - if r.Name == "" { - return "", nil - } - - inspect, err := cli.ContainerInspect(ctx, r.Name, client.ContainerInspectOptions{}) - if err != nil { - if errdefs.IsNotFound(err) { - return "", nil // Container doesn't exist, but that's not an error - } - return "", errors.Wrapf(err, "cannot inspect Docker container %q", r.Name) - } - - return inspect.Container.ID, nil -} - -func (r *RuntimeDocker) createContainer(ctx context.Context, cli containerClient) (string, error) { - r.log.Debug("Starting Docker container runtime setup", "image", r.Image) - - // Let Docker automatically allocate an available port on the bind address. - // This avoids race conditions and works reliably with Docker daemons. - port := network.MustParsePort(fmt.Sprintf("%d/tcp", FunctionPort)) - - cfg := &container.Config{ - Image: r.Image, - Cmd: []string{"--insecure"}, - ExposedPorts: network.PortSet{port: struct{}{}}, - Env: r.Env, - // Labels are only set at creation time. A pre-existing named container - // that is reused is not relabeled, since Docker can't change labels on - // an existing container. - Labels: functionContainerLabels(r.Session, r.Cleanup), - } - hcfg := &container.HostConfig{} - var ncfg *network.NetworkingConfig - - if r.Network != "" { - // When a Docker network is specified, the function container joins - // that network and is reached via its Docker hostname on the function - // port. No host port bindings are needed. - ncfg = &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{ - r.Network: {}, - }, - } - r.log.Debug("Connecting container to Docker network", "network", r.Network) - } else { - // PortBinding.HostIP is a netip.Addr in the Moby API types, where it - // was a string; reject a malformed bind address here rather than - // letting it reach the daemon as a zero value. - bindAddr, err := netip.ParseAddr(r.BindAddress) - if err != nil { - return "", errors.Wrapf(err, "invalid bind address %q", r.BindAddress) - } - hcfg.PortBindings = network.PortMap{ - port: []network.PortBinding{{ - HostIP: bindAddr, - HostPort: "0", // "0" => engine allocates an ephemeral port - }}, - } - } - - options, err := r.getPullOptions() - if err != nil { - // We can continue to pull an image if we don't have the PullOptions with RegistryAuth - // as long as the image is from a public registry. Therefore, we log the error message and continue. - r.log.Info("Cannot get pull options", "image", r.Image, "err", err) - } - - if r.PullPolicy == AnnotationValueRuntimeDockerPullPolicyAlways { - r.log.Debug("Pulling image with pullPolicy: Always", "image", r.Image) - - err = PullImage(ctx, cli, r.Image, options) - if err != nil { - return "", errors.Wrapf(err, "cannot pull Docker image %q", r.Image) - } - } - - r.log.Debug("Creating Docker container", "image", r.Image, "name", r.Name, "network", r.Network) - - rsp, err := cli.ContainerCreate(ctx, client.ContainerCreateOptions{ - Config: cfg, - HostConfig: hcfg, - NetworkingConfig: ncfg, - Name: r.Name, - }) - if err != nil { - // TODO: If Docker ever exposes a structured way to distinguish - // network-not-found from image-not-found (e.g. via errdefs or a typed - // error), handle the network case here to avoid an unnecessary image - // pull attempt. For now, both surface as errdefs.IsNotFound and - // string-matching on the daemon message is too fragile. - if !errdefs.IsNotFound(err) { - // Non-image-not-found error: could be network misconfiguration, - // daemon permissions, etc. - if r.Network != "" { - return "", errors.Wrapf(err, "cannot create Docker container for image %q on network %q; verify the network exists and is accessible by the Docker daemon", r.Image, r.Network) - } - return "", errors.Wrapf(err, "cannot create Docker container for image %q", r.Image) - } - if r.PullPolicy == AnnotationValueRuntimeDockerPullPolicyNever { - // Image not found and we're not allowed to pull it. - return "", errors.Wrapf(err, "cannot create Docker container: image %q not found and pull policy is %q", r.Image, r.PullPolicy) - } - - // The image was not found, but we're allowed to pull it. - r.log.Debug("Image not found, pulling", "image", r.Image) - - err = PullImage(ctx, cli, r.Image, options) - if err != nil { - return "", errors.Wrapf(err, "cannot pull Docker image %q", r.Image) - } - - rsp, err = cli.ContainerCreate(ctx, client.ContainerCreateOptions{ - Config: cfg, - HostConfig: hcfg, - NetworkingConfig: ncfg, - Name: r.Name, - }) - if err != nil { - if r.Network != "" { - return "", errors.Wrapf(err, "cannot create Docker container for image %q on network %q; verify the network exists and is accessible by the Docker daemon", r.Image, r.Network) - } - return "", errors.Wrapf(err, "cannot create Docker container for image %q", r.Image) - } - } - - return rsp.ID, nil -} - -// startContainer ensures the container is running and returns its address. -func (r *RuntimeDocker) startContainer(ctx context.Context, cli containerClient, containerID string) (string, error) { - // Start the container (idempotent - safe to call on running containers) - if _, err := cli.ContainerStart(ctx, containerID, client.ContainerStartOptions{}); err != nil { - return "", errors.Wrap(err, "cannot start Docker container") - } - - // When using a Docker network, containers are resolvable by name within - // the network. Inspect the container to validate it is attached to the - // specified network and to get its DNS-resolvable name (Docker's embedded - // DNS resolves container names and hostnames, but not container IDs). - if r.Network != "" { - inspect, err := cli.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{}) - if err != nil { - return "", errors.Wrap(err, "cannot inspect Docker container") - } - if _, ok := inspect.Container.NetworkSettings.Networks[r.Network]; !ok { - return "", errors.Errorf("container %q is not connected to Docker network %q; verify the %q annotation value matches an existing network", strings.TrimPrefix(inspect.Container.Name, "/"), r.Network, AnnotationKeyRuntimeDockerNetwork) - } - hostname := strings.TrimPrefix(inspect.Container.Name, "/") - if hostname == "" { - return "", errors.Errorf("cannot determine hostname for container %q on network %q", containerID, r.Network) - } - addr := net.JoinHostPort(hostname, fmt.Sprintf("%d", FunctionPort)) - r.log.Debug("Function container reachable on network", "network", r.Network, "address", addr) - return addr, nil - } - - // Inspect the container to get the actual allocated host port. - inspect, err := cli.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{}) - if err != nil { - return "", errors.Wrap(err, "cannot inspect Docker container") - } - - // Look up the specific function port instead of taking the first one - p := network.MustParsePort(fmt.Sprintf("%d/tcp", FunctionPort)) - if len(inspect.Container.NetworkSettings.Ports[p]) == 0 { - return "", errors.Errorf("container %q has no published binding for port %s", r.Name, p.Port()) - } - - binding := inspect.Container.NetworkSettings.Ports[p][0] - host := r.Target - if host == "" && binding.HostIP.IsValid() { - host = binding.HostIP.String() - } - if host == "" { - return "", errors.Errorf("container %q has port binding for %s but no host address", r.Name, p.Port()) - } - - return net.JoinHostPort(host, binding.HostPort), nil -} - -func (r *RuntimeDocker) getPullOptions() (client.ImagePullOptions, error) { - // Resolve auth token by looking into keychain - ref, err := name.ParseReference(r.Image) - if err != nil { - return client.ImagePullOptions{}, errors.Wrapf(err, "Image is not a valid reference %s", r.Image) - } - - auth, err := r.Keychain.Resolve(ref.Context().Registry) - if err != nil { - return client.ImagePullOptions{}, errors.Wrapf(err, "Cannot resolve auth for %s", ref.Context().RegistryStr()) - } - - authConfig, err := auth.Authorization() - if err != nil { - return client.ImagePullOptions{}, errors.Wrapf(err, "Cannot get auth config for %s", ref.Context().RegistryStr()) - } - - token, err := authConfig.MarshalJSON() - if err != nil { - return client.ImagePullOptions{}, errors.Wrapf(err, "Cannot marshal auth config for %s", ref.Context().RegistryStr()) - } - - return client.ImagePullOptions{ - RegistryAuth: base64.StdEncoding.EncodeToString(token), - }, nil -} - // Start a Function as a Docker container. func (r *RuntimeDocker) Start(ctx context.Context) (RuntimeContext, error) { - cli := r.dockerClient - if cli == nil { - c, err := client.New(client.FromEnv) - if err != nil { - return RuntimeContext{}, errors.Wrap(err, "cannot create Docker client using environment variables") - } - cli = c - } - - // Try to find an existing container with the supplied container name. - containerID, err := r.findContainer(ctx, cli) - if err != nil { - return RuntimeContext{}, err - } - - // Create new container if not found. - if containerID == "" { - containerID, err = r.createContainer(ctx, cli) - if err != nil { - return RuntimeContext{}, err - } - } - - containerAddr, err := r.startContainer(ctx, cli, containerID) + c := &containerruntime.Container{ + Image: r.Image, + Name: r.Name, + Cmd: []string{"--insecure"}, + Port: FunctionPort, + Env: r.Env, + Labels: functionContainerLabels(r.Session, r.Cleanup), + BindAddress: r.BindAddress, + Target: r.Target, + Network: r.Network, + PullPolicy: r.PullPolicy, + Keychain: r.Keychain, + Log: r.log, + Client: r.dockerClient, + } + + running, err := c.Start(ctx) if err != nil { return RuntimeContext{}, err } - // Inline stop function - stop := func(ctx context.Context) error { - grace := int(containerStopGracePeriod / time.Second) - stopOpts := client.ContainerStopOptions{Timeout: &grace} - - switch r.Cleanup { - case AnnotationValueRuntimeDockerCleanupOrphan: - return nil - case AnnotationValueRuntimeDockerCleanupStop: - if err := stopContainer(ctx, cli, containerID, stopOpts); err != nil { - return errors.Wrap(err, "cannot stop Docker container") - } - case AnnotationValueRuntimeDockerCleanupRemove: - // Give the container a chance to exit gracefully, then force - // remove it whether or not the stop succeeded, so a container - // that's slow to exit on SIGTERM can't cause removal to be - // skipped. A stop failure only matters if removal fails too. - var stopErr error - if err := stopContainer(ctx, cli, containerID, stopOpts); err != nil { - stopErr = errors.Wrap(err, "cannot stop Docker container") - } - if err := removeContainer(ctx, cli, containerID); err != nil { - return errors.Join(errors.Wrap(err, "cannot remove Docker container"), stopErr) - } - } - - return nil - } - - return RuntimeContext{Target: containerAddr, Stop: stop}, nil -} - -// stopContainer stops a container, bounding the request by -// containerStopTimeout. -func stopContainer(ctx context.Context, cli containerCleanupClient, containerID string, opts client.ContainerStopOptions) error { - ctx, cancel := context.WithTimeout(ctx, containerStopTimeout) - defer cancel() - - _, err := cli.ContainerStop(ctx, containerID, opts) - return err -} - -// removeContainer force removes a container, bounding the request by -// containerRemoveTimeout. -func removeContainer(ctx context.Context, cli containerCleanupClient, containerID string) error { - ctx, cancel := context.WithTimeout(ctx, containerRemoveTimeout) - defer cancel() - - _, err := cli.ContainerRemove(ctx, containerID, client.ContainerRemoveOptions{Force: true}) - return err -} - -type pullClient interface { - ImagePull(ctx context.Context, ref string, options client.ImagePullOptions) (client.ImagePullResponse, error) -} - -// PullImage pulls the supplied image using the supplied client. It blocks until -// the image has either finished pulling or hit an error. -func PullImage(ctx context.Context, p pullClient, image string, options client.ImagePullOptions) error { - out, err := p.ImagePull(ctx, image, options) - if err != nil { - return err - } - defer out.Close() //nolint:errcheck // TODO(negz): Can this error? - - // Each line read from out is a JSON object containing the status of the - // pull - similar to the progress bars you'd see if running docker pull. It - // seems that consuming all of this output is the best way to block until - // the image is actually pulled before we try to run it. - _, err = io.Copy(io.Discard, out) - - return err + return RuntimeContext{ + Target: running.Addr, + Stop: func(ctx context.Context) error { + return running.Stop(ctx, r.Cleanup) + }, + }, nil } diff --git a/cmd/crossplane/render/runtime_docker_test.go b/cmd/crossplane/render/runtime_docker_test.go index 612e089c..4e3b62e3 100644 --- a/cmd/crossplane/render/runtime_docker_test.go +++ b/cmd/crossplane/render/runtime_docker_test.go @@ -18,7 +18,6 @@ package render import ( "context" - "io" "net" "strconv" "testing" @@ -36,69 +35,10 @@ import ( "github.com/crossplane/crossplane-runtime/v2/pkg/logging" pkgv1 "github.com/crossplane/crossplane/apis/v2/pkg/v1" -) - -type mockPullClient struct { - MockPullImage func(_ context.Context, ref string, options client.ImagePullOptions) (client.ImagePullResponse, error) -} - -func (m *mockPullClient) ImagePull(ctx context.Context, ref string, options client.ImagePullOptions) (client.ImagePullResponse, error) { - return m.MockPullImage(ctx, ref, options) -} - -var _ pullClient = &mockPullClient{} - -type mockContainerClient struct { - MockImagePull func(ctx context.Context, ref string, options client.ImagePullOptions) (client.ImagePullResponse, error) - MockContainerInspect func(ctx context.Context, containerID string, options client.ContainerInspectOptions) (client.ContainerInspectResult, error) - MockContainerCreate func(ctx context.Context, options client.ContainerCreateOptions) (client.ContainerCreateResult, error) - MockContainerStart func(ctx context.Context, containerID string, options client.ContainerStartOptions) (client.ContainerStartResult, error) - MockContainerStop func(ctx context.Context, containerID string, options client.ContainerStopOptions) (client.ContainerStopResult, error) - MockContainerRemove func(ctx context.Context, containerID string, options client.ContainerRemoveOptions) (client.ContainerRemoveResult, error) -} - -func (m *mockContainerClient) ImagePull(ctx context.Context, ref string, options client.ImagePullOptions) (client.ImagePullResponse, error) { - return m.MockImagePull(ctx, ref, options) -} - -func (m *mockContainerClient) ContainerInspect(ctx context.Context, containerID string, options client.ContainerInspectOptions) (client.ContainerInspectResult, error) { - return m.MockContainerInspect(ctx, containerID, options) -} - -func (m *mockContainerClient) ContainerCreate(ctx context.Context, options client.ContainerCreateOptions) (client.ContainerCreateResult, error) { - return m.MockContainerCreate(ctx, options) -} -func (m *mockContainerClient) ContainerStart(ctx context.Context, containerID string, options client.ContainerStartOptions) (client.ContainerStartResult, error) { - return m.MockContainerStart(ctx, containerID, options) -} - -func (m *mockContainerClient) ContainerStop(ctx context.Context, containerID string, options client.ContainerStopOptions) (client.ContainerStopResult, error) { - return m.MockContainerStop(ctx, containerID, options) -} - -func (m *mockContainerClient) ContainerRemove(ctx context.Context, containerID string, options client.ContainerRemoveOptions) (client.ContainerRemoveResult, error) { - return m.MockContainerRemove(ctx, containerID, options) -} - -var _ containerClient = &mockContainerClient{} - -// imagePullDone is a MockImagePull that reports an image pull that has -// already completed. -func imagePullDone(context.Context, string, client.ImagePullOptions) (client.ImagePullResponse, error) { - return pullDoneResponse{}, nil -} - -// pullDoneResponse is an ImagePullResponse whose body is empty. Only the -// io.ReadCloser methods PullImage uses are implemented; the embedded -// interface is nil. -type pullDoneResponse struct { - client.ImagePullResponse -} - -func (pullDoneResponse) Read([]byte) (int, error) { return 0, io.EOF } - -func (pullDoneResponse) Close() error { return nil } + "github.com/crossplane/cli/v2/internal/containerruntime" + "github.com/crossplane/cli/v2/internal/containerruntime/fake" +) // createContainerReturns returns a MockContainerCreate that creates a // container with the supplied ID. @@ -356,7 +296,7 @@ func TestRuntimeDockerStart(t *testing.T) { type args struct { pullPolicy DockerPullPolicy - cli *mockContainerClient + cli *fake.Client } type want struct { rctx RuntimeContext @@ -372,7 +312,7 @@ func TestRuntimeDockerStart(t *testing.T) { reason: "Start should create, start, and inspect the Function's container through the injected client, without pulling an image that is present.", args: args{ pullPolicy: AnnotationValueRuntimeDockerPullPolicyIfNotPresent, - cli: &mockContainerClient{ + cli: &fake.Client{ MockContainerCreate: createContainerReturns(containerID), MockContainerStart: startContainer(containerID), MockContainerInspect: inspectContainerOnNetwork(containerID, containerName, dockerNetwork), @@ -386,12 +326,12 @@ func TestRuntimeDockerStart(t *testing.T) { reason: "Start should pull the Function's image through the injected client when the pull policy is Always.", args: args{ pullPolicy: AnnotationValueRuntimeDockerPullPolicyAlways, - cli: &mockContainerClient{ + cli: &fake.Client{ MockImagePull: func(ctx context.Context, ref string, options client.ImagePullOptions) (client.ImagePullResponse, error) { if diff := cmp.Diff(image, ref); diff != "" { return nil, errors.Errorf("ImagePull(...): -want ref, +got ref:\n%s", diff) } - return imagePullDone(ctx, ref, options) + return fake.ImagePullDone(ctx, ref, options) }, MockContainerCreate: createContainerReturns(containerID), MockContainerStart: startContainer(containerID), @@ -406,7 +346,7 @@ func TestRuntimeDockerStart(t *testing.T) { reason: "Start should return an error, without creating a container, when the injected client cannot pull the image.", args: args{ pullPolicy: AnnotationValueRuntimeDockerPullPolicyAlways, - cli: &mockContainerClient{ + cli: &fake.Client{ MockImagePull: func(context.Context, string, client.ImagePullOptions) (client.ImagePullResponse, error) { return nil, errBoom }, @@ -420,7 +360,7 @@ func TestRuntimeDockerStart(t *testing.T) { reason: "Start should return an error when the injected client cannot start the container.", args: args{ pullPolicy: AnnotationValueRuntimeDockerPullPolicyIfNotPresent, - cli: &mockContainerClient{ + cli: &fake.Client{ MockContainerCreate: createContainerReturns(containerID), MockContainerStart: func(context.Context, string, client.ContainerStartOptions) (client.ContainerStartResult, error) { return client.ContainerStartResult{}, errBoom @@ -577,7 +517,7 @@ func TestRuntimeDockerStop(t *testing.T) { for name, tc := range cases { t.Run(name, func(t *testing.T) { - cli := &mockContainerClient{ + cli := &fake.Client{ MockContainerCreate: createContainerReturns(containerID), MockContainerStart: startContainer(containerID), MockContainerInspect: inspectContainerOnNetwork(containerID, containerName, dockerNetwork), @@ -594,8 +534,8 @@ func TestRuntimeDockerStop(t *testing.T) { } // The stop request must have its own deadline, so a stop // that hangs can't stall cleanup. - if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > containerStopTimeout { - t.Errorf("\n%s\nContainerStop(...): want a deadline within %s, got deadline %v (set: %t)", tc.reason, containerStopTimeout, deadline, ok) + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > containerruntime.StopTimeout { + t.Errorf("\n%s\nContainerStop(...): want a deadline within %s, got deadline %v (set: %t)", tc.reason, containerruntime.StopTimeout, deadline, ok) } return stop(ctx, id, options) } @@ -610,8 +550,8 @@ func TestRuntimeDockerStop(t *testing.T) { } // The remove request must have its own deadline, so a stop // that hangs can't leave it without time to run. - if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > containerRemoveTimeout || ctx.Err() != nil { - t.Errorf("\n%s\nContainerRemove(...): want a live context with a deadline within %s, got deadline %v (set: %t), err %v", tc.reason, containerRemoveTimeout, deadline, ok, ctx.Err()) + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > containerruntime.RemoveTimeout || ctx.Err() != nil { + t.Errorf("\n%s\nContainerRemove(...): want a live context with a deadline within %s, got deadline %v (set: %t), err %v", tc.reason, containerruntime.RemoveTimeout, deadline, ok, ctx.Err()) } return remove(ctx, id, options) } @@ -673,7 +613,7 @@ func TestRuntimeDockerStartLabels(t *testing.T) { type args struct { cleanup DockerCleanup session string - cli *mockContainerClient + cli *fake.Client } type want struct { err error @@ -688,7 +628,7 @@ func TestRuntimeDockerStartLabels(t *testing.T) { reason: "Function containers should be marked as managed by render and record the Stop policy.", args: args{ cleanup: AnnotationValueRuntimeDockerCleanupStop, - cli: &mockContainerClient{ + cli: &fake.Client{ MockContainerCreate: createLabeledContainer(map[string]string{ LabelKeyManagedBy: LabelValueManagedByCrossplane, LabelKeySession: sessionID(), @@ -703,7 +643,7 @@ func TestRuntimeDockerStartLabels(t *testing.T) { reason: "Function containers should be marked as managed by render and record the Remove policy.", args: args{ cleanup: AnnotationValueRuntimeDockerCleanupRemove, - cli: &mockContainerClient{ + cli: &fake.Client{ MockContainerCreate: createLabeledContainer(map[string]string{ LabelKeyManagedBy: LabelValueManagedByCrossplane, LabelKeySession: sessionID(), @@ -719,7 +659,7 @@ func TestRuntimeDockerStartLabels(t *testing.T) { args: args{ cleanup: AnnotationValueRuntimeDockerCleanupRemove, session: "my-session", - cli: &mockContainerClient{ + cli: &fake.Client{ MockContainerCreate: createLabeledContainer(map[string]string{ LabelKeyManagedBy: LabelValueManagedByCrossplane, LabelKeySession: "my-session", @@ -734,7 +674,7 @@ func TestRuntimeDockerStartLabels(t *testing.T) { reason: "Function containers should record the Orphan policy so sweeps can skip them.", args: args{ cleanup: AnnotationValueRuntimeDockerCleanupOrphan, - cli: &mockContainerClient{ + cli: &fake.Client{ MockContainerCreate: createLabeledContainer(map[string]string{ LabelKeyManagedBy: LabelValueManagedByCrossplane, LabelKeySession: sessionID(), diff --git a/cmd/crossplane/simulate/simulate.go b/cmd/crossplane/simulate/simulate.go index a31b8749..29b5166a 100644 --- a/cmd/crossplane/simulate/simulate.go +++ b/cmd/crossplane/simulate/simulate.go @@ -52,6 +52,7 @@ import ( pkgv1beta1 "github.com/crossplane/crossplane/apis/v2/pkg/v1beta1" "github.com/crossplane/cli/v2/cmd/crossplane/common/load" + "github.com/crossplane/cli/v2/internal/containerruntime" "github.com/crossplane/cli/v2/internal/diffserver" "github.com/crossplane/cli/v2/internal/docker" "github.com/crossplane/cli/v2/internal/kube" @@ -88,14 +89,14 @@ const ( type Cmd struct { Filename string `help:"YAML files or directory with managed resources, or '-' for stdin (comma-separated)." predictor:"yaml_file_or_directory" required:"" short:"f"` - Providers string `help:"YAML file or directory with Provider manifests to run as diff servers, overriding discovery from the cluster. Supports per-provider runtime annotations." name:"providers" optional:"" predictor:"yaml_file_or_directory" type:"path"` - ImageConfigs string `help:"YAML file with ImageConfig manifests that rewrite provider image paths for local pulls. Defaults to the project's imageConfigs when a project file is present." name:"image-configs" optional:"" predictor:"yaml_file" type:"existingfile"` - ProjectFile string `help:"Path to a project definition file to read imageConfigs from. Defaults to ${project_file} when present." name:"project-file" optional:""` + Providers string `help:"YAML file or directory with Provider manifests to run as diff servers, overriding discovery from the cluster. Supports per-provider runtime annotations." name:"providers" optional:"" predictor:"yaml_file_or_directory" type:"path"` + ImageConfigs string `help:"YAML file with ImageConfig manifests that rewrite provider image paths for local pulls. Defaults to the project's imageConfigs when a project file is present." name:"image-configs" optional:"" predictor:"yaml_file" type:"existingfile"` + ProjectFile string `help:"Path to a project definition file to read imageConfigs from. Defaults to ${project_file} when present." name:"project-file" optional:""` SessionID string `help:"Session ID to apply as the simulate.crossplane.io/session label on diff server containers. Defaults to a random per-process ID." placeholder:"ID"` - Context string `default:"" help:"Kubernetes context." name:"context" predictor:"context" short:"c"` - CacheDir string `default:"~/.crossplane/cache" help:"Cache directory for downloaded provider packages." name:"cache-dir" predictor:"directory"` + Context string `default:"" help:"Kubernetes context." name:"context" predictor:"context" short:"c"` + CacheDir string `default:"~/.crossplane/cache" help:"Cache directory for downloaded provider packages." name:"cache-dir" predictor:"directory"` Timeout time.Duration `default:"2m" help:"Per-resource plan timeout."` - Output string `default:"plan" enum:"plan,json,yaml" help:"Output format. One of: plan, json, yaml." name:"output" short:"o"` + Output string `default:"plan" enum:"plan,json,yaml" help:"Output format. One of: plan, json, yaml." name:"output" short:"o"` Impersonation kube.ImpersonationFlags `embed:""` } @@ -231,7 +232,7 @@ func (c *Cmd) imageConfigs() ([]pkgv1beta1.ImageConfig, error) { if explicit { return nil, errors.Wrapf(err, "cannot read project file %q", name) } - return nil, nil //nolint:nilerr // Best effort: no project file means no project image configs. + return nil, nil // Best effort: no project file means no project image configs. } projFS := afero.NewBasePathFs(afero.NewOsFs(), filepath.Dir(projFilePath)) proj, err := projectfile.Parse(projFS, filepath.Base(projFilePath)) @@ -273,6 +274,11 @@ func loadProviderInputs(path string) ([]diffserver.Input, error) { switch cleanup := diffserver.Cleanup(annotations[AnnotationKeyDockerCleanup]); cleanup { case diffserver.CleanupRemove, diffserver.CleanupOrphan, "": in.Cleanup = cleanup + case containerruntime.CleanupStop: + // Render's Stop policy is deliberately unsupported: a stopped + // diff server serves nothing, and restarting one costs nearly as + // much as starting it. + fallthrough default: return nil, errors.Errorf("provider %q has an invalid %s annotation %q: must be %s or %s", o.GetName(), AnnotationKeyDockerCleanup, cleanup, diffserver.CleanupRemove, diffserver.CleanupOrphan) diff --git a/cmd/crossplane/simulate/simulate_test.go b/cmd/crossplane/simulate/simulate_test.go index f0d0fe6b..51f9bdf5 100644 --- a/cmd/crossplane/simulate/simulate_test.go +++ b/cmd/crossplane/simulate/simulate_test.go @@ -21,6 +21,7 @@ import ( "net/netip" "os" "path/filepath" + "strings" "testing" "time" @@ -139,6 +140,26 @@ spec: if _, err := loadProviderInputs(notAProvider); err == nil { t.Errorf("loadProviderInputs(...): want an error for non-Provider manifests") } + + // Render's Stop cleanup policy is deliberately unsupported for diff + // servers, so the annotation that would request it is rejected. + stopCleanup := filepath.Join(dir, "stop.yaml") + manifest = `--- +apiVersion: pkg.crossplane.io/v1 +kind: Provider +metadata: + name: provider-aws-ec2 + annotations: + simulate.crossplane.io/runtime-docker-cleanup: Stop +spec: + package: xpkg.upbound.io/upbound/provider-aws-ec2:v2.9.0 +` + if err := os.WriteFile(stopCleanup, []byte(manifest), 0o600); err != nil { + t.Fatalf("cannot write manifest: %v", err) + } + if _, err := loadProviderInputs(stopCleanup); err == nil || !strings.Contains(err.Error(), "invalid") { + t.Errorf("loadProviderInputs(...): want an invalid-annotation error for the Stop cleanup policy, got %v", err) + } } func TestSimulateOne(t *testing.T) { diff --git a/internal/containerruntime/container.go b/internal/containerruntime/container.go new file mode 100644 index 00000000..5caf05dc --- /dev/null +++ b/internal/containerruntime/container.go @@ -0,0 +1,454 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package containerruntime + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "net" + "net/netip" + "strings" + "time" + + "github.com/containerd/errdefs" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/network" + "github.com/moby/moby/client" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + "github.com/crossplane/crossplane-runtime/v2/pkg/logging" +) + +// A Container runs one package image as a local Docker container serving a +// gRPC API on Port. +type Container struct { + // Image to run. + Image string + + // Name of the container. When set, an existing container with this name + // is reused (and restarted if needed) instead of creating a new one. + // Empty means Docker assigns a random name and nothing is reused. + Name string + + // Cmd is the command the container runs, e.g. a server subcommand and + // its flags. + Cmd []string + + // Port is the TCP port the server binds inside the container. + Port int + + // Env is the list of environment variables to set for the container. + Env []string + + // Labels are applied at creation time. A pre-existing named container + // that is reused is not relabeled, since Docker can't change labels on + // an existing container. + Labels map[string]string + + // BindAddress is the host address Docker publishes Port to. Use 0.0.0.0 + // to publish to all host interfaces for remote Docker access. + BindAddress string + + // Target is the host address to use when connecting to the container. + // If empty, it defaults to the published HostIP from Docker inspect. + // When published on 0.0.0.0, set this explicitly (e.g. the remote + // Docker host). + Target string + + // Network is the Docker network to connect the container to. When empty + // (the default), the container uses the default Docker network and is + // reached via host port bindings. When set, the container joins the + // specified network and is reached via its Docker hostname on Port. + Network string + + // PullPolicy controls how the image is pulled. Empty means PullDefault. + PullPolicy PullPolicy + + // Keychain to use for pulling images from private registries. Nil means + // the default keychain. + Keychain authn.Keychain + + // Log is the logger for this container's lifecycle. Nil means no logs. + Log logging.Logger + + // Client manages the container. Production callers leave it nil and + // Start builds a real client from the environment. Tests substitute a + // fake to exercise container lifecycle and cleanup handling without a + // real Docker daemon. + Client Client +} + +// Client is the subset of the Docker client a Container uses to pull its +// image and manage its container. +type Client interface { + PullClient + CleanupClient + ContainerInspect(ctx context.Context, containerID string, options client.ContainerInspectOptions) (client.ContainerInspectResult, error) + ContainerCreate(ctx context.Context, options client.ContainerCreateOptions) (client.ContainerCreateResult, error) + ContainerStart(ctx context.Context, containerID string, options client.ContainerStartOptions) (client.ContainerStartResult, error) +} + +// CleanupClient is the subset of the Docker client Running.Stop uses to +// clean up a container. +type CleanupClient interface { + ContainerStop(ctx context.Context, containerID string, options client.ContainerStopOptions) (client.ContainerStopResult, error) + ContainerRemove(ctx context.Context, containerID string, options client.ContainerRemoveOptions) (client.ContainerRemoveResult, error) +} + +// PullClient is the subset of the Docker client PullImage uses. +type PullClient interface { + ImagePull(ctx context.Context, ref string, options client.ImagePullOptions) (client.ImagePullResponse, error) +} + +var _ Client = (*client.Client)(nil) + +// Running is a handle to a started container. +type Running struct { + // Addr is the host:port address the container's server is reachable on. + Addr string + + // ID of the container. + ID string + + // Reused reports whether Start found an existing container by name + // rather than creating a new one. + Reused bool + + cli CleanupClient +} + +// Start ensures the container is running and returns a handle carrying the +// address to reach it on and its cleanup lifecycle. +func (c *Container) Start(ctx context.Context) (*Running, error) { + cli := c.Client + if cli == nil { + mc, err := client.New(client.FromEnv) + if err != nil { + return nil, errors.Wrap(err, "cannot create Docker client using environment variables") + } + cli = mc + } + if c.Log == nil { + c.Log = logging.NewNopLogger() + } + if c.Keychain == nil { + c.Keychain = authn.DefaultKeychain + } + if c.PullPolicy == "" { + c.PullPolicy = PullDefault + } + + // Try to find an existing container with the supplied container name. + containerID, err := c.find(ctx, cli) + if err != nil { + return nil, err + } + reused := containerID != "" + + // Create new container if not found. + if containerID == "" { + containerID, err = c.create(ctx, cli) + if err != nil { + return nil, err + } + } + + addr, err := c.start(ctx, cli, containerID) + if err != nil { + return nil, err + } + + return &Running{Addr: addr, ID: containerID, Reused: reused, cli: cli}, nil +} + +// Stop finishes the container according to the cleanup policy: Orphan leaves +// it running, Stop stops it with the grace period, and Remove stops it and +// then force removes it even if the graceful stop fails. +func (r *Running) Stop(ctx context.Context, cleanup Cleanup) error { + grace := int(StopGracePeriod / time.Second) + stopOpts := client.ContainerStopOptions{Timeout: &grace} + + switch cleanup { + case CleanupOrphan: + return nil + case CleanupStop: + if err := stopContainer(ctx, r.cli, r.ID, stopOpts); err != nil { + return errors.Wrap(err, "cannot stop Docker container") + } + case CleanupRemove: + // Give the container a chance to exit gracefully, then force remove + // it whether or not the stop succeeded, so a container that's slow + // to exit on SIGTERM can't cause removal to be skipped. A stop + // failure only matters if removal fails too. + var stopErr error + if err := stopContainer(ctx, r.cli, r.ID, stopOpts); err != nil { + stopErr = errors.Wrap(err, "cannot stop Docker container") + } + if err := removeContainer(ctx, r.cli, r.ID); err != nil { + return errors.Join(errors.Wrap(err, "cannot remove Docker container"), stopErr) + } + } + + return nil +} + +func (c *Container) find(ctx context.Context, cli Client) (string, error) { + if c.Name == "" { + return "", nil + } + + inspect, err := cli.ContainerInspect(ctx, c.Name, client.ContainerInspectOptions{}) + if err != nil { + if errdefs.IsNotFound(err) { + return "", nil // Container doesn't exist, but that's not an error + } + return "", errors.Wrapf(err, "cannot inspect Docker container %q", c.Name) + } + + return inspect.Container.ID, nil +} + +func (c *Container) create(ctx context.Context, cli Client) (string, error) { + c.Log.Debug("Starting Docker container runtime setup", "image", c.Image) + + // Let Docker automatically allocate an available port on the bind address. + // This avoids race conditions and works reliably with Docker daemons. + port := network.MustParsePort(fmt.Sprintf("%d/tcp", c.Port)) + + cfg := &container.Config{ + Image: c.Image, + Cmd: c.Cmd, + ExposedPorts: network.PortSet{port: struct{}{}}, + Env: c.Env, + Labels: c.Labels, + } + hcfg := &container.HostConfig{} + var ncfg *network.NetworkingConfig + + if c.Network != "" { + // When a Docker network is specified, the container joins that + // network and is reached via its Docker hostname on the container + // port. No host port bindings are needed. + ncfg = &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + c.Network: {}, + }, + } + c.Log.Debug("Connecting container to Docker network", "network", c.Network) + } else { + // PortBinding.HostIP is a netip.Addr in the Moby API types, where it + // was a string; reject a malformed bind address here rather than + // letting it reach the daemon as a zero value. + bindAddr, err := netip.ParseAddr(c.BindAddress) + if err != nil { + return "", errors.Wrapf(err, "invalid bind address %q", c.BindAddress) + } + hcfg.PortBindings = network.PortMap{ + port: []network.PortBinding{{ + HostIP: bindAddr, + HostPort: "0", // "0" => engine allocates an ephemeral port + }}, + } + } + + options, err := c.getPullOptions() + if err != nil { + // We can continue to pull an image if we don't have the PullOptions with RegistryAuth + // as long as the image is from a public registry. Therefore, we log the error message and continue. + c.Log.Info("Cannot get pull options", "image", c.Image, "err", err) + } + + if c.PullPolicy == PullAlways { + c.Log.Debug("Pulling image with pullPolicy: Always", "image", c.Image) + + err = PullImage(ctx, cli, c.Image, options) + if err != nil { + return "", errors.Wrapf(err, "cannot pull Docker image %q", c.Image) + } + } + + c.Log.Debug("Creating Docker container", "image", c.Image, "name", c.Name, "network", c.Network) + + rsp, err := cli.ContainerCreate(ctx, client.ContainerCreateOptions{ + Config: cfg, + HostConfig: hcfg, + NetworkingConfig: ncfg, + Name: c.Name, + }) + if err != nil { + // TODO: If Docker ever exposes a structured way to distinguish + // network-not-found from image-not-found (e.g. via errdefs or a typed + // error), handle the network case here to avoid an unnecessary image + // pull attempt. For now, both surface as errdefs.IsNotFound and + // string-matching on the daemon message is too fragile. + if !errdefs.IsNotFound(err) { + // Non-image-not-found error: could be network misconfiguration, + // daemon permissions, etc. + if c.Network != "" { + return "", errors.Wrapf(err, "cannot create Docker container for image %q on network %q; verify the network exists and is accessible by the Docker daemon", c.Image, c.Network) + } + return "", errors.Wrapf(err, "cannot create Docker container for image %q", c.Image) + } + if c.PullPolicy == PullNever { + // Image not found and we're not allowed to pull it. + return "", errors.Wrapf(err, "cannot create Docker container: image %q not found and pull policy is %q", c.Image, c.PullPolicy) + } + + // The image was not found, but we're allowed to pull it. + c.Log.Debug("Image not found, pulling", "image", c.Image) + + err = PullImage(ctx, cli, c.Image, options) + if err != nil { + return "", errors.Wrapf(err, "cannot pull Docker image %q", c.Image) + } + + rsp, err = cli.ContainerCreate(ctx, client.ContainerCreateOptions{ + Config: cfg, + HostConfig: hcfg, + NetworkingConfig: ncfg, + Name: c.Name, + }) + if err != nil { + if c.Network != "" { + return "", errors.Wrapf(err, "cannot create Docker container for image %q on network %q; verify the network exists and is accessible by the Docker daemon", c.Image, c.Network) + } + return "", errors.Wrapf(err, "cannot create Docker container for image %q", c.Image) + } + } + + return rsp.ID, nil +} + +// start ensures the container is running and returns its address. +func (c *Container) start(ctx context.Context, cli Client, containerID string) (string, error) { + // Start the container (idempotent - safe to call on running containers) + if _, err := cli.ContainerStart(ctx, containerID, client.ContainerStartOptions{}); err != nil { + return "", errors.Wrap(err, "cannot start Docker container") + } + + // When using a Docker network, containers are resolvable by name within + // the network. Inspect the container to validate it is attached to the + // specified network and to get its DNS-resolvable name (Docker's embedded + // DNS resolves container names and hostnames, but not container IDs). + if c.Network != "" { + inspect, err := cli.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{}) + if err != nil { + return "", errors.Wrap(err, "cannot inspect Docker container") + } + if _, ok := inspect.Container.NetworkSettings.Networks[c.Network]; !ok { + return "", errors.Errorf("container %q is not connected to Docker network %q; verify the network exists and the container is attached to it", strings.TrimPrefix(inspect.Container.Name, "/"), c.Network) + } + hostname := strings.TrimPrefix(inspect.Container.Name, "/") + if hostname == "" { + return "", errors.Errorf("cannot determine hostname for container %q on network %q", containerID, c.Network) + } + addr := net.JoinHostPort(hostname, fmt.Sprintf("%d", c.Port)) + c.Log.Debug("Container reachable on network", "network", c.Network, "address", addr) + return addr, nil + } + + // Inspect the container to get the actual allocated host port. + inspect, err := cli.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{}) + if err != nil { + return "", errors.Wrap(err, "cannot inspect Docker container") + } + + // Look up the specific container port instead of taking the first one + p := network.MustParsePort(fmt.Sprintf("%d/tcp", c.Port)) + if len(inspect.Container.NetworkSettings.Ports[p]) == 0 { + return "", errors.Errorf("container %q has no published binding for port %s", c.Name, p.Port()) + } + + binding := inspect.Container.NetworkSettings.Ports[p][0] + host := c.Target + if host == "" && binding.HostIP.IsValid() { + host = binding.HostIP.String() + } + if host == "" { + return "", errors.Errorf("container %q has port binding for %s but no host address", c.Name, p.Port()) + } + + return net.JoinHostPort(host, binding.HostPort), nil +} + +func (c *Container) getPullOptions() (client.ImagePullOptions, error) { + // Resolve auth token by looking into keychain + ref, err := name.ParseReference(c.Image) + if err != nil { + return client.ImagePullOptions{}, errors.Wrapf(err, "Image is not a valid reference %s", c.Image) + } + + auth, err := c.Keychain.Resolve(ref.Context().Registry) + if err != nil { + return client.ImagePullOptions{}, errors.Wrapf(err, "Cannot resolve auth for %s", ref.Context().RegistryStr()) + } + + authConfig, err := auth.Authorization() + if err != nil { + return client.ImagePullOptions{}, errors.Wrapf(err, "Cannot get auth config for %s", ref.Context().RegistryStr()) + } + + token, err := authConfig.MarshalJSON() + if err != nil { + return client.ImagePullOptions{}, errors.Wrapf(err, "Cannot marshal auth config for %s", ref.Context().RegistryStr()) + } + + return client.ImagePullOptions{ + RegistryAuth: base64.StdEncoding.EncodeToString(token), + }, nil +} + +// stopContainer stops a container, bounding the request by StopTimeout. +func stopContainer(ctx context.Context, cli CleanupClient, containerID string, opts client.ContainerStopOptions) error { + ctx, cancel := context.WithTimeout(ctx, StopTimeout) + defer cancel() + + _, err := cli.ContainerStop(ctx, containerID, opts) + return err +} + +// removeContainer force removes a container, bounding the request by +// RemoveTimeout. +func removeContainer(ctx context.Context, cli CleanupClient, containerID string) error { + ctx, cancel := context.WithTimeout(ctx, RemoveTimeout) + defer cancel() + + _, err := cli.ContainerRemove(ctx, containerID, client.ContainerRemoveOptions{Force: true}) + return err +} + +// PullImage pulls the supplied image using the supplied client. It blocks until +// the image has either finished pulling or hit an error. +func PullImage(ctx context.Context, p PullClient, image string, options client.ImagePullOptions) error { + out, err := p.ImagePull(ctx, image, options) + if err != nil { + return err + } + defer out.Close() //nolint:errcheck // TODO(negz): Can this error? + + // Each line read from out is a JSON object containing the status of the + // pull - similar to the progress bars you'd see if running docker pull. It + // seems that consuming all of this output is the best way to block until + // the image is actually pulled before we try to run it. + _, err = io.Copy(io.Discard, out) + + return err +} diff --git a/internal/containerruntime/containerruntime.go b/internal/containerruntime/containerruntime.go new file mode 100644 index 00000000..3fbe8e69 --- /dev/null +++ b/internal/containerruntime/containerruntime.go @@ -0,0 +1,125 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package containerruntime runs Crossplane package images (Functions, +// Providers) as local Docker containers serving a gRPC API. It carries the +// lifecycle render and simulate share: find-or-create a container by name, +// start it, resolve the address to reach it on, and clean it up per a +// cleanup policy, labeling every container so leftovers can be identified +// and swept. +package containerruntime + +import ( + "sync" + "time" + + "k8s.io/apimachinery/pkg/util/rand" +) + +// Cleanup specifies what to do with a package's container once its run is +// done. +type Cleanup string + +const ( + // CleanupStop stops the container, waiting up to StopGracePeriod for it + // to exit on SIGTERM before Docker kills it. + CleanupStop Cleanup = "Stop" + + // CleanupRemove stops the container, waiting up to StopGracePeriod for + // it to exit on SIGTERM, then force removes it. The container is removed + // even if the graceful stop fails. + CleanupRemove Cleanup = "Remove" + + // CleanupOrphan leaves the container running, so a later run can find it + // by name and reuse it without paying startup. + CleanupOrphan Cleanup = "Orphan" + + // CleanupDefault is the policy callers use when none is specified. + CleanupDefault = CleanupRemove +) + +// PullPolicy controls how a package's runtime image is pulled by Docker. +type PullPolicy string + +const ( + // PullAlways always pulls the image. + PullAlways PullPolicy = "Always" + + // PullNever never pulls the image. + PullNever PullPolicy = "Never" + + // PullIfNotPresent pulls the image if it's not present. + PullIfNotPresent PullPolicy = "IfNotPresent" + + // PullDefault is the policy used when none is specified. + PullDefault = PullIfNotPresent +) + +const ( + // StopGracePeriod is how long the Stop and Remove cleanup policies wait + // for a container to exit on SIGTERM before Docker kills it. + StopGracePeriod = 3 * time.Second + + // StopTimeout bounds the ContainerStop request: the grace period plus + // time for Docker to kill the container. + StopTimeout = StopGracePeriod + 2*time.Second + + // RemoveTimeout bounds the ContainerRemove request. It's separate from + // StopTimeout so a stop that hangs can't leave removal without time to + // run. + RemoveTimeout = 10 * time.Second +) + +// Label key suffixes appended to a caller's annotation domain (e.g. +// "render.crossplane.io" becomes "render.crossplane.io/managed-by"), so each +// command keeps its documented label keys while the labeling logic lives in +// one place. +const ( + labelSuffixManagedBy = "/managed-by" + labelSuffixSession = "/session" + labelSuffixCleanup = "/cleanup" + + // LabelValueManagedBy is the value of every /managed-by label. + LabelValueManagedBy = "crossplane-cli" +) + +// SessionID returns an identifier unique to this process. It is the session +// ID used for Docker resources when the caller doesn't supply one. +// +//nolint:gochecknoglobals // Must be stable for the life of the process. +var SessionID = sync.OnceValue(func() string { return rand.String(12) }) + +// ManagedLabels returns the labels marking a Docker resource as managed by +// the crossplane CLI under the given annotation domain: +// /managed-by and /session. An empty session falls back to +// the per-process SessionID. +func ManagedLabels(domain, session string) map[string]string { + if session == "" { + session = SessionID() + } + return map[string]string{ + domain + labelSuffixManagedBy: LabelValueManagedBy, + domain + labelSuffixSession: session, + } +} + +// ContainerLabels returns ManagedLabels plus the effective cleanup policy +// recorded as /cleanup, the full label set for a package container. +func ContainerLabels(domain, session string, cleanup Cleanup) map[string]string { + l := ManagedLabels(domain, session) + l[domain+labelSuffixCleanup] = string(cleanup) + return l +} diff --git a/internal/containerruntime/fake/fake.go b/internal/containerruntime/fake/fake.go new file mode 100644 index 00000000..f72687b4 --- /dev/null +++ b/internal/containerruntime/fake/fake.go @@ -0,0 +1,88 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package fake provides a fake Docker client for testing code built on +// containerruntime without a real Docker daemon. +package fake + +import ( + "context" + "io" + + "github.com/moby/moby/client" + + "github.com/crossplane/cli/v2/internal/containerruntime" +) + +// Client implements containerruntime.Client with injectable behavior per +// method. +type Client struct { + MockImagePull func(ctx context.Context, ref string, options client.ImagePullOptions) (client.ImagePullResponse, error) + MockContainerInspect func(ctx context.Context, containerID string, options client.ContainerInspectOptions) (client.ContainerInspectResult, error) + MockContainerCreate func(ctx context.Context, options client.ContainerCreateOptions) (client.ContainerCreateResult, error) + MockContainerStart func(ctx context.Context, containerID string, options client.ContainerStartOptions) (client.ContainerStartResult, error) + MockContainerStop func(ctx context.Context, containerID string, options client.ContainerStopOptions) (client.ContainerStopResult, error) + MockContainerRemove func(ctx context.Context, containerID string, options client.ContainerRemoveOptions) (client.ContainerRemoveResult, error) +} + +var _ containerruntime.Client = &Client{} + +// ImagePull calls MockImagePull. +func (c *Client) ImagePull(ctx context.Context, ref string, options client.ImagePullOptions) (client.ImagePullResponse, error) { + return c.MockImagePull(ctx, ref, options) +} + +// ContainerInspect calls MockContainerInspect. +func (c *Client) ContainerInspect(ctx context.Context, containerID string, options client.ContainerInspectOptions) (client.ContainerInspectResult, error) { + return c.MockContainerInspect(ctx, containerID, options) +} + +// ContainerCreate calls MockContainerCreate. +func (c *Client) ContainerCreate(ctx context.Context, options client.ContainerCreateOptions) (client.ContainerCreateResult, error) { + return c.MockContainerCreate(ctx, options) +} + +// ContainerStart calls MockContainerStart. +func (c *Client) ContainerStart(ctx context.Context, containerID string, options client.ContainerStartOptions) (client.ContainerStartResult, error) { + return c.MockContainerStart(ctx, containerID, options) +} + +// ContainerStop calls MockContainerStop. +func (c *Client) ContainerStop(ctx context.Context, containerID string, options client.ContainerStopOptions) (client.ContainerStopResult, error) { + return c.MockContainerStop(ctx, containerID, options) +} + +// ContainerRemove calls MockContainerRemove. +func (c *Client) ContainerRemove(ctx context.Context, containerID string, options client.ContainerRemoveOptions) (client.ContainerRemoveResult, error) { + return c.MockContainerRemove(ctx, containerID, options) +} + +// ImagePullDone is a MockImagePull that reports an image pull that has +// already completed. +func ImagePullDone(context.Context, string, client.ImagePullOptions) (client.ImagePullResponse, error) { + return pullDoneResponse{}, nil +} + +// pullDoneResponse is an ImagePullResponse whose body is empty. Only the +// io.ReadCloser methods PullImage uses are implemented; the embedded +// interface is nil. +type pullDoneResponse struct { + client.ImagePullResponse +} + +func (pullDoneResponse) Read([]byte) (int, error) { return 0, io.EOF } + +func (pullDoneResponse) Close() error { return nil } diff --git a/internal/diffserver/manager.go b/internal/diffserver/manager.go index 8f9b3b74..09caf56a 100644 --- a/internal/diffserver/manager.go +++ b/internal/diffserver/manager.go @@ -24,7 +24,6 @@ package diffserver import ( "context" "fmt" - "net" "net/netip" "strings" "sync" @@ -36,7 +35,6 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/rand" "github.com/crossplane/crossplane-runtime/v2/pkg/errors" "github.com/crossplane/crossplane-runtime/v2/pkg/logging" @@ -44,7 +42,7 @@ import ( pkgmetav1 "github.com/crossplane/crossplane/apis/v2/pkg/meta/v1" - "github.com/crossplane/cli/v2/internal/docker" + "github.com/crossplane/cli/v2/internal/containerruntime" clixpkg "github.com/crossplane/cli/v2/internal/xpkg" ) @@ -68,59 +66,45 @@ const ( // Cleanup is how a diff server container is cleaned up when the manager // stops, following render's per-function cleanup policies. -type Cleanup string +type Cleanup = containerruntime.Cleanup const ( // CleanupRemove is the default: stop and remove the container. - CleanupRemove Cleanup = "Remove" + CleanupRemove = containerruntime.CleanupRemove // CleanupOrphan leaves the container running, so a later run finds it by // its deterministic name and reuses it without paying startup. Render's // function runtimes support a third policy, Stop; a stopped diff server // serves nothing and restarting costs nearly as much as starting, so // simulate omits it. - CleanupOrphan Cleanup = "Orphan" + CleanupOrphan = containerruntime.CleanupOrphan ) +// labelDomain prefixes the labels simulate applies to the Docker containers +// it creates. +const labelDomain = "simulate.crossplane.io" + // Labels applied to the Docker containers simulate creates, mirroring // render's, so leftovers (a crash, or an Orphan cleanup policy) can be // identified and swept. const ( // LabelKeyManagedBy is applied to every diff server container. Its value // is always LabelValueManagedBy. - LabelKeyManagedBy = "simulate.crossplane.io/managed-by" + LabelKeyManagedBy = labelDomain + "/managed-by" // LabelValueManagedBy is the value of LabelKeyManagedBy. - LabelValueManagedBy = "crossplane-cli" + LabelValueManagedBy = containerruntime.LabelValueManagedBy // LabelKeySession identifies the run that created a container. Orphaned // containers outlive their session by design; the label records which run // created them. - LabelKeySession = "simulate.crossplane.io/session" + LabelKeySession = labelDomain + "/session" // LabelKeyCleanup records the effective cleanup policy a container was // created with. - LabelKeyCleanup = "simulate.crossplane.io/cleanup" + LabelKeyCleanup = labelDomain + "/cleanup" ) -// sessionID returns an identifier unique to this process, matching render's -// per-process Docker session IDs. -// -//nolint:gochecknoglobals // Must be stable for the life of the process. -var sessionID = sync.OnceValue(func() string { return rand.String(12) }) - -// managedLabels returns the labels applied to every container the manager -// creates. An empty session falls back to the per-process session ID. -func managedLabels(session string) map[string]string { - if session == "" { - session = sessionID() - } - return map[string]string{ - LabelKeyManagedBy: LabelValueManagedBy, - LabelKeySession: session, - } -} - // Server is a running diff server for one provider package. type Server struct { Image string @@ -130,7 +114,7 @@ type Server struct { conn *grpc.ClientConn client diffv1alpha1.PlanServiceClient - reused bool + running *containerruntime.Running cleanup Cleanup } @@ -153,21 +137,11 @@ type Input struct { Cleanup Cleanup } -// containerRuntime is the slice of internal/docker the manager uses, -// extracted so tests can fake container lifecycle. -type containerRuntime interface { - StartContainer(ctx context.Context, name, image string, opts ...docker.StartContainerOption) (string, error) - StartContainerByID(ctx context.Context, id string) error - StopContainerByID(ctx context.Context, id string) error - GetRunningContainerIDByName(ctx context.Context, name string) (id string, found, running bool, err error) - GetHostPort(ctx context.Context, id string, port int) (string, error) -} - // Manager starts diff server containers for provider packages and routes // resources to them. type Manager struct { pkgs runtimexpkg.Client - rt containerRuntime + cli containerruntime.Client log logging.Logger readyTimeout time.Duration publishAddr netip.Addr @@ -181,9 +155,11 @@ type Manager struct { // Option configures a Manager. type Option func(*Manager) -// WithContainerRuntime replaces the container runtime, for testing. -func WithContainerRuntime(rt containerRuntime) Option { - return func(m *Manager) { m.rt = rt } +// WithDockerClient replaces the Docker client the manager runs containers +// with, for testing. Nil (the default) means a real client built from the +// environment. +func WithDockerClient(cli containerruntime.Client) Option { + return func(m *Manager) { m.cli = cli } } // WithReadyTimeout sets how long to wait for a started diff server to answer. @@ -203,7 +179,6 @@ func WithSessionID(session string) Option { func NewManager(pkgs runtimexpkg.Client, log logging.Logger, opts ...Option) *Manager { m := &Manager{ pkgs: pkgs, - rt: dockerRuntime{}, log: log, readyTimeout: defaultReadyTimeout, publishAddr: netip.MustParseAddr("127.0.0.1"), @@ -307,71 +282,43 @@ func (m *Manager) startOne(ctx context.Context, in Input) (*Server, *Skipped) { cleanup = CleanupRemove } - labels := managedLabels(m.session) - labels[LabelKeyCleanup] = string(cleanup) + ctr := &containerruntime.Container{ + Image: runImage, + Name: containerName(pkg.Digest), + // --debug precedes the subcommand: it is the provider's global flag. + Cmd: []string{"--debug", "internal", "diff-server", "--address", fmt.Sprintf(":%d", containerPort)}, + Port: containerPort, + Labels: containerruntime.ContainerLabels(labelDomain, m.session, cleanup), + BindAddress: publishAddr.String(), + Target: in.Target, + Log: m.log, + Client: m.cli, + } - id, reused, err := m.ensureContainer(ctx, containerName(pkg.Digest), runImage, publishAddr, labels) + running, err := ctr.Start(ctx) if err != nil { return nil, &Skipped{Image: image, Reason: fmt.Sprintf("cannot start diff server container: %s", err)} } - server, err := m.connect(ctx, id, image, in.Target) + server, err := m.connect(ctx, running, image) if err != nil { // An orphaned container is left behind even on failure, for debugging. - if !reused && cleanup != CleanupOrphan { - if serr := m.rt.StopContainerByID(ctx, id); serr != nil { - m.log.Info("Cannot clean up failed diff server container", "container", id, "err", serr) + if !running.Reused && cleanup != CleanupOrphan { + if serr := running.Stop(ctx, CleanupRemove); serr != nil { + m.log.Info("Cannot clean up failed diff server container", "container", running.ID, "err", serr) } } return nil, &Skipped{Image: image, Reason: err.Error()} } server.GroupKinds = gks - server.reused = reused server.cleanup = cleanup - m.log.Debug("Diff server ready", "image", image, "address", server.Address, "reused", reused, "cleanup", cleanup) + m.log.Debug("Diff server ready", "image", image, "address", server.Address, "reused", running.Reused, "cleanup", cleanup) return server, nil } -// ensureContainer finds a live container with the deterministic name for this -// package digest and reuses it, or starts a new one. -func (m *Manager) ensureContainer(ctx context.Context, name, image string, publishAddr netip.Addr, labels map[string]string) (id string, reused bool, err error) { - id, found, running, err := m.rt.GetRunningContainerIDByName(ctx, name) - if err != nil { - return "", false, err - } - if found { - if !running { - if err := m.rt.StartContainerByID(ctx, id); err != nil { - return "", false, err - } - } - return id, true, nil - } - - id, err = m.rt.StartContainer(ctx, name, image, - // --debug precedes the subcommand: it is the provider's global flag. - docker.StartWithCommand([]string{"--debug", "internal", "diff-server", "--address", fmt.Sprintf(":%d", containerPort)}), - docker.StartWithPublishedPort(publishAddr, containerPort), - docker.StartWithLabels(labels), - ) - return id, false, err -} - -func (m *Manager) connect(ctx context.Context, containerID, image, target string) (*Server, error) { - addr, err := m.rt.GetHostPort(ctx, containerID, containerPort) - if err != nil { - return nil, errors.Wrap(err, "cannot determine diff server address") - } - if target != "" { - _, port, err := net.SplitHostPort(addr) - if err != nil { - return nil, errors.Wrapf(err, "cannot parse diff server address %q", addr) - } - addr = net.JoinHostPort(target, port) - } - - conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) +func (m *Manager) connect(ctx context.Context, running *containerruntime.Running, image string) (*Server, error) { + conn, err := grpc.NewClient(running.Addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, errors.Wrap(err, "cannot create gRPC client") } @@ -384,10 +331,11 @@ func (m *Manager) connect(ctx context.Context, containerID, image, target string return &Server{ Image: image, - Address: addr, - ContainerID: containerID, + Address: running.Addr, + ContainerID: running.ID, conn: conn, client: client, + running: running, }, nil } @@ -450,10 +398,7 @@ func (m *Manager) Stop(ctx context.Context) error { if s.conn != nil { _ = s.conn.Close() } - if s.cleanup == CleanupOrphan { - continue - } - if err := m.rt.StopContainerByID(ctx, s.ContainerID); err != nil { + if err := s.running.Stop(ctx, s.cleanup); err != nil { errs = append(errs, errors.Wrapf(err, "cannot stop diff server container %s", s.ContainerID)) } } @@ -467,30 +412,3 @@ func containerName(digest string) string { } return containerPrefix + d } - -// dockerRuntime implements containerRuntime with the real Docker daemon. -type dockerRuntime struct{} - -func (dockerRuntime) StartContainer(ctx context.Context, name, image string, opts ...docker.StartContainerOption) (string, error) { - return docker.StartContainer(ctx, name, image, opts...) -} - -func (dockerRuntime) StartContainerByID(ctx context.Context, id string) error { - return docker.StartContainerByID(ctx, id) -} - -func (dockerRuntime) StopContainerByID(ctx context.Context, id string) error { - return docker.StopContainerByID(ctx, id) -} - -func (dockerRuntime) GetRunningContainerIDByName(ctx context.Context, name string) (string, bool, bool, error) { - c, found, err := docker.GetContainerByName(ctx, name, true) - if err != nil || !found { - return "", false, false, err - } - return c.ID, true, strings.EqualFold(string(c.State), "running"), nil -} - -func (dockerRuntime) GetHostPort(ctx context.Context, id string, port int) (string, error) { - return docker.GetHostPort(ctx, id, port) -} diff --git a/internal/diffserver/manager_test.go b/internal/diffserver/manager_test.go index b9ef42be..d9fb4c6b 100644 --- a/internal/diffserver/manager_test.go +++ b/internal/diffserver/manager_test.go @@ -18,14 +18,20 @@ package diffserver import ( "context" + "fmt" "io" "net" + "net/netip" "strings" "sync" "testing" "time" + "github.com/containerd/errdefs" diffv1alpha1 "github.com/crossplane/upjet/v2/proto/diff/v1alpha1" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/network" + "github.com/moby/moby/client" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -35,7 +41,7 @@ import ( runtimexpkg "github.com/crossplane/crossplane-runtime/v2/pkg/xpkg" "github.com/crossplane/crossplane-runtime/v2/pkg/xpkg/parser" - "github.com/crossplane/cli/v2/internal/docker" + "github.com/crossplane/cli/v2/internal/containerruntime/fake" ) const ( @@ -152,48 +158,76 @@ func (f *fakePkgClient) ListVersions(_ context.Context, _ string, _ ...runtimexp return nil, nil } -// fakeRuntime fakes container lifecycle, pointing every container at addr. -type fakeRuntime struct { +// fakeDocker fakes container lifecycle at the Docker client level, publishing +// every container's diff server port on addr. +type fakeDocker struct { mu sync.Mutex - addr string - existing map[string]fakeContainer // name -> container - started []string // names passed to StartContainer - stopped []string // ids passed to StopContainerByID + addr string // host:port every container's port binds to + existing map[string]string // name -> id of pre-existing containers + known map[string]bool // ids inspect resolves port bindings for + created []string // names passed to ContainerCreate + stopped []string // ids passed to ContainerStop + removed []string // ids passed to ContainerRemove } -type fakeContainer struct { - id string - running bool -} - -func (f *fakeRuntime) StartContainer(_ context.Context, name, _ string, _ ...docker.StartContainerOption) (string, error) { - f.mu.Lock() - defer f.mu.Unlock() - f.started = append(f.started, name) - return "new-" + name, nil -} - -func (f *fakeRuntime) StopContainerByID(_ context.Context, id string) error { - f.mu.Lock() - defer f.mu.Unlock() - f.stopped = append(f.stopped, id) - return nil -} - -func (f *fakeRuntime) StartContainerByID(_ context.Context, _ string) error { return nil } - -func (f *fakeRuntime) GetRunningContainerIDByName(_ context.Context, name string) (string, bool, bool, error) { - f.mu.Lock() - defer f.mu.Unlock() - c, ok := f.existing[name] - if !ok { - return "", false, false, nil +func newFakeDocker(addr string, existing map[string]string) *fakeDocker { + f := &fakeDocker{addr: addr, existing: existing, known: map[string]bool{}} + for _, id := range existing { + f.known[id] = true } - return c.id, true, c.running, nil + return f } -func (f *fakeRuntime) GetHostPort(_ context.Context, _ string, _ int) (string, error) { - return f.addr, nil +func (f *fakeDocker) client() *fake.Client { + return &fake.Client{ + MockContainerInspect: func(_ context.Context, ref string, _ client.ContainerInspectOptions) (client.ContainerInspectResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + if id, ok := f.existing[ref]; ok { + return client.ContainerInspectResult{Container: container.InspectResponse{ID: id}}, nil + } + if f.known[ref] { + host, port, err := net.SplitHostPort(f.addr) + if err != nil { + return client.ContainerInspectResult{}, err + } + p := network.MustParsePort(fmt.Sprintf("%d/tcp", containerPort)) + return client.ContainerInspectResult{Container: container.InspectResponse{ + ID: ref, + NetworkSettings: &container.NetworkSettings{ + Ports: network.PortMap{p: []network.PortBinding{{ + HostIP: netip.MustParseAddr(host), + HostPort: port, + }}}, + }, + }}, nil + } + return client.ContainerInspectResult{}, errdefs.ErrNotFound + }, + MockContainerCreate: func(_ context.Context, options client.ContainerCreateOptions) (client.ContainerCreateResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + id := "new-" + options.Name + f.created = append(f.created, options.Name) + f.known[id] = true + return client.ContainerCreateResult{ID: id}, nil + }, + MockContainerStart: func(_ context.Context, _ string, _ client.ContainerStartOptions) (client.ContainerStartResult, error) { + return client.ContainerStartResult{}, nil + }, + MockContainerStop: func(_ context.Context, id string, _ client.ContainerStopOptions) (client.ContainerStopResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.stopped = append(f.stopped, id) + return client.ContainerStopResult{}, nil + }, + MockContainerRemove: func(_ context.Context, id string, _ client.ContainerRemoveOptions) (client.ContainerRemoveResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.removed = append(f.removed, id) + return client.ContainerRemoveResult{}, nil + }, + } } // readyServer serves a real PlanService on a loopback listener that answers @@ -230,8 +264,8 @@ func TestManagerStart(t *testing.T) { }} t.Run("NoCapabilitySkipsWithoutStarting", func(t *testing.T) { - rt := &fakeRuntime{} - m := NewManager(pkgs, log, WithContainerRuntime(rt)) + fd := newFakeDocker("", nil) + m := NewManager(pkgs, log, WithDockerClient(fd.client())) skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-s3:v2.9.0"})) if err != nil { @@ -240,14 +274,14 @@ func TestManagerStart(t *testing.T) { if len(skipped) != 1 || !strings.Contains(skipped[0].Reason, "DiffServer capability") { t.Errorf("Start(...): want one capability skip, got %+v", skipped) } - if len(rt.started) != 0 { + if len(fd.created) != 0 { t.Errorf("Start(...): no container should start for a package without the capability") } }) t.Run("RoutesBuiltFromPackageCRDs", func(t *testing.T) { - rt := &fakeRuntime{addr: servePlanService(t)} - m := NewManager(pkgs, log, WithContainerRuntime(rt)) + fd := newFakeDocker(servePlanService(t), nil) + m := NewManager(pkgs, log, WithDockerClient(fd.client())) skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"})) if err != nil { @@ -270,39 +304,36 @@ func TestManagerStart(t *testing.T) { } wantName := containerPrefix + "0123456789ab" - if len(rt.started) != 1 || rt.started[0] != wantName { - t.Errorf("Start(...): want container %q started, got %v", wantName, rt.started) + if len(fd.created) != 1 || fd.created[0] != wantName { + t.Errorf("Start(...): want container %q started, got %v", wantName, fd.created) } if err := m.Stop(ctx); err != nil { t.Fatalf("Stop(...): unexpected error: %v", err) } - if len(rt.stopped) != 1 { - t.Errorf("Stop(...): want the container stopped, got %v", rt.stopped) + if len(fd.removed) != 1 { + t.Errorf("Stop(...): want the container removed, got %v", fd.removed) } }) - t.Run("ReusesRunningContainerByName", func(t *testing.T) { - rt := &fakeRuntime{ - addr: servePlanService(t), - existing: map[string]fakeContainer{ - containerPrefix + "0123456789ab": {id: "existing-id", running: true}, - }, - } - m := NewManager(pkgs, log, WithContainerRuntime(rt)) + t.Run("ReusesContainerByName", func(t *testing.T) { + fd := newFakeDocker(servePlanService(t), map[string]string{ + containerPrefix + "0123456789ab": "existing-id", + }) + m := NewManager(pkgs, log, WithDockerClient(fd.client())) skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"})) if err != nil || len(skipped) != 0 { t.Fatalf("Start(...): unexpected error or skips: %v %+v", err, skipped) } - if len(rt.started) != 0 { - t.Errorf("Start(...): want the existing container reused, got new containers %v", rt.started) + if len(fd.created) != 0 { + t.Errorf("Start(...): want the existing container reused, got new containers %v", fd.created) } }) t.Run("OrphanCleanupLeavesContainersRunning", func(t *testing.T) { - rt := &fakeRuntime{addr: servePlanService(t)} - m := NewManager(pkgs, log, WithContainerRuntime(rt)) + fd := newFakeDocker(servePlanService(t), nil) + m := NewManager(pkgs, log, WithDockerClient(fd.client())) if _, err := m.Start(ctx, []Input{{Image: "xpkg.example.org/provider-aws-ec2:v2.9.0", Cleanup: CleanupOrphan}}); err != nil { t.Fatalf("Start(...): unexpected error: %v", err) @@ -310,8 +341,8 @@ func TestManagerStart(t *testing.T) { if err := m.Stop(ctx); err != nil { t.Fatalf("Stop(...): unexpected error: %v", err) } - if len(rt.stopped) != 0 { - t.Errorf("Stop(...): want containers left running with the Orphan cleanup policy, got stops %v", rt.stopped) + if len(fd.stopped) != 0 || len(fd.removed) != 0 { + t.Errorf("Stop(...): want containers left running with the Orphan cleanup policy, got stops %v removes %v", fd.stopped, fd.removed) } }) @@ -323,8 +354,8 @@ func TestManagerStart(t *testing.T) { } t.Cleanup(func() { _ = lis.Close() }) - rt := &fakeRuntime{addr: lis.Addr().String()} - m := NewManager(pkgs, log, WithContainerRuntime(rt), WithReadyTimeout(700*time.Millisecond)) + fd := newFakeDocker(lis.Addr().String(), nil) + m := NewManager(pkgs, log, WithDockerClient(fd.client()), WithReadyTimeout(700*time.Millisecond)) skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"})) if err != nil { @@ -333,8 +364,8 @@ func TestManagerStart(t *testing.T) { if len(skipped) != 1 || !strings.Contains(skipped[0].Reason, "ready") { t.Errorf("Start(...): want a readiness skip, got %+v", skipped) } - if len(rt.stopped) != 1 { - t.Errorf("Start(...): want the unready container cleaned up, got stops %v", rt.stopped) + if len(fd.removed) != 1 { + t.Errorf("Start(...): want the unready container cleaned up, got removes %v", fd.removed) } }) } diff --git a/internal/docker/docker.go b/internal/docker/docker.go index e97e5894..9ca70b70 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -27,8 +27,6 @@ import ( "fmt" "io" "maps" - "net" - "net/netip" "path" "path/filepath" "slices" @@ -270,64 +268,6 @@ func StartWithCopyFiles(tarball []byte, path string) StartContainerOption { } } -// StartWithLabels adds labels to the container. -func StartWithLabels(labels map[string]string) StartContainerOption { - return func(cfg *startContainerConfig) { - if cfg.containerConfig.Labels == nil { - cfg.containerConfig.Labels = map[string]string{} - } - maps.Copy(cfg.containerConfig.Labels, labels) - } -} - -// StartWithPublishedPort publishes the container's TCP containerPort on an -// ephemeral port on hostIP; the engine allocates the host port. Use -// GetHostPort after the container starts to learn the allocated address. -func StartWithPublishedPort(hostIP netip.Addr, containerPort int) StartContainerOption { - return func(cfg *startContainerConfig) { - port := network.MustParsePort(fmt.Sprintf("%d/tcp", containerPort)) - if cfg.containerConfig.ExposedPorts == nil { - cfg.containerConfig.ExposedPorts = network.PortSet{} - } - cfg.containerConfig.ExposedPorts[port] = struct{}{} - if cfg.hostConfig == nil { - cfg.hostConfig = &container.HostConfig{} - } - if cfg.hostConfig.PortBindings == nil { - cfg.hostConfig.PortBindings = network.PortMap{} - } - cfg.hostConfig.PortBindings[port] = []network.PortBinding{{ - HostIP: hostIP, - HostPort: "0", // "0" => engine allocates an ephemeral port - }} - } -} - -// GetHostPort returns the host address (host:port) a container's published -// TCP containerPort is bound to. -func GetHostPort(ctx context.Context, containerID string, containerPort int) (string, error) { - cli, err := NewClient() - if err != nil { - return "", err - } - - inspect, err := cli.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{}) - if err != nil { - return "", errors.Wrap(err, "cannot inspect container") - } - - port := network.MustParsePort(fmt.Sprintf("%d/tcp", containerPort)) - bindings := inspect.Container.NetworkSettings.Ports[port] - if len(bindings) == 0 { - return "", errors.Errorf("container %q has no published binding for port %s", containerID, port.Port()) - } - binding := bindings[0] - if !binding.HostIP.IsValid() { - return "", errors.Errorf("container %q has a port binding for %s but no host address", containerID, port.Port()) - } - return net.JoinHostPort(binding.HostIP.String(), binding.HostPort), nil -} - // StartWithEnv adds environment variables that will be passed to the container. func StartWithEnv(env ...string) StartContainerOption { return func(cfg *startContainerConfig) { diff --git a/internal/plan/output.go b/internal/plan/output.go index dc049f6d..102c9250 100644 --- a/internal/plan/output.go +++ b/internal/plan/output.go @@ -114,20 +114,20 @@ type Renderer interface { func NewRenderer(format string) (Renderer, error) { switch format { case FormatPlan: - return PlanRenderer{}, nil + return planRenderer{}, nil case FormatJSON: - return JSONRenderer{}, nil + return jsonRenderer{}, nil case FormatYAML: - return YAMLRenderer{}, nil + return yamlRenderer{}, nil } return nil, errors.Errorf("unknown output format %q", format) } -// JSONRenderer renders the serialized Result as indented JSON. -type JSONRenderer struct{} +// jsonRenderer renders the serialized Result as indented JSON. +type jsonRenderer struct{} // Render writes the plans to w as JSON. -func (JSONRenderer) Render(w io.Writer, plans []*ResourcePlan) error { +func (jsonRenderer) Render(w io.Writer, plans []*ResourcePlan) error { out, err := json.MarshalIndent(NewResult(plans), "", " ") if err != nil { return errors.Wrap(err, "cannot marshal plans to JSON") @@ -136,11 +136,11 @@ func (JSONRenderer) Render(w io.Writer, plans []*ResourcePlan) error { return err } -// YAMLRenderer renders the serialized Result as YAML. -type YAMLRenderer struct{} +// yamlRenderer renders the serialized Result as YAML. +type yamlRenderer struct{} // Render writes the plans to w as YAML. -func (YAMLRenderer) Render(w io.Writer, plans []*ResourcePlan) error { +func (yamlRenderer) Render(w io.Writer, plans []*ResourcePlan) error { out, err := yaml.Marshal(NewResult(plans)) if err != nil { return errors.Wrap(err, "cannot marshal plans to YAML") @@ -149,12 +149,12 @@ func (YAMLRenderer) Render(w io.Writer, plans []*ResourcePlan) error { return err } -// PlanRenderer renders the human-readable plan format: a header and change +// planRenderer renders the human-readable plan format: a header and change // tree per resource, then a summary with warnings. -type PlanRenderer struct{} +type planRenderer struct{} // Render writes the plans to w in the human-readable plan format. -func (PlanRenderer) Render(w io.Writer, plans []*ResourcePlan) error { +func (planRenderer) Render(w io.Writer, plans []*ResourcePlan) error { for _, p := range plans { _, _ = fmt.Fprintln(w, header(p)) if p.Error != "" { From d2bb57950e05447dc26f7d0e01495383b5325920 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Tue, 20 Oct 2026 15:32:23 +0200 Subject: [PATCH 8/9] fix(simulate): address review feedback on plan actions and spinner ux Signed-off-by: Christopher Haar --- cmd/crossplane/simulate/simulate.go | 47 ++++++++++++++---------- cmd/crossplane/simulate/simulate_test.go | 14 ++++++- internal/containerruntime/container.go | 32 +++++++++++++--- internal/diffserver/manager.go | 26 +++++++++++-- internal/diffserver/manager_test.go | 10 ++--- internal/plan/output.go | 5 +++ internal/plan/output_test.go | 3 ++ internal/plan/plan_test.go | 9 ++++- internal/plan/proto.go | 22 +++++++++++ internal/plan/proto_test.go | 36 ++++++++++++++++++ internal/terminal/spinner.go | 46 +++++++++++++++++++++++ 11 files changed, 213 insertions(+), 37 deletions(-) diff --git a/cmd/crossplane/simulate/simulate.go b/cmd/crossplane/simulate/simulate.go index 29b5166a..bfdea6a3 100644 --- a/cmd/crossplane/simulate/simulate.go +++ b/cmd/crossplane/simulate/simulate.go @@ -52,6 +52,7 @@ import ( pkgv1beta1 "github.com/crossplane/crossplane/apis/v2/pkg/v1beta1" "github.com/crossplane/cli/v2/cmd/crossplane/common/load" + "github.com/crossplane/cli/v2/internal/async" "github.com/crossplane/cli/v2/internal/containerruntime" "github.com/crossplane/cli/v2/internal/diffserver" "github.com/crossplane/cli/v2/internal/docker" @@ -182,9 +183,9 @@ func (c *Cmd) Run(ctx context.Context, k *kong.Context, logger logging.Logger, s mgr := diffserver.NewManager(pkgClient, logger, diffserver.WithSessionID(c.SessionID)) var skipped []diffserver.Skipped - if err := sp.WrapWithSuccessSpinner("Starting provider diff servers", func() error { + if err := sp.WrapAsyncWithSuccessSpinners(func(ch async.EventChannel) error { var err error - skipped, err = mgr.Start(ctx, inputs) + skipped, err = mgr.Start(ctx, inputs, ch) return err }); err != nil { return errors.Wrap(err, "cannot start diff servers") @@ -448,24 +449,9 @@ func (c *objectStoreCache) gather(ctx context.Context, kubeClient client.Client, } } - // The ProviderConfig's group relates to the resource's group, but not - // uniformly: most providers serve it one level up (s3.aws.upbound.io → - // aws.upbound.io), others in the resource's own group - // (kubernetes.crossplane.io). Probe from the resource's group toward the - // root and take the first group that serves the kind. - group := desired.GroupVersionKind().Group - var mapping *meta.RESTMapping - for { - m, err := rmapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}) - if err == nil { - mapping = m - break - } - i := strings.Index(group, ".") - if i < 0 { - return nil - } - group = group[i+1:] + mapping := providerConfigMapping(rmapper, desired.GroupVersionKind().Group, kind) + if mapping == nil { + return nil } key := client.ObjectKey{Name: name} @@ -493,6 +479,27 @@ func (c *objectStoreCache) gather(ctx context.Context, kubeClient client.Client, return store } +// providerConfigMapping returns the REST mapping for the ProviderConfig kind +// the given resource group's provider serves, or nil when no group serves it. +// +// The ProviderConfig's group relates to the resource's group, but not +// uniformly: most providers serve it one level up (s3.aws.upbound.io → +// aws.upbound.io), others in the resource's own group +// (kubernetes.crossplane.io). Probe from the resource's group toward the root +// and take the first group that serves the kind. +func providerConfigMapping(rmapper meta.RESTMapper, group, kind string) *meta.RESTMapping { + for { + if m, err := rmapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}); err == nil { + return m + } + i := strings.Index(group, ".") + if i < 0 { + return nil + } + group = group[i+1:] + } +} + // relevantProviderImages returns the package images of exactly the providers // that serve the given resources, so simulate never starts a diff server it // will not route to. Discovery follows the ownership chain on the cluster: diff --git a/cmd/crossplane/simulate/simulate_test.go b/cmd/crossplane/simulate/simulate_test.go index 51f9bdf5..b5f9f7f0 100644 --- a/cmd/crossplane/simulate/simulate_test.go +++ b/cmd/crossplane/simulate/simulate_test.go @@ -31,6 +31,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -48,6 +49,10 @@ import ( var vpcGVK = schema.GroupVersionKind{Group: "ec2.aws.upbound.io", Version: "v1beta1", Kind: "VPC"} +func fieldValue(v string) *diffv1alpha1.FieldValue { + return &diffv1alpha1.FieldValue{Kind: &diffv1alpha1.FieldValue_Value{Value: structpb.NewStringValue(v)}} +} + func vpc(name, cidr string) *unstructured.Unstructured { return &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "ec2.aws.upbound.io/v1beta1", @@ -177,7 +182,14 @@ func TestSimulateOne(t *testing.T) { t.Run("RoutedUpdate", func(t *testing.T) { live := vpc("my-vpc", "10.0.0.0/16") kube := fake.NewClientBuilder().WithScheme(s).WithObjects(live).Build() - pc := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_UPDATE}} + pc := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{ + Action: diffv1alpha1.Action_ACTION_UPDATE, + Changes: []*diffv1alpha1.FieldChange{{ + Field: "spec.forProvider.cidrBlock", + Actual: fieldValue("10.0.0.0/16"), + Planned: fieldValue("10.1.0.0/16"), + }}, + }} got := cmd.simulateOne(ctx, kube, rm, "default", &fakeRouter{client: pc}, newObjectStoreCache(), vpc("my-vpc", "10.1.0.0/16")) diff --git a/internal/containerruntime/container.go b/internal/containerruntime/container.go index 5caf05dc..20e8cace 100644 --- a/internal/containerruntime/container.go +++ b/internal/containerruntime/container.go @@ -35,6 +35,8 @@ import ( "github.com/crossplane/crossplane-runtime/v2/pkg/errors" "github.com/crossplane/crossplane-runtime/v2/pkg/logging" + + "github.com/crossplane/cli/v2/internal/async" ) // A Container runs one package image as a local Docker container serving a @@ -94,6 +96,11 @@ type Container struct { // fake to exercise container lifecycle and cleanup handling without a // real Docker daemon. Client Client + + // Events receives progress events for the container's long-running steps, + // today only the image pull, which happens exactly when the daemon does + // not already have the image. Nil means no events are sent. + Events async.EventChannel } // Client is the subset of the Docker client a Container uses to pull its @@ -279,9 +286,8 @@ func (c *Container) create(ctx context.Context, cli Client) (string, error) { if c.PullPolicy == PullAlways { c.Log.Debug("Pulling image with pullPolicy: Always", "image", c.Image) - err = PullImage(ctx, cli, c.Image, options) - if err != nil { - return "", errors.Wrapf(err, "cannot pull Docker image %q", c.Image) + if err := c.pullImage(ctx, cli, options); err != nil { + return "", err } } @@ -315,9 +321,8 @@ func (c *Container) create(ctx context.Context, cli Client) (string, error) { // The image was not found, but we're allowed to pull it. c.Log.Debug("Image not found, pulling", "image", c.Image) - err = PullImage(ctx, cli, c.Image, options) - if err != nil { - return "", errors.Wrapf(err, "cannot pull Docker image %q", c.Image) + if err := c.pullImage(ctx, cli, options); err != nil { + return "", err } rsp, err = cli.ContainerCreate(ctx, client.ContainerCreateOptions{ @@ -337,6 +342,21 @@ func (c *Container) create(ctx context.Context, cli Client) (string, error) { return rsp.ID, nil } +// pullImage pulls the container's image, reporting the pull on the Events +// channel so callers can show it as its own step: it is the dominant cost of +// a cold start, and it only happens at all when the daemon does not have the +// image yet. +func (c *Container) pullImage(ctx context.Context, cli Client, options client.ImagePullOptions) error { + desc := "Pulling image " + c.Image + c.Events.SendEvent(desc, async.EventStatusStarted) + if err := PullImage(ctx, cli, c.Image, options); err != nil { + c.Events.SendEvent(desc, async.EventStatusFailure) + return errors.Wrapf(err, "cannot pull Docker image %q", c.Image) + } + c.Events.SendEvent(desc, async.EventStatusSuccess) + return nil +} + // start ensures the container is running and returns its address. func (c *Container) start(ctx context.Context, cli Client, containerID string) (string, error) { // Start the container (idempotent - safe to call on running containers) diff --git a/internal/diffserver/manager.go b/internal/diffserver/manager.go index 09caf56a..a65923b0 100644 --- a/internal/diffserver/manager.go +++ b/internal/diffserver/manager.go @@ -42,6 +42,7 @@ import ( pkgmetav1 "github.com/crossplane/crossplane/apis/v2/pkg/meta/v1" + "github.com/crossplane/cli/v2/internal/async" "github.com/crossplane/cli/v2/internal/containerruntime" clixpkg "github.com/crossplane/cli/v2/internal/xpkg" ) @@ -194,8 +195,10 @@ func NewManager(pkgs runtimexpkg.Client, log logging.Logger, opts ...Option) *Ma // every package that declares the DiffServer capability, in parallel and best // effort: an image that cannot serve (no capability, fetch or startup // failure) is returned as skipped rather than failing the run, since its -// resources degrade to the approximate client-side diff. -func (m *Manager) Start(ctx context.Context, inputs []Input) ([]Skipped, error) { +// resources degrade to the approximate client-side diff. Progress is reported +// per image on events (nil means no reporting), with the package pull and the +// server start as separate steps because the pull dominates a cold start. +func (m *Manager) Start(ctx context.Context, inputs []Input, events async.EventChannel) ([]Skipped, error) { var ( wg sync.WaitGroup mu sync.Mutex @@ -205,7 +208,7 @@ func (m *Manager) Start(ctx context.Context, inputs []Input) ([]Skipped, error) wg.Add(1) go func(in Input) { defer wg.Done() - s, skip := m.startOne(ctx, in) + s, skip := m.startOne(ctx, in, events) mu.Lock() defer mu.Unlock() if skip != nil { @@ -243,13 +246,19 @@ func (m *Manager) register(s *Server) { } } -func (m *Manager) startOne(ctx context.Context, in Input) (*Server, *Skipped) { +func (m *Manager) startOne(ctx context.Context, in Input, events async.EventChannel) (*Server, *Skipped) { image := in.Image + // The package pull is its own reported step: on a cold cache it downloads + // the full provider package and dwarfs everything that follows. + pull := "Pulling provider package " + image + events.SendEvent(pull, async.EventStatusStarted) pkg, err := m.pkgs.Get(ctx, image) if err != nil { + events.SendEvent(pull, async.EventStatusFailure) return nil, &Skipped{Image: image, Reason: fmt.Sprintf("cannot fetch package: %s", err)} } + events.SendEvent(pull, async.EventStatusSuccess) meta := pkg.GetMeta() if meta == nil || !pkgmetav1.CapabilitiesContainFuzzyMatch(meta.GetCapabilities(), CapabilityDiffServer) { @@ -293,15 +302,23 @@ func (m *Manager) startOne(ctx context.Context, in Input) (*Server, *Skipped) { Target: in.Target, Log: m.log, Client: m.cli, + // The container reports its own image pull on the same channel, as a + // separate step: Docker pulls the image again even when the package + // cache is warm, and only when the daemon does not have it yet. + Events: events, } + start := "Starting diff server for " + image + events.SendEvent(start, async.EventStatusStarted) running, err := ctr.Start(ctx) if err != nil { + events.SendEvent(start, async.EventStatusFailure) return nil, &Skipped{Image: image, Reason: fmt.Sprintf("cannot start diff server container: %s", err)} } server, err := m.connect(ctx, running, image) if err != nil { + events.SendEvent(start, async.EventStatusFailure) // An orphaned container is left behind even on failure, for debugging. if !running.Reused && cleanup != CleanupOrphan { if serr := running.Stop(ctx, CleanupRemove); serr != nil { @@ -310,6 +327,7 @@ func (m *Manager) startOne(ctx context.Context, in Input) (*Server, *Skipped) { } return nil, &Skipped{Image: image, Reason: err.Error()} } + events.SendEvent(start, async.EventStatusSuccess) server.GroupKinds = gks server.cleanup = cleanup diff --git a/internal/diffserver/manager_test.go b/internal/diffserver/manager_test.go index d9fb4c6b..75a36c1d 100644 --- a/internal/diffserver/manager_test.go +++ b/internal/diffserver/manager_test.go @@ -267,7 +267,7 @@ func TestManagerStart(t *testing.T) { fd := newFakeDocker("", nil) m := NewManager(pkgs, log, WithDockerClient(fd.client())) - skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-s3:v2.9.0"})) + skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-s3:v2.9.0"}), nil) if err != nil { t.Fatalf("Start(...): unexpected error: %v", err) } @@ -283,7 +283,7 @@ func TestManagerStart(t *testing.T) { fd := newFakeDocker(servePlanService(t), nil) m := NewManager(pkgs, log, WithDockerClient(fd.client())) - skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"})) + skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"}), nil) if err != nil { t.Fatalf("Start(...): unexpected error: %v", err) } @@ -322,7 +322,7 @@ func TestManagerStart(t *testing.T) { }) m := NewManager(pkgs, log, WithDockerClient(fd.client())) - skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"})) + skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"}), nil) if err != nil || len(skipped) != 0 { t.Fatalf("Start(...): unexpected error or skips: %v %+v", err, skipped) } @@ -335,7 +335,7 @@ func TestManagerStart(t *testing.T) { fd := newFakeDocker(servePlanService(t), nil) m := NewManager(pkgs, log, WithDockerClient(fd.client())) - if _, err := m.Start(ctx, []Input{{Image: "xpkg.example.org/provider-aws-ec2:v2.9.0", Cleanup: CleanupOrphan}}); err != nil { + if _, err := m.Start(ctx, []Input{{Image: "xpkg.example.org/provider-aws-ec2:v2.9.0", Cleanup: CleanupOrphan}}, nil); err != nil { t.Fatalf("Start(...): unexpected error: %v", err) } if err := m.Stop(ctx); err != nil { @@ -357,7 +357,7 @@ func TestManagerStart(t *testing.T) { fd := newFakeDocker(lis.Addr().String(), nil) m := NewManager(pkgs, log, WithDockerClient(fd.client()), WithReadyTimeout(700*time.Millisecond)) - skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"})) + skipped, err := m.Start(ctx, Images([]string{"xpkg.example.org/provider-aws-ec2:v2.9.0"}), nil) if err != nil { t.Fatalf("Start(...): unexpected error: %v", err) } diff --git a/internal/plan/output.go b/internal/plan/output.go index 102c9250..3e91b386 100644 --- a/internal/plan/output.go +++ b/internal/plan/output.go @@ -214,6 +214,11 @@ func symbol(p *ResourcePlan) string { func renderChangeTree(w io.Writer, changes []FieldChange) { var prev []string for _, c := range sortedChanges(changes) { + if c.Actual == nil && c.Planned == nil { + // No value on either side renders no leaf line, so printing its + // parent keys would leave them dangling with nothing beneath. + continue + } segs := strings.Split(c.Path, ".") if len(segs) == 0 { continue diff --git a/internal/plan/output_test.go b/internal/plan/output_test.go index a373c7ee..64ad4f6f 100644 --- a/internal/plan/output_test.go +++ b/internal/plan/output_test.go @@ -48,6 +48,9 @@ func testPlans() []*ResourcePlan { { APIVersion: "kms.aws.upbound.io/v1beta1", Kind: "Key", Name: "my-key", Action: ActionNoOp, + // A change with no value on either side renders nothing, not even + // its parent keys. + Changes: []FieldChange{{Path: "spec.forProvider.tags.cost-center"}}, }, { APIVersion: "s3.aws.upbound.io/v1beta2", Kind: "Bucket", Name: "my-bucket", diff --git a/internal/plan/plan_test.go b/internal/plan/plan_test.go index 3e2cf2c4..ee8e01cc 100644 --- a/internal/plan/plan_test.go +++ b/internal/plan/plan_test.go @@ -53,7 +53,14 @@ func TestPlan(t *testing.T) { } t.Run("RequestCarriesAllState", func(t *testing.T) { - c := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_UPDATE}} + c := &fakePlanClient{rsp: &diffv1alpha1.PlanResponse{ + Action: diffv1alpha1.Action_ACTION_UPDATE, + Changes: []*diffv1alpha1.FieldChange{{ + Field: "spec.forProvider.cidrBlock", + Actual: value("10.0.0.0/16"), + Planned: value("10.1.0.0/16"), + }}, + }} got, err := Plan(context.Background(), c, desired, actual, []*structpb.Struct{secret}) if err != nil { diff --git a/internal/plan/proto.go b/internal/plan/proto.go index 008343ea..d56e7dd1 100644 --- a/internal/plan/proto.go +++ b/internal/plan/proto.go @@ -66,9 +66,31 @@ func FromResponse(desired *unstructured.Unstructured, rsp *diffv1alpha1.PlanResp } } + // A diff server can answer update while reporting no field changes: the + // Terraform diff the action is decided on can consist entirely of flatmap + // count keys ("tags.%", "serviceEndpoints.#"), which carry a collection's + // size rather than a field and are never reported as changes. An update + // that changes no field is a no-op to the user, so report it as one. A + // replace is left alone: it warns about a forced replacement even when the + // field that forces it is not reported. + if p.Action == ActionUpdate && !hasVisibleChange(p.Changes) { + p.Action = ActionNoOp + } + return p, nil } +// hasVisibleChange reports whether any change carries a value on either side. +// A change with neither has nothing to say about the field it names. +func hasVisibleChange(changes []FieldChange) bool { + for _, c := range changes { + if c.Actual != nil || c.Planned != nil { + return true + } + } + return false +} + func fromFieldValue(fv *diffv1alpha1.FieldValue) *Value { if fv == nil { return nil diff --git a/internal/plan/proto_test.go b/internal/plan/proto_test.go index f05d1fbb..d857825e 100644 --- a/internal/plan/proto_test.go +++ b/internal/plan/proto_test.go @@ -136,6 +136,42 @@ func TestFromResponse(t *testing.T) { }, }}, }, + "UpdateWithoutChanges": { + reason: "An update reporting no field changes changes nothing the user can see, so it should decode as a no-op.", + args: args{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_UPDATE}}, + want: want{plan: &ResourcePlan{ + APIVersion: "ec2.aws.upbound.io/v1beta1", + Kind: "VPC", + Name: "my-vpc", + Action: ActionNoOp, + }}, + }, + "UpdateWithValuelessChanges": { + reason: "An update whose changes all carry no value on either side should decode as a no-op, keeping the changes.", + args: args{rsp: &diffv1alpha1.PlanResponse{ + Action: diffv1alpha1.Action_ACTION_UPDATE, + Changes: []*diffv1alpha1.FieldChange{{ + Field: "spec.forProvider.cidrBlock", + }}, + }}, + want: want{plan: &ResourcePlan{ + APIVersion: "ec2.aws.upbound.io/v1beta1", + Kind: "VPC", + Name: "my-vpc", + Action: ActionNoOp, + Changes: []FieldChange{{Path: "spec.forProvider.cidrBlock"}}, + }}, + }, + "ReplaceWithoutChanges": { + reason: "A replace reporting no field changes still warns about a forced replacement, so it should stay a replace.", + args: args{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_REPLACE}}, + want: want{plan: &ResourcePlan{ + APIVersion: "ec2.aws.upbound.io/v1beta1", + Kind: "VPC", + Name: "my-vpc", + Action: ActionReplace, + }}, + }, "NoOp": { reason: "A no-op should decode with no changes.", args: args{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_NO_OP}}, diff --git a/internal/terminal/spinner.go b/internal/terminal/spinner.go index a3fcf54d..036d617c 100644 --- a/internal/terminal/spinner.go +++ b/internal/terminal/spinner.go @@ -279,11 +279,56 @@ func (m *MultiSpinner) Start() { tea.WithInput(nil), tea.WithoutSignalHandler(), tea.WithOutput(m.out), + tea.WithEnvironment(noQueryEnviron()), ) go runProgram(m.program) } +// noQueryEnviron returns the process environment adjusted so bubbletea skips +// querying the terminal for synchronized output (mode 2026) and grapheme +// clustering (mode 2027) support. The spinners deliberately run without an +// input reader (tea.WithInput(nil)) so the terminal stays in cooked mode and +// Ctrl+C keeps reaching main's signal handler - but then nothing reads the +// terminal's replies to those queries either, and the tty echoes them as +// garbage like "^[[?2026;2$y". Bubbletea has no option to suppress the +// queries; it only skips them for environments its heuristic distrusts, so +// present one: an SSH session on a generic terminal. +func noQueryEnviron() []string { + environ := os.Environ() + env := make([]string, 0, len(environ)+2) + hasColorterm, hadWTSession := false, false + for _, kv := range environ { + k, v, _ := strings.Cut(kv, "=") + switch k { + case "SSH_TTY": + continue // Re-added below. + case "COLORTERM": + hasColorterm = true + case "WT_SESSION": + // Windows Terminal is queried regardless of SSH. Dropping the + // variable also drops the truecolor support color detection infers + // from it, which COLORTERM restores. + hadWTSession = true + continue + case "TERM": + // Terminals bubbletea queries by name regardless of SSH. They all + // accept the generic profile, and declare COLORTERM themselves. + for _, t := range []string{"ghostty", "wezterm", "alacritty", "kitty", "rio"} { + if strings.Contains(v, t) { + kv = "TERM=xterm-256color" + break + } + } + } + env = append(env, kv) + } + if hadWTSession && !hasColorterm { + env = append(env, "COLORTERM=truecolor") + } + return append(env, "SSH_TTY=/dev/null") +} + // runProgram runs p until it quits. It doesn't handle signals: main's // signal.NotifyContext does that once for all commands by cancelling the // command context. Spinner lifetime is handled once by the SpinnerPrinter @@ -417,6 +462,7 @@ func (ss *SuccessSpinner) Start() { tea.WithOutput(ss.out), tea.WithInput(nil), tea.WithoutSignalHandler(), + tea.WithEnvironment(noQueryEnviron()), ) go runProgram(ss.program) From d5aeea4788cb23a1ae46550ba78fbc0f15713a0d Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Tue, 20 Oct 2026 15:39:46 +0200 Subject: [PATCH 9/9] fix(simulate): address review feedback on plan actions and spinner ux Signed-off-by: Christopher Haar --- internal/plan/proto_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/plan/proto_test.go b/internal/plan/proto_test.go index d857825e..76a89429 100644 --- a/internal/plan/proto_test.go +++ b/internal/plan/proto_test.go @@ -137,7 +137,7 @@ func TestFromResponse(t *testing.T) { }}, }, "UpdateWithoutChanges": { - reason: "An update reporting no field changes changes nothing the user can see, so it should decode as a no-op.", + reason: "An update reporting no field changes alters nothing the user can see, so it should decode as a no-op.", args: args{rsp: &diffv1alpha1.PlanResponse{Action: diffv1alpha1.Action_ACTION_UPDATE}}, want: want{plan: &ResourcePlan{ APIVersion: "ec2.aws.upbound.io/v1beta1",