From 877a1d551ba67b3b902d246aab6ade2ba350dfe4 Mon Sep 17 00:00:00 2001 From: Jonathan Ogilvie Date: Fri, 30 Oct 2026 14:43:09 -0400 Subject: [PATCH 1/8] test(integration): add envtest fixtures package for integration tests Add internal/testutils/integration, an importable package of fixtures for tests that run CLI commands against an envtest API server: Start sets up a cluster serving Crossplane's CRDs and those generated from a test's XRDs, with its manifests as Crossplane would have left them; CRDsForXRD and PinnedCrossplaneCRDsDir generate and locate CRDs; ReapOrphanedEnvtest kills envtest servers an earlier run left behind. The cluster fixtures build only with the integration tag. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Jonathan Ogilvie --- go.mod | 10 +- go.sum | 26 +- internal/testutils/integration/cluster.go | 247 +++++ internal/testutils/integration/crds.go | 103 ++ internal/testutils/integration/crds_test.go | 336 ++++++ internal/testutils/integration/doc.go | 31 + internal/testutils/integration/fixtures.go | 987 ++++++++++++++++++ internal/testutils/integration/reaper.go | 163 +++ internal/testutils/integration/reaper_test.go | 71 ++ nix/vendor-hashes.nix | 2 +- 10 files changed, 1971 insertions(+), 5 deletions(-) create mode 100644 internal/testutils/integration/cluster.go create mode 100644 internal/testutils/integration/crds.go create mode 100644 internal/testutils/integration/crds_test.go create mode 100644 internal/testutils/integration/doc.go create mode 100644 internal/testutils/integration/fixtures.go create mode 100644 internal/testutils/integration/reaper.go create mode 100644 internal/testutils/integration/reaper_test.go diff --git a/go.mod b/go.mod index 225b496b..d9a88d2f 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/oapi-codegen/oapi-codegen/v2 v2.8.0 github.com/pkg/errors v0.9.1 github.com/posener/complete v1.2.3 + github.com/shirou/gopsutil/v4 v4.26.9 github.com/spf13/afero v1.15.0 github.com/willabides/kongplete v0.4.0 golang.org/x/mod v0.41.0 @@ -147,6 +148,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dprotaso/go-yit v0.0.0-20250513223454-5ece0c5aa76c // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.11.1 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/evanphx/json-patch v5.9.11+incompatible // indirect @@ -165,6 +167,7 @@ require ( github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/analysis v0.25.2 // indirect github.com/go-openapi/errors v0.22.7 // indirect github.com/go-openapi/jsonpointer v0.23.1 // indirect @@ -224,6 +227,7 @@ require ( github.com/lib/pq v1.10.9 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/lucasb-eyer/go-colorful v1.4.1 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.27 // indirect @@ -251,6 +255,7 @@ require ( github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect @@ -284,6 +289,8 @@ require ( github.com/theupdateframework/go-tuf v0.7.0 // indirect github.com/theupdateframework/go-tuf/v2 v2.4.2-0.20260407074541-7e8f69f906ef // indirect github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect github.com/transparency-dev/formats v0.1.1 // indirect github.com/transparency-dev/merkle v0.0.2 // indirect github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect @@ -294,6 +301,7 @@ require ( github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yuin/goldmark v1.8.4 // indirect github.com/yuin/goldmark-emoji v1.0.6 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect go.opentelemetry.io/otel v1.45.0 // indirect @@ -307,7 +315,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect golang.org/x/crypto v0.58.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect diff --git a/go.sum b/go.sum index ef561020..b45da616 100644 --- a/go.sum +++ b/go.sum @@ -276,6 +276,8 @@ github.com/dprotaso/go-yit v0.0.0-20250513223454-5ece0c5aa76c h1:EMwsP/vaHQDLhAX github.com/dprotaso/go-yit v0.0.0-20250513223454-5ece0c5aa76c/go.mod h1:lHwJo6jMevQL9tNpW6vLyhkK13bYHBcoh9tUakMhbnE= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.11.1 h1:2zpWRSQNVKN4eKsKO9eM1ILDgWfYMY9GwqRmK6XeQ/0= +github.com/ebitengine/purego v0.11.1/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emicklei/dot v1.11.0 h1:zsrhCuFHAJge/aZIC4N4LdHy5tqYu4tWEaUzIwdYj4Y= @@ -334,6 +336,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.25.2 h1:I0vy4n3alz+DHTiN1PRhCb7QZxkK6g5YmswZKv2TKuw= github.com/go-openapi/analysis v0.25.2/go.mod h1:Uhs1t/2XR10EnwONYILGEzw8gcfGIG5Xk5K2AxnhqDo= github.com/go-openapi/errors v0.22.7 h1:JLFBGC0Apwdzw3484MmBqspjPbwa2SHvpDm0u5aGhUA= @@ -439,6 +443,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-containerregistry v0.22.1 h1:RZuuSYhTvlDvtsK+NkutoCZ//C0X2ebLK8X8l3ULs84= @@ -568,6 +573,8 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= @@ -669,6 +676,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU= +github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= @@ -712,6 +721,8 @@ github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= +github.com/shirou/gopsutil/v4 v4.26.9 h1:CaBo/hBFqvlJoLvAQEKdHdf5GD97/MN2ACfyv63+2Ig= +github.com/shirou/gopsutil/v4 v4.26.9/go.mod h1:nKH+8wX2zxr/mDbsR+AAy6Qd2z4ZOQ7wfQaEp+yrYmg= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sigstore/cosign/v2 v2.6.3 h1:1W+rZWz0zkTfqmTmYBOQS/Jt97NKz1OCzxTV2YAp1+s= @@ -772,8 +783,8 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/theupdateframework/go-tuf v0.7.0 h1:CqbQFrWo1ae3/I0UCblSbczevCCbS31Qvs5LdxRWqRI= @@ -790,6 +801,10 @@ github.com/tink-crypto/tink-go/v2 v2.6.0 h1:+KHNBHhWH33Vn+igZWcsgdEPUxKwBMEe0QC6 github.com/tink-crypto/tink-go/v2 v2.6.0/go.mod h1:2WbBA6pfNsAfBwDCggboaHeB2X29wkU8XHtGwh2YIk8= github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0= github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/transparency-dev/formats v0.1.1 h1:4bVHJc+KdBgpA1OJD1yjI+g0i5Z1graCppTMH8lWKJI= github.com/transparency-dev/formats v0.1.1/go.mod h1:qtZ8goRuJ8FTBG9c9+Bj0rn2rUG7eG/AUTkr+Aw3jFw= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= @@ -824,6 +839,8 @@ github.com/yuin/goldmark v1.8.4 h1:oat/nd3U6NeQqFEL3xpEJq7d7c86NI+DbSNGAs4xnjA= github.com/yuin/goldmark v1.8.4/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs= github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= github.com/zalando/go-keyring v0.2.3/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= go.etcd.io/etcd/api/v3 v3.6.8 h1:gqb1VN92TAI6G2FiBvWcqKtHiIjr4SU2GdXxTwyexbM= @@ -892,8 +909,9 @@ go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s= go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -940,6 +958,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -947,6 +966,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/testutils/integration/cluster.go b/internal/testutils/integration/cluster.go new file mode 100644 index 00000000..6f760923 --- /dev/null +++ b/internal/testutils/integration/cluster.go @@ -0,0 +1,247 @@ +//go:build integration + +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + rbacv1 "k8s.io/api/rbac/v1" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + cgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + xpextv1 "github.com/crossplane/crossplane/apis/v2/apiextensions/v1" + xpextv2 "github.com/crossplane/crossplane/apis/v2/apiextensions/v2" + pkgv1 "github.com/crossplane/crossplane/apis/v2/pkg/v1" +) + +// XRDAPIVersion is the apiextensions.crossplane.io version of the XRDs a Setup's XRs are defined by. It decides +// where Start writes an XR's resourceRefs. +type XRDAPIVersion int + +const ( + // XRDV2 XRs list their composed resources at spec.crossplane.resourceRefs. It is the zero value, so the + // default. + XRDV2 XRDAPIVersion = iota + // XRDV1 (legacy) XRs list their composed resources at spec.resourceRefs. + XRDV1 +) + +// FieldManagerApply is a manifest to server-side apply under a specific field manager. +type FieldManagerApply struct { + File string + FieldManager string +} + +// Setup is the state Start puts in a fresh cluster, in the order its fields are listed. Every path is a YAML file +// of one or more manifests, relative to the test's working directory (its package directory). +type Setup struct { + // CRDDirs are directories of plain CRDs to install besides Crossplane's own, such as stand-ins for managed + // resources. XR and claim CRDs don't go here: Start generates them from the XRDs in SetupFiles, as Crossplane + // would. + CRDDirs []string + // SetupFiles are created in order, each filled in with what Crossplane would have written to it (see + // Start). + SetupFiles []string + // UnestablishedXRDs are XRD manifests created after SetupFiles but, unlike an XRD in SetupFiles, given no + // generated CRDs: an XRD Crossplane has not established yet, so its XR and claim types are not served. + UnestablishedXRDs []string + // CrossplaneManagedResources are the XRs that exist in the cluster, with what they compose. Every composed + // resource is server-side applied under crossplaneFieldManager and given a controller ownerReference to its + // XR's real UID, and each XR lists its composed resources in its resourceRefs, as Crossplane leaves them. Use + // Composes for an XR without nested XRs. Put a composed resource in SetupFiles instead only when it + // deliberately has no controlling XR. + CrossplaneManagedResources []HierarchicalOwnershipRelation + // XRDAPIVersion is the version of the XRDs that define CrossplaneManagedResources' XRs. + XRDAPIVersion XRDAPIVersion + // FieldManagerApplies are server-side applied, in order, each under its own field manager. Use it when a + // test depends on which manager owns a field: a plain Create records only the test client as owner, and the + // apiserver drops managedFields supplied on Create. + FieldManagerApplies []FieldManagerApply + // DeleteAfterSetup are files whose resources are deleted once everything else is set up. With a finalizer, + // which no controller clears under envtest, this leaves them Terminating. + DeleteAfterSetup []string + // RBACRules, when set, makes Start also provision a user bound to a ClusterRole granting only these rules + // (plus the discovery and self-review access the apiserver's bootstrap RBAC grants every authenticated + // user), returned as Cluster.UserConfig. + RBACRules []rbacv1.PolicyRule +} + +// Cluster is a running envtest cluster that Start has set up. +type Cluster struct { + // Config is the cluster admin's. + Config *rest.Config + // UserConfig is that of a user granted only the Setup's RBACRules; nil when it sets none. + UserConfig *rest.Config + // Kubeconfig is a kubeconfig file's contents for the cluster admin, for commands that load one. + Kubeconfig []byte + // Client is the cluster admin's. + Client client.Client + // Context is the test's context, bounded by Start's timeout. + Context context.Context //nolint:containedctx // Handed to the test with the cluster it bounds. +} + +// Start starts an envtest cluster and puts s in place. It serves Crossplane's own CRDs, at the version go.mod +// pins (PinnedCrossplaneCRDsDir), s.CRDDirs' CRDs, and the CRDs CRDsForXRD generates from the XRDs in +// s.SetupFiles. +// +// No Crossplane controller runs under envtest, so Start does what they would have to the resources it sets up, +// wherever a manifest leaves it out: XRDs control their generated CRDs; each Composition has a current +// CompositionRevision, controlled by it; a revision without a name is named and hash-labelled from its +// Composition; an XR and everything it composes carry its composite and claim labels; and an XR that names a +// Composition but no revision references its current one. A value a manifest sets always wins, which is how a +// test sets up a state Crossplane wouldn't leave. +// +// Start skips the test when the environment lacks envtest's binaries (KUBEBUILDER_ASSETS) or Crossplane's CRDs +// (CROSSPLANE_CRDS_DIR); `nix run .#integration-test` provides both. The cluster is stopped and the context +// cancelled when the test ends. +func Start(t *testing.T, s Setup, timeout time.Duration) *Cluster { + t.Helper() + + // envtest needs kube-apiserver and etcd binaries it cannot download itself; without them it would fail to + // start rather than skip. + if !envtestBinariesAvailable() { + t.Skip("env: no envtest binaries (KUBEBUILDER_ASSETS unset); run via nix run .#integration-test") + } + + if os.Getenv(crossplaneCRDsDirEnv) == "" { + t.Skipf("env: %s unset; run via nix run .#integration-test, or set it to Crossplane's cluster/crds", + crossplaneCRDsDirEnv) + } + + crds, err := envtestCRDs(append([]string{PinnedCrossplaneCRDsDir(t)}, s.CRDDirs...), s.SetupFiles) + if err != nil { + t.Fatalf("failed to assemble CRDs for the test environment: %v", err) + } + + testEnv := &envtest.Environment{ + CRDs: crds, + // Each cluster needs its own scheme, because envtest modifies it while installing CRDs. + Scheme: newScheme(), + // Leaving ControlPlane unset lets envtest start its own apiserver and etcd on random ports, so tests + // can run their clusters in parallel. + } + + cfg, err := testEnv.Start() + if err != nil { + t.Fatalf("failed to start test environment: %v", err) + } + + t.Cleanup(func() { + if err := testEnv.Stop(); err != nil { + t.Logf("failed to stop test environment: %v", err) + } + }) + + k8sClient, err := client.New(cfg, client.Options{Scheme: testEnv.Scheme}) + if err != nil { + t.Fatalf("failed to create client: %v", err) + } + + ctx, cancel := context.WithTimeout(t.Context(), timeout) + t.Cleanup(cancel) + + defaults := newCrossplaneDefaults(k8sClient, crds) + + if err := applyResourcesFromFiles(ctx, k8sClient, defaults, s.SetupFiles); err != nil { + t.Fatalf("failed to setup resources: %v", err) + } + + if err := applyResourcesFromFiles(ctx, k8sClient, defaults, s.UnestablishedXRDs); err != nil { + t.Fatalf("failed to setup unestablished XRDs: %v", err) + } + + if err := reconcileXRDCRDOwnership(ctx, k8sClient); err != nil { + t.Fatalf("failed to make XRDs the controllers of their CRDs: %v", err) + } + + if err := reconcileCompositionRevisions(ctx, k8sClient); err != nil { + t.Fatalf("failed to reconcile CompositionRevisions: %v", err) + } + + // The Crossplane-managed resources are in place before FieldManagerApplies run, since those model another + // manager writing to objects that already exist. + if len(s.CrossplaneManagedResources) > 0 { + if err := applyHierarchicalOwnership(ctx, k8sClient, defaults, s.XRDAPIVersion, s.CrossplaneManagedResources); err != nil { + t.Fatalf("failed to setup Crossplane-managed resources: %v", err) + } + } + + for _, a := range s.FieldManagerApplies { + resources, err := ReadResourcesFromFile(a.File) + if err != nil { + t.Fatalf("failed to read %s: %v", a.File, err) + } + + for _, r := range resources { + if err := applyResourceWithSSA(ctx, k8sClient, r, a.FieldManager); err != nil { + t.Fatalf("failed to setup field-managed resources: %v", err) + } + } + } + + if err := deleteResourcesFromFiles(ctx, k8sClient, s.DeleteAfterSetup); err != nil { + t.Fatalf("failed to delete resources: %v", err) + } + + cluster := &Cluster{Config: cfg, Kubeconfig: testEnv.KubeConfig, Client: k8sClient, Context: ctx} + + if s.RBACRules != nil { + cluster.UserConfig, err = addUserWithRules(ctx, testEnv, k8sClient, cfg, s.RBACRules) + if err != nil { + t.Fatalf("failed to set up a user with the setup's RBAC rules: %v", err) + } + } + + return cluster +} + +// newScheme returns a scheme of the Kubernetes and Crossplane types a test cluster serves. +func newScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + + _ = cgoscheme.AddToScheme(scheme) + _ = xpextv1.AddToScheme(scheme) + _ = xpextv2.AddToScheme(scheme) + _ = pkgv1.AddToScheme(scheme) + _ = extv1.AddToScheme(scheme) + + return scheme +} + +// envtestBinariesAvailable reports whether envtest can find a kube-apiserver to start: in KUBEBUILDER_ASSETS, or +// at envtest's built-in default. +func envtestBinariesAvailable() bool { + if dir := os.Getenv("KUBEBUILDER_ASSETS"); dir != "" { + if _, err := os.Stat(filepath.Join(dir, "kube-apiserver")); err == nil { //nolint:gosec // Path set by the test runner. + return true + } + } + + _, err := os.Stat("/usr/local/kubebuilder/bin/kube-apiserver") + + return err == nil +} diff --git a/internal/testutils/integration/crds.go b/internal/testutils/integration/crds.go new file mode 100644 index 00000000..c176fce0 --- /dev/null +++ b/internal/testutils/integration/crds.go @@ -0,0 +1,103 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "os" + "testing" + + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + un "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + + xpextv1 "github.com/crossplane/crossplane/apis/v2/apiextensions/v1" + + clicrd "github.com/crossplane/cli/v2/pkg/crd" + clixr "github.com/crossplane/cli/v2/pkg/xr" +) + +// CRDsForXRD returns the CRDs Crossplane generates from an XRD manifest: the composite resource's CRD and, +// when the XRD sets spec.claimNames, the claim's. Generation is upstream's (crossplane-runtime pkg/xcrd, via +// the crossplane CLI's ConvertToCRDs), so a test cluster gets the same CRDs a real one would. +// +// xcrd consumes the XRD as Crossplane's controller reads it: at apiextensions.crossplane.io/v1, with the +// apiserver's defaults applied. A manifest has neither, and the defaults decide what is generated (the +// policy defaults the CRDs declare, and the scope of an XRD that omits it, which is Namespaced at v2 but +// LegacyCluster at v1). So the manifest is first defaulted exactly as the apiserver would: with the schema +// of its own version, then relabelled to v1 (the XRD CRD converts with strategy None, so only apiVersion +// changes) and defaulted with v1's. Both schemas come from xrdCRD, the CompositeResourceDefinition CRD. +// +// The generated CRDs carry no owner reference: xcrd points one at the XRD, but the XRD has not been +// created yet when a test installs its CRDs, so it has no UID and an apiserver would reject the reference. +// +// The input is not modified. +func CRDsForXRD(xrd *un.Unstructured, xrdCRD *extv1.CustomResourceDefinition) ([]*extv1.CustomResourceDefinition, error) { + if gk := xrd.GroupVersionKind().GroupKind(); gk != xpextv1.CompositeResourceDefinitionGroupVersionKind.GroupKind() { + return nil, errors.Errorf("%s %q is not a CompositeResourceDefinition", gk, xrd.GetName()) + } + + defaulted := xrd.DeepCopy() + + if err := clixr.ApplyCRDDefaults(defaulted.Object, defaulted.GetAPIVersion(), *xrdCRD); err != nil { + return nil, errors.Wrapf(err, "cannot default XRD %q at %s", xrd.GetName(), xrd.GetAPIVersion()) + } + + read := xpextv1.CompositeResourceDefinitionGroupVersionKind.GroupVersion().String() + if defaulted.GetAPIVersion() != read { + defaulted.SetAPIVersion(read) + + if err := clixr.ApplyCRDDefaults(defaulted.Object, read, *xrdCRD); err != nil { + return nil, errors.Wrapf(err, "cannot default XRD %q at %s", xrd.GetName(), read) + } + } + + crds, err := clicrd.ConvertToCRDs([]*un.Unstructured{defaulted}) + if err != nil { + return nil, errors.Wrapf(err, "cannot generate CRDs for XRD %q", xrd.GetName()) + } + + for _, crd := range crds { + crd.SetOwnerReferences(nil) + } + + return crds, nil +} + +// crossplaneCRDsDirEnv names the environment variable that holds the directory PinnedCrossplaneCRDsDir returns. +const crossplaneCRDsDirEnv = "CROSSPLANE_CRDS_DIR" + +// PinnedCrossplaneCRDsDir returns the directory of Crossplane's own CRDs (Compositions, XRDs, Functions, …) at the +// version of github.com/crossplane/crossplane/apis/v2 that go.mod pins, so they match the xcrd and the rest of the +// Crossplane code under test. nix/crossplane-crds.nix fetches them, and both `nix run .#test` and the sandboxed +// `test` check export their location as CROSSPLANE_CRDS_DIR. The test fails with how to provide them if missing. +func PinnedCrossplaneCRDsDir(tb testing.TB) string { + tb.Helper() + + dir := os.Getenv(crossplaneCRDsDirEnv) + if dir == "" { + tb.Fatalf("%s is not set; run the tests with `nix run .#test` (or `./nix.sh run .#test`), which fetches "+ + "Crossplane's CRDs at the version go.mod pins", crossplaneCRDsDirEnv) + } + + if info, err := os.Stat(dir); err != nil || !info.IsDir() { //nolint:gosec // Test-only path set by the test runner. + tb.Fatalf("%s=%s is not a directory of Crossplane's CRDs; run the tests with `nix run .#test`", + crossplaneCRDsDirEnv, dir) + } + + return dir +} diff --git a/internal/testutils/integration/crds_test.go b/internal/testutils/integration/crds_test.go new file mode 100644 index 00000000..90c9d22f --- /dev/null +++ b/internal/testutils/integration/crds_test.go @@ -0,0 +1,336 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + un "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +// The XRD CRD's file in PinnedCrossplaneCRDsDir, the same file the integration suite installs. +const xrdCRDFile = "apiextensions.crossplane.io_compositeresourcedefinitions.yaml" + +// policyField is where a generated CRD declares one of Crossplane's policy fields, and the default it +// declares there. A zero value means the CRD does not declare the field at all. +type policyField struct { + Path string + Default string +} + +// crdShape is the part of a generated CRD these tests pin down: identity, scope, and the two policy +// fields whose defaults differ between an XR's CRD and its claim's. +type crdShape struct { + Name string + Scope extv1.ResourceScope + CompositionUpdatePolicy policyField + CompositeDeletePolicy policyField +} + +func TestCRDsForXRD(t *testing.T) { + xrdCRD := loadXRDCRD(t) + + tests := map[string]struct { + reason string + xrd string + want []crdShape + wantErr bool + }{ + "V1XRDWithClaimNames": { + reason: "A v1 XRD yields a legacy XR CRD that defaults compositionUpdatePolicy and has no compositeDeletePolicy, " + + "and a claim CRD that does not default compositionUpdatePolicy but defaults compositeDeletePolicy, " + + "both from the defaults the apiserver applies to the XRD itself", + xrd: ` +apiVersion: apiextensions.crossplane.io/v1 +kind: CompositeResourceDefinition +metadata: + name: xthings.example.org +spec: + group: example.org + names: + kind: XThing + plural: xthings + claimNames: + kind: Thing + plural: things + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + field: + type: string +`, + want: []crdShape{ + { + Name: "xthings.example.org", + Scope: extv1.ClusterScoped, + CompositionUpdatePolicy: policyField{Path: "spec.compositionUpdatePolicy", Default: `"Automatic"`}, + }, + { + Name: "things.example.org", + Scope: extv1.NamespaceScoped, + CompositionUpdatePolicy: policyField{Path: "spec.compositionUpdatePolicy"}, + CompositeDeletePolicy: policyField{Path: "spec.compositeDeletePolicy", Default: `"Background"`}, + }, + }, + }, + "V1XRDWithExplicitPolicies": { + reason: "Policies the XRD sets explicitly become the generated CRDs' defaults", + xrd: ` +apiVersion: apiextensions.crossplane.io/v1 +kind: CompositeResourceDefinition +metadata: + name: xthings.example.org +spec: + group: example.org + names: + kind: XThing + plural: xthings + claimNames: + kind: Thing + plural: things + defaultCompositionUpdatePolicy: Manual + defaultCompositeDeletePolicy: Foreground + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object +`, + want: []crdShape{ + { + Name: "xthings.example.org", + Scope: extv1.ClusterScoped, + CompositionUpdatePolicy: policyField{Path: "spec.compositionUpdatePolicy", Default: `"Manual"`}, + }, + { + Name: "things.example.org", + Scope: extv1.NamespaceScoped, + CompositionUpdatePolicy: policyField{Path: "spec.compositionUpdatePolicy"}, + CompositeDeletePolicy: policyField{Path: "spec.compositeDeletePolicy", Default: `"Foreground"`}, + }, + }, + }, + "V2XRDWithoutScope": { + reason: "A v2 XRD that omits scope is Namespaced (its own version's default, not v1's LegacyCluster), " + + "nests the machinery under spec.crossplane, defaults compositionUpdatePolicy, and has no claim CRD", + xrd: ` +apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +metadata: + name: xthings.example.org +spec: + group: example.org + names: + kind: XThing + plural: xthings + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object +`, + want: []crdShape{ + { + Name: "xthings.example.org", + Scope: extv1.NamespaceScoped, + CompositionUpdatePolicy: policyField{Path: "spec.crossplane.compositionUpdatePolicy", Default: `"Automatic"`}, + }, + }, + }, + "V2ClusterXRD": { + reason: "A v2 Cluster XRD yields a cluster-scoped XR CRD that still nests the machinery under spec.crossplane", + xrd: ` +apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +metadata: + name: xthings.example.org +spec: + group: example.org + scope: Cluster + names: + kind: XThing + plural: xthings + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object +`, + want: []crdShape{ + { + Name: "xthings.example.org", + Scope: extv1.ClusterScoped, + CompositionUpdatePolicy: policyField{Path: "spec.crossplane.compositionUpdatePolicy", Default: `"Automatic"`}, + }, + }, + }, + "NotAnXRD": { + reason: "Anything but an XRD is a caller error, never silently ignored", + xrd: ` +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: xthings.example.org +`, + wantErr: true, + }, + "UnknownXRDVersion": { + reason: "An XRD at a version the XRD CRD does not serve cannot be defaulted, so it is an error", + xrd: ` +apiVersion: apiextensions.crossplane.io/v9 +kind: CompositeResourceDefinition +metadata: + name: xthings.example.org +spec: + group: example.org + names: + kind: XThing + plural: xthings + versions: [] +`, + wantErr: true, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + xrd := &un.Unstructured{} + if err := yaml.Unmarshal([]byte(tt.xrd), &xrd.Object); err != nil { + t.Fatalf("cannot parse test XRD: %v", err) + } + + original := xrd.DeepCopy() + + crds, err := CRDsForXRD(xrd, xrdCRD) + + if tt.wantErr { + if err == nil { + t.Fatalf("\n%s\nCRDsForXRD(...): want error, got nil", tt.reason) + } + + return + } + + if err != nil { + t.Fatalf("\n%s\nCRDsForXRD(...): unexpected error: %v", tt.reason, err) + } + + got := make([]crdShape, 0, len(crds)) + for _, crd := range crds { + got = append(got, shapeOf(t, crd)) + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("\n%s\nCRDsForXRD(...): -want, +got:\n%s", tt.reason, diff) + } + + for _, crd := range crds { + if refs := crd.GetOwnerReferences(); len(refs) != 0 { + t.Errorf("\n%s\nCRDsForXRD(...): CRD %q has owner references %v; the XRD has no UID yet, so "+ + "an apiserver would reject them", tt.reason, crd.GetName(), refs) + } + } + + if diff := cmp.Diff(original.Object, xrd.Object); diff != "" { + t.Errorf("\n%s\nCRDsForXRD(...) mutated its input: -before, +after:\n%s", tt.reason, diff) + } + }) + } +} + +func loadXRDCRD(t *testing.T) *extv1.CustomResourceDefinition { + t.Helper() + + // Like Start, skip rather than fail when the CRDs are not provided; nix run .#test and the test check set it. + if os.Getenv(crossplaneCRDsDirEnv) == "" { + t.Skipf("%s is not set; run the tests with `nix run .#test`", crossplaneCRDsDirEnv) + } + + data, err := os.ReadFile(filepath.Join(PinnedCrossplaneCRDsDir(t), xrdCRDFile)) + if err != nil { + t.Fatalf("cannot read the XRD CRD: %v", err) + } + + crd := &extv1.CustomResourceDefinition{} + if err := yaml.Unmarshal(data, crd); err != nil { + t.Fatalf("cannot parse the XRD CRD: %v", err) + } + + return crd +} + +func shapeOf(t *testing.T, crd *extv1.CustomResourceDefinition) crdShape { + t.Helper() + + if len(crd.Spec.Versions) != 1 { + t.Fatalf("CRD %q: want exactly one version, got %d", crd.GetName(), len(crd.Spec.Versions)) + } + + spec := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + + return crdShape{ + Name: crd.GetName(), + Scope: crd.Spec.Scope, + CompositionUpdatePolicy: findPolicyField(spec, "compositionUpdatePolicy"), + CompositeDeletePolicy: findPolicyField(spec, "compositeDeletePolicy"), + } +} + +// findPolicyField looks for a policy field where Crossplane puts it: directly under spec for a legacy XR +// or a claim, or under spec.crossplane for a modern XR. +func findPolicyField(spec extv1.JSONSchemaProps, name string) policyField { + for _, parent := range []struct { + path string + props extv1.JSONSchemaProps + }{ + {path: "spec", props: spec}, + {path: "spec.crossplane", props: spec.Properties["crossplane"]}, + } { + field, ok := parent.props.Properties[name] + if !ok { + continue + } + + f := policyField{Path: parent.path + "." + name} + if field.Default != nil { + f.Default = string(field.Default.Raw) + } + + return f + } + + return policyField{} +} diff --git a/internal/testutils/integration/doc.go b/internal/testutils/integration/doc.go new file mode 100644 index 00000000..560c729e --- /dev/null +++ b/internal/testutils/integration/doc.go @@ -0,0 +1,31 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package integration provides fixtures for integration tests that run Crossplane CLI commands against an +// envtest API server (a real kube-apiserver and etcd, with no controllers). +// +// Start starts a cluster serving Crossplane's CRDs and the XR and claim CRDs generated from a test's XRDs, and +// sets up the test's manifests as Crossplane would have left them. ReapOrphanedEnvtest, called from TestMain, +// kills envtest servers an earlier run left behind. +// +// Start and the fixtures behind it build only with the integration tag, as do the tests that use them, since they +// need envtest's binaries. Run them with `nix run .#integration-test`, which provides those binaries and +// Crossplane's CRDs at the version go.mod pins. CRDsForXRD, PinnedCrossplaneCRDsDir and ReapOrphanedEnvtest are +// untagged, so their own tests run with the unit tests. +// +// Integration tests are any test files with `//go:build integration`. The integration-test app runs +// `go test -tags integration` on every package containing one, so unit tests in those packages also run there. +package integration diff --git a/internal/testutils/integration/fixtures.go b/internal/testutils/integration/fixtures.go new file mode 100644 index 00000000..35d544ec --- /dev/null +++ b/internal/testutils/integration/fixtures.go @@ -0,0 +1,987 @@ +//go:build integration + +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "bytes" + "context" + "fmt" + "io" + "maps" + "os" + "slices" + "strings" + "time" + + authorizationv1 "k8s.io/api/authorization/v1" + rbacv1 "k8s.io/api/rbac/v1" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + un "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + "github.com/crossplane/crossplane-runtime/v2/pkg/meta" + + xpextv1 "github.com/crossplane/crossplane/apis/v2/apiextensions/v1" +) + +// crossplaneFieldManager is the field manager Start server-side applies composed resources under, as Crossplane +// applies them in production. +const crossplaneFieldManager = "apiextensions.crossplane.io/composed/integration-test" + +// CompositeReconcilerFieldManager stands for Crossplane's composite reconciler, which records the +// composition and revision it selects for an XR with an Update of the XR (crossplane +// internal/controller/apiextensions/composite/api.go), not as part of any composed apply. Use it in a +// FieldManagerApply to set up an XR as that reconciler leaves it. +const CompositeReconcilerFieldManager = "crossplane" + +// relationshipFieldOwner is the field owner of the Updates that link a composed resource to its XR: the +// controller ownerReference and the XR's resourceRefs. +const relationshipFieldOwner = "integration.test" + +// compositeReconcilerFields are the fields the composite reconciler writes that way, at the v2 and v1 +// paths: compositionRef and compositionRevisionRef. +var compositeReconcilerFields = [][]string{ //nolint:gochecknoglobals // Read-only table of field paths. + {"spec", "crossplane", "compositionRef"}, //nolint:goconst // A field path reads better spelled out. + {"spec", "crossplane", "compositionRevisionRef"}, + {"spec", "compositionRef"}, + {"spec", "compositionRevisionRef"}, +} + +// HierarchicalOwnershipRelation is an XR and what it composes, as Setup.CrossplaneManagedResources sets them up. +// Each file holds exactly one resource. +type HierarchicalOwnershipRelation struct { + // OwnerFile holds the XR. + OwnerFile string + // OwnedFiles holds the XR's composed resources, each mapped to what it composes in turn when it is a nested + // XR, or to nil. + OwnedFiles map[string]*HierarchicalOwnershipRelation +} + +// Composes is the one-level HierarchicalOwnershipRelation: an XR and the composed resources it +// controls. Spell out the struct only for a tree deeper than that, i.e. one with nested XRs. +func Composes(xrFile string, composedFiles ...string) HierarchicalOwnershipRelation { + owned := make(map[string]*HierarchicalOwnershipRelation, len(composedFiles)) + for _, f := range composedFiles { + owned[f] = nil + } + + return HierarchicalOwnershipRelation{OwnerFile: xrFile, OwnedFiles: owned} +} + +// setOwnerReference adds an owner reference to the resource. +func setOwnerReference(resource, owner *un.Unstructured) { + // Create owner reference + ownerRef := metav1.OwnerReference{ + APIVersion: owner.GetAPIVersion(), + Kind: owner.GetKind(), + Name: owner.GetName(), + UID: owner.GetUID(), + Controller: new(true), + BlockOwnerDeletion: new(true), + } + + // Set the owner reference + resource.SetOwnerReferences([]metav1.OwnerReference{ownerRef}) +} + +// addResourceRef adds a reference to the child resource in the parent's resourceRefs array. +func addResourceRef(parent, child *un.Unstructured, xrAPIVersion XRDAPIVersion) error { + // Create the resource reference + ref := map[string]any{ + "apiVersion": child.GetAPIVersion(), + "kind": child.GetKind(), + "name": child.GetName(), + } + + // A namespaced XR composes only in its own namespace, so its resourceRefs have no namespace field + // (xcrd omits it from the schema); a cluster-scoped XR's name the namespace of a namespaced child. + if ns := child.GetNamespace(); ns != "" && parent.GetNamespace() == "" { + ref["namespace"] = ns + } + + path := resourceRefsPath(xrAPIVersion) + + // Get current resourceRefs or initialize if not present + resourceRefs, found, err := un.NestedSlice(parent.Object, path...) + if err != nil { + return errors.Wrap(err, "cannot get resourceRefs from parent") + } + + if !found || resourceRefs == nil { + resourceRefs = []any{} + } + + // A fixture may already list the reference, as Crossplane would have written it; don't repeat it. + for _, existing := range resourceRefs { + if m, ok := existing.(map[string]any); ok && + m["apiVersion"] == ref["apiVersion"] && m["kind"] == ref["kind"] && + m["name"] == ref["name"] && m["namespace"] == ref["namespace"] { + return nil + } + } + + // Add the new reference and update the parent + resourceRefs = append(resourceRefs, ref) + + return un.SetNestedSlice(parent.Object, resourceRefs, path...) +} + +// applyResourcesFromFiles creates the resources in YAML files in order, each filled in by d just before +// it is created, so it can derive from what precedes it. No resource should already exist. +func applyResourcesFromFiles(ctx context.Context, c client.Client, d *crossplaneDefaults, paths []string) error { + for _, path := range paths { + resources, err := ReadResourcesFromFile(path) + if err != nil { + return fmt.Errorf("failed to read resources from %s: %w", path, err) + } + + for _, r := range resources { + if err := d.fill(ctx, r, nil); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + + if err := createResources(ctx, c, []*un.Unstructured{r}); err != nil { + return err + } + } + } + + return nil +} + +// deleteResourcesFromFiles issues a delete for every resource in the supplied files. Resources +// carrying a finalizer survive the call with metadata.deletionTimestamp set, which is how tests +// observe a resource mid-deletion under envtest (no controllers run to clear the finalizer). +func deleteResourcesFromFiles(ctx context.Context, c client.Client, paths []string) error { + for _, path := range paths { + resources, err := ReadResourcesFromFile(path) + if err != nil { + return fmt.Errorf("failed to read resources from %s: %w", path, err) + } + + for _, resource := range resources { + if err := c.Delete(ctx, resource.DeepCopy()); err != nil { + return fmt.Errorf("failed to delete resource %s/%s: %w", + resource.GetNamespace(), resource.GetName(), err) + } + } + } + + return nil +} + +// AwaitAdmissionDenial blocks until every resource in the supplied manifest is refused by a +// ValidatingAdmissionPolicy on a dry-run create. An empty path is a no-op. +// +// A VAP does not take effect the instant it is created: the apiserver's admission plugin learns +// about policies and bindings through an informer, so for a short window after setup the cluster +// still accepts what the policy will later refuse. Any test that depends on a rejection must wait +// for that window to close, or it silently becomes a test of nothing. +// +// The gate is deliberately narrow: only an error naming ValidatingAdmissionPolicy counts as +// "enforcing". Accepting any rejection would let an unrelated failure (a schema error in the canary, +// a missing namespace) masquerade as a working policy. +func AwaitAdmissionDenial(ctx context.Context, c client.Client, path string) error { + if path == "" { + return nil + } + + resources, err := ReadResourcesFromFile(path) + if err != nil { + return fmt.Errorf("failed to read resources from %s: %w", path, err) + } + + // Bounded independently of the caller's context so a slow policy rollout fails here with a + // clear message instead of consuming the whole test's budget and timing out inside the command under test. + waitCtx, cancel := context.WithTimeout(ctx, admissionPolicyPropagationTimeout) + defer cancel() + + for _, r := range resources { + if err := awaitOneAdmissionDenial(waitCtx, c, r); err != nil { + return err + } + } + + return nil +} + +// admissionPolicyPropagationTimeout bounds the wait for a ValidatingAdmissionPolicy to start +// enforcing. Measured at ~600ms on envtest 1.32; the margin is for loaded CI machines. +const admissionPolicyPropagationTimeout = 30 * time.Second + +func awaitOneAdmissionDenial(ctx context.Context, c client.Client, canary *un.Unstructured) error { + for attempts := 1; ; attempts++ { + err := c.Create(ctx, canary.DeepCopy(), client.DryRunAll) + + switch { + case err != nil && strings.Contains(err.Error(), "ValidatingAdmissionPolicy"): + return nil + case err != nil: + return fmt.Errorf("dry-run create of canary %s/%s failed for a reason unrelated to admission policy: %w", + canary.GetKind(), canary.GetName(), err) + } + + if ctx.Err() != nil { + return fmt.Errorf("canary %s/%s was still accepted after %d attempts; no ValidatingAdmissionPolicy is enforcing: %w", + canary.GetKind(), canary.GetName(), attempts, ctx.Err()) + } + + time.Sleep(100 * time.Millisecond) + } +} + +// ReadResourcesFromFile reads the resources in a YAML file of one or more documents, skipping empty ones. +func ReadResourcesFromFile(path string) ([]*un.Unstructured, error) { + data, err := os.ReadFile(path) //nolint:gosec // Reads the test's own fixtures. + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %w", path, err) + } + + // Use a YAML decoder to handle multiple documents + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + + var resources []*un.Unstructured + + for { + resource := &un.Unstructured{} + + err := decoder.Decode(resource) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + + return nil, fmt.Errorf("failed to decode YAML document from %s: %w", path, err) + } + + // Skip empty documents + if len(resource.Object) == 0 { + continue + } + + resources = append(resources, resource) + } + + return resources, nil +} + +// xrdCRDName is the CRD that defines CompositeResourceDefinitions. It must be among the plain CRDs. +const xrdCRDName = "compositeresourcedefinitions.apiextensions.crossplane.io" + +// envtestCRDs returns every CRD an integration test's apiserver must serve: the plain CRDs under crdDirs, +// plus the CRDs Crossplane generates from each XRD that setupFiles declare. In a real cluster an XR's or a +// claim's CRD exists only because its XRD does, and Crossplane generates it; deriving them here the same +// way keeps test CRDs from drifting away from what a cluster would serve. +// +// A CRD declared twice is an error rather than a silent overwrite (envtest would install the later copy over +// the earlier one). Most likely it is a hand-written copy of a CRD that is now generated, which must go. +func envtestCRDs(crdDirs, setupFiles []string) ([]*extv1.CustomResourceDefinition, error) { + opts := envtest.CRDInstallOptions{Paths: crdDirs, ErrorIfPathMissing: true} + if err := envtest.ReadCRDFiles(&opts); err != nil { + return nil, fmt.Errorf("cannot read CRDs from %v: %w", crdDirs, err) + } + + var xrdCRD *extv1.CustomResourceDefinition + + // Keyed by both CRD name and kind.group: two CRDs of different names that serve the same kind + // collide just as surely (the apiserver never establishes the second). + sources := make(map[string]string, 2*len(opts.CRDs)) + for _, crd := range opts.CRDs { + source := fmt.Sprintf("a CRD file under %v", crdDirs) + sources[crd.GetName()] = source + sources[crd.Spec.Names.Kind+"."+crd.Spec.Group] = source + + if crd.GetName() == xrdCRDName { + xrdCRD = crd + } + } + + if xrdCRD == nil { + return nil, fmt.Errorf("no CRD %q under %v", xrdCRDName, crdDirs) + } + + crds := opts.CRDs + + for _, path := range setupFiles { + resources, err := ReadResourcesFromFile(path) + if err != nil { + return nil, err + } + + for _, r := range resources { + if r.GroupVersionKind().GroupKind() != xpextv1.CompositeResourceDefinitionGroupVersionKind.GroupKind() { + continue + } + + generated, err := CRDsForXRD(r, xrdCRD) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + + for _, crd := range generated { + source := fmt.Sprintf("XRD %q in %s", r.GetName(), path) + + for _, key := range []string{crd.GetName(), crd.Spec.Names.Kind + "." + crd.Spec.Group} { + if existing, ok := sources[key]; ok { + return nil, fmt.Errorf("CRD %q (%s) is generated from %s but %s also declares it; "+ + "a generated CRD must not also be hand-written or generated twice", + crd.GetName(), key, source, existing) + } + + sources[key] = source + } + + crds = append(crds, crd) + } + } + } + + return crds, nil +} + +// addUserWithRules provisions an envtest user bound to a ClusterRole granting exactly rules, and returns +// its config once the apiserver's RBAC authorizer honours the binding. envtest's apiserver authorizes +// with RBAC, so the user can do only what rules allow, plus what bootstrap RBAC grants every +// authenticated user (discovery, and reviewing its own access). +func addUserWithRules(ctx context.Context, env *envtest.Environment, c client.Client, admin *rest.Config, rules []rbacv1.PolicyRule) (*rest.Config, error) { + const name = "least-privilege" + + role := &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: name}, Rules: rules} + if err := c.Create(ctx, role); err != nil { + return nil, fmt.Errorf("cannot create ClusterRole %s: %w", name, err) + } + + binding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: name}, + Subjects: []rbacv1.Subject{{APIGroup: rbacv1.GroupName, Kind: rbacv1.UserKind, Name: name}}, + } + if err := c.Create(ctx, binding); err != nil { + return nil, fmt.Errorf("cannot create ClusterRoleBinding %s: %w", name, err) + } + + user, err := env.AddUser(envtest.User{Name: name}, &rest.Config{QPS: admin.QPS, Burst: admin.Burst}) + if err != nil { + return nil, fmt.Errorf("cannot add user %s: %w", name, err) + } + + cfg := user.Config() + + kube, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, fmt.Errorf("cannot create a client for user %s: %w", name, err) + } + + // The authorizer learns of the binding through an informer, so wait until the user holds the first + // rule's first grant before running anything as the user. + first := rules[0] + review := &authorizationv1.SelfSubjectAccessReview{Spec: authorizationv1.SelfSubjectAccessReviewSpec{ + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Group: first.APIGroups[0], Resource: first.Resources[0], Verb: first.Verbs[0], + }, + }} + + for { + got, err := kube.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, review, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("cannot review user %s's access: %w", name, err) + } + + if got.Status.Allowed { + return cfg, nil + } + + select { + case <-ctx.Done(): + return nil, fmt.Errorf("user %s never gained the access its ClusterRoleBinding grants: %w", name, ctx.Err()) + case <-time.After(100 * time.Millisecond): + } + } +} + +// createResources creates all resources in the cluster +// Assumes resources don't already exist - fails if they do. +func createResources(ctx context.Context, c client.Client, resources []*un.Unstructured) error { + for _, resource := range resources { + err := c.Create(ctx, resource.DeepCopy()) + if err != nil { + if apierrors.IsAlreadyExists(err) { + return fmt.Errorf("resource %s/%s of kind %s already exists - test setup error", + resource.GetNamespace(), resource.GetName(), resource.GetKind()) + } + + return fmt.Errorf("failed to create resource %s/%s: %w", + resource.GetNamespace(), resource.GetName(), err) + } + } + + return nil +} + +// applyResourceWithSSA applies a resource using Server-Side Apply with the specified field manager. +// This simulates how Crossplane applies composed resources, allowing us to test field ownership behavior. +func applyResourceWithSSA(ctx context.Context, c client.Client, resource *un.Unstructured, fieldManager string) error { + // Use Apply to perform Server-Side Apply + resourceCopy := resource.DeepCopy() + + // For SSA, we need to set the resource version to empty to indicate this is an apply + resourceCopy.SetResourceVersion("") + + applyConfig := client.ApplyConfigurationFromUnstructured(resourceCopy) + + err := c.Apply(ctx, applyConfig, client.FieldOwner(fieldManager), client.ForceOwnership) + if err != nil { + return fmt.Errorf("failed to apply resource %s/%s with field manager %s: %w", + resource.GetNamespace(), resource.GetName(), fieldManager, err) + } + + return nil +} + +// applyHierarchicalOwnership applies a hierarchical ownership structure. +func applyHierarchicalOwnership(ctx context.Context, c client.Client, d *crossplaneDefaults, xrdAPIVersion XRDAPIVersion, hierarchies []HierarchicalOwnershipRelation) error { + // Map to store created resources by file path + createdResources := make(map[string]*un.Unstructured) + // Map to track parent-child relationships for establishing resourceRefs + parentChildRelationships := make(map[string]string) // child file -> parent file + + // First pass: Create all resources and collect parent-child relationships + err := createAllResourcesInHierarchy(ctx, c, d, hierarchies, createdResources, parentChildRelationships) + if err != nil { + return err + } + + // Second pass: Apply all owner references and resource refs between parents and children + err = applyAllRelationships(ctx, c, createdResources, parentChildRelationships, xrdAPIVersion) + if err != nil { + return err + } + + return nil +} + +// createAllResourcesInHierarchy creates all resources in a hierarchy and tracks relationships. +func createAllResourcesInHierarchy(ctx context.Context, c client.Client, d *crossplaneDefaults, + hierarchies []HierarchicalOwnershipRelation, + createdResources map[string]*un.Unstructured, + parentChildRelationships map[string]string, +) error { + for _, hierarchy := range hierarchies { + // Create the owner resource first. Only a root is still uncreated here, since a nested owner was + // applied as its parent's composed resource before the recursion reached it. A root XR is + // created the way a user's manifest would be, never under Crossplane's composed field manager: + // its spec belongs to whoever applied it. + owner, err := createResourceFromFile(ctx, c, d, hierarchy.OwnerFile, nil, createdResources, "") + if err != nil { + return err + } + + // Create all owned resources without setting references yet + for ownedFile, childHierarchy := range hierarchy.OwnedFiles { + // Track the parent-child relationship + parentChildRelationships[ownedFile] = hierarchy.OwnerFile + + // Create the owned resource without setting references + _, err := createResourceFromFile(ctx, c, d, ownedFile, owner, createdResources, crossplaneFieldManager) + if err != nil { + return err + } + + // Process nested hierarchies recursively + if childHierarchy != nil && len(childHierarchy.OwnedFiles) > 0 { + childHierarchies := []HierarchicalOwnershipRelation{ + { + OwnerFile: ownedFile, + OwnedFiles: childHierarchy.OwnedFiles, + }, + } + + err := createAllResourcesInHierarchy(ctx, c, d, childHierarchies, + createdResources, parentChildRelationships) + if err != nil { + return err + } + } + } + } + + return nil +} + +// createResourceFromFile creates the one resource a file holds, filled in by d as parent's composed +// resource (parent is nil for a root). With a fieldManager it is applied with Server-Side Apply (SSA) +// under that manager: pass crossplaneFieldManager for a composed resource, as Crossplane applies composed +// resources in production, so tests see SSA-related behaviors like field removal. With none it is +// created plainly, as a setup file is. +func createResourceFromFile(ctx context.Context, c client.Client, d *crossplaneDefaults, path string, + parent *un.Unstructured, createdResources map[string]*un.Unstructured, fieldManager string, +) (*un.Unstructured, error) { + // Check if we've already processed this resource + if resource, exists := createdResources[path]; exists { + return resource, nil + } + + // Read the resource from file + resources, err := ReadResourcesFromFile(path) + if err != nil { + return nil, err + } + + // The hierarchy is keyed by file, so a second document would silently go unowned. + if len(resources) != 1 { + return nil, fmt.Errorf("file %s holds %d resources; an ownership hierarchy needs exactly one per file", path, len(resources)) + } + + resource := resources[0] + + if err := d.fill(ctx, resource, parent); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + + // A nested XR's composition refs are the nested XR's own reconciler's, never its parent's: the + // parent's composed apply doesn't carry them, so it must not own them: Crossplane's managedFields + // would otherwise show the parent's field manager owning fields only the nested XR's reconciler + // writes. Apply them separately, as that reconciler writes them. + applied := resource + + var reconcilerFields map[int]any + if fieldManager == crossplaneFieldManager { + applied, reconcilerFields = withoutCompositeReconcilerFields(resource) + } + + if fieldManager != "" { + if err := applyResourceWithSSA(ctx, c, applied, fieldManager); err != nil { + return nil, fmt.Errorf("failed to apply resource with SSA: %w", err) + } + } else if err := createResources(ctx, c, resources); err != nil { + return nil, err + } + + // Get the resource back from the server + serverResource := &un.Unstructured{} + serverResource.SetGroupVersionKind(resource.GroupVersionKind()) + + if err := c.Get(ctx, client.ObjectKey{ + Name: resource.GetName(), + Namespace: resource.GetNamespace(), + }, serverResource); err != nil { + return nil, fmt.Errorf("failed to get created resource: %w", err) + } + + if len(reconcilerFields) > 0 { + for i, v := range reconcilerFields { + if err := un.SetNestedField(serverResource.Object, v, compositeReconcilerFields[i]...); err != nil { + return nil, fmt.Errorf("cannot set %v on %s: %w", compositeReconcilerFields[i], path, err) + } + } + + if err := c.Update(ctx, serverResource, client.FieldOwner(CompositeReconcilerFieldManager)); err != nil { + return nil, fmt.Errorf("failed to write composite reconciler fields of %s: %w", path, err) + } + } + + // Store and return the created resource + createdResources[path] = serverResource + + return serverResource, nil +} + +// withoutCompositeReconcilerFields returns a copy of resource without the compositeReconcilerFields it +// sets, and those fields' values keyed by their index in compositeReconcilerFields. resource is not +// mutated. +func withoutCompositeReconcilerFields(resource *un.Unstructured) (*un.Unstructured, map[int]any) { + out := resource.DeepCopy() + removed := map[int]any{} + + for i, path := range compositeReconcilerFields { + if v, found, _ := un.NestedFieldCopy(out.Object, path...); found { + removed[i] = v + un.RemoveNestedField(out.Object, path...) + } + } + + return out, removed +} + +// applyAllRelationships applies all owner references and resource refs. +func applyAllRelationships(ctx context.Context, c client.Client, + createdResources map[string]*un.Unstructured, + parentChildRelationships map[string]string, + xrdAPIVersion XRDAPIVersion, +) error { + // Process all parent-child relationships + for childFile, parentFile := range parentChildRelationships { + childResource := createdResources[childFile] + parentResource := createdResources[parentFile] + + if childResource == nil || parentResource == nil { + return fmt.Errorf("missing resource in relationship: parent=%s, child=%s", + parentFile, childFile) + } + + // 1. Set the owner reference in the child's metadata + err := setOwnerReferenceAndUpdate(ctx, c, parentResource, childResource) + if err != nil { + return err + } + + // 2. Add the child resource reference to the parent + err = addResourceRefAndUpdate(ctx, c, parentResource, childResource, xrdAPIVersion) + if err != nil { + return err + } + } + + return nil +} + +// setOwnerReferenceAndUpdate sets the owner reference in the child and updates it. +// Uses a separate field manager for owner references to avoid conflicts with SSA-created resources. +func setOwnerReferenceAndUpdate(ctx context.Context, c client.Client, + owner *un.Unstructured, child *un.Unstructured, +) error { + // Get the latest version of the child + latestChild := &un.Unstructured{} + latestChild.SetGroupVersionKind(child.GroupVersionKind()) + + err := c.Get(ctx, client.ObjectKey{ + Name: child.GetName(), + Namespace: child.GetNamespace(), + }, latestChild) + if err != nil { + return fmt.Errorf("failed to get child resource: %w", err) + } + + // Set the owner reference + setOwnerReference(latestChild, owner) + + // Use a separate field manager for owner references to avoid conflicts with the + // Crossplane field manager that created the resource via SSA. + err = c.Update(ctx, latestChild, client.FieldOwner(relationshipFieldOwner)) + if err != nil { + return fmt.Errorf("failed to update child with owner reference: %w", err) + } + + return nil +} + +// addResourceRefAndUpdate adds a resource reference to the owner and updates it. +func addResourceRefAndUpdate(ctx context.Context, c client.Client, + owner *un.Unstructured, owned *un.Unstructured, xrdAPIVersion XRDAPIVersion, +) error { + // Get the latest version of the owner + latestOwner := &un.Unstructured{} + latestOwner.SetGroupVersionKind(owner.GroupVersionKind()) + + err := c.Get(ctx, client.ObjectKey{ + Name: owner.GetName(), + Namespace: owner.GetNamespace(), + }, latestOwner) + if err != nil { + return fmt.Errorf("failed to get owner for updating references: %w", err) + } + + // Add the resource reference + err = addResourceRef(latestOwner, owned, xrdAPIVersion) + if err != nil { + return fmt.Errorf("unable to add resource ref: %w", err) + } + + // Use a separate field manager for resource refs to avoid conflicts with the + // Crossplane field manager that created the resource via SSA. + want, _, _ := un.NestedSlice(latestOwner.Object, resourceRefsPath(xrdAPIVersion)...) + + err = c.Update(ctx, latestOwner, client.FieldOwner(relationshipFieldOwner)) + if err != nil { + return fmt.Errorf("failed to update owner with resource reference: %w", err) + } + + // The apiserver prunes a field its schema lacks, so resourceRefs written at the wrong API version's + // path would vanish silently and leave the XR composing nothing. + if got, _, _ := un.NestedSlice(latestOwner.Object, resourceRefsPath(xrdAPIVersion)...); len(got) != len(want) { + return fmt.Errorf("%s %s kept %d of %d resourceRefs at %v: is Setup.XRDAPIVersion that of the XRD?", + owner.GetKind(), owner.GetName(), len(got), len(want), resourceRefsPath(xrdAPIVersion)) + } + + return nil +} + +// resourceRefsPath is where an XR of the given XRD API version lists its composed resources. +func resourceRefsPath(xrdAPIVersion XRDAPIVersion) []string { + if xrdAPIVersion == XRDV1 { + return []string{"spec", "resourceRefs"} + } + + return []string{"spec", "crossplane", "resourceRefs"} +} + +// reconcileCompositionRevisions does what Crossplane's composition revision controller would, since +// envtest runs none: it gives each revision a controller reference to its Composition, and creates the +// revision of a Composition's current hash if no fixture declares it. +func reconcileCompositionRevisions(ctx context.Context, c client.Client) error { + comps := &xpextv1.CompositionList{} + revs := &xpextv1.CompositionRevisionList{} + + if err := c.List(ctx, comps); err != nil { + return err + } + + if err := c.List(ctx, revs); err != nil { + return err + } + + for i := range comps.Items { + comp := &comps.Items[i] + _, current := revisionIdentity(comp.GetName(), comp.Hash()) + owner := meta.AsController(meta.TypedReferenceTo(comp, xpextv1.CompositionGroupVersionKind)) + latest, found := int64(0), false + + for j := range revs.Items { + rev := &revs.Items[j] + if rev.GetLabels()[xpextv1.LabelCompositionName] != comp.GetName() { + continue + } + + latest = max(latest, rev.Spec.Revision) + found = found || rev.GetLabels()[xpextv1.LabelCompositionHash] == current + + meta.AddOwnerReference(rev, owner) + + if err := c.Update(ctx, rev); err != nil { + return err + } + } + + if !found { + if err := c.Create(ctx, newCompositionRevision(comp, latest+1, owner)); err != nil { + return err + } + } + } + + return nil +} + +// reconcileXRDCRDOwnership does what Crossplane's definition controllers would, since envtest runs none: it +// makes each XRD the controller of the XR and claim CRDs generated from it. The CRDs are installed before +// any XRD exists, so CRDsForXRD cannot give them the reference xcrd does. An XRD whose CRD does not exist +// (one in Setup.UnestablishedXRDs) is skipped. +func reconcileXRDCRDOwnership(ctx context.Context, c client.Client) error { + xrds := &xpextv1.CompositeResourceDefinitionList{} + if err := c.List(ctx, xrds); err != nil { + return err + } + + for i := range xrds.Items { + xrd := &xrds.Items[i] + owner := meta.AsController(meta.TypedReferenceTo(xrd, xpextv1.CompositeResourceDefinitionGroupVersionKind)) + + names := []string{xrd.Spec.Names.Plural + "." + xrd.Spec.Group} + if xrd.Spec.ClaimNames != nil { + names = append(names, xrd.Spec.ClaimNames.Plural+"."+xrd.Spec.Group) + } + + for _, name := range names { + crd := &extv1.CustomResourceDefinition{} + + err := c.Get(ctx, client.ObjectKey{Name: name}, crd) + if apierrors.IsNotFound(err) { + continue + } + + if err != nil { + return err + } + + meta.AddOwnerReference(crd, owner) + + if err := c.Update(ctx, crd); err != nil { + return err + } + } + } + + return nil +} + +// newCompositionRevision is crossplane's composition.NewCompositionRevision, which is internal. +func newCompositionRevision(comp *xpextv1.Composition, revision int64, owner metav1.OwnerReference) *xpextv1.CompositionRevision { + name, hash := revisionIdentity(comp.GetName(), comp.Hash()) + + rev := &xpextv1.CompositionRevision{Spec: (&xpextv1.GeneratedRevisionSpecConverter{}).ToRevisionSpec(comp.Spec)} + rev.Spec.Revision = revision + rev.SetName(name) + rev.SetLabels(maps.Clone(comp.GetLabels())) + meta.AddLabels(rev, map[string]string{xpextv1.LabelCompositionName: comp.GetName(), xpextv1.LabelCompositionHash: hash}) + rev.SetAnnotations(maps.Clone(comp.GetAnnotations())) + meta.AddOwnerReference(rev, owner) + + return rev +} + +// revisionIdentity is the name and hash label NewCompositionRevision gives the revision of a Composition +// with the given name and Composition.Hash(). +func revisionIdentity(compName, hash string) (string, string) { + hash = hash[:min(63, len(hash))] + return compName + "-" + hash[:min(7, len(hash))], hash +} + +// The labels Crossplane gives an XR and everything composed beneath it. +const ( + labelComposite = "crossplane.io/composite" + labelClaimName = "crossplane.io/claim-name" + labelClaimNamespace = "crossplane.io/claim-namespace" +) + +// crossplaneDefaults fills in, on each manifest Start sets up, what Crossplane would have written to it +// wherever the fixture leaves it out, just as envtestCRDs derives XR and claim CRDs from their XRDs. A +// value the manifest sets always wins: that is how a test sets up a state Crossplane wouldn't leave, such as +// a stale or pinned revision ref, or another XR's labels. +type crossplaneDefaults struct { + c client.Client + xrKinds map[schema.GroupKind]bool // the XR kinds the setup's XRDs define +} + +func newCrossplaneDefaults(c client.Client, crds []*extv1.CustomResourceDefinition) *crossplaneDefaults { + d := &crossplaneDefaults{c: c, xrKinds: map[schema.GroupKind]bool{}} + + for _, crd := range crds { + if slices.Contains(crd.Spec.Names.Categories, "composite") { + d.xrKinds[schema.GroupKind{Group: crd.Spec.Group, Kind: crd.Spec.Names.Kind}] = true + } + } + + return d +} + +// fill fills in what Crossplane would have written to o, where o doesn't set it: +// - a CompositionRevision's name and hash label, which NewCompositionRevision derives from the +// Composition it was cut from: the one its crossplane.io/composition-name label names, with the +// revision's own spec, labels and annotations; +// - an XR's crossplane.io/composite label, its own name, and for a claim's backing XR the claim labels +// the claim syncer sets from the claim its claimRef names; or, on a resource an XR composes (a nested +// XR included), the XR's composite and claim labels, as RenderComposedResourceMetadata copies them; +// - the compositionRevisionRef of an XR that names a Composition but neither a revision nor a +// revision selector: the Composition's current revision, which reconcileCompositionRevisions creates. +func (d *crossplaneDefaults) fill(ctx context.Context, o, parent *un.Unstructured) error { + gk := o.GroupVersionKind().GroupKind() + if gk == xpextv1.CompositionRevisionGroupVersionKind.GroupKind() && o.GetName() == "" { + return d.nameRevision(ctx, o) + } + + var composite, claimName, claimNamespace string + + switch { + case parent != nil: + composite = parent.GetLabels()[labelComposite] + claimName, claimNamespace = parent.GetLabels()[labelClaimName], parent.GetLabels()[labelClaimNamespace] + case d.xrKinds[gk]: + composite = o.GetName() + // The claim syncer labels a claim's backing XR with the claim, which its claimRef names. + claimName, _, _ = un.NestedString(o.Object, "spec", "claimRef", "name") + claimNamespace, _, _ = un.NestedString(o.Object, "spec", "claimRef", "namespace") + } + + labels := map[string]string{} + if composite != "" { + labels[labelComposite] = composite + } + + // Like RenderComposedResourceMetadata, set the claim labels only as a pair. + if claimName != "" && claimNamespace != "" { + labels[labelClaimName], labels[labelClaimNamespace] = claimName, claimNamespace + } + + maps.Copy(labels, o.GetLabels()) + + if len(labels) > 0 { + o.SetLabels(labels) + } + + // The refs are at spec.crossplane in a v2 XR, and at spec in a v1 one. + for _, base := range [][]string{{"spec", "crossplane"}, {"spec"}} { + field := func(name ...string) []string { return slices.Concat(base, name) } + + comp, named, _ := un.NestedString(o.Object, field("compositionRef", "name")...) + _, hasRef, _ := un.NestedFieldNoCopy(o.Object, field("compositionRevisionRef")...) + _, hasSelector, _ := un.NestedFieldNoCopy(o.Object, field("compositionRevisionSelector")...) + + if !d.xrKinds[gk] || !named || hasRef || hasSelector { + continue + } + + current := &xpextv1.Composition{} + if err := d.c.Get(ctx, client.ObjectKey{Name: comp}, current); err != nil { + return fmt.Errorf("cannot derive the compositionRevisionRef of %s %s: %w", o.GetKind(), o.GetName(), err) + } + + name, _ := revisionIdentity(comp, current.Hash()) + + return un.SetNestedField(o.Object, name, field("compositionRevisionRef", "name")...) + } + + return nil +} + +// nameRevision names and hash-labels a CompositionRevision as NewCompositionRevision would have, from the +// Composition it was cut from: its spec, as the apiserver defaults it, and its own labels and annotations. +func (d *crossplaneDefaults) nameRevision(ctx context.Context, o *un.Unstructured) error { + compName := o.GetLabels()[xpextv1.LabelCompositionName] + if compName == "" { + return errors.Errorf("a CompositionRevision without a name needs a %s label to be named from", xpextv1.LabelCompositionName) + } + + probe := o.DeepCopy() + probe.SetGenerateName("revision-") + + if err := d.c.Create(ctx, probe, client.DryRunAll); err != nil { + return err + } + + rev := &xpextv1.CompositionRevision{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(probe.Object, rev); err != nil { + return err + } + + src := &xpextv1.Composition{Spec: (&xpextv1.GeneratedRevisionSpecConverter{}).FromRevisionSpec(rev.Spec)} + labels := maps.Clone(rev.GetLabels()) + delete(labels, xpextv1.LabelCompositionName) + delete(labels, xpextv1.LabelCompositionHash) + + if len(labels) > 0 { + src.SetLabels(labels) + } + + src.SetAnnotations(rev.GetAnnotations()) + + name, hash := revisionIdentity(compName, src.Hash()) + o.SetName(name) + meta.AddLabels(o, map[string]string{xpextv1.LabelCompositionHash: hash}) + + return nil +} diff --git a/internal/testutils/integration/reaper.go b/internal/testutils/integration/reaper.go new file mode 100644 index 00000000..31bc0942 --- /dev/null +++ b/internal/testutils/integration/reaper.go @@ -0,0 +1,163 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/shirou/gopsutil/v4/process" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +// isServer reports whether name is that of a long-running server envtest starts. +func isServer(name string) bool { + return name == "kube-apiserver" || name == "etcd" +} + +// candidate is an orphaned process named like an envtest server. +type candidate struct { + proc *process.Process + // exe is the absolute path of the process's executable. + exe string + // cmdline is the process's command line. + cmdline string +} + +// ReapOrphanedEnvtest SIGKILLs every envtest server orphaned by an earlier run, and reports through logf what it +// killed and what it could not. Call it from TestMain, before m.Run. +// +// envtest runs kube-apiserver and etcd as children of the test binary, and only a deferred Environment.Stop ends +// them. When the binary dies without running its defers (the -timeout panic, a kill, a Ctrl+C), they are reparented +// to PID 1 (or to a subreaper, in which case they are not reaped) and run until someone kills them, piling up across +// runs (crossplane-contrib/crossplane-diff#524). A process whose parent is PID 1 has lost the test binary that +// started it, so it cannot belong to a live run. +// ReapOrphanedEnvtest is best-effort: it never fails, and it skips any process it cannot inspect. +func ReapOrphanedEnvtest(logf func(format string, args ...any)) { + pids, err := process.Pids() + if err != nil { + logf("envtest reaper: cannot list processes: %v", err) + return + } + + // Look up the executable only of orphans named like an envtest server. + var candidates []candidate + + for _, pid := range pids { + // Not process.NewProcess, which first checks the PID exists and fetches its creation time: most of the + // scan's cost, for nothing Kill uses. A PID that has exited meanwhile fails Ppid and is skipped. + p := &process.Process{Pid: pid} + + if ppid, err := p.Ppid(); err != nil || ppid != 1 { + continue + } + + if name, err := p.Name(); err != nil || !isServer(name) { + continue + } + + exe, err := p.Exe() + if err != nil { + continue + } + + if cmdline, err := p.Cmdline(); err == nil { + candidates = append(candidates, candidate{proc: p, exe: exe, cmdline: cmdline}) + } + } + + var killed []string + + for _, c := range selectServers(candidates, binaryDirs()) { + if err := c.proc.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) { + logf("envtest reaper: cannot kill %s (pid %d): %v", c.exe, c.proc.Pid, err) + continue + } + + killed = append(killed, fmt.Sprintf("%s (pid %d)", filepath.Base(c.exe), c.proc.Pid)) + } + + if len(killed) > 0 { + logf("envtest reaper: killed %d envtest server(s) orphaned by an earlier run: %s", + len(killed), strings.Join(killed, ", ")) + } +} + +// selectServers returns the candidates whose executable is an envtest server directly in one of dirs, and whose +// command line carries envtest's fingerprint (hasEnvtestDir), so a system etcd or kube-apiserver in a shared dir +// that KUBEBUILDER_ASSETS names is spared. +func selectServers(candidates []candidate, dirs []string) []candidate { + var selected []candidate + + for _, c := range candidates { + exe := filepath.Clean(c.exe) + if filepath.IsAbs(exe) && isServer(filepath.Base(exe)) && slices.Contains(dirs, filepath.Dir(exe)) && + hasEnvtestDir(c.cmdline) { + selected = append(selected, c) + } + } + + return selected +} + +// hasEnvtestDir reports whether cmdline passes etcd a --data-dir, or kube-apiserver a --cert-dir, that envtest +// created. envtest creates them when they are unset, as Start leaves them: controller-runtime's process.go uses +// os.MkdirTemp("", "k8s_test_framework_"). +func hasEnvtestDir(cmdline string) bool { + envtestDirPrefix := filepath.Join(os.TempDir(), "k8s_test_framework_") + + for arg := range strings.FieldsSeq(cmdline) { + for _, flag := range []string{"--data-dir=", "--cert-dir="} { + if dir, ok := strings.CutPrefix(arg, flag); ok && strings.HasPrefix(dir, envtestDirPrefix) { + return true + } + } + } + + return false +} + +// binaryDirs returns every directory a test run could have started envtest's servers from: KUBEBUILDER_ASSETS, each +// version in setup-envtest's store, and envtest's built-in default, /usr/local/kubebuilder/bin. All are +// included, not only the one envtest would use now, because an orphan may come from a run configured differently. +// Each is also included with its symlinks resolved, as the process table may report the resolved path. +func binaryDirs() []string { + dirs := []string{"/usr/local/kubebuilder/bin"} + + if v := os.Getenv("KUBEBUILDER_ASSETS"); v != "" { + dirs = append(dirs, filepath.Clean(v)) + } + + if store, err := envtest.SetupEnvtestDefaultBinaryAssetsDirectory(); err == nil { + versions, _ := os.ReadDir(store) // A missing store has no versions. + for _, v := range versions { + dirs = append(dirs, filepath.Join(store, v.Name())) + } + } + + for _, d := range dirs { + if r, err := filepath.EvalSymlinks(d); err == nil && r != d { + dirs = append(dirs, r) + } + } + + return dirs +} diff --git a/internal/testutils/integration/reaper_test.go b/internal/testutils/integration/reaper_test.go new file mode 100644 index 00000000..111425b4 --- /dev/null +++ b/internal/testutils/integration/reaper_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func TestSelectServers(t *testing.T) { + const store = "/Users/u/Library/Application Support/io.kubebuilder.envtest/k8s/1.32.0-darwin-arm64" + + dirs := []string{store, "/usr/local/kubebuilder/bin"} + envtest := "etcd --data-dir=" + filepath.Join(os.TempDir(), "k8s_test_framework_123") + " --listen-peer-urls=x" + apiserver := "kube-apiserver --cert-dir=" + filepath.Join(os.TempDir(), "k8s_test_framework_456") + + // ReapOrphanedEnvtest passes only PPID-1 processes named like a server; the executable path and the command + // line's envtest data or cert dir decide the rest. + candidates := []candidate{ + {exe: store + "/kube-apiserver", cmdline: apiserver}, // orphaned envtest apiserver + {exe: store + "/etcd", cmdline: envtest}, // orphaned envtest etcd + {exe: "/usr/local/kubebuilder/bin/etcd", cmdline: envtest}, // orphaned envtest etcd (envtest's default dir) + {exe: store + "/../1.32.0-darwin-arm64/etcd", cmdline: envtest}, // unclean path to an envtest binary + {exe: "/opt/homebrew/bin/etcd", cmdline: envtest}, // a real etcd daemon + {exe: "/usr/local/bin/kube-apiserver", cmdline: envtest}, // a real apiserver + {exe: store + "/kubectl", cmdline: envtest}, // a non-server envtest binary + {exe: "/usr/local/kubebuilder/bin-x/etcd", cmdline: envtest}, // a sibling dir sharing a prefix + {exe: store + "/sub/etcd", cmdline: envtest}, // nested below an envtest dir + {exe: "etcd", cmdline: envtest}, // a bare name, not a path + {exe: "", cmdline: envtest}, // unknown executable + {exe: store + "/etcd", cmdline: "etcd --data-dir=/var/lib/etcd"}, // a system etcd in an envtest dir + {exe: store + "/kube-apiserver", cmdline: "kube-apiserver --cert-dir=/k8s_test_framework_1"}, // not under TempDir + {exe: store + "/etcd"}, // unknown command line + } + + selected := selectServers(candidates, dirs) + + got := make([]string, 0, len(selected)) + for _, c := range selected { + got = append(got, c.exe) + } + + want := []string{ + store + "/kube-apiserver", + store + "/etcd", + "/usr/local/kubebuilder/bin/etcd", + store + "/../1.32.0-darwin-arm64/etcd", + } + + if diff := cmp.Diff(want, got, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("selectServers() -want +got:\n%s", diff) + } +} diff --git a/nix/vendor-hashes.nix b/nix/vendor-hashes.nix index 80768838..2c9b5b1d 100644 --- a/nix/vendor-hashes.nix +++ b/nix/vendor-hashes.nix @@ -12,5 +12,5 @@ # hand.) { # Root module: github.com/crossplane/cli/v2 - root = "sha256-7JUmy8760wUG8zT6c++z2szgqpbXfbRipWLahMrm33s="; + root = "sha256-3NtCcYfyFUkUO6Yzma7s8G2lqWCjC3alqcKRylcRNbM="; } From 56c286e0866846793f5cabb3283ff7014e243885 Mon Sep 17 00:00:00 2001 From: Jonathan Ogilvie Date: Fri, 30 Oct 2026 14:43:09 -0400 Subject: [PATCH 2/8] build(nix): add integration-test app and CI job Pin envtest binaries and Crossplane's CRDs in Nix, add an integration-test app that runs the integration-tagged tests against them, and run it in a new integration-tests CI job with Go module caching. The unit tests get CROSSPLANE_CRDS_DIR for the CRD helper tests, and lint covers the integration tag. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Jonathan Ogilvie --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++ .gitignore | 3 +++ .golangci.yml | 4 +++ flake.nix | 1 + nix/apps.nix | 57 +++++++++++++++++++++++++++++++++++++++- nix/checks.nix | 9 ++++++- nix/crossplane-crds.nix | 13 +++++++++ nix/envtest-assets.nix | 51 +++++++++++++++++++++++++++++++++++ 8 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 nix/crossplane-crds.nix create mode 100644 nix/envtest-assets.nix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bdacd8df..19f388bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,6 +111,46 @@ jobs: file: result/coverage.txt token: ${{ secrets.CODECOV_TOKEN }} + # The integration tests run CLI commands against an envtest API server. They + # run outside the Nix sandbox (and can reach the runner's Docker daemon), so + # they use the integration-test app rather than a check, and cache Go modules + # and build output themselves. + integration-tests: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install Nix + uses: cachix/install-nix-action@75c75c31ebece974e48ebc36b4ad125bd97bd97b # v31 + + - name: Setup Cachix + uses: cachix/cachix-action@38b082610b782e7e93e209c35fd730d399dee866 # v17 + with: + name: crossplane + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + + - name: Cache Go modules and build cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-integration-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-integration- + + - name: Run Integration Tests + run: nix run .#integration-test --option warn-dirty false + + - name: Publish Integration Test Coverage + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7 + with: + flags: integration + file: coverage-integration.txt + token: ${{ secrets.CODECOV_TOKEN }} + # Build all artifacts build-artifacts: permissions: diff --git a/.gitignore b/.gitignore index ebb4d21e..a9a87de0 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,6 @@ AGENTS.md # Ignore local dev environment variables .env + +# Integration test coverage (nix run .#integration-test) +/coverage-integration.txt diff --git a/.golangci.yml b/.golangci.yml index 86bfb643..c38cc7b8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -6,6 +6,10 @@ run: # runs our unit tests. Lint them anyway, so they don't rot. build-tags: - compilegate + # The integration tests need envtest's binaries, so they sit behind the + # integration tag and run in their own target, not the unit tests. Lint + # them anyway. + - integration output: formats: diff --git a/flake.nix b/flake.nix index 05c6b38d..8eeecc3c 100644 --- a/flake.nix +++ b/flake.nix @@ -137,6 +137,7 @@ in { test = apps.test { }; + integration-test = apps.integration-test { }; lint = apps.lint { }; generate = apps.generate { }; tidy = apps.tidy { }; diff --git a/nix/apps.nix b/nix/apps.nix index b4b25a60..c4cada17 100644 --- a/nix/apps.nix +++ b/nix/apps.nix @@ -11,8 +11,14 @@ # ''${SOME_VAR} -> ${SOME_VAR} (shell variable, escaped) # # Each app declares its tool dependencies via runtimeInputs, with inheritPath -# set to false. This ensures apps only use explicitly declared tools. +# set to false. This ensures apps only use explicitly declared tools. The +# exception is integration-test, which keeps the host PATH (see there). { pkgs }: +let + # envtest binaries for the integration tests; null only on systems + # without an envtest build, where the tests skip. + envtestAssets = import ./envtest-assets.nix { inherit pkgs; }; +in { # Run Go unit tests. test = _: { @@ -25,12 +31,61 @@ inheritPath = false; text = '' export CGO_ENABLED=0 + export CROSSPLANE_CRDS_DIR="''${CROSSPLANE_CRDS_DIR:-${ + import ./crossplane-crds.nix { inherit pkgs; } + }}" go test -covermode=count ./... "$@" ''; } ); }; + # Run the integration tests: CLI commands against an envtest API server. + # envtest binaries and Crossplane's CRDs come from the Nix store. Unlike the + # sandboxed `test` check this runs outside the Nix sandbox, and it keeps the + # host PATH (Nix inputs first) so tests that run functions (#436's simulate + # tests) can reach the host's Docker and its credential helpers + # (docker-credential-*). CI runs it in the integration-tests job in + # .github/workflows/ci.yml. + integration-test = _: { + type = "app"; + meta.description = "Run integration tests against envtest"; + program = pkgs.lib.getExe ( + pkgs.writeShellApplication { + name = "crossplane-cli-integration-test"; + runtimeInputs = [ pkgs.unstable.go_1_26 ]; + inheritPath = true; + text = '' + export CGO_ENABLED=0 + export CROSSPLANE_CRDS_DIR="''${CROSSPLANE_CRDS_DIR:-${ + import ./crossplane-crds.nix { inherit pkgs; } + }}" + ${pkgs.lib.optionalString ( + envtestAssets != null + ) ''export KUBEBUILDER_ASSETS="''${KUBEBUILDER_ASSETS:-${envtestAssets}}"''} + # Integration tests are the test files with `//go:build integration`. + # Run every package that has one: its test files with the tag, minus + # those without it. go list applies Go's build-constraint rules. + # Unit tests in those packages run here too. + tests='{{range .TestGoFiles}}{{$.ImportPath}} {{.}}{{"\n"}}{{end}}{{range .XTestGoFiles}}{{$.ImportPath}} {{.}}{{"\n"}}{{end}}' + lists=$(mktemp -d) + go list -f "$tests" ./... | sort >"$lists/untagged" + go list -tags integration -f "$tests" ./... | sort >"$lists/tagged" + mapfile -t pkgs < <(comm -13 "$lists/untagged" "$lists/tagged" | cut -d' ' -f1 | sort -u) + rm -rf "$lists" + if [ "''${#pkgs[@]}" -eq 0 ]; then + echo "integration-test: found no packages with //go:build integration test files" >&2 + exit 1 + fi + # -coverpkg credits the tests' coverage to the packages they exercise; + # without it the profile only covers the test packages themselves. + go test -tags integration -covermode=count \ + -coverpkg=./... -coverprofile=coverage-integration.txt "''${pkgs[@]}" "$@" + ''; + } + ); + }; + # Run linters with auto-fix. Formats code first, then reports remaining issues. lint = _: { type = "app"; diff --git a/nix/checks.nix b/nix/checks.nix index 26b9ad27..eb58f760 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -16,6 +16,7 @@ let # Go builders backed by a single shared per-module vendor cache. # See nix/go-builders.nix. inherit (import ./go-builders.nix { inherit pkgs self; }) buildRoot; + crossplaneCRDs = import ./crossplane-crds.nix { inherit pkgs; }; in { # Run Go unit tests with coverage @@ -26,7 +27,13 @@ in inherit version; src = self; - env.CGO_ENABLED = "0"; + # Runs untagged unit tests only; the integration tests (envtest) run + # through the integration-test app. CROSSPLANE_CRDS_DIR serves the + # untagged CRD helper tests in internal/testutils/integration. + env = { + CGO_ENABLED = "0"; + CROSSPLANE_CRDS_DIR = crossplaneCRDs; + }; dontBuild = true; diff --git a/nix/crossplane-crds.nix b/nix/crossplane-crds.nix new file mode 100644 index 00000000..19380044 --- /dev/null +++ b/nix/crossplane-crds.nix @@ -0,0 +1,13 @@ +# Crossplane's own CRDs (Compositions, XRDs, Functions, ...) for the +# integration tests and the CRD helpers' unit tests. Pinned to the +# github.com/crossplane/crossplane/apis/v2 version in go.mod; bump rev and +# hash together when that changes. Tests find them through CROSSPLANE_CRDS_DIR. +{ pkgs }: +pkgs.fetchFromGitHub { + owner = "crossplane"; + repo = "crossplane"; + rev = "v2.4.1"; + sparseCheckout = [ "cluster/crds" ]; + hash = "sha256-+jJSqzsVmhw4eU93tFJi0Hcn/iVXc5EZ4Ktp/TYO+s0="; +} ++ "/cluster/crds" diff --git a/nix/envtest-assets.nix b/nix/envtest-assets.nix new file mode 100644 index 00000000..11f734ef --- /dev/null +++ b/nix/envtest-assets.nix @@ -0,0 +1,51 @@ +# envtest control-plane binaries (kube-apiserver, etcd, kubectl) for the +# integration tests, from the controller-tools envtest release. Tests find them +# through KUBEBUILDER_ASSETS. null is returned only for systems without an +# envtest build, so callers can skip wiring them. +{ pkgs }: +let + version = "1.30.3"; + platforms = { + x86_64-linux = { + os = "linux"; + arch = "amd64"; + hash = "sha256-DDwRyCoyUfBEaw5WPmYRmfrVEhDpmscC1MIroilgo9I="; + }; + aarch64-linux = { + os = "linux"; + arch = "arm64"; + hash = "sha256-Z26hrMSpoO04w9dr7CH2/b3v6U6QyunAj6vqD+opk5M="; + }; + x86_64-darwin = { + os = "darwin"; + arch = "amd64"; + hash = "sha256-hixH51R+1/LyMOkABsvdtSa/Os6mNhruixg4NzlSdJ4="; + }; + aarch64-darwin = { + os = "darwin"; + arch = "arm64"; + hash = "sha256-sRcsK819TPzkb2guCqw/neEei7oSoWPe2maYEJY1myQ="; + }; + }; + target = platforms.${pkgs.stdenv.hostPlatform.system} or null; +in +if target == null then + null +else + pkgs.runCommand "envtest-assets-${version}" + { + nativeBuildInputs = [ pkgs.gnutar ]; + meta = { + description = "envtest control-plane binaries (Kubernetes v${version})"; + platforms = builtins.attrNames platforms; + }; + } + '' + mkdir -p $out + tar -xzf ${ + pkgs.fetchurl { + url = "https://github.com/kubernetes-sigs/controller-tools/releases/download/envtest-v${version}/envtest-v${version}-${target.os}-${target.arch}.tar.gz"; + inherit (target) hash; + } + } --strip-components=2 -C $out controller-tools/envtest + '' From 8bc2996105dbb019c22c3e7b813ff9ed709886a6 Mon Sep 17 00:00:00 2001 From: Jonathan Ogilvie Date: Fri, 30 Oct 2026 14:43:09 -0400 Subject: [PATCH 3/8] test(trace): trace a claim's tree against envtest Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Jonathan Ogilvie --- .../integration/composed-failing.yaml | 16 +++ .../testdata/integration/composed-ready.yaml | 19 +++ .../integration/crds/nopresource.yaml | 49 ++++++++ .../trace/testdata/integration/setup.yaml | 60 ++++++++++ .../trace/testdata/integration/xr.yaml | 15 +++ .../trace/trace_integration_test.go | 108 ++++++++++++++++++ 6 files changed, 267 insertions(+) create mode 100644 cmd/crossplane/trace/testdata/integration/composed-failing.yaml create mode 100644 cmd/crossplane/trace/testdata/integration/composed-ready.yaml create mode 100644 cmd/crossplane/trace/testdata/integration/crds/nopresource.yaml create mode 100644 cmd/crossplane/trace/testdata/integration/setup.yaml create mode 100644 cmd/crossplane/trace/testdata/integration/xr.yaml create mode 100644 cmd/crossplane/trace/trace_integration_test.go diff --git a/cmd/crossplane/trace/testdata/integration/composed-failing.yaml b/cmd/crossplane/trace/testdata/integration/composed-failing.yaml new file mode 100644 index 00000000..191f8d87 --- /dev/null +++ b/cmd/crossplane/trace/testdata/integration/composed-failing.yaml @@ -0,0 +1,16 @@ +apiVersion: nop.example.org/v1alpha1 +kind: NopResource +metadata: + annotations: + crossplane.io/composition-resource-name: failing + name: test-claim-xr-failing +spec: + forProvider: + configData: claim-value +status: + conditions: + - type: Synced + status: "False" + reason: ReconcileError + message: cannot observe external resource + lastTransitionTime: "2026-01-01T00:00:00Z" diff --git a/cmd/crossplane/trace/testdata/integration/composed-ready.yaml b/cmd/crossplane/trace/testdata/integration/composed-ready.yaml new file mode 100644 index 00000000..4d14cc79 --- /dev/null +++ b/cmd/crossplane/trace/testdata/integration/composed-ready.yaml @@ -0,0 +1,19 @@ +apiVersion: nop.example.org/v1alpha1 +kind: NopResource +metadata: + annotations: + crossplane.io/composition-resource-name: ready + name: test-claim-xr-ready +spec: + forProvider: + configData: claim-value +status: + conditions: + - type: Synced + status: "True" + reason: ReconcileSuccess + lastTransitionTime: "2026-01-01T00:00:00Z" + - type: Ready + status: "True" + reason: Available + lastTransitionTime: "2026-01-01T00:00:00Z" diff --git a/cmd/crossplane/trace/testdata/integration/crds/nopresource.yaml b/cmd/crossplane/trace/testdata/integration/crds/nopresource.yaml new file mode 100644 index 00000000..57a5af64 --- /dev/null +++ b/cmd/crossplane/trace/testdata/integration/crds/nopresource.yaml @@ -0,0 +1,49 @@ +# A stand-in for a provider's managed resource, which the XR composes. It has +# no status subresource, so a fixture's status conditions are stored on create. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: nopresources.nop.example.org +spec: + group: nop.example.org + names: + kind: NopResource + listKind: NopResourceList + plural: nopresources + singular: nopresource + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + forProvider: + type: object + properties: + configData: + type: string + status: + type: object + properties: + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastTransitionTime: + type: string + format: date-time diff --git a/cmd/crossplane/trace/testdata/integration/setup.yaml b/cmd/crossplane/trace/testdata/integration/setup.yaml new file mode 100644 index 00000000..64841d32 --- /dev/null +++ b/cmd/crossplane/trace/testdata/integration/setup.yaml @@ -0,0 +1,60 @@ +# A v1 XRD with a claim, its Composition, and a claim bound to the XR in +# xr.yaml, as Crossplane's claim syncer binds it. +apiVersion: apiextensions.crossplane.io/v1 +kind: CompositeResourceDefinition +metadata: + name: xnops.trace.example.org +spec: + group: trace.example.org + names: + kind: XNop + plural: xnops + claimNames: + kind: NopClaim + plural: nopclaims + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + coolField: + type: string +--- +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: xnops.trace.example.org +spec: + compositeTypeRef: + apiVersion: trace.example.org/v1alpha1 + kind: XNop + mode: Pipeline + pipeline: + - step: generate-resources + functionRef: + name: function-dummy +--- +apiVersion: v1 +kind: Namespace +metadata: + name: test-namespace +--- +apiVersion: trace.example.org/v1alpha1 +kind: NopClaim +metadata: + name: test-claim + namespace: test-namespace +spec: + coolField: claim-value + compositionRef: + name: xnops.trace.example.org + resourceRef: + apiVersion: trace.example.org/v1alpha1 + kind: XNop + name: test-claim-xr diff --git a/cmd/crossplane/trace/testdata/integration/xr.yaml b/cmd/crossplane/trace/testdata/integration/xr.yaml new file mode 100644 index 00000000..ef80c620 --- /dev/null +++ b/cmd/crossplane/trace/testdata/integration/xr.yaml @@ -0,0 +1,15 @@ +# The XR backing the claim in setup.yaml. Start fills in its resourceRefs from +# the composed resources it is set up with. +apiVersion: trace.example.org/v1alpha1 +kind: XNop +metadata: + name: test-claim-xr +spec: + claimRef: + apiVersion: trace.example.org/v1alpha1 + kind: NopClaim + name: test-claim + namespace: test-namespace + coolField: claim-value + compositionRef: + name: xnops.trace.example.org diff --git a/cmd/crossplane/trace/trace_integration_test.go b/cmd/crossplane/trace/trace_integration_test.go new file mode 100644 index 00000000..ee90df65 --- /dev/null +++ b/cmd/crossplane/trace/trace_integration_test.go @@ -0,0 +1,108 @@ +//go:build integration + +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trace + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/alecthomas/kong" + "github.com/google/go-cmp/cmp" + + "github.com/crossplane/crossplane-runtime/v2/pkg/logging" + + "github.com/crossplane/cli/v2/internal/testutils/integration" +) + +func TestMain(m *testing.M) { + integration.ReapOrphanedEnvtest(func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) }) + os.Exit(m.Run()) +} + +// TestTraceIntegration traces a claim through its XR to the resources the XR composes, read from a real API +// server, and checks the tree the default printer draws. +func TestTraceIntegration(t *testing.T) { + cluster := integration.Start(t, integration.Setup{ + CRDDirs: []string{"testdata/integration/crds"}, + SetupFiles: []string{"testdata/integration/setup.yaml"}, + CrossplaneManagedResources: []integration.HierarchicalOwnershipRelation{ + integration.Composes("testdata/integration/xr.yaml", + "testdata/integration/composed-ready.yaml", + "testdata/integration/composed-failing.yaml"), + }, + XRDAPIVersion: integration.XRDV1, + }, 2*time.Minute) + + kubeconfig := filepath.Join(t.TempDir(), "kubeconfig") + if err := os.WriteFile(kubeconfig, cluster.Kubeconfig, 0o600); err != nil { + t.Fatalf("cannot write kubeconfig: %v", err) + } + + // trace loads its kubeconfig the way kubectl does. + t.Setenv("KUBECONFIG", kubeconfig) + + var c Cmd + + parser, err := kong.New(&c) + if err != nil { + t.Fatalf("kong.New(): %v", err) + } + + kongCtx, err := parser.Parse([]string{"nopclaim", "test-claim", "--namespace=test-namespace"}) + if err != nil { + t.Fatalf("Parse(): %v", err) + } + + var stdout bytes.Buffer + kongCtx.Stdout = &stdout + + if err := c.Run(kongCtx, logging.NewNopLogger()); err != nil { + t.Fatalf("Run(): %v", err) + } + + // Children sort by apiVersion, kind and name, so failing comes before ready. No controller runs under envtest, + // so the claim and XR have no conditions. + //nolint:dupword // Columns of the table repeat values. + want := ` +NAME SYNCED READY STATUS +NopClaim/test-claim (test-namespace) - - +└─ XNop/test-claim-xr - - + ├─ NopResource/test-claim-xr-failing False - ReconcileError: cannot observe external resource + └─ NopResource/test-claim-xr-ready True True Available +` + + if diff := cmp.Diff(trimLines(want), trimLines(stdout.String())); diff != "" { + t.Errorf("trace output: -want, +got:\n%s", diff) + } +} + +// trimLines drops leading and trailing blank lines, and the padding tabwriter leaves at the end of a line. +func trimLines(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + for i, l := range lines { + lines[i] = strings.TrimRight(l, " ") + } + + return strings.Join(lines, "\n") +} From 8d820e5194ce354e231a0574b49d1796ebbe8524 Mon Sep 17 00:00:00 2001 From: Jonathan Ogilvie Date: Sat, 31 Oct 2026 14:39:06 -0400 Subject: [PATCH 4/8] test(integration): fail rather than skip when envtest inputs are missing Integration tests only build with the integration tag, so reaching Start without envtest's binaries or Crossplane's CRDs means a broken environment. Fail loudly instead of skipping. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Jonathan Ogilvie --- internal/testutils/integration/cluster.go | 13 +++++++------ internal/testutils/integration/doc.go | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/internal/testutils/integration/cluster.go b/internal/testutils/integration/cluster.go index 6f760923..6fb62598 100644 --- a/internal/testutils/integration/cluster.go +++ b/internal/testutils/integration/cluster.go @@ -115,20 +115,21 @@ type Cluster struct { // Composition but no revision references its current one. A value a manifest sets always wins, which is how a // test sets up a state Crossplane wouldn't leave. // -// Start skips the test when the environment lacks envtest's binaries (KUBEBUILDER_ASSETS) or Crossplane's CRDs -// (CROSSPLANE_CRDS_DIR); `nix run .#integration-test` provides both. The cluster is stopped and the context +// Start fails the test when the environment lacks envtest's binaries (KUBEBUILDER_ASSETS) or Crossplane's CRDs +// (CROSSPLANE_CRDS_DIR), rather than skipping it, so a broken environment can't pass silently; `nix run +// .#integration-test` provides both. The cluster is stopped and the context // cancelled when the test ends. func Start(t *testing.T, s Setup, timeout time.Duration) *Cluster { t.Helper() - // envtest needs kube-apiserver and etcd binaries it cannot download itself; without them it would fail to - // start rather than skip. + // envtest needs kube-apiserver and etcd binaries it cannot download itself. Only a run that opted in with + // the integration tag gets here, so a missing binary is a broken environment, not a reason to skip. if !envtestBinariesAvailable() { - t.Skip("env: no envtest binaries (KUBEBUILDER_ASSETS unset); run via nix run .#integration-test") + t.Fatal("env: no envtest binaries (KUBEBUILDER_ASSETS unset or without kube-apiserver); run via nix run .#integration-test, or set KUBEBUILDER_ASSETS to a directory of envtest's binaries") } if os.Getenv(crossplaneCRDsDirEnv) == "" { - t.Skipf("env: %s unset; run via nix run .#integration-test, or set it to Crossplane's cluster/crds", + t.Fatalf("env: %s unset; run via nix run .#integration-test, or set it to Crossplane's cluster/crds", crossplaneCRDsDirEnv) } diff --git a/internal/testutils/integration/doc.go b/internal/testutils/integration/doc.go index 560c729e..c2ad8e2a 100644 --- a/internal/testutils/integration/doc.go +++ b/internal/testutils/integration/doc.go @@ -23,8 +23,8 @@ limitations under the License. // // Start and the fixtures behind it build only with the integration tag, as do the tests that use them, since they // need envtest's binaries. Run them with `nix run .#integration-test`, which provides those binaries and -// Crossplane's CRDs at the version go.mod pins. CRDsForXRD, PinnedCrossplaneCRDsDir and ReapOrphanedEnvtest are -// untagged, so their own tests run with the unit tests. +// Crossplane's CRDs at the version go.mod pins; Start fails, rather than skips, a test run without them. +// CRDsForXRD, PinnedCrossplaneCRDsDir and ReapOrphanedEnvtest are untagged, so their own tests run with the unit tests. // // Integration tests are any test files with `//go:build integration`. The integration-test app runs // `go test -tags integration` on every package containing one, so unit tests in those packages also run there. From 78a965e455a7db3e856cb86ca24c2e0bb01c1df2 Mon Sep 17 00:00:00 2001 From: Jonathan Ogilvie Date: Sat, 31 Oct 2026 14:39:24 -0400 Subject: [PATCH 5/8] test(integration): bound Start by the test's context, not its own timeout Start's per-call timeout duplicated go test's -timeout, which already bounds t.Context(). Drop the parameter and use the test's context. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Jonathan Ogilvie --- cmd/crossplane/trace/trace_integration_test.go | 3 +-- internal/testutils/integration/cluster.go | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cmd/crossplane/trace/trace_integration_test.go b/cmd/crossplane/trace/trace_integration_test.go index ee90df65..7b90bb45 100644 --- a/cmd/crossplane/trace/trace_integration_test.go +++ b/cmd/crossplane/trace/trace_integration_test.go @@ -25,7 +25,6 @@ import ( "path/filepath" "strings" "testing" - "time" "github.com/alecthomas/kong" "github.com/google/go-cmp/cmp" @@ -52,7 +51,7 @@ func TestTraceIntegration(t *testing.T) { "testdata/integration/composed-failing.yaml"), }, XRDAPIVersion: integration.XRDV1, - }, 2*time.Minute) + }) kubeconfig := filepath.Join(t.TempDir(), "kubeconfig") if err := os.WriteFile(kubeconfig, cluster.Kubeconfig, 0o600); err != nil { diff --git a/internal/testutils/integration/cluster.go b/internal/testutils/integration/cluster.go index 6fb62598..a97a0c38 100644 --- a/internal/testutils/integration/cluster.go +++ b/internal/testutils/integration/cluster.go @@ -23,7 +23,6 @@ import ( "os" "path/filepath" "testing" - "time" rbacv1 "k8s.io/api/rbac/v1" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -100,7 +99,7 @@ type Cluster struct { Kubeconfig []byte // Client is the cluster admin's. Client client.Client - // Context is the test's context, bounded by Start's timeout. + // Context is the test's (t.Context()), cancelled when the test ends; go test's -timeout bounds the run. Context context.Context //nolint:containedctx // Handed to the test with the cluster it bounds. } @@ -119,7 +118,7 @@ type Cluster struct { // (CROSSPLANE_CRDS_DIR), rather than skipping it, so a broken environment can't pass silently; `nix run // .#integration-test` provides both. The cluster is stopped and the context // cancelled when the test ends. -func Start(t *testing.T, s Setup, timeout time.Duration) *Cluster { +func Start(t *testing.T, s Setup) *Cluster { t.Helper() // envtest needs kube-apiserver and etcd binaries it cannot download itself. Only a run that opted in with @@ -162,8 +161,7 @@ func Start(t *testing.T, s Setup, timeout time.Duration) *Cluster { t.Fatalf("failed to create client: %v", err) } - ctx, cancel := context.WithTimeout(t.Context(), timeout) - t.Cleanup(cancel) + ctx := t.Context() defaults := newCrossplaneDefaults(k8sClient, crds) From 9bbfcf7efce468ba063b207111c8a4403402f70f Mon Sep 17 00:00:00 2001 From: Jonathan Ogilvie Date: Sat, 31 Oct 2026 14:40:42 -0400 Subject: [PATCH 6/8] build(nix): bump envtest to Kubernetes 1.35 Match the k8s.io/api v0.35 minor that go.mod pins. 1.35.0 is the only 1.35 envtest release controller-tools publishes. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Jonathan Ogilvie --- nix/envtest-assets.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nix/envtest-assets.nix b/nix/envtest-assets.nix index 11f734ef..7b346e12 100644 --- a/nix/envtest-assets.nix +++ b/nix/envtest-assets.nix @@ -4,27 +4,27 @@ # envtest build, so callers can skip wiring them. { pkgs }: let - version = "1.30.3"; + version = "1.35.0"; platforms = { x86_64-linux = { os = "linux"; arch = "amd64"; - hash = "sha256-DDwRyCoyUfBEaw5WPmYRmfrVEhDpmscC1MIroilgo9I="; + hash = "sha256-VxZxne8Uo/7D7SheXoxCgOYmiFQDm1BzqW6MCtr7HAI="; }; aarch64-linux = { os = "linux"; arch = "arm64"; - hash = "sha256-Z26hrMSpoO04w9dr7CH2/b3v6U6QyunAj6vqD+opk5M="; + hash = "sha256-UFf7Re7PJGkp2naLIdMkNLjJbiKnjvbN/pEvGmeq5Fo="; }; x86_64-darwin = { os = "darwin"; arch = "amd64"; - hash = "sha256-hixH51R+1/LyMOkABsvdtSa/Os6mNhruixg4NzlSdJ4="; + hash = "sha256-5zP3Lv/IqAdvLI64kt5K60u1TqAggoCM4+UfgPL/heI="; }; aarch64-darwin = { os = "darwin"; arch = "arm64"; - hash = "sha256-sRcsK819TPzkb2guCqw/neEei7oSoWPe2maYEJY1myQ="; + hash = "sha256-PGsevXRbgtrtR2BftWX3xnDIozRLV6N3qRTQE7a57vA="; }; }; target = platforms.${pkgs.stdenv.hostPlatform.system} or null; From 49b460769abd43d623bc414ac6953c690c484e17 Mon Sep 17 00:00:00 2001 From: Jonathan Ogilvie Date: Sat, 31 Oct 2026 14:46:44 -0400 Subject: [PATCH 7/8] build(nix): have tidy keep the Crossplane CRDs pin in sync with go.mod After go mod tidy, tidy sets nix/crossplane-crds.nix's rev to go.mod's crossplane/crossplane/apis/v2 version and refreshes its hash the way it refreshes the vendor hash, so renovate's go.mod bumps carry it along. It fails rather than guess when the module is replaced or at a pseudo-version. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Jonathan Ogilvie --- flake.nix | 7 ++++++ nix/apps.nix | 52 ++++++++++++++++++++++++++++++----------- nix/crossplane-crds.nix | 6 +++-- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/flake.nix b/flake.nix index 8eeecc3c..2b822576 100644 --- a/flake.nix +++ b/flake.nix @@ -110,6 +110,13 @@ # Vendor-dependency derivation, used by `nix run .#tidy` to refresh # the hash in nix/vendor-hashes.nix. crossplane-vendor = build.vendor.root; + # Crossplane's CRDs, used by `nix run .#tidy` to refresh the hash in + # nix/crossplane-crds.nix. Wrapped in a derivation because that file + # evaluates to a path inside the fetched source, and packages must be + # derivations; building it fetches (and hash-checks) that source. + crossplane-crds = pkgs.runCommand "crossplane-crds" { } '' + ln -s ${import ./nix/crossplane-crds.nix { inherit pkgs; }} $out + ''; } ); diff --git a/nix/apps.nix b/nix/apps.nix index c4cada17..ac58c063 100644 --- a/nix/apps.nix +++ b/nix/apps.nix @@ -181,40 +181,66 @@ in echo "Running go mod tidy..." go mod tidy - # Refresh a module's vendor hash in nix/vendor-hashes.nix by resetting - # it to a placeholder and reading the real hash back from the build + # Refresh a hash (a Go vendor hash, or a fetcher's) by resetting it + # to a placeholder and reading the real hash back from the build # error - the same trick nix-update uses for Go vendor hashes. refresh() { - key="$1" # attribute in nix/vendor-hashes.nix - attr="$2" # flake package that builds that module's vendor dir + file="$1" # .nix file holding the hash + key="$2" # attribute in $file + attr="$3" # flake package whose build checks that hash orig="" got="" # Remember the current value so we can restore it if anything fails. - orig=$(sed -n "s|.*$key = \"\(sha256-[^\"]*\)\";.*|\1|p" nix/vendor-hashes.nix | head -n1) + orig=$(sed -n "s|.*$key = \"\(sha256-[^\"]*\)\";.*|\1|p" "$file" | head -n1) # Set a placeholder so the build is forced to report the real hash. - sed -i "s|$key = \"sha256-[^\"]*\";|$key = \"${pkgs.lib.fakeHash}\";|" nix/vendor-hashes.nix + sed -i "s|$key = \"sha256-[^\"]*\";|$key = \"${pkgs.lib.fakeHash}\";|" "$file" - # --option eval-cache false: we just edited nix/vendor-hashes.nix, - # and Nix's flake eval cache can otherwise serve a stale evaluation - # that ignores the change and reports no/old hash. + # --option eval-cache false: we just edited $file, and Nix's flake + # eval cache can otherwise serve a stale evaluation that ignores + # the change and reports no/old hash. got=$(nix build ".#$attr" --no-link --option eval-cache false 2>&1 \ | sed -n 's|.*got:[[:space:]]*\(sha256-[A-Za-z0-9+/=]*\).*|\1|p' \ | head -n1 || true) if [ -z "$got" ]; then # Build failed for some other reason (e.g. network); restore the # previous value rather than leaving the placeholder committed. - sed -i "s|$key = \"sha256-[^\"]*\";|$key = \"$orig\";|" nix/vendor-hashes.nix - echo "ERROR: could not determine vendor hash for '$key' (restored previous value)" >&2 + sed -i "s|$key = \"sha256-[^\"]*\";|$key = \"$orig\";|" "$file" + echo "ERROR: could not determine hash for '$key' in $file (restored previous value)" >&2 exit 1 fi - sed -i "s|$key = \"sha256-[^\"]*\";|$key = \"$got\";|" nix/vendor-hashes.nix + sed -i "s|$key = \"sha256-[^\"]*\";|$key = \"$got\";|" "$file" echo " $key = $got" } echo "Refreshing Go vendor hashes..." - refresh root crossplane-vendor + refresh nix/vendor-hashes.nix root crossplane-vendor + + # Keep nix/crossplane-crds.nix at the crossplane/crossplane tag that + # go.mod's apis/v2 version names (apis/vX and vX tag the same commit). + echo "Syncing Crossplane CRDs with go.mod..." + mod=github.com/crossplane/crossplane/apis/v2 + want=$(go list -m -f '{{if .Replace}}replaced{{else}}{{.Version}}{{end}}' "$mod") + if [ "$want" = replaced ]; then + echo "ERROR: $mod is replaced in go.mod; set rev and hash in nix/crossplane-crds.nix by hand" >&2 + exit 1 + fi + # A pseudo-version (...-yyyymmddhhmmss-<12 hex>) names a commit, not + # a tag, so there is no tag to pin; pin its commit by hand. + if [[ "$want" =~ [0-9]{14}-[0-9a-f]{12}$ ]]; then + echo "ERROR: $mod is at pseudo-version $want; set rev to its commit SHA and refresh hash in nix/crossplane-crds.nix by hand" >&2 + exit 1 + fi + have=$(sed -n 's|.*rev = "\([^"]*\)";.*|\1|p' nix/crossplane-crds.nix | head -n1) + if [ "$have" != "$want" ]; then + sed -i "s|rev = \"$have\";|rev = \"$want\";|" nix/crossplane-crds.nix + if ! (refresh nix/crossplane-crds.nix hash crossplane-crds); then + sed -i "s|rev = \"$want\";|rev = \"$have\";|" nix/crossplane-crds.nix + exit 1 + fi + echo " rev = $want" + fi echo "Done" ''; diff --git a/nix/crossplane-crds.nix b/nix/crossplane-crds.nix index 19380044..da0385be 100644 --- a/nix/crossplane-crds.nix +++ b/nix/crossplane-crds.nix @@ -1,7 +1,9 @@ # Crossplane's own CRDs (Compositions, XRDs, Functions, ...) for the # integration tests and the CRD helpers' unit tests. Pinned to the -# github.com/crossplane/crossplane/apis/v2 version in go.mod; bump rev and -# hash together when that changes. Tests find them through CROSSPLANE_CRDS_DIR. +# github.com/crossplane/crossplane/apis/v2 version in go.mod; `nix run .#tidy` +# (which renovate runs after go.mod bumps) keeps rev and hash in sync. It +# fails, leaving them for you to set, when that version is a pseudo-version. +# Tests find them through CROSSPLANE_CRDS_DIR. { pkgs }: pkgs.fetchFromGitHub { owner = "crossplane"; From 3c9fd477032b5edd035557850b297721111a8a9c Mon Sep 17 00:00:00 2001 From: Jonathan Ogilvie Date: Sat, 31 Oct 2026 17:25:19 -0400 Subject: [PATCH 8/8] build(nix): have tidy keep the envtest pin in sync with go.mod tidy now sets nix/envtest-assets.nix to the newest envtest release for go.mod's k8s.io/api minor, taking each platform's sha512 from controller-tools' envtest-releases.yaml, the index setup-envtest verifies downloads against. The fetchurl hashes move from sha256 to sha512 to match. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Jonathan Ogilvie --- nix/apps.nix | 55 ++++++++++++++++++++++++++++++++++++++++++ nix/envtest-assets.nix | 12 ++++++--- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/nix/apps.nix b/nix/apps.nix index ac58c063..43998fa5 100644 --- a/nix/apps.nix +++ b/nix/apps.nix @@ -173,6 +173,10 @@ in pkgs.git pkgs.gnused pkgs.coreutils + pkgs.curl + pkgs.yq-go + pkgs.gnugrep + pkgs.diffutils ]; inheritPath = false; text = '' @@ -242,6 +246,57 @@ in echo " rev = $want" fi + # Keep nix/envtest-assets.nix at the newest envtest release for the + # Kubernetes minor go.mod's k8s.io/api names (v0.35.x -> 1.35), with + # hashes from the index setup-envtest verifies downloads against. + echo "Syncing envtest assets with go.mod..." + file=nix/envtest-assets.nix + mod=k8s.io/api + want=$(go list -m -f '{{if .Replace}}replaced{{else}}{{.Version}}{{end}}' "$mod") + if [ "$want" = replaced ]; then + echo "ERROR: $mod is replaced in go.mod; set version and hashes in $file by hand" >&2 + exit 1 + fi + if [[ "$want" =~ [0-9]{14}-[0-9a-f]{12}$ ]] || ! [[ "$want" =~ ^v0\.([0-9]+)\. ]]; then + echo "ERROR: $mod is at $want, not a v0.. release; set version and hashes in $file by hand" >&2 + exit 1 + fi + minor="''${BASH_REMATCH[1]}" + index=$(curl -fsSL https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/main/envtest-releases.yaml) || { + echo "ERROR: could not fetch the envtest release index ($file left unchanged)" >&2 + exit 1 + } + # Newest v1.. with no pre-release suffix. + version=$(yq '.releases | keys | .[]' <<<"$index" \ + | sed -n "s|^v\(1\.$minor\.[0-9][0-9]*\)$|\1|p" | sort -t. -k3,3n | tail -n1) + if [ -z "$version" ]; then + echo "ERROR: no envtest release for Kubernetes 1.$minor in the index ($file left unchanged)" >&2 + exit 1 + fi + # Edit a copy and only replace $file once every hash is in hand, so + # a failure leaves the old file in place. + tmp=$(mktemp) + trap 'rm -f "$tmp"' EXIT + sed "s|version = \"[^\"]*\";|version = \"$version\";|" "$file" >"$tmp" + for p in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64; do + os="''${p%/*}" arch="''${p#*/}" + hex=$(yq ".releases.\"v$version\".\"envtest-v$version-$os-$arch.tar.gz\".hash" <<<"$index") + if ! [[ "$hex" =~ ^[0-9a-f]{128}$ ]]; then + echo "ERROR: no sha512 for envtest v$version $os/$arch in the index ($file left unchanged)" >&2 + exit 1 + fi + sri=$(nix hash convert --hash-algo sha512 --to sri "$hex") + sed -i "/os = \"$os\";/{n;/arch = \"$arch\";/{n;s|hash = \"[^\"]*\";|hash = \"$sri\";|}}" "$tmp" + grep -q "hash = \"$sri\";" "$tmp" || { + echo "ERROR: could not set the $os/$arch hash in $file (left unchanged)" >&2 + exit 1 + } + done + if ! cmp -s "$tmp" "$file"; then + cat "$tmp" >"$file" + echo " version = $version" + fi + echo "Done" ''; } diff --git a/nix/envtest-assets.nix b/nix/envtest-assets.nix index 7b346e12..ef82bb54 100644 --- a/nix/envtest-assets.nix +++ b/nix/envtest-assets.nix @@ -2,6 +2,10 @@ # integration tests, from the controller-tools envtest release. Tests find them # through KUBEBUILDER_ASSETS. null is returned only for systems without an # envtest build, so callers can skip wiring them. +# +# Don't edit version or the hashes by hand: `nix run .#tidy` sets version to the +# newest envtest release for go.mod's k8s.io/api minor, and the sha512 hashes +# from controller-tools' envtest-releases.yaml index. { pkgs }: let version = "1.35.0"; @@ -9,22 +13,22 @@ let x86_64-linux = { os = "linux"; arch = "amd64"; - hash = "sha256-VxZxne8Uo/7D7SheXoxCgOYmiFQDm1BzqW6MCtr7HAI="; + hash = "sha512-EwNpwW8HbnJNCJGJr67elgMW9fXeps9XvnpPxvCcdzQokxklCXkOQFbhFuIy3/gy7YY/W9Vdy1XTjzq4NIKKEQ=="; }; aarch64-linux = { os = "linux"; arch = "arm64"; - hash = "sha256-UFf7Re7PJGkp2naLIdMkNLjJbiKnjvbN/pEvGmeq5Fo="; + hash = "sha512-5T4riDmPW5UD4/B02CotywkMcIs0lAhIYHzmWBOKXUolli4EKraDzMAmqKbJDAvn9ljkLd4Ihzadc8O2ji/IbA=="; }; x86_64-darwin = { os = "darwin"; arch = "amd64"; - hash = "sha256-5zP3Lv/IqAdvLI64kt5K60u1TqAggoCM4+UfgPL/heI="; + hash = "sha512-/MxYO6bTIsiKjFb3h2CQp61jRgBGpLyuQUCTsjzminXxcsp0hLfHR1twdlfspRCKitP9heG09wpsmcovItvWsg=="; }; aarch64-darwin = { os = "darwin"; arch = "arm64"; - hash = "sha256-PGsevXRbgtrtR2BftWX3xnDIozRLV6N3qRTQE7a57vA="; + hash = "sha512-u10Ls5dZVjMbCqDAOZVbTE3GxcKI5a82k2TH0vvqwRoCUifauxJ1aQgLJZ3Sm2l8x3+ner0nriZyHdsj6O4GEw=="; }; }; target = platforms.${pkgs.stdenv.hostPlatform.system} or null;