From b5777de36e9115107b56682a5c14718df93bbdfa Mon Sep 17 00:00:00 2001 From: bdchatham Date: Tue, 4 Aug 2026 19:05:03 -0700 Subject: [PATCH] feat(nightly): refuse to run the harness against a stale sei-chain image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suites read SEID_IMAGE, SEID_IMAGE_MOCK and SEID_IMAGE_CHAOS from env, and Flux's image automation advances those values in git. Once Flux pushes its bump branch it reports Ready, so every stage after that — the bump PR opening, its checks, the merge — can fail while the automation still looks healthy. The manifests keep the old tag, the suites pass, and nothing else in the fleet distinguishes that from a good run. TestMain now parses the date out of each nightly tag before any suite starts and exits non-zero if one is more than 36h behind. The date is structurally guaranteed by the ImagePolicy that selects the tag, so this needs no registry call, no cluster read and no added RBAC, and it holds whichever upstream stage broke — the image build or the bump. Failing here rather than mid-run means the existing NightlyHarnessSuiteFailed alert covers the whole class, and the two-and-a-half-hour run is not spent on a build nobody advanced. Scope: the check skips any tag that is not nightly-shaped, so the upgrade suites' pinned commit images are untouched, and it is inert without SEI_NODE_CLUSTER so local runs are unaffected. SEID_IMAGE_MAX_AGE_HOURS overrides the budget. Co-Authored-By: Claude Opus 5 (1M context) --- test/integration/harness_test.go | 67 +++++++++++++++++++++++ test/integration/image_freshness_test.go | 68 ++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 test/integration/image_freshness_test.go diff --git a/test/integration/harness_test.go b/test/integration/harness_test.go index e826be9..878d50c 100644 --- a/test/integration/harness_test.go +++ b/test/integration/harness_test.go @@ -31,6 +31,7 @@ import ( "net/http" "net/url" "os" + "regexp" "strconv" "strings" "sync/atomic" @@ -318,6 +319,11 @@ var suitesStarted atomic.Int64 // bounds only the zero-match case; positive proof that every expected suite ran // is a completion-count SLI owned by the metrics layer, not this binary. func TestMain(m *testing.M) { + if msg := staleSeidImages(time.Now().UTC()); msg != "" { + fmt.Fprintln(os.Stderr, msg) + os.Exit(1) + } + code := m.Run() if code == 0 && os.Getenv("SEI_NODE_CLUSTER") != "" && suitesStarted.Load() == 0 { fmt.Fprintln(os.Stderr, "integration guard: -test.run selected zero suites against a live cluster — "+ @@ -354,6 +360,67 @@ func openClient(ctx context.Context, t *testing.T) *sei.Client { return c } +// seidImageEnvVars are the image inputs Flux's image automation advances. The +// upgrade-suite images are deliberately absent: they pin specific commits, carry +// no date, and must not be read as stale. +var seidImageEnvVars = []string{"SEID_IMAGE", "SEID_IMAGE_MOCK", "SEID_IMAGE_CHAOS"} + +// nightlyTagDate extracts the date from a nightly tag. The alternation mirrors the +// three filterTags patterns in the platform repo's +// clusters/harbor/flux-system/image-automation.yaml; a tag shape added there needs +// adding here, or its image silently stops being freshness-checked. A tag that does +// not match — a pinned commit SHA, a local build — is skipped rather than failed, +// which is what keeps this from firing on the upgrade images or on a developer +// pointing the suite at their own build. +var nightlyTagDate = regexp.MustCompile( + `:(?:mock-|mock_chain_validation-mock_balances-)?nightly-([0-9]{8})-[0-9a-f]{7}$`) + +// seidImageMaxAge is how far behind the image under test may fall before the run is +// refused. The bump lands about 01:40 UTC and the suite runs at 08:00, so a healthy +// image is hours old; 36h tolerates one skipped upstream build without tolerating a +// stopped pipeline. Override with SEID_IMAGE_MAX_AGE_HOURS. +const seidImageMaxAge = 36 * time.Hour + +// staleSeidImages returns a diagnostic if any image input carries a nightly tag +// older than the budget, and "" if every one is fresh, unset, or not nightly-tagged. +// A stalled image bump otherwise reads as a green suite against an old build, since +// nothing downstream distinguishes that from a healthy run. +// +// The date comes from the tag, which the selecting ImagePolicy guarantees carries +// one, so this needs no registry call, no cluster read and no added RBAC. Local runs +// are exempt: without SEI_NODE_CLUSTER every suite skips anyway. +func staleSeidImages(now time.Time) string { + if os.Getenv("SEI_NODE_CLUSTER") == "" { + return "" + } + maxAge := seidImageMaxAge + if h, err := strconv.Atoi(os.Getenv("SEID_IMAGE_MAX_AGE_HOURS")); err == nil && h > 0 { + maxAge = time.Duration(h) * time.Hour + } + var stale []string + for _, key := range seidImageEnvVars { + m := nightlyTagDate.FindStringSubmatch(os.Getenv(key)) + if m == nil { + continue + } + built, err := time.Parse("20060102", m[1]) + if err != nil { + continue + } + if age := now.Sub(built); age > maxAge { + stale = append(stale, fmt.Sprintf(" %s is %s old (%s)", key, age.Round(time.Hour), os.Getenv(key))) + } + } + if len(stale) == 0 { + return "" + } + return "integration guard: refusing to run against a stale sei-chain image — the image bump is stuck, " + + "not the chain. Every suite would pass and report green on a build nobody has advanced.\n" + + strings.Join(stale, "\n") + + "\nCheck Flux's harbor image automation and whether its bump PR merged: " + + "`flux get image update -n flux-system` and any open flux-image-updates PR." +} + // envOr returns the env var or a fallback (for local runs). func envOr(key, fallback string) string { if v := os.Getenv(key); v != "" { diff --git a/test/integration/image_freshness_test.go b/test/integration/image_freshness_test.go new file mode 100644 index 0000000..f969a28 --- /dev/null +++ b/test/integration/image_freshness_test.go @@ -0,0 +1,68 @@ +//go:build integration + +package integration + +import ( + "strings" + "testing" + "time" +) + +// TestStaleSeidImages pins the gate's two edges: which tag shapes it reads, and +// which it must leave alone. Runs without a cluster. +func TestStaleSeidImages(t *testing.T) { + const ( + fresh = "189176372795.dkr.ecr.us-east-2.amazonaws.com/sei/sei-chain:nightly-20260805-0d9c675" + stale = "189176372795.dkr.ecr.us-east-2.amazonaws.com/sei/sei-chain:nightly-20260731-2d2628f" + // The upgrade suites pin commits, carry no date, and must never trip this. + pinned = "189176372795.dkr.ecr.us-east-2.amazonaws.com/sei/sei-chain:fbc0d9342ca28887958013170e4020d93cacdbfa" + ) + now := time.Date(2026, 8, 5, 8, 0, 0, 0, time.UTC) + + for _, tc := range []struct { + name string + env map[string]string + wantStale bool + }{ + {"same-day image passes", map[string]string{"SEID_IMAGE": fresh}, false}, + {"five-day-old image is refused", map[string]string{"SEID_IMAGE": stale}, true}, + {"pinned commit tag is not date-checked", map[string]string{"SEID_IMAGE": pinned}, false}, + {"unset image is not checked", map[string]string{}, false}, + {"mock flavour is checked too", map[string]string{ + "SEID_IMAGE": fresh, + "SEID_IMAGE_MOCK": strings.Replace(stale, "nightly-", "mock-nightly-", 1), + }, true}, + {"chaos flavour is checked too", map[string]string{ + "SEID_IMAGE": fresh, + "SEID_IMAGE_CHAOS": strings.Replace(stale, "nightly-", + "mock_chain_validation-mock_balances-nightly-", 1), + }, true}, + {"budget is overridable", map[string]string{ + "SEID_IMAGE": stale, + "SEID_IMAGE_MAX_AGE_HOURS": "240", + }, false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("SEI_NODE_CLUSTER", "harbor") + for _, k := range append(seidImageEnvVars, "SEID_IMAGE_MAX_AGE_HOURS") { + t.Setenv(k, "") + } + for k, v := range tc.env { + t.Setenv(k, v) + } + got := staleSeidImages(now) != "" + if got != tc.wantStale { + t.Fatalf("staleSeidImages stale=%v, want %v", got, tc.wantStale) + } + }) + } + + // Without a cluster every suite skips, so the gate must not fire locally. + t.Run("local run is exempt", func(t *testing.T) { + t.Setenv("SEI_NODE_CLUSTER", "") + t.Setenv("SEID_IMAGE", stale) + if msg := staleSeidImages(now); msg != "" { + t.Fatalf("expected no gate without a cluster, got: %s", msg) + } + }) +}