From 9795e0900416dc8e36a2d1746ded4845ed52b068 Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Sat, 27 Jun 2026 18:56:20 -0400 Subject: [PATCH] ci(e2e): shard scenarios across a retrying matrix Run the 76 multistep scenarios as a fail-fast:false matrix of 5 serial shards instead of one serial 45-minute job, so a flake is an isolated ~9-minute single-shard rerun. gotestsum --rerun-fails=2 gives each leg 3 attempts, rerunning only failed scenarios. workflow_dispatch shard and scenario inputs collapse the matrix to one leg or one scenario for targeted runs. The Integration Gate aggregates all legs; the workflow name and gate identity are unchanged. Signed-off-by: Joshua Temple --- .github/workflows/e2e.yaml | 120 ++++++++++++++++++++-- e2e/e2e_test.go | 53 +++++++++- e2e/harness/shard.go | 110 +++++++++++++++++++++ e2e/harness/shard_test.go | 198 +++++++++++++++++++++++++++++++++++++ e2e/init_scaffold_test.go | 1 + e2e/multi_repo_test.go | 42 +++++++- 6 files changed, 512 insertions(+), 12 deletions(-) create mode 100644 e2e/harness/shard.go create mode 100644 e2e/harness/shard_test.go diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 90435397..28a3f3ae 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -35,6 +35,14 @@ on: description: 'Subtest parallelism (lower = slower but more reliable)' required: false default: '1' + shard: + description: 'Run only this single shard index 0-4 (blank = full matrix)' + required: false + default: '' + scenario: + description: 'Run only scenarios whose subtest name matches this -run pattern (blank = all)' + required: false + default: '' # A superseded run on the same ref is cancelled rather than left to burn a full # ~27min testcontainers slot. Keyed on github.ref so each branch/PR/tag is its @@ -77,11 +85,66 @@ jobs: echo "code=${{ steps.filter.outputs.code }}" >> "$GITHUB_OUTPUT" fi - e2e: - name: E2E Tests + plan: + name: Plan Shards needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest + permissions: + contents: read + outputs: + shards: ${{ steps.compute.outputs.shards }} + steps: + - id: compute + # Build the matrix shard list. A manual `shard` input collapses the + # matrix to that single leg for targeted reruns. A `scenario` input + # without a shard also collapses to a single leg, because the test code + # bypasses sharding when a scenario filter is set and would otherwise run + # the same scenarios on every leg. Otherwise the full set of five shards + # runs. The shard input is validated to one digit 0-4 so it cannot + # inject arbitrary JSON into the matrix expression. + env: + SHARD_INPUT: ${{ github.event.inputs.shard }} + SCENARIO_INPUT: ${{ github.event.inputs.scenario }} + run: | + if [ -n "$SHARD_INPUT" ]; then + case "$SHARD_INPUT" in + [0-4]) + echo "shards=[$SHARD_INPUT]" >> "$GITHUB_OUTPUT" + exit 0 + ;; + *) + echo "Invalid shard input '$SHARD_INPUT'; expected a single index 0-4." >&2 + exit 1 + ;; + esac + fi + if [ -n "$SCENARIO_INPUT" ]; then + echo 'shards=[0]' >> "$GITHUB_OUTPUT" + exit 0 + fi + echo 'shards=[0,1,2,3,4]' >> "$GITHUB_OUTPUT" + + e2e: + # The scenario suite is split across N runners (one matrix leg per shard). + # Each leg runs its slice serially (E2E_PARALLEL=1) to keep the per-box + # reliability floor, so a single environmental flake fails one short leg + # rather than the whole multi-hour suite. E2E_SHARD_TOTAL must match the + # length of the shard list below; the Go side reads both values from the + # environment and falls back to running everything when they are unset. + name: E2E Tests (shard ${{ matrix.shard }}) + needs: [changes, plan] + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + timeout-minutes: 35 + strategy: + # One shard's flake must not cancel the others; each is independently + # rerunnable from the failed matrix leg. + fail-fast: false + matrix: + # Computed by the plan job: the full five-shard set, or a single shard + # when the workflow_dispatch `shard` input collapses the matrix. + shard: ${{ fromJSON(needs.plan.outputs.shards) }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -110,12 +173,18 @@ jobs: echo "=== Disk ===" df -h / + - name: Install gotestsum + # gotestsum drives the rerun-on-failure self-heal: on a failed run it + # reruns only the individual failed scenarios (not the whole shard), + # giving three total attempts per leg before the shard fails. + run: go install gotest.tools/gotestsum@v1.13.0 + - name: Run e2e tests working-directory: e2e env: - # Default 60m to give 32 scenarios (4 cores × ~6min ÷ 2 parallel) - # enough headroom. Override per-dispatch as needed. - E2E_TIMEOUT: ${{ github.event.inputs.timeout || '60m' }} + # Each shard runs roughly a fifth of the suite, so 30m of per-package + # headroom is ample. Override per-dispatch as needed. + E2E_TIMEOUT: ${{ github.event.inputs.timeout || '30m' }} # Cap subtest parallelism. The GitHub runner has 4 cores / ~7.9GB # RAM. Each scenario spins up gitea + act + N job containers; at # the default GOMAXPROCS=4, four scenarios concurrently exhaust @@ -124,11 +193,29 @@ jobs: # load throttles gitea and destabilises act runs, so the default is # serial (1); raise it per-dispatch only when chasing wall-clock. E2E_PARALLEL: ${{ github.event.inputs.parallel || '1' }} + # Scenario sharding. The Go side sorts scenarios by name and selects + # those whose position modulo the total equals this index, so the + # union of all legs is the whole suite with no overlap. Keep + # E2E_SHARD_TOTAL equal to the matrix shard-list length. + E2E_SHARD_INDEX: ${{ matrix.shard }} + E2E_SHARD_TOTAL: 5 + # Optional single-scenario filter for manual debugging. The test code + # reads it as a regular expression over scenario names and, when set, + # bypasses sharding so one leg runs only the matching scenarios. It is + # consumed only by the Go test process, never by the shell. + E2E_SCENARIO: ${{ github.event.inputs.scenario }} run: | - go test -v \ + # --rerun-fails=2 gives three total attempts and reruns only the + # scenarios that failed; the leg passes if every scenario passes + # within those attempts. + gotestsum \ + --format standard-verbose \ + --rerun-fails=2 \ + --rerun-fails-report rerun-report.txt \ + --packages=./... \ + -- \ -timeout "$E2E_TIMEOUT" \ - -parallel "$E2E_PARALLEL" \ - ./... + -parallel "$E2E_PARALLEL" # On a retry-exhausted scenario the harness writes the last attempt's raw # act stdout/stderr to e2e/_artifacts/-attempt.log so the @@ -141,7 +228,9 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: e2e-crash-evidence + # Per-shard name: upload-artifact rejects duplicate names across + # matrix legs, so each shard writes its own evidence artifact. + name: e2e-crash-evidence-shard-${{ matrix.shard }} path: e2e/_artifacts/ if-no-files-found: ignore retention-days: 14 @@ -150,8 +239,11 @@ jobs: # This always-run context is the branch-protection required check. It mirrors # the heavy E2E result when E2E runs and passes cleanly when E2E is correctly # skipped on non-code changes, avoiding the required-but-skipped deadlock. + # needs.e2e aggregates every shard in the matrix: its result is success only + # when all legs pass and failure if any single shard fails, so this gate + # stays the one required-check identity over the whole sharded suite. name: Integration Gate - needs: [changes, e2e] + needs: [changes, plan, e2e] if: always() runs-on: ubuntu-latest permissions: @@ -160,6 +252,7 @@ jobs: - name: Check Integration Status env: CHANGES_RESULT: ${{ needs.changes.result }} + PLAN_RESULT: ${{ needs.plan.result }} E2E_RESULT: ${{ needs.e2e.result }} run: | # The detection job must succeed for its `code` output to be trusted. @@ -169,6 +262,13 @@ jobs: echo "Integration gate: change-detection result=$CHANGES_RESULT, failing the gate." exit 1 fi + # The plan job computes the shard matrix. When no code changed it is + # correctly skipped (alongside e2e); any other non-success is a real + # failure that must not be waved through. + if [ "$PLAN_RESULT" != "success" ] && [ "$PLAN_RESULT" != "skipped" ]; then + echo "Integration gate: shard-plan result=$PLAN_RESULT, failing the gate." + exit 1 + fi # E2E success means a code change passed the heavy suite. E2E skipped # means no code path changed, which is a legitimate pass. if [ "$E2E_RESULT" = "success" ] || [ "$E2E_RESULT" = "skipped" ]; then diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index ce376be9..e338b78f 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -2,7 +2,10 @@ package e2e import ( "context" + "os" + "regexp" "runtime" + "strings" "testing" "time" @@ -10,6 +13,15 @@ import ( "github.com/stretchr/testify/require" ) +// envScenarioFilter, when set to a regular expression, narrows the run to +// scenarios whose name matches it. It is a single-scenario debugging switch: it +// bypasses shard selection so one matrix leg can run exactly the target +// scenarios regardless of which shard would normally own them. +const envScenarioFilter = "E2E_SCENARIO" + +// scenarioFilter returns the trimmed scenario-name pattern, or "" when unset. +func scenarioFilter() string { return strings.TrimSpace(os.Getenv(envScenarioFilter)) } + func TestMultiStepScenarios(t *testing.T) { if testing.Short() { t.Skip("skipping E2E tests") @@ -19,8 +31,25 @@ func TestMultiStepScenarios(t *testing.T) { scenarios, err := harness.DiscoverMultiStepScenarios("scenarios") require.NoError(t, err) + if pattern := scenarioFilter(); pattern != "" { + // Debug switch: filter the full set by name and bypass sharding so a + // single leg runs only the matching scenarios. + re, err := regexp.Compile(pattern) + require.NoError(t, err, "invalid %s pattern", envScenarioFilter) + scenarios = filterScenariosByName(scenarios, re) + t.Logf("scenario filter %q selected %d scenario(s)", pattern, len(scenarios)) + } else { + // Select this CI shard's slice. Outside a sharded matrix the default + // (E2E_SHARD_TOTAL unset = 1) returns every scenario, so a local run is + // unchanged. The scenario Description embeds the unique source path, + // giving a stable round-robin distribution across shards. + shard, err := harness.ShardFromEnv() + require.NoError(t, err) + scenarios = harness.SelectShard(scenarios, scenarioShardKey, shard) + } + if len(scenarios) == 0 { - t.Log("No multi-step scenarios found") + t.Log("No multi-step scenarios selected for this leg") return } @@ -43,6 +72,28 @@ func TestMultiStepScenarios(t *testing.T) { } } +// filterScenariosByName keeps only the scenarios whose name matches re. +func filterScenariosByName(scenarios []*harness.MultiStepScenario, re *regexp.Regexp) []*harness.MultiStepScenario { + out := make([]*harness.MultiStepScenario, 0, len(scenarios)) + for _, s := range scenarios { + if re.MatchString(s.Name) { + out = append(out, s) + } + } + return out +} + +// scenarioShardKey returns the stable key used to order scenarios before they +// are distributed across CI shards. Discovery sets Description to the unique +// source path (optionally suffixed with the scenario description), so it is a +// reliable, collision-free sort key even when two scenarios share a name. +func scenarioShardKey(s *harness.MultiStepScenario) string { + if s.Description != "" { + return s.Description + } + return s.Name +} + // DefaultParallelism returns recommended parallel test count func DefaultParallelism() int { cpus := runtime.NumCPU() diff --git a/e2e/harness/shard.go b/e2e/harness/shard.go new file mode 100644 index 00000000..805c17e4 --- /dev/null +++ b/e2e/harness/shard.go @@ -0,0 +1,110 @@ +package harness + +import ( + "fmt" + "hash/fnv" + "os" + "sort" + "strconv" +) + +// Environment variables that select a scenario shard for one CI matrix leg. +const ( + envShardIndex = "E2E_SHARD_INDEX" + envShardTotal = "E2E_SHARD_TOTAL" +) + +// ShardConfig partitions the e2e scenario set across parallel CI runners. The +// default value (Index 0, Total 1) selects the whole set, so a plain local +// `go test` run is unaffected and every selection becomes a no-op. +type ShardConfig struct { + // Index is the zero-based position of this shard within the matrix. + Index int + // Total is the number of shards the suite is split across. + Total int +} + +// ShardFromEnv builds a ShardConfig from E2E_SHARD_INDEX and E2E_SHARD_TOTAL. +// Unset or blank variables fall back to the single-shard default (0 of 1). It +// returns an error when the values are non-numeric or out of range so a +// misconfigured matrix leg fails loudly instead of silently dropping scenarios. +func ShardFromEnv() (ShardConfig, error) { + cfg := ShardConfig{Index: 0, Total: 1} + + total, ok, err := lookupShardInt(envShardTotal) + if err != nil { + return cfg, err + } + if ok { + cfg.Total = total + } + + index, ok, err := lookupShardInt(envShardIndex) + if err != nil { + return cfg, err + } + if ok { + cfg.Index = index + } + + if cfg.Total < 1 { + return cfg, fmt.Errorf("%s must be >= 1, got %d", envShardTotal, cfg.Total) + } + if cfg.Index < 0 || cfg.Index >= cfg.Total { + return cfg, fmt.Errorf("%s must be in [0, %d), got %d", envShardIndex, cfg.Total, cfg.Index) + } + return cfg, nil +} + +// lookupShardInt reads an integer environment variable. The boolean reports +// whether the variable was set to a non-empty value. +func lookupShardInt(name string) (int, bool, error) { + raw, ok := os.LookupEnv(name) + if !ok || raw == "" { + return 0, false, nil + } + v, err := strconv.Atoi(raw) + if err != nil { + return 0, false, fmt.Errorf("%s=%q is not an integer: %w", name, raw, err) + } + return v, true, nil +} + +// SelectShard returns the items that belong to this shard. Items are first +// sorted by key for a stable, run-independent ordering, then distributed +// round-robin (position modulo Total) so heavy scenarios spread across shards +// instead of clustering in one contiguous block. The partition is by position, +// so the union of all shards equals the input and no item appears twice, even +// when keys collide. With Total <= 1 it returns the input unchanged. +func SelectShard[T any](items []T, key func(T) string, cfg ShardConfig) []T { + if cfg.Total <= 1 { + return items + } + + sorted := make([]T, len(items)) + copy(sorted, items) + sort.SliceStable(sorted, func(i, j int) bool { + return key(sorted[i]) < key(sorted[j]) + }) + + out := make([]T, 0, len(sorted)/cfg.Total+1) + for i, item := range sorted { + if i%cfg.Total == cfg.Index { + out = append(out, item) + } + } + return out +} + +// Owns reports whether a uniquely named, standalone scenario belongs to this +// shard. Each name maps to exactly one shard through a stable hash, so across +// the full matrix every named scenario runs once and only once. With Total <= 1 +// it always returns true. +func (c ShardConfig) Owns(name string) bool { + if c.Total <= 1 { + return true + } + h := fnv.New32a() + _, _ = h.Write([]byte(name)) + return int(h.Sum32()%uint32(c.Total)) == c.Index +} diff --git a/e2e/harness/shard_test.go b/e2e/harness/shard_test.go new file mode 100644 index 00000000..516972eb --- /dev/null +++ b/e2e/harness/shard_test.go @@ -0,0 +1,198 @@ +package harness + +import ( + "fmt" + "os" + "sort" + "testing" +) + +func TestShardFromEnv(t *testing.T) { + tests := []struct { + name string + index string // "" means leave the variable unset + total string + want ShardConfig + wantError bool + }{ + {name: "unset defaults to whole suite", want: ShardConfig{Index: 0, Total: 1}}, + {name: "valid mid shard", index: "2", total: "5", want: ShardConfig{Index: 2, Total: 5}}, + {name: "valid last shard", index: "4", total: "5", want: ShardConfig{Index: 4, Total: 5}}, + {name: "total without index defaults index zero", total: "3", want: ShardConfig{Index: 0, Total: 3}}, + {name: "non-numeric total", total: "five", wantError: true}, + {name: "non-numeric index", index: "x", total: "5", wantError: true}, + {name: "zero total", total: "0", wantError: true}, + {name: "negative index", index: "-1", total: "5", wantError: true}, + {name: "index equal to total", index: "5", total: "5", wantError: true}, + {name: "index above total", index: "9", total: "5", wantError: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + setOrUnset(t, envShardIndex, tt.index) + setOrUnset(t, envShardTotal, tt.total) + + got, err := ShardFromEnv() + if tt.wantError { + if err == nil { + t.Fatalf("expected error, got config %+v", got) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Fatalf("ShardFromEnv() = %+v, want %+v", got, tt.want) + } + }) + } +} + +// setOrUnset sets an environment variable for the duration of the test, or +// clears it when value is empty so the default (unset) path is exercised. The +// prior value is restored on cleanup so cases do not leak into one another. +func setOrUnset(t *testing.T, name, value string) { + t.Helper() + prev, had := os.LookupEnv(name) + t.Cleanup(func() { + if had { + _ = os.Setenv(name, prev) + return + } + _ = os.Unsetenv(name) + }) + if value == "" { + if err := os.Unsetenv(name); err != nil { + t.Fatal(err) + } + return + } + if err := os.Setenv(name, value); err != nil { + t.Fatal(err) + } +} + +func TestSelectShard_TotalOneIsNoOp(t *testing.T) { + items := makeKeyed(10) + got := SelectShard(items, keyOf, ShardConfig{Index: 0, Total: 1}) + if len(got) != len(items) { + t.Fatalf("Total=1 changed the set: got %d, want %d", len(got), len(items)) + } + for i := range items { + if got[i] != items[i] { + t.Fatalf("Total=1 reordered items at %d", i) + } + } +} + +func TestSelectShard_Deterministic(t *testing.T) { + items := makeKeyed(37) + cfg := ShardConfig{Index: 1, Total: 4} + first := SelectShard(items, keyOf, cfg) + // Re-run against a shuffled copy: a stable sort by key must yield the same + // selection regardless of input order. + shuffled := make([]keyed, len(items)) + copy(shuffled, items) + sort.SliceStable(shuffled, func(i, j int) bool { return shuffled[i].name > shuffled[j].name }) + second := SelectShard(shuffled, keyOf, cfg) + + if len(first) != len(second) { + t.Fatalf("non-deterministic length: %d vs %d", len(first), len(second)) + } + for i := range first { + if first[i] != second[i] { + t.Fatalf("non-deterministic selection at %d: %v vs %v", i, first[i], second[i]) + } + } +} + +func TestSelectShard_PartitionIsCompleteBalancedAndDisjoint(t *testing.T) { + for _, total := range []int{2, 3, 5, 8} { + for _, count := range []int{0, 1, 7, 76} { + t.Run(fmt.Sprintf("total=%d/count=%d", total, count), func(t *testing.T) { + items := makeKeyed(count) + + seen := make(map[string]int, count) + sizes := make([]int, total) + for idx := 0; idx < total; idx++ { + shard := SelectShard(items, keyOf, ShardConfig{Index: idx, Total: total}) + sizes[idx] = len(shard) + for _, it := range shard { + seen[it.name]++ + } + } + + // Complete: every item selected exactly once (union, no overlap). + if len(seen) != count { + t.Fatalf("union covered %d items, want %d", len(seen), count) + } + for name, n := range seen { + if n != 1 { + t.Fatalf("item %q appeared in %d shards, want 1", name, n) + } + } + + // Balanced: shard sizes differ by at most one. + minSize, maxSize := sizes[0], sizes[0] + for _, s := range sizes { + if s < minSize { + minSize = s + } + if s > maxSize { + maxSize = s + } + } + if maxSize-minSize > 1 { + t.Fatalf("unbalanced shards %v (spread %d)", sizes, maxSize-minSize) + } + }) + } + } +} + +func TestShardConfig_Owns(t *testing.T) { + const total = 5 + names := []string{ + "TestInitScaffoldOrchestratesAndPromotes", + "TestMultiRepoScenario_SatelliteNotifiesPrimary", + "TestMultiRepoScenario_ExternalStatePromotion", + "TestMultiRepoScenario_MixedDeploys", + } + + // Total=1 owns everything. + whole := ShardConfig{Index: 0, Total: 1} + for _, n := range names { + if !whole.Owns(n) { + t.Fatalf("Total=1 should own %q", n) + } + } + + // Across the matrix each name is owned by exactly one shard. + for _, n := range names { + owners := 0 + for idx := 0; idx < total; idx++ { + if (ShardConfig{Index: idx, Total: total}).Owns(n) { + owners++ + } + } + if owners != 1 { + t.Fatalf("name %q owned by %d shards, want 1", n, owners) + } + } +} + +type keyed struct { + name string +} + +func keyOf(k keyed) string { return k.name } + +func makeKeyed(n int) []keyed { + items := make([]keyed, n) + for i := 0; i < n; i++ { + // Zero-padded so lexical order is independent of width. + items[i] = keyed{name: fmt.Sprintf("scenario-%04d", i)} + } + return items +} diff --git a/e2e/init_scaffold_test.go b/e2e/init_scaffold_test.go index e41fe0eb..6be70e24 100644 --- a/e2e/init_scaffold_test.go +++ b/e2e/init_scaffold_test.go @@ -42,6 +42,7 @@ func TestInitScaffoldOrchestratesAndPromotes(t *testing.T) { if testing.Short() { t.Skip("skipping E2E tests") } + requireShardOwns(t) const project = "cascade-init-demo" envs := []string{"dev", "prod"} diff --git a/e2e/multi_repo_test.go b/e2e/multi_repo_test.go index db9a4dd3..ac271cd5 100644 --- a/e2e/multi_repo_test.go +++ b/e2e/multi_repo_test.go @@ -17,6 +17,10 @@ func TestMultiRepoScenarios(t *testing.T) { t.Skip("skipping multi-repo e2e tests in short mode") } + if pattern := scenarioFilter(); pattern != "" { + t.Skipf("%s=%q targets multi-step scenarios; skipping multi-repo suite", envScenarioFilter, pattern) + } + // Find the scenarios directory scenariosDir := filepath.Join("scenarios", "multi-repo") if _, err := os.Stat(scenariosDir); os.IsNotExist(err) { @@ -26,8 +30,16 @@ func TestMultiRepoScenarios(t *testing.T) { scenarios, err := harness.DiscoverMultiRepoScenarios(scenariosDir) require.NoError(t, err, "failed to discover scenarios") + // Select only this shard's slice so the heavy per-repo setup runs once + // across the matrix rather than once per leg. The default (unset) shard + // returns the full set, leaving local runs unchanged. Description embeds + // the unique source path, so the round-robin distribution is stable. + shard, err := harness.ShardFromEnv() + require.NoError(t, err) + scenarios = harness.SelectShard(scenarios, multiRepoShardKey, shard) + if len(scenarios) == 0 { - t.Skip("no multi-repo scenarios found") + t.Skip("no multi-repo scenarios for this shard") } for _, scenario := range scenarios { @@ -39,6 +51,31 @@ func TestMultiRepoScenarios(t *testing.T) { } } +// multiRepoShardKey returns the stable, collision-free key used to order +// multi-repo scenarios before they are distributed across CI shards. Discovery +// sets Description to the unique source path. +func multiRepoShardKey(s *harness.MultiRepoScenario) string { + if s.Description != "" { + return s.Description + } + return s.Name +} + +// requireShardOwns skips a standalone, singly-named heavyweight test unless the +// active shard owns it. Each test name hashes to exactly one shard, so across +// the matrix it runs once; the unset default owns everything for local runs. +func requireShardOwns(t *testing.T) { + t.Helper() + if pattern := scenarioFilter(); pattern != "" { + t.Skipf("%s=%q targets multi-step scenarios; skipping standalone test", envScenarioFilter, pattern) + } + shard, err := harness.ShardFromEnv() + require.NoError(t, err) + if !shard.Owns(t.Name()) { + t.Skipf("assigned to a different shard (this leg is %d of %d)", shard.Index, shard.Total) + } +} + func runMultiRepoScenario(t *testing.T, scenario *harness.MultiRepoScenario) { // Real per-repo workflow generation (clone + build + generate + push + // converge for each repo) plus the external-update run under act is heavy; @@ -62,6 +99,7 @@ func TestMultiRepoScenario_SatelliteNotifiesPrimary(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") } + requireShardOwns(t) scenarioPath := filepath.Join("scenarios", "multi-repo", "satellite-notifies-primary.yaml") if _, err := os.Stat(scenarioPath); os.IsNotExist(err) { @@ -82,6 +120,7 @@ func TestMultiRepoScenario_ExternalStatePromotion(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") } + requireShardOwns(t) scenarioPath := filepath.Join("scenarios", "multi-repo", "external-state-promotion.yaml") if _, err := os.Stat(scenarioPath); os.IsNotExist(err) { @@ -102,6 +141,7 @@ func TestMultiRepoScenario_MixedDeploys(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") } + requireShardOwns(t) scenarioPath := filepath.Join("scenarios", "multi-repo", "mixed-deploys.yaml") if _, err := os.Stat(scenarioPath); os.IsNotExist(err) {