feat(coverage): gate generated workflow kinds on executing coverage

CONTRIBUTING.md

@@ -54,6 +54,7 @@ cascade holds to a few conventions in its own codebase and in the workflows it g
 - **Additive manifest changes**: new fields are always optional with sensible defaults, so existing manifest files keep working across minor version bumps.
 - **Path fields reach every path sink**: a manifest field that widens which files a component reacts to must thread through all three places a path is consumed, or it is a silent bug. The emitted `on: push` paths filter fires the workflow, per-callback change detection decides which builds and deploys run, and the version commit range decides the bump. A field that reaches only some of these triggers a run that then no-ops, or bumps a version whose builds skip as unchanged. When you add such a field, add a test that asserts the shared path reaches each sink.
 - **A breaking generator or validation change moves with the fleet, in the same change**: a change that makes a previously valid manifest invalid, such as rejecting a field `parse-config` used to accept, is breaking even when it ships as a `fix:`. The fleet repin re-stamps every example repository onto the release-candidate binary and regenerates its workflows before any suite runs, so a manifest still carrying the now-rejected shape fails that repin, not the intended test. Before landing a validation change that can reject something that used to pass, scan the fleet example repos for that shape and migrate any that use it in the same pull request, alongside the doc's migration note. This generalizes the existing rule that a fleet suite and the eligibility logic it exercises are one coupled unit (see [Making a change](#making-a-change)); it applies to validation, not only to eligibility.
+- **Every generated workflow kind carries executing coverage**: each workflow the generator emits (`orchestrate`, `promote`, `external-update`, and the `cascade-` lanes) is mapped to the e2e scenarios and fleet lanes that run it in `internal/coverage/registry.yaml`. The coverage gate derives the emitted kinds straight from the generator source and fails when an emitted kind has no registry entry, so a new generated workflow cannot ship without a scenario or lane that exercises it. When you add a generated workflow kind, add its entry pointing at the scenario or fleet lane that runs it; a referenced scenario or lane that does not exist also fails the gate.
 - **Callback isolation**: generated workflows call your workflows via `workflow_call`, and cascade never reaches into your callback logic.
 - **Metadata courier**: cascade passes artifact identifiers and versions between stages. It never touches your container registry, package registry, or the systems you deploy to directly.
 

internal/coverage/coverage_test.go

@@ -0,0 +1,186 @@
+package coverage
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// emittedWorkflowKinds derives the set of generated workflow kinds directly from
+// the generator source. It parses every non-test Go file under internal/generate
+// and collects the basename of each .github/workflows/<name>.yaml string literal
+// that classifies as a generated kind. This is the code-derived denominator: it
+// never reads the registry, so the registry cannot define its own completeness.
+func emittedWorkflowKinds(t *testing.T) map[string]struct{} {
+	t.Helper()
+
+	genDir := filepath.Join("..", "generate")
+	entries, err := os.ReadDir(genDir)
+	if err != nil {
+		t.Fatalf("reading generator source dir %s: %v", genDir, err)
+	}
+
+	kinds := map[string]struct{}{}
+	fset := token.NewFileSet()
+	for _, e := range entries {
+		name := e.Name()
+		if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
+			continue
+		}
+		f, perr := parser.ParseFile(fset, filepath.Join(genDir, name), nil, 0)
+		if perr != nil {
+			t.Fatalf("parsing %s: %v", name, perr)
+		}
+		ast.Inspect(f, func(n ast.Node) bool {
+			lit, ok := n.(*ast.BasicLit)
+			if !ok || lit.Kind != token.STRING {
+				return true
+			}
+			val, uerr := strconv.Unquote(lit.Value)
+			if uerr != nil {
+				return true
+			}
+			if kind, ok := workflowKindFromLiteral(val); ok {
+				kinds[kind] = struct{}{}
+			}
+			return true
+		})
+	}
+
+	if len(kinds) == 0 {
+		t.Fatal("derived no generated workflow kinds from internal/generate; the scan is broken")
+	}
+	return kinds
+}
+
+func sortedKeys(m map[string]struct{}) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// TestCoverage_EveryEmittedWorkflowKindHasExecutingCoverage is the coverage
+// gate: it derives the emitted workflow kinds from the generator source and
+// requires each to carry at least one executing-coverage reference in the
+// registry. A generated workflow that no scenario, harness test, or fleet lane
+// runs fails here.
+func TestCoverage_EveryEmittedWorkflowKindHasExecutingCoverage(t *testing.T) {
+	kinds := emittedWorkflowKinds(t)
+
+	reg, err := Load()
+	if err != nil {
+		t.Fatalf("loading coverage registry: %v", err)
+	}
+
+	for _, kind := range sortedKeys(kinds) {
+		cov, ok := reg.Kinds[kind]
+		if !ok || cov.Refs() == 0 {
+			t.Errorf("generated workflow kind %q has no executing coverage in internal/coverage/registry.yaml; "+
+				"add an e2e scenario under e2e/scenarios/, an e2e harness test, or a fleet lane that runs it", kind)
+		}
+	}
+
+	// A registry entry for a kind the generator no longer emits is stale and
+	// masks a rename, so flag it rather than let it linger.
+	for kind := range reg.Kinds {
+		if _, ok := kinds[kind]; !ok {
+			t.Errorf("registry records workflow kind %q that the generator no longer emits; remove or rename its entry", kind)
+		}
+	}
+}
+
+// TestCoverage_RegistryReferencesResolve keeps the registry honest: every
+// referenced scenario file exists under e2e/scenarios/, every referenced harness
+// test exists under e2e/, and every referenced fleet lane is a known lane. A
+// reference that stops resolving fails here, so the registry cannot rot.
+func TestCoverage_RegistryReferencesResolve(t *testing.T) {
+	reg, err := Load()
+	if err != nil {
+		t.Fatalf("loading coverage registry: %v", err)
+	}
+
+	scenariosDir := filepath.Join("..", "..", "e2e", "scenarios")
+	e2eDir := filepath.Join("..", "..", "e2e")
+
+	for _, kind := range sortedRegistryKinds(reg) {
+		cov := reg.Kinds[kind]
+		if strings.TrimSpace(cov.Summary) == "" {
+			t.Errorf("kind %q has no summary", kind)
+		}
+		for _, s := range cov.Scenarios {
+			if _, statErr := os.Stat(filepath.Join(scenariosDir, s)); statErr != nil {
+				t.Errorf("kind %q references scenario %q that does not exist under e2e/scenarios/", kind, s)
+			}
+		}
+		for _, tf := range cov.E2ETests {
+			if _, statErr := os.Stat(filepath.Join(e2eDir, tf)); statErr != nil {
+				t.Errorf("kind %q references e2e test %q that does not exist under e2e/", kind, tf)
+			}
+		}
+		for _, lane := range cov.FleetLanes {
+			if _, ok := KnownFleetLanes[lane]; !ok {
+				t.Errorf("kind %q references fleet lane %q that is not a known lane", kind, lane)
+			}
+		}
+	}
+}
+
+// TestWorkflowKindFromLiteral_TemplatedOnlyKindIsCaptured drives the real
+// derivation function to prove a component-templated path literal with no bare
+// sibling still yields its bare kind, so a future workflow emitted only through a
+// format string cannot slip past the coverage denominator. It also proves no
+// printf verb ever survives into a derived kind.
+func TestWorkflowKindFromLiteral_TemplatedOnlyKindIsCaptured(t *testing.T) {
+	// A synthetic templated-only literal (no bare cascade-synthetic.yaml
+	// sibling) must resolve to its bare kind.
+	kind, ok := workflowKindFromLiteral(".github/workflows/cascade-synthetic-%s.yaml")
+	if !ok {
+		t.Fatalf("templated-only literal derived no kind; the gate would skip such a workflow")
+	}
+	if kind != "cascade-synthetic" {
+		t.Errorf("derived kind = %q, want %q", kind, "cascade-synthetic")
+	}
+	if strings.ContainsRune(kind, '%') {
+		t.Errorf("derived kind %q retains a printf verb; a phantom kind escaped", kind)
+	}
+
+	// Repeated trailing templated segments strip down to the same bare stem.
+	if k, ok := workflowKindFromLiteral(".github/workflows/cascade-synthetic-%d-%s.yaml"); !ok || k != "cascade-synthetic" {
+		t.Errorf("multi-segment literal derived (%q, %v), want (cascade-synthetic, true)", k, ok)
+	}
+
+	// The existing bare-literal path still resolves to its kind.
+	if k, ok := workflowKindFromLiteral(".github/workflows/cascade-rollback.yaml"); !ok || k != "cascade-rollback" {
+		t.Errorf("bare literal derived (%q, %v), want (cascade-rollback, true)", k, ok)
+	}
+
+	// A raw %-bearing name with no strippable trailing segment never becomes a
+	// kind, so no phantom cascade-...-%s kind is ever produced.
+	for _, bad := range []string{
+		".github/workflows/cascade-foo%s.yaml",
+		".github/workflows/cascade-%2s.yaml",
+		".github/workflows/%s.yaml",
+	} {
+		if k, ok := workflowKindFromLiteral(bad); ok {
+			t.Errorf("literal %q produced phantom kind %q, want no kind", bad, k)
+		}
+	}
+}
+
+func sortedRegistryKinds(reg *Registry) []string {
+	keys := make([]string, 0, len(reg.Kinds))
+	for k := range reg.Kinds {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}

internal/coverage/registry.go

@@ -0,0 +1,123 @@
+// Package coverage records, for every workflow kind the generator emits, the
+// executing coverage that exercises it: e2e scenarios, e2e harness tests, and
+// fleet lanes. The map is a positive record of what runs each generated
+// workflow. A companion test derives the full set of emitted workflow kinds
+// from the generator source and fails when an emitted kind has no entry here, so
+// a new generated workflow cannot ship without a scenario or lane that runs it.
+package coverage
+
+import (
+	_ "embed"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// workflowPathLit matches a repo-relative generated-workflow path literal and
+// captures its name segment. The name may carry printf verbs, as in a
+// component-templated cascade-<kind>-%s.yaml path, which workflowKindFromLiteral
+// strips to recover the bare kind. Anchoring the whole literal excludes stub
+// references and doc examples that are not full .github/workflows/<name>.yaml
+// literals.
+var workflowPathLit = regexp.MustCompile(`^\.github/workflows/([a-z0-9%-]+)\.yaml$`)
+
+// templatedSegment matches one trailing "-%<verb>" printf segment on a workflow
+// name, covering verbs such as "-%s", "-%v", and "-%02d".
+var templatedSegment = regexp.MustCompile(`-%[0-9]*[a-z]$`)
+
+// coreWorkflowKinds are the three generated workflows whose names predate the
+// cascade- lane-naming convention. Every other generated workflow is cascade-
+// prefixed, so these plus that prefix classify a name as a generated kind.
+var coreWorkflowKinds = map[string]struct{}{
+	"orchestrate":     {},
+	"promote":         {},
+	"external-update": {},
+}
+
+// isGeneratedKind reports whether a workflow name is a cascade-generated
+// workflow kind rather than a user stub (build.yaml, deploy.yaml) or a doc
+// example.
+func isGeneratedKind(name string) bool {
+	if _, ok := coreWorkflowKinds[name]; ok {
+		return true
+	}
+	return strings.HasPrefix(name, "cascade-")
+}
+
+// workflowKindFromLiteral extracts the cascade-generated workflow kind named by
+// a string literal shaped like ".github/workflows/<name>.yaml". It resolves both
+// bare literals (cascade-rollback.yaml -> cascade-rollback) and component-
+// templated literals whose name carries a trailing printf segment
+// (cascade-rollback-%s.yaml -> cascade-rollback), stripping any repeated
+// trailing "-%<verb>" segments to recover the bare stem. It returns ok only when
+// the stem classifies as a generated kind and no printf verb survives, so a raw
+// %-bearing string never becomes a phantom kind.
+func workflowKindFromLiteral(val string) (string, bool) {
+	m := workflowPathLit.FindStringSubmatch(val)
+	if m == nil {
+		return "", false
+	}
+	name := m[1]
+	for templatedSegment.MatchString(name) {
+		name = templatedSegment.ReplaceAllString(name, "")
+	}
+	if name == "" || strings.ContainsRune(name, '%') {
+		return "", false
+	}
+	if !isGeneratedKind(name) {
+		return "", false
+	}
+	return name, true
+}
+
+//go:embed registry.yaml
+var registryYAML []byte
+
+// Coverage lists the executing references that exercise a single workflow kind.
+// A kind is covered when it names at least one scenario, harness test, or fleet
+// lane that runs the generated workflow.
+type Coverage struct {
+	Summary    string   `yaml:"summary"`
+	Scenarios  []string `yaml:"scenarios,omitempty"`
+	E2ETests   []string `yaml:"e2e_tests,omitempty"`
+	FleetLanes []string `yaml:"fleet_lanes,omitempty"`
+}
+
+// Refs reports the number of executing-coverage references recorded for a kind.
+func (c Coverage) Refs() int {
+	return len(c.Scenarios) + len(c.E2ETests) + len(c.FleetLanes)
+}
+
+// Registry maps each generated workflow kind to its executing coverage.
+type Registry struct {
+	Kinds map[string]Coverage `yaml:"kinds"`
+}
+
+// KnownFleetLanes is the canonical fleet roster, mirroring the single source of
+// truth in .github/workflows/fleet-e2e.yaml (the Select lanes step). A fleet
+// lane reference in the registry must name one of these.
+var KnownFleetLanes = map[string]struct{}{
+	"primary":           {},
+	"artifact-a":        {},
+	"artifact-b":        {},
+	"4env":              {},
+	"3env":              {},
+	"2env":              {},
+	"single-env":        {},
+	"release-only":      {},
+	"no-env":            {},
+	"callbacks":         {},
+	"rollback-dispatch": {},
+	"monorepo":          {},
+}
+
+// Load parses the embedded coverage registry.
+func Load() (*Registry, error) {
+	var r Registry
+	if err := yaml.Unmarshal(registryYAML, &r); err != nil {
+		return nil, fmt.Errorf("parsing coverage registry: %w", err)
+	}
+	return &r, nil
+}

internal/coverage/registry.yaml

@@ -0,0 +1,79 @@
+# Workflow-kind coverage registry.
+#
+# Every workflow kind the generator emits is listed here with the executing
+# coverage that exercises it: the e2e scenarios under e2e/scenarios/, the e2e
+# harness tests under e2e/, and the fleet lanes that run it against real GitHub.
+# The companion test derives the full set of emitted kinds straight from the
+# generator source, so an emitted workflow that is not recorded here fails the
+# build. A referenced scenario, harness test, or fleet lane that does not exist
+# also fails the build, so an entry cannot drift away from what it points at.
+#
+# When you add a generated workflow kind, add its entry here pointing at the
+# scenario or fleet lane that runs it.
+kinds:
+  orchestrate:
+    summary: Trunk orchestration workflow that drafts, versions, builds, and deploys the first environment.
+    scenarios:
+      - 02-two-env-repo.yaml
+      - 03-three-env-repo.yaml
+
+  promote:
+    summary: Environment-to-environment promotion workflow for multi-environment repositories.
+    scenarios:
+      - 04-cascade-promotion.yaml
+      - 50-component-promote-fanout.yaml
+
+  external-update:
+    summary: Primary-repo coordination workflow that adopts deploy updates from external repositories.
+    scenarios:
+      - 21-cross-repo-callback.yaml
+      - multi-repo/external-update-deploys-component.yaml
+
+  cascade-validate:
+    summary: Opt-in pull_request check that validates the manifest before merge.
+    scenarios:
+      - 14-validate-check.yaml
+
+  cascade-merge-queue:
+    summary: Opt-in merge-queue validation lane fired by the merge_group trigger.
+    scenarios:
+      - 15-merge-queue.yaml
+
+  cascade-pr-preview:
+    summary: Opt-in pull_request preview-environment workflow.
+    scenarios:
+      - 16-pr-preview.yaml
+
+  cascade-drift-check:
+    summary: Opt-in pull_request lane that runs verify and fails the check on generated-workflow drift.
+    scenarios:
+      - 28-drift-check.yaml
+
+  cascade-drift-comment:
+    summary: Fork-safe workflow_run companion that posts the drift-check result as a sticky pull-request comment.
+    scenarios:
+      - 28-drift-check.yaml
+
+  cascade-hotfix:
+    summary: Hotfix workflow that applies a targeted fix to a promoted environment and rejoins the trunk line.
+    scenarios:
+      - hotfix/hotfix-multi-env-clean.yaml
+      - hotfix/hotfix-clean-apply.yaml
+      - 53-component-hotfix-rollback-fanout.yaml
+
+  cascade-rollback:
+    summary: Rollback workflow that re-points a promoted environment at a prior deployment.
+    scenarios:
+      - rollback/rollback-deploys-prior.yaml
+      - 32-rollback-repository-dispatch.yaml
+      - 53-component-hotfix-rollback-fanout.yaml
+
+  cascade-reconcile-check:
+    summary: Opt-in pull_request detector for an external governed-pin change to a generated workflow.
+    e2e_tests:
+      - pin_reconcile_test.go
+
+  cascade-reconcile-companion:
+    summary: workflow_run companion that adopts an external governed-pin change back into the manifest.
+    e2e_tests:
+      - pin_reconcile_test.go