From dbec6ccdb62d24b2d126514eaffd35f4c94e481e Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Sat, 4 Jul 2026 15:30:37 -0400 Subject: [PATCH] test(generate): make actionlint normalization test hermetic The actionlint normalization test staged sibling callback stubs into its temp dir but never anchored actionlint's project root there. actionlint resolves ./-prefixed reusable-workflow references relative to the nearest ancestor .git directory, so when the temp tree was not anchored it walked up to an unrelated repository (or found none) and silently skipped local reusable-workflow resolution. The staged stubs were ignored and the generated uses: references went unvalidated, letting the test pass for the wrong reason and mask a generator regression. Anchor the project root with a bare .git marker in the temp tree so resolution is self-contained regardless of where TMPDIR lives, declare the environment workflow_call input the generator wires into every reusable call so resolved references validate cleanly, and add a negative-control test that proves reusable-workflow resolution is genuinely active. Signed-off-by: Joshua Temple --- internal/generate/generator_normalize_test.go | 138 ++++++++++++++---- 1 file changed, 109 insertions(+), 29 deletions(-) diff --git a/internal/generate/generator_normalize_test.go b/internal/generate/generator_normalize_test.go index 3be591b4..1bda2647 100644 --- a/internal/generate/generator_normalize_test.go +++ b/internal/generate/generator_normalize_test.go @@ -4,6 +4,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" "github.com/stablekernel/cascade/internal/config" @@ -53,32 +54,79 @@ func TestNormalizeWorkflowPath(t *testing.T) { } } -func TestNormalizeWorkflowPath_ActionlintClean(t *testing.T) { - // Verify that a bare-path build callback generates an actionlint-clean workflow. - actionlint, err := exec.LookPath("actionlint") - if err != nil { - // Try the known homebrew path. - actionlint = "/opt/homebrew/bin/actionlint" - if _, statErr := os.Stat(actionlint); statErr != nil { - t.Skip("actionlint not found; skipping actionlint integration test") - } +// locateActionlint returns the actionlint binary path, or skips the test when +// the linter is not installed in the runner. +func locateActionlint(t *testing.T) string { + t.Helper() + if p, err := exec.LookPath("actionlint"); err == nil { + return p + } + // Fall back to the known homebrew path. + const brew = "/opt/homebrew/bin/actionlint" + if _, err := os.Stat(brew); err == nil { + return brew } + t.Skip("actionlint not found; skipping actionlint integration test") + return "" +} - // Stage bare-filename stub callback workflows at the normalized location - // (.github/workflows/), which is where the generator discovers - // inputs/outputs from and emits the uses: reference to. +// stageActionlintProject builds a self-contained project tree rooted at a fresh +// temp dir and returns (projectDir, workflowsDir). +// +// It stages the bare-filename callback stubs at the normalized location +// (.github/workflows/), which is where the generator discovers +// inputs/outputs from and emits the uses: reference to. Each stub declares the +// environment workflow_call input that the generator wires into every reusable +// call, so a resolved uses: reference validates cleanly against its callback. +func stageActionlintProject(t *testing.T) (projectDir, workflowsDir string) { + t.Helper() dir := t.TempDir() - wfDir := filepath.Join(dir, ".github", "workflows") - if mkErr := os.MkdirAll(wfDir, 0o755); mkErr != nil { - t.Fatalf("MkdirAll: %v", mkErr) + // Anchor actionlint's project-root detection to this temp tree. actionlint + // resolves ./-prefixed reusable-workflow references relative to the nearest + // ancestor .git directory; without a marker here it would walk up to an + // unrelated repository (or find none) and silently skip local + // reusable-workflow resolution, leaving the staged sibling stubs ignored and + // the uses: references unvalidated. A bare .git directory is enough for the + // project-root heuristic and keeps the tree fully self-contained. + if err := os.MkdirAll(filepath.Join(dir, ".git"), 0o755); err != nil { + t.Fatalf("MkdirAll .git anchor: %v", err) } - stub := []byte("on:\n workflow_call:\n") - if writeErr := os.WriteFile(filepath.Join(wfDir, "build.yaml"), stub, 0o644); writeErr != nil { - t.Fatalf("WriteFile build stub: %v", writeErr) + wfDir := filepath.Join(dir, ".github", "workflows") + if err := os.MkdirAll(wfDir, 0o755); err != nil { + t.Fatalf("MkdirAll workflows: %v", err) } - if writeErr := os.WriteFile(filepath.Join(wfDir, "deploy.yaml"), stub, 0o644); writeErr != nil { - t.Fatalf("WriteFile deploy stub: %v", writeErr) + stub := []byte("on:\n" + + " workflow_call:\n" + + " inputs:\n" + + " environment:\n" + + " type: string\n" + + " required: true\n") + for _, name := range []string{"build.yaml", "deploy.yaml"} { + if err := os.WriteFile(filepath.Join(wfDir, name), stub, 0o644); err != nil { + t.Fatalf("WriteFile %s stub: %v", name, err) + } } + return dir, wfDir +} + +// runActionlint runs actionlint over path with the optional external-linter +// integrations (shellcheck and pyflakes) disabled: these tests govern workflow +// structure and uses: reference validity, not the style of cascade-owned run: +// scripts (which carry their own pre-existing SC2129-style notes). Leaving those +// integrations on the default-enabled setting makes the result depend on whether +// (and which version of) shellcheck/pyflakes happens to be installed in the +// runner, which is a source of cross-environment flakiness. -no-color keeps the +// captured output stable for failure messages. +func runActionlint(t *testing.T, bin, path string) (string, error) { + t.Helper() + out, err := exec.Command(bin, "-shellcheck=", "-pyflakes=", "-no-color", path).CombinedOutput() + return string(out), err +} + +func TestNormalizeWorkflowPath_ActionlintClean(t *testing.T) { + // Verify that a bare-path build callback generates an actionlint-clean workflow. + actionlint := locateActionlint(t) + dir, wfDir := stageActionlintProject(t) cfg := &config.TrunkConfig{ TrunkBranch: "main", @@ -102,16 +150,48 @@ func TestNormalizeWorkflowPath_ActionlintClean(t *testing.T) { t.Fatalf("WriteFile: %v", writeErr) } - // Disable the optional external-linter integrations (shellcheck and - // pyflakes): this test governs workflow structure and uses: reference - // validity, not the style of cascade-owned run: scripts (which carry their - // own pre-existing SC2129-style notes). Leaving these integrations on the - // default-enabled setting makes the result depend on whether (and which - // version of) shellcheck/pyflakes happens to be installed in the runner, - // which is the source of the cross-environment flakiness. -no-color keeps - // the captured output stable for the failure message. - out, runErr := exec.Command(actionlint, "-shellcheck=", "-pyflakes=", "-no-color", path).CombinedOutput() + out, runErr := runActionlint(t, actionlint, path) if runErr != nil { t.Errorf("actionlint found errors in generated workflow:\n%s", out) } } + +// TestNormalizeWorkflowPath_ActionlintHermetic is a negative control that guards +// the hermeticity of the actionlint harness: it proves actionlint actually +// resolves and validates the local reusable-workflow references staged in the +// temp project tree, rather than silently skipping them. actionlint anchors its +// project root by the nearest ancestor .git directory; when the temp tree is not +// anchored it walks up to an unrelated repository (or finds none) and skips local +// reusable-workflow resolution entirely, so the staged sibling stubs are ignored +// and the uses: references go unvalidated. A clean actionlint run over a workflow +// that passes an input the callback does not declare therefore means the harness +// is not resolving siblings, which is the non-hermetic false-signal defect this +// test exists to catch. +func TestNormalizeWorkflowPath_ActionlintHermetic(t *testing.T) { + actionlint := locateActionlint(t) + _, wfDir := stageActionlintProject(t) + + broken := "name: orchestrate\n" + + "on:\n" + + " push:\n" + + " branches: [main]\n" + + "jobs:\n" + + " app:\n" + + " uses: ./.github/workflows/build.yaml\n" + + " with:\n" + + " environment: staging\n" + + " undefined_input: x\n" + path := filepath.Join(wfDir, "orchestrate.yaml") + if writeErr := os.WriteFile(path, []byte(broken), 0o644); writeErr != nil { + t.Fatalf("WriteFile: %v", writeErr) + } + + out, runErr := runActionlint(t, actionlint, path) + if runErr == nil { + t.Fatalf("expected actionlint to reject an undefined reusable-workflow input, but it "+ + "passed: the harness is not resolving sibling workflows (non-hermetic):\n%s", out) + } + if !strings.Contains(out, "undefined_input") { + t.Errorf("actionlint failed for an unexpected reason; want an undefined-input error, got:\n%s", out) + } +}