feat: build supervisor-proposed variations side by side and continue the one chosen (#10)

cmd/envctl/limits_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"strings"
 	"testing"
 	"time"
@@ -70,3 +71,30 @@ func TestRunShowReportsEffectiveNodeAgentModels(t *testing.T) {
 		t.Fatalf("expected code to inherit the run-wide model:\n%s", out.String())
 	}
 }
+
+func TestRunShowJSONCarriesTheSupervisorsProposedVariations(t *testing.T) {
+	c, err := workflow.Parse([]byte("version: 2\nproject: demo\nrepositories: [{id: app, url: /source}]\nworkflow: {template: feature}\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := workflow.NewRun("demo", "ship it", "dev", c, time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+	r.Current().Checkpoints["design"] = workflow.Checkpoint{ID: "cp_design", Node: "design", Result: workflow.Result{
+		Review: workflow.Review{Accepted: true, Summary: "sound", Variations: []workflow.Variation{{Name: "polling", Rationale: "no webhooks"}, {Name: "streaming", Rationale: "high volume"}}},
+	}}
+	cmd := &cobra.Command{}
+	var out bytes.Buffer
+	cmd.SetOut(&out)
+	if err = printRun(cmd, &globals{jsonOut: true}, r); err != nil {
+		t.Fatal(err)
+	}
+	var decoded workflow.Run
+	if err = json.Unmarshal(out.Bytes(), &decoded); err != nil {
+		t.Fatal(err)
+	}
+	if v := decoded.Current().Checkpoints["design"].Result.Review.Variations; len(v) != 2 || v[0].Name != "polling" {
+		t.Fatalf("variations lost from run show --json: %+v", v)
+	}
+}

cmd/envctl/variations.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/spf13/cobra"
+
+	"github.com/sam-bretz/envctl/internal/daemon"
+	"github.com/sam-bretz/envctl/internal/review"
+	"github.com/sam-bretz/envctl/internal/workflow"
+)
+
+func variationsCmd(g *globals) *cobra.Command {
+	return &cobra.Command{Use: "variations <run>", Short: "Compare a run's variations side by side", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error {
+		client, err := connect(cmd.Context(), g)
+		if err != nil {
+			return err
+		}
+		run, err := client.Get(cmd.Context(), args[0])
+		if err != nil {
+			return err
+		}
+		compared, err := review.CompareVariations(cmd.Context(), client, run)
+		if err != nil {
+			return err
+		}
+		if g.jsonOut {
+			return json.NewEncoder(cmd.OutOrStdout()).Encode(compared)
+		}
+		_, err = fmt.Fprint(cmd.OutOrStdout(), review.VariationsText(compared))
+		return err
+	}}
+}
+
+func chooseCmd(g *globals) *cobra.Command {
+	var variation string
+	c := &cobra.Command{Use: "choose <run>", Short: "Continue one variation to approval and retire the others", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error {
+		client, err := connect(cmd.Context(), g)
+		if err != nil {
+			return err
+		}
+		run, err := client.Get(cmd.Context(), args[0])
+		if err != nil {
+			return err
+		}
+		target, err := run.FindVariation(variation)
+		if err != nil {
+			return err
+		}
+		result, err := client.Action(cmd.Context(), run.ID, daemon.ActionRequest{
+			OperationID: workflow.ID("op"), ExpectedVersion: run.Version, Revision: target.ID, Action: "choose",
+		})
+		if err != nil {
+			return err
+		}
+		return printRun(cmd, g, result)
+	}}
+	c.Flags().StringVar(&variation, "variation", "", "variation name or revision ID to continue")
+	_ = c.MarkFlagRequired("variation")
+	return c
+}
+
+// printVariationStatus says a comparison is running or waiting, and how to act
+// on it, without printing the whole comparison into run show.
+func printVariationStatus(out io.Writer, r *workflow.Run) {
+	groups := r.VariationGroups()
+	if len(groups) == 0 {
+		return
+	}
+	summaries := r.CompareVariations(groups[len(groups)-1])
+	fmt.Fprintln(out, "  variations:")
+	waiting := false
+	for _, s := range summaries {
+		fmt.Fprintf(out, "    %-16s %s  %s\n", s.Name, s.Status, s.Revision)
+		waiting = waiting || s.Status == workflow.VariationReady || s.Status == workflow.VariationBuilding
+	}
+	if waiting {
+		fmt.Fprintf(out, "    compare with: envctl run variations %s\n    continue one: envctl run choose %s --variation <name>\n", r.ID, r.ID)
+	}
+}

cmd/envctl/variations_test.go

@@ -0,0 +1,126 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/sam-bretz/envctl/internal/review"
+	"github.com/sam-bretz/envctl/internal/workflow"
+)
+
+type fakeDiffer struct{ unavailable map[string]bool }
+
+func (f fakeDiffer) Diff(_ context.Context, _ string, req review.Request) (review.Comparison, error) {
+	if f.unavailable[req.Revision] {
+		return review.Comparison{}, errors.New("source bundle missing")
+	}
+	return review.Comparison{Repositories: []review.RepositoryDiff{{Patch: "diff --git a/x b/x\n--- a/x\n+++ b/x\n+one\n+two\n-three\n"}}}, nil
+}
+
+func comparedRun(t *testing.T) *workflow.Run {
+	t.Helper()
+	c, err := workflow.Parse([]byte("version: 2\nproject: demo\nrepositories: [{id: app, url: /source}]\nworkflow: {template: feature, nodes: {design: {variations: 3}}}\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := workflow.NewRun("demo", "ship it", "dev", c, time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, node := range []string{"task", "plan", "design"} {
+		r.Current().Checkpoints[node] = workflow.Checkpoint{ID: "cp_" + node, Node: node, Result: workflow.Result{Review: workflow.Review{Summary: "sound"}}}
+	}
+	if err = r.Branch(r.CurrentRevision, "design", []workflow.Variation{{Name: "polling", Rationale: "no webhooks upstream"}, {Name: "streaming", Rationale: "at high volume"}}, time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	for i := range r.Revisions {
+		v := &r.Revisions[i]
+		for _, node := range []string{"design", "code", "qa"} {
+			v.Checkpoints[node] = workflow.Checkpoint{ID: "cp_" + node + v.ID, Node: node, Result: workflow.Result{
+				Summary: v.Variant.Name + " " + node, Commits: map[string]string{"app": strings.Repeat("a", 40)},
+			}}
+		}
+		v.Checkpoints["qa"] = workflow.Checkpoint{ID: "cp_qa" + v.ID, Node: "qa", Result: workflow.Result{
+			Summary: v.Variant.Name + " qa", Commits: map[string]string{"app": strings.Repeat("b", 40)},
+			Checks: []workflow.CheckResult{{Name: "unit", Passed: v.Variant.Name != "streaming", ExitCode: 1}},
+		}}
+		v.Attempts = []workflow.Attempt{{ID: "a_" + v.ID, Node: "qa", State: "checkpointed", Usage: &workflow.Usage{Input: int64(1000 * (i + 1))}}}
+	}
+	return r
+}
+
+func TestRunVariationsShowsEachCandidateSideBySide(t *testing.T) {
+	r := comparedRun(t)
+	streaming, err := r.FindVariation("streaming")
+	if err != nil {
+		t.Fatal(err)
+	}
+	compared, err := review.CompareVariations(context.Background(), fakeDiffer{unavailable: map[string]bool{streaming.ID: true}}, r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(compared) != 3 {
+		t.Fatalf("want 3 candidates, got %d", len(compared))
+	}
+	text := review.VariationsText(compared)
+	for _, want := range []string{
+		"as proposed  [ready to choose]",
+		"polling  [ready to choose]", "why: no webhooks upstream",
+		"code: polling code",
+		// The change is sized from retained evidence against the last stage
+		// with commits, and a missing bundle is reported rather than hidden.
+		"change: 1 files, +2 -1", "change: unavailable (source bundle missing)",
+		"commit app: bbbbbbbbbbbb",
+		"check qa/unit: passed", "check qa/unit: failed (exit 1)",
+		"tokens: ",
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("missing %q in:\n%s", want, text)
+		}
+	}
+}
+
+func TestAVariationIsFoundByNameOrRevision(t *testing.T) {
+	r := comparedRun(t)
+	byName, err := r.FindVariation("Polling")
+	if err != nil || byName.Variant.Name != "polling" {
+		t.Fatalf("name lookup: %v %+v", err, byName)
+	}
+	if byID, err := r.FindVariation(byName.ID); err != nil || byID.ID != byName.ID {
+		t.Fatalf("revision lookup: %v", err)
+	}
+	if _, err := r.FindVariation("nope"); err == nil {
+		t.Fatal("found a variation that does not exist")
+	}
+}
+
+func TestRunVariationsExplainsARunWithoutAny(t *testing.T) {
+	c, _ := workflow.Parse([]byte("version: 2\nproject: demo\nrepositories: [{id: app, url: /source}]\nworkflow: {template: feature}\n"))
+	r, _ := workflow.NewRun("demo", "x", "dev", c, time.Now())
+	if _, err := review.CompareVariations(context.Background(), fakeDiffer{}, r); err == nil || !strings.Contains(err.Error(), "variations in envctl.yaml") {
+		t.Fatalf("a run without variations was not explained: %v", err)
+	}
+}
+
+func TestRunShowSaysAComparisonIsWaitingAndHowToAct(t *testing.T) {
+	r := comparedRun(t)
+	var out strings.Builder
+	printVariationStatus(&out, r)
+	for _, want := range []string{"variations:", "polling", "ready to choose", "envctl run variations " + r.ID, "envctl run choose " + r.ID + " --variation <name>"} {
+		if !strings.Contains(out.String(), want) {
+			t.Fatalf("missing %q:\n%s", want, out.String())
+		}
+	}
+	polling, _ := r.FindVariation("polling")
+	if err := r.Choose(polling.ID, time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	out.Reset()
+	printVariationStatus(&out, r)
+	if strings.Contains(out.String(), "envctl run choose") {
+		t.Fatalf("still says to choose after a choice was made:\n%s", out.String())
+	}
+}

cmd/envctl/workflow.go

@@ -302,6 +302,7 @@ func runCmd(g *globals) *cobra.Command {
 	for _, kind := range []string{"message", "ask", "rewind", "cancel", "close", "approve", "priority"} {
 		c.AddCommand(runActionCmd(g, kind))
 	}
+	c.AddCommand(variationsCmd(g), chooseCmd(g))
 	plugins := &cobra.Command{Use: "plugin", Short: "Change invocation plugins and reopen Plan in a new revision"}
 	for _, item := range []struct{ name, action string }{{"add", "plugin-attach"}, {"remove", "plugin-remove"}} {
 		command := runActionCmd(g, item.action)
@@ -502,6 +503,7 @@ func printRun(cmd *cobra.Command, g *globals, r *workflow.Run) error {
 	fmt.Fprintf(out, "  VM: %s\n", vm)
 	printAttention(out, r)
 	printQuestions(out, rev)
+	printVariationStatus(out, r)
 	printRuntime(out, "", rev.Runtime)
 	for node, child := range rev.ChildRuntimes {
 		if child != nil && (child.Runtime.PreviewURL != "" || len(child.Runtime.Services) > 0) {

docs/src/content/docs/config-reference.mdx

@@ -121,6 +121,39 @@ Rules for every workflow:
 
 Pick a workflow with `envctl run create --workflow small`, with `tab` in the dashboard's new-run input, or with the `workflow` argument of the MCP `envctl_create` tool. `envctl run workflows` lists them with their stages. A run stores only the workflow it selected, and a rewind with `--config` keeps using that workflow.
 
+## Workflow variations (version 2, experimental)
+
+`workflow.nodes.<id>.variations` lets that stage's supervisor propose alternative approaches it judged credible, recorded alongside its acceptance so a choice between designs is not made silently by one agent.
+
+```yaml
+workflow:
+  template: feature
+  nodes:
+    design:
+      variations: 3
+```
+
+| Field | Type | Default | Notes |
+| --- | --- | --- | --- |
+| `workflow.nodes.<id>.variations` | integer | `0` (off) | `0`, or `2` to `3`: the most alternatives the supervisor may propose. Unset keeps the configuration digest unchanged. |
+
+The supervisor proposes variations only when it accepts the work, each with a name and a rationale, and proposes none when the approach taken is clearly right.
+
+When it does, envctl builds each one. The accepted work continues as **as proposed**, and every alternative gets its own sibling revision, restored from before that stage, with the alternative given to its worker as steering. Each runs that stage and everything after it through QA in its own worktree and VM, so variations cannot touch each other's source or data. **No variation reaches its approved change**: none can be approved or published until you choose.
+
+```bash
+envctl run variations <run>                         # side by side
+envctl run choose <run> --variation streaming       # continue one
+```
+
+`run variations` shows each variation's rationale, stage summaries, the size of its change (from retained evidence, against the last stage with commits), the checks it ran and whether they passed, and its token usage; `--json` for the structured form. In the dashboard, the run summary says a comparison is waiting; `v` opens it and `C` chooses. You can steer or ask a variation while it builds.
+
+Choosing continues that variation to its approved change and makes it the run's current revision. The others become **not chosen**: their VMs are released and their checkpoints stay reviewable, but they are never published.
+
+Every variation counts toward `limits.run_tokens`, and `limits.vms` still bounds how many hold a VM at once, so the rest queue. A variation that has finished and is waiting to be chosen gives up its VM while a sibling is queued for one; comparing needs no VM, and choosing a variation that gave its up provisions a new one and restores its source from its checkpoints. A variation never branches again, so `variations` bounds the work.
+
+How and why it works this way is in the [supervisor-proposed variations design](/envctl/design/supervisor-variations/).
+
 ## Workflow limits (version 2, experimental)
 
 Version 2 workflow configurations set revision-wide limits under `limits`. A workflow node can override the attempt budget for its own assignments under `workflow.nodes.<id>.limits`. See the [workflow runtime design](/envctl/design/workflow-runtime/#child-runtimes-for-parallel-execution).

docs/src/content/docs/design/supervisor-variations.mdx

@@ -0,0 +1,68 @@
+---
+title: Supervisor-proposed variations
+description: How a supervisor's alternative approaches become isolated, comparable, choosable runs.
+---
+
+Status: implemented. Tracks [#10](https://github.com/sam-bretz/envctl/issues/10). This document records how variations work and why, including what building it turned up.
+
+## The problem
+
+When a stage has two or three credible approaches, the worker picks one and the supervisor accepts or corrects it. The alternatives are never built, so "which approach is better?" is a judgement one agent makes silently rather than evidence a person compares.
+
+## Proposals
+
+A stage opts in with `variations: N`. Its supervisor may then propose that many alternatives alongside accepting the work, each with a name and a rationale, through `agent.AssessmentSchemaFor`; every other stage keeps the unchanged contract. Proposals are made **alongside** acceptance, never instead of it, so the attempt state machine is untouched and a stage whose supervisor proposes nothing behaves exactly as before.
+
+Two invariants are tested and hold throughout:
+
+- **Approval binding.** `Result.WorkDigest` clears `Review`, so recording variations never changes what an approval is bound to.
+- **Configuration identity.** `variations: 0` restores an existing configuration's digest exactly.
+
+## The constraint that shapes everything
+
+`Revision.Checkpoints` is a `map[string]Checkpoint` keyed by node: exactly one result per stage. Variations need several competing results for the same stages. Three ways to represent that:
+
+1. **Variation-scoped node IDs** (`code@polling`), expanding the DAG at runtime. Rejected: it mutates a configuration that is frozen into the revision and breaks its digest identity, and node-keyed maps appear throughout the engine.
+2. **Nested checkpoints** (`map[node]map[variation]Checkpoint`). Rejected: it changes every reader of checkpoints for a feature most runs never use.
+3. **Sibling revisions.** Each variation is its own `Revision`, forked from the same parent. **Recommended.**
+
+Sibling revisions fit because isolation is already per revision. A revision has its own checkpoints, attempts, runtime and source pins, and its output branch is derived from its ID (`OutputBranch(repo, rev.ID)`), so two variations cannot write to the same branch. `envctl run diff` already compares checkpoints across revisions, which is most of the side-by-side view. And the run token ceiling already counts usage across all revisions, so variations are bounded by `limits.run_tokens` without new accounting.
+
+What does not fit yet: revisions today form a linear chain. `Rewind` marks the previous revision `superseded`, and `Run.CurrentRevision` names exactly one. Variations need several revisions active at once, with none superseding the others until a person chooses.
+
+## Where variations branch
+
+The issue left open whether variations branch at Design or at Code, and whether the person or the supervisor picks the stage. **The node that proposes them is the branch point, and the person chooses it by where they set `variations`.** The supervisor chooses *which* alternatives; it never chooses where branching happens.
+
+Each variation reruns the proposing node with its name and rationale added as steering, then continues downstream through QA. Branching at Design therefore produces a separate design, implementation and test run per alternative; branching at Code keeps the accepted design and varies only the implementation. Both are legitimate, and the cost difference between them is exactly the choice a person should be making.
+
+## Flow
+
+1. An opted-in stage's supervisor accepts the work and proposes two or three variations.
+2. The coordinator forks one sibling revision per variation from the checkpoint *before* the proposing node. The accepted work continues as its own variation, "as proposed".
+3. Each variation runs to QA in its own worktree and runtime. **None reaches the approved-change gate**: no variation can publish before a person chooses.
+4. The dashboard and `run show --json` show every finished variation side by side: summary, supervisor rationale, diff, the checks run against which commit, and token usage.
+5. A new `choose` action makes one variation current and continues it to approval. The others move to a new terminal state, `not-chosen`, and stay reviewable as history. They are never published.
+
+## Risks
+
+- **Capacity.** Each variation holds a VM. `limits.vms` must bound how many run at once, with the rest queued rather than failing. On a single machine this matters more than tokens.
+- **Publication.** The only guard against a variation opening a pull request is that it never reaches approved-change before `choose`. That guard needs its own test, because a regression would publish work nobody picked.
+- **Cost.** A Design-to-QA pass has measured around 10M tokens, so three variations from Design is roughly 30M. `variations` is capped at 3 for this reason, and the run token ceiling stops the rest.
+
+## What building it turned up
+
+- **A deadlock with the default limits.** `limits.vms` defaults to 2, but even two proposals make three revisions, and a finished variation waiting to be chosen kept its VM. The last sibling could never start. A finished, undecided variation now parks — releases its VM — while a sibling is queued, and `Run.VMCount` does not count a parked variation, since it stays `active` so it is not mistaken for a retired one. Choosing a parked variation queues it for a VM again, the same path a rewound revision resumes by.
+- **Steering the wrong revision.** `Run.Message` wrote to the current revision whatever was addressed. During a comparison that is a different variation, so a message would silently land on another candidate. It now writes to the addressed revision.
+- **The current-revision assumption.** The engine and daemon guarded work with `CurrentRevision` checks in eight places, which would have left every sibling queued forever. They now accept undecided variations. The daemon's guard admits a variation only for `message`, `ask` and `choose`: a `rewind` addressed to a variation would otherwise rewind the current revision out from under the comparison.
+
+## Delivery
+
+| Piece | Where |
+| --- | --- |
+| Opt-in, supervisor contract, durable proposals | `Node.Variations`, `agent.AssessmentSchemaFor`, `Review.Variations` |
+| Branching into sibling revisions | `Run.Branch`, triggered in the mutation that accepts the proposing stage |
+| Holding back approval | `Revision.ReadyNodes` skips `change` stages while undecided; publication refuses as well |
+| Choosing | `Run.Choose`, daemon action `choose`, `envctl run choose`, `C` in the dashboard |
+| Side-by-side comparison | `Run.CompareVariations`, `review.CompareVariations`, `envctl run variations`, `v` in the dashboard |
+| Capacity | parking in `Engine.parkVariation`, `Revision.Parked`, `Run.VMCount` |