Secrets broker: encrypted secrets, admin policy, audit

README.md

@@ -4,8 +4,8 @@ Core of [Opifica](https://github.com/opifica), an open-source CI/CD system built
 where every job runs in a fresh virtual machine.
 
 > **Status:** early development. A pipeline runs end to end: the controller schedules its jobs, a worker checks
-> out the exact commit and runs each job's commands through the agent, and the output is stored. Real VMs,
-> secrets and Git providers come next; see [docs/ROADMAP.md](docs/ROADMAP.md).
+> out the exact commit and runs each job's commands through the agent with the secrets the admin allows, and
+> the output is stored. Real VMs and Git providers come next; see [docs/ROADMAP.md](docs/ROADMAP.md).
 > See the [organisation page](https://github.com/opifica) for the architecture.
 
 ## What lives here
@@ -21,7 +21,8 @@ where every job runs in a fresh virtual machine.
 | `cmd/controller` | Join tokens, worker joining over TLS, the Connect stream, worker revocation | Working |
 | `internal/controller/scheduler` | Runs and jobs, matching jobs to workers, leases, retries, cancellation | Working |
 | `internal/controller/logs` | Job output on the controller's disk | Working |
-| | Secrets, Git providers, admin API | Not started |
+| `internal/controller/secrets` | Encrypted secrets, handed only to jobs the admin's policy allows; every request audited | Working |
+| | Git providers, admin API | Not started |
 | `cmd/agent`, `internal/agent` | Runs inside each VM: checkout, running steps, masking secrets in output, killing process trees on cancel and timeout | Working |
 | `cmd/devworker` | A worker without VMs for trying Opifica on one machine (no isolation) | Working |
 | `cmd/signer` | Signing service for release artifacts | Not started |
@@ -146,6 +147,20 @@ bin/controller run show <run-id>
 bin/controller run logs <run-id>
 ```
 
+### Secrets
+
+The admin decides who gets a secret; a pipeline can only ask for it. The value is read from standard input:
+
+```sh
+printf %s "$TOKEN" | bin/controller secret set DEPLOY_TOKEN -repo 'acme/*' -pool trusted -ref refs/heads/main
+bin/controller secret list                  # policies, never values
+bin/controller secret access DEPLOY_TOKEN   # who asked, and whether they got it
+```
+
+A job gets `DEPLOY_TOKEN` only if it lists it under `secrets:`, the step that needs it names it too, and the run
+matches the policy: repository, pool and ref. Pull requests get no secrets unless the secret is set with
+`-allow-pull-requests`. Values are masked as `***` in job output.
+
 ## Stack
 
 Go · Protobuf + ConnectRPC · PostgreSQL · S3-compatible object storage · CEL · OIDC

cmd/controller/main.go

@@ -3,7 +3,7 @@
 //
 // Implemented so far: the database, the built-in certificate authority, join
 // tokens, Join, the Connect stream, the scheduler (runs, jobs, leases,
-// retries, cancellation) and job logs. Secrets and Git providers come next.
+// retries, cancellation), job logs and secrets. Git providers come next.
 package main
 
 import (
@@ -27,6 +27,7 @@ import (
 	"github.com/opifica/opifica/internal/controller/ca"
 	"github.com/opifica/opifica/internal/controller/logs"
 	"github.com/opifica/opifica/internal/controller/scheduler"
+	"github.com/opifica/opifica/internal/controller/secrets"
 	"github.com/opifica/opifica/internal/controller/store"
 	"github.com/opifica/opifica/internal/controller/workers"
 	"github.com/opifica/opifica/internal/server"
@@ -45,18 +46,26 @@ Commands:
   run show       Show a run and its jobs
   run cancel     Cancel a run
   run logs       Print the output of a run's jobs
+  secret set     Store a secret and who may use it (value from stdin)
+  secret list    List secrets and their policies (never values)
+  secret delete  Delete a secret
+  secret access  Show who asked for a secret, and whether they got it
   version        Print the version
 
 Common options (or environment variables):
   -database-url  PostgreSQL URL (OPIFICA_DATABASE_URL)
-  -data-dir      Directory for the CA key and certificate (OPIFICA_DATA_DIR, default ./data)
+  -data-dir      Directory for the CA, the secrets key and job logs (OPIFICA_DATA_DIR, default ./data)
 `
 
 func main() {
-	os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
+	os.Exit(runWithInput(os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
 }
 
 func run(args []string, stdout, stderr io.Writer) int {
+	return runWithInput(args, strings.NewReader(""), stdout, stderr)
+}
+
+func runWithInput(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
 	if len(args) == 0 {
 		fmt.Fprint(stderr, usage)
 		return 2
@@ -72,6 +81,8 @@ func run(args []string, stdout, stderr io.Writer) int {
 		return workerRevoke(args[2:], stdout, stderr)
 	case args[0] == "run" && len(args) > 1:
 		return runCommand(args[1], args[2:], stdout, stderr)
+	case args[0] == "secret" && len(args) > 1:
+		return secretCommand(args[1], args[2:], stdin, stdout, stderr)
 	case args[0] == "version":
 		fmt.Fprintln(stdout, version.Version)
 		return 0
@@ -142,8 +153,14 @@ func serve(args []string, stderr io.Writer) int {
 		return 1
 	}
 	log.Info("certificate authority ready", "fingerprint", svc.CA.Fingerprint())
+	key, err := secrets.LoadOrCreateKey(c.dataDir)
+	if err != nil {
+		log.Error("loading secrets key", "err", err)
+		return 1
+	}
 	sched := scheduler.New(svc.Store, svc.Sessions(), log.With("component", "scheduler"), scheduler.Config{
-		Logs: &logs.Store{Dir: filepath.Join(c.dataDir, "logs")},
+		Logs:    &logs.Store{Dir: filepath.Join(c.dataDir, "logs")},
+		Secrets: &secrets.Manager{Store: svc.Store, Key: key},
 	})
 	svc.Messages = sched
 	go sched.Run(ctx)

cmd/controller/main_test.go

@@ -15,6 +15,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 	"testing"
 	"time"
 
@@ -176,3 +177,41 @@ func TestRunCommands(t *testing.T) {
 		t.Errorf("after cancel:\n%s", out.String())
 	}
 }
+
+func TestSecretCommands(t *testing.T) {
+	common := []string{"-database-url", pgtest.New(t), "-data-dir", t.TempDir()}
+	var out, errOut bytes.Buffer
+	cmd := func(stdin string, args ...string) int {
+		out.Reset()
+		errOut.Reset()
+		return runWithInput(append(args, common...), strings.NewReader(stdin), &out, &errOut)
+	}
+
+	if code := cmd("very-secret-value\n", "secret", "set", "DEPLOY_TOKEN", "-repo", "acme/*", "-pool", "release", "-ref", "refs/tags/v*"); code != 0 {
+		t.Fatalf("secret set: exit %d: %s", code, errOut.String())
+	}
+	if !strings.Contains(out.String(), "(17 bytes)") {
+		t.Errorf("the trailing newline should not be stored: %s", out.String())
+	}
+	if code := cmd("", "secret", "list"); code != 0 || !regexp.MustCompile(`DEPLOY_TOKEN\s+acme/\*\s+release\s+refs/tags/v\*\s+no`).MatchString(out.String()) {
+		t.Errorf("secret list (exit %d):\n%s", code, out.String())
+	}
+	if strings.Contains(out.String(), "very-secret") {
+		t.Error("secret list shows the value")
+	}
+	if code := cmd("x", "secret", "set", "lowercase", "-repo", "*", "-pool", "trusted"); code == 0 {
+		t.Error("accepted an invalid name")
+	}
+	if code := cmd("x", "secret", "set", "NO_POOL", "-repo", "*"); code == 0 {
+		t.Error("accepted a secret without a pool")
+	}
+	if code := cmd("", "secret", "access", "DEPLOY_TOKEN"); code != 0 || !strings.Contains(out.String(), "RESULT") {
+		t.Errorf("secret access (exit %d): %s", code, errOut.String())
+	}
+	if code := cmd("", "secret", "delete", "DEPLOY_TOKEN"); code != 0 {
+		t.Errorf("secret delete: exit %d: %s", code, errOut.String())
+	}
+	if code := cmd("", "secret", "delete", "DEPLOY_TOKEN"); code == 0 {
+		t.Error("deleting a missing secret succeeded")
+	}
+}

cmd/controller/secrets.go

@@ -0,0 +1,194 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/opifica/opifica/internal/controller/secrets"
+	"github.com/opifica/opifica/internal/controller/store/db"
+)
+
+type listFlag []string
+
+func (l *listFlag) String() string     { return strings.Join(*l, ",") }
+func (l *listFlag) Set(v string) error { *l = append(*l, v); return nil }
+
+func secretCommand(sub string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	switch sub {
+	case "set":
+		return secretSet(args, stdin, stdout, stderr)
+	case "list":
+		return secretList(args, stdout, stderr)
+	case "delete":
+		return secretDelete(args, stdout, stderr)
+	case "access":
+		return secretAccess(args, stdout, stderr)
+	}
+	fmt.Fprintf(stderr, "controller: unknown command \"secret %s\"\n\n%s", sub, usage)
+	return 2
+}
+
+func withSecrets(c common, stderr io.Writer, fn func(ctx context.Context, m *secrets.Manager) error) int {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	svc, closeFn, err := c.open(ctx)
+	if err != nil {
+		fmt.Fprintf(stderr, "controller: %v\n", err)
+		return 1
+	}
+	defer closeFn()
+	key, err := secrets.LoadOrCreateKey(c.dataDir)
+	if err != nil {
+		fmt.Fprintf(stderr, "controller: loading secrets key: %v\n", err)
+		return 1
+	}
+	if err := fn(ctx, &secrets.Manager{Store: svc.Store, Key: key}); err != nil {
+		fmt.Fprintf(stderr, "controller: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
+func secretSet(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("secret set", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	var c common
+	c.register(fs)
+	var repos, pools, refs listFlag
+	fs.Var(&repos, "repo", "repository glob that may use it, e.g. acme/* or * (repeatable, required)")
+	fs.Var(&pools, "pool", "pool that may use it (repeatable, required)")
+	fs.Var(&refs, "ref", "ref glob that may use it, e.g. refs/heads/main or refs/tags/v* (repeatable; default any)")
+	allowPR := fs.Bool("allow-pull-requests", false, "let pull requests use it (they run unreviewed code)")
+	desc := fs.String("description", "", "note shown in secret list")
+	fs.Usage = func() {
+		fmt.Fprintln(stderr, "Usage: controller secret set [options] NAME < value\n\nThe value is read from standard input, never from the command line.\n\nOptions:")
+		fs.PrintDefaults()
+	}
+	pos, err := parseWithArgs(fs, args)
+	if err != nil {
+		return 2
+	}
+	if len(pos) != 1 {
+		fs.Usage()
+		return 2
+	}
+	if f, ok := stdin.(*os.File); ok {
+		if info, err := f.Stat(); err == nil && info.Mode()&os.ModeCharDevice != 0 {
+			fmt.Fprintln(stderr, "Enter the value, then press Ctrl-D:")
+		}
+	}
+	value, err := io.ReadAll(io.LimitReader(stdin, secrets.MaxValueSize+1))
+	if err != nil {
+		fmt.Fprintf(stderr, "controller: reading value: %v\n", err)
+		return 1
+	}
+	// "echo value | ..." adds a newline nobody means to store.
+	value = bytes.TrimSuffix(bytes.TrimSuffix(value, []byte("\n")), []byte("\r"))
+
+	return withSecrets(c, stderr, func(ctx context.Context, m *secrets.Manager) error {
+		if err := m.Set(ctx, pos[0], value, secrets.Policy{
+			Repos: repos, Pools: pools, Refs: refs, AllowPullRequests: *allowPR, Description: *desc,
+		}); err != nil {
+			return err
+		}
+		fmt.Fprintf(stdout, "Secret %s saved (%d bytes).\n", pos[0], len(value))
+		return nil
+	})
+}
+
+func secretList(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("secret list", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	var c common
+	c.register(fs)
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+	return withSecrets(c, stderr, func(ctx context.Context, m *secrets.Manager) error {
+		list, err := m.Store.ListSecrets(ctx)
+		if err != nil {
+			return err
+		}
+		tw := tabwriter.NewWriter(stdout, 0, 4, 2, ' ', 0)
+		fmt.Fprintln(tw, "NAME\tREPOS\tPOOLS\tREFS\tPULL REQUESTS\tUPDATED\tDESCRIPTION")
+		for _, s := range list {
+			refs, pr := "any", "no"
+			if len(s.Refs) > 0 {
+				refs = strings.Join(s.Refs, ",")
+			}
+			if s.AllowPullRequests {
+				pr = "yes"
+			}
+			fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", s.Name, strings.Join(s.Repos, ","), strings.Join(s.Pools, ","),
+				refs, pr, s.UpdatedAt.Local().Format(time.DateTime), s.Description)
+		}
+		return tw.Flush()
+	})
+}
+
+func secretDelete(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("secret delete", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	var c common
+	c.register(fs)
+	pos, err := parseWithArgs(fs, args)
+	if err != nil || len(pos) != 1 {
+		fmt.Fprintln(stderr, "Usage: controller secret delete [options] NAME")
+		return 2
+	}
+	return withSecrets(c, stderr, func(ctx context.Context, m *secrets.Manager) error {
+		n, err := m.Store.DeleteSecret(ctx, pos[0])
+		if err != nil {
+			return err
+		}
+		if n == 0 {
+			return fmt.Errorf("no secret named %s", pos[0])
+		}
+		fmt.Fprintf(stdout, "Secret %s deleted.\n", pos[0])
+		return nil
+	})
+}
+
+func secretAccess(args []string, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("secret access", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	var c common
+	c.register(fs)
+	limit := fs.Int("n", 50, "number of entries")
+	pos, err := parseWithArgs(fs, args)
+	if err != nil || len(pos) != 1 {
+		fmt.Fprintln(stderr, "Usage: controller secret access [options] NAME")
+		return 2
+	}
+	return withSecrets(c, stderr, func(ctx context.Context, m *secrets.Manager) error {
+		list, err := m.Store.ListSecretAccesses(ctx, db.ListSecretAccessesParams{SecretName: pos[0], Limit: int32(*limit)})
+		if err != nil {
+			return err
+		}
+		tw := tabwriter.NewWriter(stdout, 0, 4, 2, ' ', 0)
+		fmt.Fprintln(tw, "TIME\tRESULT\tRUN\tJOB\tWORKER\tREASON")
+		for _, a := range list {
+			result := "denied"
+			if a.Granted {
+				result = "granted"
+			}
+			fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", a.At.Local().Format(time.DateTime), result,
+				shortID(a.RunID), shortID(a.JobID), shortID(a.WorkerID), a.Reason)
+		}
+		return tw.Flush()
+	})
+}
+
+func shortID[T fmt.Stringer](id *T) string {
+	if id == nil {
+		return "-"
+	}
+	return (*id).String()[:8]
+}

docs/ROADMAP.md

@@ -37,7 +37,9 @@ result are stored. Tested in ordinary CI, using the fake hypervisor with a real
   safe with several controllers (`FOR UPDATE SKIP LOCKED`)
 - [x] **Runs:** create a run from a pipeline file and an event; track job and run state; cancel
 - [x] **Logs:** receive log chunks, drop duplicates after reconnects, store them on disk (`controller run logs`)
-- [ ] **Secrets:** encrypted store; requests checked against the job token and policy
+- [x] **Secrets:** encrypted store (AES-256-GCM); requests checked against the worker holding the job, the job
+  token, the job's `secrets:` list and the admin's policy (repositories, pools, refs, pull requests); every
+  request audited (`controller secret set|list|delete|access`)
 - [ ] **Admin config:** static file for pools, routing (event → pool) and secret policy
 - [x] **TLS** on the worker API, with worker client certificates accepted
 - [x] **mTLS required** for `Connect`; the certificate must be the worker's current one
@@ -94,6 +96,7 @@ Can run alongside M2 and M3.
 ## M5 · Supply-chain security
 
 - [ ] Signing service: release artifacts signed with keys in an HSM or KMS; keys never reach a VM
+- [ ] Secrets key in a KMS or HSM instead of a file; key rotation
 - [ ] Signed provenance for every job (SLSA)
 - [ ] Optional rebuild-and-compare for releases on two different workers
 - [ ] Cache rules: untrusted jobs can never write caches that trusted or release jobs read