diff --git a/README.md b/README.md index 1ebc3e9..847b079 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ Core of [Opifica](https://github.com/opifica), an open-source CI/CD system built where every job runs in a fresh virtual machine. > **Status:** early development. A pipeline runs end to end: the controller schedules its jobs, a worker checks -> out the exact commit and runs each job's commands through the agent, and the output is stored. Real VMs, -> secrets and Git providers come next; see [docs/ROADMAP.md](docs/ROADMAP.md). +> out the exact commit and runs each job's commands through the agent with the secrets the admin allows, and +> the output is stored. Real VMs and Git providers come next; see [docs/ROADMAP.md](docs/ROADMAP.md). > See the [organisation page](https://github.com/opifica) for the architecture. ## What lives here @@ -21,7 +21,8 @@ where every job runs in a fresh virtual machine. | `cmd/controller` | Join tokens, worker joining over TLS, the Connect stream, worker revocation | Working | | `internal/controller/scheduler` | Runs and jobs, matching jobs to workers, leases, retries, cancellation | Working | | `internal/controller/logs` | Job output on the controller's disk | Working | -| | Secrets, Git providers, admin API | Not started | +| `internal/controller/secrets` | Encrypted secrets, handed only to jobs the admin's policy allows; every request audited | Working | +| | Git providers, admin API | Not started | | `cmd/agent`, `internal/agent` | Runs inside each VM: checkout, running steps, masking secrets in output, killing process trees on cancel and timeout | Working | | `cmd/devworker` | A worker without VMs for trying Opifica on one machine (no isolation) | Working | | `cmd/signer` | Signing service for release artifacts | Not started | @@ -146,6 +147,20 @@ bin/controller run show bin/controller run logs ``` +### Secrets + +The admin decides who gets a secret; a pipeline can only ask for it. The value is read from standard input: + +```sh +printf %s "$TOKEN" | bin/controller secret set DEPLOY_TOKEN -repo 'acme/*' -pool trusted -ref refs/heads/main +bin/controller secret list # policies, never values +bin/controller secret access DEPLOY_TOKEN # who asked, and whether they got it +``` + +A job gets `DEPLOY_TOKEN` only if it lists it under `secrets:`, the step that needs it names it too, and the run +matches the policy: repository, pool and ref. Pull requests get no secrets unless the secret is set with +`-allow-pull-requests`. Values are masked as `***` in job output. + ## Stack Go · Protobuf + ConnectRPC · PostgreSQL · S3-compatible object storage · CEL · OIDC diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 752e676..5536454 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -3,7 +3,7 @@ // // Implemented so far: the database, the built-in certificate authority, join // tokens, Join, the Connect stream, the scheduler (runs, jobs, leases, -// retries, cancellation) and job logs. Secrets and Git providers come next. +// retries, cancellation), job logs and secrets. Git providers come next. package main import ( @@ -27,6 +27,7 @@ import ( "github.com/opifica/opifica/internal/controller/ca" "github.com/opifica/opifica/internal/controller/logs" "github.com/opifica/opifica/internal/controller/scheduler" + "github.com/opifica/opifica/internal/controller/secrets" "github.com/opifica/opifica/internal/controller/store" "github.com/opifica/opifica/internal/controller/workers" "github.com/opifica/opifica/internal/server" @@ -45,18 +46,26 @@ Commands: run show Show a run and its jobs run cancel Cancel a run run logs Print the output of a run's jobs + secret set Store a secret and who may use it (value from stdin) + secret list List secrets and their policies (never values) + secret delete Delete a secret + secret access Show who asked for a secret, and whether they got it version Print the version Common options (or environment variables): -database-url PostgreSQL URL (OPIFICA_DATABASE_URL) - -data-dir Directory for the CA key and certificate (OPIFICA_DATA_DIR, default ./data) + -data-dir Directory for the CA, the secrets key and job logs (OPIFICA_DATA_DIR, default ./data) ` func main() { - os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) + os.Exit(runWithInput(os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) } func run(args []string, stdout, stderr io.Writer) int { + return runWithInput(args, strings.NewReader(""), stdout, stderr) +} + +func runWithInput(args []string, stdin io.Reader, stdout, stderr io.Writer) int { if len(args) == 0 { fmt.Fprint(stderr, usage) return 2 @@ -72,6 +81,8 @@ func run(args []string, stdout, stderr io.Writer) int { return workerRevoke(args[2:], stdout, stderr) case args[0] == "run" && len(args) > 1: return runCommand(args[1], args[2:], stdout, stderr) + case args[0] == "secret" && len(args) > 1: + return secretCommand(args[1], args[2:], stdin, stdout, stderr) case args[0] == "version": fmt.Fprintln(stdout, version.Version) return 0 @@ -142,8 +153,14 @@ func serve(args []string, stderr io.Writer) int { return 1 } log.Info("certificate authority ready", "fingerprint", svc.CA.Fingerprint()) + key, err := secrets.LoadOrCreateKey(c.dataDir) + if err != nil { + log.Error("loading secrets key", "err", err) + return 1 + } sched := scheduler.New(svc.Store, svc.Sessions(), log.With("component", "scheduler"), scheduler.Config{ - Logs: &logs.Store{Dir: filepath.Join(c.dataDir, "logs")}, + Logs: &logs.Store{Dir: filepath.Join(c.dataDir, "logs")}, + Secrets: &secrets.Manager{Store: svc.Store, Key: key}, }) svc.Messages = sched go sched.Run(ctx) diff --git a/cmd/controller/main_test.go b/cmd/controller/main_test.go index 4fee3ce..b90ce61 100644 --- a/cmd/controller/main_test.go +++ b/cmd/controller/main_test.go @@ -15,6 +15,7 @@ import ( "os" "path/filepath" "regexp" + "strings" "testing" "time" @@ -176,3 +177,41 @@ func TestRunCommands(t *testing.T) { t.Errorf("after cancel:\n%s", out.String()) } } + +func TestSecretCommands(t *testing.T) { + common := []string{"-database-url", pgtest.New(t), "-data-dir", t.TempDir()} + var out, errOut bytes.Buffer + cmd := func(stdin string, args ...string) int { + out.Reset() + errOut.Reset() + return runWithInput(append(args, common...), strings.NewReader(stdin), &out, &errOut) + } + + if code := cmd("very-secret-value\n", "secret", "set", "DEPLOY_TOKEN", "-repo", "acme/*", "-pool", "release", "-ref", "refs/tags/v*"); code != 0 { + t.Fatalf("secret set: exit %d: %s", code, errOut.String()) + } + if !strings.Contains(out.String(), "(17 bytes)") { + t.Errorf("the trailing newline should not be stored: %s", out.String()) + } + if code := cmd("", "secret", "list"); code != 0 || !regexp.MustCompile(`DEPLOY_TOKEN\s+acme/\*\s+release\s+refs/tags/v\*\s+no`).MatchString(out.String()) { + t.Errorf("secret list (exit %d):\n%s", code, out.String()) + } + if strings.Contains(out.String(), "very-secret") { + t.Error("secret list shows the value") + } + if code := cmd("x", "secret", "set", "lowercase", "-repo", "*", "-pool", "trusted"); code == 0 { + t.Error("accepted an invalid name") + } + if code := cmd("x", "secret", "set", "NO_POOL", "-repo", "*"); code == 0 { + t.Error("accepted a secret without a pool") + } + if code := cmd("", "secret", "access", "DEPLOY_TOKEN"); code != 0 || !strings.Contains(out.String(), "RESULT") { + t.Errorf("secret access (exit %d): %s", code, errOut.String()) + } + if code := cmd("", "secret", "delete", "DEPLOY_TOKEN"); code != 0 { + t.Errorf("secret delete: exit %d: %s", code, errOut.String()) + } + if code := cmd("", "secret", "delete", "DEPLOY_TOKEN"); code == 0 { + t.Error("deleting a missing secret succeeded") + } +} diff --git a/cmd/controller/secrets.go b/cmd/controller/secrets.go new file mode 100644 index 0000000..1d9a562 --- /dev/null +++ b/cmd/controller/secrets.go @@ -0,0 +1,194 @@ +package main + +import ( + "bytes" + "context" + "flag" + "fmt" + "io" + "os" + "strings" + "text/tabwriter" + "time" + + "github.com/opifica/opifica/internal/controller/secrets" + "github.com/opifica/opifica/internal/controller/store/db" +) + +type listFlag []string + +func (l *listFlag) String() string { return strings.Join(*l, ",") } +func (l *listFlag) Set(v string) error { *l = append(*l, v); return nil } + +func secretCommand(sub string, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + switch sub { + case "set": + return secretSet(args, stdin, stdout, stderr) + case "list": + return secretList(args, stdout, stderr) + case "delete": + return secretDelete(args, stdout, stderr) + case "access": + return secretAccess(args, stdout, stderr) + } + fmt.Fprintf(stderr, "controller: unknown command \"secret %s\"\n\n%s", sub, usage) + return 2 +} + +func withSecrets(c common, stderr io.Writer, fn func(ctx context.Context, m *secrets.Manager) error) int { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + svc, closeFn, err := c.open(ctx) + if err != nil { + fmt.Fprintf(stderr, "controller: %v\n", err) + return 1 + } + defer closeFn() + key, err := secrets.LoadOrCreateKey(c.dataDir) + if err != nil { + fmt.Fprintf(stderr, "controller: loading secrets key: %v\n", err) + return 1 + } + if err := fn(ctx, &secrets.Manager{Store: svc.Store, Key: key}); err != nil { + fmt.Fprintf(stderr, "controller: %v\n", err) + return 1 + } + return 0 +} + +func secretSet(args []string, stdin io.Reader, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("secret set", flag.ContinueOnError) + fs.SetOutput(stderr) + var c common + c.register(fs) + var repos, pools, refs listFlag + fs.Var(&repos, "repo", "repository glob that may use it, e.g. acme/* or * (repeatable, required)") + fs.Var(&pools, "pool", "pool that may use it (repeatable, required)") + fs.Var(&refs, "ref", "ref glob that may use it, e.g. refs/heads/main or refs/tags/v* (repeatable; default any)") + allowPR := fs.Bool("allow-pull-requests", false, "let pull requests use it (they run unreviewed code)") + desc := fs.String("description", "", "note shown in secret list") + fs.Usage = func() { + fmt.Fprintln(stderr, "Usage: controller secret set [options] NAME < value\n\nThe value is read from standard input, never from the command line.\n\nOptions:") + fs.PrintDefaults() + } + pos, err := parseWithArgs(fs, args) + if err != nil { + return 2 + } + if len(pos) != 1 { + fs.Usage() + return 2 + } + if f, ok := stdin.(*os.File); ok { + if info, err := f.Stat(); err == nil && info.Mode()&os.ModeCharDevice != 0 { + fmt.Fprintln(stderr, "Enter the value, then press Ctrl-D:") + } + } + value, err := io.ReadAll(io.LimitReader(stdin, secrets.MaxValueSize+1)) + if err != nil { + fmt.Fprintf(stderr, "controller: reading value: %v\n", err) + return 1 + } + // "echo value | ..." adds a newline nobody means to store. + value = bytes.TrimSuffix(bytes.TrimSuffix(value, []byte("\n")), []byte("\r")) + + return withSecrets(c, stderr, func(ctx context.Context, m *secrets.Manager) error { + if err := m.Set(ctx, pos[0], value, secrets.Policy{ + Repos: repos, Pools: pools, Refs: refs, AllowPullRequests: *allowPR, Description: *desc, + }); err != nil { + return err + } + fmt.Fprintf(stdout, "Secret %s saved (%d bytes).\n", pos[0], len(value)) + return nil + }) +} + +func secretList(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("secret list", flag.ContinueOnError) + fs.SetOutput(stderr) + var c common + c.register(fs) + if err := fs.Parse(args); err != nil { + return 2 + } + return withSecrets(c, stderr, func(ctx context.Context, m *secrets.Manager) error { + list, err := m.Store.ListSecrets(ctx) + if err != nil { + return err + } + tw := tabwriter.NewWriter(stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "NAME\tREPOS\tPOOLS\tREFS\tPULL REQUESTS\tUPDATED\tDESCRIPTION") + for _, s := range list { + refs, pr := "any", "no" + if len(s.Refs) > 0 { + refs = strings.Join(s.Refs, ",") + } + if s.AllowPullRequests { + pr = "yes" + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", s.Name, strings.Join(s.Repos, ","), strings.Join(s.Pools, ","), + refs, pr, s.UpdatedAt.Local().Format(time.DateTime), s.Description) + } + return tw.Flush() + }) +} + +func secretDelete(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("secret delete", flag.ContinueOnError) + fs.SetOutput(stderr) + var c common + c.register(fs) + pos, err := parseWithArgs(fs, args) + if err != nil || len(pos) != 1 { + fmt.Fprintln(stderr, "Usage: controller secret delete [options] NAME") + return 2 + } + return withSecrets(c, stderr, func(ctx context.Context, m *secrets.Manager) error { + n, err := m.Store.DeleteSecret(ctx, pos[0]) + if err != nil { + return err + } + if n == 0 { + return fmt.Errorf("no secret named %s", pos[0]) + } + fmt.Fprintf(stdout, "Secret %s deleted.\n", pos[0]) + return nil + }) +} + +func secretAccess(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("secret access", flag.ContinueOnError) + fs.SetOutput(stderr) + var c common + c.register(fs) + limit := fs.Int("n", 50, "number of entries") + pos, err := parseWithArgs(fs, args) + if err != nil || len(pos) != 1 { + fmt.Fprintln(stderr, "Usage: controller secret access [options] NAME") + return 2 + } + return withSecrets(c, stderr, func(ctx context.Context, m *secrets.Manager) error { + list, err := m.Store.ListSecretAccesses(ctx, db.ListSecretAccessesParams{SecretName: pos[0], Limit: int32(*limit)}) + if err != nil { + return err + } + tw := tabwriter.NewWriter(stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "TIME\tRESULT\tRUN\tJOB\tWORKER\tREASON") + for _, a := range list { + result := "denied" + if a.Granted { + result = "granted" + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", a.At.Local().Format(time.DateTime), result, + shortID(a.RunID), shortID(a.JobID), shortID(a.WorkerID), a.Reason) + } + return tw.Flush() + }) +} + +func shortID[T fmt.Stringer](id *T) string { + if id == nil { + return "-" + } + return (*id).String()[:8] +} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 6d10433..29183ec 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -37,7 +37,9 @@ result are stored. Tested in ordinary CI, using the fake hypervisor with a real safe with several controllers (`FOR UPDATE SKIP LOCKED`) - [x] **Runs:** create a run from a pipeline file and an event; track job and run state; cancel - [x] **Logs:** receive log chunks, drop duplicates after reconnects, store them on disk (`controller run logs`) -- [ ] **Secrets:** encrypted store; requests checked against the job token and policy +- [x] **Secrets:** encrypted store (AES-256-GCM); requests checked against the worker holding the job, the job + token, the job's `secrets:` list and the admin's policy (repositories, pools, refs, pull requests); every + request audited (`controller secret set|list|delete|access`) - [ ] **Admin config:** static file for pools, routing (event → pool) and secret policy - [x] **TLS** on the worker API, with worker client certificates accepted - [x] **mTLS required** for `Connect`; the certificate must be the worker's current one @@ -94,6 +96,7 @@ Can run alongside M2 and M3. ## M5 · Supply-chain security - [ ] Signing service: release artifacts signed with keys in an HSM or KMS; keys never reach a VM +- [ ] Secrets key in a KMS or HSM instead of a file; key rotation - [ ] Signed provenance for every job (SLSA) - [ ] Optional rebuild-and-compare for releases on two different workers - [ ] Cache rules: untrusted jobs can never write caches that trusted or release jobs read diff --git a/internal/controller/scheduler/scheduler.go b/internal/controller/scheduler/scheduler.go index 45e6a5f..2b7b523 100644 --- a/internal/controller/scheduler/scheduler.go +++ b/internal/controller/scheduler/scheduler.go @@ -51,6 +51,13 @@ type Config struct { Interval time.Duration // Logs stores job output. Nil drops it. Logs *logs.Store + // Secrets answers secret requests from jobs. Nil denies them all. + Secrets SecretBroker +} + +// SecretBroker decides secret requests; see secrets.Manager. +type SecretBroker interface { + Request(ctx context.Context, worker uuid.UUID, req *opificav1.SecretRequest) (value []byte, deniedReason string, err error) } // Scheduler implements workers.MessageHandler. @@ -482,12 +489,8 @@ func (s *Scheduler) HandleWorkerMessage(ctx context.Context, worker uuid.UUID, m case *opificav1.ConnectRequest_JobUpdate: s.jobUpdate(ctx, worker, m.JobUpdate) case *opificav1.ConnectRequest_SecretRequest: - // Until the secrets broker exists, answer so the job doesn't hang. return s.sessions.Send(ctx, worker, &opificav1.ConnectResponse{Message: &opificav1.ConnectResponse_SecretResponse{ - SecretResponse: &opificav1.SecretResponse{ - RequestId: m.SecretRequest.GetRequestId(), - Result: &opificav1.SecretResponse_DeniedReason{DeniedReason: "secrets are not implemented yet"}, - }, + SecretResponse: s.secret(ctx, worker, m.SecretRequest), }}) case *opificav1.ConnectRequest_LogChunk: s.logChunk(ctx, worker, m.LogChunk) @@ -495,6 +498,31 @@ func (s *Scheduler) HandleWorkerMessage(ctx context.Context, worker uuid.UUID, m return nil } +// secret answers a secret request. The job always gets an answer, so it +// never hangs waiting. +func (s *Scheduler) secret(ctx context.Context, worker uuid.UUID, req *opificav1.SecretRequest) *opificav1.SecretResponse { + resp := &opificav1.SecretResponse{RequestId: req.GetRequestId()} + deny := func(reason string) *opificav1.SecretResponse { + resp.Result = &opificav1.SecretResponse_DeniedReason{DeniedReason: reason} + return resp + } + if s.cfg.Secrets == nil { + return deny("secrets are not configured on this controller") + } + value, reason, err := s.cfg.Secrets.Request(ctx, worker, req) + switch { + case err != nil: + s.log.Error("checking secret request", "secret", req.GetName(), "job", req.GetJobId(), "err", err) + return deny("the controller could not check this request; see its log") + case reason != "": + s.log.Warn("secret denied", "secret", req.GetName(), "job", req.GetJobId(), "worker", worker, "reason", reason) + return deny(reason) + } + s.log.Info("secret granted", "secret", req.GetName(), "job", req.GetJobId(), "worker", worker) + resp.Result = &opificav1.SecretResponse_Value{Value: value} + return resp +} + func (s *Scheduler) logChunk(ctx context.Context, worker uuid.UUID, c *opificav1.LogChunk) { if s.cfg.Logs == nil { return diff --git a/internal/controller/secrets/secrets.go b/internal/controller/secrets/secrets.go new file mode 100644 index 0000000..02b6f48 --- /dev/null +++ b/internal/controller/secrets/secrets.go @@ -0,0 +1,300 @@ +// Package secrets stores secrets encrypted and hands them to jobs that the +// admin's policy allows. +// +// A job never decides what it gets. It can only ask for secrets its +// pipeline lists, with the job token of its current attempt, through the +// worker that holds it; the policy set by the admin for each secret then +// decides by repository, pool, ref and whether the run is a pull request. +// Every decision is recorded. +// +// Values are encrypted with AES-256-GCM under a key kept in a file only the +// controller can read. Moving the key into a KMS or HSM is planned +// (docs/ROADMAP.md, M5). +package secrets + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "errors" + "fmt" + "os" + gopath "path" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/google/uuid" + "google.golang.org/protobuf/proto" + + opificav1 "github.com/opifica/opifica/gen/opifica/v1" + "github.com/opifica/opifica/internal/controller/store" + "github.com/opifica/opifica/internal/controller/store/db" +) + +const keyFile = "secrets.key" + +// Key encrypts and decrypts secret values. +type Key struct { + id string + aead cipher.AEAD +} + +// LoadOrCreateKey loads the secrets key from dir, creating it on first use. +func LoadOrCreateKey(dir string) (*Key, error) { + path := filepath.Join(dir, keyFile) + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + raw = make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return nil, err + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if errors.Is(err, os.ErrExist) { + return LoadOrCreateKey(dir) // another process created it first + } + if err != nil { + return nil, err + } + if _, err := f.Write(raw); err != nil { + f.Close() + return nil, err + } + if err := f.Close(); err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + return NewKey(raw) +} + +// NewKey wraps a 32-byte AES-256 key. +func NewKey(raw []byte) (*Key, error) { + if len(raw) != 32 { + return nil, fmt.Errorf("secrets key must be 32 bytes, got %d", len(raw)) + } + block, err := aes.NewCipher(raw) + if err != nil { + return nil, err + } + aead, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + sum := sha256.Sum256(raw) + return &Key{id: hex.EncodeToString(sum[:8]), aead: aead}, nil +} + +// ID identifies the key without revealing it. +func (k *Key) ID() string { return k.id } + +// seal encrypts value. The secret's name is bound as additional data, so a +// ciphertext copied into another secret's row won't decrypt. +func (k *Key) seal(name string, value []byte) (ciphertext, nonce []byte, err error) { + nonce = make([]byte, k.aead.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return nil, nil, err + } + return k.aead.Seal(nil, nonce, value, []byte(name)), nonce, nil +} + +func (k *Key) open(name string, ciphertext, nonce []byte) ([]byte, error) { + v, err := k.aead.Open(nil, nonce, ciphertext, []byte(name)) + if err != nil { + return nil, errors.New("secret can't be decrypted: wrong key, or the stored value was altered") + } + return v, nil +} + +// Policy decides which runs may use a secret. +type Policy struct { + // Repos are repository globs. "*" alone matches every repository. + Repos []string + // Pools the run must be in. + Pools []string + // Refs are ref globs ("refs/heads/main", "refs/tags/v*"). Empty: any. + Refs []string + // AllowPullRequests lets pull request runs use the secret. Off by + // default: a pull request runs code nobody has reviewed yet. + AllowPullRequests bool + Description string +} + +var ( + nameRe = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*$`) + poolRe = regexp.MustCompile(`^[a-z][a-z0-9-]{0,62}$`) +) + +// MaxValueSize is the largest secret value accepted. +const MaxValueSize = 64 << 10 + +// Manager stores secrets and answers requests for them. +type Manager struct { + Store *store.Store + Key *Key +} + +// Set stores a secret, replacing any secret of the same name. +func (m *Manager) Set(ctx context.Context, name string, value []byte, p Policy) error { + if !nameRe.MatchString(name) { + return fmt.Errorf("invalid secret name %q (use UPPER_SNAKE_CASE)", name) + } + if len(value) == 0 || len(value) > MaxValueSize { + return fmt.Errorf("secret value must be 1 to %d bytes", MaxValueSize) + } + if len(p.Repos) == 0 { + return errors.New("at least one repository pattern is required (use '*' for all)") + } + if len(p.Pools) == 0 { + return errors.New("at least one pool is required") + } + for _, pool := range p.Pools { + if !poolRe.MatchString(pool) { + return fmt.Errorf("invalid pool name %q", pool) + } + } + for _, g := range append(slices.Clone(p.Repos), p.Refs...) { + if _, err := match(g, ""); err != nil { + return fmt.Errorf("invalid pattern %q: %v", g, err) + } + } + ct, nonce, err := m.Key.seal(name, value) + if err != nil { + return err + } + refs := p.Refs + if refs == nil { + refs = []string{} + } + _, err = m.Store.UpsertSecret(ctx, db.UpsertSecretParams{ + Name: name, Ciphertext: ct, Nonce: nonce, KeyID: m.Key.ID(), + Repos: p.Repos, Pools: p.Pools, Refs: refs, AllowPullRequests: p.AllowPullRequests, Description: p.Description, + }) + return err +} + +// Request asks for a secret on behalf of a job. It returns the value, or a +// reason the request was denied. Errors are for failures of the controller +// itself (e.g. the database), not denials. +func (m *Manager) Request(ctx context.Context, worker uuid.UUID, req *opificav1.SecretRequest) ([]byte, string, error) { + var jobID, runID *uuid.UUID + value, reason, err := m.decide(ctx, worker, req, &jobID, &runID) + if err != nil { + return nil, "", err + } + if rerr := m.Store.RecordSecretAccess(ctx, db.RecordSecretAccessParams{ + SecretName: req.GetName(), JobID: jobID, RunID: runID, WorkerID: &worker, + Granted: reason == "", Reason: reason, + }); rerr != nil { + // Without an audit record, don't hand out the secret. + return nil, "", fmt.Errorf("recording secret access: %w", rerr) + } + return value, reason, nil +} + +func (m *Manager) decide(ctx context.Context, worker uuid.UUID, req *opificav1.SecretRequest, jobOut, runOut **uuid.UUID) ([]byte, string, error) { + const notHeld = "the job is not running on this worker" + jobID, err := uuid.Parse(req.GetJobId()) + if err != nil { + return nil, notHeld, nil + } + job, err := m.Store.GetJob(ctx, jobID) + if store.IsNotFound(err) { + return nil, notHeld, nil + } + if err != nil { + return nil, "", err + } + *jobOut, *runOut = &job.ID, &job.RunID + + // 1. The worker asking holds the job right now. + if job.WorkerID == nil || *job.WorkerID != worker || (job.State != "assigned" && job.State != "running") { + return nil, notHeld, nil + } + // 2. The token belongs to the job's current attempt. + got := sha256.Sum256([]byte(req.GetJobToken())) + if len(job.TokenHash) == 0 || subtle.ConstantTimeCompare(got[:], job.TokenHash) != 1 { + return nil, "invalid job token", nil + } + // 3. The pipeline lists the secret for this job. + spec := &opificav1.Job{} + if err := proto.Unmarshal(job.Spec, spec); err != nil { + return nil, "", fmt.Errorf("stored job spec: %w", err) + } + if !slices.Contains(spec.GetAllowedSecrets(), req.GetName()) { + return nil, "the job's secrets don't include " + req.GetName(), nil + } + // 4. The admin's policy allows it for this run. + sec, err := m.Store.GetSecret(ctx, req.GetName()) + if store.IsNotFound(err) { + return nil, "no secret named " + req.GetName(), nil + } + if err != nil { + return nil, "", err + } + run, err := m.Store.GetRun(ctx, job.RunID) + if err != nil { + return nil, "", err + } + if reason := allowed(sec, run); reason != "" { + return nil, reason, nil + } + value, err := m.Key.open(sec.Name, sec.Ciphertext, sec.Nonce) + if err != nil { + return nil, "", err + } + return value, "", nil +} + +// allowed returns why the policy denies sec to run, or "" if it allows it. +func allowed(sec db.Secret, run db.Run) string { + if !matchAny(sec.Repos, run.Repo) { + return fmt.Sprintf("not allowed for repository %s", run.Repo) + } + if !slices.Contains(sec.Pools, run.Pool) { + return fmt.Sprintf("not allowed in pool %s", run.Pool) + } + if run.EventType == "pull_request" { + if !sec.AllowPullRequests { + return "not available to pull requests" + } + } else if len(sec.Refs) > 0 && !matchAny(sec.Refs, run.Ref) { + return fmt.Sprintf("not allowed for %s", run.Ref) + } + return "" +} + +func matchAny(patterns []string, s string) bool { + for _, p := range patterns { + if ok, _ := match(p, s); ok { + return true + } + } + return false +} + +// match is a glob match where "*" alone matches everything, a trailing +// "/**" matches everything below a prefix, and otherwise "*" stays within +// one path segment. +func match(pattern, s string) (bool, error) { + switch { + case pattern == "*" || pattern == "**": + return true, nil + case strings.HasSuffix(pattern, "/**"): + prefix := strings.TrimSuffix(pattern, "/**") + if _, err := gopath.Match(prefix, ""); err != nil { + return false, err + } + return s == prefix || strings.HasPrefix(s, prefix+"/"), nil + } + return gopath.Match(pattern, s) +} diff --git a/internal/controller/secrets/secrets_test.go b/internal/controller/secrets/secrets_test.go new file mode 100644 index 0000000..6fcda2d --- /dev/null +++ b/internal/controller/secrets/secrets_test.go @@ -0,0 +1,96 @@ +package secrets + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/opifica/opifica/internal/controller/store/db" +) + +func TestKeyRoundTrip(t *testing.T) { + dir := t.TempDir() + k, err := LoadOrCreateKey(dir) + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(filepath.Join(dir, keyFile)) + if err != nil || info.Mode().Perm() != 0o600 { + t.Errorf("key file mode = %v, %v; want 0600", info.Mode().Perm(), err) + } + k2, err := LoadOrCreateKey(dir) + if err != nil || k2.ID() != k.ID() { + t.Fatalf("reloading gave a different key: %v", err) + } + + ct, nonce, err := k.seal("DEPLOY_KEY", []byte("hunter2")) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(ct, []byte("hunter2")) { + t.Fatal("ciphertext contains the plaintext") + } + if v, err := k2.open("DEPLOY_KEY", ct, nonce); err != nil || string(v) != "hunter2" { + t.Errorf("open = %q, %v", v, err) + } + + // A ciphertext moved to another secret's row must not decrypt. + if _, err := k.open("OTHER_SECRET", ct, nonce); err == nil { + t.Error("ciphertext decrypted under another secret's name") + } + tampered := bytes.Clone(ct) + tampered[0] ^= 1 + if _, err := k.open("DEPLOY_KEY", tampered, nonce); err == nil { + t.Error("tampered ciphertext decrypted") + } + other, _ := LoadOrCreateKey(t.TempDir()) + if _, err := other.open("DEPLOY_KEY", ct, nonce); err == nil { + t.Error("decrypted with a different key") + } + + // Every encryption uses a fresh nonce. + ct2, nonce2, _ := k.seal("DEPLOY_KEY", []byte("hunter2")) + if bytes.Equal(nonce, nonce2) || bytes.Equal(ct, ct2) { + t.Error("two encryptions of the same value are identical") + } +} + +func TestNewKeyRejectsBadLength(t *testing.T) { + if _, err := NewKey(make([]byte, 16)); err == nil { + t.Error("accepted a 16-byte key") + } +} + +func TestPolicy(t *testing.T) { + sec := db.Secret{ + Repos: []string{"acme/*"}, Pools: []string{"release"}, + Refs: []string{"refs/heads/main", "refs/tags/v*"}, + } + run := func(repo, pool, event, ref string) db.Run { + return db.Run{Repo: repo, Pool: pool, EventType: event, Ref: ref} + } + tests := []struct { + name string + sec db.Secret + run db.Run + want string // "" means allowed + }{ + {"allowed push", sec, run("acme/app", "release", "push", "refs/heads/main"), ""}, + {"allowed tag", sec, run("acme/app", "release", "tag", "refs/tags/v1.2.0"), ""}, + {"other repo", sec, run("evil/app", "release", "push", "refs/heads/main"), "not allowed for repository evil/app"}, + {"nested repo isn't matched by acme/*", sec, run("acme/team/app", "release", "push", "refs/heads/main"), "not allowed for repository acme/team/app"}, + {"other pool", sec, run("acme/app", "untrusted", "push", "refs/heads/main"), "not allowed in pool untrusted"}, + {"other branch", sec, run("acme/app", "release", "push", "refs/heads/feature"), "not allowed for refs/heads/feature"}, + {"pull request", sec, run("acme/app", "release", "pull_request", ""), "not available to pull requests"}, + {"pull request allowed", db.Secret{Repos: []string{"*"}, Pools: []string{"untrusted"}, AllowPullRequests: true}, + run("any/repo", "untrusted", "pull_request", ""), ""}, + {"no refs means any ref", db.Secret{Repos: []string{"acme/**"}, Pools: []string{"trusted"}}, + run("acme/team/app", "trusted", "push", "refs/heads/anything"), ""}, + } + for _, tt := range tests { + if got := allowed(tt.sec, tt.run); got != tt.want { + t.Errorf("%s: allowed() = %q, want %q", tt.name, got, tt.want) + } + } +} diff --git a/internal/controller/store/db/models.go b/internal/controller/store/db/models.go index 468e5a4..99a7002 100644 --- a/internal/controller/store/db/models.go +++ b/internal/controller/store/db/models.go @@ -67,6 +67,31 @@ type Run struct { FinishedAt *time.Time } +type Secret struct { + Name string + Ciphertext []byte + Nonce []byte + KeyID string + Repos []string + Pools []string + Refs []string + AllowPullRequests bool + Description string + CreatedAt time.Time + UpdatedAt time.Time +} + +type SecretAccess struct { + ID int64 + SecretName string + JobID *uuid.UUID + RunID *uuid.UUID + WorkerID *uuid.UUID + Granted bool + Reason string + At time.Time +} + type Worker struct { ID uuid.UUID Hostname string diff --git a/internal/controller/store/db/secrets.sql.go b/internal/controller/store/db/secrets.sql.go new file mode 100644 index 0000000..6400a5b --- /dev/null +++ b/internal/controller/store/db/secrets.sql.go @@ -0,0 +1,209 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: secrets.sql + +package db + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +const deleteSecret = `-- name: DeleteSecret :execrows +DELETE FROM secrets WHERE name = $1 +` + +func (q *Queries) DeleteSecret(ctx context.Context, name string) (int64, error) { + result, err := q.db.Exec(ctx, deleteSecret, name) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getSecret = `-- name: GetSecret :one +SELECT name, ciphertext, nonce, key_id, repos, pools, refs, allow_pull_requests, description, created_at, updated_at FROM secrets WHERE name = $1 +` + +func (q *Queries) GetSecret(ctx context.Context, name string) (Secret, error) { + row := q.db.QueryRow(ctx, getSecret, name) + var i Secret + err := row.Scan( + &i.Name, + &i.Ciphertext, + &i.Nonce, + &i.KeyID, + &i.Repos, + &i.Pools, + &i.Refs, + &i.AllowPullRequests, + &i.Description, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const listSecretAccesses = `-- name: ListSecretAccesses :many +SELECT id, secret_name, job_id, run_id, worker_id, granted, reason, at FROM secret_accesses WHERE secret_name = $1 ORDER BY at DESC, id DESC LIMIT $2 +` + +type ListSecretAccessesParams struct { + SecretName string + Limit int32 +} + +func (q *Queries) ListSecretAccesses(ctx context.Context, arg ListSecretAccessesParams) ([]SecretAccess, error) { + rows, err := q.db.Query(ctx, listSecretAccesses, arg.SecretName, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SecretAccess + for rows.Next() { + var i SecretAccess + if err := rows.Scan( + &i.ID, + &i.SecretName, + &i.JobID, + &i.RunID, + &i.WorkerID, + &i.Granted, + &i.Reason, + &i.At, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listSecrets = `-- name: ListSecrets :many +SELECT name, repos, pools, refs, allow_pull_requests, description, created_at, updated_at +FROM secrets ORDER BY name +` + +type ListSecretsRow struct { + Name string + Repos []string + Pools []string + Refs []string + AllowPullRequests bool + Description string + CreatedAt time.Time + UpdatedAt time.Time +} + +func (q *Queries) ListSecrets(ctx context.Context) ([]ListSecretsRow, error) { + rows, err := q.db.Query(ctx, listSecrets) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListSecretsRow + for rows.Next() { + var i ListSecretsRow + if err := rows.Scan( + &i.Name, + &i.Repos, + &i.Pools, + &i.Refs, + &i.AllowPullRequests, + &i.Description, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const recordSecretAccess = `-- name: RecordSecretAccess :exec +INSERT INTO secret_accesses (secret_name, job_id, run_id, worker_id, granted, reason) +VALUES ($1, $2, $3, $4, $5, $6) +` + +type RecordSecretAccessParams struct { + SecretName string + JobID *uuid.UUID + RunID *uuid.UUID + WorkerID *uuid.UUID + Granted bool + Reason string +} + +func (q *Queries) RecordSecretAccess(ctx context.Context, arg RecordSecretAccessParams) error { + _, err := q.db.Exec(ctx, recordSecretAccess, + arg.SecretName, + arg.JobID, + arg.RunID, + arg.WorkerID, + arg.Granted, + arg.Reason, + ) + return err +} + +const upsertSecret = `-- name: UpsertSecret :one +INSERT INTO secrets (name, ciphertext, nonce, key_id, repos, pools, refs, allow_pull_requests, description) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +ON CONFLICT (name) DO UPDATE +SET ciphertext = excluded.ciphertext, nonce = excluded.nonce, key_id = excluded.key_id, + repos = excluded.repos, pools = excluded.pools, refs = excluded.refs, + allow_pull_requests = excluded.allow_pull_requests, description = excluded.description, + updated_at = now() +RETURNING name, ciphertext, nonce, key_id, repos, pools, refs, allow_pull_requests, description, created_at, updated_at +` + +type UpsertSecretParams struct { + Name string + Ciphertext []byte + Nonce []byte + KeyID string + Repos []string + Pools []string + Refs []string + AllowPullRequests bool + Description string +} + +func (q *Queries) UpsertSecret(ctx context.Context, arg UpsertSecretParams) (Secret, error) { + row := q.db.QueryRow(ctx, upsertSecret, + arg.Name, + arg.Ciphertext, + arg.Nonce, + arg.KeyID, + arg.Repos, + arg.Pools, + arg.Refs, + arg.AllowPullRequests, + arg.Description, + ) + var i Secret + err := row.Scan( + &i.Name, + &i.Ciphertext, + &i.Nonce, + &i.KeyID, + &i.Repos, + &i.Pools, + &i.Refs, + &i.AllowPullRequests, + &i.Description, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} diff --git a/internal/controller/store/migrations/0003_secrets.sql b/internal/controller/store/migrations/0003_secrets.sql new file mode 100644 index 0000000..f15420b --- /dev/null +++ b/internal/controller/store/migrations/0003_secrets.sql @@ -0,0 +1,35 @@ +-- Secrets are encrypted with the controller's secrets key (AES-256-GCM). +-- The admin's policy for each secret decides which runs may use it; the +-- pipeline file can only ask. +CREATE TABLE secrets ( + name text PRIMARY KEY CHECK (name ~ '^[A-Z_][A-Z0-9_]*$'), + ciphertext bytea NOT NULL, + nonce bytea NOT NULL, + -- Which secrets key encrypted it, for key rotation. + key_id text NOT NULL, + -- Repository globs, e.g. {acme/*}. '*' alone matches every repository. + repos text[] NOT NULL CHECK (cardinality(repos) > 0), + pools text[] NOT NULL CHECK (cardinality(pools) > 0), + -- Ref globs, e.g. {refs/heads/main, refs/tags/v*}. Empty: any ref. + refs text[] NOT NULL DEFAULT '{}', + -- Pull requests run code that hasn't been reviewed yet: no secrets + -- unless the admin explicitly allows it. + allow_pull_requests boolean NOT NULL DEFAULT false, + description text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +-- Every secret request, granted or not. +CREATE TABLE secret_accesses ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + secret_name text NOT NULL, + job_id uuid, + run_id uuid, + worker_id uuid, + granted boolean NOT NULL, + reason text NOT NULL DEFAULT '', + at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX secret_accesses_name ON secret_accesses (secret_name, at DESC); diff --git a/internal/controller/store/queries/secrets.sql b/internal/controller/store/queries/secrets.sql new file mode 100644 index 0000000..110b3f8 --- /dev/null +++ b/internal/controller/store/queries/secrets.sql @@ -0,0 +1,26 @@ +-- name: UpsertSecret :one +INSERT INTO secrets (name, ciphertext, nonce, key_id, repos, pools, refs, allow_pull_requests, description) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +ON CONFLICT (name) DO UPDATE +SET ciphertext = excluded.ciphertext, nonce = excluded.nonce, key_id = excluded.key_id, + repos = excluded.repos, pools = excluded.pools, refs = excluded.refs, + allow_pull_requests = excluded.allow_pull_requests, description = excluded.description, + updated_at = now() +RETURNING *; + +-- name: GetSecret :one +SELECT * FROM secrets WHERE name = $1; + +-- name: ListSecrets :many +SELECT name, repos, pools, refs, allow_pull_requests, description, created_at, updated_at +FROM secrets ORDER BY name; + +-- name: DeleteSecret :execrows +DELETE FROM secrets WHERE name = $1; + +-- name: RecordSecretAccess :exec +INSERT INTO secret_accesses (secret_name, job_id, run_id, worker_id, granted, reason) +VALUES ($1, $2, $3, $4, $5, $6); + +-- name: ListSecretAccesses :many +SELECT * FROM secret_accesses WHERE secret_name = $1 ORDER BY at DESC, id DESC LIMIT $2; diff --git a/internal/integration/connect_test.go b/internal/integration/connect_test.go index fde168a..8009ce2 100644 --- a/internal/integration/connect_test.go +++ b/internal/integration/connect_test.go @@ -23,6 +23,7 @@ import ( "github.com/opifica/opifica/internal/controller/ca" "github.com/opifica/opifica/internal/controller/logs" "github.com/opifica/opifica/internal/controller/scheduler" + "github.com/opifica/opifica/internal/controller/secrets" "github.com/opifica/opifica/internal/controller/store" "github.com/opifica/opifica/internal/controller/workers" "github.com/opifica/opifica/internal/server" @@ -40,11 +41,12 @@ func quiet() *slog.Logger { } type env struct { - t *testing.T - svc *workers.Service - sched *scheduler.Scheduler - logs *logs.Store - url string + t *testing.T + svc *workers.Service + sched *scheduler.Scheduler + logs *logs.Store + secrets *secrets.Manager + url string } // newEnv starts a controller with short timeouts on a random port. @@ -68,8 +70,13 @@ func newEnv(t *testing.T) *env { }, } logStore := &logs.Store{Dir: t.TempDir()} + key, err := secrets.LoadOrCreateKey(t.TempDir()) + if err != nil { + t.Fatal(err) + } + secretMgr := &secrets.Manager{Store: st, Key: key} sched := scheduler.New(st, svc.Sessions(), quiet(), scheduler.Config{ - Lease: 400 * time.Millisecond, Interval: 20 * time.Millisecond, Logs: logStore, + Lease: 400 * time.Millisecond, Interval: 20 * time.Millisecond, Logs: logStore, Secrets: secretMgr, }) svc.Messages = sched go sched.Run(ctx) @@ -91,7 +98,7 @@ func newEnv(t *testing.T) *env { <-done st.Close() }) - return &env{t: t, svc: svc, sched: sched, logs: logStore, url: "https://" + ln.Addr().String()} + return &env{t: t, svc: svc, sched: sched, logs: logStore, secrets: secretMgr, url: "https://" + ln.Addr().String()} } func info() *opificav1.WorkerInfo { diff --git a/internal/integration/scheduler_test.go b/internal/integration/scheduler_test.go index acf4702..2af2e49 100644 --- a/internal/integration/scheduler_test.go +++ b/internal/integration/scheduler_test.go @@ -46,6 +46,7 @@ type fakeWorker struct { received []*opificav1.Job maxRunning int behave func(*opificav1.Job) outcome + secretResp chan *opificav1.SecretResponse } type workerOpts struct { @@ -81,7 +82,8 @@ func (e *env) fakeWorker(name string, o workerOpts) *fakeWorker { } id := e.joinWith(o.pool, wi) - fw := &fakeWorker{name: name, id: id, running: map[string]chan struct{}{}, behave: o.behave} + fw := &fakeWorker{name: name, id: id, running: map[string]chan struct{}{}, behave: o.behave, + secretResp: make(chan *opificav1.SecretResponse, 8)} fw.runningClient = e.start(id, func(c *worker.ClientConfig) { c.Info = wi c.Status = fw.status @@ -122,6 +124,8 @@ func (fw *fakeWorker) onMessage(_ context.Context, m *opificav1.ConnectResponse) close(c) } fw.mu.Unlock() + case *opificav1.ConnectResponse_SecretResponse: + fw.secretResp <- msg.SecretResponse } } diff --git a/internal/integration/secrets_test.go b/internal/integration/secrets_test.go new file mode 100644 index 0000000..6b5c8ec --- /dev/null +++ b/internal/integration/secrets_test.go @@ -0,0 +1,193 @@ +package integration + +import ( + "context" + "regexp" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + opificav1 "github.com/opifica/opifica/gen/opifica/v1" + "github.com/opifica/opifica/internal/controller/scheduler" + "github.com/opifica/opifica/internal/controller/secrets" + "github.com/opifica/opifica/internal/controller/store/db" + "github.com/opifica/opifica/internal/pipeline" +) + +const secretValue = "s3cr3t-deploy-token-value" + +func (e *env) setSecret(name string, p secrets.Policy) { + e.t.Helper() + if err := e.secrets.Set(context.Background(), name, []byte(secretValue), p); err != nil { + e.t.Fatal(err) + } +} + +const deployPipeline = ` +on: [push, pull_request, tag] +jobs: + deploy: + runs-on: OS + secrets: [DEPLOY_TOKEN] + steps: + - run: | + echo "token is $DEPLOY_TOKEN" + # Check the real value arrived, without the pipeline containing it. + test "${#DEPLOY_TOKEN}" = 25 + case "$DEPLOY_TOKEN" in s3cr3t-deploy-*) ;; *) exit 1 ;; esac + secrets: [DEPLOY_TOKEN] + - name: only the step that asked gets it + run: test -z "$DEPLOY_TOKEN" +` + +func TestE2ESecretGrantedAndMasked(t *testing.T) { + e := newEnv(t) + e.realWorker(1) + e.setSecret("DEPLOY_TOKEN", secrets.Policy{Repos: []string{"acme/*"}, Pools: []string{"trusted"}, Refs: []string{"refs/heads/main"}}) + repo, sha := gitRepo(t, map[string]string{"README": "x"}) + runID := e.startRun(repo, sha, strings.ReplaceAll(deployPipeline, "OS", hostOS())) + + run, jobs := e.waitRun(runID) + log := e.jobLog(runID, "deploy") + if run.State != "succeeded" { + t.Fatalf("run %s (%q)\n%s", run.State, jobs["deploy"].Error, log) + } + wantIn(t, log, "token is ***") + if strings.Contains(log, secretValue) { + t.Errorf("the secret value is in the stored log:\n%s", log) + } + + acc, err := e.svc.Store.ListSecretAccesses(context.Background(), dbAccessParams("DEPLOY_TOKEN")) + if err != nil || len(acc) != 1 || !acc[0].Granted || *acc[0].JobID != jobs["deploy"].ID { + t.Errorf("audit = %+v, %v; want one granted access for the job", acc, err) + } +} + +// The admin's policy decides, whatever the pipeline asks for. +func TestE2ESecretPolicyDenials(t *testing.T) { + tests := []struct { + name string + policy secrets.Policy + event pipeline.Event + want string + }{ + {"other pool", secrets.Policy{Repos: []string{"*"}, Pools: []string{"release"}}, + pipeline.Event{Type: pipeline.EventPush, Branch: "main", Ref: "refs/heads/main"}, "not allowed in pool trusted"}, + {"other repository", secrets.Policy{Repos: []string{"other/*"}, Pools: []string{"trusted"}}, + pipeline.Event{Type: pipeline.EventPush, Branch: "main", Ref: "refs/heads/main"}, "not allowed for repository acme/app"}, + {"other branch", secrets.Policy{Repos: []string{"*"}, Pools: []string{"trusted"}, Refs: []string{"refs/heads/main"}}, + pipeline.Event{Type: pipeline.EventPush, Branch: "feature", Ref: "refs/heads/feature"}, "not allowed for refs/heads/feature"}, + {"pull request", secrets.Policy{Repos: []string{"*"}, Pools: []string{"trusted"}}, + pipeline.Event{Type: pipeline.EventPullRequest, Branch: "evil", BaseBranch: "main"}, "not available to pull requests"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newEnv(t) + e.realWorker(1) + e.setSecret("DEPLOY_TOKEN", tt.policy) + repo, sha := gitRepo(t, map[string]string{"README": "x"}) + ev := tt.event + ev.SHA = sha + run, err := e.sched.CreateRun(context.Background(), scheduler.RunRequest{ + Repo: "acme/app", CloneURL: repo, Pool: "trusted", Event: ev, + Pipeline: []byte(strings.ReplaceAll(deployPipeline, "OS", hostOS())), + }) + if err != nil { + t.Fatal(err) + } + got, jobs := e.waitRun(run.ID) + j := jobs["deploy"] + if got.State != "failed" || !strings.Contains(j.Error, tt.want) || j.Attempt != 1 { + t.Errorf("run %s, attempt %d, error %q; want failed with %q", got.State, j.Attempt, j.Error, tt.want) + } + // The step's output, not its "==> Step" header line. + if regexp.MustCompile(`(?m)^token is`).MatchString(e.jobLog(run.ID, "deploy")) { + t.Error("the step ran without its secret") + } + }) + } +} + +// Requests that try to get around the checks, sent straight over a +// worker's connection. +func TestSecretRequestChecks(t *testing.T) { + e := newEnv(t) + e.setSecret("DEPLOY_TOKEN", secrets.Policy{Repos: []string{"*"}, Pools: []string{"trusted"}}) + e.setSecret("OTHER_TOKEN", secrets.Policy{Repos: []string{"*"}, Pools: []string{"trusted"}}) + + holder := e.fakeWorker("holder", workerOpts{behave: func(*opificav1.Job) outcome { return outcome{hang: true} }}) + e.createRun("on: push\njobs:\n deploy:\n runs-on: linux\n secrets: [DEPLOY_TOKEN]\n steps: [{ run: x }]\n", pipeline.Event{}, "") + eventually(t, "the job to reach the worker", func() bool { return len(holder.jobs()) == 1 }) + job := holder.jobs()[0] + eventually(t, "the job to be running", func() bool { + j, err := e.svc.Store.GetJob(context.Background(), uuid.MustParse(job.GetId())) + return err == nil && j.State == "running" + }) + + ask := func(t *testing.T, fw *fakeWorker, jobID, token, name string) *opificav1.SecretResponse { + t.Helper() + fw.Send(&opificav1.ConnectRequest{Message: &opificav1.ConnectRequest_SecretRequest{SecretRequest: &opificav1.SecretRequest{ + RequestId: uuid.NewString(), JobId: jobID, JobToken: token, Name: name, + }}}) + select { + case r := <-fw.secretResp: + return r + case <-time.After(5 * time.Second): + t.Fatal("no answer to the secret request") + return nil + } + } + + if r := ask(t, holder, job.GetId(), job.GetJobToken(), "DEPLOY_TOKEN"); string(r.GetValue()) != secretValue { + t.Fatalf("legitimate request denied: %q", r.GetDeniedReason()) + } + + other := e.fakeWorker("other", workerOpts{}) + select { + case <-other.Connected(): + case <-time.After(5 * time.Second): + t.Fatal("second worker did not connect") + } + for _, tt := range []struct { + name string + fw *fakeWorker + jobID, token, want string + }{ + {"secret the job doesn't list", holder, job.GetId(), job.GetJobToken(), "don't include OTHER_TOKEN"}, + {"wrong job token", holder, job.GetId(), "opt_guessed", "invalid job token"}, + {"another worker's job", other, job.GetId(), job.GetJobToken(), "not running on this worker"}, + {"unknown job", holder, uuid.NewString(), job.GetJobToken(), "not running on this worker"}, + } { + t.Run(tt.name, func(t *testing.T) { + name := "DEPLOY_TOKEN" + if strings.Contains(tt.name, "doesn't list") { + name = "OTHER_TOKEN" + } + r := ask(t, tt.fw, tt.jobID, tt.token, name) + if r.GetValue() != nil || !strings.Contains(r.GetDeniedReason(), tt.want) { + t.Errorf("got value %q, denied %q; want denial %q", r.GetValue(), r.GetDeniedReason(), tt.want) + } + }) + } + + // Every request was recorded, granted or not. + acc, err := e.svc.Store.ListSecretAccesses(context.Background(), dbAccessParams("DEPLOY_TOKEN")) + if err != nil { + t.Fatal(err) + } + granted := 0 + for _, a := range acc { + if a.Granted { + granted++ + } + } + if len(acc) != 4 || granted != 1 { + t.Errorf("%d DEPLOY_TOKEN accesses recorded (%d granted), want 4 (1 granted)", len(acc), granted) + } +} + +func dbAccessParams(name string) db.ListSecretAccessesParams { + return db.ListSecretAccessesParams{SecretName: name, Limit: 100} +}