diff --git a/.github/workflows/dispose-retained-email-copy.yaml b/.github/workflows/dispose-retained-email-copy.yaml index e40add58c..2d71da431 100644 --- a/.github/workflows/dispose-retained-email-copy.yaml +++ b/.github/workflows/dispose-retained-email-copy.yaml @@ -4,7 +4,7 @@ name: Dispose Verified Retained Email Copy workflow_dispatch: inputs: confirm: - description: "Type exactly: dispose-retained-email-copy" + description: "Type dispose-retained-email-copy or dispose-fresh-retained-email-copy" required: true type: string @@ -28,7 +28,7 @@ jobs: env: CONFIRM: ${{ inputs.confirm }} run: | - if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$CONFIRM" != dispose-retained-email-copy ]]; then + if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || ( "$CONFIRM" != dispose-retained-email-copy && "$CONFIRM" != dispose-fresh-retained-email-copy ) ]]; then printf '%s\n' 'EMAIL_RETAINED_DISPOSAL=UNVERIFIED stage=dispatch' >&2 exit 2 fi @@ -70,6 +70,7 @@ jobs: GH_TOKEN: ${{ github.token }} EMAIL_RETAINED_READER_UID: ${{ secrets.EMAIL_RETAINED_READER_UID }} EMAIL_RETAINED_CLAIM_UID: ${{ secrets.EMAIL_RETAINED_CLAIM_UID }} + EMAIL_FRESH_RECOVERY_WITNESS: ${{ inputs.confirm == 'dispose-fresh-retained-email-copy' && secrets.EMAIL_FRESH_RECOVERY_WITNESS || '' }} KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} EMAIL_RETAINED_COPY_CONFIRM: ${{ inputs.confirm }} @@ -78,7 +79,11 @@ jobs: set -euo pipefail umask 077 stage=private-workspace - fail() { printf 'EMAIL_RETAINED_DISPOSAL=UNVERIFIED stage=%s\n' "$stage" >&2; exit 2; } + receipt=EMAIL_RETAINED_DISPOSAL + if [[ "$EMAIL_RETAINED_COPY_CONFIRM" == dispose-fresh-retained-email-copy ]]; then + receipt=EMAIL_FRESH_RETAINED_DISPOSAL + fi + fail() { printf '%s=UNVERIFIED stage=%s cleanup=NOT_STARTED\n' "$receipt" "$stage" >&2; exit 2; } [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail private_dir="$(mktemp -d /dev/shm/email-retained-disposal.XXXXXXXX)" trap 'rm -rf "$private_dir"' EXIT diff --git a/internal/namespaceinventory/credentials.go b/internal/namespaceinventory/credentials.go index 13d4875a0..fabb4c708 100644 --- a/internal/namespaceinventory/credentials.go +++ b/internal/namespaceinventory/credentials.go @@ -19,6 +19,9 @@ var credentialPath = regexp.MustCompile(`^/dev/shm/longhorn-namespace-inventory\ var sourcePattern = regexp.MustCompile(`^[0-9a-f]{40}$`) var runPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`) +// FreshEmailDisposalConfirm selects the fixed fresh Email recovery copy. +const FreshEmailDisposalConfirm = "dispose-fresh-retained-email-copy" + // Dispatch is the verified source commit and run of a protected observation. type Dispatch struct{ Source, Run string } @@ -88,11 +91,14 @@ func LoadRetainedEmailClient(getenv func(string) string) (*http.Client, *url.URL // confirmed, maintainer-dispatched disposal workflow on main in this repository, with a // well-formed source commit and run id. func AuthorizeRetainedEmailDispatch(getenv func(string) string) error { + confirm := getenv("EMAIL_RETAINED_COPY_CONFIRM") + if confirm != "dispose-retained-email-copy" && confirm != FreshEmailDisposalConfirm { + return errors.New("protected Email disposal dispatch is unverified") + } for key, want := range map[string]string{ "GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", "GITHUB_REPOSITORY": "devantler-tech/platform", "GITHUB_RUN_ATTEMPT": "1", "RUNNER_ENVIRONMENT": "github-hosted", "GITHUB_ACTOR": "devantler", - "GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main", - "EMAIL_RETAINED_COPY_CONFIRM": "dispose-retained-email-copy", + "GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main", } { if getenv(key) != want { return errors.New("protected Email disposal dispatch is unverified") diff --git a/internal/namespaceinventory/inventory.go b/internal/namespaceinventory/inventory.go index 9a7bf113b..5c876cf0e 100644 --- a/internal/namespaceinventory/inventory.go +++ b/internal/namespaceinventory/inventory.go @@ -107,12 +107,19 @@ func ObserveRetainedEmailRecovery(ctx context.Context, client *http.Client, endp return observeNamespace(ctx, client, endpoint, "email-data-recovery-41561215415", uid) } +// ObserveFreshRetainedEmailRecovery inventories the fixed fresh restore with +// its separately witnessed namespace identity. +func ObserveFreshRetainedEmailRecovery(ctx context.Context, client *http.Client, endpoint *url.URL, uid string) (Inventory, error) { + return observeNamespace(ctx, client, endpoint, "email-data-recovery-42818236850", uid) +} + func observeNamespace(ctx context.Context, client *http.Client, endpoint *url.URL, namespace, uid string) (Inventory, error) { if endpoint == nil { return Inventory{}, errors.New("authenticated API root is required") } root := *endpoint - if client == nil || root.Scheme != "https" || root.Host == "" || root.Opaque != "" || root.ForceQuery || root.User != nil || root.RawQuery != "" || root.Fragment != "" || (root.Path != "" && root.Path != "/") || (!namespacePattern.MatchString(namespace) && namespace != "longhorn-system" && namespace != "email-data-recovery-41561215415") || len(namespace) > 63 || uid == "" || len(uid) > 128 { + fixed := namespace == "longhorn-system" || namespace == "email-data-recovery-41561215415" || namespace == "email-data-recovery-42818236850" + if client == nil || root.Scheme != "https" || root.Host == "" || root.Opaque != "" || root.ForceQuery || root.User != nil || root.RawQuery != "" || root.Fragment != "" || (root.Path != "" && root.Path != "/") || (!namespacePattern.MatchString(namespace) && !fixed) || len(namespace) > 63 || uid == "" || len(uid) > 128 { return Inventory{}, errors.New("fixed benchmark namespace and authenticated API root are required") } ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) diff --git a/internal/namespaceinventory/inventory_test.go b/internal/namespaceinventory/inventory_test.go index a7f3d2117..4a1f01b79 100644 --- a/internal/namespaceinventory/inventory_test.go +++ b/internal/namespaceinventory/inventory_test.go @@ -97,6 +97,21 @@ func TestRetainedEmailInventoryCoversAllServedKinds(t *testing.T) { } } +func TestFreshRetainedEmailInventoryHasItsOwnFixedNamespace(t *testing.T) { + const namespace = "email-data-recovery-42818236850" + result, err := observeFixtureInNamespace(t, "", namespace) + if err != nil || len(result.Bindings()) != 2 { + t.Fatalf("fixed fresh Email inventory failed: %v", err) + } + result, err = observeFixtureInNamespace(t, "namespace replaced", namespace) + if err == nil || len(result.Bindings()) != 0 { + t.Fatal("a replaced fresh namespace was accepted") + } + if result, err = Observe(context.Background(), nil, nil, namespace, "uid"); err == nil || len(result.Bindings()) != 0 { + t.Fatal("fresh namespace admitted by the benchmark API") + } +} + // One stored Event is served by core v1 and events.k8s.io under the same UID. // That alias is one object; any other identity difference stays a conflict. func TestEventAliasesAreOneObject(t *testing.T) { @@ -315,6 +330,8 @@ func observeFixtureInNamespace(t *testing.T, failure, namespace string) (Invento result, err = CalibrateLonghornSystem(ctx, server.Client(), endpoint, "namespace-uid") case "email-data-recovery-41561215415": result, err = ObserveRetainedEmailRecovery(ctx, server.Client(), endpoint, "namespace-uid") + case "email-data-recovery-42818236850": + result, err = ObserveFreshRetainedEmailRecovery(ctx, server.Client(), endpoint, "namespace-uid") default: result, err = Observe(ctx, server.Client(), endpoint, testNamespace, "namespace-uid") } diff --git a/scripts/verify-email-data-recovery/README.md b/scripts/verify-email-data-recovery/README.md index 8f38a7f06..bafe33764 100644 --- a/scripts/verify-email-data-recovery/README.md +++ b/scripts/verify-email-data-recovery/README.md @@ -100,6 +100,12 @@ encrypted scratch claim. The observer compares the one diagnostic from the protected run's creation window on both copies. Its receipt reports message parity and leaves disposal to a separately reviewed operation. +After the fresh parity check passes, the confirmed +`dispose-fresh-retained-email-copy` mode repeats the graph, inventory and +message checks before disposal. It requires the witnessed backup to remain +valid through the cleanup budget. It releases the fresh reader and scratch +while retaining the source messages and backup, and reports its own result. + The pass receipt covers message recovery, encrypted disposable storage, source retention and disposal of the restored copy. It does not establish an independent cluster rebuild, archive migration, transport delivery, OIDC, or production diff --git a/scripts/verify-email-data-recovery/fresh_observation.go b/scripts/verify-email-data-recovery/fresh_observation.go index 5e230e226..794333e29 100644 --- a/scripts/verify-email-data-recovery/fresh_observation.go +++ b/scripts/verify-email-data-recovery/fresh_observation.go @@ -87,6 +87,9 @@ func observeFreshRecoveryGraph(ctx context.Context, d *recoveryDriver, w *freshR backupRef := ownedReference{"Backup", d.i.namespace + "-backup", "velero", w.Backup.UID} downloadRef := ownedReference{"DataDownload", d.i.namespace + "-download", "velero", w.Download.UID} for _, ref := range []ownedReference{backupRef, downloadRef} { + if tracked, present := d.refs[ref.key()]; present && tracked != ref { + return "fresh-witness" + } d.refs[ref.key()] = ref } backup, err := d.owned(ctx, backupRef) @@ -116,7 +119,7 @@ func observeFreshRecoveryGraph(ctx context.Context, d *recoveryDriver, w *freshR return "fresh-transfer" } claimRef, err := d.ref("PersistentVolumeClaim", "data", d.i.namespace) - if err != nil { + if err != nil || claimRef.uid != w.Claim.UID { return "fresh-storage" } claim, err := d.owned(ctx, claimRef) diff --git a/scripts/verify-email-data-recovery/fresh_observation_test.go b/scripts/verify-email-data-recovery/fresh_observation_test.go index a4d3cd3d2..451db70aa 100644 --- a/scripts/verify-email-data-recovery/fresh_observation_test.go +++ b/scripts/verify-email-data-recovery/fresh_observation_test.go @@ -97,3 +97,44 @@ func TestFreshGraphBindsTheUnexpiredBackupToSeparateEncryptedScratch(t *testing. }) } } + +func TestFreshDisposalRequiresLiveNamespaceAndTimeToFinish(t *testing.T) { + w, err := parseFreshRecoveryWitness(freshFixtureWitness) + if err != nil { + t.Fatal("invalid test witness") + } + i := freshRetainedRecovery + _, _, source, _ := backupProofFixture() + beforeExpiry := time.Date(2026, 11, 11, 12, 43, 0, 0, time.UTC) + for _, tc := range []struct { + name, stage string + replace, drift bool + horizon time.Time + }{ + {name: "exact graph", horizon: beforeExpiry}, + {name: "backup expires during cleanup", stage: "fresh-backup", horizon: beforeExpiry.Add(10 * time.Minute)}, + {name: "live namespace replaced", stage: "fresh-namespace", replace: true, horizon: beforeExpiry}, + {name: "frozen backup identity differs", stage: "fresh-witness", drift: true, horizon: beforeExpiry}, + } { + t.Run(tc.name, func(t *testing.T) { + api := freshGraphFixture(w) + ns := recoveryManifest(i, "Namespace", i.namespace, "", "v1") + child(ns, "metadata")["uid"] = w.Namespace.UID + if tc.replace { + child(ns, "metadata")["uid"] = "replacement" + } + api["Namespace"] = ns + nsRef := ownedReference{"Namespace", i.namespace, "", w.Namespace.UID} + claim := ownedReference{"PersistentVolumeClaim", "data", i.namespace, w.Claim.UID} + backup := ownedReference{"Backup", i.namespace + "-backup", "velero", w.Backup.UID} + if tc.drift { + backup.uid = "replacement" + } + refs := map[string]ownedReference{nsRef.key(): nsRef, claim.key(): claim, backup.key(): backup} + d := &recoveryDriver{i: i, k: observationReadOnly{base: api, parity: true, fresh: w}, source: source, refs: refs} + if stage := verifyFreshDisposalGraph(context.Background(), d, w, tc.horizon); stage != tc.stage || d.refs[backup.key()] != backup { + t.Fatalf("expected stable backup binding and stage %q, got %q", tc.stage, stage) + } + }) + } +} diff --git a/scripts/verify-email-data-recovery/retained_auxiliary.go b/scripts/verify-email-data-recovery/retained_auxiliary.go index ae58c4001..e72973448 100644 --- a/scripts/verify-email-data-recovery/retained_auxiliary.go +++ b/scripts/verify-email-data-recovery/retained_auxiliary.go @@ -19,7 +19,7 @@ var ( // reference was removed; this helper never adopts or deletes an auxiliary. func (d *recoveryDriver) retainedAuxiliary(ctx context.Context, binding namespaceinventory.Binding, bindings map[string]namespaceinventory.Binding, readerUID string) (string, error) { fail := func() (string, error) { return "", errors.New("retained-auxiliary") } - if d == nil || d.k == nil || d.i != retainedRecovery || ctx.Err() != nil || binding.Namespace != retainedRecovery.namespace || len(binding.Name) > 253 || !retainedAuxiliaryName.MatchString(binding.Name) || !retainedAuxiliaryUID.MatchString(binding.UID) || !retainedAuxiliaryUID.MatchString(readerUID) { + if d == nil || d.k == nil || (d.i != retainedRecovery && d.i != freshRetainedRecovery) || ctx.Err() != nil || binding.Namespace != d.i.namespace || len(binding.Name) > 253 || !retainedAuxiliaryName.MatchString(binding.Name) || !retainedAuxiliaryUID.MatchString(binding.UID) || !retainedAuxiliaryUID.MatchString(readerUID) { return fail() } api := "" @@ -51,7 +51,7 @@ func (d *recoveryDriver) retainedAuxiliary(ctx context.Context, binding namespac } } } - parent, ok := retainedAuxiliaryOwner(owner, binding.Kind, bindings, readerUID) + parent, ok := retainedAuxiliaryOwner(d.i, owner, binding.Kind, bindings, readerUID) if !ok { return fail() } @@ -64,8 +64,11 @@ func (d *recoveryDriver) retainedAuxiliary(ctx context.Context, binding namespac return parent, nil } -func retainedAuxiliaryOwner(owner object, childKind string, bindings map[string]namespaceinventory.Binding, readerUID string) (string, bool) { - ns := retainedRecovery.namespace +func retainedAuxiliaryOwner(target invocation, owner object, childKind string, bindings map[string]namespaceinventory.Binding, readerUID string) (string, bool) { + if target != retainedRecovery && target != freshRetainedRecovery { + return "", false + } + ns := target.namespace if text(owner, "kind") == "Pod" && text(owner, "name") == "reader" && text(owner, "uid") == readerUID { // The independent reader witness remains usable while its frozen child // awaits GC after Pod deletion. No current Pod is newly adopted here. diff --git a/scripts/verify-email-data-recovery/retained_auxiliary_test.go b/scripts/verify-email-data-recovery/retained_auxiliary_test.go index 37c92eaeb..48c4e1e16 100644 --- a/scripts/verify-email-data-recovery/retained_auxiliary_test.go +++ b/scripts/verify-email-data-recovery/retained_auxiliary_test.go @@ -14,7 +14,11 @@ const ( ) func retainedAuxiliaryFixture(kind, parentKind string) (*recoveryDriver, namespaceinventory.Binding, map[string]namespaceinventory.Binding, object) { - ns := retainedRecovery.namespace + return retainedAuxiliaryFixtureFor(retainedRecovery, kind, parentKind) +} + +func retainedAuxiliaryFixtureFor(target invocation, kind, parentKind string) (*recoveryDriver, namespaceinventory.Binding, map[string]namespaceinventory.Binding, object) { + ns := target.namespace binding := namespaceinventory.Binding{Name: "diagnostic-child", Namespace: ns, UID: "child-canary", Kind: kind, Resource: "policyreports.wgpolicyk8s.io"} api := "wgpolicyk8s.io/v1alpha2" if kind == "CiliumEndpoint" { @@ -33,10 +37,73 @@ func retainedAuxiliaryFixture(kind, parentKind string) (*recoveryDriver, namespa "ServiceAccount/" + ns + "/default": {Name: "default", Namespace: ns, UID: auxiliarySAUID, Kind: "ServiceAccount", Resource: "serviceaccounts"}, } k := &cleanupAPI{objects: map[string]object{cleanupKey(binding.Resource, ns, binding.Name): actual}} - d := &recoveryDriver{i: retainedRecovery, k: k, refs: map[string]ownedReference{}} + d := &recoveryDriver{i: target, k: k, refs: map[string]ownedReference{}} return d, binding, bindings, actual } +func TestFreshRetainedAuxiliaryBindsTheSelectedFixedCopy(t *testing.T) { + for _, tc := range []struct{ kind, parentKind, want string }{ + {"CiliumEndpoint", "Pod", "Pod/email-data-recovery-42818236850/reader"}, + {"PolicyReport", "Pod", "Pod/email-data-recovery-42818236850/reader"}, + {"PolicyReport", "ServiceAccount", "ServiceAccount/email-data-recovery-42818236850/default"}, + } { + t.Run(tc.kind+"/"+tc.parentKind, func(t *testing.T) { + d, binding, bindings, _ := retainedAuxiliaryFixtureFor(freshRetainedRecovery, tc.kind, tc.parentKind) + parent, err := d.retainedAuxiliary(context.Background(), binding, bindings, auxiliaryReaderUID) + if err != nil || parent != tc.want { + t.Fatal("fresh child lost its exact selected-copy parent binding") + } + if tc.parentKind == "Pod" { + delete(bindings, tc.want) + parent, err = d.retainedAuxiliary(context.Background(), binding, bindings, auxiliaryReaderUID) + if err != nil || parent != tc.want { + t.Fatal("fresh child lost its independent reader witness during GC") + } + } + if len(d.refs) != 0 || len(d.k.(*cleanupAPI).objects) != 1 { + t.Fatal("fresh auxiliary observation adopted or mutated a resource") + } + }) + } +} + +func TestFreshRetainedAuxiliaryRefusesCrossCopyBindings(t *testing.T) { + for _, failure := range []string{"original invocation", "foreign invocation", "original binding namespace", "original object namespace", "original report scope", "original serviceaccount binding", "wrong owner UID", "replaced child UID"} { + t.Run(failure, func(t *testing.T) { + d, binding, bindings, actual := retainedAuxiliaryFixtureFor(freshRetainedRecovery, "PolicyReport", "ServiceAccount") + switch failure { + case "original invocation": + d.i = retainedRecovery + case "foreign invocation": + d.i.run = "99999999999" + case "original binding namespace": + binding.Namespace = retainedRecovery.namespace + case "original object namespace": + child(actual, "metadata")["namespace"] = retainedRecovery.namespace + case "original report scope": + child(actual, "scope")["namespace"] = retainedRecovery.namespace + case "original serviceaccount binding": + key := "ServiceAccount/email-data-recovery-42818236850/default" + parent := bindings[key] + delete(bindings, key) + parent.Namespace = retainedRecovery.namespace + bindings["ServiceAccount/"+retainedRecovery.namespace+"/default"] = parent + case "wrong owner UID": + items(child(actual, "metadata"), "ownerReferences")[0]["uid"] = "different-serviceaccount-owner" + case "replaced child UID": + child(actual, "metadata")["uid"] = "replacement-child" + } + parent, err := d.retainedAuxiliary(context.Background(), binding, bindings, auxiliaryReaderUID) + if err == nil || parent != "" { + t.Fatal("fresh child accepted a cross-copy or changed identity") + } + if len(d.refs) != 0 || len(d.k.(*cleanupAPI).objects) != 1 { + t.Fatal("refused fresh observation adopted or mutated a resource") + } + }) + } +} + // Removing owner identity, canonical GVR or scope checks would authorize a // foreign controller child as disposable namespace-generated state. func TestRetainedAuxiliaryBindsOnlyObservedControllerChildren(t *testing.T) { diff --git a/scripts/verify-email-data-recovery/retained_disposal_test.go b/scripts/verify-email-data-recovery/retained_disposal_test.go index a8282cae3..3b490b0f0 100644 --- a/scripts/verify-email-data-recovery/retained_disposal_test.go +++ b/scripts/verify-email-data-recovery/retained_disposal_test.go @@ -96,6 +96,26 @@ func TestRetainedDisposalRefusesWrongDispatchAndLostOutputBeforeAccess(t *testin } } +func TestFreshDisposalRequiresFreshWitnessBeforeAccess(t *testing.T) { + for _, tc := range []struct{ name, witness, stage string }{ + {"missing", "", "fresh-witness"}, + {"old run", strings.Replace(freshFixtureWitness, "42818236850", "41561215415", 1), "fresh-witness"}, + {"exact witness", freshFixtureWitness, "credentials"}, + } { + t.Run(tc.name, func(t *testing.T) { + env := retainedDispatch() + env["EMAIL_RETAINED_COPY_CONFIRM"] = "dispose-fresh-retained-email-copy" + env["EMAIL_FRESH_RECOVERY_WITNESS"] = tc.witness + var out bytes.Buffer + k := &refusedAPI{} + if runRetainedDisposal(context.Background(), func(key string) string { return env[key] }, k, &out) != 2 || + k.calls != 0 || !strings.Contains(out.String(), "EMAIL_FRESH_RETAINED_DISPOSAL=UNVERIFIED stage="+tc.stage+" cleanup=NOT_STARTED") { + t.Fatal("fresh disposal did not fail closed at the expected stage") + } + }) + } +} + func TestRetainedBackupAcceptsObservedDefaultsAndRefusesExpandedScope(t *testing.T) { want := child(backupManifest(retainedRecovery), "spec") actual := object{} diff --git a/scripts/verify-email-data-recovery/retained_entry.go b/scripts/verify-email-data-recovery/retained_entry.go index 7b440252d..994c9752b 100644 --- a/scripts/verify-email-data-recovery/retained_entry.go +++ b/scripts/verify-email-data-recovery/retained_entry.go @@ -13,13 +13,18 @@ import ( "github.com/devantler-tech/platform/internal/namespaceinventory" ) -// Disposal is distinct from the lost original trial. It preserves all source -// messages and the backup and never emits the fresh recovery PASS receipt. +// Disposal preserves all source messages and the backup. It gives the fixed +// original and fresh copies separate receipts without rewriting either run. func runRetainedDisposal(ctx context.Context, env func(string) string, k kubernetes, output io.Writer) int { + fresh := env("EMAIL_RETAINED_COPY_CONFIRM") == namespaceinventory.FreshEmailDisposalConfirm + receipt := "EMAIL_RETAINED_DISPOSAL" + if fresh { + receipt = "EMAIL_FRESH_RETAINED_DISPOSAL" + } stage := "dispatch" cleanup := "NOT_STARTED" fail := func() int { - _, _ = fmt.Fprintf(output, "EMAIL_RETAINED_DISPOSAL=UNVERIFIED stage=%s cleanup=%s\n", stage, cleanup) + _, _ = fmt.Fprintf(output, "%s=UNVERIFIED stage=%s cleanup=%s\n", receipt, stage, cleanup) return 2 } if namespaceinventory.AuthorizeRetainedEmailDispatch(env) != nil { @@ -27,10 +32,28 @@ func runRetainedDisposal(ctx context.Context, env func(string) string, k kuberne } uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) readerUID, claimUID := env("EMAIL_RETAINED_READER_UID"), env("EMAIL_RETAINED_CLAIM_UID") + target := retainedRecovery + observeInventory := namespaceinventory.ObserveRetainedEmailRecovery + var witness *freshRecoveryWitness + var window *retainedMessageWindow + start := "EMAIL_RETAINED_DISPOSAL=STARTED source_writes=0 original_trial=UNVERIFIED" + if fresh { + var err error + stage = "fresh-witness" + witness, err = parseFreshRecoveryWitness(env("EMAIL_FRESH_RECOVERY_WITNESS")) + if err != nil { + return fail() + } + target = freshRetainedRecovery + readerUID, claimUID = witness.Reader.UID, witness.Claim.UID + window = freshRecoveryWindow() + observeInventory = namespaceinventory.ObserveFreshRetainedEmailRecovery + start = "EMAIL_FRESH_RETAINED_DISPOSAL=STARTED source_writes=0" + } if !uuid.MatchString(readerUID) || !uuid.MatchString(claimUID) || readerUID == claimUID { return fail() } - if _, err := fmt.Fprintln(output, "EMAIL_RETAINED_DISPOSAL=STARTED source_writes=0 original_trial=UNVERIFIED"); err != nil { + if _, err := fmt.Fprintln(output, start); err != nil { return 2 } ctx, cancel := context.WithTimeout(ctx, 8*time.Minute) @@ -40,7 +63,7 @@ func runRetainedDisposal(ctx context.Context, env func(string) string, k kuberne if err != nil { return fail() } - d := &recoveryDriver{i: retainedRecovery, k: k, refs: map[string]ownedReference{}} + d := &recoveryDriver{i: target, k: k, refs: map[string]ownedReference{}} stage = "source" if d.source, err = d.readSource(ctx); err != nil { return fail() @@ -52,10 +75,12 @@ func runRetainedDisposal(ctx context.Context, env func(string) string, k kuberne if err = d.adoptRetainedGraph(ctx, readerUID, claimUID); err != nil { return fail() } - if err = d.observeRestoredStorage(ctx); err != nil { - return fail() - } - if err = d.retainedSnapshot(ctx, ""); err != nil { + const cleanupLimit = 10 * time.Minute + if witness != nil { + if stage = verifyFreshDisposalGraph(ctx, d, witness, time.Now().Add(cleanupLimit)); stage != "" { + return fail() + } + } else if d.observeRestoredStorage(ctx) != nil || d.retainedSnapshot(ctx, "") != nil { return fail() } stage = "complete-inventory" @@ -69,7 +94,7 @@ func runRetainedDisposal(ctx context.Context, env func(string) string, k kuberne if e != nil { return e } - inventory, e := namespaceinventory.ObserveRetainedEmailRecovery(ctx, client, endpoint, ns.uid) + inventory, e := observeInventory(ctx, client, endpoint, ns.uid) if e != nil { return e } @@ -164,30 +189,51 @@ func runRetainedDisposal(ctx context.Context, env func(string) string, k kuberne if err != nil { return fail() } - if _, err = verifyRetainedMessage(ctx, reader, source); err != nil { + if _, err = verifyRetainedMessageInWindow(ctx, reader, source, window); err != nil { return fail() } if err = d.sourceRetained(ctx); err != nil || ctx.Err() != nil { return fail() } - if _, err = fmt.Fprintln(output, "EMAIL_RETAINED_COPY=VERIFIED source_writes=0 original_trial=UNVERIFIED"); err != nil { + copyReceipt := "EMAIL_RETAINED_COPY=VERIFIED source_writes=0 original_trial=UNVERIFIED" + pass := "EMAIL_RETAINED_DISPOSAL=PASS source_writes=0 source=RETAINED backup=RETAINED original_trial=UNVERIFIED" + if witness != nil { + if stage = verifyFreshDisposalGraph(ctx, d, witness, time.Now().Add(cleanupLimit)); stage != "" { + return fail() + } + copyReceipt = "EMAIL_FRESH_RETAINED_COPY=VERIFIED source_writes=0" + pass = "EMAIL_FRESH_RETAINED_DISPOSAL=PASS diagnostic=VERIFIED cleanup=VERIFIED source_writes=0 source=RETAINED backup=RETAINED" + } + if _, err = fmt.Fprintln(output, copyReceipt); err != nil { return 2 } stopReader() stopSource() stage = "owned-disposal" cleanup = "UNVERIFIED" - cleanupCtx, finish := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Minute) + cleanupCtx, finish := context.WithTimeout(context.WithoutCancel(ctx), cleanupLimit) defer finish() if err = d.cleanupRestore(cleanupCtx); err != nil { return fail() } - if _, err = fmt.Fprintln(output, "EMAIL_RETAINED_DISPOSAL=PASS source_writes=0 source=RETAINED backup=RETAINED original_trial=UNVERIFIED"); err != nil { + if _, err = fmt.Fprintln(output, pass); err != nil { return 2 } return 0 } +func verifyFreshDisposalGraph(ctx context.Context, d *recoveryDriver, w *freshRecoveryWitness, mustRemainUntil time.Time) string { + ns, err := d.ref("Namespace", d.i.namespace, "") + if err != nil || d.i != freshRetainedRecovery || ns.uid != w.Namespace.UID { + return "fresh-namespace" + } + actual, err := d.owned(ctx, ns) + if err != nil || child(actual, "metadata")["deletionTimestamp"] != nil { + return "fresh-namespace" + } + return observeFreshRecoveryGraph(ctx, d, w, mustRemainUntil) +} + // retainedTransientResource reports an Event, from either Event API. The // kubelet and controllers record Events for the reader's own lifecycle at any // time; they hold no recovered data and expire on their own, so they are neither diff --git a/scripts/verify-email-data-recovery/retained_workflow_test.go b/scripts/verify-email-data-recovery/retained_workflow_test.go index 55cd26df7..c3a52fb18 100644 --- a/scripts/verify-email-data-recovery/retained_workflow_test.go +++ b/scripts/verify-email-data-recovery/retained_workflow_test.go @@ -62,12 +62,20 @@ func TestRetainedDisposalWorkflowMainGuardAndCustody(t *testing.T) { } }) } + cmd := exec.Command("/bin/bash", "--noprofile", "--norc", "-c", guard.Run+"\nprintf FRESH_DISPOSAL_ADMITTED\n") + env["CONFIRM"] = "dispose-fresh-retained-email-copy" + for key, value := range env { + cmd.Env = append(cmd.Env, key+"="+value) + } + if out, err := cmd.CombinedOutput(); err != nil || string(out) != "FRESH_DISPOSAL_ADMITTED" { + t.Fatal("fixed fresh disposal dispatch refused") + } checkout := job.Steps[1] if checkout.With["ref"] != "${{ github.sha }}" || checkout.With["persist-credentials"] != false || !strings.HasPrefix(checkout.Uses, "actions/checkout@") { t.Fatal("checkout lost immutable credential-free binding") } for i, step := range job.Steps { - if i < 5 && (step.Env["KUBE_CONFIG"] != "" || step.Env["HCLOUD_TOKEN"] != "") { + if i < 5 && (step.Env["KUBE_CONFIG"] != "" || step.Env["HCLOUD_TOKEN"] != "" || step.Env["EMAIL_FRESH_RECOVERY_WITNESS"] != "") { t.Fatal("credential restored before build") } } @@ -78,6 +86,9 @@ func TestRetainedDisposalWorkflowMainGuardAndCustody(t *testing.T) { if final.Env["EMAIL_RETAINED_READER_UID"] != "${{ secrets.EMAIL_RETAINED_READER_UID }}" || final.Env["EMAIL_RETAINED_CLAIM_UID"] != "${{ secrets.EMAIL_RETAINED_CLAIM_UID }}" || !strings.Contains(final.Run, `./scripts/verify-prod-recovery-source.sh "$GITHUB_SHA"`) { t.Fatal("disposal lost independent identities or current reviewed source guard") } + if final.Env["EMAIL_FRESH_RECOVERY_WITNESS"] != "${{ inputs.confirm == 'dispose-fresh-retained-email-copy' && secrets.EMAIL_FRESH_RECOVERY_WITNESS || '' }}" { + t.Fatal("fresh witness was not isolated to its exact disposal confirmation") + } for _, step := range job.Steps { if strings.Contains(step.Uses, "upload-artifact") { t.Fatal("private observation output became a public artifact")