From e53bd45bfc3dd2688c20bbe2ab49db0645459191 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 03:58:54 +0100 Subject: [PATCH 1/2] fix(email): release the verified fresh recovery copy Part of #6041. Reuse the reviewed fixed witness and repeat read-only parity before UID-fenced disposal. Preserve source, backup, the earlier retained copy and key-until-backend-absence ordering. --- .../dispose-retained-email-copy.yaml | 11 ++- internal/namespaceinventory/credentials.go | 10 ++- internal/namespaceinventory/inventory.go | 9 ++- internal/namespaceinventory/inventory_test.go | 17 +++++ scripts/verify-email-data-recovery/README.md | 6 ++ .../fresh_observation.go | 5 +- .../fresh_observation_test.go | 41 ++++++++++ .../retained_disposal_test.go | 20 +++++ .../retained_entry.go | 74 +++++++++++++++---- .../retained_workflow_test.go | 13 +++- 10 files changed, 184 insertions(+), 22 deletions(-) diff --git a/.github/workflows/dispose-retained-email-copy.yaml b/.github/workflows/dispose-retained-email-copy.yaml index e40add58c6..2d71da431e 100644 --- a/.github/workflows/dispose-retained-email-copy.yaml +++ b/.github/workflows/dispose-retained-email-copy.yaml @@ -4,7 +4,7 @@ name: Dispose Verified Retained Email Copy workflow_dispatch: inputs: confirm: - description: "Type exactly: dispose-retained-email-copy" + description: "Type dispose-retained-email-copy or dispose-fresh-retained-email-copy" required: true type: string @@ -28,7 +28,7 @@ jobs: env: CONFIRM: ${{ inputs.confirm }} run: | - if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$CONFIRM" != dispose-retained-email-copy ]]; then + if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || ( "$CONFIRM" != dispose-retained-email-copy && "$CONFIRM" != dispose-fresh-retained-email-copy ) ]]; then printf '%s\n' 'EMAIL_RETAINED_DISPOSAL=UNVERIFIED stage=dispatch' >&2 exit 2 fi @@ -70,6 +70,7 @@ jobs: GH_TOKEN: ${{ github.token }} EMAIL_RETAINED_READER_UID: ${{ secrets.EMAIL_RETAINED_READER_UID }} EMAIL_RETAINED_CLAIM_UID: ${{ secrets.EMAIL_RETAINED_CLAIM_UID }} + EMAIL_FRESH_RECOVERY_WITNESS: ${{ inputs.confirm == 'dispose-fresh-retained-email-copy' && secrets.EMAIL_FRESH_RECOVERY_WITNESS || '' }} KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} EMAIL_RETAINED_COPY_CONFIRM: ${{ inputs.confirm }} @@ -78,7 +79,11 @@ jobs: set -euo pipefail umask 077 stage=private-workspace - fail() { printf 'EMAIL_RETAINED_DISPOSAL=UNVERIFIED stage=%s\n' "$stage" >&2; exit 2; } + receipt=EMAIL_RETAINED_DISPOSAL + if [[ "$EMAIL_RETAINED_COPY_CONFIRM" == dispose-fresh-retained-email-copy ]]; then + receipt=EMAIL_FRESH_RETAINED_DISPOSAL + fi + fail() { printf '%s=UNVERIFIED stage=%s cleanup=NOT_STARTED\n' "$receipt" "$stage" >&2; exit 2; } [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail private_dir="$(mktemp -d /dev/shm/email-retained-disposal.XXXXXXXX)" trap 'rm -rf "$private_dir"' EXIT diff --git a/internal/namespaceinventory/credentials.go b/internal/namespaceinventory/credentials.go index 13d4875a05..fabb4c7080 100644 --- a/internal/namespaceinventory/credentials.go +++ b/internal/namespaceinventory/credentials.go @@ -19,6 +19,9 @@ var credentialPath = regexp.MustCompile(`^/dev/shm/longhorn-namespace-inventory\ var sourcePattern = regexp.MustCompile(`^[0-9a-f]{40}$`) var runPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`) +// FreshEmailDisposalConfirm selects the fixed fresh Email recovery copy. +const FreshEmailDisposalConfirm = "dispose-fresh-retained-email-copy" + // Dispatch is the verified source commit and run of a protected observation. type Dispatch struct{ Source, Run string } @@ -88,11 +91,14 @@ func LoadRetainedEmailClient(getenv func(string) string) (*http.Client, *url.URL // confirmed, maintainer-dispatched disposal workflow on main in this repository, with a // well-formed source commit and run id. func AuthorizeRetainedEmailDispatch(getenv func(string) string) error { + confirm := getenv("EMAIL_RETAINED_COPY_CONFIRM") + if confirm != "dispose-retained-email-copy" && confirm != FreshEmailDisposalConfirm { + return errors.New("protected Email disposal dispatch is unverified") + } for key, want := range map[string]string{ "GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", "GITHUB_REPOSITORY": "devantler-tech/platform", "GITHUB_RUN_ATTEMPT": "1", "RUNNER_ENVIRONMENT": "github-hosted", "GITHUB_ACTOR": "devantler", - "GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main", - "EMAIL_RETAINED_COPY_CONFIRM": "dispose-retained-email-copy", + "GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main", } { if getenv(key) != want { return errors.New("protected Email disposal dispatch is unverified") diff --git a/internal/namespaceinventory/inventory.go b/internal/namespaceinventory/inventory.go index 9a7bf113bf..5c876cf0ed 100644 --- a/internal/namespaceinventory/inventory.go +++ b/internal/namespaceinventory/inventory.go @@ -107,12 +107,19 @@ func ObserveRetainedEmailRecovery(ctx context.Context, client *http.Client, endp return observeNamespace(ctx, client, endpoint, "email-data-recovery-41561215415", uid) } +// ObserveFreshRetainedEmailRecovery inventories the fixed fresh restore with +// its separately witnessed namespace identity. +func ObserveFreshRetainedEmailRecovery(ctx context.Context, client *http.Client, endpoint *url.URL, uid string) (Inventory, error) { + return observeNamespace(ctx, client, endpoint, "email-data-recovery-42818236850", uid) +} + func observeNamespace(ctx context.Context, client *http.Client, endpoint *url.URL, namespace, uid string) (Inventory, error) { if endpoint == nil { return Inventory{}, errors.New("authenticated API root is required") } root := *endpoint - if client == nil || root.Scheme != "https" || root.Host == "" || root.Opaque != "" || root.ForceQuery || root.User != nil || root.RawQuery != "" || root.Fragment != "" || (root.Path != "" && root.Path != "/") || (!namespacePattern.MatchString(namespace) && namespace != "longhorn-system" && namespace != "email-data-recovery-41561215415") || len(namespace) > 63 || uid == "" || len(uid) > 128 { + fixed := namespace == "longhorn-system" || namespace == "email-data-recovery-41561215415" || namespace == "email-data-recovery-42818236850" + if client == nil || root.Scheme != "https" || root.Host == "" || root.Opaque != "" || root.ForceQuery || root.User != nil || root.RawQuery != "" || root.Fragment != "" || (root.Path != "" && root.Path != "/") || (!namespacePattern.MatchString(namespace) && !fixed) || len(namespace) > 63 || uid == "" || len(uid) > 128 { return Inventory{}, errors.New("fixed benchmark namespace and authenticated API root are required") } ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) diff --git a/internal/namespaceinventory/inventory_test.go b/internal/namespaceinventory/inventory_test.go index a7f3d2117f..4a1f01b79f 100644 --- a/internal/namespaceinventory/inventory_test.go +++ b/internal/namespaceinventory/inventory_test.go @@ -97,6 +97,21 @@ func TestRetainedEmailInventoryCoversAllServedKinds(t *testing.T) { } } +func TestFreshRetainedEmailInventoryHasItsOwnFixedNamespace(t *testing.T) { + const namespace = "email-data-recovery-42818236850" + result, err := observeFixtureInNamespace(t, "", namespace) + if err != nil || len(result.Bindings()) != 2 { + t.Fatalf("fixed fresh Email inventory failed: %v", err) + } + result, err = observeFixtureInNamespace(t, "namespace replaced", namespace) + if err == nil || len(result.Bindings()) != 0 { + t.Fatal("a replaced fresh namespace was accepted") + } + if result, err = Observe(context.Background(), nil, nil, namespace, "uid"); err == nil || len(result.Bindings()) != 0 { + t.Fatal("fresh namespace admitted by the benchmark API") + } +} + // One stored Event is served by core v1 and events.k8s.io under the same UID. // That alias is one object; any other identity difference stays a conflict. func TestEventAliasesAreOneObject(t *testing.T) { @@ -315,6 +330,8 @@ func observeFixtureInNamespace(t *testing.T, failure, namespace string) (Invento result, err = CalibrateLonghornSystem(ctx, server.Client(), endpoint, "namespace-uid") case "email-data-recovery-41561215415": result, err = ObserveRetainedEmailRecovery(ctx, server.Client(), endpoint, "namespace-uid") + case "email-data-recovery-42818236850": + result, err = ObserveFreshRetainedEmailRecovery(ctx, server.Client(), endpoint, "namespace-uid") default: result, err = Observe(ctx, server.Client(), endpoint, testNamespace, "namespace-uid") } diff --git a/scripts/verify-email-data-recovery/README.md b/scripts/verify-email-data-recovery/README.md index 8f38a7f065..bafe33764a 100644 --- a/scripts/verify-email-data-recovery/README.md +++ b/scripts/verify-email-data-recovery/README.md @@ -100,6 +100,12 @@ encrypted scratch claim. The observer compares the one diagnostic from the protected run's creation window on both copies. Its receipt reports message parity and leaves disposal to a separately reviewed operation. +After the fresh parity check passes, the confirmed +`dispose-fresh-retained-email-copy` mode repeats the graph, inventory and +message checks before disposal. It requires the witnessed backup to remain +valid through the cleanup budget. It releases the fresh reader and scratch +while retaining the source messages and backup, and reports its own result. + The pass receipt covers message recovery, encrypted disposable storage, source retention and disposal of the restored copy. It does not establish an independent cluster rebuild, archive migration, transport delivery, OIDC, or production diff --git a/scripts/verify-email-data-recovery/fresh_observation.go b/scripts/verify-email-data-recovery/fresh_observation.go index 5e230e226e..794333e294 100644 --- a/scripts/verify-email-data-recovery/fresh_observation.go +++ b/scripts/verify-email-data-recovery/fresh_observation.go @@ -87,6 +87,9 @@ func observeFreshRecoveryGraph(ctx context.Context, d *recoveryDriver, w *freshR backupRef := ownedReference{"Backup", d.i.namespace + "-backup", "velero", w.Backup.UID} downloadRef := ownedReference{"DataDownload", d.i.namespace + "-download", "velero", w.Download.UID} for _, ref := range []ownedReference{backupRef, downloadRef} { + if tracked, present := d.refs[ref.key()]; present && tracked != ref { + return "fresh-witness" + } d.refs[ref.key()] = ref } backup, err := d.owned(ctx, backupRef) @@ -116,7 +119,7 @@ func observeFreshRecoveryGraph(ctx context.Context, d *recoveryDriver, w *freshR return "fresh-transfer" } claimRef, err := d.ref("PersistentVolumeClaim", "data", d.i.namespace) - if err != nil { + if err != nil || claimRef.uid != w.Claim.UID { return "fresh-storage" } claim, err := d.owned(ctx, claimRef) diff --git a/scripts/verify-email-data-recovery/fresh_observation_test.go b/scripts/verify-email-data-recovery/fresh_observation_test.go index a4d3cd3d27..451db70aa4 100644 --- a/scripts/verify-email-data-recovery/fresh_observation_test.go +++ b/scripts/verify-email-data-recovery/fresh_observation_test.go @@ -97,3 +97,44 @@ func TestFreshGraphBindsTheUnexpiredBackupToSeparateEncryptedScratch(t *testing. }) } } + +func TestFreshDisposalRequiresLiveNamespaceAndTimeToFinish(t *testing.T) { + w, err := parseFreshRecoveryWitness(freshFixtureWitness) + if err != nil { + t.Fatal("invalid test witness") + } + i := freshRetainedRecovery + _, _, source, _ := backupProofFixture() + beforeExpiry := time.Date(2026, 11, 11, 12, 43, 0, 0, time.UTC) + for _, tc := range []struct { + name, stage string + replace, drift bool + horizon time.Time + }{ + {name: "exact graph", horizon: beforeExpiry}, + {name: "backup expires during cleanup", stage: "fresh-backup", horizon: beforeExpiry.Add(10 * time.Minute)}, + {name: "live namespace replaced", stage: "fresh-namespace", replace: true, horizon: beforeExpiry}, + {name: "frozen backup identity differs", stage: "fresh-witness", drift: true, horizon: beforeExpiry}, + } { + t.Run(tc.name, func(t *testing.T) { + api := freshGraphFixture(w) + ns := recoveryManifest(i, "Namespace", i.namespace, "", "v1") + child(ns, "metadata")["uid"] = w.Namespace.UID + if tc.replace { + child(ns, "metadata")["uid"] = "replacement" + } + api["Namespace"] = ns + nsRef := ownedReference{"Namespace", i.namespace, "", w.Namespace.UID} + claim := ownedReference{"PersistentVolumeClaim", "data", i.namespace, w.Claim.UID} + backup := ownedReference{"Backup", i.namespace + "-backup", "velero", w.Backup.UID} + if tc.drift { + backup.uid = "replacement" + } + refs := map[string]ownedReference{nsRef.key(): nsRef, claim.key(): claim, backup.key(): backup} + d := &recoveryDriver{i: i, k: observationReadOnly{base: api, parity: true, fresh: w}, source: source, refs: refs} + if stage := verifyFreshDisposalGraph(context.Background(), d, w, tc.horizon); stage != tc.stage || d.refs[backup.key()] != backup { + t.Fatalf("expected stable backup binding and stage %q, got %q", tc.stage, stage) + } + }) + } +} diff --git a/scripts/verify-email-data-recovery/retained_disposal_test.go b/scripts/verify-email-data-recovery/retained_disposal_test.go index a8282cae3a..3b490b0f04 100644 --- a/scripts/verify-email-data-recovery/retained_disposal_test.go +++ b/scripts/verify-email-data-recovery/retained_disposal_test.go @@ -96,6 +96,26 @@ func TestRetainedDisposalRefusesWrongDispatchAndLostOutputBeforeAccess(t *testin } } +func TestFreshDisposalRequiresFreshWitnessBeforeAccess(t *testing.T) { + for _, tc := range []struct{ name, witness, stage string }{ + {"missing", "", "fresh-witness"}, + {"old run", strings.Replace(freshFixtureWitness, "42818236850", "41561215415", 1), "fresh-witness"}, + {"exact witness", freshFixtureWitness, "credentials"}, + } { + t.Run(tc.name, func(t *testing.T) { + env := retainedDispatch() + env["EMAIL_RETAINED_COPY_CONFIRM"] = "dispose-fresh-retained-email-copy" + env["EMAIL_FRESH_RECOVERY_WITNESS"] = tc.witness + var out bytes.Buffer + k := &refusedAPI{} + if runRetainedDisposal(context.Background(), func(key string) string { return env[key] }, k, &out) != 2 || + k.calls != 0 || !strings.Contains(out.String(), "EMAIL_FRESH_RETAINED_DISPOSAL=UNVERIFIED stage="+tc.stage+" cleanup=NOT_STARTED") { + t.Fatal("fresh disposal did not fail closed at the expected stage") + } + }) + } +} + func TestRetainedBackupAcceptsObservedDefaultsAndRefusesExpandedScope(t *testing.T) { want := child(backupManifest(retainedRecovery), "spec") actual := object{} diff --git a/scripts/verify-email-data-recovery/retained_entry.go b/scripts/verify-email-data-recovery/retained_entry.go index 7b440252df..994c9752b2 100644 --- a/scripts/verify-email-data-recovery/retained_entry.go +++ b/scripts/verify-email-data-recovery/retained_entry.go @@ -13,13 +13,18 @@ import ( "github.com/devantler-tech/platform/internal/namespaceinventory" ) -// Disposal is distinct from the lost original trial. It preserves all source -// messages and the backup and never emits the fresh recovery PASS receipt. +// Disposal preserves all source messages and the backup. It gives the fixed +// original and fresh copies separate receipts without rewriting either run. func runRetainedDisposal(ctx context.Context, env func(string) string, k kubernetes, output io.Writer) int { + fresh := env("EMAIL_RETAINED_COPY_CONFIRM") == namespaceinventory.FreshEmailDisposalConfirm + receipt := "EMAIL_RETAINED_DISPOSAL" + if fresh { + receipt = "EMAIL_FRESH_RETAINED_DISPOSAL" + } stage := "dispatch" cleanup := "NOT_STARTED" fail := func() int { - _, _ = fmt.Fprintf(output, "EMAIL_RETAINED_DISPOSAL=UNVERIFIED stage=%s cleanup=%s\n", stage, cleanup) + _, _ = fmt.Fprintf(output, "%s=UNVERIFIED stage=%s cleanup=%s\n", receipt, stage, cleanup) return 2 } if namespaceinventory.AuthorizeRetainedEmailDispatch(env) != nil { @@ -27,10 +32,28 @@ func runRetainedDisposal(ctx context.Context, env func(string) string, k kuberne } uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) readerUID, claimUID := env("EMAIL_RETAINED_READER_UID"), env("EMAIL_RETAINED_CLAIM_UID") + target := retainedRecovery + observeInventory := namespaceinventory.ObserveRetainedEmailRecovery + var witness *freshRecoveryWitness + var window *retainedMessageWindow + start := "EMAIL_RETAINED_DISPOSAL=STARTED source_writes=0 original_trial=UNVERIFIED" + if fresh { + var err error + stage = "fresh-witness" + witness, err = parseFreshRecoveryWitness(env("EMAIL_FRESH_RECOVERY_WITNESS")) + if err != nil { + return fail() + } + target = freshRetainedRecovery + readerUID, claimUID = witness.Reader.UID, witness.Claim.UID + window = freshRecoveryWindow() + observeInventory = namespaceinventory.ObserveFreshRetainedEmailRecovery + start = "EMAIL_FRESH_RETAINED_DISPOSAL=STARTED source_writes=0" + } if !uuid.MatchString(readerUID) || !uuid.MatchString(claimUID) || readerUID == claimUID { return fail() } - if _, err := fmt.Fprintln(output, "EMAIL_RETAINED_DISPOSAL=STARTED source_writes=0 original_trial=UNVERIFIED"); err != nil { + if _, err := fmt.Fprintln(output, start); err != nil { return 2 } ctx, cancel := context.WithTimeout(ctx, 8*time.Minute) @@ -40,7 +63,7 @@ func runRetainedDisposal(ctx context.Context, env func(string) string, k kuberne if err != nil { return fail() } - d := &recoveryDriver{i: retainedRecovery, k: k, refs: map[string]ownedReference{}} + d := &recoveryDriver{i: target, k: k, refs: map[string]ownedReference{}} stage = "source" if d.source, err = d.readSource(ctx); err != nil { return fail() @@ -52,10 +75,12 @@ func runRetainedDisposal(ctx context.Context, env func(string) string, k kuberne if err = d.adoptRetainedGraph(ctx, readerUID, claimUID); err != nil { return fail() } - if err = d.observeRestoredStorage(ctx); err != nil { - return fail() - } - if err = d.retainedSnapshot(ctx, ""); err != nil { + const cleanupLimit = 10 * time.Minute + if witness != nil { + if stage = verifyFreshDisposalGraph(ctx, d, witness, time.Now().Add(cleanupLimit)); stage != "" { + return fail() + } + } else if d.observeRestoredStorage(ctx) != nil || d.retainedSnapshot(ctx, "") != nil { return fail() } stage = "complete-inventory" @@ -69,7 +94,7 @@ func runRetainedDisposal(ctx context.Context, env func(string) string, k kuberne if e != nil { return e } - inventory, e := namespaceinventory.ObserveRetainedEmailRecovery(ctx, client, endpoint, ns.uid) + inventory, e := observeInventory(ctx, client, endpoint, ns.uid) if e != nil { return e } @@ -164,30 +189,51 @@ func runRetainedDisposal(ctx context.Context, env func(string) string, k kuberne if err != nil { return fail() } - if _, err = verifyRetainedMessage(ctx, reader, source); err != nil { + if _, err = verifyRetainedMessageInWindow(ctx, reader, source, window); err != nil { return fail() } if err = d.sourceRetained(ctx); err != nil || ctx.Err() != nil { return fail() } - if _, err = fmt.Fprintln(output, "EMAIL_RETAINED_COPY=VERIFIED source_writes=0 original_trial=UNVERIFIED"); err != nil { + copyReceipt := "EMAIL_RETAINED_COPY=VERIFIED source_writes=0 original_trial=UNVERIFIED" + pass := "EMAIL_RETAINED_DISPOSAL=PASS source_writes=0 source=RETAINED backup=RETAINED original_trial=UNVERIFIED" + if witness != nil { + if stage = verifyFreshDisposalGraph(ctx, d, witness, time.Now().Add(cleanupLimit)); stage != "" { + return fail() + } + copyReceipt = "EMAIL_FRESH_RETAINED_COPY=VERIFIED source_writes=0" + pass = "EMAIL_FRESH_RETAINED_DISPOSAL=PASS diagnostic=VERIFIED cleanup=VERIFIED source_writes=0 source=RETAINED backup=RETAINED" + } + if _, err = fmt.Fprintln(output, copyReceipt); err != nil { return 2 } stopReader() stopSource() stage = "owned-disposal" cleanup = "UNVERIFIED" - cleanupCtx, finish := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Minute) + cleanupCtx, finish := context.WithTimeout(context.WithoutCancel(ctx), cleanupLimit) defer finish() if err = d.cleanupRestore(cleanupCtx); err != nil { return fail() } - if _, err = fmt.Fprintln(output, "EMAIL_RETAINED_DISPOSAL=PASS source_writes=0 source=RETAINED backup=RETAINED original_trial=UNVERIFIED"); err != nil { + if _, err = fmt.Fprintln(output, pass); err != nil { return 2 } return 0 } +func verifyFreshDisposalGraph(ctx context.Context, d *recoveryDriver, w *freshRecoveryWitness, mustRemainUntil time.Time) string { + ns, err := d.ref("Namespace", d.i.namespace, "") + if err != nil || d.i != freshRetainedRecovery || ns.uid != w.Namespace.UID { + return "fresh-namespace" + } + actual, err := d.owned(ctx, ns) + if err != nil || child(actual, "metadata")["deletionTimestamp"] != nil { + return "fresh-namespace" + } + return observeFreshRecoveryGraph(ctx, d, w, mustRemainUntil) +} + // retainedTransientResource reports an Event, from either Event API. The // kubelet and controllers record Events for the reader's own lifecycle at any // time; they hold no recovered data and expire on their own, so they are neither diff --git a/scripts/verify-email-data-recovery/retained_workflow_test.go b/scripts/verify-email-data-recovery/retained_workflow_test.go index 55cd26df7e..c3a52fb185 100644 --- a/scripts/verify-email-data-recovery/retained_workflow_test.go +++ b/scripts/verify-email-data-recovery/retained_workflow_test.go @@ -62,12 +62,20 @@ func TestRetainedDisposalWorkflowMainGuardAndCustody(t *testing.T) { } }) } + cmd := exec.Command("/bin/bash", "--noprofile", "--norc", "-c", guard.Run+"\nprintf FRESH_DISPOSAL_ADMITTED\n") + env["CONFIRM"] = "dispose-fresh-retained-email-copy" + for key, value := range env { + cmd.Env = append(cmd.Env, key+"="+value) + } + if out, err := cmd.CombinedOutput(); err != nil || string(out) != "FRESH_DISPOSAL_ADMITTED" { + t.Fatal("fixed fresh disposal dispatch refused") + } checkout := job.Steps[1] if checkout.With["ref"] != "${{ github.sha }}" || checkout.With["persist-credentials"] != false || !strings.HasPrefix(checkout.Uses, "actions/checkout@") { t.Fatal("checkout lost immutable credential-free binding") } for i, step := range job.Steps { - if i < 5 && (step.Env["KUBE_CONFIG"] != "" || step.Env["HCLOUD_TOKEN"] != "") { + if i < 5 && (step.Env["KUBE_CONFIG"] != "" || step.Env["HCLOUD_TOKEN"] != "" || step.Env["EMAIL_FRESH_RECOVERY_WITNESS"] != "") { t.Fatal("credential restored before build") } } @@ -78,6 +86,9 @@ func TestRetainedDisposalWorkflowMainGuardAndCustody(t *testing.T) { if final.Env["EMAIL_RETAINED_READER_UID"] != "${{ secrets.EMAIL_RETAINED_READER_UID }}" || final.Env["EMAIL_RETAINED_CLAIM_UID"] != "${{ secrets.EMAIL_RETAINED_CLAIM_UID }}" || !strings.Contains(final.Run, `./scripts/verify-prod-recovery-source.sh "$GITHUB_SHA"`) { t.Fatal("disposal lost independent identities or current reviewed source guard") } + if final.Env["EMAIL_FRESH_RECOVERY_WITNESS"] != "${{ inputs.confirm == 'dispose-fresh-retained-email-copy' && secrets.EMAIL_FRESH_RECOVERY_WITNESS || '' }}" { + t.Fatal("fresh witness was not isolated to its exact disposal confirmation") + } for _, step := range job.Steps { if strings.Contains(step.Uses, "upload-artifact") { t.Fatal("private observation output became a public artifact") From 9e24db1bae7159ea8509939d088bb9371542a466 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 04:51:42 +0100 Subject: [PATCH 2/2] fix(email): bind cleanup children to the selected restore Part of #6041. Keep auxiliary ownership and scope checks tied to either reviewed fixed recovery copy, including parent garbage collection. Add fresh-copy acceptance and cross-copy refusal coverage. --- .../retained_auxiliary.go | 11 +-- .../retained_auxiliary_test.go | 71 ++++++++++++++++++- 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/scripts/verify-email-data-recovery/retained_auxiliary.go b/scripts/verify-email-data-recovery/retained_auxiliary.go index ae58c4001b..e729734482 100644 --- a/scripts/verify-email-data-recovery/retained_auxiliary.go +++ b/scripts/verify-email-data-recovery/retained_auxiliary.go @@ -19,7 +19,7 @@ var ( // reference was removed; this helper never adopts or deletes an auxiliary. func (d *recoveryDriver) retainedAuxiliary(ctx context.Context, binding namespaceinventory.Binding, bindings map[string]namespaceinventory.Binding, readerUID string) (string, error) { fail := func() (string, error) { return "", errors.New("retained-auxiliary") } - if d == nil || d.k == nil || d.i != retainedRecovery || ctx.Err() != nil || binding.Namespace != retainedRecovery.namespace || len(binding.Name) > 253 || !retainedAuxiliaryName.MatchString(binding.Name) || !retainedAuxiliaryUID.MatchString(binding.UID) || !retainedAuxiliaryUID.MatchString(readerUID) { + if d == nil || d.k == nil || (d.i != retainedRecovery && d.i != freshRetainedRecovery) || ctx.Err() != nil || binding.Namespace != d.i.namespace || len(binding.Name) > 253 || !retainedAuxiliaryName.MatchString(binding.Name) || !retainedAuxiliaryUID.MatchString(binding.UID) || !retainedAuxiliaryUID.MatchString(readerUID) { return fail() } api := "" @@ -51,7 +51,7 @@ func (d *recoveryDriver) retainedAuxiliary(ctx context.Context, binding namespac } } } - parent, ok := retainedAuxiliaryOwner(owner, binding.Kind, bindings, readerUID) + parent, ok := retainedAuxiliaryOwner(d.i, owner, binding.Kind, bindings, readerUID) if !ok { return fail() } @@ -64,8 +64,11 @@ func (d *recoveryDriver) retainedAuxiliary(ctx context.Context, binding namespac return parent, nil } -func retainedAuxiliaryOwner(owner object, childKind string, bindings map[string]namespaceinventory.Binding, readerUID string) (string, bool) { - ns := retainedRecovery.namespace +func retainedAuxiliaryOwner(target invocation, owner object, childKind string, bindings map[string]namespaceinventory.Binding, readerUID string) (string, bool) { + if target != retainedRecovery && target != freshRetainedRecovery { + return "", false + } + ns := target.namespace if text(owner, "kind") == "Pod" && text(owner, "name") == "reader" && text(owner, "uid") == readerUID { // The independent reader witness remains usable while its frozen child // awaits GC after Pod deletion. No current Pod is newly adopted here. diff --git a/scripts/verify-email-data-recovery/retained_auxiliary_test.go b/scripts/verify-email-data-recovery/retained_auxiliary_test.go index 37c92eaebb..48c4e1e160 100644 --- a/scripts/verify-email-data-recovery/retained_auxiliary_test.go +++ b/scripts/verify-email-data-recovery/retained_auxiliary_test.go @@ -14,7 +14,11 @@ const ( ) func retainedAuxiliaryFixture(kind, parentKind string) (*recoveryDriver, namespaceinventory.Binding, map[string]namespaceinventory.Binding, object) { - ns := retainedRecovery.namespace + return retainedAuxiliaryFixtureFor(retainedRecovery, kind, parentKind) +} + +func retainedAuxiliaryFixtureFor(target invocation, kind, parentKind string) (*recoveryDriver, namespaceinventory.Binding, map[string]namespaceinventory.Binding, object) { + ns := target.namespace binding := namespaceinventory.Binding{Name: "diagnostic-child", Namespace: ns, UID: "child-canary", Kind: kind, Resource: "policyreports.wgpolicyk8s.io"} api := "wgpolicyk8s.io/v1alpha2" if kind == "CiliumEndpoint" { @@ -33,10 +37,73 @@ func retainedAuxiliaryFixture(kind, parentKind string) (*recoveryDriver, namespa "ServiceAccount/" + ns + "/default": {Name: "default", Namespace: ns, UID: auxiliarySAUID, Kind: "ServiceAccount", Resource: "serviceaccounts"}, } k := &cleanupAPI{objects: map[string]object{cleanupKey(binding.Resource, ns, binding.Name): actual}} - d := &recoveryDriver{i: retainedRecovery, k: k, refs: map[string]ownedReference{}} + d := &recoveryDriver{i: target, k: k, refs: map[string]ownedReference{}} return d, binding, bindings, actual } +func TestFreshRetainedAuxiliaryBindsTheSelectedFixedCopy(t *testing.T) { + for _, tc := range []struct{ kind, parentKind, want string }{ + {"CiliumEndpoint", "Pod", "Pod/email-data-recovery-42818236850/reader"}, + {"PolicyReport", "Pod", "Pod/email-data-recovery-42818236850/reader"}, + {"PolicyReport", "ServiceAccount", "ServiceAccount/email-data-recovery-42818236850/default"}, + } { + t.Run(tc.kind+"/"+tc.parentKind, func(t *testing.T) { + d, binding, bindings, _ := retainedAuxiliaryFixtureFor(freshRetainedRecovery, tc.kind, tc.parentKind) + parent, err := d.retainedAuxiliary(context.Background(), binding, bindings, auxiliaryReaderUID) + if err != nil || parent != tc.want { + t.Fatal("fresh child lost its exact selected-copy parent binding") + } + if tc.parentKind == "Pod" { + delete(bindings, tc.want) + parent, err = d.retainedAuxiliary(context.Background(), binding, bindings, auxiliaryReaderUID) + if err != nil || parent != tc.want { + t.Fatal("fresh child lost its independent reader witness during GC") + } + } + if len(d.refs) != 0 || len(d.k.(*cleanupAPI).objects) != 1 { + t.Fatal("fresh auxiliary observation adopted or mutated a resource") + } + }) + } +} + +func TestFreshRetainedAuxiliaryRefusesCrossCopyBindings(t *testing.T) { + for _, failure := range []string{"original invocation", "foreign invocation", "original binding namespace", "original object namespace", "original report scope", "original serviceaccount binding", "wrong owner UID", "replaced child UID"} { + t.Run(failure, func(t *testing.T) { + d, binding, bindings, actual := retainedAuxiliaryFixtureFor(freshRetainedRecovery, "PolicyReport", "ServiceAccount") + switch failure { + case "original invocation": + d.i = retainedRecovery + case "foreign invocation": + d.i.run = "99999999999" + case "original binding namespace": + binding.Namespace = retainedRecovery.namespace + case "original object namespace": + child(actual, "metadata")["namespace"] = retainedRecovery.namespace + case "original report scope": + child(actual, "scope")["namespace"] = retainedRecovery.namespace + case "original serviceaccount binding": + key := "ServiceAccount/email-data-recovery-42818236850/default" + parent := bindings[key] + delete(bindings, key) + parent.Namespace = retainedRecovery.namespace + bindings["ServiceAccount/"+retainedRecovery.namespace+"/default"] = parent + case "wrong owner UID": + items(child(actual, "metadata"), "ownerReferences")[0]["uid"] = "different-serviceaccount-owner" + case "replaced child UID": + child(actual, "metadata")["uid"] = "replacement-child" + } + parent, err := d.retainedAuxiliary(context.Background(), binding, bindings, auxiliaryReaderUID) + if err == nil || parent != "" { + t.Fatal("fresh child accepted a cross-copy or changed identity") + } + if len(d.refs) != 0 || len(d.k.(*cleanupAPI).objects) != 1 { + t.Fatal("refused fresh observation adopted or mutated a resource") + } + }) + } +} + // Removing owner identity, canonical GVR or scope checks would authorize a // foreign controller child as disposable namespace-generated state. func TestRetainedAuxiliaryBindsOnlyObservedControllerChildren(t *testing.T) {