Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions .github/workflows/dispose-retained-email-copy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ name: Dispose Verified Retained Email Copy
workflow_dispatch:
inputs:
confirm:
description: "Type exactly: dispose-retained-email-copy"
description: "Type dispose-retained-email-copy or dispose-fresh-retained-email-copy"
required: true
type: string

Expand All @@ -28,7 +28,7 @@ jobs:
env:
CONFIRM: ${{ inputs.confirm }}
run: |
if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$CONFIRM" != dispose-retained-email-copy ]]; then
if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_WORKFLOW_REF" != devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || ( "$CONFIRM" != dispose-retained-email-copy && "$CONFIRM" != dispose-fresh-retained-email-copy ) ]]; then
printf '%s\n' 'EMAIL_RETAINED_DISPOSAL=UNVERIFIED stage=dispatch' >&2
exit 2
fi
Expand Down Expand Up @@ -70,6 +70,7 @@ jobs:
GH_TOKEN: ${{ github.token }}
EMAIL_RETAINED_READER_UID: ${{ secrets.EMAIL_RETAINED_READER_UID }}
EMAIL_RETAINED_CLAIM_UID: ${{ secrets.EMAIL_RETAINED_CLAIM_UID }}
EMAIL_FRESH_RECOVERY_WITNESS: ${{ inputs.confirm == 'dispose-fresh-retained-email-copy' && secrets.EMAIL_FRESH_RECOVERY_WITNESS || '' }}
KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
EMAIL_RETAINED_COPY_CONFIRM: ${{ inputs.confirm }}
Expand All @@ -78,7 +79,11 @@ jobs:
set -euo pipefail
umask 077
stage=private-workspace
fail() { printf 'EMAIL_RETAINED_DISPOSAL=UNVERIFIED stage=%s\n' "$stage" >&2; exit 2; }
receipt=EMAIL_RETAINED_DISPOSAL
if [[ "$EMAIL_RETAINED_COPY_CONFIRM" == dispose-fresh-retained-email-copy ]]; then
receipt=EMAIL_FRESH_RETAINED_DISPOSAL
fi
fail() { printf '%s=UNVERIFIED stage=%s cleanup=NOT_STARTED\n' "$receipt" "$stage" >&2; exit 2; }
[[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail
private_dir="$(mktemp -d /dev/shm/email-retained-disposal.XXXXXXXX)"
trap 'rm -rf "$private_dir"' EXIT
Expand Down
10 changes: 8 additions & 2 deletions internal/namespaceinventory/credentials.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ var credentialPath = regexp.MustCompile(`^/dev/shm/longhorn-namespace-inventory\
var sourcePattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
var runPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`)

// FreshEmailDisposalConfirm selects the fixed fresh Email recovery copy.
const FreshEmailDisposalConfirm = "dispose-fresh-retained-email-copy"

// Dispatch is the verified source commit and run of a protected observation.
type Dispatch struct{ Source, Run string }

Expand Down Expand Up @@ -88,11 +91,14 @@ func LoadRetainedEmailClient(getenv func(string) string) (*http.Client, *url.URL
// confirmed, maintainer-dispatched disposal workflow on main in this repository, with a
// well-formed source commit and run id.
func AuthorizeRetainedEmailDispatch(getenv func(string) string) error {
confirm := getenv("EMAIL_RETAINED_COPY_CONFIRM")
if confirm != "dispose-retained-email-copy" && confirm != FreshEmailDisposalConfirm {
return errors.New("protected Email disposal dispatch is unverified")
}
for key, want := range map[string]string{
"GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", "GITHUB_REPOSITORY": "devantler-tech/platform",
"GITHUB_RUN_ATTEMPT": "1", "RUNNER_ENVIRONMENT": "github-hosted", "GITHUB_ACTOR": "devantler",
"GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main",
"EMAIL_RETAINED_COPY_CONFIRM": "dispose-retained-email-copy",
"GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/dispose-retained-email-copy.yaml@refs/heads/main",
} {
if getenv(key) != want {
return errors.New("protected Email disposal dispatch is unverified")
Expand Down
9 changes: 8 additions & 1 deletion internal/namespaceinventory/inventory.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,19 @@ func ObserveRetainedEmailRecovery(ctx context.Context, client *http.Client, endp
return observeNamespace(ctx, client, endpoint, "email-data-recovery-41561215415", uid)
}

// ObserveFreshRetainedEmailRecovery inventories the fixed fresh restore with
// its separately witnessed namespace identity.
func ObserveFreshRetainedEmailRecovery(ctx context.Context, client *http.Client, endpoint *url.URL, uid string) (Inventory, error) {
return observeNamespace(ctx, client, endpoint, "email-data-recovery-42818236850", uid)
}

func observeNamespace(ctx context.Context, client *http.Client, endpoint *url.URL, namespace, uid string) (Inventory, error) {
if endpoint == nil {
return Inventory{}, errors.New("authenticated API root is required")
}
root := *endpoint
if client == nil || root.Scheme != "https" || root.Host == "" || root.Opaque != "" || root.ForceQuery || root.User != nil || root.RawQuery != "" || root.Fragment != "" || (root.Path != "" && root.Path != "/") || (!namespacePattern.MatchString(namespace) && namespace != "longhorn-system" && namespace != "email-data-recovery-41561215415") || len(namespace) > 63 || uid == "" || len(uid) > 128 {
fixed := namespace == "longhorn-system" || namespace == "email-data-recovery-41561215415" || namespace == "email-data-recovery-42818236850"
if client == nil || root.Scheme != "https" || root.Host == "" || root.Opaque != "" || root.ForceQuery || root.User != nil || root.RawQuery != "" || root.Fragment != "" || (root.Path != "" && root.Path != "/") || (!namespacePattern.MatchString(namespace) && !fixed) || len(namespace) > 63 || uid == "" || len(uid) > 128 {
return Inventory{}, errors.New("fixed benchmark namespace and authenticated API root are required")
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
Expand Down
17 changes: 17 additions & 0 deletions internal/namespaceinventory/inventory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,21 @@ func TestRetainedEmailInventoryCoversAllServedKinds(t *testing.T) {
}
}

func TestFreshRetainedEmailInventoryHasItsOwnFixedNamespace(t *testing.T) {
const namespace = "email-data-recovery-42818236850"
result, err := observeFixtureInNamespace(t, "", namespace)
if err != nil || len(result.Bindings()) != 2 {
t.Fatalf("fixed fresh Email inventory failed: %v", err)
}
result, err = observeFixtureInNamespace(t, "namespace replaced", namespace)
if err == nil || len(result.Bindings()) != 0 {
t.Fatal("a replaced fresh namespace was accepted")
}
if result, err = Observe(context.Background(), nil, nil, namespace, "uid"); err == nil || len(result.Bindings()) != 0 {
t.Fatal("fresh namespace admitted by the benchmark API")
}
}

// One stored Event is served by core v1 and events.k8s.io under the same UID.
// That alias is one object; any other identity difference stays a conflict.
func TestEventAliasesAreOneObject(t *testing.T) {
Expand Down Expand Up @@ -315,6 +330,8 @@ func observeFixtureInNamespace(t *testing.T, failure, namespace string) (Invento
result, err = CalibrateLonghornSystem(ctx, server.Client(), endpoint, "namespace-uid")
case "email-data-recovery-41561215415":
result, err = ObserveRetainedEmailRecovery(ctx, server.Client(), endpoint, "namespace-uid")
case "email-data-recovery-42818236850":
result, err = ObserveFreshRetainedEmailRecovery(ctx, server.Client(), endpoint, "namespace-uid")
default:
result, err = Observe(ctx, server.Client(), endpoint, testNamespace, "namespace-uid")
}
Expand Down
6 changes: 6 additions & 0 deletions scripts/verify-email-data-recovery/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,12 @@ encrypted scratch claim. The observer compares the one diagnostic from the
protected run's creation window on both copies. Its receipt reports message
parity and leaves disposal to a separately reviewed operation.

After the fresh parity check passes, the confirmed
`dispose-fresh-retained-email-copy` mode repeats the graph, inventory and
message checks before disposal. It requires the witnessed backup to remain
valid through the cleanup budget. It releases the fresh reader and scratch
while retaining the source messages and backup, and reports its own result.

The pass receipt covers message recovery, encrypted disposable storage, source
retention and disposal of the restored copy. It does not establish an independent
cluster rebuild, archive migration, transport delivery, OIDC, or production
Expand Down
5 changes: 4 additions & 1 deletion scripts/verify-email-data-recovery/fresh_observation.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ func observeFreshRecoveryGraph(ctx context.Context, d *recoveryDriver, w *freshR
backupRef := ownedReference{"Backup", d.i.namespace + "-backup", "velero", w.Backup.UID}
downloadRef := ownedReference{"DataDownload", d.i.namespace + "-download", "velero", w.Download.UID}
for _, ref := range []ownedReference{backupRef, downloadRef} {
if tracked, present := d.refs[ref.key()]; present && tracked != ref {
return "fresh-witness"
}
d.refs[ref.key()] = ref
}
backup, err := d.owned(ctx, backupRef)
Expand Down Expand Up @@ -116,7 +119,7 @@ func observeFreshRecoveryGraph(ctx context.Context, d *recoveryDriver, w *freshR
return "fresh-transfer"
}
claimRef, err := d.ref("PersistentVolumeClaim", "data", d.i.namespace)
if err != nil {
if err != nil || claimRef.uid != w.Claim.UID {
return "fresh-storage"
}
claim, err := d.owned(ctx, claimRef)
Expand Down
41 changes: 41 additions & 0 deletions scripts/verify-email-data-recovery/fresh_observation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,3 +97,44 @@ func TestFreshGraphBindsTheUnexpiredBackupToSeparateEncryptedScratch(t *testing.
})
}
}

func TestFreshDisposalRequiresLiveNamespaceAndTimeToFinish(t *testing.T) {
w, err := parseFreshRecoveryWitness(freshFixtureWitness)
if err != nil {
t.Fatal("invalid test witness")
}
i := freshRetainedRecovery
_, _, source, _ := backupProofFixture()
beforeExpiry := time.Date(2026, 11, 11, 12, 43, 0, 0, time.UTC)
for _, tc := range []struct {
name, stage string
replace, drift bool
horizon time.Time
}{
{name: "exact graph", horizon: beforeExpiry},
{name: "backup expires during cleanup", stage: "fresh-backup", horizon: beforeExpiry.Add(10 * time.Minute)},
{name: "live namespace replaced", stage: "fresh-namespace", replace: true, horizon: beforeExpiry},
{name: "frozen backup identity differs", stage: "fresh-witness", drift: true, horizon: beforeExpiry},
} {
t.Run(tc.name, func(t *testing.T) {
api := freshGraphFixture(w)
ns := recoveryManifest(i, "Namespace", i.namespace, "", "v1")
child(ns, "metadata")["uid"] = w.Namespace.UID
if tc.replace {
child(ns, "metadata")["uid"] = "replacement"
}
api["Namespace"] = ns
nsRef := ownedReference{"Namespace", i.namespace, "", w.Namespace.UID}
claim := ownedReference{"PersistentVolumeClaim", "data", i.namespace, w.Claim.UID}
backup := ownedReference{"Backup", i.namespace + "-backup", "velero", w.Backup.UID}
if tc.drift {
backup.uid = "replacement"
}
refs := map[string]ownedReference{nsRef.key(): nsRef, claim.key(): claim, backup.key(): backup}
d := &recoveryDriver{i: i, k: observationReadOnly{base: api, parity: true, fresh: w}, source: source, refs: refs}
if stage := verifyFreshDisposalGraph(context.Background(), d, w, tc.horizon); stage != tc.stage || d.refs[backup.key()] != backup {
t.Fatalf("expected stable backup binding and stage %q, got %q", tc.stage, stage)
}
})
}
}
11 changes: 7 additions & 4 deletions scripts/verify-email-data-recovery/retained_auxiliary.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ var (
// reference was removed; this helper never adopts or deletes an auxiliary.
func (d *recoveryDriver) retainedAuxiliary(ctx context.Context, binding namespaceinventory.Binding, bindings map[string]namespaceinventory.Binding, readerUID string) (string, error) {
fail := func() (string, error) { return "", errors.New("retained-auxiliary") }
if d == nil || d.k == nil || d.i != retainedRecovery || ctx.Err() != nil || binding.Namespace != retainedRecovery.namespace || len(binding.Name) > 253 || !retainedAuxiliaryName.MatchString(binding.Name) || !retainedAuxiliaryUID.MatchString(binding.UID) || !retainedAuxiliaryUID.MatchString(readerUID) {
if d == nil || d.k == nil || (d.i != retainedRecovery && d.i != freshRetainedRecovery) || ctx.Err() != nil || binding.Namespace != d.i.namespace || len(binding.Name) > 253 || !retainedAuxiliaryName.MatchString(binding.Name) || !retainedAuxiliaryUID.MatchString(binding.UID) || !retainedAuxiliaryUID.MatchString(readerUID) {
return fail()
}
api := ""
Expand Down Expand Up @@ -51,7 +51,7 @@ func (d *recoveryDriver) retainedAuxiliary(ctx context.Context, binding namespac
}
}
}
parent, ok := retainedAuxiliaryOwner(owner, binding.Kind, bindings, readerUID)
parent, ok := retainedAuxiliaryOwner(d.i, owner, binding.Kind, bindings, readerUID)
if !ok {
return fail()
}
Expand All @@ -64,8 +64,11 @@ func (d *recoveryDriver) retainedAuxiliary(ctx context.Context, binding namespac
return parent, nil
}

func retainedAuxiliaryOwner(owner object, childKind string, bindings map[string]namespaceinventory.Binding, readerUID string) (string, bool) {
ns := retainedRecovery.namespace
func retainedAuxiliaryOwner(target invocation, owner object, childKind string, bindings map[string]namespaceinventory.Binding, readerUID string) (string, bool) {
if target != retainedRecovery && target != freshRetainedRecovery {
return "", false
}
ns := target.namespace
if text(owner, "kind") == "Pod" && text(owner, "name") == "reader" && text(owner, "uid") == readerUID {
// The independent reader witness remains usable while its frozen child
// awaits GC after Pod deletion. No current Pod is newly adopted here.
Expand Down
71 changes: 69 additions & 2 deletions scripts/verify-email-data-recovery/retained_auxiliary_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@ const (
)

func retainedAuxiliaryFixture(kind, parentKind string) (*recoveryDriver, namespaceinventory.Binding, map[string]namespaceinventory.Binding, object) {
ns := retainedRecovery.namespace
return retainedAuxiliaryFixtureFor(retainedRecovery, kind, parentKind)
}

func retainedAuxiliaryFixtureFor(target invocation, kind, parentKind string) (*recoveryDriver, namespaceinventory.Binding, map[string]namespaceinventory.Binding, object) {
ns := target.namespace
binding := namespaceinventory.Binding{Name: "diagnostic-child", Namespace: ns, UID: "child-canary", Kind: kind, Resource: "policyreports.wgpolicyk8s.io"}
api := "wgpolicyk8s.io/v1alpha2"
if kind == "CiliumEndpoint" {
Expand All @@ -33,10 +37,73 @@ func retainedAuxiliaryFixture(kind, parentKind string) (*recoveryDriver, namespa
"ServiceAccount/" + ns + "/default": {Name: "default", Namespace: ns, UID: auxiliarySAUID, Kind: "ServiceAccount", Resource: "serviceaccounts"},
}
k := &cleanupAPI{objects: map[string]object{cleanupKey(binding.Resource, ns, binding.Name): actual}}
d := &recoveryDriver{i: retainedRecovery, k: k, refs: map[string]ownedReference{}}
d := &recoveryDriver{i: target, k: k, refs: map[string]ownedReference{}}
return d, binding, bindings, actual
}

func TestFreshRetainedAuxiliaryBindsTheSelectedFixedCopy(t *testing.T) {
for _, tc := range []struct{ kind, parentKind, want string }{
{"CiliumEndpoint", "Pod", "Pod/email-data-recovery-42818236850/reader"},
{"PolicyReport", "Pod", "Pod/email-data-recovery-42818236850/reader"},
{"PolicyReport", "ServiceAccount", "ServiceAccount/email-data-recovery-42818236850/default"},
} {
t.Run(tc.kind+"/"+tc.parentKind, func(t *testing.T) {
d, binding, bindings, _ := retainedAuxiliaryFixtureFor(freshRetainedRecovery, tc.kind, tc.parentKind)
parent, err := d.retainedAuxiliary(context.Background(), binding, bindings, auxiliaryReaderUID)
if err != nil || parent != tc.want {
t.Fatal("fresh child lost its exact selected-copy parent binding")
}
if tc.parentKind == "Pod" {
delete(bindings, tc.want)
parent, err = d.retainedAuxiliary(context.Background(), binding, bindings, auxiliaryReaderUID)
if err != nil || parent != tc.want {
t.Fatal("fresh child lost its independent reader witness during GC")
}
}
if len(d.refs) != 0 || len(d.k.(*cleanupAPI).objects) != 1 {
t.Fatal("fresh auxiliary observation adopted or mutated a resource")
}
})
}
}

func TestFreshRetainedAuxiliaryRefusesCrossCopyBindings(t *testing.T) {
for _, failure := range []string{"original invocation", "foreign invocation", "original binding namespace", "original object namespace", "original report scope", "original serviceaccount binding", "wrong owner UID", "replaced child UID"} {
t.Run(failure, func(t *testing.T) {
d, binding, bindings, actual := retainedAuxiliaryFixtureFor(freshRetainedRecovery, "PolicyReport", "ServiceAccount")
switch failure {
case "original invocation":
d.i = retainedRecovery
case "foreign invocation":
d.i.run = "99999999999"
case "original binding namespace":
binding.Namespace = retainedRecovery.namespace
case "original object namespace":
child(actual, "metadata")["namespace"] = retainedRecovery.namespace
case "original report scope":
child(actual, "scope")["namespace"] = retainedRecovery.namespace
case "original serviceaccount binding":
key := "ServiceAccount/email-data-recovery-42818236850/default"
parent := bindings[key]
delete(bindings, key)
parent.Namespace = retainedRecovery.namespace
bindings["ServiceAccount/"+retainedRecovery.namespace+"/default"] = parent
case "wrong owner UID":
items(child(actual, "metadata"), "ownerReferences")[0]["uid"] = "different-serviceaccount-owner"
case "replaced child UID":
child(actual, "metadata")["uid"] = "replacement-child"
}
parent, err := d.retainedAuxiliary(context.Background(), binding, bindings, auxiliaryReaderUID)
if err == nil || parent != "" {
t.Fatal("fresh child accepted a cross-copy or changed identity")
}
if len(d.refs) != 0 || len(d.k.(*cleanupAPI).objects) != 1 {
t.Fatal("refused fresh observation adopted or mutated a resource")
}
})
}
}

// Removing owner identity, canonical GVR or scope checks would authorize a
// foreign controller child as disposable namespace-generated state.
func TestRetainedAuxiliaryBindsOnlyObservedControllerChildren(t *testing.T) {
Expand Down
20 changes: 20 additions & 0 deletions scripts/verify-email-data-recovery/retained_disposal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,26 @@ func TestRetainedDisposalRefusesWrongDispatchAndLostOutputBeforeAccess(t *testin
}
}

func TestFreshDisposalRequiresFreshWitnessBeforeAccess(t *testing.T) {
for _, tc := range []struct{ name, witness, stage string }{
{"missing", "", "fresh-witness"},
{"old run", strings.Replace(freshFixtureWitness, "42818236850", "41561215415", 1), "fresh-witness"},
{"exact witness", freshFixtureWitness, "credentials"},
} {
t.Run(tc.name, func(t *testing.T) {
env := retainedDispatch()
env["EMAIL_RETAINED_COPY_CONFIRM"] = "dispose-fresh-retained-email-copy"
env["EMAIL_FRESH_RECOVERY_WITNESS"] = tc.witness
var out bytes.Buffer
k := &refusedAPI{}
if runRetainedDisposal(context.Background(), func(key string) string { return env[key] }, k, &out) != 2 ||
k.calls != 0 || !strings.Contains(out.String(), "EMAIL_FRESH_RETAINED_DISPOSAL=UNVERIFIED stage="+tc.stage+" cleanup=NOT_STARTED") {
t.Fatal("fresh disposal did not fail closed at the expected stage")
}
})
}
}

func TestRetainedBackupAcceptsObservedDefaultsAndRefusesExpandedScope(t *testing.T) {
want := child(backupManifest(retainedRecovery), "spec")
actual := object{}
Expand Down
Loading
Loading