devantler · Oct 29, 2026 · Oct 28, 2026 · Oct 28, 2026 · Oct 28, 2026 · Oct 28, 2026
@@ -119,7 +119,7 @@ jobs:
           [[ "$ready" == true ]] || fail
           export OPENBAO_RECOVERY_DIR="$proof_dir"
           stage=collection
-          if ! "$binary" collect > "$proof_dir/collection-result" 2>"$proof_dir/collection.log"; then
+          if ! "$binary" collect-email > "$proof_dir/collection-result" 2>"$proof_dir/collection.log"; then
             # Raw diagnostics stay private; only these closed failure codes leave scratch storage.
             grep -E '^OPENBAO_BACKUP_INPUTS=UNVERIFIED stage=(private-workspace|snapshot-job-read|snapshot-job-admission|live-health|deployment-inputs|backup-target|recovery-inputs|recovery-backup-read|recovery-backup-admission|recovery-key-download|live-records|snapshot-download|private-handoff|collection)$' "$proof_dir/collection.log" >&2 || true
             fail
@@ -135,7 +135,7 @@ jobs:
           image=$("$binary" image) || fail
           if ! docker pull "$image" > "$proof_dir/pull.log" 2>&1; then fail; fi
           stage=isolated-restore
-          if ! bash scripts/run-openbao-backup-sandbox.sh verify "$proof_dir" "$binary" > "$proof_dir/verification-result" 2>"$proof_dir/verification.log"; then
+          if ! bash scripts/run-openbao-backup-sandbox.sh verify-email "$proof_dir" "$binary" > "$proof_dir/verification-result" 2>"$proof_dir/verification.log"; then
             grep -E '^OPENBAO_BACKUP_RESTORE=FAIL stage=(mode|sandbox|server|initialization|inputs|snapshot|restore|seal|unseal|records|membership|restored_state|fixture|verification|runtime)$' "$proof_dir/verification.log" >&2 || true
             fail
           fi

@@ -109,7 +109,16 @@ all three records permits a later consumer-path switch. A plan or an `UNSEEDED`
 result is not proof of migration.
 
 Consumer changes, exact-path authorization and readback must land after the copy
-is verified. Retire the old active keys only after the consumers and recovery
+is verified. Readers and non-overwriting seed writers use the same three
+`infrastructure/email/` records. Their Kubernetes Secret names and generator
+caches remain unchanged. The synchronizer has no write grant for the retained
+legacy records, and no list, delete or wildcard Email grant.
+
+The independent recovery workflow compares the canonical records using its fixed
+`collect-email` and `verify-email` modes. Its legacy modes remain available to
+the guarded one-time copy workflow and retained-history recovery; the canonical
+mode never falls back to a legacy record when a canonical record is absent.
+Retire the old active keys only after the consumers and recovery
 path are proven; preserve recoverable versions rather than destroying metadata.
 The preparation mechanism can then be removed. Rebuilding from an empty store
 is distinct from moving existing credentials and must not replace restored

@@ -11,8 +11,14 @@ storage destination before downloading its exact archive with an immutable
 conditional read. Archive size and expansion are bounded. Only the fixed recovery
 resource is decoded in memory; nothing is extracted to the host filesystem.
 Duplicate or conflicting resource copies, incomplete gzip data, bad checksums
-and unrelated locations refuse success. The data snapshot retains its separate
-freshness requirement from the repaired snapshot job.
+and unrelated locations refuse success. Legacy custody-copy evidence retains its
+specific completed snapshot-job window. Active canonical recovery instead reads
+the newest independent snapshot from the complete bounded listing within the
+last 24 hours, with at most five minutes of clock skew. It does not accept a
+producer window from an input or fall back to an older matching snapshot. This
+allows a completed backup to cover records created after the one-time copy's
+snapshot. Every selected canonical record must still survive the actual restore;
+a recent snapshot that lacks a copied record is not recovery evidence.
 
 The verifier starts the pinned server release in a fresh, network-disabled
 container. The container runs as the non-root runner user, with a read-only root
@@ -24,6 +30,11 @@ rebinding this recovery verifier to the approved running release.
 Only a fixed loopback API is available. The verifier initializes its own temporary
 Raft store, restores the supplied snapshot there, and requires the supplied root
 token to recover every field in all three existing Email credential records.
+The normal main-only workflow selects the three `infrastructure/email/`
+records for both collection and restoration. It refuses missing or conflicting
+canonical data rather than accepting a retained `apps/private-mail/` copy.
+Explicit legacy modes remain for the guarded custody-copy operation and older
+recovery evidence. Neither mode accepts an arbitrary credential path.
 It checks that the restored server has exactly one local voting leader. It then
 seals that disposable server, unseals it with the supplied recovery key and repeats
 the state, membership and record checks. A prior automatic unseal cannot hide a
@@ -37,18 +48,20 @@ vocabulary and retains client diagnostics privately. Cleanup binds an owned
 container to a random label before removing it, including a lost start response.
 An unreadable cleanup result is explicitly unverified.
 
-CI creates a synthetic snapshot with the actual pinned server, archives its
-recovery credentials and removes the original credential files. Every restore
+CI creates synthetic snapshots for both custody layouts with the actual pinned
+server, archives their recovery credentials and removes the original credential
+files. Every restore
 obtains its credentials from the archive reader. It requires successful
 restoration, rejects a corrupted snapshot, rejects an altered expected record,
-and rejects an archived wrong unseal key. Local tests without the explicit native
-test setting skip that container exercise and are not evidence that a restore worked.
+rejects an archived wrong unseal key, and refuses expectations for the other
+custody layout. Local tests without the explicit native test setting skip that
+container exercise and are not evidence that a restore worked.
 
 Do not point this program at production: it has no configurable API address.
 Do not run production handoffs manually through this staging wrapper. The reviewed
 main-only workflow must obtain the fixed inputs, enforce private RAM custody and
 freshness, bind the running server version, and record the final production proof
-before #5513 can close or operational custody changes proceed.
+before operational custody changes proceed.
 
 The separate `Copy Email Credential Custody` workflow repeats this complete
 independent recovery before copying any existing Email credential. It accepts
@@ -60,3 +73,17 @@ totals. It never deletes the legacy records or switches consumers. The existing
 `Verify Email Credential Custody` workflow remains a read-only plan. Storage-key
 generation, consumer switching and encrypted-volume activation are separate
 operations after their own recovery coverage is verified.
+
+Production explicitly uses retained credential custody: its overlay removes
+the three Email PushSecrets while retaining the durable ExternalSecret readers.
+Regenerated local caches therefore cannot create a missing canonical record
+after an older backup is restored. Restore OpenBao first, then run the guarded
+legacy custody copy when the restored backup predates the move. A conflicting
+canonical value remains a refusal; do not delete history to make copying pass.
+
+The fresh-install administration base still generates and seeds all three
+accounts automatically with `IfNotExists` and retained remote history. Use that
+fresh bootstrap bundle only for an explicitly declared new installation, never
+while an existing installation awaits restoration. A fresh production bootstrap
+must restore the retained-custody overlay before routine operation. Missing
+records or regenerated caches alone never establish fresh-install intent.
@@ -14,5 +14,5 @@ spec:
   data:
     - secretKey: password
       remoteRef:
-        key: apps/private-mail/agent
+        key: infrastructure/email/agent
         property: password
@@ -14,5 +14,5 @@ spec:
   data:
     - secretKey: password
       remoteRef:
-        key: apps/private-mail/maintainer
+        key: infrastructure/email/maintainer
         property: password
@@ -17,5 +17,5 @@ spec:
     - match:
         secretKey: password
         remoteRef:
-          remoteKey: apps/private-mail/agent
+          remoteKey: infrastructure/email/agent
           property: password
@@ -17,5 +17,5 @@ spec:
     - match:
         secretKey: password
         remoteRef:
-          remoteKey: apps/private-mail/maintainer
+          remoteKey: infrastructure/email/maintainer
           property: password
@@ -18,5 +18,5 @@ spec:
   data:
     - secretKey: password
       remoteRef:
-        key: apps/private-mail/admin
+        key: infrastructure/email/admin
         property: password
@@ -18,5 +18,5 @@ spec:
     - match:
         secretKey: password
         remoteRef:
-          remoteKey: apps/private-mail/admin
+          remoteKey: infrastructure/email/admin
           property: password
@@ -1047,13 +1047,13 @@ spec:
               }
               # Company Email preparation preserves three durable credentials.
               # Exact-key read/seed only; no mailbox data or wildcard grant.
-              path "secret/data/apps/private-mail/admin" {
+              path "secret/data/infrastructure/email/admin" {
                 capabilities = ["create", "update", "read"]
               }
-              path "secret/data/apps/private-mail/maintainer" {
+              path "secret/data/infrastructure/email/maintainer" {
                 capabilities = ["create", "update", "read"]
               }
-              path "secret/data/apps/private-mail/agent" {
+              path "secret/data/infrastructure/email/agent" {
                 capabilities = ["create", "update", "read"]
               }
               path "secret/data/apps/actual-budget/*" {
@@ -1101,13 +1101,13 @@ spec:
               path "secret/metadata/apps/actual-budget/*" {
                 capabilities = ["create", "update", "read"]
               }
-              path "secret/metadata/apps/private-mail/admin" {
+              path "secret/metadata/infrastructure/email/admin" {
                 capabilities = ["create", "update", "read"]
               }
-              path "secret/metadata/apps/private-mail/maintainer" {
+              path "secret/metadata/infrastructure/email/maintainer" {
                 capabilities = ["create", "update", "read"]
               }
-              path "secret/metadata/apps/private-mail/agent" {
+              path "secret/metadata/infrastructure/email/agent" {
                 capabilities = ["create", "update", "read"]
               }
               path "secret/metadata/apps/actual-budget/*" {

@@ -3,3 +3,9 @@ kind: Kustomization
 resources:
   - ../../../../bases/apps/private-mail-administration
   - secret-settings.enc.yaml
+patches:
+  # Production has retained credential custody. Never infer a fresh installation
+  # from missing records while an older backup is being restored.
+  - path: patches/disable-admin-credential-seeding.yaml
+  - path: patches/disable-maintainer-credential-seeding.yaml
+  - path: patches/disable-agent-credential-seeding.yaml
@@ -0,0 +1,6 @@
+apiVersion: external-secrets.io/v1alpha1
+kind: PushSecret
+metadata:
+  name: private-mail-admin
+  namespace: private-mail
+$patch: delete
@@ -0,0 +1,6 @@
+apiVersion: external-secrets.io/v1alpha1
+kind: PushSecret
+metadata:
+  name: private-mail-agent
+  namespace: private-mail
+$patch: delete
@@ -0,0 +1,6 @@
+apiVersion: external-secrets.io/v1alpha1
+kind: PushSecret
+metadata:
+  name: private-mail-maintainer
+  namespace: private-mail
+$patch: delete
@@ -0,0 +1,27 @@
+package main
+
+import (
+	"testing"
+	"time"
+)
+
+func TestCanonicalRecoveryUsesRecentIndependentBackupsAfterCustodyChanges(t *testing.T) {
+	previous := credentialPaths
+	t.Cleanup(func() { credentialPaths = previous })
+	_, credentialPaths, _ = credentialMode("collect-email")
+	now := time.Now().UTC()
+	start, end, err := freshRecoveryWindow(nil, now)
+	if err != nil || !start.Equal(now.Add(-24*time.Hour)) || !end.Equal(now.Add(5*time.Minute)) {
+		t.Fatal("canonical recovery is still bound to the obsolete one-time copy snapshot")
+	}
+	if _, _, err := freshRecoveryWindow([]byte("undeclared-window"), now); err == nil {
+		t.Fatal("canonical recovery accepted an externally supplied evidence window")
+	}
+	if _, _, err := freshRecoveryWindow(nil, time.Time{}); err == nil {
+		t.Fatal("canonical recovery accepted an absent current clock")
+	}
+	_, credentialPaths, _ = credentialMode("collect")
+	if _, _, err := freshRecoveryWindow(nil, now); err == nil {
+		t.Fatal("legacy copy recovery lost its specific completed-job prerequisite")
+	}
+}
@@ -196,12 +196,16 @@ func collectRecoveryInputs(dir string) (result error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()
 	stage = "snapshot-job-read"
-	job, err := recoveryObject(ctx, "job", "openbao", "vault-snapshot-init")
-	if err != nil {
-		return errEvidence
+	var job []byte
+	if !canonicalRecoverySet() {
+		var err error
+		job, err = recoveryObject(ctx, "job", "openbao", "vault-snapshot-init")
+		if err != nil {
+			return errEvidence
+		}
 	}
 	stage = "snapshot-job-admission"
-	start, end, err := freshJobWindow(job, time.Now().UTC())
+	start, end, err := freshRecoveryWindow(job, time.Now().UTC())
 	if err != nil {
 		return errEvidence
 	}

@@ -0,0 +1,24 @@
+package main
+
+import "slices"
+
+// credentialMode selects only a declared credential set. Legacy modes remain
+// available to the one-time copy workflow and retained-history recovery.
+func credentialMode(requested string) (string, []string, bool) {
+	mode := map[string]string{
+		"image": "image", "collect": "collect", "verify": "verify", "fixture": "fixture",
+		"collect-email": "collect", "verify-email": "verify", "fixture-email": "fixture",
+	}[requested]
+	if mode == "" {
+		return "", nil, false
+	}
+	if requested != mode {
+		return mode, []string{"infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent"}, true
+	}
+	return mode, []string{"apps/private-mail/admin", "apps/private-mail/maintainer", "apps/private-mail/agent"}, true
+}
+
+func canonicalRecoverySet() bool {
+	_, paths, ok := credentialMode("collect-email")
+	return ok && slices.Equal(credentialPaths, paths)
+}
@@ -0,0 +1,30 @@
+package main
+
+import (
+	"slices"
+	"testing"
+)
+
+func TestCredentialModesKeepCanonicalRecoverySeparateFromLegacyCopy(t *testing.T) {
+	legacy := []string{"apps/private-mail/admin", "apps/private-mail/maintainer", "apps/private-mail/agent"}
+	canonical := []string{"infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent"}
+	for _, mode := range []string{"collect", "verify", "fixture"} {
+		for _, suffix := range []string{"", "-email"} {
+			t.Run(mode+suffix, func(t *testing.T) {
+				base, paths, ok := credentialMode(mode + suffix)
+				want := legacy
+				if suffix != "" {
+					want = canonical
+				}
+				if !ok || base != mode || !slices.Equal(paths, want) {
+					t.Fatal("recovery mode selected the wrong fixed credential set")
+				}
+			})
+		}
+	}
+	for _, mode := range []string{"", "image-email", "verify-email-email", "collect-other", "verify/email", "../collect", "fixture-email\n"} {
+		if _, _, ok := credentialMode(mode); ok {
+			t.Fatal("unsupported recovery mode was accepted")
+		}
+	}
+}
@@ -64,8 +64,18 @@ func backupTarget(base, cluster, secret, keys []byte) (target, error) {
 	return result, nil
 }
 
-// freshJobWindow requires a recently completed consumer of the replacement storage.
-func freshJobWindow(raw []byte, now time.Time) (time.Time, time.Time, error) {
+// freshRecoveryWindow requires a recently completed consumer of the replacement storage.
+func freshRecoveryWindow(raw []byte, now time.Time) (time.Time, time.Time, error) {
+	// Active custody can be newer than the one-time copy's snapshot. Its full
+	// records must survive an actual restore from the newest bounded independent
+	// backup, with no user-supplied producer window. Legacy copy keeps its exact
+	// completed-job binding below.
+	if canonicalRecoverySet() {
+		if len(raw) != 0 || now.IsZero() {
+			return time.Time{}, time.Time{}, errEvidence
+		}
+		return now.Add(-24 * time.Hour), now.Add(5 * time.Minute), nil
+	}
 	var job struct {
 		Kind     string `json:"kind"`
 		Metadata struct {

@@ -77,7 +77,7 @@ func TestFreshSnapshotRequiresCompletedReplacementJob(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			_, _, err = freshJobWindow(encoded, now)
+			_, _, err = freshRecoveryWindow(encoded, now)
 			if (err == nil) != (test == "complete") {
 				t.Fatal("incorrect fresh-job verdict")
 			}

@@ -430,23 +430,29 @@ func inputFailureStage(err error) string {
 }
 
 func main() {
-	if len(os.Args) == 2 && os.Args[1] == "collect" {
+	if len(os.Args) != 2 {
+		fmt.Fprintln(os.Stderr, "OPENBAO_BACKUP_RESTORE=FAIL stage=mode")
+		os.Exit(2)
+	}
+	mode, paths, ok := credentialMode(os.Args[1])
+	if !ok {
+		fmt.Fprintln(os.Stderr, "OPENBAO_BACKUP_RESTORE=FAIL stage=mode")
+		os.Exit(2)
+	}
+	credentialPaths = paths
+	if mode == "collect" {
 		if err := collectRecoveryInputs(os.Getenv("OPENBAO_RECOVERY_DIR")); err != nil {
 			fmt.Fprintf(os.Stderr, "OPENBAO_BACKUP_INPUTS=UNVERIFIED stage=%s\n", inputFailureStage(err))
 			os.Exit(2)
 		}
 		fmt.Println("OPENBAO_BACKUP_INPUTS=PASS")
 		return
 	}
-	if len(os.Args) == 2 && os.Args[1] == "image" {
+	if mode == "image" {
 		fmt.Println(serverImage)
 		return
 	}
-	if len(os.Args) != 2 {
-		fmt.Fprintln(os.Stderr, "OPENBAO_BACKUP_RESTORE=FAIL stage=mode")
-		os.Exit(2)
-	}
-	err := sandbox(os.Args[1])
+	err := sandbox(mode)
 	if err != nil {
 		stage := "verification"
 		var known stageError
@@ -456,7 +462,7 @@ func main() {
 		fmt.Printf("OPENBAO_BACKUP_RESTORE=FAIL stage=%s\n", stage)
 		os.Exit(2)
 	}
-	if os.Args[1] == "fixture" {
+	if mode == "fixture" {
 		fmt.Println("OPENBAO_BACKUP_FIXTURE=PASS")
 	} else {
 		fmt.Printf("OPENBAO_BACKUP_RESTORE=PASS records=3 version=%s\n", serverVersion)