diff --git a/.github/workflows/verify-openbao-backup-recovery.yaml b/.github/workflows/verify-openbao-backup-recovery.yaml index c944d41492..f6a1220352 100644 --- a/.github/workflows/verify-openbao-backup-recovery.yaml +++ b/.github/workflows/verify-openbao-backup-recovery.yaml @@ -119,7 +119,7 @@ jobs: [[ "$ready" == true ]] || fail export OPENBAO_RECOVERY_DIR="$proof_dir" stage=collection - if ! "$binary" collect > "$proof_dir/collection-result" 2>"$proof_dir/collection.log"; then + if ! "$binary" collect-email > "$proof_dir/collection-result" 2>"$proof_dir/collection.log"; then # Raw diagnostics stay private; only these closed failure codes leave scratch storage. grep -E '^OPENBAO_BACKUP_INPUTS=UNVERIFIED stage=(private-workspace|snapshot-job-read|snapshot-job-admission|live-health|deployment-inputs|backup-target|recovery-inputs|recovery-backup-read|recovery-backup-admission|recovery-key-download|live-records|snapshot-download|private-handoff|collection)$' "$proof_dir/collection.log" >&2 || true fail @@ -135,7 +135,7 @@ jobs: image=$("$binary" image) || fail if ! docker pull "$image" > "$proof_dir/pull.log" 2>&1; then fail; fi stage=isolated-restore - if ! bash scripts/run-openbao-backup-sandbox.sh verify "$proof_dir" "$binary" > "$proof_dir/verification-result" 2>"$proof_dir/verification.log"; then + if ! bash scripts/run-openbao-backup-sandbox.sh verify-email "$proof_dir" "$binary" > "$proof_dir/verification-result" 2>"$proof_dir/verification.log"; then grep -E '^OPENBAO_BACKUP_RESTORE=FAIL stage=(mode|sandbox|server|initialization|inputs|snapshot|restore|seal|unseal|records|membership|restored_state|fixture|verification|runtime)$' "$proof_dir/verification.log" >&2 || true fail fi diff --git a/docs/adr/email.md b/docs/adr/email.md index 25ffa5e2de..6d1d73c9be 100644 --- a/docs/adr/email.md +++ b/docs/adr/email.md @@ -109,7 +109,16 @@ all three records permits a later consumer-path switch. A plan or an `UNSEEDED` result is not proof of migration. Consumer changes, exact-path authorization and readback must land after the copy -is verified. Retire the old active keys only after the consumers and recovery +is verified. Readers and non-overwriting seed writers use the same three +`infrastructure/email/` records. Their Kubernetes Secret names and generator +caches remain unchanged. The synchronizer has no write grant for the retained +legacy records, and no list, delete or wildcard Email grant. + +The independent recovery workflow compares the canonical records using its fixed +`collect-email` and `verify-email` modes. Its legacy modes remain available to +the guarded one-time copy workflow and retained-history recovery; the canonical +mode never falls back to a legacy record when a canonical record is absent. +Retire the old active keys only after the consumers and recovery path are proven; preserve recoverable versions rather than destroying metadata. The preparation mechanism can then be removed. Rebuilding from an empty store is distinct from moving existing credentials and must not replace restored diff --git a/docs/dr/openbao-recovery-proof.md b/docs/dr/openbao-recovery-proof.md index 4c816039d4..87584a9790 100644 --- a/docs/dr/openbao-recovery-proof.md +++ b/docs/dr/openbao-recovery-proof.md @@ -11,8 +11,14 @@ storage destination before downloading its exact archive with an immutable conditional read. Archive size and expansion are bounded. Only the fixed recovery resource is decoded in memory; nothing is extracted to the host filesystem. Duplicate or conflicting resource copies, incomplete gzip data, bad checksums -and unrelated locations refuse success. The data snapshot retains its separate -freshness requirement from the repaired snapshot job. +and unrelated locations refuse success. Legacy custody-copy evidence retains its +specific completed snapshot-job window. Active canonical recovery instead reads +the newest independent snapshot from the complete bounded listing within the +last 24 hours, with at most five minutes of clock skew. It does not accept a +producer window from an input or fall back to an older matching snapshot. This +allows a completed backup to cover records created after the one-time copy's +snapshot. Every selected canonical record must still survive the actual restore; +a recent snapshot that lacks a copied record is not recovery evidence. The verifier starts the pinned server release in a fresh, network-disabled container. The container runs as the non-root runner user, with a read-only root @@ -24,6 +30,11 @@ rebinding this recovery verifier to the approved running release. Only a fixed loopback API is available. The verifier initializes its own temporary Raft store, restores the supplied snapshot there, and requires the supplied root token to recover every field in all three existing Email credential records. +The normal main-only workflow selects the three `infrastructure/email/` +records for both collection and restoration. It refuses missing or conflicting +canonical data rather than accepting a retained `apps/private-mail/` copy. +Explicit legacy modes remain for the guarded custody-copy operation and older +recovery evidence. Neither mode accepts an arbitrary credential path. It checks that the restored server has exactly one local voting leader. It then seals that disposable server, unseals it with the supplied recovery key and repeats the state, membership and record checks. A prior automatic unseal cannot hide a @@ -37,18 +48,20 @@ vocabulary and retains client diagnostics privately. Cleanup binds an owned container to a random label before removing it, including a lost start response. An unreadable cleanup result is explicitly unverified. -CI creates a synthetic snapshot with the actual pinned server, archives its -recovery credentials and removes the original credential files. Every restore +CI creates synthetic snapshots for both custody layouts with the actual pinned +server, archives their recovery credentials and removes the original credential +files. Every restore obtains its credentials from the archive reader. It requires successful restoration, rejects a corrupted snapshot, rejects an altered expected record, -and rejects an archived wrong unseal key. Local tests without the explicit native -test setting skip that container exercise and are not evidence that a restore worked. +rejects an archived wrong unseal key, and refuses expectations for the other +custody layout. Local tests without the explicit native test setting skip that +container exercise and are not evidence that a restore worked. Do not point this program at production: it has no configurable API address. Do not run production handoffs manually through this staging wrapper. The reviewed main-only workflow must obtain the fixed inputs, enforce private RAM custody and freshness, bind the running server version, and record the final production proof -before #5513 can close or operational custody changes proceed. +before operational custody changes proceed. The separate `Copy Email Credential Custody` workflow repeats this complete independent recovery before copying any existing Email credential. It accepts @@ -60,3 +73,17 @@ totals. It never deletes the legacy records or switches consumers. The existing `Verify Email Credential Custody` workflow remains a read-only plan. Storage-key generation, consumer switching and encrypted-volume activation are separate operations after their own recovery coverage is verified. + +Production explicitly uses retained credential custody: its overlay removes +the three Email PushSecrets while retaining the durable ExternalSecret readers. +Regenerated local caches therefore cannot create a missing canonical record +after an older backup is restored. Restore OpenBao first, then run the guarded +legacy custody copy when the restored backup predates the move. A conflicting +canonical value remains a refusal; do not delete history to make copying pass. + +The fresh-install administration base still generates and seeds all three +accounts automatically with `IfNotExists` and retained remote history. Use that +fresh bootstrap bundle only for an explicitly declared new installation, never +while an existing installation awaits restoration. A fresh production bootstrap +must restore the retained-custody overlay before routine operation. Missing +records or regenerated caches alone never establish fresh-install intent. diff --git a/k8s/bases/apps/private-mail-administration/external-secret-agent.yaml b/k8s/bases/apps/private-mail-administration/external-secret-agent.yaml index 33ddd28c31..3b90690c37 100644 --- a/k8s/bases/apps/private-mail-administration/external-secret-agent.yaml +++ b/k8s/bases/apps/private-mail-administration/external-secret-agent.yaml @@ -14,5 +14,5 @@ spec: data: - secretKey: password remoteRef: - key: apps/private-mail/agent + key: infrastructure/email/agent property: password diff --git a/k8s/bases/apps/private-mail-administration/external-secret-maintainer.yaml b/k8s/bases/apps/private-mail-administration/external-secret-maintainer.yaml index a6d29b09bf..5a2dbf9626 100644 --- a/k8s/bases/apps/private-mail-administration/external-secret-maintainer.yaml +++ b/k8s/bases/apps/private-mail-administration/external-secret-maintainer.yaml @@ -14,5 +14,5 @@ spec: data: - secretKey: password remoteRef: - key: apps/private-mail/maintainer + key: infrastructure/email/maintainer property: password diff --git a/k8s/bases/apps/private-mail-administration/push-secret-agent.yaml b/k8s/bases/apps/private-mail-administration/push-secret-agent.yaml index 52d2c01221..24c52757b9 100644 --- a/k8s/bases/apps/private-mail-administration/push-secret-agent.yaml +++ b/k8s/bases/apps/private-mail-administration/push-secret-agent.yaml @@ -17,5 +17,5 @@ spec: - match: secretKey: password remoteRef: - remoteKey: apps/private-mail/agent + remoteKey: infrastructure/email/agent property: password diff --git a/k8s/bases/apps/private-mail-administration/push-secret-maintainer.yaml b/k8s/bases/apps/private-mail-administration/push-secret-maintainer.yaml index 8b5def8335..62e060ad8c 100644 --- a/k8s/bases/apps/private-mail-administration/push-secret-maintainer.yaml +++ b/k8s/bases/apps/private-mail-administration/push-secret-maintainer.yaml @@ -17,5 +17,5 @@ spec: - match: secretKey: password remoteRef: - remoteKey: apps/private-mail/maintainer + remoteKey: infrastructure/email/maintainer property: password diff --git a/k8s/bases/apps/private-mail/external-secret-admin.yaml b/k8s/bases/apps/private-mail/external-secret-admin.yaml index 8b9a80db2f..2c9bcbadd6 100644 --- a/k8s/bases/apps/private-mail/external-secret-admin.yaml +++ b/k8s/bases/apps/private-mail/external-secret-admin.yaml @@ -18,5 +18,5 @@ spec: data: - secretKey: password remoteRef: - key: apps/private-mail/admin + key: infrastructure/email/admin property: password diff --git a/k8s/bases/apps/private-mail/push-secret-admin.yaml b/k8s/bases/apps/private-mail/push-secret-admin.yaml index 3bca85f7ff..778199f5a5 100644 --- a/k8s/bases/apps/private-mail/push-secret-admin.yaml +++ b/k8s/bases/apps/private-mail/push-secret-admin.yaml @@ -18,5 +18,5 @@ spec: - match: secretKey: password remoteRef: - remoteKey: apps/private-mail/admin + remoteKey: infrastructure/email/admin property: password diff --git a/k8s/bases/infrastructure/vault-config/job.yaml b/k8s/bases/infrastructure/vault-config/job.yaml index f611207a2e..55dc07cc80 100644 --- a/k8s/bases/infrastructure/vault-config/job.yaml +++ b/k8s/bases/infrastructure/vault-config/job.yaml @@ -1047,13 +1047,13 @@ spec: } # Company Email preparation preserves three durable credentials. # Exact-key read/seed only; no mailbox data or wildcard grant. - path "secret/data/apps/private-mail/admin" { + path "secret/data/infrastructure/email/admin" { capabilities = ["create", "update", "read"] } - path "secret/data/apps/private-mail/maintainer" { + path "secret/data/infrastructure/email/maintainer" { capabilities = ["create", "update", "read"] } - path "secret/data/apps/private-mail/agent" { + path "secret/data/infrastructure/email/agent" { capabilities = ["create", "update", "read"] } path "secret/data/apps/actual-budget/*" { @@ -1101,13 +1101,13 @@ spec: path "secret/metadata/apps/actual-budget/*" { capabilities = ["create", "update", "read"] } - path "secret/metadata/apps/private-mail/admin" { + path "secret/metadata/infrastructure/email/admin" { capabilities = ["create", "update", "read"] } - path "secret/metadata/apps/private-mail/maintainer" { + path "secret/metadata/infrastructure/email/maintainer" { capabilities = ["create", "update", "read"] } - path "secret/metadata/apps/private-mail/agent" { + path "secret/metadata/infrastructure/email/agent" { capabilities = ["create", "update", "read"] } path "secret/metadata/apps/actual-budget/*" { diff --git a/k8s/providers/hetzner/apps/private-mail/kustomization.yaml b/k8s/providers/hetzner/apps/private-mail/kustomization.yaml index cd99a861eb..5b297fe705 100644 --- a/k8s/providers/hetzner/apps/private-mail/kustomization.yaml +++ b/k8s/providers/hetzner/apps/private-mail/kustomization.yaml @@ -3,3 +3,9 @@ kind: Kustomization resources: - ../../../../bases/apps/private-mail-administration - secret-settings.enc.yaml +patches: + # Production has retained credential custody. Never infer a fresh installation + # from missing records while an older backup is being restored. + - path: patches/disable-admin-credential-seeding.yaml + - path: patches/disable-maintainer-credential-seeding.yaml + - path: patches/disable-agent-credential-seeding.yaml diff --git a/k8s/providers/hetzner/apps/private-mail/patches/disable-admin-credential-seeding.yaml b/k8s/providers/hetzner/apps/private-mail/patches/disable-admin-credential-seeding.yaml new file mode 100644 index 0000000000..82c0757ccc --- /dev/null +++ b/k8s/providers/hetzner/apps/private-mail/patches/disable-admin-credential-seeding.yaml @@ -0,0 +1,6 @@ +apiVersion: external-secrets.io/v1alpha1 +kind: PushSecret +metadata: + name: private-mail-admin + namespace: private-mail +$patch: delete diff --git a/k8s/providers/hetzner/apps/private-mail/patches/disable-agent-credential-seeding.yaml b/k8s/providers/hetzner/apps/private-mail/patches/disable-agent-credential-seeding.yaml new file mode 100644 index 0000000000..8ef350618a --- /dev/null +++ b/k8s/providers/hetzner/apps/private-mail/patches/disable-agent-credential-seeding.yaml @@ -0,0 +1,6 @@ +apiVersion: external-secrets.io/v1alpha1 +kind: PushSecret +metadata: + name: private-mail-agent + namespace: private-mail +$patch: delete diff --git a/k8s/providers/hetzner/apps/private-mail/patches/disable-maintainer-credential-seeding.yaml b/k8s/providers/hetzner/apps/private-mail/patches/disable-maintainer-credential-seeding.yaml new file mode 100644 index 0000000000..9fb521ea8b --- /dev/null +++ b/k8s/providers/hetzner/apps/private-mail/patches/disable-maintainer-credential-seeding.yaml @@ -0,0 +1,6 @@ +apiVersion: external-secrets.io/v1alpha1 +kind: PushSecret +metadata: + name: private-mail-maintainer + namespace: private-mail +$patch: delete diff --git a/scripts/openbao-backup-recovery/canonical_window_test.go b/scripts/openbao-backup-recovery/canonical_window_test.go new file mode 100644 index 0000000000..f88fc4996f --- /dev/null +++ b/scripts/openbao-backup-recovery/canonical_window_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "testing" + "time" +) + +func TestCanonicalRecoveryUsesRecentIndependentBackupsAfterCustodyChanges(t *testing.T) { + previous := credentialPaths + t.Cleanup(func() { credentialPaths = previous }) + _, credentialPaths, _ = credentialMode("collect-email") + now := time.Now().UTC() + start, end, err := freshRecoveryWindow(nil, now) + if err != nil || !start.Equal(now.Add(-24*time.Hour)) || !end.Equal(now.Add(5*time.Minute)) { + t.Fatal("canonical recovery is still bound to the obsolete one-time copy snapshot") + } + if _, _, err := freshRecoveryWindow([]byte("undeclared-window"), now); err == nil { + t.Fatal("canonical recovery accepted an externally supplied evidence window") + } + if _, _, err := freshRecoveryWindow(nil, time.Time{}); err == nil { + t.Fatal("canonical recovery accepted an absent current clock") + } + _, credentialPaths, _ = credentialMode("collect") + if _, _, err := freshRecoveryWindow(nil, now); err == nil { + t.Fatal("legacy copy recovery lost its specific completed-job prerequisite") + } +} diff --git a/scripts/openbao-backup-recovery/collector.go b/scripts/openbao-backup-recovery/collector.go index 30c53f2e22..947439283a 100644 --- a/scripts/openbao-backup-recovery/collector.go +++ b/scripts/openbao-backup-recovery/collector.go @@ -196,12 +196,16 @@ func collectRecoveryInputs(dir string) (result error) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() stage = "snapshot-job-read" - job, err := recoveryObject(ctx, "job", "openbao", "vault-snapshot-init") - if err != nil { - return errEvidence + var job []byte + if !canonicalRecoverySet() { + var err error + job, err = recoveryObject(ctx, "job", "openbao", "vault-snapshot-init") + if err != nil { + return errEvidence + } } stage = "snapshot-job-admission" - start, end, err := freshJobWindow(job, time.Now().UTC()) + start, end, err := freshRecoveryWindow(job, time.Now().UTC()) if err != nil { return errEvidence } diff --git a/scripts/openbao-backup-recovery/credential_mode.go b/scripts/openbao-backup-recovery/credential_mode.go new file mode 100644 index 0000000000..61915a1e29 --- /dev/null +++ b/scripts/openbao-backup-recovery/credential_mode.go @@ -0,0 +1,24 @@ +package main + +import "slices" + +// credentialMode selects only a declared credential set. Legacy modes remain +// available to the one-time copy workflow and retained-history recovery. +func credentialMode(requested string) (string, []string, bool) { + mode := map[string]string{ + "image": "image", "collect": "collect", "verify": "verify", "fixture": "fixture", + "collect-email": "collect", "verify-email": "verify", "fixture-email": "fixture", + }[requested] + if mode == "" { + return "", nil, false + } + if requested != mode { + return mode, []string{"infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent"}, true + } + return mode, []string{"apps/private-mail/admin", "apps/private-mail/maintainer", "apps/private-mail/agent"}, true +} + +func canonicalRecoverySet() bool { + _, paths, ok := credentialMode("collect-email") + return ok && slices.Equal(credentialPaths, paths) +} diff --git a/scripts/openbao-backup-recovery/credential_mode_test.go b/scripts/openbao-backup-recovery/credential_mode_test.go new file mode 100644 index 0000000000..ba951ee766 --- /dev/null +++ b/scripts/openbao-backup-recovery/credential_mode_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "slices" + "testing" +) + +func TestCredentialModesKeepCanonicalRecoverySeparateFromLegacyCopy(t *testing.T) { + legacy := []string{"apps/private-mail/admin", "apps/private-mail/maintainer", "apps/private-mail/agent"} + canonical := []string{"infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent"} + for _, mode := range []string{"collect", "verify", "fixture"} { + for _, suffix := range []string{"", "-email"} { + t.Run(mode+suffix, func(t *testing.T) { + base, paths, ok := credentialMode(mode + suffix) + want := legacy + if suffix != "" { + want = canonical + } + if !ok || base != mode || !slices.Equal(paths, want) { + t.Fatal("recovery mode selected the wrong fixed credential set") + } + }) + } + } + for _, mode := range []string{"", "image-email", "verify-email-email", "collect-other", "verify/email", "../collect", "fixture-email\n"} { + if _, _, ok := credentialMode(mode); ok { + t.Fatal("unsupported recovery mode was accepted") + } + } +} diff --git a/scripts/openbao-backup-recovery/host.go b/scripts/openbao-backup-recovery/host.go index a9952ff4ae..653e317575 100644 --- a/scripts/openbao-backup-recovery/host.go +++ b/scripts/openbao-backup-recovery/host.go @@ -64,8 +64,18 @@ func backupTarget(base, cluster, secret, keys []byte) (target, error) { return result, nil } -// freshJobWindow requires a recently completed consumer of the replacement storage. -func freshJobWindow(raw []byte, now time.Time) (time.Time, time.Time, error) { +// freshRecoveryWindow requires a recently completed consumer of the replacement storage. +func freshRecoveryWindow(raw []byte, now time.Time) (time.Time, time.Time, error) { + // Active custody can be newer than the one-time copy's snapshot. Its full + // records must survive an actual restore from the newest bounded independent + // backup, with no user-supplied producer window. Legacy copy keeps its exact + // completed-job binding below. + if canonicalRecoverySet() { + if len(raw) != 0 || now.IsZero() { + return time.Time{}, time.Time{}, errEvidence + } + return now.Add(-24 * time.Hour), now.Add(5 * time.Minute), nil + } var job struct { Kind string `json:"kind"` Metadata struct { diff --git a/scripts/openbao-backup-recovery/host_test.go b/scripts/openbao-backup-recovery/host_test.go index d5ba474b47..d08123760d 100644 --- a/scripts/openbao-backup-recovery/host_test.go +++ b/scripts/openbao-backup-recovery/host_test.go @@ -77,7 +77,7 @@ func TestFreshSnapshotRequiresCompletedReplacementJob(t *testing.T) { if err != nil { t.Fatal(err) } - _, _, err = freshJobWindow(encoded, now) + _, _, err = freshRecoveryWindow(encoded, now) if (err == nil) != (test == "complete") { t.Fatal("incorrect fresh-job verdict") } diff --git a/scripts/openbao-backup-recovery/main.go b/scripts/openbao-backup-recovery/main.go index 2c873b6dfd..be4e3de143 100644 --- a/scripts/openbao-backup-recovery/main.go +++ b/scripts/openbao-backup-recovery/main.go @@ -430,7 +430,17 @@ func inputFailureStage(err error) string { } func main() { - if len(os.Args) == 2 && os.Args[1] == "collect" { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "OPENBAO_BACKUP_RESTORE=FAIL stage=mode") + os.Exit(2) + } + mode, paths, ok := credentialMode(os.Args[1]) + if !ok { + fmt.Fprintln(os.Stderr, "OPENBAO_BACKUP_RESTORE=FAIL stage=mode") + os.Exit(2) + } + credentialPaths = paths + if mode == "collect" { if err := collectRecoveryInputs(os.Getenv("OPENBAO_RECOVERY_DIR")); err != nil { fmt.Fprintf(os.Stderr, "OPENBAO_BACKUP_INPUTS=UNVERIFIED stage=%s\n", inputFailureStage(err)) os.Exit(2) @@ -438,15 +448,11 @@ func main() { fmt.Println("OPENBAO_BACKUP_INPUTS=PASS") return } - if len(os.Args) == 2 && os.Args[1] == "image" { + if mode == "image" { fmt.Println(serverImage) return } - if len(os.Args) != 2 { - fmt.Fprintln(os.Stderr, "OPENBAO_BACKUP_RESTORE=FAIL stage=mode") - os.Exit(2) - } - err := sandbox(os.Args[1]) + err := sandbox(mode) if err != nil { stage := "verification" var known stageError @@ -456,7 +462,7 @@ func main() { fmt.Printf("OPENBAO_BACKUP_RESTORE=FAIL stage=%s\n", stage) os.Exit(2) } - if os.Args[1] == "fixture" { + if mode == "fixture" { fmt.Println("OPENBAO_BACKUP_FIXTURE=PASS") } else { fmt.Printf("OPENBAO_BACKUP_RESTORE=PASS records=3 version=%s\n", serverVersion) diff --git a/scripts/openbao-backup-recovery/native_test.go b/scripts/openbao-backup-recovery/native_test.go index f1896e7eac..0c1797a45e 100644 --- a/scripts/openbao-backup-recovery/native_test.go +++ b/scripts/openbao-backup-recovery/native_test.go @@ -14,6 +14,15 @@ import ( // TestPinnedServerRestoresSnapshotAndRejectsInvalidRecovery exercises actual Raft state. func TestPinnedServerRestoresSnapshotAndRejectsInvalidRecovery(t *testing.T) { + testPinnedRecovery(t, "") +} + +func TestPinnedServerRestoresCanonicalEmailCredentials(t *testing.T) { + testPinnedRecovery(t, "-email") +} + +func testPinnedRecovery(t *testing.T, suffix string) { + t.Helper() if os.Getenv("OPENBAO_BACKUP_RECOVERY_NATIVE_TEST") != "1" { t.Skip("actual pinned server proof runs in Linux CI") } @@ -45,7 +54,7 @@ func TestPinnedServerRestoresSnapshotAndRejectsInvalidRecovery(t *testing.T) { out, err := cmd.CombinedOutput() return string(out), err } - if out, err := run("fixture", fixture); err != nil || strings.TrimSpace(out) != "OPENBAO_BACKUP_FIXTURE=PASS" { + if out, err := run("fixture"+suffix, fixture); err != nil || strings.TrimSpace(out) != "OPENBAO_BACKUP_FIXTURE=PASS" { t.Fatalf("native synthetic snapshot creation failed: %v %s", err, out) } rootToken, err := privateInput(filepath.Join(fixture, "root-token"), 16384) @@ -69,7 +78,7 @@ func TestPinnedServerRestoresSnapshotAndRejectsInvalidRecovery(t *testing.T) { t.Fatal("could not remove original synthetic credentials") } } - for _, test := range []string{"matching", "corrupt-snapshot", "different-record", "wrong-unseal-key"} { + for _, test := range []string{"matching", "corrupt-snapshot", "different-record", "wrong-unseal-key", "wrong-custody-scope"} { t.Run(test, func(t *testing.T) { dir, err := os.MkdirTemp("/dev/shm", "openbao-recovery-case-") if err != nil { @@ -117,19 +126,42 @@ func TestPinnedServerRestoresSnapshotAndRejectsInvalidRecovery(t *testing.T) { if err != nil || json.Unmarshal(b, &records) != nil { t.Fatal("invalid synthetic records") } - records[credentialPaths[1]] = json.RawMessage(`{"username":"changed-synthetic-user","password":"` + strings.Repeat("d", 48) + `","retainedField":"synthetic-fixture"}`) + path := "apps/private-mail/maintainer" + if suffix == "-email" { + path = "infrastructure/email/maintainer" + } + records[path] = json.RawMessage(`{"username":"changed-synthetic-user","password":"` + strings.Repeat("d", 48) + `","retainedField":"synthetic-fixture"}`) b, err = json.Marshal(records) if err != nil || os.WriteFile(filepath.Join(dir, "expected.json"), b, 0600) != nil { t.Fatal("could not change synthetic expected record") } + case "wrong-custody-scope": + b, err := os.ReadFile(filepath.Join(dir, "expected.json")) + var records map[string]json.RawMessage + if err != nil || json.Unmarshal(b, &records) != nil { + t.Fatal("invalid synthetic records") + } + wrongScope := map[string]json.RawMessage{} + for key, value := range records { + if suffix == "-email" { + key = strings.Replace(key, "infrastructure/email/", "apps/private-mail/", 1) + } else { + key = strings.Replace(key, "apps/private-mail/", "infrastructure/email/", 1) + } + wrongScope[key] = value + } + b, err = json.Marshal(wrongScope) + if err != nil || os.WriteFile(filepath.Join(dir, "expected.json"), b, 0600) != nil { + t.Fatal("could not change synthetic custody scope") + } } - out, err := run("verify", dir) + out, err := run("verify"+suffix, dir) if test == "matching" { if err != nil || strings.TrimSpace(out) != "OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3" { t.Fatalf("actual snapshot restoration not proven: %v %s", err, out) } } else { - stage := map[string]string{"corrupt-snapshot": "restore", "different-record": "records", "wrong-unseal-key": "unseal"}[test] + stage := map[string]string{"corrupt-snapshot": "restore", "different-record": "records", "wrong-unseal-key": "unseal", "wrong-custody-scope": "inputs"}[test] if err == nil || strings.TrimSpace(out) != "OPENBAO_BACKUP_RESTORE=FAIL stage="+stage { t.Fatalf("native refusal did not prove its intended failure: %v %s", err, out) } diff --git a/scripts/openbao-backup-recovery/workflow_runtime_test.go b/scripts/openbao-backup-recovery/workflow_runtime_test.go index 7530169c5b..0573cf851c 100644 --- a/scripts/openbao-backup-recovery/workflow_runtime_test.go +++ b/scripts/openbao-backup-recovery/workflow_runtime_test.go @@ -62,7 +62,7 @@ exec /bin/sleep 30 "runner/openbao-recovery-tools/verifier": `#!/bin/bash set -eu if [[ $1 == image ]]; then printf '%s\n' '` + serverImage + `'; exit 0; fi -[[ $1 == collect ]] || exit 91 +[[ $1 == collect-email ]] || exit 91 printf '%s' "$OPENBAO_RECOVERY_DIR" > "$FIXTURE_DIR/proof-path" for name in root-token unseal-key expected.json snapshot.snap; do printf '%s' synthetic > "$OPENBAO_RECOVERY_DIR/$name"; done if [[ $FIXTURE_CASE == collection-refused ]]; then printf '%s\n' PRIVATE_SYNTHETIC_PAYLOAD; exit 91; fi @@ -74,7 +74,7 @@ printf '%s\n' OPENBAO_BACKUP_INPUTS=PASS "scripts/run-openbao-backup-sandbox.sh": `#!/bin/bash set -eu printf '%s' reached > "$FIXTURE_DIR/sandbox-started" -[[ $1 == verify ]] || exit 91 +[[ $1 == verify-email ]] || exit 91 if [[ -e $2/kubeconfig ]]; then printf '%s' exposed > "$FIXTURE_DIR/kubeconfig-exposed"; printf '%s\n' PRIVATE_SYNTHETIC_CLUSTER_CREDENTIAL >&2; exit 91; fi for name in root-token unseal-key expected.json snapshot.snap; do [[ -s $2/$name ]] || exit 91; done if [[ $FIXTURE_CASE == restore-stage ]]; then printf '%s\n' 'OPENBAO_BACKUP_RESTORE=FAIL stage=unseal' >&2; exit 91; fi diff --git a/scripts/run-openbao-backup-sandbox.sh b/scripts/run-openbao-backup-sandbox.sh index bc3e842ea2..cba3219706 100755 --- a/scripts/run-openbao-backup-sandbox.sh +++ b/scripts/run-openbao-backup-sandbox.sh @@ -8,7 +8,7 @@ fail() { printf '%s\n' 'OPENBAO_BACKUP_RESTORE=FAIL stage=runtime' >&2; exit 2; mode=$1 proof_dir=$2 binary=$3 -[[ $mode == verify || $mode == fixture ]] || fail +[[ $mode == verify || $mode == fixture || $mode == verify-email || $mode == fixture-email ]] || fail [[ $proof_dir == /* && -d $proof_dir && ! -L $proof_dir && $binary == /* && -f $binary && ! -L $binary && -x $binary ]] || fail [[ $(cd "$proof_dir" && pwd -P) == "$proof_dir" && $(stat -c %a "$proof_dir") == 700 && $(stat -c %u "$proof_dir") == "$(id -u)" ]] || fail image=$("$binary" image) || fail @@ -56,7 +56,7 @@ arguments=(run --rm --name "$name" --label "platform.devantler.tech/recovery-pro --tmpfs "/openbao/logs:ro,nosuid,nodev,noexec,size=1m,mode=0700,uid=$(id -u),gid=$(id -g)" --mount "type=bind,source=$binary,target=/verifier,readonly" --entrypoint /verifier) -if [[ $mode == fixture ]]; then +if [[ $mode == fixture || $mode == fixture-email ]]; then arguments+=(--env OPENBAO_RECOVERY_SYNTHETIC=1 --mount "type=bind,source=$proof_dir,target=/export") else arguments+=(--mount "type=bind,source=$proof_dir,target=/proof,readonly") @@ -65,9 +65,9 @@ fi rc=0 docker "${arguments[@]}" "$image" "$mode" >"$proof_dir/sandbox-result" 2>"$proof_dir/sandbox-error.log" || rc=$? result=$(cat "$proof_dir/sandbox-result") -if [[ $rc == 0 && $mode == fixture && $result == OPENBAO_BACKUP_FIXTURE=PASS ]]; then +if [[ $rc == 0 && ( $mode == fixture || $mode == fixture-email ) && $result == OPENBAO_BACKUP_FIXTURE=PASS ]]; then validated_result=$result -elif [[ $rc == 0 && $mode == verify && $result == 'OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3' ]]; then +elif [[ $rc == 0 && ( $mode == verify || $mode == verify-email ) && $result == 'OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3' ]]; then validated_result=$result else # All container and API diagnostics remain private. Only a fixed helper diff --git a/scripts/tests/arc-staging/mail_integration_test.go b/scripts/tests/arc-staging/mail_integration_test.go index 3ec2894fbe..1593cbaa9e 100644 --- a/scripts/tests/arc-staging/mail_integration_test.go +++ b/scripts/tests/arc-staging/mail_integration_test.go @@ -35,11 +35,11 @@ func TestPrivateMailCredentialsHaveExactSeedAndReadAuthorization(t *testing.T) { t.Fatal("credential synchronizer must receive the seed/read policy") } _, policy := captureBaoWrite(t, "bao", "policy", "write", "vault-seed-write") - entries := regexp.MustCompile(`(?s)path\s+"(secret/(?:data|metadata)/apps/private-mail/[^\"]+)"\s*\{\s*capabilities\s*=\s*(\[[^\]]+\])\s*\}`).FindAllStringSubmatch(policy, -1) + entries := regexp.MustCompile(`(?s)path\s+"(secret/(?:data|metadata)/infrastructure/email/[^\"]+)"\s*\{\s*capabilities\s*=\s*(\[[^\]]+\])\s*\}`).FindAllStringSubmatch(policy, -1) want := map[string]bool{} for _, kind := range []string{"data", "metadata"} { for _, account := range []string{"admin", "maintainer", "agent"} { - want["secret/"+kind+"/apps/private-mail/"+account] = true + want["secret/"+kind+"/infrastructure/email/"+account] = true } } if len(entries) != len(want) { @@ -59,6 +59,27 @@ func TestPrivateMailCredentialsHaveExactSeedAndReadAuthorization(t *testing.T) { t.Fatal("mail credential grant must seed and read without list or delete authority") } } + if strings.Contains(policy, "apps/private-mail/") { + t.Fatal("the synchronizer must no longer write retained legacy credentials") + } + for _, account := range []string{"admin", "maintainer", "agent"} { + directory := "k8s/bases/apps/private-mail-administration/" + if account == "admin" { + directory = "k8s/bases/apps/private-mail/" + } + read := readYAML(t, directory+"external-secret-"+account+".yaml") + seed := readYAML(t, directory+"push-secret-"+account+".yaml") + readData := mailField[[]any](t, read, "spec", "data") + seedData := mailField[[]any](t, seed, "spec", "data") + equal(t, len(readData), 1) + equal(t, len(seedData), 1) + equal(t, mailField[string](t, readData[0], "remoteRef", "key"), "infrastructure/email/"+account) + equal(t, mailField[string](t, seedData[0], "match", "remoteRef", "remoteKey"), "infrastructure/email/"+account) + equal(t, mailField[string](t, read, "spec", "target", "name"), "private-mail-"+account) + equal(t, mailField[string](t, seed, "spec", "selector", "secret", "name"), "private-mail-generated-"+account) + equal(t, mailField[string](t, seed, "spec", "updatePolicy"), "IfNotExists") + equal(t, mailField[string](t, seed, "spec", "deletionPolicy"), "None") + } } // Match the first node rule exactly as declared, including catch-all precedence. diff --git a/scripts/tests/vault-email-credentials/seed_mode_test.go b/scripts/tests/vault-email-credentials/seed_mode_test.go new file mode 100644 index 0000000000..b1310408dc --- /dev/null +++ b/scripts/tests/vault-email-credentials/seed_mode_test.go @@ -0,0 +1,75 @@ +package emailcredentials_test + +import ( + "bytes" + "io" + "os/exec" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestProductionRecoveryCannotSeedFromRegeneratedCredentialCaches(t *testing.T) { + for _, test := range []struct { + path string + seed bool + }{ + {"k8s/bases/apps/private-mail-administration", true}, + {"k8s/providers/hetzner/apps/private-mail", false}, + } { + t.Run(test.path, func(t *testing.T) { + command := exec.Command("kubectl", "kustomize", test.path) + command.Dir = repositoryRoot() + raw, err := command.Output() + if err != nil { + t.Fatal("cannot render the declared credential mode") + } + decoder := yaml.NewDecoder(bytes.NewReader(raw)) + seeded, readers := map[string]bool{}, map[string]bool{} + wanted := map[string]bool{"infrastructure/email/admin": true, "infrastructure/email/maintainer": true, "infrastructure/email/agent": true} + for { + var object struct { + Kind string + Metadata struct{ Namespace string } + Spec struct { + UpdatePolicy string `yaml:"updatePolicy"` + DeletionPolicy string `yaml:"deletionPolicy"` + Data []struct { + RemoteRef struct{ Key string } `yaml:"remoteRef"` + Match struct { + RemoteRef struct { + RemoteKey string `yaml:"remoteKey"` + } `yaml:"remoteRef"` + } + } + } + } + if err := decoder.Decode(&object); err == io.EOF { + break + } else if err != nil { + t.Fatal("cannot decode the declared credential mode") + } + if object.Metadata.Namespace != "private-mail" { + continue + } + for _, data := range object.Spec.Data { + if object.Kind == "PushSecret" && wanted[data.Match.RemoteRef.RemoteKey] { + if !test.seed { + t.Fatal("restored production can seed a missing durable record from a regenerated cache") + } + if seeded[data.Match.RemoteRef.RemoteKey] || object.Spec.UpdatePolicy != "IfNotExists" || object.Spec.DeletionPolicy != "None" { + t.Fatal("fresh initialization lost its create-only retained seed") + } + seeded[data.Match.RemoteRef.RemoteKey] = true + } + if object.Kind == "ExternalSecret" && wanted[data.RemoteRef.Key] { + readers[data.RemoteRef.Key] = true + } + } + } + if len(readers) != 3 || test.seed && len(seeded) != 3 || !test.seed && len(seeded) != 0 { + t.Fatal("credential modes lost their complete durable readers or fresh initialization") + } + }) + } +}