From f5ab4e2037e3f49f82ccd6a68acb71643b07a5c9 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 19:20:13 +0100 Subject: [PATCH 1/7] feat(homepage): give mail its own Email section Part of #5387. Keep the enforced service-group catalogue aligned with the layout and move the existing Stalwart link without changing its destination. --- k8s/bases/apps/homepage/config-map.yaml | 2 ++ k8s/bases/apps/private-mail-administration/http-route.yaml | 2 +- .../best-practices/restrict-homepage-service-groups.yaml | 3 ++- scripts/tests/private-mail-preparation/production_test.go | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/k8s/bases/apps/homepage/config-map.yaml b/k8s/bases/apps/homepage/config-map.yaml index 4d60151981..9b1ce8cba6 100644 --- a/k8s/bases/apps/homepage/config-map.yaml +++ b/k8s/bases/apps/homepage/config-map.yaml @@ -119,6 +119,8 @@ data: icon: mdi-book-open-page-variant Clients: icon: mdi-briefcase-account + Email: + icon: mdi-email-outline Personal: icon: mdi-web # Bookmark groups (bookmarks.yaml). Listing them here is what gives them a diff --git a/k8s/bases/apps/private-mail-administration/http-route.yaml b/k8s/bases/apps/private-mail-administration/http-route.yaml index 69a93253d6..ae19e0d8d8 100644 --- a/k8s/bases/apps/private-mail-administration/http-route.yaml +++ b/k8s/bases/apps/private-mail-administration/http-route.yaml @@ -7,7 +7,7 @@ metadata: gethomepage.dev/enabled: "true" gethomepage.dev/name: Stalwart gethomepage.dev/description: Private mail administration. - gethomepage.dev/group: Personal + gethomepage.dev/group: Email gethomepage.dev/icon: mdi-email-lock gethomepage.dev/href: https://mail.${domain}/admin gethomepage.dev/pod-selector: app=private-mail diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-homepage-service-groups.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-homepage-service-groups.yaml index 898979eb85..cfd60850d4 100644 --- a/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-homepage-service-groups.yaml +++ b/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-homepage-service-groups.yaml @@ -91,7 +91,7 @@ spec: layout (k8s/bases/apps/homepage/config-map.yaml): Internal Developer Platform, Kubernetes, Observability, Security, Diagnostics, Device Management, Infrastructure as Code, Cloud, Network, Home, Finance, - Analytics, Data, Business, Products, Clients, or Personal. + Analytics, Data, Business, Products, Clients, Email, or Personal. An undeclared group still renders, but sorts below every declared group with a generic icon. Bookmark group names are deliberately not accepted here. @@ -118,4 +118,5 @@ spec: - Business - Products - Clients + - Email - Personal diff --git a/scripts/tests/private-mail-preparation/production_test.go b/scripts/tests/private-mail-preparation/production_test.go index b133abfeb6..01e9f5656d 100644 --- a/scripts/tests/private-mail-preparation/production_test.go +++ b/scripts/tests/private-mail-preparation/production_test.go @@ -31,7 +31,7 @@ func TestProductionAdministrationUsesEncryptedSettings(t *testing.T) { } route := object(t, objects, "HTTPRoute", "private-mail") annotations := mapping(t, mapping(t, route["metadata"])["annotations"]) - if annotations["gethomepage.dev/name"] != "Stalwart" || annotations["gethomepage.dev/group"] != "Personal" { + if annotations["gethomepage.dev/name"] != "Stalwart" || annotations["gethomepage.dev/group"] != "Email" { t.Fatal("production must register the Stalwart Homepage entry") } for _, o := range objects { From 4ca291b8fcd29a71a6513ec27e4ad2d2b75b5971 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 20:18:48 +0100 Subject: [PATCH 2/7] fix(mail): declare the authentication recovery baseline Part of #5387. Explicitly recover local authentication and the managed administrator permissions before the OIDC transition. Readback rejects stale external authentication and a suspended administrator; extend the native replay fixture without replacing its accounts or mail. --- .../authentication_baseline_test.go | 85 +++++++++++++++++++ scripts/private-mail-config/config.go | 6 ++ .../tests/test-private-mail-configuration.sh | 3 + 3 files changed, 94 insertions(+) create mode 100644 scripts/private-mail-config/authentication_baseline_test.go diff --git a/scripts/private-mail-config/authentication_baseline_test.go b/scripts/private-mail-config/authentication_baseline_test.go new file mode 100644 index 0000000000..cff724bcda --- /dev/null +++ b/scripts/private-mail-config/authentication_baseline_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +// Exercise the rendered policy through native JMAP readback, so a successful +// replay cannot silently retain a future external directory or a suspended +// recovery account. The protocol fixture contains no real credentials. +func TestRebuildVerifiesTheLocalAuthenticationBaseline(t *testing.T) { + plan, err := render(fixture()) + if err != nil { + t.Fatal(err) + } + ops, err := planOperations(plan) + if err != nil { + t.Fatal(err) + } + var administrator, authentication operation + for _, op := range ops { + if op.Object == "Account" && op.Type == "upsert" { + administrator = operation{Type: "upsert", Object: "Account", Value: map[string]any{"administrator": op.Value["administrator"]}} + administrator.Value["administrator"].(map[string]any)["domainId"] = "fixture-domain" + } + if op.Object == "Authentication" && op.Type == "update" { + authentication = op + } + } + for _, tc := range []struct { + name string + op operation + actual map[string]any + valid bool + }{ + {"administrator recovered", administrator, map[string]any{"id": "fixture-admin", "@type": "User", "name": "operator", "domainId": "fixture-domain", "roles": map[string]any{"@type": "Admin"}, "permissions": map[string]any{"@type": "Inherit"}}, true}, + {"administrator still suspended", administrator, map[string]any{"id": "fixture-admin", "@type": "User", "name": "operator", "domainId": "fixture-domain", "roles": map[string]any{"@type": "Admin"}, "permissions": map[string]any{"@type": "Replace", "enabledPermissions": map[string]any{}, "disabledPermissions": map[string]any{}}}, false}, + {"local directory recovered", authentication, map[string]any{"id": "singleton", "directoryId": nil}, true}, + {"external directory retained", authentication, map[string]any{"id": "singleton", "directoryId": "future-provider"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.op.Object == "" { + t.Fatal("the rebuild plan does not declare the authentication baseline") + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var call struct { + Methods [][]json.RawMessage `json:"methodCalls"` + } + if r.URL.Path != "/jmap" || json.NewDecoder(r.Body).Decode(&call) != nil || len(call.Methods) != 1 || len(call.Methods[0]) != 3 { + w.WriteHeader(http.StatusBadRequest) + return + } + var method, tag string + if json.Unmarshal(call.Methods[0][0], &method) != nil || json.Unmarshal(call.Methods[0][2], &tag) != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + var result any + switch method { + case "x:Account/query": + result = map[string]any{"ids": []string{"fixture-admin"}, "total": 1, "position": 0} + case "x:Account/get", "x:Authentication/get": + result = map[string]any{"list": []any{tc.actual}, "notFound": []string{}} + default: + w.WriteHeader(http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"methodResponses": []any{[]any{method, result, tag}}}) + })) + defer server.Close() + raw, err := json.Marshal(tc.op) + if err != nil { + t.Fatal(err) + } + m := management{client: server.Client(), url: server.URL, password: "synthetic-test-only"} + if (m.verify(context.Background(), append(raw, '\n')) == nil) != tc.valid { + t.Fatal("readback accepted stale authentication state or rejected the recovered baseline") + } + }) + } +} diff --git a/scripts/private-mail-config/config.go b/scripts/private-mail-config/config.go index 0de977ac4c..b152738edc 100644 --- a/scripts/private-mail-config/config.go +++ b/scripts/private-mail-config/config.go @@ -68,6 +68,9 @@ func render(v inputs) ([]byte, error) { return map[string]any{"@type": "User", "name": name, "domainId": "#domain", "credentials": map[string]any{"0": map[string]any{"@type": "Password", "secret": password}}, "roles": map[string]any{"@type": role}} } administrator := account(v.Administrator, v.administratorPassword, "Admin") + // Replaying the local baseline must recover its managed administrator even + // after a later identity cutover has suspended the ordinary account. + administrator["permissions"] = map[string]any{"@type": "Inherit"} maintainer := account(v.Maintainer, v.maintainerPassword, "User") if v.MaintainerAliases != nil { maintainer["aliases"] = receivingAliases(v.MaintainerAliases, "#domain") @@ -87,6 +90,9 @@ func render(v inputs) ([]byte, error) { return map[string]any{"@type": "update", "object": object, "value": value} } operations := []map[string]any{ + // Declare the existing local authentication state explicitly. A main + // recovery must clear a later external directory rather than retain it. + update("Authentication", map[string]any{"directoryId": nil}), upsert("Domain", []string{"name"}, map[string]any{"domain": map[string]any{"name": v.Domain, "isEnabled": true, "allowRelaying": false, "catchAllAddress": nil, "subAddressing": map[string]any{"@type": "Disabled"}, "dkimManagement": map[string]any{"@type": "Automatic"}, "dnsManagement": map[string]any{"@type": "Manual"}}}), upsert("Account", []string{"name", "domainId"}, map[string]any{"administrator": administrator, "maintainer": maintainer, "agents": agents}), // Address the exact ID returned by the upsert and replace the whole diff --git a/scripts/tests/test-private-mail-configuration.sh b/scripts/tests/test-private-mail-configuration.sh index 283d1d5fc4..6372579a1e 100755 --- a/scripts/tests/test-private-mail-configuration.sh +++ b/scripts/tests/test-private-mail-configuration.sh @@ -302,8 +302,11 @@ native_cli update Domain "$domain_id" --file /proof/restore-alias-patch.json >"$ # Seed a previously wider permission and an extra expression branch. Replay # must clear both maps rather than preserving keys absent from the new plan. +# Also suspend the managed administrator: replay must restore its inherited +# access, proved by the regular administrator's requests after serving starts. printf '%s\n' '{"@type":"upsert","object":"Domain","matchOn":["name"],"value":{"domain":{"name":"example.test"}}}' \ '{"@type":"upsert","object":"Account","matchOn":["name","domainId"],"value":{"agents":{"@type":"User","name":"agents","domainId":"#domain","permissions":{"@type":"Replace","enabledPermissions":{"jmapEmailImport":true,"jmapEmailCreate":true}}}}}' \ + '{"@type":"upsert","object":"Account","matchOn":["name","domainId"],"value":{"administrator":{"@type":"User","name":"operator","domainId":"#domain","permissions":{"@type":"Replace","enabledPermissions":{},"disabledPermissions":{}}}}}' \ '{"@type":"update","object":"MtaStageRcpt","value":{"rewrite":{"match":{"9":{"if":"true","then":"'"'"'operator@example.test'"'"'"}}}}}' >"$scratch/wider.ndjson" native_cli apply --file /proof/wider.ndjson --json >"$scratch/wider-apply.jsonl" 2>"$scratch/wider-apply.err" prepare_plan "$scratch/replay.ndjson" From a21eeec0550fe1a42a74a2a359702df8d38992e4 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 21:04:46 +0100 Subject: [PATCH 3/7] feat(email): name company email consistently on Homepage Part of #5387. Use Email for the tile and section, describe company correspondence in the operator decision, and preserve retained workload and storage identifiers. --- docs/adr/private-mail.md | 13 +++++++------ .../private-mail-administration/http-route.yaml | 4 ++-- .../private-mail-preparation/administration_test.go | 4 ++-- .../private-mail-preparation/production_test.go | 4 ++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/adr/private-mail.md b/docs/adr/private-mail.md index 4261eb5f64..d3b5fc5920 100644 --- a/docs/adr/private-mail.md +++ b/docs/adr/private-mail.md @@ -1,8 +1,8 @@ -# Private mail: isolated preparation before activation +# Company email: isolated preparation before activation ## Decision -Use Stalwart Community on the existing platform for the private mail capability +Use Stalwart Community on the existing platform for the company email capability requested in #5030. The preparation slice is #5288. Version v0.16.27 was released on 18 October 2026; its source is available under AGPL-3.0-only or the vendor's commercial alternative. This deployment uses the Community functionality and @@ -13,7 +13,7 @@ Stalwart supplies SMTP, IMAP/JMAP, DKIM and sender authentication together. Its supported declarative path uses a small datastore configuration and the official CLI to reconcile the remaining settings through its management API. Keeping this capability on the existing platform avoids a separate mail-only repository or an -unnecessary external processor of private message contents. +unnecessary external processor of confidential company correspondence. Provider discovery was performed before selection: the Directory had no self-hosted-mail result. The server's exact release, licensing, source and @@ -72,8 +72,8 @@ signer requirements. The administration Service and HTTPS route select only normal management port 8080. The network policy admits only gateway ingress there, explicitly denies recovery and prepared mail ports from every source, and retains all-egress -denial. The route registers a Stalwart Homepage entry at its authenticated -administration path. Inclusion prepares browser evaluation; it does not prove +denial. The route registers an Email Homepage entry in the Email section at its +authenticated administration path. Inclusion prepares browser evaluation; it does not prove the Homepage entry, authentication or any production behavior until observed. Public mail, DNS migration, credential custody, history import and restoration remain separate activation requirements below. @@ -106,7 +106,8 @@ listeners: - An inbox test with passing SPF/DKIM/DMARC, forged-reply and open-relay refusal, a refused unauthorized recipient, an exercised backup and restore, and the maintainer-performed credential handoff to the host's Keychain. -- A visible Stalwart entry on Homepage at `https://platform.devantler.tech`, +- A visible Email entry in the Email section on Homepage at + `https://platform.devantler.tech`, with a working link to the authenticated administration interface. Register it with the serving administration route and verify the tile and navigation in a real browser after activation. The recovery API stays internal. diff --git a/k8s/bases/apps/private-mail-administration/http-route.yaml b/k8s/bases/apps/private-mail-administration/http-route.yaml index ae19e0d8d8..32758844aa 100644 --- a/k8s/bases/apps/private-mail-administration/http-route.yaml +++ b/k8s/bases/apps/private-mail-administration/http-route.yaml @@ -5,8 +5,8 @@ metadata: namespace: private-mail annotations: gethomepage.dev/enabled: "true" - gethomepage.dev/name: Stalwart - gethomepage.dev/description: Private mail administration. + gethomepage.dev/name: Email + gethomepage.dev/description: Company email administration with Stalwart. gethomepage.dev/group: Email gethomepage.dev/icon: mdi-email-lock gethomepage.dev/href: https://mail.${domain}/admin diff --git a/scripts/tests/private-mail-preparation/administration_test.go b/scripts/tests/private-mail-preparation/administration_test.go index 3c8e907c1e..e12927b64b 100644 --- a/scripts/tests/private-mail-preparation/administration_test.go +++ b/scripts/tests/private-mail-preparation/administration_test.go @@ -175,8 +175,8 @@ func TestAdministrationExposesOnlyNormalManagement(t *testing.T) { route := object(t, objects, "HTTPRoute", "private-mail") meta := mapping(t, route["metadata"]) annotations := mapping(t, meta["annotations"]) - if annotations["gethomepage.dev/enabled"] != "true" || annotations["gethomepage.dev/name"] != "Stalwart" || annotations["gethomepage.dev/href"] != "https://mail.${domain}/admin" { - t.Fatal("the serving route must register a usable Stalwart Homepage entry") + if annotations["gethomepage.dev/enabled"] != "true" || annotations["gethomepage.dev/name"] != "Email" || annotations["gethomepage.dev/href"] != "https://mail.${domain}/admin" { + t.Fatal("the serving route must register a usable Email Homepage entry") } if !reflect.DeepEqual(mapping(t, route["spec"])["hostnames"], []any{"mail.${domain}"}) { t.Fatal("administration and its Homepage link must use the same hostname") diff --git a/scripts/tests/private-mail-preparation/production_test.go b/scripts/tests/private-mail-preparation/production_test.go index 01e9f5656d..679285a2a7 100644 --- a/scripts/tests/private-mail-preparation/production_test.go +++ b/scripts/tests/private-mail-preparation/production_test.go @@ -31,8 +31,8 @@ func TestProductionAdministrationUsesEncryptedSettings(t *testing.T) { } route := object(t, objects, "HTTPRoute", "private-mail") annotations := mapping(t, mapping(t, route["metadata"])["annotations"]) - if annotations["gethomepage.dev/name"] != "Stalwart" || annotations["gethomepage.dev/group"] != "Email" { - t.Fatal("production must register the Stalwart Homepage entry") + if annotations["gethomepage.dev/name"] != "Email" || annotations["gethomepage.dev/group"] != "Email" { + t.Fatal("production must register the Email entry in the Email section") } for _, o := range objects { m, _ := o["metadata"].(map[string]any) From dcabf4f64e54c261a77233095af66395fd8ffbd8 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 21:52:29 +0100 Subject: [PATCH 4/7] feat(email): prepare durable credential relocation Part of #5387. Declare read-only planning of the three current credentials before their infrastructure/email relocation. Future copy uses conditional creation and readback, preserves current values and source records, and refuses conflicting or unreadable state. Cover lost replies and pinned native API replay; retain an Email documentation compatibility link. --- .github/workflows/ci.yaml | 7 + docs/adr/email.md | 348 ++++++++++++++ docs/adr/private-mail.md | 316 +------------ .../config-map-email-credentials.yaml | 152 ++++++ .../infrastructure/vault-config/job.yaml | 50 ++ .../vault-config/kustomization.yaml | 1 + scripts/private-mail-config/bootstrap.go | 8 +- scripts/private-mail-config/config.go | 2 +- scripts/private-mail-config/main.go | 2 +- .../vault-email-credentials/migration_test.go | 431 ++++++++++++++++++ .../vault-email-credentials/native_test.go | 151 ++++++ 11 files changed, 1148 insertions(+), 320 deletions(-) create mode 100644 docs/adr/email.md create mode 100644 k8s/bases/infrastructure/vault-config/config-map-email-credentials.yaml create mode 100644 scripts/tests/vault-email-credentials/migration_test.go create mode 100644 scripts/tests/vault-email-credentials/native_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fff58b5e45..19a5dd7458 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -145,6 +145,7 @@ jobs: k8s: - 'k8s/**' - 'scripts/tests/private-mail-preparation/**' + - 'scripts/tests/vault-email-credentials/**' - 'scripts/tests/test-private-mail-server.sh' - 'scripts/tests/test-private-mail-configuration.sh' - 'scripts/private-mail-config/**' @@ -3360,6 +3361,12 @@ jobs: - name: 🔒 Validate Crossplane provider egress policy if: needs.changes.outputs.k8s == 'true' run: bash scripts/tests/test-crossplane-egress-policy.sh + - name: ✉️ Prove Email credential preparation in its pinned container + if: needs.changes.outputs.k8s == 'true' + timeout-minutes: 5 + env: + EMAIL_CREDENTIAL_CONTAINER_TEST: '1' + run: go test -count=1 -v ./scripts/tests/vault-email-credentials - name: ✉️ Prove native mail accounts and recipient restrictions if: needs.changes.outputs.k8s == 'true' timeout-minutes: 10 diff --git a/docs/adr/email.md b/docs/adr/email.md new file mode 100644 index 0000000000..dd74747556 --- /dev/null +++ b/docs/adr/email.md @@ -0,0 +1,348 @@ +# Company email: isolated preparation before activation + +## Decision + +Use Stalwart Community on the existing platform for the company email capability +requested in #5030. The preparation slice is #5288. Version v0.16.27 was released +on 18 October 2026; its source is available under AGPL-3.0-only or the vendor's +commercial alternative. This deployment uses the Community functionality and +introduces no paid service or Enterprise dependency. The published image index +is pinned by digest as well as release tag. + +Stalwart supplies SMTP, IMAP/JMAP, DKIM and sender authentication together. Its +supported declarative path uses a small datastore configuration and the official +CLI to reconcile the remaining settings through its management API. Keeping this +capability on the existing platform avoids a separate mail-only repository or an +unnecessary external processor of confidential company correspondence. + +Provider discovery was performed before selection: the Directory had no +self-hosted-mail result. The server's exact release, licensing, source and +official deployment documentation were then inspected. An external relay remains +a separate decision if direct outbound delivery cannot be established. + +## Shared preparation resources + +The `k8s/bases/apps/private-mail` bundle supplies the shared storage and +initialization resources. The shared and local application layers omit mail; +production includes this base through the administration bundle below. Rendering +the base directly tests preparation, while the provider renders verify that only +production enables normal administration. + +The preparation base alone starts only the internal recovery management +API. It has no public Service or route, mail listener or outbound network grant. +Only a setup workload in the same namespace can reach its management port. The +server receives no Kubernetes API token. Its single local datastore owner is +explicitly exempted from the serving-workload replica floor; this is not the +availability decision for public mail. + +An External Secrets password generator creates the initial credential. A +non-overwriting PushSecret preserves it in OpenBao; the server consumes only the +credential read back from OpenBao. Losing the generator cache cannot replace the +durable administrator. Supplying the recovery administrator before first start +prevents Stalwart from generating a temporary password into its logs. No human +or agent reads the credential as part of this preparation. + +The namespace and claim retain the platform's current prune protection. The +claim is not a backup or restore proof. Preparation neither migrates nor receives +messages, and must not be used as evidence that an existing volume is empty. + +## Administration opt-in + +The separate `k8s/bases/apps/private-mail-administration` bundle inherits the +preparation resources. Production includes it with encrypted settings; the +shared and local application layers keep mail disabled. +Its initializer must finish and stop recovery before Kubernetes starts the +normal Stalwart process. Both use the same immutable published bootstrap image. +Only the initializer receives the private settings and account passwords; +normal serving receives the datastore and certificate files. Each process has +its own memory-backed temporary directory, and neither receives Kubernetes API +credentials. + +Maintainer and agent passwords follow the existing administrator's durable +generation pattern: create a cache once, push only if absent, then consume the +value read back from OpenBao. Deleting a generator cache cannot rotate an +existing account password. Missing settings, credentials or certificate files +block startup. Production settings remain encrypted outside the shared bundle. +The credential synchronizer receives exact-key seed and read permissions for +all three durable credentials, with no mail-path wildcard, list or delete grant. +Admission and node image verification route only the exact bootstrap image +repository to its dedicated main-only publisher; other images keep their own +signer requirements. + +The administration Service and HTTPS route select only normal management port +8080. The network policy admits only gateway ingress there, explicitly denies +recovery and prepared mail ports from every source, and retains all-egress +denial. The route registers an Email Homepage entry in the Email section at its +authenticated administration path. Inclusion prepares browser evaluation; it does not prove +the Homepage entry, authentication or any production behavior until observed. +Public mail, DNS migration, credential custody, history import and restoration +remain separate activation requirements below. + +## Email credential custody + +The canonical operator-facing name is **Email**, for company email +infrastructure. Its credential destination is the `secret/` mount under +`infrastructure/email/admin`, `infrastructure/email/maintainer` and +`infrastructure/email/agent`. Retained namespace, datastore and image identifiers +keep their existing `private-mail` names during this migration. + +The OpenBao configuration Job first runs the declared credential preparation in +`plan` mode. It reads the current durable records under `apps/private-mail/`, +compares any existing destination, and reports pending copies without changing +credentials or consumer paths. It uses the configuration Job's existing private +credential file and bounded memory-backed scratch space; credentials and API +responses are never logged. Missing or unreadable data is distinguished from a +valid plan, and a different destination fails closed. + +After snapshot recovery has been verified, a separate reviewed revision may +activate `copy` mode. Creation uses KV v2 compare-and-set with version zero and +copies the complete current data object. It reads back and compares the result, +rechecks the source, and verifies a same-value creation race rather than +overwriting it. Numeric extra fields are refused to avoid JSON precision loss. +An uncertain result remains retryable. Existing source records +and their history remain intact. Only a native production `VERIFIED` result for +all three records permits a later consumer-path switch. A plan or an `UNSEEDED` +result is not proof of migration. + +Consumer changes, exact-path authorization and readback must land after the copy +is verified. Retire the old active keys only after the consumers and recovery +path are proven; preserve recoverable versions rather than destroying metadata. +The preparation mechanism can then be removed. Rebuilding from an empty store +is distinct from moving existing credentials and must not replace restored +credentials with a generator cache. + +## Activation and completion + +The parent #5030 remains open until the service delivers and receives real mail. +Activation must supply and validate all of the following before opening public +listeners: + +- Migration of the existing custom-domain mailbox from iCloud, as requested on + 24 October 2026. Inventory the domain's other addresses privately, preserve + their receiving continuity during retirement, and preserve existing messages. + Verify the destination, imported folders and rollback before changing the + domain-wide MX records through the owning DNS repository. +- Usable public TLS, MX, SPF, DKIM and DMARC. The mail hostname can be a subdomain; + the mailbox domain is the existing custom domain. Keep iCloud and its history + intact until the migration is verified. Never delete shared iCloud history as + a side effect of moving this mailbox. +- Verified maintainer-owned recipients and a server-enforced restriction for the + agents' account, covering SMTP and any permitted JMAP submission path. +- An observed outbound path. Hetzner's default port restriction is a provider + fact, not proof of the account's current state. Prefer direct delivery when + available; a provider unblock or an external relay needs its own resolved step. +- Idempotent configuration through the official CLI, a permanent administrator, + and removal of the preparation recovery variables before normal operation. +- Replies tied to an unguessable message token and the server's evaluated DMARC + result for an approved sender. Sender-provided authentication headers are not + evidence of authentication. +- An inbox test with passing SPF/DKIM/DMARC, forged-reply and open-relay refusal, + a refused unauthorized recipient, an exercised backup and restore, and the + maintainer-performed credential handoff to the host's Keychain. +- A visible Email entry in the Email section on Homepage at + `https://platform.devantler.tech`, + with a working link to the authenticated administration interface. Register + it with the serving administration route and verify the tile and navigation + in a real browser after activation. The recovery API stays internal. + +The public daily GitHub ask channel stays the accepted channel. Deploying mail +does not enable spend stewardship or designate its financial decision channel. +Those remain separate maintainer-owned decisions. + +The owned bootstrap process uses recovery management port 18080, separate from +normal administration on 8080. The pinned server supports a recovery port +override but [binds it to every interface](https://github.com/stalwartlabs/stalwart/blob/v0.16.27/crates/common/src/config/server/listener.rs#L59-L84), +so the administration overlay excludes 18080 from every Service and ingress +grant, including during initialization and retries. Readiness alone is not that +boundary. Bootstrap sets the override explicitly rather than inheriting it, and +stops and collects its recovery child before permitting normal serving. +Production exposes only normal administration after this clean recovery exit. +Public mail transport and migration remain separate activation requirements. + +## Prepared account configuration + +The default-off command `go run ./scripts/private-mail-config` prepares a native +CLI plan from private mounted files. It neither connects to a server nor starts +one. Its required flags are `--input-dir` and `--output-file`, both absolute +paths. The output must be a new file in a private directory; it is created with +mode 0600 and must live in memory-backed storage when used with real inputs. +The command emits fixed verdicts, never the plan or its inputs. + +The input directory contains `settings.json`, `administrator-password`, +`maintainer-password`, `agent-password` and `listener-inventory.json`. Settings +contain only `domain`, `hostname`, `maintainer`, `agent` and `administrator`; +account names are distinct local parts. Passwords are separate generated +40–128 character alphanumeric values, with no trailing newline. Unknown or +duplicate settings, invalid names, broad file permissions and an existing output +are refused. Exact production addresses stay in private operator configuration. + +Obtain the listener inventory with the committed +`scripts/private-mail-config/inventory-request.json` through the authenticated +management API in the same exclusive recovery session. Its bounded query/get +checks totals, returned IDs, missing objects and every listener's bind, protocol, +TLS and proxy settings. Preparation permits an empty fresh store or an exact +subset of the four prepared profiles. Unknown listeners, old default listeners, +incomplete reads and enabled hooks, milters or webhooks require reconciliation; +they are never silently removed. After apply, require all four exact profiles. +An empty CLI query is not this completeness proof. + +The same bounded inventory also covers every local domain and its aliases. +The reserved rejection domain must be absent from both, and external directory +resolution is refused. A reserved public DNS suffix alone cannot prove that a +mail server will refuse local delivery. The settings cannot register a reserved +`.invalid` domain as the mailbox domain. + +The inventory also covers every HTTP application. It accepts no application or +the pinned server's single default Stalwart Web Interface, with exactly the +`/admin` and `/account` prefixes. A missing or partial application census, +another application, an unknown resource, an external OAuth client or a custom +unpack directory fences initialization. The plan preserves the interface's +identity and replaces its route map with the two approved prefixes. +Known unversioned local resources and earlier digest-qualified resources are +accepted only as input to reconciliation to the active image's verified pin. + +Apply the private plan with the pinned official Stalwart CLI 1.0.13, using its +environment-based authentication from mounted credentials. Capture both output +streams privately, check every operation and read back the resulting policy. +Do not enable debug output, print plans or put passwords in command arguments. +The plan updates the agent account by the exact ID returned by its upsert, +preserving accounts, mail and certificates on replay. It replaces the complete +permission set and expression maps so stale grants or recipient branches cannot +survive. The normal server reads TLS files at `/etc/stalwart/tls/tls.crt` and +`/etc/stalwart/tls/tls.key`; its default certificate references that object. + +The human account retains normal authenticated SMTP and IMAP over TLS. Personal +Inbox history has no digest expiry policy. The separate agent account has only +the reviewed authentication, send/receive and JMAP read permissions. Its exact +approved recipient is enforced before local recipient resolution; other local, +remote and subaddress recipients are refused. MX offers no authentication; +submission requires TLS and authentication before accepting MAIL and enforces +the sender identity. Post-RCPT scripts are disabled. The HTTP upload endpoint is +denied because its native path does not honor the JMAP mutation permission list. +This supports IMAP/SMTP mail apps; full human JMAP attachment upload is not part +of this configuration. + +The native fixture `scripts/tests/test-private-mail-configuration.sh` starts its +own pinned server in explicit recovery, applies and reapplies the real plan, +exercises wider-policy and unexpected-listener negatives, and tests allowed +delivery, denied recipients/relay/writes/upload and human IMAP/TLS. It repeats +configuration after real fixture mail exists and reads that message again. +These are disposable `.test` accounts, not proof of public mail delivery. + +The explicit `bootstrap` subcommand owns an exclusive recovery child and the +datastore-volume lock. It reads mounted inputs, queries the real inventory, +applies the plan through the pinned CLI, requires a successful receipt for every +operation and reads the resulting policy back. Permission, listener-bind and +expression maps require exact equality, including removal of stale entries. +All native output is kept out of logs. A cancellation or failure stops and +collects the owned child; a forced kill cannot produce success. A clean recovery +shutdown is required before exit zero can allow a separate serving container to +start. The command never starts that serving container itself. + +The bootstrap image contains the exact upstream server and CLI plus the static +Go initializer and the pinned Web Interface v1.0.11 bundle. Both image build +paths stage the bundle and its Community AGPL license only after verifying +their committed SHA-256 checksums. The image includes the upstream source URL. +Bootstrap verifies the bundle again before starting recovery, refusing missing, +altered, oversized or symbolic-link assets. The application uses the image's +local file rather than downloading executable browser assets at runtime. +The filename and configured resource URL include the bundle's digest. Changing +the pin therefore changes the native resource URL and invalidates Stalwart's +cached ZIP while preserving the Application identity; verifying an image file +at a permanent URL alone would still permit a previously cached interface. + +Its default entrypoint is bootstrap; it cannot silently fall +through to an unconfigured normal server. It expects private inputs under +`/etc/private-mail`, the datastore at `/var/lib/stalwart`, native configuration +and TLS at `/etc/stalwart`, and memory-backed `/tmp`. The dedicated publisher +builds without publishing for PRs. Protected-main publication adds SBOM, +provenance and a verified Cosign signature to the immutable image digest. +Production manifests must use that signed digest and keep public activation +separate. API liveness or closed mail ports alone are insufficient. + +The native fixture also runs the actual bootstrap image, rejects unsafe stored +inventory, and serves both interfaces from that same image on an internal +network without external egress. It compares every script and stylesheet +referenced by each entry page with the pinned bundle, proves authenticated +management access and refuses the same request without authentication. It +primes an old cached interface, reapplies bootstrap with the real image, and +requires the pinned pages and assets to replace that cache without changing +the Application identity. It +proves normal authentication after recovery stops, and repeats +bootstrap after receiving mail to verify history survives. A failed or +interrupted apply is not atomic and must keep serving stopped. This +configuration slice changes no running workload, public route or DNS record. +Client setup, private credential handoff, migration, public sender-authentication +proof and a production backup restore remain in #5030. + +### Receiving continuity + +Private settings may optionally declare `maintainerAliases`, a list of up to +16 distinct local address names on the configured domain. They become receiving +aliases of the maintainer mailbox, without separate credentials or broader +agent-sending permission. Omission leaves existing aliases unmanaged. An +explicit empty list cannot erase a nonempty native alias set. + +The inventory includes complete bounded Account and MailingList query/get +responses: both primary addresses and disabled aliases reserve ownership. +Configured aliases require an empty or exactly approved existing maintainer +alias set; any competing owner, unexpected alias or incomplete census refuses +the plan. Bootstrap repeats this check on fresh data from its exclusively owned +recovery child immediately before apply. A saved standalone plan and inventory +are snapshots, not an atomic reconcile: the pinned CLI has no revision +precondition, so they must not authorize a later retry after live drift. + +Readback compares the entire alias collection, resolving domain references and +native defaults without depending on numeric entry order. The native fixture +exercises actual delivery to both aliases, unknown-address refusal, competing +User/Group/MailingList ownership, stale snapshots, and omission/replay preserving +the retained account and delivered mail. Deployment and DNS remain separate. + +## Deployment sequence + +Deliver the dedicated mail publisher's admission and node verification rules, +together with exact-key credential seed/read permissions, in a prerequisite +revision while production mail administration remains disabled. Verify the +normal protected deployment and the exact declared rules on every node before a +separate revision enables administration. + +That prerequisite also owns the namespace and claim through the storage-only +`k8s/bases/apps/private-mail-storage` bundle. It starts no process and exposes no +service. Preparation and administration inherit those same persistent identities. +When activation replaces the production storage-only reference, remove that direct +reference so the inherited resources are declared only once. A failed activation +can then return to a reviewed main that still owns the retained storage, keeping +it managed without deleting data or weakening the orphan-ownership check. + +The storage-only claim carries a `storage-stage: prepared` marker. Its explicit +Flux health decision accepts Pending only for this exact claim on the default +`longhorn-wffc` class before provisioning has selected a node or volume. Lost or +terminating claims do not pass. The serving bundle removes the marker, so its +claim must be Bound; all other claims retain their binding requirement. Health +waiting remains enabled throughout, without changing the retained claim's spec. + +The production pipeline reconciles applications before its full node configuration +update. Combining the prerequisite with an application that already needs the new +node rule makes that rollout wait on itself. Separate reviewed revisions preserve +the existing deployment sequence and signer isolation without a manual node +configuration change. Public mail activation and migration retain their separate +acceptance requirements. + +After the protected rollout, the dispatch-only **Verify Private Mail Storage +Recovery** workflow checks the original failed deployment's ownership window. +Run it from `main` with confirmation `verify-private-mail-storage-recovery`. +It uses the existing production reader while holding the deployment lock, makes +no cluster changes, and prints only a fixed verdict. Inventory, errors and +temporary credential files are not published. A missing read or incomplete scan +is unverified, never a pass. This verifies ownership recovery, not mail backup +or restore acceptance. + +## Sources + +- [Pinned release](https://github.com/stalwartlabs/stalwart/releases/tag/v0.16.27) +- [Pinned community license](https://github.com/stalwartlabs/stalwart/blob/v0.16.27/LICENSES/AGPL-3.0-only.txt) +- [Pinned Web Interface release](https://github.com/stalwartlabs/webui/releases/tag/v1.0.11) +- [Web Interface Community license](https://github.com/stalwartlabs/webui/blob/v1.0.11/LICENSES/AGPL-3.0-only.txt) +- [Declarative deployments](https://stalw.art/docs/configuration/declarative-deployments/) +- [Kubernetes deployment and restricted security context](https://stalw.art/docs/cluster/orchestration/kubernetes/) +- [Hetzner outbound mail policy](https://docs.hetzner.com/cloud/servers/faq/#why-can-i-not-send-any-mails-from-my-server) diff --git a/docs/adr/private-mail.md b/docs/adr/private-mail.md index d3b5fc5920..a2865d776d 100644 --- a/docs/adr/private-mail.md +++ b/docs/adr/private-mail.md @@ -1,315 +1,3 @@ -# Company email: isolated preparation before activation +# Company email -## Decision - -Use Stalwart Community on the existing platform for the company email capability -requested in #5030. The preparation slice is #5288. Version v0.16.27 was released -on 18 October 2026; its source is available under AGPL-3.0-only or the vendor's -commercial alternative. This deployment uses the Community functionality and -introduces no paid service or Enterprise dependency. The published image index -is pinned by digest as well as release tag. - -Stalwart supplies SMTP, IMAP/JMAP, DKIM and sender authentication together. Its -supported declarative path uses a small datastore configuration and the official -CLI to reconcile the remaining settings through its management API. Keeping this -capability on the existing platform avoids a separate mail-only repository or an -unnecessary external processor of confidential company correspondence. - -Provider discovery was performed before selection: the Directory had no -self-hosted-mail result. The server's exact release, licensing, source and -official deployment documentation were then inspected. An external relay remains -a separate decision if direct outbound delivery cannot be established. - -## Shared preparation resources - -The `k8s/bases/apps/private-mail` bundle supplies the shared storage and -initialization resources. The shared and local application layers omit mail; -production includes this base through the administration bundle below. Rendering -the base directly tests preparation, while the provider renders verify that only -production enables normal administration. - -The preparation base alone starts only the internal recovery management -API. It has no public Service or route, mail listener or outbound network grant. -Only a setup workload in the same namespace can reach its management port. The -server receives no Kubernetes API token. Its single local datastore owner is -explicitly exempted from the serving-workload replica floor; this is not the -availability decision for public mail. - -An External Secrets password generator creates the initial credential. A -non-overwriting PushSecret preserves it in OpenBao; the server consumes only the -credential read back from OpenBao. Losing the generator cache cannot replace the -durable administrator. Supplying the recovery administrator before first start -prevents Stalwart from generating a temporary password into its logs. No human -or agent reads the credential as part of this preparation. - -The namespace and claim retain the platform's current prune protection. The -claim is not a backup or restore proof. Preparation neither migrates nor receives -messages, and must not be used as evidence that an existing volume is empty. - -## Administration opt-in - -The separate `k8s/bases/apps/private-mail-administration` bundle inherits the -preparation resources. Production includes it with encrypted settings; the -shared and local application layers keep mail disabled. -Its initializer must finish and stop recovery before Kubernetes starts the -normal Stalwart process. Both use the same immutable published bootstrap image. -Only the initializer receives the private settings and account passwords; -normal serving receives the datastore and certificate files. Each process has -its own memory-backed temporary directory, and neither receives Kubernetes API -credentials. - -Maintainer and agent passwords follow the existing administrator's durable -generation pattern: create a cache once, push only if absent, then consume the -value read back from OpenBao. Deleting a generator cache cannot rotate an -existing account password. Missing settings, credentials or certificate files -block startup. Production settings remain encrypted outside the shared bundle. -The credential synchronizer receives exact-key seed and read permissions for -all three durable credentials, with no mail-path wildcard, list or delete grant. -Admission and node image verification route only the exact bootstrap image -repository to its dedicated main-only publisher; other images keep their own -signer requirements. - -The administration Service and HTTPS route select only normal management port -8080. The network policy admits only gateway ingress there, explicitly denies -recovery and prepared mail ports from every source, and retains all-egress -denial. The route registers an Email Homepage entry in the Email section at its -authenticated administration path. Inclusion prepares browser evaluation; it does not prove -the Homepage entry, authentication or any production behavior until observed. -Public mail, DNS migration, credential custody, history import and restoration -remain separate activation requirements below. - -## Activation and completion - -The parent #5030 remains open until the service delivers and receives real mail. -Activation must supply and validate all of the following before opening public -listeners: - -- Migration of the existing custom-domain mailbox from iCloud, as requested on - 24 October 2026. Inventory the domain's other addresses privately, preserve - their receiving continuity during retirement, and preserve existing messages. - Verify the destination, imported folders and rollback before changing the - domain-wide MX records through the owning DNS repository. -- Usable public TLS, MX, SPF, DKIM and DMARC. The mail hostname can be a subdomain; - the mailbox domain is the existing custom domain. Keep iCloud and its history - intact until the migration is verified. Never delete shared iCloud history as - a side effect of moving this mailbox. -- Verified maintainer-owned recipients and a server-enforced restriction for the - agents' account, covering SMTP and any permitted JMAP submission path. -- An observed outbound path. Hetzner's default port restriction is a provider - fact, not proof of the account's current state. Prefer direct delivery when - available; a provider unblock or an external relay needs its own resolved step. -- Idempotent configuration through the official CLI, a permanent administrator, - and removal of the preparation recovery variables before normal operation. -- Replies tied to an unguessable message token and the server's evaluated DMARC - result for an approved sender. Sender-provided authentication headers are not - evidence of authentication. -- An inbox test with passing SPF/DKIM/DMARC, forged-reply and open-relay refusal, - a refused unauthorized recipient, an exercised backup and restore, and the - maintainer-performed credential handoff to the host's Keychain. -- A visible Email entry in the Email section on Homepage at - `https://platform.devantler.tech`, - with a working link to the authenticated administration interface. Register - it with the serving administration route and verify the tile and navigation - in a real browser after activation. The recovery API stays internal. - -The public daily GitHub ask channel stays the accepted channel. Deploying mail -does not enable spend stewardship or designate its financial decision channel. -Those remain separate maintainer-owned decisions. - -The owned bootstrap process uses recovery management port 18080, separate from -normal administration on 8080. The pinned server supports a recovery port -override but [binds it to every interface](https://github.com/stalwartlabs/stalwart/blob/v0.16.27/crates/common/src/config/server/listener.rs#L59-L84), -so the administration overlay excludes 18080 from every Service and ingress -grant, including during initialization and retries. Readiness alone is not that -boundary. Bootstrap sets the override explicitly rather than inheriting it, and -stops and collects its recovery child before permitting normal serving. -Production exposes only normal administration after this clean recovery exit. -Public mail transport and migration remain separate activation requirements. - -## Prepared account configuration - -The default-off command `go run ./scripts/private-mail-config` prepares a native -CLI plan from private mounted files. It neither connects to a server nor starts -one. Its required flags are `--input-dir` and `--output-file`, both absolute -paths. The output must be a new file in a private directory; it is created with -mode 0600 and must live in memory-backed storage when used with real inputs. -The command emits fixed verdicts, never the plan or its inputs. - -The input directory contains `settings.json`, `administrator-password`, -`maintainer-password`, `agent-password` and `listener-inventory.json`. Settings -contain only `domain`, `hostname`, `maintainer`, `agent` and `administrator`; -account names are distinct local parts. Passwords are separate generated -40–128 character alphanumeric values, with no trailing newline. Unknown or -duplicate settings, invalid names, broad file permissions and an existing output -are refused. Exact production addresses stay in private operator configuration. - -Obtain the listener inventory with the committed -`scripts/private-mail-config/inventory-request.json` through the authenticated -management API in the same exclusive recovery session. Its bounded query/get -checks totals, returned IDs, missing objects and every listener's bind, protocol, -TLS and proxy settings. Preparation permits an empty fresh store or an exact -subset of the four prepared profiles. Unknown listeners, old default listeners, -incomplete reads and enabled hooks, milters or webhooks require reconciliation; -they are never silently removed. After apply, require all four exact profiles. -An empty CLI query is not this completeness proof. - -The same bounded inventory also covers every local domain and its aliases. -The reserved rejection domain must be absent from both, and external directory -resolution is refused. A reserved public DNS suffix alone cannot prove that a -mail server will refuse local delivery. The settings cannot register a reserved -`.invalid` domain as the mailbox domain. - -The inventory also covers every HTTP application. It accepts no application or -the pinned server's single default Stalwart Web Interface, with exactly the -`/admin` and `/account` prefixes. A missing or partial application census, -another application, an unknown resource, an external OAuth client or a custom -unpack directory fences initialization. The plan preserves the interface's -identity and replaces its route map with the two approved prefixes. -Known unversioned local resources and earlier digest-qualified resources are -accepted only as input to reconciliation to the active image's verified pin. - -Apply the private plan with the pinned official Stalwart CLI 1.0.13, using its -environment-based authentication from mounted credentials. Capture both output -streams privately, check every operation and read back the resulting policy. -Do not enable debug output, print plans or put passwords in command arguments. -The plan updates the agent account by the exact ID returned by its upsert, -preserving accounts, mail and certificates on replay. It replaces the complete -permission set and expression maps so stale grants or recipient branches cannot -survive. The normal server reads TLS files at `/etc/stalwart/tls/tls.crt` and -`/etc/stalwart/tls/tls.key`; its default certificate references that object. - -The human account retains normal authenticated SMTP and IMAP over TLS. Personal -Inbox history has no digest expiry policy. The separate agent account has only -the reviewed authentication, send/receive and JMAP read permissions. Its exact -approved recipient is enforced before local recipient resolution; other local, -remote and subaddress recipients are refused. MX offers no authentication; -submission requires TLS and authentication before accepting MAIL and enforces -the sender identity. Post-RCPT scripts are disabled. The HTTP upload endpoint is -denied because its native path does not honor the JMAP mutation permission list. -This supports IMAP/SMTP mail apps; full human JMAP attachment upload is not part -of this configuration. - -The native fixture `scripts/tests/test-private-mail-configuration.sh` starts its -own pinned server in explicit recovery, applies and reapplies the real plan, -exercises wider-policy and unexpected-listener negatives, and tests allowed -delivery, denied recipients/relay/writes/upload and human IMAP/TLS. It repeats -configuration after real fixture mail exists and reads that message again. -These are disposable `.test` accounts, not proof of public mail delivery. - -The explicit `bootstrap` subcommand owns an exclusive recovery child and the -datastore-volume lock. It reads mounted inputs, queries the real inventory, -applies the plan through the pinned CLI, requires a successful receipt for every -operation and reads the resulting policy back. Permission, listener-bind and -expression maps require exact equality, including removal of stale entries. -All native output is kept out of logs. A cancellation or failure stops and -collects the owned child; a forced kill cannot produce success. A clean recovery -shutdown is required before exit zero can allow a separate serving container to -start. The command never starts that serving container itself. - -The bootstrap image contains the exact upstream server and CLI plus the static -Go initializer and the pinned Web Interface v1.0.11 bundle. Both image build -paths stage the bundle and its Community AGPL license only after verifying -their committed SHA-256 checksums. The image includes the upstream source URL. -Bootstrap verifies the bundle again before starting recovery, refusing missing, -altered, oversized or symbolic-link assets. The application uses the image's -local file rather than downloading executable browser assets at runtime. -The filename and configured resource URL include the bundle's digest. Changing -the pin therefore changes the native resource URL and invalidates Stalwart's -cached ZIP while preserving the Application identity; verifying an image file -at a permanent URL alone would still permit a previously cached interface. - -Its default entrypoint is bootstrap; it cannot silently fall -through to an unconfigured normal server. It expects private inputs under -`/etc/private-mail`, the datastore at `/var/lib/stalwart`, native configuration -and TLS at `/etc/stalwart`, and memory-backed `/tmp`. The dedicated publisher -builds without publishing for PRs. Protected-main publication adds SBOM, -provenance and a verified Cosign signature to the immutable image digest. -Production manifests must use that signed digest and keep public activation -separate. API liveness or closed mail ports alone are insufficient. - -The native fixture also runs the actual bootstrap image, rejects unsafe stored -inventory, and serves both interfaces from that same image on an internal -network without external egress. It compares every script and stylesheet -referenced by each entry page with the pinned bundle, proves authenticated -management access and refuses the same request without authentication. It -primes an old cached interface, reapplies bootstrap with the real image, and -requires the pinned pages and assets to replace that cache without changing -the Application identity. It -proves normal authentication after recovery stops, and repeats -bootstrap after receiving mail to verify history survives. A failed or -interrupted apply is not atomic and must keep serving stopped. This -configuration slice changes no running workload, public route or DNS record. -Client setup, private credential handoff, migration, public sender-authentication -proof and a production backup restore remain in #5030. - -### Receiving continuity - -Private settings may optionally declare `maintainerAliases`, a list of up to -16 distinct local address names on the configured domain. They become receiving -aliases of the maintainer mailbox, without separate credentials or broader -agent-sending permission. Omission leaves existing aliases unmanaged. An -explicit empty list cannot erase a nonempty native alias set. - -The inventory includes complete bounded Account and MailingList query/get -responses: both primary addresses and disabled aliases reserve ownership. -Configured aliases require an empty or exactly approved existing maintainer -alias set; any competing owner, unexpected alias or incomplete census refuses -the plan. Bootstrap repeats this check on fresh data from its exclusively owned -recovery child immediately before apply. A saved standalone plan and inventory -are snapshots, not an atomic reconcile: the pinned CLI has no revision -precondition, so they must not authorize a later retry after live drift. - -Readback compares the entire alias collection, resolving domain references and -native defaults without depending on numeric entry order. The native fixture -exercises actual delivery to both aliases, unknown-address refusal, competing -User/Group/MailingList ownership, stale snapshots, and omission/replay preserving -the retained account and delivered mail. Deployment and DNS remain separate. - -## Deployment sequence - -Deliver the dedicated mail publisher's admission and node verification rules, -together with exact-key credential seed/read permissions, in a prerequisite -revision while production mail administration remains disabled. Verify the -normal protected deployment and the exact declared rules on every node before a -separate revision enables administration. - -That prerequisite also owns the namespace and claim through the storage-only -`k8s/bases/apps/private-mail-storage` bundle. It starts no process and exposes no -service. Preparation and administration inherit those same persistent identities. -When activation replaces the production storage-only reference, remove that direct -reference so the inherited resources are declared only once. A failed activation -can then return to a reviewed main that still owns the retained storage, keeping -it managed without deleting data or weakening the orphan-ownership check. - -The storage-only claim carries a `storage-stage: prepared` marker. Its explicit -Flux health decision accepts Pending only for this exact claim on the default -`longhorn-wffc` class before provisioning has selected a node or volume. Lost or -terminating claims do not pass. The serving bundle removes the marker, so its -claim must be Bound; all other claims retain their binding requirement. Health -waiting remains enabled throughout, without changing the retained claim's spec. - -The production pipeline reconciles applications before its full node configuration -update. Combining the prerequisite with an application that already needs the new -node rule makes that rollout wait on itself. Separate reviewed revisions preserve -the existing deployment sequence and signer isolation without a manual node -configuration change. Public mail activation and migration retain their separate -acceptance requirements. - -After the protected rollout, the dispatch-only **Verify Private Mail Storage -Recovery** workflow checks the original failed deployment's ownership window. -Run it from `main` with confirmation `verify-private-mail-storage-recovery`. -It uses the existing production reader while holding the deployment lock, makes -no cluster changes, and prints only a fixed verdict. Inventory, errors and -temporary credential files are not published. A missing read or incomplete scan -is unverified, never a pass. This verifies ownership recovery, not mail backup -or restore acceptance. - -## Sources - -- [Pinned release](https://github.com/stalwartlabs/stalwart/releases/tag/v0.16.27) -- [Pinned community license](https://github.com/stalwartlabs/stalwart/blob/v0.16.27/LICENSES/AGPL-3.0-only.txt) -- [Pinned Web Interface release](https://github.com/stalwartlabs/webui/releases/tag/v1.0.11) -- [Web Interface Community license](https://github.com/stalwartlabs/webui/blob/v1.0.11/LICENSES/AGPL-3.0-only.txt) -- [Declarative deployments](https://stalw.art/docs/configuration/declarative-deployments/) -- [Kubernetes deployment and restricted security context](https://stalw.art/docs/cluster/orchestration/kubernetes/) -- [Hetzner outbound mail policy](https://docs.hetzner.com/cloud/servers/faq/#why-can-i-not-send-any-mails-from-my-server) +The current decision and recovery procedure are in [Company email](email.md). diff --git a/k8s/bases/infrastructure/vault-config/config-map-email-credentials.yaml b/k8s/bases/infrastructure/vault-config/config-map-email-credentials.yaml new file mode 100644 index 0000000000..667b0c2a8f --- /dev/null +++ b/k8s/bases/infrastructure/vault-config/config-map-email-credentials.yaml @@ -0,0 +1,152 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vault-email-credentials + namespace: openbao +data: + # Plan is the deployed default. Copy is a later reviewed activation after + # snapshot recovery is verified; it never switches consumers or deletes keys. + prepare-email-credentials.sh: | + #!/bin/sh + set -eu + umask 077 + + fail() { + printf '%s\n' 'EMAIL_CUSTODY_PREPARATION=FAIL' >&2 + exit 1 + } + + mode=$EMAIL_CUSTODY_MODE + case "$mode" in plan|copy) ;; *) fail ;; esac + [ -n "$BAO_ADDR" ] || fail + [ -f "$EMAIL_CUSTODY_TOKEN_FILE" ] || fail + [ -d "$EMAIL_CUSTODY_WORK_DIR" ] || fail + work=$(mktemp -d "$EMAIL_CUSTODY_WORK_DIR/email-credentials.XXXXXXXX") || fail + trap 'rm -rf "$work"' 0 + trap 'exit 1' HUP INT TERM + token=$(cat "$EMAIL_CUSTODY_TOKEN_FILE") || fail + case "$token" in ''|*[!A-Za-z0-9._-]*) fail ;; esac + printf 'X-Vault-Token: %s\nContent-Type: application/json\n' "$token" > "$work/headers" + unset token + + # Headers and JSON bodies are files in private memory, never process arguments. + # Disabling curl's user configuration also prevents accidental trace logging. + request() { + request_method=$1 + request_path=$2 + request_input=$3 + set -- --disable --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 20 --max-filesize 1048576 \ + --header "@$work/headers" --output "$work/response.json" \ + --write-out '%{http_code}' + case "$request_method" in + GET) ;; + POST) set -- "$@" --request POST --data-binary "@$request_input" ;; + *) fail ;; + esac + status=$(curl "$@" "$BAO_ADDR/v1/$request_path" 2> "$work/request-error") || fail + } + + read_record() { + record_path=$1 + record_output=$2 + : > "$record_output" + request GET "secret/data/$record_path" '' + case "$status" in + 404) return 4 ;; + 200) ;; + *) fail ;; + esac + # Preserve the complete data object, including any additional recovery fields. + # Invalid or unreadable credentials are not equivalent to an absent key. + # Refuse numeric extras rather than risk precision changes in JSON tooling. + if ! jq -ceS '.data.data | select(type == "object") | + select(all(..; type != "number")) | + select(.password | type == "string") | + select(.password | test("^[A-Za-z0-9]{40,128}$"))' \ + "$work/response.json" > "$record_output" 2> "$work/json-error"; then + fail + fi + } + + existing=0 + copied=0 + pending=0 + unseeded=0 + report() { + verdict=PLANNED + if [ "$unseeded" -gt 0 ]; then + verdict=UNSEEDED + elif [ "$mode" = copy ]; then + verdict=VERIFIED + fi + printf 'EMAIL_CUSTODY_PREPARATION=%s mode=%s existing=%s copied=%s pending=%s unseeded=%s\n' \ + "$verdict" "$mode" "$existing" "$copied" "$pending" "$unseeded" + } + + # This init container precedes normal configuration. A new installation may + # not have its KV mount yet; report it explicitly without creating passwords. + request GET sys/mounts '' + [ "$status" = 200 ] || fail + if ! jq -e '.data | type == "object"' "$work/response.json" > /dev/null 2> "$work/json-error"; then + fail + fi + if jq -e '.data["secret/"] == null' "$work/response.json" > /dev/null; then + unseeded=3 + report + exit 0 + fi + if ! jq -e '.data["secret/"] | .type == "kv" and .options.version == "2"' \ + "$work/response.json" > /dev/null 2> "$work/json-error"; then + fail + fi + + for account in admin maintainer agent; do + source_path=apps/private-mail/$account + destination_path=infrastructure/email/$account + source_present=false + destination_present=false + if read_record "$source_path" "$work/source.json"; then + source_present=true + else + [ "$?" = 4 ] || fail + fi + if read_record "$destination_path" "$work/destination.json"; then + destination_present=true + else + [ "$?" = 4 ] || fail + fi + + if [ "$source_present" = false ]; then + # A populated destination without its source is not a proven migration. + [ "$destination_present" = false ] || fail + unseeded=$((unseeded + 1)) + continue + fi + if [ "$destination_present" = true ]; then + cmp -s "$work/source.json" "$work/destination.json" || fail + existing=$((existing + 1)) + elif [ "$mode" = plan ]; then + pending=$((pending + 1)) + else + if ! jq -cn --slurpfile value "$work/source.json" \ + '{options: {cas: 0}, data: $value[0]}' > "$work/create.json" 2> "$work/json-error"; then + fail + fi + request POST "secret/data/$destination_path" "$work/create.json" + creation_status=$status + case "$creation_status" in 200|204|400) ;; *) fail ;; esac + # Read back successful writes AND conditional-create races. Never replace + # a different destination or silently accept a lost/uncertain write. + read_record "$destination_path" "$work/destination.json" || fail + cmp -s "$work/source.json" "$work/destination.json" || fail + if [ "$creation_status" = 400 ]; then + existing=$((existing + 1)) + else + copied=$((copied + 1)) + fi + fi + read_record "$source_path" "$work/source-after.json" || fail + cmp -s "$work/source.json" "$work/source-after.json" || fail + done + report diff --git a/k8s/bases/infrastructure/vault-config/job.yaml b/k8s/bases/infrastructure/vault-config/job.yaml index be710362b0..cfb31685d0 100644 --- a/k8s/bases/infrastructure/vault-config/job.yaml +++ b/k8s/bases/infrastructure/vault-config/job.yaml @@ -126,6 +126,13 @@ spec: secret: secretName: vault-config-oidc optional: true + - name: email-credentials + configMap: + name: vault-email-credentials + - name: email-credential-work + emptyDir: + medium: Memory + sizeLimit: 16Mi # Local scratch for the newest raft snapshot, populated by the # fetch-snapshot initContainer from the OFF-CLUSTER mirror (Cloudflare R2 # in prod, in-cluster MinIO locally) -- NOT the single-attach RWO @@ -513,6 +520,49 @@ spec: else echo "Vault was already initialized — no new keys to store." fi + # Read-only preparation for company Email credential custody. Reuse the + # configuration Job's existing private root-token file; no new identity, + # Kubernetes grant, password generation, consumer switch or deletion. + - name: prepare-email-credentials + image: docker.io/alpine/k8s:1.37.1@sha256:712dc5385d1c1a3f83b747fa9e40f64d433a97f72ff42f9755ff7dd1f62846e4 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + env: + - name: BAO_ADDR + value: http://openbao-active.openbao.svc.cluster.local:8200 + - name: EMAIL_CUSTODY_MODE + value: plan + - name: EMAIL_CUSTODY_TOKEN_FILE + value: /shared/root-token + - name: EMAIL_CUSTODY_WORK_DIR + value: /work + volumeMounts: + - name: shared + mountPath: /shared + readOnly: true + - name: email-credentials + mountPath: /vault-config/email + readOnly: true + - name: email-credential-work + mountPath: /work + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 250m + memory: 128Mi + command: + - /bin/sh + - -eu + - /vault-config/email/prepare-email-credentials.sh containers: - name: vault-config image: quay.io/openbao/openbao:2.5.3@sha256:fdc6da21ca6963560c32336fd7feb9cf2d5e52668f1a1647205a4b41171f0806 diff --git a/k8s/bases/infrastructure/vault-config/kustomization.yaml b/k8s/bases/infrastructure/vault-config/kustomization.yaml index 6e355d2b38..06d88a4dec 100644 --- a/k8s/bases/infrastructure/vault-config/kustomization.yaml +++ b/k8s/bases/infrastructure/vault-config/kustomization.yaml @@ -6,5 +6,6 @@ resources: - role.yaml - role-binding.yaml - job.yaml + - config-map-email-credentials.yaml - external-secret.yaml - cilium-network-policy.yaml diff --git a/scripts/private-mail-config/bootstrap.go b/scripts/private-mail-config/bootstrap.go index 73b3ca7bc6..1f5bf05758 100644 --- a/scripts/private-mail-config/bootstrap.go +++ b/scripts/private-mail-config/bootstrap.go @@ -32,7 +32,7 @@ func bootstrapCommand(args []string, log io.Writer) int { f.SetOutput(io.Discard) dir := f.String("input-dir", "/etc/private-mail", "mounted private inputs") if f.Parse(args) != nil || f.NArg() != 0 || !filepath.IsAbs(*dir) { - _, _ = fmt.Fprintln(log, "Private mail bootstrap refused invalid arguments.") + _, _ = fmt.Fprintln(log, "Email bootstrap refused invalid arguments.") return 2 } ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) @@ -41,7 +41,7 @@ func bootstrapCommand(args []string, log io.Writer) int { defer timeout() applicationDigest, err := pinnedApplicationDigest() if err != nil { - _, _ = fmt.Fprintln(log, "Private mail bootstrap refused invalid administration asset metadata.") + _, _ = fmt.Fprintln(log, "Email bootstrap refused invalid administration asset metadata.") return 2 } runtime := bootstrapRuntime{ @@ -51,10 +51,10 @@ func bootstrapCommand(args []string, log io.Writer) int { applicationFile: applicationAssetPath(applicationDigest), applicationDigest: applicationDigest, } if err := bootstrap(ctx, *dir, runtime); err != nil { - _, _ = fmt.Fprintln(log, "Private mail bootstrap failed; serving must remain stopped.", err) + _, _ = fmt.Fprintln(log, "Email bootstrap failed; serving must remain stopped.", err) return 2 } - if _, err := fmt.Fprintln(log, "Private mail configuration verified; recovery process stopped."); err != nil { + if _, err := fmt.Fprintln(log, "Email configuration verified; recovery process stopped."); err != nil { return 2 } return 0 diff --git a/scripts/private-mail-config/config.go b/scripts/private-mail-config/config.go index b152738edc..5acc7e4144 100644 --- a/scripts/private-mail-config/config.go +++ b/scripts/private-mail-config/config.go @@ -10,7 +10,7 @@ import ( "strings" ) -var errConfiguration = errors.New("private mail configuration is incomplete or invalid") +var errConfiguration = errors.New("email configuration is incomplete or invalid") type settings struct { Domain string `json:"domain"` diff --git a/scripts/private-mail-config/main.go b/scripts/private-mail-config/main.go index 6156608e25..4b14d0cb74 100644 --- a/scripts/private-mail-config/main.go +++ b/scripts/private-mail-config/main.go @@ -153,7 +153,7 @@ func command(args []string, log io.Writer) int { _ = os.Remove(*output) return fail() } - if _, err = fmt.Fprintln(log, "Private mail plan prepared; activation is separate."); err != nil { + if _, err = fmt.Fprintln(log, "Email plan prepared; activation is separate."); err != nil { return 2 } return 0 diff --git a/scripts/tests/vault-email-credentials/migration_test.go b/scripts/tests/vault-email-credentials/migration_test.go new file mode 100644 index 0000000000..8af16131cf --- /dev/null +++ b/scripts/tests/vault-email-credentials/migration_test.go @@ -0,0 +1,431 @@ +package emailcredentials_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "sync" + "testing" + + "gopkg.in/yaml.v3" +) + +const syntheticToken = "test-only-root-token" + +type apiFixture struct { + mu sync.Mutex + keys map[string]map[string]any + writes int + denied string + missingKV bool + invalidKV bool + race string + changeAfter bool + loseReply bool +} + +func credentials() map[string]map[string]any { + keys := map[string]map[string]any{} + for _, account := range []string{"admin", "maintainer", "agent"} { + keys["apps/private-mail/"+account] = map[string]any{ + "password": strings.Repeat("A", 48) + account, + "recovery": map[string]any{"preserve": true}, + } + } + return keys +} + +func (f *apiFixture) serve(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if r.Header.Get("X-Vault-Token") != syntheticToken { + http.Error(w, `{"errors":["authentication refused"]}`, http.StatusForbidden) + return + } + if r.Method == http.MethodGet && r.URL.Path == "/v1/sys/mounts" { + mounts := map[string]any{} + if !f.missingKV { + version := "2" + if f.invalidKV { + version = "1" + } + mounts["secret/"] = map[string]any{"type": "kv", "options": map[string]string{"version": version}} + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": mounts}) + return + } + key, ok := strings.CutPrefix(r.URL.Path, "/v1/secret/data/") + if !ok || !allowedKey(key) { + http.Error(w, `{"errors":["unexpected path"]}`, http.StatusBadRequest) + return + } + if key == f.denied { + http.Error(w, `{"errors":["read refused"]}`, http.StatusForbidden) + return + } + if r.Method == http.MethodGet { + data, exists := f.keys[key] + if !exists { + http.Error(w, `{"errors":[]}`, http.StatusNotFound) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{ + "data": data, "metadata": map[string]any{"version": 1}, + }}) + return + } + if r.Method != http.MethodPost || !strings.HasPrefix(key, "infrastructure/email/") { + http.Error(w, `{"errors":["unexpected mutation"]}`, http.StatusMethodNotAllowed) + return + } + var request struct { + Options struct { + CAS *int `json:"cas"` + } `json:"options"` + Data map[string]any `json:"data"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil || request.Options.CAS == nil || *request.Options.CAS != 0 { + http.Error(w, `{"errors":["creation must be conditional"]}`, http.StatusBadRequest) + return + } + if f.race != "" { + data := request.Data + if f.race == "different" { + data = map[string]any{"password": strings.Repeat("Z", 48)} + } + f.keys[key] = data + f.race = "" + http.Error(w, `{"errors":["check-and-set conflict"]}`, http.StatusBadRequest) + return + } + if _, exists := f.keys[key]; exists { + http.Error(w, `{"errors":["check-and-set conflict"]}`, http.StatusBadRequest) + return + } + f.keys[key] = request.Data + f.writes++ + if f.loseReply { + f.loseReply = false + connection, _, err := w.(http.Hijacker).Hijack() + if err == nil { + _ = connection.Close() + } + return + } + if f.changeAfter { + f.keys["apps/private-mail/admin"] = map[string]any{"password": strings.Repeat("B", 48)} + f.changeAfter = false + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"version": 1}}) +} + +func allowedKey(key string) bool { + for _, prefix := range []string{"apps/private-mail/", "infrastructure/email/"} { + for _, account := range []string{"admin", "maintainer", "agent"} { + if key == prefix+account { + return true + } + } + } + return false +} + +func repositoryRoot() string { + _, file, _, _ := runtime.Caller(0) + return filepath.Clean(filepath.Join(filepath.Dir(file), "../../..")) +} + +func runMigration(t *testing.T, fixture *apiFixture, mode string) (string, error) { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(fixture.serve)) + defer server.Close() + return runMigrationAt(t, server.URL, mode) +} + +func runMigrationAt(t *testing.T, serverURL, mode string) (string, error) { + t.Helper() + manifest, err := os.ReadFile(filepath.Join(repositoryRoot(), "k8s/bases/infrastructure/vault-config/config-map-email-credentials.yaml")) + if err != nil { + t.Fatal("missing declarative Email credential preparation", err) + } + var config struct{ Data map[string]string } + if err := yaml.Unmarshal(manifest, &config); err != nil { + t.Fatal(err) + } + script := config.Data["prepare-email-credentials.sh"] + if script == "" { + t.Fatal("missing executable credential preparation") + } + dir := t.TempDir() + if err := os.Chmod(dir, 0o700); err != nil { + t.Fatal(err) + } + for name, content := range map[string]string{"prepare.sh": script, "token": syntheticToken} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + environment := []string{ + "PATH=" + os.Getenv("PATH"), "HOME=" + dir, "LC_ALL=C", + "BAO_ADDR=" + serverURL, "EMAIL_CUSTODY_MODE=" + mode, + "EMAIL_CUSTODY_TOKEN_FILE=" + filepath.Join(dir, "token"), + "EMAIL_CUSTODY_WORK_DIR=" + dir, + } + command := exec.Command("/bin/sh", "-eu", filepath.Join(dir, "prepare.sh")) + if os.Getenv("EMAIL_CREDENTIAL_CONTAINER_TEST") == "1" { + // Linux CI runs the identical fixture through the exact deployed image. + // Bind only synthetic private inputs, and grant no container capabilities. + if runtime.GOOS != "linux" { + t.Fatal("native container proof requires the Linux CI host network") + } + args := []string{ + "run", "--rm", "--network=host", "--read-only", "--cap-drop=ALL", + "--security-opt=no-new-privileges", "--user", fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), + "--mount", "type=bind,source=" + dir + ",target=" + dir, + "--entrypoint", "/bin/sh", + } + for _, value := range environment[1:] { + args = append(args, "--env", value) + } + args = append(args, deployedToolingImage(t), "-eu", filepath.Join(dir, "prepare.sh")) + command = exec.Command("docker", args...) + } + command.Env = environment + out, runErr := command.CombinedOutput() + if strings.Contains(string(out), syntheticToken) || strings.Contains(string(out), strings.Repeat("A", 48)) || strings.Contains(string(out), serverURL) { + t.Fatal("credential preparation emitted private material") + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 { + t.Fatal("credential scratch material survived the operation") + } + return string(out), runErr +} + +func deployedToolingImage(t *testing.T) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(repositoryRoot(), "k8s/bases/infrastructure/vault-config/job.yaml")) + if err != nil { + t.Fatal(err) + } + var job struct { + Spec struct { + Template struct { + Spec struct { + InitContainers []struct{ Name, Image string } `yaml:"initContainers"` + } + } + } + } + if err := yaml.Unmarshal(data, &job); err != nil { + t.Fatal(err) + } + for _, c := range job.Spec.Template.Spec.InitContainers { + if c.Name == "prepare-email-credentials" && strings.HasPrefix(c.Image, "docker.io/alpine/k8s:") && strings.Contains(c.Image, "@sha256:") { + return c.Image + } + } + t.Fatal("native proof must use the exact declared tooling image") + return "" +} + +func TestPlanIsReadOnlyAndCopyPreservesEveryCurrentField(t *testing.T) { + fixture := &apiFixture{keys: credentials()} + out, err := runMigration(t, fixture, "plan") + if err != nil || !strings.Contains(out, "=PLANNED mode=plan existing=0 copied=0 pending=3 unseeded=0") || fixture.writes != 0 { + t.Fatalf("plan must report three pending copies without writing: %v %s", err, out) + } + for attempt := 0; attempt < 2; attempt++ { + out, err = runMigration(t, fixture, "copy") + if err != nil || fixture.writes != 3 { + t.Fatalf("copy and replay must create exactly three records: %v %s", err, out) + } + for _, account := range []string{"admin", "maintainer", "agent"} { + if !reflect.DeepEqual(fixture.keys["apps/private-mail/"+account], fixture.keys["infrastructure/email/"+account]) { + t.Fatal("copy changed the current credential or discarded another field") + } + } + } + if !strings.Contains(out, "=VERIFIED mode=copy existing=3 copied=0 pending=0 unseeded=0") { + t.Fatal("replay must distinguish existing verified values from new copies") + } +} + +func TestMigrationRefusesConflictsAndUnreadableState(t *testing.T) { + for _, name := range []string{"destination-conflict", "read-refused", "invalid-password", "numeric-field", "invalid-mount", "different-race", "source-rotation", "invalid-mode"} { + t.Run(name, func(t *testing.T) { + fixture := &apiFixture{keys: credentials()} + mode := "copy" + switch name { + case "destination-conflict": + fixture.keys["infrastructure/email/admin"] = map[string]any{"password": strings.Repeat("Z", 48)} + case "read-refused": + fixture.denied = "apps/private-mail/admin" + case "invalid-password": + fixture.keys["apps/private-mail/admin"]["password"] = "invalid" + case "numeric-field": + fixture.keys["apps/private-mail/admin"]["extra"] = json.Number("9007199254740993") + case "invalid-mount": + fixture.invalidKV = true + case "different-race": + fixture.race = "different" + case "source-rotation": + fixture.changeAfter = true + case "invalid-mode": + mode = "replace" + } + out, err := runMigration(t, fixture, mode) + if err == nil || strings.Contains(out, "=PASS") { + t.Fatalf("uncertain state must fail closed: %s", out) + } + if name != "source-rotation" && fixture.writes != 0 { + t.Fatal("refusal wrote a credential") + } + }) + } +} + +func TestConcurrentIdenticalCreationIsVerifiedWithoutOverwrite(t *testing.T) { + fixture := &apiFixture{keys: credentials(), race: "same"} + out, err := runMigration(t, fixture, "copy") + if err != nil || fixture.writes != 2 || !strings.Contains(out, "existing=1 copied=2") { + t.Fatalf("identical conditional-create race must be read back and compared: %v %s", err, out) + } +} + +func TestLostWriteReplyIsRecoveredByReadbackWithoutRotation(t *testing.T) { + fixture := &apiFixture{keys: credentials(), loseReply: true} + if _, err := runMigration(t, fixture, "copy"); err == nil || fixture.writes != 1 { + t.Fatal("a committed write with a lost reply must remain an uncertain result") + } + out, err := runMigration(t, fixture, "copy") + if err != nil || fixture.writes != 3 || !strings.Contains(out, "=VERIFIED mode=copy existing=1 copied=2") { + t.Fatalf("retry must verify the committed value and copy only missing records: %v %s", err, out) + } + for _, account := range []string{"admin", "maintainer", "agent"} { + if !reflect.DeepEqual(fixture.keys["apps/private-mail/"+account], fixture.keys["infrastructure/email/"+account]) { + t.Fatal("uncertain-write recovery changed or erased a credential") + } + } +} + +func TestUnseededInstallationIsExplicitAndNeverCreatesCredentials(t *testing.T) { + for _, missingMount := range []bool{false, true} { + t.Run(fmt.Sprint(missingMount), func(t *testing.T) { + fixture := &apiFixture{keys: map[string]map[string]any{}, missingKV: missingMount} + out, err := runMigration(t, fixture, "plan") + if err != nil || fixture.writes != 0 || !strings.Contains(out, "=UNSEEDED") || !strings.Contains(out, "unseeded=3") { + t.Fatalf("fresh installation must report unseeded state: %v %s", err, out) + } + }) + } +} + +func TestDeclaredPreparationRunsReadOnlyWithPrivateMemoryScratch(t *testing.T) { + out, err := exec.Command("kubectl", "kustomize", filepath.Join(repositoryRoot(), "k8s/bases/infrastructure/vault-config")).CombinedOutput() + if err != nil { + t.Fatalf("configuration bundle must render: %v", err) + } + type volume struct { + Name string + EmptyDir *struct { + Medium string + SizeLimit string `yaml:"sizeLimit"` + } `yaml:"emptyDir"` + } + type container struct { + Name, Image string + Command []string + Env []struct{ Name, Value string } + VolumeMounts []struct { + Name string + MountPath string `yaml:"mountPath"` + ReadOnly bool `yaml:"readOnly"` + } `yaml:"volumeMounts"` + SecurityContext struct { + ReadOnlyRootFilesystem bool `yaml:"readOnlyRootFilesystem"` + RunAsNonRoot bool `yaml:"runAsNonRoot"` + AllowPrivilegeEscalation *bool `yaml:"allowPrivilegeEscalation"` + } `yaml:"securityContext"` + } + var foundConfig, foundContainer bool + decoder := yaml.NewDecoder(bytes.NewReader(out)) + for { + var object struct { + Kind string + Metadata struct{ Name string } + Spec struct { + Template struct { + Spec struct { + InitContainers []container `yaml:"initContainers"` + Volumes []volume + } + } + } + } + if err := decoder.Decode(&object); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + if object.Kind == "ConfigMap" && object.Metadata.Name == "vault-email-credentials" { + foundConfig = true + } + if object.Kind != "Job" || object.Metadata.Name != "vault-config" { + continue + } + for _, c := range object.Spec.Template.Spec.InitContainers { + if c.Name != "prepare-email-credentials" { + continue + } + foundContainer = true + if c.Image != deployedToolingImage(t) || !reflect.DeepEqual(c.Command, []string{"/bin/sh", "-eu", "/vault-config/email/prepare-email-credentials.sh"}) { + t.Fatal("preparation must execute the declared script with the existing pinned tooling image") + } + env := map[string]string{} + for _, v := range c.Env { + env[v.Name] = v.Value + } + if len(env) != 4 || env["EMAIL_CUSTODY_MODE"] != "plan" || env["EMAIL_CUSTODY_TOKEN_FILE"] != "/shared/root-token" || env["EMAIL_CUSTODY_WORK_DIR"] != "/work" { + t.Fatal("deployed preparation must only plan and reuse the existing private credential file") + } + if !c.SecurityContext.ReadOnlyRootFilesystem || !c.SecurityContext.RunAsNonRoot || c.SecurityContext.AllowPrivilegeEscalation == nil || *c.SecurityContext.AllowPrivilegeEscalation { + t.Fatal("preparation must preserve the restricted container boundary") + } + mounts := map[string]bool{} + for _, m := range c.VolumeMounts { + mounts[m.Name+":"+m.MountPath] = m.ReadOnly + } + if len(mounts) != 3 || !mounts["shared:/shared"] || !mounts["email-credentials:/vault-config/email"] { + t.Fatal("credential and script mounts must remain read only") + } + foundScratch := false + for _, v := range object.Spec.Template.Spec.Volumes { + if v.Name == "email-credential-work" && v.EmptyDir != nil && v.EmptyDir.Medium == "Memory" && v.EmptyDir.SizeLimit == "16Mi" { + if _, ok := mounts["email-credential-work:/work"]; ok { + foundScratch = true + } + } + } + if !foundScratch { + t.Fatal("credential scratch must use bounded memory-backed storage") + } + } + } + if !foundConfig || !foundContainer { + t.Fatal("declarative Email preparation is not wired into the configuration Job") + } +} diff --git a/scripts/tests/vault-email-credentials/native_test.go b/scripts/tests/vault-email-credentials/native_test.go new file mode 100644 index 0000000000..4c1ffd44ec --- /dev/null +++ b/scripts/tests/vault-email-credentials/native_test.go @@ -0,0 +1,151 @@ +package emailcredentials_test + +import ( + "bytes" + "encoding/json" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" + "time" + + "gopkg.in/yaml.v3" +) + +func TestPinnedOpenBaoPreservesValuesAndVersions(t *testing.T) { + if os.Getenv("EMAIL_CREDENTIAL_CONTAINER_TEST") != "1" { + t.Skip("real pinned OpenBao and tooling containers run in Linux CI") + } + if runtime.GOOS != "linux" { + t.Fatal("native container proof requires Linux host networking") + } + manifest, err := os.ReadFile(filepath.Join(repositoryRoot(), "k8s/bases/infrastructure/vault-config/job.yaml")) + if err != nil { + t.Fatal(err) + } + var job struct { + Spec struct { + Template struct { + Spec struct { + Containers []struct{ Name, Image string } + } + } + } + } + if err := yaml.Unmarshal(manifest, &job); err != nil { + t.Fatal(err) + } + image := "" + for _, c := range job.Spec.Template.Spec.Containers { + if c.Name == "vault-config" && strings.HasPrefix(c.Image, "quay.io/openbao/openbao:") && strings.Contains(c.Image, "@sha256:") { + image = c.Image + } + } + if image == "" { + t.Fatal("native test must use the declared pinned OpenBao version") + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + address := listener.Addr().String() + _ = listener.Close() + command := exec.Command("docker", "run", "--rm", "--detach", "--network=host", "--read-only", + "--cap-drop=ALL", "--security-opt=no-new-privileges", "--memory=256m", "--cpus=1", "--pids-limit=128", + "--tmpfs", "/tmp:rw,nosuid,size=32m", "--env", "HOME=/tmp", "--entrypoint", "bao", image, + "server", "-dev", "-dev-root-token-id="+syntheticToken, "-dev-listen-address="+address, "-dev-no-store-token") + command.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir()} + id, err := command.Output() + if err != nil || !regexp.MustCompile(`^[a-f0-9]{64}$`).Match(bytes.TrimSpace(id)) { + t.Fatal("isolated native OpenBao container could not start") + } + t.Cleanup(func() { + if err := exec.Command("docker", "rm", "--force", strings.TrimSpace(string(id))).Run(); err != nil { + t.Error("owned native OpenBao container cleanup failed") + } + }) + baseURL := "http://" + address + client := &http.Client{Timeout: 2 * time.Second} + deadline := time.Now().Add(20 * time.Second) + for { + response, err := client.Get(baseURL + "/v1/sys/health") + ready := err == nil && response.StatusCode == http.StatusOK + if response != nil { + _ = response.Body.Close() + } + if ready { + break + } + if time.Now().After(deadline) { + t.Fatal("isolated native OpenBao did not become ready") + } + time.Sleep(100 * time.Millisecond) + } + request := func(method, key string, payload any) (int, map[string]any) { + t.Helper() + var input io.Reader + if payload != nil { + body, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + input = bytes.NewReader(body) + } + req, err := http.NewRequest(method, baseURL+"/v1/secret/data/"+key, input) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Vault-Token", syntheticToken) + req.Header.Set("Content-Type", "application/json") + response, err := client.Do(req) + if err != nil { + t.Fatal("native synthetic credential request failed") + } + defer response.Body.Close() + var result map[string]any + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil { + t.Fatal("native credential response was invalid") + } + return response.StatusCode, result + } + current := credentials() + for key, data := range current { + status, _ := request(http.MethodPost, key, map[string]any{"options": map[string]int{"cas": 0}, "data": data}) + if status != http.StatusOK { + t.Fatal("native synthetic source could not be seeded") + } + } + for _, mode := range []string{"plan", "copy", "copy"} { + out, err := runMigrationAt(t, baseURL, mode) + if err != nil { + t.Fatalf("native preparation or replay failed: %v %s", err, out) + } + for key, data := range current { + destination := strings.Replace(key, "apps/private-mail/", "infrastructure/email/", 1) + status, result := request(http.MethodGet, destination, nil) + if mode == "plan" { + if status != http.StatusNotFound { + t.Fatal("native plan changed a destination") + } + continue + } + state, _ := result["data"].(map[string]any) + metadata, _ := state["metadata"].(map[string]any) + if status != http.StatusOK || !reflect.DeepEqual(state["data"], data) || metadata["version"] != float64(1) { + t.Fatal("native copy or replay changed fields or created another version") + } + sourceStatus, source := request(http.MethodGet, key, nil) + sourceState, _ := source["data"].(map[string]any) + if sourceStatus != http.StatusOK || !reflect.DeepEqual(sourceState["data"], data) { + t.Fatal("native copy altered the existing credential") + } + } + } +} From e72105f791dd5f7ae842e10b6b9f27beb1abe974 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 23:10:04 +0100 Subject: [PATCH 5/7] fix(email): verify credential custody from guarded workflow --- .github/workflows/ci.yaml | 6 +- .../publish-private-mail-bootstrap.yaml | 2 +- .../verify-email-credential-custody.yaml | 114 +++++++++++++ .../verify-private-mail-storage-recovery.yaml | 2 +- docs/adr/email.md | 16 +- .../config-map-email-credentials.yaml | 152 ----------------- .../infrastructure/vault-config/job.yaml | 52 +----- .../vault-config/kustomization.yaml | 1 - scripts/prepare-email-credentials.sh | 144 ++++++++++++++++ .../vault-email-credentials/migration_test.go | 161 +----------------- .../vault-email-credentials/native_test.go | 4 +- .../vault-email-credentials/workflow_test.go | 133 +++++++++++++++ 12 files changed, 414 insertions(+), 373 deletions(-) create mode 100644 .github/workflows/verify-email-credential-custody.yaml delete mode 100644 k8s/bases/infrastructure/vault-config/config-map-email-credentials.yaml create mode 100755 scripts/prepare-email-credentials.sh create mode 100644 scripts/tests/vault-email-credentials/workflow_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 19a5dd7458..81f7d56e6d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -146,6 +146,8 @@ jobs: - 'k8s/**' - 'scripts/tests/private-mail-preparation/**' - 'scripts/tests/vault-email-credentials/**' + - 'scripts/prepare-email-credentials.sh' + - '.github/workflows/verify-email-credential-custody.yaml' - 'scripts/tests/test-private-mail-server.sh' - 'scripts/tests/test-private-mail-configuration.sh' - 'scripts/private-mail-config/**' @@ -3361,11 +3363,11 @@ jobs: - name: 🔒 Validate Crossplane provider egress policy if: needs.changes.outputs.k8s == 'true' run: bash scripts/tests/test-crossplane-egress-policy.sh - - name: ✉️ Prove Email credential preparation in its pinned container + - name: ✉️ Verify Email credential custody against pinned OpenBao if: needs.changes.outputs.k8s == 'true' timeout-minutes: 5 env: - EMAIL_CREDENTIAL_CONTAINER_TEST: '1' + EMAIL_CREDENTIAL_NATIVE_TEST: '1' run: go test -count=1 -v ./scripts/tests/vault-email-credentials - name: ✉️ Prove native mail accounts and recipient restrictions if: needs.changes.outputs.k8s == 'true' diff --git a/.github/workflows/publish-private-mail-bootstrap.yaml b/.github/workflows/publish-private-mail-bootstrap.yaml index bc0cb20dc4..50cc4f36b4 100644 --- a/.github/workflows/publish-private-mail-bootstrap.yaml +++ b/.github/workflows/publish-private-mail-bootstrap.yaml @@ -1,4 +1,4 @@ -name: Publish Private Mail Bootstrap +name: Publish Email Bootstrap on: pull_request: diff --git a/.github/workflows/verify-email-credential-custody.yaml b/.github/workflows/verify-email-credential-custody.yaml new file mode 100644 index 0000000000..7d3c75fd9c --- /dev/null +++ b/.github/workflows/verify-email-credential-custody.yaml @@ -0,0 +1,114 @@ +name: Verify Email Credential Custody + +"on": + workflow_dispatch: + inputs: + confirm: + description: "Type exactly: verify-email-credential-custody" + required: true + type: string + +permissions: {} +concurrency: + group: prod-deploy + cancel-in-progress: false + queue: max + +jobs: + verify: + name: Verify Email credential custody without copying + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: prod + permissions: + contents: read + steps: + - name: Require main and typed confirmation + shell: bash + env: + CONFIRM: ${{ inputs.confirm }} + run: | + if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$CONFIRM" != verify-email-credential-custody ]]; then + printf '%s\n' 'EMAIL_CUSTODY_VERIFICATION=UNVERIFIED' >&2 + exit 2 + fi + + - name: Check out the confirmed revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Install the repository-approved Kubernetes reader + shell: bash + env: + KUBECTL_VERSION: v1.36.2 + KUBECTL_SHA256: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/email-custody-kubectl" + curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path" + printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check + mkdir -p "$RUNNER_TEMP/email-custody-tools" + install -m 0755 "$kubectl_path" "$RUNNER_TEMP/email-custody-tools/kubectl" + printf '%s\n' "$RUNNER_TEMP/email-custody-tools" >> "$GITHUB_PATH" + + - name: Compare all three credential records without copying + shell: bash + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + EMAIL_CUSTODY_MODE: plan + run: | + set +x + set -euo pipefail + umask 077 + fail() { printf '%s\n' 'EMAIL_CUSTODY_VERIFICATION=UNVERIFIED' >&2; exit 2; } + [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail + proof_dir="$(mktemp -d /dev/shm/email-custody.XXXXXXXX)" + forward_pid='' + cleanup() { + if [[ -n "$forward_pid" ]]; then + kill "$forward_pid" 2>/dev/null || true + wait "$forward_pid" 2>/dev/null || true + fi + rm -rf "$proof_dir" + } + trap cleanup EXIT + [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail + export KUBECONFIG="$proof_dir/kubeconfig" + printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" + unset KUBE_CONFIG + ./scripts/use-prod-stable-api-endpoint.sh > /dev/null 2>"$proof_dir/endpoint.log" || fail + unset HCLOUD_TOKEN + if ! kubectl -n openbao get secret openbao-unseal -o jsonpath='{.data.root-token}' 2>"$proof_dir/reader.log" | + base64 --decode > "$proof_dir/token" 2>"$proof_dir/decode.log"; then + fail + fi + [[ -s "$proof_dir/token" ]] || fail + kubectl -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200 > "$proof_dir/forward.log" 2>&1 & + forward_pid=$! + ready=false + for ((attempt=0; attempt<20; attempt++)); do + kill -0 "$forward_pid" 2>/dev/null || fail + if curl --disable --silent --fail --noproxy '*' --connect-timeout 1 --max-time 2 \ + http://127.0.0.1:38200/v1/sys/health > /dev/null 2>"$proof_dir/health.log"; then + ready=true + break + fi + sleep 1 + done + [[ "$ready" == true ]] || fail + export BAO_ADDR=http://127.0.0.1:38200 + export EMAIL_CUSTODY_MODE=plan + export EMAIL_CUSTODY_TOKEN_FILE="$proof_dir/token" + export EMAIL_CUSTODY_WORK_DIR="$proof_dir" + bash scripts/prepare-email-credentials.sh > "$proof_dir/result" 2>"$proof_dir/helper.log" || fail + result="$(cat "$proof_dir/result")" + if [[ "$result" =~ ^EMAIL_CUSTODY_PREPARATION=PLANNED\ mode=plan\ existing=([0-3])\ copied=0\ pending=([0-3])\ unseeded=0$ ]]; then + [[ $((BASH_REMATCH[1] + BASH_REMATCH[2])) == 3 ]] || fail + else + fail + fi + printf '%s\n' "$result" diff --git a/.github/workflows/verify-private-mail-storage-recovery.yaml b/.github/workflows/verify-private-mail-storage-recovery.yaml index e20a5e6766..861c808bb0 100644 --- a/.github/workflows/verify-private-mail-storage-recovery.yaml +++ b/.github/workflows/verify-private-mail-storage-recovery.yaml @@ -1,4 +1,4 @@ -name: Verify Private Mail Storage Recovery +name: Verify Email Storage Recovery "on": workflow_dispatch: diff --git a/docs/adr/email.md b/docs/adr/email.md index dd74747556..25ffa5e2de 100644 --- a/docs/adr/email.md +++ b/docs/adr/email.md @@ -86,13 +86,17 @@ infrastructure. Its credential destination is the `secret/` mount under `infrastructure/email/agent`. Retained namespace, datastore and image identifiers keep their existing `private-mail` names during this migration. -The OpenBao configuration Job first runs the declared credential preparation in -`plan` mode. It reads the current durable records under `apps/private-mail/`, +The main-only **Verify Email Credential Custody** workflow runs the canonical +`scripts/prepare-email-credentials.sh` in fixed `plan` mode. Dispatch it on `main` +with confirmation `verify-email-credential-custody`. It reads the current durable records under `apps/private-mail/`, compares any existing destination, and reports pending copies without changing -credentials or consumer paths. It uses the configuration Job's existing private -credential file and bounded memory-backed scratch space; credentials and API +credentials or consumer paths. It reuses the existing recovery credential over +a loopback connection, with private memory-backed scratch space; credentials and API responses are never logged. Missing or unreadable data is distinguished from a -valid plan, and a different destination fails closed. +valid plan, and a different destination fails closed. The workflow rejects an +unseeded or incomplete result and exposes only validated counters for all three +records. It has no copy input and cannot activate migration. No credential-handling +program is embedded in a Kubernetes ConfigMap or added to the configuration Job. After snapshot recovery has been verified, a separate reviewed revision may activate `copy` mode. Creation uses KV v2 compare-and-set with version zero and @@ -328,7 +332,7 @@ the existing deployment sequence and signer isolation without a manual node configuration change. Public mail activation and migration retain their separate acceptance requirements. -After the protected rollout, the dispatch-only **Verify Private Mail Storage +After the protected rollout, the dispatch-only **Verify Email Storage Recovery** workflow checks the original failed deployment's ownership window. Run it from `main` with confirmation `verify-private-mail-storage-recovery`. It uses the existing production reader while holding the deployment lock, makes diff --git a/k8s/bases/infrastructure/vault-config/config-map-email-credentials.yaml b/k8s/bases/infrastructure/vault-config/config-map-email-credentials.yaml deleted file mode 100644 index 667b0c2a8f..0000000000 --- a/k8s/bases/infrastructure/vault-config/config-map-email-credentials.yaml +++ /dev/null @@ -1,152 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: vault-email-credentials - namespace: openbao -data: - # Plan is the deployed default. Copy is a later reviewed activation after - # snapshot recovery is verified; it never switches consumers or deletes keys. - prepare-email-credentials.sh: | - #!/bin/sh - set -eu - umask 077 - - fail() { - printf '%s\n' 'EMAIL_CUSTODY_PREPARATION=FAIL' >&2 - exit 1 - } - - mode=$EMAIL_CUSTODY_MODE - case "$mode" in plan|copy) ;; *) fail ;; esac - [ -n "$BAO_ADDR" ] || fail - [ -f "$EMAIL_CUSTODY_TOKEN_FILE" ] || fail - [ -d "$EMAIL_CUSTODY_WORK_DIR" ] || fail - work=$(mktemp -d "$EMAIL_CUSTODY_WORK_DIR/email-credentials.XXXXXXXX") || fail - trap 'rm -rf "$work"' 0 - trap 'exit 1' HUP INT TERM - token=$(cat "$EMAIL_CUSTODY_TOKEN_FILE") || fail - case "$token" in ''|*[!A-Za-z0-9._-]*) fail ;; esac - printf 'X-Vault-Token: %s\nContent-Type: application/json\n' "$token" > "$work/headers" - unset token - - # Headers and JSON bodies are files in private memory, never process arguments. - # Disabling curl's user configuration also prevents accidental trace logging. - request() { - request_method=$1 - request_path=$2 - request_input=$3 - set -- --disable --silent --show-error --noproxy '*' \ - --connect-timeout 5 --max-time 20 --max-filesize 1048576 \ - --header "@$work/headers" --output "$work/response.json" \ - --write-out '%{http_code}' - case "$request_method" in - GET) ;; - POST) set -- "$@" --request POST --data-binary "@$request_input" ;; - *) fail ;; - esac - status=$(curl "$@" "$BAO_ADDR/v1/$request_path" 2> "$work/request-error") || fail - } - - read_record() { - record_path=$1 - record_output=$2 - : > "$record_output" - request GET "secret/data/$record_path" '' - case "$status" in - 404) return 4 ;; - 200) ;; - *) fail ;; - esac - # Preserve the complete data object, including any additional recovery fields. - # Invalid or unreadable credentials are not equivalent to an absent key. - # Refuse numeric extras rather than risk precision changes in JSON tooling. - if ! jq -ceS '.data.data | select(type == "object") | - select(all(..; type != "number")) | - select(.password | type == "string") | - select(.password | test("^[A-Za-z0-9]{40,128}$"))' \ - "$work/response.json" > "$record_output" 2> "$work/json-error"; then - fail - fi - } - - existing=0 - copied=0 - pending=0 - unseeded=0 - report() { - verdict=PLANNED - if [ "$unseeded" -gt 0 ]; then - verdict=UNSEEDED - elif [ "$mode" = copy ]; then - verdict=VERIFIED - fi - printf 'EMAIL_CUSTODY_PREPARATION=%s mode=%s existing=%s copied=%s pending=%s unseeded=%s\n' \ - "$verdict" "$mode" "$existing" "$copied" "$pending" "$unseeded" - } - - # This init container precedes normal configuration. A new installation may - # not have its KV mount yet; report it explicitly without creating passwords. - request GET sys/mounts '' - [ "$status" = 200 ] || fail - if ! jq -e '.data | type == "object"' "$work/response.json" > /dev/null 2> "$work/json-error"; then - fail - fi - if jq -e '.data["secret/"] == null' "$work/response.json" > /dev/null; then - unseeded=3 - report - exit 0 - fi - if ! jq -e '.data["secret/"] | .type == "kv" and .options.version == "2"' \ - "$work/response.json" > /dev/null 2> "$work/json-error"; then - fail - fi - - for account in admin maintainer agent; do - source_path=apps/private-mail/$account - destination_path=infrastructure/email/$account - source_present=false - destination_present=false - if read_record "$source_path" "$work/source.json"; then - source_present=true - else - [ "$?" = 4 ] || fail - fi - if read_record "$destination_path" "$work/destination.json"; then - destination_present=true - else - [ "$?" = 4 ] || fail - fi - - if [ "$source_present" = false ]; then - # A populated destination without its source is not a proven migration. - [ "$destination_present" = false ] || fail - unseeded=$((unseeded + 1)) - continue - fi - if [ "$destination_present" = true ]; then - cmp -s "$work/source.json" "$work/destination.json" || fail - existing=$((existing + 1)) - elif [ "$mode" = plan ]; then - pending=$((pending + 1)) - else - if ! jq -cn --slurpfile value "$work/source.json" \ - '{options: {cas: 0}, data: $value[0]}' > "$work/create.json" 2> "$work/json-error"; then - fail - fi - request POST "secret/data/$destination_path" "$work/create.json" - creation_status=$status - case "$creation_status" in 200|204|400) ;; *) fail ;; esac - # Read back successful writes AND conditional-create races. Never replace - # a different destination or silently accept a lost/uncertain write. - read_record "$destination_path" "$work/destination.json" || fail - cmp -s "$work/source.json" "$work/destination.json" || fail - if [ "$creation_status" = 400 ]; then - existing=$((existing + 1)) - else - copied=$((copied + 1)) - fi - fi - read_record "$source_path" "$work/source-after.json" || fail - cmp -s "$work/source.json" "$work/source-after.json" || fail - done - report diff --git a/k8s/bases/infrastructure/vault-config/job.yaml b/k8s/bases/infrastructure/vault-config/job.yaml index cfb31685d0..2ff97c16ec 100644 --- a/k8s/bases/infrastructure/vault-config/job.yaml +++ b/k8s/bases/infrastructure/vault-config/job.yaml @@ -126,13 +126,6 @@ spec: secret: secretName: vault-config-oidc optional: true - - name: email-credentials - configMap: - name: vault-email-credentials - - name: email-credential-work - emptyDir: - medium: Memory - sizeLimit: 16Mi # Local scratch for the newest raft snapshot, populated by the # fetch-snapshot initContainer from the OFF-CLUSTER mirror (Cloudflare R2 # in prod, in-cluster MinIO locally) -- NOT the single-attach RWO @@ -520,49 +513,6 @@ spec: else echo "Vault was already initialized — no new keys to store." fi - # Read-only preparation for company Email credential custody. Reuse the - # configuration Job's existing private root-token file; no new identity, - # Kubernetes grant, password generation, consumer switch or deletion. - - name: prepare-email-credentials - image: docker.io/alpine/k8s:1.37.1@sha256:712dc5385d1c1a3f83b747fa9e40f64d433a97f72ff42f9755ff7dd1f62846e4 - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - env: - - name: BAO_ADDR - value: http://openbao-active.openbao.svc.cluster.local:8200 - - name: EMAIL_CUSTODY_MODE - value: plan - - name: EMAIL_CUSTODY_TOKEN_FILE - value: /shared/root-token - - name: EMAIL_CUSTODY_WORK_DIR - value: /work - volumeMounts: - - name: shared - mountPath: /shared - readOnly: true - - name: email-credentials - mountPath: /vault-config/email - readOnly: true - - name: email-credential-work - mountPath: /work - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 250m - memory: 128Mi - command: - - /bin/sh - - -eu - - /vault-config/email/prepare-email-credentials.sh containers: - name: vault-config image: quay.io/openbao/openbao:2.5.3@sha256:fdc6da21ca6963560c32336fd7feb9cf2d5e52668f1a1647205a4b41171f0806 @@ -1086,7 +1036,7 @@ spec: path "secret/data/apps/crossview/*" { capabilities = ["create", "update", "read"] } - # Private mail preparation preserves three durable credentials. + # Company Email preparation preserves three durable credentials. # Exact-key read/seed only; no mailbox data or wildcard grant. path "secret/data/apps/private-mail/admin" { capabilities = ["create", "update", "read"] diff --git a/k8s/bases/infrastructure/vault-config/kustomization.yaml b/k8s/bases/infrastructure/vault-config/kustomization.yaml index 06d88a4dec..6e355d2b38 100644 --- a/k8s/bases/infrastructure/vault-config/kustomization.yaml +++ b/k8s/bases/infrastructure/vault-config/kustomization.yaml @@ -6,6 +6,5 @@ resources: - role.yaml - role-binding.yaml - job.yaml - - config-map-email-credentials.yaml - external-secret.yaml - cilium-network-policy.yaml diff --git a/scripts/prepare-email-credentials.sh b/scripts/prepare-email-credentials.sh new file mode 100755 index 0000000000..7c2f884bea --- /dev/null +++ b/scripts/prepare-email-credentials.sh @@ -0,0 +1,144 @@ +#!/bin/bash +set -eu +umask 077 + +fail() { + printf '%s\n' 'EMAIL_CUSTODY_PREPARATION=FAIL' >&2 + exit 1 +} + +mode=$EMAIL_CUSTODY_MODE +case "$mode" in plan|copy) ;; *) fail ;; esac +[ -n "$BAO_ADDR" ] || fail +[ -f "$EMAIL_CUSTODY_TOKEN_FILE" ] || fail +[ -d "$EMAIL_CUSTODY_WORK_DIR" ] || fail +work=$(mktemp -d "$EMAIL_CUSTODY_WORK_DIR/email-credentials.XXXXXXXX") || fail +trap 'rm -rf "$work"' 0 +trap 'exit 1' HUP INT TERM +token=$(cat "$EMAIL_CUSTODY_TOKEN_FILE") || fail +case "$token" in ''|*[!A-Za-z0-9._-]*) fail ;; esac +printf 'X-Vault-Token: %s\nContent-Type: application/json\n' "$token" > "$work/headers" +unset token + +# Headers and JSON bodies are files in private memory, never process arguments. +# Disabling curl's user configuration also prevents accidental trace logging. +request() { + request_method=$1 + request_path=$2 + request_input=$3 + set -- --disable --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 20 --max-filesize 1048576 \ + --header "@$work/headers" --output "$work/response.json" \ + --write-out '%{http_code}' + case "$request_method" in + GET) ;; + POST) set -- "$@" --request POST --data-binary "@$request_input" ;; + *) fail ;; + esac + status=$(curl "$@" "$BAO_ADDR/v1/$request_path" 2> "$work/request-error") || fail +} + +read_record() { + record_path=$1 + record_output=$2 + : > "$record_output" + request GET "secret/data/$record_path" '' + case "$status" in + 404) return 4 ;; + 200) ;; + *) fail ;; + esac + # Preserve the complete data object, including any additional recovery fields. + # Invalid or unreadable credentials are not equivalent to an absent key. + # Refuse numeric extras rather than risk precision changes in JSON tooling. + if ! jq -ceS '.data.data | select(type == "object") | + select(all(..; type != "number")) | + select(.password | type == "string") | + select(.password | test("^[A-Za-z0-9]{40,128}$"))' \ + "$work/response.json" > "$record_output" 2> "$work/json-error"; then + fail + fi +} + +existing=0 +copied=0 +pending=0 +unseeded=0 +report() { + verdict=PLANNED + if [ "$unseeded" -gt 0 ]; then + verdict=UNSEEDED + elif [ "$mode" = copy ]; then + verdict=VERIFIED + fi + printf 'EMAIL_CUSTODY_PREPARATION=%s mode=%s existing=%s copied=%s pending=%s unseeded=%s\n' \ + "$verdict" "$mode" "$existing" "$copied" "$pending" "$unseeded" +} + +# A fresh installation may not have its KV mount yet. Report it explicitly +# without creating passwords or mistaking absence for a verified migration. +request GET sys/mounts '' +[ "$status" = 200 ] || fail +if ! jq -e '.data | type == "object"' "$work/response.json" > /dev/null 2> "$work/json-error"; then + fail +fi +if jq -e '.data["secret/"] == null' "$work/response.json" > /dev/null; then + unseeded=3 + report + exit 0 +fi +if ! jq -e '.data["secret/"] | .type == "kv" and .options.version == "2"' \ + "$work/response.json" > /dev/null 2> "$work/json-error"; then + fail +fi + +for account in admin maintainer agent; do + source_path=apps/private-mail/$account + destination_path=infrastructure/email/$account + source_present=false + destination_present=false + if read_record "$source_path" "$work/source.json"; then + source_present=true + else + [ "$?" = 4 ] || fail + fi + if read_record "$destination_path" "$work/destination.json"; then + destination_present=true + else + [ "$?" = 4 ] || fail + fi + + if [ "$source_present" = false ]; then + # A populated destination without its source is not a proven migration. + [ "$destination_present" = false ] || fail + unseeded=$((unseeded + 1)) + continue + fi + if [ "$destination_present" = true ]; then + cmp -s "$work/source.json" "$work/destination.json" || fail + existing=$((existing + 1)) + elif [ "$mode" = plan ]; then + pending=$((pending + 1)) + else + if ! jq -cn --slurpfile value "$work/source.json" \ + '{options: {cas: 0}, data: $value[0]}' > "$work/create.json" 2> "$work/json-error"; then + fail + fi + request POST "secret/data/$destination_path" "$work/create.json" + creation_status=$status + case "$creation_status" in 200|204|400) ;; *) fail ;; esac + # Read back successful writes AND conditional-create races. Never replace + # a different destination or silently accept a lost/uncertain write. + read_record "$destination_path" "$work/destination.json" || fail + cmp -s "$work/source.json" "$work/destination.json" || fail + if [ "$creation_status" = 400 ]; then + existing=$((existing + 1)) + else + copied=$((copied + 1)) + fi + fi + read_record "$source_path" "$work/source-after.json" || fail + cmp -s "$work/source.json" "$work/source-after.json" || fail +done +report + diff --git a/scripts/tests/vault-email-credentials/migration_test.go b/scripts/tests/vault-email-credentials/migration_test.go index 8af16131cf..2d4911a7e7 100644 --- a/scripts/tests/vault-email-credentials/migration_test.go +++ b/scripts/tests/vault-email-credentials/migration_test.go @@ -1,10 +1,8 @@ package emailcredentials_test import ( - "bytes" "encoding/json" "fmt" - "io" "net/http" "net/http/httptest" "os" @@ -15,8 +13,6 @@ import ( "strings" "sync" "testing" - - "gopkg.in/yaml.v3" ) const syntheticToken = "test-only-root-token" @@ -154,23 +150,15 @@ func runMigration(t *testing.T, fixture *apiFixture, mode string) (string, error func runMigrationAt(t *testing.T, serverURL, mode string) (string, error) { t.Helper() - manifest, err := os.ReadFile(filepath.Join(repositoryRoot(), "k8s/bases/infrastructure/vault-config/config-map-email-credentials.yaml")) + script, err := os.ReadFile(filepath.Join(repositoryRoot(), "scripts/prepare-email-credentials.sh")) if err != nil { - t.Fatal("missing declarative Email credential preparation", err) - } - var config struct{ Data map[string]string } - if err := yaml.Unmarshal(manifest, &config); err != nil { - t.Fatal(err) - } - script := config.Data["prepare-email-credentials.sh"] - if script == "" { - t.Fatal("missing executable credential preparation") + t.Fatal("missing canonical Email credential preparation", err) } dir := t.TempDir() if err := os.Chmod(dir, 0o700); err != nil { t.Fatal(err) } - for name, content := range map[string]string{"prepare.sh": script, "token": syntheticToken} { + for name, content := range map[string]string{"prepare.sh": string(script), "token": syntheticToken} { if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil { t.Fatal(err) } @@ -181,25 +169,7 @@ func runMigrationAt(t *testing.T, serverURL, mode string) (string, error) { "EMAIL_CUSTODY_TOKEN_FILE=" + filepath.Join(dir, "token"), "EMAIL_CUSTODY_WORK_DIR=" + dir, } - command := exec.Command("/bin/sh", "-eu", filepath.Join(dir, "prepare.sh")) - if os.Getenv("EMAIL_CREDENTIAL_CONTAINER_TEST") == "1" { - // Linux CI runs the identical fixture through the exact deployed image. - // Bind only synthetic private inputs, and grant no container capabilities. - if runtime.GOOS != "linux" { - t.Fatal("native container proof requires the Linux CI host network") - } - args := []string{ - "run", "--rm", "--network=host", "--read-only", "--cap-drop=ALL", - "--security-opt=no-new-privileges", "--user", fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), - "--mount", "type=bind,source=" + dir + ",target=" + dir, - "--entrypoint", "/bin/sh", - } - for _, value := range environment[1:] { - args = append(args, "--env", value) - } - args = append(args, deployedToolingImage(t), "-eu", filepath.Join(dir, "prepare.sh")) - command = exec.Command("docker", args...) - } + command := exec.Command("bash", "-eu", filepath.Join(dir, "prepare.sh")) command.Env = environment out, runErr := command.CombinedOutput() if strings.Contains(string(out), syntheticToken) || strings.Contains(string(out), strings.Repeat("A", 48)) || strings.Contains(string(out), serverURL) { @@ -215,33 +185,6 @@ func runMigrationAt(t *testing.T, serverURL, mode string) (string, error) { return string(out), runErr } -func deployedToolingImage(t *testing.T) string { - t.Helper() - data, err := os.ReadFile(filepath.Join(repositoryRoot(), "k8s/bases/infrastructure/vault-config/job.yaml")) - if err != nil { - t.Fatal(err) - } - var job struct { - Spec struct { - Template struct { - Spec struct { - InitContainers []struct{ Name, Image string } `yaml:"initContainers"` - } - } - } - } - if err := yaml.Unmarshal(data, &job); err != nil { - t.Fatal(err) - } - for _, c := range job.Spec.Template.Spec.InitContainers { - if c.Name == "prepare-email-credentials" && strings.HasPrefix(c.Image, "docker.io/alpine/k8s:") && strings.Contains(c.Image, "@sha256:") { - return c.Image - } - } - t.Fatal("native proof must use the exact declared tooling image") - return "" -} - func TestPlanIsReadOnlyAndCopyPreservesEveryCurrentField(t *testing.T) { fixture := &apiFixture{keys: credentials()} out, err := runMigration(t, fixture, "plan") @@ -333,99 +276,3 @@ func TestUnseededInstallationIsExplicitAndNeverCreatesCredentials(t *testing.T) }) } } - -func TestDeclaredPreparationRunsReadOnlyWithPrivateMemoryScratch(t *testing.T) { - out, err := exec.Command("kubectl", "kustomize", filepath.Join(repositoryRoot(), "k8s/bases/infrastructure/vault-config")).CombinedOutput() - if err != nil { - t.Fatalf("configuration bundle must render: %v", err) - } - type volume struct { - Name string - EmptyDir *struct { - Medium string - SizeLimit string `yaml:"sizeLimit"` - } `yaml:"emptyDir"` - } - type container struct { - Name, Image string - Command []string - Env []struct{ Name, Value string } - VolumeMounts []struct { - Name string - MountPath string `yaml:"mountPath"` - ReadOnly bool `yaml:"readOnly"` - } `yaml:"volumeMounts"` - SecurityContext struct { - ReadOnlyRootFilesystem bool `yaml:"readOnlyRootFilesystem"` - RunAsNonRoot bool `yaml:"runAsNonRoot"` - AllowPrivilegeEscalation *bool `yaml:"allowPrivilegeEscalation"` - } `yaml:"securityContext"` - } - var foundConfig, foundContainer bool - decoder := yaml.NewDecoder(bytes.NewReader(out)) - for { - var object struct { - Kind string - Metadata struct{ Name string } - Spec struct { - Template struct { - Spec struct { - InitContainers []container `yaml:"initContainers"` - Volumes []volume - } - } - } - } - if err := decoder.Decode(&object); err == io.EOF { - break - } else if err != nil { - t.Fatal(err) - } - if object.Kind == "ConfigMap" && object.Metadata.Name == "vault-email-credentials" { - foundConfig = true - } - if object.Kind != "Job" || object.Metadata.Name != "vault-config" { - continue - } - for _, c := range object.Spec.Template.Spec.InitContainers { - if c.Name != "prepare-email-credentials" { - continue - } - foundContainer = true - if c.Image != deployedToolingImage(t) || !reflect.DeepEqual(c.Command, []string{"/bin/sh", "-eu", "/vault-config/email/prepare-email-credentials.sh"}) { - t.Fatal("preparation must execute the declared script with the existing pinned tooling image") - } - env := map[string]string{} - for _, v := range c.Env { - env[v.Name] = v.Value - } - if len(env) != 4 || env["EMAIL_CUSTODY_MODE"] != "plan" || env["EMAIL_CUSTODY_TOKEN_FILE"] != "/shared/root-token" || env["EMAIL_CUSTODY_WORK_DIR"] != "/work" { - t.Fatal("deployed preparation must only plan and reuse the existing private credential file") - } - if !c.SecurityContext.ReadOnlyRootFilesystem || !c.SecurityContext.RunAsNonRoot || c.SecurityContext.AllowPrivilegeEscalation == nil || *c.SecurityContext.AllowPrivilegeEscalation { - t.Fatal("preparation must preserve the restricted container boundary") - } - mounts := map[string]bool{} - for _, m := range c.VolumeMounts { - mounts[m.Name+":"+m.MountPath] = m.ReadOnly - } - if len(mounts) != 3 || !mounts["shared:/shared"] || !mounts["email-credentials:/vault-config/email"] { - t.Fatal("credential and script mounts must remain read only") - } - foundScratch := false - for _, v := range object.Spec.Template.Spec.Volumes { - if v.Name == "email-credential-work" && v.EmptyDir != nil && v.EmptyDir.Medium == "Memory" && v.EmptyDir.SizeLimit == "16Mi" { - if _, ok := mounts["email-credential-work:/work"]; ok { - foundScratch = true - } - } - } - if !foundScratch { - t.Fatal("credential scratch must use bounded memory-backed storage") - } - } - } - if !foundConfig || !foundContainer { - t.Fatal("declarative Email preparation is not wired into the configuration Job") - } -} diff --git a/scripts/tests/vault-email-credentials/native_test.go b/scripts/tests/vault-email-credentials/native_test.go index 4c1ffd44ec..da34e61f61 100644 --- a/scripts/tests/vault-email-credentials/native_test.go +++ b/scripts/tests/vault-email-credentials/native_test.go @@ -20,8 +20,8 @@ import ( ) func TestPinnedOpenBaoPreservesValuesAndVersions(t *testing.T) { - if os.Getenv("EMAIL_CREDENTIAL_CONTAINER_TEST") != "1" { - t.Skip("real pinned OpenBao and tooling containers run in Linux CI") + if os.Getenv("EMAIL_CREDENTIAL_NATIVE_TEST") != "1" { + t.Skip("real pinned OpenBao API proof runs in Linux CI") } if runtime.GOOS != "linux" { t.Fatal("native container proof requires Linux host networking") diff --git a/scripts/tests/vault-email-credentials/workflow_test.go b/scripts/tests/vault-email-credentials/workflow_test.go new file mode 100644 index 0000000000..6a9c437982 --- /dev/null +++ b/scripts/tests/vault-email-credentials/workflow_test.go @@ -0,0 +1,133 @@ +package emailcredentials_test + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +type verificationStep struct { + Name, Uses, Run, Shell string + Env map[string]string + With map[string]any +} + +func verificationSteps(t *testing.T) []verificationStep { + t.Helper() + data, err := os.ReadFile(filepath.Join(repositoryRoot(), ".github/workflows/verify-email-credential-custody.yaml")) + if err != nil { + t.Fatal("missing guarded Email custody verification workflow", err) + } + var workflow struct { + On map[string]any + Permissions map[string]string + Jobs map[string]struct { + RunsOn string `yaml:"runs-on"` + Environment string + Permissions map[string]string + Steps []verificationStep + } + } + if err := yaml.Unmarshal(data, &workflow); err != nil { + t.Fatal(err) + } + job, ok := workflow.Jobs["verify"] + if len(workflow.On) != 1 || workflow.On["workflow_dispatch"] == nil || len(workflow.Permissions) != 0 || !ok || len(workflow.Jobs) != 1 { + t.Fatal("custody verification must be dispatch-only with no ambient write permissions") + } + if job.RunsOn != "ubuntu-latest" || job.Environment != "prod" || len(job.Permissions) != 1 || job.Permissions["contents"] != "read" || len(job.Steps) != 4 { + t.Fatal("verification must retain its hosted, protected, read-only execution boundary") + } + checkout := job.Steps[1] + if checkout.Uses != "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" || checkout.With["ref"] != "${{ github.sha }}" || checkout.With["persist-credentials"] != false { + t.Fatal("verification must check out the confirmed immutable revision without persisted credentials") + } + return job.Steps +} + +func TestVerificationGuardRunsBeforeCheckoutOrSecrets(t *testing.T) { + steps := verificationSteps(t) + guard := steps[0] + if guard.Uses != "" || guard.Shell != "bash" || len(guard.Env) != 1 || guard.Env["CONFIRM"] != "${{ inputs.confirm }}" { + t.Fatal("the first step must guard dispatch before any checkout or secret access") + } + for _, test := range []struct { + name, event, ref, repo, confirm string + allowed bool + }{ + {"approved", "workflow_dispatch", "refs/heads/main", "devantler-tech/platform", "verify-email-credential-custody", true}, + {"pull request", "pull_request", "refs/heads/main", "devantler-tech/platform", "verify-email-credential-custody", false}, + {"branch", "workflow_dispatch", "refs/heads/codex/other", "devantler-tech/platform", "verify-email-credential-custody", false}, + {"repository", "workflow_dispatch", "refs/heads/main", "other/platform", "verify-email-credential-custody", false}, + {"confirmation", "workflow_dispatch", "refs/heads/main", "devantler-tech/platform", "copy", false}, + {"shell input", "workflow_dispatch", "refs/heads/main", "devantler-tech/platform", "$(exit 0)", false}, + } { + t.Run(test.name, func(t *testing.T) { + command := exec.Command("bash", "-eu", "-c", guard.Run) + command.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_EVENT_NAME=" + test.event, "GITHUB_REF=" + test.ref, "GITHUB_REPOSITORY=" + test.repo, "CONFIRM=" + test.confirm} + if err := command.Run(); (err == nil) != test.allowed { + t.Fatal("dispatch guard admitted an unconfirmed execution or refused main") + } + }) + } +} + +func TestVerificationWrapperRefusesIncompleteProofAndNeverCopies(t *testing.T) { + step := verificationSteps(t)[3] + if len(step.Env) != 3 || step.Env["KUBE_CONFIG"] != "${{ secrets.KUBE_CONFIG }}" || step.Env["HCLOUD_TOKEN"] != "${{ secrets.HCLOUD_TOKEN }}" || step.Env["EMAIL_CUSTODY_MODE"] != "plan" { + t.Fatal("verification must use only existing credentials and force read-only planning") + } + if runtime.GOOS != "linux" { + t.Skip("the unchanged workflow wrapper requires Linux /dev/shm; Linux CI executes all refusal cases") + } + for _, test := range []struct { + name, proof string + allowed bool + }{ + {"pending", "EMAIL_CUSTODY_PREPARATION=PLANNED mode=plan existing=0 copied=0 pending=3 unseeded=0", true}, + {"matching", "EMAIL_CUSTODY_PREPARATION=PLANNED mode=plan existing=3 copied=0 pending=0 unseeded=0", true}, + {"mixed", "EMAIL_CUSTODY_PREPARATION=PLANNED mode=plan existing=1 copied=0 pending=2 unseeded=0", true}, + {"missing", "EMAIL_CUSTODY_PREPARATION=UNSEEDED mode=plan existing=0 copied=0 pending=0 unseeded=3", false}, + {"incomplete", "EMAIL_CUSTODY_PREPARATION=PLANNED mode=plan existing=1 copied=0 pending=1 unseeded=0", false}, + {"copy", "EMAIL_CUSTODY_PREPARATION=VERIFIED mode=copy existing=0 copied=3 pending=0 unseeded=0", false}, + {"private output", "unexpected private response", false}, + } { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + for _, path := range []string{"bin", "scripts"} { + if err := os.Mkdir(filepath.Join(dir, path), 0o700); err != nil { + t.Fatal(err) + } + } + files := map[string]string{ + "bin/findmnt": "#!/bin/bash\nprintf 'tmpfs\\n'\n", + "bin/curl": "#!/bin/bash\nexit 0\n", + "bin/kubectl": "#!/bin/bash\nset -eu\ncase \"$*\" in\n'-n openbao get secret openbao-unseal -o jsonpath={.data.root-token}') printf 'dGVzdC1vbmx5LXJvb3QtdG9rZW4=';;\n'-n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200') exec sleep 60;;\n*) exit 91;;\nesac\n", + "scripts/use-prod-stable-api-endpoint.sh": "#!/bin/bash\nexit 0\n", + "scripts/prepare-email-credentials.sh": "#!/bin/bash\nset -eu\n[[ $EMAIL_CUSTODY_MODE == plan && $EMAIL_CUSTODY_WORK_DIR == /dev/shm/email-custody.* && -s $EMAIL_CUSTODY_TOKEN_FILE ]] || exit 92\nprintf '%s\\n' \"$SYNTHETIC_PROOF\"\n", + } + for name, content := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o700); err != nil { + t.Fatal(err) + } + } + // The wrapper itself runs unchanged. Only its external reader/API and + // helper boundary are synthetic; any unexpected kubectl action fails. + command := exec.Command("bash", "-eu", "-c", step.Run) + command.Dir = dir + command.Env = []string{"PATH=" + filepath.Join(dir, "bin") + ":" + os.Getenv("PATH"), "HOME=" + dir, "KUBE_CONFIG=synthetic-kubeconfig", "HCLOUD_TOKEN=synthetic-provider-token", "EMAIL_CUSTODY_MODE=copy", "SYNTHETIC_PROOF=" + test.proof} + out, err := command.CombinedOutput() + if (err == nil) != test.allowed { + t.Fatalf("verification wrapper accepted incomplete proof or refused all-three plan: %v %s", err, out) + } + if strings.Contains(string(out), "synthetic-") || strings.Contains(string(out), "unexpected private response") { + t.Fatal("verification wrapper exposed private input or unvalidated output") + } + }) + } +} From e1c38d7983155b2378ae026a16cf8da13e401a17 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 23:13:52 +0100 Subject: [PATCH 6/7] fix(email): use reviewed endpoint selection wrapper --- .github/workflows/verify-email-credential-custody.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify-email-credential-custody.yaml b/.github/workflows/verify-email-credential-custody.yaml index 7d3c75fd9c..c37f58c59f 100644 --- a/.github/workflows/verify-email-credential-custody.yaml +++ b/.github/workflows/verify-email-credential-custody.yaml @@ -80,7 +80,13 @@ jobs: export KUBECONFIG="$proof_dir/kubeconfig" printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" unset KUBE_CONFIG - ./scripts/use-prod-stable-api-endpoint.sh > /dev/null 2>"$proof_dir/endpoint.log" || fail + if ! ( + if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then + exit 1 + fi + ) 2>"$proof_dir/endpoint.log"; then + fail + fi unset HCLOUD_TOKEN if ! kubectl -n openbao get secret openbao-unseal -o jsonpath='{.data.root-token}' 2>"$proof_dir/reader.log" | base64 --decode > "$proof_dir/token" 2>"$proof_dir/decode.log"; then From 6eaa901c05076fde7da05c76e7168e941924cd0b Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 27 Oct 2026 00:37:40 +0100 Subject: [PATCH 7/7] test(email): require deliberate credential refusal verdict --- scripts/private-mail-config/bootstrap.go | 3 +++ scripts/private-mail-config/config.go | 1 + scripts/private-mail-config/main.go | 4 ++++ .../private-mail-preparation/production_test.go | 1 + .../tests/vault-email-credentials/migration_test.go | 13 ++++++++++++- .../tests/vault-email-credentials/native_test.go | 1 + .../tests/vault-email-credentials/workflow_test.go | 3 +++ 7 files changed, 25 insertions(+), 1 deletion(-) diff --git a/scripts/private-mail-config/bootstrap.go b/scripts/private-mail-config/bootstrap.go index 1f5bf05758..f4cde1ef5a 100644 --- a/scripts/private-mail-config/bootstrap.go +++ b/scripts/private-mail-config/bootstrap.go @@ -27,6 +27,7 @@ type bootstrapRuntime struct { startup, shutdown time.Duration } +// bootstrapCommand runs the bounded bootstrap lifecycle using private file inputs. func bootstrapCommand(args []string, log io.Writer) int { f := flag.NewFlagSet("bootstrap", flag.ContinueOnError) f.SetOutput(io.Discard) @@ -62,6 +63,7 @@ func bootstrapCommand(args []string, log io.Writer) int { type boundedOutput struct{ bytes.Buffer } +// Write retains only the configured amount of bootstrap diagnostics. func (b *boundedOutput) Write(p []byte) (int, error) { if b.Len()+len(p) > 1048576 { return 0, errConfiguration @@ -202,6 +204,7 @@ prepared: return nil } +// stopOwned terminates the bootstrap child within the cleanup grace period. func stopOwned(child *exec.Cmd, done <-chan error, grace time.Duration) error { select { case <-done: diff --git a/scripts/private-mail-config/config.go b/scripts/private-mail-config/config.go index 5acc7e4144..273206d9cc 100644 --- a/scripts/private-mail-config/config.go +++ b/scripts/private-mail-config/config.go @@ -29,6 +29,7 @@ var domainPattern = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?: var localPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,31}$`) var passwordPattern = regexp.MustCompile(`^[a-zA-Z0-9]{40,128}$`) +// render builds staged settings and account changes from validated private inputs. func render(v inputs) ([]byte, error) { if !domainPattern.MatchString(v.Domain) || strings.HasSuffix(v.Domain, ".invalid") || len(v.Domain) > 200 || !domainPattern.MatchString(v.Hostname) || !strings.HasSuffix(v.Hostname, "."+v.Domain) || len(v.Hostname) > 253 { return nil, errConfiguration diff --git a/scripts/private-mail-config/main.go b/scripts/private-mail-config/main.go index 4b14d0cb74..ba6ad4019d 100644 --- a/scripts/private-mail-config/main.go +++ b/scripts/private-mail-config/main.go @@ -10,6 +10,7 @@ import ( "path/filepath" ) +// privateInput reads a bounded input without including its contents in errors. func privateInput(dir, name string, limit int64) (data []byte, result error) { // Projected Kubernetes Secrets use symlinks; require the resolved target to // remain under the private input directory rather than rejecting projections. @@ -46,6 +47,7 @@ func privateInput(dir, name string, limit int64) (data []byte, result error) { return b, nil } +// loadInputs validates the settings and credential files needed for configuration. func loadInputs(dir string) (inputs, error) { var v inputs fail := func() (inputs, error) { return inputs{}, errConfiguration } @@ -115,6 +117,7 @@ func loadInputs(dir string) (inputs, error) { return v, nil } +// command handles the Email configuration CLI without echoing private input values. func command(args []string, log io.Writer) int { if len(args) > 0 && args[0] == "bootstrap" { return bootstrapCommand(args[1:], log) @@ -159,4 +162,5 @@ func command(args []string, log io.Writer) int { return 0 } +// main preserves the configuration command's exit status. func main() { os.Exit(command(os.Args[1:], os.Stderr)) } diff --git a/scripts/tests/private-mail-preparation/production_test.go b/scripts/tests/private-mail-preparation/production_test.go index 679285a2a7..81df5c2031 100644 --- a/scripts/tests/private-mail-preparation/production_test.go +++ b/scripts/tests/private-mail-preparation/production_test.go @@ -5,6 +5,7 @@ import ( "testing" ) +// TestProductionAdministrationUsesEncryptedSettings checks the production configuration boundary. func TestProductionAdministrationUsesEncryptedSettings(t *testing.T) { objects := render(t, "k8s/providers/hetzner/apps") sts := object(t, objects, "StatefulSet", "private-mail") diff --git a/scripts/tests/vault-email-credentials/migration_test.go b/scripts/tests/vault-email-credentials/migration_test.go index 2d4911a7e7..893d800718 100644 --- a/scripts/tests/vault-email-credentials/migration_test.go +++ b/scripts/tests/vault-email-credentials/migration_test.go @@ -29,6 +29,7 @@ type apiFixture struct { loseReply bool } +// credentials supplies synthetic records with extra fields that must survive copying. func credentials() map[string]map[string]any { keys := map[string]map[string]any{} for _, account := range []string{"admin", "maintainer", "agent"} { @@ -40,6 +41,7 @@ func credentials() map[string]map[string]any { return keys } +// serve models KV v2 reads, conditional creation, races and uncertain replies. func (f *apiFixture) serve(w http.ResponseWriter, r *http.Request) { f.mu.Lock() defer f.mu.Unlock() @@ -125,6 +127,7 @@ func (f *apiFixture) serve(w http.ResponseWriter, r *http.Request) { _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"version": 1}}) } +// allowedKey rejects fixture requests outside the six declared migration paths. func allowedKey(key string) bool { for _, prefix := range []string{"apps/private-mail/", "infrastructure/email/"} { for _, account := range []string{"admin", "maintainer", "agent"} { @@ -136,11 +139,13 @@ func allowedKey(key string) bool { return false } +// repositoryRoot binds fixtures to this test's checkout rather than the caller's directory. func repositoryRoot() string { _, file, _, _ := runtime.Caller(0) return filepath.Clean(filepath.Join(filepath.Dir(file), "../../..")) } +// runMigration executes the canonical program against an isolated synthetic API. func runMigration(t *testing.T, fixture *apiFixture, mode string) (string, error) { t.Helper() server := httptest.NewServer(http.HandlerFunc(fixture.serve)) @@ -148,6 +153,7 @@ func runMigration(t *testing.T, fixture *apiFixture, mode string) (string, error return runMigrationAt(t, server.URL, mode) } +// runMigrationAt confines inputs and checks output privacy and scratch cleanup. func runMigrationAt(t *testing.T, serverURL, mode string) (string, error) { t.Helper() script, err := os.ReadFile(filepath.Join(repositoryRoot(), "scripts/prepare-email-credentials.sh")) @@ -185,6 +191,7 @@ func runMigrationAt(t *testing.T, serverURL, mode string) (string, error) { return string(out), runErr } +// TestPlanIsReadOnlyAndCopyPreservesEveryCurrentField verifies copying and idempotent replay. func TestPlanIsReadOnlyAndCopyPreservesEveryCurrentField(t *testing.T) { fixture := &apiFixture{keys: credentials()} out, err := runMigration(t, fixture, "plan") @@ -207,6 +214,7 @@ func TestPlanIsReadOnlyAndCopyPreservesEveryCurrentField(t *testing.T) { } } +// TestMigrationRefusesConflictsAndUnreadableState requires a deliberate refusal. func TestMigrationRefusesConflictsAndUnreadableState(t *testing.T) { for _, name := range []string{"destination-conflict", "read-refused", "invalid-password", "numeric-field", "invalid-mount", "different-race", "source-rotation", "invalid-mode"} { t.Run(name, func(t *testing.T) { @@ -231,7 +239,7 @@ func TestMigrationRefusesConflictsAndUnreadableState(t *testing.T) { mode = "replace" } out, err := runMigration(t, fixture, mode) - if err == nil || strings.Contains(out, "=PASS") { + if err == nil || !strings.Contains(out, "EMAIL_CUSTODY_PREPARATION=FAIL") { t.Fatalf("uncertain state must fail closed: %s", out) } if name != "source-rotation" && fixture.writes != 0 { @@ -241,6 +249,7 @@ func TestMigrationRefusesConflictsAndUnreadableState(t *testing.T) { } } +// TestConcurrentIdenticalCreationIsVerifiedWithoutOverwrite preserves another writer's value. func TestConcurrentIdenticalCreationIsVerifiedWithoutOverwrite(t *testing.T) { fixture := &apiFixture{keys: credentials(), race: "same"} out, err := runMigration(t, fixture, "copy") @@ -249,6 +258,7 @@ func TestConcurrentIdenticalCreationIsVerifiedWithoutOverwrite(t *testing.T) { } } +// TestLostWriteReplyIsRecoveredByReadbackWithoutRotation recovers an uncertain commit. func TestLostWriteReplyIsRecoveredByReadbackWithoutRotation(t *testing.T) { fixture := &apiFixture{keys: credentials(), loseReply: true} if _, err := runMigration(t, fixture, "copy"); err == nil || fixture.writes != 1 { @@ -265,6 +275,7 @@ func TestLostWriteReplyIsRecoveredByReadbackWithoutRotation(t *testing.T) { } } +// TestUnseededInstallationIsExplicitAndNeverCreatesCredentials prevents false migration proof. func TestUnseededInstallationIsExplicitAndNeverCreatesCredentials(t *testing.T) { for _, missingMount := range []bool{false, true} { t.Run(fmt.Sprint(missingMount), func(t *testing.T) { diff --git a/scripts/tests/vault-email-credentials/native_test.go b/scripts/tests/vault-email-credentials/native_test.go index da34e61f61..ae22987e78 100644 --- a/scripts/tests/vault-email-credentials/native_test.go +++ b/scripts/tests/vault-email-credentials/native_test.go @@ -19,6 +19,7 @@ import ( "gopkg.in/yaml.v3" ) +// TestPinnedOpenBaoPreservesValuesAndVersions verifies CAS and replay on the deployed API version. func TestPinnedOpenBaoPreservesValuesAndVersions(t *testing.T) { if os.Getenv("EMAIL_CREDENTIAL_NATIVE_TEST") != "1" { t.Skip("real pinned OpenBao API proof runs in Linux CI") diff --git a/scripts/tests/vault-email-credentials/workflow_test.go b/scripts/tests/vault-email-credentials/workflow_test.go index 6a9c437982..b6d3844bdf 100644 --- a/scripts/tests/vault-email-credentials/workflow_test.go +++ b/scripts/tests/vault-email-credentials/workflow_test.go @@ -17,6 +17,7 @@ type verificationStep struct { With map[string]any } +// verificationSteps validates the dispatch-only workflow and its execution boundary. func verificationSteps(t *testing.T) []verificationStep { t.Helper() data, err := os.ReadFile(filepath.Join(repositoryRoot(), ".github/workflows/verify-email-credential-custody.yaml")) @@ -50,6 +51,7 @@ func verificationSteps(t *testing.T) []verificationStep { return job.Steps } +// TestVerificationGuardRunsBeforeCheckoutOrSecrets executes permitted and refused dispatches. func TestVerificationGuardRunsBeforeCheckoutOrSecrets(t *testing.T) { steps := verificationSteps(t) guard := steps[0] @@ -77,6 +79,7 @@ func TestVerificationGuardRunsBeforeCheckoutOrSecrets(t *testing.T) { } } +// TestVerificationWrapperRefusesIncompleteProofAndNeverCopies rejects unexamined or partial results. func TestVerificationWrapperRefusesIncompleteProofAndNeverCopies(t *testing.T) { step := verificationSteps(t)[3] if len(step.Env) != 3 || step.Env["KUBE_CONFIG"] != "${{ secrets.KUBE_CONFIG }}" || step.Env["HCLOUD_TOKEN"] != "${{ secrets.HCLOUD_TOKEN }}" || step.Env["EMAIL_CUSTODY_MODE"] != "plan" {