feat(email): prepare Email credential custody and recovery

.github/workflows/ci.yaml

@@ -144,6 +144,9 @@ jobs:
             k8s:
               - 'k8s/**'
               - 'scripts/tests/private-mail-preparation/**'
+              - 'scripts/tests/vault-email-credentials/**'
+              - 'scripts/prepare-email-credentials.sh'
+              - '.github/workflows/verify-email-credential-custody.yaml'
               - 'scripts/tests/test-private-mail-server.sh'
               - 'scripts/tests/test-private-mail-configuration.sh'
               - 'scripts/private-mail-config/**'
@@ -3308,6 +3311,12 @@ jobs:
       - name: 🔒 Validate Crossplane provider egress policy
         if: needs.changes.outputs.k8s == 'true'
         run: bash scripts/tests/test-crossplane-egress-policy.sh
+      - name: ✉️ Verify Email credential custody against pinned OpenBao
+        if: needs.changes.outputs.k8s == 'true'
+        timeout-minutes: 5
+        env:
+          EMAIL_CREDENTIAL_NATIVE_TEST: '1'
+        run: go test -count=1 -v ./scripts/tests/vault-email-credentials
       - name: ✉️ Prove native mail accounts and recipient restrictions
         if: needs.changes.outputs.k8s == 'true'
         timeout-minutes: 10

.github/workflows/publish-private-mail-bootstrap.yaml

@@ -1,4 +1,4 @@
-name: Publish Private Mail Bootstrap
+name: Publish Email Bootstrap
 
 on:
   pull_request:

.github/workflows/verify-email-credential-custody.yaml

@@ -0,0 +1,120 @@
+name: Verify Email Credential Custody
+
+"on":
+  workflow_dispatch:
+    inputs:
+      confirm:
+        description: "Type exactly: verify-email-credential-custody"
+        required: true
+        type: string
+
+permissions: {}
+concurrency:
+  group: prod-deploy
+  cancel-in-progress: false
+  queue: max
+
+jobs:
+  verify:
+    name: Verify Email credential custody without copying
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    environment: prod
+    permissions:
+      contents: read
+    steps:
+      - name: Require main and typed confirmation
+        shell: bash
+        env:
+          CONFIRM: ${{ inputs.confirm }}
+        run: |
+          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$CONFIRM" != verify-email-credential-custody ]]; then
+            printf '%s\n' 'EMAIL_CUSTODY_VERIFICATION=UNVERIFIED' >&2
+            exit 2
+          fi
+
+      - name: Check out the confirmed revision
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+
+      - name: Install the repository-approved Kubernetes reader
+        shell: bash
+        env:
+          KUBECTL_VERSION: v1.36.2
+          KUBECTL_SHA256: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82
+        run: |
+          set -euo pipefail
+          kubectl_path="$RUNNER_TEMP/email-custody-kubectl"
+          curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \
+            "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path"
+          printf '%s  %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check
+          mkdir -p "$RUNNER_TEMP/email-custody-tools"
+          install -m 0755 "$kubectl_path" "$RUNNER_TEMP/email-custody-tools/kubectl"
+          printf '%s\n' "$RUNNER_TEMP/email-custody-tools" >> "$GITHUB_PATH"
+
+      - name: Compare all three credential records without copying
+        shell: bash
+        env:
+          KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
+          EMAIL_CUSTODY_MODE: plan
+        run: |
+          set +x
+          set -euo pipefail
+          umask 077
+          fail() { printf '%s\n' 'EMAIL_CUSTODY_VERIFICATION=UNVERIFIED' >&2; exit 2; }
+          [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail
+          proof_dir="$(mktemp -d /dev/shm/email-custody.XXXXXXXX)"
+          forward_pid=''
+          cleanup() {
+            if [[ -n "$forward_pid" ]]; then
+              kill "$forward_pid" 2>/dev/null || true
+              wait "$forward_pid" 2>/dev/null || true
+            fi
+            rm -rf "$proof_dir"
+          }
+          trap cleanup EXIT
+          [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail
+          export KUBECONFIG="$proof_dir/kubeconfig"
+          printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG"
+          unset KUBE_CONFIG
+          if ! (
+            if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then
+              exit 1
+            fi
+          ) 2>"$proof_dir/endpoint.log"; then
+            fail
+          fi
+          unset HCLOUD_TOKEN
+          if ! kubectl -n openbao get secret openbao-unseal -o jsonpath='{.data.root-token}' 2>"$proof_dir/reader.log" |
+              base64 --decode > "$proof_dir/token" 2>"$proof_dir/decode.log"; then
+            fail
+          fi
+          [[ -s "$proof_dir/token" ]] || fail
+          kubectl -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200 > "$proof_dir/forward.log" 2>&1 &
+          forward_pid=$!
+          ready=false
+          for ((attempt=0; attempt<20; attempt++)); do
+            kill -0 "$forward_pid" 2>/dev/null || fail
+            if curl --disable --silent --fail --noproxy '*' --connect-timeout 1 --max-time 2 \
+                http://127.0.0.1:38200/v1/sys/health > /dev/null 2>"$proof_dir/health.log"; then
+              ready=true
+              break
+            fi
+            sleep 1
+          done
+          [[ "$ready" == true ]] || fail
+          export BAO_ADDR=http://127.0.0.1:38200
+          export EMAIL_CUSTODY_MODE=plan
+          export EMAIL_CUSTODY_TOKEN_FILE="$proof_dir/token"
+          export EMAIL_CUSTODY_WORK_DIR="$proof_dir"
+          bash scripts/prepare-email-credentials.sh > "$proof_dir/result" 2>"$proof_dir/helper.log" || fail
+          result="$(cat "$proof_dir/result")"
+          if [[ "$result" =~ ^EMAIL_CUSTODY_PREPARATION=PLANNED\ mode=plan\ existing=([0-3])\ copied=0\ pending=([0-3])\ unseeded=0$ ]]; then
+            [[ $((BASH_REMATCH[1] + BASH_REMATCH[2])) == 3 ]] || fail
+          else
+            fail
+          fi
+          printf '%s\n' "$result"

.github/workflows/verify-private-mail-storage-recovery.yaml

@@ -1,4 +1,4 @@
-name: Verify Private Mail Storage Recovery
+name: Verify Email Storage Recovery
 
 "on":
   workflow_dispatch: