diff --git a/.github/workflows/prepare-email-storage-key.yaml b/.github/workflows/prepare-email-storage-key.yaml new file mode 100644 index 0000000000..36205ae3a1 --- /dev/null +++ b/.github/workflows/prepare-email-storage-key.yaml @@ -0,0 +1,132 @@ +name: Prepare Email Storage Key + +"on": + workflow_dispatch: + inputs: + confirm: + description: "Type exactly: prepare-email-storage-key" + required: true + type: string + +permissions: {} +concurrency: + group: prod-deploy + cancel-in-progress: false + queue: max + +jobs: + prepare: + name: Prepare the Email storage key after independent recovery + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: prod + permissions: + contents: read + steps: + - name: Require the confirmed first-attempt main dispatch + shell: bash + env: + CONFIRM: ${{ inputs.confirm }} + run: | + if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_RUN_ATTEMPT" != 1 || "$CONFIRM" != prepare-email-storage-key ]]; then + printf '%s\n' 'EMAIL_STORAGE_KEY=UNVERIFIED' >&2 + exit 2 + fi + + - name: Check out the confirmed immutable revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Set up the repository Go version + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Build the reviewed recovery reader before restoring credentials + shell: bash + run: go build -o "$RUNNER_TEMP/verify-openbao-recovery" ./scripts/openbao-backup-recovery + + - name: Install the repository-approved Kubernetes client + shell: bash + env: + KUBECTL_VERSION: v1.36.2 + KUBECTL_SHA256: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/email-custody-kubectl" + curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path" + printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check + mkdir -p "$RUNNER_TEMP/email-custody-tools" + install -m 0755 "$kubectl_path" "$RUNNER_TEMP/email-custody-tools/kubectl" + printf '%s\n' "$RUNNER_TEMP/email-custody-tools" >> "$GITHUB_PATH" + + - name: Prepare only after independent recovery succeeds + shell: bash + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + EMAIL_STORAGE_CONFIRM: ${{ inputs.confirm }} + run: | + set +x + set -euo pipefail + umask 077 + stage=private-workspace + fail() { printf 'EMAIL_STORAGE_KEY=UNVERIFIED stage=%s\n' "$stage" >&2; exit 2; } + [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail + private_dir="$(mktemp -d /dev/shm/email-storage-key.XXXXXXXX)" + proof_dir="$private_dir/recovery" + mkdir -m 0700 "$proof_dir" + forward_pid='' + cleanup() { + if [[ -n "$forward_pid" ]]; then + kill "$forward_pid" 2>/dev/null || true + wait "$forward_pid" 2>/dev/null || true + fi + rm -rf "$private_dir" + } + trap cleanup EXIT + [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail + export KUBECONFIG="$private_dir/kubeconfig" + printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" + unset KUBE_CONFIG + stage=stable-api + if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then + fail + fi + unset HCLOUD_TOKEN + stage=live-connection + kubectl --context=admin@prod -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200 > "$private_dir/forward.log" 2>&1 & + forward_pid=$! + ready=false + for ((attempt=0; attempt<20; attempt++)); do + kill -0 "$forward_pid" 2>/dev/null || fail + if curl --disable --silent --fail --noproxy '*' --connect-timeout 1 --max-time 2 \ + http://127.0.0.1:38200/v1/sys/health > /dev/null 2>"$private_dir/health.log"; then + ready=true + break + fi + sleep 1 + done + [[ "$ready" == true ]] || fail + binary="$RUNNER_TEMP/verify-openbao-recovery" + export OPENBAO_RECOVERY_DIR="$proof_dir" + stage=collection + if ! "$binary" collect-email > "$private_dir/collection-result" 2>"$private_dir/collection.log"; then fail; fi + [[ $(cat "$private_dir/collection-result") == OPENBAO_BACKUP_INPUTS=PASS ]] || fail + stage=image + image=$("$binary" image) || fail + if ! docker pull "$image" > "$private_dir/pull.log" 2>&1; then fail; fi + stage=isolated-restore + if ! bash scripts/run-openbao-backup-sandbox.sh verify-email "$proof_dir" "$binary" > "$private_dir/verification-result" 2>"$private_dir/verification.log"; then fail; fi + [[ $(cat "$private_dir/verification-result") == 'OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3' ]] || fail + stage='storage-key' + kill -0 "$forward_pid" 2>/dev/null || fail + if ! "$binary" seed-email-storage > "$private_dir/seed-result" 2>"$private_dir/seed.log"; then fail; fi + stage='result-format' + result=$(cat "$private_dir/seed-result") + [[ "$result" =~ ^EMAIL_STORAGE_KEY=PREPARED\ state=(CREATED|EXISTING)\ recovered=false$ ]] || fail + printf '%s\n' "$result" diff --git a/.github/workflows/refresh-email-recovery-backup.yaml b/.github/workflows/refresh-email-recovery-backup.yaml new file mode 100644 index 0000000000..af378b23a4 --- /dev/null +++ b/.github/workflows/refresh-email-recovery-backup.yaml @@ -0,0 +1,88 @@ +name: Refresh Email Recovery Backup + +"on": + workflow_dispatch: + inputs: + confirm: + description: "Type exactly: refresh-email-recovery-backup" + required: true + type: string + +permissions: {} +concurrency: + group: prod-deploy + cancel-in-progress: false + queue: max + +jobs: + refresh: + name: Run the current snapshot and off-cluster mirror once + runs-on: ubuntu-latest + timeout-minutes: 40 + environment: prod + permissions: + contents: read + steps: + - name: Require the confirmed first-attempt main dispatch + shell: bash + env: + CONFIRM: ${{ inputs.confirm }} + run: | + if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$CONFIRM" != refresh-email-recovery-backup ]]; then + printf '%s\n' 'EMAIL_BACKUP_REFRESH=UNVERIFIED stage=dispatch' >&2 + exit 2 + fi + + - name: Check out the confirmed immutable revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Check the reviewed driver syntax before restoring credentials + shell: bash + run: bash -n scripts/refresh-email-recovery-backup.sh + + - name: Install the repository-approved Kubernetes client + shell: bash + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/email-backup-kubectl" + curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \ + https://dl.k8s.io/release/v1.36.2/bin/linux/amd64/kubectl -o "$kubectl_path" + printf '%s %s\n' 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 "$kubectl_path" | sha256sum --check + mkdir -p "$RUNNER_TEMP/email-backup-tools" + install -m 0755 "$kubectl_path" "$RUNNER_TEMP/email-backup-tools/kubectl" + printf '%s\n' "$RUNNER_TEMP/email-backup-tools" >> "$GITHUB_PATH" + + - name: Refresh only the reviewed daily backup template + shell: bash + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + EMAIL_BACKUP_CONFIRM: ${{ inputs.confirm }} + run: | + set +x + set -euo pipefail + umask 077 + ulimit -c 0 + stage=workspace + fail() { printf 'EMAIL_BACKUP_REFRESH=UNVERIFIED stage=%s\n' "$stage" >&2; exit 2; } + [[ $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail + stage=current-source + latest=$(git ls-remote --exit-code https://github.com/devantler-tech/platform.git refs/heads/main) || fail + [[ "$latest" == "$GITHUB_SHA"$'\trefs/heads/main' ]] || fail + private_dir=$(mktemp -d /dev/shm/email-backup-refresh.XXXXXXXX) + trap 'rm -rf "$private_dir"' EXIT + stage=private-inputs + [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail + export KUBECONFIG="$private_dir/kubeconfig" + printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" + unset KUBE_CONFIG + stage=stable-api + if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then + fail + fi + unset HCLOUD_TOKEN + export EMAIL_BACKUP_WORK_DIR="$private_dir" + bash scripts/refresh-email-recovery-backup.sh diff --git a/.github/workflows/verify-email-storage-key-recovery.yaml b/.github/workflows/verify-email-storage-key-recovery.yaml new file mode 100644 index 0000000000..7fc8959e36 --- /dev/null +++ b/.github/workflows/verify-email-storage-key-recovery.yaml @@ -0,0 +1,141 @@ +name: Verify Email Storage Key Recovery + +"on": + workflow_dispatch: + inputs: + confirm: + description: "Type exactly: verify-email-storage-key-recovery" + required: true + type: string + +permissions: {} +concurrency: + group: prod-deploy + cancel-in-progress: false + queue: max + +jobs: + verify: + name: Restore an independent backup without changing production + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: prod + permissions: + contents: read + steps: + - name: Require main and typed confirmation + shell: bash + env: + CONFIRM: ${{ inputs.confirm }} + run: | + if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$GITHUB_RUN_ATTEMPT" != 1 || "$CONFIRM" != verify-email-storage-key-recovery ]]; then + printf '%s\n' 'OPENBAO_BACKUP_RESTORE=UNVERIFIED' >&2 + exit 2 + fi + + - name: Check out the confirmed revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Set up the repository Go version + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Install the repository-approved Kubernetes reader + shell: bash + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/openbao-recovery-kubectl" + curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \ + https://dl.k8s.io/release/v1.36.2/bin/linux/amd64/kubectl -o "$kubectl_path" + printf '%s %s\n' 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 "$kubectl_path" | sha256sum --check + mkdir -p "$RUNNER_TEMP/openbao-recovery-tools" + install -m 0755 "$kubectl_path" "$RUNNER_TEMP/openbao-recovery-tools/kubectl" + printf '%s\n' "$RUNNER_TEMP/openbao-recovery-tools" >> "$GITHUB_PATH" + CGO_ENABLED=0 go build -trimpath -o "$RUNNER_TEMP/openbao-recovery-tools/verifier" ./scripts/openbao-backup-recovery + + - name: Read the fixed backup inputs and prove isolated recovery + shell: bash + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + run: | + set +x + set -euo pipefail + umask 077 + ulimit -c 0 + stage=workspace + fail() { + printf 'OPENBAO_BACKUP_STAGE=%s\n' "$stage" >&2 + printf '%s\n' 'OPENBAO_BACKUP_RESTORE=UNVERIFIED' >&2 + exit 2 + } + [[ $(id -u) != 0 && $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail + stage=current-source + latest=$(git ls-remote --exit-code https://github.com/devantler-tech/platform.git refs/heads/main) || fail + [[ "$latest" == "$GITHUB_SHA"$'\trefs/heads/main' ]] || fail + proof_dir=$(mktemp -d /dev/shm/openbao-recovery.XXXXXXXX) + forward_pid='' + cleanup() { + if [[ -n "$forward_pid" ]]; then + kill "$forward_pid" 2>/dev/null || true + wait "$forward_pid" 2>/dev/null || true + fi + rm -rf "$proof_dir" + } + trap cleanup EXIT + binary="$RUNNER_TEMP/openbao-recovery-tools/verifier" + stage=private-inputs + [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail + export KUBECONFIG="$proof_dir/kubeconfig" + printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" + unset KUBE_CONFIG + stage=stable-api + if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then + fail + fi + unset HCLOUD_TOKEN + stage=live-connection + kubectl --context=admin@prod -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200 > "$proof_dir/forward.log" 2>&1 & + forward_pid=$! + ready=false + for ((attempt=0; attempt<20; attempt++)); do + kill -0 "$forward_pid" 2>/dev/null || fail + if curl --disable --silent --fail --noproxy '*' --connect-timeout 1 --max-time 2 \ + http://127.0.0.1:38200/v1/sys/health > /dev/null 2>"$proof_dir/health.log"; then + ready=true + break + fi + sleep 1 + done + [[ "$ready" == true ]] || fail + export OPENBAO_RECOVERY_DIR="$proof_dir" + stage=collection + if ! "$binary" collect-email-storage > "$proof_dir/collection-result" 2>"$proof_dir/collection.log"; then + # Raw diagnostics stay private; only these closed failure codes leave scratch storage. + grep -E '^OPENBAO_BACKUP_INPUTS=UNVERIFIED stage=(private-workspace|snapshot-job-read|snapshot-job-admission|live-health|deployment-inputs|backup-target|recovery-inputs|recovery-backup-read|recovery-backup-admission|recovery-key-download|live-records|snapshot-download|private-handoff|collection)$' "$proof_dir/collection.log" >&2 || true + fail + fi + [[ $(cat "$proof_dir/collection-result") == OPENBAO_BACKUP_INPUTS=PASS ]] || fail + stage=private-handoff + kill "$forward_pid" 2>/dev/null || fail + wait "$forward_pid" 2>/dev/null || true + forward_pid='' + unset OPENBAO_RECOVERY_DIR KUBECONFIG + rm -- "$proof_dir/kubeconfig" + stage=image + image=$("$binary" image) || fail + if ! docker pull "$image" > "$proof_dir/pull.log" 2>&1; then fail; fi + stage=isolated-restore + if ! bash scripts/run-openbao-backup-sandbox.sh verify-email-storage "$proof_dir" "$binary" > "$proof_dir/verification-result" 2>"$proof_dir/verification.log"; then + grep -E '^OPENBAO_BACKUP_RESTORE=FAIL stage=(mode|sandbox|server|initialization|inputs|snapshot|restore|seal|unseal|records|membership|restored_state|fixture|verification|runtime)$' "$proof_dir/verification.log" >&2 || true + fail + fi + stage=proof-format + result=$(cat "$proof_dir/verification-result") + [[ "$result" == 'OPENBAO_BACKUP_RESTORE=PASS records=4 version=2.6.3' ]] || fail + printf '%s\n' "$result" diff --git a/docs/dr/openbao-recovery-proof.md b/docs/dr/openbao-recovery-proof.md index 87584a9790..e015dc8cfa 100644 --- a/docs/dr/openbao-recovery-proof.md +++ b/docs/dr/openbao-recovery-proof.md @@ -87,3 +87,54 @@ fresh bootstrap bundle only for an explicitly declared new installation, never while an existing installation awaits restoration. A fresh production bootstrap must restore the retained-custody overlay before routine operation. Missing records or regenerated caches alone never establish fresh-install intent. + +## Preparing the Email storage key + +`Prepare Email Storage Key` is a protected first-attempt, main-only operation, +confirmed with `prepare-email-storage-key`. Before creating anything, it repeats +an independent restore of the three canonical Email accounts and compares those complete values +with their fixed infrastructure destinations. The private connection files stay +outside the restore sandbox. A fresh key uses 256 bits of system randomness and +a create-only KV-v2 write. Existing valid keys are preserved. Deleted history, +conflicts, changed account values, uncertain writes and changed key readback +refuse success without retrying or replacing the key. + +Its `EMAIL_STORAGE_KEY=PREPARED` receipt explicitly says `recovered=false`. +Preparation does not select or migrate an Email volume. The service key lives in +OpenBao, while the existing approved independent recovery credentials recover +OpenBao itself. No per-client certificate or new host-only recovery copy is +created. + +After the configured off-cluster backup contains the new record, confirm +`verify-email-storage-key-recovery` in `Verify Email Storage Key Recovery`. +Its fixed collection and restoration modes include all three canonical accounts +and `infrastructure/email/storage`, whose sole field is the exact 64-character +hexadecimal passphrase. The normal three-account and legacy modes remain +separate. Native Linux tests restore all four synthetic records with the pinned +server and refuse an altered storage passphrase, corrupt snapshot, wrong unseal +key or wrong custody scope. + +Only the actual independent result +`OPENBAO_BACKUP_RESTORE=PASS records=4 version=2.6.3` clears this key's recovery +prerequisite. A prepared key, a recent snapshot missing the key, a skipped native +test or an offline fixture is insufficient. Encrypted-volume canary, data +migration, restoration and activation remain separate reviewed operations; keep +the current mailbox volume until those checks and rollback are proven. + +### Refresh the durable snapshot after custody changes + +`Refresh Email Recovery Backup` is a confirmed, first-attempt main-only operation +with confirmation `refresh-email-recovery-backup`. It starts one run of the +current reviewed daily snapshot template, retaining its CronJob controller owner +and its existing service account, recovery volume and off-cluster mirror. It does +not change the schedule, retention, account credentials or storage key. An active +run, unexpected template, changed source, uncertain creation or replacement Job +refuses a completion claim; the operation never deletes uncertain resources. + +Run it after the canonical credential copy and again after storage-key +preparation. A completed snapshot and mirror is still `restore=UNVERIFIED`: +follow it with `Verify OpenBao Backup Recovery` for the three canonical records, +or `Verify Email Storage Key Recovery` for all four records after key preparation. +Only the corresponding independent restore clears recovery. The temporary +backup uses the daily template's existing bounded deadline and normal CronJob +history management. diff --git a/scripts/openbao-backup-recovery/canonical_window_test.go b/scripts/openbao-backup-recovery/canonical_window_test.go index f88fc4996f..5aeeebc9f3 100644 --- a/scripts/openbao-backup-recovery/canonical_window_test.go +++ b/scripts/openbao-backup-recovery/canonical_window_test.go @@ -8,17 +8,19 @@ import ( func TestCanonicalRecoveryUsesRecentIndependentBackupsAfterCustodyChanges(t *testing.T) { previous := credentialPaths t.Cleanup(func() { credentialPaths = previous }) - _, credentialPaths, _ = credentialMode("collect-email") now := time.Now().UTC() - start, end, err := freshRecoveryWindow(nil, now) - if err != nil || !start.Equal(now.Add(-24*time.Hour)) || !end.Equal(now.Add(5*time.Minute)) { - t.Fatal("canonical recovery is still bound to the obsolete one-time copy snapshot") - } - if _, _, err := freshRecoveryWindow([]byte("undeclared-window"), now); err == nil { - t.Fatal("canonical recovery accepted an externally supplied evidence window") - } - if _, _, err := freshRecoveryWindow(nil, time.Time{}); err == nil { - t.Fatal("canonical recovery accepted an absent current clock") + for _, mode := range []string{"collect-email", "collect-email-storage"} { + _, credentialPaths, _ = credentialMode(mode) + start, end, err := freshRecoveryWindow(nil, now) + if err != nil || !start.Equal(now.Add(-24*time.Hour)) || !end.Equal(now.Add(5*time.Minute)) { + t.Fatal("canonical recovery is still bound to the obsolete one-time copy snapshot") + } + if _, _, err := freshRecoveryWindow([]byte("undeclared-window"), now); err == nil { + t.Fatal("canonical recovery accepted an externally supplied evidence window") + } + if _, _, err := freshRecoveryWindow(nil, time.Time{}); err == nil { + t.Fatal("canonical recovery accepted an absent current clock") + } } _, credentialPaths, _ = credentialMode("collect") if _, _, err := freshRecoveryWindow(nil, now); err == nil { diff --git a/scripts/openbao-backup-recovery/credential_mode.go b/scripts/openbao-backup-recovery/credential_mode.go index 61915a1e29..0671b66797 100644 --- a/scripts/openbao-backup-recovery/credential_mode.go +++ b/scripts/openbao-backup-recovery/credential_mode.go @@ -8,17 +8,28 @@ func credentialMode(requested string) (string, []string, bool) { mode := map[string]string{ "image": "image", "collect": "collect", "verify": "verify", "fixture": "fixture", "collect-email": "collect", "verify-email": "verify", "fixture-email": "fixture", + "collect-email-storage": "collect", "verify-email-storage": "verify", "fixture-email-storage": "fixture", + "seed-email-storage": "seed-storage", }[requested] if mode == "" { return "", nil, false } if requested != mode { - return mode, []string{"infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent"}, true + paths := []string{"infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent"} + switch requested { + case "collect-email-storage", "verify-email-storage", "fixture-email-storage": + paths = append(paths, storageCredentialPath) + } + return mode, paths, true } return mode, []string{"apps/private-mail/admin", "apps/private-mail/maintainer", "apps/private-mail/agent"}, true } func canonicalRecoverySet() bool { _, paths, ok := credentialMode("collect-email") + if ok && slices.Equal(credentialPaths, paths) { + return true + } + _, paths, ok = credentialMode("collect-email-storage") return ok && slices.Equal(credentialPaths, paths) } diff --git a/scripts/openbao-backup-recovery/evidence.go b/scripts/openbao-backup-recovery/evidence.go index 8eb59b2e31..8c49e81cd4 100644 --- a/scripts/openbao-backup-recovery/evidence.go +++ b/scripts/openbao-backup-recovery/evidence.go @@ -95,8 +95,8 @@ func recordsEqual(expected, actual map[string]json.RawMessage) bool { return false } for _, path := range credentialPaths { - e, ok1 := recordValue(expected[path]) - a, ok2 := recordValue(actual[path]) + e, ok1 := recordValueAtPath(path, expected[path]) + a, ok2 := recordValueAtPath(path, actual[path]) if !ok1 || !ok2 || !reflect.DeepEqual(e, a) { return false } diff --git a/scripts/openbao-backup-recovery/main.go b/scripts/openbao-backup-recovery/main.go index be4e3de143..1b14cd46b0 100644 --- a/scripts/openbao-backup-recovery/main.go +++ b/scripts/openbao-backup-recovery/main.go @@ -177,7 +177,7 @@ func (a *baoAPI) credentialRecords(ctx context.Context, token string) (map[strin if err != nil || status != 200 || json.Unmarshal(b, &response) != nil || response.Data.Metadata.Version < 1 || response.Data.Metadata.Deleted != "" || response.Data.Metadata.Destroyed { return nil, errEvidence } - if _, ok := recordValue(response.Data.Data); !ok { + if _, ok := recordValueAtPath(path, response.Data.Data); !ok { return nil, errEvidence } records[path] = response.Data.Data @@ -321,7 +321,9 @@ func (a *baoAPI) createFixture(ctx context.Context, root, key, dir string) error // native restoration proves both minimum-schema admission and retention // of every optional field. data := map[string]any{"password": hex.EncodeToString(random)} - if i > 0 { + if path == storageCredentialPath { + data = map[string]any{"passphrase": hex.EncodeToString(random)} + } else if i > 0 { data["username"] = fmt.Sprintf("synthetic-%d", i) data["retainedField"] = "synthetic-fixture" } @@ -440,6 +442,9 @@ func main() { os.Exit(2) } credentialPaths = paths + if mode == "seed-storage" { + os.Exit(runStorageSeed(os.Getenv, os.Stdout)) + } if mode == "collect" { if err := collectRecoveryInputs(os.Getenv("OPENBAO_RECOVERY_DIR")); err != nil { fmt.Fprintf(os.Stderr, "OPENBAO_BACKUP_INPUTS=UNVERIFIED stage=%s\n", inputFailureStage(err)) @@ -465,6 +470,6 @@ func main() { if mode == "fixture" { fmt.Println("OPENBAO_BACKUP_FIXTURE=PASS") } else { - fmt.Printf("OPENBAO_BACKUP_RESTORE=PASS records=3 version=%s\n", serverVersion) + fmt.Printf("OPENBAO_BACKUP_RESTORE=PASS records=%d version=%s\n", len(credentialPaths), serverVersion) } } diff --git a/scripts/openbao-backup-recovery/native_test.go b/scripts/openbao-backup-recovery/native_test.go index 0c1797a45e..d7dd0f7511 100644 --- a/scripts/openbao-backup-recovery/native_test.go +++ b/scripts/openbao-backup-recovery/native_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "fmt" "os" "os/exec" "path/filepath" @@ -21,6 +22,10 @@ func TestPinnedServerRestoresCanonicalEmailCredentials(t *testing.T) { testPinnedRecovery(t, "-email") } +func TestPinnedServerRestoresEmailStorageKeyAndAccountCredentials(t *testing.T) { + testPinnedRecovery(t, "-email-storage") +} + func testPinnedRecovery(t *testing.T, suffix string) { t.Helper() if os.Getenv("OPENBAO_BACKUP_RECOVERY_NATIVE_TEST") != "1" { @@ -126,11 +131,17 @@ func testPinnedRecovery(t *testing.T, suffix string) { if err != nil || json.Unmarshal(b, &records) != nil { t.Fatal("invalid synthetic records") } - path := "apps/private-mail/maintainer" - if suffix == "-email" { - path = "infrastructure/email/maintainer" + if suffix == "-email-storage" { + // Preserve all three account expectations and alter the actual + // storage key, so native restoration reaches key comparison. + records[storageCredentialPath] = json.RawMessage(`{"passphrase":"` + strings.Repeat("d", 64) + `"}`) + } else { + path := "apps/private-mail/maintainer" + if suffix != "" { + path = "infrastructure/email/maintainer" + } + records[path] = json.RawMessage(`{"username":"changed-synthetic-user","password":"` + strings.Repeat("d", 48) + `","retainedField":"synthetic-fixture"}`) } - records[path] = json.RawMessage(`{"username":"changed-synthetic-user","password":"` + strings.Repeat("d", 48) + `","retainedField":"synthetic-fixture"}`) b, err = json.Marshal(records) if err != nil || os.WriteFile(filepath.Join(dir, "expected.json"), b, 0600) != nil { t.Fatal("could not change synthetic expected record") @@ -143,7 +154,7 @@ func testPinnedRecovery(t *testing.T, suffix string) { } wrongScope := map[string]json.RawMessage{} for key, value := range records { - if suffix == "-email" { + if suffix != "" { key = strings.Replace(key, "infrastructure/email/", "apps/private-mail/", 1) } else { key = strings.Replace(key, "apps/private-mail/", "infrastructure/email/", 1) @@ -157,7 +168,11 @@ func testPinnedRecovery(t *testing.T, suffix string) { } out, err := run("verify"+suffix, dir) if test == "matching" { - if err != nil || strings.TrimSpace(out) != "OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3" { + count := 3 + if suffix == "-email-storage" { + count = 4 + } + if err != nil || strings.TrimSpace(out) != fmt.Sprintf("OPENBAO_BACKUP_RESTORE=PASS records=%d version=2.6.3", count) { t.Fatalf("actual snapshot restoration not proven: %v %s", err, out) } } else { diff --git a/scripts/openbao-backup-recovery/storage_entry.go b/scripts/openbao-backup-recovery/storage_entry.go new file mode 100644 index 0000000000..f57fa046d5 --- /dev/null +++ b/scripts/openbao-backup-recovery/storage_entry.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "path/filepath" + "regexp" + "time" +) + +func runStorageSeed(env func(string) string, out io.Writer) int { + fail := func(stage string) int { + _, _ = fmt.Fprintf(out, "EMAIL_STORAGE_KEY=UNVERIFIED stage=%s\n", stage) + return 2 + } + if env("GITHUB_REPOSITORY") != "devantler-tech/platform" || env("GITHUB_EVENT_NAME") != "workflow_dispatch" || env("GITHUB_REF") != "refs/heads/main" || env("GITHUB_RUN_ATTEMPT") != "1" || env("EMAIL_STORAGE_CONFIRM") != "prepare-email-storage-key" || !regexp.MustCompile(`^[a-f0-9]{40}$`).MatchString(env("GITHUB_SHA")) { + return fail("guard") + } + dir := env("OPENBAO_RECOVERY_DIR") + if !recoveryDirectory(dir) { + return fail("inputs") + } + root, err := privateInput(filepath.Join(dir, "root-token"), 16384) + if err != nil { + return fail("inputs") + } + defer clear(root) + raw, err := privateInput(filepath.Join(dir, "expected.json"), 1<<20) + if err != nil { + return fail("inputs") + } + defer clear(raw) + var expected map[string]json.RawMessage + if json.Unmarshal(raw, &expected) != nil || len(expected) != 3 { + return fail("inputs") + } + api, err := newAPI("http://127.0.0.1:38200") + if err != nil { + return fail("preparation") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + state, err := api.prepareStorage(ctx, string(root), expected, rand.Reader) + if err != nil || state != "CREATED" && state != "EXISTING" { + return fail("preparation") + } + _, _ = fmt.Fprintf(out, "EMAIL_STORAGE_KEY=PREPARED state=%s recovered=false\n", state) + return 0 +} diff --git a/scripts/openbao-backup-recovery/storage_entry_test.go b/scripts/openbao-backup-recovery/storage_entry_test.go new file mode 100644 index 0000000000..dc71e6b4d5 --- /dev/null +++ b/scripts/openbao-backup-recovery/storage_entry_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestStorageSeedModeUsesOnlyTheRecoveredCanonicalAccounts(t *testing.T) { + mode, paths, ok := credentialMode("seed-email-storage") + want := []string{"infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent"} + if !ok || mode != "seed-storage" || !slices.Equal(paths, want) { + t.Fatal("seed mode did not bind the recovered account prerequisite") + } +} + +func TestStorageSeedEntrypointRefusesDispatchGuardsBeforePrivateReads(t *testing.T) { + binary := filepath.Join(t.TempDir(), "verifier") + build := exec.Command("go", "build", "-o", binary, ".") + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("cannot build the owned entrypoint: %v %s", err, output) + } + valid := map[string]string{"GITHUB_REPOSITORY": "devantler-tech/platform", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", "GITHUB_SHA": strings.Repeat("a", 40), "GITHUB_RUN_ATTEMPT": "1", "EMAIL_STORAGE_CONFIRM": "prepare-email-storage-key"} + for _, changed := range []string{"GITHUB_REPOSITORY", "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_SHA", "GITHUB_RUN_ATTEMPT", "EMAIL_STORAGE_CONFIRM", "private-inputs"} { + t.Run(changed, func(t *testing.T) { + command := exec.Command(binary, "seed-email-storage") + command.Env = append(os.Environ(), "OPENBAO_RECOVERY_DIR=/nonexistent-synthetic-storage-inputs") + for name, value := range valid { + if name == changed { + value = "unapproved" + } + command.Env = append(command.Env, name+"="+value) + } + output, err := command.CombinedOutput() + stage := "guard" + if changed == "private-inputs" { + stage = "inputs" + } + if err == nil || strings.TrimSpace(string(output)) != "EMAIL_STORAGE_KEY=UNVERIFIED stage="+stage { + t.Fatal("entrypoint did not refuse before accessing private inputs") + } + }) + } +} diff --git a/scripts/openbao-backup-recovery/storage_recovery.go b/scripts/openbao-backup-recovery/storage_recovery.go new file mode 100644 index 0000000000..e9250c9361 --- /dev/null +++ b/scripts/openbao-backup-recovery/storage_recovery.go @@ -0,0 +1,26 @@ +package main + +import ( + "encoding/json" + "regexp" + + "github.com/devantler-tech/platform/internal/strictjson" +) + +const storageCredentialPath = "infrastructure/email/storage" + +func recordValueAtPath(path string, raw json.RawMessage) (map[string]any, bool) { + if path == storageCredentialPath { + var value map[string]string + if strictjson.Validate(raw, map[string][]string{"": {"passphrase"}}) != nil || json.Unmarshal(raw, &value) != nil || len(value) != 1 || !regexp.MustCompile(`^[a-f0-9]{64}$`).MatchString(value["passphrase"]) { + return nil, false + } + return map[string]any{"passphrase": value["passphrase"]}, true + } + switch path { + case "apps/private-mail/admin", "apps/private-mail/maintainer", "apps/private-mail/agent", "infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent": + return recordValue(raw) + default: + return nil, false + } +} diff --git a/scripts/openbao-backup-recovery/storage_recovery_test.go b/scripts/openbao-backup-recovery/storage_recovery_test.go new file mode 100644 index 0000000000..2426aec5f7 --- /dev/null +++ b/scripts/openbao-backup-recovery/storage_recovery_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "encoding/json" + "slices" + "strings" + "testing" +) + +func TestStorageRecoveryRequiresTheThreeAccountsAndTheStorageKey(t *testing.T) { + want := []string{"infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent", "infrastructure/email/storage"} + for _, mode := range []string{"collect", "verify", "fixture"} { + base, paths, ok := credentialMode(mode + "-email-storage") + if !ok || base != mode || !slices.Equal(paths, want) { + t.Fatal("storage recovery did not select exactly the four declared records") + } + } + for _, mode := range []string{"image-email-storage", "verify-email-storage-storage", "collect-storage", "verify-email-storage\n"} { + if _, _, ok := credentialMode(mode); ok { + t.Fatal("storage recovery admitted an undeclared mode") + } + } +} + +func TestStorageRecoveryComparesTheActualPassphraseAndRefusesAccountSubstitution(t *testing.T) { + previous := credentialPaths + t.Cleanup(func() { credentialPaths = previous }) + credentialPaths = []string{"infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent", "infrastructure/email/storage"} + records := map[string]json.RawMessage{} + for _, path := range credentialPaths[:3] { + records[path] = json.RawMessage(`{"password":"` + strings.Repeat("a", 64) + `"}`) + } + valid := json.RawMessage(`{"passphrase":"` + strings.Repeat("b", 64) + `"}`) + records[credentialPaths[3]] = valid + if !recordsEqual(records, records) { + t.Fatal("complete storage recovery records were refused") + } + actual := map[string]json.RawMessage{} + for path, value := range records { + actual[path] = value + } + for index, invalid := range []string{ + `{"passphrase":"` + strings.Repeat("c", 64) + `"}`, + `{"password":"` + strings.Repeat("b", 64) + `"}`, + `{"passphrase":"short"}`, + `{"passphrase":"` + strings.Repeat("b", 64) + `","extra":"ignored"}`, + `{"passphrase":"` + strings.Repeat("b", 64) + `","passphrase":"` + strings.Repeat("b", 64) + `"}`, + `{"Passphrase":"` + strings.Repeat("b", 64) + `"}`, + } { + actual[credentialPaths[3]] = json.RawMessage(invalid) + if recordsEqual(records, actual) { + t.Fatal("storage recovery accepted an altered or ambiguous key") + } + if index > 0 && recordsEqual(actual, actual) { + t.Fatal("matching malformed storage expectations were accepted") + } + } + actual[credentialPaths[3]] = valid + actual[credentialPaths[0]] = valid + if recordsEqual(records, actual) { + t.Fatal("storage key was accepted in place of an account credential") + } +} diff --git a/scripts/openbao-backup-recovery/storage_seed.go b/scripts/openbao-backup-recovery/storage_seed.go new file mode 100644 index 0000000000..5eb63a42fb --- /dev/null +++ b/scripts/openbao-backup-recovery/storage_seed.go @@ -0,0 +1,123 @@ +package main + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "reflect" + "regexp" + "slices" +) + +func (a *baoAPI) storageRecord(ctx context.Context, token string) (json.RawMessage, int64, int, error) { + status, raw, err := a.request(ctx, http.MethodGet, "/v1/secret/data/"+storageCredentialPath, token, nil, 16384) + defer clear(raw) + if err != nil || status != http.StatusOK { + return nil, 0, status, errEvidence + } + var response struct { + Data struct { + Data json.RawMessage `json:"data"` + Metadata struct { + Version int64 `json:"version"` + Deleted string `json:"deletion_time"` + Destroyed bool `json:"destroyed"` + } `json:"metadata"` + } `json:"data"` + } + if json.Unmarshal(raw, &response) != nil || response.Data.Metadata.Version < 1 || response.Data.Metadata.Deleted != "" || response.Data.Metadata.Destroyed { + return nil, 0, status, errEvidence + } + if _, ok := recordValueAtPath(storageCredentialPath, response.Data.Data); !ok { + return nil, 0, status, errEvidence + } + return response.Data.Data, response.Data.Metadata.Version, status, nil +} + +// prepareStorage never rotates a key, overwrites history or retries an uncertain write. +// Its expected account records must already have survived independent recovery. +func (a *baoAPI) prepareStorage(ctx context.Context, token string, expected map[string]json.RawMessage, entropy io.Reader) (string, error) { + canonical := []string{"infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent"} + if !slices.Equal(credentialPaths, canonical) || len(expected) != 3 || entropy == nil || !regexp.MustCompile(`^[A-Za-z0-9._-]{20,256}$`).MatchString(token) { + return "", errEvidence + } + // Bind the three independently restored canonical records to the live + // canonical accounts before creating anything. Legacy proofs cannot replace + // recovery of the paths the operational consumers now read. + verified := map[string]json.RawMessage{} + for _, path := range canonical { + if _, ok := recordValueAtPath(path, expected[path]); !ok { + return "", errEvidence + } + verified[path] = expected[path] + } + health, err := a.health(ctx) + if err != nil || !*health.Initialized || *health.Sealed { + return "", errEvidence + } + status, raw, err := a.request(ctx, http.MethodGet, "/v1/sys/mounts", token, nil, 1<<20) + var mounts struct { + Data map[string]struct { + Type string `json:"type"` + Options map[string]string `json:"options"` + } `json:"data"` + } + if err != nil || status != http.StatusOK || json.Unmarshal(raw, &mounts) != nil || mounts.Data["secret/"].Type != "kv" || mounts.Data["secret/"].Options["version"] != "2" { + return "", errEvidence + } + before, err := a.credentialRecords(ctx, token) + if err != nil || !recordsEqual(verified, before) { + return "", errEvidence + } + value, version, status, err := a.storageRecord(ctx, token) + defer func() { clear(value) }() + state := "EXISTING" + if err != nil { + if status != http.StatusNotFound { + return "", errEvidence + } + // A deleted or destroyed value is not a fresh key. Never recreate history. + status, _, err = a.request(ctx, http.MethodGet, "/v1/secret/metadata/"+storageCredentialPath, token, nil, 16384) + if err != nil || status != http.StatusNotFound { + return "", errEvidence + } + random := make([]byte, 32) + if _, err := io.ReadFull(entropy, random); err != nil { + clear(random) + return "", errEvidence + } + value, err = json.Marshal(map[string]string{"passphrase": hex.EncodeToString(random)}) + clear(random) + if err != nil { + return "", errEvidence + } + body, err := json.Marshal(map[string]any{"data": value, "options": map[string]int{"cas": 0}}) + if err != nil { + return "", errEvidence + } + status, raw, err = a.request(ctx, http.MethodPost, "/v1/secret/data/"+storageCredentialPath, token, bytes.NewReader(body), 16384) + clear(body) + var created struct { + Data struct{ Version int64 } `json:"data"` + } + if err != nil || status != http.StatusOK || json.Unmarshal(raw, &created) != nil || created.Data.Version != 1 { + return "", errEvidence + } + version, state = 1, "CREATED" + } + readback, afterVersion, _, err := a.storageRecord(ctx, token) + defer clear(readback) + left, ok1 := recordValueAtPath(storageCredentialPath, value) + right, ok2 := recordValueAtPath(storageCredentialPath, readback) + if err != nil || !ok1 || !ok2 || afterVersion != version || !reflect.DeepEqual(left, right) { + return "", errEvidence + } + after, err := a.credentialRecords(ctx, token) + if err != nil || !recordsEqual(verified, after) { + return "", errEvidence + } + return state, nil +} diff --git a/scripts/openbao-backup-recovery/storage_seed_test.go b/scripts/openbao-backup-recovery/storage_seed_test.go new file mode 100644 index 0000000000..bc294d2b2a --- /dev/null +++ b/scripts/openbao-backup-recovery/storage_seed_test.go @@ -0,0 +1,166 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +type storagePreparer interface { + prepareStorage(context.Context, string, map[string]json.RawMessage, io.Reader) (string, error) +} + +func TestStorageSeedCreatesOnceAndRefusesUncertaintyOrChangedInputs(t *testing.T) { + previous := credentialPaths + t.Cleanup(func() { credentialPaths = previous }) + credentialPaths = []string{"infrastructure/email/admin", "infrastructure/email/maintainer", "infrastructure/email/agent"} + for _, scenario := range []string{"create", "existing", "conflict", "uncertain", "malformed-existing", "deleted-existing", "entropy", "changed-before", "changed-after", "changed-readback", "missing-mount", "wrong-version", "unbound-scope"} { + t.Run(scenario, func(t *testing.T) { + posts, adminReads := 0, 0 + var stored map[string]string + if scenario == "existing" { + stored = map[string]string{"passphrase": strings.Repeat("b", 64)} + } + if scenario == "malformed-existing" { + stored = map[string]string{"password": strings.Repeat("b", 64)} + } + expected := map[string]json.RawMessage{} + for _, path := range credentialPaths { + expected[path] = json.RawMessage(`{"password":"` + strings.Repeat("a", 64) + `"}`) + } + restored := map[string]json.RawMessage{} + for path, value := range expected { + restored[path] = value + } + if scenario == "unbound-scope" { + restored = map[string]json.RawMessage{} + for path, value := range expected { + restored[strings.Replace(path, "infrastructure/email/", "apps/private-mail/", 1)] = value + } + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/sys/health" && r.Header.Get("X-Vault-Token") != "synthetic-seed-token-only" { + t.Error("seed used the wrong credential") + w.WriteHeader(http.StatusForbidden) + return + } + encode := func(value any) { _ = json.NewEncoder(w).Encode(value) } + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v1/sys/health": + version := serverVersion + if scenario == "wrong-version" { + version = "unexpected" + } + encode(map[string]any{"initialized": true, "sealed": false, "version": version}) + case r.Method == http.MethodGet && r.URL.Path == "/v1/sys/mounts": + version := "2" + if scenario == "missing-mount" { + version = "1" + } + encode(map[string]any{"data": map[string]any{"secret/": map[string]any{"type": "kv", "options": map[string]string{"version": version}}}}) + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/v1/secret/data/") && r.URL.Path != "/v1/secret/data/"+storageCredentialPath: + path := strings.TrimPrefix(r.URL.Path, "/v1/secret/data/") + value, ok := expected[path] + if !ok { + t.Error("seed read an undeclared account") + w.WriteHeader(http.StatusNotFound) + return + } + if path == credentialPaths[0] { + adminReads++ + if scenario == "changed-before" || scenario == "changed-after" && adminReads > 1 { + value = json.RawMessage(`{"password":"` + strings.Repeat("c", 64) + `"}`) + } + } + encode(map[string]any{"data": map[string]any{"data": value, "metadata": map[string]any{"version": 1}}}) + case r.Method == http.MethodGet && r.URL.Path == "/v1/secret/metadata/"+storageCredentialPath: + if scenario == "deleted-existing" { + encode(map[string]any{"data": map[string]any{"current_version": 1}}) + } else { + w.WriteHeader(http.StatusNotFound) + } + case r.URL.Path == "/v1/secret/data/"+storageCredentialPath: + switch r.Method { + case http.MethodPost: + posts++ + var body struct { + Data map[string]string `json:"data"` + Options struct{ CAS *int } `json:"options"` + } + if json.NewDecoder(r.Body).Decode(&body) != nil || body.Options.CAS == nil || *body.Options.CAS != 0 || len(body.Data) != 1 || body.Data["passphrase"] != strings.Repeat("03", 32) { + t.Error("seed did not use the generated key and create-only CAS") + w.WriteHeader(http.StatusBadRequest) + return + } + if scenario == "conflict" { + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, "PRIVATE_SYNTHETIC_REMOTE_ERROR") + return + } + stored = body.Data + if scenario == "uncertain" { + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, "PRIVATE_SYNTHETIC_REMOTE_ERROR") + return + } + encode(map[string]any{"data": map[string]any{"version": 1}}) + case http.MethodGet: + if stored == nil { + w.WriteHeader(http.StatusNotFound) + return + } + version := 1 + if scenario == "changed-readback" { + version = 2 + } + encode(map[string]any{"data": map[string]any{"data": stored, "metadata": map[string]any{"version": version}}}) + default: + t.Error("seed used an undeclared method") + w.WriteHeader(http.StatusMethodNotAllowed) + } + default: + t.Error("seed used an undeclared operation") + w.WriteHeader(http.StatusForbidden) + } + })) + defer server.Close() + api, err := newAPI(server.URL) + if err != nil { + t.Fatal("cannot prepare the loopback fixture") + } + preparer, ok := any(api).(storagePreparer) + if !ok { + t.Fatal("bounded storage seed operation is missing") + } + var entropy io.Reader = bytes.NewReader(bytes.Repeat([]byte{3}, 32)) + if scenario == "entropy" || scenario == "existing" { + entropy = strings.NewReader("") + } + state, err := preparer.prepareStorage(context.Background(), "synthetic-seed-token-only", restored, entropy) + if scenario == "create" || scenario == "existing" { + want := "CREATED" + if scenario == "existing" { + want = "EXISTING" + } + if err != nil || state != want { + t.Fatal("confirmed storage preparation did not preserve the expected result") + } + } else if err == nil || state != "" || strings.Contains(err.Error(), "PRIVATE_SYNTHETIC") { + t.Fatal("unsafe storage preparation was accepted or reflected private data") + } + wantPosts := 0 + switch scenario { + case "create", "conflict", "uncertain", "changed-after", "changed-readback": + wantPosts = 1 + } + if posts != wantPosts { + t.Fatal("seed replaced, retried, or wrote before its prerequisites") + } + }) + } +} diff --git a/scripts/refresh-email-recovery-backup.sh b/scripts/refresh-email-recovery-backup.sh new file mode 100755 index 0000000000..389e01a61c --- /dev/null +++ b/scripts/refresh-email-recovery-backup.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Run the reviewed daily snapshot template once, without changing cadence or credentials. +set -euo pipefail +umask 077 +stage=dispatch +work='' +watchers=() +fail() { printf 'EMAIL_BACKUP_REFRESH=UNVERIFIED stage=%s\n' "$stage" >&2; exit 2; } +cleanup() { + for pid in "${watchers[@]}"; do kill "$pid" 2>/dev/null || true; done + for pid in "${watchers[@]}"; do wait "$pid" 2>/dev/null || true; done + if [[ -n "$work" ]]; then rm -rf "$work"; fi +} +trap cleanup EXIT +[[ ${BASH_VERSINFO[0]} -ge 5 && ${GITHUB_EVENT_NAME:-} == workflow_dispatch && ${GITHUB_REF:-} == refs/heads/main && ${GITHUB_REPOSITORY:-} == devantler-tech/platform && ${GITHUB_RUN_ATTEMPT:-} == 1 && ${EMAIL_BACKUP_CONFIRM:-} == refresh-email-recovery-backup && ${GITHUB_RUN_ID:-} =~ ^[1-9][0-9]{0,19}$ ]] || fail +stage=workspace +[[ -d ${EMAIL_BACKUP_WORK_DIR:-} && ! -L "$EMAIL_BACKUP_WORK_DIR" ]] || fail +work=$(mktemp -d "$EMAIL_BACKUP_WORK_DIR/snapshot-refresh.XXXXXXXX") || fail +kc=(kubectl --context admin@prod --namespace openbao --request-timeout=25s) +job="vault-snapshot-refresh-${GITHUB_RUN_ID}-1" +stage=source +"${kc[@]}" get cronjob vault-snapshot -o json > "$work/source.json" 2> "$work/reader.log" || fail +jq -e ' + .kind=="CronJob" and .metadata.name=="vault-snapshot" and .metadata.namespace=="openbao" and + (.metadata.uid|test("^[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}$")) and + (.metadata.resourceVersion|type=="string" and length>0) and + .metadata.labels["kustomize.toolkit.fluxcd.io/name"]=="infrastructure" and + .metadata.labels["kustomize.toolkit.fluxcd.io/namespace"]=="flux-system" and + .spec.suspend!=true and .spec.concurrencyPolicy=="Forbid" and + ((.status.active//[])|length==0) and + .spec.jobTemplate.spec.activeDeadlineSeconds==1800 and + .spec.jobTemplate.spec.backoffLimit==3 and + (.spec.jobTemplate.spec.template.spec as $p | + $p.serviceAccountName=="vault-snapshot" and $p.restartPolicy=="OnFailure" and + $p.automountServiceAccountToken==false and $p.hostNetwork!=true and + ($p.initContainers|length)==2 and ($p.containers|length)==1 and + $p.initContainers[0].name=="install-mirror-tools" and + $p.initContainers[0].image=="docker.io/library/busybox:1.38.0-musl@sha256:ea2b9914a16a4ac1981994af97b318f7c7d4db76b580c56177f08bf76f4a0be8" and + $p.initContainers[1].name=="snapshot" and + $p.initContainers[1].image=="quay.io/openbao/openbao:2.5.3@sha256:fdc6da21ca6963560c32336fd7feb9cf2d5e52668f1a1647205a4b41171f0806" and + $p.containers[0].name=="mirror" and + $p.containers[0].image=="quay.io/minio/aistor/mc:RELEASE.2026-03-12T04-18-55Z@sha256:6c33dc0fbf65c362be95003cd010ed95a41c556500833ea139f86de40c4c4e9f" and + ([$p.containers[0].env[]|select(.name=="REQUIRE_REMOTE_SNAPSHOT")|.value]==["true"]) and + ([$p.volumes[]|select(.persistentVolumeClaim)|{name,claim:.persistentVolumeClaim.claimName}]==[{name:"snapshots",claim:"vault-snapshots-recovery"}])) +' "$work/source.json" > /dev/null 2> "$work/admission.log" || fail +# Construct exactly the observed template and retain its CronJob controller owner. +# Native history cleanup retains that ownership; a manual create still needs +# its own unfinished-Job check because Forbid governs scheduled creation. +jq --arg name "$job" ' + . as $source | + {apiVersion:"batch/v1",kind:"Job",metadata:{name:$name,namespace:"openbao", + labels:((.spec.jobTemplate.metadata.labels//{})+{"platform.devantler.tech/email-backup-refresh":"true"}), + annotations:((.spec.jobTemplate.metadata.annotations//{})+{"cronjob.kubernetes.io/instantiate":"manual"}), + ownerReferences:[{apiVersion:"batch/v1",kind:"CronJob",name:$source.metadata.name,uid:$source.metadata.uid,controller:true}]}, + spec:.spec.jobTemplate.spec} +' "$work/source.json" > "$work/job.json" 2> "$work/manifest.log" || fail +"${kc[@]}" get cronjob vault-snapshot -o json > "$work/rebound.json" 2> "$work/reader.log" || fail +jq -e --slurpfile before "$work/source.json" '.metadata.uid==$before[0].metadata.uid and .metadata.resourceVersion==$before[0].metadata.resourceVersion and .spec==$before[0].spec and ((.status.active//[])|length==0)' "$work/rebound.json" > /dev/null 2> "$work/admission.log" || fail +stage=overlap +"${kc[@]}" get jobs --chunk-size=0 -o json > "$work/jobs.json" 2> "$work/reader.log" || fail +jq -e --slurpfile source "$work/source.json" ' + (((.apiVersion=="v1" and .kind=="List") or (.apiVersion=="batch/v1" and .kind=="JobList")) and + (.items|type)=="array" and (.metadata.continue//"")=="") and + all(.items[]; + .apiVersion=="batch/v1" and .kind=="Job" and .metadata.namespace=="openbao" and + (.metadata.uid|test("^[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}$")) and + (if any(.metadata.ownerReferences[]?; .apiVersion=="batch/v1" and .kind=="CronJob" and .uid==$source[0].metadata.uid and .controller==true) + then .metadata.deletionTimestamp==null and (.status.active//0)==0 and + ([.status.conditions[]?|select((.type=="Complete" or .type=="Failed") and .status=="True")]|length)==1 + else true end)) +' "$work/jobs.json" > /dev/null 2> "$work/admission.log" || fail +stage=create-unknown +"${kc[@]}" create -f "$work/job.json" -o json > "$work/created.json" 2> "$work/create.log" || fail +jq -e --arg name "$job" --slurpfile source "$work/source.json" ' + .kind=="Job" and .metadata.name==$name and .metadata.namespace=="openbao" and + (.metadata.uid|test("^[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}$")) and + any(.metadata.ownerReferences[]; .kind=="CronJob" and .uid==$source[0].metadata.uid and .controller==true) +' "$work/created.json" > /dev/null 2> "$work/admission.log" || fail +stage=completion +"${kc[@]}" wait --for=condition=Complete "job/$job" --timeout=1850s > "$work/complete.log" 2>&1 & +watchers+=("$!") +"${kc[@]}" wait --for=condition=Failed "job/$job" --timeout=1850s > "$work/failed.log" 2>&1 & +watchers+=("$!") +# A native watch only wakes the readback; it is never itself a success verdict. +wait -n "${watchers[@]}" 2> "$work/wait.log" || true +for pid in "${watchers[@]}"; do kill "$pid" 2>/dev/null || true; done +for pid in "${watchers[@]}"; do wait "$pid" 2>/dev/null || true; done +watchers=() +"${kc[@]}" get job "$job" -o json > "$work/completed.json" 2> "$work/reader.log" || fail +jq -e --slurpfile created "$work/created.json" ' + .metadata.uid==$created[0].metadata.uid and .metadata.name==$created[0].metadata.name and .metadata.namespace=="openbao" and + .metadata.ownerReferences==$created[0].metadata.ownerReferences and .status.succeeded==1 and + any(.status.conditions[]; .type=="Complete" and .status=="True") and + all(.status.conditions[]; .type!="Failed" or .status!="True") +' "$work/completed.json" > /dev/null 2> "$work/admission.log" || fail +stage=pod-proof +uid=$(jq -er '.metadata.uid' "$work/created.json") || fail +"${kc[@]}" get pods -l "batch.kubernetes.io/controller-uid=$uid" -o json > "$work/pods.json" 2> "$work/reader.log" || fail +jq -e --arg uid "$uid" ' + [.items[]|select(any(.metadata.ownerReferences[]?;.kind=="Job" and .uid==$uid and .controller==true))|select(.status.phase=="Succeeded")] + | length==1 and all(.[]; + ([.status.initContainerStatuses[]|select(.state.terminated.exitCode==0)|.name]|sort)==["install-mirror-tools","snapshot"] and + ([.status.containerStatuses[]|select(.state.terminated.exitCode==0)|.name]==["mirror"])) +' "$work/pods.json" > /dev/null 2> "$work/admission.log" || fail +printf '%s\n' 'EMAIL_BACKUP_REFRESH=COMPLETED snapshot=1 mirror=1 restore=UNVERIFIED' diff --git a/scripts/run-openbao-backup-sandbox.sh b/scripts/run-openbao-backup-sandbox.sh index cba3219706..44d94543db 100755 --- a/scripts/run-openbao-backup-sandbox.sh +++ b/scripts/run-openbao-backup-sandbox.sh @@ -8,7 +8,14 @@ fail() { printf '%s\n' 'OPENBAO_BACKUP_RESTORE=FAIL stage=runtime' >&2; exit 2; mode=$1 proof_dir=$2 binary=$3 -[[ $mode == verify || $mode == fixture || $mode == verify-email || $mode == fixture-email ]] || fail +record_count=3 +case "$mode" in + verify|fixture|verify-email|fixture-email) ;; + verify-email-storage|fixture-email-storage) record_count=4 ;; + *) fail ;; +esac +fixture_mode=false +case "$mode" in fixture|fixture-email|fixture-email-storage) fixture_mode=true ;; esac [[ $proof_dir == /* && -d $proof_dir && ! -L $proof_dir && $binary == /* && -f $binary && ! -L $binary && -x $binary ]] || fail [[ $(cd "$proof_dir" && pwd -P) == "$proof_dir" && $(stat -c %a "$proof_dir") == 700 && $(stat -c %u "$proof_dir") == "$(id -u)" ]] || fail image=$("$binary" image) || fail @@ -56,7 +63,7 @@ arguments=(run --rm --name "$name" --label "platform.devantler.tech/recovery-pro --tmpfs "/openbao/logs:ro,nosuid,nodev,noexec,size=1m,mode=0700,uid=$(id -u),gid=$(id -g)" --mount "type=bind,source=$binary,target=/verifier,readonly" --entrypoint /verifier) -if [[ $mode == fixture || $mode == fixture-email ]]; then +if [[ $fixture_mode == true ]]; then arguments+=(--env OPENBAO_RECOVERY_SYNTHETIC=1 --mount "type=bind,source=$proof_dir,target=/export") else arguments+=(--mount "type=bind,source=$proof_dir,target=/proof,readonly") @@ -65,9 +72,9 @@ fi rc=0 docker "${arguments[@]}" "$image" "$mode" >"$proof_dir/sandbox-result" 2>"$proof_dir/sandbox-error.log" || rc=$? result=$(cat "$proof_dir/sandbox-result") -if [[ $rc == 0 && ( $mode == fixture || $mode == fixture-email ) && $result == OPENBAO_BACKUP_FIXTURE=PASS ]]; then +if [[ $rc == 0 && $fixture_mode == true && $result == OPENBAO_BACKUP_FIXTURE=PASS ]]; then validated_result=$result -elif [[ $rc == 0 && ( $mode == verify || $mode == verify-email ) && $result == 'OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3' ]]; then +elif [[ $rc == 0 && $fixture_mode == false && $result == "OPENBAO_BACKUP_RESTORE=PASS records=$record_count version=2.6.3" ]]; then validated_result=$result else # All container and API diagnostics remain private. Only a fixed helper diff --git a/scripts/tests/vault-email-credentials/backup_refresh_test.go b/scripts/tests/vault-email-credentials/backup_refresh_test.go new file mode 100644 index 0000000000..b44106f9cb --- /dev/null +++ b/scripts/tests/vault-email-credentials/backup_refresh_test.go @@ -0,0 +1,255 @@ +package emailcredentials_test + +import ( + "encoding/json" + "gopkg.in/yaml.v3" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestBackupRefreshRequiresTheOwnedCompletedSnapshotAndMirror(t *testing.T) { + for _, scenario := range []string{"complete", "owned-unfinished", "owned-complete", "foreign-unfinished", "jobs-read-failed", "jobs-partial", "active", "image", "pvc", "changed", "create-unknown", "failed", "replacement", "pod-missing"} { + t.Run(scenario, func(t *testing.T) { + work := t.TempDir() + if err := os.Chmod(work, 0700); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(repositoryRoot(), "k8s/bases/infrastructure/vault-backup/cron-job.yaml")) + if err != nil { + t.Fatal(err) + } + var cron map[string]any + if yaml.Unmarshal(raw, &cron) != nil { + t.Fatal("cron fixture") + } + meta := cron["metadata"].(map[string]any) + meta["uid"] = "11111111-1111-4111-8111-111111111111" + meta["resourceVersion"] = "100" + meta["labels"] = map[string]any{"kustomize.toolkit.fluxcd.io/name": "infrastructure", "kustomize.toolkit.fluxcd.io/namespace": "flux-system"} + spec := cron["spec"].(map[string]any) + spec["jobTemplate"].(map[string]any)["metadata"] = map[string]any{"labels": map[string]any{"example.invalid/retain": "yes"}, "annotations": map[string]any{"example.invalid/retain": "yes"}} + template := spec["jobTemplate"].(map[string]any)["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any) + volumes := template["volumes"].([]any) + volumes[0].(map[string]any)["persistentVolumeClaim"].(map[string]any)["claimName"] = "vault-snapshots-recovery" + mirror := template["containers"].([]any)[0].(map[string]any) + mirror["env"].([]any)[1].(map[string]any)["value"] = "true" + if scenario == "active" { + cron["status"] = map[string]any{"active": []any{map[string]any{"name": "other"}}} + } + if scenario == "image" { + mirror["image"] = "unreviewed.invalid/image:latest" + } + if scenario == "pvc" { + volumes[0].(map[string]any)["persistentVolumeClaim"].(map[string]any)["claimName"] = "other" + } + data, err := json.Marshal(cron) + if err != nil { + t.Fatal(err) + } + if os.WriteFile(filepath.Join(work, "cron.json"), data, 0600) != nil { + t.Fatal("fixture") + } + bin := filepath.Join(work, "bin") + if os.Mkdir(bin, 0700) != nil { + t.Fatal("bin") + } + fake := `#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> "$FIXTURE/calls" +shift 5 +case "$1" in + get) + case "$2" in + cronjob) + if [[ "$SCENARIO" == changed && -f "$FIXTURE/read" ]]; then jq '.metadata.resourceVersion="101"' "$FIXTURE/cron.json"; else cat "$FIXTURE/cron.json"; fi + touch "$FIXTURE/read" ;; + jobs) + if [[ "$SCENARIO" == jobs-read-failed ]];then printf SENSITIVE_FIXTURE_VALUE >&2;exit 2;fi + jq -n --arg scenario "$SCENARIO" '{apiVersion:"v1",kind:"List",metadata:{continue:(if $scenario=="jobs-partial" then "opaque" else "" end)},items:(if ($scenario=="owned-unfinished" or $scenario=="owned-complete" or $scenario=="foreign-unfinished") then [{apiVersion:"batch/v1",kind:"Job",metadata:{uid:"66666666-6666-4666-8666-666666666666",namespace:"openbao",ownerReferences:[{apiVersion:"batch/v1",kind:"CronJob",name:"vault-snapshot",uid:(if $scenario=="foreign-unfinished" then "99999999-9999-4999-8999-999999999999" else "11111111-1111-4111-8111-111111111111" end),controller:true}]},status:(if $scenario=="owned-complete" then {succeeded:1,conditions:[{type:"Complete",status:"True"}]} else {} end)}] else [] end)}' ;; + job) + if [[ "$SCENARIO" == replacement ]];then jq '.metadata.uid="33333333-3333-4333-8333-333333333333"' "$FIXTURE/created.json"; else cat "$FIXTURE/created.json";fi ;; + pods) + if [[ "$SCENARIO" == pod-missing ]];then printf '{"items":[]}';else + jq '{items:[{metadata:{ownerReferences:[{kind:"Job",uid:.metadata.uid,controller:true}]},status:{phase:"Succeeded",initContainerStatuses:[{name:"install-mirror-tools",state:{terminated:{exitCode:0}}},{name:"snapshot",state:{terminated:{exitCode:0}}}],containerStatuses:[{name:"mirror",state:{terminated:{exitCode:0}}}]}}]}' "$FIXTURE/created.json";fi ;; + *) exit 2 ;; + esac ;; + create) + cat "$3" > "$FIXTURE/planned.json" + if [[ "$SCENARIO" == create-unknown ]];then printf 'SENSITIVE_FIXTURE_VALUE' >&2;exit 2;fi + jq '.metadata.uid="22222222-2222-4222-8222-222222222222"|.status={succeeded:1,conditions:[{type:"Complete",status:"True"}]}' "$FIXTURE/planned.json" > "$FIXTURE/created.json" + if [[ "$SCENARIO" == failed ]];then jq '.status={failed:1,conditions:[{type:"Failed",status:"True"}]}' "$FIXTURE/created.json" > "$FIXTURE/failed.json";mv "$FIXTURE/failed.json" "$FIXTURE/created.json";fi + cat "$FIXTURE/created.json" ;; + wait) exit 0 ;; + *) printf 'SENSITIVE_FIXTURE_VALUE' >&2;exit 2 ;; +esac +` + if os.WriteFile(filepath.Join(bin, "kubectl"), []byte(fake), 0700) != nil { + t.Fatal("kubectl fixture") + } + cmd := exec.Command("bash", filepath.Join(repositoryRoot(), "scripts/refresh-email-recovery-backup.sh")) + cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH"), "FIXTURE="+work, "SCENARIO="+scenario, "EMAIL_BACKUP_WORK_DIR="+work, "GITHUB_EVENT_NAME=workflow_dispatch", "GITHUB_REF=refs/heads/main", "GITHUB_REPOSITORY=devantler-tech/platform", "GITHUB_RUN_ID=12345", "GITHUB_RUN_ATTEMPT=1", "EMAIL_BACKUP_CONFIRM=refresh-email-recovery-backup") + output, err := cmd.CombinedOutput() + success := scenario == "complete" || scenario == "owned-complete" || scenario == "foreign-unfinished" + if (err == nil) != success { + t.Fatalf("scenario result: error=%v output=%s", err, output) + } + if strings.Contains(string(output), "SENSITIVE_FIXTURE_VALUE") || strings.Contains(string(output), "cloudflarestorage") || strings.Contains(string(output), "11111111") { + t.Fatal("private input escaped") + } + if success && !strings.Contains(string(output), "EMAIL_BACKUP_REFRESH=COMPLETED snapshot=1 mirror=1 restore=UNVERIFIED") { + t.Fatal("missing bounded completion receipt") + } + calls, _ := os.ReadFile(filepath.Join(work, "calls")) + if strings.Contains(string(calls), " delete ") { + t.Fatal("unknown resource was deleted") + } + early := scenario == "owned-unfinished" || scenario == "jobs-read-failed" || scenario == "jobs-partial" || scenario == "active" || scenario == "image" || scenario == "pvc" || scenario == "changed" + if early && strings.Contains(string(calls), " create ") { + t.Fatal("unsafe source created a job") + } + if success { + planned, err := os.ReadFile(filepath.Join(work, "planned.json")) + if err != nil { + t.Fatal(err) + } + var job map[string]any + if json.Unmarshal(planned, &job) != nil { + t.Fatal("job") + } + jm := job["metadata"].(map[string]any) + if jm["name"] != "vault-snapshot-refresh-12345-1" || jm["namespace"] != "openbao" { + t.Fatal("job identity") + } + if jm["labels"].(map[string]any)["example.invalid/retain"] != "yes" || jm["annotations"].(map[string]any)["example.invalid/retain"] != "yes" { + t.Fatal("reviewed template metadata lost") + } + owner := jm["ownerReferences"].([]any)[0].(map[string]any) + if owner["uid"] != meta["uid"] || owner["controller"] != true { + t.Fatal("CronJob concurrency ownership lost") + } + } + }) + } +} + +func TestBackupRefreshWorkflowRequiresTheConfirmedNativeMainDispatch(t *testing.T) { + data, err := os.ReadFile(filepath.Join(repositoryRoot(), ".github/workflows/refresh-email-recovery-backup.yaml")) + if err != nil { + t.Fatal(err) + } + var w struct { + On map[string]any + Permissions map[string]string + Concurrency struct { + Group string + Cancel bool `yaml:"cancel-in-progress"` + Queue string + } + Jobs map[string]struct { + Environment string + Permissions map[string]string + Steps []verificationStep + } + } + if yaml.Unmarshal(data, &w) != nil || len(w.On) != 1 || w.On["workflow_dispatch"] == nil || len(w.Permissions) != 0 || w.Concurrency.Group != "prod-deploy" || w.Concurrency.Cancel || w.Concurrency.Queue != "max" || len(w.Jobs) != 1 { + t.Fatal("unprotected refresh workflow") + } + job, ok := w.Jobs["refresh"] + if !ok || job.Environment != "prod" || len(job.Permissions) != 1 || job.Permissions["contents"] != "read" || len(job.Steps) < 2 { + t.Fatal("refresh authorization") + } + guard := job.Steps[0] + if guard.Uses != "" || len(guard.Env) != 1 || guard.Env["CONFIRM"] != "${{ inputs.confirm }}" { + t.Fatal("guard must run before checkout and credentials") + } + for _, bad := range []string{"", "event", "ref", "repo", "attempt", "runner", "confirmation"} { + t.Run(bad, func(t *testing.T) { + e, ref, repo, attempt, runner, confirm := "workflow_dispatch", "refs/heads/main", "devantler-tech/platform", "1", "github-hosted", "refresh-email-recovery-backup" + switch bad { + case "event": + e = "pull_request" + case "ref": + ref = "refs/heads/other" + case "repo": + repo = "other/repo" + case "attempt": + attempt = "2" + case "runner": + runner = "self-hosted" + case "confirmation": + confirm = "" + } + cmd := exec.Command("bash", "-c", guard.Run) + cmd.Env = append(os.Environ(), "GITHUB_EVENT_NAME="+e, "GITHUB_REF="+ref, "GITHUB_REPOSITORY="+repo, "GITHUB_RUN_ATTEMPT="+attempt, "RUNNER_ENVIRONMENT="+runner, "CONFIRM="+confirm) + output, err := cmd.CombinedOutput() + if (err == nil) != (bad == "") { + t.Fatalf("guard result %v: %s", err, output) + } + }) + } +} + +// The normalizer's mask command must reach Actions even when endpoint selection fails. +func TestStorageRecoveryCallersPreserveEndpointMaskCommands(t *testing.T) { + for _, name := range []string{"prepare-email-storage-key.yaml", "verify-email-storage-key-recovery.yaml", "refresh-email-recovery-backup.yaml"} { + t.Run(name, func(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(repositoryRoot(), ".github/workflows", name)) + if err != nil { + t.Fatal(err) + } + var w struct { + Jobs map[string]struct{ Steps []verificationStep } + } + if yaml.Unmarshal(raw, &w) != nil { + t.Fatal("workflow") + } + run := "" + for _, job := range w.Jobs { + for _, step := range job.Steps { + if strings.Contains(step.Run, "./scripts/use-prod-stable-api-endpoint.sh") { + if run != "" { + t.Fatal("multiple privileged callers") + } + run = step.Run + } + } + } + if run == "" { + t.Fatal("missing endpoint normalizer") + } + work := t.TempDir() + bin := filepath.Join(work, "bin") + scripts := filepath.Join(work, "scripts") + ram := filepath.Join(work, "ram") + for _, dir := range []string{bin, scripts, ram} { + if os.Mkdir(dir, 0700) != nil { + t.Fatal("workspace") + } + } + stubs := map[string]string{ + filepath.Join(bin, "findmnt"): "#!/bin/sh\nprintf 'tmpfs\\n'\n", + filepath.Join(bin, "git"): "#!/bin/sh\nprintf '%s\\trefs/heads/main\\n' \"$GITHUB_SHA\"\n", + filepath.Join(scripts, "use-prod-stable-api-endpoint.sh"): "#!/bin/sh\nprintf '%s\\n' '::add-mask::fixture-private-endpoint' >&2\nexit 2\n", + } + for path, body := range stubs { + if os.WriteFile(path, []byte(body), 0700) != nil { + t.Fatal("stub") + } + } + run = strings.ReplaceAll(run, "/dev/shm", ram) + cmd := exec.Command("bash", "-c", run) + cmd.Dir = work + cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH"), "GITHUB_SHA="+strings.Repeat("a", 40), "GITHUB_EVENT_NAME=workflow_dispatch", "GITHUB_REF=refs/heads/main", "GITHUB_REPOSITORY=devantler-tech/platform", "GITHUB_RUN_ID=12345", "GITHUB_RUN_ATTEMPT=1", "RUNNER_ENVIRONMENT=github-hosted", "RUNNER_TEMP="+work, "KUBE_CONFIG=synthetic-kubeconfig", "HCLOUD_TOKEN=synthetic-token") + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatal("failed endpoint selection was accepted") + } + if !strings.Contains(string(output), "::add-mask::fixture-private-endpoint") { + t.Fatalf("endpoint mask swallowed by caller: %s", output) + } + }) + } +} diff --git a/scripts/tests/vault-email-credentials/storage_workflow_test.go b/scripts/tests/vault-email-credentials/storage_workflow_test.go new file mode 100644 index 0000000000..df86cc69da --- /dev/null +++ b/scripts/tests/vault-email-credentials/storage_workflow_test.go @@ -0,0 +1,151 @@ +package emailcredentials_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func storageKeySteps(t *testing.T, file, jobName string) []verificationStep { + t.Helper() + raw, err := os.ReadFile(filepath.Join(repositoryRoot(), ".github/workflows", file)) + if err != nil { + t.Fatal("guarded storage-key workflow is missing") + } + var workflow struct { + On map[string]any + Permissions map[string]string + Jobs map[string]struct { + Environment string + Permissions map[string]string + Steps []verificationStep + } + } + if yaml.Unmarshal(raw, &workflow) != nil || len(workflow.On) != 1 || workflow.On["workflow_dispatch"] == nil || len(workflow.Permissions) != 0 || len(workflow.Jobs) != 1 { + t.Fatal("storage key operation admits an ambient trigger or permission") + } + job, ok := workflow.Jobs[jobName] + if !ok || job.Environment != "prod" || len(job.Permissions) != 1 || job.Permissions["contents"] != "read" || len(job.Steps) < 3 { + t.Fatal("storage key operation lost its protected boundary") + } + return job.Steps +} + +func TestStoragePreparationRequiresIndependentRestoreBeforeSeeding(t *testing.T) { + steps := storageKeySteps(t, "prepare-email-storage-key.yaml", "prepare") + var step verificationStep + for _, candidate := range steps { + if candidate.Env["KUBE_CONFIG"] != "" { + step = candidate + } + } + if step.Run == "" || len(step.Env) != 3 || step.Env["KUBE_CONFIG"] != "${{ secrets.KUBE_CONFIG }}" || step.Env["HCLOUD_TOKEN"] != "${{ secrets.HCLOUD_TOKEN }}" || step.Env["EMAIL_STORAGE_CONFIRM"] != "${{ inputs.confirm }}" { + t.Fatal("storage preparation admits an undeclared credential or confirmation") + } + for _, scenario := range []string{"create", "existing", "collection-failed", "restore-failed", "restore-format", "seed-failed", "seed-format"} { + t.Run(scenario, func(t *testing.T) { + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal("cannot resolve the fixture workspace") + } + for _, name := range []string{"bin", "scripts"} { + if os.Mkdir(filepath.Join(dir, name), 0700) != nil { + t.Fatal("cannot prepare the fixture workspace") + } + } + files := map[string]string{ + "bin/findmnt": "#!/bin/sh\nprintf '%s\\n' tmpfs\n", + "bin/mktemp": "#!/bin/bash\n[[ $* == '-d /dev/shm/email-storage-key.XXXXXXXX' ]] || exit 91\nexec /usr/bin/mktemp -d \"$FIXTURE_DIR/private.XXXXXXXX\"\n", + "bin/kubectl": "#!/bin/bash\n[[ $* == '--context=admin@prod -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200' ]] || exit 91\nexec /bin/sleep 30\n", + "bin/curl": "#!/bin/sh\nexit 0\n", + "bin/docker": "#!/bin/bash\n[[ $* == 'pull synthetic-pinned-server' ]] || exit 91\n", + "scripts/use-prod-stable-api-endpoint.sh": "#!/bin/sh\nexit 0\n", + "verify-openbao-recovery": `#!/bin/bash +set -eu +case "$*" in + collect-email) + [[ -s $KUBECONFIG && $KUBECONFIG != "$OPENBAO_RECOVERY_DIR/"* ]] || exit 91 + printf '%s' "$OPENBAO_RECOVERY_DIR" > "$FIXTURE_DIR/scratch" + [[ $FIXTURE_CASE != collection-failed ]] || exit 7 + printf '%s' synthetic-recovered-root > "$OPENBAO_RECOVERY_DIR/root-token" + printf '%s' synthetic-recovered-records > "$OPENBAO_RECOVERY_DIR/expected.json" + printf '%s\n' OPENBAO_BACKUP_INPUTS=PASS;; + image) printf '%s\n' synthetic-pinned-server;; + seed-email-storage) + [[ -f $FIXTURE_DIR/restored && $EMAIL_STORAGE_CONFIRM == prepare-email-storage-key && -s $OPENBAO_RECOVERY_DIR/root-token && -s $OPENBAO_RECOVERY_DIR/expected.json ]] || exit 91 + touch "$FIXTURE_DIR/seeded" + [[ $FIXTURE_CASE != seed-failed ]] || exit 7 + if [[ $FIXTURE_CASE == seed-format ]]; then printf '%s\n' PRIVATE_SYNTHETIC_PAYLOAD + elif [[ $FIXTURE_CASE == existing ]]; then printf '%s\n' 'EMAIL_STORAGE_KEY=PREPARED state=EXISTING recovered=false' + else printf '%s\n' 'EMAIL_STORAGE_KEY=PREPARED state=CREATED recovered=false'; fi;; + *) exit 91;; +esac +`, + "scripts/run-openbao-backup-sandbox.sh": `#!/bin/bash +set -eu +[[ $1 == verify-email && $2 == "$OPENBAO_RECOVERY_DIR" && $3 == "$RUNNER_TEMP/verify-openbao-recovery" && ! -e $2/kubeconfig ]] || exit 91 +[[ $FIXTURE_CASE != restore-failed ]] || exit 7 +if [[ $FIXTURE_CASE == restore-format ]]; then printf '%s\n' PRIVATE_SYNTHETIC_PAYLOAD; else + touch "$FIXTURE_DIR/restored" + printf '%s\n' 'OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3' +fi +`, + } + for name, body := range files { + if os.WriteFile(filepath.Join(dir, name), []byte(body), 0700) != nil { + t.Fatal("cannot prepare a fixture command") + } + } + command := exec.Command("bash", "-eu", "-c", step.Run) + command.Dir = dir + command.Env = []string{"PATH=" + filepath.Join(dir, "bin") + ":" + os.Getenv("PATH"), "RUNNER_TEMP=" + dir, "KUBE_CONFIG=synthetic-kubeconfig", "HCLOUD_TOKEN=synthetic-provider", "EMAIL_STORAGE_CONFIRM=prepare-email-storage-key", "FIXTURE_DIR=" + dir, "FIXTURE_CASE=" + scenario} + output, err := command.CombinedOutput() + allowed := scenario == "create" || scenario == "existing" + if (err == nil) != allowed || strings.Contains(string(output), "PRIVATE_SYNTHETIC") { + t.Fatal("storage workflow accepted incomplete recovery or exposed private output") + } + _, seeded := os.Stat(filepath.Join(dir, "seeded")) + wantSeed := allowed || scenario == "seed-failed" || scenario == "seed-format" + if (seeded == nil) != wantSeed { + t.Fatal("storage seed bypassed or never followed independent restoration") + } + if scratch, err := os.ReadFile(filepath.Join(dir, "scratch")); err == nil { + if _, err := os.Stat(filepath.Dir(string(scratch))); !os.IsNotExist(err) { + t.Fatal("storage workflow retained private connection or recovery inputs") + } + } + }) + } +} + +func TestStorageKeyWorkflowsRefuseUnconfirmedExecutionBeforeCheckout(t *testing.T) { + for _, workflow := range []struct{ file, job, confirmation string }{ + {"prepare-email-storage-key.yaml", "prepare", "prepare-email-storage-key"}, + {"verify-email-storage-key-recovery.yaml", "verify", "verify-email-storage-key-recovery"}, + } { + t.Run(workflow.job, func(t *testing.T) { + guard := storageKeySteps(t, workflow.file, workflow.job)[0] + if guard.Uses != "" || guard.Env["CONFIRM"] != "${{ inputs.confirm }}" || len(guard.Env) != 1 { + t.Fatal("storage key confirmation must precede checkout and secret access") + } + valid := map[string]string{"GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", "GITHUB_REPOSITORY": "devantler-tech/platform", "GITHUB_RUN_ATTEMPT": "1", "CONFIRM": workflow.confirmation} + for _, changed := range []string{"confirmed", "GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_REPOSITORY", "GITHUB_RUN_ATTEMPT", "CONFIRM"} { + command := exec.Command("bash", "-eu", "-c", guard.Run) + command.Env = []string{"PATH=" + os.Getenv("PATH")} + for name, value := range valid { + if name == changed { + value = "unapproved" + } + command.Env = append(command.Env, name+"="+value) + } + if err := command.Run(); (err == nil) != (changed == "confirmed") { + t.Fatal("storage key workflow admitted an unconfirmed execution") + } + } + }) + } +}