diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4b4ab77aa6..f5ab51637c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1004,16 +1004,6 @@ jobs: shellcheck scripts/verify-wedding-shared-backup-coverage.sh scripts/verify-wedding-shared-backup-coverage-pod.sh scripts/remove-wedding-shared-backup-pod.sh scripts/verify-shared-backup-consumers.sh scripts/tests/test-verify-wedding-shared-backup-coverage.sh bash scripts/tests/test-verify-wedding-shared-backup-coverage.sh - # The Crossview, Headlamp and Backstage leftovers, including retained - # database volumes, are deleted on this script's word, and deleted data - # cannot be restored. These cases keep an undeployed removal, a foreign - # object, attached or unrelated storage, a failed read or a survivor from - # passing. - - name: ๐Ÿงน Validate the unused-app retirement - run: | - shellcheck scripts/retire-unused-apps.sh scripts/tests/test-retire-unused-apps.sh - bash scripts/tests/test-retire-unused-apps.sh - - name: Validate generated autoscaler cleanup run: | shellcheck scripts/cleanup-orphan-generated-vpas.sh scripts/tests/test-cleanup-orphan-generated-vpas.sh diff --git a/.github/workflows/retire-unused-apps.yaml b/.github/workflows/retire-unused-apps.yaml deleted file mode 100644 index cd9b6beeb6..0000000000 --- a/.github/workflows/retire-unused-apps.yaml +++ /dev/null @@ -1,67 +0,0 @@ -name: Retire Unused App Leftovers - -# One approved retirement of Crossview, Headlamp (#2496) and the retained -# Backstage database (#4266), dormant until an explicit main dispatch. Flux -# keeps prune-protected Namespaces, HelmReleases, Clusters and claims after -# their manifests leave Git, and Retain volumes keep their data; this deletes -# exactly those leftovers once the removal is deployed. Source CI and the merge -# queue never execute this workflow. -on: - workflow_dispatch: - inputs: - confirm: - description: "Type exactly: retire-crossview-headlamp-backstage" - required: true - type: string - -permissions: {} -concurrency: - group: prod-deploy - cancel-in-progress: false - queue: max - -jobs: - retirement: - name: ๐Ÿงน Delete the retired apps' protected leftovers - runs-on: ubuntu-latest - timeout-minutes: 90 - environment: prod - permissions: - contents: read - env: - RETIREMENT_CONFIRM: ${{ inputs.confirm }} - steps: - - name: Require the approved first-attempt main dispatch - shell: bash - run: | - set -euo pipefail - [[ "${GITHUB_REPOSITORY}" == devantler-tech/platform && "${GITHUB_EVENT_NAME}" == workflow_dispatch && - "${GITHUB_REF}" == refs/heads/main && "${GITHUB_RUN_ATTEMPT}" == 1 && - "${RETIREMENT_CONFIRM}" == retire-crossview-headlamp-backstage ]] - - - name: ๐Ÿ“‘ Checkout the reviewed dispatch revision - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - ref: ${{ github.sha }} - - - name: ๐Ÿ”‘ Restore production kubeconfig - shell: bash - env: - KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} - run: | - set -euo pipefail - test -n "${KUBE_CONFIG}" - umask 077 - mkdir -p ~/.kube - printf '%s' "${KUBE_CONFIG}" >~/.kube/config - chmod 600 ~/.kube/config - - - name: ๐ŸŽฏ Select stable production API endpoint - shell: bash - env: - HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} - run: ./scripts/use-prod-stable-api-endpoint.sh - - - name: Prove the removal deployed, delete the leftovers and verify none remain - run: bash scripts/retire-unused-apps.sh diff --git a/AGENTS.md b/AGENTS.md index 1e3629157f..477b0eeb40 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -650,11 +650,11 @@ never owns the claims a controller creates, such as CloudNativePG instance claims, so removing a manifest leaves them in place, and a `Retain` volume and its storage-backend volume outlive the claim. Retiring that data is an explicit, reviewed deletion of the owning resource, its claims, their volumes and the -backend volumes. The one-off `Retire Unused App Leftovers` workflow does this -for the retired Backstage database on the maintainer's decision of 26 October -2026 (#4266), together with the Crossview and Headlamp leftovers (#2496). No -required opt-out is removed from a managed manifest or an existing protected -object. +backend volumes. A one-off `Retire Unused App Leftovers` workflow did this for +the retired Backstage database on the maintainer's decision of 26 October 2026 +(#4266), together with the Crossview and Headlamp leftovers (#2496), and was +removed once its run proved nothing remained (#5391). No required opt-out is +removed from a managed manifest or an existing protected object. This is the rule in force today, and it is being replaced. [`docs/adr/storage-retention.md`](docs/adr/storage-retention.md) records the decision diff --git a/docs/developer-platform.md b/docs/developer-platform.md index d529acdbbe..45aa3d1a8d 100644 --- a/docs/developer-platform.md +++ b/docs/developer-platform.md @@ -35,7 +35,8 @@ how it worked. The portal's retained database was no longer needed. On 26 October 2026 the maintainer decided to delete it rather than keep a recovery copy, which closes the open retention question -([#5122](https://github.com/devantler-tech/platform/issues/5122)). The one-off -**Retire Unused App Leftovers** workflow deletes the database, its claims, their +([#5122](https://github.com/devantler-tech/platform/issues/5122)). A one-off +**Retire Unused App Leftovers** workflow deleted the database, its claims, their retained volumes and the `backstage` namespace -([#4266](https://github.com/devantler-tech/platform/issues/4266)). +([#4266](https://github.com/devantler-tech/platform/issues/4266)) and was then +removed ([#5391](https://github.com/devantler-tech/platform/issues/5391)). diff --git a/k8s/bases/infrastructure/vault-config/job.yaml b/k8s/bases/infrastructure/vault-config/job.yaml index 2ff97c16ec..c3aa3d90f8 100644 --- a/k8s/bases/infrastructure/vault-config/job.yaml +++ b/k8s/bases/infrastructure/vault-config/job.yaml @@ -692,6 +692,19 @@ spec: echo "WARNING: secret/infrastructure/velero/repo is not populated yet -- leaving the old copy at secret/infrastructure/backup/velero-repo in place until the PushSecret has synced." fi + # --- 1g. Retire Crossview's generated credentials --- + # Crossview, its retained database and their storage were deleted + # by the explicit retirement (#2496), so nothing can need these + # credentials any more. Their PushSecrets never delete the remote + # copies, so destroy every version plus the metadata here. + # Deleting absent metadata succeeds, so a later run is a no-op; + # any other error fails the Job (set -e) instead of reading as + # "already retired". + for retired in apps/crossview/db apps/crossview/app apps/crossview/admin; do + echo "Retiring secret/$retired..." + bao kv metadata delete -mount=secret "$retired" + done + # --- 2. Enable Kubernetes auth --- if ! bao auth list -format=json | grep -q '"kubernetes/"'; then echo "Enabling Kubernetes auth method..." @@ -894,18 +907,9 @@ spec: } POLICY - # crossview reads its bundled-Postgres password, app session secret, - # and break-glass admin password from OpenBao via the shared - # `external-secrets` ClusterSecretStore role (apps/crossview/{db,app, - # admin}); the values are generated and seeded by - # infrastructure/vault-seed. (Its OIDC client secret is read from the - # shared infrastructure/oidc/dex path via infra-oidc-readonly, also on - # that role โ€” no crossview-specific grant needed.) - bao policy write app-crossview-readonly - <<'POLICY' - path "secret/data/apps/crossview/*" { - capabilities = ["read"] - } - POLICY + # Crossview was removed from the platform; drop its retired read + # policy. Deleting an absent policy succeeds, so re-runs are no-ops. + bao policy delete app-crossview-readonly # actual-budget reads its Enable Banking bank-sync credentials from # OpenBao via the shared `external-secrets` ClusterSecretStore role @@ -1033,9 +1037,6 @@ spec: path "secret/data/apps/actual-budget/*" { capabilities = ["create", "update", "read"] } - path "secret/data/apps/crossview/*" { - capabilities = ["create", "update", "read"] - } # Company Email preparation preserves three durable credentials. # Exact-key read/seed only; no mailbox data or wildcard grant. path "secret/data/apps/private-mail/admin" { @@ -1092,9 +1093,6 @@ spec: path "secret/metadata/apps/actual-budget/*" { capabilities = ["create", "update", "read"] } - path "secret/metadata/apps/crossview/*" { - capabilities = ["create", "update", "read"] - } path "secret/metadata/apps/private-mail/admin" { capabilities = ["create", "update", "read"] } @@ -1124,7 +1122,7 @@ spec: bao write auth/kubernetes/role/external-secrets \ bound_service_account_names=external-secrets \ bound_service_account_namespaces=external-secrets \ - policies=infra-backup-readonly,infra-dns-readonly,infra-monitoring-readonly,infra-hcloud-readonly,infra-tls-readonly,infra-ghcr-readonly,app-fleetdm-readonly,app-wedding-readonly,app-umami-readonly,app-actual-budget-readonly,app-crossview-readonly,app-ascoachingogvaner-readonly,infra-oidc-readonly,vault-seed-write \ + policies=infra-backup-readonly,infra-dns-readonly,infra-monitoring-readonly,infra-hcloud-readonly,infra-tls-readonly,infra-ghcr-readonly,app-fleetdm-readonly,app-wedding-readonly,app-umami-readonly,app-actual-budget-readonly,app-ascoachingogvaner-readonly,infra-oidc-readonly,vault-seed-write \ ttl=1h # Dedicated role for the fleetdm VaultDynamicSecret generator. diff --git a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-admin-password.yaml b/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-admin-password.yaml deleted file mode 100644 index 6ad8d97a2d..0000000000 --- a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-admin-password.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -# crossview break-glass `admin` login password โ€” generate once, cache, mirror to -# OpenBao. Read back via crossview-credentials (apps/crossview/admin) and seeded -# into the boot-time admin user (createAdmin, session mode). -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: generated-crossview-admin-password - namespace: openbao -spec: - refreshInterval: "0" - target: - name: generated-crossview-admin-password - creationPolicy: Owner - dataFrom: - - sourceRef: - generatorRef: - apiVersion: generators.external-secrets.io/v1alpha1 - kind: Password - name: crossview-admin-password diff --git a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-db-password.yaml b/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-db-password.yaml deleted file mode 100644 index 4444fa1986..0000000000 --- a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-db-password.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# crossview bundled-Postgres password โ€” generate once, cache, mirror to OpenBao. -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: generated-crossview-db-password - namespace: openbao -spec: - refreshInterval: "0" - target: - name: generated-crossview-db-password - creationPolicy: Owner - dataFrom: - - sourceRef: - generatorRef: - apiVersion: generators.external-secrets.io/v1alpha1 - kind: Password - name: crossview-db-password diff --git a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-session-secret.yaml b/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-session-secret.yaml deleted file mode 100644 index 6937842c0a..0000000000 --- a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-session-secret.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# crossview app session signing key โ€” generate once, cache, mirror to OpenBao. -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: generated-crossview-session-secret - namespace: openbao -spec: - refreshInterval: "0" - target: - name: generated-crossview-session-secret - creationPolicy: Owner - dataFrom: - - sourceRef: - generatorRef: - apiVersion: generators.external-secrets.io/v1alpha1 - kind: Password - name: crossview-session-secret diff --git a/k8s/bases/infrastructure/vault-seed/kustomization.yaml b/k8s/bases/infrastructure/vault-seed/kustomization.yaml index 997ad6f3ca..e924834de9 100644 --- a/k8s/bases/infrastructure/vault-seed/kustomization.yaml +++ b/k8s/bases/infrastructure/vault-seed/kustomization.yaml @@ -10,9 +10,6 @@ resources: - password-umami-app-secret.yaml - password-umami-admin-password.yaml - password-umami-ascoachingogvaner-password.yaml - - password-crossview-db-password.yaml - - password-crossview-session-secret.yaml - - password-crossview-admin-password.yaml - external-secret-generated-fleetdm-mysql-root-password.yaml - push-secret-push-fleetdm-mysql-passwords.yaml - external-secret-generated-fleetdm-mysql-replication-password.yaml @@ -29,12 +26,6 @@ resources: - push-secret-push-umami-admin-password.yaml - external-secret-generated-umami-ascoachingogvaner-password.yaml - push-secret-push-umami-ascoachingogvaner-password.yaml - - external-secret-generated-crossview-db-password.yaml - - push-secret-push-crossview-db-password.yaml - - external-secret-generated-crossview-session-secret.yaml - - push-secret-push-crossview-session-secret.yaml - - external-secret-generated-crossview-admin-password.yaml - - push-secret-push-crossview-admin-password.yaml - push-secret-seed-r2-credentials.yaml - push-secret-seed-wedding-db-backup-r2.yaml - push-secret-seed-github-app.yaml diff --git a/k8s/bases/infrastructure/vault-seed/password-crossview-admin-password.yaml b/k8s/bases/infrastructure/vault-seed/password-crossview-admin-password.yaml deleted file mode 100644 index eeda75b4b6..0000000000 --- a/k8s/bases/infrastructure/vault-seed/password-crossview-admin-password.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -# crossview's break-glass native `admin` login password (ADMIN_PASSWORD), -# replacing the chart's well-known admin/password default. symbols:0 keeps it -# safe to pass as an env var without escaping. -apiVersion: generators.external-secrets.io/v1alpha1 -kind: Password -metadata: - name: crossview-admin-password - namespace: openbao -spec: - length: 32 - digits: 8 - symbols: 0 - noUpper: false - allowRepeat: true diff --git a/k8s/bases/infrastructure/vault-seed/password-crossview-db-password.yaml b/k8s/bases/infrastructure/vault-seed/password-crossview-db-password.yaml deleted file mode 100644 index 88140413e4..0000000000 --- a/k8s/bases/infrastructure/vault-seed/password-crossview-db-password.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# crossview's bundled-Postgres password. symbols:0 keeps it safe to pass as -# POSTGRES_PASSWORD / in the app's DB connection without escaping. -apiVersion: generators.external-secrets.io/v1alpha1 -kind: Password -metadata: - name: crossview-db-password - namespace: openbao -spec: - length: 32 - digits: 8 - symbols: 0 - noUpper: false - allowRepeat: true diff --git a/k8s/bases/infrastructure/vault-seed/password-crossview-session-secret.yaml b/k8s/bases/infrastructure/vault-seed/password-crossview-session-secret.yaml deleted file mode 100644 index 3fe5295f1d..0000000000 --- a/k8s/bases/infrastructure/vault-seed/password-crossview-session-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -# crossview's app session signing key (SESSION_SECRET). -apiVersion: generators.external-secrets.io/v1alpha1 -kind: Password -metadata: - name: crossview-session-secret - namespace: openbao -spec: - length: 64 - digits: 16 - symbols: 0 - noUpper: false - allowRepeat: true diff --git a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-admin-password.yaml b/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-admin-password.yaml deleted file mode 100644 index 79843e57bc..0000000000 --- a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-admin-password.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: external-secrets.io/v1alpha1 -kind: PushSecret -metadata: - name: push-crossview-admin-password - namespace: openbao -spec: - refreshInterval: 1h - secretStoreRefs: - - name: openbao - kind: ClusterSecretStore - selector: - secret: - name: generated-crossview-admin-password - data: - - match: - secretKey: password - remoteRef: - remoteKey: apps/crossview/admin - property: admin-password diff --git a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-db-password.yaml b/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-db-password.yaml deleted file mode 100644 index 8b1ff4676a..0000000000 --- a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-db-password.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: external-secrets.io/v1alpha1 -kind: PushSecret -metadata: - name: push-crossview-db-password - namespace: openbao -spec: - refreshInterval: 1h - secretStoreRefs: - - name: openbao - kind: ClusterSecretStore - selector: - secret: - name: generated-crossview-db-password - data: - - match: - secretKey: password - remoteRef: - remoteKey: apps/crossview/db - property: db-password diff --git a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-session-secret.yaml b/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-session-secret.yaml deleted file mode 100644 index b5404994f6..0000000000 --- a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-session-secret.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: external-secrets.io/v1alpha1 -kind: PushSecret -metadata: - name: push-crossview-session-secret - namespace: openbao -spec: - refreshInterval: 1h - secretStoreRefs: - - name: openbao - kind: ClusterSecretStore - selector: - secret: - name: generated-crossview-session-secret - data: - - match: - secretKey: password - remoteRef: - remoteKey: apps/crossview/app - property: session-secret diff --git a/scripts/retire-unused-apps.sh b/scripts/retire-unused-apps.sh deleted file mode 100755 index a7af8440f0..0000000000 --- a/scripts/retire-unused-apps.sh +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env bash -# Deletes what Flux deliberately leaves behind once Crossview, Headlamp (#2496) -# and the retained Backstage database (#4266) are gone from Git. -# -# Production disables Flux pruning on every Namespace, HelmRelease, claim and -# CloudNativePG Cluster, so removing the manifests stops reconciling them but -# keeps them running. AGENTS.md ("Persistence retirement is always two-stage") -# makes the final deletion an explicit step. This is that step, for exactly this -# set, deleted in this order: -# -# 1. HelmReleases crossview/crossview and headlamp/headlamp: the chart -# uninstall removes their workloads; -# 2. CloudNativePG Cluster backstage/backstage-db: its instances stop; -# 3. Namespaces crossview, headlamp and backstage, with every claim and object -# still inside them; -# 4. volumes that were bound to claims in those namespaces. A Delete policy -# reclaims them on its own; a Retain volume (the Backstage database) is -# deleted once Released; -# 5. Longhorn volumes last used by claims in those namespaces, once detached: -# a Retain volume's backing data survives its volume object otherwise; -# 6. the Kubescape scan summaries named after the namespaces, which outlive -# the namespaces they describe. -# -# The Backstage data has no backup: deleting it was the maintainer's decision -# of 26 October 2026, which replaced keeping a recovery copy (#5122). -# -# Nothing is deleted until the infrastructure-controllers, infrastructure and -# apps Kustomizations are Ready for their current generations and their -# inventories own no object of the three apps: that is the proof the removal -# has deployed, so nothing recreates them. Each Namespace, HelmRelease and -# Cluster must be owned by the apps Kustomization, and every volume and Longhorn -# volume must name one of the three namespaces as its last claim. Re-running -# after a partial run is safe: absent objects are skipped, and storage is found -# again by its recorded claim namespace. The run ends by proving the whole set, -# and the retired bindings, are gone. -# -# PUBLIC LOG: workflow logs are public, so kubectl's own error text, which can -# quote the server it reached, is discarded and each failure is named here. -# -# Exit codes: 0 everything is gone, 1 a refusal or leftover, 2 an input could not -# be read. -set -euo pipefail - -readonly confirm_value='retire-crossview-headlamp-backstage' -readonly context='admin@prod' -kubectl_bin="${KUBECTL:-kubectl}" -poll_seconds="${RETIREMENT_POLL_SECONDS:-10}" -poll_attempts="${RETIREMENT_POLL_ATTEMPTS:-90}" -readonly kubectl_bin poll_seconds poll_attempts -# A layer retrying an unrelated failing health check passes through a -# Progressing phase every retry interval, so its state is sampled until it -# settles rather than judged at one instant. -settle_seconds="${RETIREMENT_SETTLE_SECONDS:-15}" -settle_attempts="${RETIREMENT_SETTLE_ATTEMPTS:-40}" -readonly settle_seconds settle_attempts -readonly releases=(crossview/crossview headlamp/headlamp) -readonly clusters=(backstage/backstage-db) -readonly namespaces=(crossview headlamp backstage) -readonly retired_pattern='crossview|headlamp|backstage' -# Crossview's Vault seed chain stays declared until this retirement has run, so -# a healed failed deploy never regenerates credentials that Crossview's retained -# database was initialised with. A later revision removes it (#5391). -readonly retained_seed_pattern='^openbao_(generated-|push-)?crossview-(db-password|session-secret|admin-password)_' -readonly longhorn_namespace=longhorn-system -readonly summary_kinds=( - configurationscansummaries.spdx.softwarecomposition.kubescape.io - vulnerabilitysummaries.spdx.softwarecomposition.kubescape.io -) -readonly retired_bindings=(crossview-cluster-reader crossview-view headlamp-cluster-reader) -namespaces_json="$(printf '%s\n' "${namespaces[@]}" | jq -R . | jq -sc .)" -readonly namespaces_json - -fail() { - printf 'FAIL: %s\n' "$1" >&2 - exit 1 -} - -unknown() { - printf 'UNKNOWN: %s\n' "$1" >&2 - exit 2 -} - -k() { - "${kubectl_bin}" --context "${context}" --request-timeout=30s "$@" 2>/dev/null -} - -# Prints the object's JSON, or nothing when it does not exist. A failed read is -# UNKNOWN, never "absent". Call it only in a plain assignment: inside `[ ]` or -# a condition its failure would be read as "absent". -read_object() { - local out - out="$(k get "$@" --ignore-not-found -o json)" || unknown "cannot read $*" - printf '%s' "${out}" -} - -# Prints the names of the volumes whose claim was in a retired namespace. -retired_volumes() { - local out - out="$(k get pv -o json)" || unknown 'cannot list volumes' - jq -r --argjson ns "${namespaces_json}" \ - '.items[] | select(.spec.claimRef.namespace as $n | $ns | index($n)) | .metadata.name' <<<"${out}" -} - -# Prints " " for the Longhorn volumes last claimed in a retired namespace. -retired_longhorn_volumes() { - local out - out="$(k -n "${longhorn_namespace}" get volumes.longhorn.io -o json)" || - unknown 'cannot list Longhorn volumes' - jq -r --argjson ns "${namespaces_json}" ' - .items[] | select(.status.kubernetesStatus.namespace as $n | $ns | index($n)) | - "\(.metadata.name) \(.status.state // "unknown")"' <<<"${out}" -} - -require_apps_owned() { - local description="$1" object="$2" - jq -e '.metadata.labels["kustomize.toolkit.fluxcd.io/name"] == "apps" and - .metadata.labels["kustomize.toolkit.fluxcd.io/namespace"] == "flux-system"' \ - <<<"${object}" >/dev/null || - fail "${description} is not owned by the apps Kustomization; refusing to delete it" -} - -# Deletes an apps-owned object and waits for its finalizers. -delete_owned() { # - local description="$1" timeout="$2" object - shift 2 - object="$(read_object "$@")" - if [ -z "${object}" ]; then - printf '%s is already gone\n' "${description}" - return - fi - require_apps_owned "${description}" "${object}" - printf 'Deleting %s\n' "${description}" - k delete "$@" --ignore-not-found --wait --timeout="${timeout}" >/dev/null || - fail "${description} was not deleted" -} - -[ "${RETIREMENT_CONFIRM:-}" = "${confirm_value}" ] || - unknown "set RETIREMENT_CONFIRM=${confirm_value} to run this retirement" -command -v jq >/dev/null || unknown 'jq is required' - -# 0. The removal is deployed: every layer that carried part of an app applies -# its current revision without owning anything of them. Flux records a -# layer's inventory right after a successful apply, before health checks, -# so a layer whose only failure is HealthCheckFailed still proves what its -# current revision applied; an unrelated unhealthy workload must not block -# this retirement. Any other not-Ready reason, such as a failed build or -# apply, leaves the inventory unproven and is refused. -for layer in infrastructure-controllers infrastructure apps; do - attempt=0 - while :; do - kustomization="$(k -n flux-system get kustomization "${layer}" -o json)" || - unknown "cannot read the ${layer} Kustomization" - jq -e ' - (.status.conditions // [] | any(.type == "Ready" and - (.status == "True" or (.status == "False" and .reason == "HealthCheckFailed")))) and - .status.observedGeneration == .metadata.generation and - (.status.inventory.entries | type == "array" and length > 0) - ' <<<"${kustomization}" >/dev/null && break - attempt=$((attempt + 1)) - [ "${attempt}" -lt "${settle_attempts}" ] || - fail "the ${layer} Kustomization has not applied its current generation with a non-empty inventory" - sleep "${settle_seconds}" - done - if jq -e '.status.conditions // [] | any(.type == "Ready" and .status == "False")' <<<"${kustomization}" >/dev/null; then - printf 'Note: the %s Kustomization applied its revision but an unrelated health check is failing\n' "${layer}" - fi - owned="$(jq -r --arg p "${retired_pattern}" --arg seed "${retained_seed_pattern}" \ - '[.status.inventory.entries[].id | select(test($p) and (test($seed) | not))] | join(" ")' <<<"${kustomization}")" - [ -z "${owned}" ] || - fail "the deployed ${layer} inventory still owns ${owned}; deploy the removal before retiring the leftovers" -done - -# 1โ€“3. Owners first, so no controller recreates what the namespaces hold. -for release in "${releases[@]}"; do - delete_owned "HelmRelease ${release}" 10m helmrelease -n "${release%%/*}" "${release#*/}" -done -for cluster in "${clusters[@]}"; do - delete_owned "Cluster ${cluster}" 10m clusters.postgresql.cnpg.io -n "${cluster%%/*}" "${cluster#*/}" -done -for namespace in "${namespaces[@]}"; do - delete_owned "Namespace ${namespace}" 15m namespace "${namespace}" -done - -# 4. Volumes: wait until each is reclaimed or Released, then delete the Released. -# Every pass counts, so a volume that outlives its deletion cannot spin. -attempt=0 -while :; do - volumes="$(retired_volumes)" - [ -n "${volumes}" ] || break - attempt=$((attempt + 1)) - [ "${attempt}" -le "${poll_attempts}" ] || - fail "volumes were neither reclaimed nor deleted: ${volumes//$'\n'/ }" - for volume in ${volumes}; do - object="$(read_object pv "${volume}")" - [ -n "${object}" ] || continue - phase="$(jq -r '.status.phase // ""' <<<"${object}")" - [ "${phase}" = Released ] || continue - printf 'Deleting released volume %s\n' "${volume}" - k delete pv "${volume}" --ignore-not-found --wait --timeout=5m >/dev/null || - fail "volume ${volume} was not deleted" - done - sleep "${poll_seconds}" -done - -# 5. Longhorn volumes: delete each once detached. A Delete-policy volume may -# already be on its way out, which deleting again does not disturb. -attempt=0 -while :; do - backing="$(retired_longhorn_volumes)" - [ -n "${backing}" ] || break - attempt=$((attempt + 1)) - [ "${attempt}" -le "${poll_attempts}" ] || - fail "Longhorn volumes did not detach or were not deleted: ${backing//$'\n'/; }" - while read -r name state; do - [ "${state}" = detached ] || continue - printf 'Deleting Longhorn volume %s\n' "${name}" - k -n "${longhorn_namespace}" delete volumes.longhorn.io "${name}" --ignore-not-found --wait --timeout=5m >/dev/null || - fail "Longhorn volume ${name} was not deleted" - done <<<"${backing}" - sleep "${poll_seconds}" -done - -# 6. The scan summaries are cluster-scoped and named after the namespaces. -for kind in "${summary_kinds[@]}"; do - k delete "${kind}" "${namespaces[@]}" --ignore-not-found >/dev/null || - fail "the ${kind} records could not be deleted" -done - -# Prove the whole set is gone. -leftovers=() -check_absent() { - local description="$1" object - shift - object="$(read_object "$@")" - [ -z "${object}" ] || leftovers+=("${description}") -} -for release in "${releases[@]}"; do - check_absent "HelmRelease ${release}" helmrelease -n "${release%%/*}" "${release#*/}" -done -for cluster in "${clusters[@]}"; do - check_absent "Cluster ${cluster}" clusters.postgresql.cnpg.io -n "${cluster%%/*}" "${cluster#*/}" -done -for namespace in "${namespaces[@]}"; do - check_absent "Namespace ${namespace}" namespace "${namespace}" - for kind in "${summary_kinds[@]}"; do - check_absent "${kind} ${namespace}" "${kind}" "${namespace}" - done -done -for binding in "${retired_bindings[@]}"; do - check_absent "ClusterRoleBinding ${binding}" clusterrolebinding "${binding}" -done -volumes="$(retired_volumes)" -[ -z "${volumes}" ] || leftovers+=("volumes ${volumes//$'\n'/ }") -backing="$(retired_longhorn_volumes)" -[ -z "${backing}" ] || leftovers+=("Longhorn volumes ${backing//$'\n'/; }") -[ "${#leftovers[@]}" -eq 0 ] || fail "still present: $(printf '%s; ' "${leftovers[@]}")" - -printf 'PASS: Crossview, Headlamp and Backstage leave nothing behind in production\n' diff --git a/scripts/tests/test-retire-unused-apps.sh b/scripts/tests/test-retire-unused-apps.sh deleted file mode 100755 index 396f0ec481..0000000000 --- a/scripts/tests/test-retire-unused-apps.sh +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env bash -# Offline regression for scripts/retire-unused-apps.sh against a fake kubectl -# whose objects live as files. It proves the retirement refuses before the -# removal is deployed in every layer, refuses objects Flux `apps` does not own, -# deletes owners before namespaces and storage, deletes only storage last -# claimed in a retired namespace, waits for Longhorn volumes to detach, is -# idempotent, never reads a failed read as "absent", and reports any leftover. -# No cluster or credential is used. -set -euo pipefail - -root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly root_dir -readonly script="${root_dir}/scripts/retire-unused-apps.sh" -work_dir="$(mktemp -d)" -readonly work_dir -trap 'rm -rf "${work_dir}"' EXIT - -fail() { - printf 'FAIL: %s\n' "$1" >&2 - exit 1 -} - -command -v jq >/dev/null || fail 'jq is required' - -readonly lh=volumes.longhorn.io -readonly cnpg=clusters.postgresql.cnpg.io - -# The fake stores one JSON file per object under $STATE/// -# (namespace "_" for cluster-scoped kinds), logs every call as " -# ", answers a nameless get with a list, and models the side effects the -# script relies on: deleting a namespace deletes its objects, a Delete-policy -# volume goes with its claim, and a Retain volume becomes Released while its -# Longhorn volume detaches. -cat >"${work_dir}/kubectl" <<'FAKE' -#!/usr/bin/env bash -set -euo pipefail -namespace=_ verb='' kind='' names=() -while [ "$#" -gt 0 ]; do - case "$1" in - --request-timeout=*|--ignore-not-found|--wait|--timeout=*) ;; - -o|--context) shift ;; - -n) namespace="$2"; shift ;; - get|delete) verb="$1" ;; - *) if [ -z "${kind}" ] && [ -n "${verb}" ]; then kind="$1"; else names+=("$1"); fi ;; - esac - shift -done -printf '%s %s %s\n' "${verb}" "${kind}" "${names[*]:-}" >>"${STATE}/calls" -if [ -n "${FAIL_READ:-}" ] && [ "${verb}" = get ] && [ "${kind}" = "${FAIL_READ}" ]; then - echo 'error: the server is currently unable to handle the request' >&2 - exit 1 -fi -if [ "${verb}" = get ] && [ "${#names[@]}" -eq 0 ]; then - find "${STATE}/${kind}/${namespace}" -type f -exec cat {} + 2>/dev/null | jq -s '{items: .}' - exit 0 -fi -for name in "${names[@]}"; do - file="${STATE}/${kind}/${namespace}/${name}" - case "${verb}" in - get) - if [ "${kind}" = kustomization ] && [ -f "${STATE}/${name}.first.json" ]; then cat "${STATE}/${name}.first.json"; rm -f "${STATE}/${name}.first.json"; - elif [ "${kind}" = kustomization ]; then cat "${STATE}/${name}.json" 2>/dev/null || cat "${STATE}/apps.json"; elif [ -f "${file}" ]; then cat "${file}"; fi ;; - delete) - [ -n "${KEEP:-}" ] && [ "${kind}/${name}" = "${KEEP}" ] && continue - rm -f "${file}" - if [ "${kind}" = namespace ]; then - rm -rf "${STATE}"/*/"${name}" - for pv in "${STATE}"/pv/_/*; do - [ -f "${pv}" ] && [ "$(jq -r .spec.claimRef.namespace "${pv}")" = "${name}" ] || continue - volume="${STATE}/volumes.longhorn.io/longhorn-system/$(basename "${pv}")" - if [ "$(jq -r .spec.persistentVolumeReclaimPolicy "${pv}")" = Delete ]; then - rm -f "${pv}" "${volume}" - else - jq '.status.phase = "Released"' "${pv}" >"${pv}.tmp" && mv "${pv}.tmp" "${pv}" - [ -n "${STAY_ATTACHED:-}" ] || { jq '.status.state = "detached"' "${volume}" >"${volume}.tmp" && mv "${volume}.tmp" "${volume}"; } - fi - done - fi ;; - esac -done -FAKE -chmod +x "${work_dir}/kubectl" - -owned='{"metadata":{"labels":{"kustomize.toolkit.fluxcd.io/name":"apps","kustomize.toolkit.fluxcd.io/namespace":"flux-system"}}}' - -put() { # - mkdir -p "$1/$2/$3" - printf '%s' "$5" >"$1/$2/$3/$4" -} - -volume() { # - put "$1" pv _ "$2" "$(jq -nc --arg n "$2" --arg ns "$3" --arg c "$4" --arg p "$5" \ - '{metadata:{name:$n},spec:{claimRef:{namespace:$ns,name:$c},persistentVolumeReclaimPolicy:$p},status:{phase:"Bound"}}')" - put "$1" "${lh}" longhorn-system "$2" "$(jq -nc --arg n "$2" --arg ns "$3" --arg c "$4" \ - '{metadata:{name:$n},status:{state:"attached",kubernetesStatus:{namespace:$ns,pvcName:$c,pvName:$n}}}')" -} - -seed() { # - local state="${work_dir}/$1" kind - rm -rf "${state}" - mkdir -p "${state}" - jq -n --argjson ids "$2" '{metadata:{generation:4},status:{observedGeneration:4, - conditions:[{type:"Ready",status:"True"}],inventory:{entries:[$ids[] | {id: ., v:"v1"}]}}}' \ - >"${state}/apps.json" - put "${state}" helmrelease crossview crossview "${owned}" - put "${state}" helmrelease headlamp headlamp "${owned}" - put "${state}" "${cnpg}" backstage backstage-db "${owned}" - for namespace in crossview headlamp backstage; do - put "${state}" namespace _ "${namespace}" "${owned}" - done - put "${state}" pvc crossview crossview-postgres-pvc '{}' - volume "${state}" pvc-crossview crossview crossview-postgres-pvc Delete - put "${state}" pvc backstage backstage-db-3 '{}' - volume "${state}" pvc-backstage-3 backstage backstage-db-3 Retain - volume "${state}" pvc-backstage-4 backstage backstage-db-4 Retain - # Unrelated storage must survive every run. - volume "${state}" pvc-wedding wedding-app wedding-db-1 Retain - for kind in configurationscansummaries.spdx.softwarecomposition.kubescape.io \ - vulnerabilitysummaries.spdx.softwarecomposition.kubescape.io; do - for namespace in crossview headlamp backstage; do - put "${state}" "${kind}" _ "${namespace}" '{}' - done - done - printf '%s' "${state}" -} - -run() { # [env...] -> sets status and output - local state="$1" - shift - if output="$(env STATE="${state}" KUBECTL="${work_dir}/kubectl" RETIREMENT_POLL_SECONDS=0 \ - RETIREMENT_POLL_ATTEMPTS=3 RETIREMENT_SETTLE_SECONDS=0 RETIREMENT_SETTLE_ATTEMPTS=3 RETIREMENT_CONFIRM=retire-crossview-headlamp-backstage "$@" \ - bash "${script}" 2>&1)"; then - status=0 - else - status=$? - fi -} - -expect() { # : the run ended with status and printed needle - if [ "${status}" -ne "$1" ] || ! grep -Fq -- "$2" <<<"${output}"; then - fail "$3, got ${status}: ${output}" - fi -} - -deleted() { # - grep -c '^delete ' "$1/calls" 2>/dev/null || true -} - -survivors() { # : every object file left, relative to the state - (cd "$1" && find . -path ./calls -prune -o -name '*.json' -prune -o -type f -print | sort) -} - -deployed='["flux-system_apps_kustomize.toolkit.fluxcd.io_Kustomization","whoami_whoami_helm.toolkit.fluxcd.io_HelmRelease"]' - -# Without the exact confirmation nothing is read. -state="$(seed unconfirmed "${deployed}")" -run "${state}" RETIREMENT_CONFIRM=retire-crossview-headlamp-2496 -[ "${status}" -eq 2 ] || fail "an unconfirmed run must be UNKNOWN, got ${status}: ${output}" -[ ! -e "${state}/calls" ] || fail 'an unconfirmed run must not call the cluster' - -# Before the removal deploys, an inventory still owns an app: refuse. -for id in crossview_crossview_helm.toolkit.fluxcd.io_HelmRelease _headlamp-cluster-reader_rbac.authorization.k8s.io_ClusterRoleBinding \ - _backstage__Namespace; do - state="$(seed undeployed "[\"${id}\"]")" - run "${state}" - expect 1 'still owns' "an undeployed removal owning ${id} must be refused" - [ "$(deleted "${state}")" -eq 0 ] || fail "an undeployed removal owning ${id} must delete nothing" -done - -# Every layer that carried part of an app must have deployed the removal. -state="$(seed infrastructure-undeployed "${deployed}")" -jq '.status.inventory.entries += [{id: "observability_crossview-login-alerter_batch_CronJob", v: "v1"}]' \ - "${state}/apps.json" >"${state}/infrastructure.json" -run "${state}" -expect 1 'deployed infrastructure inventory still owns observability_crossview-login-alerter' \ - 'a removal not yet deployed to infrastructure must be refused' -[ "$(deleted "${state}")" -eq 0 ] || fail 'an undeployed infrastructure removal must delete nothing' - -# A Kustomization that is not Ready for its generation proves nothing. -state="$(seed stale "${deployed}")" -jq '.status.observedGeneration = 3' "${state}/apps.json" >"${state}/apps.tmp" && mv "${state}/apps.tmp" "${state}/apps.json" -run "${state}" -expect 1 'has not applied its current generation' 'a stale apps Kustomization must be refused' -[ "$(deleted "${state}")" -eq 0 ] || fail 'a stale apps Kustomization must delete nothing' - -# A failed build or apply leaves the inventory unproven, so it is refused. -state="$(seed apply-failed "${deployed}")" -jq '.status.conditions = [{type: "Ready", status: "False", reason: "ReconciliationFailed"}]' "${state}/apps.json" >"${state}/apps.tmp" && mv "${state}/apps.tmp" "${state}/apps.json" -run "${state}" -expect 1 'has not applied its current generation' 'a failed apply must be refused' -[ "$(deleted "${state}")" -eq 0 ] || fail 'a failed apply must delete nothing' - -# An unrelated workload failing its health check does not block the retirement: -# the apply succeeded, so the inventory still proves the removal deployed. -state="$(seed unrelated-health "${deployed}")" -jq '.status.conditions = [{type: "Ready", status: "False", reason: "HealthCheckFailed"}]' "${state}/apps.json" >"${state}/apps.tmp" && mv "${state}/apps.tmp" "${state}/apps.json" -run "${state}" -expect 0 'an unrelated health check is failing' 'an unrelated health failure must not block the retirement' - -# A layer read mid-retry (Progressing) is sampled again until it settles. -state="$(seed retrying "${deployed}")" -jq '.status.conditions = [{type: "Ready", status: "False", reason: "HealthCheckFailed"}]' "${state}/apps.json" >"${state}/apps.tmp" && mv "${state}/apps.tmp" "${state}/apps.json" -jq '.status.conditions = [{type: "Ready", status: "Unknown", reason: "Progressing"}]' "${state}/apps.json" >"${state}/apps.first.json" -run "${state}" -expect 0 'an unrelated health check is failing' 'a layer that settles after a retry must not block the retirement' -[ "$(grep -c '^get kustomization apps' "${state}/calls")" -ge 2 ] || fail 'a Progressing layer must be sampled again' - -# An object of the same name that Flux apps does not own is never deleted. -state="$(seed foreign "${deployed}")" -put "${state}" "${cnpg}" backstage backstage-db '{"metadata":{"labels":{}}}' -run "${state}" -expect 1 'Cluster backstage/backstage-db is not owned' 'a foreign database must be refused' -[ -f "${state}/${cnpg}/backstage/backstage-db" ] || fail 'a foreign database must survive' -[ -f "${state}/namespace/_/backstage" ] || fail 'a refusal must stop before the namespaces' - -# A failed read is UNKNOWN and never reads as "already gone". -state="$(seed failed-read "${deployed}")" -run "${state}" FAIL_READ=helmrelease -expect 2 'UNKNOWN: cannot read helmrelease' 'a failed read must stop the run as UNKNOWN' -[ "$(deleted "${state}")" -eq 0 ] || fail 'a failed read must delete nothing' - -# The same holds for the storage listings and the final proof. -for kind in pv "${lh}" clusterrolebinding; do - state="$(seed "failed-${kind}" "${deployed}")" - run "${state}" FAIL_READ="${kind}" - [ "${status}" -eq 2 ] || fail "a failed ${kind} read must be UNKNOWN, got ${status}: ${output}" - if [ "${kind}" = pv ] && grep -q "^delete ${lh} " "${state}/calls"; then - fail 'a failed volume read must stop the run before any Longhorn volume is deleted' - fi -done - -# A Longhorn volume that never detaches is not deleted under a workload. -state="$(seed attached "${deployed}")" -run "${state}" STAY_ATTACHED=1 -expect 1 'Longhorn volumes did not detach' 'an attached Longhorn volume must stop the run' -[ -f "${state}/${lh}/longhorn-system/pvc-backstage-3" ] || fail 'an attached Longhorn volume must survive' - -# Crossview's Vault seed chain stays declared until a later revision; its exact -# objects do not block the retirement, but a near-miss name still does. -seed_ids='"openbao_crossview-db-password_generators.external-secrets.io_Password", - "openbao_generated-crossview-session-secret_external-secrets.io_ExternalSecret", - "openbao_push-crossview-admin-password_external-secrets.io_PushSecret"' -state="$(seed seed-near-miss "${deployed}")" -jq ".status.inventory.entries += [{id: \"openbao_push-crossview-oidc-secret_external-secrets.io_PushSecret\", v: \"v1\"}]" \ - "${state}/apps.json" >"${state}/infrastructure.json" -run "${state}" -expect 1 'still owns openbao_push-crossview-oidc-secret' 'a seed outside the retained chain must be refused' - -# The deployed removal retires everything in order and leaves only unrelated -# storage behind. -state="$(seed deployed "${deployed}")" -jq "[${seed_ids}] as \$seed | .status.inventory.entries += [\$seed[] | {id: ., v: \"v1\"}]" \ - "${state}/apps.json" >"${state}/infrastructure.json" -run "${state}" -expect 0 'PASS:' 'the deployed retirement must pass' -order="$(awk '$1 == "delete" {print $2}' "${state}/calls" | uniq | tr '\n' ' ')" -[ "${order}" = "helmrelease ${cnpg} namespace pv ${lh} configurationscansummaries.spdx.softwarecomposition.kubescape.io vulnerabilitysummaries.spdx.softwarecomposition.kubescape.io " ] || - fail "unexpected deletion order: ${order}" -[ "$(survivors "${state}")" = "$(printf './pv/_/pvc-wedding\n./%s/longhorn-system/pvc-wedding' "${lh}")" ] || - fail "only unrelated storage may survive, got: $(survivors "${state}")" - -# A second run finds nothing to delete but still verifies. -rm -f "${state}/calls" -run "${state}" -expect 0 'already gone' 'a repeat run must pass' -[ "$(deleted "${state}")" -eq 2 ] || fail 'a repeat run may only re-issue the idempotent summary deletions' - -# A rerun after the volume objects went still finds the retained Longhorn data. -state="$(seed partial "${deployed}")" -run "${state}" KEEP="${lh}/pvc-backstage-4" -expect 1 'Longhorn volumes did not detach or were not deleted: pvc-backstage-4 detached' \ - 'a Longhorn volume that outlives its deletion must fail the run' -[ ! -e "${state}/pv/_/pvc-backstage-4" ] || fail 'its volume object must already be gone for this case' -run "${state}" -expect 0 'Deleting Longhorn volume pvc-backstage-4' 'a rerun must delete the Longhorn volume found by its claim namespace' - -# An object that survives its deletion is reported, never passed. -state="$(seed leftover "${deployed}")" -run "${state}" KEEP=configurationscansummaries.spdx.softwarecomposition.kubescape.io/headlamp -expect 1 'still present: configurationscansummaries.spdx.softwarecomposition.kubescape.io headlamp' \ - 'a surviving object must fail the run' - -# A retired binding Flux failed to prune is reported too. -state="$(seed binding "${deployed}")" -put "${state}" clusterrolebinding _ crossview-view '{}' -run "${state}" -expect 1 'ClusterRoleBinding crossview-view' 'a surviving binding must fail the run' - -printf 'PASS: the unused-app retirement waits for the deployed removal, deletes only owned leftovers and their storage, and proves nothing remains\n' diff --git a/tests/restrict-homepage-service-groups/kyverno-test.yaml b/tests/restrict-homepage-service-groups/kyverno-test.yaml index 1ed1f45cd8..9088c035bd 100644 --- a/tests/restrict-homepage-service-groups/kyverno-test.yaml +++ b/tests/restrict-homepage-service-groups/kyverno-test.yaml @@ -14,7 +14,7 @@ results: rule: declared-service-group resources: - observability/declared-group - - crossview/infrastructure-as-code-group + - github-config/infrastructure-as-code-group - client-site/business-and-clients-group - personal-site/personal-group - openbao/secrets-management-group diff --git a/tests/restrict-homepage-service-groups/resources.yaml b/tests/restrict-homepage-service-groups/resources.yaml index 51229ce006..15ee84a78d 100644 --- a/tests/restrict-homepage-service-groups/resources.yaml +++ b/tests/restrict-homepage-service-groups/resources.yaml @@ -13,19 +13,19 @@ spec: hostnames: - observability.platform.devantler.tech --- -# The group this change adds. Fails before the layout entry exists, passes after. +# A second declared group, on another namespace. Fails without its layout entry. apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: infrastructure-as-code-group - namespace: crossview + namespace: github-config annotations: gethomepage.dev/enabled: "true" - gethomepage.dev/name: Crossview + gethomepage.dev/name: GitHub Config gethomepage.dev/group: Infrastructure as Code spec: hostnames: - - crossview.platform.devantler.tech + - github-config.platform.devantler.tech --- # The exact drift that motivated the policy: an undeclared group, which Homepage # accepts silently and renders below the bookmark rail.