From 1184b22bb7f31b3a443c3f7a40a50087718304f4 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 26 Oct 2026 18:25:51 +0100 Subject: [PATCH 1/6] fix(platform): retire the unused Crossview and Headlamp UIs Neither cluster UI is in use, and the KSail web UI is taking over their role (devantler-tech/ksail#5827, devantler-tech/ksail#8011). Remove both apps and everything that existed only for them: SSO routes and Dex callbacks, network allowances, Coroot risk acceptances and alerting, the Crossview login sensor, Kubescape exception entries and the Headlamp-only exceptions mirror with its generator mode, Renovate rules, local hosts, docs and the approved authorization ledger entries. The vault-config Job now deletes Crossview's retired read policy and destroys its generated credentials, which their PushSecrets never remove. The hosted Docker registry-cache setup moves to a shared helper so the Wedding runtime fixture keeps it without the Crossview fixture. The Longhorn client maintenance journal schema still names Crossview: its completed journal is validated against that scope on every deploy, and the completed path reads no Crossview object. Fixes #2496 Co-Authored-By: Claude Opus 5.5 --- .github/renovate.json | 20 +- .github/workflows/ci.yaml | 80 +- .github/workflows/dr-rebuild.yaml | 4 +- AGENTS.md | 4 +- docs/TEMPLATING.md | 2 +- docs/architecture.md | 4 +- docs/dr/alerting.md | 51 +- docs/dr/cnpg-storageclass-migration.md | 2 +- docs/dr/velero-cnpg.md | 2 +- docs/progressive-delivery.md | 2 +- docs/runtime-security.md | 2 +- docs/secret-rotation.md | 2 +- hosts | 2 - k8s/bases/apps/README.md | 4 +- .../apps/crossview/cilium-network-policy.yaml | 58 - .../cluster-role-binding-cluster-reader.yaml | 19 - .../crossview/cluster-role-binding-view.yaml | 18 - k8s/bases/apps/crossview/external-secret.yaml | 59 - k8s/bases/apps/crossview/helm-release.yaml | 303 ---- k8s/bases/apps/crossview/helm-repository.yaml | 7 - k8s/bases/apps/crossview/http-route.yaml | 39 - k8s/bases/apps/crossview/kustomization.yaml | 12 - k8s/bases/apps/crossview/namespace.yaml | 7 - k8s/bases/apps/headlamp/README.md | 6 - .../apps/headlamp/cilium-network-policy.yaml | 44 - .../apps/headlamp/cluster-role-binding.yaml | 27 - k8s/bases/apps/headlamp/external-secret.yaml | 18 - k8s/bases/apps/headlamp/helm-release.yaml | 137 -- k8s/bases/apps/headlamp/helm-repository.yaml | 7 - k8s/bases/apps/headlamp/http-route.yaml | 30 - k8s/bases/apps/headlamp/kustomization.yaml | 11 - k8s/bases/apps/headlamp/namespace.yaml | 7 - k8s/bases/apps/kustomization.yaml | 5 - .../cron-job-cronjob-failure-alert.yaml | 8 +- ...e-cronjob-failure-alert-observability.yaml | 8 +- .../validate-replica-floor.yaml | 8 +- .../cluster-roles/cluster-reader.yaml | 2 +- .../controller-rbac.yaml | 12 +- .../health-probes.yaml | 11 +- .../secret-reader-rbac.yaml | 10 +- .../service-account-tokens.yaml | 16 +- ...los-cis-control-plane-false-positives.yaml | 8 - .../talos-cis-worker-false-positives.yaml | 8 - .../wildcard-rbac.yaml | 5 +- .../auth-proxy/cilium-network-policy.yaml | 8 - .../controllers/auth-proxy/config-map.yaml | 20 - .../dex/cilium-network-policy.yaml | 21 - .../controllers/dex/helm-release.yaml | 16 +- .../flux-operator/helm-release.yaml | 7 +- .../config-map-headlamp-exceptions.yaml | 1474 ----------------- .../controllers/kubescape/kustomization.yaml | 1 - .../oauth2-proxy/reference-grant.yaml | 3 - .../security-progress/config-map.yaml | 2 +- .../security-progress/dashboards.json | 2 +- .../infrastructure/vault-config/job.yaml | 37 +- ...et-generated-crossview-admin-password.yaml | 20 - ...ecret-generated-crossview-db-password.yaml | 18 - ...et-generated-crossview-session-secret.yaml | 18 - .../vault-seed/kustomization.yaml | 9 - .../password-crossview-admin-password.yaml | 15 - .../password-crossview-db-password.yaml | 14 - .../password-crossview-session-secret.yaml | 13 - ...-secret-push-crossview-admin-password.yaml | 20 - ...ush-secret-push-crossview-db-password.yaml | 20 - ...-secret-push-crossview-session-secret.yaml | 20 - .../push-secret-seed-dex-client-secret.yaml | 5 +- k8s/clusters/prod/bootstrap/config-map.yaml | 8 +- .../headlamp/patches/trust-platform-ca.yaml | 21 - k8s/providers/docker/apps/kustomization.yaml | 8 +- .../cilium-network-policy-login-alerter.yaml | 26 - .../apps/crossview/cilium-network-policy.yaml | 25 - .../config-map-coroot-monitor-init.yaml | 50 - .../external-secret-coroot-monitor.yaml | 21 - .../crossview/password-coroot-monitor.yaml | 11 - .../patches/enable-coroot-monitoring.yaml | 107 -- .../patches/enable-durable-storage.yaml | 130 -- .../patches/enable-user-namespaces.yaml | 3 - .../patches/enable-user-namespaces.yaml | 39 - k8s/providers/hetzner/apps/kustomization.yaml | 56 +- .../cloudflare-origin-ca-trust-bundle.yaml | 9 +- .../alertmanager/cilium-network-policy.yaml | 17 +- .../alertmanager/helm-release.yaml | 23 +- .../ksail-operator/patches/enable-oidc.yaml | 2 +- .../install-default-runtime-rules.yaml | 2 +- .../route-runtime-detection-alerts.yaml | 13 +- .../controllers/kustomization.yaml | 13 +- .../coroot/config-map-risk-acceptances.yaml | 28 +- .../coroot/cron-job-alert-autosuppressor.yaml | 29 +- .../cron-job-crossview-login-alerter.yaml | 251 --- .../flux-notifications/alert.yaml | 6 - .../hetzner/infrastructure/kustomization.yaml | 4 - .../headlamp-configmap.yaml.tmpl | 69 - scripts/generate-kubescape-exceptions/main.go | 185 +-- .../main_test.go | 197 +-- .../guard-crossplane-provider-name-parity.sh | 42 +- .../lib/configure-hosted-registry-cache.sh | 48 + .../tests/test-arc-listener-public-content.sh | 6 +- .../tests/test-coroot-alert-autosuppressor.sh | 24 +- .../test-coroot-risk-acceptance-reconciler.sh | 27 +- scripts/tests/test-cronjob-failure-alert.sh | 8 +- ...t-crossview-coroot-postgres-integration.sh | 229 --- .../test-crossview-login-alerter-runtime.sh | 366 ---- scripts/tests/test-crossview-login-alerter.sh | 453 ----- ...test-crossview-postgres-durable-storage.sh | 207 --- ...t-guard-crossplane-provider-name-parity.sh | 87 +- .../tests/test-headlamp-oidc-version-gate.sh | 93 -- scripts/tests/test-headlamp-plugin-removal.sh | 200 --- .../tests/test-human-api-access-boundary.sh | 49 +- .../test-wedding-backup-denial-runtime.sh | 5 +- .../approved-surface-history.md | 35 + .../approved-surface.txt | 20 +- scripts/validate-eks-ci-role-policy/main.go | 2 - .../kyverno-test.yaml | 2 +- .../resources.yaml | 8 +- .../resources.yaml | 4 +- 115 files changed, 312 insertions(+), 5810 deletions(-) delete mode 100644 k8s/bases/apps/crossview/cilium-network-policy.yaml delete mode 100644 k8s/bases/apps/crossview/cluster-role-binding-cluster-reader.yaml delete mode 100644 k8s/bases/apps/crossview/cluster-role-binding-view.yaml delete mode 100644 k8s/bases/apps/crossview/external-secret.yaml delete mode 100644 k8s/bases/apps/crossview/helm-release.yaml delete mode 100644 k8s/bases/apps/crossview/helm-repository.yaml delete mode 100644 k8s/bases/apps/crossview/http-route.yaml delete mode 100644 k8s/bases/apps/crossview/kustomization.yaml delete mode 100644 k8s/bases/apps/crossview/namespace.yaml delete mode 100644 k8s/bases/apps/headlamp/README.md delete mode 100644 k8s/bases/apps/headlamp/cilium-network-policy.yaml delete mode 100644 k8s/bases/apps/headlamp/cluster-role-binding.yaml delete mode 100644 k8s/bases/apps/headlamp/external-secret.yaml delete mode 100644 k8s/bases/apps/headlamp/helm-release.yaml delete mode 100644 k8s/bases/apps/headlamp/helm-repository.yaml delete mode 100644 k8s/bases/apps/headlamp/http-route.yaml delete mode 100644 k8s/bases/apps/headlamp/kustomization.yaml delete mode 100644 k8s/bases/apps/headlamp/namespace.yaml delete mode 100644 k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml delete mode 100644 k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-admin-password.yaml delete mode 100644 k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-db-password.yaml delete mode 100644 k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-session-secret.yaml delete mode 100644 k8s/bases/infrastructure/vault-seed/password-crossview-admin-password.yaml delete mode 100644 k8s/bases/infrastructure/vault-seed/password-crossview-db-password.yaml delete mode 100644 k8s/bases/infrastructure/vault-seed/password-crossview-session-secret.yaml delete mode 100644 k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-admin-password.yaml delete mode 100644 k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-db-password.yaml delete mode 100644 k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-session-secret.yaml delete mode 100644 k8s/providers/docker/apps/headlamp/patches/trust-platform-ca.yaml delete mode 100644 k8s/providers/hetzner/apps/crossview/cilium-network-policy-login-alerter.yaml delete mode 100644 k8s/providers/hetzner/apps/crossview/cilium-network-policy.yaml delete mode 100644 k8s/providers/hetzner/apps/crossview/config-map-coroot-monitor-init.yaml delete mode 100644 k8s/providers/hetzner/apps/crossview/external-secret-coroot-monitor.yaml delete mode 100644 k8s/providers/hetzner/apps/crossview/password-coroot-monitor.yaml delete mode 100644 k8s/providers/hetzner/apps/crossview/patches/enable-coroot-monitoring.yaml delete mode 100644 k8s/providers/hetzner/apps/crossview/patches/enable-durable-storage.yaml delete mode 100644 k8s/providers/hetzner/apps/crossview/patches/enable-user-namespaces.yaml delete mode 100644 k8s/providers/hetzner/apps/headlamp/patches/enable-user-namespaces.yaml delete mode 100644 k8s/providers/hetzner/infrastructure/coroot/cron-job-crossview-login-alerter.yaml delete mode 100644 scripts/generate-kubescape-exceptions/headlamp-configmap.yaml.tmpl create mode 100755 scripts/tests/lib/configure-hosted-registry-cache.sh delete mode 100755 scripts/tests/test-crossview-coroot-postgres-integration.sh delete mode 100755 scripts/tests/test-crossview-login-alerter-runtime.sh delete mode 100755 scripts/tests/test-crossview-login-alerter.sh delete mode 100755 scripts/tests/test-crossview-postgres-durable-storage.sh delete mode 100755 scripts/tests/test-headlamp-oidc-version-gate.sh delete mode 100755 scripts/tests/test-headlamp-plugin-removal.sh diff --git a/.github/renovate.json b/.github/renovate.json index baf9f6bd83..644f1a68b1 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -131,18 +131,7 @@ }, { "customType": "regex", - "description": "Track the bundled Postgres image tag pinned for the Crossview app's chart-bundled database (k8s/bases/apps/crossview/helm-release.yaml). The crossview chart defaults database.image.tag to `latest`; pinning it lives in the Flux HelmRelease's inline spec.values, which the built-in flux manager (chart version only) and the kubernetes/helm-values managers (values.yaml files only) never see — so a custom manager is the only way to keep the pin current. Matches the `# renovate:` comment and the tag line that follows it, resolving from the docker datasource (Docker Hub library/postgres).", - "managerFilePatterns": [ - "/^k8s/bases/apps/crossview/helm-release\\.ya?ml$/" - ], - "matchStrings": [ - "# renovate: datasource=(?\\S+) depName=(?\\S+)\\s+tag:\\s*\"(?[^\"]+)\"" - ], - "versioningTemplate": "docker" - }, - { - "customType": "regex", - "description": "Track the PostgreSQL operand image pinned on the platform-owned CloudNativePG Clusters (coroot-db, umami-db — each a cluster.yaml). These pin spec.imageName explicitly instead of riding the CNPG operator default because the latest GA operator (v1.29.1) still defaults to postgresql:18.3-system-trixie — a now-frozen tag (last rebuilt 2026-05-11) carrying fixable base-OS CVEs (openssl/libssl3, libgnutls30, libc6, python3.13) — and the default only advances to 18.4 in the not-yet-GA v1.30, so the built-in flux manager bumping the cloudnative-pg operator chart does NOT clear them. spec.imageName lives in the Cluster CR (not a chart version or a values.yaml), so the flux, kubernetes, and helm-values managers never see it; a custom manager is the only way to keep it current. Matches the `# renovate:` comment and the imageName line that follows it (mirroring the crossview manager above), resolving from the docker datasource (ghcr.io/cloudnative-pg/postgresql); docker versioning tracks the 18.4-system-trixie -> 18.5-system-trixie minor line while preserving the -system-trixie flavor suffix. The tenant-owned wedding-db pins its own imageName in the external wedding-app artifact and is intentionally not covered here. NOTE (platform#2149): this manager's file pattern originally targeted a `postgres-cluster.yaml` filename that has never existed in this repo — the actual files are `cluster.yaml` — so the manager silently matched nothing since it was authored; fixed to the real filename.", + "description": "Track the PostgreSQL operand image pinned on the platform-owned CloudNativePG Clusters (coroot-db, umami-db — each a cluster.yaml). These pin spec.imageName explicitly instead of riding the CNPG operator default because the latest GA operator (v1.29.1) still defaults to postgresql:18.3-system-trixie — a now-frozen tag (last rebuilt 2026-05-11) carrying fixable base-OS CVEs (openssl/libssl3, libgnutls30, libc6, python3.13) — and the default only advances to 18.4 in the not-yet-GA v1.30, so the built-in flux manager bumping the cloudnative-pg operator chart does NOT clear them. spec.imageName lives in the Cluster CR (not a chart version or a values.yaml), so the flux, kubernetes, and helm-values managers never see it; a custom manager is the only way to keep it current. Matches the `# renovate:` comment and the imageName line that follows it, resolving from the docker datasource (ghcr.io/cloudnative-pg/postgresql); docker versioning tracks the 18.4-system-trixie -> 18.5-system-trixie minor line while preserving the -system-trixie flavor suffix. The tenant-owned wedding-db pins its own imageName in the external wedding-app artifact and is intentionally not covered here. NOTE (platform#2149): this manager's file pattern originally targeted a `postgres-cluster.yaml` filename that has never existed in this repo — the actual files are `cluster.yaml` — so the manager silently matched nothing since it was authored; fixed to the real filename.", "managerFilePatterns": ["/(^|/)cluster\\.ya?ml$/"], "matchStrings": [ "# renovate: datasource=(?\\S+) depName=(?\\S+)\\s+imageName:\\s*\\S+:(?\\S+)" @@ -322,13 +311,6 @@ "allowedVersions": "/^v?1\\.14\\.0$/", "automerge": false }, - { - "description": "headlamp is HARD-PINNED at 0.42.0 (k8s/bases/apps/headlamp/helm-release.yaml): 0.43.0 introduced an in-cluster OIDC login regression in Safari, and the same broken popup handoff was reproduced after upgrading to 0.44.0 — the session cookie succeeds, but Safari ITP suppresses the cross-window localStorage storage event after the popup's cross-site Dex/GitHub hops, so the popup never redirects its opener (upstream kubernetes-sigs/headlamp#6353; fix PR #6355 unmerged; tracked in #2438). Keep the last Safari-verified release until a released upstream fix is manually verified end-to-end; automerge:false placed last so an unverified Headlamp update cannot inherit the minor/patch automerge defaults above.", - "matchDatasources": ["helm"], - "matchPackageNames": ["headlamp"], - "allowedVersions": "/^v?0\\.42\\.0$/", - "automerge": false - }, { "description": "KubeVirt and CDI ship as VENDORED upstream operator manifests (k8s/bases/infrastructure/controllers/kubevirt/kubevirt-operator.yaml, .../cdi/cdi-operator.yaml) — thousands of lines of CRDs, RBAC and the operator Deployment vendored as one unit from each release's published asset. The kubernetes manager only sees the `image:` lines inside them, so a Renovate bump rewrites the operator image alone and silently skews it against the CRDs/RBAC vendored next to it; the system-test would still pass bring-up (operators tolerate startup skew) so automerge would land the skew unseen. Keep the update PR as the new-release signal but require a maintainer to re-vendor the full manifest from the matching release before merging. automerge:false placed last so it overrides the major/minor/patch automerge defaults above.", "matchPackageNames": ["quay.io/kubevirt/**"], diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fff58b5e45..88abb5efa8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -115,7 +115,6 @@ jobs: k8s: ${{ steps.filter.outputs.k8s }} bridge_validation: ${{ steps.filter.outputs.bridge_validation }} talos: ${{ steps.filter.outputs.talos }} - crossview: ${{ steps.filter.outputs.crossview }} queue_receipt: ${{ steps.queue-receipt.outputs.receipt }} steps: - name: 📑 Checkout @@ -228,7 +227,6 @@ jobs: - 'tests/validate-pod-security-provider-deployment/**' - 'scripts/tests/test-pvc-prune-safety.sh' - 'scripts/tests/test-flux-kustomization-prune.sh' - - 'scripts/tests/test-headlamp-plugin-removal.sh' - 'scripts/tests/test-github-config-role-activation-parity.sh' - 'scripts/tests/test-restrict-tenant-secret-stores.sh' - 'scripts/tests/test-tenant-kro-pull-secret.sh' @@ -456,10 +454,6 @@ jobs: - 'scripts/tests/test-cilium-mutual-auth-policy.sh' - 'scripts/tests/test-cilium-mutual-auth-policy-regressions.sh' - 'scripts/tests/test-coroot-postgres-scrape-policy.sh' - - 'scripts/tests/test-crossview-coroot-postgres-integration.sh' - - 'scripts/tests/test-crossview-postgres-durable-storage.sh' - - 'scripts/tests/test-crossview-login-alerter.sh' - - 'scripts/tests/test-crossview-login-alerter-runtime.sh' - 'scripts/tests/test-coroot-api-key-contract.sh' - 'scripts/tests/test-coroot-availability-remediation.sh' - 'scripts/tests/test-descheduler-availability.sh' @@ -743,21 +737,6 @@ jobs: - 'go.sum' - '.github/workflows/ci.yaml' - '.github/workflows/cd.yaml' - # What the Docker reproduction of the Crossview login outage reads: - # the chart pin and values, the prod apps overlay and the shared - # components it applies to every HelmRelease, the sensor and its test. - crossview: - - 'k8s/bases/apps/crossview/**' - - 'k8s/bases/apps/kustomization.yaml' - - 'k8s/bases/components/annotations-transformers/**' - - 'k8s/bases/components/helmrelease-drift-detection/**' - - 'k8s/bases/components/helmrelease-flux-defaults/**' - - 'k8s/providers/hetzner/apps/crossview/**' - - 'k8s/providers/hetzner/apps/kustomization.yaml' - - 'k8s/providers/hetzner/infrastructure/coroot/cron-job-crossview-login-alerter.yaml' - - 'k8s/clusters/prod/bootstrap/config-map.yaml' - - 'scripts/tests/test-crossview-login-alerter-runtime.sh' - - '.github/workflows/ci.yaml' # The full offline fixture suite runs alongside changes, which retains the real production # render and ShellCheck. Both deployment and the required gate wait for these regressions. @@ -1008,7 +987,7 @@ jobs: run: | shellcheck scripts/verify-wedding-backup-denial.sh scripts/verify-wedding-backup-denial-pod.sh scripts/tests/test-verify-wedding-backup-denial.sh bash scripts/tests/test-verify-wedding-backup-denial.sh - shellcheck scripts/tests/test-wedding-backup-denial-runtime.sh + shellcheck scripts/tests/test-wedding-backup-denial-runtime.sh scripts/tests/lib/configure-hosted-registry-cache.sh bash scripts/tests/test-wedding-backup-denial-runtime.sh # The stale shared copy of the Wedding catalogue is removed on this proof's @@ -1272,16 +1251,10 @@ jobs: go test ./scripts/validate-concurrency-queue go run ./scripts/validate-concurrency-queue "${workflows[@]}" - # This guard spans the HelmRelease and Renovate policy, so it must run - # without routing policy-only changes into the production k8s deploy path. - - name: 🔐 Validate Headlamp OIDC version gate - run: | - shellcheck scripts/tests/test-headlamp-oidc-version-gate.sh - bash scripts/tests/test-headlamp-oidc-version-gate.sh - # The renderer's Helm SDK follows the deployed controller, not Helm's own - # releases. This spans the build guard and the Renovate policy, so it runs - # here for the same reason as the Headlamp gate above (#4738). + # releases. This spans the build guard and the Renovate policy, so it must + # run without routing policy-only changes into the production k8s deploy + # path (#4738). - name: 📌 Validate the controller Helm SDK Renovate pin run: | shellcheck scripts/tests/test-controller-helm-sdk-renovate-pin.sh @@ -2018,10 +1991,9 @@ jobs: - name: 🧩 Validate embedded JSON blobs # ConfigMaps that embed whole JSON documents as block scalars (e.g. the - # Headlamp Kubescape exceptions ConfigMap's exceptionPolicies) are opaque - # strings to schema validation, so a stray comma ships silently and only - # surfaces at consumption time — where an empty exceptions view reads - # identical to a clean posture. Local Go tests and validation need no + # Coroot dashboards) are opaque strings to schema validation, so a stray + # comma ships silently and only surfaces at consumption time. Local Go + # tests and validation need no # cluster or credentials. See scripts/validate-embedded-json/ (#2480). run: | shellcheck scripts/validate-embedded-json.sh @@ -2162,12 +2134,6 @@ jobs: shellcheck scripts/tests/test-world-zone-trial.sh bash scripts/tests/test-world-zone-trial.sh - - name: 🧩 Validate Headlamp plugin removal - if: needs.changes.outputs.k8s == 'true' - # Removing runtime-downloaded plugins must also prune upgrade state and - # preserve Crossview's authenticated fresh-install entry point. - run: bash scripts/tests/test-headlamp-plugin-removal.sh - - name: 🔑 Validate github-config Role covers the activated MR kinds if: needs.changes.outputs.k8s == 'true' # The tenant Role enumerates managed-resource kinds instead of `*`, so @@ -2375,35 +2341,6 @@ jobs: run: | shellcheck scripts/tests/test-coroot-postgres-scrape-policy.sh bash scripts/tests/test-coroot-postgres-scrape-policy.sh - shellcheck scripts/tests/test-crossview-coroot-postgres-integration.sh - bash scripts/tests/test-crossview-coroot-postgres-integration.sh - - - name: 💾 Validate Crossview PostgreSQL durable storage - if: needs.changes.outputs.k8s == 'true' - # Render the pinned chart through Flux's production post-renderers so - # persistence cannot regress to an emptyDir or an overlapping rollout. - run: | - shellcheck scripts/tests/test-crossview-postgres-durable-storage.sh - bash scripts/tests/test-crossview-postgres-durable-storage.sh - - - name: 🔓 Validate the Crossview login alert - if: needs.changes.outputs.k8s == 'true' - # Crossview's only probe never touches its database, so an empty schema - # broke every sign-in for six days with the pod Ready (#3315). This pins - # the sensor's verdicts and wiring against stubbed answers. - run: | - shellcheck scripts/tests/test-crossview-login-alerter.sh scripts/tests/test-crossview-login-alerter-runtime.sh - bash scripts/tests/test-crossview-login-alerter.sh - - - name: 🔓 Reproduce the Crossview login outage - if: needs.changes.outputs.crossview == 'true' - # Measured at about 30s; the bound only stops a hung pull holding the job. - timeout-minutes: 15 - # Runs the pinned app and database in Docker, empties the database - # under the running app and runs the deployed sensor against it, so a - # chart bump that changes what /api/auth/check reports fails here. Only - # Crossview changes pay for the image pulls. - run: bash scripts/tests/test-crossview-login-alerter-runtime.sh - name: 🔑 Validate Coroot API key and registry egress if: needs.changes.outputs.k8s == 'true' @@ -3093,8 +3030,7 @@ jobs: # enforced and except-only findings suppressed, the threshold gates the # residual REAL posture instead of sitting ~15 points under it. # - # Two surfaces apply these exceptions: this scan, and the Headlamp view - # via the generated headlamp-exceptions ConfigMap. The kubescape-operator's + # Only this scan applies these exceptions. The kubescape-operator's # stored scan resources (workloadconfigurationscans and the two summary # kinds) do NOT — their appliedIgnoreRules is empty, so they carry # pre-exception verdicts. Never read a posture number off those to judge diff --git a/.github/workflows/dr-rebuild.yaml b/.github/workflows/dr-rebuild.yaml index bfb39f6483..e4ef66884c 100644 --- a/.github/workflows/dr-rebuild.yaml +++ b/.github/workflows/dr-rebuild.yaml @@ -19,7 +19,7 @@ # without it the step prints the manual Scenario 9 instructions) # # Known limits (documented in the runbook): -# * Per-app PVC data (headlamp, actual-budget) is NOT rehydrated into +# * Per-app PVC data (actual-budget) is NOT rehydrated into # already-running pods — Velero skips existing resources. Recover an app's # data with the per-app reset dance (runbook Scenario 5). # * CNPG databases (umami-db) recover from their own barman backups — @@ -636,6 +636,6 @@ jobs: echo "- Load balancer IP: ${IP:-}" echo "- DNS: external-dns (policy: sync) repoints the Cloudflare records automatically once HTTPRoutes are Ready — verify with: kubectl -n external-dns logs deploy/external-dns" echo "- CNPG databases (umami-db): recover from barman/R2 per docs/dr/runbook.md Scenario 5 (kubectl cnpg restore)." - echo "- Per-app PVC data (headlamp, actual-budget): already-running pods keep their fresh volumes; use the per-app reset dance from runbook Scenario 5 if old data is needed." + echo "- Per-app PVC data (actual-budget): already-running pods keep their fresh volumes; use the per-app reset dance from runbook Scenario 5 if old data is needed." kubectl get kustomizations.kustomize.toolkit.fluxcd.io -A -o wide kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded 2>/dev/null | head -30 diff --git a/AGENTS.md b/AGENTS.md index 8154f855aa..48ac588c75 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -149,7 +149,7 @@ The scan is a **hard gate**: it fails the PR if the combined compliance score dr - **ksail is Renovate-managed** (the Setup step, grouped `ksail` with the deploy pins). It was previously frozen at 7.65.0 because 7.66.x parallelised the in-process Helm render and made it racy — two distinct symptoms of the same regression: `ksail workload validate` non-deterministically corrupted the render with varying YAML parse errors ([devantler-tech/ksail#5362](https://github.com/devantler-tech/ksail/issues/5362), closed — contained since KSail v7.163.1 by the [ksail#5978](https://github.com/devantler-tech/ksail/issues/5978) stream-splitting fix, which is what let the temporary `--skip-helm-render` workaround be removed), and the scan's compliance score swung run-to-run ([devantler-tech/ksail#5371](https://github.com/devantler-tech/ksail/issues/5371), closed). Both are resolved upstream, so the pin is lifted back onto the latest release. Tripwire (kept in sync with the comments in `.github/workflows/ci.yaml`): if `validate` output or the `scan` score varies run-to-run again, re-add `--skip-helm-render` and reopen ksail#5362 (or re-pin to a known-good version, reopening #5371 if only the score swings). - **52 of the 76 excepted controls are in neither framework, so the gate still cannot see them.** NSA-CISA covers 17 of the 76; adding MITRE ATT&CK brings 7 more under the gate — C-0007, C-0015, C-0031, C-0037, C-0045, C-0048, C-0053 (measured 2026-08-10, ksail 7.178.20). The residue is tracked on [#2823](https://github.com/devantler-tech/platform/issues/2823). Two consequences: an exception naming a control outside both frameworks is inert **here** whether or not it is correct, so never read "the score did not move" as "the exception is broken"; and a finding on such a control reaches neither the gate nor Code Scanning. -- **The threshold is a regression floor, not the actual score — and the scan runs WITH the platform's justified exceptions applied.** The `ClusterSecurityException` CRs (`k8s/bases/infrastructure/cluster-security-exceptions/`) are the single source of truth, and are converted at scan time into Kubescape's native exceptions format by [`scripts/generate-kubescape-exceptions`](scripts/generate-kubescape-exceptions) (fail-closed: an unrecognised CR shape aborts the scan step rather than silently dropping or widening an exception; Go unit tests alongside it), so runtime-enforced (Kyverno mutation, `CiliumNetworkPolicy`) and except-only findings (e.g. **C-0002**, the KubeVirt operator's `pods/exec` RBAC) no longer depress the score and the floor gates the residual REAL posture (#2264). The score has historically been **environment-dependent** (Linux CI runner vs macOS — a gap that is *not* the render mode, the framework cache, or PR-merge content, all ruled out) and shifts with the ksail render, so **CI is the source of truth** (re-baseline the floor after a ksail bump, from the combined and per-framework scores the scan step prints in its log and job summary); the observed CI reference with exceptions applied is **≈98.9%** (2026-07-11, ksail 7.165.2), with the floor a few points under it. Only **two** surfaces apply these exceptions: this scan, and the Headlamp view via the generated `headlamp-exceptions` ConfigMap. The kubescape-operator's stored scan resources (`workloadconfigurationscans` and the two summary kinds) do **not** — their `appliedIgnoreRules` is empty — so a posture number read off those says nothing about whether an exception works. They also outlive the objects they describe for every non-workload kind, so a stored result is not proof that its object still exists either; `scripts/report-kubescape-scan-orphans.sh` classifies them against the live cluster (#3697, [`docs/kubescape-result-coverage.md`](docs/kubescape-result-coverage.md)). A new justified exception is added as a CSE CR (kind+name-scoped, minimal — see the existing CRs' conventions), never by lowering the floor: **ratchet up** as genuine gaps close; never lower it. +- **The threshold is a regression floor, not the actual score — and the scan runs WITH the platform's justified exceptions applied.** The `ClusterSecurityException` CRs (`k8s/bases/infrastructure/cluster-security-exceptions/`) are the single source of truth, and are converted at scan time into Kubescape's native exceptions format by [`scripts/generate-kubescape-exceptions`](scripts/generate-kubescape-exceptions) (fail-closed: an unrecognised CR shape aborts the scan step rather than silently dropping or widening an exception; Go unit tests alongside it), so runtime-enforced (Kyverno mutation, `CiliumNetworkPolicy`) and except-only findings (e.g. **C-0002**, the KubeVirt operator's `pods/exec` RBAC) no longer depress the score and the floor gates the residual REAL posture (#2264). The score has historically been **environment-dependent** (Linux CI runner vs macOS — a gap that is *not* the render mode, the framework cache, or PR-merge content, all ruled out) and shifts with the ksail render, so **CI is the source of truth** (re-baseline the floor after a ksail bump, from the combined and per-framework scores the scan step prints in its log and job summary); the observed CI reference with exceptions applied is **≈98.9%** (2026-07-11, ksail 7.165.2), with the floor a few points under it. Only this scan applies these exceptions. The kubescape-operator's stored scan resources (`workloadconfigurationscans` and the two summary kinds) do **not** — their `appliedIgnoreRules` is empty — so a posture number read off those says nothing about whether an exception works. They also outlive the objects they describe for every non-workload kind, so a stored result is not proof that its object still exists either; `scripts/report-kubescape-scan-orphans.sh` classifies them against the live cluster (#3697, [`docs/kubescape-result-coverage.md`](docs/kubescape-result-coverage.md)). A new justified exception is added as a CSE CR (kind+name-scoped, minimal — see the existing CRs' conventions), never by lowering the floor: **ratchet up** as genuine gaps close; never lower it. Repository-wide shell and source guards run unconditionally in `validate-repository-guards` on both pull requests and merge groups. This job @@ -290,7 +290,7 @@ ksail workload reconcile The local cluster is a **thin manual test-bed** — a small Talos cluster for trying a component out before promoting it to prod, not a full prod stand-in. By default it brings up only the **core infrastructure** (Cilium + Gateway API, CoreDNS, cert-manager/trust-manager, Flux, metrics-server, Kyverno + cluster-policies, VPA, OpenBao + External Secrets, the Dex/oauth2-proxy/auth-proxy SSO stack, and CloudNativePG) — enough for a working, reachable cluster with prod-like admission, secrets and SSO. Core infrastructure UIs are reachable via the `hosts` file's `*.platform.lan` → `127.0.0.1` mappings (e.g. `dex.platform.lan`, `flux.platform.lan`). -Heavier/optional infrastructure (observability, progressive delivery, autoscaling, backup/Velero + MinIO, runtime security, the VM stack, …) is **opt-in**: uncomment the controller you want — plus its `infrastructure`-layer resources and patch where noted — in the docker provider overlays (`k8s/providers/docker/infrastructure/controllers/kustomization.yaml` and `…/infrastructure/kustomization.yaml`), which carry copy-paste templates. **Apps** are opt-in the same way: replace `resources: []` in the docker apps overlay (`k8s/providers/docker/apps/kustomization.yaml`) with the entries you want — its comments carry a copy-paste template, including the `patches:` block needed for `actual-budget`/`headlamp`. After any change, re-run `ksail workload push` + `ksail workload reconcile`. Only then do the opt-in routes respond — the apex `https://platform.lan` (served by the homepage app) and per-app subdomains such as `headlamp.platform.lan` or `whoami.platform.lan`. +Heavier/optional infrastructure (observability, progressive delivery, autoscaling, backup/Velero + MinIO, runtime security, the VM stack, …) is **opt-in**: uncomment the controller you want — plus its `infrastructure`-layer resources and patch where noted — in the docker provider overlays (`k8s/providers/docker/infrastructure/controllers/kustomization.yaml` and `…/infrastructure/kustomization.yaml`), which carry copy-paste templates. **Apps** are opt-in the same way: replace `resources: []` in the docker apps overlay (`k8s/providers/docker/apps/kustomization.yaml`) with the entries you want — its comments carry a copy-paste template, including the `patches:` block needed for `actual-budget`. After any change, re-run `ksail workload push` + `ksail workload reconcile`. Only then do the opt-in routes respond — the apex `https://platform.lan` (served by the homepage app) and per-app subdomains such as `headlamp.platform.lan` or `whoami.platform.lan`. **Cleanup:** diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md index 729df6107b..6f1e76dd6a 100644 --- a/docs/TEMPLATING.md +++ b/docs/TEMPLATING.md @@ -93,7 +93,7 @@ See `.github/workflows/` for the exact names. - `k8s/bases/infrastructure/` — Cilium, cert-manager, Kyverno, alerting configs, OpenBao vault, External Secrets Operator, ClusterSecretStore, vault-config Job, vault-seed PushSecrets, vault-backup CronJob. -- `k8s/bases/apps/` — reference applications (homepage, whoami, headlamp). +- `k8s/bases/apps/` — reference applications (homepage, whoami). - `k8s/providers/{docker,hetzner}/` — provider-specific assembly of bases. Changes here are "platform changes" — upstream them instead of forking them. diff --git a/docs/architecture.md b/docs/architecture.md index b177c60468..8d93933704 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -66,8 +66,8 @@ ships only core controllers by default and makes additional components opt-in. Application manifests live in [`k8s/bases/apps/`](../k8s/bases/apps). -- Homepage (dashboard), Headlamp (Kubernetes UI), Umami (analytics), Actual - Budget (budgeting), `whoami` (debug) +- Homepage (dashboard), Umami (analytics), Actual Budget (budgeting), `whoami` + (debug) - The [apps assembly](../k8s/bases/apps/kustomization.yaml) declares which optional applications are enabled. - **Tenants** — apps deployed from their own repositories (`ascoachingogvaner`, diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md index fcf43e55d6..5df619ebb8 100644 --- a/docs/dr/alerting.md +++ b/docs/dr/alerting.md @@ -148,23 +148,6 @@ stays quiet by design, exactly as the old Alertmanager did. `VolumeAttachment`, because the strand lives in the cloud provider while the Kubernetes objects can look consistent; it is read-only, and force-detach stays a manual step (#2754). -- **A Crossview login that cannot work alerts.** Crossview's sign-in was broken - for six days in August 2026 while every probe stayed green: its database came - back empty, and the chart's only probe (`/api/health`) never touches the - database (#3315). - `providers/hetzner/infrastructure/coroot/cron-job-crossview-login-alerter.yaml` - reads the unauthenticated `/api/auth/check` every 15 minutes. A bootstrapped - Crossview always has the admin it creates at startup, so four reads two - minutes apart that all answer `"hasAdmin":false` and `"hasUsers":false` post - `CrossviewLoginBroken` through the Alertmanager to Slack. One read with - either `true` ends the run healthy, and a - run that never gets a clear answer fails, which the CronJob failure detector - above reports. The same answer comes back when the app cannot reach its - database, so the alert's runbook checks the database and the app's database - errors first: a Ready database alone does not prove schema loss. A database - that returns with its data needs nothing else, while confirmed schema loss - needs the app rolled so it re-runs its bootstrap, and the runbook names the - annotation to bump. - **kube-apiserver audit logs are searchable in Coroot again.** Coroot's node-agent ingests container logs/traces, not host audit-log files, so the previous alloy-audit → Loki pipeline was removed with the migration. The @@ -180,42 +163,22 @@ stays quiet by design, exactly as the old Alertmanager did. ## Kubescape runtime-detection alerts The Kubescape node-agent's **runtime-detection** alerts (rule violations, -malware) are the one signal that does **not** fit the Coroot model, because their -only first-class dashboard — the **Headlamp Kubescape plugin's "Runtime -Detection → Alerts" tab** — reads exclusively from a Prometheus **Alertmanager** -(`GET /api/v2/alerts`, filtered on `alertname="KubescapeRuleViolated"`). It -cannot read Kubescape's storage CRs and cannot query Coroot's Prometheus (a -metrics store, not an Alertmanager). So a **single, minimal Alertmanager** is -reintroduced *scoped to Kubescape* — not a re-adoption of the Prometheus stack. -It lives in the `kubescape` namespace, prod-only +malware) are the one signal that does **not** fit the Coroot model, because the +node-agent can push them only to a Prometheus **Alertmanager**. It cannot write +to Coroot's Prometheus (a metrics store, not an Alertmanager). So a **single, +minimal Alertmanager** is reintroduced *scoped to Kubescape* — not a re-adoption +of the Prometheus stack. It lives in the `kubescape` namespace, prod-only (`providers/hetzner/infrastructure/controllers/alertmanager/`), ~10m CPU / 32Mi RAM, ephemeral (no PVC). -The node-agent fans each alert out to **all three destinations** (wired in +The node-agent fans each alert out to **both destinations** (wired in `providers/hetzner/infrastructure/controllers/kubescape/patches/`): | Destination | Path | | --- | --- | -| **Headlamp plugin** | `nodeAgent.config.alertManagerExporterUrls` → the Alertmanager, which the plugin queries. | -| **Slack** | the Alertmanager's `slack_configs` receiver → the shared `${alertmanager_webhook_url}` incoming-webhook (same channel as Coroot/Flux). | +| **Slack** | `nodeAgent.config.alertManagerExporterUrls` → the Alertmanager's `slack_configs` receiver → the shared `${alertmanager_webhook_url}` incoming-webhook (same channel as Coroot/Flux). | | **Coroot** | `nodeAgent.config.stdoutExporter` (on by default) → Coroot's eBPF log capture surfaces the alert in the **Logs** view (Coroot CE has no inbound alert receiver). | -**One manual step (Headlamp).** The plugin's Alertmanager address is a -**per-user, per-browser** setting (stored in `localStorage`; there is no -declarative/Helm way to seed it — [headlamp#3979](https://github.com/kubernetes-sigs/headlamp/issues/3979)). -Each operator sets it **once** in Headlamp → *Settings → Plugins → Kubescape*, -in the `namespace/service:port` form the plugin validates: - -```text -kubescape/alertmanager:9093 -``` - -The plugin reaches it through the Kubernetes API server's Service proxy, so the -logged-in user needs `get`/`create` on `services/proxy` in the `kubescape` -namespace (satisfied by the admin binding). Until it is set, the tab shows -"Alertmanager URL is not configured" — the data source now exists, only the -per-user pointer is manual. - ## Dead-man's-switch (off-cluster heartbeat) In-cluster alerting cannot tell you the cluster is down — it's down too. A tiny diff --git a/docs/dr/cnpg-storageclass-migration.md b/docs/dr/cnpg-storageclass-migration.md index 59e0a0293a..f7fc58eaae 100644 --- a/docs/dr/cnpg-storageclass-migration.md +++ b/docs/dr/cnpg-storageclass-migration.md @@ -112,7 +112,7 @@ backup from the prerequisites is the last-resort recovery path (`restore-drill.m flow above is the plugin-free equivalent of `kubectl cnpg destroy`. - Longhorn volumes are replicated across storage nodes and are **not** Hetzner servers, so this migration does not consume hcloud server-quota slots. -- Non-CNPG longhorn PVCs (`kubescape-storage`, `headlamp`, `actual-budget`) can be migrated with +- Non-CNPG longhorn PVCs (`kubescape-storage`, `actual-budget`) can be migrated with the same delete-PVC-after-backup pattern **only** where the workload can tolerate recreation from a backup — they have no built-in replica to re-clone from, so back up first and treat each individually. diff --git a/docs/dr/velero-cnpg.md b/docs/dr/velero-cnpg.md index a8a8aa1188..960f7048bf 100644 --- a/docs/dr/velero-cnpg.md +++ b/docs/dr/velero-cnpg.md @@ -153,7 +153,7 @@ CSI snapshots need cluster-wide plumbing that the hetzner overlay adds: > ×`longhorn_replica_count`) and a node-agent pod copies it to R2, then deletes > both. On the space-constrained cx23 workers this transiently consumes up to > the source volume's size × replica count per PVC during the 02:17 run — fine -> for the current set (umami 5Gi, actual-budget 10Gi, headlamp 256Mi) but worth +> for the current set (umami 5Gi, actual-budget 10Gi) but worth > watching if larger Longhorn volumes are added. ## CloudNativePG diff --git a/docs/progressive-delivery.md b/docs/progressive-delivery.md index 6a9dab5681..643bb86ea3 100644 --- a/docs/progressive-delivery.md +++ b/docs/progressive-delivery.md @@ -69,7 +69,7 @@ Promotion vs rollback is gated on: | Workload | Reason | | --- | --- | -| whoami, headlamp, actual-budget, fleetdm (parked), hubble-ui | Single-replica by design (always-on; auto-vpa right-sizes them). headlamp = single-pod in-memory OIDC; actual-budget = single-writer file DB; hubble-ui/whoami = single-replica UIs — none can run concurrent canary pods. fleetdm is disabled since 2026-06-03. | +| whoami, actual-budget, fleetdm (parked), hubble-ui | Single-replica by design (always-on; auto-vpa right-sizes them). actual-budget = single-writer file DB; hubble-ui/whoami = single-replica UIs — none can run concurrent canary pods. fleetdm is disabled since 2026-06-03. | | openbao | StatefulSet — Flagger only manages Deployments / DaemonSets. | | coroot UI, hubble-ui | Operator-reconciled Deployments — the operator fights Flagger for ownership. | | dex, oauth2-proxy, flux-operator | Critical SSO / GitOps — too risky to canary. | diff --git a/docs/runtime-security.md b/docs/runtime-security.md index 8df01bc3a5..d743d3d215 100644 --- a/docs/runtime-security.md +++ b/docs/runtime-security.md @@ -79,7 +79,7 @@ only when `alertCRD.installDefault` is true. Production declares that switch, [`install-default-runtime-rules.yaml`](../k8s/providers/hetzner/infrastructure/controllers/kubescape/patches/install-default-runtime-rules.yaml), so the agent keeps building profiles but raises no alerts: an empty alert route means nothing was checked. Turning the switch on adds exactly those two objects -and sends alerts through the existing Alertmanager (Slack, Headlamp) and stdout +and sends alerts through the existing Alertmanager (Slack) and stdout (Coroot) routes. `scripts/tests/test-kubescape-runtime-rules-switch.sh` renders the pinned chart in both states and refuses an undeclared switch, an on state that binds no enabled rule, and an on state that changes anything else (#4968). diff --git a/docs/secret-rotation.md b/docs/secret-rotation.md index 22c3893401..71a804957f 100644 --- a/docs/secret-rotation.md +++ b/docs/secret-rotation.md @@ -24,7 +24,7 @@ reviewed PR. - **Two sourcing paths coexist (mid-migration):** - **OpenBao → ESO**: generators (the `password-*` files in `k8s/bases/infrastructure/vault-seed/`) seed OpenBao KV once; `ExternalSecret`s sync to consumer namespaces (1h refresh). Used by - fleetdm DB/redis/license, headlamp/actual-budget OIDC, cloudflare token, R2. + fleetdm DB/redis/license, actual-budget OIDC, cloudflare token, R2. - **SOPS → Flux postBuild substitution**: `${dex_client_secret}`, `${flux_web_client_secret}`, `${oauth2_proxy_cookie_secret}` etc. are still read from `k8s/clusters/*/bootstrap/secret.enc.yaml`. **Dex diff --git a/hosts b/hosts index 61ddf9e8cb..94fc7930c5 100644 --- a/hosts +++ b/hosts @@ -1,10 +1,8 @@ 127.0.0.1 ascoachingogvaner.platform.lan 127.0.0.1 budget.platform.lan -127.0.0.1 crossview.platform.lan 127.0.0.1 dex.platform.lan 127.0.0.1 fleetdm.platform.lan 127.0.0.1 flux.platform.lan -127.0.0.1 headlamp.platform.lan 127.0.0.1 oauth2-proxy.platform.lan 127.0.0.1 platform.lan 127.0.0.1 wedding.platform.lan diff --git a/k8s/bases/apps/README.md b/k8s/bases/apps/README.md index f7555a9c3d..228ce2575b 100644 --- a/k8s/bases/apps/README.md +++ b/k8s/bases/apps/README.md @@ -11,11 +11,9 @@ A folder that it does not list is not included by this base. | ----------------------------------------------------- | ------------------------------------------------------------------------------------------- | | [`actual-budget/`](actual-budget) | Active. Can also be switched on locally. | | [`ascoachingogvaner/`](ascoachingogvaner) | Active in production only. Tenant: its manifests ship from its own repository. | -| [`crossview/`](crossview) | Active in production only; it reads Crossplane resources. | | [`data-product-controller/`](data-product-controller) | Active in production only. | | [`fleetdm/`](fleetdm) | Retained, not deployed since 3 June 2026. See its [README](fleetdm/README.md). | | [`github-config/`](github-config) | Active in production only. See [GitHub management](../../../docs/github-management.md). | -| [`headlamp/`](headlamp) | Active. Can also be switched on locally. | | [`homepage/`](homepage) | Active. Can also be switched on locally. | | [`umami/`](umami) | Active in production only; its database needs production storage. | | [`wedding-app/`](wedding-app) | Active in production only. Tenant. | @@ -25,7 +23,7 @@ The production overlay ([`providers/hetzner/apps`](../../providers/hetzner/apps)) includes this whole base and adds production-only apps. The local overlay ([`providers/docker/apps`](../../providers/docker/apps)) deploys no apps by -default; its `kustomization.yaml` carries the lines to copy for the four apps +default; its `kustomization.yaml` carries the lines to copy for the three apps that can run locally. To host a new application, follow [tenant onboarding](../../../docs/TENANTS.md). diff --git a/k8s/bases/apps/crossview/cilium-network-policy.yaml b/k8s/bases/apps/crossview/cilium-network-policy.yaml deleted file mode 100644 index a15d52097c..0000000000 --- a/k8s/bases/apps/crossview/cilium-network-policy.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: allow-crossview - namespace: crossview -spec: - endpointSelector: {} - ingress: - # Public route traffic after the shared oauth2-proxy maintainer-team gate. - # auth-proxy performs the Host-based hop from oauth2-proxy to Crossview; - # Crossview's native session middleware then supplies its app session. - - fromEndpoints: - - matchLabels: - app: auth-proxy - k8s:io.kubernetes.pod.namespace: oauth2-proxy - toPorts: - - ports: - - port: "3001" - protocol: TCP - # Intra-namespace: app <-> bundled Postgres (and any Helm-test pods). - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: crossview - egress: - # Kube API — Crossview reads Crossplane CRDs, claims, composites, managed - # resources, and providers. - - toEntities: - - kube-apiserver - # Dex OIDC server-side calls: the authorization-code -> token exchange and the - # userinfo lookup (config.sso.oidc tokenURL / userInfoURL in helm-release.yaml) - # go to the IN-CLUSTER Dex Service on :5556, which the pod can always resolve — - # unlike the public dex.${domain} URL, which the browser (not this pod) hits - # for the authorization redirect. Mirrors the oauth2-proxy egress. - - toEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: dex - toPorts: - - ports: - - port: "5556" - protocol: TCP - # Intra-namespace: app -> bundled Postgres (5432). - - toEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: crossview - # DNS resolution (engages Cilium's L7 DNS proxy). - - toEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: kube-system - k8s-app: kube-dns - toPorts: - - ports: - - port: "53" - protocol: UDP - - port: "53" - protocol: TCP - rules: - dns: - - matchPattern: "*" diff --git a/k8s/bases/apps/crossview/cluster-role-binding-cluster-reader.yaml b/k8s/bases/apps/crossview/cluster-role-binding-cluster-reader.yaml deleted file mode 100644 index fa38e75d9e..0000000000 --- a/k8s/bases/apps/crossview/cluster-role-binding-cluster-reader.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -# Safe cluster-scoped and custom-resource reads for Crossview. The shared -# cluster-reader role enumerates the core resources and API groups that may be -# displayed; it excludes Secrets and API groups whose specs may embed them. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: crossview-cluster-reader - labels: - app.kubernetes.io/name: crossview - app.kubernetes.io/managed-by: ksail -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-reader -subjects: - - kind: ServiceAccount - name: crossview-sa - namespace: crossview diff --git a/k8s/bases/apps/crossview/cluster-role-binding-view.yaml b/k8s/bases/apps/crossview/cluster-role-binding-view.yaml deleted file mode 100644 index 83969bdb75..0000000000 --- a/k8s/bases/apps/crossview/cluster-role-binding-view.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# Namespaced workload/config reads for the Crossview backend. The built-in -# `view` role deliberately excludes Secrets and write/exec verbs. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: crossview-view - labels: - app.kubernetes.io/name: crossview - app.kubernetes.io/managed-by: ksail -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: view -subjects: - - kind: ServiceAccount - name: crossview-sa - namespace: crossview diff --git a/k8s/bases/apps/crossview/external-secret.yaml b/k8s/bases/apps/crossview/external-secret.yaml deleted file mode 100644 index 2c6b1cd2c9..0000000000 --- a/k8s/bases/apps/crossview/external-secret.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# crossview-credentials: every secret Crossview needs, materialized from OpenBao -# by External Secrets and consumed by the HelmRelease through secretKeyRef -# (POSTGRES_PASSWORD / DB_PASS, SESSION_SECRET, ADMIN_PASSWORD, OIDC_CLIENT_SECRET): -# -# - db-password the bundled Postgres password (apps/crossview/db) -# - session-secret the app's session-cookie signing key (apps/crossview/app) -# - admin-password the break-glass native `admin` login password -# (apps/crossview/admin) — replaces the chart's well-known -# admin/password default; the boot-time admin user -# (createAdmin, session mode) is seeded from it -# - oidc-client-secret the SHARED Dex confidential-client secret -# (infrastructure/oidc/dex, property client-secret) — the -# same value oauth2-proxy / headlamp / actual-budget read, -# seeded into OpenBao from the SOPS bootstrap secret by the -# seed-dex-client-secret PushSecret. Crossview authenticates -# to Dex as the `public-client` (see helm-release.yaml), so -# it reuses this secret rather than minting a new one. -# -# db-password / session-secret / admin-password are generated once and seeded into -# OpenBao by the ESO Password generators + PushSecrets in infrastructure/vault-seed -# (remoteKeys apps/crossview/{db,app,admin}) — the same generate-once-then-mirror -# pattern umami uses — so nothing is committed to this public repo. Stable values -# keep Postgres data readable, login sessions valid, and the admin login working -# across chart upgrades and pod restarts. -# -# All four paths are read through the shared `external-secrets` ClusterSecretStore -# role, which already bundles app-crossview-readonly (apps/crossview/*) AND -# infra-oidc-readonly (infrastructure/oidc/*) — see infrastructure/vault-config/job.yaml -# — so no policy change is needed to read the Dex client secret here. -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: crossview-credentials - namespace: crossview -spec: - refreshInterval: 1h - secretStoreRef: - name: openbao - kind: ClusterSecretStore - target: - name: crossview-credentials - creationPolicy: Owner - data: - - secretKey: db-password # checkov:skip=CKV_SECRET_6:names the key in the synced Secret; the value lives in OpenBao - remoteRef: - key: apps/crossview/db - property: db-password - - secretKey: session-secret - remoteRef: - key: apps/crossview/app - property: session-secret - - secretKey: admin-password # checkov:skip=CKV_SECRET_6:names the key in the synced Secret; the value lives in OpenBao - remoteRef: - key: apps/crossview/admin - property: admin-password - - secretKey: oidc-client-secret # checkov:skip=CKV_SECRET_6:names the key in the synced Secret; the value lives in OpenBao - remoteRef: - key: infrastructure/oidc/dex - property: client-secret diff --git a/k8s/bases/apps/crossview/helm-release.yaml b/k8s/bases/apps/crossview/helm-release.yaml deleted file mode 100644 index 877ba8480f..0000000000 --- a/k8s/bases/apps/crossview/helm-release.yaml +++ /dev/null @@ -1,303 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: crossview - namespace: crossview - labels: - helm.toolkit.fluxcd.io/remediation: enabled -spec: - interval: 10m - timeout: 10m - chart: - spec: - chart: crossview - # 4.5.0 carries the upstream OIDC fix this deployment depends on: an empty - # `issuer` now skips discovery entirely, so the explicit split-horizon - # endpoints below are used verbatim (see config.sso.oidc). - version: 4.6.0 - sourceRef: - kind: HelmRepository - name: crossview - # Harden against Kubescape C-0017 (immutable container filesystem). The chart - # exposes app/database.containerSecurityContext but NO extraVolumes/ - # extraVolumeMounts knob, and it hardcodes only /app/logs (app) and - # /var/run/postgresql (postgres) as emptyDir volumes — so both the - # readOnlyRootFilesystem flip AND the extra writable mounts postgres needs are - # applied here as a post-render kustomize patch (strategic-merge merges the - # named containers/volumes additively; every other securityContext field — - # capabilities.drop, runAsUser, seccompProfile — is preserved). - postRenderStrategy: nohooks - postRenderers: - - kustomize: - patches: - # App (Go server) + wait-for-db init — flip root FS read-only. The - # crossview binary is a static CGO-free Go server that writes NOTHING to - # the filesystem at runtime (state lives in Postgres; sessions are a - # signed cookie, no server-side session files), so no extra writable - # mount is needed — the chart's hardcoded /app/logs emptyDir already - # covers the one path it mounts. The wait-for-db init only runs - # `psql -c "SELECT 1"` over TCP, so it is read-only-safe too. - - target: - kind: Deployment - name: crossview - patch: | - apiVersion: apps/v1 - kind: Deployment - metadata: - name: crossview - annotations: - # The chart wires every OIDC value in as env from - # crossview-config via env[].valueFrom.configMapKeyRef, and env - # is resolved once at container creation -- so a ConfigMap - # change alone NEVER reaches the running process. Without this - # annotation the pod keeps serving whatever OIDC_CALLBACK_URL it - # started with, which is how a reconciled Dex registration and a - # stale in-pod callback silently disagree and every login fails - # with "Unregistered redirect_uri". Reloader (already deployed - # cluster-wide) rolls the Deployment when the named ConfigMap - # changes; same convention as auth-proxy, oauth2-proxy and - # homepage. - configmap.reloader.stakater.com/reload: crossview-config - spec: - template: - spec: - initContainers: - - name: wait-for-db - securityContext: - readOnlyRootFilesystem: true - containers: - - name: crossview - securityContext: - readOnlyRootFilesystem: true - # Bundled Postgres (postgres:18.6). The provider-neutral base keeps - # persistence disabled, so its data directory is an emptyDir; the - # production overlay enables Longhorn and replaces this exact patch. - # Flip the root FS read-only and add the writable mounts the ephemeral - # base genuinely needs: - # /var/lib/postgresql the image VOLUME. PG18 initdb writes the fresh - # cluster to PGDATA=/var/lib/postgresql/18/docker; - # with persistence off the chart mounts nothing - # here, so it must be a writable emptyDir. - # /tmp the entrypoint mktemp's the initdb pwfile here. - # /var/run/postgresql (the socket dir) is already a chart emptyDir. - - target: - kind: Deployment - name: crossview-postgres - patch: | - apiVersion: apps/v1 - kind: Deployment - metadata: - name: crossview-postgres - spec: - template: - spec: - containers: - - name: postgres - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - name: postgres-data - mountPath: /var/lib/postgresql - - name: tmp - mountPath: /tmp - volumes: - - name: postgres-data - emptyDir: - sizeLimit: 2Gi - - name: tmp - emptyDir: - sizeLimit: 128Mi - # https://github.com/crossplane-contrib/crossview/blob/main/helm/crossview/values.yaml - values: - global: - namespace: crossview - # The namespace is managed declaratively (namespace.yaml) so it carries the - # Pod Security Standards labels; don't let the chart create it. - createNamespace: false - # The upstream chart grants its ServiceAccount get/list/watch over every - # resource in every API group. That includes core Secrets, so a dashboard - # session would bypass the human OIDC boundary. Repository-managed bindings - # below replace it with built-in view + the platform's Secret-free - # cluster-reader role. - rbac: - create: false - # Crossview creates its tables (and the break-glass admin) once, at its OWN - # startup, so an empty database only gains a schema when the APP restarts — - # never when Postgres alone does. - # - # Bump this value to make the app re-run that bootstrap against whatever - # database is currently live; the changed pod template is what rolls it. It is - # the only lever available — the chart is third-party and its `/api/health` - # probe does not touch the database, so the pod stays Ready on an empty - # schema. In prod the crossview-login-alerter CronJob - # (providers/hetzner/infrastructure/coroot/) notices instead: it alerts when - # /api/auth/check keeps answering that it finds neither its admin nor any - # user (#3315). - # - # Note this rolls ONLY the app. That is deliberate: it re-bootstraps against - # the running database, with no second Deployment reconciling underneath it. - # A change that also replaces the database must sequence the two, because the - # app's `wait-for-db` can succeed against the outgoing Postgres pod and - # bootstrap a schema that disappears when the Service cuts over. - podAnnotations: - platform.devantler.tech/db-bootstrap: "2026-08-22-schema-recreate" - app: - replicas: ${crossview_replicas:=1} - # Crossview is a bursty, near-idle dashboard: ~14Mi RSS while nobody is - # looking at it, but a single dashboard load lists Crossplane resources - # cluster-wide (every managed resource across all provider CRDs, plus - # composites/claims/XRDs/providers/functions) and spikes to ~240Mi+ — a - # ~17x jump — with the frontend firing those lists concurrently. - # - # auto-vpa (the Kyverno-generated VerticalPodAutoscaler, controlledValues: - # RequestsAndLimits) right-sizes the memory REQUEST to that low idle/steady - # histogram and, preserving the authored limit:request ratio, drags the - # LIMIT down with it. At the chart's default 4:1 ratio (1Gi:256Mi) a - # transient-low request recommendation (~100Mi, e.g. after a VPA-recommender - # restart) pins the limit near ~400Mi — below the load burst — and the pod - # is OOMKilled mid-request. Every dashboard widget then shows "Error - # loading", so Crossview looks like it lost kube-API access when it hasn't: - # the SA token, the read-only view + cluster-reader RBAC, and network are - # all fine. - # - # Fix (same auto-vpa footgun the homepage HelmRelease documents): author - # resources explicitly with a WIDE memory ratio (8:1). VPA still scales the - # request to the observed working set, but the preserved 8:1 ratio keeps the - # limit generous at every request level — ~800Mi even at the ~100Mi low - # bootstrap, ~1.9Gi at the ~240Mi steady target — so the burst is absorbed. - # The limit is a ceiling, not a reservation (idle still reserves only the - # request), and the auto-vpa maxAllowed (6Gi) remains the runaway/leak - # guard. CPU stays at the chart default: compressible, VPA-managed, never - # the OOM cause. - resources: - requests: - cpu: 250m - memory: 256Mi - limits: - cpu: "1" - memory: 2Gi - extraEnv: - # Crossview keys several behaviours off NODE_ENV (lib/env.go -> - # Environment): with it set to `production` the session-cookie middleware - # marks the cookie Secure (api/middlewares/session.go), CORS debug logging - # is off, and gin runs in release mode. Crossview is served over HTTPS at - # crossview.${domain}, so the Secure cookie is valid in every environment. - - name: NODE_ENV - value: "production" - # Keep the reusable base provider-neutral. Production enables a protected - # longhorn-wffc PVC in its Crossview overlay and couples it to a serialized, - # no-surge rollout plus an explicit database-only hostUsers choice (#3319). - # The Docker provider does not deploy Crossview by default and therefore - # acquires no storage-class dependency. A disposable consumer that opts into - # this base directly keeps the chart's ephemeral behavior. - database: - # Pin the chart's bundled Postgres image. The crossview chart defaults - # database.image.tag to `latest` (a disallow-latest-tag policy violation, - # and a reliability footgun: a pod restart can silently pull a new Postgres - # major). Pin to the current stable — what `latest` resolves to today — and - # let Renovate keep it current via the customManager scoped to this file in - # .github/renovate.json. (Same image drives the chart's wait-for-db init.) - image: - # renovate: datasource=docker depName=postgres - tag: "18.6" - persistence: - enabled: false - # Every Crossview secret is sourced from the crossview-credentials Secret - # (external-secret.yaml) via secretKeyRef. That Secret is materialized from - # OpenBao by External Secrets — the values are generated once and seeded by - # the ESO Password generators + PushSecrets in infrastructure/vault-seed - # (apps/crossview/{db,app,admin}), and oidc-client-secret is the shared Dex - # client secret read from infrastructure/oidc/dex — so nothing is committed - # to this public repo. - # - # dbPassword the chart's bundled Postgres refuses to initialize - # without POSTGRES_PASSWORD (it CrashLoopBackOffs with - # "superuser password is not specified", stalling the app - # in Init and hanging the whole `apps` Kustomization). - # sessionSecret signs the session cookie set after OIDC / admin login; - # a stable value keeps logins valid across pod restarts. - # adminPassword the break-glass native `admin` login (createAdmin runs - # at boot in session mode). Overriding it here means the - # chart no longer emits the well-known admin/password - # default into the crossview-secrets Secret — that Secret - # now carries only the non-secret admin USERNAME. - # OIDCClientSecret the Dex confidential-client secret for the OIDC login - # (config.sso.oidc below); Crossview presents it as the - # shared `public-client`. - secrets: - dbPassword: - secretKeyRef: - name: crossview-credentials - key: db-password - sessionSecret: - secretKeyRef: - name: crossview-credentials - key: session-secret - adminPassword: - secretKeyRef: - name: crossview-credentials - key: admin-password - OIDCClientSecret: - secretKeyRef: - name: crossview-credentials - key: oidc-client-secret - config: - server: - cors: - # The public HTTPS route is the post-login redirect target and replaces - # the removed runtime-downloaded Headlamp plugin entry point. - origin: https://crossview.${domain} - # Crossview keeps its OWN application session after the outer - # oauth2-proxy maintainer-team gate. Mode `session` is mandatory for the - # OIDC flow below: - # the SessionAuthMiddleware gates the /api/kubernetes/* data routes on a - # signed session cookie, the cookie store is ONLY wired up when - # AUTH_MODE=session (api/middlewares/session.go), and the boot-time admin - # user is seeded only in session mode (createAdmin, commands/serve.go). The - # public /api/auth/* + /api/health routes stay unauthenticated so login and - # kubelet probes work. - auth: - mode: session - # Native OIDC against Dex — the platform's single OIDC IdP (Dex -> GitHub). - # Crossview's Login page shows a "Sign in with OIDC" button when this is - # enabled (src/presentation/pages/Login.jsx); clicking it runs - # /api/auth/oidc -> Dex -> GitHub -> /api/auth/oidc/callback, which sets the - # session cookie. Crossview reuses the shared confidential `public-client` - # (clientId below + OIDCClientSecret from infrastructure/oidc/dex), exactly - # like oauth2-proxy / headlamp; its callback is registered on that client in - # controllers/dex/helm-release.yaml. - # - # Split-horizon endpoints, mirroring oauth2-proxy (skip_oidc_discovery): - # - authorizationURL is the PUBLIC Dex URL — the browser is redirected - # there and resolves it via /etc/hosts (local) or public DNS (prod). - # - tokenURL / userInfoURL are the IN-CLUSTER Dex Service — these are - # server-side calls from the Crossview pod, which cannot resolve - # dex.${domain} on the local/docker cluster and should not hairpin the - # LoadBalancer in prod. - # The split works because `issuer` is explicitly EMPTY: since crossview - # v4.5.0 an empty issuer SKIPS OIDC discovery entirely (fixed upstream for - # exactly this split-horizon shape), so the explicit endpoints below are - # used verbatim. Before 4.5.0 the chart forced a non-empty issuer and a - # successful discovery overrode these endpoints — the old fail-closed - # localhost-issuer workaround that used to be documented here is obsolete. - # Crossview validates the user via the userinfo endpoint, not the - # id_token, so no JWKS/issuer match is required. - # - # SECURITY NOTE: the public HTTPRoute is fronted by oauth2-proxy's - # devantler-tech:maintainers allowed_groups gate before auth-proxy reaches - # this service. Crossview then maps the same user's native OIDC session to - # a non-admin `user` on the read-only dashboard (the first login is the - # break-glass `admin` seeded at boot, so SSO users are never `admin`). The - # ServiceAccount is independently restricted to view + cluster-reader, so - # neither authentication layer can turn the dashboard into a Secret-read - # proxy. - sso: - oidc: - enabled: true - clientId: public-client - issuer: "" - authorizationURL: https://dex.${domain}/auth - tokenURL: http://dex.dex.svc.cluster.local:5556/token - userInfoURL: http://dex.dex.svc.cluster.local:5556/userinfo - # Registered exactly on the shared Dex client in controllers/dex. - callbackURL: https://crossview.${domain}/api/auth/oidc/callback diff --git a/k8s/bases/apps/crossview/helm-repository.yaml b/k8s/bases/apps/crossview/helm-repository.yaml deleted file mode 100644 index fac3f15994..0000000000 --- a/k8s/bases/apps/crossview/helm-repository.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository -metadata: - name: crossview - namespace: crossview -spec: - url: https://crossplane-contrib.github.io/crossview/ diff --git a/k8s/bases/apps/crossview/http-route.yaml b/k8s/bases/apps/crossview/http-route.yaml deleted file mode 100644 index ef32dd5b3f..0000000000 --- a/k8s/bases/apps/crossview/http-route.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Route for Crossview at crossview.${domain} — the authenticated replacement for -# the removed runtime-downloaded Headlamp plugin. Public traffic first passes -# through oauth2-proxy's maintainer-team gate and auth-proxy's Host router, then -# Crossview establishes its own Dex-backed application session. -# -# Prod/hetzner only in practice: Crossview is opt-in on the local Docker overlay, -# so this route renders only where the app is deployed. -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: crossview - namespace: crossview - annotations: - gethomepage.dev/enabled: "true" - gethomepage.dev/name: Crossview - gethomepage.dev/description: Crossplane resource explorer. - gethomepage.dev/group: Infrastructure as Code - gethomepage.dev/icon: mdi-graph-outline - gethomepage.dev/pod-selector: app.kubernetes.io/name=crossview,app.kubernetes.io/component=app -spec: - parentRefs: - - name: platform - namespace: kube-system - sectionName: https - hostnames: - - crossview.${domain} - rules: - - filters: - - type: ResponseHeaderModifier - responseHeaderModifier: - set: - - name: Strict-Transport-Security - value: max-age=63072000; includeSubDomains; preload - - name: Content-Security-Policy - value: "frame-ancestors 'self'" - backendRefs: - - name: oauth2-proxy - namespace: oauth2-proxy - port: 80 diff --git a/k8s/bases/apps/crossview/kustomization.yaml b/k8s/bases/apps/crossview/kustomization.yaml deleted file mode 100644 index 72f533d2e4..0000000000 --- a/k8s/bases/apps/crossview/kustomization.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - namespace.yaml - - helm-repository.yaml - - helm-release.yaml - - external-secret.yaml - - cilium-network-policy.yaml - - cluster-role-binding-view.yaml - - cluster-role-binding-cluster-reader.yaml - - http-route.yaml diff --git a/k8s/bases/apps/crossview/namespace.yaml b/k8s/bases/apps/crossview/namespace.yaml deleted file mode 100644 index 7efe6f9921..0000000000 --- a/k8s/bases/apps/crossview/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: crossview - labels: - pod-security.kubernetes.io/enforce: restricted - pod-security.kubernetes.io/enforce-version: latest diff --git a/k8s/bases/apps/headlamp/README.md b/k8s/bases/apps/headlamp/README.md deleted file mode 100644 index f8387b1e10..0000000000 --- a/k8s/bases/apps/headlamp/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Headlamp - -Headlamp is a lightweight Kubernetes UI which supports multiple clusters and is extensible at its core. - -- [Documentation](https://headlamp.dev/docs/latest/) -- [Helm Chart](https://github.com/headlamp-k8s/headlamp/tree/main/charts/headlamp) diff --git a/k8s/bases/apps/headlamp/cilium-network-policy.yaml b/k8s/bases/apps/headlamp/cilium-network-policy.yaml deleted file mode 100644 index 3a5387a889..0000000000 --- a/k8s/bases/apps/headlamp/cilium-network-policy.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: allow-headlamp - namespace: headlamp -spec: - endpointSelector: {} - ingress: - # Gateway ingress (direct from Cilium envoy) - - fromEntities: - - ingress - toPorts: - - ports: - - port: "4466" - protocol: TCP - egress: - # Kube API for dashboard - - toEntities: - - kube-apiserver - # Dex OIDC (via Cloudflare proxy). Dynamic plugin downloads are disabled - # in helm-release.yaml, so Headlamp no longer needs Artifact Hub, GitHub - # release assets, or npm egress at runtime. - - toFQDNs: - - matchName: "dex.${domain}" - toPorts: - - ports: - - port: "443" - protocol: TCP - # DNS resolution. rules.dns engages Cilium's L7 DNS proxy so the toFQDNs - # allow-list above can be enforced; matchPattern "*" leaves resolution - # itself unrestricted (the FQDN list constrains where traffic may go). - - toEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: kube-system - k8s-app: kube-dns - toPorts: - - ports: - - port: "53" - protocol: UDP - - port: "53" - protocol: TCP - rules: - dns: - - matchPattern: "*" diff --git a/k8s/bases/apps/headlamp/cluster-role-binding.yaml b/k8s/bases/apps/headlamp/cluster-role-binding.yaml deleted file mode 100644 index 2491413e2b..0000000000 --- a/k8s/bases/apps/headlamp/cluster-role-binding.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -# Read-only cluster RBAC for the Headlamp backend ServiceAccount. Replaces the -# chart's default cluster-admin ClusterRoleBinding, which is disabled in -# helm-release.yaml (clusterRoleBinding.create: false). Headlamp authenticates -# user requests with the logged-in user's OIDC token, so its ServiceAccount only -# needs the platform's read-only cluster-reader role — not a `*/*/*` god-role -# (Kubescape CIS-5.1.3 / C-0187). -# -# This is a separate, differently-named binding (not headlamp-admin) on purpose: -# a ClusterRoleBinding's roleRef is immutable, so the existing cluster-admin -# binding cannot be retargeted in place. Helm removes the old binding and this -# one is created fresh, avoiding the "cannot change roleRef" upgrade failure. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: headlamp-cluster-reader - labels: - app.kubernetes.io/name: headlamp - app.kubernetes.io/managed-by: ksail -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-reader -subjects: - - kind: ServiceAccount - name: headlamp - namespace: headlamp diff --git a/k8s/bases/apps/headlamp/external-secret.yaml b/k8s/bases/apps/headlamp/external-secret.yaml deleted file mode 100644 index 88ade0ec36..0000000000 --- a/k8s/bases/apps/headlamp/external-secret.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: headlamp-oidc - namespace: headlamp -spec: - refreshInterval: 1h - secretStoreRef: - name: openbao - kind: ClusterSecretStore - target: - name: headlamp-oidc - creationPolicy: Owner - data: - - secretKey: client-secret # checkov:skip=CKV_SECRET_6:names the key in the synced Secret; the value lives in OpenBao - remoteRef: - key: infrastructure/oidc/dex - property: client-secret diff --git a/k8s/bases/apps/headlamp/helm-release.yaml b/k8s/bases/apps/headlamp/helm-release.yaml deleted file mode 100644 index 35697e71c1..0000000000 --- a/k8s/bases/apps/headlamp/helm-release.yaml +++ /dev/null @@ -1,137 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: headlamp - namespace: headlamp - labels: - helm.toolkit.fluxcd.io/helm-test: enabled - helm.toolkit.fluxcd.io/remediation: enabled -spec: - interval: 10m - timeout: 10m - chart: - spec: - chart: headlamp - # 0.43.0 introduced a Safari OIDC regression and 0.44.0 still reproduces - # it: the session succeeds, but ITP suppresses the cross-window storage - # event that redirects the opener. Pin the last Safari-verified release - # until upstream #6355 ships and its replacement is verified (#2438). - version: 0.42.0 - sourceRef: - kind: HelmRepository - name: headlamp - valuesFrom: - - kind: Secret - name: headlamp-oidc - valuesKey: client-secret - targetPath: config.oidc.clientSecret - optional: true - postRenderStrategy: nohooks - postRenderers: - - kustomize: - patches: - - target: - kind: Deployment - name: headlamp - patch: | - - op: add - path: /metadata/annotations/secret.reloader.stakater.com~1reload - value: headlamp-oidc - - op: add - path: /spec/strategy - value: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - # Chart hardcodes liveness/readiness with K8s defaults - # (timeoutSeconds: 1, failureThreshold: 3, periodSeconds: 10). - # Headlamp is KEDA-scaled to 0 in prod; every cold start logs - # 1-3 Unhealthy probe warnings while the Go binary initialises. - # Add a startupProbe so liveness/readiness are gated until the - # main container is actually serving. - # initialDelaySeconds skips past the cold-start window so the - # first probe lands on a serving container — zero failure - # events during the merge-queue's 90s steady-state Warning - # check. - - op: add - path: /spec/template/spec/containers/0/startupProbe - value: - httpGet: - path: / - port: http - initialDelaySeconds: 20 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 12 # 60s grace beyond initial delay - # https://github.com/kubernetes-sigs/headlamp/blob/main/charts/headlamp/values.yaml - values: - replicaCount: ${headlamp_replicas:=1} - # Use chart-native values for the main container's writable directories. - # When pluginsManager and its PVC are disabled, the chart renders no volumes - # parent at all; JSON post-render appends therefore fail before the - # Deployment reaches Kubernetes. - volumes: - - name: tmp-dir - emptyDir: {} - - name: headlamp-config - emptyDir: {} - volumeMounts: - - name: tmp-dir - mountPath: /tmp - - name: headlamp-config - mountPath: /home/headlamp/.config - podDisruptionBudget: - enabled: true - # Drain-safe (#1880/#1882): maxUnavailable: 1 lets the pod be evicted - # during a node drain (minAvailable would block a single-replica pod). - minAvailable: null - maxUnavailable: 1 - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchLabels: - app.kubernetes.io/name: headlamp - app.kubernetes.io/instance: headlamp - # Least-privilege RBAC. The chart default binds Headlamp's ServiceAccount to - # cluster-admin, but Headlamp runs with OIDC (config.oidc below; - # unsafeUseServiceAccountToken defaults to false), so every Kubernetes API - # request is authenticated with the logged-in USER's token — the pod - # ServiceAccount is not used to serve user requests. - # - # Disable the chart's ClusterRoleBinding and bind the SA to the read-only - # cluster-reader ClusterRole via our own binding (cluster-role-binding.yaml). - # We deliberately do NOT just set clusterRoleName: a ClusterRoleBinding's - # roleRef is IMMUTABLE, so retargeting the existing headlamp-admin binding - # fails the Helm upgrade ("cannot change roleRef"). Disabling create lets Helm - # delete the old cluster-admin binding, and our separate, differently-named - # binding is created fresh — no `*/*/*` god-role (clears Kubescape CIS-5.1.3 / - # C-0187), while still covering any baseline cluster reads the backend makes. - clusterRoleBinding: - create: false - config: - # Disable dynamic plugin loading until Headlamp supports committing - # plugin artifact integrity (digest/checksum) in GitOps state. The - # pluginsManager resolves Artifact Hub metadata at runtime by name/version - # only, which can install unverified same-origin JavaScript into the - # dashboard if an upstream package or delivery path is compromised. - watchPlugins: false - oidc: - clientID: public-client - issuerURL: https://dex.${domain} - scopes: "profile,email,groups" - pluginsManager: - enabled: false - persistentVolumeClaim: - enabled: false - podSecurityContext: - fsGroup: 101 - runAsUser: 100 - runAsGroup: 101 - securityContext: - runAsUser: 100 - runAsGroup: 101 - ingress: - enabled: false diff --git a/k8s/bases/apps/headlamp/helm-repository.yaml b/k8s/bases/apps/headlamp/helm-repository.yaml deleted file mode 100644 index b6439682af..0000000000 --- a/k8s/bases/apps/headlamp/helm-repository.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository -metadata: - name: headlamp - namespace: headlamp -spec: - url: https://kubernetes-sigs.github.io/headlamp/ diff --git a/k8s/bases/apps/headlamp/http-route.yaml b/k8s/bases/apps/headlamp/http-route.yaml deleted file mode 100644 index 18542ce18b..0000000000 --- a/k8s/bases/apps/headlamp/http-route.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: headlamp - namespace: headlamp - annotations: - gethomepage.dev/enabled: "true" - gethomepage.dev/name: Headlamp - gethomepage.dev/description: A web UI for managing Kubernetes clusters. - gethomepage.dev/group: Kubernetes - gethomepage.dev/icon: https://pbs.twimg.com/profile_images/1537480067227566080/waXG0X7n_400x400.jpg - gethomepage.dev/href: https://headlamp.${domain} - gethomepage.dev/pod-selector: app.kubernetes.io/name=headlamp -spec: - parentRefs: - - name: platform - namespace: kube-system - sectionName: https - hostnames: - - headlamp.${domain} - rules: - - backendRefs: - - name: headlamp - port: 80 - filters: - - type: ResponseHeaderModifier - responseHeaderModifier: - set: - - name: Strict-Transport-Security - value: max-age=63072000; includeSubDomains; preload diff --git a/k8s/bases/apps/headlamp/kustomization.yaml b/k8s/bases/apps/headlamp/kustomization.yaml deleted file mode 100644 index a91de0427a..0000000000 --- a/k8s/bases/apps/headlamp/kustomization.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - namespace.yaml - - external-secret.yaml - - helm-release.yaml - - helm-repository.yaml - - cluster-role-binding.yaml - - http-route.yaml - - cilium-network-policy.yaml diff --git a/k8s/bases/apps/headlamp/namespace.yaml b/k8s/bases/apps/headlamp/namespace.yaml deleted file mode 100644 index e832f82983..0000000000 --- a/k8s/bases/apps/headlamp/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: headlamp - labels: - pod-security.kubernetes.io/enforce: restricted - pod-security.kubernetes.io/enforce-version: latest diff --git a/k8s/bases/apps/kustomization.yaml b/k8s/bases/apps/kustomization.yaml index 5f0b750e20..db7170201f 100644 --- a/k8s/bases/apps/kustomization.yaml +++ b/k8s/bases/apps/kustomization.yaml @@ -4,10 +4,6 @@ kind: Kustomization resources: - actual-budget/ - ascoachingogvaner/ - # Crossplane resource dashboard (Crossview). Prod-only in practice: it reads - # Crossplane CRDs, and Crossplane runs only on the Hetzner provider. The - # docker provider deploys no apps, so it is naturally excluded from local. - - crossview/ - data-product-controller/ # fleetdm disabled 2026-06-03 — not in use. The manifests under # bases/apps/fleetdm/ are retained; re-enable by uncommenting the line below. @@ -21,7 +17,6 @@ resources: # Prod-only in practice (its Crossplane provider is Hetzner-only); the docker # provider deploys no apps, so it is naturally excluded from local clusters. - github-config/ - - headlamp/ - homepage/ - umami/ - wedding-app/ diff --git a/k8s/bases/components/coroot-cronjob-failure-alert/cron-job-cronjob-failure-alert.yaml b/k8s/bases/components/coroot-cronjob-failure-alert/cron-job-cronjob-failure-alert.yaml index 9c9f3c8ce4..1eeb77584e 100644 --- a/k8s/bases/components/coroot-cronjob-failure-alert/cron-job-cronjob-failure-alert.yaml +++ b/k8s/bases/components/coroot-cronjob-failure-alert/cron-job-cronjob-failure-alert.yaml @@ -6,9 +6,9 @@ # namespace/name:failure-threshold:stale-seconds. Umami absorbs one failed # 15-minute tick; the daily Kubescape cleanup alerts on its first failed run and # allows 26 hours before a missing completion is stale. The 15-minute -# stranded-autoscaler-node and crossview-login alerters (hetzner overlay) use -# Umami's thresholds: a sensor that cannot read its target or deliver alerts -# must not look like a healthy one. +# stranded-autoscaler-node alerter (hetzner overlay) uses Umami's thresholds: +# a sensor that cannot read its target or deliver alerts must not look like a +# healthy one. # # FAILURE THRESHOLDS ARE PER TARGET. Umami absorbs a single failed tick (the # weekly database-credential rotation produces exactly that); the daily @@ -86,7 +86,7 @@ spec: # Space-separated namespace/name:failure-threshold:stale-seconds # entries. Each namespace needs its own Role and RoleBinding. - name: WATCH - value: "umami/umami-provision-tenants:2:3600 kubescape/kubescape-hostdata-cleanup:1:93600 observability/stranded-autoscaler-node-alerter:2:3600 observability/crossview-login-alerter:2:3600" + value: "umami/umami-provision-tenants:2:3600 kubescape/kubescape-hostdata-cleanup:1:93600 observability/stranded-autoscaler-node-alerter:2:3600" - name: FAILURE_THRESHOLD # Default for a WATCH entry without target-specific tuning. value: "2" diff --git a/k8s/bases/components/coroot-cronjob-failure-alert/role-cronjob-failure-alert-observability.yaml b/k8s/bases/components/coroot-cronjob-failure-alert/role-cronjob-failure-alert-observability.yaml index 25653c2271..fca9f26cf2 100644 --- a/k8s/bases/components/coroot-cronjob-failure-alert/role-cronjob-failure-alert-observability.yaml +++ b/k8s/bases/components/coroot-cronjob-failure-alert/role-cronjob-failure-alert-observability.yaml @@ -1,7 +1,6 @@ -# Lets the detector read the stranded-autoscaler-node-alerter and -# crossview-login-alerter CronJobs (hetzner overlay) and the Jobs they created, -# so a sensor that keeps failing is reported instead of reading as a clean -# cluster. Read-only and resource-name scoped where Kubernetes supports it; Jobs +# Lets the detector read the stranded-autoscaler-node-alerter CronJob (hetzner +# overlay) and the Jobs it created, so a sensor that keeps failing is reported +# instead of reading as a clean cluster. Read-only and resource-name scoped where Kubernetes supports it; Jobs # require namespace-wide list for owner-reference filtering. --- apiVersion: rbac.authorization.k8s.io/v1 @@ -16,7 +15,6 @@ rules: - cronjobs resourceNames: - stranded-autoscaler-node-alerter - - crossview-login-alerter verbs: - get - apiGroups: diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/validate-replica-floor.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/validate-replica-floor.yaml index 0781761618..6977e26749 100644 --- a/k8s/bases/infrastructure/cluster-policies/best-practices/validate-replica-floor.yaml +++ b/k8s/bases/infrastructure/cluster-policies/best-practices/validate-replica-floor.yaml @@ -100,12 +100,11 @@ spec: # whose outage delays only an idempotent loop (production may still opt # them into HA); and components single by design from the June 2026 HA # sweep -- SPIRE needs an external datastore for real HA, CNPG-I exposes - # its gRPC readiness endpoint only on the elected leader, and Crossview with - # its bundled Postgres is one non-HA unit until the datastore is replaced. + # its gRPC readiness endpoint only on the elected leader. - name: namedExempt expression: >- object.metadata.name in [ - 'actual-budget-actualbudget', 'fleet', 'headlamp', 'whoami', + 'actual-budget-actualbudget', 'fleet', 'whoami', 'hubble-ui', 'longhorn-ui', 'homepage', 'umami-umami'] || object.metadata.name in [ 'external-dns', 'origin-ca-issuer', 'flagger-loadtester'] || @@ -116,8 +115,7 @@ spec: object.metadata.name in [ 'cluster-autoscaler-hetzner-cluster-autoscaler', 'keda-operator-metrics-apiserver', 'longhorn-driver-deployer', - 'spire-server', 'plugin-barman-cloud', 'crossview', - 'crossview-postgres'] + 'spire-server', 'plugin-barman-cloud'] - name: exempt expression: >- variables.workloadExempt || variables.templateExempt || diff --git a/k8s/bases/infrastructure/cluster-roles/cluster-reader.yaml b/k8s/bases/infrastructure/cluster-roles/cluster-reader.yaml index 6a226f8d63..cb6469ab66 100644 --- a/k8s/bases/infrastructure/cluster-roles/cluster-reader.yaml +++ b/k8s/bases/infrastructure/cluster-roles/cluster-reader.yaml @@ -1,7 +1,7 @@ # Read-only ClusterRole for the resources the built-in `view` role omits. # Bound ALONGSIDE `view` (see cluster-role-bindings/oidc-cluster-reader.yaml) so an # OIDC identity gets a COMPLETE read-only picture of the cluster — including -# every safe-to-read custom resource, so the Headlamp UI can display them: +# every safe-to-read custom resource: # - `view` → namespaced workloads/config (pods, deployments, # configmaps, pod logs, events, ...), minus Secrets. # - `cluster-reader` → cluster-scoped core resources `view` omits (nodes, diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/controller-rbac.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/controller-rbac.yaml index ea6c2311af..e1d5a33b86 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/controller-rbac.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/controller-rbac.yaml @@ -2,9 +2,8 @@ # Exempt controllers and middleware that legitimately need broad RBAC. # Infrastructure controllers (Flux, Kyverno, cert-manager, CNPG, Velero, # KEDA, Coroot, KubeVirt, CDI, Longhorn) and platform middleware -# (headlamp cluster dashboard, homepage service discovery, fleetdm device -# management, dex identity, oauth2-proxy auth, -# reloader config watcher) all require cluster-wide read or read/write +# (homepage service discovery, fleetdm device management, dex identity, +# oauth2-proxy auth, reloader config watcher) all require cluster-wide read or read/write # access to function correctly. apiVersion: kubescape.io/v1beta1 kind: ClusterSecurityException @@ -17,8 +16,7 @@ spec: (Flux), policy engines (Kyverno), certificate managers (cert-manager), database operators (CNPG), backup controllers (Velero), autoscalers (KEDA), observability (Coroot), virtualisation (KubeVirt, - CDI), storage (Longhorn), cluster dashboards (headlamp), service - discovery (homepage), device management (fleetdm), identity (dex), + CDI), storage (Longhorn), service discovery (homepage), device management (fleetdm), identity (dex), auth proxies (oauth2-proxy), and config watchers (reloader). posture: - controlID: C-0053 @@ -82,8 +80,7 @@ spec: # read-only client-side oracle in docs/kubescape-exception-oracle.md. It # hydrates each summary with an individual GET and matches the generated # exception artifact against the kubescape.io/workload-* labels; it does - # not parse kind or name out of wlid. Headlamp is fixed at the root (SA - # scoped to read-only cluster-reader); Flux and Velero are exempted + # not parse kind or name out of wlid. Flux and Velero are exempted # by-design in wildcard-rbac.yaml. - controlID: C-0188 action: ignore @@ -107,7 +104,6 @@ spec: - longhorn-system - external-dns - oauth2-proxy - - headlamp - kubelet-serving-cert-approver - reloader - dex diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/health-probes.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/health-probes.yaml index 170713758e..bc33ed11b0 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/health-probes.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/health-probes.yaml @@ -3,9 +3,9 @@ # Many controllers, sidecars, and init containers do not expose health # endpoints. Adding probes to upstream charts is impractical and they # typically have their own readiness mechanisms (leader election, watch -# loops). App namespaces (headlamp, homepage) are included because their -# charts deploy init containers and sidecars that lack probe endpoints, -# while main application containers have probes configured. +# loops). The homepage app namespace is included because its chart deploys +# init containers and sidecars that lack probe endpoints, while the main +# application container has probes configured. apiVersion: kubescape.io/v1beta1 kind: ClusterSecurityException metadata: @@ -41,7 +41,6 @@ spec: - reloader - external-dns - kubelet-serving-cert-approver - - headlamp - kubevirt - cdi - dex @@ -51,7 +50,7 @@ spec: # workloads genuinely lack probe endpoints (verified failing C-0018 / # C-0056 in the cluster Kubescape operator): the external-secrets # operator/webhook/cert-controller and the Crossplane control-plane - # (core, rbac-manager, provider pods). crossview / ksail-operator / - # flagger DO configure probes, so they are deliberately NOT listed. + # (core, rbac-manager, provider pods). ksail-operator / flagger DO + # configure probes, so they are deliberately NOT listed. - external-secrets - crossplane-system diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/secret-reader-rbac.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/secret-reader-rbac.yaml index 6066cb5ae2..659502f6b4 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/secret-reader-rbac.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/secret-reader-rbac.yaml @@ -27,17 +27,11 @@ # broad built-in `ClusterRole/edit`: their pods run as the `default` SA, so # these edit-bound SAs are dormant namespace-wide secret grants = genuine # over-privilege to remove, not to silence. -# - `crossview-sa` (`crossview-role` grants `*/* get,list,watch`, i.e. reads all -# Secret contents for a topology viewer) — tighten upstream in the crossview -# chart, don't exempt. # - The break-glass admin groups `system:masters` (cluster-admin) and # `crossplane:masters` (crossplane-admin): surfacing the highest-privilege # identities is the control working as intended. # - `longhorn-support-bundle` (an on-demand diagnostic SA bound to cluster-admin) # — same treatment as in wildcard-rbac.yaml. -# -# Keep this list and the Headlamp mirror -# (controllers/kubescape/config-map-headlamp-exceptions.yaml) in sync by hand. apiVersion: kubescape.io/v1beta1 kind: ClusterSecurityException metadata: @@ -51,8 +45,8 @@ spec: databases). Matched by the specific RBAC object (Role/ClusterRole, or the binding for cluster-admin holders) so a new/accidental secret-reader elsewhere still trips C-0015. Over-privileged or admin identities - (unused tenant `edit` SAs, crossview's wildcard read, the cluster-admin / - crossplane admin groups) are intentionally left flagged. + (unused tenant `edit` SAs, the cluster-admin / crossplane admin groups) + are intentionally left flagged. posture: - controlID: C-0015 action: ignore diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/service-account-tokens.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/service-account-tokens.yaml index 828dd49fb8..fe280203d6 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/service-account-tokens.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/service-account-tokens.yaml @@ -2,8 +2,8 @@ # Exempt service account token auto-mount controls for infrastructure # and middleware namespaces. Controllers and platform middleware that # manage cluster resources (Flux, Kyverno, cert-manager, KEDA, VPA, -# headlamp dashboard, homepage service discovery, fleetdm device -# management, etc.) genuinely need API tokens to function. +# homepage service discovery, fleetdm device management, etc.) genuinely +# need API tokens to function. apiVersion: kubescape.io/v1beta1 kind: ClusterSecurityException metadata: @@ -12,10 +12,9 @@ spec: reason: >- Infrastructure controllers and platform middleware need Kubernetes API access via SA tokens. This includes GitOps controllers, policy - engines, certificate managers, autoscalers, monitoring, cluster - dashboards (headlamp), service discovery (homepage), device - management (fleetdm), identity providers (dex), auth proxies - (oauth2-proxy), and config watchers (reloader). The platform uses + engines, certificate managers, autoscalers, monitoring, service + discovery (homepage), device management (fleetdm), identity providers + (dex), auth proxies (oauth2-proxy), and config watchers (reloader). The platform uses RBAC to restrict what each SA can do. posture: - controlID: C-0261 @@ -57,7 +56,6 @@ spec: - dex - fleetdm - homepage - - headlamp - oauth2-proxy # Controllers / middleware and dashboards added since this list was # last updated, same accepted pattern as the entries above (verified @@ -65,10 +63,8 @@ spec: # external-secrets (the operator that syncs OpenBao -> Secrets, so it # genuinely needs secret access), crossplane & ksail-operator # (cluster-state reconcilers), flagger (progressive-delivery - # controller), and crossview (read-only Crossplane dashboard, like - # headlamp). RBAC still scopes what each SA can do. + # controller). RBAC still scopes what each SA can do. - external-secrets - crossplane-system - ksail-operator - flagger-system - - crossview diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/talos-cis-control-plane-false-positives.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/talos-cis-control-plane-false-positives.yaml index 4b2239d1c7..865e6995c8 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/talos-cis-control-plane-false-positives.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/talos-cis-control-plane-false-positives.yaml @@ -29,14 +29,6 @@ kind: ClusterSecurityException metadata: name: talos-cis-control-plane-false-positives annotations: - # Keep this exception out of the Headlamp mirror ConfigMap. The plugin only - # evaluates WORKLOAD posture scans, and every control below is a host-scanner - # finding on the control-plane node (file permissions/ownership, API-server - # flags) that never attaches to a workload — so it cannot be expressed as a - # workload exception. Without this marker the generator would emit a - # cluster-wide `kind: ".*"` designator for it, which the plugin would apply - # to every workload in the cluster. - platform.devantler.tech/headlamp-mirror: exclude # Cluster-wide on purpose: every control here is a host-scanner finding on # the control-plane node, which has no workload, namespace or kind to scope # against. See the NO-`match:` note above. diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/talos-cis-worker-false-positives.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/talos-cis-worker-false-positives.yaml index 0998e7824a..863cf342ec 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/talos-cis-worker-false-positives.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/talos-cis-worker-false-positives.yaml @@ -32,14 +32,6 @@ kind: ClusterSecurityException metadata: name: talos-cis-worker-false-positives annotations: - # Keep this exception out of the Headlamp mirror ConfigMap. The plugin only - # evaluates WORKLOAD posture scans, and every control below is a host-scanner - # finding on the worker node (kubelet config file permissions/ownership and - # kubelet flags) that never attaches to a workload — so it cannot be - # expressed as a workload exception. Without this marker the generator would - # emit a cluster-wide `kind: ".*"` designator for it, which the plugin would - # apply to every workload in the cluster. - platform.devantler.tech/headlamp-mirror: exclude # Cluster-wide on purpose: every control here is a host-scanner finding on # the worker node, which has no workload, namespace or kind to scope # against. See the NO-`match:` note above. diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/wildcard-rbac.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/wildcard-rbac.yaml index 7efba81a47..cc10de4e70 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/wildcard-rbac.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/wildcard-rbac.yaml @@ -21,9 +21,8 @@ # velero-server Role. The chart's RBAC is all-or-nothing (no scoped mode) and # restore must create/patch arbitrary resource kinds. # -# NOT exempted (intentionally): Headlamp is fixed at the root (its SA is scoped to -# the read-only cluster-reader role in k8s/bases/apps/headlamp/helm-release.yaml), -# and longhorn-support-bundle is a transient diagnostic SA created on demand. +# NOT exempted (intentionally): longhorn-support-bundle is a transient diagnostic +# SA created on demand. apiVersion: kubescape.io/v1beta1 kind: ClusterSecurityException metadata: diff --git a/k8s/bases/infrastructure/controllers/auth-proxy/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/auth-proxy/cilium-network-policy.yaml index 76e043b782..06ad4ac777 100644 --- a/k8s/bases/infrastructure/controllers/auth-proxy/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/auth-proxy/cilium-network-policy.yaml @@ -38,14 +38,6 @@ spec: - ports: - port: "5006" protocol: TCP - # Crossview upstream (cluster resource explorer; Service :80 -> pod :3001). - - toEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: crossview - toPorts: - - ports: - - port: "3001" - protocol: TCP # Homepage upstream - toEndpoints: - matchLabels: diff --git a/k8s/bases/infrastructure/controllers/auth-proxy/config-map.yaml b/k8s/bases/infrastructure/controllers/auth-proxy/config-map.yaml index 18739333f9..28d1367be1 100644 --- a/k8s/bases/infrastructure/controllers/auth-proxy/config-map.yaml +++ b/k8s/bases/infrastructure/controllers/auth-proxy/config-map.yaml @@ -40,22 +40,6 @@ data: rule: "Host(`budget.${domain}`)" entryPoints: ["web"] service: actual-budget - # Crossview also has native OIDC, but the outer oauth2-proxy hop keeps - # its cluster-wide read surface restricted to the maintainer team. - crossview: - rule: "Host(`crossview.${domain}`)" - entryPoints: ["web"] - # Crossview builds its OIDC redirect_uri from the incoming request - # rather than from OIDC_CALLBACK_URL -- proven in prod: with the env - # set to the https public callback it still sent http://. Every hop in - # front of it is plaintext (Gateway terminates TLS, then oauth2-proxy, - # then this entryPoint), and Traefik sets X-Forwarded-Proto from its - # OWN entryPoint scheme, so without this the app sees http and Dex - # rejects the redirect_uri it did not register. This router is only - # ever reached through the public HTTPS Gateway, so asserting https - # here states a fact about the traffic, not a preference. - middlewares: ["forwarded-proto-https"] - service: crossview # Kyverno Policy Reporter UI. Prod-only (the app lives in the # `infrastructure` layer, pulled in full only by the hetzner overlay), so # this router is inert on local/CI -- no HTTPRoute sends @@ -114,10 +98,6 @@ data: loadBalancer: servers: - url: "http://ksail-operator.ksail-operator.svc.cluster.local:8080" - crossview: - loadBalancer: - servers: - - url: "http://crossview-service.crossview.svc.cluster.local:80" # Policy Reporter UI (policy-reporter-ui Service :8080). Prod-only and # inert on local (the policy-reporter namespace has no pods there). policy-reporter: diff --git a/k8s/bases/infrastructure/controllers/dex/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/dex/cilium-network-policy.yaml index 4b9fbeae1f..045f976fde 100644 --- a/k8s/bases/infrastructure/controllers/dex/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/dex/cilium-network-policy.yaml @@ -13,27 +13,6 @@ spec: - ports: - port: "5556" protocol: TCP - # OIDC token exchange from Headlamp backend (hairpin via gateway ClusterIP - # preserves the originating pod identity instead of using the 'ingress' entity) - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: headlamp - toPorts: - - ports: - - port: "5556" - protocol: TCP - # OIDC token + userinfo exchange from the Crossview backend. Crossview does - # native OIDC against Dex (apps/crossview/helm-release.yaml, config.sso.oidc) - # and dials the in-cluster Dex Service directly on :5556, so Dex sees the - # crossview pod identity here — matching crossview's egress allow. Inert on - # local/CI where the crossview namespace is absent (crossview is prod-only). - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: crossview - toPorts: - - ports: - - port: "5556" - protocol: TCP # gRPC from oauth2-proxy - fromEndpoints: - matchLabels: diff --git a/k8s/bases/infrastructure/controllers/dex/helm-release.yaml b/k8s/bases/infrastructure/controllers/dex/helm-release.yaml index de6315bdf2..e3d61bacff 100644 --- a/k8s/bases/infrastructure/controllers/dex/helm-release.yaml +++ b/k8s/bases/infrastructure/controllers/dex/helm-release.yaml @@ -152,8 +152,6 @@ spec: - "https://analytics.${domain}:443/oauth2/callback" - "https://budget.${domain}/oauth2/callback" - "https://budget.${domain}:443/oauth2/callback" - - "https://crossview.${domain}/oauth2/callback" - - "https://crossview.${domain}:443/oauth2/callback" - "https://data-products.${domain}/oauth2/callback" - "https://data-products.${domain}:443/oauth2/callback" - "https://hubble.${domain}/oauth2/callback" @@ -166,7 +164,6 @@ spec: - "https://policy-reporter.${domain}:443/oauth2/callback" - "https://vault.${domain}/oauth2/callback" - "https://vault.${domain}:443/oauth2/callback" - - "https://headlamp.${domain}/oidc-callback" # actual-budget (sync-server) hardcodes its OIDC callback path as # /openid/callback (ACTUAL_OPENID_SERVER_HOSTNAME=https://budget.${domain}). - "https://budget.${domain}/openid/callback" @@ -175,9 +172,6 @@ spec: # this paired with the admin role's allowed_redirect_uris. - "http://localhost:8250/oidc/callback" - "https://ksail.${domain}/api/v1/auth/callback" - # Crossview authenticates natively against Dex on its public HTTPS - # route; Dex matches this callback exactly. - - "https://crossview.${domain}/api/auth/oidc/callback" trustedPeers: - kubectl - name: kubectl @@ -202,14 +196,12 @@ spec: - name: devantler-tech # A LOGIN requirement for every Dex client, not a per-app filter: # an org member outside `maintainers` fails at Dex before reaching - # any app — oauth2-proxy's UIs, Headlamp, OpenBao (UI and CLI) - # and kubectl alike. It is kept deliberately as a second layer. + # any app — oauth2-proxy's UIs, OpenBao (UI and CLI) and + # kubectl alike. It is kept deliberately as a second layer. # Each sensitive client also enforces the rule itself: # oauth2-proxy's allowed_groups gates the browser UIs routed - # through it (the native-OIDC apps behind it included); Headlamp - # is routed directly but acts only with the user's own token, - # which the OIDC RBAC bindings authorize for the admin alone; - # and OpenBao's oidc role binds the maintainers group. Dex has + # through it (the native-OIDC apps behind it included), and + # OpenBao's oidc role binds the maintainers group. Dex has # no per-client group filter to replace it with. # To give a collaborator access, add them to `maintainers`. To # admit non-maintainers anywhere, first re-check that each client diff --git a/k8s/bases/infrastructure/controllers/flux-operator/helm-release.yaml b/k8s/bases/infrastructure/controllers/flux-operator/helm-release.yaml index f21c4cb98b..db2af928f4 100644 --- a/k8s/bases/infrastructure/controllers/flux-operator/helm-release.yaml +++ b/k8s/bases/infrastructure/controllers/flux-operator/helm-release.yaml @@ -16,11 +16,8 @@ spec: timeout: 10m values: web: - # The Flux Operator's bundled Status web UI is disabled: Flux is surfaced - # in-cluster through Headlamp's `headlamp_flux` plugin (see - # bases/apps/headlamp), so a standalone Flux dashboard is redundant. This - # release is the operator/reconciler only and must stay running at all - # times. + # The Flux Operator's bundled Status web UI is disabled: this release is + # the operator/reconciler only and must stay running at all times. enabled: false # Give the operator container the uid AND gid its own image declares, so # Kubescape C-0013 (non-root containers) passes on its own terms rather than diff --git a/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml b/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml deleted file mode 100644 index 25f0b80f09..0000000000 --- a/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml +++ /dev/null @@ -1,1474 +0,0 @@ ---- -# GENERATED FILE — DO NOT EDIT BY HAND. -# -# Regenerate with, from the repository root: -# go run ./scripts/generate-kubescape-exceptions \ -# -format headlamp-configmap \ -# -o k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml -# -# CI regenerates this file and fails on any difference, so a hand edit is -# reverted by the next run rather than silently kept. Change the -# ClusterSecurityException CRs in k8s/bases/infrastructure/cluster-security-exceptions/ -# instead — they are the single source of truth — or the template at -# scripts/generate-kubescape-exceptions/headlamp-configmap.yaml.tmpl for this -# prose. -# -# WHY THIS FILE EXISTS: the pinned plugin (headlamp_kubescape v0.11.2) has its OWN -# exception mechanism — ConfigMaps labelled `kubescape.io/custom-object: exceptions` — -# and does NOT read the kubescape.io ClusterSecurityException/SecurityException CRDs. -# (The CRD-native "Security Exceptions" view exists only on the plugin's unreleased -# main branch.) Without this ConfigMap the plugin's Exceptions page shows -# "No data to be shown" and its compliance view recomputes scores client-side -# with zero exceptions, so every control the CRs except still renders as failing. -# -# This file is a presentation-layer FALLBACK for the Headlamp dashboard ONLY. -# DELETE it (plus the manual step below, and the -format headlamp-configmap mode) -# once the scanner is confirmed marking excepted controls in prod, OR once the -# plugin is bumped to a release that reads the CRDs natively — whichever lands -# first. As of 2026-08-28 neither holds: the deployed scanner is v4.0.12, which -# should write status=passed/subStatus=w-exceptions into the stored -# workloadconfigurationscansummaries, but a sweep of all ~2200 summaries found no -# object carrying that subStatus (platform#2586, platform#2823). -# -# WHAT IS MIRRORED: every ClusterSecurityException CR, except those annotated -# `platform.devantler.tech/headlamp-mirror: exclude`. That marker is for -# host-scanner exceptions (CIS file permissions/ownership, kubelet and API-server -# flags) whose controls never attach to a workload, so they cannot be expressed as -# a workload exception — and whose cluster-wide designator would otherwise except -# those controls for every workload in the cluster. The exclusion is declared on -# the CR because there is no structural way to infer it: nine legitimately-mirrored -# policies also render a bare wildcard. -# -# Matching semantics (apply-exceptions.ts): a control is excepted for a workload -# when a policy's posturePolicies regex matches the controlID AND a resources -# regex matches the workload's kind/name/namespace. Values are JS RegExp, anchored -# by the generator for exactness. A CR with no `spec.match` renders as -# `kind: ".*"`, which matches every workload. -# -# IMPORTANT (interim manual step, needed ONLY until a scan marks the summaries): -# the v0.11.2 plugin applies ONE selected group at a time and recomputes scores -# client-side, so a viewer must pick the "platform-exceptions" group from the -# Compliance page's exceptions selector for these to take effect. Once the scanner -# writes the exception status this is no longer required (the plugin renders the -# summary status directly). -apiVersion: v1 -kind: ConfigMap -metadata: - name: headlamp-exceptions - namespace: kubescape - labels: - kubescape.io/custom-object: exceptions - app.kubernetes.io/name: platform-exceptions - app.kubernetes.io/managed-by: headlamp -data: - name: platform-exceptions - description: >- - Mirror of the platform ClusterSecurityException CRs for the Headlamp Kubescape - plugin (v0.11.2 reads ConfigMaps, not CRDs). - exceptionPolicies: | - [ - { - "name": "admission-controllers", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": ".*" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0036$" - }, - { - "controlID": "^C-0039$" - } - ], - "reason": "Admission webhook timeouts are configured by upstream Helm charts (Kyverno, cert-manager, VPA, KEDA) at appropriate values for their operation. Changing these would cause admission failures." - }, - { - "name": "arc-openbao-listener", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ConfigMap$", - "name": "^arc-openbao-listener$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0012$" - } - ], - "reason": "C-0012's generic credential-text pattern matches the mandatory native OpenBao TLS private-key file setting. Its value is a public path to a mounted Secret, never credential content. Enforced restrict-arc-openbao-listener admission and unconditional source tests permit exactly the audited public listener in openbao, reject extra or binary data and reject this name elsewhere. Other ConfigMaps, controls and genuine credentials remain scanned." - }, - { - "name": "batch-workloads", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Job$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^CronJob$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0018$" - }, - { - "controlID": "^C-0056$" - } - ], - "reason": "Readiness and liveness probes are inapplicable to run-to-completion batch workloads (Jobs/CronJobs): the pod terminates on completion and never serves traffic. Matched by workload kind so long-running workloads still require probes." - }, - { - "name": "cilium-network-policies", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": ".*" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0054$" - }, - { - "controlID": "^C-0030$" - }, - { - "controlID": "^C-0260$" - } - ], - "reason": "Platform uses CiliumNetworkPolicies for all network segmentation. Kubescape does not recognise CiliumNetworkPolicy as equivalent to Kubernetes NetworkPolicy." - }, - { - "name": "controller-rbac", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "namespace": "^(flux-system|kyverno|cert-manager|cnpg-system|velero|keda|observability|kubescape|vertical-pod-autoscaler|kubevirt|cdi|longhorn-system|external-dns|oauth2-proxy|headlamp|kubelet-serving-cert-approver|reloader|dex|fleetdm|homepage)$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0053$" - }, - { - "controlID": "^C-0037$" - }, - { - "controlID": "^C-0031$" - }, - { - "controlID": "^C-0063$" - }, - { - "controlID": "^C-0035$" - }, - { - "controlID": "^C-0188$" - } - ], - "reason": "Infrastructure controllers and platform middleware require broad RBAC to manage their target resources cluster-wide. This includes GitOps (Flux), policy engines (Kyverno), certificate managers (cert-manager), database operators (CNPG), backup controllers (Velero), autoscalers (KEDA), observability (Coroot), virtualisation (KubeVirt, CDI), storage (Longhorn), cluster dashboards (headlamp), service discovery (homepage), device management (fleetdm), identity (dex), auth proxies (oauth2-proxy), and config watchers (reloader)." - }, - { - "name": "delete-capable-rbac-by-design", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cdi-sa$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cert-manager-controller-issuers$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cert-manager-controller-clusterissuers$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cert-manager-controller-certificates$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cert-manager-controller-challenges$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cloudnative-pg$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cluster-reconciler-flux-system$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^flux-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^coroot-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^crossplane-admin$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^crossplane:provider:provider-upjet-github-[0-9a-f]+:system$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^flagger$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^kro:controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^ksail-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^kube-prometheus-stack-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^kubevirt-apiserver$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^kubevirt-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^longhorn-bind$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^reloader-reloader-role-binding$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cdi-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^kubevirt-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^RoleBinding$", - "name": "^ascoachingogvaner$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^RoleBinding$", - "name": "^kyverno:admission-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^RoleBinding$", - "name": "^kyverno:cleanup-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^RoleBinding$", - "name": "^longhorn$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^RoleBinding$", - "name": "^velero-server$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^RoleBinding$", - "name": "^wedding-app$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^RoleBinding$", - "name": "^cdi-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^RoleBinding$", - "name": "^kubevirt-operator-rolebinding$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0007$" - } - ], - "reason": "Infrastructure controllers, operators and tenant deployers legitimately require delete permissions by design (GitOps reconciliation, operator lifecycle, backup/restore, storage, virtualisation, cert/DB/secret management). Each offending Role/binding is matched by kind+name so C-0007 still catches any new or accidental delete-capable grant." - }, - { - "name": "exec-into-container-rbac", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": "^RoleBinding$", - "name": "^velero-server$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^velero-server$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^velero-server$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cloudnative-pg$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cluster-reconciler-flux-system$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^flux-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^kubevirt-operator$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0002$" - } - ], - "reason": "Velero (all-or-nothing backup/restore RBAC + cluster-admin), CloudNativePG (operator execs into Postgres pods — explicit pods/exec grant), the Flux GitOps reconcilers kustomize-controller, helm-controller (both via the cluster-reconciler-flux-system binding), and flux-operator (cluster-admin trust model), and KubeVirt's virt-operator (explicit pods/exec grant to manage VM lifecycles) legitimately hold pods/exec by design. The specific RBAC bindings are matched by kind+name so C-0002 still catches any new exec grant." - }, - { - "name": "gitops-managed-cronjobs", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "namespace": "^(kube-system|kubescape|longhorn-system|observability|openbao|umami|fleetdm)$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0026$" - } - ], - "reason": "C-0026 lists every CronJob for manual approval and has no hardening remedy. All CronJobs on this cluster are GitOps-managed platform components delivered by Flux after PR review (the review is the approval): Cilium Hubble cert rotation, the Kubescape scan schedulers / stale host-data cleanup, Longhorn stale-node cleanup, the observability heartbeat / alerting jobs (CNPG degraded cluster, stranded volume attach, stranded autoscaler node, reconcile-CronJob failure, Crossplane sync, unhealthy Crossplane provider, posture-scan staleness, prune-protected orphans, Velero maintenance OOM) / Coroot reconcilers (alert autosuppression, cloud pricing, application health, risk acceptance), the OpenBao raft-snapshot backup, the Umami database-login rotation, and Umami tenant provisioning. Namespace-scoped to the infrastructure namespaces that run these, so a CronJob in any unexpected namespace still surfaces for review." - }, - { - "name": "health-probes", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "namespace": "^(flux-system|kyverno|cert-manager|cnpg-system|velero|keda|observability|kubescape|vertical-pod-autoscaler|longhorn-system|kube-system|reloader|external-dns|kubelet-serving-cert-approver|headlamp|kubevirt|cdi|dex|oauth2-proxy|homepage|external-secrets|crossplane-system)$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0018$" - }, - { - "controlID": "^C-0056$" - } - ], - "reason": "Infrastructure controllers, middleware, sidecars, and init containers often lack health probe endpoints. Upstream charts manage availability through leader election and watch mechanisms. App namespaces are included for their init containers and sidecars, not main containers." - }, - { - "name": "helm-chart-metadata", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": ".*" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0076$" - }, - { - "controlID": "^C-0077$" - } - ], - "reason": "Third-party Helm charts use their own label conventions. Adding all kubescape-expected labels would require patching every chart and would break on upgrades." - }, - { - "name": "image-verification", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": ".*" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0237$" - } - ], - "reason": "First-party images (ghcr.io/devantler-tech/*) are cosign-signed and verified at Pod admission by Kyverno's verify-app-images policy, at the Talos containerd pull layer (ImageVerificationConfig), and at the Flux OCI source layer. Third-party chart images are not all signed or verified at admission, so this cluster-wide control still fails on unsigned workload images." - }, - { - "name": "infrastructure-privileged", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "namespace": "^(kube-system|longhorn-system|observability|velero|kubevirt|cdi)$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0057$" - }, - { - "controlID": "^C-0048$" - }, - { - "controlID": "^C-0045$" - }, - { - "controlID": "^C-0041$" - }, - { - "controlID": "^C-0038$" - }, - { - "controlID": "^C-0044$" - }, - { - "controlID": "^C-0046$" - }, - { - "controlID": "^C-0016$" - }, - { - "controlID": "^C-0013$" - }, - { - "controlID": "^C-0017$" - }, - { - "controlID": "^C-0055$" - } - ], - "reason": "CNI (Cilium), distributed storage (Longhorn), node telemetry (Coroot node-agent), identity (SPIRE agent), backup (Velero node-agent), and virtualisation (KubeVirt, CDI) require host-level access by design." - }, - { - "name": "kubescape-privileged", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": "^DaemonSet$", - "name": "^node-agent$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Deployment$", - "name": "^storage$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0057$" - }, - { - "controlID": "^C-0048$" - }, - { - "controlID": "^C-0045$" - }, - { - "controlID": "^C-0041$" - }, - { - "controlID": "^C-0038$" - }, - { - "controlID": "^C-0044$" - }, - { - "controlID": "^C-0046$" - }, - { - "controlID": "^C-0016$" - }, - { - "controlID": "^C-0013$" - }, - { - "controlID": "^C-0017$" - }, - { - "controlID": "^C-0055$" - } - ], - "reason": "Kubescape's node-agent needs privileged host access (eBPF runtime detection) and the storage aggregated APIserver writes its backing store at runtime; scoped to those two workloads so the operator, kubevuln, and scheduler in the same namespace stay subject to these controls." - }, - { - "name": "pod-security-mutations", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "namespace": "^([^cfklov].*|c|c[^dh].*|cd|cd[^i].*|cdi.+|ch|ch[^a].*|cha|cha[^o].*|chao|chao[^s].*|chaos|chaos[^\\-].*|chaos-|chaos-[^m].*|chaos-m|chaos-m[^e].*|chaos-me|chaos-me[^s].*|chaos-mes|chaos-mes[^h].*|chaos-mesh.+|f|f[^l].*|fl|fl[^u].*|flu|flu[^x].*|flux|flux[^\\-].*|flux-|flux-[^s].*|flux-s|flux-s[^y].*|flux-sy|flux-sy[^s].*|flux-sys|flux-sys[^t].*|flux-syst|flux-syst[^e].*|flux-syste|flux-syste[^m].*|flux-system.+|k|k[^u].*|ku|ku[^b].*|kub|kub[^e].*|kube|kube[^\\-sv].*|kube-|kube-[^nps].*|kube-n|kube-n[^o].*|kube-no|kube-no[^d].*|kube-nod|kube-nod[^e].*|kube-node|kube-node[^\\-].*|kube-node-|kube-node-[^l].*|kube-node-l|kube-node-l[^e].*|kube-node-le|kube-node-le[^a].*|kube-node-lea|kube-node-lea[^s].*|kube-node-leas|kube-node-leas[^e].*|kube-node-lease.+|kube-p|kube-p[^u].*|kube-pu|kube-pu[^b].*|kube-pub|kube-pub[^l].*|kube-publ|kube-publ[^i].*|kube-publi|kube-publi[^c].*|kube-public.+|kube-s|kube-s[^y].*|kube-sy|kube-sy[^s].*|kube-sys|kube-sys[^t].*|kube-syst|kube-syst[^e].*|kube-syste|kube-syste[^m].*|kube-system.+|kubes|kubes[^c].*|kubesc|kubesc[^a].*|kubesca|kubesca[^p].*|kubescap|kubescap[^e].*|kubescape.+|kubev|kubev[^i].*|kubevi|kubevi[^r].*|kubevir|kubevir[^t].*|kubevirt.+|l|l[^o].*|lo|lo[^cn].*|loc|loc[^a].*|loca|loca[^l].*|local|local[^\\-].*|local-|local-[^p].*|local-p|local-p[^a].*|local-pa|local-pa[^t].*|local-pat|local-pat[^h].*|local-path|local-path[^\\-].*|local-path-|local-path-[^s].*|local-path-s|local-path-s[^t].*|local-path-st|local-path-st[^o].*|local-path-sto|local-path-sto[^r].*|local-path-stor|local-path-stor[^a].*|local-path-stora|local-path-stora[^g].*|local-path-storag|local-path-storag[^e].*|local-path-storage.+|lon|lon[^g].*|long|long[^h].*|longh|longh[^o].*|longho|longho[^r].*|longhor|longhor[^n].*|longhorn|longhorn[^\\-].*|longhorn-|longhorn-[^s].*|longhorn-s|longhorn-s[^y].*|longhorn-sy|longhorn-sy[^s].*|longhorn-sys|longhorn-sys[^t].*|longhorn-syst|longhorn-syst[^e].*|longhorn-syste|longhorn-syste[^m].*|longhorn-system.+|o|o[^b].*|ob|ob[^s].*|obs|obs[^e].*|obse|obse[^r].*|obser|obser[^v].*|observ|observ[^a].*|observa|observa[^b].*|observab|observab[^i].*|observabi|observabi[^l].*|observabil|observabil[^i].*|observabili|observabili[^t].*|observabilit|observabilit[^y].*|observability.+|v|v[^e].*|ve|ve[^l].*|vel|vel[^e].*|vele|vele[^r].*|veler|veler[^o].*|velero.+)$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0013$" - }, - { - "controlID": "^C-0016$" - }, - { - "controlID": "^C-0055$" - }, - { - "controlID": "^C-0211$" - } - ], - "reason": "Kyverno mutate policy \"add-security-context\" injects allowPrivilegeEscalation: false, drop ALL capabilities and seccompProfile at container admission, and runAsNonRoot, runAsUser and runAsGroup at pod admission. The Deployment spec does not contain them and Kubescape scans the stored spec, not the live pod. C-0211 is included here; its remaining gaps outside those namespaces are scoped to named elevated workloads in pod-security-mutations-residual. The namespace exception cannot suppress a new workload where nothing injects these fields. readOnlyRootFilesystem (C-0017) is handled separately — set in-spec per workload." - }, - { - "name": "pod-security-mutations-residual", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Deployment$", - "name": "^csi-attacher$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Deployment$", - "name": "^csi-provisioner$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Deployment$", - "name": "^csi-resizer$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Deployment$", - "name": "^csi-snapshotter$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Deployment$", - "name": "^longhorn-driver-deployer$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^DaemonSet$", - "name": "^engine-image-ei-[0-9a-f]+$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^DaemonSet$", - "name": "^longhorn-csi-plugin$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^DaemonSet$", - "name": "^longhorn-manager$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^DaemonSet$", - "name": "^audit-log-forwarder-agent$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^DaemonSet$", - "name": "^coroot-node-agent$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^DaemonSet$", - "name": "^node-agent$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0211$" - } - ], - "reason": "C-0211's privilege-adjacent fields remain absent from eleven elevated storage, observability and backup workload templates. These exact workloads are excepted while other workloads regain a real C-0211 verdict." - }, - { - "name": "secret-reader-rbac", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^crd-controller-flux-system$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cluster-reconciler-flux-system$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^flux-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^velero-server$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^cert-manager-cainjector$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^cert-manager-controller-certificates$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^cert-manager-controller-challenges$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^cert-manager-controller-clusterissuers$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^cert-manager-controller-issuers$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^cert-manager-controller-orders$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^origin-ca-issuer-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^simply-dns-webhook:secret-access$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^cert-manager-webhook:dynamic-serving$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^trust-manager$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^cloudnative-pg$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^plugin-barman-cloud$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^coroot-db$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^coroot-db-barman-cloud$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^umami-db$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^umami-db-barman-cloud$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^wedding-db$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^wedding-db-barman-cloud$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^external-secrets-cert-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^external-secrets-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^crossplane$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^crossplane:provider:provider-aws-iam-[0-9a-f]+:system$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^crossplane:provider:upbound-provider-family-aws-[0-9a-f]+:system$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^crossplane:provider:provider-upjet-github-[0-9a-f]+:system$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^crossplane:provider:provider-upjet-unifi-[0-9a-f]+:system$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^kyverno:admission-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^kyverno:background-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^kyverno:cleanup-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^kyverno:reports-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^coroot-cluster-agent$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^coroot-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^kube-prometheus-stack-grafana-clusterrole$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^kube-prometheus-stack-kube-state-metrics$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^kube-prometheus-stack-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^hcloud-csi-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^longhorn-role$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^longhorn$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^cilium-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^cilium-gateway-secrets$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^cilium-tlsinterception-secrets$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^hubble-generate-certs$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^keda-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^keda-operator-certs$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^system:controller:generic-garbage-collector$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^system:controller:namespace-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^system:controller:resourcequota-controller$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^system:kube-controller-manager$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^system:controller:bootstrap-signer$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^system:controller:token-cleaner$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^kubescape$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^flagger$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^ksail-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRole$", - "name": "^reloader-reloader-role$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^vault-config$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^velero-server$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0015$" - } - ], - "reason": "Controllers, operators and databases that read Kubernetes Secrets by design (cert-manager, External Secrets, CloudNativePG + Barman backups, Crossplane and its providers, Kyverno, Flux, KEDA, Longhorn, Cilium, the Kubescape operator, the built-in kube-controller-manager controllers, and the tenant databases). Matched by the specific RBAC object (Role/ClusterRole, or the binding for cluster-admin holders) so a new/accidental secret-reader elsewhere still trips C-0015. Over-privileged or admin identities (unused tenant `edit` SAs, crossview's wildcard read, the cluster-admin / crossplane admin groups) are intentionally left flagged." - }, - { - "name": "service-account-tokens", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "namespace": "^(kube-system|kube-public|kube-node-lease|flux-system|kyverno|cert-manager|cnpg-system|velero|keda|observability|kubescape|vertical-pod-autoscaler|kubevirt|cdi|longhorn-system|kubelet-serving-cert-approver|external-dns|reloader|dex|fleetdm|homepage|headlamp|oauth2-proxy|external-secrets|crossplane-system|ksail-operator|flagger-system|crossview)$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0261$" - }, - { - "controlID": "^C-0207$" - }, - { - "controlID": "^C-0012$" - }, - { - "controlID": "^C-0255$" - }, - { - "controlID": "^C-0034$" - }, - { - "controlID": "^C-0190$" - } - ], - "reason": "Infrastructure controllers and platform middleware need Kubernetes API access via SA tokens. This includes GitOps controllers, policy engines, certificate managers, autoscalers, monitoring, cluster dashboards (headlamp), service discovery (homepage), device management (fleetdm), identity providers (dex), auth proxies (oauth2-proxy), and config watchers (reloader). The platform uses RBAC to restrict what each SA can do." - }, - { - "name": "umami-db-rotation-service-account-token", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": "^CronJob$", - "name": "^umami-db-rotation$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0034$" - } - ], - "reason": "The Umami database-login rotation CronJob needs its service-account token to authenticate to OpenBao's Kubernetes auth method. Its OpenBao role permits only reading the two login colours' rotation metadata and rotating them, and the ServiceAccount holds no Kubernetes RBAC." - }, - { - "name": "umami-provisioning-service-account-token", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": "^CronJob$", - "name": "^umami-provision-tenants$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Job$", - "name": "^umami-provision-tenants-bootstrap$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0034$" - } - ], - "reason": "The Umami scheduled and bootstrap provisioners need their service-account token to coordinate through the single umami-provision-tenants Lease. Their namespaced Role permits only get and update on that named Lease." - }, - { - "name": "upstream-chart-defaults", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": ".*" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0210$" - }, - { - "controlID": "^C-0258$" - }, - { - "controlID": "^C-0259$" - }, - { - "controlID": "^C-0266$" - } - ], - "reason": "These controls flag behaviors defined by upstream Helm charts or the underlying infrastructure (Talos etcd encryption, container base image UIDs, hostPath usage by operators). Fixing them requires forking charts or changing infrastructure-level configuration." - }, - { - "name": "vpa-managed-resources", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": ".*" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0268$" - }, - { - "controlID": "^C-0269$" - }, - { - "controlID": "^C-0270$" - }, - { - "controlID": "^C-0271$" - } - ], - "reason": "VPA manages resource requests and limits at pod admission time. Deployment specs intentionally omit static values to allow VPA right-sizing. Kubescape scans the stored Deployment spec, not the mutated pod spec." - }, - { - "name": "wildcard-rbac-by-design", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^cluster-reconciler-flux-system$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^flux-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^ClusterRoleBinding$", - "name": "^velero-server$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^velero-server$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0187$" - } - ], - "reason": "Flux GitOps controllers and Velero legitimately require cluster-admin / wildcard RBAC by design: arbitrary-manifest reconciliation (Flux) and cluster-wide backup/restore (Velero). The specific bindings are matched by kind+name so C-0187 still catches any new wildcard or cluster-admin grant." - }, - { - "name": "world-at-ruin-trial-operator", - "policyType": "postureExceptionPolicy", - "actions": [ - "disable" - ], - "resources": [ - { - "designatorType": "Attributes", - "attributes": { - "kind": "^RoleBinding$", - "name": "^world-at-ruin-zone-trial-operator$" - } - }, - { - "designatorType": "Attributes", - "attributes": { - "kind": "^Role$", - "name": "^world-at-ruin-zone-trial-operator$" - } - } - ], - "posturePolicies": [ - { - "controlID": "^C-0002$" - } - ], - "reason": "The configured administrator's private scripted zone trial intentionally grants namespaced exec to run its short-lived admission-token helper while retaining the global reader roles and withholding Secret API access. RBAC cannot constrain exec to one command. The uniquely named trial Role and binding are retired before production credentials or player data are added." - } - ] diff --git a/k8s/bases/infrastructure/controllers/kubescape/kustomization.yaml b/k8s/bases/infrastructure/controllers/kubescape/kustomization.yaml index 2e56495abd..6239c850d2 100644 --- a/k8s/bases/infrastructure/controllers/kubescape/kustomization.yaml +++ b/k8s/bases/infrastructure/controllers/kubescape/kustomization.yaml @@ -18,4 +18,3 @@ resources: # workload. The patched image looks the object up before removing its record. - cluster-role-storage-scan-record-cleanup.yaml - cluster-role-binding-storage-scan-record-cleanup.yaml - - config-map-headlamp-exceptions.yaml diff --git a/k8s/bases/infrastructure/controllers/oauth2-proxy/reference-grant.yaml b/k8s/bases/infrastructure/controllers/oauth2-proxy/reference-grant.yaml index f472215f53..1c3be16f81 100644 --- a/k8s/bases/infrastructure/controllers/oauth2-proxy/reference-grant.yaml +++ b/k8s/bases/infrastructure/controllers/oauth2-proxy/reference-grant.yaml @@ -12,9 +12,6 @@ spec: - group: gateway.networking.k8s.io kind: HTTPRoute namespace: actual-budget - - group: gateway.networking.k8s.io - kind: HTTPRoute - namespace: crossview - group: gateway.networking.k8s.io kind: HTTPRoute namespace: homepage diff --git a/k8s/bases/infrastructure/coroot/components/security-progress/config-map.yaml b/k8s/bases/infrastructure/coroot/components/security-progress/config-map.yaml index c258d1d64f..4ea911fc33 100644 --- a/k8s/bases/infrastructure/coroot/components/security-progress/config-map.yaml +++ b/k8s/bases/infrastructure/coroot/components/security-progress/config-map.yaml @@ -1448,7 +1448,7 @@ data: "panels": [ { "name": "Failed controls to remediate next", - "description": "Up to 30 controls grouped by severity and failed control-resource outcome count. Raw operator results do not automatically apply the platform's CI or Headlamp exceptions.", + "description": "Up to 30 controls grouped by severity and failed control-resource outcome count. Raw operator results do not automatically apply the platform's CI exceptions.", "source": { "metrics": { "queries": [ diff --git a/k8s/bases/infrastructure/coroot/components/security-progress/dashboards.json b/k8s/bases/infrastructure/coroot/components/security-progress/dashboards.json index decd94965a..a944b8cd19 100644 --- a/k8s/bases/infrastructure/coroot/components/security-progress/dashboards.json +++ b/k8s/bases/infrastructure/coroot/components/security-progress/dashboards.json @@ -823,7 +823,7 @@ "panels": [ { "name": "Failed controls to remediate next", - "description": "Up to 30 controls grouped by severity and failed control-resource outcome count. Raw operator results do not automatically apply the platform's CI or Headlamp exceptions.", + "description": "Up to 30 controls grouped by severity and failed control-resource outcome count. Raw operator results do not automatically apply the platform's CI exceptions.", "source": { "metrics": { "queries": [ diff --git a/k8s/bases/infrastructure/vault-config/job.yaml b/k8s/bases/infrastructure/vault-config/job.yaml index be710362b0..72f256b5c2 100644 --- a/k8s/bases/infrastructure/vault-config/job.yaml +++ b/k8s/bases/infrastructure/vault-config/job.yaml @@ -692,6 +692,20 @@ spec: echo "WARNING: secret/infrastructure/velero/repo is not populated yet -- leaving the old copy at secret/infrastructure/backup/velero-repo in place until the PushSecret has synced." fi + # --- 1g. Retire Crossview's generated credentials --- + # Crossview was removed from the platform. Its database, session and + # break-glass admin credentials were seeded by PushSecrets that never + # delete their remote copies, so destroy every version plus the + # metadata here. Idempotent: a later run finds nothing to remove. + for retired in apps/crossview/db apps/crossview/app apps/crossview/admin; do + if bao kv metadata get -mount=secret "$retired" >/dev/null 2>&1; then + echo "Retiring secret/$retired..." + bao kv metadata delete -mount=secret "$retired" + else + echo "No secret at secret/$retired (already retired)." + fi + done + # --- 2. Enable Kubernetes auth --- if ! bao auth list -format=json | grep -q '"kubernetes/"'; then echo "Enabling Kubernetes auth method..." @@ -894,18 +908,9 @@ spec: } POLICY - # crossview reads its bundled-Postgres password, app session secret, - # and break-glass admin password from OpenBao via the shared - # `external-secrets` ClusterSecretStore role (apps/crossview/{db,app, - # admin}); the values are generated and seeded by - # infrastructure/vault-seed. (Its OIDC client secret is read from the - # shared infrastructure/oidc/dex path via infra-oidc-readonly, also on - # that role — no crossview-specific grant needed.) - bao policy write app-crossview-readonly - <<'POLICY' - path "secret/data/apps/crossview/*" { - capabilities = ["read"] - } - POLICY + # Crossview was removed from the platform; drop its retired read + # policy. Deleting an absent policy succeeds, so re-runs are no-ops. + bao policy delete app-crossview-readonly # actual-budget reads its Enable Banking bank-sync credentials from # OpenBao via the shared `external-secrets` ClusterSecretStore role @@ -1033,9 +1038,6 @@ spec: path "secret/data/apps/actual-budget/*" { capabilities = ["create", "update", "read"] } - path "secret/data/apps/crossview/*" { - capabilities = ["create", "update", "read"] - } # Private mail preparation preserves three durable credentials. # Exact-key read/seed only; no mailbox data or wildcard grant. path "secret/data/apps/private-mail/admin" { @@ -1092,9 +1094,6 @@ spec: path "secret/metadata/apps/actual-budget/*" { capabilities = ["create", "update", "read"] } - path "secret/metadata/apps/crossview/*" { - capabilities = ["create", "update", "read"] - } path "secret/metadata/apps/private-mail/admin" { capabilities = ["create", "update", "read"] } @@ -1124,7 +1123,7 @@ spec: bao write auth/kubernetes/role/external-secrets \ bound_service_account_names=external-secrets \ bound_service_account_namespaces=external-secrets \ - policies=infra-backup-readonly,infra-dns-readonly,infra-monitoring-readonly,infra-hcloud-readonly,infra-tls-readonly,infra-ghcr-readonly,app-fleetdm-readonly,app-wedding-readonly,app-umami-readonly,app-actual-budget-readonly,app-crossview-readonly,app-ascoachingogvaner-readonly,infra-oidc-readonly,vault-seed-write \ + policies=infra-backup-readonly,infra-dns-readonly,infra-monitoring-readonly,infra-hcloud-readonly,infra-tls-readonly,infra-ghcr-readonly,app-fleetdm-readonly,app-wedding-readonly,app-umami-readonly,app-actual-budget-readonly,app-ascoachingogvaner-readonly,infra-oidc-readonly,vault-seed-write \ ttl=1h # Dedicated role for the fleetdm VaultDynamicSecret generator. diff --git a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-admin-password.yaml b/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-admin-password.yaml deleted file mode 100644 index 6ad8d97a2d..0000000000 --- a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-admin-password.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -# crossview break-glass `admin` login password — generate once, cache, mirror to -# OpenBao. Read back via crossview-credentials (apps/crossview/admin) and seeded -# into the boot-time admin user (createAdmin, session mode). -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: generated-crossview-admin-password - namespace: openbao -spec: - refreshInterval: "0" - target: - name: generated-crossview-admin-password - creationPolicy: Owner - dataFrom: - - sourceRef: - generatorRef: - apiVersion: generators.external-secrets.io/v1alpha1 - kind: Password - name: crossview-admin-password diff --git a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-db-password.yaml b/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-db-password.yaml deleted file mode 100644 index 4444fa1986..0000000000 --- a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-db-password.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# crossview bundled-Postgres password — generate once, cache, mirror to OpenBao. -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: generated-crossview-db-password - namespace: openbao -spec: - refreshInterval: "0" - target: - name: generated-crossview-db-password - creationPolicy: Owner - dataFrom: - - sourceRef: - generatorRef: - apiVersion: generators.external-secrets.io/v1alpha1 - kind: Password - name: crossview-db-password diff --git a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-session-secret.yaml b/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-session-secret.yaml deleted file mode 100644 index 6937842c0a..0000000000 --- a/k8s/bases/infrastructure/vault-seed/external-secret-generated-crossview-session-secret.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# crossview app session signing key — generate once, cache, mirror to OpenBao. -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: generated-crossview-session-secret - namespace: openbao -spec: - refreshInterval: "0" - target: - name: generated-crossview-session-secret - creationPolicy: Owner - dataFrom: - - sourceRef: - generatorRef: - apiVersion: generators.external-secrets.io/v1alpha1 - kind: Password - name: crossview-session-secret diff --git a/k8s/bases/infrastructure/vault-seed/kustomization.yaml b/k8s/bases/infrastructure/vault-seed/kustomization.yaml index 997ad6f3ca..e924834de9 100644 --- a/k8s/bases/infrastructure/vault-seed/kustomization.yaml +++ b/k8s/bases/infrastructure/vault-seed/kustomization.yaml @@ -10,9 +10,6 @@ resources: - password-umami-app-secret.yaml - password-umami-admin-password.yaml - password-umami-ascoachingogvaner-password.yaml - - password-crossview-db-password.yaml - - password-crossview-session-secret.yaml - - password-crossview-admin-password.yaml - external-secret-generated-fleetdm-mysql-root-password.yaml - push-secret-push-fleetdm-mysql-passwords.yaml - external-secret-generated-fleetdm-mysql-replication-password.yaml @@ -29,12 +26,6 @@ resources: - push-secret-push-umami-admin-password.yaml - external-secret-generated-umami-ascoachingogvaner-password.yaml - push-secret-push-umami-ascoachingogvaner-password.yaml - - external-secret-generated-crossview-db-password.yaml - - push-secret-push-crossview-db-password.yaml - - external-secret-generated-crossview-session-secret.yaml - - push-secret-push-crossview-session-secret.yaml - - external-secret-generated-crossview-admin-password.yaml - - push-secret-push-crossview-admin-password.yaml - push-secret-seed-r2-credentials.yaml - push-secret-seed-wedding-db-backup-r2.yaml - push-secret-seed-github-app.yaml diff --git a/k8s/bases/infrastructure/vault-seed/password-crossview-admin-password.yaml b/k8s/bases/infrastructure/vault-seed/password-crossview-admin-password.yaml deleted file mode 100644 index eeda75b4b6..0000000000 --- a/k8s/bases/infrastructure/vault-seed/password-crossview-admin-password.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -# crossview's break-glass native `admin` login password (ADMIN_PASSWORD), -# replacing the chart's well-known admin/password default. symbols:0 keeps it -# safe to pass as an env var without escaping. -apiVersion: generators.external-secrets.io/v1alpha1 -kind: Password -metadata: - name: crossview-admin-password - namespace: openbao -spec: - length: 32 - digits: 8 - symbols: 0 - noUpper: false - allowRepeat: true diff --git a/k8s/bases/infrastructure/vault-seed/password-crossview-db-password.yaml b/k8s/bases/infrastructure/vault-seed/password-crossview-db-password.yaml deleted file mode 100644 index 88140413e4..0000000000 --- a/k8s/bases/infrastructure/vault-seed/password-crossview-db-password.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# crossview's bundled-Postgres password. symbols:0 keeps it safe to pass as -# POSTGRES_PASSWORD / in the app's DB connection without escaping. -apiVersion: generators.external-secrets.io/v1alpha1 -kind: Password -metadata: - name: crossview-db-password - namespace: openbao -spec: - length: 32 - digits: 8 - symbols: 0 - noUpper: false - allowRepeat: true diff --git a/k8s/bases/infrastructure/vault-seed/password-crossview-session-secret.yaml b/k8s/bases/infrastructure/vault-seed/password-crossview-session-secret.yaml deleted file mode 100644 index 3fe5295f1d..0000000000 --- a/k8s/bases/infrastructure/vault-seed/password-crossview-session-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -# crossview's app session signing key (SESSION_SECRET). -apiVersion: generators.external-secrets.io/v1alpha1 -kind: Password -metadata: - name: crossview-session-secret - namespace: openbao -spec: - length: 64 - digits: 16 - symbols: 0 - noUpper: false - allowRepeat: true diff --git a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-admin-password.yaml b/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-admin-password.yaml deleted file mode 100644 index 79843e57bc..0000000000 --- a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-admin-password.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: external-secrets.io/v1alpha1 -kind: PushSecret -metadata: - name: push-crossview-admin-password - namespace: openbao -spec: - refreshInterval: 1h - secretStoreRefs: - - name: openbao - kind: ClusterSecretStore - selector: - secret: - name: generated-crossview-admin-password - data: - - match: - secretKey: password - remoteRef: - remoteKey: apps/crossview/admin - property: admin-password diff --git a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-db-password.yaml b/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-db-password.yaml deleted file mode 100644 index 8b1ff4676a..0000000000 --- a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-db-password.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: external-secrets.io/v1alpha1 -kind: PushSecret -metadata: - name: push-crossview-db-password - namespace: openbao -spec: - refreshInterval: 1h - secretStoreRefs: - - name: openbao - kind: ClusterSecretStore - selector: - secret: - name: generated-crossview-db-password - data: - - match: - secretKey: password - remoteRef: - remoteKey: apps/crossview/db - property: db-password diff --git a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-session-secret.yaml b/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-session-secret.yaml deleted file mode 100644 index b5404994f6..0000000000 --- a/k8s/bases/infrastructure/vault-seed/push-secret-push-crossview-session-secret.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: external-secrets.io/v1alpha1 -kind: PushSecret -metadata: - name: push-crossview-session-secret - namespace: openbao -spec: - refreshInterval: 1h - secretStoreRefs: - - name: openbao - kind: ClusterSecretStore - selector: - secret: - name: generated-crossview-session-secret - data: - - match: - secretKey: password - remoteRef: - remoteKey: apps/crossview/app - property: session-secret diff --git a/k8s/bases/infrastructure/vault-seed/push-secret-seed-dex-client-secret.yaml b/k8s/bases/infrastructure/vault-seed/push-secret-seed-dex-client-secret.yaml index 77d4ecaed7..a2ffd5eeb7 100644 --- a/k8s/bases/infrastructure/vault-seed/push-secret-seed-dex-client-secret.yaml +++ b/k8s/bases/infrastructure/vault-seed/push-secret-seed-dex-client-secret.yaml @@ -2,9 +2,8 @@ # OIDC client/cookie secrets — single source of truth is the SOPS bootstrap # secret (variables-cluster), NOT a generator. Dex's staticClients and # oauth2-proxy consume these very values via Flux post-build substitution -# (${dex_client_secret} / ${oauth2_proxy_cookie_secret}), while headlamp / -# actual-budget / the -# OpenBao OIDC auth method read them back from these vault paths. When the +# (${dex_client_secret} / ${oauth2_proxy_cookie_secret}), while actual-budget +# and the OpenBao OIDC auth method read them back from these vault paths. When the # vault copy was generated independently (the pre-2026-06-10 design), the # two copies could never match — dex rejected every vault-sourced client # secret. Seeding from the same SOPS key makes both sides identical. diff --git a/k8s/clusters/prod/bootstrap/config-map.yaml b/k8s/clusters/prod/bootstrap/config-map.yaml index 07836e2c0f..d12bcaf9af 100644 --- a/k8s/clusters/prod/bootstrap/config-map.yaml +++ b/k8s/clusters/prod/bootstrap/config-map.yaml @@ -58,11 +58,9 @@ data: tetragon_operator_replicas: "2" # Always-on UIs (the KEDA HTTPScaledObjects that used to scale these to zero # were removed). Static single replica each; auto-vpa right-sizes them at idle - # so they stay cheap. headlamp: OIDC login state is in-memory per pod, so a 2nd - # pod would break the callback — must stay single (see validate-replica-floor). - # whoami: diagnostic echo, one replica is plenty. fleetdm: disabled in prod - # since 2026-06-03, so this var is idle until re-enable. - headlamp_replicas: "1" + # so they stay cheap. whoami: diagnostic echo, one replica is plenty. + # fleetdm: disabled in prod since 2026-06-03, so this var is idle until + # re-enable. whoami_replicas: "1" fleetdm_replicas: "1" # Standalone MySQL (no replication). The bitnami replication topology causes diff --git a/k8s/providers/docker/apps/headlamp/patches/trust-platform-ca.yaml b/k8s/providers/docker/apps/headlamp/patches/trust-platform-ca.yaml deleted file mode 100644 index e96f3c962b..0000000000 --- a/k8s/providers/docker/apps/headlamp/patches/trust-platform-ca.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: headlamp - namespace: headlamp -spec: - # Required by the HelmRelease v2 schema so ksail (>= 7.26.0) can validate this - # opt-in patch standalone; matches the base, so it's a no-op once merged in. - interval: 2m - values: - config: - extraArgs: - - "-oidc-ca-file=/etc/ssl/certs/platform-ca/ca-certificates.crt" - volumes: - - name: trust-bundle - configMap: - name: platform-ca - volumeMounts: - - name: trust-bundle - mountPath: /etc/ssl/certs/platform-ca - readOnly: true diff --git a/k8s/providers/docker/apps/kustomization.yaml b/k8s/providers/docker/apps/kustomization.yaml index ae91f79f5f..6ea2271b93 100644 --- a/k8s/providers/docker/apps/kustomization.yaml +++ b/k8s/providers/docker/apps/kustomization.yaml @@ -7,13 +7,12 @@ kind: Kustomization # # To enable one or more apps locally, replace `resources: []` below with the # entries you want (and add the matching `patches:` block for any app that has -# one — currently actual-budget and headlamp), then run: +# one — currently actual-budget), then run: # ksail workload push && ksail workload reconcile # # Available apps (copy the lines you need): # resources: # - ../../../bases/apps/actual-budget/ # also add the actual-budget patch below -# - ../../../bases/apps/headlamp/ # also add the headlamp patch below # - ../../../bases/apps/homepage/ # - ../../../bases/apps/whoami/ # @@ -23,11 +22,6 @@ kind: Kustomization # name: actual-budget # namespace: actual-budget # path: actual-budget/patches/shrink-persistence.yaml -# - target: -# kind: HelmRelease -# name: headlamp -# namespace: headlamp -# path: headlamp/patches/trust-platform-ca.yaml # # Tenant RGD pilot (#1932 step 3) — opt-in like the apps above. Enabling it has # the KRO controller expand ascoachingogvaner's full control-plane skeleton from diff --git a/k8s/providers/hetzner/apps/crossview/cilium-network-policy-login-alerter.yaml b/k8s/providers/hetzner/apps/crossview/cilium-network-policy-login-alerter.yaml deleted file mode 100644 index 13d26cc8df..0000000000 --- a/k8s/providers/hetzner/apps/crossview/cilium-network-policy-login-alerter.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Let only the crossview-login-alerter CronJob (observability, hetzner overlay) -# reach the Crossview app on 3001, where it reads the unauthenticated -# /api/auth/check to notice a database that has lost its schema (#3315). -# Additive to allow-crossview: that base policy keeps public traffic behind -# oauth2-proxy and auth-proxy, and this prod-only policy opens one more path, -# scoped to the alerter's pod label, to the app pods only. The observability -# side already allows egress to every namespace (allow-coroot). -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: allow-login-alerter - namespace: crossview -spec: - endpointSelector: - matchLabels: - app.kubernetes.io/name: crossview - app.kubernetes.io/component: app - ingress: - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: observability - app: crossview-login-alerter - toPorts: - - ports: - - port: "3001" - protocol: TCP diff --git a/k8s/providers/hetzner/apps/crossview/cilium-network-policy.yaml b/k8s/providers/hetzner/apps/crossview/cilium-network-policy.yaml deleted file mode 100644 index 1f824f439f..0000000000 --- a/k8s/providers/hetzner/apps/crossview/cilium-network-policy.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Let only the Coroot cluster-agent reach the bundled Crossview PostgreSQL pod -# on 5432. This is additive to allow-crossview: that base policy keeps app and -# database traffic inside the namespace, while this prod-only policy opens the -# one cross-namespace path used by the declarative PostgreSQL integration. -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: allow-coroot-postgres-scrape - namespace: crossview -spec: - endpointSelector: - matchLabels: - app.kubernetes.io/component: postgres - ingress: - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: observability - app.kubernetes.io/managed-by: coroot-operator - app.kubernetes.io/part-of: coroot - app.kubernetes.io/component: coroot-cluster-agent - io.cilium.k8s.policy.serviceaccount: coroot-cluster-agent - toPorts: - - ports: - - port: "5432" - protocol: TCP diff --git a/k8s/providers/hetzner/apps/crossview/config-map-coroot-monitor-init.yaml b/k8s/providers/hetzner/apps/crossview/config-map-coroot-monitor-init.yaml deleted file mode 100644 index 6e9e27f26a..0000000000 --- a/k8s/providers/hetzner/apps/crossview/config-map-coroot-monitor-init.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# The Crossview chart uses the upstream postgres image directly rather than -# CloudNativePG, so its monitoring role and extension must be initialized by the -# image entrypoint. The script runs only for a fresh data directory; Crossview's -# database was ephemeral before #3319; after the durable-storage migration the -# same script initializes only a genuinely fresh claim. psql imports the -# password from the process environment so the credential never appears in the -# command line or this public ConfigMap. -apiVersion: v1 -kind: ConfigMap -metadata: - name: crossview-postgres-coroot-monitor-init - namespace: crossview -data: - init-coroot-monitor.sh: | - #!/usr/bin/env bash - set -Eeuo pipefail - - # Annotation-based Coroot targets always connect to the postgres - # maintenance database, so the extension must exist there. - psql \ - --username "$POSTGRES_USER" \ - --dbname postgres \ - --no-psqlrc \ - --set=ON_ERROR_STOP=1 <<'SQL' - \getenv coroot_monitor_password COROOT_MONITOR_PASSWORD - CREATE EXTENSION IF NOT EXISTS pg_stat_statements; - SELECT format( - 'CREATE ROLE coroot_monitor LOGIN PASSWORD %L', - :'coroot_monitor_password' - ) - WHERE NOT EXISTS ( - SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'coroot_monitor' - ) \gexec - ALTER ROLE coroot_monitor LOGIN PASSWORD :'coroot_monitor_password'; - GRANT pg_monitor TO coroot_monitor; - - -- This durable marker distinguishes the new PVC-backed cluster from the - -- outgoing ephemeral cluster during the one-time cutover. The Crossview - -- app's init container waits for it before allowing the app process to - -- recreate its schema. Because docker-entrypoint-initdb.d runs only for an - -- empty data directory, the marker cannot appear on the outgoing cluster. - CREATE SCHEMA IF NOT EXISTS platform_bootstrap; - CREATE TABLE IF NOT EXISTS platform_bootstrap.gates ( - name text PRIMARY KEY, - completed_at timestamptz NOT NULL DEFAULT clock_timestamp() - ); - INSERT INTO platform_bootstrap.gates (name) - VALUES ('durable-storage-v1') - ON CONFLICT (name) DO NOTHING; - SQL diff --git a/k8s/providers/hetzner/apps/crossview/external-secret-coroot-monitor.yaml b/k8s/providers/hetzner/apps/crossview/external-secret-coroot-monitor.yaml deleted file mode 100644 index 738fc6e393..0000000000 --- a/k8s/providers/hetzner/apps/crossview/external-secret-coroot-monitor.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: crossview-postgres-coroot-monitor - namespace: crossview -spec: - refreshInterval: "0" - target: - name: crossview-postgres-coroot-monitor - creationPolicy: Owner - template: - type: kubernetes.io/basic-auth - data: - username: coroot_monitor - password: "{{ .password }}" - dataFrom: - - sourceRef: - generatorRef: - apiVersion: generators.external-secrets.io/v1alpha1 - kind: Password - name: crossview-postgres-coroot-monitor diff --git a/k8s/providers/hetzner/apps/crossview/password-coroot-monitor.yaml b/k8s/providers/hetzner/apps/crossview/password-coroot-monitor.yaml deleted file mode 100644 index d0c61ab541..0000000000 --- a/k8s/providers/hetzner/apps/crossview/password-coroot-monitor.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: generators.external-secrets.io/v1alpha1 -kind: Password -metadata: - name: crossview-postgres-coroot-monitor - namespace: crossview -spec: - length: 32 - digits: 8 - symbols: 0 - noUpper: false - allowRepeat: true diff --git a/k8s/providers/hetzner/apps/crossview/patches/enable-coroot-monitoring.yaml b/k8s/providers/hetzner/apps/crossview/patches/enable-coroot-monitoring.yaml deleted file mode 100644 index aa7f08a738..0000000000 --- a/k8s/providers/hetzner/apps/crossview/patches/enable-coroot-monitoring.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# Prod-only Coroot integration for the PostgreSQL Deployment rendered by the -# Crossview chart. Unlike the platform CNPG databases, this operand has no -# operator-managed role or extension API, so the chart's supported pod -# annotations are combined with one post-render patch for server settings and -# its docker-entrypoint init hook. -- op: add - path: /spec/values/database/podAnnotations - value: - coroot.com/postgres-scrape: "true" - coroot.com/postgres-scrape-port: "5432" - coroot.com/postgres-scrape-credentials-secret-name: crossview-postgres-coroot-monitor - coroot.com/postgres-scrape-credentials-secret-username-key: username - coroot.com/postgres-scrape-credentials-secret-password-key: password - # The chart's bundled PostgreSQL server does not enable TLS. Pod-to-pod - # traffic remains encrypted by the platform's Cilium WireGuard layer. - coroot.com/postgres-scrape-param-sslmode: disable -- op: add - path: /spec/values/database/extraEnv - value: - - name: COROOT_MONITOR_PASSWORD - valueFrom: - secretKeyRef: - name: crossview-postgres-coroot-monitor - key: password -- op: add - path: /spec/postRenderers/0/kustomize/patches/- - value: - target: - kind: Deployment - name: crossview-postgres - patch: |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: crossview-postgres - spec: - # The data directory is an emptyDir. A serialized RollingUpdate prevents - # the old and new database pods from serving different ephemeral - # clusters behind one Service during this settings-changing rollout. - # Keep the strategy type compatible with the live Deployment: changing - # it to Recreate through Helm's server-side apply leaves Kubernetes' - # defaulted rollingUpdate child in place and is rejected by the API. - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - template: - spec: - containers: - - name: postgres - args: - - -c - - shared_preload_libraries=pg_stat_statements - - -c - - track_io_timing=on - volumeMounts: - - name: coroot-monitor-init - mountPath: /docker-entrypoint-initdb.d - readOnly: true - volumes: - - name: coroot-monitor-init - configMap: - name: crossview-postgres-coroot-monitor-init - defaultMode: 365 -- op: add - path: /spec/postRenderers/0/kustomize/patches/- - value: - target: - kind: Deployment - name: crossview - patch: |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: crossview - spec: - template: - spec: - initContainers: - - name: wait-for-db - args: - - | - until PGPASSWORD="$DB_PASS" psql -X -w \ - -h crossview-postgres \ - -p "$DB_PORT" \ - -U "$DB_USER" \ - -d postgres \ - -tAc "SELECT 1 - FROM pg_catalog.pg_roles AS monitor - JOIN pg_catalog.pg_auth_members AS membership - ON membership.member = monitor.oid - JOIN pg_catalog.pg_roles AS granted_role - ON granted_role.oid = membership.roleid - WHERE monitor.rolname = 'coroot_monitor' - AND monitor.rolcanlogin - AND granted_role.rolname = 'pg_monitor' - AND EXISTS ( - SELECT 1 - FROM pg_catalog.pg_extension - WHERE extname = 'pg_stat_statements' - )" \ - 2>/dev/null | grep -qx '1'; do - echo "Waiting for the instrumented database to be ready..." - sleep 2 - done - echo "Instrumented database is ready and accepting connections!" diff --git a/k8s/providers/hetzner/apps/crossview/patches/enable-durable-storage.yaml b/k8s/providers/hetzner/apps/crossview/patches/enable-durable-storage.yaml deleted file mode 100644 index 32b8bd548b..0000000000 --- a/k8s/providers/hetzner/apps/crossview/patches/enable-durable-storage.yaml +++ /dev/null @@ -1,130 +0,0 @@ -# Production-only durable storage for Crossview's bundled PostgreSQL. The base -# stays ephemeral so Docker/local installs do not acquire a Longhorn dependency. -# Longhorn replicates the block volume across three nodes; PostgreSQL remains a -# single process because Crossview is a low-criticality read-only dashboard. -- op: replace - path: /spec/values/database/persistence/enabled - value: true -- op: add - path: /spec/values/database/persistence/size - value: 2Gi -- op: add - path: /spec/values/database/persistence/accessMode - value: ReadWriteOnce -- op: add - path: /spec/values/database/persistence/storageClass - value: longhorn-wffc -# Roll the application in the same release, but gate its startup on the marker -# written only while the new PVC-backed PostgreSQL cluster is initialized. A -# plain pod-template bump is insufficient: its wait-for-db container could -# otherwise connect to the outgoing ephemeral database before Service cutover. -- op: replace - path: /spec/values/podAnnotations/platform.devantler.tech~1db-bootstrap - value: 2026-09-15-durable-storage-v1 -# Replace the base's ephemeral database post-render patch. Replacing the patch -# text (rather than trying to delete its nested list items in a later renderer) -# lets the chart-owned postgres-storage mount survive strategic merge intact. -- op: replace - path: /spec/postRenderers/0/kustomize/patches/1/patch - value: | - apiVersion: apps/v1 - kind: Deployment - metadata: - name: crossview-postgres - spec: - # A ReadWriteOnce database volume cannot safely overlap old and new pods. - # Keep the live strategy type and serialize the rollout: for one replica, - # zero surge plus one unavailable terminates the old pod before replacement - # just like Recreate, without an incompatible strategy-type transition. - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - template: - spec: - # The namespace remains opted into user namespaces for the app. - # Longhorn's volume is not idmapped, so only the database explicitly - # retains the host user namespace. - hostUsers: true - # Match the storage consumers' tier so they cannot preempt the database. - # Unlike CNPG, a Deployment rolls on the changed Pod template directly. - priorityClassName: hcloud-volume-workload - containers: - - name: postgres - securityContext: - readOnlyRootFilesystem: true - volumeMounts: - - name: tmp - mountPath: /tmp - volumes: - - name: tmp - emptyDir: - sizeLimit: 128Mi -# The PVC is Helm-owned. Flux prune protection on the HelmRelease does not stop -# Helm deleting a child that disappears from a later chart render, so retain the -# claim explicitly and retire it only through the repository's two-stage flow. -- op: add - path: /spec/postRenderers/0/kustomize/patches/- - value: - target: - kind: PersistentVolumeClaim - name: crossview-postgres-pvc - patch: |- - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: crossview-postgres-pvc - annotations: - helm.sh/resource-policy: keep -# The Coroot integration patch already strengthens wait-for-db to require the -# monitoring role and extension. This later patch additionally requires the -# durable init marker, so the replacement app cannot bootstrap against the -# outgoing ephemeral database even if both Deployments reconcile concurrently. -- op: add - path: /spec/postRenderers/0/kustomize/patches/- - value: - target: - kind: Deployment - name: crossview - patch: |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: crossview - spec: - template: - spec: - initContainers: - - name: wait-for-db - args: - - | - until PGPASSWORD="$DB_PASS" psql -X -w \ - -h crossview-postgres \ - -p "$DB_PORT" \ - -U "$DB_USER" \ - -d postgres \ - -tAc "SELECT 1 - FROM pg_catalog.pg_roles AS monitor - JOIN pg_catalog.pg_auth_members AS membership - ON membership.member = monitor.oid - JOIN pg_catalog.pg_roles AS granted_role - ON granted_role.oid = membership.roleid - WHERE monitor.rolname = 'coroot_monitor' - AND monitor.rolcanlogin - AND granted_role.rolname = 'pg_monitor' - AND EXISTS ( - SELECT 1 - FROM pg_catalog.pg_extension - WHERE extname = 'pg_stat_statements' - ) - AND EXISTS ( - SELECT 1 - FROM platform_bootstrap.gates - WHERE name = 'durable-storage-v1' - )" \ - 2>/dev/null | grep -qx '1'; do - echo "Waiting for the durable instrumented database to be ready..." - sleep 2 - done - echo "Durable instrumented database is ready and accepting connections!" diff --git a/k8s/providers/hetzner/apps/crossview/patches/enable-user-namespaces.yaml b/k8s/providers/hetzner/apps/crossview/patches/enable-user-namespaces.yaml deleted file mode 100644 index cee75375e5..0000000000 --- a/k8s/providers/hetzner/apps/crossview/patches/enable-user-namespaces.yaml +++ /dev/null @@ -1,3 +0,0 @@ -# Retired and intentionally unreferenced. Crossview's Longhorn-backed database -# requires the namespace to remain outside user-namespace enforcement; the live -# production kustomization therefore does not include this fragment. diff --git a/k8s/providers/hetzner/apps/headlamp/patches/enable-user-namespaces.yaml b/k8s/providers/hetzner/apps/headlamp/patches/enable-user-namespaces.yaml deleted file mode 100644 index cd9d260e43..0000000000 --- a/k8s/providers/hetzner/apps/headlamp/patches/enable-user-namespaces.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Opt headlamp into the label-gated user-namespace rules (#1807, #2843). -# -# Headlamp carries this label for a different reason than the other five -# namespaces, and the difference is the whole point of the change. -# -# Elsewhere the label drives the `add-user-namespaces` MUTATE rule, which -# injects hostUsers: false through a +(hostUsers) conditional anchor. That -# anchor cannot reach headlamp: its chart renders `hostUsers` from a -# values.yaml default of true, so the field is authored rather than absent. -# The value is set instead by the exact-target HelmRelease values patch in -# ../../kustomization.yaml, and that patch remains what actually hardens the -# workload. The mutate rule is a no-op here either way. -# -# What the label adds is the ENFORCE RATCHET — the hostUsers validation in -# validate-host-restrictions.yaml, which is scoped by this -# same label. Without it headlamp was the one user-namespaced workload whose -# isolation rested on a single line of kustomize config: drop that values -# patch in a chart bump, a bad conflict resolution or an overshooting revert, -# and the chart default of true returns, admission accepts it, no policy -# report is produced, and the namespace does not look wrong because it was -# never labelled. With the label, that same regression is rejected at -# admission instead. -# -# Safe to apply now: the pilot (#2651) is verified and closed, the running pod -# already sets hostUsers: false explicitly and so passes the hostUsers check -# unchanged, headlamp is the only workload in the namespace, and it holds no -# CNPG pods, so the policies' cnpg.io/cluster exclusion is moot here. Runtime plugin -# downloads and their Longhorn PVC have since been removed; the disposable -# userns-longhorn-smoke Job retains the storage proof independently (#2619). -# -# Hetzner-only, like the others: this label lives in the Hetzner overlay so the -# nested local Docker provider stays default-off. ---- -apiVersion: v1 -kind: Namespace -metadata: - name: headlamp - labels: - pod-security.devantler.tech/user-namespaces: enabled diff --git a/k8s/providers/hetzner/apps/kustomization.yaml b/k8s/providers/hetzner/apps/kustomization.yaml index a619fa1816..a0585ef7d9 100644 --- a/k8s/providers/hetzner/apps/kustomization.yaml +++ b/k8s/providers/hetzner/apps/kustomization.yaml @@ -36,10 +36,9 @@ resources: # ephemeral PVC and Longhorn volume are garbage-collected. # - userns-longhorn-smoke/ # Prod-only Coroot Postgres integrations for platform-owned databases: - # the Cluster patches below configure CNPG annotations, roles and extensions; - # the Crossview resources and HelmRelease patch configure its bundled - # PostgreSQL Deployment. Additive CiliumNetworkPolicies let only the - # observability cluster-agent scrape each database on 5432. + # the Cluster patches below configure CNPG annotations, roles and extensions. + # Additive CiliumNetworkPolicies let only the observability cluster-agent + # scrape each database on 5432. # coroot-db needs none — it is intra-namespace to the agent and already # covered by the base allow-coroot policy. The wedding-app TENANT owns its own # equivalent in its repo's deploy/ (cluster.yaml + networkpolicy.yaml), since @@ -47,16 +46,6 @@ resources: - umami/cilium-network-policy.yaml - umami/password-coroot-monitor.yaml - umami/external-secret-coroot-monitor.yaml - # Crossview bundles a plain PostgreSQL Deployment instead of CNPG. These - # resources supply its init hook, dedicated credential, and scrape-only - # network path; the HelmRelease patch below wires them into the chart output. - - crossview/config-map-coroot-monitor-init.yaml - - crossview/cilium-network-policy.yaml - - crossview/password-coroot-monitor.yaml - - crossview/external-secret-coroot-monitor.yaml - # Admits only the prod crossview-login-alerter CronJob (observability) to the - # app's port, so an empty database is reported without anyone logging in. - - crossview/cilium-network-policy-login-alerter.yaml components: - ../../../bases/components/annotations-transformers # Platform-wide HelmRelease drift detection (mode: enabled) — see @@ -99,15 +88,6 @@ patches: # merge: inheritedMetadata scrape annotations + pg_stat_statements params). # These self-identify, so no target. - path: umami/patches/enable-coroot-monitoring.yaml - # Crossview is a Helm-rendered Deployment, so its JSON6902 patch adds chart - # values and appends post-render patches without replacing the base list. - - target: - group: helm.toolkit.fluxcd.io - version: v2 - kind: HelmRelease - name: crossview - namespace: crossview - path: crossview/patches/enable-coroot-monitoring.yaml # umami's dedicated pg_monitor role is a JSON6902 append onto the existing # managed.roles list (so it doesn't replace / desync the OpenBao-rotation enforcer); a # JSON6902 patch has no embedded identity, so it keeps an explicit target. @@ -124,35 +104,5 @@ patches: # workloads apart from the Actual Budget pilot component above; in mixed # namespaces (umami) the rule itself excludes the # CNPG database pods via their cnpg.io/cluster label. - # Headlamp takes an exact-target values patch instead of a namespace label - # (#2651). Its chart renders `hostUsers: {{ .Values.hostUsers }}` from a - # values.yaml default of true, so the value is authored rather than absent and - # the add-security-context rule's +(hostUsers) conditional anchor — which only - # fills in a MISSING value — cannot reach it. Setting the chart value is what - # actually changes the rendered Deployment. - # Headlamp uses only ephemeral writable volumes now that runtime plugin - # downloads are disabled. The disposable userns-longhorn-smoke Job still - # carries the separate Longhorn idmapped-mount proof for future stateful - # workloads; this Headlamp activation no longer depends on that storage path. - - target: - group: helm.toolkit.fluxcd.io - version: v2 - kind: HelmRelease - name: headlamp - namespace: headlamp - patch: |- - - op: add - path: /spec/values/hostUsers - value: false - path: whoami/patches/enable-user-namespaces.yaml - path: homepage/patches/enable-user-namespaces.yaml - - target: - group: helm.toolkit.fluxcd.io - version: v2 - kind: HelmRelease - name: crossview - namespace: crossview - path: crossview/patches/enable-durable-storage.yaml - # Headlamp's label is the enforce ratchet only — the values patch above is - # what sets hostUsers, and the mutate rule cannot reach an authored value. - - path: headlamp/patches/enable-user-namespaces.yaml diff --git a/k8s/providers/hetzner/infrastructure/cluster-issuers/cloudflare-origin-ca-trust-bundle.yaml b/k8s/providers/hetzner/infrastructure/cluster-issuers/cloudflare-origin-ca-trust-bundle.yaml index ba155b685d..29936d3bc7 100644 --- a/k8s/providers/hetzner/infrastructure/cluster-issuers/cloudflare-origin-ca-trust-bundle.yaml +++ b/k8s/providers/hetzner/infrastructure/cluster-issuers/cloudflare-origin-ca-trust-bundle.yaml @@ -1,8 +1,9 @@ # Distributes the Cloudflare Origin ECC CA root + system CAs to all namespaces -# as the ConfigMap "cloudflare-origin-ca". Required so that in-cluster clients -# (e.g. Headlamp) can verify TLS connections to the Cilium gateway when CoreDNS -# resolves *.platform.devantler.tech directly to the in-cluster service instead -# of routing through Cloudflare's publicly-trusted edge. +# as the ConfigMap "cloudflare-origin-ca", so that in-cluster clients can +# verify TLS connections to the Cilium gateway when CoreDNS resolves +# *.platform.devantler.tech directly to the in-cluster service instead of +# routing through Cloudflare's publicly-trusted edge. No workload mounts it +# today; Headlamp, its last consumer, was removed. apiVersion: trust.cert-manager.io/v1alpha1 kind: Bundle metadata: diff --git a/k8s/providers/hetzner/infrastructure/controllers/alertmanager/cilium-network-policy.yaml b/k8s/providers/hetzner/infrastructure/controllers/alertmanager/cilium-network-policy.yaml index 933e98c848..7e68ac44e8 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/alertmanager/cilium-network-policy.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/alertmanager/cilium-network-policy.yaml @@ -16,10 +16,9 @@ spec: matchLabels: app.kubernetes.io/name: alertmanager ingress: - # The Headlamp Kubescape plugin reads GET /api/v2/alerts through the - # Kubernetes API server's Service proxy (/api/v1/namespaces/kubescape/ - # services/alertmanager:9093/proxy/...), so the connection to :9093 - # originates from the API server. Mirror the entity set the base + # Reads of GET /api/v2/alerts through the Kubernetes API server's Service + # proxy (/api/v1/namespaces/kubescape/services/alertmanager:9093/proxy/...) + # originate from the API server. Mirror the entity set the base # allow-kubescape policy uses for the webhook/host-scanner ports. - fromEntities: - kube-apiserver @@ -52,16 +51,6 @@ spec: - ports: - port: "9093" protocol: TCP - # The crossview-login-alerter CronJob (observability) POSTs /api/v2/alerts - # the same way when Crossview's database has lost its schema (#3315). - - fromEndpoints: - - matchLabels: - k8s:io.kubernetes.pod.namespace: observability - app: crossview-login-alerter - toPorts: - - ports: - - port: "9093" - protocol: TCP egress: # Post alert notifications to the Slack incoming-webhook. Pinned by FQDN # (not world:443) so a compromised pod cannot exfiltrate elsewhere — the same diff --git a/k8s/providers/hetzner/infrastructure/controllers/alertmanager/helm-release.yaml b/k8s/providers/hetzner/infrastructure/controllers/alertmanager/helm-release.yaml index b438a6798f..0777c8f803 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/alertmanager/helm-release.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/alertmanager/helm-release.yaml @@ -8,17 +8,14 @@ metadata: spec: # WHY THIS EXISTS (prod-only). The platform's general alerting was migrated off # the kube-prometheus-stack onto Coroot (docs/dr/alerting.md), so there is no - # Alertmanager in the cluster. But the Kubescape node-agent's runtime-detection - # alerts have exactly one first-class dashboard — the Headlamp Kubescape - # plugin's "Runtime Detection > Alerts" tab — and that tab reads ONLY from a - # Prometheus Alertmanager `GET /api/v2/alerts` (filtered on - # alertname="KubescapeRuleViolated"); it cannot read Kubescape storage CRs and - # cannot query Coroot's Prometheus (a metrics store, not an Alertmanager). So a - # minimal Alertmanager cluster is reintroduced here — scoped to the kubescape - # namespace and to runtime alerts only, NOT a general re-adoption of the - # Prometheus stack. The service both RECEIVES the node-agent's pushed alerts - # (POST /api/v2/alerts, wired via the nodeAgent.config.alertManagerExporterUrls - # patch in ../kubescape/patches/) and SERVES them to the Headlamp plugin (GET), + # Alertmanager in the cluster. But the Kubescape node-agent can only push its + # runtime-detection alerts to a Prometheus Alertmanager, and Coroot's + # Prometheus is a metrics store, not an Alertmanager. So a minimal Alertmanager + # cluster is reintroduced here — scoped to the kubescape namespace and to + # pushed alerts only, NOT a general re-adoption of the Prometheus stack. The + # service RECEIVES the node-agent's pushed alerts (POST /api/v2/alerts, wired + # via the nodeAgent.config.alertManagerExporterUrls patch in + # ../kubescape/patches/) and the platform's alerting CronJobs' alerts, # and routes them to Slack (slack_configs → the shared incoming-webhook). The # third destination, Coroot, is fed independently by the node-agent's stdout # exporter (Coroot's eBPF log capture surfaces the alerts in its Logs view; @@ -56,8 +53,8 @@ spec: values: # Deterministic in-cluster names. Producers post to every peer through the # headless Service (alertmanager-N.alertmanager-headless.kubescape.svc:9093), - # because received alerts are not replicated between peers; the Headlamp - # plugin reads through the Service alertmanager.kubescape.svc:9093. + # because received alerts are not replicated between peers; readers use the + # Service alertmanager.kubescape.svc:9093. fullnameOverride: alertmanager replicaCount: 2 # Keep the two peers on distinct nodes. The cluster-wide spread-pods policy diff --git a/k8s/providers/hetzner/infrastructure/controllers/ksail-operator/patches/enable-oidc.yaml b/k8s/providers/hetzner/infrastructure/controllers/ksail-operator/patches/enable-oidc.yaml index 300b9bdb2c..49dabae8c6 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/ksail-operator/patches/enable-oidc.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/ksail-operator/patches/enable-oidc.yaml @@ -7,7 +7,7 @@ # local/CI the *.platform.lan host-file names are not in-cluster resolvable. # # Reuses the shared `public-client` confidential client (same as oauth2-proxy / -# Vault / Headlamp) so no new SOPS secret is needed -- the redirect URI is +# Vault) so no new SOPS secret is needed -- the redirect URI is # registered on that client in the dex HelmRelease. This merges into spec.values # (it does not replace the base values). apiVersion: helm.toolkit.fluxcd.io/v2 diff --git a/k8s/providers/hetzner/infrastructure/controllers/kubescape/patches/install-default-runtime-rules.yaml b/k8s/providers/hetzner/infrastructure/controllers/kubescape/patches/install-default-runtime-rules.yaml index f407afa717..3a5c6466e9 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kubescape/patches/install-default-runtime-rules.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kubescape/patches/install-default-runtime-rules.yaml @@ -18,7 +18,7 @@ spec: # defaults it to false. Turning it on adds exactly those two objects and # nothing else (no RBAC, workload or CRD change). The binding covers every # pod outside `excludeNamespaces`, and alerts then flow to the existing - # Alertmanager (Slack, Headlamp) and stdout (Coroot) routes. + # Alertmanager (Slack) and stdout (Coroot) routes. # # It stays off until the rollout is decided: the anomaly rules depend on # complete application profiles, several workloads have only partial ones, diff --git a/k8s/providers/hetzner/infrastructure/controllers/kubescape/patches/route-runtime-detection-alerts.yaml b/k8s/providers/hetzner/infrastructure/controllers/kubescape/patches/route-runtime-detection-alerts.yaml index 67f7ff9e67..9ab816ec22 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kubescape/patches/route-runtime-detection-alerts.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kubescape/patches/route-runtime-detection-alerts.yaml @@ -9,20 +9,19 @@ spec: values: nodeAgent: config: - # Fan the runtime-detection alerts out to their three destinations + # Fan the runtime-detection alerts out to their two destinations # (prod-only — runtimeDetection is disabled on the docker/local overlay). # - # 1. Headlamp + 2. Slack — via the in-cluster Alertmanager deployed in - # ../alertmanager/. The node-agent POSTs alerts to it; the Headlamp - # Kubescape plugin's "Runtime Detection > Alerts" tab reads them back - # (GET /api/v2/alerts), and the Alertmanager Slack receiver forwards - # them to the shared incoming-webhook. The value is HOST:PORT ONLY + # 1. Slack — via the in-cluster Alertmanager deployed in + # ../alertmanager/. The node-agent POSTs alerts to it, and the + # Alertmanager Slack receiver forwards them to the shared + # incoming-webhook. The value is HOST:PORT ONLY # (no scheme, no path) — the node-agent's Prometheus Alertmanager # client appends /api/v2/alerts itself. Alertmanager HA does not # replicate received alerts between peers, so each StatefulSet pod # is listed through the headless Service; the load-balanced # Service would deliver every alert to only one peer. - # 3. Coroot — via stdout. Coroot CE has no inbound alert receiver, but + # 2. Coroot — via stdout. Coroot CE has no inbound alert receiver, but # its eBPF node-agent captures this pod's stdout, so the alerts show # up in Coroot's Logs view (and its log-pattern inspection). This is # the chart default (stdoutExporter: true); pinned here so the Coroot diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index 9d393e7988..536378384e 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -12,12 +12,11 @@ resources: # registration is repaired. Flux reconciles the controller with its watch # scope limited to arc-runners; the organization pool stays outside aggregates. - ../../../../bases/infrastructure/controllers/actions-runner-controller/ - # Prod-only: a minimal Alertmanager scoped to Kubescape runtime-detection - # alerts. The Headlamp Kubescape plugin's "Runtime Detection > Alerts" tab - # reads ONLY from a Prometheus Alertmanager (GET /api/v2/alerts), which the - # Coroot migration removed from the cluster; this reintroduces one, fed by the - # node-agent (kubescape/patches/ below) and routing to Slack. See its - # helm-release.yaml header. Prod-only (runtimeDetection is off on docker/local). + # Prod-only: a minimal Alertmanager for pushed alerts. The Kubescape + # node-agent can push runtime-detection alerts only to a Prometheus + # Alertmanager, which the Coroot migration removed from the cluster; this + # reintroduces one, fed by the node-agent (kubescape/patches/ below) and + # routing to Slack. See its helm-release.yaml header. Prod-only (runtimeDetection is off on docker/local). - alertmanager/ # Cilium is supplied by the base controller aggregate. Authentication must # be scoped inside each workload's narrow allow rule: an empty @@ -156,7 +155,7 @@ patches: - path: ksail-operator/patches/enable-oidc.yaml # Prod-only: point the Kubescape node-agent's runtime-detection alerts at the # in-cluster Alertmanager (alertmanager/ above) + keep the stdout exporter on, - # fanning them out to the Headlamp plugin, Slack, and Coroot — see file. + # fanning them out to Slack and Coroot — see file. - path: kubescape/patches/route-runtime-detection-alerts.yaml # Prod-only release switch for the node-agent's default detection rules # (#4968). OFF: runtime detection runs with no rules until it is turned on. diff --git a/k8s/providers/hetzner/infrastructure/coroot/config-map-risk-acceptances.yaml b/k8s/providers/hetzner/infrastructure/coroot/config-map-risk-acceptances.yaml index 0c7544f267..3a072d9a22 100644 --- a/k8s/providers/hetzner/infrastructure/coroot/config-map-risk-acceptances.yaml +++ b/k8s/providers/hetzner/infrastructure/coroot/config-map-risk-acceptances.yaml @@ -10,9 +10,7 @@ # # Four initial warnings are deliberately absent: Alertmanager and the Longhorn # CSI snapshotter get warm replicas, Origin CA Issuer is disabled, and #3818 -# removes the superseded OpenCost usage scraper. Crossview PostgreSQL is accepted -# only after #3829 delivered a retained, three-node-replicated Longhorn volume -# and live Coroot PostgreSQL collection. +# removes the superseded OpenCost usage scraper. apiVersion: v1 kind: ConfigMap metadata: @@ -69,24 +67,6 @@ data: "type": "single-instance-app", "reason": "platform#3812: telemetry exporter is off the serving path; a short scrape gap is acceptable" }, - { - "application": "crossview:Deployment:crossview", - "category": "Availability", - "type": "single-instance-app", - "reason": "platform#3812: low-criticality internal UI; fast rescheduling meets its recovery objective" - }, - { - "application": "crossview:Deployment:crossview-postgres", - "category": "Availability", - "type": "single-instance-app", - "reason": "platform#3812: single-writer low-criticality database on retained cross-node storage; Kubernetes rescheduling meets its recovery objective" - }, - { - "application": "crossview:Deployment:crossview-postgres", - "category": "Availability", - "type": "unreplicated-database", - "reason": "platform#3812: low-criticality dashboard database on a cross-node replicated Longhorn volume; pod rescheduling meets its recovery objective" - }, { "application": "kube-system:Deployment:descheduler", "category": "Availability", @@ -111,12 +91,6 @@ data: "type": "single-instance-app", "reason": "platform#3812: operator is off the serving path; the Flux controllers continue reconciling workloads" }, - { - "application": "headlamp:Deployment:headlamp", - "category": "Availability", - "type": "single-instance-app", - "reason": "platform#3812: operator-only UI; the Kubernetes API remains available during rescheduling" - }, { "application": "kube-system:Deployment:hubble-ui", "category": "Availability", diff --git a/k8s/providers/hetzner/infrastructure/coroot/cron-job-alert-autosuppressor.yaml b/k8s/providers/hetzner/infrastructure/coroot/cron-job-alert-autosuppressor.yaml index 258545b1b0..a471b29849 100644 --- a/k8s/providers/hetzner/infrastructure/coroot/cron-job-alert-autosuppressor.yaml +++ b/k8s/providers/hetzner/infrastructure/coroot/cron-job-alert-autosuppressor.yaml @@ -68,12 +68,8 @@ # BuildFailed labels, and exact cancellation shapes qualify; timeouts, # apply/authorization errors, and dependency failures remain visible. # -# 5. crossview-monitor-secret-create-race - Coroot and External Secrets can race -# to create the generated crossview-postgres-coroot-monitor Secret during a -# database roll. The loser reports AlreadyExists, then the ExternalSecret -# immediately reaches Ready/SecretSynced and continues owning the target. -# Only that exact object, reason, source, and AlreadyExists message qualify; -# forbidden, refresh, provider, and all other ExternalSecret errors remain. +# 5. Retired with Crossview: the generated Secret it exempted no longer +# exists, so the exact race it matched cannot recur. # # 6. cilium-dns-search-expansion - Cilium's DNS proxy aggregates resolver # attempts from every workload into the Cilium DaemonSet application. The @@ -363,27 +359,6 @@ spec: ids="$ids $id"; log_info "match flux-status-context-canceled: $id" done - # Exemption 5 - the exact ExternalSecret target-creation race - # during Crossview PostgreSQL reconciliation. - for id in $(printf '%s' "$ALERTS" | jq -r ' - def details: (.details // [] | map({key: .name, value: .value}) | from_entries); - def labels($d): ($d.Labels | split("\n") | map(capture("^(?[^=]+)=\"(?.*)\"$")) | from_entries); - .data.alerts[] - | select(.suppressed==false and .resolved_at==null and .rule_id=="kubernetes-events") - | details as $d - | labels($d) as $l - | select(($l | keys | sort) == ["cluster","kind","name","namespace","reason","source"] - and $l.cluster=="platform" - and $l.kind=="ExternalSecret" - and $l.name=="crossview-postgres-coroot-monitor" - and $l.namespace=="crossview" - and $l.reason=="UpdateFailed" - and $l.source=="external-secrets" - and $d["Event message"]=="secrets \"crossview-postgres-coroot-monitor\" already exists") - | .id'); do - ids="$ids $id"; log_info "match crossview-monitor-secret-create-race: $id" - done - # Exemption 6 - Kubernetes resolver search-list candidates # observed by Cilium's cluster-wide DNS proxy. Preserve every # other DNS check and every other application's NXDOMAINs. diff --git a/k8s/providers/hetzner/infrastructure/coroot/cron-job-crossview-login-alerter.yaml b/k8s/providers/hetzner/infrastructure/coroot/cron-job-crossview-login-alerter.yaml deleted file mode 100644 index bdc7997d08..0000000000 --- a/k8s/providers/hetzner/infrastructure/coroot/cron-job-crossview-login-alerter.yaml +++ /dev/null @@ -1,251 +0,0 @@ -# Alerts when Crossview's login cannot work because the app cannot find its -# users in its database: the schema is gone, or the database is unreachable. -# -# WHY THIS EXISTS -# -# Crossview creates its tables and its break-glass admin once, at its own -# startup. When its database comes back empty underneath a running app, every -# OIDC sign-in fails with `relation "users" does not exist`, yet the pod stays -# Ready: the chart's liveness and readiness probe is `/api/health`, which never -# touches the database. That is how login stayed broken for six days in August -# 2026 with every probe green (platform#3315). Durable storage (platform#3319) -# makes an empty database rarer; it does not make one visible. -# -# WHAT IT CHECKS -# -# `GET /api/auth/check` is unauthenticated and answers 200 in both states, so -# no HTTP probe can tell them apart. Its body can: a bootstrapped Crossview -# always has the admin it seeds at startup, so `"hasAdmin":true` or -# `"hasUsers":true` shows the app can read its users table and is healthy. -# Both `false` means it cannot: the empty schema showing through, as in the -# outage, or a database it cannot reach. Every sign-in fails in both cases, so -# both alert; the runbook says how to tell them apart, because only the first -# needs the app rolled. -# -# NO FLAPPING. One healthy read ends the run. An alert needs every one of -# SAMPLES reads, SAMPLE_INTERVAL_SECONDS apart, to answer both false: six -# minutes, longer than a planned database restart takes to detach, reattach and -# start its volume, so routine maintenance does not page. A lost schema is -# persistent: it lasts until the app restarts. -# -# NO SILENT ZERO. A read that is not a 200 with boolean `hasAdmin` and -# `hasUsers` is not an answer, so a run that never gets a definitive one fails the Job instead of -# reporting health. This CronJob is on the `cronjob-failure-alert` watch list -# (bases/components/coroot-cronjob-failure-alert), which alerts in Slack when two -# runs in a row fail or no run has finished for an hour. That also covers an -# upstream change to the endpoint's response shape. -# -# DELIVERY follows cron-job-stranded-autoscaler-node-alerter.yaml: POST to every -# Alertmanager peer's v2 API, which fans out to Slack. The alert carries an -# `endsAt` of three missed runs, so it resolves once Crossview can read its users -# again and a broken Job degrades to "the alert resolves", never a stuck alert. -# -# READ-ONLY BY CONSTRUCTION. It reads one public endpoint and needs no -# Kubernetes API access, so no ServiceAccount token is mounted. Restoring login -# stays a reviewed change: see the runbook annotation below. -# -# PROD-ONLY: Crossview, Alertmanager and the Slack webhook are wired only in the -# hetzner overlay. providers/hetzner/apps/crossview/cilium-network-policy-login-alerter.yaml -# admits this pod (and only this pod label) to Crossview's port. Retire this -# CronJob together with Crossview. -apiVersion: batch/v1 -kind: CronJob -metadata: - name: crossview-login-alerter - namespace: observability - annotations: - # SAMPLES and SAMPLE_INTERVAL_SECONDS below are pod environment variables, - # not Flux inputs. The infrastructure Kustomization runs post-build - # substitution, which would otherwise blank them. This resource has no Flux - # substitutions of its own. - kustomize.toolkit.fluxcd.io/substitute: disabled -spec: - # Every 15 minutes: an outage that took six days to notice now surfaces within - # about twenty-five. Must stay well under the 45-minute endsAt below. - schedule: "*/15 * * * *" - concurrencyPolicy: Forbid - startingDeadlineSeconds: 60 - successfulJobsHistoryLimit: 1 - # Must be at least the cronjob-failure-alert threshold for this target (2), or - # the detector cannot observe the streak it alerts on. - failedJobsHistoryLimit: 3 - jobTemplate: - spec: - backoffLimit: 0 - # Worst case: --retry-max-time only stops NEW attempts, so a read can take - # its 30s retry budget plus one more 10s attempt (40s), and a delivery 20s - # plus 10s (30s) per Alertmanager peer. SAMPLES reads, (SAMPLES - 1) waits - # and two peers: 4*40 + 3*120 + 2*30 = 580s. - activeDeadlineSeconds: 720 - template: - metadata: - labels: - app: crossview-login-alerter - spec: - restartPolicy: Never - # Reads one unauthenticated HTTP endpoint; it has no use for a token. - automountServiceAccountToken: false - securityContext: - runAsNonRoot: true - runAsUser: 65532 - fsGroup: 65532 - seccompProfile: - type: RuntimeDefault - # The two non-privilege C-0211 fields the cluster-wide - # add-security-context mutation cannot supply in observability - # (#3239). Values match add-security-context.yaml. - fsGroupChangePolicy: OnRootMismatch - containers: - - name: alerter - # curl + jq, digest-pinned (same image as the other alerters). - # observability is exempt from disallow-latest-tag. - image: docker.io/badouralix/curl-jq:latest@sha256:1e7c0284e24572ace7170df9fc91f15fd3b79ebf056d4dde17244d5d74bbfabc - imagePullPolicy: IfNotPresent - env: - # Every read must answer that no user is readable before the - # alert fires: four reads two minutes apart span six minutes. - - name: SAMPLES - value: "4" - - name: SAMPLE_INTERVAL_SECONDS - value: "120" - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - # CIS-5.7.3 (C-0211); a documented no-op on this AppArmor cluster. - seLinuxOptions: {} - capabilities: - drop: - - ALL - resources: - requests: - cpu: 5m - memory: 16Mi - ephemeral-storage: 1Mi - limits: - cpu: 100m - memory: 64Mi - ephemeral-storage: 8Mi - volumeMounts: - - name: scratch - mountPath: /tmp - command: - - /bin/sh - - -c - - | - set -eu - # Alertmanager HA does not replicate received alerts between - # peers, so every alert goes to each StatefulSet pod directly. - AM_PEERS="http://alertmanager-0.alertmanager-headless.kubescape.svc.cluster.local.:9093 http://alertmanager-1.alertmanager-headless.kubescape.svc.cluster.local.:9093" - CHECK_URL="http://crossview-service.crossview.svc.cluster.local.:80/api/auth/check" - BODY="/tmp/auth-check.json" - - case "$SAMPLES" in - [1-9]) ;; - *) echo "ERROR: SAMPLES must be a single digit from 1 to 9 (got '$SAMPLES')"; exit 1 ;; - esac - case "$SAMPLE_INTERVAL_SECONDS" in - ''|*[!0-9]*) echo "ERROR: SAMPLE_INTERVAL_SECONDS must be a non-negative integer (got '$SAMPLE_INTERVAL_SECONDS')"; exit 1 ;; - esac - if [ "${#SAMPLE_INTERVAL_SECONDS}" -gt 3 ]; then - echo "ERROR: SAMPLE_INTERVAL_SECONDS must stay below 1000 to fit the Job deadline" - exit 1 - fi - - missing=0 - sample=0 - observed="" - while [ "$sample" -lt "$SAMPLES" ]; do - sample=$((sample + 1)) - if [ "$sample" -gt 1 ]; then - sleep "$SAMPLE_INTERVAL_SECONDS" - fi - rm -f "$BODY" - # A refused or timed-out connection says nothing about the - # schema, so it is recorded as no answer, never as health. - code=$(curl -sS --retry 3 --retry-delay 2 --retry-all-errors \ - --max-time 10 --retry-max-time 30 -o "$BODY" -w '%{http_code}' \ - "$CHECK_URL") || code="000" - if [ "$code" != "200" ] || [ ! -s "$BODY" ]; then - echo "sample ${sample}/${SAMPLES}: no answer (HTTP ${code}) from ${CHECK_URL}" - continue - fi - # Only boolean hasAdmin and hasUsers are an answer. Anything - # else means the endpoint's contract changed, which must not - # read as health. Either one true proves the users table is - # readable, so a deleted break-glass admin alone never pages. - verdict=$(jq -r ' - if type == "object" and (.hasAdmin | type) == "boolean" - and (.hasUsers | type) == "boolean" - then (if .hasAdmin or .hasUsers then "readable" else "missing" end) - else "unrecognised" end' "$BODY" 2>/dev/null) || verdict="unrecognised" - case "$verdict" in - readable) - echo "sample ${sample}/${SAMPLES}: $(jq -c '{hasAdmin, hasUsers}' "$BODY"); Crossview can read its users" - exit 0 - ;; - missing) - missing=$((missing + 1)) - observed=$(jq -c '{authMode, hasAdmin, hasUsers}' "$BODY") - echo "sample ${sample}/${SAMPLES}: no readable user ${observed}" - ;; - *) - echo "sample ${sample}/${SAMPLES}: HTTP 200 without boolean hasAdmin and hasUsers" - ;; - esac - done - - if [ "$missing" -ne "$SAMPLES" ]; then - echo "ERROR: ${missing} of ${SAMPLES} reads found no readable user and none found one; that is neither healthy nor a confirmed outage" - exit 1 - fi - - # endsAt = three missed runs after this POST. jq formats it: - # BusyBox date has no relative offsets, and only `+%s` is - # portable. Computed after sampling, so the hold-over is not - # shortened by the time the reads took. - ENDS=$(jq -nr --argjson t "$(date -u +%s)" '($t + 2700) | todate') - case "$ENDS" in - ????-??-??T??:??:??Z) ;; - *) echo "ERROR: could not compute endsAt (got '$ENDS')"; exit 1 ;; - esac - - alerts=$(jq -cn --arg e "$ENDS" --arg observed "$observed" \ - --arg n "$SAMPLES" --arg s "$SAMPLE_INTERVAL_SECONDS" '[{ - labels: { - alertname: "CrossviewLoginBroken", - severity: "warning", - namespace: "crossview" - }, - annotations: { - summary: "Crossview cannot read its users, so every sign-in fails", - description: "GET /api/auth/check answered \($observed) on all \($n) reads, \($s)s apart. A bootstrapped Crossview always has the break-glass admin it creates at its own startup, so no admin and no users means the app cannot read its users table: its database has lost the schema, or the app cannot reach it. OIDC sign-in fails either way while the pod stays Ready, because its probe (/api/health) never touches the database.", - runbook: "Check the crossview-postgres Deployment in the crossview namespace and database errors reported by Crossview. Restore database reachability or authentication if those fail; the app reconnects on its own once the database is back with its data. A Ready database alone does not prove schema loss. If the errors confirm a missing users table, roll the app so it re-runs its one-shot schema bootstrap by bumping the platform.devantler.tech/db-bootstrap value in providers/hetzner/apps/crossview/patches/enable-durable-storage.yaml, then find out why the database came back empty (crossview-postgres-pvc). The alert resolves on its own once a read finds a user again." - }, - endsAt: $e - }]') - echo "Crossview cannot read its users; alerting" - - # --fail makes a rejected POST observable; the Job fails only - # when no peer accepted the alert. - delivered=0 - for am in $AM_PEERS; do - if curl -sS --retry 3 --retry-delay 2 --retry-all-errors --max-time 10 --retry-max-time 20 \ - --fail -X POST -H 'Content-Type: application/json' \ - -d "$alerts" "$am/api/v2/alerts" >/dev/null; then - delivered=$((delivered + 1)) - else - echo "WARN: $am rejected the POST" - fi - done - if [ "$delivered" -eq 0 ]; then - echo "ERROR: no alertmanager peer accepted the POST; the alert was NOT delivered" - exit 1 - fi - echo "posted the alert to $delivered alertmanager peer(s)" - volumes: - - name: scratch - emptyDir: - sizeLimit: 1Mi diff --git a/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml b/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml index 8a40c2d715..14af5fbe49 100644 --- a/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml +++ b/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml @@ -105,9 +105,6 @@ spec: - kind: HelmRelease name: "*" namespace: crossplane-system - - kind: HelmRelease - name: "*" - namespace: crossview - kind: HelmRelease name: "*" namespace: data-product-controller @@ -126,9 +123,6 @@ spec: - kind: HelmRelease name: "*" namespace: flux-system - - kind: HelmRelease - name: "*" - namespace: headlamp - kind: HelmRelease name: "*" namespace: homepage diff --git a/k8s/providers/hetzner/infrastructure/kustomization.yaml b/k8s/providers/hetzner/infrastructure/kustomization.yaml index 86edf56773..d09f568dec 100644 --- a/k8s/providers/hetzner/infrastructure/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/kustomization.yaml @@ -122,10 +122,6 @@ resources: - coroot/service-account-stranded-autoscaler-node-alerter.yaml - coroot/cluster-role-stranded-autoscaler-node-alerter.yaml - coroot/cluster-role-binding-stranded-autoscaler-node-alerter.yaml - # Prod-only: alerts through Alertmanager when Crossview cannot read its admin - # or any other user: the schema is missing or the database is unreachable, - # while its probe stays green (#3315). See the file header. - - coroot/cron-job-crossview-login-alerter.yaml # Prod-only: the CloudNativePG datastore + backup wiring that lets the Coroot # SERVER run HA (replicas: 2 — set in the coroot-patch below). The operator # only honours replicas > 1 when spec.postgres is set, so this Postgres is the diff --git a/scripts/generate-kubescape-exceptions/headlamp-configmap.yaml.tmpl b/scripts/generate-kubescape-exceptions/headlamp-configmap.yaml.tmpl deleted file mode 100644 index 87df063273..0000000000 --- a/scripts/generate-kubescape-exceptions/headlamp-configmap.yaml.tmpl +++ /dev/null @@ -1,69 +0,0 @@ ---- -# GENERATED FILE — DO NOT EDIT BY HAND. -# -# Regenerate with, from the repository root: -# go run ./scripts/generate-kubescape-exceptions \ -# -format headlamp-configmap \ -# -o k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml -# -# CI regenerates this file and fails on any difference, so a hand edit is -# reverted by the next run rather than silently kept. Change the -# ClusterSecurityException CRs in k8s/bases/infrastructure/cluster-security-exceptions/ -# instead — they are the single source of truth — or the template at -# scripts/generate-kubescape-exceptions/headlamp-configmap.yaml.tmpl for this -# prose. -# -# WHY THIS FILE EXISTS: the pinned plugin (headlamp_kubescape v0.11.2) has its OWN -# exception mechanism — ConfigMaps labelled `kubescape.io/custom-object: exceptions` — -# and does NOT read the kubescape.io ClusterSecurityException/SecurityException CRDs. -# (The CRD-native "Security Exceptions" view exists only on the plugin's unreleased -# main branch.) Without this ConfigMap the plugin's Exceptions page shows -# "No data to be shown" and its compliance view recomputes scores client-side -# with zero exceptions, so every control the CRs except still renders as failing. -# -# This file is a presentation-layer FALLBACK for the Headlamp dashboard ONLY. -# DELETE it (plus the manual step below, and the -format headlamp-configmap mode) -# once the scanner is confirmed marking excepted controls in prod, OR once the -# plugin is bumped to a release that reads the CRDs natively — whichever lands -# first. As of 2026-08-28 neither holds: the deployed scanner is v4.0.12, which -# should write status=passed/subStatus=w-exceptions into the stored -# workloadconfigurationscansummaries, but a sweep of all ~2200 summaries found no -# object carrying that subStatus (platform#2586, platform#2823). -# -# WHAT IS MIRRORED: every ClusterSecurityException CR, except those annotated -# `platform.devantler.tech/headlamp-mirror: exclude`. That marker is for -# host-scanner exceptions (CIS file permissions/ownership, kubelet and API-server -# flags) whose controls never attach to a workload, so they cannot be expressed as -# a workload exception — and whose cluster-wide designator would otherwise except -# those controls for every workload in the cluster. The exclusion is declared on -# the CR because there is no structural way to infer it: nine legitimately-mirrored -# policies also render a bare wildcard. -# -# Matching semantics (apply-exceptions.ts): a control is excepted for a workload -# when a policy's posturePolicies regex matches the controlID AND a resources -# regex matches the workload's kind/name/namespace. Values are JS RegExp, anchored -# by the generator for exactness. A CR with no `spec.match` renders as -# `kind: ".*"`, which matches every workload. -# -# IMPORTANT (interim manual step, needed ONLY until a scan marks the summaries): -# the v0.11.2 plugin applies ONE selected group at a time and recomputes scores -# client-side, so a viewer must pick the "platform-exceptions" group from the -# Compliance page's exceptions selector for these to take effect. Once the scanner -# writes the exception status this is no longer required (the plugin renders the -# summary status directly). -apiVersion: v1 -kind: ConfigMap -metadata: - name: headlamp-exceptions - namespace: kubescape - labels: - kubescape.io/custom-object: exceptions - app.kubernetes.io/name: platform-exceptions - app.kubernetes.io/managed-by: headlamp -data: - name: platform-exceptions - description: >- - Mirror of the platform ClusterSecurityException CRs for the Headlamp Kubescape - plugin (v0.11.2 reads ConfigMaps, not CRDs). - exceptionPolicies: | -{{ .Policies }} diff --git a/scripts/generate-kubescape-exceptions/main.go b/scripts/generate-kubescape-exceptions/main.go index 544b984914..8c03b55ebc 100644 --- a/scripts/generate-kubescape-exceptions/main.go +++ b/scripts/generate-kubescape-exceptions/main.go @@ -24,8 +24,6 @@ package main import ( - "bytes" - _ "embed" "encoding/json" "errors" "flag" @@ -36,7 +34,6 @@ import ( "regexp" "sort" "strings" - "text/template" "gopkg.in/yaml.v3" ) @@ -47,21 +44,6 @@ const ( exceptionKind = "ClusterSecurityException" designatorTypeAttr = "Attributes" - // mirrorAnnotation marks whether an exception belongs in the Headlamp - // mirror ConfigMap. The Headlamp Kubescape plugin only evaluates workload - // posture scans, so a host-scanner exception (CIS file permissions, kubelet - // flags) has no workload to attach to. There is no structural way to tell - // those apart — a CR with no `spec.match` is cluster-wide for the offline CI - // scan and meaningless for the plugin, and nine legitimately-mirrored - // policies also render a bare wildcard — so the distinction is an editorial - // judgement that has to be declared on the CR. - mirrorAnnotation = "platform.devantler.tech/headlamp-mirror" - // mirrorExclude is the only accepted mirrorAnnotation value. Any other value - // fails closed rather than being read as "include": a typo'd marker must not - // silently push a host-scanner exception into the mirror, where its - // cluster-wide designator would except that control for every workload. - mirrorExclude = "exclude" - // clusterWideAnnotation declares that an exception is MEANT to suppress its // controls for every workload in the cluster. An omitted `spec.match` is // what produces that scope, so without this marker the widest exception the @@ -71,17 +53,10 @@ const ( // exactly like a compliant one — so the scope has to be stated, not inferred // from an absence. clusterWideAnnotation = "platform.devantler.tech/cluster-wide" - // clusterWideDeclared is the only accepted clusterWideAnnotation value. As - // with mirrorExclude, any other value fails closed rather than being read as - // a declaration: a typo must not grant cluster-wide suppression. + // clusterWideDeclared is the only accepted clusterWideAnnotation value. Any + // other value fails closed rather than being read as a declaration: a typo + // must not grant cluster-wide suppression. clusterWideDeclared = "declared" - - formatKubescape = "kubescape" - formatConfigMap = "headlamp-configmap" - - // mirrorConfigMapPath is the generated mirror, repo-root-relative. The drift - // test regenerates it and names it in the failure message it prints. - mirrorConfigMapPath = "k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml" ) // designator identifies the Kubernetes resources covered by an exception. @@ -104,11 +79,6 @@ type policy struct { Resources []designator `json:"resources"` PosturePolicies []posturePolicy `json:"posturePolicies"` Reason string `json:"reason,omitempty"` - - // mirrorExcluded records the CR's mirrorAnnotation decision. It is never - // serialised: the offline CI scan consumes every exception, and Kubescape's - // schema has no such field, so emitting it would change the scan input. - mirrorExcluded bool } // cseErrorf builds the fail-closed error naming the offending CR. @@ -201,10 +171,8 @@ type namespacePrefix struct { } // escapeRegexClassRune uses the small character-class escape vocabulary shared -// by both Go/OPA's RE2 implementation and JavaScript RegExp. The latter matters -// because the generated Headlamp fallback evaluates the same designators in the -// browser; RE2-only hex-brace escapes would make its entire exception group -// unparsable. +// by RE2 (Go/OPA) and other common regex engines, rather than RE2-only +// hex-brace escapes, so a generated designator stays portable. func escapeRegexClassRune(r rune) string { switch r { case '\\', '-', ']', '^': @@ -423,45 +391,6 @@ func resolveMatch(match map[string]any, path, name string) ([]designator, error) }}, nil } -// resolveMirrorExclusion reads the CR's Headlamp-mirror marker. -// -// Absent annotation => mirrored, which is the safe default: the mirror is a -// presentation fallback, and showing an exception the CRs do grant is a display -// choice, whereas dropping one makes the dashboard report an excepted workload -// as failing. Any value other than mirrorExclude fails closed. -func resolveMirrorExclusion(metadata map[string]any, path, name string) (bool, error) { - rawAnnotations, present := metadata["annotations"] - if !present || rawAnnotations == nil { - return false, nil - } - - // Present but not a mapping must fail closed rather than read as "no - // marker": silently ignoring a malformed annotations block would drop the - // exclusion and mirror a host exception, whose cluster-wide designator then - // excepts that control for every workload — the exact widening this marker - // exists to prevent. - annotations, ok := rawAnnotations.(map[string]any) - if !ok { - return false, cseErrorf(path, name, "metadata.annotations must be a mapping, got %v", rawAnnotations) - } - - raw, ok := annotations[mirrorAnnotation] - if !ok { - return false, nil - } - - value, ok := raw.(string) - if !ok { - return false, cseErrorf(path, name, "%s must be a string, got %v", mirrorAnnotation, raw) - } - - if value != mirrorExclude { - return false, cseErrorf(path, name, "unsupported %s value %q (only %q is recognised)", mirrorAnnotation, value, mirrorExclude) - } - - return true, nil -} - // resolveClusterWideDeclaration reads the CR's cluster-wide scope marker. // // Absent annotation => not declared, which is the safe default: an exception @@ -473,9 +402,9 @@ func resolveClusterWideDeclaration(metadata map[string]any, path, name string) ( return false, nil } - // Same fail-closed reasoning as resolveMirrorExclusion: a malformed - // annotations block must not be read as "no marker", because here that - // reading is the permissive one. + // Present but not a mapping must fail closed: a malformed annotations block + // must not be read as "no marker", because here that reading is the + // permissive one. annotations, ok := rawAnnotations.(map[string]any) if !ok { return false, cseErrorf(path, name, "metadata.annotations must be a mapping, got %v", rawAnnotations) @@ -512,11 +441,6 @@ func convertDocument(doc any, path string) (*policy, error) { return nil, cseErrorf(path, "", "missing metadata.name") } - mirrorExcluded, err := resolveMirrorExclusion(metadata, path, name) - if err != nil { - return nil, err - } - spec, _ := document["spec"].(map[string]any) if _, ok := spec["expiresAt"]; ok { return nil, cseErrorf(path, name, "spec.expiresAt cannot be preserved in Kubescape exceptions") @@ -601,7 +525,6 @@ func convertDocument(doc any, path string) (*policy, error) { Actions: []string{"disable"}, Resources: resources, PosturePolicies: policies, - mirrorExcluded: mirrorExcluded, } if reason, ok := spec["reason"].(string); ok && strings.TrimSpace(reason) != "" { @@ -718,81 +641,10 @@ func render(policies []policy) ([]byte, error) { return append(encoded, '\n'), nil } -//go:embed headlamp-configmap.yaml.tmpl -var headlampConfigMapTemplate string - -// mirrored returns the policies the Headlamp plugin should see. -func mirrored(policies []policy) []policy { - kept := make([]policy, 0, len(policies)) - - for _, candidate := range policies { - if candidate.mirrorExcluded { - continue - } - - kept = append(kept, candidate) - } - - return kept -} - -// renderHeadlampConfigMap renders the mirror ConfigMap for the Headlamp plugin. -// -// The whole file is generated rather than the JSON block alone: the mirror and -// the generator express the same exception set in different-but-equivalent forms -// (the hand-written mirror alternation-collapsed resource lists, and wrote an -// unscoped CR as `namespace: ".*"` where the generator emits `kind: ".*"`), so a -// drift gate comparing the two by value would fire on day one. Letting the -// generator own the file makes the comparison a byte diff. -func renderHeadlampConfigMap(policies []policy) ([]byte, error) { - kept := mirrored(policies) - if len(kept) == 0 { - return nil, errors.New("every exception is annotated " + mirrorAnnotation + ": " + mirrorExclude + - ", which would leave the Headlamp plugin with no exceptions at all") - } - - encoded, err := json.MarshalIndent(kept, "", " ") - if err != nil { - return nil, fmt.Errorf("marshal mirror policies: %w", err) - } - - // The JSON sits under `exceptionPolicies: |`, so every line — including the - // blank ones a marshaller never emits but a future one might — carries the - // block's four-space indent. - var indented strings.Builder - - for _, line := range strings.Split(string(encoded), "\n") { - if line == "" { - indented.WriteString("\n") - - continue - } - - indented.WriteString(" " + line + "\n") - } - - parsed, err := template.New("headlamp-configmap").Parse(headlampConfigMapTemplate) - if err != nil { - return nil, fmt.Errorf("parse ConfigMap template: %w", err) - } - - var out bytes.Buffer - if err := parsed.Execute(&out, struct{ Policies string }{ - Policies: strings.TrimRight(indented.String(), "\n"), - }); err != nil { - return nil, fmt.Errorf("render ConfigMap template: %w", err) - } - - return out.Bytes(), nil -} - -// main converts the configured exception directory and writes the chosen format. +// main converts the configured exception directory and writes the exceptions. func main() { output := flag.String("o", "", "output file (stdout if omitted)") flag.StringVar(output, "output", "", "output file (stdout if omitted)") - format := flag.String("format", formatKubescape, - "output format: "+formatKubescape+" (Kubescape exceptions JSON for the offline scan) or "+ - formatConfigMap+" (the Headlamp mirror ConfigMap)") flag.Parse() directory := defaultDir @@ -806,17 +658,7 @@ func main() { os.Exit(1) } - var rendered []byte - - switch *format { - case formatKubescape: - rendered, err = render(policies) - case formatConfigMap: - rendered, err = renderHeadlampConfigMap(policies) - default: - err = fmt.Errorf("unknown -format %q (want %s or %s)", *format, formatKubescape, formatConfigMap) - } - + rendered, err := render(policies) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -833,10 +675,5 @@ func main() { os.Exit(1) } - written := len(policies) - if *format == formatConfigMap { - written = len(mirrored(policies)) - } - - fmt.Fprintf(os.Stderr, "wrote %d exception policies to %s\n", written, *output) + fmt.Fprintf(os.Stderr, "wrote %d exception policies to %s\n", len(policies), *output) } diff --git a/scripts/generate-kubescape-exceptions/main_test.go b/scripts/generate-kubescape-exceptions/main_test.go index 1d7c6eed0b..acd77b703d 100644 --- a/scripts/generate-kubescape-exceptions/main_test.go +++ b/scripts/generate-kubescape-exceptions/main_test.go @@ -573,129 +573,15 @@ func TestGenerateRejectsEmptyDirectory(t *testing.T) { } } -// mirrorNames returns the policy names the Headlamp mirror would carry. -func mirrorNames(t *testing.T, dir string) []string { - t.Helper() - - policies, err := generate(dir) - if err != nil { - t.Fatalf("generate: %v", err) - } - - names := make([]string, 0, len(policies)) - for _, kept := range mirrored(policies) { - names = append(names, kept.Name) - } - - return names -} - -// TestMirrorExcludesAnnotatedException verifies the annotation keeps a -// host-scanner exception out of the Headlamp mirror while leaving it in the -// Kubescape scan input — the two consumers must not be conflated. -func TestMirrorExcludesAnnotatedException(t *testing.T) { - dir := writeCSE(t, ` -kind: ClusterSecurityException -metadata: - name: host-only - annotations: - platform.devantler.tech/headlamp-mirror: exclude - platform.devantler.tech/cluster-wide: declared -spec: - posture: [{controlID: C-0092, action: ignore}] ---- -kind: ClusterSecurityException -metadata: - name: workload-scoped -spec: - match: {namespaceSelector: {matchExpressions: [{key: kubernetes.io/metadata.name, operator: In, values: [flux-system]}]}} - posture: [{controlID: C-0002, action: ignore}] -`) - - policies, err := generate(dir) - if err != nil { - t.Fatalf("generate: %v", err) - } - - // The offline CI scan must still receive BOTH: excluding a host exception - // from the scan would make its controls fail the compliance gate. - if len(policies) != 2 { - t.Fatalf("Kubescape output must keep every exception, got %d", len(policies)) - } - - names := mirrorNames(t, dir) - if len(names) != 1 || names[0] != "workload-scoped" { - t.Errorf("mirror = %v, want only [workload-scoped]", names) - } -} - -// TestMirrorDefaultsToIncluded verifies an unannotated CR is mirrored: dropping -// an exception makes the dashboard report an excepted workload as failing. -func TestMirrorDefaultsToIncluded(t *testing.T) { - dir := writeCSE(t, ` -kind: ClusterSecurityException -metadata: {name: no-annotations} -spec: - match: {namespaceSelector: {matchExpressions: [{key: kubernetes.io/metadata.name, operator: In, values: [flux-system]}]}} - posture: [{controlID: C-0002, action: ignore}] -`) - - if names := mirrorNames(t, dir); len(names) != 1 { - t.Errorf("mirror = %v, want the unannotated exception to be mirrored", names) - } -} - -// TestMirrorAnnotationFailsClosed verifies a marker this converter does not -// recognise aborts instead of being read as "include" — a typo must never push -// a cluster-wide host exception into the workload dashboard. -func TestMirrorAnnotationFailsClosed(t *testing.T) { - t.Parallel() - - cases := map[string]struct { - value string - want string - }{ - "unknown value": {value: "excluded", want: "unsupported platform.devantler.tech/headlamp-mirror value"}, - "list not string": {value: "[exclude]", want: "must be a string"}, - "include is not a value": {value: "include", want: "unsupported platform.devantler.tech/headlamp-mirror value"}, - "empty value": {value: `""`, want: "unsupported platform.devantler.tech/headlamp-mirror value"}, - "boolean not string": {value: "true", want: "must be a string"}, - } - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - t.Parallel() - - _, err := generate(writeCSE(t, ` -kind: ClusterSecurityException -metadata: - name: marked - annotations: - platform.devantler.tech/headlamp-mirror: `+tc.value+` -spec: - posture: [{controlID: C-0002, action: ignore}] -`)) - if err == nil { - t.Fatalf("want a fail-closed error containing %q, got none", tc.want) - } - - if !strings.Contains(err.Error(), tc.want) { - t.Errorf("error = %q, want it to contain %q", err, tc.want) - } - }) - } -} - -// TestMirrorMalformedAnnotationsFailClosed verifies a malformed annotations -// block aborts instead of being read as "no marker". Reading it as absent would -// silently drop the exclusion and mirror a host-scanner exception, whose -// cluster-wide designator then excepts that control for every workload. -func TestMirrorMalformedAnnotationsFailClosed(t *testing.T) { +// TestMalformedAnnotationsFailClosed verifies a malformed annotations block +// aborts instead of being read as "no marker". Reading it as absent would +// silently drop a cluster-wide declaration, so the converter refuses it. +func TestMalformedAnnotationsFailClosed(t *testing.T) { _, err := generate(writeCSE(t, ` kind: ClusterSecurityException metadata: name: malformed-annotations - annotations: "platform.devantler.tech/headlamp-mirror=exclude" + annotations: "platform.devantler.tech/cluster-wide=declared" spec: posture: [{controlID: C-0092, action: ignore}] `)) @@ -704,11 +590,11 @@ spec: } } -// TestMirrorEmptyAnnotationsAreNotAMarker verifies an explicitly-empty -// annotations block is still just "no marker", so the fail-closed check above -// does not reject ordinary CRs. -func TestMirrorEmptyAnnotationsAreNotAMarker(t *testing.T) { - dir := writeCSE(t, ` +// TestEmptyAnnotationsAreNotAMarker verifies an explicitly-empty annotations +// block is still just "no marker", so the fail-closed check above does not +// reject ordinary CRs. +func TestEmptyAnnotationsAreNotAMarker(t *testing.T) { + policies, err := generate(writeCSE(t, ` kind: ClusterSecurityException metadata: name: empty-annotations @@ -716,70 +602,13 @@ metadata: spec: match: {namespaceSelector: {matchExpressions: [{key: kubernetes.io/metadata.name, operator: In, values: [flux-system]}]}} posture: [{controlID: C-0002, action: ignore}] -`) - - if names := mirrorNames(t, dir); len(names) != 1 { - t.Errorf("mirror = %v, want the CR mirrored (empty annotations are not a marker)", names) - } -} - -// TestMirrorRefusesEmptyResult verifies an all-excluded set fails closed rather -// than writing a ConfigMap that silently shows the dashboard no exceptions. -func TestMirrorRefusesEmptyResult(t *testing.T) { - dir := writeCSE(t, ` -kind: ClusterSecurityException -metadata: - name: host-only - annotations: - platform.devantler.tech/headlamp-mirror: exclude - platform.devantler.tech/cluster-wide: declared -spec: - posture: [{controlID: C-0092, action: ignore}] -`) - - policies, err := generate(dir) - if err != nil { - t.Fatalf("generate: %v", err) - } - - if _, err := renderHeadlampConfigMap(policies); err == nil { - t.Fatal("want an error when every exception is excluded from the mirror") - } -} - -// TestCommittedMirrorIsUpToDate is the drift gate. The Headlamp mirror used to -// be hand-maintained and drifted in the direction that HIDES findings — it once -// excepted C-0017 for every workload in the cluster while the CRs excepted it in -// three narrow places (platform#2586, fixed by #2837). Regenerating here means a -// CR change that is not mirrored fails CI instead of silently desynchronising -// the dashboard from the exceptions actually in force. -func TestCommittedMirrorIsUpToDate(t *testing.T) { - dir := filepath.Join("..", "..", defaultDir) - if _, err := os.Stat(dir); err != nil { - t.Skipf("exceptions dir not present: %v", err) - } - - committedPath := filepath.Join("..", "..", mirrorConfigMapPath) - - committed, err := os.ReadFile(committedPath) - if err != nil { - t.Fatalf("read committed mirror: %v", err) - } - - policies, err := generate(dir) +`)) if err != nil { t.Fatalf("generate: %v", err) } - regenerated, err := renderHeadlampConfigMap(policies) - if err != nil { - t.Fatalf("render mirror: %v", err) - } - - if string(committed) != string(regenerated) { - t.Errorf("%s is out of date.\n\nRegenerate it with:\n"+ - " go run ./scripts/generate-kubescape-exceptions -format %s -o %s\n", - mirrorConfigMapPath, formatConfigMap, mirrorConfigMapPath) + if len(policies) != 1 { + t.Errorf("policies = %v, want the CR converted (empty annotations are not a marker)", policies) } } diff --git a/scripts/guard-crossplane-provider-name-parity.sh b/scripts/guard-crossplane-provider-name-parity.sh index 50a530faed..86a671be69 100755 --- a/scripts/guard-crossplane-provider-name-parity.sh +++ b/scripts/guard-crossplane-provider-name-parity.sh @@ -2,7 +2,7 @@ # # Fail when a Crossplane Provider's NAME stops agreeing with the two places that # depend on it: the name Crossplane derives from its package, and the Kubescape -# secret-reader exceptions that match its generated ClusterRole by regex. +# secret-reader exception that matches its generated ClusterRole by regex. # # WHY THE NAME MATTERS (platform#3454, a live prod outage). # @@ -26,17 +26,17 @@ # # Crossplane names a provider's generated RBAC after the provider REVISION — # `crossplane:provider:-:system` — so the provider name -# is a prefix of a cluster-scoped ClusterRole name. Two Kubescape exception lists -# match that ClusterRole by anchored regex to suppress C-0015 ("List Kubernetes -# secrets") for providers that read credential Secrets by design. Renaming a provider -# without updating both regexes does not fail anything: the manifests still build, the -# provider still runs, and the only symptom is a C-0015 finding reappearing days later -# with no obvious cause. The two lists are kept in sync BY HAND (their own headers say -# so), which is exactly the arrangement that drifts. +# is a prefix of a cluster-scoped ClusterRole name. The Kubescape secret-reader +# exception matches that ClusterRole by anchored regex to suppress C-0015 ("List +# Kubernetes secrets") for providers that read credential Secrets by design. Renaming +# a provider without updating its regex does not fail anything: the manifests still +# build, the provider still runs, and the only symptom is a C-0015 finding reappearing +# days later with no obvious cause. The list is kept BY HAND, which is exactly the +# arrangement that drifts. # # 🔴 FAIL CLOSED. Every input this guard cannot read is exit 2, never a quiet pass. -# An empty provider set compared against empty exception files is the failure mode -# that makes a guard look green forever, so the provider set and both exception files +# An empty provider set compared against an empty exception file is the failure mode +# that makes a guard look green forever, so the provider set and the exception file # are required to be non-empty before any comparison happens. # # 🔴 SCOPE BY apiVersion, NOT BY `kind: Provider`. `notification.toolkit.fluxcd.io` @@ -46,7 +46,8 @@ # # Exit codes: # 0 every pkg.crossplane.io Provider is canonically named (or explicitly -# dispositioned below) and carries its ClusterRole regex in both exception lists +# dispositioned below) and carries its ClusterRole regex in the secret-reader +# exception # 1 a drift was found # 2 an input could not be read, or a required set was empty @@ -55,7 +56,6 @@ set -euo pipefail root="${1:-k8s}" secret_reader="$root/bases/infrastructure/cluster-security-exceptions/secret-reader-rbac.yaml" -headlamp_mirror="$root/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml" die() { printf 'guard-crossplane-provider-name-parity: %s\n' "$1" >&2 @@ -64,9 +64,7 @@ die() { [ -d "$root" ] || die "root '$root' is not a directory" [ -r "$secret_reader" ] || die "cannot read the secret-reader exception '$secret_reader'" -[ -r "$headlamp_mirror" ] || die "cannot read the Headlamp exception mirror '$headlamp_mirror'" [ -s "$secret_reader" ] || die "the secret-reader exception '$secret_reader' is empty" -[ -s "$headlamp_mirror" ] || die "the Headlamp exception mirror '$headlamp_mirror' is empty" # Providers deliberately NOT carrying their derived name. Each is safe ONLY while no # installed package declares it as a dependency: nothing resolves them by name today, @@ -166,14 +164,12 @@ while IFS="$(printf '\t')" read -r name pkg file; do fi regex="^crossplane:provider:${name}-[0-9a-f]+:system\$" - for f in "$secret_reader" "$headlamp_mirror"; do - if ! grep -qF -- "$regex" "$f"; then - printf 'guard-crossplane-provider-name-parity: %s has no secret-reader exception entry in %s\n' "$name" "$f" >&2 - printf ' Expected the anchored ClusterRole pattern: %s\n' "$regex" >&2 - printf ' Crossplane names provider RBAC after the revision, so renaming a provider silently orphans its exception and C-0015 reappears with no failing check.\n' >&2 - status=1 - fi - done + if ! grep -qF -- "$regex" "$secret_reader"; then + printf 'guard-crossplane-provider-name-parity: %s has no secret-reader exception entry in %s\n' "$name" "$secret_reader" >&2 + printf ' Expected the anchored ClusterRole pattern: %s\n' "$regex" >&2 + printf ' Crossplane names provider RBAC after the revision, so renaming a provider silently orphans its exception and C-0015 reappears with no failing check.\n' >&2 + status=1 + fi done <&2; exit 1; +fi +if [ "${GITHUB_ACTIONS:-}" != true ] || + [ "${RUNNER_ENVIRONMENT:-}" != github-hosted ] || + [ "${RUNNER_OS:-}" != Linux ]; then + printf 'FAIL: registry cache setup requires a hosted Linux runner\n' >&2; exit 1; +fi +umask 077 +cache_dir="$(mktemp -d)" +trap 'rm -rf "${cache_dir}"' EXIT +docker ps -aq >"${cache_dir}/containers" +[ ! -s "${cache_dir}/containers" ] || { + printf 'FAIL: registry cache setup requires an unused Docker daemon\n' >&2; exit 1; +} +if sudo test -e /etc/docker/daemon.json; then + sudo cat /etc/docker/daemon.json | tee "${cache_dir}/original.json" >/dev/null +else + sudo test ! -e /etc/docker/daemon.json + printf '{}\n' >"${cache_dir}/original.json" +fi +# Slurp rejects empty or multiple documents rather than treating either as +# an empty daemon configuration. No configuration values reach the job log. +jq -e -s ' + if length != 1 or (.[0] | type) != "object" then error("invalid daemon configuration") + else .[0] end | + if has("registry-mirrors") and + (.["registry-mirrors"] | type != "array" or + any(.[]; type != "string" or length == 0)) + then error("invalid registry mirrors") else . end | + .["registry-mirrors"] = (["https://mirror.gcr.io"] + + [(.["registry-mirrors"] // [])[] | select(. != "https://mirror.gcr.io")]) +' "${cache_dir}/original.json" >"${cache_dir}/daemon.json" 2>/dev/null || { + printf 'FAIL: invalid daemon or registry cache configuration\n' >&2; exit 1; +} +sudo dockerd --validate --config-file="${cache_dir}/daemon.json" >/dev/null +sudo mkdir -p /etc/docker +sudo install -m 600 "${cache_dir}/daemon.json" /etc/docker/daemon.json +sudo systemctl restart docker +docker info --format '{{json .RegistryConfig.Mirrors}}' | + jq -e 'type == "array" and any(.[]; . == "https://mirror.gcr.io/" or . == "https://mirror.gcr.io")' >/dev/null +printf 'PASS: hosted Docker registry cache configured and read back\n' diff --git a/scripts/tests/test-arc-listener-public-content.sh b/scripts/tests/test-arc-listener-public-content.sh index 6105dea974..8bd051a10d 100755 --- a/scripts/tests/test-arc-listener-public-content.sh +++ b/scripts/tests/test-arc-listener-public-content.sh @@ -42,14 +42,10 @@ scan justified --exceptions "$scratch/exceptions.json" "$listener" jq -e '.summaryDetails.controls[] | select(.controlID == "C-0012") | .ResourceCounters.failedResources == 0 and .subStatusCounters.ignoredResources == 1' \ "$scratch/justified.json" >/dev/null || fail unapplied-disposition -scan mirror k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml -jq -e '.summaryDetails.controls[] | select(.controlID == "C-0012") | - .ResourceCounters.passedResources == 1 and .ResourceCounters.failedResources == 0 and .subStatusCounters.ignoredResources == 0' \ - "$scratch/mirror.json" >/dev/null || fail new-mirror-false-positive jq '.metadata.name = "synthetic-credential-control" | .data = {password:"synthetic PRIVATE KEY"}' \ "$scratch/public.json" >"$scratch/credential-input.json" scan credential --exceptions "$scratch/exceptions.json" "$scratch/credential-input.json" jq -e '.summaryDetails.controls[] | select(.controlID == "C-0012") | .ResourceCounters.failedResources == 1 and .subStatusCounters.ignoredResources == 0' \ "$scratch/credential.json" >/dev/null || fail hidden-genuine-credential -printf 'ARC public listener: native admission 1 pass/8 denies; scanner false positive, exact disposition, unexcepted mirror and credential control verified.\n' +printf 'ARC public listener: native admission 1 pass/8 denies; scanner false positive, exact disposition and credential control verified.\n' diff --git a/scripts/tests/test-coroot-alert-autosuppressor.sh b/scripts/tests/test-coroot-alert-autosuppressor.sh index c704e1e14d..0dddfacfd8 100755 --- a/scripts/tests/test-coroot-alert-autosuppressor.sh +++ b/scripts/tests/test-coroot-alert-autosuppressor.sh @@ -444,17 +444,6 @@ cat >"${event_dir}/alerts.json" <<'JSON' {"name":"Labels","value":"cluster=\"platform\"\nkind=\"Kustomization\"\nname=\"github-config\"\nnamespace=\"github-config\"\nreason=\"ReconciliationFailed\"\nsource=\"kustomize-controller\""} ] }, - { - "id":"monitor-secret-race", - "suppressed":false, - "resolved_at":null, - "rule_id":"kubernetes-events", - "application_id":"95rsc5yp:crossview:Deployment:external-secrets", - "details":[ - {"name":"Event message","value":"secrets \"crossview-postgres-coroot-monitor\" already exists"}, - {"name":"Labels","value":"cluster=\"platform\"\nkind=\"ExternalSecret\"\nname=\"crossview-postgres-coroot-monitor\"\nnamespace=\"crossview\"\nreason=\"UpdateFailed\"\nsource=\"external-secrets\""} - ] - }, { "id":"openbao-snapshot-device-race", "suppressed":false, @@ -576,17 +565,6 @@ cat >"${event_dir}/alerts.json" <<'JSON' {"name":"Labels","value":"cluster=\"platform\"\nkind=\"Kustomization\"\nname=\"github-config\"\nnamespace=\"github-config\"\nreason=\"ReconciliationFailed\"\nsource=\"kustomize-controller\""} ] }, - { - "id":"near-match-secret", - "suppressed":false, - "resolved_at":null, - "rule_id":"kubernetes-events", - "application_id":"95rsc5yp:crossview:Deployment:external-secrets", - "details":[ - {"name":"Event message","value":"secrets \"crossview-postgres-coroot-monitor\" is forbidden"}, - {"name":"Labels","value":"cluster=\"platform\"\nkind=\"ExternalSecret\"\nname=\"crossview-postgres-coroot-monitor\"\nnamespace=\"crossview\"\nreason=\"UpdateFailed\"\nsource=\"external-secrets\""} - ] - }, { "id":"near-match-snapshot-device", "suppressed":false, @@ -636,7 +614,7 @@ JSON run_scenario "${event_dir}" >/dev/null [ -f "${event_dir}/suppressed.json" ] || fail "the exact by-design Kubernetes lifecycle events were not suppressed" -jq -e '.ids | sort == ["flux-build-canceled", "flux-dryrun-canceled", "flux-health-canceled", "flux-health-canceled-milliseconds", "flux-status-canceled", "longhorn-clone-awaiting-healthy", "longhorn-volume-failed-delete", "monitor-secret-race", "openbao-snapshot-device-race", "snapshot-content-cleanup"]' \ +jq -e '.ids | sort == ["flux-build-canceled", "flux-dryrun-canceled", "flux-health-canceled", "flux-health-canceled-milliseconds", "flux-status-canceled", "longhorn-clone-awaiting-healthy", "longhorn-volume-failed-delete", "openbao-snapshot-device-race", "snapshot-content-cleanup"]' \ "${event_dir}/suppressed.json" >/dev/null || { jq -c '.ids | sort' "${event_dir}/suppressed.json" >&2 fail "the event exemptions were broader than their exact label and message contracts" diff --git a/scripts/tests/test-coroot-risk-acceptance-reconciler.sh b/scripts/tests/test-coroot-risk-acceptance-reconciler.sh index cd77234fac..b2dc1a958f 100755 --- a/scripts/tests/test-coroot-risk-acceptance-reconciler.sh +++ b/scripts/tests/test-coroot-risk-acceptance-reconciler.sh @@ -39,7 +39,7 @@ acceptances_file="${work_root}/risks.json" yq e -r '.data."risks.json"' "${acceptances_manifest}" >"${acceptances_file}" jq -e ' - type == "array" and length == 41 and + type == "array" and length == 37 and all(.[] | select(has("application")); (.application | type == "string") and (.application | split(":") | length == 3) and @@ -77,13 +77,10 @@ expected_applications=( 'crossplane-system:Deployment:crossplane' 'crossplane-system:Deployment:crossplane-rbac-manager' 'observability:Deployment:crossplane-sync-exporter' - 'crossview:Deployment:crossview' - 'crossview:Deployment:crossview-postgres' 'kube-system:Deployment:descheduler' 'external-dns:Deployment:external-dns' 'ascoachingogvaner:Deployment:external-dns' 'flux-system:Deployment:flux-operator' - 'headlamp:Deployment:headlamp' 'kube-system:Deployment:hubble-ui' 'longhorn-system:Deployment:longhorn-driver-deployer' 'longhorn-system:InstanceManager:instance-manager-0c0363730e10c9272de1f53e608f8860' @@ -112,20 +109,6 @@ for application in "${expected_applications[@]}"; do fail "missing reviewed risk acceptance for ${application}" done -if ! jq -e ' - . as $acceptances | - ["unreplicated-database", "single-instance-app"] | - all(.[]; . as $type | - any($acceptances[]; - .application == "crossview:Deployment:crossview-postgres" and - .category == "Availability" and - .type == $type - ) - ) -' "${acceptances_file}" >/dev/null; then - fail 'Crossview PostgreSQL must bind both reviewed availability risks exactly' -fi - if ! jq -e ' any(.[]; .application == "longhorn-system:Deployment:longhorn-driver-deployer" and @@ -184,16 +167,16 @@ fi setup_scenario() { local name="$1" mode="$2" dir - local crossview_reason + local whoami_reason dir="${work_root}/${name}" mkdir -p "${dir}/bin" - crossview_reason="$(jq -r '.[] | select(.application == "crossview:Deployment:crossview") | .reason' "${acceptances_file}")" + whoami_reason="$(jq -r '.[] | select(.application == "whoami:Deployment:whoami") | .reason' "${acceptances_file}")" jq -n '{data:{projects:[{id:"95rsc5yp",name:"platform"}]}}' >"${dir}/user.json" if [ "${mode}" = 'normal' ]; then - jq -n --arg crossview_reason "${crossview_reason}" '{data:{risks:[ + jq -n --arg whoami_reason "${whoami_reason}" '{data:{risks:[ {application_id:"95rsc5yp:actual-budget:Deployment:actual-budget-actualbudget",key:{category:"Availability",type:"single-instance-app"}}, - {application_id:"95rsc5yp:crossview:Deployment:crossview",key:{category:"Availability",type:"single-instance-app"},dismissal:{reason:$crossview_reason}}, + {application_id:"95rsc5yp:whoami:Deployment:whoami",key:{category:"Availability",type:"single-instance-app"},dismissal:{reason:$whoami_reason}}, {application_id:"95rsc5yp:retired:Deployment:old-tool",key:{category:"Availability",type:"single-instance-app"},dismissal:{reason:"manual dismissal outside GitOps"}}, {application_id:"95rsc5yp:new-service:Deployment:new-service",key:{category:"Availability",type:"single-instance-app"}}, {application_id:"95rsc5yp:database:StatefulSet:database",key:{category:"Security",type:"db-internet-exposure"},dismissal:{reason:"owned by a separate security policy"}} diff --git a/scripts/tests/test-cronjob-failure-alert.sh b/scripts/tests/test-cronjob-failure-alert.sh index 5fef9a7b58..e69f2a43d9 100755 --- a/scripts/tests/test-cronjob-failure-alert.sh +++ b/scripts/tests/test-cronjob-failure-alert.sh @@ -54,8 +54,8 @@ env_value() { yq eval "${container_path}.env[] | select(.name == \"$1\") | .valu fail 'binding must grant only the detector ServiceAccount' pass 'RBAC is get on the one watched CronJob and list on Jobs in umami' -[ "$(env_value WATCH)" = 'umami/umami-provision-tenants:2:3600 kubescape/kubescape-hostdata-cleanup:1:93600 observability/stranded-autoscaler-node-alerter:2:3600 observability/crossview-login-alerter:2:3600' ] || - fail 'watch list must include the Umami and Kubescape reconcile loops and the stranded-node and Crossview login sensors' +[ "$(env_value WATCH)" = 'umami/umami-provision-tenants:2:3600 kubescape/kubescape-hostdata-cleanup:1:93600 observability/stranded-autoscaler-node-alerter:2:3600' ] || + fail 'watch list must include the Umami and Kubescape reconcile loops and the stranded-node sensor' [ "$(yq eval '.metadata.namespace' "$kubescape_role")" = kubescape ] || fail 'Kubescape Role must be namespaced to kubescape' [ "$(yq eval '[.rules[] | select(.resources[] == "cronjobs") | .resourceNames[]] | join(",")' "$kubescape_role")" = kubescape-hostdata-cleanup ] || fail 'the Kubescape CronJob grant must name only the host-data cleanup' @@ -71,8 +71,8 @@ observability_role="${bundle_dir}/role-cronjob-failure-alert-observability.yaml" observability_binding="${bundle_dir}/role-binding-cronjob-failure-alert-observability.yaml" [ "$(yq eval '.metadata.namespace' "$observability_role")" = observability ] || fail 'observability Role must be namespaced to observability' -[ "$(yq eval '[.rules[] | select(.resources[] == "cronjobs") | .resourceNames[]] | join(",")' "$observability_role")" = stranded-autoscaler-node-alerter,crossview-login-alerter ] || - fail 'the observability CronJob grant must name only the stranded-node and Crossview login sensors' +[ "$(yq eval '[.rules[] | select(.resources[] == "cronjobs") | .resourceNames[]] | join(",")' "$observability_role")" = stranded-autoscaler-node-alerter ] || + fail 'the observability CronJob grant must name only the stranded-node sensor' [ "$(yq eval '[.rules[] | select(.resources[] == "cronjobs") | .verbs[]] | join(",")' "$observability_role")" = get ] || fail 'the observability CronJob grant must be get only' [ "$(yq eval '[.rules[] | select(.resources[] == "jobs") | .verbs[]] | join(",")' "$observability_role")" = list ] || diff --git a/scripts/tests/test-crossview-coroot-postgres-integration.sh b/scripts/tests/test-crossview-coroot-postgres-integration.sh deleted file mode 100755 index 9886546246..0000000000 --- a/scripts/tests/test-crossview-coroot-postgres-integration.sh +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly root_dir - -fail() { - printf 'FAIL: %s\n' "$1" >&2 - exit 1 -} - -command -v kubectl >/dev/null 2>&1 || fail 'kubectl is required to render the production apps overlay' -command -v yq >/dev/null 2>&1 || fail 'yq v4 is required to inspect the rendered integration' - -rendered="$(kubectl kustomize "${root_dir}/k8s/providers/hetzner/apps")" || - fail 'the production apps overlay must render successfully' - -snapshot="$( - printf '%s\n' "${rendered}" | - yq ea -o=json -I=0 '[.] | { - "helmRelease": map(select( - (.kind == "HelmRelease") and - (.metadata.name == "crossview") and - (.metadata.namespace == "crossview") - ))[0], - "password": map(select( - (.kind == "Password") and - (.metadata.name == "crossview-postgres-coroot-monitor") and - (.metadata.namespace == "crossview") - ))[0], - "externalSecret": map(select( - (.kind == "ExternalSecret") and - (.metadata.name == "crossview-postgres-coroot-monitor") and - (.metadata.namespace == "crossview") - ))[0], - "networkPolicy": map(select( - (.kind == "CiliumNetworkPolicy") and - (.metadata.name == "allow-coroot-postgres-scrape") and - (.metadata.namespace == "crossview") - ))[0], - "configMap": map(select( - (.kind == "ConfigMap") and - (.metadata.name == "crossview-postgres-coroot-monitor-init") and - (.metadata.namespace == "crossview") - ))[0] - }' - -)" || fail 'the rendered Crossview integration resources must be inspectable' -readonly snapshot - -printf '%s\n' "${snapshot}" | - yq e -e ' - (.helmRelease.spec.values.database.podAnnotations."coroot.com/postgres-scrape" == "true") and - (.helmRelease.spec.values.database.podAnnotations."coroot.com/postgres-scrape-port" == "5432") and - (.helmRelease.spec.values.database.podAnnotations."coroot.com/postgres-scrape-credentials-secret-name" == "crossview-postgres-coroot-monitor") and - (.helmRelease.spec.values.database.podAnnotations."coroot.com/postgres-scrape-credentials-secret-username-key" == "username") and - (.helmRelease.spec.values.database.podAnnotations."coroot.com/postgres-scrape-credentials-secret-password-key" == "password") and - (.helmRelease.spec.values.database.podAnnotations."coroot.com/postgres-scrape-param-sslmode" == "disable") and - ([ - (.helmRelease.spec.values.database.extraEnv // [])[] | - select( - (.name == "COROOT_MONITOR_PASSWORD") and - (.valueFrom.secretKeyRef.name == "crossview-postgres-coroot-monitor") and - (.valueFrom.secretKeyRef.key == "password") - ) - ] | length == 1) - ' - >/dev/null || - fail 'the rendered Crossview HelmRelease must configure Coroot discovery with its dedicated credential' - -postgres_patch="$( - printf '%s\n' "${snapshot}" | - yq e -r ' - .helmRelease.spec.postRenderers[].kustomize.patches[] | - select((.target.kind == "Deployment") and (.target.name == "crossview-postgres")) | - select(.patch | contains("shared_preload_libraries")) | - .patch - ' - -)" || fail 'the rendered Crossview PostgreSQL post-render patch must be inspectable' - -printf '%s\n' "${postgres_patch}" | - yq e -e ' - (.spec.strategy.type == "RollingUpdate") and - (.spec.strategy.rollingUpdate.maxSurge == 0) and - (.spec.strategy.rollingUpdate.maxUnavailable == 1) and - ([ - .spec.template.spec.containers[] | - select( - (.name == "postgres") and - (.args | length == 4) and - (.args[0] == "-c") and - (.args[1] == "shared_preload_libraries=pg_stat_statements") and - (.args[2] == "-c") and - (.args[3] == "track_io_timing=on") - ) | - .volumeMounts[] | - select( - (.name == "coroot-monitor-init") and - (.mountPath == "/docker-entrypoint-initdb.d") and - (.readOnly == true) - ) - ] | length == 1) and - ([ - .spec.template.spec.volumes[] | - select( - (.name == "coroot-monitor-init") and - (.configMap.name == "crossview-postgres-coroot-monitor-init") and - (.configMap.defaultMode == 365) - ) - ] | length == 1) - ' - >/dev/null || - fail 'the PostgreSQL post-render patch must serialize ephemeral-database replacement, preload statistics, enable I/O timing, and mount an executable init hook' - -app_patch="$( - printf '%s\n' "${snapshot}" | - yq e -r ' - .helmRelease.spec.postRenderers[].kustomize.patches[] | - select((.target.kind == "Deployment") and (.target.name == "crossview")) | - select(.patch | contains("pg_stat_statements")) | - .patch - ' - -)" || fail 'the rendered Crossview rollout-safety patch must be inspectable' - -printf '%s\n' "${app_patch}" | - yq e -e ' - ([ - .spec.template.spec.initContainers[] | - select(.name == "wait-for-db") | - .args[] | - select( - contains("pg_stat_statements") and - contains("-d postgres") and - contains("FROM pg_catalog.pg_roles AS monitor") and - contains("monitor.rolcanlogin") and - contains("pg_catalog.pg_auth_members AS membership") and - contains("granted_role.rolname =") and - contains("pg_monitor") - ) - ] | length == 1) - ' - >/dev/null || - fail 'the Crossview app must verify the extension, login role, and pg_monitor membership before bootstrapping' - -printf '%s\n' "${snapshot}" | - yq e -e ' - (.password.spec.length == 32) and (.password.spec.symbols == 0) - ' - >/dev/null || - fail 'the production overlay must generate a stable Coroot monitoring password' - -printf '%s\n' "${snapshot}" | - yq e -e ' - (.externalSecret.spec.refreshInterval == "0") and - (.externalSecret.spec.target.name == "crossview-postgres-coroot-monitor") and - (.externalSecret.spec.target.template.type == "kubernetes.io/basic-auth") and - (.externalSecret.spec.target.template.data.username == "coroot_monitor") and - (.externalSecret.spec.target.template.data.password == "{{ .password }}") - ' - >/dev/null || - fail 'the generated credential must be exposed as a create-once basic-auth Secret' - -printf '%s\n' "${snapshot}" | - yq e -e ' - (.networkPolicy.spec.endpointSelector.matchLabels."app.kubernetes.io/component" == "postgres") and - ([ - .networkPolicy.spec.ingress[] | - select( - ([ - .fromEndpoints[].matchLabels | - select( - (."k8s:io.kubernetes.pod.namespace" == "observability") and - (."app.kubernetes.io/managed-by" == "coroot-operator") and - (."app.kubernetes.io/part-of" == "coroot") and - (."app.kubernetes.io/component" == "coroot-cluster-agent") and - (."io.cilium.k8s.policy.serviceaccount" == "coroot-cluster-agent") and - ((. | length) == 5) - ) - ] | length == 1) and - ([ - .toPorts[].ports[] | - select((.port == "5432") and (.protocol == "TCP")) - ] | length == 1) - ) - ] | length == 1) - ' - >/dev/null || - fail 'the scrape policy must allow only the Coroot cluster-agent to reach the Crossview PostgreSQL pod' - -init_script="$( - printf '%s\n' "${snapshot}" | - yq e -r '.configMap.data."init-coroot-monitor.sh"' - -)" || fail 'the rendered PostgreSQL init hook must be inspectable' - -tmp_dir="$(mktemp -d)" -readonly tmp_dir -trap 'rm -rf "${tmp_dir}"' EXIT - -printf '%s\n' "${init_script}" >"${tmp_dir}/init-coroot-monitor.sh" -cat >"${tmp_dir}/psql" <<'FAKE_PSQL' -#!/usr/bin/env bash -set -euo pipefail -printf '%s\n' "$*" >"${CAPTURE_ARGS}" -printf '%s\n' "${COROOT_MONITOR_PASSWORD}" >"${CAPTURE_PASSWORD}" -cat >"${CAPTURE_STDIN}" -FAKE_PSQL -chmod +x "${tmp_dir}/psql" - -PATH="${tmp_dir}:${PATH}" \ - CAPTURE_ARGS="${tmp_dir}/args" \ - CAPTURE_PASSWORD="${tmp_dir}/password" \ - CAPTURE_STDIN="${tmp_dir}/stdin" \ - POSTGRES_USER=postgres \ - POSTGRES_DB=crossview \ - COROOT_MONITOR_PASSWORD=test-password \ - bash "${tmp_dir}/init-coroot-monitor.sh" || - fail 'the rendered PostgreSQL init hook must execute successfully' - -grep -q -- '--username postgres --dbname postgres' "${tmp_dir}/args" || - fail 'the init hook must create pg_stat_statements in the maintenance database Coroot connects to' -if grep -q -- 'test-password' "${tmp_dir}/args"; then - fail 'the init hook must not expose the generated credential in process arguments' -fi -grep -qx -- 'test-password' "${tmp_dir}/password" || - fail 'the init hook must pass the generated credential to psql through its environment' -grep -Fq -- '\getenv coroot_monitor_password COROOT_MONITOR_PASSWORD' "${tmp_dir}/stdin" || - fail 'the init hook must import the generated credential inside psql' -grep -q -- 'CREATE EXTENSION IF NOT EXISTS pg_stat_statements' "${tmp_dir}/stdin" || - fail 'the init hook must create pg_stat_statements' -grep -q -- 'CREATE ROLE coroot_monitor' "${tmp_dir}/stdin" || - fail 'the init hook must create the dedicated monitoring role' -grep -q -- 'GRANT pg_monitor TO coroot_monitor' "${tmp_dir}/stdin" || - fail 'the init hook must grant only the PostgreSQL monitoring role' - -printf 'PASS: Crossview PostgreSQL renders a least-privilege Coroot integration with rollout safety\n' diff --git a/scripts/tests/test-crossview-login-alerter-runtime.sh b/scripts/tests/test-crossview-login-alerter-runtime.sh deleted file mode 100755 index 3eed0548b0..0000000000 --- a/scripts/tests/test-crossview-login-alerter-runtime.sh +++ /dev/null @@ -1,366 +0,0 @@ -#!/usr/bin/env bash - -# Reproduce the #3315 outage against the real pinned Crossview app and its -# bundled PostgreSQL, and run the deployed login sensor, in its own pinned -# image, against it: -# -# 1. a freshly bootstrapped app: the sensor reports healthy; -# 2. the database replaced by an empty one under the running app, which is the -# August 2026 trigger (an emptyDir lost with its pod): `/api/health` still -# answers 200 while the schema is gone, and the sensor alerts; -# 3. the app restarted: it re-runs its schema bootstrap and the sensor is -# healthy again, so restarting is what restores login; -# 4. the database stopped: the sensor never reports health; -# 5. the same database started again with its data: the app recovers without a -# restart, which is why the runbook checks the database before rolling. -# -# Images, configuration and the Service contract come from rendering the pinned -# chart with the production values, so a chart bump re-proves the sensor against -# the new app. The HelmRelease post-renderers are not applied: they harden the -# containers and gate the app's init container on markers that the database's -# init hook writes into any fresh data directory, which -# test-crossview-postgres-durable-storage.sh and -# test-crossview-coroot-postgres-integration.sh pin. What is reproduced here is -# the app's own bootstrap, which is what the outage and its fix turn on. Every -# container runs on an internal Docker network with no route out; only the image -# pulls use the network. Requires docker. -set -euo pipefail - -# #4693: use Google's documented Docker Hub cache without changing the pinned -# images or the outage assertions. This mode is invoked only before the hosted -# fixture starts; local and self-hosted Docker daemons are never reconfigured. -if [ "$#" -gt 0 ]; then - if [ "$#" != 1 ] || [ "$1" != --configure-registry-cache ]; then - printf 'FAIL: unexpected runtime fixture argument\n' >&2; exit 1; - fi - if [ "${GITHUB_ACTIONS:-}" != true ] || - [ "${RUNNER_ENVIRONMENT:-}" != github-hosted ] || - [ "${RUNNER_OS:-}" != Linux ]; then - printf 'FAIL: registry cache setup requires a hosted Linux runner\n' >&2; exit 1; - fi - umask 077 - cache_dir="$(mktemp -d)" - trap 'rm -rf "${cache_dir}"' EXIT - docker ps -aq >"${cache_dir}/containers" - [ ! -s "${cache_dir}/containers" ] || { - printf 'FAIL: registry cache setup requires an unused Docker daemon\n' >&2; exit 1; - } - if sudo test -e /etc/docker/daemon.json; then - sudo cat /etc/docker/daemon.json | tee "${cache_dir}/original.json" >/dev/null - else - sudo test ! -e /etc/docker/daemon.json - printf '{}\n' >"${cache_dir}/original.json" - fi - # Slurp rejects empty or multiple documents rather than treating either as - # an empty daemon configuration. No configuration values reach the job log. - jq -e -s ' - if length != 1 or (.[0] | type) != "object" then error("invalid daemon configuration") - else .[0] end | - if has("registry-mirrors") and - (.["registry-mirrors"] | type != "array" or - any(.[]; type != "string" or length == 0)) - then error("invalid registry mirrors") else . end | - .["registry-mirrors"] = (["https://mirror.gcr.io"] + - [(.["registry-mirrors"] // [])[] | select(. != "https://mirror.gcr.io")]) - ' "${cache_dir}/original.json" >"${cache_dir}/daemon.json" 2>/dev/null || { - printf 'FAIL: invalid daemon or registry cache configuration\n' >&2; exit 1; - } - sudo dockerd --validate --config-file="${cache_dir}/daemon.json" >/dev/null - sudo mkdir -p /etc/docker - sudo install -m 600 "${cache_dir}/daemon.json" /etc/docker/daemon.json - sudo systemctl restart docker - docker info --format '{{json .RegistryConfig.Mirrors}}' | - jq -e 'type == "array" and any(.[]; . == "https://mirror.gcr.io/" or . == "https://mirror.gcr.io")' >/dev/null - printf 'PASS: hosted Docker registry cache configured and read back\n' - exit 0 -fi -if [ "${GITHUB_ACTIONS:-}" = true ] && - [ "${RUNNER_ENVIRONMENT:-}" = github-hosted ] && - [ "${RUNNER_OS:-}" = Linux ]; then - bash "${BASH_SOURCE[0]}" --configure-registry-cache -fi - -root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly root_dir -readonly manifest="${root_dir}/k8s/providers/hetzner/infrastructure/coroot/cron-job-crossview-login-alerter.yaml" -readonly app_policy="${root_dir}/k8s/providers/hetzner/apps/crossview/cilium-network-policy-login-alerter.yaml" -work_dir="$(mktemp -d)" -readonly work_dir -readonly suffix="$$" -readonly net="crossview-login-${suffix}" -readonly db="crossview-db-${suffix}" -readonly app="crossview-app-${suffix}" -readonly probe="crossview-probe-${suffix}" - -# shellcheck disable=SC2317,SC2329 # Invoked by EXIT. -cleanup() { - docker rm -f "${app}" "${db}" "${probe}" >/dev/null 2>&1 || true - docker network rm "${net}" >/dev/null 2>&1 || true - rm -rf "${work_dir}" -} -trap cleanup EXIT - -fail() { - printf 'FAIL: %s\n' "$1" >&2 - if docker inspect "${app}" >/dev/null 2>&1; then - printf -- '--- crossview app log (last 40 lines)\n' >&2 - docker logs --tail 40 "${app}" >&2 2>&1 || true - fi - exit 1 -} -pass() { printf 'ok: %s\n' "$1"; } - -for tool in docker helm jq kubectl yq; do - command -v "${tool}" >/dev/null 2>&1 || fail "${tool} is required" -done - -# ---- Render the pinned chart with the production values ---------------------- -kubectl kustomize "${root_dir}/k8s/providers/hetzner/apps" | yq ea -o=json '[.]' | - jq -e '.[] | select(.kind == "HelmRelease" and .metadata.namespace == "crossview" - and .metadata.name == "crossview")' >"${work_dir}/release.json" || - fail 'the production apps overlay must carry the Crossview HelmRelease' -helm pull "$(jq -r '.spec.chart.spec.chart' "${work_dir}/release.json")" \ - --repo "$(yq -r '.spec.url' "${root_dir}/k8s/bases/apps/crossview/helm-repository.yaml")" \ - --version "$(jq -r '.spec.chart.spec.version' "${work_dir}/release.json")" \ - --destination "${work_dir}" >/dev/null -# Resolve the numeric Flux substitutions Helm must consume, as the durable -# storage test does; string substitutions stay literal and are harmless here. -yq -o=json '.data' "${root_dir}/k8s/clusters/prod/bootstrap/config-map.yaml" >"${work_dir}/variables.json" -jq --slurpfile variables "${work_dir}/variables.json" '.spec.values | walk( - if type == "string" and test("^\\$\\{[a-z_]+:=[0-9]+\\}$") then - capture("^\\$\\{(?[a-z_]+):=(?[0-9]+)\\}$") | - ($variables[0][.name] // .fallback) | tonumber - else . end)' "${work_dir}/release.json" >"${work_dir}/values.json" -if ! helm template crossview "${work_dir}"/crossview-*.tgz --namespace crossview \ - --values "${work_dir}/values.json" >"${work_dir}/chart.yaml" 2>"${work_dir}/helm.err"; then - cat "${work_dir}/helm.err" >&2 - fail 'the pinned chart must render with the production values' -fi -yq ea -o=json '[.]' "${work_dir}/chart.yaml" >"${work_dir}/chart.json" - -# A missing piece of the chart contract names itself instead of ending the step -# silently through set -e. -pick() { jq -e "$1" "${work_dir}/chart.json" || fail "the rendered chart has no match for: $1"; } -app_container="$(pick '[.[] | select(.kind == "Deployment" and .metadata.name == "crossview") - | .spec.template.spec.containers[] | select(.name == "crossview")] | first')" -db_container="$(pick '[.[] | select(.kind == "Deployment" and .metadata.name == "crossview-postgres") - | .spec.template.spec.containers[] | select(.name == "postgres")] | first')" -config="$(pick '[.[] | select(.kind == "ConfigMap" and .metadata.name == "crossview-config") | .data] | first')" -service="$(pick '[.[] | select(.kind == "Service" and .metadata.name == "crossview-service")] | first')" -app_labels="$(pick '[.[] | select(.kind == "Deployment" and .metadata.name == "crossview") - | .spec.template.metadata.labels] | first')" -app_image="$(jq -r '.image' <<<"${app_container}")" -db_image="$(jq -r '.image' <<<"${db_container}")" -sensor_image="$(yq -r '.spec.jobTemplate.spec.template.spec.containers[] | select(.name == "alerter") | .image' "${manifest}")" - -# ---- The sensor's address is the chart's Service, and the policy admits it --- -service_port="$(jq -r '.spec.ports[] | select(.name == "http") | .port' <<<"${service}")" -target_port="$(jq -r '.spec.ports[] | select(.name == "http") | .targetPort' <<<"${service}")" -container_port="$(jq -r '.ports[] | select(.name == "http") | .containerPort' <<<"${app_container}")" -if [ "${service_port}" != 80 ] || [ "${target_port}" != "${container_port}" ]; then - fail "the sensor reads crossview-service:80, but the chart serves ${service_port} -> ${target_port}" -fi -[ "$(yq -r '.spec.ingress[0].toPorts[0].ports[0].port' "${app_policy}")" = "${container_port}" ] || - fail "the network policy must admit the sensor to the app's container port ${container_port}" -subset_of_app_labels() { - jq -e --argjson labels "${app_labels}" 'to_entries | length > 0 and all(. as $e | $labels[$e.key] == $e.value)' -} -yq -o=json '.spec.endpointSelector.matchLabels' "${app_policy}" | subset_of_app_labels >/dev/null || - fail 'the network policy must select the rendered app pods' -jq '.spec.selector' <<<"${service}" | subset_of_app_labels >/dev/null || - fail 'crossview-service must route to the rendered app pods' -pass "the sensor reads the chart's Service and the policy admits it to port ${container_port}" - -# ---- Configuration exactly as the chart wires it ----------------------------- -db_password="db-$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" -admin_password="admin-$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" -session_secret="session-$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" -admin_username="$(pick '[.[] | select(.kind == "Secret" and .metadata.name == "crossview-secrets") - | (.stringData.adminusername // (.data.adminusername | @base64d))] | first' | jq -r .)" -[ -n "${admin_username}" ] || fail 'the chart must name the break-glass admin' -jq -r --argjson config "${config}" --arg db "${db_password}" --arg admin "${admin_password}" \ - --arg session "${session_secret}" --arg user "${admin_username}" ' - .env[] | - if .value != null then "\(.name)=\(.value)" - elif .valueFrom.configMapKeyRef != null then "\(.name)=\($config[.valueFrom.configMapKeyRef.key] // "")" - elif .valueFrom.secretKeyRef.key == "db-password" then "\(.name)=\($db)" - elif .valueFrom.secretKeyRef.key == "admin-password" then "\(.name)=\($admin)" - elif .valueFrom.secretKeyRef.key == "session-secret" then "\(.name)=\($session)" - elif .valueFrom.secretKeyRef.key == "adminusername" then "\(.name)=\($user)" - elif .valueFrom.secretKeyRef.key == "oidc-client-secret" then "\(.name)=unused-in-this-test" - else error("unmapped env \(.name)") end' <<<"${app_container}" >"${work_dir}/app.env" -db_name="$(jq -r '.DB_NAME' <<<"${config}")" -db_user="$(jq -r '.DB_USER' <<<"${config}")" -db_host="$(jq -r '.DB_HOST' <<<"${config}")" -jq -r --arg db "${db_password}" '.env[] | - if .name == "POSTGRES_PASSWORD" then "\(.name)=\($db)" - elif .value != null then "\(.name)=\(.value)" - else empty end' <<<"${db_container}" >"${work_dir}/db.env" -grep -qx "POSTGRES_DB=${db_name}" "${work_dir}/db.env" || - fail 'the bundled database must create the database the app connects to' - -# ---- The sensor, unchanged except for the Service port and Alertmanager ----- -mkdir -p "${work_dir}/sensor/bin" "${work_dir}/capture" -chmod 0777 "${work_dir}/capture" -yq -r '.spec.jobTemplate.spec.template.spec.containers[] | select(.name == "alerter") | .command[2]' "${manifest}" | - sed 's#crossview-service.crossview.svc.cluster.local.:80/#crossview-service.crossview.svc.cluster.local.:'"${container_port}"'/#' \ - >"${work_dir}/sensor/sensor.sh" -grep -Fq "crossview-service.crossview.svc.cluster.local.:${container_port}/api/auth/check" "${work_dir}/sensor/sensor.sh" || - fail 'could not point the sensor at the container port' -real_curl="$(docker run --rm --entrypoint /bin/sh "${sensor_image}" -c 'command -v curl')" -[ -n "${real_curl}" ] || fail 'the sensor image must provide curl' -# Alertmanager is not part of this reproduction: capture what the sensor posts. -cat >"${work_dir}/sensor/bin/curl" <"/capture/alerts-\${peer%%.*}.json" - exit 0 ;; - esac -done -exec ${real_curl} "\$@" -STUB -chmod 0755 "${work_dir}/sensor/bin/curl" -chmod -R a+rX "${work_dir}/sensor" - -samples="$(yq -r '.spec.jobTemplate.spec.template.spec.containers[] | select(.name == "alerter") - | .env[] | select(.name == "SAMPLES") | .value' "${manifest}")" - -# Run the sensor the way the CronJob does: same image, user and read-only root. -run_sensor() { - rm -f "${work_dir}"/capture/alerts-*.json - sensor_status=0 - docker run --rm --network "${net}" --user 65532:65532 --read-only \ - --tmpfs /tmp:size=1m,mode=1777 --cap-drop ALL --security-opt no-new-privileges \ - -v "${work_dir}/sensor:/sensor:ro" -v "${work_dir}/capture:/capture" \ - -e PATH="/sensor/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ - -e SAMPLES="${samples}" -e SAMPLE_INTERVAL_SECONDS=2 \ - --entrypoint /bin/sh "${sensor_image}" /sensor/sensor.sh \ - >"${work_dir}/sensor.log" 2>&1 || sensor_status=$? - sed 's/^/ sensor: /' "${work_dir}/sensor.log" -} -alerted() { ls "${work_dir}"/capture/alerts-*.json >/dev/null 2>&1; } - -# One request from inside the network, through the sensor image's own curl in a -# long-lived helper container. -app_get() { - docker exec "${probe}" "${real_curl}" -sS --max-time 5 -o /dev/stdout -w '\n%{http_code}' \ - "http://crossview-service.crossview.svc.cluster.local.:${container_port}$1" 2>/dev/null || true -} -wait_for_app() { - for _ in $(seq 1 60); do - [ "$(app_get /api/health | tail -n 1)" = 200 ] && return 0 - sleep 2 - done - fail 'the Crossview app never answered /api/health' -} -start_db() { - docker run -d --name "${db}" --network "${net}" --network-alias "${db_host}" \ - --env-file "${work_dir}/db.env" "${db_image}" >/dev/null - # Only the final server listens on TCP; the image's init-time server does not. - for _ in $(seq 1 60); do - db_query 'SELECT 1' >/dev/null 2>&1 && return 0 - sleep 2 - done - fail 'the bundled PostgreSQL never accepted TCP connections' -} -db_query() { - docker exec -e PGPASSWORD="${db_password}" "${db}" \ - psql -h 127.0.0.1 -U "${db_user}" -d "${db_name}" -tAc "$1" -} -users_table() { db_query "SELECT to_regclass('public.users') IS NOT NULL"; } - -docker network create --internal "${net}" >/dev/null -docker run -d --name "${probe}" --network "${net}" --user 65532:65532 \ - --entrypoint /bin/sh "${sensor_image}" -c 'sleep 3600' >/dev/null -start_db -# As in production: non-root, read-only root, only the chart's /app/logs writable. -docker run -d --name "${app}" --network "${net}" \ - --network-alias crossview-service.crossview.svc.cluster.local \ - --user 1000:1000 --read-only --tmpfs /app/logs:mode=1777 \ - --cap-drop ALL --security-opt no-new-privileges \ - --env-file "${work_dir}/app.env" "${app_image}" >/dev/null -wait_for_app - -expect_healthy() { - run_sensor - if [ "${sensor_status}" != 0 ] || alerted; then - fail "$1" - fi - grep -q 'can read its users' "${work_dir}/sensor.log" || fail 'the healthy verdict must be explicit' -} - -# 1. Bootstrapped. -printf '1. bootstrapped: /api/auth/check -> %s\n' "$(app_get /api/auth/check | head -n 1)" -[ "$(users_table)" = t ] || fail 'the app must create its users table at startup' -expect_healthy 'a freshly bootstrapped Crossview must read as healthy' -pass 'a bootstrapped Crossview reads as healthy' - -# 2. The database comes back empty under the running app. -docker rm -f "${db}" >/dev/null -start_db -[ "$(users_table)" = f ] || fail 'the replacement database must start without the schema' -for _ in $(seq 1 30); do - [ "$(app_get /api/auth/check | tail -n 1)" = 200 ] && break - sleep 2 -done -printf '2. schema gone: /api/health -> %s, /api/auth/check -> %s\n' \ - "$(app_get /api/health | tail -n 1)" "$(app_get /api/auth/check | head -n 1)" -[ "$(app_get /api/health | tail -n 1)" = 200 ] || fail 'the reproduction needs the probe to stay green, as it did in the outage' -run_sensor -[ "${sensor_status}" = 0 ] || fail 'the sensor must deliver its alert' -alerted || fail 'an empty database under a running app must alert' -jq -e 'length == 1 and .[0].labels.alertname == "CrossviewLoginBroken" - and (.[0].annotations.description | contains("\"hasAdmin\":false"))' \ - "${work_dir}/capture/alerts-0.json" >/dev/null || fail 'the alert must carry the answer that triggered it' -pass 'an empty database alerts while /api/health stays green' - -# 3. Restarting the app re-runs its bootstrap. -docker restart "${app}" >/dev/null -wait_for_app -printf '3. restarted: /api/auth/check -> %s\n' "$(app_get /api/auth/check | head -n 1)" -[ "$(users_table)" = t ] || fail 'restarting the app must recreate its users table' -expect_healthy 'a restarted Crossview must read as healthy again' -pass 'restarting the app restores the schema and clears the signal' - -# 4. A database the app cannot reach is never health. Sign-in fails here too, -# so either verdict that is not health is a signal: an alert, or a failed Job -# that the CronJob failure detector reports. -docker stop "${db}" >/dev/null -printf '4. database stopped: /api/auth/check -> %s\n' "$(app_get /api/auth/check | tr '\n' ' ')" -run_sensor -if [ "${sensor_status}" = 0 ] && ! alerted; then - fail 'an unreachable database must not read as healthy' -fi -if grep -q 'can read its users' "${work_dir}/sensor.log"; then - fail 'an unreachable database reported readable users' -fi -if alerted; then - printf ' -> the sensor alerted\n' -else - printf ' -> the sensor failed the Job (exit %s)\n' "${sensor_status}" -fi -pass 'an unreachable database does not read as healthy' - -# 5. The database comes back with its data: no restart is needed. -docker start "${db}" >/dev/null -for _ in $(seq 1 60); do - db_query 'SELECT 1' >/dev/null 2>&1 && break - sleep 2 -done -[ "$(users_table)" = t ] || fail 'the restarted database must still hold the schema' -for _ in $(seq 1 30); do - app_get /api/auth/check | head -n 1 | grep -q '"hasAdmin":true' && break - sleep 2 -done -printf '5. database back: /api/auth/check -> %s\n' "$(app_get /api/auth/check | head -n 1)" -expect_healthy 'the app must recover on its own once its database is back with its data' -pass 'a database that returns with its data needs no app restart' - -printf 'PASS: the Crossview login sensor detects the #3315 outage on the real app\n' diff --git a/scripts/tests/test-crossview-login-alerter.sh b/scripts/tests/test-crossview-login-alerter.sh deleted file mode 100755 index e4dad9f158..0000000000 --- a/scripts/tests/test-crossview-login-alerter.sh +++ /dev/null @@ -1,453 +0,0 @@ -#!/usr/bin/env bash - -# Exercise the deployed Crossview login sensor (#3315) against stubbed -# /api/auth/check answers and captured Alertmanager deliveries, and pin the -# wiring that lets it reach both. Only network calls, sleeps and in-pod mount -# paths are replaced; the script under test is the one the CronJob runs. -# -# The broken-state body below is the one Crossview served during the August 2026 -# outage, recorded on #3315. test-crossview-login-alerter-runtime.sh reproduces -# that state against the real pinned app and database. -set -euo pipefail - -root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly root_dir -readonly manifest="${root_dir}/k8s/providers/hetzner/infrastructure/coroot/cron-job-crossview-login-alerter.yaml" -readonly app_policy="${root_dir}/k8s/providers/hetzner/apps/crossview/cilium-network-policy-login-alerter.yaml" -readonly alertmanager_policy="${root_dir}/k8s/providers/hetzner/infrastructure/controllers/alertmanager/cilium-network-policy.yaml" -readonly watcher="${root_dir}/k8s/bases/components/coroot-cronjob-failure-alert/cron-job-cronjob-failure-alert.yaml" -readonly watcher_role="${root_dir}/k8s/bases/components/coroot-cronjob-failure-alert/role-cronjob-failure-alert-observability.yaml" -readonly alerting_doc="${root_dir}/docs/dr/alerting.md" -work_dir="$(mktemp -d)" -readonly work_dir -trap 'rm -rf "${work_dir}"' EXIT - -fail() { printf 'FAIL: %s\n' "$1" >&2; exit 1; } -pass() { printf 'ok: %s\n' "$1"; } - -for tool in jq kubectl yq; do - command -v "${tool}" >/dev/null 2>&1 || fail "${tool} is required" -done - -# The runtime fixture configures only an ephemeral hosted daemon. Exercise the -# actual command boundary, including failed setup, without touching this host. -mkdir -p "${work_dir}/cache-bin" -cat >"${work_dir}/cache-bin/sudo" <<'STUB' -#!/usr/bin/env bash -set -euo pipefail -printf '%s\n' "$1" >>"${CACHE_CASE}/calls" -case "$1" in - test) - [ "${CACHE_MODE:-}" != test-failed ] || exit 2 - if [ "$2" = ! ]; then test ! -e "${CACHE_CASE}/original.json" - else test -e "${CACHE_CASE}/original.json"; fi ;; - cat) cat "${CACHE_CASE}/original.json" ;; - dockerd) [ "${CACHE_MODE:-}" != invalid-daemon ] ;; - mkdir) : ;; - install) - [ "${CACHE_MODE:-}" != install-failed ] || exit 1 - cp "$4" "${CACHE_CASE}/installed.json" ;; - systemctl) [ "${CACHE_MODE:-}" != restart-failed ] ;; - *) exit 93 ;; -esac -STUB -cat >"${work_dir}/cache-bin/docker" <<'STUB' -#!/usr/bin/env bash -set -euo pipefail -printf 'docker-%s\n' "$1" >>"${CACHE_CASE}/calls" -case "$1" in - ps) - [ "$2" = -aq ] || exit 95 - [ "${CACHE_MODE:-}" != list-failed ] || exit 1 - [ "${CACHE_MODE:-}" != occupied ] || printf 'fixture-container\n' ;; - info) - [ "${CACHE_MODE:-}" != readback-failed ] || exit 1 - if [ "${CACHE_MODE:-}" = missing-mirror ] || - [ ! -e "${CACHE_CASE}/installed.json" ] || - ! grep -qx systemctl "${CACHE_CASE}/calls"; then printf '[]\n' - else printf '["https://mirror.gcr.io/"]\n'; fi ;; - *) exit 94 ;; -esac -STUB -chmod +x "${work_dir}/cache-bin/sudo" "${work_dir}/cache-bin/docker" -cache_case() { - cache_dir="${work_dir}/cache-$1" - mkdir -p "${cache_dir}" - : >"${cache_dir}/calls" - if [ "$#" -ge 2 ]; then printf '%s\n' "$2" >"${cache_dir}/original.json"; fi -} -cache_run() { - cache_rc=0 - PATH="${work_dir}/cache-bin:${PATH}" CACHE_CASE="${cache_dir}" CACHE_MODE="${1:-}" \ - GITHUB_ACTIONS="${4:-true}" RUNNER_ENVIRONMENT="${2:-github-hosted}" RUNNER_OS="${3:-Linux}" \ - bash "${root_dir}/scripts/tests/test-crossview-login-alerter-runtime.sh" --configure-registry-cache \ - >"${cache_dir}/output" 2>&1 || cache_rc=$? -} -cache_case preserve '{"debug":false,"log-driver":"local","registry-mirrors":["https://mirror.example.invalid","https://mirror.gcr.io"]}' -cache_run -[ "${cache_rc}" = 0 ] || fail 'hosted cache setup must succeed' -jq -e '.debug == false and .["log-driver"] == "local" and - .["registry-mirrors"] == ["https://mirror.gcr.io","https://mirror.example.invalid"]' \ - "${cache_dir}/installed.json" >/dev/null || fail 'cache setup must preserve unrelated settings and mirrors' -grep -q '^systemctl$' "${cache_dir}/calls" || fail 'the daemon must restart before cache readback' -cache_case absent -cache_run -[ "${cache_rc}" = 0 ] || fail 'an absent daemon file must be initialized' -for config in 'not-json' '[]' 'null' '{} {}' '{"registry-mirrors":null}' '{"registry-mirrors":"bad"}' '{"registry-mirrors":[7]}'; do - cache_case malformed "${config}" - cache_run - [ "${cache_rc}" != 0 ] || fail 'malformed daemon configuration must fail' - [ ! -e "${cache_dir}/installed.json" ] || fail 'malformed configuration must not be installed' -done -for mode in occupied list-failed test-failed invalid-daemon install-failed restart-failed readback-failed missing-mirror; do - cache_case "${mode}" '{}' - cache_run "${mode}" - [ "${cache_rc}" != 0 ] || fail "${mode} must fail instead of admitting the runtime fixture" -done -for runner in self-hosted ''; do - cache_case not-hosted '{}' - cache_run '' "${runner:-local}" - [ "${cache_rc}" != 0 ] || fail 'non-hosted daemon changes must be refused' - [ ! -s "${cache_dir}/calls" ] || fail 'non-hosted refusal must precede all daemon commands' -done -cache_case not-linux '{}' -cache_run '' github-hosted macOS -if [ "${cache_rc}" = 0 ] || [ -s "${cache_dir}/calls" ]; then - fail 'non-Linux daemon changes must be refused' -fi -cache_case not-actions '{}' -cache_run '' github-hosted Linux false -if [ "${cache_rc}" = 0 ] || [ -s "${cache_dir}/calls" ]; then - fail 'local daemon changes must be refused' -fi -pass 'hosted registry cache preserves configuration and fails closed without changing local daemons' - -# The independent Wedding fixture builds a pinned Docker Hub toolbox too. Its -# hosted setup must finish before any build; failed setup must stop the fixture. -cache_runtime_run() { - cache_rc=0 - PATH="${work_dir}/cache-bin:${PATH}" CACHE_CASE="${cache_dir}" CACHE_MODE="${1:-}" \ - GITHUB_ACTIONS="${2-true}" RUNNER_ENVIRONMENT="${3-github-hosted}" RUNNER_OS="${4-Linux}" \ - bash "${root_dir}/scripts/tests/${5-test-wedding-backup-denial-runtime.sh}" \ - >"${cache_dir}/output" 2>&1 || cache_rc=$? -} -cache_case wedding-hosted '{}' -cache_runtime_run -[ "${cache_rc}" = 94 ] || fail 'the hosted Wedding fixture must reach the stubbed build only after setup' -awk '/^docker-info$/ { readback=1 } /^docker-build$/ { if (!readback) exit 1; built=1 } END { if (!built) exit 1 }' \ - "${cache_dir}/calls" || fail 'the Wedding build must follow cache readback' -cache_case wedding-setup-failed '{}' -cache_runtime_run invalid-daemon -if [ "${cache_rc}" = 0 ] || grep -qx docker-build "${cache_dir}/calls"; then - fail 'failed cache setup must stop the Wedding fixture before its build' -fi -cache_case wedding-local '{}' -cache_runtime_run '' false -if [ "${cache_rc}" != 94 ] || [ -e "${cache_dir}/installed.json" ]; then - fail 'a local Wedding fixture must build without reconfiguring Docker' -fi -pass 'both hosted runtime fixtures configure the cache before pulling pinned images' - -# Exercise each ordinary fixture's automatic setup decision, not just the -# explicit hosted-only setup mode. Stop Crossview at rendering so these cases -# never use Docker or fetch an image; Wedding stops at its stubbed build. -cat >"${work_dir}/cache-bin/kubectl" <<'STUB' -#!/usr/bin/env bash -set -euo pipefail -[ "$1" = kustomize ] || exit 97 -printf 'fixture-render\n' >>"${CACHE_CASE}/calls" -exit 96 -STUB -chmod +x "${work_dir}/cache-bin/kubectl" -for fixture in test-wedding-backup-denial-runtime.sh test-crossview-login-alerter-runtime.sh; do - case "${fixture}" in - test-wedding*) expected_rc=94; boundary='docker-build' ;; - *) expected_rc=1; boundary='fixture-render' ;; - esac - for runner_case in local self-hosted non-linux missing-environment missing-os; do - actions=true; environment=github-hosted; runner_os=Linux - case "${runner_case}" in - local) actions=false ;; - self-hosted) environment=self-hosted ;; - non-linux) runner_os=macOS ;; - missing-environment) environment='' ;; - missing-os) runner_os='' ;; - esac - cache_case "${fixture}-${runner_case}" '{}' - cache_runtime_run '' "${actions}" "${environment}" "${runner_os}" "${fixture}" - [ "${cache_rc}" = "${expected_rc}" ] || fail "${fixture}: ${runner_case} must continue the ordinary fixture" - [ ! -e "${cache_dir}/installed.json" ] || fail "${fixture}: ${runner_case} must not reconfigure Docker" - grep -qx "${boundary}" "${cache_dir}/calls" || fail "${fixture}: ${runner_case} never reached its ordinary runtime boundary" - if grep -qE '^(test|cat|dockerd|mkdir|install|systemctl|docker-ps|docker-info)$' "${cache_dir}/calls"; then - fail "${fixture}: ${runner_case} must not attempt hosted cache setup" - fi - done - cache_case "${fixture}-automatic-hosted" '{}' - cache_runtime_run '' true github-hosted Linux "${fixture}" - [ "${cache_rc}" = "${expected_rc}" ] || fail "${fixture}: hosted setup must continue the ordinary fixture" - awk -v boundary="${boundary}" '/^docker-info$/ { readback=1 } $0 == boundary { if (!readback) exit 1; reached=1 } END { if (!reached) exit 1 }' \ - "${cache_dir}/calls" || fail "${fixture}: hosted cache readback must precede the ordinary fixture" -done -rm "${work_dir}/cache-bin/kubectl" -pass 'automatic cache setup leaves local and self-hosted runtime fixtures usable' - -readonly pod='.spec.jobTemplate.spec.template.spec' -readonly container="${pod}.containers[] | select(.name == \"alerter\")" -env_value() { yq -r "${container} | .env[] | select(.name == \"$1\") | .value" "${manifest}"; } -script_body="$(yq -r "${container} | .command[2]" "${manifest}")" -readonly script_body -if [ -z "${script_body}" ] || [ "${script_body}" = null ]; then - fail 'could not extract the sensor script' -fi - -# ---- Wiring ---------------------------------------------------------------- -# Render what Flux applies, so a file that exists but is not referenced fails. -render() { kubectl kustomize "${root_dir}/$1" | yq ea -o=json -I=0 '[.]'; } -render k8s/providers/hetzner/infrastructure >"${work_dir}/prod-infrastructure.json" -render k8s/providers/hetzner/apps >"${work_dir}/prod-apps.json" -render k8s/providers/docker/infrastructure >"${work_dir}/local-infrastructure.json" -render k8s/providers/docker/apps >"${work_dir}/local-apps.json" - -jq -e '[.[] | select(.kind == "CronJob" and .metadata.namespace == "observability" - and .metadata.name == "crossview-login-alerter")] | length == 1' \ - "${work_dir}/prod-infrastructure.json" >/dev/null || - fail 'the production infrastructure layer must deploy the sensor' -jq -e '[.[] | select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "crossview" - and .metadata.name == "allow-login-alerter")] | length == 1' \ - "${work_dir}/prod-apps.json" >/dev/null || - fail 'the production apps layer must admit the sensor to Crossview' -for layer in local-infrastructure local-apps; do - jq -e '[.[] | select(.metadata.name == "crossview-login-alerter" - or .metadata.name == "allow-login-alerter")] | length == 0' \ - "${work_dir}/${layer}.json" >/dev/null || - fail "the local provider (${layer}) has no Crossview or Alertmanager, so it must not carry the sensor" -done -pass 'the sensor and its network path deploy to production only' - -label="$(yq -r '.spec.jobTemplate.spec.template.metadata.labels.app' "${manifest}")" -[ "${label}" = crossview-login-alerter ] || fail "unexpected sensor pod label '${label}'" -policy_json() { yq -o=json "$1" "${app_policy}" | jq -cS .; } -[ "$(policy_json '.spec.endpointSelector')" = \ - '{"matchLabels":{"app.kubernetes.io/component":"app","app.kubernetes.io/name":"crossview"}}' ] || - fail 'the Crossview policy must select only the app pods' -[ "$(policy_json '.spec')" = \ - "{\"endpointSelector\":{\"matchLabels\":{\"app.kubernetes.io/component\":\"app\",\"app.kubernetes.io/name\":\"crossview\"}},\"ingress\":[{\"fromEndpoints\":[{\"matchLabels\":{\"app\":\"${label}\",\"k8s:io.kubernetes.pod.namespace\":\"observability\"}}],\"toPorts\":[{\"ports\":[{\"port\":\"3001\",\"protocol\":\"TCP\"}]}]}]}" ] || - fail 'the Crossview policy must admit only the sensor pod label to port 3001, and allow nothing else' -yq -e ".spec.ingress[] | select(.fromEndpoints[0].matchLabels.app == \"${label}\" - and .fromEndpoints[0].matchLabels.\"k8s:io.kubernetes.pod.namespace\" == \"observability\" - and .toPorts[0].ports[0].port == \"9093\")" "${alertmanager_policy}" >/dev/null || - fail 'Alertmanager must admit the sensor pod label to its v2 API' -for target in \ - 'http://crossview-service.crossview.svc.cluster.local.:80/api/auth/check' \ - 'http://alertmanager-0.alertmanager-headless.kubescape.svc.cluster.local.:9093' \ - 'http://alertmanager-1.alertmanager-headless.kubescape.svc.cluster.local.:9093'; do - grep -Fq "${target}" <<<"${script_body}" || fail "target is not an absolute cluster DNS name: ${target}" -done -pass 'both network policies admit exactly the sensor pod, on absolute DNS names' - -yq -e '.metadata.annotations."kustomize.toolkit.fluxcd.io/substitute" == "disabled"' "${manifest}" >/dev/null || - fail 'Flux substitution would blank the shell variables the script reads' -yq -e "${pod}.automountServiceAccountToken == false" "${manifest}" >/dev/null || - fail 'the sensor needs no Kubernetes API access, so it must not mount a token' -yq -e "${container} | .securityContext.readOnlyRootFilesystem == true" "${manifest}" >/dev/null || - fail 'the sensor must not need a writable root filesystem' -scratch="$(yq -r "${container} | .volumeMounts[] | select(.mountPath == \"/tmp\") | .name" "${manifest}")" -yq -e "${pod}.volumes[] | select(.name == \"${scratch}\") | .emptyDir.sizeLimit != null" "${manifest}" >/dev/null || - fail 'the sensor scratch space must be a bounded emptyDir' -[ "$(yq -r "${pod}.securityContext.fsGroup" "${manifest}")" = "$(yq -r "${pod}.securityContext.runAsUser" "${manifest}")" ] || - fail 'the non-root sensor must be able to write its scratch volume' -pass 'the sensor runs read-only, without a token, on bounded scratch' - -# curl restarts --max-time on every retry, so only --retry-max-time bounds a call. -curl_calls="$(grep -c 'curl -sS' <<<"${script_body}")" -[ "${curl_calls}" = 2 ] || fail "expected one read and one delivery call, found ${curl_calls} curl calls" -# shellcheck disable=SC2016 # matches the script's literal "$BODY" -grep -Fq -- '--max-time 10 --retry-max-time 30 -o "$BODY"' <<<"${script_body}" || - fail 'each read must bound its retries to 30s' -grep -Fq -- '--max-time 10 --retry-max-time 20' <<<"${script_body}" || - fail 'each delivery must bound its retries to 20s' -samples="$(env_value SAMPLES)" -interval="$(env_value SAMPLE_INTERVAL_SECONDS)" -deadline="$(yq -r '.spec.jobTemplate.spec.activeDeadlineSeconds' "${manifest}")" -# --retry-max-time stops only NEW attempts, so one call can run its retry budget -# plus one more --max-time attempt: 30+10 per read, 20+10 per Alertmanager peer. -worst=$((samples * 40 + (samples - 1) * interval + 2 * 30)) -[ "${deadline}" -gt "${worst}" ] || fail "the Job deadline (${deadline}s) must cover the worst case (${worst}s)" -# One healthy read ends a run; a page needs every read over this window false, -# so a planned database restart (volume detach, reattach, start) is quiet. -if [ "${samples}" -lt 3 ] || [ $(((samples - 1) * interval)) -lt 300 ]; then - fail 'an alert must need at least three false reads spanning five minutes' -fi -pass "the deadline covers the worst case (${worst}s of ${deadline}s) and the alert needs a sustained answer" - -watch_entry="$(yq -r '.spec.jobTemplate.spec.template.spec.containers[0].env[] | select(.name == "WATCH") | .value' "${watcher}" | - tr ' ' '\n' | grep '^observability/crossview-login-alerter:' || true)" -[ "${watch_entry}" = 'observability/crossview-login-alerter:2:3600' ] || - fail 'the sensor must be on the CronJob failure detector watch list, or a broken sensor reads as a healthy login' -yq -e '.rules[] | select(.resources[] == "cronjobs") | .resourceNames[] | select(. == "crossview-login-alerter")' \ - "${watcher_role}" >/dev/null || fail 'the failure detector must be allowed to read the sensor CronJob' -[ "$(yq -r '.spec.failedJobsHistoryLimit' "${manifest}")" -ge 2 ] || - fail 'the sensor must retain at least as many failed Jobs as the detector threshold' -grep -Fq 'providers/hetzner/infrastructure/coroot/cron-job-crossview-login-alerter.yaml' "${alerting_doc}" || - fail 'the alerting runbook must point at the sensor' -pass 'a failing sensor is reported by the CronJob failure detector' - -# ---- Behaviour ------------------------------------------------------------- -mkdir -p "${work_dir}/bin" -cat >"${work_dir}/bin/curl" <<'STUB' -#!/usr/bin/env bash -set -euo pipefail -output='' payload='' url='' -while [ "$#" -gt 0 ]; do - case "$1" in - -o) output="$2"; shift 2 ;; - -d) payload="$2"; shift 2 ;; - -w|-H|-X|--retry|--retry-delay|--max-time|--retry-max-time) shift 2 ;; - http*) url="$1"; shift ;; - *) shift ;; - esac -done -case "$url" in - http://crossview-service.crossview.svc.cluster.local.:80/api/auth/check) - n=$(($(cat "${CASE_DIR}/reads" 2>/dev/null || echo 0) + 1)) - printf '%s\n' "$n" >"${CASE_DIR}/reads" - answer="${CASE_DIR}/read-${n}" - [ -e "${answer}.code" ] || answer="${CASE_DIR}/read-default" - code="$(cat "${answer}.code")" - if [ "$code" = refused ]; then - printf '000' - exit 7 - fi - cat "${answer}.body" >"$output" - printf '%s' "$code" ;; - http://alertmanager-[01].alertmanager-headless.kubescape.svc.cluster.local.:9093/api/v2/alerts) - peer="${url#http://alertmanager-}" - peer="${peer%%.*}" - printf '%s\n' "$payload" >"${CASE_DIR}/alerts-${peer}.json" - [ ! -e "${CASE_DIR}/reject-${peer}" ] || exit 22 ;; - *) exit 93 ;; -esac -STUB -cat >"${work_dir}/bin/sleep" <<'STUB' -#!/usr/bin/env bash -printf '%s\n' "$1" >>"${CASE_DIR}/sleeps" -STUB -chmod +x "${work_dir}/bin/curl" "${work_dir}/bin/sleep" - -readonly healthy_body='{"authMode":"session","authenticated":false,"hasAdmin":true,"hasUsers":true}' -# Verbatim from the August 2026 outage (#3315). -readonly broken_body='{"authMode":"session","authenticated":false,"hasAdmin":false,"hasUsers":false}' - -# answer [body] -answer() { - printf '%s\n' "$2" >"${case_dir}/read-$1.code" - printf '%s\n' "${3:-}" >"${case_dir}/read-$1.body" -} -new_case() { - case_dir="${work_dir}/case-$1" - mkdir -p "${case_dir}/tmp" - printf '%s\n' "${script_body//\/tmp\//${case_dir}/tmp/}" >"${case_dir}/sensor.sh" - answer default 200 "${healthy_body}" -} -run_case() { - result=0 - PATH="${work_dir}/bin:${PATH}" CASE_DIR="${case_dir}" \ - SAMPLES="${SAMPLES_OVERRIDE-${samples}}" SAMPLE_INTERVAL_SECONDS="${interval}" \ - sh "${case_dir}/sensor.sh" >"${case_dir}/output.log" 2>&1 || result=$? -} -reads() { cat "${case_dir}/reads" 2>/dev/null || echo 0; } -no_alert() { ! ls "${case_dir}"/alerts-*.json >/dev/null 2>&1 || fail "$1 must not alert"; } - -new_case healthy -run_case -[ "${result}" = 0 ] || fail 'a bootstrapped Crossview must pass' -[ "$(reads)" = 1 ] || fail 'one healthy read must end the run' -[ ! -e "${case_dir}/sleeps" ] || fail 'a healthy run must not wait' -no_alert 'a bootstrapped Crossview' -grep -q 'can read its users' "${case_dir}/output.log" || fail 'a healthy verdict must be explicit' -pass 'a bootstrapped Crossview is healthy after one read' - -new_case schema-missing -answer default 200 "${broken_body}" -run_case -[ "${result}" = 0 ] || fail 'a delivered alert must succeed' -[ "$(reads)" = "${samples}" ] || fail 'every sample must be read before alerting' -expected_sleeps='' -for ((i = 1; i < samples; i++)); do expected_sleeps="${expected_sleeps}${interval} "; done -[ "$(tr '\n' ' ' <"${case_dir}/sleeps")" = "${expected_sleeps}" ] || - fail 'samples must be spaced by SAMPLE_INTERVAL_SECONDS' -for peer in 0 1; do - jq -e --arg observed '{"authMode":"session","hasAdmin":false,"hasUsers":false}' ' - length == 1 and - .[0].labels == {alertname: "CrossviewLoginBroken", severity: "warning", namespace: "crossview"} and - (.[0].annotations.description | contains($observed)) and - (.[0].annotations.runbook | contains("crossview-postgres") and contains("platform.devantler.tech/db-bootstrap")) and - (.[0].endsAt | test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$")) - ' "${case_dir}/alerts-${peer}.json" >/dev/null || fail "alertmanager-${peer} must receive the schema alert" -done -pass 'the outage answer on every read alerts both Alertmanager peers' - -new_case transient -answer 1 200 "${broken_body}" -answer default 200 "${healthy_body}" -run_case -[ "${result}" = 0 ] || fail 'a database blip under a healthy app must pass' -[ "$(reads)" = 2 ] || fail 'the first healthy read must end the run' -no_alert 'a single false read' -pass 'a single read without users does not page' - -# The users table is readable even when the break-glass admin was removed, so -# sign-in works and nothing pages. -new_case admin-removed -answer default 200 '{"authMode":"session","authenticated":false,"hasAdmin":false,"hasUsers":true}' -run_case -[ "${result}" = 0 ] || fail 'readable users without the admin must pass' -[ "$(reads)" = 1 ] || fail 'a readable users table must end the run' -no_alert 'a removed admin with readable users' -pass 'a removed admin alone does not page' - -# No answer is never health and never a confirmed outage: the Job fails, and the -# CronJob failure detector reports a sensor that keeps failing. -for problem in refused unavailable no-field no-users-field string-field not-json mixed; do - new_case "${problem}" - case "${problem}" in - refused) answer default refused ;; - unavailable) answer default 503 '{"error":"unavailable"}' ;; - no-field) answer default 200 '{"authMode":"session","authenticated":false}' ;; - no-users-field) answer default 200 '{"authMode":"session","authenticated":false,"hasAdmin":false}' ;; - string-field) answer default 200 '{"authMode":"session","hasAdmin":"true","hasUsers":true}' ;; - not-json) answer default 200 'sign in' ;; - mixed) - answer 1 200 "${broken_body}" - answer 2 refused - answer default 200 "${broken_body}" ;; - esac - run_case - [ "${result}" != 0 ] || fail "${problem} must fail the Job instead of reporting a verdict" - [ "$(reads)" = "${samples}" ] || fail "${problem} must still take every sample" - no_alert "${problem}" - ! grep -q 'can read its users' "${case_dir}/output.log" || fail "${problem} reported health" -done -pass 'refusals, error statuses and unrecognised bodies fail the Job without a verdict' - -new_case one-peer-down -answer default 200 "${broken_body}" -touch "${case_dir}/reject-0" -run_case -[ "${result}" = 0 ] || fail 'one accepting Alertmanager peer is a delivered alert' -new_case both-peers-down -answer default 200 "${broken_body}" -touch "${case_dir}/reject-0" "${case_dir}/reject-1" -run_case -[ "${result}" != 0 ] || fail 'an alert no peer accepted must fail the Job' -pass 'delivery fails only when no Alertmanager peer accepts the alert' - -for bad in 0 10 '' x; do - new_case "bad-samples-${bad:-empty}" - SAMPLES_OVERRIDE="${bad}" run_case - [ "${result}" != 0 ] || fail "SAMPLES='${bad}' must be refused" - [ "$(reads)" = 0 ] || fail "SAMPLES='${bad}' must be refused before any read" -done -pass 'an invalid sample count is refused before any read' - -printf 'PASS: the Crossview login sensor alerts on a schemaless database and only then\n' diff --git a/scripts/tests/test-crossview-postgres-durable-storage.sh b/scripts/tests/test-crossview-postgres-durable-storage.sh deleted file mode 100755 index 6556d75a5b..0000000000 --- a/scripts/tests/test-crossview-postgres-durable-storage.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env bash - -# Exercise the pinned Crossview chart through the production Flux -# post-renderers. A HelmRelease-only assertion cannot prove that the database -# Deployment consumes the intended PVC or that its single-writer rollout is -# safe. -set -euo pipefail - -root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -readonly root_dir -scratch_dir="$(mktemp -d)" -readonly scratch_dir -trap 'rm -rf "${scratch_dir}"' EXIT - -fail() { printf 'FAIL: %s\n' "$1" >&2; exit 1; } - -for tool in helm jq kubectl yq; do - command -v "${tool}" >/dev/null || fail "${tool} is required" -done - -render_overlay() { - local overlay="$1" - local destination="$2" - - kubectl kustomize "${overlay}" | - yq ea -o=json '[.]' | - jq -S 'sort_by([.apiVersion, .kind, .metadata.namespace, .metadata.name])' \ - >"${destination}" -} - -extract_release() { - local resources="$1" - local destination="$2" - - jq -e '.[] | select( - .kind == "HelmRelease" and - .metadata.namespace == "crossview" and - .metadata.name == "crossview" - )' "${resources}" >"${destination}" -} - -render_overlay "${root_dir}/k8s/providers/hetzner/apps" "${scratch_dir}/prod.json" -render_overlay "${root_dir}/k8s/providers/docker/apps" "${scratch_dir}/local.json" -render_overlay "${root_dir}/k8s/bases/apps/crossview" "${scratch_dir}/base.json" -extract_release "${scratch_dir}/prod.json" "${scratch_dir}/prod-release.json" -extract_release "${scratch_dir}/base.json" "${scratch_dir}/base-release.json" - -jq -e ' - (.spec.values.database.persistence.enabled == true) and - (.spec.values.database.persistence.size == "2Gi") and - (.spec.values.database.persistence.accessMode == "ReadWriteOnce") and - (.spec.values.database.persistence.storageClass == "longhorn-wffc") and - (.metadata.annotations["kustomize.toolkit.fluxcd.io/prune"] == "disabled") -' "${scratch_dir}/prod-release.json" >/dev/null || - fail 'production Crossview PostgreSQL must declare a protected 2Gi longhorn-wffc RWO volume' - -jq -e ' - (.spec.values.database.persistence.enabled == false) and - (.spec.values.database.persistence.storageClass // null) == null -' "${scratch_dir}/base-release.json" >/dev/null || - fail 'the reusable Crossview base must keep persistence disabled without a Longhorn dependency' - -jq -e '[.[] | select( - .kind == "HelmRelease" and - .metadata.namespace == "crossview" and - .metadata.name == "crossview" -)] | length == 0' "${scratch_dir}/local.json" >/dev/null || - fail 'the default Docker provider must remain free of the production-only Crossview release' - -jq -e '[.[] | select( - .kind == "Namespace" and .metadata.name == "crossview" -) | ( - (.metadata.labels["pod-security.devantler.tech/user-namespaces"] // null) == null and - .metadata.annotations["kustomize.toolkit.fluxcd.io/prune"] == "disabled" -)] == [true]' "${scratch_dir}/prod.json" >/dev/null || - fail 'the mixed Crossview namespace must leave userns enforcement disabled while PostgreSQL uses Longhorn' - -chart_name="$(jq -r '.spec.chart.spec.chart' "${scratch_dir}/prod-release.json")" -chart_version="$(jq -r '.spec.chart.spec.version' "${scratch_dir}/prod-release.json")" -chart_repo="$(yq -r '.spec.url' "${root_dir}/k8s/bases/apps/crossview/helm-repository.yaml")" -helm pull "${chart_name}" --repo "${chart_repo}" --version "${chart_version}" \ - --destination "${scratch_dir}" -readonly chart_archive="${scratch_dir}/${chart_name}-${chart_version}.tgz" - -# Resolve the numeric production substitutions that Helm must consume. Other -# substitutions are string-valued URLs or credentials and are safe to preserve -# literally for this chart render. -yq -o=json '.data' "${root_dir}/k8s/clusters/prod/bootstrap/config-map.yaml" \ - >"${scratch_dir}/variables.json" -jq --slurpfile variables "${scratch_dir}/variables.json" '.spec.values | walk( - if type == "string" and test("^\\$\\{[a-z_]+:=[0-9]+\\}$") then - capture("^\\$\\{(?[a-z_]+):=(?[0-9]+)\\}$") | - ($variables[0][.name] // .fallback) | tonumber - else . end -)' "${scratch_dir}/prod-release.json" >"${scratch_dir}/values.json" - -helm template crossview "${chart_archive}" --namespace crossview \ - --values "${scratch_dir}/values.json" >"${scratch_dir}/resources.yaml" - -renderer_count="$(jq '.spec.postRenderers | length' "${scratch_dir}/prod-release.json")" -for ((renderer_index = 0; renderer_index < renderer_count; renderer_index++)); do - render_dir="${scratch_dir}/renderer-${renderer_index}" - mkdir "${render_dir}" - cp "${scratch_dir}/resources.yaml" "${render_dir}/resources.yaml" - jq --argjson index "${renderer_index}" '{ - apiVersion: "kustomize.config.k8s.io/v1beta1", - kind: "Kustomization", - resources: ["resources.yaml"] - } + .spec.postRenderers[$index].kustomize' \ - "${scratch_dir}/prod-release.json" >"${render_dir}/kustomization.yaml" - kubectl kustomize "${render_dir}" >"${scratch_dir}/next.yaml" - mv "${scratch_dir}/next.yaml" "${scratch_dir}/resources.yaml" -done - -yq ea -o=json '[.]' "${scratch_dir}/resources.yaml" | - jq -S 'sort_by([.apiVersion, .kind, .metadata.namespace, .metadata.name])' \ - >"${scratch_dir}/workload.json" - -jq -e '[.[] | select( - .kind == "PersistentVolumeClaim" and - .metadata.namespace == "crossview" and - .metadata.name == "crossview-postgres-pvc" -) | ( - .spec.storageClassName == "longhorn-wffc" and - .spec.accessModes == ["ReadWriteOnce"] and - .spec.resources.requests.storage == "2Gi" and - .metadata.annotations["helm.sh/resource-policy"] == "keep" -)] == [true]' "${scratch_dir}/workload.json" >/dev/null || - fail 'the pinned chart must render the exact retained Crossview PostgreSQL PVC contract' - -jq -e '[.[] | select( - .kind == "ConfigMap" and - .metadata.namespace == "crossview" and - .metadata.name == "crossview-postgres-coroot-monitor-init" -) | .data["init-coroot-monitor.sh"] | contains("durable-storage-v1")] == [true]' \ - "${scratch_dir}/prod.json" >/dev/null || - fail 'the fresh persistent database must create the durable-storage bootstrap gate' - -database_filter='select( - .kind == "Deployment" and - .metadata.namespace == "crossview" and - .metadata.name == "crossview-postgres" -)' - -jq -e "[.[] | ${database_filter}] | length == 1" \ - "${scratch_dir}/workload.json" >/dev/null || - fail 'the pinned chart must render exactly one Crossview PostgreSQL Deployment' - -jq -e "[.[] | ${database_filter} | - .spec.template.spec.priorityClassName == \"hcloud-volume-workload\" -] == [true]" "${scratch_dir}/workload.json" >/dev/null || - fail 'the existing durable Crossview database must roll onto the protected storage priority' - -jq -e "[.[] | ${database_filter} | ( - .spec.replicas == 1 and - .spec.strategy.type == \"RollingUpdate\" and - .spec.strategy.rollingUpdate.maxSurge == 0 and - .spec.strategy.rollingUpdate.maxUnavailable == 1 -)] == [true]" "${scratch_dir}/workload.json" >/dev/null || - fail 'the single-writer PostgreSQL rollout must terminate the old pod before creating its replacement' - -jq -e "[.[] | ${database_filter} | - .spec.template.spec.hostUsers == true -] == [true]" "${scratch_dir}/workload.json" >/dev/null || - fail 'only the PVC-backed PostgreSQL Deployment must explicitly retain host users' - -jq -e "[.[] | ${database_filter} | ( - ([.spec.template.spec.volumes[] | select( - .name == \"postgres-storage\" and - .persistentVolumeClaim.claimName == \"crossview-postgres-pvc\" - )] | length == 1) and - ([.spec.template.spec.volumes[] | select(.name == \"postgres-data\")] | length == 0) -)] == [true]" "${scratch_dir}/workload.json" >/dev/null || - fail 'the database pod must consume only the chart-owned persistent data volume' - -jq -e "[.[] | ${database_filter} | ( - ([.spec.template.spec.containers[] | select(.name == \"postgres\") | - .volumeMounts[] | select( - .name == \"postgres-storage\" and .mountPath == \"/var/lib/postgresql\" - )] | length == 1) and - ([.spec.template.spec.containers[] | select(.name == \"postgres\") | - .volumeMounts[] | select(.mountPath == \"/var/lib/postgresql\") - ] | length == 1) -)] == [true]" "${scratch_dir}/workload.json" >/dev/null || { - jq -c "[.[] | ${database_filter} | - .spec.template.spec.containers[] | select(.name == \"postgres\") | - .volumeMounts - ]" "${scratch_dir}/workload.json" >&2 - fail 'the database container must mount exactly one persistent data directory' -} - -jq -e '[.[] | select( - .kind == "Deployment" and - .metadata.namespace == "crossview" and - .metadata.name == "crossview" -) | ( - (.spec.template.spec.hostUsers // null) == null and - .spec.template.metadata.annotations["platform.devantler.tech/db-bootstrap"] == - "2026-09-15-durable-storage-v1" and - ([.spec.template.spec.initContainers[] | select(.name == "wait-for-db") | - .args[] | contains("durable-storage-v1") - ] | any) -)] == [true]' \ - "${scratch_dir}/workload.json" >/dev/null || - fail 'the Crossview app must restart only after the durable database bootstrap gate exists' - -printf 'PASS: Crossview PostgreSQL renders durable single-writer storage without changing the local provider\n' diff --git a/scripts/tests/test-guard-crossplane-provider-name-parity.sh b/scripts/tests/test-guard-crossplane-provider-name-parity.sh index a7571ce722..2e595c9f09 100755 --- a/scripts/tests/test-guard-crossplane-provider-name-parity.sh +++ b/scripts/tests/test-guard-crossplane-provider-name-parity.sh @@ -4,12 +4,12 @@ # # exit 0 every pkg.crossplane.io Provider carries the name Crossplane derives from # its package (or is explicitly dispositioned) AND its generated ClusterRole -# regex appears in both Kubescape secret-reader exception lists +# regex appears in the Kubescape secret-reader exception # exit 1 a name drifted from its derived name, or an exception entry is orphaned # exit 2 the guard could not check — missing root, unreadable/empty exception list, # or NO pkg.crossplane.io Provider found at all # -# Three of these are anti-vacuity or scoping cases rather than behaviours: +# Two of these are anti-vacuity or scoping cases rather than behaviours: # # * A tree with no Crossplane Provider must be exit 2, NOT a clean exit 0. A # selector that matched nothing looks exactly like a healthy tree, which is the @@ -18,8 +18,6 @@ # provider). It has no package and no generated ClusterRole, so matching on # `kind: Provider` alone would report a false failure on a correct tree. The # mixed fixture asserts the reported count excludes it. -# * An orphaned exception must be caught in EACH list independently — the two are -# synced by hand, so the realistic drift is one updated and the other forgotten. # # Every fixture carries its OWN provider name, so an assertion can only be satisfied # by the case it belongs to. @@ -80,17 +78,14 @@ assert_not_contains() { #