diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 93b0dcf762..4f5622a650 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -928,6 +928,8 @@ jobs: - 'scripts/tests/test-coroot-final-remediation.sh' - 'scripts/tests/test-cnpg-degraded-alert.sh' - 'scripts/tests/test-stranded-volume-attach-alert.sh' + - 'scripts/tests/test-crossplane-provider-unhealthy-alert.sh' + - 'scripts/tests/fixtures/crossplane-provider-list/**' - 'scripts/tests/test-prune-protected-orphan-alert.sh' - 'scripts/tests/test-coroot-alert-autosuppressor.sh' - 'scripts/tests/test-coroot-application-health-reconciler.sh' @@ -2222,6 +2224,15 @@ jobs: # verdict and both silent-zero guards are pinned by behaviour. run: bash scripts/tests/test-stranded-volume-attach-alert.sh + - name: ๐Ÿฉบ Validate the unhealthy Crossplane provider alert + if: needs.changes.outputs.k8s == 'true' + # Flux does not gate on Provider health, so this CronJob is the only + # signal for a Provider fleet that is not Healthy (#3454, #3457). Runs its + # real script against a stubbed API and delivery, so the grace period, + # both silent-zero guards and the credential handling are pinned by + # behaviour. + run: bash scripts/tests/test-crossplane-provider-unhealthy-alert.sh + - name: ๐Ÿงน Validate the prune-protected-orphan alert if: needs.changes.outputs.k8s == 'true' # The only signal that a retirement's manual delete step was missed โ€” diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md index 83ffeb0309..fcf43e55d6 100644 --- a/docs/dr/alerting.md +++ b/docs/dr/alerting.md @@ -105,6 +105,18 @@ stays quiet by design, exactly as the old Alertmanager did. remove this coverage. The archive branch was added after wedding-db remained 3/3 Ready while Barman rejected WAL from a replacement system ID for roughly three days. +- **An unhealthy Crossplane Provider fleet alerts on its own.** kstatus has no + readiness rule for a Crossplane `Provider`, so the `wait: true` gate above + does not cover one: during #3454 every Provider was `Healthy=False` with no + provider pod running while `infrastructure` stayed Ready. + `bases/infrastructure/controllers/coroot/cron-job-crossplane-provider-unhealthy-alert.yaml` + lists every `providers.pkg.crossplane.io` every 30 minutes and posts to the + same Slack webhook when a Provider's `Healthy` condition has not been `True` + (False, Unknown or missing) for more than 15 minutes, keyed on that + condition's `lastTransitionTime` (creation time when the condition is + missing). A provider upgrade is unhealthy for about a minute, inside the + grace. It is read-only (`get`/`list` on Providers only), exits cleanly on a + cluster without Crossplane, and is independent of any Flux health gate. - **Velero maintenance OOMKills remain visible after a successful retry.** `bases/components/coroot-velero-maintenance-oom-alert/cron-job-velero-maintenance-oom-alert.yaml` checks retained repository-maintenance pod status every 30 minutes. It alerts diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/gitops-managed-cronjobs.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/gitops-managed-cronjobs.yaml index 065b855415..8183e70e5b 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/gitops-managed-cronjobs.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/gitops-managed-cronjobs.yaml @@ -28,6 +28,8 @@ # allowlist (platform) # * observability/cronjob-failure-alert โ€” alerts on a reconcile CronJob failing on consecutive runs # or no longer finishing (platform) +# * observability/crossplane-provider-unhealthy-alert โ€” alerts on a Crossplane Provider not Healthy +# past its grace period (platform) # * observability/crossplane-sync-alerter โ€” alerts on a Crossplane managed resource that reports # Ready but has stopped syncing (platform) # * observability/posture-staleness-alert โ€” alerts when Kubescape's configuration-scan data @@ -67,7 +69,8 @@ spec: rotation, the Kubescape scan schedulers / stale host-data cleanup, Longhorn stale-node cleanup, the observability heartbeat / alerting jobs (CNPG degraded cluster, stranded volume attach, stranded autoscaler node, reconcile-CronJob - failure, Crossplane sync, posture-scan staleness, prune-protected orphans, + failure, Crossplane sync, unhealthy Crossplane provider, posture-scan + staleness, prune-protected orphans, Velero maintenance OOM) / Coroot reconcilers (alert autosuppression, cloud pricing, application health, risk acceptance), the OpenBao raft-snapshot backup, the Umami database-login rotation, and Umami tenant diff --git a/k8s/bases/infrastructure/controllers/coroot/cluster-role-binding-crossplane-provider-unhealthy-alert.yaml b/k8s/bases/infrastructure/controllers/coroot/cluster-role-binding-crossplane-provider-unhealthy-alert.yaml new file mode 100644 index 0000000000..d7fbd2969f --- /dev/null +++ b/k8s/bases/infrastructure/controllers/coroot/cluster-role-binding-crossplane-provider-unhealthy-alert.yaml @@ -0,0 +1,16 @@ +# Grants the crossplane-provider-unhealthy-alert ServiceAccount its read-only view +# of Crossplane Providers. See +# cluster-role-crossplane-provider-unhealthy-alert.yaml for the scope rationale. +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: crossplane-provider-unhealthy-alert +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: crossplane-provider-unhealthy-alert +subjects: + - kind: ServiceAccount + name: crossplane-provider-unhealthy-alert + namespace: observability diff --git a/k8s/bases/infrastructure/controllers/coroot/cluster-role-crossplane-provider-unhealthy-alert.yaml b/k8s/bases/infrastructure/controllers/coroot/cluster-role-crossplane-provider-unhealthy-alert.yaml new file mode 100644 index 0000000000..0828e7e99f --- /dev/null +++ b/k8s/bases/infrastructure/controllers/coroot/cluster-role-crossplane-provider-unhealthy-alert.yaml @@ -0,0 +1,18 @@ +# What the crossplane-provider-unhealthy-alert CronJob may read. +# +# Read-only, and ONLY the Crossplane Provider package. Providers are +# cluster-scoped, so a ClusterRole is required; it grants get/list on that single +# resource type and nothing else. +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: crossplane-provider-unhealthy-alert +rules: + - apiGroups: + - pkg.crossplane.io + resources: + - providers + verbs: + - get + - list diff --git a/k8s/bases/infrastructure/controllers/coroot/cron-job-crossplane-provider-unhealthy-alert.yaml b/k8s/bases/infrastructure/controllers/coroot/cron-job-crossplane-provider-unhealthy-alert.yaml new file mode 100644 index 0000000000..26fd2e3034 --- /dev/null +++ b/k8s/bases/infrastructure/controllers/coroot/cron-job-crossplane-provider-unhealthy-alert.yaml @@ -0,0 +1,229 @@ +# Alerts on a Crossplane `Provider` whose `Healthy` condition has not been +# `True` (False, Unknown or missing) for longer than a grace period. +# +# WHY THIS EXISTS AS A SEPARATE CHECK +# +# kstatus has no readiness rule for a Crossplane `Provider`, so the +# `wait: true` health gate described in docs/dr/alerting.md does not gate on +# one: an unhealthy Provider fleet leaves `infrastructure` Ready=True. During +# #3454 every Provider was Healthy=False, no provider pod ran, and Flux and every +# GitHub surface stayed green; Coroot inspections see no crashloop because there +# is no pod at all. This check reads Provider health directly through the API +# server and is independent of every Flux health gate (#3457). +# +# It follows cron-job-cnpg-degraded-alert.yaml exactly โ€” same hardening, webhook +# handling, silent-zero guards, delivery and repeat-every-tick behaviour. Read +# that file for the rationale behind each guard; it is not repeated here. +# +# GRACE PERIOD โ€” `Healthy`'s `lastTransitionTime` IS A REAL TRANSITION TIME +# +# The grace period keys on the Healthy condition's `lastTransitionTime`. +# crossplane-runtime keeps a condition's stored timestamp whenever type, status, +# reason and message are unchanged. Measured on this cluster, with the +# 2026-10-17 samples taken on Crossplane v2.4.2 (re-measure after a Crossplane +# minor upgrade, since this rule is what makes the grace reachable): +# * during #3454 (2026-08-29T15:09:53Z) all five Providers were Healthy=False +# with `lastTransitionTime` stable at 14:51:40โ€“14:51:42Z for ~18 minutes +# across an active reconcile loop; +# * a healthy fleet of four Providers, sampled on prod at +# 2026-10-17T17:31:40Z and 50s later, had byte-identical Healthy +# `lastTransitionTime` values (and an unchanged `resourceVersion`, so the +# healthy fleet is not rewriting its status); +# * the provider-upjet-unifi v1.1.2 upgrade on 2026-10-17 went Installed=True +# at 16:54:05Z and Healthy=True at 16:55:06Z โ€” about one minute not healthy +# while the new revision started. +# So the grace is reachable, and an ordinary upgrade stays well inside the 900s +# grace and does not page. (This is the property the CNPG header warns must be +# re-measured before keying a Crossplane condition on it.) +# +# A Provider with no Healthy condition at all uses its creationTimestamp as the +# conservative clock, so a package that never reports health still alerts once +# the grace has passed โ€” the same rule as the CNPG archiver branch. +# +# On a cluster without Crossplane the Provider API returns 404 and the check +# exits 0. Lives in the observability namespace to reuse the allow-coroot +# CiliumNetworkPolicy's kube-apiserver, hooks.slack.com:443 and DNS egress. +# Unset (local/CI) the webhook defaults to the reserved example.invalid host, +# which the script skips; every other delivery failure fails the Job. +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: crossplane-provider-unhealthy-alert + namespace: observability + annotations: + checkov.io/skip1: "CKV_K8S_38=the check container reads its SA token to list Crossplane Providers through the API server" + # `${GRACE_SECONDS}` below is the container's own env var, not a Flux + # post-build variable; substitution would blank it (see the CNPG CronJob). + kustomize.toolkit.fluxcd.io/substitute: disabled +spec: + schedule: "*/30 * * * *" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 60 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 180 + template: + metadata: + labels: + app: crossplane-provider-unhealthy-alert + spec: + restartPolicy: Never + serviceAccountName: crossplane-provider-unhealthy-alert + # This check calls the Kubernetes API, so it needs its token. C-0034 + # is already excepted for the observability namespace. + automountServiceAccountToken: true + securityContext: + runAsNonRoot: true + runAsUser: 65532 + # fsGroup makes the mode-0440 webhook file readable by this + # non-root container (see the CNPG CronJob). + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: check + image: docker.io/badouralix/curl-jq:latest@sha256:1e7c0284e24572ace7170df9fc91f15fd3b79ebf056d4dde17244d5d74bbfabc + imagePullPolicy: IfNotPresent + env: + - name: GRACE_SECONDS + value: "900" + command: + - /bin/sh + - -c + - | + set -eu + API=https://kubernetes.default.svc + SA=/var/run/secrets/kubernetes.io/serviceaccount + + # Read once from the mounted Secret, never from env + # (checkov CKV_K8S_35). + WEBHOOK_URL=$(cat /etc/crossplane-provider-unhealthy-alert/url) + + # The value becomes a curl config directive, where a newline + # would start a second directive: reject control characters. + if [ "$(printf '%s' "$WEBHOOK_URL" | tr -d '[:cntrl:]')" != "$WEBHOOK_URL" ]; then + echo "ERROR: webhook URL contains control characters" >&2 + exit 1 + fi + # The token travels in the URL path and egress is allowed by + # port, not scheme: deliver only to https://. + case "$WEBHOOK_URL" in + https://*) ;; + *) + echo "ERROR: webhook URL must use https://" >&2 + exit 1 + ;; + esac + + # A failed or malformed API read must fail the Job, never + # read as "every provider healthy". + code=$(curl -sS -o /tmp/body -w '%{http_code}' \ + --cacert "$SA/ca.crt" \ + -H "Authorization: Bearer $(cat "$SA/token")" \ + "$API/apis/pkg.crossplane.io/v1/providers") + if [ "$code" = "404" ]; then + echo "Crossplane Provider CRD not installed on this cluster; nothing to check." + exit 0 + fi + if [ "$code" != "200" ]; then + echo "ERROR: kube API returned HTTP $code listing Crossplane providers" >&2 + exit 1 + fi + # Require the items ARRAY, not merely the key: {"items":{}} + # would otherwise count zero providers as healthy. + if ! jq -e '(.items | type) == "array"' /tmp/body >/dev/null; then + echo "ERROR: unexpected response shape from kube API" >&2 + exit 1 + fi + + total=$(jq -r '.items | length' /tmp/body) + unhealthy=$(jq -r --argjson grace "${GRACE_SECONDS}" \ + --argjson now "$(date -u +%s)" ' + .items[] + | . as $p + | (($p.status.conditions // []) | map(select(.type == "Healthy")) | first) as $healthy + | ($healthy.status // "missing") as $status + | ($healthy.lastTransitionTime // $p.metadata.creationTimestamp) as $since + | (if $since == null + then false + else (($now - ($since | fromdate)) >= $grace) + end) as $pastGrace + | if ($status != "True" and $pastGrace) + then "โ€ข `\($p.metadata.name)` โ€” Healthy=\($status) since \($since)" + + (if (($healthy.message // "") | length) > 0 + then ": \($healthy.message | .[0:300])" + else "" + end) + else empty + end + ' /tmp/body) + + if [ -z "$unhealthy" ]; then + echo "Checked $total Crossplane provider(s); none unhealthy beyond ${GRACE_SECONDS}s." + exit 0 + fi + + echo "Unhealthy Crossplane provider(s) detected:" + echo "$unhealthy" + + text=$(printf '*Unhealthy Crossplane provider(s)*\n\n%s\n\n_A Provider has not been Healthy for longer than %ss. Its managed resources are not being reconciled; Flux does not gate on Provider health, so nothing else reports this. Inspect `kubectl get providers.pkg.crossplane.io` and the crossplane-system pods._' \ + "$unhealthy" "${GRACE_SECONDS}") + jq -n --arg t "$text" '{text: $t}' > /tmp/payload.json + + # Only the RFC 2606 placeholder host may skip delivery. + case "$WEBHOOK_URL" in + https://example.invalid/*) + echo "No Slack webhook configured; alert not delivered." + exit 0 + ;; + esac + + # No `|| true`: a failed POST must fail the Job. The URL + # reaches curl on stdin as an unquoted config value, never in + # argv (see the CNPG CronJob for both rationales). + printf 'url = %s\n' "$WEBHOOK_URL" | + curl -sf --max-time 15 \ + --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \ + -H 'Content-Type: application/json' \ + --data @/tmp/payload.json \ + --config - + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + # CIS-5.7.3 (C-0211); a documented no-op on this AppArmor cluster. + seLinuxOptions: {} + capabilities: + drop: + - ALL + volumeMounts: + # Writable scratch for the API response and the payload. + - name: tmp + mountPath: /tmp + - name: webhook + mountPath: /etc/crossplane-provider-unhealthy-alert + readOnly: true + resources: + requests: + cpu: 5m + memory: 32Mi + limits: + # CPU limit set for kubescape C-0270. + cpu: 100m + memory: 64Mi + volumes: + - name: tmp + emptyDir: {} + # 288 is 0440, written in decimal so YAML 1.1 and 1.2 readers agree + # (see the CNPG CronJob). + - name: webhook + secret: + secretName: crossplane-provider-unhealthy-alert-webhook + defaultMode: 288 diff --git a/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml b/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml index 75298d99c8..e57d2113cb 100644 --- a/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml +++ b/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml @@ -38,6 +38,13 @@ resources: - cluster-role-stranded-volume-attach-alert.yaml - cluster-role-binding-stranded-volume-attach-alert.yaml - cron-job-stranded-volume-attach-alert.yaml + # Unhealthy-Crossplane-Provider alert (#3457): Flux does not gate on Provider + # health, so a provider fleet outage otherwise reads green. 404 off-Crossplane. + - service-account-crossplane-provider-unhealthy-alert.yaml + - secret-crossplane-provider-unhealthy-alert.yaml + - cluster-role-crossplane-provider-unhealthy-alert.yaml + - cluster-role-binding-crossplane-provider-unhealthy-alert.yaml + - cron-job-crossplane-provider-unhealthy-alert.yaml # The prune-protected-orphan alert (#3503) deliberately lives in its own # folder, controllers/prune-protected-orphan-alert/, so retiring this one # cannot prune the check that would report what it leaves behind. diff --git a/k8s/bases/infrastructure/controllers/coroot/secret-crossplane-provider-unhealthy-alert.yaml b/k8s/bases/infrastructure/controllers/coroot/secret-crossplane-provider-unhealthy-alert.yaml new file mode 100644 index 0000000000..cc4e316489 --- /dev/null +++ b/k8s/bases/infrastructure/controllers/coroot/secret-crossplane-provider-unhealthy-alert.yaml @@ -0,0 +1,16 @@ +# Slack incoming-webhook for the crossplane-provider-unhealthy-alert CronJob. It +# is the same ${alertmanager_webhook_url} the cnpg-degraded-alert Secret carries, +# held in a Secret rather than inline env for the same reasons (see +# secret-cnpg-degraded-alert.yaml). +# +# The inline default keeps `ksail workload validate` (which has no SOPS access) +# building and leaves local/CI inert. +--- +apiVersion: v1 +kind: Secret +metadata: + name: crossplane-provider-unhealthy-alert-webhook + namespace: observability +type: Opaque +stringData: + url: ${alertmanager_webhook_url:=https://example.invalid/no-slack-configured} diff --git a/k8s/bases/infrastructure/controllers/coroot/service-account-crossplane-provider-unhealthy-alert.yaml b/k8s/bases/infrastructure/controllers/coroot/service-account-crossplane-provider-unhealthy-alert.yaml new file mode 100644 index 0000000000..fc6c82f979 --- /dev/null +++ b/k8s/bases/infrastructure/controllers/coroot/service-account-crossplane-provider-unhealthy-alert.yaml @@ -0,0 +1,10 @@ +# Identity for the crossplane-provider-unhealthy-alert CronJob, which lists the +# cluster-scoped Crossplane Providers. See +# cron-job-crossplane-provider-unhealthy-alert.yaml for why the check exists and +# cluster-role-crossplane-provider-unhealthy-alert.yaml for what it may read. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: crossplane-provider-unhealthy-alert + namespace: observability diff --git a/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml b/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml index b3f26d9df4..25f0b80f09 100644 --- a/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml +++ b/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml @@ -507,7 +507,7 @@ data: "controlID": "^C-0026$" } ], - "reason": "C-0026 lists every CronJob for manual approval and has no hardening remedy. All CronJobs on this cluster are GitOps-managed platform components delivered by Flux after PR review (the review is the approval): Cilium Hubble cert rotation, the Kubescape scan schedulers / stale host-data cleanup, Longhorn stale-node cleanup, the observability heartbeat / alerting jobs (CNPG degraded cluster, stranded volume attach, stranded autoscaler node, reconcile-CronJob failure, Crossplane sync, posture-scan staleness, prune-protected orphans, Velero maintenance OOM) / Coroot reconcilers (alert autosuppression, cloud pricing, application health, risk acceptance), the OpenBao raft-snapshot backup, the Umami database-login rotation, and Umami tenant provisioning. Namespace-scoped to the infrastructure namespaces that run these, so a CronJob in any unexpected namespace still surfaces for review." + "reason": "C-0026 lists every CronJob for manual approval and has no hardening remedy. All CronJobs on this cluster are GitOps-managed platform components delivered by Flux after PR review (the review is the approval): Cilium Hubble cert rotation, the Kubescape scan schedulers / stale host-data cleanup, Longhorn stale-node cleanup, the observability heartbeat / alerting jobs (CNPG degraded cluster, stranded volume attach, stranded autoscaler node, reconcile-CronJob failure, Crossplane sync, unhealthy Crossplane provider, posture-scan staleness, prune-protected orphans, Velero maintenance OOM) / Coroot reconcilers (alert autosuppression, cloud pricing, application health, risk acceptance), the OpenBao raft-snapshot backup, the Umami database-login rotation, and Umami tenant provisioning. Namespace-scoped to the infrastructure namespaces that run these, so a CronJob in any unexpected namespace still surfaces for review." }, { "name": "health-probes", diff --git a/scripts/tests/fixtures/crossplane-provider-list/provider-list-api-shape.json b/scripts/tests/fixtures/crossplane-provider-list/provider-list-api-shape.json new file mode 100644 index 0000000000..44df713918 --- /dev/null +++ b/scripts/tests/fixtures/crossplane-provider-list/provider-list-api-shape.json @@ -0,0 +1,198 @@ +{ + "apiVersion": "pkg.crossplane.io/v1", + "kind": "ProviderList", + "metadata": { + "continue": "", + "resourceVersion": "20000" + }, + "items": [ + { + "apiVersion": "pkg.crossplane.io/v1", + "kind": "Provider", + "metadata": { + "creationTimestamp": "2026-06-29T05:17:58Z", + "generation": 3, + "labels": { + "kustomize.toolkit.fluxcd.io/name": "infrastructure", + "kustomize.toolkit.fluxcd.io/namespace": "flux-system" + }, + "name": "provider-aws-iam", + "resourceVersion": "100001", + "uid": "00000000-0000-4000-8000-000000000001" + }, + "spec": { + "package": "xpkg.example.invalid/example/provider-aws-iam:v1.0.0", + "packagePullPolicy": "IfNotPresent", + "revisionActivationPolicy": "Automatic", + "revisionHistoryLimit": 1, + "runtimeConfigRef": { + "apiVersion": "pkg.crossplane.io/v1beta1", + "kind": "DeploymentRuntimeConfig", + "name": "default" + } + }, + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-16T18:02:03Z", + "observedGeneration": 3, + "reason": "HealthyPackageRevision", + "status": "True", + "type": "Healthy" + }, + { + "lastTransitionTime": "2026-10-16T18:02:03Z", + "observedGeneration": 3, + "reason": "ActivePackageRevision", + "status": "True", + "type": "Installed" + } + ], + "currentIdentifier": "xpkg.example.invalid/example/provider-aws-iam:v1.0.0", + "currentRevision": "provider-aws-iam-0123456789ab", + "resolvedPackage": "xpkg.example.invalid/example/provider-aws-iam:v1.0.0" + } + }, + { + "apiVersion": "pkg.crossplane.io/v1", + "kind": "Provider", + "metadata": { + "creationTimestamp": "2026-06-18T11:13:01Z", + "generation": 3, + "labels": { + "kustomize.toolkit.fluxcd.io/name": "infrastructure", + "kustomize.toolkit.fluxcd.io/namespace": "flux-system" + }, + "name": "provider-upjet-github", + "resourceVersion": "100002", + "uid": "00000000-0000-4000-8000-000000000002" + }, + "spec": { + "package": "xpkg.example.invalid/example/provider-upjet-github:v1.0.0", + "packagePullPolicy": "IfNotPresent", + "revisionActivationPolicy": "Automatic", + "revisionHistoryLimit": 1, + "runtimeConfigRef": { + "apiVersion": "pkg.crossplane.io/v1beta1", + "kind": "DeploymentRuntimeConfig", + "name": "default" + } + }, + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-13T05:29:11Z", + "observedGeneration": 3, + "reason": "HealthyPackageRevision", + "status": "True", + "type": "Healthy" + }, + { + "lastTransitionTime": "2026-10-13T05:29:11Z", + "observedGeneration": 3, + "reason": "ActivePackageRevision", + "status": "True", + "type": "Installed" + } + ], + "currentIdentifier": "xpkg.example.invalid/example/provider-upjet-github:v1.0.0", + "currentRevision": "provider-upjet-github-0123456789ab", + "resolvedPackage": "xpkg.example.invalid/example/provider-upjet-github:v1.0.0" + } + }, + { + "apiVersion": "pkg.crossplane.io/v1", + "kind": "Provider", + "metadata": { + "creationTimestamp": "2026-06-18T11:13:01Z", + "generation": 3, + "labels": { + "kustomize.toolkit.fluxcd.io/name": "infrastructure", + "kustomize.toolkit.fluxcd.io/namespace": "flux-system" + }, + "name": "provider-upjet-unifi", + "resourceVersion": "100003", + "uid": "00000000-0000-4000-8000-000000000003" + }, + "spec": { + "package": "xpkg.example.invalid/example/provider-upjet-unifi:v1.0.0", + "packagePullPolicy": "IfNotPresent", + "revisionActivationPolicy": "Automatic", + "revisionHistoryLimit": 1, + "runtimeConfigRef": { + "apiVersion": "pkg.crossplane.io/v1beta1", + "kind": "DeploymentRuntimeConfig", + "name": "default" + } + }, + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-17T16:55:06Z", + "observedGeneration": 3, + "reason": "HealthyPackageRevision", + "status": "True", + "type": "Healthy" + }, + { + "lastTransitionTime": "2026-10-17T16:55:06Z", + "observedGeneration": 3, + "reason": "ActivePackageRevision", + "status": "True", + "type": "Installed" + } + ], + "currentIdentifier": "xpkg.example.invalid/example/provider-upjet-unifi:v1.0.0", + "currentRevision": "provider-upjet-unifi-0123456789ab", + "resolvedPackage": "xpkg.example.invalid/example/provider-upjet-unifi:v1.0.0" + } + }, + { + "apiVersion": "pkg.crossplane.io/v1", + "kind": "Provider", + "metadata": { + "creationTimestamp": "2026-06-29T05:17:58Z", + "generation": 3, + "labels": { + "kustomize.toolkit.fluxcd.io/name": "infrastructure", + "kustomize.toolkit.fluxcd.io/namespace": "flux-system" + }, + "name": "upbound-provider-family-aws", + "resourceVersion": "100004", + "uid": "00000000-0000-4000-8000-000000000004" + }, + "spec": { + "package": "xpkg.example.invalid/example/upbound-provider-family-aws:v1.0.0", + "packagePullPolicy": "IfNotPresent", + "revisionActivationPolicy": "Automatic", + "revisionHistoryLimit": 1, + "runtimeConfigRef": { + "apiVersion": "pkg.crossplane.io/v1beta1", + "kind": "DeploymentRuntimeConfig", + "name": "default" + } + }, + "status": { + "conditions": [ + { + "lastTransitionTime": "2026-10-16T18:01:40Z", + "observedGeneration": 3, + "reason": "HealthyPackageRevision", + "status": "True", + "type": "Healthy" + }, + { + "lastTransitionTime": "2026-10-16T18:01:40Z", + "observedGeneration": 3, + "reason": "ActivePackageRevision", + "status": "True", + "type": "Installed" + } + ], + "currentIdentifier": "xpkg.example.invalid/example/upbound-provider-family-aws:v1.0.0", + "currentRevision": "upbound-provider-family-aws-0123456789ab", + "resolvedPackage": "xpkg.example.invalid/example/upbound-provider-family-aws:v1.0.0" + } + } + ] +} diff --git a/scripts/tests/test-crossplane-provider-unhealthy-alert.sh b/scripts/tests/test-crossplane-provider-unhealthy-alert.sh new file mode 100755 index 0000000000..6d23c6ff72 --- /dev/null +++ b/scripts/tests/test-crossplane-provider-unhealthy-alert.sh @@ -0,0 +1,385 @@ +#!/usr/bin/env bash + +# Contract for the crossplane-provider-unhealthy-alert CronJob (#3457). +# +# Flux does not gate on Crossplane Provider health, so during #3454 every +# Provider sat Healthy=False while everything else read green. This CronJob is +# the only signal for that shape. It follows the cnpg-degraded-alert pattern, +# and this test follows test-cnpg-degraded-alert.sh: the script is extracted +# from the manifest and run against a stubbed kube API and a stubbed curl. +# +# Asserted in both directions: +# FIRES on a Provider not Healthy (False, Unknown or missing) past the grace. +# STAYS QUIET on a healthy fleet and on a Provider unhealthy only inside the +# grace, which is what a provider upgrade looks like. +# FAILS LOUDLY on a non-200 or malformed API read and on a failed delivery, +# instead of recording a silent zero. + +set -euo pipefail + +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +readonly root_dir +readonly coroot_dir="${root_dir}/k8s/bases/infrastructure/controllers/coroot" +readonly manifest="${coroot_dir}/cron-job-crossplane-provider-unhealthy-alert.yaml" +readonly cluster_role="${coroot_dir}/cluster-role-crossplane-provider-unhealthy-alert.yaml" +readonly ci_workflow="${root_dir}/.github/workflows/ci.yaml" +readonly mount_path='/etc/crossplane-provider-unhealthy-alert' + +# Not a hooks.slack.com-shaped literal (GitHub push protection blocks those); +# `.invalid` is RFC 2606 reserved, so it can never resolve. +readonly REAL_WEBHOOK='https://hooks.test.invalid/delivery-target' +readonly PLACEHOLDER_WEBHOOK='https://example.invalid/no-slack-configured' + +fail() { + printf 'FAIL: %s\n' "$1" >&2 + exit 1 +} + +pass() { + printf 'ok โ€” %s\n' "$1" +} + +for tool in yq jq; do + command -v "${tool}" >/dev/null 2>&1 || { + printf '::error::%s is required to run this contract test.\n' "${tool}" >&2 + exit 64 + } +done +[ -f "${manifest}" ] || fail "manifest not found: ${manifest}" + +# --------------------------------------------------------------------------- +# Structural assertions. +# --------------------------------------------------------------------------- + +container_path='.spec.jobTemplate.spec.template.spec.containers[0]' +pod_path='.spec.jobTemplate.spec.template.spec' + +script_body="$(yq eval "${container_path}.command[2]" "${manifest}")" +[ -n "${script_body}" ] && [ "${script_body}" != "null" ] || + fail "could not extract the container script from ${manifest}" + +if yq eval "${container_path}.env" "${manifest}" | grep -q 'secretKeyRef'; then + fail "CKV_K8S_35: the webhook must arrive as a mounted file, not a secretKeyRef" +fi +[ "$(yq eval "${container_path}.volumeMounts[] | select(.name == \"webhook\") | .mountPath" "${manifest}")" = "${mount_path}" ] || + fail "the webhook volume is not mounted at ${mount_path}" +[ "$(yq eval "${container_path}.volumeMounts[] | select(.name == \"webhook\") | .readOnly" "${manifest}")" = "true" ] || + fail "the webhook mount must be readOnly" +[ "$(yq eval "${pod_path}.volumes[] | select(.name == \"webhook\") | .secret.secretName" "${manifest}")" = "crossplane-provider-unhealthy-alert-webhook" ] || + fail "the webhook volume does not name the crossplane-provider-unhealthy-alert-webhook Secret" +# Mode and fsGroup are one invariant: kubelet owns the file root:fsGroup. +[ "$(yq eval "${pod_path}.volumes[] | select(.name == \"webhook\") | .secret.defaultMode" "${manifest}")" = "288" ] || + fail "expected the webhook volume defaultMode 0440 (288 decimal)" +[ "$(yq eval "${pod_path}.securityContext.fsGroup" "${manifest}")" = "$(yq eval "${pod_path}.securityContext.runAsUser" "${manifest}")" ] || + fail "fsGroup must match runAsUser, or the mounted webhook is unreadable" +pass "webhook arrives as a readOnly 0440 file readable by the pod's fsGroup" + +[ "$(yq eval '.metadata.annotations."kustomize.toolkit.fluxcd.io/substitute"' "${manifest}")" = "disabled" ] || + fail "Flux substitution must be disabled, or \${GRACE_SECONDS} is blanked on deploy" +pass "Flux substitution is disabled for the CronJob" + +# Line-anchored: the curl call spans continuation lines (see the CNPG test). +if grep -Eq '^[[:space:]]*"\$\{?WEBHOOK_URL\}?"[[:space:]]*$' <<<"${script_body}" || + grep -Eq 'curl.*"\$\{?WEBHOOK_URL\}?"' <<<"${script_body}"; then + fail "the webhook URL is passed to curl as an argument" +fi +grep -Fq -- '--config -' <<<"${script_body}" || + fail "the delivery does not read its URL from a stdin config" +pass "webhook URL is never passed to curl in argv" + +# Least privilege: exactly get/list on providers.pkg.crossplane.io. +[ "$(yq eval -o=json -I=0 '.rules' "${cluster_role}")" = '[{"apiGroups":["pkg.crossplane.io"],"resources":["providers"],"verbs":["get","list"]}]' ] || + fail "the ClusterRole must grant only get/list on providers.pkg.crossplane.io" +pass "the ClusterRole is read-only on Providers alone" + +# --------------------------------------------------------------------------- +# Behavioural scenarios. +# --------------------------------------------------------------------------- + +# Under /tmp so the /tmp/ rewrite below reproduces the Linux CI condition. +work_root="$(mktemp -d /tmp/tmp.XXXXXXXXXX)" +trap 'rm -rf "${work_root}"' EXIT + +# The stub tells the API read (-o ) from the delivery, as the CNPG test does. +setup_scenario() { + local name="$1" api_code="$2" api_body="$3" webhook="$4" + local dir="${work_root}/${name}" + + mkdir -p "${dir}/bin" "${dir}/sa" "${dir}/webhook" "${dir}/tmp" + printf 'fake-ca' >"${dir}/sa/ca.crt" + printf 'fake-token' >"${dir}/sa/token" + printf '%s' "${webhook}" >"${dir}/webhook/url" + printf '%s' "${api_body}" >"${dir}/api-body.json" + printf '%s' "${api_code}" >"${dir}/api-code" + + cat >"${dir}/bin/curl" <<'STUB' +#!/usr/bin/env bash +set -uo pipefail +dir="${SCENARIO_DIR}" +printf '%s\n' "$*" >>"${dir}/curl-argv.log" +out_file="" +prev="" +for arg in "$@"; do + [ "${prev}" = "-o" ] && out_file="${arg}" + prev="${arg}" +done +if [ -n "${out_file}" ]; then + printf '%s\n' "$*" >"${dir}/api-request.log" + cat "${dir}/api-body.json" >"${out_file}" + printf '%s' "$(cat "${dir}/api-code")" + exit 0 +fi +cat >>"${dir}/curl-stdin.log" +for arg in "$@"; do + case "${arg}" in + @*) cp "${arg#@}" "${dir}/delivered-payload.json" 2>/dev/null || true ;; + esac +done +printf 'delivered\n' >>"${dir}/deliveries.log" +exit "${STUB_DELIVERY_EXIT:-0}" +STUB + chmod +x "${dir}/bin/curl" + printf '%s' "${dir}" +} + +# Redirect only the in-cluster paths; the `/tmp/` rewrite must run first (see +# the CNPG test for the doubled-path bug that ordering prevents). +run_scenario() { + local dir="$1" + local patched="${dir}/script.sh" + + { + printf '%s\n' 'set -eu' + printf 'GRACE_SECONDS=%s\n' 900 + printf 'SA_OVERRIDE=%q\n' "${dir}/sa" + # shellcheck disable=SC2016 # `$SA_OVERRIDE` is emitted INTO the patched script. + printf '%s\n' "${script_body}" | + sed -e 's#^\( *\)SA=/var/run/secrets/kubernetes.io/serviceaccount$#\1SA="$SA_OVERRIDE"#' \ + -e 's#/tmp/#'"${dir}"'/tmp/#g' \ + -e 's#'"${mount_path}"'/url#'"${dir}"'/webhook/url#g' + } >"${patched}" + + # shellcheck disable=SC2016 # matching the LITERAL `$SA_OVERRIDE`. + grep -q 'SA="\$SA_OVERRIDE"' "${patched}" || + fail "serviceaccount path redirection did not apply" + grep -q "${dir}/webhook/url" "${patched}" || + fail "webhook path redirection did not apply" + ! grep -q -- "${dir}.*${dir}" "${patched}" || + fail "a redirection rewrote another rule's output" + + ( + cd "${dir}" + PATH="${dir}/bin:${PATH}" SCENARIO_DIR="${dir}" bash "${patched}" >"${dir}/stdout.log" 2>"${dir}/stderr.log" + ) +} + +deliveries() { + local dir="$1" + [ -f "${dir}/deliveries.log" ] && wc -l <"${dir}/deliveries.log" | tr -d ' ' || printf '0' +} + +expect_alert() { + local name="$1" body="$2" pattern="$3" + local dir + dir="$(setup_scenario "${name}" 200 "${body}" "${REAL_WEBHOOK}")" + run_scenario "${dir}" || fail "scenario '${name}': script exited non-zero: $(cat "${dir}/stderr.log")" + [ "$(deliveries "${dir}")" = "1" ] || + fail "scenario '${name}': expected exactly 1 delivery, got $(deliveries "${dir}")" + jq -e --arg p "${pattern}" '.text | test($p)' "${dir}/delivered-payload.json" >/dev/null || + fail "scenario '${name}': payload does not match /${pattern}/" + printf '%s' "${dir}" +} + +expect_quiet() { + local name="$1" body="$2" + local dir + dir="$(setup_scenario "${name}" 200 "${body}" "${REAL_WEBHOOK}")" + run_scenario "${dir}" || fail "scenario '${name}': script exited non-zero: $(cat "${dir}/stderr.log")" + [ "$(deliveries "${dir}")" = "0" ] || fail "scenario '${name}': expected no delivery" + printf '%s' "${dir}" +} + +expect_failure() { + local name="$1" code="$2" body="$3" webhook="$4" stderr_pattern="$5" + local dir + dir="$(setup_scenario "${name}" "${code}" "${body}" "${webhook}")" + if run_scenario "${dir}"; then + fail "scenario '${name}': expected the Job to fail" + fi + [ "$(deliveries "${dir}")" = "0" ] || fail "scenario '${name}': delivered despite failing" + grep -q "${stderr_pattern}" "${dir}/stderr.log" || + fail "scenario '${name}': stderr does not report '${stderr_pattern}'" +} + +old_stamp="$(date -u -r 0 +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d @0 +%Y-%m-%dT%H:%M:%SZ)" +now_stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +readonly old_stamp now_stamp +readonly incident_message='cannot initialize dependency graph from the packages in the lock: node xpkg.upbound.io/upbound/provider-family-aws already exists' + +# provider +# An empty status omits the Healthy condition entirely. +provider() { + jq -n --arg n "$1" --arg s "$2" --arg t "$3" --arg c "$4" --arg m "${incident_message}" '{ + apiVersion: "pkg.crossplane.io/v1", + kind: "Provider", + metadata: { name: $n, creationTimestamp: $c }, + spec: { package: ("xpkg.upbound.io/upbound/" + $n + ":v2.0.0") }, + status: { + conditions: ( + [{ type: "Installed", status: "True", reason: "ActivePackageRevision", lastTransitionTime: $c }] + + (if $s == "" then [] + else [{ type: "Healthy", status: $s, + reason: (if $s == "True" then "HealthyPackageRevision" else "UnhealthyPackageRevision" end), + lastTransitionTime: $t } + + (if $s == "True" then {} else { message: $m } end)] + end) + ) + } + }' +} + +fleet() { + jq -s '{ apiVersion: "pkg.crossplane.io/v1", kind: "ProviderList", items: . }' +} + +healthy_fleet="$( + { + provider provider-family-aws True "${old_stamp}" "${old_stamp}" + provider provider-aws-iam True "${old_stamp}" "${old_stamp}" + } | fleet +)" +incident_fleet="$( + { + provider provider-family-aws False "${old_stamp}" "${old_stamp}" + provider provider-aws-iam False "${old_stamp}" "${old_stamp}" + } | fleet +)" +readonly healthy_fleet incident_fleet + +# 1 โ€” the #3454 shape: every Provider Healthy=False past the grace. One delivery +# naming each Provider and its message; the URL travels on stdin, not argv. +dir="$(expect_alert incident "${incident_fleet}" 'provider-family-aws. โ€” Healthy=False since')" +jq -e '.text | test("provider-aws-iam") and test("already exists")' "${dir}/delivered-payload.json" >/dev/null || + fail "scenario 'incident': payload does not name every unhealthy Provider with its message" +grep -Fq "/apis/pkg.crossplane.io/v1/providers" "${dir}/api-request.log" || + fail "scenario 'incident': the script did not list providers.pkg.crossplane.io/v1" +grep -Fq "${REAL_WEBHOOK}" "${dir}/curl-stdin.log" || + fail "scenario 'incident': the webhook URL never reached curl's stdin config" +if grep -Fq "${REAL_WEBHOOK}" "${dir}/curl-argv.log"; then + fail "scenario 'incident': the webhook URL appeared in curl's argv" +fi +pass "an unhealthy-past-grace fleet delivers one alert naming every Provider" + +# 2 โ€” Unknown is not healthy. +expect_alert unknown "$(provider provider-upjet-unifi Unknown "${old_stamp}" "${old_stamp}" | fleet)" \ + 'provider-upjet-unifi. โ€” Healthy=Unknown' >/dev/null +pass "Healthy=Unknown past the grace alerts" + +# 3 โ€” a Provider that never reports Healthy alerts once its creation is past the +# grace, and stays quiet while it is new. +expect_alert missing_old "$(provider provider-upjet-unifi "" "" "${old_stamp}" | fleet)" \ + 'provider-upjet-unifi. โ€” Healthy=missing since' >/dev/null +expect_quiet missing_new "$(provider provider-upjet-unifi "" "" "${now_stamp}" | fleet)" >/dev/null +pass "a missing Healthy condition is timed from creationTimestamp" + +# 4 โ€” GUARDRAIL: mid-upgrade. The new revision is Installed but not yet Healthy, +# and the transition is recent (about a minute on the 2026-10-17 upgrade). +dir="$(expect_quiet mid_upgrade "$( + { + provider provider-family-aws True "${old_stamp}" "${old_stamp}" + provider provider-upjet-unifi False "${now_stamp}" "${old_stamp}" + } | fleet +)")" +grep -q 'none unhealthy beyond 900s' "${dir}/stdout.log" || + fail "scenario 'mid_upgrade': the quiet verdict was not reported" +pass "a Provider unhealthy only inside the grace (an upgrade) does not page" + +# 5 โ€” a healthy fleet is quiet and reports what it checked. +dir="$(expect_quiet healthy "${healthy_fleet}")" +grep -q 'Checked 2 Crossplane provider' "${dir}/stdout.log" || + fail "scenario 'healthy': the checked count was not reported" +pass "a healthy fleet is quiet and reports what it checked" + +# 6 โ€” the reserved placeholder host stays inert. +dir="$(setup_scenario placeholder 200 "${incident_fleet}" "${PLACEHOLDER_WEBHOOK}")" +run_scenario "${dir}" || fail "scenario 'placeholder': script exited non-zero" +[ "$(deliveries "${dir}")" = "0" ] || fail "scenario 'placeholder': delivered to example.invalid" +grep -q 'No Slack webhook configured' "${dir}/stdout.log" || + fail "scenario 'placeholder': the skip was not reported" +pass "the example.invalid placeholder stays inert" + +# 7 โ€” a cluster without Crossplane is a supported state. +dir="$(setup_scenario no_crd 404 '{}' "${REAL_WEBHOOK}")" +run_scenario "${dir}" || fail "scenario 'no_crd': a 404 should exit 0" +[ "$(deliveries "${dir}")" = "0" ] || fail "scenario 'no_crd': alerted with no Provider CRD" +pass "a cluster without the Provider CRD exits cleanly" + +# 8 โ€” SILENT-ZERO guards: each must fail the Job, never read as healthy. +expect_failure api_error 500 '{}' "${REAL_WEBHOOK}" 'kube API returned HTTP 500' +pass "a failing kube API read fails the Job" +expect_failure bad_shape 200 '{"unexpected":true}' "${REAL_WEBHOOK}" 'unexpected response shape' +expect_failure items_not_array 200 '{"items":{}}' "${REAL_WEBHOOK}" 'unexpected response shape' +pass "a malformed API response, including a non-array items, fails the Job" + +# 9 โ€” webhook guards: a control character or a non-https URL fails before delivery. +expect_failure url_control_chars 200 '{"items":[]}' "${REAL_WEBHOOK} +output = ${work_root}/injected-output" 'control characters' +[ ! -e "${work_root}/injected-output" ] || fail "scenario 'url_control_chars': injected directive took effect" +expect_failure plain_http 200 "${incident_fleet}" 'http://hooks.test.invalid:443/delivery-target' 'must use https://' +pass "a control character or non-https webhook URL fails before any delivery" + +# 10 โ€” a failed delivery fails the Job. +dir="$(setup_scenario delivery_fails 200 "${incident_fleet}" "${REAL_WEBHOOK}")" +if STUB_DELIVERY_EXIT=22 run_scenario "${dir}"; then + fail "scenario 'delivery_fails': a failed webhook POST was reported as success" +fi +pass "a failed webhook delivery fails the Job" + +# 11 โ€” the full API response shape. provider()/fleet() build only the fields the +# script reads, so they cannot show that the extra fields a real response carries +# leave the verdict alone. This fixture has exactly the structure of a +# `GET /apis/pkg.crossplane.io/v1/providers` body read from prod on 2026-10-17 +# (Crossplane v2.4.2, metadata.managedFields aside): every field path that +# response carried, including the list metadata and each Provider's spec and +# status extras. The values are invented; production data is not committed. +readonly shape_fleet_file="${root_dir}/scripts/tests/fixtures/crossplane-provider-list/provider-list-api-shape.json" +[ -f "${shape_fleet_file}" ] || fail "ProviderList shape fixture not found: ${shape_fleet_file}" +shape_fleet="$(cat "${shape_fleet_file}")" +jq -e '.kind == "ProviderList" and (.items | length) == 4 + and all(.items[]; any(.status.conditions[]; .type == "Healthy" and .status == "True"))' \ + <<<"${shape_fleet}" >/dev/null || + fail "the shape fixture is not the four-Provider healthy ProviderList this scenario asserts on" +dir="$(expect_quiet shape_healthy "${shape_fleet}")" +grep -q 'Checked 4 Crossplane provider' "${dir}/stdout.log" || + fail "scenario 'shape_healthy': the script did not count the four Providers" +grep -q 'none unhealthy beyond 900s' "${dir}/stdout.log" || + fail "scenario 'shape_healthy': the quiet verdict was not reported" +pass "a full-shape ProviderList reads as four healthy Providers" + +# The same objects with one Healthy condition flipped, keeping its +# lastTransitionTime (already past the grace) and every other field. +dir="$(expect_alert shape_one_unhealthy "$( + jq --arg m "${incident_message}" '(.items[] | select(.metadata.name == "upbound-provider-family-aws") + | .status.conditions[] | select(.type == "Healthy")) + |= (.status = "False" | .reason = "UnhealthyPackageRevision" | .message = $m)' <<<"${shape_fleet}" +)" 'upbound-provider-family-aws. โ€” Healthy=False since')" +if jq -e '.text | test("provider-aws-iam|provider-upjet-github|provider-upjet-unifi")' "${dir}/delivered-payload.json" >/dev/null; then + fail "scenario 'shape_one_unhealthy': a healthy Provider was named in the alert" +fi +pass "one unhealthy Provider in the full shape alerts and names only that Provider" + +# --------------------------------------------------------------------------- +# Wiring. +# --------------------------------------------------------------------------- + +grep -Fq "'scripts/tests/test-crossplane-provider-unhealthy-alert.sh'" "${ci_workflow}" || + fail "this test is not in the CI paths filter" +grep -Fq "bash scripts/tests/test-crossplane-provider-unhealthy-alert.sh" "${ci_workflow}" || + fail "this test is never executed by a CI step" +for f in service-account secret cluster-role cluster-role-binding cron-job; do + grep -Fq -- "- ${f}-crossplane-provider-unhealthy-alert.yaml" "${coroot_dir}/kustomization.yaml" || + fail "${f}-crossplane-provider-unhealthy-alert.yaml is not in the coroot kustomization" +done +pass "the contract is wired into CI and every resource is deployed" + +printf '\nAll crossplane-provider-unhealthy-alert contract assertions passed.\n' diff --git a/scripts/validate-eks-ci-role-policy/approved-surface-history.md b/scripts/validate-eks-ci-role-policy/approved-surface-history.md index 025b05ace6..17b230abfc 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface-history.md +++ b/scripts/validate-eks-ci-role-policy/approved-surface-history.md @@ -3576,3 +3576,19 @@ two changed entries and one added entry against `409c6cdc`: The generator records the real Ready/SourceVerified product observation and retires its publication bootstrap into an immutable binding. No observation is synthesized. Existing tenant RBAC and admission limits are unchanged. + +## 2026-10-17 โ€” Alert on an unhealthy Crossplane provider fleet + +For #3457, the validator measures two added entries against `f4ed0a8e`, and +nothing changed or removed: + +- ClusterRole crossplane-provider-unhealthy-alert is added as + `6a8845e6cd798bea6c9dd0353f5ac34dc93e3f2ccdda4e4d92288705e0dc6a45`. +- ClusterRoleBinding crossplane-provider-unhealthy-alert is added as + `1cc87f7bfd2b06426b26df11488ceb00e9a22e749bfca05814c1c9a0edc7be9a`. + +The ClusterRole grants read-only `get` and `list` on +`providers.pkg.crossplane.io` and nothing else. The binding grants it to one +subject, the `observability/crossplane-provider-unhealthy-alert` +ServiceAccount, which the new alert CronJob runs as. No other RBAC, admission +scope, AWS identity, credential or permission grant changes. diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index dd5dcb457a..199d4a524f 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -154,6 +154,8 @@ rbac.authorization.k8s.io/v1|ClusterRoleBinding||ascoachingogvaner-external-dns rbac.authorization.k8s.io/v1|ClusterRoleBinding||cnpg-degraded-alert 51487655b693fe18c5d7196d0042e4bc8345a15b0ea3afe03216e69d37905940 +rbac.authorization.k8s.io/v1|ClusterRoleBinding||crossplane-provider-unhealthy-alert 1cc87f7bfd2b06426b26df11488ceb00e9a22e749bfca05814c1c9a0edc7be9a + rbac.authorization.k8s.io/v1|ClusterRoleBinding||crossplane-sync-alerter 2ff039b4399f755bb15112771e2586ac05d68b113313824972720f4a6e156147 rbac.authorization.k8s.io/v1|ClusterRoleBinding||crossplane-sync-exporter 89816b5cc430d16ae2fef6695022844c8658573eb5af90178ec8195c9601f1b1 @@ -200,6 +202,8 @@ rbac.authorization.k8s.io/v1|ClusterRole||cnpg-degraded-alert 42c958c00b28103b38 rbac.authorization.k8s.io/v1|ClusterRole||cnpg-tenant-edit 942ba69a066888eb7a82444a29f3917aa8c836c0bb800cee159e4e3bda4e3974 +rbac.authorization.k8s.io/v1|ClusterRole||crossplane-provider-unhealthy-alert 6a8845e6cd798bea6c9dd0353f5ac34dc93e3f2ccdda4e4d92288705e0dc6a45 + rbac.authorization.k8s.io/v1|ClusterRole||crossplane-sync-alerter faf47ab2d737acd58c26bbf7aa3b46f23e5b22fc52aed6acd0fcf7f434fc431f rbac.authorization.k8s.io/v1|ClusterRole||crossplane-sync-exporter 603cec5b91944155ef37677bc46e02dc418ed5e71ad7c3e843ea9b794766fa36