From b6ded35cdf220db380718dedfa1804bbd2777058 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Sat, 18 Jul 2026 11:50:18 +0200
Subject: [PATCH 1/3] feat(observability): stage Coroot-backed OpenCost handoff

---
 .../opencost/kustomization.yaml               | 22 ++++++
 .../patches/use-coroot-network-policy.yaml    |  6 ++
 .../patches/use-coroot-prometheus.yaml        | 21 ++++++
 .../kustomization.yaml                        | 69 +++++++++++++++++++
 .../docker/controllers/kustomization.yaml     |  3 +
 .../docker/infrastructure/kustomization.yaml  |  2 +
 .../hetzner/controllers/kustomization.yaml    |  3 +
 .../hetzner/infrastructure/kustomization.yaml |  2 +
 scripts/validate-observability-option.sh      | 45 ++++++++++++
 9 files changed, 173 insertions(+)
 create mode 100644 k8s/bases/infrastructure/opencost/kustomization.yaml
 create mode 100644 k8s/bases/infrastructure/opencost/patches/use-coroot-network-policy.yaml
 create mode 100644 k8s/bases/infrastructure/opencost/patches/use-coroot-prometheus.yaml
 create mode 100644 k8s/testdata/observability-option/components/remove-controller-opencost/kustomization.yaml

diff --git a/k8s/bases/infrastructure/opencost/kustomization.yaml b/k8s/bases/infrastructure/opencost/kustomization.yaml
new file mode 100644
index 0000000..f631d14
--- /dev/null
+++ b/k8s/bases/infrastructure/opencost/kustomization.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Coroot-backed OpenCost must be applied in the infrastructure layer beside
+# the Coroot custom resource. The operator creates Coroot's Prometheus only
+# after that CR exists, so keeping this HelmRelease in the earlier,
+# wait-gated controllers layer would deadlock the rollout.
+#
+# This composition is staged and default-off. Provider activation must remove
+# the legacy controller-layer HelmRelease and add this base atomically so Flux
+# never has two Kustomizations claiming HelmRelease/opencost/opencost.
+resources:
+  - ../controllers/opencost/
+patches:
+  - path: patches/use-coroot-prometheus.yaml
+  - target:
+      group: cilium.io
+      version: v2
+      kind: CiliumNetworkPolicy
+      name: allow-opencost
+      namespace: opencost
+    path: patches/use-coroot-network-policy.yaml
diff --git a/k8s/bases/infrastructure/opencost/patches/use-coroot-network-policy.yaml b/k8s/bases/infrastructure/opencost/patches/use-coroot-network-policy.yaml
new file mode 100644
index 0000000..b1fa382
--- /dev/null
+++ b/k8s/bases/infrastructure/opencost/patches/use-coroot-network-policy.yaml
@@ -0,0 +1,6 @@
+---
+- op: remove
+  path: /spec/ingress/0
+- op: replace
+  path: /spec/egress/1/toEndpoints/0/matchLabels/k8s:io.kubernetes.pod.namespace
+  value: observability
diff --git a/k8s/bases/infrastructure/opencost/patches/use-coroot-prometheus.yaml b/k8s/bases/infrastructure/opencost/patches/use-coroot-prometheus.yaml
new file mode 100644
index 0000000..6eece37
--- /dev/null
+++ b/k8s/bases/infrastructure/opencost/patches/use-coroot-prometheus.yaml
@@ -0,0 +1,21 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: opencost
+  namespace: opencost
+spec:
+  dependsOn:
+    - name: coroot-operator
+      namespace: observability
+  values:
+    opencost:
+      prometheus:
+        external:
+          url: "http://coroot-prometheus.observability.svc.cluster.local:9090"
+      # kube-prometheus-stack leaves during the Coroot cutover, including the
+      # Prometheus Operator CRDs. Coroot still scrapes OpenCost's metrics via
+      # its node agent, so no ServiceMonitor is needed.
+      metrics:
+        serviceMonitor:
+          enabled: false
diff --git a/k8s/testdata/observability-option/components/remove-controller-opencost/kustomization.yaml b/k8s/testdata/observability-option/components/remove-controller-opencost/kustomization.yaml
new file mode 100644
index 0000000..1f17468
--- /dev/null
+++ b/k8s/testdata/observability-option/components/remove-controller-opencost/kustomization.yaml
@@ -0,0 +1,69 @@
+---
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+# Model the future atomic Flux ownership transfer: the earlier controllers
+# layer gives up every OpenCost object before the later infrastructure layer
+# claims the same identities with the Coroot-backed composition.
+patches:
+  - target:
+      version: v1
+      kind: Namespace
+      name: opencost
+    patch: |
+      $patch: delete
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        name: opencost
+  - target:
+      group: source.toolkit.fluxcd.io
+      version: v1
+      kind: HelmRepository
+      name: opencost
+      namespace: opencost
+    patch: |
+      $patch: delete
+      apiVersion: source.toolkit.fluxcd.io/v1
+      kind: HelmRepository
+      metadata:
+        name: opencost
+        namespace: opencost
+  - target:
+      group: helm.toolkit.fluxcd.io
+      version: v2
+      kind: HelmRelease
+      name: opencost
+      namespace: opencost
+    patch: |
+      $patch: delete
+      apiVersion: helm.toolkit.fluxcd.io/v2
+      kind: HelmRelease
+      metadata:
+        name: opencost
+        namespace: opencost
+  - target:
+      group: gateway.networking.k8s.io
+      version: v1
+      kind: HTTPRoute
+      name: opencost
+      namespace: opencost
+    patch: |
+      $patch: delete
+      apiVersion: gateway.networking.k8s.io/v1
+      kind: HTTPRoute
+      metadata:
+        name: opencost
+        namespace: opencost
+  - target:
+      group: cilium.io
+      version: v2
+      kind: CiliumNetworkPolicy
+      name: allow-opencost
+      namespace: opencost
+    patch: |
+      $patch: delete
+      apiVersion: cilium.io/v2
+      kind: CiliumNetworkPolicy
+      metadata:
+        name: allow-opencost
+        namespace: opencost
diff --git a/k8s/testdata/observability-option/docker/controllers/kustomization.yaml b/k8s/testdata/observability-option/docker/controllers/kustomization.yaml
index 44e1e67..6dfc6c9 100644
--- a/k8s/testdata/observability-option/docker/controllers/kustomization.yaml
+++ b/k8s/testdata/observability-option/docker/controllers/kustomization.yaml
@@ -1,5 +1,8 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../providers/docker/infrastructure/controllers/
   - ../../../../bases/infrastructure/controllers/coroot/
+components:
+  - ../../components/remove-controller-opencost/
diff --git a/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml b/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml
index c1f24be..a4f54f9 100644
--- a/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml
+++ b/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml
@@ -1,5 +1,7 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../providers/docker/infrastructure/
   - ../../../../bases/infrastructure/coroot/
+  - ../../../../bases/infrastructure/opencost/
diff --git a/k8s/testdata/observability-option/hetzner/controllers/kustomization.yaml b/k8s/testdata/observability-option/hetzner/controllers/kustomization.yaml
index ffcb6aa..127edae 100644
--- a/k8s/testdata/observability-option/hetzner/controllers/kustomization.yaml
+++ b/k8s/testdata/observability-option/hetzner/controllers/kustomization.yaml
@@ -1,5 +1,8 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../providers/hetzner/infrastructure/controllers/
   - ../../../../bases/infrastructure/controllers/coroot/
+components:
+  - ../../components/remove-controller-opencost/
diff --git a/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml b/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml
index 51ccc71..e3c80c8 100644
--- a/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml
+++ b/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml
@@ -1,6 +1,8 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../providers/hetzner/infrastructure/
   - ../../../../bases/infrastructure/coroot/
+  - ../../../../bases/infrastructure/opencost/
   - ../../../../bases/infrastructure/audit-log-forwarder/
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 2681eeb..87b6ab4 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -79,6 +79,16 @@ for rendered_path in \
   assert_default_off "${rendered_path}"
 done
 
+# The default provider payloads still use the legacy kube-prometheus-backed
+# OpenCost release. This staged slice must not activate or relocate it outside
+# the explicit opt-in fixtures.
+for rendered_path in "${local_controllers_default}" "${prod_controllers_default}"; do
+  assert_resource_count "${rendered_path}" HelmRelease opencost opencost 1
+done
+for rendered_path in "${local_infrastructure_default}" "${prod_infrastructure_default}"; do
+  assert_resource_count "${rendered_path}" HelmRelease opencost opencost 0
+done
+
 render k8s/testdata/observability-option/docker/controllers/ "${docker_controllers}"
 render k8s/testdata/observability-option/docker/infrastructure/ "${docker_infrastructure}"
 render k8s/testdata/observability-option/hetzner/controllers/ "${hetzner_controllers}"
@@ -89,15 +99,18 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_resource_count "${rendered_path}" Coroot observability coroot 0
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 1
+  assert_resource_count "${rendered_path}" HelmRelease opencost opencost 0
 done
 
 assert_resource_count "${docker_infrastructure}" HelmRelease observability coroot-operator 0
 assert_resource_count "${docker_infrastructure}" Coroot observability coroot 1
 assert_resource_count "${docker_infrastructure}" HelmRelease observability audit-log-forwarder 0
+assert_resource_count "${docker_infrastructure}" HelmRelease opencost opencost 1
 
 assert_resource_count "${hetzner_infrastructure}" HelmRelease observability coroot-operator 0
 assert_resource_count "${hetzner_infrastructure}" Coroot observability coroot 1
 assert_resource_count "${hetzner_infrastructure}" HelmRelease observability audit-log-forwarder 1
+assert_resource_count "${hetzner_infrastructure}" HelmRelease opencost opencost 1
 
 coroot_key_contract="$(
   yq eval-all -o=json '.' "${hetzner_infrastructure}" |
@@ -115,6 +128,34 @@ forwarder_key_contract="$(
   yq eval-all -o=json '.' "${hetzner_infrastructure}" |
     jq -s '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "observability" and .metadata.name == "audit-log-forwarder") | .spec.values.extraEnvs[]? | select(.name == "COROOT_API_KEY") | .valueFrom.secretKeyRef | select(.name == "coroot-api-key" and .key == "key")] | length'
 )"
+opencost_dependency_contract="$(
+  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
+    jq -s '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "opencost" and .metadata.name == "opencost") | select((.spec.dependsOn | length) == 1 and .spec.dependsOn[0].name == "coroot-operator" and .spec.dependsOn[0].namespace == "observability")] | length'
+)"
+opencost_prometheus_contract="$(
+  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
+    jq -s '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "opencost" and .metadata.name == "opencost") | select(.spec.values.opencost.prometheus.external.url == "http://coroot-prometheus.observability.svc.cluster.local:9090")] | length'
+)"
+opencost_service_monitor_disabled="$(
+  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
+    jq -s '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "opencost" and .metadata.name == "opencost") | select(.spec.values.opencost.metrics.serviceMonitor.enabled == false)] | length'
+)"
+opencost_legacy_backend_count="$(
+  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
+    jq -s '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "opencost" and .metadata.name == "opencost") | .spec.values.opencost.prometheus.external.url | select(type == "string" and contains("kube-prometheus-stack"))] | length'
+)"
+opencost_coroot_egress_contract="$(
+  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
+    jq -s '[.[] | select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "opencost" and .metadata.name == "allow-opencost") | select(any(.spec.egress[]?.toEndpoints[]?; .matchLabels["k8s:io.kubernetes.pod.namespace"] == "observability"))] | length'
+)"
+opencost_legacy_egress_count="$(
+  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
+    jq -s '[.[] | select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "opencost" and .metadata.name == "allow-opencost") | select(any(.spec.egress[]?.toEndpoints[]?; .matchLabels["k8s:io.kubernetes.pod.namespace"] == "monitoring"))] | length'
+)"
+opencost_legacy_ingress_count="$(
+  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
+    jq -s '[.[] | select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "opencost" and .metadata.name == "allow-opencost") | select(any(.spec.ingress[]?.fromEndpoints[]?; .matchLabels["k8s:io.kubernetes.pod.namespace"] == "monitoring"))] | length'
+)"
 substitution_disabled="$(
   yq eval-all -o=json '.' "${hetzner_infrastructure}" |
     jq -sr '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "observability" and .metadata.name == "audit-log-forwarder") | .metadata.annotations["kustomize.toolkit.fluxcd.io/substitute"] == "disabled"] | all'
@@ -136,6 +177,10 @@ if [[ "${coroot_key_contract}" != "1" || "${coroot_agent_key_contract}" != "1" |
   echo "Coroot projects, bundled agents, and audit-log-forwarder do not share the coroot-api-key/key Secret contract" >&2
   exit 1
 fi
+if [[ "${opencost_dependency_contract}" != "2" || "${opencost_prometheus_contract}" != "2" || "${opencost_service_monitor_disabled}" != "2" || "${opencost_legacy_backend_count}" != "0" || "${opencost_coroot_egress_contract}" != "2" || "${opencost_legacy_egress_count}" != "0" || "${opencost_legacy_ingress_count}" != "0" ]]; then
+  echo "opt-in OpenCost must run in the infrastructure layer against Coroot Prometheus without a ServiceMonitor or obsolete monitoring access" >&2
+  exit 1
+fi
 if [[ "${anonymous_role_contract}" != "0" ]]; then
   echo "generic Coroot option must not grant an anonymous role; instances may opt in behind their own authentication boundary" >&2
   exit 1

From 97af39fe71ccae64fd0cebf4e935b8d7c19af768 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Sat, 18 Jul 2026 12:24:57 +0200
Subject: [PATCH 2/3] fix(observability): retire OpenCost from Coroot option

---
 .../opencost/kustomization.yaml               | 22 ------
 .../patches/use-coroot-network-policy.yaml    |  6 --
 .../patches/use-coroot-prometheus.yaml        | 21 -----
 .../kustomization.yaml                        |  5 +-
 .../docker/infrastructure/kustomization.yaml  |  2 -
 .../hetzner/infrastructure/kustomization.yaml |  2 -
 scripts/validate-observability-option.sh      | 77 +++++++++----------
 7 files changed, 38 insertions(+), 97 deletions(-)
 delete mode 100644 k8s/bases/infrastructure/opencost/kustomization.yaml
 delete mode 100644 k8s/bases/infrastructure/opencost/patches/use-coroot-network-policy.yaml
 delete mode 100644 k8s/bases/infrastructure/opencost/patches/use-coroot-prometheus.yaml

diff --git a/k8s/bases/infrastructure/opencost/kustomization.yaml b/k8s/bases/infrastructure/opencost/kustomization.yaml
deleted file mode 100644
index f631d14..0000000
--- a/k8s/bases/infrastructure/opencost/kustomization.yaml
+++ /dev/null
@@ -1,22 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-# Coroot-backed OpenCost must be applied in the infrastructure layer beside
-# the Coroot custom resource. The operator creates Coroot's Prometheus only
-# after that CR exists, so keeping this HelmRelease in the earlier,
-# wait-gated controllers layer would deadlock the rollout.
-#
-# This composition is staged and default-off. Provider activation must remove
-# the legacy controller-layer HelmRelease and add this base atomically so Flux
-# never has two Kustomizations claiming HelmRelease/opencost/opencost.
-resources:
-  - ../controllers/opencost/
-patches:
-  - path: patches/use-coroot-prometheus.yaml
-  - target:
-      group: cilium.io
-      version: v2
-      kind: CiliumNetworkPolicy
-      name: allow-opencost
-      namespace: opencost
-    path: patches/use-coroot-network-policy.yaml
diff --git a/k8s/bases/infrastructure/opencost/patches/use-coroot-network-policy.yaml b/k8s/bases/infrastructure/opencost/patches/use-coroot-network-policy.yaml
deleted file mode 100644
index b1fa382..0000000
--- a/k8s/bases/infrastructure/opencost/patches/use-coroot-network-policy.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-- op: remove
-  path: /spec/ingress/0
-- op: replace
-  path: /spec/egress/1/toEndpoints/0/matchLabels/k8s:io.kubernetes.pod.namespace
-  value: observability
diff --git a/k8s/bases/infrastructure/opencost/patches/use-coroot-prometheus.yaml b/k8s/bases/infrastructure/opencost/patches/use-coroot-prometheus.yaml
deleted file mode 100644
index 6eece37..0000000
--- a/k8s/bases/infrastructure/opencost/patches/use-coroot-prometheus.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
----
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: opencost
-  namespace: opencost
-spec:
-  dependsOn:
-    - name: coroot-operator
-      namespace: observability
-  values:
-    opencost:
-      prometheus:
-        external:
-          url: "http://coroot-prometheus.observability.svc.cluster.local:9090"
-      # kube-prometheus-stack leaves during the Coroot cutover, including the
-      # Prometheus Operator CRDs. Coroot still scrapes OpenCost's metrics via
-      # its node agent, so no ServiceMonitor is needed.
-      metrics:
-        serviceMonitor:
-          enabled: false
diff --git a/k8s/testdata/observability-option/components/remove-controller-opencost/kustomization.yaml b/k8s/testdata/observability-option/components/remove-controller-opencost/kustomization.yaml
index 1f17468..552794a 100644
--- a/k8s/testdata/observability-option/components/remove-controller-opencost/kustomization.yaml
+++ b/k8s/testdata/observability-option/components/remove-controller-opencost/kustomization.yaml
@@ -1,9 +1,8 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
-# Model the future atomic Flux ownership transfer: the earlier controllers
-# layer gives up every OpenCost object before the later infrastructure layer
-# claims the same identities with the Coroot-backed composition.
+# Stage complete OpenCost retirement only for the explicit Coroot fixtures.
+# Real provider payloads keep the legacy surface until a later activation.
 patches:
   - target:
       version: v1
diff --git a/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml b/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml
index a4f54f9..c1f24be 100644
--- a/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml
+++ b/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml
@@ -1,7 +1,5 @@
----
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../providers/docker/infrastructure/
   - ../../../../bases/infrastructure/coroot/
-  - ../../../../bases/infrastructure/opencost/
diff --git a/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml b/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml
index e3c80c8..51ccc71 100644
--- a/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml
+++ b/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml
@@ -1,8 +1,6 @@
----
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../providers/hetzner/infrastructure/
   - ../../../../bases/infrastructure/coroot/
-  - ../../../../bases/infrastructure/opencost/
   - ../../../../bases/infrastructure/audit-log-forwarder/
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 87b6ab4..e80e990 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -27,7 +27,14 @@ resource_count() {
       --arg kind "${kind}" \
       --arg namespace "${namespace}" \
       --arg name "${name}" \
-      '[.[] | select(type == "object" and .kind == $kind and .metadata.namespace == $namespace and .metadata.name == $name)] | length'
+      '[.[] | select(type == "object" and .kind == $kind and (.metadata.namespace // "") == $namespace and .metadata.name == $name)] | length'
+}
+
+opencost_resource_count() {
+  local rendered_path="$1"
+
+  yq eval-all -o=json '.' "${rendered_path}" |
+    jq -s '[.[] | select(type == "object" and ((.kind == "Namespace" and .metadata.name == "opencost") or .metadata.namespace == "opencost"))] | length'
 }
 
 assert_resource_count() {
@@ -54,6 +61,27 @@ assert_default_off() {
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 0
 }
 
+assert_opencost_present() {
+  local rendered_path="$1"
+
+  assert_resource_count "${rendered_path}" Namespace "" opencost 1
+  assert_resource_count "${rendered_path}" HelmRepository opencost opencost 1
+  assert_resource_count "${rendered_path}" HelmRelease opencost opencost 1
+  assert_resource_count "${rendered_path}" HTTPRoute opencost opencost 1
+  assert_resource_count "${rendered_path}" CiliumNetworkPolicy opencost allow-opencost 1
+}
+
+assert_opencost_absent() {
+  local rendered_path="$1"
+  local actual
+
+  actual="$(opencost_resource_count "${rendered_path}")"
+  if [[ "${actual}" != "0" ]]; then
+    echo "expected no OpenCost resources in ${rendered_path}, found ${actual}" >&2
+    return 1
+  fi
+}
+
 local_controllers_default="${tmp_dir}/local-controllers-default.yaml"
 local_infrastructure_default="${tmp_dir}/local-infrastructure-default.yaml"
 prod_controllers_default="${tmp_dir}/prod-controllers-default.yaml"
@@ -79,14 +107,13 @@ for rendered_path in \
   assert_default_off "${rendered_path}"
 done
 
-# The default provider payloads still use the legacy kube-prometheus-backed
-# OpenCost release. This staged slice must not activate or relocate it outside
-# the explicit opt-in fixtures.
+# The default provider payloads retain the complete legacy OpenCost surface.
+# Only the explicit Coroot fixtures stage its retirement.
 for rendered_path in "${local_controllers_default}" "${prod_controllers_default}"; do
-  assert_resource_count "${rendered_path}" HelmRelease opencost opencost 1
+  assert_opencost_present "${rendered_path}"
 done
 for rendered_path in "${local_infrastructure_default}" "${prod_infrastructure_default}"; do
-  assert_resource_count "${rendered_path}" HelmRelease opencost opencost 0
+  assert_opencost_absent "${rendered_path}"
 done
 
 render k8s/testdata/observability-option/docker/controllers/ "${docker_controllers}"
@@ -99,18 +126,18 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_resource_count "${rendered_path}" Coroot observability coroot 0
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 1
-  assert_resource_count "${rendered_path}" HelmRelease opencost opencost 0
+  assert_opencost_absent "${rendered_path}"
 done
 
 assert_resource_count "${docker_infrastructure}" HelmRelease observability coroot-operator 0
 assert_resource_count "${docker_infrastructure}" Coroot observability coroot 1
 assert_resource_count "${docker_infrastructure}" HelmRelease observability audit-log-forwarder 0
-assert_resource_count "${docker_infrastructure}" HelmRelease opencost opencost 1
+assert_opencost_absent "${docker_infrastructure}"
 
 assert_resource_count "${hetzner_infrastructure}" HelmRelease observability coroot-operator 0
 assert_resource_count "${hetzner_infrastructure}" Coroot observability coroot 1
 assert_resource_count "${hetzner_infrastructure}" HelmRelease observability audit-log-forwarder 1
-assert_resource_count "${hetzner_infrastructure}" HelmRelease opencost opencost 1
+assert_opencost_absent "${hetzner_infrastructure}"
 
 coroot_key_contract="$(
   yq eval-all -o=json '.' "${hetzner_infrastructure}" |
@@ -128,34 +155,6 @@ forwarder_key_contract="$(
   yq eval-all -o=json '.' "${hetzner_infrastructure}" |
     jq -s '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "observability" and .metadata.name == "audit-log-forwarder") | .spec.values.extraEnvs[]? | select(.name == "COROOT_API_KEY") | .valueFrom.secretKeyRef | select(.name == "coroot-api-key" and .key == "key")] | length'
 )"
-opencost_dependency_contract="$(
-  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
-    jq -s '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "opencost" and .metadata.name == "opencost") | select((.spec.dependsOn | length) == 1 and .spec.dependsOn[0].name == "coroot-operator" and .spec.dependsOn[0].namespace == "observability")] | length'
-)"
-opencost_prometheus_contract="$(
-  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
-    jq -s '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "opencost" and .metadata.name == "opencost") | select(.spec.values.opencost.prometheus.external.url == "http://coroot-prometheus.observability.svc.cluster.local:9090")] | length'
-)"
-opencost_service_monitor_disabled="$(
-  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
-    jq -s '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "opencost" and .metadata.name == "opencost") | select(.spec.values.opencost.metrics.serviceMonitor.enabled == false)] | length'
-)"
-opencost_legacy_backend_count="$(
-  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
-    jq -s '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "opencost" and .metadata.name == "opencost") | .spec.values.opencost.prometheus.external.url | select(type == "string" and contains("kube-prometheus-stack"))] | length'
-)"
-opencost_coroot_egress_contract="$(
-  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
-    jq -s '[.[] | select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "opencost" and .metadata.name == "allow-opencost") | select(any(.spec.egress[]?.toEndpoints[]?; .matchLabels["k8s:io.kubernetes.pod.namespace"] == "observability"))] | length'
-)"
-opencost_legacy_egress_count="$(
-  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
-    jq -s '[.[] | select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "opencost" and .metadata.name == "allow-opencost") | select(any(.spec.egress[]?.toEndpoints[]?; .matchLabels["k8s:io.kubernetes.pod.namespace"] == "monitoring"))] | length'
-)"
-opencost_legacy_ingress_count="$(
-  yq eval-all -o=json '.' "${docker_infrastructure}" "${hetzner_infrastructure}" |
-    jq -s '[.[] | select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "opencost" and .metadata.name == "allow-opencost") | select(any(.spec.ingress[]?.fromEndpoints[]?; .matchLabels["k8s:io.kubernetes.pod.namespace"] == "monitoring"))] | length'
-)"
 substitution_disabled="$(
   yq eval-all -o=json '.' "${hetzner_infrastructure}" |
     jq -sr '[.[] | select(.kind == "HelmRelease" and .metadata.namespace == "observability" and .metadata.name == "audit-log-forwarder") | .metadata.annotations["kustomize.toolkit.fluxcd.io/substitute"] == "disabled"] | all'
@@ -177,10 +176,6 @@ if [[ "${coroot_key_contract}" != "1" || "${coroot_agent_key_contract}" != "1" |
   echo "Coroot projects, bundled agents, and audit-log-forwarder do not share the coroot-api-key/key Secret contract" >&2
   exit 1
 fi
-if [[ "${opencost_dependency_contract}" != "2" || "${opencost_prometheus_contract}" != "2" || "${opencost_service_monitor_disabled}" != "2" || "${opencost_legacy_backend_count}" != "0" || "${opencost_coroot_egress_contract}" != "2" || "${opencost_legacy_egress_count}" != "0" || "${opencost_legacy_ingress_count}" != "0" ]]; then
-  echo "opt-in OpenCost must run in the infrastructure layer against Coroot Prometheus without a ServiceMonitor or obsolete monitoring access" >&2
-  exit 1
-fi
 if [[ "${anonymous_role_contract}" != "0" ]]; then
   echo "generic Coroot option must not grant an anonymous role; instances may opt in behind their own authentication boundary" >&2
   exit 1

From 67da63606b714e15974883abf90f871ce8ea3414 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Sat, 18 Jul 2026 12:56:55 +0200
Subject: [PATCH 3/3] fix(observability): remove retired OpenCost references

---
 .../kustomization.yaml                        | 162 ++++++++++++++++++
 .../docker/controllers/kustomization.yaml     |   1 +
 .../docker/infrastructure/kustomization.yaml  |   3 +
 .../hetzner/controllers/kustomization.yaml    |   1 +
 .../hetzner/infrastructure/kustomization.yaml |   3 +
 scripts/validate-observability-option.sh      |  46 ++++-
 6 files changed, 214 insertions(+), 2 deletions(-)
 create mode 100644 k8s/testdata/observability-option/components/remove-opencost-references/kustomization.yaml

diff --git a/k8s/testdata/observability-option/components/remove-opencost-references/kustomization.yaml b/k8s/testdata/observability-option/components/remove-opencost-references/kustomization.yaml
new file mode 100644
index 0000000..9330d37
--- /dev/null
+++ b/k8s/testdata/observability-option/components/remove-opencost-references/kustomization.yaml
@@ -0,0 +1,162 @@
+---
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+# Remove cross-namespace consumers and exemptions alongside the explicit
+# Coroot fixtures' OpenCost workload retirement. Targets absent from a layer
+# are ignored; the executable contract rejects any surviving reference.
+patches:
+  - target:
+      version: v1
+      kind: ConfigMap
+      name: auth-proxy-config
+      namespace: oauth2-proxy
+    patch: |
+      - op: replace
+        path: /data/dynamic.yaml
+        value: |
+          http:
+            routers:
+              homepage:
+                rule: "Host(`${domain}`)"
+                entryPoints: ["web"]
+                service: homepage
+              hubble:
+                rule: "Host(`hubble.${domain}`)"
+                entryPoints: ["web"]
+                service: hubble-ui
+              prometheus:
+                rule: "Host(`prometheus.${domain}`)"
+                entryPoints: ["web"]
+                service: prometheus
+              alertmanager:
+                rule: "Host(`alertmanager.${domain}`)"
+                entryPoints: ["web"]
+                service: alertmanager
+            services:
+              homepage:
+                loadBalancer:
+                  servers:
+                    - url: "http://homepage.homepage.svc.cluster.local:80"
+              hubble-ui:
+                loadBalancer:
+                  servers:
+                    - url: >-
+                        http://keda-add-ons-http-interceptor-proxy.keda.svc.cluster.local:8080
+              prometheus:
+                loadBalancer:
+                  servers:
+                    - url: >-
+                        http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090
+              alertmanager:
+                loadBalancer:
+                  servers:
+                    - url: >-
+                        http://kube-prometheus-stack-alertmanager.monitoring.svc.cluster.local:9093
+  - target:
+      group: cilium.io
+      version: v2
+      kind: CiliumNetworkPolicy
+      name: allow-auth-proxy
+      namespace: oauth2-proxy
+    patch: |
+      - op: test
+        path: /spec/egress/3
+        value:
+          toEndpoints:
+            - matchLabels:
+                k8s:io.kubernetes.pod.namespace: opencost
+          toPorts:
+            - ports:
+                - port: "9090"
+                  protocol: TCP
+      - op: remove
+        path: /spec/egress/3
+  - target:
+      group: cilium.io
+      version: v2
+      kind: CiliumNetworkPolicy
+      name: allow-monitoring
+      namespace: monitoring
+    patch: |
+      - op: test
+        path: /spec/ingress/1
+        value:
+          fromEndpoints:
+            - matchLabels:
+                k8s:io.kubernetes.pod.namespace: opencost
+          toPorts:
+            - ports:
+                - port: "9090"
+                  protocol: TCP
+      - op: remove
+        path: /spec/ingress/1
+  - target:
+      group: gateway.networking.k8s.io
+      version: v1beta1
+      kind: ReferenceGrant
+      name: allow-oauth2-proxy-backends
+      namespace: oauth2-proxy
+    patch: |
+      - op: test
+        path: /spec/from/4
+        value:
+          group: gateway.networking.k8s.io
+          kind: HTTPRoute
+          namespace: opencost
+      - op: remove
+        path: /spec/from/4
+  - target:
+      group: kubescape.io
+      version: v1beta1
+      kind: ClusterSecurityException
+      name: controller-rbac
+    patch: |
+      - op: test
+        path: /spec/match/namespaceSelector/matchExpressions/0/values/16
+        value: opencost
+      - op: remove
+        path: /spec/match/namespaceSelector/matchExpressions/0/values/16
+      - op: test
+        path: /spec/reason
+        value: >-
+          Infrastructure controllers and platform middleware require broad RBAC
+          to manage their target resources cluster-wide. This includes GitOps
+          (Flux), policy engines (Kyverno), certificate managers (cert-manager),
+          database operators (CNPG), backup controllers (Velero), autoscalers
+          (KEDA), monitoring (Prometheus/opencost), virtualisation (KubeVirt,
+          CDI), storage (Longhorn), cluster dashboards (headlamp), service
+          discovery (homepage), identity (dex), auth proxies (oauth2-proxy),
+          and config watchers (reloader).
+      - op: replace
+        path: /spec/reason
+        value: >-
+          Infrastructure controllers and platform middleware require broad RBAC
+          to manage their target resources cluster-wide. This includes GitOps
+          (Flux), policy engines (Kyverno), certificate managers (cert-manager),
+          database operators (CNPG), backup controllers (Velero), autoscalers
+          (KEDA), monitoring (Prometheus), virtualisation (KubeVirt, CDI),
+          storage (Longhorn), cluster dashboards (headlamp), service discovery
+          (homepage), identity (dex), auth proxies (oauth2-proxy), and config
+          watchers (reloader).
+  - target:
+      group: kubescape.io
+      version: v1beta1
+      kind: ClusterSecurityException
+      name: service-account-tokens
+    patch: |
+      - op: test
+        path: /spec/match/namespaceSelector/matchExpressions/0/values/18
+        value: opencost
+      - op: remove
+        path: /spec/match/namespaceSelector/matchExpressions/0/values/18
+  - target:
+      group: kubescape.io
+      version: v1beta1
+      kind: ClusterSecurityException
+      name: health-probes
+    patch: |
+      - op: test
+        path: /spec/match/namespaceSelector/matchExpressions/0/values/14
+        value: opencost
+      - op: remove
+        path: /spec/match/namespaceSelector/matchExpressions/0/values/14
diff --git a/k8s/testdata/observability-option/docker/controllers/kustomization.yaml b/k8s/testdata/observability-option/docker/controllers/kustomization.yaml
index 6dfc6c9..6c3e308 100644
--- a/k8s/testdata/observability-option/docker/controllers/kustomization.yaml
+++ b/k8s/testdata/observability-option/docker/controllers/kustomization.yaml
@@ -6,3 +6,4 @@ resources:
   - ../../../../bases/infrastructure/controllers/coroot/
 components:
   - ../../components/remove-controller-opencost/
+  - ../../components/remove-opencost-references/
diff --git a/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml b/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml
index c1f24be..8005522 100644
--- a/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml
+++ b/k8s/testdata/observability-option/docker/infrastructure/kustomization.yaml
@@ -1,5 +1,8 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../providers/docker/infrastructure/
   - ../../../../bases/infrastructure/coroot/
+components:
+  - ../../components/remove-opencost-references/
diff --git a/k8s/testdata/observability-option/hetzner/controllers/kustomization.yaml b/k8s/testdata/observability-option/hetzner/controllers/kustomization.yaml
index 127edae..90c369c 100644
--- a/k8s/testdata/observability-option/hetzner/controllers/kustomization.yaml
+++ b/k8s/testdata/observability-option/hetzner/controllers/kustomization.yaml
@@ -6,3 +6,4 @@ resources:
   - ../../../../bases/infrastructure/controllers/coroot/
 components:
   - ../../components/remove-controller-opencost/
+  - ../../components/remove-opencost-references/
diff --git a/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml b/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml
index 51ccc71..63e86fe 100644
--- a/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml
+++ b/k8s/testdata/observability-option/hetzner/infrastructure/kustomization.yaml
@@ -1,6 +1,9 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../../../../providers/hetzner/infrastructure/
   - ../../../../bases/infrastructure/coroot/
   - ../../../../bases/infrastructure/audit-log-forwarder/
+components:
+  - ../../components/remove-opencost-references/
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index e80e990..ec8a0d2 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -37,6 +37,13 @@ opencost_resource_count() {
     jq -s '[.[] | select(type == "object" and ((.kind == "Namespace" and .metadata.name == "opencost") or .metadata.namespace == "opencost"))] | length'
 }
 
+opencost_reference_count() {
+  local rendered_path="$1"
+
+  yq eval-all -o=json '.' "${rendered_path}" |
+    jq -s '[.[] | select(type == "object" and (tojson | test("opencost"; "i")))] | length'
+}
+
 assert_resource_count() {
   local rendered_path="$1"
   local kind="$2"
@@ -71,7 +78,7 @@ assert_opencost_present() {
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy opencost allow-opencost 1
 }
 
-assert_opencost_absent() {
+assert_opencost_resources_absent() {
   local rendered_path="$1"
   local actual
 
@@ -82,6 +89,38 @@ assert_opencost_absent() {
   fi
 }
 
+assert_opencost_absent() {
+  local rendered_path="$1"
+  local actual
+
+  assert_opencost_resources_absent "${rendered_path}"
+  actual="$(opencost_reference_count "${rendered_path}")"
+  if [[ "${actual}" != "0" ]]; then
+    echo "expected no OpenCost references in ${rendered_path}, found ${actual} resource(s)" >&2
+    return 1
+  fi
+}
+
+assert_auth_proxy_without_opencost() {
+  local default_rendered_path="$1"
+  local opt_in_rendered_path="$2"
+  local expected
+  local actual
+
+  expected="$(
+    yq eval-all -o=json 'select(.kind == "ConfigMap" and .metadata.namespace == "oauth2-proxy" and .metadata.name == "auth-proxy-config") | .data."dynamic.yaml" | from_yaml' "${default_rendered_path}" |
+      jq -S -c 'del(.http.routers.opencost, .http.services.opencost)'
+  )"
+  actual="$(
+    yq eval-all -o=json 'select(.kind == "ConfigMap" and .metadata.namespace == "oauth2-proxy" and .metadata.name == "auth-proxy-config") | .data."dynamic.yaml" | from_yaml' "${opt_in_rendered_path}" |
+      jq -S -c '.'
+  )"
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "opt-in auth-proxy config must equal its provider default minus the OpenCost router and service" >&2
+    return 1
+  fi
+}
+
 local_controllers_default="${tmp_dir}/local-controllers-default.yaml"
 local_infrastructure_default="${tmp_dir}/local-infrastructure-default.yaml"
 prod_controllers_default="${tmp_dir}/prod-controllers-default.yaml"
@@ -113,7 +152,7 @@ for rendered_path in "${local_controllers_default}" "${prod_controllers_default}
   assert_opencost_present "${rendered_path}"
 done
 for rendered_path in "${local_infrastructure_default}" "${prod_infrastructure_default}"; do
-  assert_opencost_absent "${rendered_path}"
+  assert_opencost_resources_absent "${rendered_path}"
 done
 
 render k8s/testdata/observability-option/docker/controllers/ "${docker_controllers}"
@@ -121,6 +160,9 @@ render k8s/testdata/observability-option/docker/infrastructure/ "${docker_infras
 render k8s/testdata/observability-option/hetzner/controllers/ "${hetzner_controllers}"
 render k8s/testdata/observability-option/hetzner/infrastructure/ "${hetzner_infrastructure}"
 
+assert_auth_proxy_without_opencost "${local_controllers_default}" "${docker_controllers}"
+assert_auth_proxy_without_opencost "${prod_controllers_default}" "${hetzner_controllers}"
+
 for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_resource_count "${rendered_path}" HelmRelease observability coroot-operator 1
   assert_resource_count "${rendered_path}" Coroot observability coroot 0