fix(observability): measure endpoint creation queueing safely

.github/workflows/ci.yaml

@@ -379,6 +379,9 @@ jobs:
               - 'scripts/measure-datapath-cpu-throttling.sh'
               - 'scripts/tests/test-measure-datapath-cpu-throttling.sh'
               - '.github/workflows/measure-datapath-cpu-throttling.yaml'
+              - 'scripts/read-cilium-api-limiter.sh'
+              - 'scripts/tests/test-read-cilium-api-limiter.sh'
+              - '.github/workflows/read-cilium-api-limiter.yaml'
               - 'scripts/generate-kubescape-exceptions/**'
               - 'scripts/tests/test-kubescape-self-hosted-scan-persistence.sh'
               - 'scripts/tests/test-kubescape-storage-hotfix.sh'
@@ -2316,6 +2319,12 @@ jobs:
           shellcheck scripts/measure-datapath-cpu-throttling.sh scripts/tests/test-measure-datapath-cpu-throttling.sh
           bash scripts/tests/test-measure-datapath-cpu-throttling.sh
 
+      - name: Validate finite Cilium API limiter reads
+        if: needs.changes.outputs.k8s == 'true'
+        run: |
+          shellcheck scripts/read-cilium-api-limiter.sh scripts/tests/test-read-cilium-api-limiter.sh
+          bash scripts/tests/test-read-cilium-api-limiter.sh
+
       - name: 🔐 Validate Cilium mutual-auth policy scope
         if: needs.changes.outputs.k8s == 'true'
         # An authentication stanza is still an allow rule. Reject a cluster-wide

.github/workflows/read-cilium-api-limiter.yaml

@@ -0,0 +1,73 @@
+name: Read Cilium API Limiter
+
+# Finite read-only evidence for #4329. The runner starts only the fixed `cilium-dbg metrics
+# list` command in current cilium-agent containers. It changes no listener, setting or policy.
+# KUBE_CONFIG has the established admin@prod pipeline identity; the agent's reader cannot
+# reach this surface. Reviewed main code, fixed targets/command, a typed confirmation and
+# sanitized output bound this job. No raw metrics, pod identities or kubectl errors are uploaded.
+on:
+  workflow_dispatch:
+    inputs:
+      confirm:
+        description: "Type exactly: read-production-cilium-api-limiter"
+        required: true
+        type: string
+
+permissions: {}
+
+concurrency:
+  group: prod-deploy
+  cancel-in-progress: false
+  queue: max
+
+jobs:
+  read:
+    name: Read Cilium endpoint creation timing
+    runs-on: ubuntu-latest
+    timeout-minutes: 65
+    environment: prod
+    permissions:
+      contents: read
+    steps:
+      - name: Require the reviewed main branch
+        shell: bash
+        env:
+          RUN_REF: ${{ github.ref }}
+        run: |
+          set -euo pipefail
+          [[ "${RUN_REF}" == refs/heads/main ]] || { printf 'Dispatch from reviewed main.\n' >&2; exit 1; }
+
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+
+      - name: Require explicit confirmation
+        shell: bash
+        env:
+          CONFIRM: ${{ inputs.confirm }}
+        run: |
+          set -euo pipefail
+          [[ "${CONFIRM}" == read-production-cilium-api-limiter ]] || { printf 'Confirmation did not match.\n' >&2; exit 1; }
+
+      - name: Restore kubeconfig
+        shell: bash
+        env:
+          KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+        run: |
+          set -euo pipefail
+          umask 077
+          mkdir -p ~/.kube
+          printf '%s' "${KUBE_CONFIG}" > ~/.kube/config
+          chmod 600 ~/.kube/config
+
+      - name: Select stable prod API endpoint
+        shell: bash
+        env:
+          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
+        run: ./scripts/use-prod-stable-api-endpoint.sh >/dev/null
+
+      - name: Read bounded limiter samples
+        shell: bash
+        run: ./scripts/read-cilium-api-limiter.sh --context admin@prod

docs/cilium-api-limiter-diagnosis.md

@@ -0,0 +1,37 @@
+# Cilium endpoint creation timing
+
+The **Read Cilium API Limiter** workflow collects five snapshots of the endpoint
+creation limiter from each stable ready Cilium agent. Dispatch it from reviewed
+`main` with `confirm=read-production-cilium-api-limiter`. It uses the established
+production pipeline identity and serializes with deployments. No listener,
+limiter setting, network policy or workload is changed.
+
+The fixed command is `cilium-dbg metrics list --output json --match-pattern
+'^cilium_api_limiter_'`. The implementation follows the
+[Cilium v1.20.3 CLI](https://github.com/cilium/cilium/blob/v1.20.3/cilium-dbg/cmd/metrics_list.go)
+and [limiter observer](https://github.com/cilium/cilium/blob/v1.20.3/pkg/rate/metrics/metrics.go).
+The diagnostic refuses other agent versions. A later version needs a source audit
+and matching fixture updates before dispatch.
+
+Public output contains opaque agent indices, capture times, endpoint-create
+rolling mean/estimated processing durations, mean/min/max waiting durations,
+in-flight/limit values, rate/burst settings, adjustment factor and the aggregate
+processed-request counter. Raw responses and identity fields stay in a private
+temporary directory which is removed on exit. No raw artifact is uploaded.
+
+`MEASURED` requires a complete finite metric set for every sample and the same
+ready DaemonSet, Pod UID, node name, container ID, image ID and restart count at
+both ends. Missing metrics, a failed read or a changed identity is
+`INCONCLUSIVE` (exit 3). There is no successful partial result. Opaque indices
+are local to one run and cannot be joined to another run or to Coroot machines.
+
+These gauges summarize limiter observations; they are not per-request traces or
+latency percentiles. Compare waiting and processing values, counter movement,
+the finite runtime logs and the same Coroot incident window before choosing a
+change. A measured sample alone establishes neither request attribution nor
+SLO recovery. Keep the incident open until its original objective recovers.
+
+Offline validation: `bash scripts/tests/test-read-cilium-api-limiter.sh` uses a
+fake Kubernetes client, checks the exact reads and privacy boundary, exercises
+missing/replaced/unsupported agents and malformed/partial/oversized metrics,
+and executes both workflow guards with accepted and rejected inputs.

scripts/read-cilium-api-limiter.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+# Reviewed workflow only: five finite, fixed Cilium CLI metric reads per ready agent (#4329).
+# No API listener, setting, workload or policy is changed. Raw responses stay in a private
+# temporary directory. Output is delayed until every read and identity comparison succeeds.
+# Cilium v1.20.3 sources: cilium-dbg/cmd/metrics_list.go, pkg/metrics/cmd.go,
+# pkg/metrics/metrics.go and pkg/rate/metrics/metrics.go. These are rolling gauges,
+# not a latency distribution or proof of Coroot request attribution or recovery.
+set -euo pipefail
+if [[ "$#" -ne 2 || "$1" != --context || "$2" != admin@prod ]]; then
+  printf 'Usage: %s --context admin@prod\n' "$(basename "$0")" >&2
+  exit 1
+fi
+for tool in kubectl jq timeout head; do
+  command -v "${tool}" >/dev/null 2>&1 || { printf 'Required diagnostic tool is unavailable.\n' >&2; exit 1; }
+done
+umask 077
+work_dir="$(mktemp -d)"
+trap 'rm -rf "${work_dir}"' EXIT
+inconclusive() { printf 'INCONCLUSIVE: %s\n' "$1"; exit 3; }
+
+# Bound every stream and request. Neither failed kubectl output nor parser diagnostics are public.
+read_json() {
+  local destination="$1"
+  shift
+  if ! timeout 20s kubectl --context admin@prod --request-timeout 15s "$@" \
+    2>"${work_dir}/read.stderr" | head -c 1048577 >"${destination}"; then return 1; fi
+  [[ "$(wc -c <"${destination}")" -le 1048576 ]] &&
+    jq -e -s 'length==1 and .[0]!=null' "${destination}" >/dev/null 2>&1
+}
+roster() {
+  local prefix="$1"
+  read_json "${prefix}.ds" get daemonset cilium --namespace kube-system --output json || return 1
+  read_json "${prefix}.pods" get pods --namespace kube-system --selector k8s-app=cilium --output json || return 1
+  jq -e -S -c --slurpfile ds "${prefix}.ds" '
+    $ds[0] as $d |
+    [$d.spec.template.spec.containers[] | select(.name=="cilium-agent")] as $dc |
+    if ($d.metadata.uid|type)!="string" or ($d.metadata.uid|length)==0 or
+      ($dc|length)!=1 or
+      ($dc[0].image|test("^quay.io/cilium/cilium:v1\\.20\\.3(@sha256:[a-f0-9]{64})?$"))!=true or
+      ($d.status.desiredNumberScheduled|type)!="number" or
+      $d.status.desiredNumberScheduled<1 or $d.status.desiredNumberScheduled>32 or
+      $d.status.desiredNumberScheduled!=$d.status.numberAvailable or
+      $d.status.desiredNumberScheduled!=$d.status.updatedNumberScheduled or
+      $d.status.numberMisscheduled!=0 or $d.status.observedGeneration!=$d.metadata.generation or
+      (.items|type)!="array" or (.items|length)!=$d.status.desiredNumberScheduled
+    then error("incomplete daemonset") else . end |
+    [.items[] |
+      . as $p |
+      [.spec.containers[]|select(.name=="cilium-agent")] as $c |
+      [.status.containerStatuses[]|select(.name=="cilium-agent")] as $s |
+      if (.metadata.name|test("^cilium-[a-z0-9]+$"))!=true or
+        (.metadata.uid|type)!="string" or (.metadata.uid|length)==0 or
+        .metadata.namespace!="kube-system" or .metadata.deletionTimestamp!=null or
+        (.metadata.ownerReferences|map(select(.kind=="DaemonSet" and .name=="cilium" and
+          .uid==$d.metadata.uid and .controller==true))|length)!=1 or
+        .status.phase!="Running" or (.spec.nodeName|type)!="string" or (.spec.nodeName|length)==0 or
+        ($c|length)!=1 or $c[0].image!=$dc[0].image or ($s|length)!=1 or $s[0].ready!=true or
+        ($s[0].state.running|type)!="object" or ($s[0].restartCount|type)!="number" or
+        ($s[0].containerID|type)!="string" or ($s[0].containerID|length)==0 or
+        ($s[0].imageID|type)!="string" or ($s[0].imageID|length)==0
+      then error("incomplete agent") else
+        {name:$p.metadata.name,uid:$p.metadata.uid,node:$p.spec.nodeName,
+         container:$s[0].containerID,image:$s[0].imageID,restarts:$s[0].restartCount}
+      end] | sort_by(.name) |
+    if (map(.name)|unique|length)!=length or (map(.node)|unique|length)!=length
+    then error("duplicate agent") else {daemonset:$d.metadata.uid,generation:$d.metadata.generation,agents:.} end
+  ' "${prefix}.pods" >"${prefix}.identity" 2>/dev/null
+}
+
+roster "${work_dir}/before" || inconclusive 'the ready agent roster could not be established.'
+jq -r '.agents[].name' "${work_dir}/before.identity" >"${work_dir}/names"
+: >"${work_dir}/safe"
+for sample in 1 2 3 4 5; do
+  [[ "${sample}" -eq 1 ]] || sleep 15
+  agent=0
+  while IFS= read -r pod; do
+    agent=$((agent + 1))
+    read_json "${work_dir}/metrics" exec --namespace kube-system "${pod}" --container cilium-agent \
+      -- cilium-dbg metrics list --output json --match-pattern '^cilium_api_limiter_' ||
+      inconclusive 'an agent metric read failed.'
+    captured_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+    if ! jq -e -c --argjson sample "${sample}" --argjson agent "${agent}" --arg time "${captured_at}" '
+      def metric_key: .name+"/"+(.labels.value//"");
+      ["cilium_api_limiter_processing_duration_seconds/mean",
+       "cilium_api_limiter_processing_duration_seconds/estimated",
+       "cilium_api_limiter_wait_duration_seconds/mean",
+       "cilium_api_limiter_wait_duration_seconds/min",
+       "cilium_api_limiter_wait_duration_seconds/max",
+       "cilium_api_limiter_requests_in_flight/in-flight",
+       "cilium_api_limiter_requests_in_flight/limit",
+       "cilium_api_limiter_rate_limit/limit",
+       "cilium_api_limiter_rate_limit/burst",
+       "cilium_api_limiter_adjustment_factor/"] | sort as $expected |
+      input |
+      if type!="array" or length>10000 then error("invalid metrics") else . end |
+      map(select(.labels.api_call=="endpoint-create")) as $endpoint |
+      [$endpoint[]|select(.name!="cilium_api_limiter_processed_requests_total")|
+        select(.name as $name | $expected|any(startswith($name+"/")))] as $gauges |
+      [$endpoint[]|select(.name=="cilium_api_limiter_processed_requests_total")] as $counters |
+      if ($gauges|map(metric_key)|sort)!=$expected or ($counters|length)==0 or
+        (all(($gauges+$counters)[]; (.value|type)=="number" and .value>=0 and .value<=1e12))!=true or
+        (all($gauges[]; (.labels|keys)==(if .name=="cilium_api_limiter_adjustment_factor"
+          then ["api_call"] else ["api_call","value"] end)))!=true or
+        (all($counters[]; (.labels|keys)==["api_call","outcome","return_code"]))!=true or
+        ($counters|map(.labels)|unique|length)!=($counters|length)
+      then error("incomplete metric set") else
+        {sample:$sample,agent:$agent,capturedAt:$time,apiCall:"endpoint-create",
+         metrics:([$gauges[]|{metric:.name,stat:(.labels.value//"current"),value:.value}]+
+          [{metric:"cilium_api_limiter_processed_requests_total",stat:"all",value:($counters|map(.value)|add)}])}
+      end
+    ' --null-input "${work_dir}/metrics" >>"${work_dir}/safe" 2>"${work_dir}/parse.stderr"; then
+      inconclusive 'a complete finite metric set could not be established.'
+    fi
+  done <"${work_dir}/names"
+done
+roster "${work_dir}/after" || inconclusive 'the final agent roster could not be established.'
+cmp -s "${work_dir}/before.identity" "${work_dir}/after.identity" ||
+  inconclusive 'agent identity or container state changed during collection.'
+cat "${work_dir}/safe"
+printf 'MEASURED: five complete samples per stable ready agent; rolling gauges only.\n'