From 1a44dc8213d59a4a8c55b23eaf376b7b8d13c91b Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 27 Oct 2026 07:18:57 +0100 Subject: [PATCH] fix(retirement): sample each layer until it settles before judging it The second Retire Unused App Leftovers run refused again, before deleting anything. With the unrelated private-mail workload failing (#5412), the apps Kustomization retries its health checks every two minutes (retryInterval 2m) and reports Ready=Unknown/Progressing during each attempt, which can last minutes. The run read it in that window. Read each layer up to 40 times, 15 seconds apart, and proceed once it reports Ready or an unrelated HealthCheckFailed at its current generation. The acceptance rule is unchanged, and a layer that never settles is still refused. Part of #2496 Co-Authored-By: Claude Opus 5.5 --- scripts/retire-unused-apps.sh | 30 +++++++++++++++++------- scripts/tests/test-retire-unused-apps.sh | 13 ++++++++-- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/scripts/retire-unused-apps.sh b/scripts/retire-unused-apps.sh index 269cff6092..a7af8440f0 100755 --- a/scripts/retire-unused-apps.sh +++ b/scripts/retire-unused-apps.sh @@ -47,6 +47,12 @@ kubectl_bin="${KUBECTL:-kubectl}" poll_seconds="${RETIREMENT_POLL_SECONDS:-10}" poll_attempts="${RETIREMENT_POLL_ATTEMPTS:-90}" readonly kubectl_bin poll_seconds poll_attempts +# A layer retrying an unrelated failing health check passes through a +# Progressing phase every retry interval, so its state is sampled until it +# settles rather than judged at one instant. +settle_seconds="${RETIREMENT_SETTLE_SECONDS:-15}" +settle_attempts="${RETIREMENT_SETTLE_ATTEMPTS:-40}" +readonly settle_seconds settle_attempts readonly releases=(crossview/crossview headlamp/headlamp) readonly clusters=(backstage/backstage-db) readonly namespaces=(crossview headlamp backstage) @@ -140,15 +146,21 @@ command -v jq >/dev/null || unknown 'jq is required' # this retirement. Any other not-Ready reason, such as a failed build or # apply, leaves the inventory unproven and is refused. for layer in infrastructure-controllers infrastructure apps; do - kustomization="$(k -n flux-system get kustomization "${layer}" -o json)" || - unknown "cannot read the ${layer} Kustomization" - jq -e ' - (.status.conditions // [] | any(.type == "Ready" and - (.status == "True" or (.status == "False" and .reason == "HealthCheckFailed")))) and - .status.observedGeneration == .metadata.generation and - (.status.inventory.entries | type == "array" and length > 0) - ' <<<"${kustomization}" >/dev/null || - fail "the ${layer} Kustomization has not applied its current generation with a non-empty inventory" + attempt=0 + while :; do + kustomization="$(k -n flux-system get kustomization "${layer}" -o json)" || + unknown "cannot read the ${layer} Kustomization" + jq -e ' + (.status.conditions // [] | any(.type == "Ready" and + (.status == "True" or (.status == "False" and .reason == "HealthCheckFailed")))) and + .status.observedGeneration == .metadata.generation and + (.status.inventory.entries | type == "array" and length > 0) + ' <<<"${kustomization}" >/dev/null && break + attempt=$((attempt + 1)) + [ "${attempt}" -lt "${settle_attempts}" ] || + fail "the ${layer} Kustomization has not applied its current generation with a non-empty inventory" + sleep "${settle_seconds}" + done if jq -e '.status.conditions // [] | any(.type == "Ready" and .status == "False")' <<<"${kustomization}" >/dev/null; then printf 'Note: the %s Kustomization applied its revision but an unrelated health check is failing\n' "${layer}" fi diff --git a/scripts/tests/test-retire-unused-apps.sh b/scripts/tests/test-retire-unused-apps.sh index f85748de41..396f0ec481 100755 --- a/scripts/tests/test-retire-unused-apps.sh +++ b/scripts/tests/test-retire-unused-apps.sh @@ -58,7 +58,8 @@ for name in "${names[@]}"; do file="${STATE}/${kind}/${namespace}/${name}" case "${verb}" in get) - if [ "${kind}" = kustomization ]; then cat "${STATE}/${name}.json" 2>/dev/null || cat "${STATE}/apps.json"; elif [ -f "${file}" ]; then cat "${file}"; fi ;; + if [ "${kind}" = kustomization ] && [ -f "${STATE}/${name}.first.json" ]; then cat "${STATE}/${name}.first.json"; rm -f "${STATE}/${name}.first.json"; + elif [ "${kind}" = kustomization ]; then cat "${STATE}/${name}.json" 2>/dev/null || cat "${STATE}/apps.json"; elif [ -f "${file}" ]; then cat "${file}"; fi ;; delete) [ -n "${KEEP:-}" ] && [ "${kind}/${name}" = "${KEEP}" ] && continue rm -f "${file}" @@ -127,7 +128,7 @@ run() { # [env...] -> sets status and output local state="$1" shift if output="$(env STATE="${state}" KUBECTL="${work_dir}/kubectl" RETIREMENT_POLL_SECONDS=0 \ - RETIREMENT_POLL_ATTEMPTS=3 RETIREMENT_CONFIRM=retire-crossview-headlamp-backstage "$@" \ + RETIREMENT_POLL_ATTEMPTS=3 RETIREMENT_SETTLE_SECONDS=0 RETIREMENT_SETTLE_ATTEMPTS=3 RETIREMENT_CONFIRM=retire-crossview-headlamp-backstage "$@" \ bash "${script}" 2>&1)"; then status=0 else @@ -196,6 +197,14 @@ jq '.status.conditions = [{type: "Ready", status: "False", reason: "HealthCheckF run "${state}" expect 0 'an unrelated health check is failing' 'an unrelated health failure must not block the retirement' +# A layer read mid-retry (Progressing) is sampled again until it settles. +state="$(seed retrying "${deployed}")" +jq '.status.conditions = [{type: "Ready", status: "False", reason: "HealthCheckFailed"}]' "${state}/apps.json" >"${state}/apps.tmp" && mv "${state}/apps.tmp" "${state}/apps.json" +jq '.status.conditions = [{type: "Ready", status: "Unknown", reason: "Progressing"}]' "${state}/apps.json" >"${state}/apps.first.json" +run "${state}" +expect 0 'an unrelated health check is failing' 'a layer that settles after a retry must not block the retirement' +[ "$(grep -c '^get kustomization apps' "${state}/calls")" -ge 2 ] || fail 'a Progressing layer must be sampled again' + # An object of the same name that Flux apps does not own is never deleted. state="$(seed foreign "${deployed}")" put "${state}" "${cnpg}" backstage backstage-db '{"metadata":{"labels":{}}}'