Skip to content

fix(retirement): sample each layer until it settles before judging it #5430

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
devantler merged 1 commit into from
Oct 27, 2026
Merged

fix(retirement): sample each layer until it settles before judging it #5430

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 21 additions & 9 deletions scripts/retire-unused-apps.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ kubectl_bin="${KUBECTL:-kubectl}"
poll_seconds="${RETIREMENT_POLL_SECONDS:-10}"
poll_attempts="${RETIREMENT_POLL_ATTEMPTS:-90}"
readonly kubectl_bin poll_seconds poll_attempts
# A layer retrying an unrelated failing health check passes through a
# Progressing phase every retry interval, so its state is sampled until it
# settles rather than judged at one instant.
settle_seconds="${RETIREMENT_SETTLE_SECONDS:-15}"
settle_attempts="${RETIREMENT_SETTLE_ATTEMPTS:-40}"
readonly settle_seconds settle_attempts
readonly releases=(crossview/crossview headlamp/headlamp)
readonly clusters=(backstage/backstage-db)
readonly namespaces=(crossview headlamp backstage)
Expand Down Expand Up @@ -140,15 +146,21 @@ command -v jq >/dev/null || unknown 'jq is required'
# this retirement. Any other not-Ready reason, such as a failed build or
# apply, leaves the inventory unproven and is refused.
for layer in infrastructure-controllers infrastructure apps; do
kustomization="$(k -n flux-system get kustomization "${layer}" -o json)" ||
unknown "cannot read the ${layer} Kustomization"
jq -e '
(.status.conditions // [] | any(.type == "Ready" and
(.status == "True" or (.status == "False" and .reason == "HealthCheckFailed")))) and
.status.observedGeneration == .metadata.generation and
(.status.inventory.entries | type == "array" and length > 0)
' <<<"${kustomization}" >/dev/null ||
fail "the ${layer} Kustomization has not applied its current generation with a non-empty inventory"
attempt=0
while :; do
kustomization="$(k -n flux-system get kustomization "${layer}" -o json)" ||
unknown "cannot read the ${layer} Kustomization"
jq -e '
(.status.conditions // [] | any(.type == "Ready" and
(.status == "True" or (.status == "False" and .reason == "HealthCheckFailed")))) and
.status.observedGeneration == .metadata.generation and
(.status.inventory.entries | type == "array" and length > 0)
' <<<"${kustomization}" >/dev/null && break
attempt=$((attempt + 1))
[ "${attempt}" -lt "${settle_attempts}" ] ||
fail "the ${layer} Kustomization has not applied its current generation with a non-empty inventory"
sleep "${settle_seconds}"
done
if jq -e '.status.conditions // [] | any(.type == "Ready" and .status == "False")' <<<"${kustomization}" >/dev/null; then
printf 'Note: the %s Kustomization applied its revision but an unrelated health check is failing\n' "${layer}"
fi
Expand Down
13 changes: 11 additions & 2 deletions scripts/tests/test-retire-unused-apps.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,8 @@ for name in "${names[@]}"; do
file="${STATE}/${kind}/${namespace}/${name}"
case "${verb}" in
get)
if [ "${kind}" = kustomization ]; then cat "${STATE}/${name}.json" 2>/dev/null || cat "${STATE}/apps.json"; elif [ -f "${file}" ]; then cat "${file}"; fi ;;
if [ "${kind}" = kustomization ] && [ -f "${STATE}/${name}.first.json" ]; then cat "${STATE}/${name}.first.json"; rm -f "${STATE}/${name}.first.json";
elif [ "${kind}" = kustomization ]; then cat "${STATE}/${name}.json" 2>/dev/null || cat "${STATE}/apps.json"; elif [ -f "${file}" ]; then cat "${file}"; fi ;;
delete)
[ -n "${KEEP:-}" ] && [ "${kind}/${name}" = "${KEEP}" ] && continue
rm -f "${file}"
Expand Down Expand Up @@ -127,7 +128,7 @@ run() { # <state> [env...] -> sets status and output
local state="$1"
shift
if output="$(env STATE="${state}" KUBECTL="${work_dir}/kubectl" RETIREMENT_POLL_SECONDS=0 \
RETIREMENT_POLL_ATTEMPTS=3 RETIREMENT_CONFIRM=retire-crossview-headlamp-backstage "$@" \
RETIREMENT_POLL_ATTEMPTS=3 RETIREMENT_SETTLE_SECONDS=0 RETIREMENT_SETTLE_ATTEMPTS=3 RETIREMENT_CONFIRM=retire-crossview-headlamp-backstage "$@" \
bash "${script}" 2>&1)"; then
status=0
else
Expand Down Expand Up @@ -196,6 +197,14 @@ jq '.status.conditions = [{type: "Ready", status: "False", reason: "HealthCheckF
run "${state}"
expect 0 'an unrelated health check is failing' 'an unrelated health failure must not block the retirement'

# A layer read mid-retry (Progressing) is sampled again until it settles.
state="$(seed retrying "${deployed}")"
jq '.status.conditions = [{type: "Ready", status: "False", reason: "HealthCheckFailed"}]' "${state}/apps.json" >"${state}/apps.tmp" && mv "${state}/apps.tmp" "${state}/apps.json"
jq '.status.conditions = [{type: "Ready", status: "Unknown", reason: "Progressing"}]' "${state}/apps.json" >"${state}/apps.first.json"
run "${state}"
expect 0 'an unrelated health check is failing' 'a layer that settles after a retry must not block the retirement'
[ "$(grep -c '^get kustomization apps' "${state}/calls")" -ge 2 ] || fail 'a Progressing layer must be sampled again'

# An object of the same name that Flux apps does not own is never deleted.
state="$(seed foreign "${deployed}")"
put "${state}" "${cnpg}" backstage backstage-db '{"metadata":{"labels":{}}}'
Expand Down
Loading