diff --git a/scripts/retire-unused-apps.sh b/scripts/retire-unused-apps.sh index 269cff6092..a7af8440f0 100755 --- a/scripts/retire-unused-apps.sh +++ b/scripts/retire-unused-apps.sh @@ -47,6 +47,12 @@ kubectl_bin="${KUBECTL:-kubectl}" poll_seconds="${RETIREMENT_POLL_SECONDS:-10}" poll_attempts="${RETIREMENT_POLL_ATTEMPTS:-90}" readonly kubectl_bin poll_seconds poll_attempts +# A layer retrying an unrelated failing health check passes through a +# Progressing phase every retry interval, so its state is sampled until it +# settles rather than judged at one instant. +settle_seconds="${RETIREMENT_SETTLE_SECONDS:-15}" +settle_attempts="${RETIREMENT_SETTLE_ATTEMPTS:-40}" +readonly settle_seconds settle_attempts readonly releases=(crossview/crossview headlamp/headlamp) readonly clusters=(backstage/backstage-db) readonly namespaces=(crossview headlamp backstage) @@ -140,15 +146,21 @@ command -v jq >/dev/null || unknown 'jq is required' # this retirement. Any other not-Ready reason, such as a failed build or # apply, leaves the inventory unproven and is refused. for layer in infrastructure-controllers infrastructure apps; do - kustomization="$(k -n flux-system get kustomization "${layer}" -o json)" || - unknown "cannot read the ${layer} Kustomization" - jq -e ' - (.status.conditions // [] | any(.type == "Ready" and - (.status == "True" or (.status == "False" and .reason == "HealthCheckFailed")))) and - .status.observedGeneration == .metadata.generation and - (.status.inventory.entries | type == "array" and length > 0) - ' <<<"${kustomization}" >/dev/null || - fail "the ${layer} Kustomization has not applied its current generation with a non-empty inventory" + attempt=0 + while :; do + kustomization="$(k -n flux-system get kustomization "${layer}" -o json)" || + unknown "cannot read the ${layer} Kustomization" + jq -e ' + (.status.conditions // [] | any(.type == "Ready" and + (.status == "True" or (.status == "False" and .reason == "HealthCheckFailed")))) and + .status.observedGeneration == .metadata.generation and + (.status.inventory.entries | type == "array" and length > 0) + ' <<<"${kustomization}" >/dev/null && break + attempt=$((attempt + 1)) + [ "${attempt}" -lt "${settle_attempts}" ] || + fail "the ${layer} Kustomization has not applied its current generation with a non-empty inventory" + sleep "${settle_seconds}" + done if jq -e '.status.conditions // [] | any(.type == "Ready" and .status == "False")' <<<"${kustomization}" >/dev/null; then printf 'Note: the %s Kustomization applied its revision but an unrelated health check is failing\n' "${layer}" fi diff --git a/scripts/tests/test-retire-unused-apps.sh b/scripts/tests/test-retire-unused-apps.sh index f85748de41..396f0ec481 100755 --- a/scripts/tests/test-retire-unused-apps.sh +++ b/scripts/tests/test-retire-unused-apps.sh @@ -58,7 +58,8 @@ for name in "${names[@]}"; do file="${STATE}/${kind}/${namespace}/${name}" case "${verb}" in get) - if [ "${kind}" = kustomization ]; then cat "${STATE}/${name}.json" 2>/dev/null || cat "${STATE}/apps.json"; elif [ -f "${file}" ]; then cat "${file}"; fi ;; + if [ "${kind}" = kustomization ] && [ -f "${STATE}/${name}.first.json" ]; then cat "${STATE}/${name}.first.json"; rm -f "${STATE}/${name}.first.json"; + elif [ "${kind}" = kustomization ]; then cat "${STATE}/${name}.json" 2>/dev/null || cat "${STATE}/apps.json"; elif [ -f "${file}" ]; then cat "${file}"; fi ;; delete) [ -n "${KEEP:-}" ] && [ "${kind}/${name}" = "${KEEP}" ] && continue rm -f "${file}" @@ -127,7 +128,7 @@ run() { # [env...] -> sets status and output local state="$1" shift if output="$(env STATE="${state}" KUBECTL="${work_dir}/kubectl" RETIREMENT_POLL_SECONDS=0 \ - RETIREMENT_POLL_ATTEMPTS=3 RETIREMENT_CONFIRM=retire-crossview-headlamp-backstage "$@" \ + RETIREMENT_POLL_ATTEMPTS=3 RETIREMENT_SETTLE_SECONDS=0 RETIREMENT_SETTLE_ATTEMPTS=3 RETIREMENT_CONFIRM=retire-crossview-headlamp-backstage "$@" \ bash "${script}" 2>&1)"; then status=0 else @@ -196,6 +197,14 @@ jq '.status.conditions = [{type: "Ready", status: "False", reason: "HealthCheckF run "${state}" expect 0 'an unrelated health check is failing' 'an unrelated health failure must not block the retirement' +# A layer read mid-retry (Progressing) is sampled again until it settles. +state="$(seed retrying "${deployed}")" +jq '.status.conditions = [{type: "Ready", status: "False", reason: "HealthCheckFailed"}]' "${state}/apps.json" >"${state}/apps.tmp" && mv "${state}/apps.tmp" "${state}/apps.json" +jq '.status.conditions = [{type: "Ready", status: "Unknown", reason: "Progressing"}]' "${state}/apps.json" >"${state}/apps.first.json" +run "${state}" +expect 0 'an unrelated health check is failing' 'a layer that settles after a retry must not block the retirement' +[ "$(grep -c '^get kustomization apps' "${state}/calls")" -ge 2 ] || fail 'a Progressing layer must be sampled again' + # An object of the same name that Flux apps does not own is never deleted. state="$(seed foreign "${deployed}")" put "${state}" "${cnpg}" backstage backstage-db '{"metadata":{"labels":{}}}'