devantler · Sep 17, 2026 · Sep 17, 2026 · Sep 17, 2026
@@ -52,6 +52,8 @@ jobs:
           shellcheck scripts/report-prod-convergence.sh
           shellcheck scripts/tests/test-report-prod-convergence.sh
           bash scripts/tests/test-report-prod-convergence.sh
+          shellcheck scripts/redeploy-main-when-behind.sh scripts/tests/test-redeploy-main-when-behind.sh
+          bash scripts/tests/test-redeploy-main-when-behind.sh
 
       # The Wedding backup catalogue must be proven complete before the database
       # switches to its dedicated bucket, and continuous after it. The evaluator's

@@ -437,17 +437,26 @@ jobs:
   # resolver merged in #3846). After a merge group lands, prod should already hold its
   # artifact; #2838 describes the interleaving where a heal overwrote it and nothing
   # deployed it again. This job only READS the signed provenance of `:latest` and
-  # annotates the run — it never deploys. Acting on BEHIND/DIVERGED is the next slice.
+  # annotates the run; its last step deploys main on a BEHIND verdict.
   #
   # Warns rather than fails on BEHIND/DIVERGED: a queued merge group deploys before it
   # merges, so a push can briefly see the next group's artifact. UNKNOWN fails, so a
   # broken check never reads as a converged prod.
+  #
+  # It takes the prod-deploy lock so it reads `:latest` only after any deploy or heal
+  # already in flight has finished publishing. Read earlier, a heal that checked out
+  # main before this push landed could still overwrite prod after a CONVERGED reading.
   observe-prod-convergence:
     name: 🧭 Observe Production Convergence
     runs-on: ubuntu-latest
+    concurrency:
+      group: prod-deploy
+      cancel-in-progress: false
+      queue: max
     permissions:
       contents: read # checkout repository history for the ancestry check
       packages: read # read the private manifests artifact and its attestations from GHCR
+      actions: write # list and dispatch CD runs on main when prod is behind
     steps:
       - name: 📑 Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -468,13 +477,33 @@ jobs:
           shellcheck scripts/report-prod-convergence.sh
           shellcheck scripts/tests/test-report-prod-convergence.sh
           bash scripts/tests/test-report-prod-convergence.sh
+          shellcheck scripts/redeploy-main-when-behind.sh scripts/tests/test-redeploy-main-when-behind.sh
+          bash scripts/tests/test-redeploy-main-when-behind.sh
 
       - name: 🔑 Log in to GHCR (read-only)
         env:
           REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: printf '%s' "${REGISTRY_TOKEN}" | docker login ghcr.io --username "${GITHUB_ACTOR}" --password-stdin
 
       - name: 🧭 Report production convergence
+        id: report
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: ./scripts/report-prod-convergence.sh
+
+      # Closes #2838: when prod is BEHIND main, deploy main through the manual CD
+      # workflow, which carries every production gate and queues on the same lock.
+      # DIVERGED does not deploy — it is what a queued merge group that has deployed but
+      # not yet merged looks like, and redeploying main would roll that group back. The
+      # race in #2838 reads BEHIND once the later group merges. A failed report writes no
+      # verdict, so a broken check never deploys.
+      #
+      # This runs inside the lock so the decision is serialized: the script skips when a
+      # CD run for main's tip is already pending, and waits until its own dispatch is
+      # listed before the lock is released to the next observer.
+      - name: 🔁 Redeploy main when prod is behind
+        if: steps.report.outputs.verdict == 'BEHIND'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REDEPLOY_REPOSITORY: ${{ github.repository }}
+        run: ./scripts/redeploy-main-when-behind.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+#
+# Deploys main through the CD workflow after the convergence report reads BEHIND (#3869).
+#
+# The caller holds the prod-deploy lock, so no other observer can decide at the same
+# time. Two rules keep a burst of pushes from stacking redundant deploys:
+#
+#   - skip when a CD run that has not finished already deploys main's current tip; a
+#     later observer then sees that run instead of dispatching its own
+#   - after dispatching, wait until the new run is listed, so the lock is only
+#     released once the next observer can see it
+#
+# A CD run for an older commit does not count: it would leave prod behind the tip.
+#
+#   exit 0  a CD run for main's tip was dispatched, or one was already pending
+#   exit 1  the tip, the run list, the dispatch, or the new run could not be confirmed
+
+set -uo pipefail
+
+repository="${REDEPLOY_REPOSITORY:?REDEPLOY_REPOSITORY is required}"
+main_ref="${REDEPLOY_MAIN_REF:-origin/main}"
+attempts="${REDEPLOY_POLL_ATTEMPTS:-30}"
+interval="${REDEPLOY_POLL_INTERVAL:-2}"
+
+fail() {
+  printf '::error title=Redeploy of main not confirmed::%s\n' "$1"
+  exit 1
+}
+
+# Runs of cd.yaml on main that have not finished, as `<id> <head sha>` lines.
+unfinished_runs() {
+  gh run list --repo "$repository" --workflow cd.yaml --branch main --limit 100 \
+    --json databaseId,headSha,status \
+    --jq '.[] | select(.status != "completed") | "\(.databaseId) \(.headSha)"'
+}
+
+tip="$(git rev-parse --verify --quiet "${main_ref}^{commit}")" || fail "could not resolve ${main_ref}"
+[[ "$tip" =~ ^[0-9a-f]{40}$ ]] || fail "${main_ref} resolved to '${tip}', not a commit"
+
+before="$(unfinished_runs)" || fail "could not list CD runs on main"
+
+if TIP="$tip" awk '$2 == ENVIRON["TIP"] { found = 1 } END { exit !found }' <<<"$before"; then
+  printf '::notice title=Redeploy already pending::a CD run for %s has not finished; not dispatching another\n' "$tip"
+  exit 0
+fi
+
+gh workflow run cd.yaml --repo "$repository" --ref main || fail "could not dispatch cd.yaml on main"
+
+for ((attempt = 1; attempt <= attempts; attempt++)); do
+  if after="$(unfinished_runs)" &&
+    BEFORE="$before" awk '
+      BEGIN { n = split(ENVIRON["BEFORE"], lines, "\n"); for (i = 1; i <= n; i++) { split(lines[i], f, " "); seen[f[1]] = 1 } }
+      NF && !($1 in seen) { found = 1 }
+      END { exit !found }
+    ' <<<"$after"; then
+    printf '::notice title=Redeploying main::dispatched CD because prod is behind %s\n' "$tip"
+    exit 0
+  fi
+  sleep "$interval"
+done
+
+fail "dispatched cd.yaml on main, but no new CD run appeared after ${attempts} checks"
@@ -2,9 +2,11 @@
 #
 # Reports whether production runs what main says, after a push to main (#3848).
 #
-# Observe-only: it resolves the digest `:latest` names, asks
-# scripts/resolve-prod-convergence for a verdict, and annotates the run. It never
-# deploys. The follow-up slice of #3848 acts on the verdict.
+# It resolves the digest `:latest` names, asks scripts/resolve-prod-convergence for
+# a verdict, annotates the run, and writes `verdict=<CONVERGED|BEHIND|DIVERGED>` to
+# $GITHUB_OUTPUT. It never deploys itself; the workflow redeploys main on BEHIND
+# (#3869). No verdict is written when the check fails, so a broken check can never
+# trigger a deploy.
 #
 #   exit 0  CONVERGED, or BEHIND/DIVERGED reported as a warning
 #   exit 1  the verdict could not be established (UNKNOWN, an unreadable digest,
@@ -74,5 +76,7 @@ case "${rc}:${verdict}" in
     ;;
 esac
 
+printf 'verdict=%s\n' "$verdict" >>"${GITHUB_OUTPUT:-/dev/null}"
+
 # shellcheck disable=SC2016 # the backticks are Markdown code spans, not expansions
 printf '### 🧭 Production convergence\n\n`%s` for `%s@%s` against `%s`.\n' "$output" "$subject" "$digest" "$main_ref" >>"$summary"
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+#
+# Pins scripts/redeploy-main-when-behind.sh against stub `gh` and `git` commands.
+#
+# The stub run list answers from a sequence of snapshots: the first call is the list
+# before any dispatch, later calls are the lists seen while waiting for the new run.
+
+set -uo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+script="$repo_root/scripts/redeploy-main-when-behind.sh"
+scratch="$(mktemp -d)"
+trap 'rm -rf "$scratch"' EXIT
+
+failures=0
+assertions=0
+
+[ -x "$script" ] || { printf 'FAIL: %s is not executable\n' "$script"; exit 1; }
+
+tip="$(printf 'a%.0s' {1..40})"
+older="$(printf 'b%.0s' {1..40})"
+
+mkdir -p "$scratch/bin"
+cat >"$scratch/bin/gh" <<'EOF'
+#!/usr/bin/env bash
+printf 'gh %s\n' "$*" >>"$STUB_LOG"
+case "$1 $2" in
+  "run list")
+    [ "${STUB_LIST_RC:-0}" = 0 ] || exit "$STUB_LIST_RC"
+    calls=$(( $(cat "$STUB_CALLS") + 1 )); printf '%s' "$calls" >"$STUB_CALLS"
+    if [ "$calls" = 1 ]; then printf '%b' "$STUB_BEFORE"; else printf '%b' "$STUB_AFTER"; fi
+    ;;
+  "workflow run") exit "${STUB_DISPATCH_RC:-0}" ;;
+  *) exit 99 ;;
+esac
+EOF
+cat >"$scratch/bin/git" <<'EOF'
+#!/usr/bin/env bash
+printf 'git %s\n' "$*" >>"$STUB_LOG"
+[ -n "${STUB_TIP:-}" ] || exit 1
+printf '%s\n' "$STUB_TIP"
+EOF
+chmod +x "$scratch/bin/gh" "$scratch/bin/git"
+
+# <label> <expected-rc> <expected-annotation> <tip> <before> <after> <list-rc> <dispatch-rc> <dispatch-expected: yes|no>
+run_case() {
+  local label="$1" want_rc="$2" want_annotation="$3" want_dispatch="$9" out rc dispatched=no
+  : >"$scratch/log"
+  printf '0' >"$scratch/calls"
+  if out="$(PATH="$scratch/bin:$PATH" STUB_LOG="$scratch/log" STUB_CALLS="$scratch/calls" \
+    STUB_TIP="$4" STUB_BEFORE="$5" STUB_AFTER="$6" STUB_LIST_RC="$7" STUB_DISPATCH_RC="$8" \
+    REDEPLOY_REPOSITORY=devantler-tech/platform REDEPLOY_POLL_ATTEMPTS=3 REDEPLOY_POLL_INTERVAL=0 \
+    "$script" 2>&1)"; then rc=0; else rc=$?; fi
+  grep -Fq 'gh workflow run cd.yaml --repo devantler-tech/platform --ref main' "$scratch/log" && dispatched=yes
+
+  assertions=$((assertions + 1))
+  if [ "$rc" = "$want_rc" ] && [[ "$out" == *"$want_annotation"* ]] && [ "$dispatched" = "$want_dispatch" ]; then
+    printf '  ok   %s (exit %s, dispatched=%s)\n' "$label" "$rc" "$dispatched"
+  else
+    printf '  FAIL %s: expected exit %s, %s, dispatched=%s; got exit %s, dispatched=%s:\n%s\n' \
+      "$label" "$want_rc" "$want_annotation" "$want_dispatch" "$rc" "$dispatched" "$out"
+    failures=$((failures + 1))
+  fi
+}
+
+run_case "no pending run dispatches and waits for the new run" 0 "::notice title=Redeploying main::" \
+  "$tip" "" "101 $tip\n" 0 0 yes
+run_case "a pending run for the tip is not duplicated" 0 "::notice title=Redeploy already pending::" \
+  "$tip" "100 $tip\n" "100 $tip\n" 0 0 no
+run_case "a pending run for an older commit does not count" 0 "::notice title=Redeploying main::" \
+  "$tip" "100 $older\n" "100 $older\n101 $tip\n" 0 0 yes
+run_case "the old run alone never confirms the dispatch" 1 "no new CD run appeared" \
+  "$tip" "100 $older\n" "100 $older\n" 0 0 yes
+run_case "an unresolvable tip fails before listing or dispatching" 1 "could not resolve" \
+  "" "" "" 0 0 no
+run_case "an unreadable run list fails closed without dispatching" 1 "could not list CD runs" \
+  "$tip" "" "" 1 0 no
+run_case "a refused dispatch fails" 1 "could not dispatch" \
+  "$tip" "" "" 0 1 yes
+
+printf '%s assertions, %s failures\n' "$assertions" "$failures"
+[ "$failures" -eq 0 ]
@@ -52,13 +52,14 @@ EOF
 chmod +x "$scratch/bin/docker" "$scratch/bin/go" "$scratch/bin/gh" "$scratch/bin/git"
 export STUB_ATTESTED="" STUB_PRESENT=""
 
-run_case() { # <label> <expected-rc> <expected-annotation> <digest> <docker-rc> <resolver-rc> <resolver-out>
-  local label="$1" want_rc="$2" want_annotation="$3" out rc
+run_case() { # <label> <expected-rc> <expected-annotation> <digest> <docker-rc> <resolver-rc> <resolver-out> <expected-output>
+  local label="$1" want_rc="$2" want_annotation="$3" want_output="$8" out rc got_output
   : >"$scratch/log"
   : >"$scratch/summary"
+  : >"$scratch/output"
   if out="$(cd "$repo_root" && PATH="$scratch/bin:$PATH" STUB_LOG="$scratch/log" \
     STUB_DIGEST="$4" STUB_DOCKER_RC="$5" STUB_RESOLVER_RC="$6" STUB_RESOLVER_OUT="$7" \
-    GITHUB_STEP_SUMMARY="$scratch/summary" "$script" 2>&1)"; then rc=0; else rc=$?; fi
+    GITHUB_STEP_SUMMARY="$scratch/summary" GITHUB_OUTPUT="$scratch/output" "$script" 2>&1)"; then rc=0; else rc=$?; fi
 
   assertions=$((assertions + 1))
   if [ "$rc" = "$want_rc" ] && [[ "$out" == *"$want_annotation"* ]] && [ -s "$scratch/summary" ]; then
@@ -68,6 +69,17 @@ run_case() { # <label> <expected-rc> <expected-annotation> <digest> <docker-rc>
       "$label" "$want_rc" "$want_annotation" "$rc" "$out"
     failures=$((failures + 1))
   fi
+
+  # The workflow redeploys main on this output, so it must be exactly the verdict,
+  # and empty whenever the verdict could not be established.
+  got_output="$(cat "$scratch/output")"
+  assertions=$((assertions + 1))
+  if [ "$got_output" = "$want_output" ]; then
+    printf '  ok   %s writes output [%s]\n' "$label" "$want_output"
+  else
+    printf '  FAIL %s: expected output [%s], got [%s]\n' "$label" "$want_output" "$got_output"
+    failures=$((failures + 1))
+  fi
 }
 
 assert_log() { # <label> <needle>
@@ -81,25 +93,25 @@ assert_log() { # <label> <needle>
 }
 
 run_case "converged is a notice" 0 "::notice title=Production converged::CONVERGED" \
-  "$good_digest" 0 0 "CONVERGED attested commit is main"
+  "$good_digest" 0 0 "CONVERGED attested commit is main" "verdict=CONVERGED"
 assert_log "resolver receives the digest :latest names" "--digest $good_digest"
 assert_log "digest is read from :latest" "ghcr.io/devantler-tech/platform/manifests:latest"
 
 run_case "behind warns without failing" 0 "::warning title=Production is not on main::BEHIND" \
-  "$good_digest" 0 1 "BEHIND deploy input changed since"
+  "$good_digest" 0 1 "BEHIND deploy input changed since" "verdict=BEHIND"
 run_case "diverged warns without failing" 0 "::warning title=Production is not on main::DIVERGED" \
-  "$good_digest" 0 1 "DIVERGED not reachable from main"
+  "$good_digest" 0 1 "DIVERGED not reachable from main" "verdict=DIVERGED"
 
 run_case "unknown fails" 1 "::error title=Production convergence unknown::" \
-  "$good_digest" 0 2 "UNKNOWN attestation could not be verified"
+  "$good_digest" 0 2 "UNKNOWN attestation could not be verified" ""
 run_case "unreadable digest fails" 1 "could not read the digest" \
-  "$good_digest" 1 0 "CONVERGED never asked"
+  "$good_digest" 1 0 "CONVERGED never asked" ""
 run_case "malformed digest fails" 1 "not a sha256 digest" \
-  "sha256:short" 0 0 "CONVERGED never asked"
+  "sha256:short" 0 0 "CONVERGED never asked" ""
 run_case "exit 0 without CONVERGED fails" 1 "resolver exited 0" \
-  "$good_digest" 0 0 "BEHIND contradicts exit 0"
+  "$good_digest" 0 0 "BEHIND contradicts exit 0" ""
 run_case "exit 1 claiming CONVERGED fails" 1 "resolver exited 1" \
-  "$good_digest" 0 1 "CONVERGED contradicts exit 1"
+  "$good_digest" 0 1 "CONVERGED contradicts exit 1" ""
 
 assert_no_log() { # <label> <needle>
   assertions=$((assertions + 1))
@@ -116,14 +128,45 @@ present_commit="$(printf 'c%.0s' {1..40})"
 STUB_ATTESTED="$missing_commit $present_commit" STUB_PRESENT="$present_commit"
 export STUB_ATTESTED STUB_PRESENT
 run_case "attested commits are prepared before resolving" 0 "CONVERGED" \
-  "$good_digest" 0 0 "CONVERGED healed artifact"
+  "$good_digest" 0 0 "CONVERGED healed artifact" "verdict=CONVERGED"
 assert_log "a missing attested commit is fetched" "git fetch --quiet --no-tags origin $missing_commit"
 assert_no_log "a present attested commit is not fetched" "origin $present_commit"
 
 STUB_ATTESTED="not-a-commit"
 run_case "a malformed attested value is never fetched" 0 "CONVERGED" \
-  "$good_digest" 0 0 "CONVERGED ignored"
+  "$good_digest" 0 0 "CONVERGED ignored" "verdict=CONVERGED"
 assert_no_log "malformed value reaches no fetch" "git fetch"
 
+
+# The workflow acts on the verdict written above. Only BEHIND may deploy: DIVERGED is
+# what a merge group that deployed before merging looks like, and a failed check writes
+# no verdict at all. The decision runs inside the observer, which holds the prod-deploy
+# lock, so no two observers can decide at once and none reads `:latest` while a heal is
+# still about to overwrite it.
+workflow="$repo_root/.github/workflows/validate-main.yaml"
+redeploy='.jobs.observe-prod-convergence.steps[] | select(.run == "./scripts/redeploy-main-when-behind.sh")'
+assert_workflow() { # <label> <yq-expression> <expected>
+  local got
+  assertions=$((assertions + 1))
+  if got="$(yq -r "$2" "$workflow")" && [ "$got" = "$3" ]; then
+    printf '  ok   %s\n' "$1"
+  else
+    printf '  FAIL %s: expected [%s], got [%s]\n' "$1" "$3" "${got:-}"
+    failures=$((failures + 1))
+  fi
+}
+assert_workflow "the report step carries the id the redeploy reads" \
+  '.jobs.observe-prod-convergence.steps[] | select(.run == "./scripts/report-prod-convergence.sh") | .id' 'report'
+assert_workflow "the observer waits on the prod-deploy lock" \
+  '.jobs.observe-prod-convergence.concurrency.group' 'prod-deploy'
+assert_workflow "the observer never cancels a queued deploy" \
+  '.jobs.observe-prod-convergence.concurrency.cancel-in-progress' 'false'
+assert_workflow "only BEHIND redeploys" \
+  "$redeploy | .if" "steps.report.outputs.verdict == 'BEHIND'"
+assert_workflow "the redeploy is the locked job's last step, after the report" \
+  '.jobs.observe-prod-convergence.steps[-1].run' './scripts/redeploy-main-when-behind.sh'
+assert_workflow "no separate job decides outside the lock" \
+  '[.jobs[] | .steps[]? | select(.run // "" | test("workflow run cd.yaml|redeploy-main-when-behind.sh$"))] | length' '1'
+
 printf '%s assertions, %s failures\n' "$assertions" "$failures"
 [ "$failures" -eq 0 ]