feat(pm-dashboard-backup): per-step run ledger in .last-run.json (#1854 rule 2)

nixos/home/pm-dashboard-backup-record.test.sh

@@ -60,6 +60,12 @@ case "$cmd" in
           exit 42
         fi
         ;;
+      drive:*)
+        if [[ "${RCLONE_FAIL:-}" == "DRIVE" ]]; then
+          echo "stub rclone: forced Drive failure" >&2
+          exit 42
+        fi
+        ;;
     esac
     cp "$src" "$store/$(key "$dest")"
     ;;
@@ -80,18 +86,31 @@ STUB
 chmod +x "$BIN/bws" "$BIN/journalctl" "$BIN/rclone"
 
 run_scenario() {
-  local name="$1" fail_mode="$2"
+  local name="$1" fail_mode="$2" dump_mode="${3:-absent}"
   local home="$TMP_ROOT/$name/home"
   mkdir -p "$home/backups/dashboard" "$home/.local/state/pm-aggregator" \
     "$home/atlas/meta-quota-export"
+  local dump_script="$home/no-such-dump-script.sh"
+  if [[ "$dump_mode" == "ok" ]]; then
+    dump_script="$home/dump-ok.sh"
+    { printf '#!%s\n' "$BASH"
+      printf '%s\n' 'mkdir -p "$HOME/backups/db-dumps/$1"'
+      printf '%s\n' 'printf "{}\n" > "$HOME/backups/db-dumps/$1/dump.sql"'
+    } >"$dump_script"
+    chmod +x "$dump_script"
+  elif [[ "$dump_mode" == "fail" ]]; then
+    dump_script="$home/dump-fail.sh"
+    { printf '#!%s\n' "$BASH"; printf '%s\n' 'exit 3'; } >"$dump_script"
+    chmod +x "$dump_script"
+  fi
   for f in actions.json comments.json fleet-feed.json quota-current.json registry.json quota-gatherer-state.json quota-gatherer-heartbeat.json; do
     printf '{}\n' >"$home/.local/state/pm-aggregator/$f"
   done
   printf '{}\n' >"$home/atlas/meta-quota-export/latest.json"
   printf '{}\n' >"$home/atlas/meta-quota-export/history.jsonl"
   HOME="$home" PATH="$BIN:$PATH" BWS_ACCESS_TOKEN="<redacted>" \
     RCLONE_FAIL="$fail_mode" RCLONE_STUB_DIR="$TMP_ROOT/$name/rclone-store" \
-    PM_DB_DUMP_SCRIPT="$home/no-such-dump-script.sh" \
+    PM_DB_DUMP_SCRIPT="$dump_script" \
     bash "$SCRIPT" 2026-10-28 >"$TMP_ROOT/$name.log" 2>&1
   RC=$?
   SCEN_HOME="$home"
@@ -120,6 +139,14 @@ assert rec["failing_step"] is None, rec
 assert rec["archive_sha256"] == sha, (rec, sha)
 assert rec["destinations"] == {"local": "ok", "r2": "ok", "drive": "ok"}, rec
 assert rec["started_at"] and rec["finished_at"], rec
+assert rec["steps"] == [
+    {"step": "validate-date", "outcome": "ok"},
+    {"step": "credentials", "outcome": "ok"},
+    {"step": "db-dump", "outcome": "not-applicable"},
+    {"step": "local-archive", "outcome": "ok"},
+    {"step": "r2-upload", "outcome": "ok"},
+    {"step": "drive-upload", "outcome": "ok"},
+], rec
 PYEOF
 
 cat >"$TMP_ROOT/assert-r2fail.py" <<'PYEOF'
@@ -131,6 +158,14 @@ assert rec["destinations"]["local"] == "ok", rec
 assert rec["destinations"]["r2"] == "failed", rec
 assert rec["destinations"]["drive"] == "not-attempted", rec
 assert rec["archive_sha256"], rec
+assert rec["steps"] == [
+    {"step": "validate-date", "outcome": "ok"},
+    {"step": "credentials", "outcome": "ok"},
+    {"step": "db-dump", "outcome": "not-applicable"},
+    {"step": "local-archive", "outcome": "ok"},
+    {"step": "r2-upload", "outcome": "failed"},
+    {"step": "drive-upload", "outcome": "not-attempted"},
+], rec
 PYEOF
 
 run_scenario happy ""
@@ -177,10 +212,66 @@ rec = json.load(open(os.environ["RECORD"]))
 assert rec["result"] == "failed", rec
 assert rec["exit_code"] == 143, rec
 assert rec["failing_step"] == "credentials", rec
+steps = {s["step"]: s["outcome"] for s in rec["steps"]}
+assert steps["validate-date"] == "ok", rec
+assert steps["credentials"] == "failed", rec
+assert steps["db-dump"] == "not-attempted", rec
+assert steps["local-archive"] == "not-attempted", rec
+assert steps["r2-upload"] == "not-attempted", rec
+assert steps["drive-upload"] == "not-attempted", rec
 SIGASSERT
   check_record sigterm "$TMP_ROOT/assert-sigterm.py"
 fi
 
+cat >"$TMP_ROOT/assert-drivefail.py" <<'PYEOF'
+import json, os
+rec = json.load(open(os.environ["RECORD"]))
+assert rec["result"] == "failed", rec
+assert rec["failing_step"] == "drive-upload", rec
+assert rec["destinations"] == {"local": "ok", "r2": "ok", "drive": "failed"}, rec
+steps = {s["step"]: s["outcome"] for s in rec["steps"]}
+assert steps["r2-upload"] == "ok", rec
+assert steps["drive-upload"] == "failed", rec
+PYEOF
+
+run_scenario drivefail DRIVE
+if [[ "$RC" -ne 0 ]]; then echo "ok: drivefail: script exited non-zero ($RC)"
+else fail "drivefail: expected non-zero exit, got 0"; fi
+check_record drivefail "$TMP_ROOT/assert-drivefail.py"
+
+cat >"$TMP_ROOT/assert-dumpfail.py" <<'PYEOF'
+import json, os
+rec = json.load(open(os.environ["RECORD"]))
+assert rec["result"] == "failed", rec
+assert rec["failing_step"] == "db-dump", rec
+assert rec["destinations"] == {"local": "not-attempted", "r2": "not-attempted", "drive": "not-attempted"}, rec
+steps = {s["step"]: s["outcome"] for s in rec["steps"]}
+assert steps["validate-date"] == "ok", rec
+assert steps["credentials"] == "ok", rec
+assert steps["db-dump"] == "failed", rec
+assert steps["local-archive"] == "not-attempted", rec
+PYEOF
+
+run_scenario dumpfail "" fail
+if [[ "$RC" -ne 0 ]]; then echo "ok: dumpfail: script exited non-zero ($RC)"
+else fail "dumpfail: expected non-zero exit, got 0"; fi
+check_record dumpfail "$TMP_ROOT/assert-dumpfail.py"
+
+cat >"$TMP_ROOT/assert-dumpok.py" <<'PYEOF'
+import json, os
+rec = json.load(open(os.environ["RECORD"]))
+assert rec["result"] == "complete", rec
+assert rec["failing_step"] is None, rec
+steps = {s["step"]: s["outcome"] for s in rec["steps"]}
+assert steps["db-dump"] == "ok", rec
+assert all(o == "ok" for o in steps.values()), rec
+PYEOF
+
+run_scenario dumpok "" ok
+if [[ "$RC" -eq 0 ]]; then echo "ok: dumpok: script exited 0"
+else fail "dumpok: expected exit 0, got $RC; log: $(cat "$TMP_ROOT/dumpok.log")"; fi
+check_record dumpok "$TMP_ROOT/assert-dumpok.py"
+
 if (( failures > 0 )); then
   echo "pm-dashboard-backup record tests: $failures failure(s)" >&2
   exit 1

nixos/home/pm-dashboard-backup.sh

@@ -69,6 +69,22 @@ DEST_LOCAL="not-attempted"
 DEST_R2="not-attempted"
 DEST_DRIVE="not-attempted"
 ARC_SHA=""
+# --- run ledger (#1854 rule 2: one failure ledger per run) ------------------
+# Every material step's outcome lands in ONE ordered ledger, serialized
+# into the run record as "steps" by write_record below. Outcomes:
+# not-attempted -> in-progress -> ok | failed, plus not-applicable for
+# a step whose precondition is absent (the optional DB dump, #1514).
+# The record's overall result is DERIVED from this ledger in
+# write_record: a run whose required steps are not all ok (db-dump may
+# be ok or not-applicable) is recorded failed even on a zero exit, so a
+# failed or interrupted step can never be recorded as complete. A step
+# is non-fatal only when named here as tolerated -- db-dump's
+# not-applicable; an undeclared swallow is a defect.
+LEDGER_ORDER="validate-date credentials db-dump local-archive r2-upload drive-upload"
+declare -A LEDGER=()
+for _s in $LEDGER_ORDER; do LEDGER[$_s]="not-attempted"; done
+ledger_set() { LEDGER["$1"]="$2"; }
+ledger_begin() { STEP="$1"; ledger_set "$1" "in-progress"; }
 # Suppressed until DATE validation passes: an invalid DATE must be
 # rejected before ANY side effect (#989, pinned by the date suite),
 # so a rejected run leaves no record and no backup root.
@@ -81,10 +97,34 @@ write_record() {
     [ "$d_r2" = "in-progress" ] && d_r2="failed"
     [ "$d_drive" = "in-progress" ] && d_drive="failed"
   fi
+  # Ledger derivation (#1854 rule 2): an in-progress step on a failed
+  # exit is the failing step; on ANY exit, the first required step that
+  # is not ok (db-dump: not ok and not not-applicable) makes the run
+  # failed and names failing_step -- the exit code alone never decides.
+  local steps_arg="" _name _outcome failing=""
+  for _name in $LEDGER_ORDER; do
+    _outcome="${LEDGER[$_name]}"
+    if [ "$rc" -ne 0 ] && [ "$_outcome" = "in-progress" ]; then _outcome="failed"; fi
+    steps_arg="${steps_arg:+$steps_arg,}$_name:$_outcome"
+    if [ -z "$failing" ]; then
+      if [ "$_outcome" = "failed" ]; then
+        failing="$_name"
+      elif [ "$rc" -eq 0 ] && [ "$_name" = "db-dump" ]; then
+        # Zero exit only: an untouched db-dump on a failed exit is not
+        # the failure -- the in-progress step (converted above) or the
+        # $STEP fallback names it, never an unattempted step.
+        { [ "$_outcome" != "ok" ] && [ "$_outcome" != "not-applicable" ]; } && failing="$_name"
+      elif [ "$rc" -eq 0 ] && [ "$_outcome" != "ok" ]; then
+        failing="$_name"
+      fi
+    fi
+  done
+  if [ -n "$failing" ]; then result="failed"; fi
   command -v python3 >/dev/null 2>&1 || return 0
   mkdir -p "$ROOT" 2>/dev/null || return 0
   RECORD_PATH="$RECORD" RECORD_RESULT="$result" RECORD_RC="$rc" \
-  RECORD_STEP="$STEP" RECORD_DATE="$DATE" RECORD_STARTED="$CREATED_AT" \
+  RECORD_STEP="${failing:-$STEP}" RECORD_STEPS="$steps_arg" \
+  RECORD_DATE="$DATE" RECORD_STARTED="$CREATED_AT" \
   RECORD_SHA="$ARC_SHA" RECORD_LOCAL="$d_local" RECORD_R2="$d_r2" \
   RECORD_DRIVE="$d_drive" \
   python3 - <<'PYEOF' 2>/dev/null || true
@@ -98,6 +138,10 @@ rec = {
     "result": os.environ["RECORD_RESULT"],
     "exit_code": int(os.environ["RECORD_RC"]),
     "failing_step": os.environ["RECORD_STEP"] if os.environ["RECORD_RESULT"] == "failed" else None,
+    "steps": [
+        {"step": pair.split(":", 1)[0], "outcome": pair.split(":", 1)[1]}
+        for pair in os.environ["RECORD_STEPS"].split(",") if pair
+    ],
     "archive_sha256": os.environ["RECORD_SHA"] or None,
     "destinations": {
         "local": os.environ["RECORD_LOCAL"],
@@ -146,6 +190,7 @@ if [[ ! "$DATE" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] \
   || [ "$(date -d "$DATE" +%F 2>/dev/null)" != "$DATE" ]; then
   fail "invalid DATE '$DATE' - expected a real calendar date as YYYY-MM-DD"
 fi
+ledger_set validate-date ok
 
 mkdir -p "$ROOT"
 RECORD_SKIP=0
@@ -181,7 +226,7 @@ if ! command -v bws >/dev/null 2>&1 && [ -x "$HOME/tools/bin/bws" ]; then
 fi
 command -v bws >/dev/null 2>&1 || fail "bws not in PATH"
 
-STEP="credentials"
+ledger_begin credentials
 # --- credentials from Bitwarden (names only in code/logs) ------------------
 if [ -z "${BWS_ACCESS_TOKEN:-}" ] && [ -f "$HOME/.config/bws/access-token" ]; then
   BWS_ACCESS_TOKEN="$(cat "$HOME/.config/bws/access-token")"
@@ -216,6 +261,7 @@ export RCLONE_CONFIG_R2BAK_REGION="auto"
 # bucket check - otherwise it falls back to CreateBucket and the upload
 # fails with AccessDenied (found 2026-10-28 via the openclaw-backup job).
 export RCLONE_CONFIG_R2BAK_NO_CHECK_BUCKET="true"
+ledger_set credentials ok
 
 
 # --- optional pm-stack DB dumps (code#1514) --------------------------------
@@ -228,16 +274,20 @@ DUMP_SCRIPT="${PM_DB_DUMP_SCRIPT:-$HOME/tools/pm-stack-db-backup.sh}"
 DUMP_DIR="$HOME/backups/db-dumps/$DATE"
 DUMP_RAN=0
 if [ -x "$DUMP_SCRIPT" ]; then
-  STEP="db-dump"
+  ledger_begin db-dump
   log "running DB dump step: $DUMP_SCRIPT"
   "$DUMP_SCRIPT" "$DATE" || fail "DB dump step failed"
   DUMP_RAN=1
+  ledger_set db-dump ok
 else
+  # Tolerated absence (named in the ledger contract above): the dump
+  # job has not landed, so the step is not-applicable, never a failure.
+  ledger_set db-dump not-applicable
   log "no DB dump script at $DUMP_SCRIPT (code#1514 pending) - dashboard state only"
 fi
 
 # === leg 1: the local archive (dashboard-backup.sh, code#1459) ============
-STEP="local-archive"
+ledger_begin local-archive
 DEST_LOCAL="in-progress"
 STAGE="$(mktemp -d)"
 mkdir -p "$STAGE/aggregator" "$STAGE/meta-quota-export" "$STAGE/journals" "$OUTDIR"
@@ -410,6 +460,7 @@ rm -rf "$VDIR"
 
 ARC_SHA="$(sha256sum "$ARCHIVE" | cut -d' ' -f1)"
 DEST_LOCAL="ok"
+ledger_set local-archive ok
 log "local OK $ARCHIVE ($(stat -c%s "$ARCHIVE") bytes, sha256 $ARC_SHA)"
 
 # --- prune local day dirs older than 30 days (only this script's own;
@@ -449,21 +500,23 @@ prune_remote() { # $1 rclone dir holding <date> subdirs, $2 label
   done
 }
 
-STEP="r2-upload"
+ledger_begin r2-upload
 DEST_R2="in-progress"
 R2_DEST="r2bak:$R2_BUCKET/$R2_PREFIX/$DATE/dashboard-$DATE.tar.zst"
 rclone copyto "$ARCHIVE" "$R2_DEST" || fail "R2 upload failed"
 verify_remote "$R2_DEST" "R2"
 prune_remote "r2bak:$R2_BUCKET/$R2_PREFIX" "R2"
 DEST_R2="ok"
+ledger_set r2-upload ok
 
-STEP="drive-upload"
+ledger_begin drive-upload
 DEST_DRIVE="in-progress"
 DRIVE_DEST="drive:$DRIVE_DIR/$DATE/dashboard-$DATE.tar.zst"
 rclone copyto "$ARCHIVE" "$DRIVE_DEST" || fail "Drive upload failed"
 verify_remote "$DRIVE_DEST" "Drive"
 prune_remote "drive:$DRIVE_DIR" "Drive"
 DEST_DRIVE="ok"
+ledger_set drive-upload ok
 STEP="complete"
 
 # --- journal cursor: advance ONLY now that all three legs have succeeded ---