From b720f885dbb3e3cfa4c0317475da1b15fc84cbc6 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 8 Jul 2026 06:41:04 -0500 Subject: [PATCH 1/3] feat(#613): relocate canary-rollout engine + bootstrap into .github (workflow disabled) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-B of epic #613: move the release-promotion / repo-bootstrap tooling into petry-projects/.github so all org tooling + standards share one canonical home (the #596 principle, extended). .github already owns the standards (standards/rulesets/, lib/ring-pins.sh, the canonical apply-repo-settings.sh); this brings the orchestrators alongside them. Relocated from .github-private (byte-identical): - scripts/canary-rollout.sh, scripts/lib/canary-rollout.sh (+ tests/canary_rollout.bats) - standards/canary-rings.json (ring registry — now co-located with lib/ring-pins.sh) - docs/{bootstrap/new-repo-validation,release/versioning,initiatives/canary-rollout-adr}.md - .github/workflows/canary-rollout.yml — SHIPS DISABLED (schedule commented out, workflow_dispatch only) so this PR lands with zero operational effect. The live cutover (re-enable schedule here + delete the copy from .github-private in the same window, to avoid a double-run race on protected channel tags) is PR-C. Reconcile fixes required by the move (bootstrap now sits next to .github's canonical 346L apply-repo-settings.sh, not .github-private's 129L subset): - bootstrap-new-repo.sh: bridge dry-run onto DRY_RUN (the var the canonical settings script reads), keeping DEV_LEAD_DRY_RUN for backward-compat. Without this a real `DRY_RUN=true` bootstrap would issue LIVE settings PATCHes — a latent bug the relocation exposes. - test_bootstrap_new_repo.bats: the real-sub-script tests now exercise the canonical settings script, which requires GH_TOKEN and fetches current state even under dry-run. Provide a dummy token (gh is stubbed) and return a non-"{}" repo object from the gh read-stub (canonical treats "{}" as a failed fetch). Assertion wording updated to the canonical script's dry-run messages. Auth prerequisite (already applied, org-admin): petry-projects/.github added to the selected-repos list for the RELEASE_MANAGER_APP_KEY secret + RELEASE_MANAGER_APP_CLIENT_ID variable, so the workflow can mint the release-manager App token once enabled. The App (id 4193127) is installed org-wide, so no per-repo install gap. Validation: canary_rollout.bats 88/88, test_bootstrap_new_repo.bats 13/13, shellcheck clean, canary-rings.json valid, workflow YAML parses (dispatch-only). Refs #613 #596 Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/canary-rollout.yml | 254 ++++++ docs/bootstrap/new-repo-validation.md | 125 +++ docs/initiatives/canary-rollout-adr.md | 98 +++ docs/release/versioning.md | 197 +++++ scripts/bootstrap-new-repo.sh | 433 ++++++++++ scripts/canary-rollout.sh | 1090 ++++++++++++++++++++++++ scripts/lib/canary-rollout.sh | 249 ++++++ standards/canary-rings.json | 471 ++++++++++ tests/canary_rollout.bats | 1063 +++++++++++++++++++++++ tests/test_bootstrap_new_repo.bats | 266 ++++++ 10 files changed, 4246 insertions(+) create mode 100644 .github/workflows/canary-rollout.yml create mode 100644 docs/bootstrap/new-repo-validation.md create mode 100644 docs/initiatives/canary-rollout-adr.md create mode 100644 docs/release/versioning.md create mode 100755 scripts/bootstrap-new-repo.sh create mode 100755 scripts/canary-rollout.sh create mode 100644 scripts/lib/canary-rollout.sh create mode 100644 standards/canary-rings.json create mode 100644 tests/canary_rollout.bats create mode 100644 tests/test_bootstrap_new_repo.bats diff --git a/.github/workflows/canary-rollout.yml b/.github/workflows/canary-rollout.yml new file mode 100644 index 000000000..1951fb619 --- /dev/null +++ b/.github/workflows/canary-rollout.yml @@ -0,0 +1,254 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Canary Rollout — ring-staged, health-gated promotion of agent releases by moving +# channel tags only. Initiative #495 · issues #501 (promotion) / #502 (rollback). +# Gate standard: .github#548 (graduated dwell/sample, robust baseline, per-candidate +# window, ring0 sample waiver, failure triage) — see scripts/canary-rollout.sh. +# +# The ONLY mutation this performs is moving a channel tag (`git tag -f` + push) and +# recording a GitHub Deployment for traceability. It never writes consumer files. +# +# Auth: the mover credential is a GitHub App installation token minted at runtime via +# actions/create-github-app-token from RELEASE_MANAGER_APP_CLIENT_ID (variable) + RELEASE_MANAGER_APP_KEY (secret) (scoped, +# rotatable, auditable) — NOT a raw PAT and NOT github.token. The App carries +# Contents:write (tag moves on the host repos), Deployments:write, Issues:write (the +# per-agent blocker issues in sync-issues; the fleet-status table goes to the job summary, +# not an issue), Actions:read (the gate reads workflow-run history across every tier repo) +# and Metadata:read. Minting +# is a HARD requirement: if the token cannot be minted the job fails — it must not +# silently degrade to github.token, which cannot move protected `.github` tags (#868). +# +# - schedule (every 4h): the ordering is autocut → promote-all → sync-issues. +# FIRST `autocut` (#1069) — the FRONT END that closes the last manual seam: gated by the +# org variable CANARY_AUTO_CUT (the single kill-switch), it cuts a new /vX.Y.Z + +# moves `next` whenever a registered reusable's blob on its host main HEAD differs from the +# current `next` candidate, seeding the soak/promote pipeline with no manual cut-release.sh. +# A freshly cut candidate SOAKS the same tick (dwell=0 < floor). It is best-effort — a +# failure here never fails the run. THEN the gated fleet auto-promote (#1045 part b). When the org +# variable CANARY_AUTO_PROMOTE == 'true' the timer runs `promote-all` — the gate +# advances each PROMOTE-ready agent one ring per run and leaves BLOCKED/REGRESSION +# agents untouched (human approval is OVERRIDE-ONLY, never a required gate). When the +# variable is unset/false it runs read-only `evaluate-all` — evaluates EVERY agent in +# canary-rings.json (fleet-wide, #1025 P1) and emits each one's per-version gate + +# health report (#502 observability). The arm variable is the single kill-switch. +# After promoting, the timer also runs `sync-issues`: it upserts one tracked issue per +# BLOCKED agent (failing-run evidence attached; REGRESSION → needs-human; auto-closed on +# clear) and renders the whole-fleet status table into the run's job summary (auto-triage). +# - workflow_dispatch: `evaluate` | `evaluate-all` | `promote` | `promote-all` | +# `rollback` | `sync-issues`, human-gated (promote/promote-all/sync-issues default to --dry-run). +# ───────────────────────────────────────────────────────────────────────────── +name: Canary Rollout + +on: + # schedule: # DISABLED until epic #613 step-4 cutover — see PR (relocated from .github-private, must not double-run) + # - cron: "0 */4 * * *" # every 4 hours — evaluate only (RE-ENABLE at cutover) + workflow_dispatch: + inputs: + command: + description: "autocut (gated cut+seed) | evaluate | evaluate-all (read-only) | promote | promote-all (gated tag move) | rollback | sync-issues | drift (read-only registry audit)" + type: choice + options: [autocut, evaluate, evaluate-all, promote, promote-all, rollback, sync-issues, drift] + default: evaluate + agent: + description: "agent to act on" + type: choice + options: [dev-lead, agent-shield, auto-rebase, dependency-audit, dependabot-automerge, dependabot-rebase, pr-review-mention] + default: dev-lead + override: + description: "promote: advance even if the gate is not PROMOTE (records a warning)" + type: boolean + default: false + dry_run: + description: "promote/rollback: show the tag move but do not push" + type: boolean + default: true + ring: + description: "rollback: which ring channel to move (e.g. ring1, stable)" + type: string + default: "" + to: + description: "rollback: target immutable release (e.g. v1.3.0)" + type: string + default: "" + +permissions: + contents: read + +concurrency: + # Scheduled runs evaluate the whole fleet (read-only), so they group under 'fleet' + # and never serialize behind a per-agent promote/rollback dispatch (#1025 P1). + group: canary-rollout-${{ github.event.inputs.agent || 'fleet' }} + cancel-in-progress: false + +jobs: + canary: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + # Mint a scoped GitHub App installation token (Contents/Deployments:write, + # Actions/Metadata:read). owner-wide so the gate can read run history across all + # tier repos and move tags on the host repos. This step FAILS the job if the App + # id/key secrets are missing or the token cannot be minted — no silent fallback. + - name: Mint GitHub App token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + # client-id (not the deprecated app-id, #1025 P3). This is the App's client ID, + # a distinct value from the numeric app id — set vars.RELEASE_MANAGER_APP_CLIENT_ID. + client-id: ${{ vars.RELEASE_MANAGER_APP_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_MANAGER_APP_KEY }} + # owner-wide (NOT a fixed repositories: list). The fleet gate reads run history + # across every tier repo, including the stable-tier '*' member (all consumers), + # which cannot be enumerated — so scoping to a repo list would break the gate (#1025 P3). + owner: ${{ github.repository_owner }} + + - name: Verify App token was minted (hard requirement) + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + if [ -z "${APP_TOKEN}" ]; then + echo "::error::Canary rollout requires a GitHub App token (RELEASE_MANAGER_APP_CLIENT_ID (variable) + RELEASE_MANAGER_APP_KEY (secret)). Refusing to fall back to github.token, which cannot move protected .github channel tags (#868)." + exit 1 + fi + + - name: Checkout (with tags) + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + fetch-depth: 0 + fetch-tags: true + # Mover identity: the App token so pushing protected channel tags is + # authorized under the release-channel-tags ruleset bypass (#868). + token: ${{ steps.app-token.outputs.token }} + + # FRONT END (#1069): cut a new candidate + move `next` whenever a registered reusable's + # blob on its host main HEAD differs from the current `next` candidate. Runs FIRST on the + # scheduled tick (before promote-all) so a freshly cut candidate begins soaking the same + # tick. Gated by CANARY_AUTO_CUT (the single kill-switch) — the script no-ops unless it is + # 'true'. Best-effort: a cut/detection hiccup must never fail the scheduled promotion run. + - name: Autocut new candidates (front end) + if: github.event_name == 'schedule' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + CANARY_AUTO_CUT: ${{ vars.CANARY_AUTO_CUT }} + run: | + set -uo pipefail + bash scripts/canary-rollout.sh autocut || echo "::warning::autocut failed (non-fatal)" + + - name: Run canary-rollout + id: run + env: + # autocut kill-switch — honored when a dispatch runs `autocut` (the script no-ops unless 'true'). + CANARY_AUTO_CUT: ${{ vars.CANARY_AUTO_CUT }} + # On the 4h timer the arm variable decides: CANARY_AUTO_PROMOTE=='true' runs the + # gated fleet auto-promote (promote-all, #1045 part b); anything else runs read-only + # fleet-wide evaluate-all (#1025 P1). Either way EVERY registry agent is covered, not + # just dev-lead. The gate — not the schedule — decides which agents actually advance. + CMD: ${{ github.event_name == 'schedule' && (vars.CANARY_AUTO_PROMOTE == 'true' && 'promote-all' || 'evaluate-all') || github.event.inputs.command }} + AGENT: ${{ github.event.inputs.agent || 'dev-lead' }} + OVERRIDE: ${{ github.event.inputs.override }} + # Scheduled promote-all is a REAL move (DRY_RUN=false); dispatch honours the input + # (which defaults to true — dispatch promote/promote-all are dry unless set false). + DRY_RUN: ${{ github.event_name == 'schedule' && 'false' || github.event.inputs.dry_run }} + RING: ${{ github.event.inputs.ring }} + TO: ${{ github.event.inputs.to }} + # cut-release.sh (cross-repo tag moves on petry-projects/.github) and the + # gate's cross-repo run-history reads both authenticate with the App token. + GH_TOKEN: ${{ steps.app-token.outputs.token }} + # One TSV line per real promotion (agent, ring, sha, owning-repo) so the deployment + # step records a deployment for EVERY move in a promote-all run, on the right host. + CANARY_PROMOTIONS_LOG: ${{ runner.temp }}/promotions.tsv + run: | + set -euo pipefail + args=() + case "$CMD" in + autocut) # gated cut+seed of new candidates (#1069) + args=(autocut) + [ "$DRY_RUN" != "false" ] && args+=(--dry-run) # default-safe: dry unless explicitly false + ;; + evaluate) args=(evaluate "$AGENT") ;; + evaluate-all) args=(evaluate-all) ;; # fleet-wide read-only (the 4h timer, #1025 P1) + promote) + args=(promote "$AGENT") + [ "$OVERRIDE" = "true" ] && args+=(--override) + [ "$DRY_RUN" != "false" ] && args+=(--dry-run) # default-safe: dry unless explicitly false + ;; + promote-all) # gated fleet auto-promote (the SCHEDULED arm, #1045b) + args=(promote-all) + [ "$OVERRIDE" = "true" ] && args+=(--override) + [ "$DRY_RUN" != "false" ] && args+=(--dry-run) # default-safe: dry unless explicitly false + ;; + rollback) + if [ -z "$RING" ] || [ -z "$TO" ]; then echo "::error::rollback needs ring + to"; exit 2; fi + args=(rollback "$AGENT" "$RING" --to "$TO") + [ "$DRY_RUN" != "false" ] && args+=(--dry-run) + ;; + sync-issues) # upsert blocker issues + dashboard (auto-triage) + args=(sync-issues) + [ "$DRY_RUN" != "false" ] && args+=(--dry-run) # default-safe: dry unless explicitly false + ;; + drift) args=(drift) ;; # read-only registry/host reusable audit (#1082) + *) echo "::error::unknown command: $CMD"; exit 2 ;; + esac + echo "::notice::canary-rollout ${args[*]}" + bash scripts/canary-rollout.sh "${args[@]}" + + # Record a GitHub Deployment per real (non-dry-run) promotion — traceability for the + # tag moves (#502). Reads the promotions log so a promote-all run records EVERY move, + # each on the repo that owns the moved commit (a cross-repo agent's SHA lives on its + # host, not GITHUB_REPOSITORY — creating it there 422s "No ref found", #1059). + # always() + best-effort per line: a traceability hiccup must never mark a SUCCESSFUL + # tag move as a failed run. + - name: Record deployment for promotions + if: always() && hashFiles(format('{0}/promotions.tsv', runner.temp)) != '' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + CANARY_PROMOTIONS_LOG: ${{ runner.temp }}/promotions.tsv + run: | + set -uo pipefail + while IFS=$'\t' read -r agent ring sha repo; do + [ -z "${agent:-}" ] && continue + echo "recording deployment: ${agent}/${ring} -> ${sha:0:12} on ${repo}" + gh api -X POST "repos/${repo}/deployments" --input - < ${sha}" + } + JSON + done < "${CANARY_PROMOTIONS_LOG}" + + # Auto-triage: turn the gate's BLOCKED verdicts into tracked work + a fleet readout. + # From the post-promotion fleet state: (1) upsert ONE idempotent issue per BLOCKED agent + # (failing-run evidence pre-attached; REGRESSION → needs-human; auto-closed on clear), and + # (2) render the whole-fleet status table into THIS run's job summary (a read-only + # snapshot — no dashboard issue, no Issues:write needed for it). Best-effort + always(): + # a GitHub-side hiccup must never fail the promotion run. The blocker ISSUES need the App + # to carry Issues:write — if absent that part warns and continues; the summary still renders. + - name: Sync blocker issues + fleet-status summary + if: always() && github.event_name == 'schedule' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + ISSUE_REPO: ${{ github.repository }} + run: | + set -uo pipefail + bash scripts/canary-rollout.sh sync-issues || echo "::warning::issue sync failed (non-fatal)" + + # Reusable-registry drift audit (#1082): the registry (.agents{}) is the MANUAL source + # of truth for what this pipeline manages — a *-reusable.yml added/renamed on a host but + # never registered ships with ZERO staged rollout (no cut/soak/gate/dashboard) until a + # human registers it. This read-only step scans every host's .github/workflows for + # *-reusable.yml, diffs against the registry, and surfaces (a) unregistered reusables and + # (b) stale registry entries whose file is gone — as ::warning:: annotations + a job-summary + # table, within one scheduled cycle. Report-only (never registers anything). Best-effort + + # always(): a drift-scan hiccup must never fail the promotion run. + - name: Reusable-registry drift audit + if: always() && github.event_name == 'schedule' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -uo pipefail + bash scripts/canary-rollout.sh drift || echo "::warning::drift audit failed (non-fatal)" diff --git a/docs/bootstrap/new-repo-validation.md b/docs/bootstrap/new-repo-validation.md new file mode 100644 index 000000000..11e4808e3 --- /dev/null +++ b/docs/bootstrap/new-repo-validation.md @@ -0,0 +1,125 @@ +# New-repo onboarding — primary path & end-to-end DRY_RUN validation + +_Epic #964 · Phase 3 (#970). Validation recorded 2026-06-28._ + +This is the recorded, end-to-end validation that the documented onboarding path — +**"Use this template" → `scripts/bootstrap-new-repo.sh` → ring confirmation** — +produces a fully org-compliant repo, exercised under `DRY_RUN` with no drift. + +## Primary onboarding path (one-click + one-command) + +1. **Use this template.** Create the repo from `petry-projects/repo-template` + (`seed-repo-template.sh` keeps that template in sync with `petry-projects/.github` + `standards/`). This seeds day-0 only: the thin-caller workflow stubs pinned to + their published `@/stable` channel tags plus the root/baseline files + (CODEOWNERS, AGENTS.md/CLAUDE.md pointers, LICENSE, SECURITY.md, `.gitignore`, + `BOOTSTRAP.md`). Then follow `BOOTSTRAP.md` for the two per-stack picks + (Dependabot stack, `ci.yml`). +2. **Run the bootstrap.** `bash scripts/bootstrap-new-repo.sh owner/new-repo` + brings repo settings, security/GHAS + secret-scanning push protection, the two + sanctioned rulesets (with required checks + bypass actors), the standard label + set, and CODEOWNERS verification to org compliance by orchestrating the existing + `apply-*` scripts. It reimplements no policy. +3. **Confirm the release ring.** The ring is an auditable choice (default + `stable`, record-only). A non-stable ring is registered in both central files + and the caller stub is repinned to the matching `@/` channel tag. + +Preview any run with `DRY_RUN=true` first — it prints the full intended state and +makes zero write API calls. + +> **Template seeds day-0 only.** Ongoing standards updates to an existing repo +> still flow through the PR-based sync (`deploy-standard-workflows.sh` / +> `aw-standards-sync.sh`), not by re-templating. The legacy manual runbook is the +> **existing-repo fallback**, not the primary path. + +## Recorded DRY_RUN walkthrough + +Captured with `gh` stubbed to a no-op reader (`echo '{}'`) so the real sub-scripts +(`apply-repo-settings.sh`, `apply-rulesets.sh`) run end-to-end without network or +writes — the same seam the bats suite uses. Executable form: +`tests/test_bootstrap_new_repo.bats` → _"e2e DRY_RUN: covers the whole +intended-state surface with no write calls (#970 AC #2/#3)"_. + +### Default ring (`stable`) + +``` +$ DRY_RUN=true GITHUB_ACTOR=octocat bash scripts/bootstrap-new-repo.sh petry-projects/acme-service +[bootstrap] repo=petry-projects/acme-service dry_run=true ring=dev-lead/stable +[bootstrap] (1/5) release ring confirmation (dev-lead/stable) + [ring-audit] repo=petry-projects/acme-service agent=dev-lead ring=stable operator=octocat at=... decision=recorded + ring=stable — record-only; no central-file change required (covered by the '*' catch-all) +[bootstrap] (2/5) repo settings + security/GHAS + push protection +[dry-run] would patch security_and_analysis on petry-projects/acme-service: secret_scanning secret_scanning_push_protection secret_scanning_ai_detection secret_scanning_non_provider_patterns dependabot_security_updates +[dry-run] would disable auto-trigger for apps 1236702 347564 on petry-projects/acme-service +[bootstrap] (3/5) sanctioned fleet rulesets (pr-quality + code-quality) +[apply-rulesets] repo=petry-projects/acme-service dir= dry_run=true + create ruleset 'code-quality' on petry-projects/acme-service + [dry-run] POST repos/petry-projects/acme-service/rulesets + create ruleset 'pr-quality' on petry-projects/acme-service + [dry-run] POST repos/petry-projects/acme-service/rulesets +[apply-rulesets] done (2 ruleset(s)) +[bootstrap] (4/5) standard label set + [dry-run] would ensure label 'needs-human-review' on petry-projects/acme-service + [dry-run] would ensure label 'ack-test-deletion' on petry-projects/acme-service + [dry-run] would ensure label 'dependencies' on petry-projects/acme-service + [dry-run] would ensure label 'automerge' on petry-projects/acme-service +[bootstrap] (5/5) verify CODEOWNERS team (@petry-projects/org-leads first owner) + [dry-run] would verify @petry-projects/org-leads is the first CODEOWNERS owner on petry-projects/acme-service + +[bootstrap] PASS — petry-projects/acme-service bootstrapped to org compliance (dry-run) +``` + +### Non-stable ring (`--ring ring1`) + +The ring step additionally registers the repo in both central files and repins the +caller stub — and asserts no drift before writing: + +``` +[bootstrap] (1/5) release ring confirmation (dev-lead/ring1) + [ring-audit] repo=petry-projects/acme-service agent=dev-lead ring=ring1 operator=octocat at=... decision=registered + [dry-run] would add petry-projects/acme-service to dev-lead 'ring1' members in canary-rings.json + [ring] would register petry-projects/acme-service in petry-projects/.github:scripts/lib/ring-pins.sh for channel dev-lead/ring1 (cross-repo PR, keeps central files in sync) + [ring] would repin petry-projects/acme-service caller stub .github/workflows/dev-lead.yml to @dev-lead/ring1 + ring consistency OK — petry-projects/acme-service sits in 'ring1' across both central files and its stub pins @dev-lead/ring1 +``` + +## Intended state covered (no drift) + +| Surface | Source of truth | Verified in walkthrough | +|---|---|---| +| Repo settings + security/GHAS | `apply-repo-settings.sh` | `security_and_analysis` patch intent | +| Secret-scanning push protection | `lib/push-protection.sh` | `secret_scanning_push_protection` in the patch set | +| Check-suite auto-trigger (Claude/CodeRabbit) | `apply-repo-settings.sh` | `would disable auto-trigger for apps 1236702 347564` | +| `pr-quality` ruleset + bypass actors | `petry-projects/.github` → `standards/rulesets/pr-quality.json` | created; bypass = OrganizationAdmin + Integration (`bypass_mode: always`) | +| `code-quality` ruleset + required checks + bypass | `petry-projects/.github` → `standards/rulesets/code-quality.json` | created; required checks SonarCloud, CodeQL, agent-shield, dependency-audit; same bypass actors | +| Required status checks | carried in the ruleset JSONs | not wired by bootstrap — live in `code-quality.json` | +| Standard labels | `bootstrap-new-repo.sh` `BOOTSTRAP_LABELS` | needs-human-review, ack-test-deletion, dependencies, automerge | +| CODEOWNERS team | new repo's `.github/CODEOWNERS` | first owner verified = `@petry-projects/org-leads` | +| Recorded ring | `standards/canary-rings.json` (+ cross-repo `ring-pins.sh`) | audited; stable record-only / non-stable registered with no drift | +| No drift | — | `DRY_RUN` emits **zero** write API calls | + +## AC #1 — cross-repo checklist cutover (petry-projects/.github) + +The two onboarding checklists live in **`petry-projects/.github`**, not in this +repo, so they are landed via the standards cross-repo PR pattern +(`STANDARDS_REPO=petry-projects/.github`, as `aw-standards-sync.sh` does), not on +this branch. The cutover content to apply: + +- **`standards/github-settings.md` — "Applying to a New Repository":** lead with + **"Use this template" + `scripts/bootstrap-new-repo.sh`** as the primary, + one-click + one-command path (it applies repo settings, both rulesets with bypass + actors + required checks, labels, CODEOWNERS, and secret-scanning push + protection). Demote the existing 9-step manual settings runbook to an + **existing-repo fallback**. +- **`standards/ci-standards.md` — "Applying to a New Repository":** same cutover — + template + bootstrap first; the 13-step CI checklist becomes the existing-repo + fallback. State explicitly that the template seeds **day-0 only** and ongoing + CI/standards updates continue to flow through the PR-based sync + (`deploy-standard-workflows.sh`), which remains the ongoing-sync path for + existing repos and is out of scope to replace. + +## See also + +- `scripts/bootstrap-new-repo.sh` — the orchestrator (issues #967, #968) +- `scripts/seed-repo-template.sh` — day-0 template seeding + the generated `BOOTSTRAP.md` (#966) +- `tests/test_bootstrap_new_repo.bats`, `tests/test_seed_repo_template.bats` — the executable validation diff --git a/docs/initiatives/canary-rollout-adr.md b/docs/initiatives/canary-rollout-adr.md new file mode 100644 index 000000000..a59599aa4 --- /dev/null +++ b/docs/initiatives/canary-rollout-adr.md @@ -0,0 +1,98 @@ +# ADR: Ring-staged, health-gated canary rollout of agent releases + +**Status:** Accepted — records the *implemented* model (initiative [#495](https://github.com/petry-projects/.github-private/issues/495), issues #499–#502, #868). Supersedes the aspirational "rings" sketch and the "Rollout status / next phase" caveat in `petry-projects/.github` → `standards/ci-standards.md`. +**Author:** dev-lead / Claude Code +**Date:** 2026-07-01 +**Scope (confirmed):** The delivery path for the versioned agents (`dev-lead`, `pr-review`, and the cross-repo `feature-ideation`) — how a new immutable release is promoted from the host repo out to the whole fleet, and how it is rolled back. Records decisions; does not re-derive the initiative analysis (see [`agentic-release-strategy.md`](./agentic-release-strategy.md)). +**Constraints (confirmed):** GitHub-native only — moving **channel tags** on first-party reusables, no external infrastructure. +The machine-readable ring map is `standards/canary-rings.json`; the gate logic is `scripts/lib/canary-rollout.sh`; the orchestrator is `scripts/canary-rollout.sh` driven by `.github/workflows/canary-rollout.yml`. This ADR mirrors those artifacts — it does not restate their contents as a second source of truth. + +--- + +## 1. Context + +The release-strategy initiative ([#495](https://github.com/petry-projects/.github-private/issues/495)) replaced a floating-`@main` deploy — where a merge was instantly live across the agents' own self-review duty *and* every consumer repo, with no canary and no rollback — +with a **versioned-release model**: immutable `/vX.Y.Z` releases plus **moving channel tags** (`/next`, `/ring0`, `/ring1`, `/stable`) that callers pin to. See [`agentic-release-strategy.md`](./agentic-release-strategy.md) for the full analysis and [`docs/release/versioning.md`](../release/versioning.md) for the tag mechanics. + +Once #499–#502 and #868 landed, the concentric rings, the soak gate, and the promotion/rollback workflow were **built and machine-backed** — no longer aspirational. This ADR records the decisions those artifacts encode so the public-repo standard can be promoted to match (and link here) instead of carrying a "not yet built" note. + +## 2. Decision 1 — the ring map is host-relative and data-driven + +[`standards/canary-rings.json`](../../standards/canary-rings.json) is the single source of truth read by [`scripts/canary-rollout.sh`](../../scripts/canary-rollout.sh). Ring membership is **host-relative**, resolved from three member tokens rather than hard-coded repo lists: + +| Channel | Order | Members (token) | Resolves to (for `dev-lead`, host = `.github-private`) | +|---|---|---|---| +| `next` | 0 | `$host` — the repo that owns the reusable | `petry-projects/.github-private` | +| `ring0` | 1 | `$org_infra` — `org_infra_repos` **minus** the host | `petry-projects/.github` | +| `ring1` | 2 | explicit list | `petry-projects/TalkTerm`, `petry-projects/bmad-bgreat-suite` | +| `stable`| 3 | `*` — every other consumer not named in an earlier ring | the rest of the fleet | + +Rationale for host-relative tokens: an agent's reusable can live in either org-infra repo (`dev-lead`/`pr-review` are hosted here; `feature-ideation`'s reusable is hosted in the public `.github`). +`next` always means "the host dogfoods first," and `ring0` always means "the *other* org-infra repo next" — so `next` + `ring0` together always cover both org-infra repos, whichever hosts the agent. Hard-coding `ring0 = .github + .github-private` would be wrong for a host that is already sitting in `next`. + +## 3. Decision 2 — soak / health gate rules + +The promotion gate is a pure, unit-tested decision core ([`scripts/lib/canary-rollout.sh`](../../scripts/lib/canary-rollout.sh); tests [`tests/canary_rollout.bats`](../../tests/canary_rollout.bats)). A candidate `vX.Y.Z` advances to the next ring only once the rings **already on** the candidate have, over a trailing **7-day** window (`SOAK_WINDOW_DAYS = 7`): + +- **Volume:** `healthy_runs >= min_healthy_runs`, where `min_healthy_runs = ceil(baseline_runs / 7)` — roughly one trailing day's worth of the prior version's run volume. +- **Quality:** candidate failure-rate `<=` baseline failure-rate (compared in integer per-mille to stay in pure-bash arithmetic). + +Two rules make the gate safe under low traffic: + +- **No synthetic floor / "we still wait."** A ring with no healthy candidate volume simply never advances — the candidate parks at the frontier rather than being waved through on an arbitrary minimum. `baseline_runs == 0` yields `min_healthy_runs == 0`, but the `healthy_runs > 0` guard still blocks promotion on zero real volume. +- **Quality is checked first.** A failure-rate breach is evaluated before the volume threshold, so a regression is never masked by "not enough runs yet." + +### Gate states + +| State | Meaning | Follow-up | +|---|---|---| +| `PROMOTE` | Quality holds **and** the volume threshold is met | Advance the next ring (innermost-out). | +| `SOAKING` | Quality holds but not enough healthy candidate runs yet | Wait — no action. | +| `INVESTIGATE` | Candidate failure-rate exceeds baseline (possible regression) | A **human** classifies it: pre-existing → log + continue; genuine regression → fix + cut a new `vX.Y.Z`, which **RESETs** the rollout. | +| `RESET` | Roll all rings back to the last known-good | A **human/override outcome** of a confirmed regression — *not* an automatic gate state. | + +### Cadence + +[`.github/workflows/canary-rollout.yml`](../../.github/workflows/canary-rollout.yml) runs the gate on a **4-hourly** schedule (`cron: "0 */4 * * *"`) in **read-only `evaluate` mode only** — it emits the per-version gate + health report (#502 observability) and **never** promotes on the timer. Promotion and rollback are `workflow_dispatch`-only and human-gated. + +## 4. Decision 3 — promotion is a single gated channel-tag move; rollback moves the tag back + +The **channel tags are the rollout state.** There is no separate state store: + +- **Promotion** = advancing one ring by **moving one channel tag** to the candidate `vX.Y.Z`, innermost-out along the ordered list `next → ring0 → ring1 → stable` (`next_channel_in_order`). The **only** mutation the orchestrator performs is `git tag -f` + push; it never writes consumer files. +- **Rollback** = moving a channel tag **back** to a prior immutable release (`rollback --to vX.Y.Z`). Because immutable `vX.Y.Z` tags never move, they are the audit trail and the rollback targets. + +The mover credential is **`GH_PAT_WORKFLOWS`** (the workflow falls back to `GITHUB_TOKEN` when the PAT is unset, but only the PAT satisfies the `release-channel-tags` ruleset bypass scoped to this workflow's identity (#868)) — promotion and rollback effectively require the PAT; agents running only as `GITHUB_TOKEN` cannot push protected release tags. +This is the sanctioned, documented exception to the SHA-pin standard for **first-party** channel tags (see [AGENTS.md → "Release channel tags & the mutable-ref exception"](../../AGENTS.md)); compliance audits must not flag `@/` pins on first-party callers as unpinned. + +## 5. Decision 4 — inner-ring repos pin to their ring channel, not `stable` + +For soak to observe a candidate, a ring's repos must actually *run* it. So stub-pin guidance is: + +| Repo class | Pins to | +|---|---| +| Host repo (dogfood) | `/next` | +| Other org-infra repo (`ring0`) | `/ring0` | +| `ring1` repos (`TalkTerm`, `bmad-bgreat-suite`) | `/ring1` | +| Everything else (`stable`) | `/stable` | + +An inner-ring repo pinned to `/stable` would only ever see a version *after* it reached the outermost ring, contributing **zero** soak signal to the gate that is supposed to protect the fleet — defeating the ring model. +The public-repo audit's `check_centralized_workflow_stubs` expected-pin map must therefore expect `/ring0|ring1` for inner-ring repos, matching the ring map in [`standards/canary-rings.json`](../../standards/canary-rings.json). + +## 6. Consequences + +- Blast radius shrinks from "whole fleet, instantly" to "one ring at a time," and rollback is a single pointer flip. +- The public `petry-projects/.github` → `standards/ci-standards.md` can now describe the **implemented** model and drop its "Rollout status / next phase" caveat, linking here for the decision rationale. + **That edit lands in the public repo** and is out of scope for a `.github-private` dev-lead run — this ADR is its in-repo counterpart and prerequisite. + **A separate pull request against `petry-projects/.github` is still required to complete that public-repo update; issue #869 should be closed by that PR, not by this one.** +- The ring map, gate thresholds, and cadence are data/code ([`standards/canary-rings.json`](../../standards/canary-rings.json), [`scripts/lib/canary-rollout.sh`](../../scripts/lib/canary-rollout.sh), [`.github/workflows/canary-rollout.yml`](../../.github/workflows/canary-rollout.yml)) — changing them is a reviewed diff, not a doc edit. + +## 7. References + +- Initiative [#495](https://github.com/petry-projects/.github-private/issues/495) — safe release strategy for agentic workflows. +- [`docs/initiatives/agentic-release-strategy.md`](./agentic-release-strategy.md) — the initiative analysis (rings, channels, health-gated promotion); §5.1 mutable-ref exception. +- [`docs/initiatives/agentic-release-strategy-orchestration.md`](./agentic-release-strategy-orchestration.md) — how the child issues (#496–#503) were delivered; the dependency DAG. +- [`docs/release/versioning.md`](../release/versioning.md) — immutable-release vs. moving-channel-tag mechanics; cross-repo reusables. +- [`standards/canary-rings.json`](../../standards/canary-rings.json) — machine-readable ring map (source of truth). +- [`scripts/lib/canary-rollout.sh`](../../scripts/lib/canary-rollout.sh) / [`scripts/canary-rollout.sh`](../../scripts/canary-rollout.sh) / [`.github/workflows/canary-rollout.yml`](../../.github/workflows/canary-rollout.yml) — gate core, orchestrator, and scheduler. +- Decisions context: [petry-projects/.github#516](https://github.com/petry-projects/.github/issues/516). diff --git a/docs/release/versioning.md b/docs/release/versioning.md new file mode 100644 index 000000000..1e6ce2b97 --- /dev/null +++ b/docs/release/versioning.md @@ -0,0 +1,197 @@ +# Agent versioning & release channels + +Status: **active** (Phase 1 of the [Safe Release Strategy](../initiatives/agentic-release-strategy.md) +initiative, epic #495). Implements issue #496. + +This defines how the **dev-lead** and **pr-review** agents are versioned and how callers select a +version. It is the foundation the rest of the initiative (rings, promotion, rollback) builds on. + +## What is versioned + +A "release" of an agent is the reusable workflow **plus the scripts it executes** — they move +together, so a version is a single repo commit that contains a known-good combination: + +| Agent | Reusable workflow | Key scripts (non-exhaustive) | +|---|---|---| +| `pr-review` | `.github/workflows/pr-review.yml` | `scripts/review-one-pr.sh`, `scripts/review-batch.sh`, `scripts/post-pr-review.sh`, `scripts/engine.sh`, `scripts/lib/*` | +| `dev-lead` | `.github/workflows/dev-lead-reusable.yml` | `scripts/dev-lead-*.sh`, `scripts/engine.sh`, `scripts/lib/*` | +| `feature-ideation` | `petry-projects/.github` → `.github/workflows/feature-ideation-reusable.yml` (**cross-repo**) | reusable-owned (lives in the public repo; this repo holds only the thin caller `.github/workflows/feature-ideation.yml`) | +| `agent-shield`, `auto-rebase`, `dependency-audit`, `dependabot-automerge`, `dependabot-rebase`, `pr-review-mention` (the six #482 reusables) | `petry-projects/.github` → `.github/workflows/-reusable.yml` (**cross-repo**) | reusable-owned (live in the public repo; this repo holds only the thin caller stubs) | + +`pr-review` and `dev-lead` both live in this repo, so a release tag points at a whole-repo commit; the +tag *name* scopes it to one agent so the two can be released and promoted independently. + +The rest are the exception: their reusables live in **`petry-projects/.github`** (this repo holds only +the thin caller stubs), so a "release" is a commit on that public repo, and their release/channel tags +must be cut **against `petry-projects/.github`, not this repo's `origin`**. This is `feature-ideation` +plus the six reusables #482 migrated to channel tags — see [Cross-repo reusables](#cross-repo-reusables) +below. + +## Tag scheme + +Two kinds of tag, per agent: + +| Kind | Format | Mutable? | Purpose | +|---|---|---|---| +| **Immutable release** | `/vMAJOR.MINOR.PATCH` | No (annotated, never moved) | Audit trail + rollback target | +| **Channel** | `/` | Yes (moved on promotion) | What callers pin to | + +Channels (Phase 1 defines `stable`; Phase 2 adds `next` and per-ring channels): + +- `/stable` — the production channel (blue). Callers in production pin here. +- `/next` — the candidate channel (green). **Live for `dev-lead`** (#499). +- `/ring0`, `/ring1`, … — per-ring channels for staged promotion. **Live for `dev-lead`** (#499/#500). + +Examples: `pr-review/v1.0.0`, `pr-review/stable`, `dev-lead/v1.0.0`, `dev-lead/stable`, +`dev-lead/next`, `dev-lead/ring0`, `dev-lead/ring1`. + +### Ring channels (live for `dev-lead`) + +The candidate (`next`) and ring channels are real moving tags, created alongside `stable`. A caller +pins **once** to its ring channel and is never edited again; a release flows outward ring-by-ring as +each ring's tag is advanced. + +Ring membership (canonical model — see [#500](https://github.com/petry-projects/.github-private/issues/500)). +`next` is **host-relative**: it always resolves to the repo that *hosts* the reusable, and `ring0` +covers the other org-infra repo, so `next` + `ring0` always span `.github` + `.github-private`, +partitioned by which one is the host: + +| Ring | Channel | Members (general) | Role | +|---|---|---|---| +| **next** | `/next` | the repo that **hosts** the reusable | canary / dogfood at the source | +| **ring0** | `/ring0` | `.github` **and** `.github-private` (host already in `next`) | org-infra self-host | +| **ring1** | `/ring1` | `TalkTerm`, `bmad-bgreat-suite` | named low-traffic consumers | +| **stable** | `/stable` | everything else | full-fleet production | + +Concretely for **`dev-lead`** (hosted in `.github-private`): `next` = `.github-private`, +`ring0` = `.github`, `ring1` = `{TalkTerm, bmad-bgreat-suite}`, `stable` = the rest. +**Production self-review/dev duty stays pinned to `stable` even within ring 0** — the agent validating +fixes is never the unvalidated candidate (the circular-dependency fix #500 targets). The intended +machine-readable source of truth is `standards/canary-rings.json`, consumed by the promotion +automation (#501). + +A staged rollout advances the channels in order — `next` → `ring0` → `ring1` → `stable` — validating +at each step (see [`runbook.md` §2c](./runbook.md#2c-staged-canary--ring-rollout)). All four channels +may sit at the same commit between releases; they diverge while a candidate is being staged. The +`check_dev_lead_stub` compliance audit accepts any `dev-lead/{stable,next,ring}` channel pin (it +rejects `@main` and frozen `@vX.Y.Z`/`@` — callers must pin a *moving* channel). pr-review still +uses `stable` only; its ring channels are pending under #499. + +`feature-ideation` uses the full per-ring channel set — `{next, ring0, ring1, stable}` — so it can be +promoted through the same canary → ring → stable model. Its tags follow the identical name scheme +(`feature-ideation/vX.Y.Z`, `feature-ideation/next`, `feature-ideation/ring0`, +`feature-ideation/ring1`, `feature-ideation/stable`) but are cut against `petry-projects/.github` +(see [Cross-repo reusables](#cross-repo-reusables)). + +The six **#482 reusables** — `agent-shield`, `auto-rebase`, `dependency-audit`, `dependabot-automerge`, +`dependabot-rebase`, `pr-review-mention` — use the same `{next, ring0, ring1, stable}` set (#870), +replacing the single-hop `/stable`-only migration #482 cut by hand. They are `.github`-hosted, so +like `feature-ideation` their tags are cut against `petry-projects/.github`. Their ring membership is an +**explicit org-owner assignment** (#870, matching #866), and it does **not** follow the host-relative +`$host`/`$org_infra` default above — `next` is `.github-private` (the dogfood lab) even though the +reusables are hosted in `.github`, which sits in `ring0`: + +| Channel | Member repo(s) | +|---|---| +| `next` | `.github-private` | +| `ring0` | `.github` (the host) | +| `ring1` | `TalkTerm`, `bmad-bgreat-suite` | +| `stable` | `markets`, `broodly`, `ContentTwin`, `google-app-scripts` (full-fleet production) | + +### Semantic versioning + +- **MAJOR** — breaking change to the caller contract (workflow inputs/secrets, required permissions, + the merge-gate behavior a consumer relies on). +- **MINOR** — backward-compatible capability (new safety check, new optional input). +- **PATCH** — backward-compatible fix (bug fix, prompt tweak, resilience hardening). + +## How callers select a version (no per-caller churn) + +GitHub does **not** allow an expression in a `uses:` ref, so version selection is **a moving channel +tag**, not a variable. Each caller pins **once** to a channel and is never edited again; promotion is a +central move of the channel tag (see the initiative doc §5.1): + +```yaml +# A consumer / self-host caller pins once to the per-agent channel: +uses: petry-projects/.github-private/.github/workflows/pr-review.yml@pr-review/stable +uses: petry-projects/.github-private/.github/workflows/dev-lead-reusable.yml@dev-lead/stable +``` + +> Shorthand: the initiative doc writes `@stable`; the concrete tag is the **per-agent** channel +> (`pr-review/stable`, `dev-lead/stable`) so the agents promote independently. + +## The v1.0.0 baseline + +The first release was cut from the production `main` at the time of issue #496: + +- `pr-review/v1.0.0`, `dev-lead/v1.0.0` — immutable baselines. +- `pr-review/stable`, `dev-lead/stable` — channels pointing at v1.0.0. + +`v1.0.0` is **"what is in production today,"** the current rollback floor — *not* a certified-perfect +version. The health-gated promotion added in Phase 2 (#501) is what makes *future* promotions to +`stable` genuinely validated before they become production. + +## Cross-repo reusables + +Everything above assumes the agent's reusable workflow lives **in this repo**, so a release tag is a +whole-repo commit here and tags are cut against this repo's `origin`. That holds for `pr-review` and +`dev-lead`. It does **not** hold for the cross-repo reusables — `feature-ideation` and the six #482 +reusables (`agent-shield`, `auto-rebase`, `dependency-audit`, `dependabot-automerge`, +`dependabot-rebase`, `pr-review-mention`): + +- Their reusables live in **`petry-projects/.github`** (`.github/workflows/-reusable.yml`); this + repo carries only the thin caller stubs `.github/workflows/.yml`. +- Therefore a release is a commit on **`petry-projects/.github`**, and the `/vX.Y.Z` immutable + + `/` tags must be cut **against that repo**, not this repo's `origin`. +- `scripts/cut-release.sh` recognizes each of these agents (`valid_agent`) and, for a cross-repo agent, + **resolves the ref and creates/moves the tags against `petry-projects/.github` via `gh api`** (#872, + wired). `--dry-run` previews the immutable + channel tag names; a live `--push` creates the annotated + `/vX.Y.Z` release object and force-moves the `/` tag on `.github`, and needs + `GH_TOKEN` with `contents:write` on that repo. (`origin/main` in `--ref` means `.github`'s `main` — the + `origin/` prefix is stripped for the cross-repo API lookup.) +- Because these channel tags live on `petry-projects/.github`, the protective ruleset that bounds them + (the mutable-ref exception) is created **there**, not on this repo — see + [`AGENTS.md`](../../AGENTS.md) "Release channel tags & the mutable-ref exception". + +> **Why not `standards/canary-rings.json` yet?** The machine-readable ring source of truth consumed by +> the promotion automation (`scripts/canary-rollout.sh`, #501) carries only the agents whose channel +> tags that automation can move. `cut-release.sh` now publishes cross-repo tags via `gh api` (#872), but +> `canary-rollout.sh` does not yet use that cross-repo path, so `feature-ideation` and the six above stay +> **absent** from `canary-rings.json` until the rollout automation is taught the same `gh api` move — +> adding them sooner would advertise a promotion the automation cannot perform. (Manual/scripted cuts via +> `cut-release.sh` work today.) + +### The six #482 reusables (ring assignment) + +PR #482 migrated these six to moving channel tags **single-hop** — it cut only `/stable` (+ a manual +off-convention `/v2.0.0`) and put all consumers on `stable`, with no `next`/`ring*`. #870 brings +them under the full `{next, ring0, ring1, stable}` model with the explicit, org-owner ring assignment in +[Ring channels](#ring-channels-live-for-dev-lead) above. Note this assignment is **explicit, not +host-relative**: `next` is `.github-private` even though the reusables are hosted in `.github` (which +sits in `ring0`). Re-cutting their releases on a sane incremental-semver basis (reconciling the manual +`/v2.0.0` tags) and cutting the `next`/`ring0`/`ring1` channels is an operational step done via +`cut-release.sh`, whose cross-repo publish path is now wired (#872). + +## Cutting / moving tags + +Use `scripts/cut-release.sh` (tested in `tests/test_cut_release.bats`) rather than ad-hoc `git tag`: + +```bash +# Cut an immutable release for an agent at a ref (default ref: origin/main): +scripts/cut-release.sh pr-review 1.1.0 --push + +# Cut a release AND advance that agent's stable channel to it (a promotion): +scripts/cut-release.sh pr-review 1.1.0 --channel stable --push + +# Preview without touching anything: +scripts/cut-release.sh pr-review 1.1.0 --channel stable --dry-run + +# Cross-repo agent (reusable in petry-projects/.github): cut against that repo via +# gh api (#872). --dry-run previews; --push publishes (needs contents:write on .github): +scripts/cut-release.sh feature-ideation 1.4.0 --channel ring0 --dry-run +scripts/cut-release.sh feature-ideation 1.4.0 --channel ring0 --push +``` + +The promote/rollback **runbook** (when to move `stable`, how to roll back, verify, gotchas) lives in +[`runbook.md`](./runbook.md). The automated, health-gated promotion workflow is issue #501; tag-protection +so only the promotion workflow may move a channel tag is issue #505. diff --git a/scripts/bootstrap-new-repo.sh b/scripts/bootstrap-new-repo.sh new file mode 100755 index 000000000..a3737ba9d --- /dev/null +++ b/scripts/bootstrap-new-repo.sh @@ -0,0 +1,433 @@ +#!/usr/bin/env bash +set -euo pipefail +# bootstrap-new-repo.sh — one-shot, DRY_RUN-aware bootstrap that brings a newly +# created repo to full org compliance by ORCHESTRATING the existing apply-* +# scripts. It reimplements no policy (issue #967, epic #964): +# +# • repo settings + security/GHAS + secret-scanning push protection come from +# scripts/apply-repo-settings.sh (which sources scripts/lib/push-protection.sh) +# • the two sanctioned fleet rulesets — pr-quality + code-quality, each carrying +# the mandatory bypass actors dependabot-automerge-petry (Integration app) + +# OrganizationAdmin, both bypass_mode "always" — come from +# scripts/apply-rulesets.sh, which sources them from petry-projects/.github +# (standards/rulesets/*.json, relocated there under #575). Required status +# checks are carried in those ruleset JSONs, not wired here. The repo-local +# `release-channel-tags` ruleset is NOT applied to bootstrapped repos — it +# protects .github-private's own pr-review/** + dev-lead/** release tags only. +# No legacy/ad-hoc `main` ruleset is created. +# • the standard label set + CODEOWNERS-team verification are bootstrap data, +# applied/verified here. +# +# Steps run in sequence and FAIL FAST: if repo settings or rulesets fail, the +# remaining steps are skipped and the script exits non-zero with a FAIL summary. +# +# It also CONFIRMS the new repo's release ring (issue #968): the ring is a +# deliberate, auditable choice (default stable). The confirmation is always +# recorded (operator + value + timestamp). A non-stable ring is registered in +# BOTH central files — standards/canary-rings.json here AND scripts/lib/ring-pins.sh +# in petry-projects/.github (cross-repo, via the aw-standards-sync.sh PR pattern, +# kept in sync) — and the shipped caller stub is repinned to the matching channel +# tag @/, so audit/fleet drift never sees the repo in two rings. +# +# Usage: +# bash scripts/bootstrap-new-repo.sh owner/new-repo +# bash scripts/bootstrap-new-repo.sh --ring ring1 owner/new-repo +# DRY_RUN=true bash scripts/bootstrap-new-repo.sh owner/new-repo +# +# Env: +# DRY_RUN "true" → print intent, make no write calls. Bridged onto +# apply-repo-settings.sh's DEV_LEAD_DRY_RUN and +# apply-rulesets.sh's DRY_RUN from this single flag. +# GH_TOKEN classic PAT with repo + admin scope (apply-repo-settings.sh +# rejects OAuth app tokens for the check-suites API). +# CODEOWNERS_TEAM expected first CODEOWNERS owner (default @petry-projects/org-leads). +# ORG passed through to apply-repo-settings.sh (default petry-projects). +# CANARY_RINGS ring SoT path (default standards/canary-rings.json next to this repo). +# STANDARDS_REPO repo holding the cross-repo ring-pins.sh (default petry-projects/.github). +# RING_PINS_PATH ring-pins.sh path within STANDARDS_REPO (default scripts/lib/ring-pins.sh). +# +# Args: +# --ring release ring to confirm (default stable). +# --agent agent whose rings/stub this repo joins (default dev-lead). +# +# Test seams: APPLY_REPO_SETTINGS / APPLY_RULESETS override the sub-script paths. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" + +DRY_RUN="${DRY_RUN:-false}" +CODEOWNERS_TEAM="${CODEOWNERS_TEAM:-@petry-projects/org-leads}" +APPLY_REPO_SETTINGS="${APPLY_REPO_SETTINGS:-${SCRIPT_DIR}/apply-repo-settings.sh}" +APPLY_RULESETS="${APPLY_RULESETS:-${SCRIPT_DIR}/apply-rulesets.sh}" +CANARY_RINGS="${CANARY_RINGS:-${SCRIPT_DIR}/../standards/canary-rings.json}" +STANDARDS_REPO="${STANDARDS_REPO:-petry-projects/.github}" +RING_PINS_PATH="${RING_PINS_PATH:-scripts/lib/ring-pins.sh}" +RING="${RING:-stable}" +RING_AGENT="${RING_AGENT:-dev-lead}" + +# Standard org label set every repo carries — the labels the shared automation +# keys on. Format: "name|hex-color|description". Applied idempotently (--force). +readonly -a BOOTSTRAP_LABELS=( + "needs-human-review|d93f0b|Escalated by automation — a human owner must review" + "ack-test-deletion|5319e7|Maintainer acknowledgement to allow deleting files under tests/" + "dependencies|0366d6|Dependency updates (Dependabot)" + "automerge|0e8a16|Eligible for auto-merge once required checks pass" +) + +_is_dry() { [ "$DRY_RUN" = "true" ]; } + +# ── Release-ring confirmation & registration (issue #968, epic #964) ────────── +# The default ring `stable` is record-only: a new repo is already covered by the +# `*` catch-all member token, so no central-file edit is needed — but the choice +# is still recorded. A non-stable ring is registered explicitly in both central +# files (canary-rings.json here + ring-pins.sh cross-repo) and the caller stub is +# repinned to @/ so the audit / fleet drift check sees one ring. + +_ring_operator() { printf '%s' "${GITHUB_ACTOR:-${USER:-unknown}}"; } +_ring_timestamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } + +# _ring_channels — the agent's known channels, one per line. +_ring_channels() { + jq -r --arg a "$1" '.agents[$a].rings[].channel' "$CANARY_RINGS" 2>/dev/null +} + +# _ring_is_valid — true when is a known channel for . +_ring_is_valid() { + local agent="$1" ring="$2" c + while IFS= read -r c; do + [ "$c" = "$ring" ] && return 0 + done < <(_ring_channels "$agent") + return 1 +} + +# _record_ring_decision — emit the auditable record +# (operator + value + timestamp), always — even when the answer is stable. +_record_ring_decision() { + local repo="$1" agent="$2" ring="$3" op ts decision line + op="$(_ring_operator)" + ts="$(_ring_timestamp)" + decision="recorded" + [ "$ring" != "stable" ] && decision="registered" + line="[ring-audit] repo=${repo} agent=${agent} ring=${ring} operator=${op} at=${ts} decision=${decision}" + echo " $line" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '%s\n' "$line" >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + fi +} + +# _canary_add_member_json — print canary-rings.json with +# added to 's members (idempotent — no-op if already present). +_canary_add_member_json() { + local repo="$1" agent="$2" ring="$3" + jq --arg a "$agent" --arg c "$ring" --arg r "$repo" ' + .agents[$a].rings |= map( + if .channel == $c and ((.members | index($r)) | not) + then .members += [$r] else . end + ) + ' "$CANARY_RINGS" +} + +# _assert_ring_consistency — read a proposed canary-rings +# JSON on stdin; succeed iff is an explicit member of and of NO +# other ring (the no-drift invariant the audit/fleet check enforces). +_assert_ring_consistency() { + local repo="$1" agent="$2" ring="$3" + jq -e --arg a "$agent" --arg c "$ring" --arg r "$repo" ' + ((.agents[$a].rings[] | select(.channel==$c) | (.members | index($r))) != null) + and (([.agents[$a].rings[] | select(.channel!=$c) | .members[] | select(.==$r)] | length) == 0) + ' >/dev/null 2>&1 +} + +# _register_ring_pins — keep the cross-repo central file +# scripts/lib/ring-pins.sh (in $STANDARDS_REPO) in sync with canary-rings.json by +# registering in the / mapping. ring-pins.sh is NOT present in +# this repo, so this is a cross-repo edit done via the aw-standards-sync.sh PR +# pattern. DRY_RUN prints the intent. +_register_ring_pins() { + local repo="$1" agent="$2" ring="$3" + echo " [ring] would register ${repo} in ${STANDARDS_REPO}:${RING_PINS_PATH} for channel ${agent}/${ring} (cross-repo PR, keeps central files in sync)" + if _is_dry; then + return 0 + fi + _cross_repo_ring_pins_pr "$repo" "$agent" "$ring" +} + +# _repin_stub — a non-stable repo's caller stub must pin +# the channel tag @/ (and agent_ref) matching its ring, or the audit +# / fleet stub-drift check flags it. DRY_RUN prints the intent. +_repin_stub() { + local repo="$1" agent="$2" ring="$3" + echo " [ring] would repin ${repo} caller stub .github/workflows/${agent}.yml to @${agent}/${ring}" + if _is_dry; then + return 0 + fi + _cross_repo_repin_stub_pr "$repo" "$agent" "$ring" +} + +# _cross_repo_file_pr <body> +# Opens a PR on <repo> setting <path> to <new_content> via the contents API + +# gh pr create (mirrors scripts/aw-standards-sync.sh). Returns non-zero on failure. +_cross_repo_file_pr() { + local repo="$1" path="$2" content="$3" branch="$4" msg="$5" title="$6" body="$7" + local default_branch base_sha sha encoded + local -a sha_arg=() + default_branch="$(gh api "repos/${repo}" --jq '.default_branch' 2>/dev/null || echo main)" + base_sha="$(gh api "repos/${repo}/git/ref/heads/${default_branch}" --jq '.object.sha' 2>/dev/null || true)" + [ -n "$base_sha" ] || { echo " [warn] ${repo}: cannot read ${default_branch} head" >&2; return 1; } + gh api "repos/${repo}/git/refs" --method POST \ + --field "ref=refs/heads/${branch}" --field "sha=${base_sha}" --silent 2>/dev/null \ + || gh api "repos/${repo}/git/ref/heads/${branch}" --silent 2>/dev/null \ + || { echo " [warn] ${repo}: cannot create branch ${branch}" >&2; return 1; } + sha="$(gh api "repos/${repo}/contents/${path}?ref=${branch}" --jq '.sha' 2>/dev/null || true)" + [ -n "$sha" ] && sha_arg=(--field "sha=${sha}") + encoded="$(printf '%s' "$content" | base64 -w 0 2>/dev/null || printf '%s' "$content" | base64)" + gh api "repos/${repo}/contents/${path}" --method PUT \ + --field "message=${msg}" --field "content=${encoded}" --field "branch=${branch}" \ + "${sha_arg[@]}" --silent 2>/dev/null \ + || { echo " [warn] ${repo}: cannot write ${path}" >&2; return 1; } + local existing_pr + if ! command -v gh > /dev/null 2>&1; then + echo " [warn] gh CLI is not installed" >&2 + return 1 + fi + existing_pr="$(gh pr list --repo "$repo" --head "$branch" --state open --json number --jq '.[0].number' 2>/dev/null || true)" + if [ -n "$existing_pr" ]; then + echo " [ring] PR #$existing_pr already open for branch $branch on $repo" + else + gh pr create --repo "$repo" --head "$branch" --base "$default_branch" \ + --title "$title" --body "$body" 2>/dev/null \ + || { echo " [warn] ${repo}: cannot open PR for ${path}" >&2; return 1; } + fi +} + +# _cross_repo_ring_pins_pr <repo> <agent> <ring> — fetch ring-pins.sh from +# $STANDARDS_REPO, register <repo> in the <ring> bash array, and open a PR. The +# assumed format is a per-ring indexed array `<ring>=( "owner/repo" … )` on one +# line (mirrors canary-rings.json per-channel members). If <repo> is already +# present it is a no-op; if the anchor is absent it errors rather than corrupt. +_cross_repo_ring_pins_pr() { + local repo="$1" agent="$2" ring="$3" content new branch + content="$(gh api "repos/${STANDARDS_REPO}/contents/${RING_PINS_PATH}" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null || true)" + [ -n "$content" ] || { echo "::error::cannot fetch ${STANDARDS_REPO}:${RING_PINS_PATH} for ring-pins registration" >&2; return 1; } + if printf '%s' "$content" | grep -qF "\"$repo\""; then + echo " [ring] ${repo} already present in ${RING_PINS_PATH} — in sync" + return 0 + fi + new="$(printf '%s' "$content" | awk -v repo="$repo" -v ring="$ring" ' + { line=$0 + if (!done && line !~ /^[[:space:]]*#/ && index(line, ring "=(") > 0 && match(line, /\)[[:space:]]*$/)) { + pre=substr(line, 1, RSTART-1); post=substr(line, RSTART) + sub(/[[:space:]]*$/, "", pre) + line = pre " \"" repo "\"" post; done=1 + } + print line } + END { if (!done) exit 3 }')" \ + || { echo "::error::could not locate a '${ring}=( … )' anchor in ${RING_PINS_PATH} — register manually" >&2; return 1; } + branch="ring-pins/${agent}-${ring}-$(printf '%s' "$repo" | tr '/' '-')" + _cross_repo_file_pr "$STANDARDS_REPO" "$RING_PINS_PATH" "$new" "$branch" \ + "chore: register ${repo} in ${agent}/${ring} ring-pins" \ + "chore: register ${repo} in ${agent}/${ring}" \ + "Registers \`${repo}\` in the \`${ring}\` ring, syncing ring-pins.sh with canary-rings.json (bootstrap, #968)." +} + +# _cross_repo_repin_stub_pr <repo> <agent> <ring> — fetch the target repo's caller +# stub and repin its channel tag + agent_ref to @<agent>/<ring>, then open a PR. +_cross_repo_repin_stub_pr() { + local repo="$1" agent="$2" ring="$3" path content new branch + path=".github/workflows/${agent}.yml" + content="$(gh api "repos/${repo}/contents/${path}" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null || true)" + [ -n "$content" ] || { echo " [warn] ${repo}: no ${path} stub to repin" >&2; return 0; } + new="$(printf '%s' "$content" \ + | sed -E "s#@${agent}/[A-Za-z0-9_-]+#@${agent}/${ring}#g; s#(agent_ref:[[:space:]]*[\"']?)${agent}/[A-Za-z0-9_-]+#\\1${agent}/${ring}#g")" + if [ "$new" = "$content" ]; then + echo " [ring] ${repo} stub already pins @${agent}/${ring}" + return 0 + fi + branch="ring-repin/${agent}-${ring}" + _cross_repo_file_pr "$repo" "$path" "$new" "$branch" \ + "chore: repin ${agent} stub to @${agent}/${ring}" \ + "chore: repin ${agent} stub to @${agent}/${ring}" \ + "Repins the \`${agent}\` caller stub to the \`${ring}\` channel to match this repo's ring (bootstrap, #968)." +} + +# step_ring <repo> — confirm + record the release ring; register a non-stable ring. +step_ring() { + local repo="${1:-}" proposed + echo "[bootstrap] (1/5) release ring confirmation (${RING_AGENT}/${RING})" + if [ ! -f "$CANARY_RINGS" ]; then + echo "::error::ring SoT not found at $CANARY_RINGS" >&2 + return 1 + fi + if ! _ring_is_valid "$RING_AGENT" "$RING"; then + echo "::error::unknown ring '${RING}' for agent '${RING_AGENT}' (valid: $(_ring_channels "$RING_AGENT" | paste -sd, -))" >&2 + return 1 + fi + _record_ring_decision "$repo" "$RING_AGENT" "$RING" + + if [ "$RING" = "stable" ]; then + echo " ring=stable — record-only; no central-file change required (covered by the '*' catch-all)" + return 0 + fi + + # Non-stable: register in canary-rings.json (this repo) and assert no drift on + # the proposed post-state before writing. + proposed="$(_canary_add_member_json "$repo" "$RING_AGENT" "$RING")" \ + || { echo "::error::failed to compute canary-rings update for ${repo}" >&2; return 1; } + if ! _assert_ring_consistency "$repo" "$RING_AGENT" "$RING" <<< "$proposed"; then + echo "::error::ring drift: ${repo} would not sit solely in '${RING}' — refusing to register" >&2 + return 1 + fi + if _is_dry; then + echo " [dry-run] would add ${repo} to ${RING_AGENT} '${RING}' members in $(basename "$CANARY_RINGS")" + else + printf '%s\n' "$proposed" > "$CANARY_RINGS" + echo " registered ${repo} in ${RING_AGENT} '${RING}' (canary-rings.json)" + fi + + _register_ring_pins "$repo" "$RING_AGENT" "$RING" || return 1 + _repin_stub "$repo" "$RING_AGENT" "$RING" || return 1 + echo " ring consistency OK — ${repo} sits in '${RING}' across both central files and its stub pins @${RING_AGENT}/${RING}" +} + +# pass_summary <repo> +pass_summary() { + local repo="${1:-}" + local mode="${2:-}" + echo "" + echo "[bootstrap] =====================================================" + echo "[bootstrap] PASS — ${repo} bootstrapped to org compliance${mode:+ (${mode})}" + echo "[bootstrap] =====================================================" +} + +# fail_summary <repo> <stage> +fail_summary() { + local repo="${1:-}" + local stage="${2:-}" + echo "" + echo "[bootstrap] =====================================================" + echo "[bootstrap] FAIL — ${repo}: '${stage}' step failed; remaining steps skipped" + echo "[bootstrap] =====================================================" +} + +# step_repo_settings <repo> — repo settings + security/GHAS + push protection. +step_repo_settings() { + local repo="${1:-}" settings_dry=false + _is_dry && settings_dry=true + echo "[bootstrap] (2/5) repo settings + security/GHAS + push protection" + # Canonical apply-repo-settings.sh (petry-projects/.github) reads DRY_RUN; bridge + # onto it. DEV_LEAD_DRY_RUN is kept for backward-compat with any older copy. + DRY_RUN="$settings_dry" DEV_LEAD_DRY_RUN="$settings_dry" bash "$APPLY_REPO_SETTINGS" "$repo" +} + +# step_rulesets <repo> — apply the sanctioned fleet rulesets (pr-quality + +# code-quality), which apply-rulesets.sh sources from petry-projects/.github. With +# no explicit RULESETS_DIR the applier is in fleet mode, so exactly those two are +# applied; release-channel-tags is applied by name only (org tag standard) and not +# part of the bootstrap fleet set. +step_rulesets() { + local repo="${1:-}" rulesets_dry=false + _is_dry && rulesets_dry=true + echo "[bootstrap] (3/5) sanctioned fleet rulesets (pr-quality + code-quality)" + DRY_RUN="$rulesets_dry" RULESETS_DIR="" bash "$APPLY_RULESETS" --repo "$repo" +} + +# step_labels <repo> — apply the standard label set (best-effort, idempotent). +step_labels() { + local repo="${1:-}" spec name color desc + echo "[bootstrap] (4/5) standard label set" + for spec in "${BOOTSTRAP_LABELS[@]}"; do + IFS='|' read -r name color desc <<<"$spec" + if _is_dry; then + echo " [dry-run] would ensure label '${name}' on ${repo}" + continue + fi + if gh label create "$name" --color "$color" --description "$desc" --force --repo "$repo" >/dev/null 2>&1; then + echo " ensured label '${name}'" + else + echo " [warn] could not ensure label '${name}' on ${repo}" >&2 + fi + done +} + +# step_codeowners <repo> — verify CODEOWNERS lists $CODEOWNERS_TEAM first. Best-effort. +step_codeowners() { + local repo="${1:-}" encoded decoded first_owner + echo "[bootstrap] (5/5) verify CODEOWNERS team (${CODEOWNERS_TEAM} first owner)" + if _is_dry; then + echo " [dry-run] would verify ${CODEOWNERS_TEAM} is the first CODEOWNERS owner on ${repo}" + return 0 + fi + encoded="" + local path + for path in .github/CODEOWNERS CODEOWNERS docs/CODEOWNERS; do + encoded="$(gh api "repos/${repo}/contents/${path}" --jq '.content' 2>/dev/null || true)" + [ -n "$encoded" ] && [ "$encoded" != "null" ] && break + encoded="" + done + if [ -z "$encoded" ]; then + echo " [warn] CODEOWNERS not found on ${repo} — cannot verify team ownership" >&2 + return 0 + fi + decoded="$(printf '%s' "$encoded" | base64 -d 2>/dev/null || true)" + # First owner on the first non-comment, non-blank rule line. + first_owner="$(printf '%s\n' "$decoded" \ + | sed 's/#.*//' \ + | awk 'NF >= 2 {print $2; exit}')" + if [ "$first_owner" = "$CODEOWNERS_TEAM" ]; then + echo " verified: ${CODEOWNERS_TEAM} is the first CODEOWNERS owner" + else + echo " [warn] CODEOWNERS first owner is '${first_owner:-<none>}', expected ${CODEOWNERS_TEAM}" >&2 + fi +} + +main() { + local repo="" + while [ $# -gt 0 ]; do + case "$1" in + --ring) [ $# -ge 2 ] || { echo "::error::--ring requires a value" >&2; return 2; }; RING="$2"; shift 2 ;; + --ring=*) RING="${1#--ring=}"; shift ;; + --agent) [ $# -ge 2 ] || { echo "::error::--agent requires a value" >&2; return 2; }; RING_AGENT="$2"; shift 2 ;; + --agent=*) RING_AGENT="${1#--agent=}"; shift ;; + --) shift; break ;; + -*) echo "::error::unknown flag: $1" >&2; return 2 ;; + *) if [ -z "$repo" ]; then repo="$1"; else echo "::error::unexpected argument: $1" >&2; return 2; fi; shift ;; + esac + done + if [ -z "$repo" ] && [ $# -gt 0 ]; then repo="$1"; fi + if [ -z "$repo" ]; then + echo "::error::usage: $0 [--ring <ring>] [--agent <agent>] owner/new-repo (DRY_RUN=true for a no-write preview)" >&2 + return 2 + fi + + local cmd + for cmd in gh jq; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "::error::${cmd} is required but not installed." >&2 + return 1 + fi + done + + if [ ! -f "$APPLY_REPO_SETTINGS" ]; then + echo "::error::apply-repo-settings.sh not found at $APPLY_REPO_SETTINGS" >&2 + return 1 + fi + + if [ ! -f "$APPLY_RULESETS" ]; then + echo "::error::apply-rulesets.sh not found at $APPLY_RULESETS" >&2 + return 1 + fi + + echo "[bootstrap] repo=${repo} dry_run=${DRY_RUN} ring=${RING_AGENT}/${RING}" + + if ! step_ring "$repo"; then fail_summary "$repo" "ring"; return 1; fi + if ! step_repo_settings "$repo"; then fail_summary "$repo" "repo-settings"; return 1; fi + if ! step_rulesets "$repo"; then fail_summary "$repo" "rulesets"; return 1; fi + if ! step_labels "$repo"; then fail_summary "$repo" "labels"; return 1; fi + if ! step_codeowners "$repo"; then fail_summary "$repo" "codeowners"; return 1; fi + + pass_summary "$repo" "$([ "$DRY_RUN" = "true" ] && echo "dry-run")" +} + +# Source-guard: tests source this to exercise individual step_* helpers. +if [ "${BASH_SOURCE[0]:-$0}" = "$0" ]; then + main "$@" +fi diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh new file mode 100755 index 000000000..868e96f97 --- /dev/null +++ b/scripts/canary-rollout.sh @@ -0,0 +1,1090 @@ +#!/usr/bin/env bash +set -euo pipefail +# canary-rollout.sh — ring-staged, health-gated promotion of agent releases by +# moving channel tags only (initiative #495, issue #501; rollback/observability #502). +# +# The ONLY action this performs is moving a channel tag (via `gh api` on the agent's host +# repo, with the release-manager App token — #1076) — it never writes consumer files. A ring advances +# only when the rings already on the candidate pass the soak/health gate (see +# scripts/lib/canary-rollout.sh for the pure decision core). +# +# Channel tags ARE the rollout state (#502): the frontier is derived from where each +# channel tag resolves; there is no separate state store. +# +# The FRONT END that closes the last manual seam (#1069): `autocut` polls each registered +# reusable and, when its blob on the host's main HEAD differs from the current `next` +# candidate, cuts a new immutable <agent>/vX.Y.Z and moves `next` onto it — seeding the soak/ +# promote pipeline with no manual `cut-release.sh`. It is gated by CANARY_AUTO_CUT (the single +# kill-switch), runs BEFORE promote-all on the scheduled tick, and is best-effort. +# +# Usage: +# canary-rollout.sh autocut [--dry-run] # cut+seed new candidates for changed reusables (gated by CANARY_AUTO_CUT) +# canary-rollout.sh evaluate <agent> # read-only gate + health report (also the #502 report) +# canary-rollout.sh evaluate-all # read-only evaluate for EVERY registry agent (fleet-wide; the 4h timer) +# canary-rollout.sh promote <agent> [--override] [--allow-pre-existing] [--dry-run] +# canary-rollout.sh promote-all [--override] [--allow-pre-existing] [--dry-run] # gated fleet auto-promote (the SCHEDULED arm, #1045b) +# canary-rollout.sh rollback <agent> <ring> --to <vX.Y.Z> [--dry-run] +# canary-rollout.sh resolve <agent> <channel> # debug: print resolved member repos +# canary-rollout.sh sync-issues [--dry-run] # blocker issue per BLOCKED agent + fleet-status table to the job summary (auto-triage) +# canary-rollout.sh drift [--emit-stub] # read-only: report *-reusable.yml on a host but unregistered, + stale registry entries (#1082) +# +# Gate standard: .github#548 — graduated per-transition dwell/sample floors over a +# per-candidate cumulative window (since the candidate's OWN vX.Y.Z cut), a robust +# spike-capped baseline for the sample target, the ring0->ring1 sample waiver, and +# candidate-regression-vs-environmental failure triage. Knobs live in the ring SoT +# under .agents[<agent>].gate (see scripts/lib/canary-rollout.sh for the pure core). +# +# Env: +# CANARY_RINGS path to ring SoT (default: standards/canary-rings.json next to this repo) +# SOAK_WINDOW_DAYS optional override of the baseline-window length in days +# (default: .gate.baseline_window_days, else 14) +# CANARY_FAILURE_CATEGORY optional triage hint (comment-cap|rate-limit|infra|data) +# CANARY_AUTO_CUT autocut kill-switch — autocut is a no-op unless this == 'true' +# GH_TOKEN credential; the workflow mints a GitHub App token and passes it here + +_HERE="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=lib/canary-rollout.sh +source "${_HERE}/lib/canary-rollout.sh" +DEFAULT_RINGS="$(cd "${_HERE}/.." && pwd)/standards/canary-rings.json" +CANARY_RINGS="${CANARY_RINGS:-$DEFAULT_RINGS}" + +# CUT_RELEASE — the cut-release.sh invoked by `autocut` to cut a new candidate. A +# sibling of this script by default; overridable so tests can stub the cut (#1069). +CUT_RELEASE="${CUT_RELEASE:-${_HERE}/cut-release.sh}" + +# THIS_REPO — the repo this checkout belongs to. Agents hosted HERE (dev-lead, pr-review) +# keep their channel/release tags in this checkout and resolve them via local git. A +# cross-repo agent (host != THIS_REPO, e.g. the #482 reusables hosted in petry-projects/ +# .github) keeps its <name>/<channel> and <name>/vX.Y.Z tags on ITS host, so those tags +# must be resolved there via `gh api` — reading local refs resolves empty and the frontier +# falsely reports "fully rolled out" (#1049). Mirrors cut-release.sh's CROSS_REPO_TARGET. +THIS_REPO="${GITHUB_REPOSITORY:-petry-projects/.github-private}" + +_jq() { jq "$@" "$CANARY_RINGS"; } +_agent_field() { _jq -r --arg a "$1" ".agents[\$a].$2"; } + +# ordered_channels <agent> — e.g. "next,ring0,ring1,stable" +ordered_channels() { + _jq -r --arg a "$1" '.agents[$a].rings | sort_by(.order) | map(.channel) | join(",")' +} + +# resolve_members <agent> <channel> — print the member repos, expanding the +# host-relative tokens $host / $org_infra / * (one repo per line). +resolve_members() { + local agent="$1" channel="$2" host t r + host="$(_agent_field "$agent" host)" + while IFS= read -r t; do + [ -z "$t" ] && continue + case "$t" in + '$host') printf '%s\n' "$host" ;; + '$org_infra') + while IFS= read -r r; do + if [ "$r" != "$host" ]; then printf '%s\n' "$r"; fi + done < <(_jq -r '.org_infra_repos[]') ;; + '*') printf '%s\n' '*' ;; + *) printf '%s\n' "$t" ;; + esac + done < <(_jq -r --arg a "$agent" --arg c "$channel" \ + '.agents[$a].rings[] | select(.channel==$c) | .members[]') + return 0 +} + +# _gh_tag_commit <repo> <tag> — echo the COMMIT sha <tag> resolves to on <repo> via the +# GitHub API, dereferencing an annotated tag object (mirrors cut-release.sh's +# gh_release_commit). Empty on any error / absent tag (never fails the caller). +_gh_tag_commit() { + local repo="$1" tag="$2" ref_info obj type + ref_info="$(gh api "repos/$repo/git/ref/tags/$tag" --jq '[(.object?.sha // "" | tostring), (.object?.type // "" | tostring)] | @tsv' 2>/dev/null)" || return 0 + [ -z "$ref_info" ] && return 0 + read -r obj type <<< "$ref_info" + if [ "$type" = "tag" ]; then + gh api "repos/$repo/git/tags/$obj" --jq '(.object?.sha // "" | tostring)' 2>/dev/null || true + else + printf '%s\n' "$obj" + fi +} + +# _gh_move_tag <repo> <tag> <sha> — force-move (or create) the lightweight ref +# refs/tags/<tag> on <repo> to <sha> via the GitHub API. THE channel-tag mover for every +# agent (promote + rollback), this-repo and cross-repo alike (#1076): the API path is +# granted the release-manager App's ruleset bypass for tag UPDATEs, whereas a local +# `git push --force` is not (013 on protected channel tags) and also fails with +# "nonexistent object" for a cross-repo host commit absent from this checkout (#1054). +# Tries PATCH (existing ref) then falls back to POST (create the ref). +_gh_move_tag() { + [ $# -lt 3 ] && return 1 + local repo="$1" tag="$2" sha="$3" + gh api -X PATCH "repos/$repo/git/refs/tags/$tag" \ + -f sha="$sha" -F force=true >/dev/null 2>&1 && return 0 + gh api -X POST "repos/$repo/git/refs" \ + -f ref="refs/tags/$tag" -f sha="$sha" >/dev/null 2>&1 +} + +# channel_commit <agent> <channel> — commit the channel tag <agent>/<channel> resolves to +# (empty if the tag does not exist). Agents hosted in THIS repo resolve against the local +# checkout; a cross-repo agent's channel tags live on ITS host, so they are resolved there +# via the GitHub API — reading local refs resolves empty and the frontier falsely reports +# "fully rolled out" (#1049). +channel_commit() { + local agent="$1" channel="$2" host + host="$(_agent_field "$agent" host)" + if [ -n "$host" ] && [ "$host" != "$THIS_REPO" ]; then + _gh_tag_commit "$host" "$agent/$channel" + return 0 + fi + git rev-parse -q --verify "refs/tags/$agent/$channel^{commit}" 2>/dev/null \ + || git rev-parse -q --verify "$agent/$channel^{commit}" 2>/dev/null || true +} + +# _gate_field <agent> <field> — read .agents[a].gate.<field> (empty if absent). +_gate_field() { _jq -r --arg a "$1" --arg f "$2" '.agents[$a].gate[$f] // empty'; } +# _gate_knob <agent> <transition_key> <field> — read a per-transition knob (empty if absent). +_gate_knob() { _jq -r --arg a "$1" --arg t "$2" --arg f "$3" '.agents[$a].gate.transitions[$t][$f] // empty'; } + +# _iso_now_minus_days <ndays> — ISO-8601 Zulu timestamp n days ago (GNU or BSD date). +_iso_now_minus_days() { + date -u -d "-${1} days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null \ + || date -u -v"-${1}d" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo "" +} +_to_z() { # normalise any parseable timestamp to ISO-8601 Zulu (empty passes through) + [ -z "${1:-}" ] && { echo ""; return 0; } + date -u -d "$1" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null \ + || date -u -v "$1" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null \ + || echo "$1" +} +_epoch() { + date -u -d "$1" +%s 2>/dev/null \ + || date -u -v "$1" +%s 2>/dev/null \ + || echo 0 +} + +# _gh_candidate_cut_date <repo> <agent> <commit> — ISO-8601 Zulu tagger date of the +# release tag <agent>/vX.Y.Z on <repo> whose (dereferenced) commit equals <commit>. The +# cross-repo analogue of the local for-each-ref path: a cross-repo agent's release tags +# live on its host, not this checkout (#1049). Empty if no matching release tag is found. +_gh_candidate_cut_date() { + local repo="$1" agent="$2" commit="$3" obj type csha cdate + while IFS=$'\t' read -r _ obj type; do + [ -z "$obj" ] && continue + if [ "$type" = "tag" ]; then + IFS=$'\t' read -r csha cdate < <(gh api "repos/$repo/git/tags/$obj" \ + --jq '[(.object?.sha // "" | tostring), (.tagger?.date // "" | tostring)] | @tsv' 2>/dev/null) || true + else + csha="$obj"; cdate="" + fi + if [ "$csha" = "$commit" ]; then _to_z "$cdate"; return 0; fi + done < <(gh api "repos/$repo/git/matching-refs/tags/$agent/v" \ + --jq '.[]? | [.ref, (.object?.sha // "" | tostring), (.object?.type // "" | tostring)] | @tsv' 2>/dev/null) + echo "" +} + +# candidate_cut_date <agent> <candidate_commit> — ISO-8601 Zulu tagger date of the +# immutable release tag <agent>/vX.Y.Z that points at the candidate commit. This is the +# per-candidate cumulative-window start (#548): health is measured since the candidate's +# OWN cut, NOT a rolling window — so a pre-cut failure of a prior version is excluded. +# For a cross-repo agent (host != THIS_REPO) the release tags live on the host, so the +# date is resolved there via the GitHub API instead of the local for-each-ref (#1049). +candidate_cut_date() { + local agent="$1" commit="$2" host obj deref cdate c + host="$(_agent_field "$agent" host)" + if [ -n "$host" ] && [ "$host" != "$THIS_REPO" ]; then + _gh_candidate_cut_date "$host" "$agent" "$commit" + return 0 + fi + while IFS='|' read -r obj deref cdate; do + c="$deref"; [ -z "$c" ] && c="$obj" + if [ "$c" = "$commit" ]; then _to_z "$cdate"; return 0; fi + done < <(git for-each-ref \ + --format='%(objectname)|%(*objectname)|%(creatordate:iso-strict)' \ + "refs/tags/${agent}/v*" 2>/dev/null) + git log -1 --format=%cI "$commit" 2>/dev/null || echo "" +} + +# _run_json <repo> <workflow> <since_z> — gh run-list JSON (conclusion,createdAt,databaseId,workflowName) for a +# repo since the given Zulu timestamp. Empty repo/wildcard → []. Never fails the caller. +_run_json() { + local repo="$1" wf="$2" since="$3" + [ -z "$repo" ] || [ "$repo" = '*' ] && { echo '[]'; return 0; } + gh run list --repo "$repo" --workflow "$wf" ${since:+--created ">=$since"} \ + -L 1000 --json conclusion,createdAt,databaseId,workflowName 2>/dev/null || echo '[]' +} + +# _tier_sample <agent> <since_z> <repo...> — EXECUTED runs (success+failure) on the +# source tier since the candidate cut. Prints "<executed> <earliest_createdAt|->". +_tier_sample() { + local agent="$1" since="$2"; shift 2 + local wf repo json executed=0 earliest="" e + wf="$(_agent_field "$agent" run_workflow)" + for repo in "$@"; do + json="$(_run_json "$repo" "$wf" "$since")" + executed=$(( executed + $(jq '[.[]?|select(.conclusion=="success" or .conclusion=="failure")]|length' 2>/dev/null <<< "$json" || echo 0) )) + e="$(jq -r '[.[]?|select(.conclusion=="success" or .conclusion=="failure")|.createdAt?]|min // empty' 2>/dev/null <<< "$json" || echo "")" + if [ -n "$e" ] && { [ -z "$earliest" ] || [[ "$e" < "$earliest" ]]; }; then earliest="$e"; fi + done + echo "$executed ${earliest:--}" +} + +# _benign_patterns <agent> — emit the per-reusable known-benign failure-class allowlist +# (#1025 P2) as TSV "<workflow_regex>\t<step_regex>", one entry per line. Empty if none. +_benign_patterns() { + _jq -r --arg a "$1" \ + '.agents[$a].gate?.benign_failure_classes // [] | .[] | [(.workflow // ""), (.step // "")] | @tsv' +} + +# Memoization cache for _run_signature: keyed by "repo:run_id". +# Avoids duplicate gh run view calls for the same (repo, run_id) across agents +# in evaluate-all (where multiple agents can share repos). +declare -A _RUN_SIG_CACHE=() + +# _run_signature <repo> <run_id> — the failed step names of a run, joined by newlines +# (the "step/error signature" the allowlist matches against). Empty repo/wildcard/id or +# any gh error → "" (fail-closed: an unknown signature is never treated as benign). +_run_signature() { + local repo="$1" id="$2" cache_key sig json + { [ -z "$repo" ] || [ "$repo" = '*' ] || [ -z "$id" ]; } && { echo ""; return 0; } + cache_key="${repo}:${id}" + if [[ -v _RUN_SIG_CACHE["$cache_key"] ]]; then + echo "${_RUN_SIG_CACHE[$cache_key]}" + return 0 + fi + json="$(gh run view "$id" --repo "$repo" --json jobs 2>/dev/null || echo '{}')" + sig="$(jq -r '[.jobs[]?|.steps[]?|select(.conclusion=="failure")|.name] | join("\n")' \ + 2>/dev/null <<< "$json" || echo "")" + _RUN_SIG_CACHE["$cache_key"]="$sig" + echo "$sig" +} + +# _failure_benign <repo> <run_id> <workflow_name> <patterns_tsv> — return 0 if this +# in-window failure matches any allowlist entry, else 1. Fail-closed on an empty signature. +_failure_benign() { + local repo="$1" rid="$2" rwf="$3" patterns="$4" sig wf_re step_re + sig="$(_run_signature "$repo" "$rid")" + [ -z "$sig" ] && return 1 + while IFS=$'\t' read -r wf_re step_re; do + [ -z "$step_re" ] && continue + if [ "$(benign_match "$rwf" "$sig" "$wf_re" "$step_re")" = "yes" ]; then return 0; fi + done <<< "$patterns" + return 1 +} + +# _cumulative_health <agent> <since_z> <apply_benign 0|1> <repo...> — failures + +# startup_failures across EVERY given tier repo since the candidate cut. When +# apply_benign=1, failures matching the per-reusable known-benign allowlist (#1025 P2) +# are counted separately and excluded from the blocking total. Prints +# "<failures> <startup_failures> <benign_excluded>". +_cumulative_health() { + local agent="$1" since="$2" apply_benign="$3"; shift 3 + local wf repo json fail=0 startup=0 benign=0 patterns="" rid rwf + wf="$(_agent_field "$agent" run_workflow)" + [ "$apply_benign" = "1" ] && patterns="$(_benign_patterns "$agent")" + for repo in "$@"; do + json="$(_run_json "$repo" "$wf" "$since")" + startup=$(( startup + $(jq '[.[]?|select(.conclusion=="startup_failure")]|length' 2>/dev/null <<< "$json" || echo 0) )) + if [ -z "$patterns" ]; then + # No benign patterns to match — count all failures with one jq pass, + # avoiding a gh run view call per failure. + fail=$(( fail + $(jq '[.[]?|select(.conclusion=="failure")]|length' 2>/dev/null <<< "$json" || echo 0) )) + else + while IFS=$'\t' read -r rid rwf; do + if _failure_benign "$repo" "$rid" "$rwf" "$patterns"; then + benign=$(( benign + 1 )) + else + fail=$(( fail + 1 )) + fi + done < <(jq -r '.[]?|select(.conclusion=="failure")|[(.databaseId // "" | tostring),(.workflowName // "")]|@tsv' 2>/dev/null <<< "$json") + fi + done + echo "$fail $startup $benign" +} + +# _baseline_daily <agent> <window_days> <repo...> — per-day EXECUTED counts on the +# source tier over the trailing window_days (exactly window_days integers, zero-filled), +# feeding the robust spike-capped baseline for the sample target (#548). +_baseline_daily() { + local agent="$1" window="$2"; shift 2 + local wf since repo json dates="" day i count out="" + wf="$(_agent_field "$agent" run_workflow)" + since="$(_iso_now_minus_days "$window")" + for repo in "$@"; do + json="$(_run_json "$repo" "$wf" "$since")" + dates+="$(jq -r '.[]?|select(.conclusion=="success" or .conclusion=="failure")|.createdAt[0:10]?' 2>/dev/null <<< "$json" || true)"$'\n' + done + for (( i=0; i<window; i++ )); do + day="$(date -u -d "-${i} days" +%Y-%m-%d 2>/dev/null || date -u -v"-${i}d" +%Y-%m-%d 2>/dev/null || echo "")" + count=$(grep -c "^${day}$" 2>/dev/null <<< "$dates" || true) + out+="${count} " + done + echo "${out% }" +} + +# _reusable_differs <agent> <candidate_commit> <prior_commit> — 1 if the agent's reusable +# workflow blob differs between the candidate SHA and the prior channel SHA, else 0. Used +# by triage to confirm a CANDIDATE REGRESSION (#548): a failure whose reusable is identical +# to the prior version is pre-existing, not introduced by the candidate. +_reusable_differs() { + local agent="$1" cand="$2" prior="$3" reusable a b + reusable="$(_agent_field "$agent" reusable)" + [ -z "$reusable" ] || [ -z "$cand" ] || [ -z "$prior" ] && { echo 0; return 0; } + a="$(git rev-parse -q --verify "${cand}:${reusable}" 2>/dev/null || echo "")" + b="$(git rev-parse -q --verify "${prior}:${reusable}" 2>/dev/null || echo "")" + [ -n "$a" ] && [ "$a" != "$b" ] && { echo 1; return 0; } + echo 0 +} + +# _frontier_state <agent> — compute the rollout frontier and graduated gate, echoing: +# "<cand> <frontier> <transition> <state> <dwell_h> <dwell_floor> <sample> <target> <cum_fail> <cum_startup> <triage>" +# frontier = first ring (after next) not yet on the candidate commit; triage is "-" +# unless state is BLOCKED (then REGRESSION | PRE_EXISTING). +_frontier_state() { + local agent="$1" + local cand chans frontier="" + cand="$(channel_commit "$agent" next)" + chans="$(ordered_channels "$agent")" + + local chan_array=() + IFS=, read -r -a chan_array <<< "$chans" + local ch + for ch in "${chan_array[@]}"; do + local c; c="$(channel_commit "$agent" "$ch")" + if [ "$ch" = "next" ] || [ "$c" = "$cand" ]; then :; else frontier="$ch"; break; fi + done + if [ -z "$frontier" ]; then + echo "$cand - - COMPLETE 0 0 0 0 0 0 0 -"; return 0 + fi + + local transition source cut_z now_epoch + transition="$(transition_key "$frontier" "$chans")" + source="${transition%%->*}" + cut_z="$(candidate_cut_date "$agent" "$cand")" + if [ -z "$cut_z" ]; then + # Cannot determine the per-candidate window start — fail closed to prevent unbounded history queries. + echo "$cand $frontier $transition BLOCKED 0 0 0 0 0 0 0 -"; return 0 + fi + now_epoch="$(date -u +%s)" + + # Source-tier repos (the tier currently running the candidate). + local src_repos=() r + while IFS= read -r r; do [ -n "$r" ] && src_repos+=("$r"); done < <(resolve_members "$agent" "$source") + + # Sample on the source tier over the per-candidate window. + local sample earliest + read -r sample earliest < <(_tier_sample "$agent" "$cut_z" "${src_repos[@]}") + + # Dwell is always measured from the candidate's own cut (tagger date), per #548 spec. + local dwell_h=0 + local cut_epoch; cut_epoch="$(_epoch "$cut_z")" + if [ "$cut_epoch" -gt 0 ]; then + dwell_h=$(( (now_epoch - cut_epoch) / 3600 )) + fi + [ "$dwell_h" -lt 0 ] && dwell_h=0 + + # Whether the candidate changed the agent's reusable vs the prior channel on the frontier. + # The known-benign allowlist is applied ONLY when the reusable is byte-identical (differs=0), + # so it can never mask a candidate-introduced regression (#1025 P2). + local prior differs apply_benign=1 + prior="$(channel_commit "$agent" "$frontier")" + differs="$(_reusable_differs "$agent" "$cand" "$prior")" + if [ "$differs" = "1" ]; then apply_benign=0; fi + + # Cumulative health across EVERY concrete tier repo since the candidate's own cut. + local all_repos=() ch3 + for ch3 in "${chan_array[@]}"; do + while IFS= read -r r; do [ -n "$r" ] && [ "$r" != '*' ] && all_repos+=("$r"); done \ + < <(resolve_members "$agent" "$ch3") + done + local cum_fail cum_startup cum_benign + read -r cum_fail cum_startup cum_benign < <(_cumulative_health "$agent" "$cut_z" "$apply_benign" "${all_repos[@]}") + + # Per-transition knobs (registry-configurable; #548 defaults live in the ring SoT). + local dwell_floor waived="false" target=0 + dwell_floor="$(_gate_knob "$agent" "$transition" dwell_hours)"; dwell_floor="${dwell_floor:-0}" + if [ "$(_gate_knob "$agent" "$transition" waive_sample)" = "true" ]; then + waived="true" + elif [ -n "$(_gate_knob "$agent" "$transition" sample_min)" ]; then + target="$(_gate_knob "$agent" "$transition" sample_min)" + else + local win frac cmin cmax spike_cap daily baseline_total + win="${SOAK_WINDOW_DAYS:-$(_gate_field "$agent" baseline_window_days)}"; win="${win:-14}" + frac="$(_gate_knob "$agent" "$transition" sample_fraction_permille)"; frac="${frac:-250}" + cmin="$(_gate_knob "$agent" "$transition" sample_clamp_min)"; cmin="${cmin:-3}" + cmax="$(_gate_knob "$agent" "$transition" sample_clamp_max)"; cmax="${cmax:-15}" + spike_cap="$(_gate_field "$agent" baseline_spike_cap_multiple)"; spike_cap="${spike_cap:-3}" + daily="$(_baseline_daily "$agent" "$win" "${src_repos[@]}")" + baseline_total=0; for c in $daily; do baseline_total=$(( baseline_total + c )); done + if [ "$baseline_total" -eq 0 ] && [ "$(_gate_knob "$agent" "$transition" waive_sample_if_no_caller)" = "true" ]; then + waived="true" # dwell-only: the source tier has no caller (#548) + else + # shellcheck disable=SC2086 + target="$(robust_sample_target "$frac" "$cmin" "$cmax" "$spike_cap" $daily)" + fi + fi + + local state; state="$(decide_graduated "$dwell_h" "$dwell_floor" "$sample" "$target" "$waived" "$cum_fail" "$cum_startup")" + + local triage="-" + if [ "$state" = "BLOCKED" ]; then + triage="$(classify_failure "$differs" "${CANARY_FAILURE_CATEGORY:-unknown}")" + fi + + echo "$cand $frontier $transition $state $dwell_h $dwell_floor $sample $target $cum_fail $cum_startup $cum_benign $triage" +} + +cmd_evaluate() { + local agent="$1" + echo "== canary-rollout evaluate: $agent (gate standard: .github#548) ==" + local cand; cand="$(channel_commit "$agent" next)" + echo "candidate (next) = ${cand:0:12} cut=$(candidate_cut_date "$agent" "$cand")" + local chan_array=() + IFS=, read -r -a chan_array <<< "$(ordered_channels "$agent")" + local ch + for ch in "${chan_array[@]}"; do + local c; c="$(channel_commit "$agent" "$ch")" + local mark=" "; [ -n "$cand" ] && [ "$c" = "$cand" ] && mark="* " + printf ' %s%-7s -> %s\n' "$mark" "$ch" "${c:0:12}" + done + read -r _cand frontier transition state dwell floor sample target cum_fail cum_startup cum_benign triage < <(_frontier_state "$agent") + echo "----" + if [ "$frontier" = "-" ]; then + echo "frontier: none — fully rolled out (all rings on candidate)." + else + gate_summary_line "$transition" "$state" "$dwell" "$floor" "$sample" "$target" "$cum_fail" "$cum_startup" "$cum_benign" + echo "decision for next ring '$frontier' [$transition]: $state" + if [ "$state" = "BLOCKED" ]; then + if [ "$triage" = "REGRESSION" ]; then + echo "::error::triage=REGRESSION — candidate changed the reusable and a run failed since cut. HALT + hold; recommend rollback." + else + echo "::warning::triage=PRE_EXISTING — failure is pre-existing/environmental. Report only; do NOT rollback, do NOT advance." + fi + fi + fi +} + +# cmd_evaluate_all — read-only evaluate for EVERY agent in the ring registry (#1025 P1). +# The 4h schedule runs this so the whole fleet is evaluated on the timer, not just one +# agent. Iterates the registry keys, so newly-registered reusables are picked up with no +# workflow change. Never mutates (evaluate is read-only). +cmd_evaluate_all() { + local agents rc=0 agent + agents="$(_jq -r '.agents | keys[]' 2>/dev/null || true)" + if [ -z "$agents" ]; then + echo "no agents registered in $CANARY_RINGS — nothing to evaluate."; return 0 + fi + echo "== canary-rollout evaluate-all: fleet-wide (gate standard: .github#548) ==" + while IFS= read -r agent; do + [ -z "$agent" ] && continue + echo "──────── agent: $agent ────────" + cmd_evaluate "$agent" || rc=$? + done <<< "$agents" + return "$rc" +} + +cmd_promote() { + local agent="$1"; shift + local override=false dry=false allow_pre_flag=false + while [ $# -gt 0 ]; do + case "$1" in + --override) override=true ;; + --dry-run) dry=true ;; + --allow-pre-existing) allow_pre_flag=true ;; + *) echo "::error::unknown promote flag: $1" >&2; return 2 ;; + esac; shift + done + read -r cand frontier transition state _dwell _floor _sample _target cum_fail _cum_startup _cum_benign triage < <(_frontier_state "$agent") + if [ "$frontier" = "-" ]; then + echo "nothing to promote — $agent is fully rolled out."; return 0 + fi + # allow_pre: advance a BLOCKED frontier ONLY when triage=PRE_EXISTING (never REGRESSION). + # Sourced from the per-reusable control block or the --allow-pre-existing flag (#1025 P2). + local allow_pre + allow_pre="$(_jq -r --arg a "$agent" '.agents[$a].gate?.control?.allow_pre_existing // false')" + [ "$allow_pre_flag" = true ] && allow_pre=true + if [ "$state" = "BLOCKED" ] && [ "$triage" = "REGRESSION" ] && [ "$override" != true ]; then + echo "::error::gate=BLOCKED (triage=REGRESSION) for '$frontier' [$transition] — candidate regression suspected; not promoting. Investigate + rollback, do not --override blindly." + return 0 + fi + local advance=false + [ "$state" = "PROMOTE" ] && advance=true + [ "$override" = true ] && advance=true + [ "$state" = "BLOCKED" ] && [ "$triage" = "PRE_EXISTING" ] && [ "$allow_pre" = true ] && advance=true + if [ "$advance" != true ]; then + echo "gate=$state for ring '$frontier' [$transition] (cum_fail=$cum_fail, triage=$triage) — not promoting. (use --override, or --allow-pre-existing for a PRE_EXISTING triage, after investigating)" + return 0 + fi + [ "$state" != "PROMOTE" ] && echo "::warning::advancing $agent/$frontier despite gate state '$state' (triage=$triage)" + # Consistent move (#1076): EVERY agent moves its channel tag via `gh api` on its HOST + # repo — never a local `git push`. A local force-push is NOT granted the release-manager + # App's ruleset bypass for a tag UPDATE, so it 013s on a protected channel tag such as + # dev-lead/next; the API path (same App token) IS honored as a bypass actor. host + # defaults to THIS_REPO for an agent whose registry entry omits it. + local host + host="$(_jq -r --arg a "$agent" '.agents[$a].host // "" | tostring')" + host="${host:-$THIS_REPO}" + echo "advancing $agent/$frontier -> ${cand:0:12} on $host" + if [ "$dry" = true ]; then + echo "[DRY-RUN] would: gh api PATCH repos/$host/git/refs/tags/$agent/$frontier sha=$cand (force)" + return 0 + fi + _gh_move_tag "$host" "$agent/$frontier" "$cand" \ + || { echo "::error::failed to move $agent/$frontier -> ${cand:0:12} on $host" >&2; return 1; } + echo "promoted $agent/$frontier -> ${cand:0:12}" + # Expose the move for the workflow's GitHub Deployment (traceability, #502). The + # deployment must be created on the repo that OWNS the moved commit: a cross-repo agent's + # candidate SHA lives on its host, NOT on THIS_REPO — creating the deployment against + # GITHUB_REPOSITORY 422s with "No ref found" (#1059). So emit the owning repo too. + local deploy_repo="$host" # the repo that OWNS the moved commit (#1059); host==THIS_REPO for this-repo agents + # GITHUB_OUTPUT is single-valued (last write wins), fine for a single `promote`. For + # `promote-all` (many promotions per run) the workflow reads CANARY_PROMOTIONS_LOG — one + # TSV line per promotion — so it can record a deployment for EVERY move, not just the last. + if [ -n "${GITHUB_OUTPUT:-}" ]; then + { echo "promoted_agent=$agent"; echo "promoted_ring=$frontier" + echo "promoted_sha=$cand"; echo "promoted_host=$deploy_repo"; } >> "$GITHUB_OUTPUT" + fi + if [ -n "${CANARY_PROMOTIONS_LOG:-}" ]; then + printf '%s\t%s\t%s\t%s\n' "$agent" "$frontier" "$cand" "$deploy_repo" >> "$CANARY_PROMOTIONS_LOG" + fi +} + +# cmd_promote_all [--override] [--allow-pre-existing] [--dry-run] — the gated fleet +# auto-promote and the SCHEDULED arm of the automation (#1045 part b). Iterates every +# registry agent and calls cmd_promote for each, so each PROMOTE-ready agent advances +# exactly one ring per run while BLOCKED/REGRESSION agents are left untouched by the gate +# unless --override is passed (the scheduled workflow never passes it). A per-agent +# failure is logged and skipped so one agent cannot halt the fleet sweep. Flags are +# forwarded verbatim to every cmd_promote. +cmd_promote_all() { + local agents rc=0 agent + agents="$(_jq -r '.agents? | keys[]?' 2>/dev/null || true)" + if [ -z "$agents" ]; then + echo "no agents registered in $CANARY_RINGS — nothing to promote."; return 0 + fi + echo "== canary-rollout promote-all: fleet-wide (gate standard: .github#548) ==" + while IFS= read -r agent; do + [ -z "$agent" ] && continue + echo "──────── agent: $agent ────────" + cmd_promote "$agent" "$@" || { rc=$?; echo "::warning::promote of $agent returned $rc (continuing fleet)"; } + done <<< "$agents" + return "$rc" +} + +cmd_rollback() { + local agent="$1" ring="$2"; shift 2 + local to="" dry=false + while [ $# -gt 0 ]; do + case "$1" in + --to) + if [ $# -lt 2 ]; then echo "::error::--to requires a value" >&2; return 2; fi + to="$2"; shift + ;; + --dry-run) dry=true ;; + *) echo "::error::unknown rollback flag: $1" >&2; return 2 ;; + esac; shift + done + [ -z "$to" ] && { echo "::error::rollback requires --to <vX.Y.Z>" >&2; return 2; } + # Consistent path (#1076), mirroring cmd_promote: the target lookup AND the move both go + # through gh api on the agent's HOST repo for every agent — never local git. host defaults + # to THIS_REPO for an agent whose registry entry omits it. + local host + host="$(_jq -r --arg a "$agent" '.agents[$a].host // "" | tostring')" + host="${host:-$THIS_REPO}" + local target + target="$(_gh_tag_commit "$host" "$agent/$to")" + [ -z "$target" ] && { echo "::error::release tag $agent/$to not found on $host" >&2; return 1; } + echo "rolling back $agent/$ring -> $to (${target:0:12}) on $host" + if [ "$dry" = true ]; then + echo "[DRY-RUN] would: gh api PATCH repos/$host/git/refs/tags/$agent/$ring sha=$target (force)" + return 0 + fi + _gh_move_tag "$host" "$agent/$ring" "$target" \ + || { echo "::error::failed to move $agent/$ring -> $to on $host" >&2; return 1; } + echo "rolled back $agent/$ring -> $to" +} + +# ── blocker-issue + fleet-status automation (auto-triage of held promotions) ──── +# The gate detects + classifies a BLOCKED promotion; sync-issues turns that signal into +# a tracked work item: one idempotent issue per BLOCKED agent (with the failing-run +# evidence pre-attached), auto-closed when the gate clears, plus the whole-fleet status +# table rendered into the run's job summary (a read-only snapshot — not an issue). Runs on +# the schedule after promote-all. All GitHub writes are best-effort — a failure here never +# fails the promotion run. +ISSUE_REPO="${ISSUE_REPO:-$THIS_REPO}" + +# _issue_find <label> <marker> — "<number>\t<STATE>" of the newest issue carrying <marker> +# in its body (matched locally over the label's issues, not GitHub search — HTML-comment +# markers are not reliably indexed). Empty if none. +_issue_find() { + local label="$1" marker="$2" + gh issue list --repo "$ISSUE_REPO" --label "$label" --state all -L 100 \ + --json number,state,body 2>/dev/null \ + | jq -r --arg m "$marker" \ + '[.[] | select((.body // "") | contains($m))] | sort_by(.number) | last + | if . == null then "" else "\(.number)\t\(.state | ascii_upcase)" end' 2>/dev/null \ + || echo "" +} + +# _gh_issue_create <title> <body> <labels_csv> — create an issue, echo its number. +_gh_issue_create() { + gh issue create --repo "$ISSUE_REPO" --title "$1" --body "$2" --label "$3" 2>/dev/null \ + | grep -oE '[0-9]+$' | tail -1 +} + +# _blocker_evidence <agent> <candidate_commit> — markdown bullets for the in-window failing +# runs (repo + run link + failed-step signature), capped at 8. Uses the SAME per-candidate +# window + tier repos the gate counts, so the evidence matches cum_fail. +_blocker_evidence() { + local agent="$1" cand="$2" wf cut_z repo json r n=0 out="" + wf="$(_agent_field "$agent" run_workflow)" + cut_z="$(candidate_cut_date "$agent" "$cand")" + [ -z "$cut_z" ] && { printf '_(no candidate cut date resolved — cannot list failing runs)_\n'; return 0; } + local chan_array=() ch all=() seen=" " dedup=() + IFS=, read -r -a chan_array <<< "$(ordered_channels "$agent")" + for ch in "${chan_array[@]}"; do + while IFS= read -r r; do [ -n "$r" ] && [ "$r" != '*' ] && all+=("$r"); done < <(resolve_members "$agent" "$ch") + done + for r in "${all[@]}"; do case "$seen" in *" $r "*) ;; *) dedup+=("$r"); seen+="$r ";; esac; done + for repo in "${dedup[@]}"; do + json="$(_run_json "$repo" "$wf" "$cut_z")" + local rid sig + while IFS= read -r rid; do + [ -z "$rid" ] && continue + if [ "$n" -ge 8 ]; then out+="- _(…more failing runs; truncated at 8)_"$'\n'; printf '%s' "$out"; return 0; fi + sig="$(_run_signature "$repo" "$rid" | tr '\n' ';' | sed 's/;$//')" + out+="- \`$repo\` — run [$rid](https://github.com/$repo/actions/runs/$rid); failed steps: ${sig:-unknown}"$'\n' + n=$((n+1)) + done < <(jq -r '.[]?|select(.conclusion=="failure" or .conclusion=="startup_failure")|(.databaseId|tostring)' 2>/dev/null <<< "$json") + done + [ -z "$out" ] && out="_(no failing runs in the per-candidate window — cum_fail may be startup_failures or a transient count)_"$'\n' + printf '%s' "$out" +} + +# _blocker_body <agent> <transition> <cand> <cum_fail> <cum_startup> <triage> <host> <evidence> +_blocker_body() { + local agent="$1" transition="$2" cand="$3" cum_fail="$4" cum_startup="$5" triage="$6" host="$7" evidence="$8" note + if [ "$triage" = "REGRESSION" ]; then + note="> ⛔ **REGRESSION** — the candidate changed the reusable and a run failed since its cut. HALT + hold; investigate and roll back rather than \`--override\`. (labelled \`needs-human\`)" + else + note="> ⚠️ **PRE_EXISTING** — the failure is pre-existing/environmental (reusable byte-identical to the prior channel). Report only; the gate will not roll back or advance. Fix-forward, and the armed timer auto-promotes once clean." + fi + cat <<EOF +<!-- canary-blocker:$agent --> +**Automated canary-rollout blocker.** The release gate is holding \`$agent\` and will not promote it until this clears. Filed + maintained by the Canary Rollout workflow (gate standard: .github#548); this issue is **regenerated each run and auto-closes** when the gate passes — do not edit the table below by hand. + +| field | value | +|---|---| +| agent | \`$agent\` | +| transition | \`$transition\` | +| candidate | \`${cand:0:12}\` | +| host repo | \`$host\` | +| cumulative failures | **$cum_fail** (startup_failures: $cum_startup) | +| triage | **$triage** | + +$note + +### Failing runs in the per-candidate window +$evidence +--- +_Whole-fleet status is in the Canary Rollout workflow run's job summary (Actions → Canary Rollout → latest run → Summary)._ +EOF +} + +# _dashboard_md <rows_md> <timestamp> — the fleet-status table. This is a read-only snapshot +# (not a work item), so it is rendered into the GitHub Actions run's job summary — NOT filed +# as an issue: it needs no Issues:write, spawns no synthetic pinned issue, and shows on the +# run's Summary page every tick. The per-agent BLOCKED issues remain issues (they are +# trackable, assignable, closable work items). +_dashboard_md() { + cat <<EOF +# Canary Rollout — fleet status + +Last updated: \`$2\` · auto-promote armed: \`${CANARY_AUTO_PROMOTE:-unset}\` · gate standard: .github#548. + +| agent | state | transition | cum_fail | triage | blocker | +|---|---|---|---|---|---| +$1 + +\`PROMOTE\`/\`COMPLETE\`/\`SOAKING\` need no action. \`BLOCKED\` opens a per-agent issue (label \`canary-blocker\`) with the failing-run evidence; it auto-closes when the gate clears. +EOF +} + +# cmd_sync_issues [--dry-run] — upsert one blocker issue per BLOCKED agent, and render the +# fleet-status table into the run's job summary (GITHUB_STEP_SUMMARY). Blocker issues are +# idempotent (marker-keyed) and auto-close when the gate clears. Best-effort: never fails the +# run. Reads ISSUE_REPO (default THIS_REPO). +cmd_sync_issues() { + local dry=false; [ "${1:-}" = "--dry-run" ] && dry=true + local agents; agents="$(_jq -r '.agents? | keys[]?' 2>/dev/null || true)" + [ -z "$agents" ] && { echo "no agents registered in $CANARY_RINGS — nothing to sync."; return 0; } + echo "== canary-rollout sync-issues: repo=$ISSUE_REPO dry=$dry (gate standard: .github#548) ==" + if [ "$dry" != true ]; then + gh label create canary-blocker --repo "$ISSUE_REPO" --color ededed --description "canary-rollout automation" >/dev/null 2>&1 || true + fi + local rows="" agent + while IFS= read -r agent; do + [ -z "$agent" ] && continue + local cand frontier transition state _d _f _s _t cum_fail cum_startup _cb triage host + read -r cand frontier transition state _d _f _s _t cum_fail cum_startup _cb triage < <(_frontier_state "$agent") + host="$(_agent_field "$agent" host)" + local blk="—" num_state num istate + # Best-effort (#1081): these substitutions call gh/jq. Under `set -euo pipefail` + # a bare assignment propagates a non-zero exit and would abort the whole step — + # before the fleet dashboard renders and before the intended fallback warnings + # below. `|| true` keeps sync-issues degrading gracefully (empty → handled). + num_state="$(_issue_find canary-blocker "<!-- canary-blocker:$agent -->" || true)" + num="${num_state%%$'\t'*}"; istate="${num_state##*$'\t'}" + if [ "$state" = "BLOCKED" ]; then + local evidence body title + evidence="$(_blocker_evidence "$agent" "$cand" || true)" + body="$(_blocker_body "$agent" "$transition" "$cand" "$cum_fail" "$cum_startup" "$triage" "$host" "$evidence")" + title="Canary blocker: $agent $transition (cum_fail=$cum_fail, $triage)" + if [ -z "$num" ]; then + if [ "$dry" = true ]; then echo " [DRY] would OPEN blocker issue for $agent ($triage)"; blk="(new)"; else + num="$(_gh_issue_create "$title" "$body" "canary-blocker" || true)" + if [ -n "$num" ]; then + [ "$triage" = "REGRESSION" ] && gh issue edit "$num" --repo "$ISSUE_REPO" --add-label needs-human >/dev/null 2>&1 || true + echo " opened blocker issue #$num for $agent"; blk="#$num" + else echo "::warning::could not open blocker issue for $agent (Issues:write on the App?)"; fi + fi + else + if [ "$dry" = true ]; then echo " [DRY] would UPDATE blocker issue #$num for $agent"; blk="#$num"; else + [ "$istate" = "OPEN" ] || gh issue reopen "$num" --repo "$ISSUE_REPO" >/dev/null 2>&1 || true + gh issue edit "$num" --repo "$ISSUE_REPO" --title "$title" --body "$body" >/dev/null 2>&1 \ + || echo "::warning::could not update blocker issue #$num for $agent" + [ "$triage" = "REGRESSION" ] && gh issue edit "$num" --repo "$ISSUE_REPO" --add-label needs-human >/dev/null 2>&1 || true + echo " updated blocker issue #$num for $agent"; blk="#$num" + fi + fi + else + # Not blocked — close a stale open blocker issue (the gate cleared). + if [ -n "$num" ] && [ "$istate" = "OPEN" ]; then + if [ "$dry" = true ]; then echo " [DRY] would CLOSE cleared blocker issue #$num for $agent ($state)"; else + gh issue close "$num" --repo "$ISSUE_REPO" \ + --comment "✅ Gate cleared — \`$agent\` is now \`$state\`. Closed automatically by canary-rollout." >/dev/null 2>&1 || true + echo " closed cleared blocker issue #$num for $agent" + fi + blk="#$num (closed)" + fi + fi + rows+="| \`$agent\` | $state | \`$transition\` | $cum_fail | $triage | $blk |"$'\n' + done <<< "$agents" + + # Render the fleet-status table into the run's job summary (a read-only snapshot, not a + # work item). Falls back to stdout when GITHUB_STEP_SUMMARY is unset (local/manual runs). + local ts dmd + ts="$(date -u +%Y-%m-%dT%H:%MZ 2>/dev/null || echo unknown)" + dmd="$(_dashboard_md "${rows%$'\n'}" "$ts")" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + if printf '\n%s\n' "$dmd" >> "$GITHUB_STEP_SUMMARY"; then + echo " wrote fleet-status table to the job summary" + else + echo "::warning::could not write the fleet-status job summary" + fi + else + echo "──── fleet status ────" + printf '%s\n' "$dmd" + fi + return 0 +} + +# ── autocut: cut a new candidate when a reusable changes on main (#1069) ──────── +# The FRONT END of the canary pipeline (the promoter's counterpart). At each scheduled +# tick — gated by CANARY_AUTO_CUT — for each registered agent it compares the reusable +# blob at the host's default-branch HEAD against the blob at the current `next` candidate; +# if they differ it cuts a new immutable <agent>/vX.Y.Z (patch bump by default) and moves +# `next` onto it via cut-release.sh, seeding the candidate into the existing soak/promote +# pipeline. Detection is done here in .github-private via the App token (which reads every +# host), so no cross-repo Actions plumbing is needed. Best-effort: never fails the run. +# +# Scope/limitation (v1): detection is on the reusable FILE blob only (the registry +# `reusable` path). A change to a shared library the reusable sources — without the +# reusable file itself changing — is not detected. Acceptable for v1; could extend to a +# path-set later. + +# _gh_default_branch <repo> — the repo's default branch name (empty on error). +_gh_default_branch() { gh api "repos/$1" --jq '.default_branch' 2>/dev/null || echo ""; } + +# _gh_head_sha <repo> <branch> — the commit SHA at <branch> HEAD (empty on error). +_gh_head_sha() { gh api "repos/$1/commits/$2" --jq '.sha' 2>/dev/null || echo ""; } + +# _gh_blob_sha <repo> <path> <ref> — the git blob SHA of <path> at <ref> (empty on error). +_gh_blob_sha() { gh api "repos/$1/contents/$2?ref=$3" --jq '.sha' 2>/dev/null || echo ""; } + +# _host_release_versions <agent> — echo every MAJOR.MINOR.PATCH (one per line) that has an +# immutable <agent>/vX.Y.Z tag on the agent's host, via the API (the App token reads every +# host; release tags are on the remote for both this-repo and cross-repo agents). +_host_release_versions() { + local agent="$1" host + host="$(_agent_field "$agent" host)" + [ -z "$host" ] && return 0 + gh api "repos/$host/git/matching-refs/tags/$agent/v" --jq '.[]?.ref' 2>/dev/null \ + | sed -n "s#^refs/tags/${agent}/v##p" || true +} + +# _autocut_bump <agent> — the configured bump level for the agent (registry knob +# .agents[a].autocut.bump), defaulting to patch. Anything but minor/major → patch. +_autocut_bump() { + local b + b="$(_jq -r --arg a "$1" '.agents[$a]?.autocut?.bump // "patch"')" + case "$b" in major|minor|patch) echo "$b" ;; *) echo "patch" ;; esac +} + +# _next_release_version <agent> <bump> — compute the next release version: bump the highest +# existing <agent>/vX.Y.Z on the host by <bump>. No existing tags → seed from 0.0.0. +_next_release_version() { + local agent="$1" bump="$2" versions highest + versions="$(_host_release_versions "$agent")" + # shellcheck disable=SC2086 + highest="$(max_semver $versions)" + [ -z "$highest" ] && highest="0.0.0" + bump_version "$highest" "$bump" +} + +# _autocut_agent <agent> <dry:true|false> — cut a new candidate for ONE agent if its reusable +# blob on the host default-branch HEAD differs from the blob at the current `next` candidate. +# Idempotent: no-op when the blobs match or main HEAD already equals the next candidate. +# Best-effort: any resolution gap is a ::warning + skip, never a hard failure. +_autocut_agent() { + local agent="$1" dry="$2" + local host reusable defbranch mainsha next_commit main_blob next_blob bump newver + host="$(_agent_field "$agent" host)" + reusable="$(_agent_field "$agent" reusable)" + if [ -z "$host" ] || [ -z "$reusable" ]; then + echo "::warning::autocut $agent: missing host/reusable in registry — skipping"; return 0 + fi + defbranch="$(_gh_default_branch "$host")"; [ -z "$defbranch" ] && defbranch="main" + mainsha="$(_gh_head_sha "$host" "$defbranch")" + if [ -z "$mainsha" ]; then + echo "::warning::autocut $agent: could not resolve $host $defbranch HEAD — skipping"; return 0 + fi + main_blob="$(_gh_blob_sha "$host" "$reusable" "$mainsha")" + if [ -z "$main_blob" ]; then + echo "::warning::autocut $agent: reusable '$reusable' not found at $host@${mainsha:0:12} — skipping"; return 0 + fi + next_commit="$(channel_commit "$agent" next)" + next_blob="" + [ -n "$next_commit" ] && next_blob="$(_gh_blob_sha "$host" "$reusable" "$next_commit")" + # Idempotency: nothing to cut when main HEAD already IS the candidate, or the reusable blob + # is byte-identical between main and the current next candidate. + if [ "$mainsha" = "$next_commit" ] || { [ -n "$next_blob" ] && [ "$main_blob" = "$next_blob" ]; }; then + echo "autocut $agent: reusable unchanged on $host (next candidate up to date) — no cut." + return 0 + fi + bump="$(_autocut_bump "$agent")" + newver="$(_next_release_version "$agent" "$bump")" + echo "autocut $agent: reusable changed on $host ($defbranch ${mainsha:0:12}) vs next ${next_commit:0:12} — cutting v$newver (bump=$bump), moving next." + if [ "$dry" = true ]; then + echo "[DRY-RUN] would: cut-release.sh $agent $newver --ref $mainsha --channel next --push" + return 0 + fi + bash "$CUT_RELEASE" "$agent" "$newver" --ref "$mainsha" --channel next --push \ + || { echo "::warning::autocut $agent: cut-release failed for v$newver (best-effort, continuing)"; return 0; } + echo "autocut $agent: cut v$newver from ${mainsha:0:12} and moved next." +} + +# cmd_autocut [--dry-run] — the scheduled front end. Gated by CANARY_AUTO_CUT (the single +# kill-switch): a clean no-op unless CANARY_AUTO_CUT == 'true'. Iterates every registry agent; +# a per-agent failure is logged and skipped so one agent cannot halt the fleet sweep. Runs +# BEFORE promote-all so a freshly cut candidate begins soaking the same tick (dwell=0 < floor +# → it SOAKS, does not promote). +cmd_autocut() { + local dry=false; [ "${1:-}" = "--dry-run" ] && dry=true + if [ "${CANARY_AUTO_CUT:-}" != "true" ]; then + echo "== canary-rollout autocut: DISABLED (CANARY_AUTO_CUT != 'true') — no-op. ==" + return 0 + fi + local agents agent + agents="$(_jq -r '.agents? | keys[]?' 2>/dev/null || true)" + [ -z "$agents" ] && { echo "no agents registered in $CANARY_RINGS — nothing to autocut."; return 0; } + echo "== canary-rollout autocut: fleet-wide dry=$dry (gate standard: .github#548) ==" + while IFS= read -r agent; do + [ -z "$agent" ] && continue + echo "──────── agent: $agent ────────" + _autocut_agent "$agent" "$dry" || echo "::warning::autocut of $agent failed (continuing fleet)" + done <<< "$agents" + return 0 +} + +# ── drift: registry vs host reusables (read-only audit, #1082) ────────────────── +# canary-rings.json's .agents{} is the MANUAL source of truth for what the canary +# pipeline manages — both autocut and evaluate-all iterate `.agents | keys`, so ANY +# reusable not registered is covered by nothing (no cut, no soak, no gate, no dashboard). +# There is otherwise no scan of the hosts' .github/workflows/*-reusable.yml, so a +# first-party reusable that was added/renamed but never registered ships with ZERO staged +# rollout until a human remembers to register it. +# +# `drift` closes that observability gap: it enumerates the *-reusable.yml on every +# registered host repo, diffs against the registry's reusable paths, and REPORTS (stdout + +# job summary) two classes of drift. It is read-only — it never registers anything (ring +# topology/members need human intent); `--emit-stub` optionally prints a scaffold +# .agents[<name>] block for a maintainer to fill in. +# - unregistered: a *-reusable.yml present on a host but absent from canary-rings.json. +# - missing-file: a registry entry whose reusable file no longer exists on its host. + +# _registered_hosts — the set of host repos to scan: the union of every agent's `host` +# and the org_infra_repos list, deduped (one repo per line). +_registered_hosts() { + _jq -r '([.agents[]?.host // empty] + (.org_infra_repos // [])) + | map(select(. != "")) | unique | .[]' +} + +# _gh_list_reusables <repo> — the full paths of *-reusable.yml files under the repo's +# .github/workflows (one per line). Reads the directory listing via the contents API and +# filters locally (mirrors _run_json: raw fetch, jq in the caller). Returns non-zero when +# the listing could NOT be enumerated (the API errored or did not return a JSON array), so +# the caller can distinguish "no reusables here" from "could not read the host" — the latter +# must NOT be treated as every registered reusable having been deleted (a false missing-file +# avalanche). A genuinely empty (but readable) workflows dir returns success with no output. +_gh_list_reusables() { + local repo="$1" json + json="$(gh api "repos/$repo/contents/.github/workflows" 2>/dev/null)" || return 1 + jq -e 'type=="array"' >/dev/null 2>&1 <<< "$json" || return 1 + jq -r '[.[]? | select(.type=="file") | select((.name // "") | endswith("-reusable.yml")) | .path] | .[]' \ + 2>/dev/null <<< "$json" || true + return 0 +} + +# _registered_reusables_for_host <host> — the reusable paths registered to <host> (one +# per line, deduped). +_registered_reusables_for_host() { + _jq -r --arg h "$1" '[.agents[]? | select(.host==$h) | .reusable // empty] | unique | .[]' +} + +# _agents_for_reusable <host> <path> — the registry agent name(s) whose (host,reusable) +# match, joined by ", " (for the missing-file finding message). +_agents_for_reusable() { + _jq -r --arg h "$1" --arg p "$2" \ + '[.agents | to_entries[] | select(.value.host==$h and .value.reusable==$p) | .key] | join(", ")' +} + +# _drift_scaffold <host> <path> — a scaffold .agents[<name>] JSON block for an unregistered +# reusable, keyed by the name derived from the filename (foo-reusable.yml → foo). Clones an +# existing agent's ring topology + gate defaults as a starting point (members need human +# intent), resets host/reusable/run_workflow, and empties the per-reusable benign allowlist +# so a new reusable does not silently inherit another agent's benign classes. --emit-stub only. +_drift_scaffold() { + local host="$1" path="$2" name + name="${path##*/}"; name="${name%-reusable.yml}" + _jq --arg h "$host" --arg p "$path" --arg n "$name" ' + (.agents | to_entries | (map(select(.value.host==$h)) + .) | .[0].value) as $tpl + | { ($n): ($tpl + | .host = $h + | .reusable = $p + | .run_workflow = "TODO: set to the reusable workflow name (the name: value)" + | (.gate.benign_failure_classes) = []) }' +} + +# cmd_drift [--emit-stub] — the read-only registry/host drift audit. Exits 0 (report-only); +# each finding is a ::warning:: annotation and a job-summary row. --emit-stub additionally +# prints a scaffold .agents[<name>] block per unregistered reusable. +cmd_drift() { + local emit_stub=false + while [ $# -gt 0 ]; do + case "$1" in + --emit-stub) emit_stub=true ;; + *) echo "::error::unknown drift flag: $1" >&2; return 2 ;; + esac; shift + done + echo "== canary-rollout drift: registry vs host reusables (read-only; gate standard: .github#548) ==" + local hosts host rows="" stubs="" u_total=0 m_total=0 + hosts="$(_registered_hosts)" + if [ -z "$hosts" ]; then + echo "no host repos resolved from $CANARY_RINGS — nothing to audit."; return 0 + fi + while IFS= read -r host; do + [ -z "$host" ] && continue + echo "──────── host: $host ────────" + local present registered unregistered missing p agents + if ! present="$(_gh_list_reusables "$host")"; then + # Could not read the host's workflows dir — skip it rather than false-flag every + # registered reusable as missing-file. Read-only + best-effort: a warning, not a failure. + echo "::warning::drift $host: could not enumerate .github/workflows (API error / no access) — skipping host this cycle" + continue + fi + registered="$(_registered_reusables_for_host "$host")" + local reg_arr=() pres_arr=() reg_str="" pres_str="" + if [ -n "$registered" ]; then mapfile -t reg_arr <<< "$registered"; fi + if [ -n "$present" ]; then mapfile -t pres_arr <<< "$present"; fi + [ "${#reg_arr[@]}" -gt 0 ] && reg_str=" ${reg_arr[*]}" + [ "${#pres_arr[@]}" -gt 0 ] && pres_str=" ${pres_arr[*]}" + echo " registered reusables (${#reg_arr[@]}):$reg_str" + echo " present *-reusable.yml (${#pres_arr[@]}):$pres_str" + # unregistered = present on the host but not in the registry. + unregistered="$(set_difference "$present" "$registered")" + while IFS= read -r p; do + [ -z "$p" ] && continue + echo "::warning::DRIFT[unregistered] $host: $p present on host but absent from canary-rings.json (no cut/soak/gate/dashboard until registered)" + rows+="| \`$host\` | unregistered | \`$p\` | not in \`.agents{}\` — register it or delete the file |"$'\n' + u_total=$((u_total + 1)) + if [ "$emit_stub" = true ]; then stubs+="$(_drift_scaffold "$host" "$p")"$'\n'; fi + done <<< "$unregistered" + # missing-file = registered in the registry but the file is gone from the host. + missing="$(set_difference "$registered" "$present")" + while IFS= read -r p; do + [ -z "$p" ] && continue + agents="$(_agents_for_reusable "$host" "$p")" + echo "::warning::DRIFT[missing-file] $host: registry agent '${agents:-?}' -> $p not found on host (deleted/renamed reusable; stale registry entry)" + rows+="| \`$host\` | missing-file | \`$p\` | registry agent \`${agents:-?}\` points at a file that no longer exists |"$'\n' + m_total=$((m_total + 1)) + done <<< "$missing" + done <<< "$hosts" + + echo "----" + local total=$((u_total + m_total)) + echo "drift summary: $u_total unregistered, $m_total missing-file ($total total findings)" + if [ "$total" -eq 0 ]; then + echo "no reusable drift detected — the registry and host reusables are in sync." + fi + if [ "$emit_stub" = true ] && [ -n "$stubs" ]; then + echo "---- scaffold .agents[<name>] stubs for unregistered reusables (--emit-stub) ----" + echo "Paste into standards/canary-rings.json under .agents, then set run_workflow + review ring members:" + printf '%s' "$stubs" + fi + + # Render the fleet-drift table into the run's job summary (a read-only snapshot). Falls + # back to stdout when GITHUB_STEP_SUMMARY is unset (local/manual runs). + local ts dmd + ts="$(date -u +%Y-%m-%dT%H:%MZ 2>/dev/null || echo unknown)" + if [ "$total" -eq 0 ]; then + dmd="$(printf '# Canary Rollout — reusable drift\n\nLast updated: `%s` · gate standard: .github#548.\n\n_No reusable drift — the registry and host `*-reusable.yml` are in sync._\n' "$ts")" + else + dmd="$(printf '# Canary Rollout — reusable drift\n\nLast updated: `%s` · gate standard: .github#548 · findings: **%s** (%s unregistered, %s missing-file).\n\n| host | class | reusable | note |\n|---|---|---|---|\n%s\n> `unregistered` ships with ZERO staged rollout until added to `.agents{}`; `missing-file` is a stale registry entry pointing at a deleted reusable. Report-only — no auto-registration.\n' "$ts" "$total" "$u_total" "$m_total" "${rows%$'\n'}")" + fi + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + if printf '\n%s\n' "$dmd" >> "$GITHUB_STEP_SUMMARY"; then + echo " wrote reusable-drift table to the job summary" + else + echo "::warning::could not write the reusable-drift job summary" + fi + fi + return 0 +} + +main() { + local cmd + for cmd in jq gh git; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "::error::Required command '$cmd' is not installed." >&2 + return 1 + fi + done + if [ -n "${SOAK_WINDOW_DAYS:-}" ] && ! [[ "${SOAK_WINDOW_DAYS}" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::SOAK_WINDOW_DAYS, when set, must be a positive integer" >&2 + return 2 + fi + local sub="${1:-}"; shift || true + case "$sub" in + evaluate) [ $# -ge 1 ] || { echo "usage: evaluate <agent>" >&2; return 2; }; cmd_evaluate "$@" ;; + evaluate-all) cmd_evaluate_all ;; + promote) [ $# -ge 1 ] || { echo "usage: promote <agent> [--override] [--allow-pre-existing] [--dry-run]" >&2; return 2; }; cmd_promote "$@" ;; + promote-all) cmd_promote_all "$@" ;; + rollback) [ $# -ge 2 ] || { echo "usage: rollback <agent> <ring> --to <vX.Y.Z>" >&2; return 2; }; cmd_rollback "$@" ;; + resolve) [ $# -ge 2 ] || { echo "usage: resolve <agent> <channel>" >&2; return 2; }; resolve_members "$@" ;; + sync-issues) cmd_sync_issues "$@" ;; # upsert blocker issues + dashboard for held promotions + autocut) cmd_autocut "$@" ;; # cut a new candidate when a reusable changes on main (#1069) + drift) cmd_drift "$@" ;; # report reusables present on a host but unregistered, + stale registry entries (#1082) + *) echo "::error::usage: canary-rollout.sh {autocut|drift|evaluate|evaluate-all|promote|promote-all|rollback|resolve|sync-issues} [args]" >&2; return 2 ;; + esac +} + +# Source-guard: tests source this file to exercise resolve_members etc. without running. +if [ "${BASH_SOURCE[0]:-$0}" = "$0" ]; then + main "$@" +fi diff --git a/scripts/lib/canary-rollout.sh b/scripts/lib/canary-rollout.sh new file mode 100644 index 000000000..0993df03b --- /dev/null +++ b/scripts/lib/canary-rollout.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +# canary-rollout.sh (lib) — pure decision core for the ring-staged, health-gated +# promotion of agent releases (initiative #495, issue #501; rollback/observability #502). +# +# This file is side-effect-free and `source`-able: it defines pure functions only +# (no I/O, no gh/git calls) so the gate logic can be unit-tested deterministically. +# The orchestrator scripts/canary-rollout.sh sources this and feeds it real numbers +# gathered from `gh`. +# +# Gate standard: .github#548 (definitive spec). Per transition, evaluate on the +# SOURCE tier (the tier currently running the candidate), counting EXECUTED runs +# (success+failure; skipped/cancelled excluded) since the candidate's OWN cut: +# - next->ring0: dwell >= 4h + sample >= clamp(round(0.25·avg), 3, 15) +# (dwell-only when the source tier has no caller) +# - ring0->ring1: dwell >= 8h + CUMULATIVE-HEALTH-ONLY (fresh sample WAIVED) +# - ring1->stable: dwell >= 12h + >= 1 ring1 run +# - ALWAYS: cumulative health = ZERO failures / ZERO startup_failures +# across EVERY tier since the candidate's own first cut. +# The dwell floors and sample fractions are registry-configurable per transition +# (see standards/canary-rings.json .gate); the numbers above are the defaults. + +# clamp <value> <lo> <hi> — echo value bounded to [lo, hi]. +clamp() { + local v="$1" lo="$2" hi="$3" + if [ "$v" -lt "$lo" ]; then echo "$lo"; return 0; fi + if [ "$v" -gt "$hi" ]; then echo "$hi"; return 0; fi + echo "$v" +} + +# round_div <numerator> <denominator> — integer half-up rounding of n/d. +# Denominator must be > 0. Pure arithmetic, no bc. +round_div() { + local n="$1" d="$2" + if [ "${d:-0}" -le 0 ]; then echo 0; return 1; fi + echo $(( (2 * n + d) / (2 * d) )) +} + +# median_x2 <nums...> — echo 2×median as an exact integer (so an even-length set's +# half-integer median stays exact). Empty set → 0. +median_x2() { + local n=$# + [ "$n" -eq 0 ] && { echo 0; return 0; } + local sorted + sorted=$(printf '%s\n' "$@" | sort -n) + local arr=() + local x + while IFS= read -r x; do arr+=("$x"); done <<< "$sorted" + local mid=$(( n / 2 )) + if [ $(( n % 2 )) -eq 1 ]; then + echo $(( 2 * arr[mid] )) + else + echo $(( arr[mid - 1] + arr[mid] )) + fi +} + +# robust_sample_target <fraction_permille> <clamp_lo> <clamp_hi> <daily_counts...> +# Robust baseline: cap each per-day executed count at 3× the median day (neutralising +# spikes like a runaway loop day), take the mean of the capped days, then the sample +# target = clamp(round(fraction · avg), lo, hi). fraction is in per-mille (250 == 0.25). +# +# All arithmetic stays exact in integers by working in "×2" units for the median: +# capped_x2(c) = min(2c, 3·median_x2) [3·median compared without rounding] +# avg = (Σ capped_x2) / (2·N) +# target_raw = round(fraction/1000 · avg) = round(fraction·Σcapped_x2 / (2000·N)) +robust_sample_target() { + local frac="$1" lo="$2" hi="$3" cap_multiple="${4:-3}"; shift 4 + local n=$# + [ "$n" -eq 0 ] && { clamp 0 "$lo" "$hi"; return 0; } + local m2 cap3 c c2 sum2=0 + m2=$(median_x2 "$@") + cap3=$(( cap_multiple * m2 )) + for c in "$@"; do + c2=$(( 2 * c )) + [ "$c2" -gt "$cap3" ] && c2=$cap3 + sum2=$(( sum2 + c2 )) + done + # target_raw = round( frac · sum2 / (2000·n) ) + local denom=$(( 2000 * n )) target_raw + target_raw=$(round_div $(( frac * sum2 )) "$denom") + clamp "$target_raw" "$lo" "$hi" +} + +# ── version-bump math (autocut front end, #1069) ────────────────────────────── +# Pure semver helpers used by `canary-rollout.sh autocut` to compute the next +# immutable release version from the highest existing <agent>/vX.Y.Z on the host. + +# bump_version <version> <patch|minor|major> — echo MAJOR.MINOR.PATCH bumped by one +# level. Unknown/absent level defaults to patch (the v1 default). Input must be a +# strict MAJOR.MINOR.PATCH; callers validate the tag before calling. +bump_version() { + local ver="$1" level="${2:-patch}" major=0 minor=0 patch=0 + IFS=. read -r major minor patch <<< "$ver" + case "$level" in + major) echo "$((major + 1)).0.0" ;; + minor) echo "${major}.$((minor + 1)).0" ;; + *) echo "${major}.${minor}.$((patch + 1))" ;; + esac +} + +# _semver_gt <a> <b> — return 0 iff semver a is strictly greater than b (compared +# by major, then minor, then patch). Equal is NOT greater. +_semver_gt() { + local a1=0 a2=0 a3=0 b1=0 b2=0 b3=0 + IFS=. read -r a1 a2 a3 <<< "$1" + IFS=. read -r b1 b2 b3 <<< "$2" + if [ "$a1" -ne "$b1" ]; then [ "$a1" -gt "$b1" ]; return; fi + if [ "$a2" -ne "$b2" ]; then [ "$a2" -gt "$b2" ]; return; fi + [ "$a3" -gt "$b3" ] +} + +# max_semver <version...> — echo the highest strict MAJOR.MINOR.PATCH among the +# args, ignoring any token that is not a strict semver. Empty if none are valid. +max_semver() { + local v hi="" + for v in "$@"; do + [[ "$v" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || continue + if [ -z "$hi" ] || _semver_gt "$v" "$hi"; then hi="$v"; fi + done + echo "$hi" +} + +# dwell_met <dwell_hours> <floor_hours> — echo 1 if the candidate has dwelled at +# least floor_hours on the source tier, else 0. +dwell_met() { + local dwell="${1:-0}" floor="${2:-0}" + if [ "$dwell" -ge "$floor" ]; then echo 1; else echo 0; fi +} + +# iso_after <a> <b> — echo "yes" if ISO-8601 UTC timestamp a is at or after b, else +# "no". Zulu ISO-8601 strings sort lexically, so a per-candidate window is a string +# comparison against the candidate's cut date (excludes pre-cut runs of prior versions). +iso_after() { + local a="$1" b="$2" + if [[ "$a" > "$b" || "$a" == "$b" ]]; then echo "yes"; else echo "no"; fi +} + +# decide_graduated <dwell_h> <dwell_floor> <sample> <sample_target> <sample_waived> \ +# <cum_failures> <cum_startup_failures> +# Pure graduated gate for one transition. Echoes exactly one of: +# BLOCKED — cumulative health breached (>=1 failure or startup_failure across any +# tier since the candidate's cut). Never advances; a human/triage +# classifies it as REGRESSION (rollback) or PRE_EXISTING (report only). +# PROMOTE — clean AND dwell floor met AND (sample target met OR sample waived). +# SOAKING — clean but dwell floor or sample target not yet met → wait. +# Health is checked first: a cumulative breach is never masked by dwell/sample. +decide_graduated() { + local dwell="${1:-0}" floor="${2:-0}" sample="${3:-0}" target="${4:-0}" + local waived="${5:-false}" cum_fail="${6:-0}" cum_startup="${7:-0}" + if [ "$cum_fail" -gt 0 ] || [ "$cum_startup" -gt 0 ]; then + echo "BLOCKED"; return 0 + fi + local dwell_ok sample_ok + dwell_ok=$(dwell_met "$dwell" "$floor") + if [ "$waived" = "true" ] || [ "$sample" -ge "$target" ]; then sample_ok=1; else sample_ok=0; fi + if [ "$dwell_ok" -eq 1 ] && [ "$sample_ok" -eq 1 ]; then + echo "PROMOTE"; return 0 + fi + echo "SOAKING" +} + +# classify_failure <reusable_differs 0|1> <category> — triage an in-window failure +# (called only when decide_graduated returns BLOCKED). Echoes: +# REGRESSION — the candidate changed the reusable (differs=1) AND the failure is +# not a known environmental class → HALT, auto-hold, recommend rollback. +# PRE_EXISTING — the reusable is byte-identical to the prior channel (differs=0), OR +# the failure is environmental (comment-cap / rate-limit / infra / data) +# → report, do NOT rollback, do NOT advance. +classify_failure() { + local differs="${1:-0}" category="${2:-unknown}" + case "$category" in + comment-cap|rate-limit|infra|data) echo "PRE_EXISTING"; return 0 ;; + esac + if [ "$differs" = "1" ]; then echo "REGRESSION"; else echo "PRE_EXISTING"; fi +} + +# benign_match <workflow_name> <failure_signature> <workflow_regex> <step_regex> +# Decide whether ONE in-window failure matches ONE known-benign failure-class allowlist +# entry (#1025 P2). A match requires a non-empty <step_regex> (guards against a match-all +# entry) that matches the failure signature (the failed step/error names), AND — when +# <workflow_regex> is non-empty — the run's workflow name. Echoes "yes" or "no". +# Pure: bash ERE only, no I/O. (Author regexes with explicit char classes, e.g. [Pp]ush, +# since ERE has no inline case-insensitivity flag.) +benign_match() { + local wf="$1" sig="$2" wf_re="$3" step_re="$4" + [ -z "$step_re" ] && { echo "no"; return 0; } + if [ -n "$wf_re" ] && ! [[ "$wf" =~ $wf_re ]]; then echo "no"; return 0; fi + if [[ "$sig" =~ $step_re ]]; then echo "yes"; else echo "no"; fi +} + +# next_channel_in_order <current_channel> <ordered_channels_csv> +# Given the frontier channel and the ordered channel list (e.g. "next,ring0,ring1,stable"), +# echo the channel that a PROMOTE advances next, or empty if already at the last ring. +next_channel_in_order() { + local current="$1" csv="$2" + local prev="" ch found="" + local chan_array=() + IFS=, read -r -a chan_array <<< "$csv" + for ch in "${chan_array[@]}"; do + if [ "$prev" = "$current" ]; then found="$ch"; break; fi + prev="$ch" + done + echo "$found" +} + +# transition_key <frontier_channel> <ordered_channels_csv> +# Echo the per-transition registry key "<source>-><frontier>", where source is the +# ring immediately below the frontier in ring order (the tier running the candidate). +# Empty if the frontier is the first channel (no source below it). +transition_key() { + local frontier="$1" csv="$2" + local prev="" ch + local chan_array=() + IFS=, read -r -a chan_array <<< "$csv" + for ch in "${chan_array[@]}"; do + if [ "$ch" = "$frontier" ] && [ -n "$prev" ]; then echo "${prev}->${frontier}"; return 0; fi + prev="$ch" + done + echo "" +} + +# ── set-diff core (drift detection, #1082) ──────────────────────────────────── +# The registry (.agents{}) is the MANUAL source of truth for what the canary pipeline +# manages; drift detection diffs it against the *-reusable.yml actually present on each +# host. Both sides of that diff are a plain set difference over whole-line strings. + +# set_difference <set_a_newlines> <set_b_newlines> — echo each line of A that is NOT +# present in B (whole-line match; order + duplicates of A preserved, blank lines dropped). +# Pure: bash only (associative-array lookup), no gh/git I/O — deterministically unit-testable. +set_difference() { + local a="$1" b="$2" line + declare -A b_map + while IFS= read -r line; do + [ -n "$line" ] && b_map["$line"]=1 + done <<< "$b" + while IFS= read -r line; do + [ -z "$line" ] && continue + if [[ -z "${b_map["$line"]:-}" ]]; then + printf '%s\n' "$line" + fi + done <<< "$a" +} + +# gate_summary_line <transition> <state> <dwell_h> <dwell_floor> <sample> <sample_target> <cum_fail> <cum_startup> [cum_benign] +# One-line human/observability row (used by `evaluate`, doubling as the #502 report). +# cum_benign (allowlisted failures excluded from cum_fail) is shown when provided. +gate_summary_line() { + printf '%-14s %-9s dwell=%sh/%sh sample=%s/%s cum_fail=%s startup=%s benign=%s\n' \ + "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "${9:-0}" +} diff --git a/standards/canary-rings.json b/standards/canary-rings.json new file mode 100644 index 000000000..c765267a5 --- /dev/null +++ b/standards/canary-rings.json @@ -0,0 +1,471 @@ +{ + "version": 1, + "description": "Ring membership for agent canary rollout (initiative #495, issues #500/#501). The promotion automation (scripts/canary-rollout.sh) reads this as the single source of truth. `next` is host-relative: it resolves to the repo that HOSTS the reusable; `ring0` covers the other org-infra repo (the host already sits in `next`).", + "org_infra_repos": [ + "petry-projects/.github", + "petry-projects/.github-private" + ], + "member_tokens": { + "$host": "the agent's host repo (the repo that owns the reusable)", + "$org_infra": "org_infra_repos minus the host (the host is already in `next`)", + "*": "every other consumer not named in an earlier ring" + }, + "_autocut_note": "autocut (#1069): `canary-rollout.sh autocut` (gated by org var CANARY_AUTO_CUT) auto-cuts a new immutable <agent>/vX.Y.Z + moves `next` when a reusable's blob on its host main HEAD differs from the current `next` candidate. Version bump defaults to patch; override per agent with the optional `.agents[<a>].autocut.bump` knob (patch|minor|major).", + "agents": { + "dev-lead": { + "host": "petry-projects/.github-private", + "reusable": ".github/workflows/dev-lead-reusable.yml", + "run_workflow": "Dev-Lead Agent", + "rings": [ + { + "channel": "next", + "order": 0, + "members": [ + "$host" + ] + }, + { + "channel": "ring0", + "order": 1, + "members": [ + "$org_infra" + ] + }, + { + "channel": "ring1", + "order": 2, + "members": [ + "petry-projects/TalkTerm", + "petry-projects/bmad-bgreat-suite" + ] + }, + { + "channel": "stable", + "order": 3, + "members": [ + "*" + ] + } + ], + "gate": { + "_standard": "petry-projects/.github#548 \u2014 graduated dwell/sample gate over a per-candidate cumulative window. These are registry-configurable per-transition knobs; the values below are the #548 defaults.", + "baseline_window_days": 14, + "baseline_spike_cap_multiple": 3, + "benign_failure_classes": [ + { + "id": "dependabot-context-dispatch", + "workflow": "Dev-Lead Agent", + "step": "[Dd]ependabot", + "reason": "#864 \u2014 a Dev-Lead run dispatched in a Dependabot context cannot read repo secrets; version-independent benign failure, not a candidate regression." + }, + { + "id": "fix-review-git-push-permission", + "workflow": "Dev-Lead Agent", + "step": "[Pp]ush", + "reason": "fix-review git push denied by branch protection / missing write scope; version-independent benign failure, not a candidate regression." + } + ], + "control": { + "allow_pre_existing": false + }, + "_benign_note": "benign_failure_classes (#1025 P2): per-reusable allowlist matched against a failed run's workflow name (`workflow` ERE) + failed-step signature (`step` ERE, matched via `gh run view`). Matches are excluded from cum_fail \u2014 but ONLY when the candidate's reusable is byte-identical to the prior channel (differs=0), so the allowlist can never mask a candidate-introduced regression. `control.allow_pre_existing`, or `promote --allow-pre-existing`, advances a BLOCKED+PRE_EXISTING frontier (never REGRESSION).", + "transitions": { + "next->ring0": { + "dwell_hours": 4, + "sample_fraction_permille": 250, + "sample_clamp_min": 3, + "sample_clamp_max": 15, + "waive_sample_if_no_caller": true + }, + "ring0->ring1": { + "dwell_hours": 8, + "waive_sample": true + }, + "ring1->stable": { + "dwell_hours": 12, + "sample_min": 1 + } + } + } + }, + "agent-shield": { + "host": "petry-projects/.github", + "reusable": ".github/workflows/agent-shield-reusable.yml", + "run_workflow": "AgentShield", + "rings": [ + { + "channel": "next", + "order": 0, + "members": [ + "petry-projects/.github-private" + ] + }, + { + "channel": "ring0", + "order": 1, + "members": [ + "petry-projects/.github" + ] + }, + { + "channel": "ring1", + "order": 2, + "members": [ + "petry-projects/TalkTerm", + "petry-projects/bmad-bgreat-suite" + ] + }, + { + "channel": "stable", + "order": 3, + "members": [ + "*" + ] + } + ], + "gate": { + "_standard": "petry-projects/.github#548 \u2014 graduated dwell/sample gate over a per-candidate cumulative window. These are registry-configurable per-transition knobs; the values below are the #548 defaults.", + "baseline_window_days": 14, + "baseline_spike_cap_multiple": 3, + "benign_failure_classes": [], + "control": { + "allow_pre_existing": false + }, + "_benign_note": "benign_failure_classes (#1025 P2): per-reusable allowlist matched against a failed run's workflow name (`workflow` ERE) + failed-step signature (`step` ERE, matched via `gh run view`). Matches are excluded from cum_fail \u2014 but ONLY when the candidate's reusable is byte-identical to the prior channel (differs=0), so the allowlist can never mask a candidate-introduced regression. `control.allow_pre_existing`, or `promote --allow-pre-existing`, advances a BLOCKED+PRE_EXISTING frontier (never REGRESSION).", + "transitions": { + "next->ring0": { + "dwell_hours": 4, + "sample_fraction_permille": 250, + "sample_clamp_min": 3, + "sample_clamp_max": 15, + "waive_sample_if_no_caller": true + }, + "ring0->ring1": { + "dwell_hours": 8, + "waive_sample": true + }, + "ring1->stable": { + "dwell_hours": 12, + "sample_min": 1 + } + } + } + }, + "auto-rebase": { + "host": "petry-projects/.github", + "reusable": ".github/workflows/auto-rebase-reusable.yml", + "run_workflow": "Auto-rebase non-Dependabot PRs", + "rings": [ + { + "channel": "next", + "order": 0, + "members": [ + "petry-projects/.github-private" + ] + }, + { + "channel": "ring0", + "order": 1, + "members": [ + "petry-projects/.github" + ] + }, + { + "channel": "ring1", + "order": 2, + "members": [ + "petry-projects/TalkTerm", + "petry-projects/bmad-bgreat-suite" + ] + }, + { + "channel": "stable", + "order": 3, + "members": [ + "*" + ] + } + ], + "gate": { + "_standard": "petry-projects/.github#548 \u2014 graduated dwell/sample gate over a per-candidate cumulative window. These are registry-configurable per-transition knobs; the values below are the #548 defaults.", + "baseline_window_days": 14, + "baseline_spike_cap_multiple": 3, + "benign_failure_classes": [], + "control": { + "allow_pre_existing": false + }, + "_benign_note": "benign_failure_classes (#1025 P2): per-reusable allowlist matched against a failed run's workflow name (`workflow` ERE) + failed-step signature (`step` ERE, matched via `gh run view`). Matches are excluded from cum_fail \u2014 but ONLY when the candidate's reusable is byte-identical to the prior channel (differs=0), so the allowlist can never mask a candidate-introduced regression. `control.allow_pre_existing`, or `promote --allow-pre-existing`, advances a BLOCKED+PRE_EXISTING frontier (never REGRESSION).", + "transitions": { + "next->ring0": { + "dwell_hours": 4, + "sample_fraction_permille": 250, + "sample_clamp_min": 3, + "sample_clamp_max": 15, + "waive_sample_if_no_caller": true + }, + "ring0->ring1": { + "dwell_hours": 8, + "waive_sample": true + }, + "ring1->stable": { + "dwell_hours": 12, + "sample_min": 1 + } + } + } + }, + "dependency-audit": { + "host": "petry-projects/.github", + "reusable": ".github/workflows/dependency-audit-reusable.yml", + "run_workflow": "Dependency audit", + "rings": [ + { + "channel": "next", + "order": 0, + "members": [ + "petry-projects/.github-private" + ] + }, + { + "channel": "ring0", + "order": 1, + "members": [ + "petry-projects/.github" + ] + }, + { + "channel": "ring1", + "order": 2, + "members": [ + "petry-projects/TalkTerm", + "petry-projects/bmad-bgreat-suite" + ] + }, + { + "channel": "stable", + "order": 3, + "members": [ + "*" + ] + } + ], + "gate": { + "_standard": "petry-projects/.github#548 \u2014 graduated dwell/sample gate over a per-candidate cumulative window. These are registry-configurable per-transition knobs; the values below are the #548 defaults.", + "baseline_window_days": 14, + "baseline_spike_cap_multiple": 3, + "benign_failure_classes": [], + "control": { + "allow_pre_existing": false + }, + "_benign_note": "benign_failure_classes (#1025 P2): per-reusable allowlist matched against a failed run's workflow name (`workflow` ERE) + failed-step signature (`step` ERE, matched via `gh run view`). Matches are excluded from cum_fail \u2014 but ONLY when the candidate's reusable is byte-identical to the prior channel (differs=0), so the allowlist can never mask a candidate-introduced regression. `control.allow_pre_existing`, or `promote --allow-pre-existing`, advances a BLOCKED+PRE_EXISTING frontier (never REGRESSION).", + "transitions": { + "next->ring0": { + "dwell_hours": 4, + "sample_fraction_permille": 250, + "sample_clamp_min": 3, + "sample_clamp_max": 15, + "waive_sample_if_no_caller": true + }, + "ring0->ring1": { + "dwell_hours": 8, + "waive_sample": true + }, + "ring1->stable": { + "dwell_hours": 12, + "sample_min": 1 + } + } + } + }, + "dependabot-automerge": { + "host": "petry-projects/.github", + "reusable": ".github/workflows/dependabot-automerge-reusable.yml", + "run_workflow": "Dependabot auto-merge", + "rings": [ + { + "channel": "next", + "order": 0, + "members": [ + "petry-projects/.github-private" + ] + }, + { + "channel": "ring0", + "order": 1, + "members": [ + "petry-projects/.github" + ] + }, + { + "channel": "ring1", + "order": 2, + "members": [ + "petry-projects/TalkTerm", + "petry-projects/bmad-bgreat-suite" + ] + }, + { + "channel": "stable", + "order": 3, + "members": [ + "*" + ] + } + ], + "gate": { + "_standard": "petry-projects/.github#548 \u2014 graduated dwell/sample gate over a per-candidate cumulative window. These are registry-configurable per-transition knobs; the values below are the #548 defaults.", + "baseline_window_days": 14, + "baseline_spike_cap_multiple": 3, + "benign_failure_classes": [], + "control": { + "allow_pre_existing": false + }, + "_benign_note": "benign_failure_classes (#1025 P2): per-reusable allowlist matched against a failed run's workflow name (`workflow` ERE) + failed-step signature (`step` ERE, matched via `gh run view`). Matches are excluded from cum_fail \u2014 but ONLY when the candidate's reusable is byte-identical to the prior channel (differs=0), so the allowlist can never mask a candidate-introduced regression. `control.allow_pre_existing`, or `promote --allow-pre-existing`, advances a BLOCKED+PRE_EXISTING frontier (never REGRESSION).", + "transitions": { + "next->ring0": { + "dwell_hours": 4, + "sample_fraction_permille": 250, + "sample_clamp_min": 3, + "sample_clamp_max": 15, + "waive_sample_if_no_caller": true + }, + "ring0->ring1": { + "dwell_hours": 8, + "waive_sample": true + }, + "ring1->stable": { + "dwell_hours": 12, + "sample_min": 1 + } + } + } + }, + "dependabot-rebase": { + "host": "petry-projects/.github", + "reusable": ".github/workflows/dependabot-rebase-reusable.yml", + "run_workflow": "Dependabot update and merge", + "rings": [ + { + "channel": "next", + "order": 0, + "members": [ + "petry-projects/.github-private" + ] + }, + { + "channel": "ring0", + "order": 1, + "members": [ + "petry-projects/.github" + ] + }, + { + "channel": "ring1", + "order": 2, + "members": [ + "petry-projects/TalkTerm", + "petry-projects/bmad-bgreat-suite" + ] + }, + { + "channel": "stable", + "order": 3, + "members": [ + "*" + ] + } + ], + "gate": { + "_standard": "petry-projects/.github#548 \u2014 graduated dwell/sample gate over a per-candidate cumulative window. These are registry-configurable per-transition knobs; the values below are the #548 defaults.", + "baseline_window_days": 14, + "baseline_spike_cap_multiple": 3, + "benign_failure_classes": [], + "control": { + "allow_pre_existing": false + }, + "_benign_note": "benign_failure_classes (#1025 P2): per-reusable allowlist matched against a failed run's workflow name (`workflow` ERE) + failed-step signature (`step` ERE, matched via `gh run view`). Matches are excluded from cum_fail \u2014 but ONLY when the candidate's reusable is byte-identical to the prior channel (differs=0), so the allowlist can never mask a candidate-introduced regression. `control.allow_pre_existing`, or `promote --allow-pre-existing`, advances a BLOCKED+PRE_EXISTING frontier (never REGRESSION).", + "transitions": { + "next->ring0": { + "dwell_hours": 4, + "sample_fraction_permille": 250, + "sample_clamp_min": 3, + "sample_clamp_max": 15, + "waive_sample_if_no_caller": true + }, + "ring0->ring1": { + "dwell_hours": 8, + "waive_sample": true + }, + "ring1->stable": { + "dwell_hours": 12, + "sample_min": 1 + } + } + }, + "soak_start_ring": "ring0" + }, + "pr-review-mention": { + "host": "petry-projects/.github", + "reusable": ".github/workflows/pr-review-mention-reusable.yml", + "run_workflow": "PR Review \u2014 Mention Trigger", + "rings": [ + { + "channel": "next", + "order": 0, + "members": [ + "petry-projects/.github-private" + ] + }, + { + "channel": "ring0", + "order": 1, + "members": [ + "petry-projects/.github" + ] + }, + { + "channel": "ring1", + "order": 2, + "members": [ + "petry-projects/TalkTerm", + "petry-projects/bmad-bgreat-suite" + ] + }, + { + "channel": "stable", + "order": 3, + "members": [ + "*" + ] + } + ], + "gate": { + "_standard": "petry-projects/.github#548 \u2014 graduated dwell/sample gate over a per-candidate cumulative window. These are registry-configurable per-transition knobs; the values below are the #548 defaults.", + "baseline_window_days": 14, + "baseline_spike_cap_multiple": 3, + "benign_failure_classes": [], + "control": { + "allow_pre_existing": false + }, + "_benign_note": "benign_failure_classes (#1025 P2): per-reusable allowlist matched against a failed run's workflow name (`workflow` ERE) + failed-step signature (`step` ERE, matched via `gh run view`). Matches are excluded from cum_fail \u2014 but ONLY when the candidate's reusable is byte-identical to the prior channel (differs=0), so the allowlist can never mask a candidate-introduced regression. `control.allow_pre_existing`, or `promote --allow-pre-existing`, advances a BLOCKED+PRE_EXISTING frontier (never REGRESSION).", + "transitions": { + "next->ring0": { + "dwell_hours": 4, + "sample_fraction_permille": 250, + "sample_clamp_min": 3, + "sample_clamp_max": 15, + "waive_sample_if_no_caller": true + }, + "ring0->ring1": { + "dwell_hours": 8, + "waive_sample": true + }, + "ring1->stable": { + "dwell_hours": 12, + "sample_min": 1 + } + } + } + } + } +} diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats new file mode 100644 index 000000000..bdaf8525d --- /dev/null +++ b/tests/canary_rollout.bats @@ -0,0 +1,1063 @@ +#!/usr/bin/env bats +# Unit tests for the canary-rollout decision core (scripts/lib/canary-rollout.sh) +# and the scripts/canary-rollout.sh orchestrator's pure paths (with gh stubs). +# Initiative #495 · issues #501 (promotion) / #502 (rollback + observability). +# Gate standard: .github#548 (graduated dwell/sample, robust baseline, +# per-candidate cumulative window, ring0 sample waiver, failure triage). + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)" +LIB="$SCRIPT_DIR/scripts/lib/canary-rollout.sh" +ORCH="$SCRIPT_DIR/scripts/canary-rollout.sh" +RINGS="$SCRIPT_DIR/standards/canary-rings.json" + +setup() { + # shellcheck source=/dev/null + source "$LIB" +} + +# ── clamp ───────────────────────────────────────────────────────────────────── +@test "clamp: within range is unchanged" { [ "$(clamp 7 3 15)" -eq 7 ]; } +@test "clamp: below floor snaps to floor" { [ "$(clamp 1 3 15)" -eq 3 ]; } +@test "clamp: above ceiling snaps to ceiling" { [ "$(clamp 25 3 15)" -eq 15 ]; } +@test "clamp: at bounds is inclusive" { [ "$(clamp 3 3 15)" -eq 3 ]; [ "$(clamp 15 3 15)" -eq 15 ]; } + +# ── round_div (banker-free half-up rounding) ────────────────────────────────── +@test "round_div: exact" { [ "$(round_div 10 5)" -eq 2 ]; } +@test "round_div: rounds half up" { [ "$(round_div 5 2)" -eq 3 ]; [ "$(round_div 7 2)" -eq 4 ]; } +@test "round_div: rounds down below half" { [ "$(round_div 4 3)" -eq 1 ]; } +@test "round_div: zero denominator → 0 + nonzero rc" { + run round_div 5 0 + [ "$status" -ne 0 ]; [ "$output" -eq 0 ] +} + +# ── median_x2 (2×median, exact integer even for even-length sets) ────────────── +@test "median_x2: odd length" { [ "$(median_x2 1 5 3)" -eq 6 ]; } # median 3 → 6 +@test "median_x2: even length sums the two middles" { [ "$(median_x2 4 4 4 4)" -eq 8 ]; } # 4+4 +@test "median_x2: unsorted input" { [ "$(median_x2 40 2 2 40 2 2)" -eq 4 ]; } # sorted middles 2,2 +@test "median_x2: empty → 0" { [ "$(median_x2)" -eq 0 ]; } + +# ── robust_sample_target: robust baseline = spike-capped mean, then clamp ────── +# fraction_permille=250 (0.25), clamp [3,15]. +@test "robust_sample_target: steady volume → round(0.25·avg)" { + # 14 days all = 40 → avg 40 → 0.25·40 = 10 → clamp 10 + set -- 40 40 40 40 40 40 40 40 40 40 40 40 40 40 + [ "$(robust_sample_target 250 3 15 3 "$@")" -eq 10 ] +} +@test "robust_sample_target: below floor clamps up to 3" { + set -- 4 4 4 4 4 4 4 4 4 4 4 4 4 4 # avg 4 → 0.25·4 = 1 → clamp 3 + [ "$(robust_sample_target 250 3 15 3 "$@")" -eq 3 ] +} +@test "robust_sample_target: above ceiling clamps down to 15" { + set -- 100 100 100 100 100 100 100 100 100 100 100 100 100 100 # 25 → clamp 15 + [ "$(robust_sample_target 250 3 15 3 "$@")" -eq 15 ] +} +@test "robust_sample_target: a 2500-run loop day is capped at 3× median (not inflated to 15)" { + # 13 low days of 2 + one 2500-run loop day. Robust baseline caps the spike at + # 3× median (=6), so the target stays a reachable 3 — NOT the 15 a raw mean gives. + set -- 2 2 2 2 2 2 2 2 2 2 2 2 2 2500 + [ "$(robust_sample_target 250 3 15 3 "$@")" -eq 3 ] + # sanity: the naive (uncapped) mean would blow past the ceiling + local sum=0 n=0 c; for c in "$@"; do sum=$((sum+c)); n=$((n+1)); done + [ "$(clamp "$(round_div $((250*sum)) $((1000*n)))" 3 15)" -eq 15 ] +} + +# ── version-bump math (autocut front end, #1069) ────────────────────────────── +@test "bump_version: patch increments the patch component" { + [ "$(bump_version 2.1.0 patch)" = "2.1.1" ] + [ "$(bump_version 0.0.0 patch)" = "0.0.1" ] +} +@test "bump_version: minor increments minor and zeroes patch" { + [ "$(bump_version 2.1.3 minor)" = "2.2.0" ] +} +@test "bump_version: major increments major and zeroes minor+patch" { + [ "$(bump_version 2.1.3 major)" = "3.0.0" ] +} +@test "bump_version: default (no/unknown level) is patch" { + [ "$(bump_version 2.1.0)" = "2.1.1" ] + [ "$(bump_version 2.1.0 bogus)" = "2.1.1" ] +} + +@test "_semver_gt: compares by major, then minor, then patch" { + run _semver_gt 2.1.1 2.1.0; [ "$status" -eq 0 ] + run _semver_gt 2.2.0 2.1.9; [ "$status" -eq 0 ] + run _semver_gt 3.0.0 2.9.9; [ "$status" -eq 0 ] + run _semver_gt 2.1.0 2.1.0; [ "$status" -ne 0 ] # equal is not greater + run _semver_gt 2.1.0 2.1.1; [ "$status" -ne 0 ] +} + +@test "max_semver: picks the highest version" { + [ "$(max_semver 2.0.5 2.1.0 2.0.9)" = "2.1.0" ] + [ "$(max_semver 1.0.0)" = "1.0.0" ] +} +@test "max_semver: ignores non-semver tokens" { + [ "$(max_semver 2.1.0 not-a-version 2.1.5 v3.0.0)" = "2.1.5" ] +} +@test "max_semver: empty input → empty" { + [ -z "$(max_semver)" ] +} + +# ── dwell_met ───────────────────────────────────────────────────────────────── +@test "dwell_met: at/over floor → 1" { [ "$(dwell_met 4 4)" -eq 1 ]; [ "$(dwell_met 9 8)" -eq 1 ]; } +@test "dwell_met: under floor → 0" { [ "$(dwell_met 3 4)" -eq 0 ]; } + +# ── iso_after (per-candidate cumulative window predicate) ────────────────────── +@test "iso_after: strictly-after and equal are 'yes'" { + [ "$(iso_after 2026-06-28T00:00:00Z 2026-06-28T00:00:00Z)" = yes ] + [ "$(iso_after 2026-06-29T00:00:00Z 2026-06-28T00:00:00Z)" = yes ] +} +@test "iso_after: excludes a pre-cut failure (pr-review-mention 06-27 before v2.1.1 06-28 cut)" { + # A run on 06-27 is NOT after the 06-28 candidate cut → must be excluded. + [ "$(iso_after 2026-06-27T10:00:00Z 2026-06-28T00:00:00Z)" = no ] +} + +# ── decide_graduated (dwell + sample + cumulative-health) ───────────────────── +# args: <dwell_h> <dwell_floor> <sample> <sample_target> <sample_waived> <cum_fail> <cum_startup> +@test "decide_graduated: dwell+sample met, clean → PROMOTE" { + [ "$(decide_graduated 5 4 8 3 false 0 0)" = "PROMOTE" ] +} +@test "decide_graduated: dwell short → SOAKING" { + [ "$(decide_graduated 3 4 8 3 false 0 0)" = "SOAKING" ] +} +@test "decide_graduated: sample short → SOAKING" { + [ "$(decide_graduated 5 4 2 3 false 0 0)" = "SOAKING" ] +} +@test "decide_graduated: any cumulative failure → BLOCKED (beats dwell+sample)" { + [ "$(decide_graduated 99 4 99 3 false 1 0)" = "BLOCKED" ] + [ "$(decide_graduated 99 4 99 3 false 0 1)" = "BLOCKED" ] +} +@test "decide_graduated: ring0→ring1 waives the fresh sample (cumulative-clean + dwell only)" { + # sample 0 but waived → PROMOTE once dwell met and clean. + [ "$(decide_graduated 9 8 0 0 true 0 0)" = "PROMOTE" ] + # still blocks on a cumulative failure even when waived. + [ "$(decide_graduated 9 8 0 0 true 1 0)" = "BLOCKED" ] + # still soaks if dwell not met. + [ "$(decide_graduated 5 8 0 0 true 0 0)" = "SOAKING" ] +} + +# ── classify_failure (triage: regression vs pre-existing/environmental) ──────── +# args: <reusable_differs 0|1> <category> +@test "classify_failure: reusable changed + non-environmental → REGRESSION" { + [ "$(classify_failure 1 unknown)" = "REGRESSION" ] +} +@test "classify_failure: reusable identical to prior version → PRE_EXISTING" { + [ "$(classify_failure 0 unknown)" = "PRE_EXISTING" ] +} +@test "classify_failure: environmental category → PRE_EXISTING even if reusable changed" { + [ "$(classify_failure 1 comment-cap)" = "PRE_EXISTING" ] + [ "$(classify_failure 1 rate-limit)" = "PRE_EXISTING" ] + [ "$(classify_failure 1 infra)" = "PRE_EXISTING" ] + [ "$(classify_failure 1 data)" = "PRE_EXISTING" ] +} + +# ── next_channel_in_order ───────────────────────────────────────────────────── +@test "next_channel_in_order: walks the ring order" { + [ "$(next_channel_in_order next 'next,ring0,ring1,stable')" = "ring0" ] + [ "$(next_channel_in_order ring0 'next,ring0,ring1,stable')" = "ring1" ] + [ "$(next_channel_in_order ring1 'next,ring0,ring1,stable')" = "stable" ] +} +@test "next_channel_in_order: last ring → empty" { + [ -z "$(next_channel_in_order stable 'next,ring0,ring1,stable')" ] +} + +# ── transition_key (source→frontier lookup key) ─────────────────────────────── +@test "transition_key: frontier maps to its source→frontier key" { + [ "$(transition_key ring0 'next,ring0,ring1,stable')" = "next->ring0" ] + [ "$(transition_key ring1 'next,ring0,ring1,stable')" = "ring0->ring1" ] + [ "$(transition_key stable 'next,ring0,ring1,stable')" = "ring1->stable" ] +} + +# ── canary-rings.json SoT shape (rings + gate knobs) ────────────────────────── +@test "canary-rings.json: valid JSON + dev-lead host + ordered rings" { + run jq -e '.agents["dev-lead"].host == "petry-projects/.github-private"' "$RINGS" + [ "$status" -eq 0 ] + run bash -c "jq -r '.agents[\"dev-lead\"].rings | sort_by(.order) | map(.channel) | join(\",\")' '$RINGS'" + [ "$output" = "next,ring0,ring1,stable" ] + run jq -e '.agents["dev-lead"].rings[] | select(.channel=="ring1") | (.members | index("petry-projects/TalkTerm")) and (.members | index("petry-projects/bmad-bgreat-suite"))' "$RINGS" + [ "$status" -eq 0 ] +} + +@test "canary-rings.json: gate block carries #548 per-transition defaults" { + # baseline window + spike cap + run jq -e '.agents["dev-lead"].gate.baseline_window_days == 14' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.agents["dev-lead"].gate.baseline_spike_cap_multiple == 3' "$RINGS" + [ "$status" -eq 0 ] + # next->ring0: 4h dwell, 0.25 fraction, clamp [3,15], dwell-only when source has no caller + run jq -e '.agents["dev-lead"].gate.transitions["next->ring0"].dwell_hours == 4' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.agents["dev-lead"].gate.transitions["next->ring0"].sample_fraction_permille == 250' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.agents["dev-lead"].gate.transitions["next->ring0"].sample_clamp_min == 3 and .agents["dev-lead"].gate.transitions["next->ring0"].sample_clamp_max == 15' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.agents["dev-lead"].gate.transitions["next->ring0"].waive_sample_if_no_caller == true' "$RINGS" + [ "$status" -eq 0 ] + # ring0->ring1: 8h dwell, sample waived (cumulative-only) + run jq -e '.agents["dev-lead"].gate.transitions["ring0->ring1"].dwell_hours == 8' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.agents["dev-lead"].gate.transitions["ring0->ring1"].waive_sample == true' "$RINGS" + [ "$status" -eq 0 ] + # ring1->stable: 12h dwell, >=1 ring1 run + run jq -e '.agents["dev-lead"].gate.transitions["ring1->stable"].dwell_hours == 12' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.agents["dev-lead"].gate.transitions["ring1->stable"].sample_min == 1' "$RINGS" + [ "$status" -eq 0 ] +} + +# ── orchestrator: resolve_members (host-relative tokens) ────────────────────── +@test "orchestrator: resolve_members expands \$host / \$org_infra / * " { + run bash -c "source '$ORCH' && CANARY_RINGS='$RINGS' resolve_members dev-lead next" + [ "$status" -eq 0 ]; [ "$output" = "petry-projects/.github-private" ] + run bash -c "source '$ORCH' && CANARY_RINGS='$RINGS' resolve_members dev-lead ring0" + [ "$status" -eq 0 ]; [ "$output" = "petry-projects/.github" ] + run bash -c "source '$ORCH' && CANARY_RINGS='$RINGS' resolve_members dev-lead ring1" + [[ "$output" == *"petry-projects/TalkTerm"* ]] + [[ "$output" == *"petry-projects/bmad-bgreat-suite"* ]] +} + +# ── orchestrator: evaluate / promote (read-only + dry-run) with stubs ───────── +_make_stub_bin() { + STUB_BIN="$(mktemp -d)"; export PATH="$STUB_BIN:$PATH" + cat > "$STUB_BIN/git" <<'GITEOF' +#!/usr/bin/env bash +case "$*" in + *"for-each-ref"*) : ;; # no release-tag date available in the stub + *"rev-parse"*"dev-lead/v1.4.0"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;; + *"rev-parse"*"dev-lead/next"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;; + *"rev-parse"*"dev-lead/ring0"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;; + *"rev-parse"*"dev-lead/ring1"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;; + *"rev-parse"*"dev-lead/stable"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;; + *"tag -f"*) : ;; + *"push"*) : ;; + *"fetch"*) : ;; + *) : ;; +esac +GITEOF + chmod +x "$STUB_BIN/git" +} + +teardown() { [ -n "${STUB_BIN:-}" ] && rm -rf "$STUB_BIN"; return 0; } + +@test "orchestrator: evaluate prints a per-ring gate report and exits 0 (read-only)" { + _make_stub_bin + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"run list"*) echo "[]" ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"dev-lead"* ]] + [[ "$output" == *"next"* ]] + [[ "$output" == *"stable"* ]] +} + +@test "orchestrator: promote --override --dry-run shows the move but never pushes" { + _make_stub_bin + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"run list"*) echo "[]" ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + local pushlog="$STUB_BIN/push.log" + cat > "$STUB_BIN/git" <<GITEOF +#!/usr/bin/env bash +case "\$*" in + *"for-each-ref"*) : ;; + *"rev-parse"*"dev-lead/ring1"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;; + *"rev-parse"*"dev-lead/stable"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;; + *"rev-parse"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;; + *"push"*) echo "\$*" >> "$pushlog" ;; + *) : ;; +esac +GITEOF + chmod +x "$STUB_BIN/git" + + run env CANARY_RINGS="$RINGS" bash "$ORCH" promote dev-lead --override --dry-run + [ "$status" -eq 0 ] + [ ! -f "$pushlog" ] + [[ "$output" == *"DRY-RUN"* ]] + [[ "$output" == *"ring1"* ]] + # dev-lead is a this-repo agent, but the move now goes through gh api on its host — the + # dry-run must show the API PATCH, never a local `git tag -f`/`git push` (#1076). + [[ "$output" == *"gh api PATCH"* ]] + [[ "$output" != *"git tag -f"* ]] +} + +@test "orchestrator: no local git tag/push in the promote/rollback move paths — gh api only (#1076)" { + # Structural guard: the channel-tag move for EVERY agent (this-repo and cross-repo) goes + # through gh api so the release-manager App's ruleset bypass is applied; a local force-push + # is not a bypass actor for a tag UPDATE and 013s on protected tags such as dev-lead/next. + ! grep -Ev '^[[:space:]]*#' "$ORCH" | grep -Eq '\bgit[[:space:]]+(tag|push)\b' +} + +# ── orchestrator: full graduated verdicts (cut date + gh run data → gate state) ─ +# Lay out next = candidate (cccc); ring0/ring1/stable = prior (bbbb): frontier = ring0, +# transition next->ring0, source = next. The release tag cccc is dated `cut_days` ago, +# so the per-candidate window (and the robust sample target) are exercised end to end. +_graduated_stub() { + local cut_days="$1" run_days_ago="$2" conclusion="$3" reusable_diff="$4" + STUB_BIN="$(mktemp -d)"; export PATH="$STUB_BIN:$PATH" + local cut_iso run_iso + cut_iso="$(date -u -d "-${cut_days} days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v"-${cut_days}d" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + run_iso="$(date -u -d "-${run_days_ago} days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v"-${run_days_ago}d" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + # gh: every run-list query returns 20 runs at run_iso with the given conclusion. + { + echo '#!/usr/bin/env bash' + echo 'case "$*" in' + printf ' *"run list"*) jq -nc --arg d "%s" --arg c "%s" '"'"'[range(20)|{conclusion:$c,createdAt:$d}]'"'"' ;;\n' "$run_iso" "$conclusion" + echo ' *) echo "{}" ;;' + echo 'esac' + } > "$STUB_BIN/gh" + chmod +x "$STUB_BIN/gh" + # git: only `next` is on the candidate (cccc); ring0/ring1/stable stay on the prior + # version (bbbb). for-each-ref yields the cccc release tag dated cut_iso; the reusable + # blob is identical (reusable_diff=0) or differs (=1) between cand and prior. + local cand_blob="reuseAAAA" prior_blob="reuseAAAA" + [ "$reusable_diff" = "1" ] && prior_blob="reuseBBBB" + { + echo '#!/usr/bin/env bash' + echo 'case "$*" in' + printf ' *"for-each-ref"*) echo "cccccccccccccccccccccccccccccccccccccccc||%s" ;;\n' "$cut_iso" + printf ' *"rev-parse"*"cccccccccccccccccccccccccccccccccccccccc:"*) echo "%s" ;;\n' "$cand_blob" + printf ' *"rev-parse"*"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:"*) echo "%s" ;;\n' "$prior_blob" + echo ' *"rev-parse"*"dev-lead/next"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;;' + echo ' *"rev-parse"*"dev-lead/ring0"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;;' + echo ' *"rev-parse"*"dev-lead/ring1"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;;' + echo ' *"rev-parse"*"dev-lead/stable"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;;' + echo ' *"rev-parse"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;;' + echo ' *) : ;;' + echo 'esac' + } > "$STUB_BIN/git" + chmod +x "$STUB_BIN/git" +} + +@test "orchestrator: PROMOTE verdict — dwell + sample met on a clean per-candidate window" { + # cut 3 days ago, runs 2 days ago, all success → dwell ≫ 4h, sample 20 ≥ target, clean. + _graduated_stub 3 2 success 0 + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"next->ring0"* ]] + [[ "$output" == *"PROMOTE"* ]] + [[ "$output" == *"decision for next ring 'ring0'"* ]] +} + +@test "orchestrator: BLOCKED + REGRESSION — in-window failure with a changed reusable" { + # A failure since the candidate cut, and the reusable differs from the prior channel + # → cumulative-health breach classified as a candidate regression (HALT + rollback). + _graduated_stub 3 2 failure 1 + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"BLOCKED"* ]] + [[ "$output" == *"REGRESSION"* ]] +} + +@test "orchestrator: BLOCKED + PRE_EXISTING — in-window failure but reusable unchanged" { + # Same failure, but the reusable is byte-identical to the prior channel → pre-existing, + # report only (do NOT rollback, do NOT advance). + _graduated_stub 3 2 failure 0 + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"BLOCKED"* ]] + [[ "$output" == *"PRE_EXISTING"* ]] +} + +# ── benign_match (per-reusable known-benign failure-class matcher, #1025 P2) ──── +# args: <workflow_name> <failure_signature> <workflow_regex> <step_regex> +@test "benign_match: workflow + step signature both match → yes" { + [ "$(benign_match 'Dev-Lead Agent' 'Push fix-review branch' 'Dev-Lead' '[Pp]ush')" = "yes" ] +} +@test "benign_match: workflow regex mismatch → no" { + [ "$(benign_match 'Other Workflow' 'Push branch' 'Dev-Lead' '[Pp]ush')" = "no" ] +} +@test "benign_match: signature does not match step regex → no" { + [ "$(benign_match 'Dev-Lead Agent' 'Compile sources' 'Dev-Lead' '[Pp]ush')" = "no" ] +} +@test "benign_match: empty step regex never matches (guards against a match-all entry)" { + [ "$(benign_match 'Dev-Lead Agent' 'anything at all' 'Dev-Lead' '')" = "no" ] +} +@test "benign_match: empty workflow regex matches any workflow" { + [ "$(benign_match 'Whatever' 'Resolve Dependabot dispatch context' '' '[Dd]ependabot')" = "yes" ] +} + +# ── canary-rings.json: benign allowlist + control block shape (#1025 P2) ──────── +@test "canary-rings.json: dev-lead gate carries a benign_failure_classes allowlist + control block" { + run jq -e '.agents["dev-lead"].gate.benign_failure_classes | type == "array" and length >= 1' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.agents["dev-lead"].gate.benign_failure_classes | all(has("id") and has("reason") and has("step"))' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.agents["dev-lead"].gate.control | has("allow_pre_existing")' "$RINGS" + [ "$status" -eq 0 ] +} + +# ── orchestrator: evaluate-all iterates the whole registry (#1025 P1) ────────── +@test "orchestrator: evaluate-all iterates every agent in the registry (fleet-wide)" { + _make_stub_bin + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"run list"*) echo "[]" ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + # A registry with a second (cloned) agent proves fleet iteration over the registry + # keys rather than a dev-lead hardcode. + local multi="$BATS_TEST_TMPDIR/rings.json" + jq '.agents["fleet-canary-test"] = .agents["dev-lead"]' "$RINGS" > "$multi" + run env CANARY_RINGS="$multi" bash "$ORCH" evaluate-all + [ "$status" -eq 0 ] + [[ "$output" == *"dev-lead"* ]] + [[ "$output" == *"fleet-canary-test"* ]] +} + +# ── orchestrator: benign-failure allowlist excludes known-benign from cum_fail ── +# Lay out next = candidate (cccc); ring0/ring1/stable = prior (bbbb). Every tier repo +# returns `failure` runs whose only failed step is <step>; `gh run view` yields that +# step so the orchestrator can build a signature and test it against the allowlist. +_benign_stub() { + local cut_days="$1" run_days_ago="$2" step="$3" reusable_diff="$4" + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + local cut_iso run_iso + cut_iso="$(date -u -d "-${cut_days} days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v"-${cut_days}d" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + run_iso="$(date -u -d "-${run_days_ago} days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v"-${run_days_ago}d" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + { + echo '#!/usr/bin/env bash' + echo 'case "$*" in' + echo ' *"run list"*)' + printf ' if [[ "$*" == *"databaseId"* ]]; then jq -nc --arg d "%s" '"'"'[range(3)|{conclusion:"failure",createdAt:$d,databaseId:(1000+.),workflowName:"Dev-Lead Agent"}]'"'"'; else jq -nc --arg d "%s" '"'"'[range(3)|{conclusion:"failure",createdAt:$d}]'"'"'; fi\n' "$run_iso" "$run_iso" + echo ' ;;' + printf ' *"run view"*) jq -nc --arg s "%s" '"'"'{jobs:[{steps:[{name:$s,conclusion:"failure"}]}]}'"'"' ;;\n' "$step" + echo ' *) echo "{}" ;;' + echo 'esac' + } > "$STUB_BIN/gh" + chmod +x "$STUB_BIN/gh" + local cand_blob="reuseAAAA" prior_blob="reuseAAAA" + [ "$reusable_diff" = "1" ] && prior_blob="reuseBBBB" + { + echo '#!/usr/bin/env bash' + echo 'case "$*" in' + printf ' *"for-each-ref"*) echo "cccccccccccccccccccccccccccccccccccccccc||%s" ;;\n' "$cut_iso" + printf ' *"rev-parse"*"cccccccccccccccccccccccccccccccccccccccc:"*) echo "%s" ;;\n' "$cand_blob" + printf ' *"rev-parse"*"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:"*) echo "%s" ;;\n' "$prior_blob" + echo ' *"rev-parse"*"dev-lead/next"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;;' + echo ' *"rev-parse"*"dev-lead/ring0"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;;' + echo ' *"rev-parse"*"dev-lead/ring1"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;;' + echo ' *"rev-parse"*"dev-lead/stable"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;;' + echo ' *"rev-parse"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;;' + echo ' *) : ;;' + echo 'esac' + } > "$STUB_BIN/git" + chmod +x "$STUB_BIN/git" +} + +@test "orchestrator: allowlisted benign failure (reusable unchanged) is excluded from cum_fail → not BLOCKED" { + # A git-push-permission failure since cut, but the reusable is byte-identical to the + # prior channel → matches the [Pp]ush benign class → excluded → gate is not BLOCKED. + _benign_stub 3 2 "Push fix-review branch" 0 + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" != *"BLOCKED"* ]] + [[ "$output" == *"benign"* ]] +} + +@test "orchestrator: benign allowlist is DISABLED when the candidate changed the reusable → BLOCKED+REGRESSION" { + # Same push failure + matching class, but the candidate changed the reusable → the + # allowlist must NOT mask a possible candidate regression. + _benign_stub 3 2 "Push fix-review branch" 1 + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"BLOCKED"* ]] + [[ "$output" == *"REGRESSION"* ]] +} + +@test "orchestrator: a non-allowlisted failure (reusable unchanged) still BLOCKS as PRE_EXISTING" { + # Failed step matches no benign class → counted → BLOCKED, triaged PRE_EXISTING. + _benign_stub 3 2 "Compile TypeScript" 0 + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"BLOCKED"* ]] + [[ "$output" == *"PRE_EXISTING"* ]] +} + +# ── orchestrator: promote --allow-pre-existing (control override, #1025 P2) ───── +@test "orchestrator: promote --allow-pre-existing advances a BLOCKED+PRE_EXISTING frontier (dry-run)" { + _graduated_stub 3 2 failure 0 + run env CANARY_RINGS="$RINGS" bash "$ORCH" promote dev-lead --allow-pre-existing --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"DRY-RUN"* ]] + [[ "$output" == *"PRE_EXISTING"* ]] + [[ "$output" != *"not promoting"* ]] +} + +@test "orchestrator: promote --allow-pre-existing REFUSES a BLOCKED+REGRESSION frontier" { + _graduated_stub 3 2 failure 1 + run env CANARY_RINGS="$RINGS" bash "$ORCH" promote dev-lead --allow-pre-existing --dry-run + [ "$status" -eq 0 ] + [[ "$output" != *"DRY-RUN"* ]] + [[ "$output" == *"REGRESSION"* ]] +} + +# ── orchestrator: cross-repo agents resolve tags on `host`, not the local checkout ─ +# (#1049) A cross-repo agent (host = petry-projects/.github) keeps its <name>/<channel> +# and <name>/vX.Y.Z tags on the HOST repo, not this checkout. `evaluate` must resolve them +# there via `gh api` (dereferencing annotated tags) — reading the LOCAL refs makes every +# ring resolve empty → "all rings equal → fully rolled out", which would falsely SKIP a +# cross-repo agent that is actually ring1 with stable on an old baseline (READY to promote). +_crossrepo_stub() { + # next=ring0=ring1 on the candidate (cccc); stable on the OLD baseline (bbbb): + # frontier = ring1, transition = ring1->stable — the ring1->stable promotion is pending. + local cut_days="$1" run_days_ago="$2" conclusion="$3" + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + local cand="cccccccccccccccccccccccccccccccccccccccc" + local old="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + local cut_iso run_iso + cut_iso="$(date -u -d "-${cut_days} days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v"-${cut_days}d" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + run_iso="$(date -u -d "-${run_days_ago} days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v"-${run_days_ago}d" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + # gh: channel tags + the release tag are resolved via the API on the HOST repo; run-list + # feeds the sample/health. Channel/release tag responses are pre-computed strings; the + # run-list branch invokes jq to generate timestamped records from the injected parameters. + # The channel tags are lightweight (object.type=commit); the release tag is annotated + # (object.type=tag), so its commit + tagger date come from a second git/tags/<obj> call. + cat > "$STUB_BIN/gh" <<GHEOF +#!/usr/bin/env bash +case "\$*" in + *"git/ref/tags/auto-rebase/next"*) echo "$cand commit" ;; + *"git/ref/tags/auto-rebase/ring0"*) echo "$cand commit" ;; + *"git/ref/tags/auto-rebase/ring1"*) echo "$cand commit" ;; + *"git/ref/tags/auto-rebase/stable"*) echo "$old commit" ;; + *"matching-refs/tags/auto-rebase/v"*) printf 'refs/tags/auto-rebase/v1.0.0\ttagobj\ttag\n' ;; + *"git/tags/tagobj"*) printf '%s\t%s\n' "$cand" "$cut_iso" ;; + *"run list"*) jq -nc --arg d "$run_iso" --arg c "$conclusion" '[range(20)|{conclusion:\$c,createdAt:\$d}]' ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + # git: a cross-repo agent has NO local refs — every git call resolves empty. If the code + # regressed to the local path, channel_commit would be empty for every ring → the frontier + # would collapse to "fully rolled out" and this test would fail (that is exactly #1049). + cat > "$STUB_BIN/git" <<'GITEOF' +#!/usr/bin/env bash +: # no local refs for a cross-repo agent +GITEOF + chmod +x "$STUB_BIN/git" +} + +@test "orchestrator: cross-repo agent resolves channel+release tags on host → ring1->stable pending, NOT 'fully rolled out' (#1049)" { + # cut 2 days ago (dwell ≫ 12h), ring1 runs 1 day ago all success → sample ≥ 1, clean. + _crossrepo_stub 2 1 success + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate auto-rebase + [ "$status" -eq 0 ] + [[ "$output" != *"fully rolled out"* ]] + [[ "$output" == *"ring1->stable"* ]] + [[ "$output" == *"PROMOTE"* ]] +} + +# ── orchestrator: promote-all — the gated fleet auto-promote (the SCHEDULED arm, #1045b) ─ +@test "orchestrator: promote-all iterates every registry agent and forwards to promote (dry-run, no push)" { + _make_stub_bin + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"run list"*) echo "[]" ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + local pushlog="$STUB_BIN/push.log" + # Reuse the dev-lead git stub but log any push so we can assert the sweep never mutates. + cat > "$STUB_BIN/git" <<GITEOF +#!/usr/bin/env bash +case "\$*" in + *"for-each-ref"*) : ;; + *"push"*) echo "\$*" >> "$pushlog" ;; + *) : ;; +esac +GITEOF + chmod +x "$STUB_BIN/git" + + run env CANARY_RINGS="$RINGS" bash "$ORCH" promote-all --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"promote-all: fleet-wide"* ]] + # Every registry agent gets its own section (the loop covers the whole fleet, not dev-lead only). + [[ "$output" == *"agent: dev-lead"* ]] + [[ "$output" == *"agent: auto-rebase"* ]] + [[ "$output" == *"agent: pr-review-mention"* ]] + # A real move is never pushed under --dry-run. + [ ! -f "$pushlog" ] +} + +# ── orchestrator: cross-repo promote MOVES the channel tag on the HOST via gh api (#1054) ─ +# A cross-repo agent (host = petry-projects/.github) keeps its channel tags on the host, so +# the promote move must go through `gh api PATCH .../git/refs/tags/...`, NOT local `git tag -f` +# (which fails "nonexistent object" for a host commit absent from this checkout — #1054). +@test "orchestrator: cross-repo promote --dry-run shows the host gh-api move, not a local git tag (#1054)" { + _crossrepo_stub 2 1 success # auto-rebase ring1->stable PROMOTE + run env CANARY_RINGS="$RINGS" bash "$ORCH" promote auto-rebase --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"DRY-RUN"* ]] + [[ "$output" == *"gh api PATCH repos/petry-projects/.github/git/refs/tags/auto-rebase/stable"* ]] + [[ "$output" != *"git tag -f"* ]] +} + +_crossrepo_promote_stub() { + # Like _crossrepo_stub but the gh stub LOGS any ref mutation (PATCH/POST) to $MOVE_LOG so + # a REAL promote can be asserted to move the tag on the host (never touching local git). + local cut_days="$1" run_days_ago="$2" conclusion="$3" + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + export MOVE_LOG="$STUB_BIN/move.log" + local cand="cccccccccccccccccccccccccccccccccccccccc" + local old="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + local cut_iso run_iso + cut_iso="$(date -u -d "-${cut_days} days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v"-${cut_days}d" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + run_iso="$(date -u -d "-${run_days_ago} days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v"-${run_days_ago}d" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + cat > "$STUB_BIN/gh" <<GHEOF +#!/usr/bin/env bash +case "\$*" in + *"-X PATCH"*"git/refs/tags/"*) echo "\$*" >> "$MOVE_LOG"; echo "{}"; exit 0 ;; + *"-X POST"*"git/refs"*) echo "\$*" >> "$MOVE_LOG"; echo "{}"; exit 0 ;; + *"git/ref/tags/auto-rebase/next"*) echo "$cand commit" ;; + *"git/ref/tags/auto-rebase/ring0"*) echo "$cand commit" ;; + *"git/ref/tags/auto-rebase/ring1"*) echo "$cand commit" ;; + *"git/ref/tags/auto-rebase/stable"*) echo "$old commit" ;; + *"matching-refs/tags/auto-rebase/v"*) printf 'refs/tags/auto-rebase/v1.0.0\ttagobj\ttag\n' ;; + *"git/tags/tagobj"*) printf '%s\t%s\n' "$cand" "$cut_iso" ;; + *"run list"*) jq -nc --arg d "$run_iso" --arg c "$conclusion" '[range(20)|{conclusion:\$c,createdAt:\$d}]' ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + # A cross-repo agent has NO local refs; if the code regressed to `git tag -f`, this stub + # would record the attempt (and the move.log would stay empty) — the test would fail. + cat > "$STUB_BIN/git" <<GITEOF +#!/usr/bin/env bash +case "\$*" in + *"tag -f"*|*"push"*) echo "LOCAL:\$*" >> "$MOVE_LOG" ;; + *) : ;; +esac +GITEOF + chmod +x "$STUB_BIN/git" +} + +@test "orchestrator: cross-repo promote (real) moves the host channel tag via gh api + records the output (#1054)" { + _crossrepo_promote_stub 2 1 success # auto-rebase ring1->stable PROMOTE + local out="$BATS_TEST_TMPDIR/gh_output"; : > "$out" + local plog="$BATS_TEST_TMPDIR/promotions.tsv"; : > "$plog" + run env CANARY_RINGS="$RINGS" GITHUB_OUTPUT="$out" CANARY_PROMOTIONS_LOG="$plog" bash "$ORCH" promote auto-rebase + [ "$status" -eq 0 ] + [[ "$output" == *"promoted auto-rebase/stable"* ]] + # The move went through gh api PATCH on the HOST, never local git tag/push. + grep -q "PATCH repos/petry-projects/.github/git/refs/tags/auto-rebase/stable" "$MOVE_LOG" + ! grep -q "^LOCAL:" "$MOVE_LOG" + # The move is exposed for the workflow's GitHub Deployment step (#502). + grep -q "promoted_agent=auto-rebase" "$out" + grep -q "promoted_ring=stable" "$out" + # promoted_host is the OWNING repo (the cross-repo host), so the deployment is created + # where the moved commit exists — not GITHUB_REPOSITORY, which would 422 "No ref found" (#1059). + grep -q "promoted_host=petry-projects/.github" "$out" + # The promotions log gets one TSV line per move (agent, ring, sha, owning-repo) so a + # promote-all run can record a deployment for EVERY promotion, not just the last. + grep -qP "^auto-rebase\tstable\t[0-9a-f]+\tpetry-projects/\.github$" "$plog" +} + +@test "orchestrator: cross-repo promote --dry-run shows the host move but touches neither git nor the API (#1054)" { + _crossrepo_promote_stub 2 1 success + run env CANARY_RINGS="$RINGS" bash "$ORCH" promote auto-rebase --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"DRY-RUN"* ]] + [[ "$output" == *"auto-rebase/stable"* ]] + [[ "$output" == *"petry-projects/.github"* ]] + [ ! -f "$MOVE_LOG" ] +} + +# ── orchestrator: sync-issues — auto-triage held promotions into tracked issues ── +# A held (BLOCKED) promotion files/updates ONE idempotent issue per agent with the failing-run +# evidence; a cleared agent's issue auto-closes; the fleet-status table is rendered to the job +# summary. dev-lead-only registry keeps the fleet loop to one agent; the gh stub logs issue ops. +_sync_stub() { + # $1 conclusion (failure→BLOCKED | success→cleared); $2 blocker-list JSON returned by `gh issue list` + local concl="$1" blocker_list="${2:-[]}" + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + export ISSUE_LOG="$STUB_BIN/issue.log"; : > "$ISSUE_LOG" + local cut_iso run_iso + cut_iso="$(date -u -d '-3 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-3d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + run_iso="$(date -u -d '-2 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-2d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + cat > "$STUB_BIN/git" <<GITEOF +#!/usr/bin/env bash +case "\$*" in + *"for-each-ref"*) echo "cccccccccccccccccccccccccccccccccccccccc||$cut_iso" ;; + *"rev-parse"*"cccc"*":"*) echo "blobAAAA" ;; + *"rev-parse"*"bbbb"*":"*) echo "blobAAAA" ;; + *"rev-parse"*"dev-lead/next"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;; + *"rev-parse"*"dev-lead/ring0"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;; + *"rev-parse"*"dev-lead/ring1"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;; + *"rev-parse"*"dev-lead/stable"*) echo "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ;; + *"rev-parse"*) echo "cccccccccccccccccccccccccccccccccccccccc" ;; + *) : ;; +esac +GITEOF + chmod +x "$STUB_BIN/git" + cat > "$STUB_BIN/gh" <<GHEOF +#!/usr/bin/env bash +case "\$*" in + *"run list"*) jq -nc --arg d "$run_iso" --arg c "$concl" '[range(3)|{conclusion:\$c,createdAt:\$d,databaseId:12345,workflowName:"Dev-Lead Agent"}]' ;; + *"run view"*) echo '{"jobs":[{"steps":[{"name":"Some step","conclusion":"failure"}]}]}' ;; + "issue list"*) echo '$blocker_list' ;; + "issue create"*) echo "CREATE|\$*" >> "$ISSUE_LOG"; echo "https://github.com/petry-projects/.github-private/issues/777" ;; + "issue edit"*) echo "EDIT|\$*" >> "$ISSUE_LOG" ;; + "issue close"*) echo "CLOSE|\$*" >> "$ISSUE_LOG" ;; + "issue reopen"*) echo "REOPEN|\$*" >> "$ISSUE_LOG" ;; + "issue pin"*) echo "PIN|\$*" >> "$ISSUE_LOG" ;; + "label create"*) : ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + SYNC_RINGS="$BATS_TEST_TMPDIR/sync-rings.json" + jq '{org_infra_repos, agents: {"dev-lead": .agents["dev-lead"]}}' "$RINGS" > "$SYNC_RINGS" +} + +@test "orchestrator: sync-issues --dry-run plans the blocker + renders status, no GitHub writes" { + _sync_stub failure '[]' # BLOCKED, nothing filed yet + # Pin GITHUB_STEP_SUMMARY to a temp file — under CI the runner sets it, so the table lands + # in the summary, not stdout; asserting the file keeps the test env-independent. + local summ="$BATS_TEST_TMPDIR/summary.md"; : > "$summ" + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" GITHUB_STEP_SUMMARY="$summ" bash "$ORCH" sync-issues --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"would OPEN blocker issue for dev-lead"* ]] + grep -q "Canary Rollout — fleet status" "$summ" # table still renders under --dry-run + [ ! -s "$ISSUE_LOG" ] # dry-run mutates nothing on GitHub +} + +@test "orchestrator: sync-issues opens ONE blocker issue (with evidence) + writes the fleet summary — no dashboard issue" { + _sync_stub failure '[]' + local summ="$BATS_TEST_TMPDIR/summary.md"; : > "$summ" + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" GITHUB_STEP_SUMMARY="$summ" bash "$ORCH" sync-issues + [ "$status" -eq 0 ] + [[ "$output" == *"opened blocker issue #777 for dev-lead"* ]] + [[ "$output" == *"wrote fleet-status table to the job summary"* ]] + # Exactly ONE issue create (the blocker) — the dashboard is NOT an issue anymore. + [ "$(grep -c '^CREATE|' "$ISSUE_LOG" || true)" -eq 1 ] + grep -q -- "--label canary-blocker" "$ISSUE_LOG" + ! grep -q -- "--label canary-dashboard" "$ISSUE_LOG" + ! grep -q "^PIN|" "$ISSUE_LOG" + # The fleet-status table landed in the job summary file. + grep -q "Canary Rollout — fleet status" "$summ" + grep -q "dev-lead" "$summ" +} + +@test "orchestrator: sync-issues survives a failing 'gh issue create' — no abort under set -e, still renders the fleet summary (#1081)" { + # Regression: the blocker-issue create failing (App lacks Issues:write, rate-limit, …) + # must NOT abort the whole step before the dashboard renders. Make `gh issue create` exit + # non-zero and assert graceful degradation: warning logged, table still written, status 0. + _sync_stub failure '[]' + sed 's#"issue create"\*).*#"issue create"*) echo "gh: HTTP 403 (Issues:write?)" >\&2; exit 1 ;;#' "$STUB_BIN/gh" > "$STUB_BIN/gh.tmp" && mv "$STUB_BIN/gh.tmp" "$STUB_BIN/gh" && chmod +x "$STUB_BIN/gh" + local summ="$BATS_TEST_TMPDIR/summary_fail.md"; : > "$summ" + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" GITHUB_STEP_SUMMARY="$summ" bash "$ORCH" sync-issues + [ "$status" -eq 0 ] + [[ "$output" == *"could not open blocker issue for dev-lead (Issues:write on the App?)"* ]] + [[ "$output" == *"wrote fleet-status table to the job summary"* ]] + grep -q "Canary Rollout — fleet status" "$summ" + grep -q "dev-lead" "$summ" +} + +@test "orchestrator: sync-issues auto-closes a cleared agent's open blocker issue" { + # dev-lead now clean (success → not BLOCKED) but an OPEN blocker issue #501 exists → close it. + _sync_stub success '[{"number":501,"state":"OPEN","body":"<!-- canary-blocker:dev-lead -->"}]' + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" bash "$ORCH" sync-issues + [ "$status" -eq 0 ] + [[ "$output" == *"closed cleared blocker issue #501 for dev-lead"* ]] + grep -q "CLOSE|.*501" "$ISSUE_LOG" +} + +@test "orchestrator: sync-issues prepends a separator newline so the fleet-status header is never concatenated to prior summary content" { + # If prior summary content lacks a trailing newline, the fleet-status header must still + # start on its own line — not be appended directly to the prior content. + _sync_stub success '[]' + local summ="$BATS_TEST_TMPDIR/summary_sep.md" + printf 'prior step output (no trailing newline)' > "$summ" + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" GITHUB_STEP_SUMMARY="$summ" bash "$ORCH" sync-issues + [ "$status" -eq 0 ] + # The header must appear at the start of a line — not concatenated onto the prior content. + grep -q '^# Canary Rollout' "$summ" + ! grep -q 'prior.*# Canary Rollout' "$summ" +} + +# ── orchestrator: autocut — auto-cut a new candidate when a reusable changes on main (#1069) ─ +# The front end of the canary pipeline: at each scheduled tick (gated by CANARY_AUTO_CUT), for +# each registered agent compare the reusable blob at the host's main HEAD against the blob at the +# current `next` candidate; if they differ, cut a new immutable vX.Y.Z (patch bump default) and +# move `next` onto it via cut-release.sh. The stub feeds: default_branch, main HEAD, the two blob +# SHAs, the `next` commit (git for a this-repo agent, gh api for a cross-repo one), the existing +# release-tag versions (matching-refs), and a cut-release.sh stand-in (CUT_RELEASE) that logs args. +_autocut_stub() { + # args: agent host reusable main_blob next_blob mainsha nextsha versions_ws [bump] + local agent="$1" host="$2" reusable="$3" MAIN_BLOB="$4" NEXT_BLOB="$5" MAINSHA="$6" NEXTSHA="$7" versions="$8" bump="${9:-}" + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + export CUT_LOG="$STUB_BIN/cut.log"; : > "$CUT_LOG" + export CUT_RELEASE="$STUB_BIN/cut-release" + cat > "$CUT_RELEASE" <<CUTEOF +#!/usr/bin/env bash +echo "\$*" >> "$CUT_LOG" +CUTEOF + chmod +x "$CUT_RELEASE" + local refs="" v + for v in $versions; do refs+="refs/tags/$agent/v$v"$'\n'; done + cat > "$STUB_BIN/gh" <<GHEOF +#!/usr/bin/env bash +case "\$*" in + *".default_branch"*) echo "main" ;; + *"contents/"*"ref=$MAINSHA"*) echo "$MAIN_BLOB" ;; + *"contents/"*"ref=$NEXTSHA"*) echo "$NEXT_BLOB" ;; + *"/commits/"*) echo "$MAINSHA" ;; + *"matching-refs/tags/$agent/v"*) printf '%s' "$refs" ;; + *"git/ref/tags/$agent/next"*) printf '%s\tcommit\n' "$NEXTSHA" ;; + *"run list"*) echo "[]" ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + cat > "$STUB_BIN/git" <<GITEOF +#!/usr/bin/env bash +case "\$*" in + *"rev-parse"*"$agent/next"*) echo "$NEXTSHA" ;; + *) : ;; +esac +GITEOF + chmod +x "$STUB_BIN/git" + AUTOCUT_RINGS="$BATS_TEST_TMPDIR/autocut-rings.json" + if [ -n "$bump" ]; then + jq --arg a "$agent" --arg b "$bump" \ + '{version, description, org_infra_repos, member_tokens, agents: {($a): (.agents[$a] + {autocut: {bump: $b}})}}' \ + "$RINGS" > "$AUTOCUT_RINGS" + else + jq --arg a "$agent" \ + '{version, description, org_infra_repos, member_tokens, agents: {($a): .agents[$a]}}' \ + "$RINGS" > "$AUTOCUT_RINGS" + fi +} + +@test "orchestrator: autocut is a no-op when CANARY_AUTO_CUT is not 'true' (kill-switch off)" { + _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \ + blobMAIN blobNEXT aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa cccccccccccccccccccccccccccccccccccccccc "2.1.0" + run env CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut + [ "$status" -eq 0 ] + [[ "$output" == *"DISABLED"* ]] + [ ! -s "$CUT_LOG" ] # nothing cut when the kill-switch is off +} + +@test "orchestrator: autocut cuts a patch-bumped version + moves next when the reusable blob differs on main" { + _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \ + blobMAIN blobNEXT aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa cccccccccccccccccccccccccccccccccccccccc "2.1.0" + run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut + [ "$status" -eq 0 ] + # Highest existing tag is v2.1.0 → patch bump → v2.1.1, cut from main HEAD, channel next, pushed. + grep -q "dev-lead 2.1.1 --ref aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa --channel next --push" "$CUT_LOG" +} + +@test "orchestrator: autocut is idempotent — identical blob on main and next is a clean no-op" { + _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \ + sameBLOB sameBLOB aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa cccccccccccccccccccccccccccccccccccccccc "2.1.0" + run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut + [ "$status" -eq 0 ] + [[ "$output" == *"no cut"* ]] + [ ! -s "$CUT_LOG" ] # nothing cut when the blob is unchanged +} + +@test "orchestrator: autocut --dry-run prints the intended cut without invoking cut-release --push" { + _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \ + blobMAIN blobNEXT aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa cccccccccccccccccccccccccccccccccccccccc "2.1.0" + run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"DRY-RUN"* ]] + [[ "$output" == *"2.1.1"* ]] + [ ! -s "$CUT_LOG" ] # dry-run never pushes a real cut +} + +@test "orchestrator: autocut honors the registry autocut.bump override (minor)" { + _autocut_stub dev-lead petry-projects/.github-private .github/workflows/dev-lead-reusable.yml \ + blobMAIN blobNEXT aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa cccccccccccccccccccccccccccccccccccccccc "2.1.0" minor + run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut + [ "$status" -eq 0 ] + # minor bump of v2.1.0 → v2.2.0 + grep -q "dev-lead 2.2.0 --ref aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa --channel next --push" "$CUT_LOG" +} + +@test "orchestrator: autocut is cross-repo aware — cuts v2.1.1 for auto-rebase from the host main HEAD (#1069)" { + # auto-rebase is hosted in petry-projects/.github; its next candidate + release tags live there, + # so the next commit is resolved via gh api (not local git) and the cut is cross-repo. + _autocut_stub auto-rebase petry-projects/.github .github/workflows/auto-rebase-reusable.yml \ + ece45480ece45480ece45480ece45480ece45480 2763750027637500276375002763750027637500 \ + ece45480ece45480ece45480ece45480ece45480 2763750027637500276375002763750027637500 "2.1.0" + run env CANARY_AUTO_CUT=true CANARY_RINGS="$AUTOCUT_RINGS" CUT_RELEASE="$CUT_RELEASE" bash "$ORCH" autocut + [ "$status" -eq 0 ] + grep -q "auto-rebase 2.1.1 --ref ece45480ece45480ece45480ece45480ece45480 --channel next --push" "$CUT_LOG" +} + +# ── set_difference (pure set-diff core for drift detection, #1082) ───────────── +# args: <set_a_newlines> <set_b_newlines> — echo lines in A that are NOT in B. +@test "set_difference: A minus B keeps only A-only elements" { + run set_difference "$(printf 'a\nb\nc\n')" "$(printf 'b\n')" + [ "$status" -eq 0 ] + [ "$output" = "$(printf 'a\nc')" ] +} +@test "set_difference: no overlap returns all of A" { + run set_difference "$(printf 'x\ny\n')" "$(printf 'p\nq\n')" + [ "$output" = "$(printf 'x\ny')" ] +} +@test "set_difference: full overlap returns nothing" { + run set_difference "$(printf 'a\nb\n')" "$(printf 'a\nb\n')" + [ -z "$output" ] +} +@test "set_difference: empty A returns nothing" { + run set_difference "" "$(printf 'a\n')" + [ -z "$output" ] +} +@test "set_difference: empty B returns all of A (nothing removed)" { + run set_difference "$(printf 'a\nb\n')" "" + [ "$output" = "$(printf 'a\nb')" ] +} +@test "set_difference: matches whole lines only (a path is not a prefix match)" { + # '.github/workflows/foo-reusable.yml' must not be swallowed by a partial 'foo'. + run set_difference "$(printf '.github/workflows/foo-reusable.yml\n')" "$(printf 'foo\n')" + [ "$output" = ".github/workflows/foo-reusable.yml" ] +} + +# ── orchestrator: drift — registry vs host reusables (read-only audit, #1082) ── +# The registry (.agents{}) is the MANUAL source of truth for what the canary pipeline +# manages. `drift` scans each registered host repo's .github/workflows/*-reusable.yml +# and diffs it against the registry so an unregistered reusable (zero staged rollout) OR +# a registry entry pointing at a deleted reusable surfaces within one scheduled cycle. +# +# The stub answers `gh api repos/<host>/contents/.github/workflows` with a per-host JSON +# array (env HOSTPRIV_JSON / HOSTPUB_JSON); the orchestrator filters *-reusable.yml itself. +_drift_stub() { + # $1 = JSON array for petry-projects/.github-private ; $2 = JSON array for petry-projects/.github + local priv_json="$1" pub_json="${2:-[]}" + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + # '.github' is a substring of '.github-private', so match the more specific repo FIRST. + cat > "$STUB_BIN/gh" <<GHEOF +#!/usr/bin/env bash +case "\$*" in + *"repos/petry-projects/.github-private/contents/.github/workflows"*) cat <<'JSON' +$priv_json +JSON + ;; + *"repos/petry-projects/.github/contents/.github/workflows"*) cat <<'JSON' +$pub_json +JSON + ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + cat > "$STUB_BIN/git" <<'GITEOF' +#!/usr/bin/env bash +: # drift never touches git +GITEOF + chmod +x "$STUB_BIN/git" +} + +# A registry with a single this-repo agent (dev-lead) so the present/registered sets are +# fully controlled: registered on .github-private = {dev-lead-reusable.yml}, none on .github. +_drift_rings_one_agent() { + DRIFT_RINGS="$BATS_TEST_TMPDIR/drift-rings.json" + jq '{version, description, org_infra_repos, member_tokens, agents: {"dev-lead": .agents["dev-lead"]}}' \ + "$RINGS" > "$DRIFT_RINGS" +} + +@test "orchestrator: drift flags a reusable present on a host but absent from the registry (unregistered)" { + _drift_rings_one_agent + # .github-private hosts an EXTRA foo-reusable.yml that no .agents{} block registers. + _drift_stub '[ + {"type":"file","name":"dev-lead-reusable.yml","path":".github/workflows/dev-lead-reusable.yml"}, + {"type":"file","name":"foo-reusable.yml","path":".github/workflows/foo-reusable.yml"}, + {"type":"file","name":"ci.yml","path":".github/workflows/ci.yml"} + ]' '[]' + run env CANARY_RINGS="$DRIFT_RINGS" bash "$ORCH" drift + [ "$status" -eq 0 ] + [[ "$output" == *"DRIFT[unregistered]"* ]] + [[ "$output" == *".github/workflows/foo-reusable.yml"* ]] + # the registered reusable and the non-reusable ci.yml are NOT flagged + [[ "$output" != *"DRIFT[unregistered] petry-projects/.github-private: .github/workflows/dev-lead-reusable.yml"* ]] + [[ "$output" != *"ci.yml present on host"* ]] +} + +@test "orchestrator: drift flags a registry entry whose reusable file no longer exists on the host (missing-file)" { + # Registry has 'ghost' pointing at a reusable that is NOT present on the host. + DRIFT_RINGS="$BATS_TEST_TMPDIR/drift-ghost.json" + jq '{version, description, org_infra_repos, member_tokens, + agents: {"ghost": (.agents["dev-lead"] + {reusable: ".github/workflows/ghost-reusable.yml"})}}' \ + "$RINGS" > "$DRIFT_RINGS" + # Host lists only an unrelated (registered-elsewhere-none) file — ghost-reusable.yml is gone. + _drift_stub '[ + {"type":"file","name":"dev-lead-reusable.yml","path":".github/workflows/dev-lead-reusable.yml"} + ]' '[]' + run env CANARY_RINGS="$DRIFT_RINGS" bash "$ORCH" drift + [ "$status" -eq 0 ] + [[ "$output" == *"DRIFT[missing-file]"* ]] + [[ "$output" == *"ghost"* ]] + [[ "$output" == *".github/workflows/ghost-reusable.yml"* ]] +} + +@test "orchestrator: drift reports NO drift when the registry and host reusables are in sync" { + _drift_rings_one_agent + # Host lists exactly the one registered reusable — nothing extra, nothing missing. + _drift_stub '[ + {"type":"file","name":"dev-lead-reusable.yml","path":".github/workflows/dev-lead-reusable.yml"} + ]' '[]' + run env CANARY_RINGS="$DRIFT_RINGS" bash "$ORCH" drift + [ "$status" -eq 0 ] + [[ "$output" != *"DRIFT["* ]] + [[ "$output" == *"no reusable drift"* ]] + [[ "$output" == *"0 unregistered, 0 missing-file"* ]] +} + +@test "orchestrator: drift --emit-stub prints a scaffold .agents[<name>] block for an unregistered reusable" { + _drift_rings_one_agent + _drift_stub '[ + {"type":"file","name":"dev-lead-reusable.yml","path":".github/workflows/dev-lead-reusable.yml"}, + {"type":"file","name":"foo-reusable.yml","path":".github/workflows/foo-reusable.yml"} + ]' '[]' + run env CANARY_RINGS="$DRIFT_RINGS" bash "$ORCH" drift --emit-stub + [ "$status" -eq 0 ] + # A scaffold JSON object keyed by the derived agent name 'foo' the maintainer can fill in. + [[ "$output" == *"\"foo\""* ]] + [[ "$output" == *"\"reusable\": \".github/workflows/foo-reusable.yml\""* ]] + [[ "$output" == *"petry-projects/.github-private"* ]] +} + +@test "orchestrator: drift skips a host it cannot enumerate — no false missing-file avalanche" { + # Registry has 'ghost' on .github-private, but the contents API returns a non-array error + # body (no access / API error). The host must be SKIPPED, NOT reported as every registered + # reusable having been deleted. + DRIFT_RINGS="$BATS_TEST_TMPDIR/drift-noaccess.json" + jq '{version, description, org_infra_repos, member_tokens, + agents: {"ghost": (.agents["dev-lead"] + {reusable: ".github/workflows/ghost-reusable.yml"})}}' \ + "$RINGS" > "$DRIFT_RINGS" + _drift_stub '{"message":"Not Found"}' '[]' + run env CANARY_RINGS="$DRIFT_RINGS" bash "$ORCH" drift + [ "$status" -eq 0 ] + [[ "$output" == *"could not enumerate"* ]] + [[ "$output" != *"DRIFT[missing-file]"* ]] + [[ "$output" == *"0 unregistered, 0 missing-file"* ]] +} + +@test "orchestrator: drift renders a fleet-drift table into the job summary when GITHUB_STEP_SUMMARY is set" { + _drift_rings_one_agent + _drift_stub '[ + {"type":"file","name":"dev-lead-reusable.yml","path":".github/workflows/dev-lead-reusable.yml"}, + {"type":"file","name":"foo-reusable.yml","path":".github/workflows/foo-reusable.yml"} + ]' '[]' + local summ="$BATS_TEST_TMPDIR/drift-summary.md"; : > "$summ" + run env CANARY_RINGS="$DRIFT_RINGS" GITHUB_STEP_SUMMARY="$summ" bash "$ORCH" drift + [ "$status" -eq 0 ] + grep -q "Canary Rollout — reusable drift" "$summ" + grep -q "foo-reusable.yml" "$summ" +} diff --git a/tests/test_bootstrap_new_repo.bats b/tests/test_bootstrap_new_repo.bats new file mode 100644 index 000000000..a141bdd0c --- /dev/null +++ b/tests/test_bootstrap_new_repo.bats @@ -0,0 +1,266 @@ +#!/usr/bin/env bats +# Unit tests for scripts/bootstrap-new-repo.sh — the DRY_RUN-aware bootstrap that +# orchestrates the existing apply-* scripts to bring a new repo to full org +# compliance (issue #967, epic #964). Mirrors tests/test_apply_rulesets.bats. + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)" +BOOTSTRAP="$SCRIPT_DIR/scripts/bootstrap-new-repo.sh" + +setup() { + STUB_BIN="$(mktemp -d)" || { echo "Failed to create STUB_BIN" >&2; exit 1; } + export PATH="$STUB_BIN:$PATH" + CALLS="$STUB_BIN/calls.log"; export CALLS + STUB_DIR="$(mktemp -d)" || { echo "Failed to create STUB_DIR" >&2; exit 1; } + # Fleet rulesets (code-quality, pr-quality) now live in petry-projects/.github + # (relocated under #575), so the REAL apply-rulesets.sh runs in fleet mode. Point + # it at a local fixture so resolution needs no network; the authoritative content + # is validated in .github. release-channel-tags stays repo-local and is NOT + # applied by bootstrap. + FLEET_DIR="$(mktemp -d)" || { echo "Failed to create FLEET_DIR" >&2; exit 1; } + printf '{"name":"code-quality","target":"branch","enforcement":"active"}\n' > "$FLEET_DIR/code-quality.json" + printf '{"name":"pr-quality","target":"branch","enforcement":"active"}\n' > "$FLEET_DIR/pr-quality.json" + export FLEET_RULESETS_DIR="$FLEET_DIR" + # Canonical apply-repo-settings.sh (petry-projects/.github) requires a token and + # fetches current state even under dry-run (to diff). gh is stubbed, so this dummy + # is never used for a real call — it only satisfies the guard. + export GH_TOKEN=dummy +} +teardown() { + [ -n "${STUB_BIN:-}" ] && rm -rf "$STUB_BIN" + [ -n "${STUB_DIR:-}" ] && rm -rf "$STUB_DIR" + [ -n "${FLEET_DIR:-}" ] && rm -rf "$FLEET_DIR" + return 0 +} + +# gh stub: records every write (POST/PUT/PATCH/label create) to $CALLS; returns a +# non-empty stub object for reads so the orchestration never fails on a read. The +# object is deliberately non-"{}" (canonical apply-repo-settings.sh treats "{}" as a +# failed settings fetch and aborts) and carries NONE of the desired setting values, +# so every dry-run diff still prints its full intent. +_stub_gh() { + cat > "$STUB_BIN/gh" <<EOF +#!/usr/bin/env bash +args="\$*" +case "\$args" in + *"--method POST"*|*"--method PUT"*|*"PATCH"*|*"label create"*|*"-X PUT"*) echo "\$args" >> "$CALLS" ;; +esac +echo '{"full_name":"stub/repo"}' +EOF + chmod +x "$STUB_BIN/gh" +} + +# Stub the two sub-scripts so behavior (exit code) is deterministic. Each stub +# logs its invocation to $CALLS and exits with $1 (default 0). +_stub_substeps() { + local settings_rc="${1:-0}" rulesets_rc="${2:-0}" + cat > "$STUB_DIR/apply-repo-settings.sh" <<EOF +#!/usr/bin/env bash +echo "apply-repo-settings called: \$*" >> "$CALLS" +exit $settings_rc +EOF + cat > "$STUB_DIR/apply-rulesets.sh" <<EOF +#!/usr/bin/env bash +echo "apply-rulesets called: \$*" >> "$CALLS" +exit $rulesets_rc +EOF + chmod +x "$STUB_DIR/apply-repo-settings.sh" "$STUB_DIR/apply-rulesets.sh" + export APPLY_REPO_SETTINGS="$STUB_DIR/apply-repo-settings.sh" + export APPLY_RULESETS="$STUB_DIR/apply-rulesets.sh" +} + +# NOTE: the codified pr-quality.json / code-quality.json shape assertions moved out +# of this suite with the JSONs themselves — the fleet rulesets are now owned by +# petry-projects/.github (standards/rulesets/, relocated under #575) and their +# content is validated there. This suite covers bootstrap's ORCHESTRATION: that it +# applies exactly the two fleet rulesets (via fleet-mode apply-rulesets.sh) and not +# the repo-local release-channel-tags. + +# ── orchestration: DRY_RUN path ─────────────────────────────────────────────── +@test "DRY_RUN: exits 0 and makes no write API calls" { + _stub_gh + run env DRY_RUN=true bash "$BOOTSTRAP" owner/new-repo + [ "$status" -eq 0 ] + [ ! -f "$CALLS" ] +} + +@test "DRY_RUN: prints intent and a PASS summary" { + _stub_gh + run env DRY_RUN=true bash "$BOOTSTRAP" owner/new-repo + [ "$status" -eq 0 ] + [[ "$output" == *"dry-run"* ]] || [[ "$output" == *"DRY_RUN"* ]] + [[ "$output" == *"owner/new-repo"* ]] + [[ "$output" == *"PASS"* ]] +} + +@test "DRY_RUN: bridges the flag onto both sub-scripts (DEV_LEAD_DRY_RUN + DRY_RUN)" { + # Use real sub-scripts but a gh stub; in dry-run neither makes write calls. + _stub_gh + run env DRY_RUN=true bash "$BOOTSTRAP" owner/new-repo + [ "$status" -eq 0 ] + # apply-repo-settings dry-run prints its own intent line; apply-rulesets too. + [[ "$output" == *"apply-rulesets"* ]] || [[ "$output" == *"rulesets"* ]] +} + +# ── orchestration: sequencing + fail-fast + summary ─────────────────────────── +@test "sequence: invokes apply-repo-settings then apply-rulesets --repo on success" { + _stub_gh + _stub_substeps 0 0 + run bash "$BOOTSTRAP" owner/new-repo + [ "$status" -eq 0 ] + grep -q "apply-repo-settings called" "$CALLS" + grep -q "apply-rulesets called: --repo owner/new-repo" "$CALLS" + # order: settings line must precede rulesets line + settings_ln="$(grep -n 'apply-repo-settings called' "$CALLS" | head -1 | cut -d: -f1)" + rulesets_ln="$(grep -n 'apply-rulesets called' "$CALLS" | head -1 | cut -d: -f1)" + [ "$settings_ln" -lt "$rulesets_ln" ] +} + +@test "fail-fast: apply-repo-settings failure stops before apply-rulesets and exits non-zero" { + _stub_gh + _stub_substeps 1 0 + run bash "$BOOTSTRAP" owner/new-repo + [ "$status" -ne 0 ] + grep -q "apply-repo-settings called" "$CALLS" + ! grep -q "apply-rulesets called" "$CALLS" + [[ "$output" == *"FAIL"* ]] +} + +@test "fail-fast: apply-rulesets failure exits non-zero with a FAIL summary" { + _stub_gh + _stub_substeps 0 1 + run bash "$BOOTSTRAP" owner/new-repo + [ "$status" -ne 0 ] + grep -q "apply-rulesets called" "$CALLS" + [[ "$output" == *"FAIL"* ]] +} + +@test "summary: success prints a PASS summary naming the repo" { + _stub_gh + _stub_substeps 0 0 + run bash "$BOOTSTRAP" owner/new-repo + [ "$status" -eq 0 ] + [[ "$output" == *"PASS"* ]] + [[ "$output" == *"owner/new-repo"* ]] +} + +# ── argument handling ───────────────────────────────────────────────────────── +@test "errors when no repo argument is given" { + _stub_gh + run bash "$BOOTSTRAP" + [ "$status" -ne 0 ] +} + +# ── release-ring confirmation + registration (#968) ─────────────────────────── +# A throwaway copy of the real ring SoT so non-stable tests can assert intent +# without mutating the tracked file. +_ring_sot_copy() { + RING_SOT="$STUB_DIR/canary-rings.json" + cp "$SCRIPT_DIR/standards/canary-rings.json" "$RING_SOT" + export CANARY_RINGS="$RING_SOT" +} + +@test "ring: defaults to stable and records an auditable decision (record-only)" { + _stub_gh + _stub_substeps 0 0 + _ring_sot_copy + run env GITHUB_ACTOR=octocat bash "$BOOTSTRAP" owner/new-repo + [ "$status" -eq 0 ] + [[ "$output" == *"[ring-audit]"* ]] + [[ "$output" == *"ring=stable"* ]] + [[ "$output" == *"operator=octocat"* ]] + [[ "$output" == *"decision=recorded"* ]] + # stable is record-only: no central-file edit intent, SoT untouched + [[ "$output" != *"would add owner/new-repo"* ]] + run diff "$SCRIPT_DIR/standards/canary-rings.json" "$RING_SOT" + [ "$status" -eq 0 ] +} + +@test "ring: rejects an unknown ring value" { + _stub_gh + _stub_substeps 0 0 + _ring_sot_copy + run bash "$BOOTSTRAP" --ring nope owner/new-repo + [ "$status" -ne 0 ] + [[ "$output" == *"unknown ring"* ]] +} + +@test "ring: non-stable (DRY_RUN) records the decision and prints both-file + stub intent" { + # Real sub-scripts under dry-run make no writes, so $CALLS stays a pure + # gh-write ledger (a stubbed sub-script would log to it and break the check). + _stub_gh + _ring_sot_copy + run env DRY_RUN=true GITHUB_ACTOR=octocat bash "$BOOTSTRAP" --ring ring1 owner/new-repo + [ "$status" -eq 0 ] + # auditable record, even in dry-run + [[ "$output" == *"[ring-audit]"* ]] + [[ "$output" == *"ring=ring1"* ]] + [[ "$output" == *"decision=registered"* ]] + # central file 1: this repo's canary-rings.json + [[ "$output" == *"would add owner/new-repo"* ]] + [[ "$output" == *"ring1"* ]] + # central file 2: cross-repo ring-pins.sh in petry-projects/.github + [[ "$output" == *"ring-pins.sh"* ]] + [[ "$output" == *"petry-projects/.github"* ]] + # stub repin to the matching channel tag + [[ "$output" == *"@dev-lead/ring1"* ]] + # dry-run makes no write API calls and does not mutate the SoT + [ ! -f "$CALLS" ] + run diff "$SCRIPT_DIR/standards/canary-rings.json" "$RING_SOT" + [ "$status" -eq 0 ] +} + +@test "ring: non-stable consistency holds — repo lands in the named ring and no other" { + _stub_gh + _stub_substeps 0 0 + _ring_sot_copy + # Proposed post-state must place owner/new-repo in ring1 and nowhere else. + run env DRY_RUN=true bash "$BOOTSTRAP" --ring ring1 owner/new-repo + [ "$status" -eq 0 ] + [[ "$output" == *"ring consistency OK"* ]] +} + +# ── end-to-end DRY_RUN validation: the full policy surface in one run (#970) ──── +# A single DRY_RUN walkthrough with the REAL sub-scripts (only `gh` stubbed) must +# describe the entire intended state — recorded ring, repo settings + GHAS + push +# protection, both sanctioned rulesets, the standard labels, CODEOWNERS team — and +# emit zero write API calls (no drift). This is the executable form of the +# end-to-end validation recorded in docs/bootstrap/new-repo-validation.md. +@test "e2e DRY_RUN: covers the whole intended-state surface with no write calls (#970 AC #2/#3)" { + _stub_gh + _ring_sot_copy + run env DRY_RUN=true GITHUB_ACTOR=octocat CANARY_RINGS="$RING_SOT" \ + bash "$BOOTSTRAP" petry-projects/acme-service + [ "$status" -eq 0 ] + + # (1/5) ring — auditable record, default stable, record-only. + [[ "$output" == *"[ring-audit]"* ]] + [[ "$output" == *"ring=stable"* ]] + [[ "$output" == *"decision=recorded"* ]] + + # (2/5) repo settings — security_and_analysis + secret-scanning push protection, + # and the Claude/CodeRabbit check-suite auto-trigger disable. + [[ "$output" == *"security_and_analysis"* ]] + [[ "$output" == *"secret_scanning_push_protection"* ]] + # canonical apply-repo-settings.sh exercises the check-suite auto-trigger step and, + # under dry-run, marks it skip (message wording: "... check-suite prefs ..."). + [[ "$output" == *"check-suite prefs"* ]] + + # (3/5) exactly the two sanctioned FLEET rulesets are applied — and NOT the + # release-channel-tags org tag standard (applied by name only, never in fleet mode). + [[ "$output" == *"pr-quality"* ]] + [[ "$output" == *"code-quality"* ]] + [[ "$output" != *"release-channel-tags"* ]] + [[ "$output" == *"done (2 ruleset(s))"* ]] + + # (4/5) the standard label set. + [[ "$output" == *"needs-human-review"* ]] + [[ "$output" == *"ack-test-deletion"* ]] + + # (5/5) CODEOWNERS team verification. + [[ "$output" == *"CODEOWNERS"* ]] + [[ "$output" == *"org-leads"* ]] + + # PASS summary + the no-drift invariant: a pure DRY_RUN makes no write API calls. + [[ "$output" == *"PASS"* ]] + [ ! -f "$CALLS" ] +} From 03ccf84a7bb81e834112c77686fa73ce18714567 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:49:03 +0000 Subject: [PATCH 2/3] chore: dev-lead update (review-changes) [skip ci-relay] --- .github/workflows/canary-rollout.yml | 4 +- .github/workflows/feature-ideation-tests.yml | 2 +- docs/bootstrap/new-repo-validation.md | 4 +- docs/initiatives/canary-rollout-adr.md | 65 ++++++++++++++------ 4 files changed, 52 insertions(+), 23 deletions(-) diff --git a/.github/workflows/canary-rollout.yml b/.github/workflows/canary-rollout.yml index 1951fb619..af371e0d7 100644 --- a/.github/workflows/canary-rollout.yml +++ b/.github/workflows/canary-rollout.yml @@ -107,7 +107,9 @@ jobs: run: | set -euo pipefail if [ -z "${APP_TOKEN}" ]; then - echo "::error::Canary rollout requires a GitHub App token (RELEASE_MANAGER_APP_CLIENT_ID (variable) + RELEASE_MANAGER_APP_KEY (secret)). Refusing to fall back to github.token, which cannot move protected .github channel tags (#868)." + echo "::error::Canary rollout requires a GitHub App token" \ + "(RELEASE_MANAGER_APP_CLIENT_ID (variable) + RELEASE_MANAGER_APP_KEY (secret))." \ + "Refusing to fall back to github.token, which cannot move protected .github channel tags (#868)." exit 1 fi diff --git a/.github/workflows/feature-ideation-tests.yml b/.github/workflows/feature-ideation-tests.yml index 9bb808286..e3ae51a12 100644 --- a/.github/workflows/feature-ideation-tests.yml +++ b/.github/workflows/feature-ideation-tests.yml @@ -82,7 +82,7 @@ jobs: - name: Lint direct prompt blocks run: bash .github/scripts/feature-ideation/lint-prompt.sh - - name: Lint caller stubs for the #571 inputs-context startup failure + - name: "Lint caller stubs for the #571 inputs-context startup failure" run: bash .github/scripts/feature-ideation/lint-caller.sh - name: Validate schema fixtures diff --git a/docs/bootstrap/new-repo-validation.md b/docs/bootstrap/new-repo-validation.md index 11e4808e3..b8c37c548 100644 --- a/docs/bootstrap/new-repo-validation.md +++ b/docs/bootstrap/new-repo-validation.md @@ -42,7 +42,7 @@ intended-state surface with no write calls (#970 AC #2/#3)"_. ### Default ring (`stable`) -``` +```text $ DRY_RUN=true GITHUB_ACTOR=octocat bash scripts/bootstrap-new-repo.sh petry-projects/acme-service [bootstrap] repo=petry-projects/acme-service dry_run=true ring=dev-lead/stable [bootstrap] (1/5) release ring confirmation (dev-lead/stable) @@ -74,7 +74,7 @@ $ DRY_RUN=true GITHUB_ACTOR=octocat bash scripts/bootstrap-new-repo.sh petry-pro The ring step additionally registers the repo in both central files and repins the caller stub — and asserts no drift before writing: -``` +```text [bootstrap] (1/5) release ring confirmation (dev-lead/ring1) [ring-audit] repo=petry-projects/acme-service agent=dev-lead ring=ring1 operator=octocat at=... decision=registered [dry-run] would add petry-projects/acme-service to dev-lead 'ring1' members in canary-rings.json diff --git a/docs/initiatives/canary-rollout-adr.md b/docs/initiatives/canary-rollout-adr.md index a59599aa4..60283b23c 100644 --- a/docs/initiatives/canary-rollout-adr.md +++ b/docs/initiatives/canary-rollout-adr.md @@ -1,24 +1,33 @@ # ADR: Ring-staged, health-gated canary rollout of agent releases -**Status:** Accepted — records the *implemented* model (initiative [#495](https://github.com/petry-projects/.github-private/issues/495), issues #499–#502, #868). Supersedes the aspirational "rings" sketch and the "Rollout status / next phase" caveat in `petry-projects/.github` → `standards/ci-standards.md`. +**Status:** Accepted — records the *implemented* model (initiative [#495](https://github.com/petry-projects/.github-private/issues/495), +issues #499–#502, #868). Supersedes the aspirational "rings" sketch and the "Rollout status / next phase" caveat in +`petry-projects/.github` → `standards/ci-standards.md`. **Author:** dev-lead / Claude Code **Date:** 2026-07-01 -**Scope (confirmed):** The delivery path for the versioned agents (`dev-lead`, `pr-review`, and the cross-repo `feature-ideation`) — how a new immutable release is promoted from the host repo out to the whole fleet, and how it is rolled back. Records decisions; does not re-derive the initiative analysis (see [`agentic-release-strategy.md`](./agentic-release-strategy.md)). +**Scope (confirmed):** The delivery path for the versioned agents (`dev-lead`, `pr-review`, and the cross-repo `feature-ideation`) — how a new +immutable release is promoted from the host repo out to the whole fleet, and how it is rolled back. Records decisions; does not re-derive +the initiative analysis (see [`agentic-release-strategy.md`](./agentic-release-strategy.md)). **Constraints (confirmed):** GitHub-native only — moving **channel tags** on first-party reusables, no external infrastructure. -The machine-readable ring map is `standards/canary-rings.json`; the gate logic is `scripts/lib/canary-rollout.sh`; the orchestrator is `scripts/canary-rollout.sh` driven by `.github/workflows/canary-rollout.yml`. This ADR mirrors those artifacts — it does not restate their contents as a second source of truth. +The machine-readable ring map is `standards/canary-rings.json`; the gate logic is `scripts/lib/canary-rollout.sh`; the orchestrator is +`scripts/canary-rollout.sh` driven by `.github/workflows/canary-rollout.yml`. This ADR mirrors those artifacts — it does not restate their contents as a second source of truth. --- ## 1. Context -The release-strategy initiative ([#495](https://github.com/petry-projects/.github-private/issues/495)) replaced a floating-`@main` deploy — where a merge was instantly live across the agents' own self-review duty *and* every consumer repo, with no canary and no rollback — -with a **versioned-release model**: immutable `<agent>/vX.Y.Z` releases plus **moving channel tags** (`<agent>/next`, `/ring0`, `/ring1`, `/stable`) that callers pin to. See [`agentic-release-strategy.md`](./agentic-release-strategy.md) for the full analysis and [`docs/release/versioning.md`](../release/versioning.md) for the tag mechanics. +The release-strategy initiative ([#495](https://github.com/petry-projects/.github-private/issues/495)) replaced a floating-`@main` deploy — +where a merge was instantly live across the agents' own self-review duty *and* every consumer repo, with no canary and no rollback — +with a **versioned-release model**: immutable `<agent>/vX.Y.Z` releases plus **moving channel tags** (`<agent>/next`, `/ring0`, `/ring1`, `/stable`) that callers pin to. +See [`agentic-release-strategy.md`](./agentic-release-strategy.md) for the full analysis and [`docs/release/versioning.md`](../release/versioning.md) for the tag mechanics. -Once #499–#502 and #868 landed, the concentric rings, the soak gate, and the promotion/rollback workflow were **built and machine-backed** — no longer aspirational. This ADR records the decisions those artifacts encode so the public-repo standard can be promoted to match (and link here) instead of carrying a "not yet built" note. +Once #499–#502 and #868 landed, the concentric rings, the soak gate, and the promotion/rollback workflow were **built and machine-backed** — no longer aspirational. +This ADR records the decisions those artifacts encode so the public-repo standard can be promoted to match (and link here) instead of carrying a "not yet built" note. ## 2. Decision 1 — the ring map is host-relative and data-driven -[`standards/canary-rings.json`](../../standards/canary-rings.json) is the single source of truth read by [`scripts/canary-rollout.sh`](../../scripts/canary-rollout.sh). Ring membership is **host-relative**, resolved from three member tokens rather than hard-coded repo lists: +[`standards/canary-rings.json`](../../standards/canary-rings.json) is the single source of truth read by +[`scripts/canary-rollout.sh`](../../scripts/canary-rollout.sh). Ring membership is **host-relative**, resolved from three member tokens rather than hard-coded repo lists: | Channel | Order | Members (token) | Resolves to (for `dev-lead`, host = `.github-private`) | |---|---|---|---| @@ -28,18 +37,23 @@ Once #499–#502 and #868 landed, the concentric rings, the soak gate, and the p | `stable`| 3 | `*` — every other consumer not named in an earlier ring | the rest of the fleet | Rationale for host-relative tokens: an agent's reusable can live in either org-infra repo (`dev-lead`/`pr-review` are hosted here; `feature-ideation`'s reusable is hosted in the public `.github`). -`next` always means "the host dogfoods first," and `ring0` always means "the *other* org-infra repo next" — so `next` + `ring0` together always cover both org-infra repos, whichever hosts the agent. Hard-coding `ring0 = .github + .github-private` would be wrong for a host that is already sitting in `next`. +`next` always means "the host dogfoods first," and `ring0` always means "the *other* org-infra repo next" — so `next` + `ring0` +together always cover both org-infra repos, whichever hosts the agent. Hard-coding `ring0 = .github + .github-private` would be wrong for a host that is already sitting in `next`. ## 3. Decision 2 — soak / health gate rules -The promotion gate is a pure, unit-tested decision core ([`scripts/lib/canary-rollout.sh`](../../scripts/lib/canary-rollout.sh); tests [`tests/canary_rollout.bats`](../../tests/canary_rollout.bats)). A candidate `vX.Y.Z` advances to the next ring only once the rings **already on** the candidate have, over a trailing **7-day** window (`SOAK_WINDOW_DAYS = 7`): +The promotion gate is a pure, unit-tested decision core ([`scripts/lib/canary-rollout.sh`](../../scripts/lib/canary-rollout.sh); +tests [`tests/canary_rollout.bats`](../../tests/canary_rollout.bats)). A candidate `vX.Y.Z` advances to the next ring only once the rings +**already on** the candidate have, over a trailing **7-day** window (`SOAK_WINDOW_DAYS = 7`): - **Volume:** `healthy_runs >= min_healthy_runs`, where `min_healthy_runs = ceil(baseline_runs / 7)` — roughly one trailing day's worth of the prior version's run volume. - **Quality:** candidate failure-rate `<=` baseline failure-rate (compared in integer per-mille to stay in pure-bash arithmetic). Two rules make the gate safe under low traffic: -- **No synthetic floor / "we still wait."** A ring with no healthy candidate volume simply never advances — the candidate parks at the frontier rather than being waved through on an arbitrary minimum. `baseline_runs == 0` yields `min_healthy_runs == 0`, but the `healthy_runs > 0` guard still blocks promotion on zero real volume. +- **No synthetic floor / "we still wait."** A ring with no healthy candidate volume simply never advances — the candidate parks at the frontier + rather than being waved through on an arbitrary minimum. `baseline_runs == 0` yields `min_healthy_runs == 0`, but the `healthy_runs > 0` guard + still blocks promotion on zero real volume. - **Quality is checked first.** A failure-rate breach is evaluated before the volume threshold, so a regression is never masked by "not enough runs yet." ### Gate states @@ -53,17 +67,25 @@ Two rules make the gate safe under low traffic: ### Cadence -[`.github/workflows/canary-rollout.yml`](../../.github/workflows/canary-rollout.yml) runs the gate on a **4-hourly** schedule (`cron: "0 */4 * * *"`) in **read-only `evaluate` mode only** — it emits the per-version gate + health report (#502 observability) and **never** promotes on the timer. Promotion and rollback are `workflow_dispatch`-only and human-gated. +[`.github/workflows/canary-rollout.yml`](../../.github/workflows/canary-rollout.yml) runs the gate on a **4-hourly** schedule +(`cron: "0 */4 * * *"`) in **read-only `evaluate` mode only** — it emits the per-version gate + health report (#502 observability) +and **never** promotes on the timer. Promotion and rollback are `workflow_dispatch`-only and human-gated. ## 4. Decision 3 — promotion is a single gated channel-tag move; rollback moves the tag back The **channel tags are the rollout state.** There is no separate state store: -- **Promotion** = advancing one ring by **moving one channel tag** to the candidate `vX.Y.Z`, innermost-out along the ordered list `next → ring0 → ring1 → stable` (`next_channel_in_order`). The **only** mutation the orchestrator performs is `git tag -f` + push; it never writes consumer files. -- **Rollback** = moving a channel tag **back** to a prior immutable release (`rollback <agent> <ring> --to vX.Y.Z`). Because immutable `vX.Y.Z` tags never move, they are the audit trail and the rollback targets. +- **Promotion** = advancing one ring by **moving one channel tag** to the candidate `vX.Y.Z`, innermost-out along the ordered list + `next → ring0 → ring1 → stable` (`next_channel_in_order`). The **only** mutation the orchestrator performs is `git tag -f` + push; it never writes consumer files. +- **Rollback** = moving a channel tag **back** to a prior immutable release (`rollback <agent> <ring> --to vX.Y.Z`). Because immutable + `vX.Y.Z` tags never move, they are the audit trail and the rollback targets. -The mover credential is **`GH_PAT_WORKFLOWS`** (the workflow falls back to `GITHUB_TOKEN` when the PAT is unset, but only the PAT satisfies the `release-channel-tags` ruleset bypass scoped to this workflow's identity (#868)) — promotion and rollback effectively require the PAT; agents running only as `GITHUB_TOKEN` cannot push protected release tags. -This is the sanctioned, documented exception to the SHA-pin standard for **first-party** channel tags (see [AGENTS.md → "Release channel tags & the mutable-ref exception"](../../AGENTS.md)); compliance audits must not flag `@<agent>/<channel>` pins on first-party callers as unpinned. +The mover credential is **`GH_PAT_WORKFLOWS`** (the workflow falls back to `GITHUB_TOKEN` when the PAT is unset, but only the PAT satisfies +the `release-channel-tags` ruleset bypass scoped to this workflow's identity (#868)) — promotion and rollback effectively require the PAT; +agents running only as `GITHUB_TOKEN` cannot push protected release tags. +This is the sanctioned, documented exception to the SHA-pin standard for **first-party** channel tags (see +[AGENTS.md → "Release channel tags & the mutable-ref exception"](../../AGENTS.md)); compliance audits must not flag `@<agent>/<channel>` pins +on first-party callers as unpinned. ## 5. Decision 4 — inner-ring repos pin to their ring channel, not `stable` @@ -76,8 +98,10 @@ For soak to observe a candidate, a ring's repos must actually *run* it. So stub- | `ring1` repos (`TalkTerm`, `bmad-bgreat-suite`) | `<agent>/ring1` | | Everything else (`stable`) | `<agent>/stable` | -An inner-ring repo pinned to `<agent>/stable` would only ever see a version *after* it reached the outermost ring, contributing **zero** soak signal to the gate that is supposed to protect the fleet — defeating the ring model. -The public-repo audit's `check_centralized_workflow_stubs` expected-pin map must therefore expect `<agent>/ring0|ring1` for inner-ring repos, matching the ring map in [`standards/canary-rings.json`](../../standards/canary-rings.json). +An inner-ring repo pinned to `<agent>/stable` would only ever see a version *after* it reached the outermost ring, contributing **zero** soak +signal to the gate that is supposed to protect the fleet — defeating the ring model. +The public-repo audit's `check_centralized_workflow_stubs` expected-pin map must therefore expect `<agent>/ring0|ring1` for inner-ring repos, +matching the ring map in [`standards/canary-rings.json`](../../standards/canary-rings.json). ## 6. Consequences @@ -85,7 +109,9 @@ The public-repo audit's `check_centralized_workflow_stubs` expected-pin map must - The public `petry-projects/.github` → `standards/ci-standards.md` can now describe the **implemented** model and drop its "Rollout status / next phase" caveat, linking here for the decision rationale. **That edit lands in the public repo** and is out of scope for a `.github-private` dev-lead run — this ADR is its in-repo counterpart and prerequisite. **A separate pull request against `petry-projects/.github` is still required to complete that public-repo update; issue #869 should be closed by that PR, not by this one.** -- The ring map, gate thresholds, and cadence are data/code ([`standards/canary-rings.json`](../../standards/canary-rings.json), [`scripts/lib/canary-rollout.sh`](../../scripts/lib/canary-rollout.sh), [`.github/workflows/canary-rollout.yml`](../../.github/workflows/canary-rollout.yml)) — changing them is a reviewed diff, not a doc edit. +- The ring map, gate thresholds, and cadence are data/code ([`standards/canary-rings.json`](../../standards/canary-rings.json), + [`scripts/lib/canary-rollout.sh`](../../scripts/lib/canary-rollout.sh), [`.github/workflows/canary-rollout.yml`](../../.github/workflows/canary-rollout.yml)) + — changing them is a reviewed diff, not a doc edit. ## 7. References @@ -94,5 +120,6 @@ The public-repo audit's `check_centralized_workflow_stubs` expected-pin map must - [`docs/initiatives/agentic-release-strategy-orchestration.md`](./agentic-release-strategy-orchestration.md) — how the child issues (#496–#503) were delivered; the dependency DAG. - [`docs/release/versioning.md`](../release/versioning.md) — immutable-release vs. moving-channel-tag mechanics; cross-repo reusables. - [`standards/canary-rings.json`](../../standards/canary-rings.json) — machine-readable ring map (source of truth). -- [`scripts/lib/canary-rollout.sh`](../../scripts/lib/canary-rollout.sh) / [`scripts/canary-rollout.sh`](../../scripts/canary-rollout.sh) / [`.github/workflows/canary-rollout.yml`](../../.github/workflows/canary-rollout.yml) — gate core, orchestrator, and scheduler. +- [`scripts/lib/canary-rollout.sh`](../../scripts/lib/canary-rollout.sh) / [`scripts/canary-rollout.sh`](../../scripts/canary-rollout.sh) / + [`.github/workflows/canary-rollout.yml`](../../.github/workflows/canary-rollout.yml) — gate core, orchestrator, and scheduler. - Decisions context: [petry-projects/.github#516](https://github.com/petry-projects/.github/issues/516). From 6491f3f231012d58e4daead3956ea466db2cdb42 Mon Sep 17 00:00:00 2001 From: Don Petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:12:33 -0500 Subject: [PATCH 3/3] fix(#613): address advisory-bot findings (Gemini/Copilot/CodeRabbit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correctness / behavior: - canary-rollout.sh _run_json: fail CLOSED on a genuine gh failure (return 1 + ::error:: instead of silent []) — a swallowed error read as "zero failures" could green-light a bad promotion. Empty-but-successful still returns [] (Gemini critical). - canary-rollout.sh _reusable_differs: host-aware — for a cross-repo agent (host != THIS_REPO, e.g. dev-lead once the engine runs from .github) the reusable blob isn't in the local checkout, so resolve it on the host via gh api (_gh_blob_sha) and fail CLOSED (unresolvable compare → differs=1, disabling the benign-failure allowlist). +3 unit tests (Copilot). this-repo agents keep the local git rev-parse path (existing tests green). - bootstrap-new-repo.sh ring-pins dup check: scope to the target ring's array line so a repo registered in a DIFFERENT ring no longer false-positives "in sync" (CodeRabbit). Portability: - bootstrap-new-repo.sh base64: encode via `base64 | tr -d '\n'` (GNU wraps, BSD has no -w0); decode via `base64 -d || base64 -D` at all three sites (Gemini/Copilot). Hardening / hygiene: - canary-rollout.yml: persist-credentials: false on checkout (tag moves use gh api, not git push, so the git credential is unused — CodeRabbit/zizmor). - lib/canary-rollout.sh _semver_gt: explicit `return $?` (Gemini); robust_sample_target docstring documents the cap_multiple arg (CodeRabbit). - bootstrap-new-repo.sh: drop the redundant local gh-CLI check (main() checks) (Gemini). - canary_rollout.bats: `run grep -c` idiom instead of `$(... || true)` (Gemini). Docs (relocated ADR/versioning canonicalized in .github — refreshed to live behavior): - canary-rollout.yml header + ADR §4: tag moves are `gh api` ref updates on the host, not `git tag -f` + push; mover credential is the release-manager GitHub App token, not a PAT. - ADR §3 gate: replace the old 7-day-window / failure-rate model with the #548 graduated dwell+sample floors and cumulative zero-failures-since-cut; gate-state table matches impl. - ADR cadence: timer runs autocut → promote-all (arm-gated by CANARY_AUTO_PROMOTE) / evaluate-all → sync-issues, not "evaluate-only, never promotes on timer". - versioning.md: the six #482 reusables are now registered (cross-repo gh api moves, #870/#1054); only feature-ideation remains absent (Copilot). Validation: canary_rollout.bats 91/91 (+3), test_bootstrap_new_repo.bats 13/13, shellcheck no new findings (22 pre-existing info-only), markdownlint 0, yamllint clean. Refs #613 #596 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .github/workflows/canary-rollout.yml | 12 ++++--- docs/initiatives/canary-rollout-adr.md | 50 +++++++++++++++----------- docs/release/versioning.md | 13 ++++--- scripts/bootstrap-new-repo.sh | 22 ++++++------ scripts/canary-rollout.sh | 35 ++++++++++++++---- scripts/lib/canary-rollout.sh | 10 +++--- tests/canary_rollout.bats | 37 ++++++++++++++++++- 7 files changed, 124 insertions(+), 55 deletions(-) diff --git a/.github/workflows/canary-rollout.yml b/.github/workflows/canary-rollout.yml index af371e0d7..a7154f7bc 100644 --- a/.github/workflows/canary-rollout.yml +++ b/.github/workflows/canary-rollout.yml @@ -4,8 +4,10 @@ # Gate standard: .github#548 (graduated dwell/sample, robust baseline, per-candidate # window, ring0 sample waiver, failure triage) — see scripts/canary-rollout.sh. # -# The ONLY mutation this performs is moving a channel tag (`git tag -f` + push) and -# recording a GitHub Deployment for traceability. It never writes consumer files. +# The ONLY mutation this performs is moving a channel tag via a `gh api` ref update +# (PATCH/POST repos/<host>/git/refs/tags/… — NOT a local `git push`, which the +# release-channel-tags ruleset would reject) and recording a GitHub Deployment for +# traceability. It never writes consumer files. # # Auth: the mover credential is a GitHub App installation token minted at runtime via # actions/create-github-app-token from RELEASE_MANAGER_APP_CLIENT_ID (variable) + RELEASE_MANAGER_APP_KEY (secret) (scoped, @@ -118,8 +120,10 @@ jobs: with: fetch-depth: 0 fetch-tags: true - # Mover identity: the App token so pushing protected channel tags is - # authorized under the release-channel-tags ruleset bypass (#868). + # Tag moves go through `gh api` with the App token (GH_TOKEN env), never a + # local `git push`, so the checkout credential is only needed to authenticate + # the fetch — don't persist it into .git/config (narrows the exposure surface). + persist-credentials: false token: ${{ steps.app-token.outputs.token }} # FRONT END (#1069): cut a new candidate + move `next` whenever a registered reusable's diff --git a/docs/initiatives/canary-rollout-adr.md b/docs/initiatives/canary-rollout-adr.md index 60283b23c..d9d73a40e 100644 --- a/docs/initiatives/canary-rollout-adr.md +++ b/docs/initiatives/canary-rollout-adr.md @@ -43,46 +43,54 @@ together always cover both org-infra repos, whichever hosts the agent. Hard-codi ## 3. Decision 2 — soak / health gate rules The promotion gate is a pure, unit-tested decision core ([`scripts/lib/canary-rollout.sh`](../../scripts/lib/canary-rollout.sh); -tests [`tests/canary_rollout.bats`](../../tests/canary_rollout.bats)). A candidate `vX.Y.Z` advances to the next ring only once the rings -**already on** the candidate have, over a trailing **7-day** window (`SOAK_WINDOW_DAYS = 7`): +tests [`tests/canary_rollout.bats`](../../tests/canary_rollout.bats)), whose definitive spec is **#548**. A candidate `vX.Y.Z` +advances one ring only when the source tier (the tier currently running it) clears, evaluated per transition against EXECUTED runs +(success+failure; skipped/cancelled excluded) since the candidate's **own** cut: -- **Volume:** `healthy_runs >= min_healthy_runs`, where `min_healthy_runs = ceil(baseline_runs / 7)` — roughly one trailing day's worth of the prior version's run volume. -- **Quality:** candidate failure-rate `<=` baseline failure-rate (compared in integer per-mille to stay in pure-bash arithmetic). +- **Dwell floor:** the candidate has held the source tier a minimum time — `next→ring0` ≥ 4h, `ring0→ring1` ≥ 8h, + `ring1→stable` ≥ 12h (registry-configurable per transition in `standards/canary-rings.json` `.gate`). +- **Sample floor:** a graduated `clamp(round(0.25·avg), 3, 15)` over a robust, spike-capped baseline. `next→ring0` is dwell-only + when the source tier has no caller; `ring0→ring1` **waives** a fresh sample and rides cumulative health; `ring1→stable` needs ≥ 1 ring1 run. +- **Cumulative health (ALWAYS):** ZERO failures and ZERO startup-failures across EVERY tier since the candidate's first cut. A failure + whose reusable blob is **unchanged** from the prior release is triaged as PRE_EXISTING, not a candidate regression. -Two rules make the gate safe under low traffic: - -- **No synthetic floor / "we still wait."** A ring with no healthy candidate volume simply never advances — the candidate parks at the frontier - rather than being waved through on an arbitrary minimum. `baseline_runs == 0` yields `min_healthy_runs == 0`, but the `healthy_runs > 0` guard - still blocks promotion on zero real volume. -- **Quality is checked first.** A failure-rate breach is evaluated before the volume threshold, so a regression is never masked by "not enough runs yet." +Two properties keep the gate safe under low traffic: it never advances on an arbitrary synthetic floor (no volume → the candidate +parks at the frontier), and any in-window failure is triaged before promotion, so a real regression is never masked by "not enough runs yet." ### Gate states | State | Meaning | Follow-up | |---|---|---| -| `PROMOTE` | Quality holds **and** the volume threshold is met | Advance the next ring (innermost-out). | -| `SOAKING` | Quality holds but not enough healthy candidate runs yet | Wait — no action. | -| `INVESTIGATE` | Candidate failure-rate exceeds baseline (possible regression) | A **human** classifies it: pre-existing → log + continue; genuine regression → fix + cut a new `vX.Y.Z`, which **RESETs** the rollout. | -| `RESET` | Roll all rings back to the last known-good | A **human/override outcome** of a confirmed regression — *not* an automatic gate state. | +| `PROMOTE` | Dwell + sample floors met and cumulative health is clean | Advance the next ring (innermost-out). | +| `SOAKING` | Health clean but the dwell/sample floor isn't met yet | Wait — no action. | +| `BLOCKED` + `REGRESSION` | In-window failure whose reusable blob **changed** vs the prior release | Fix + cut a new `vX.Y.Z`, which **RESETs** the rollout. | +| `BLOCKED` + `PRE_EXISTING` | In-window failure whose reusable is **unchanged** (pre-dates the candidate) | A **human** classifies; may `promote --allow-pre-existing` to advance past it. | +| `COMPLETE` | Candidate is already on every ring | Fully rolled out — no action. | ### Cadence -[`.github/workflows/canary-rollout.yml`](../../.github/workflows/canary-rollout.yml) runs the gate on a **4-hourly** schedule -(`cron: "0 */4 * * *"`) in **read-only `evaluate` mode only** — it emits the per-version gate + health report (#502 observability) -and **never** promotes on the timer. Promotion and rollback are `workflow_dispatch`-only and human-gated. +[`.github/workflows/canary-rollout.yml`](../../.github/workflows/canary-rollout.yml) runs every **4 hours** (`cron: "0 */4 * * *"`), +ordering **autocut → promote-all → sync-issues**. `autocut` (#1069, gated by the `CANARY_AUTO_CUT` variable) cuts a new +`<agent>/vX.Y.Z` + moves `next` whenever a registered reusable's blob on its host `main` HEAD differs from the current candidate. +Timer promotion is **arm-gated**: when `CANARY_AUTO_PROMOTE == 'true'` the run advances every PROMOTE-ready agent one ring (BLOCKED +agents untouched; human approval is OVERRIDE-only), otherwise it runs read-only `evaluate-all`, emitting each agent's gate + health +report (#502). `sync-issues` then upserts one blocker issue per BLOCKED agent and renders the fleet-status table to the job summary. +Per-agent `promote` / `rollback` remain `workflow_dispatch`-only (default `--dry-run`). ## 4. Decision 3 — promotion is a single gated channel-tag move; rollback moves the tag back The **channel tags are the rollout state.** There is no separate state store: - **Promotion** = advancing one ring by **moving one channel tag** to the candidate `vX.Y.Z`, innermost-out along the ordered list - `next → ring0 → ring1 → stable` (`next_channel_in_order`). The **only** mutation the orchestrator performs is `git tag -f` + push; it never writes consumer files. + `next → ring0 → ring1 → stable` (`next_channel_in_order`). The **only** mutation the orchestrator performs is a `gh api` ref update + (PATCH/POST `repos/<host>/git/refs/tags/…`) on the agent's host repo — never a local `git push`; it never writes consumer files. - **Rollback** = moving a channel tag **back** to a prior immutable release (`rollback <agent> <ring> --to vX.Y.Z`). Because immutable `vX.Y.Z` tags never move, they are the audit trail and the rollback targets. -The mover credential is **`GH_PAT_WORKFLOWS`** (the workflow falls back to `GITHUB_TOKEN` when the PAT is unset, but only the PAT satisfies -the `release-channel-tags` ruleset bypass scoped to this workflow's identity (#868)) — promotion and rollback effectively require the PAT; -agents running only as `GITHUB_TOKEN` cannot push protected release tags. +The mover credential is a **GitHub App installation token** (`release-manager`, minted at runtime via +`actions/create-github-app-token`): the App is a bypass actor on the `release-channel-tags` ruleset (#868), so its `gh api` ref +updates are authorized to move protected channel tags. Minting is a HARD requirement — the job fails rather than fall back to +`github.token`, which cannot move `.github` tags. This is the sanctioned, documented exception to the SHA-pin standard for **first-party** channel tags (see [AGENTS.md → "Release channel tags & the mutable-ref exception"](../../AGENTS.md)); compliance audits must not flag `@<agent>/<channel>` pins on first-party callers as unpinned. diff --git a/docs/release/versioning.md b/docs/release/versioning.md index 1e6ce2b97..da7906e02 100644 --- a/docs/release/versioning.md +++ b/docs/release/versioning.md @@ -153,13 +153,12 @@ reusables (`agent-shield`, `auto-rebase`, `dependency-audit`, `dependabot-autome (the mutable-ref exception) is created **there**, not on this repo — see [`AGENTS.md`](../../AGENTS.md) "Release channel tags & the mutable-ref exception". -> **Why not `standards/canary-rings.json` yet?** The machine-readable ring source of truth consumed by -> the promotion automation (`scripts/canary-rollout.sh`, #501) carries only the agents whose channel -> tags that automation can move. `cut-release.sh` now publishes cross-repo tags via `gh api` (#872), but -> `canary-rollout.sh` does not yet use that cross-repo path, so `feature-ideation` and the six above stay -> **absent** from `canary-rings.json` until the rollout automation is taught the same `gh api` move — -> adding them sooner would advertise a promotion the automation cannot perform. (Manual/scripted cuts via -> `cut-release.sh` work today.) +> **Now registered in `standards/canary-rings.json`.** The promotion automation +> (`scripts/canary-rollout.sh`, #501) now moves channel tags **cross-repo** via `gh api` ref updates on +> each agent's host repo (#1054), so the six reusables above are registered under the full +> `{next, ring0, ring1, stable}` model (#870) — the earlier "automation can't move cross-repo tags" +> gap is closed. `feature-ideation` remains **absent** for now: its host + ring assignment aren't settled, +> and registering an agent before then would advertise a promotion the registry can't yet describe. ### The six #482 reusables (ring assignment) diff --git a/scripts/bootstrap-new-repo.sh b/scripts/bootstrap-new-repo.sh index a3737ba9d..086c0f9fe 100755 --- a/scripts/bootstrap-new-repo.sh +++ b/scripts/bootstrap-new-repo.sh @@ -179,16 +179,15 @@ _cross_repo_file_pr() { || { echo " [warn] ${repo}: cannot create branch ${branch}" >&2; return 1; } sha="$(gh api "repos/${repo}/contents/${path}?ref=${branch}" --jq '.sha' 2>/dev/null || true)" [ -n "$sha" ] && sha_arg=(--field "sha=${sha}") - encoded="$(printf '%s' "$content" | base64 -w 0 2>/dev/null || printf '%s' "$content" | base64)" + # Portable unwrapped base64: GNU wraps at 76 cols and BSD has no -w0, so strip + # newlines explicitly — the Contents API rejects a wrapped payload. + encoded="$(printf '%s' "$content" | base64 | tr -d '\n')" gh api "repos/${repo}/contents/${path}" --method PUT \ --field "message=${msg}" --field "content=${encoded}" --field "branch=${branch}" \ "${sha_arg[@]}" --silent 2>/dev/null \ || { echo " [warn] ${repo}: cannot write ${path}" >&2; return 1; } + # gh/jq availability is checked once in main(); no need to re-check here. local existing_pr - if ! command -v gh > /dev/null 2>&1; then - echo " [warn] gh CLI is not installed" >&2 - return 1 - fi existing_pr="$(gh pr list --repo "$repo" --head "$branch" --state open --json number --jq '.[0].number' 2>/dev/null || true)" if [ -n "$existing_pr" ]; then echo " [ring] PR #$existing_pr already open for branch $branch on $repo" @@ -206,10 +205,13 @@ _cross_repo_file_pr() { # present it is a no-op; if the anchor is absent it errors rather than corrupt. _cross_repo_ring_pins_pr() { local repo="$1" agent="$2" ring="$3" content new branch - content="$(gh api "repos/${STANDARDS_REPO}/contents/${RING_PINS_PATH}" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null || true)" + content="$(gh api "repos/${STANDARDS_REPO}/contents/${RING_PINS_PATH}" --jq '.content' 2>/dev/null | { base64 -d 2>/dev/null || base64 -D 2>/dev/null; } || true)" [ -n "$content" ] || { echo "::error::cannot fetch ${STANDARDS_REPO}:${RING_PINS_PATH} for ring-pins registration" >&2; return 1; } - if printf '%s' "$content" | grep -qF "\"$repo\""; then - echo " [ring] ${repo} already present in ${RING_PINS_PATH} — in sync" + # Ring-specific: only treat as in-sync if the repo is already in THIS ring's array + # line (`<ring>=( … )`). A bare repo-wide match would false-positive when the repo is + # registered in a *different* ring, leaving canary-rings.json and ring-pins.sh divergent. + if printf '%s' "$content" | grep -F "${ring}=(" | grep -qF "\"$repo\""; then + echo " [ring] ${repo} already present in ${RING_PINS_PATH} for ${ring} — in sync" return 0 fi new="$(printf '%s' "$content" | awk -v repo="$repo" -v ring="$ring" ' @@ -234,7 +236,7 @@ _cross_repo_ring_pins_pr() { _cross_repo_repin_stub_pr() { local repo="$1" agent="$2" ring="$3" path content new branch path=".github/workflows/${agent}.yml" - content="$(gh api "repos/${repo}/contents/${path}" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null || true)" + content="$(gh api "repos/${repo}/contents/${path}" --jq '.content' 2>/dev/null | { base64 -d 2>/dev/null || base64 -D 2>/dev/null; } || true)" [ -n "$content" ] || { echo " [warn] ${repo}: no ${path} stub to repin" >&2; return 0; } new="$(printf '%s' "$content" \ | sed -E "s#@${agent}/[A-Za-z0-9_-]+#@${agent}/${ring}#g; s#(agent_ref:[[:space:]]*[\"']?)${agent}/[A-Za-z0-9_-]+#\\1${agent}/${ring}#g")" @@ -367,7 +369,7 @@ step_codeowners() { echo " [warn] CODEOWNERS not found on ${repo} — cannot verify team ownership" >&2 return 0 fi - decoded="$(printf '%s' "$encoded" | base64 -d 2>/dev/null || true)" + decoded="$(printf '%s' "$encoded" | { base64 -d 2>/dev/null || base64 -D 2>/dev/null; } || true)" # First owner on the first non-comment, non-blank rule line. first_owner="$(printf '%s\n' "$decoded" \ | sed 's/#.*//' \ diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 868e96f97..34e26e7f5 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -203,10 +203,18 @@ candidate_cut_date() { # _run_json <repo> <workflow> <since_z> — gh run-list JSON (conclusion,createdAt,databaseId,workflowName) for a # repo since the given Zulu timestamp. Empty repo/wildcard → []. Never fails the caller. _run_json() { - local repo="$1" wf="$2" since="$3" + local repo="$1" wf="$2" since="$3" out [ -z "$repo" ] || [ "$repo" = '*' ] && { echo '[]'; return 0; } - gh run list --repo "$repo" --workflow "$wf" ${since:+--created ">=$since"} \ - -L 1000 --json conclusion,createdAt,databaseId,workflowName 2>/dev/null || echo '[]' + # Fail CLOSED on a genuine gh failure (network / rate-limit / expired creds): returning + # an empty [] would read as "zero failures" and could green-light a bad promotion. A + # non-zero return halts the run under `set -e` instead. An empty-but-successful result + # (no runs) is still a valid []. + if ! out="$(gh run list --repo "$repo" --workflow "$wf" ${since:+--created ">=$since"} \ + -L 1000 --json conclusion,createdAt,databaseId,workflowName 2>/dev/null)"; then + echo "::error::_run_json: failed to fetch run list for $repo (workflow=$wf)" >&2 + return 1 + fi + echo "${out:-[]}" } # _tier_sample <agent> <since_z> <repo...> — EXECUTED runs (success+failure) on the @@ -322,12 +330,25 @@ _baseline_daily() { # by triage to confirm a CANDIDATE REGRESSION (#548): a failure whose reusable is identical # to the prior version is pre-existing, not introduced by the candidate. _reusable_differs() { - local agent="$1" cand="$2" prior="$3" reusable a b + local agent="$1" cand="$2" prior="$3" reusable a b host reusable="$(_agent_field "$agent" reusable)" [ -z "$reusable" ] || [ -z "$cand" ] || [ -z "$prior" ] && { echo 0; return 0; } - a="$(git rev-parse -q --verify "${cand}:${reusable}" 2>/dev/null || echo "")" - b="$(git rev-parse -q --verify "${prior}:${reusable}" 2>/dev/null || echo "")" - [ -n "$a" ] && [ "$a" != "$b" ] && { echo 1; return 0; } + host="$(_agent_field "$agent" host)"; host="${host:-$THIS_REPO}" + if [ "$host" = "$THIS_REPO" ]; then + # This-repo agent: the reusable blob lives in the local checkout. + a="$(git rev-parse -q --verify "${cand}:${reusable}" 2>/dev/null || echo "")" + b="$(git rev-parse -q --verify "${prior}:${reusable}" 2>/dev/null || echo "")" + else + # Cross-repo agent (#613): the reusable blob lives on the HOST repo, not locally — + # a local `git rev-parse` would resolve empty and wrongly report "unchanged". Resolve + # the blob SHA on the host via gh api, like the autocut path does. + a="$(_gh_blob_sha "$host" "$reusable" "$cand")" + b="$(_gh_blob_sha "$host" "$reusable" "$prior")" + fi + # Fail CLOSED: an unresolvable compare counts as "changed" so the benign-failure + # allowlist is disabled and a genuine candidate regression is never masked. + { [ -z "$a" ] || [ -z "$b" ]; } && { echo 1; return 0; } + [ "$a" != "$b" ] && { echo 1; return 0; } echo 0 } diff --git a/scripts/lib/canary-rollout.sh b/scripts/lib/canary-rollout.sh index 0993df03b..fc7cb5bfa 100644 --- a/scripts/lib/canary-rollout.sh +++ b/scripts/lib/canary-rollout.sh @@ -53,9 +53,9 @@ median_x2() { fi } -# robust_sample_target <fraction_permille> <clamp_lo> <clamp_hi> <daily_counts...> -# Robust baseline: cap each per-day executed count at 3× the median day (neutralising -# spikes like a runaway loop day), take the mean of the capped days, then the sample +# robust_sample_target <fraction_permille> <clamp_lo> <clamp_hi> <cap_multiple> <daily_counts...> +# Robust baseline: cap each per-day executed count at <cap_multiple>× the median day (default +# 3×, neutralising spikes like a runaway loop day), take the mean of the capped days, then the sample # target = clamp(round(fraction · avg), lo, hi). fraction is in per-mille (250 == 0.25). # # All arithmetic stays exact in integers by working in "×2" units for the median: @@ -103,8 +103,8 @@ _semver_gt() { local a1=0 a2=0 a3=0 b1=0 b2=0 b3=0 IFS=. read -r a1 a2 a3 <<< "$1" IFS=. read -r b1 b2 b3 <<< "$2" - if [ "$a1" -ne "$b1" ]; then [ "$a1" -gt "$b1" ]; return; fi - if [ "$a2" -ne "$b2" ]; then [ "$a2" -gt "$b2" ]; return; fi + if [ "$a1" -ne "$b1" ]; then [ "$a1" -gt "$b1" ]; return $?; fi + if [ "$a2" -ne "$b2" ]; then [ "$a2" -gt "$b2" ]; return $?; fi [ "$a3" -gt "$b3" ] } diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index bdaf8525d..c38eaff07 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -149,6 +149,40 @@ setup() { [ "$(classify_failure 1 data)" = "PRE_EXISTING" ] } +# ── _reusable_differs: cross-repo host-aware blob compare (#613) ─────────────── +# When an agent's host != THIS_REPO (e.g. dev-lead hosted in .github-private while the +# engine runs from .github), the reusable blob is NOT in the local checkout — the compare +# must go through `gh api` on the host. agent-shield (host = petry-projects/.github) is the +# cross-repo agent here, with THIS_REPO forced to .github-private via GITHUB_REPOSITORY. +@test "_reusable_differs: cross-repo agent resolves host blobs via gh api — differ → 1" { + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + { echo '#!/usr/bin/env bash' + echo 'case "$*" in' + echo ' *"ref=CAND"*) echo "aaaaaaaaaa" ;;' + echo ' *"ref=PRIOR"*) echo "bbbbbbbbbb" ;;' + echo ' *) echo "" ;;' + echo 'esac'; } > "$STUB_BIN/gh"; chmod +x "$STUB_BIN/gh" + run env GITHUB_REPOSITORY="petry-projects/.github-private" CANARY_RINGS="$RINGS" \ + bash -c "source '$ORCH' && _reusable_differs agent-shield CAND PRIOR" + [ "$status" -eq 0 ]; [ "$output" = "1" ] +} + +@test "_reusable_differs: cross-repo agent — identical host blob → 0" { + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + printf '#!/usr/bin/env bash\necho "samesamesame"\n' > "$STUB_BIN/gh"; chmod +x "$STUB_BIN/gh" + run env GITHUB_REPOSITORY="petry-projects/.github-private" CANARY_RINGS="$RINGS" \ + bash -c "source '$ORCH' && _reusable_differs agent-shield CAND PRIOR" + [ "$status" -eq 0 ]; [ "$output" = "0" ] +} + +@test "_reusable_differs: cross-repo agent — unresolvable blob fails CLOSED → 1" { + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + printf '#!/usr/bin/env bash\necho ""\n' > "$STUB_BIN/gh"; chmod +x "$STUB_BIN/gh" + run env GITHUB_REPOSITORY="petry-projects/.github-private" CANARY_RINGS="$RINGS" \ + bash -c "source '$ORCH' && _reusable_differs agent-shield CAND PRIOR" + [ "$status" -eq 0 ]; [ "$output" = "1" ] +} + # ── next_channel_in_order ───────────────────────────────────────────────────── @test "next_channel_in_order: walks the ring order" { [ "$(next_channel_in_order next 'next,ring0,ring1,stable')" = "ring0" ] @@ -743,7 +777,8 @@ GHEOF [[ "$output" == *"opened blocker issue #777 for dev-lead"* ]] [[ "$output" == *"wrote fleet-status table to the job summary"* ]] # Exactly ONE issue create (the blocker) — the dashboard is NOT an issue anymore. - [ "$(grep -c '^CREATE|' "$ISSUE_LOG" || true)" -eq 1 ] + run grep -c '^CREATE|' "$ISSUE_LOG" + [ "$output" -eq 1 ] grep -q -- "--label canary-blocker" "$ISSUE_LOG" ! grep -q -- "--label canary-dashboard" "$ISSUE_LOG" ! grep -q "^PIN|" "$ISSUE_LOG"