fix(qa): slug-scope QA content publish + resilient commit + lock release

.github/workflows/rebuild-content-qa.yml

@@ -268,6 +268,13 @@ jobs:
         env:
           CAP_QA_BASE_URL: ${{ secrets.CAP_SRV_URL_QA }}
           CONTENT_API_KEY_QA: ${{ secrets.CONTENT_API_KEY_QA }}
+          # Scope the publish to the changed slug on a commit-triggered (or single-slug
+          # workflow_dispatch) rebuild — publish-content.ts reads PUBLISH_SLUG as the
+          # --slug fallback. Empty on a full rebuild → whole-catalog delta. Mirrors
+          # rebuild-content.yml. Without force-publish, this makes the commit O(changed)
+          # (sub-second) instead of a ~35s full-catalog write. force-publish=true ignores
+          # the slug and re-seeds the whole catalog (--force).
+          PUBLISH_SLUG: ${{ (! inputs.force-publish) && (github.event.client_payload.slug || inputs.slug) || '' }}
 
       # #1373 — synthetic watchdog. The publish step reporting success is not
       # proof rows landed: the render-concepts 404 that hid for days (#1372)

docs/developers/architecture/build.md

@@ -53,7 +53,7 @@ flowchart TB
 
     subgraph publish[Content publish]
         PublishProd["publish-content.ts<br/>delta-aware, gzip,<br/>sha256 hash compare"]
-        PublishQa["publish-content.ts<br/>--channel qa<br/>(always --force)"]
+        PublishQa["publish-content.ts<br/>--channel qa<br/>(delta; slug-scoped via PUBLISH_SLUG,<br/>--force for full re-seed)"]
     end
 
     subgraph deployed[Deployed targets]
@@ -184,7 +184,7 @@ build:qa  → hugo --config ../hugo.qa.toml → hugo/public-qa/
               (strips Joule FAB, rating, completion buttons, progress UI)
   ├─ verify-qa-build.ts  fails the build if QA-only stripping didn't apply
   ↓
-publish-content:qa  (always --force; CONTENT_API_KEY_QA)
+publish-content:qa  (delta by default; PUBLISH_SLUG scopes to the changed slug; --force = full re-seed; CONTENT_API_KEY_QA)
   ↓
 tutorials-srv-qa /content/publish
   ↓

docs/superpowers/specs/2026-05-23-tutorials-qa-endpoint-design.md

@@ -126,8 +126,10 @@ features are out of scope and must not be reachable through the QA surface.
 - `scripts/publish-content.ts` — add `--channel <prod|qa>` flag. In `qa` mode:
   - `CAP_BASE_URL` defaults to `CAP_QA_BASE_URL` env.
   - Bearer token reads from `CONTENT_API_KEY_QA`.
-  - Always uses delta detection bypass (effectively `--force` semantics —
-    matches the existing prod gotcha).
+  - Delta by default (mirrors prod). `PUBLISH_SLUG` (dispatch slug) scopes a
+    commit-triggered rebuild to the changed slug; `--force` (workflow
+    `force-publish` input) does a full re-seed. Superseded the original
+    "always --force" design once QA gained Option B (mutable ContentCurrent).
   - Source dir: `hugo/public-qa/`.
 - `scripts/install-qa-workflows.ts` — one-shot installer that opens a PR adding
   `.github/workflows/notify-qa.yml` to each `*-Contribution` repo. Idempotent;
@@ -325,5 +327,7 @@ are deferred to deploy time.
   mirrors.
 - `scripts/parsers/github.ts:432,477` — existing `INCLUDE_CONTRIBUTION_REPOS`
   flag; QA uses inverse `ONLY_CONTRIBUTION_REPOS` semantic.
-- `feedback_publish_content_force.md` — published Tom-memory; QA always uses
-  force-publish for the same reason.
+- `feedback_publish_content_force.md` — published Tom-memory; originally QA
+  always force-published. Superseded: QA is now delta by default with
+  `PUBLISH_SLUG` scoping (safe once Option B landed); `--force` reserved for
+  full re-seeds.

scripts/__tests__/publish-client.test.ts

@@ -81,6 +81,17 @@ describe('publish-client', () => {
     expect(out).toEqual({ slug1: 'h1', slug2: 'h2' });
   });
 
+  it('fetchRemoteHashes sends a Bearer header when apiKey is given (srv-qa gates /content/hashes)', async () => {
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: true, status: 200, json: () => Promise.resolve({ slug1: 'h1' }),
+    });
+    vi.stubGlobal('fetch', fetchMock);
+    await fetchRemoteHashes({ baseUrl, apiKey: 'qa-key' });
+    const [url, opts] = fetchMock.mock.calls[0];
+    expect(url).toBe(`${baseUrl}/content/hashes`);
+    expect(opts?.headers).toEqual(expect.objectContaining({ Authorization: 'Bearer qa-key' }));
+  });
+
   it('renderConceptsPhase POSTs sessionId to /render-concepts and returns counts (#1327)', async () => {
     const fetchMock = vi.fn().mockResolvedValue({
       ok: true, status: 200,

scripts/__tests__/publish-content-qa.test.ts

@@ -20,11 +20,33 @@ describe('publish-content qa channel', () => {
   it('uses CAP_QA_BASE_URL and CONTENT_API_KEY_QA when channel=qa', () => {
     process.env.CAP_QA_BASE_URL = 'https://qa.example';
     process.env.CONTENT_API_KEY_QA = 'qa-key';
-    const cfg = resolvePublishConfig({ channel: 'qa' });
-    expect(cfg.baseUrl).toBe('https://qa.example');
-    expect(cfg.apiKey).toBe('qa-key');
-    expect(cfg.sourceDir).toMatch(/public-qa$/);
-    expect(cfg.force).toBe(true);
+    const originalArgv = process.argv;
+    process.argv = process.argv.filter(a => a !== '--force');
+    try {
+      const cfg = resolvePublishConfig({ channel: 'qa' });
+      expect(cfg.baseUrl).toBe('https://qa.example');
+      expect(cfg.apiKey).toBe('qa-key');
+      expect(cfg.sourceDir).toMatch(/public-qa$/);
+      // QA is delta-by-default now (mirrors prod). A commit-triggered rebuild sets
+      // PUBLISH_SLUG for an O(changed) publish; force is opt-in via --force
+      // (workflow force-publish input) for a full re-seed.
+      expect(cfg.force).toBe(false);
+    } finally {
+      process.argv = originalArgv;
+    }
+  });
+
+  it('force-publishes on channel=qa only when --force is passed', () => {
+    process.env.CAP_QA_BASE_URL = 'https://qa.example';
+    process.env.CONTENT_API_KEY_QA = 'qa-key';
+    const originalArgv = process.argv;
+    process.argv = [...process.argv.filter(a => a !== '--force'), '--force'];
+    try {
+      const cfg = resolvePublishConfig({ channel: 'qa' });
+      expect(cfg.force).toBe(true);
+    } finally {
+      process.argv = originalArgv;
+    }
   });
 
   it('uses CAP_BASE_URL and CONTENT_API_KEY when channel=prod', () => {
@@ -60,4 +82,20 @@ describe('publish-content qa channel', () => {
         'See scripts/check-srv-qa-route-drift.ts ALLOWLIST_ONLY_ON_SRV.',
     ).not.toBeNull();
   });
+
+  // Regression guard for the half-done #2062 fix: the QA publish step MUST forward
+  // the dispatched slug as PUBLISH_SLUG so a commit-triggered rebuild is slug-scoped
+  // (sub-second commit) instead of a ~35s full-catalog force write that intermittently
+  // 502/503s the single srv-qa instance. Without this, resolvePublishConfig's delta
+  // default is inert on QA (the workflow never narrows the publish).
+  it('QA workflow forwards the dispatched slug to the publish step as PUBLISH_SLUG', () => {
+    const wf = readFileSync(
+      join(dirname(fileURLToPath(import.meta.url)), '..', '..', '.github', 'workflows', 'rebuild-content-qa.yml'),
+      'utf8',
+    );
+    // PUBLISH_SLUG must resolve from the dispatch payload or the workflow_dispatch input,
+    // and must be empty on a force-publish re-seed (force + slug is contradictory).
+    expect(wf).toMatch(/PUBLISH_SLUG:\s*\$\{\{[^}]*github\.event\.client_payload\.slug[^}]*inputs\.slug/);
+    expect(wf).toMatch(/PUBLISH_SLUG:\s*\$\{\{[^}]*inputs\.force-publish/);
+  });
 });

scripts/__tests__/publish-retry.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { withRetry, classifyError, formatErrorChain } from '../lib/publish-retry.js';
+import { withRetry, classifyError, formatErrorChain, computeBackoff } from '../lib/publish-retry.js';
 
 describe('classifyError', () => {
   it('classifies HTTP 5xx as transient', () => {
@@ -49,6 +49,26 @@ describe('formatErrorChain', () => {
   });
 });
 
+describe('computeBackoff', () => {
+  it('returns baseMs unchanged when jitterRatio is 0 or omitted', () => {
+    expect(computeBackoff(2000)).toBe(2000);
+    expect(computeBackoff(2000, 0)).toBe(2000);
+    // rand should be ignored when there is no jitter
+    expect(computeBackoff(2000, 0, () => 0.99)).toBe(2000);
+  });
+
+  it('scales within [1-r, 1+r] of baseMs', () => {
+    // rand()=0 → factor 1-r (low end); rand()=1 → factor 1+r (high end); 0.5 → base
+    expect(computeBackoff(1000, 0.2, () => 0)).toBe(800);
+    expect(computeBackoff(1000, 0.2, () => 1)).toBe(1200);
+    expect(computeBackoff(1000, 0.2, () => 0.5)).toBe(1000);
+  });
+
+  it('never returns a negative wait', () => {
+    expect(computeBackoff(100, 5, () => 0)).toBe(0);
+  });
+});
+
 describe('withRetry', () => {
   beforeEach(() => { vi.useFakeTimers(); });
 

scripts/lib/publish-client.ts

@@ -112,8 +112,11 @@ export async function abortSession({ baseUrl, apiKey, sessionId, reason }: {
   }
 }
 
-export async function fetchRemoteHashes({ baseUrl }: { baseUrl: string }): Promise<Record<string, string>> {
-  const res = await fetch(`${baseUrl}/content/hashes`);
+export async function fetchRemoteHashes({ baseUrl, apiKey }: { baseUrl: string; apiKey?: string }): Promise<Record<string, string>> {
+  // srv-qa gates /content/hashes behind contentAuthMiddleware (unlike prod srv,
+  // where it's public-read). Send the bearer when we have one — harmless on the
+  // public prod route, required on QA or delta detection 401s.
+  const res = await fetch(`${baseUrl}/content/hashes`, apiKey ? { headers: { Authorization: `Bearer ${apiKey}` } } : undefined);
   if (!res.ok) {
     if (res.status === 503) return {};
     const err: any = new Error(`HTTP ${res.status}`);
@@ -130,10 +133,11 @@ export async function fetchRemoteHashes({ baseUrl }: { baseUrl: string }): Promi
  * Slugs whose sourceHash is null (e.g. published before PR #591) are omitted
  * from the response — drift check skips those by design.
  *
- * Public-read like /content/hashes; no auth needed.
+ * Public-read on prod srv; srv-qa gates it behind contentAuthMiddleware, so pass
+ * apiKey when publishing to QA.
  */
-export async function fetchRemoteSourceHashes({ baseUrl }: { baseUrl: string }): Promise<Record<string, string>> {
-  const res = await fetch(`${baseUrl}/content/source-hashes`);
+export async function fetchRemoteSourceHashes({ baseUrl, apiKey }: { baseUrl: string; apiKey?: string }): Promise<Record<string, string>> {
+  const res = await fetch(`${baseUrl}/content/source-hashes`, apiKey ? { headers: { Authorization: `Bearer ${apiKey}` } } : undefined);
   if (!res.ok) {
     if (res.status === 503) return {};
     if (res.status === 404) {

scripts/lib/publish-retry.ts

@@ -41,6 +41,23 @@ export interface RetryOptions {
   attempts: number;
   backoffMs: number[];
   onAttemptFail?: (attempt: number, err: any, willRetry: boolean) => void;
+  /**
+   * Optional +/- jitter as a fraction of the base delay (e.g. 0.2 = ±20%).
+   * Defaults to 0 (no jitter) so existing callers are byte-for-byte unchanged.
+   * Jitter de-synchronizes retries so all three DevRel projects' publishes don't
+   * re-hit a briefly-unhealthy backend in lockstep.
+   */
+  jitterRatio?: number;
+}
+
+/**
+ * Compute the wait before the next attempt. Pure + injectable rand for tests.
+ * With jitterRatio r, returns baseMs scaled by a factor in [1-r, 1+r], floored at 0.
+ */
+export function computeBackoff(baseMs: number, jitterRatio = 0, rand: () => number = Math.random): number {
+  if (!jitterRatio) return baseMs;
+  const factor = 1 + (rand() * 2 - 1) * jitterRatio; // [1-r, 1+r)
+  return Math.max(0, Math.round(baseMs * factor));
 }
 
 export async function withRetry<T>(fn: () => Promise<T>, opts: RetryOptions): Promise<T> {
@@ -54,7 +71,8 @@ export async function withRetry<T>(fn: () => Promise<T>, opts: RetryOptions): Pr
       const willRetry = cls === 'transient' && attempt < opts.attempts;
       opts.onAttemptFail?.(attempt, err, willRetry);
       if (!willRetry) break;
-      const wait = opts.backoffMs[Math.min(attempt - 1, opts.backoffMs.length - 1)];
+      const base = opts.backoffMs[Math.min(attempt - 1, opts.backoffMs.length - 1)];
+      const wait = computeBackoff(base, opts.jitterRatio);
       await new Promise(r => setTimeout(r, wait));
     }
   }

scripts/publish-content.ts

@@ -35,7 +35,13 @@ export function resolvePublishConfig({ channel }: { channel: Channel }): Publish
       baseUrl: process.env.CAP_QA_BASE_URL ?? 'http://localhost:4005',
       apiKey: process.env.CONTENT_API_KEY_QA,
       sourceDir: 'hugo/public-qa',
-      force: true,
+      // Delta by default (mirrors prod), so a commit-triggered QA rebuild that
+      // sets PUBLISH_SLUG publishes only the changed slug — a sub-second commit
+      // instead of a ~35s full-catalog force write that intermittently 502/503s
+      // the single srv-qa instance. `force-publish=true` (workflow input → --force)
+      // still does a full re-seed. Safe now that QA has Option B fully active
+      // (mutable ContentCurrent, O(changed) publish, server carries unchanged slugs).
+      force: process.argv.includes('--force'),
     };
   }
   return {
@@ -754,7 +760,7 @@ async function main() {
     // 2) Fetch server's source-hash map.
     let remote: Record<string, string>;
     try {
-      remote = await fetchRemoteSourceHashes({ baseUrl: opts.baseUrl });
+      remote = await fetchRemoteSourceHashes({ baseUrl: opts.baseUrl, apiKey: opts.apiKey });
     } catch (err) {
       console.error('Verify failed: cannot reach /content/source-hashes:', formatErrorChain(err));
       process.exit(1);
@@ -845,7 +851,7 @@ async function main() {
 
     // 3. Fetch /content/source-hashes
     let remote: Record<string, string>;
-    try { remote = await fetchRemoteSourceHashes({ baseUrl: opts.baseUrl }); }
+    try { remote = await fetchRemoteSourceHashes({ baseUrl: opts.baseUrl, apiKey: opts.apiKey }); }
     catch (err) {
       console.error('purge-orphans: cannot reach /content/source-hashes:', formatErrorChain(err));
       process.exit(1);
@@ -1074,7 +1080,7 @@ async function main() {
   let remoteHashes: Record<string, string> = {};
   if (mode !== 'force') {
     log(`Fetching remote hashes from ${opts.baseUrl}/content/hashes...`);
-    try { remoteHashes = await fetchRemoteHashes({ baseUrl: opts.baseUrl }); }
+    try { remoteHashes = await fetchRemoteHashes({ baseUrl: opts.baseUrl, apiKey: opts.apiKey }); }
     catch (err) {
       console.error(`Cannot reach ${opts.baseUrl}/content/hashes: ${formatErrorChain(err)}`);
       process.exit(1);
@@ -1101,7 +1107,7 @@ async function main() {
     const localSourceHashes = computeLocalSourceHashes(targetSlugs, cacheDirForHashes);
     let serverSourceHashes: Record<string, string> = {};
     try {
-      serverSourceHashes = await fetchRemoteSourceHashes({ baseUrl: opts.baseUrl });
+      serverSourceHashes = await fetchRemoteSourceHashes({ baseUrl: opts.baseUrl, apiKey: opts.apiKey });
     } catch (err) {
       console.warn(`[publish-content] #672 short-circuit disengaged: cannot reach /content/source-hashes: ${formatErrorChain(err)}`);
     }
@@ -1244,14 +1250,34 @@ async function main() {
     commit = await withRetry(
       () => commitSession({ baseUrl: opts.baseUrl, apiKey: opts.apiKey, sessionId: begin.sessionId, allowRevertSlugs }),
       {
-        attempts: 3, backoffMs: [1000, 3000, 9000],
+        // Widened from [1000,3000,9000]/3 attempts: the old schedule only ever
+        // waited 1s then 3s (~4s total window) — too short to ride out a transient
+        // gorouter 502/503 blip on the single srv-qa instance (the commit is the
+        // longest single request in the publish). ±20% jitter de-syncs retries
+        // across the three DevRel projects. 502/503/504 are already classified
+        // transient in publish-retry.ts.
+        attempts: 5, backoffMs: [2000, 5000, 10000, 20000], jitterRatio: 0.2,
         onAttemptFail: (attempt, err, willRetry) => {
-          console.error(`[publish-content] commit failed (attempt ${attempt}/3): ${formatErrorChain(err)}${willRetry ? ' — retrying' : ''}`);
+          console.error(`[publish-content] commit failed (attempt ${attempt}/5): ${formatErrorChain(err)}${willRetry ? ' — retrying' : ''}`);
         },
       }
     );
   } catch (err) {
-    console.error(`[publish-content] commit failed permanently — manifest left for GC reaper: ${formatErrorChain(err)}`);
+    console.error(`[publish-content] commit failed permanently: ${formatErrorChain(err)}`);
+    // Release the server-side publish lock + mark the manifest FAILED instead of
+    // stranding it for the 30-min TTL (which 409s every QA rebuild in that window).
+    // Mirrors the append-failure path; abort gets its own transient retry since the
+    // 502/503 that killed the commit is usually a brief blip that has cleared by now.
+    // If abort still can't reach the server, the stuck-manifest reaper is the backstop.
+    try {
+      await withRetry(
+        () => abortSession({ baseUrl: opts.baseUrl, apiKey: opts.apiKey, sessionId: begin.sessionId, reason: 'commit failed' }),
+        { attempts: 4, backoffMs: [1000, 3000, 6000], jitterRatio: 0.2 }
+      );
+      console.error('[publish-content] session aborted — publish lock released');
+    } catch (abortErr) {
+      console.error(`[publish-content] abort after commit failure also failed — manifest left for GC reaper: ${formatErrorChain(abortErr)}`);
+    }
     process.exit(1);
   }
 
@@ -1328,7 +1354,7 @@ async function main() {
   // --- auto-verify ---
   log('Verifying server state matches local...');
   let postRemote: Record<string, string>;
-  try { postRemote = await fetchRemoteHashes({ baseUrl: opts.baseUrl }); }
+  try { postRemote = await fetchRemoteHashes({ baseUrl: opts.baseUrl, apiKey: opts.apiKey }); }
   catch (err) {
     console.error(`Auto-verify warning: cannot reach /content/hashes after commit: ${formatErrorChain(err)}`);
     process.exit(0); // commit was successful; don't punish for a transient verify-fetch error