From a2e04185b3e6c9290f0dec85abc281b48234c65d Mon Sep 17 00:00:00 2001 From: pelikhan Date: Thu, 15 Oct 2026 21:27:32 -0700 Subject: [PATCH 1/3] fix: retry accepted work-queue launch verification safely Bound transient verification reads by the existing reconciliation policy, retain independently verified evidence after binding publication failures, and preserve uncertain reservations without redispatch. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../retain-accepted-queue-launch-identity.md | 5 ++ actions/setup/js/work_queue_dispatch.cjs | 27 +++++- actions/setup/js/work_queue_dispatch.test.cjs | 85 +++++++++++++++++++ docs/src/content/docs/reference/work-queue.md | 9 ++ 4 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 .changeset/retain-accepted-queue-launch-identity.md diff --git a/.changeset/retain-accepted-queue-launch-identity.md b/.changeset/retain-accepted-queue-launch-identity.md new file mode 100644 index 00000000000..8374bff8e2c --- /dev/null +++ b/.changeset/retain-accepted-queue-launch-identity.md @@ -0,0 +1,5 @@ +--- +"gh-aw": patch +--- + +Retry transient verification reads for accepted work-queue launches within existing reconciliation limits and retain verified native evidence after binding publication failures for safe recovery without redispatch. diff --git a/actions/setup/js/work_queue_dispatch.cjs b/actions/setup/js/work_queue_dispatch.cjs index 93a2876ac81..3f42a02e342 100644 --- a/actions/setup/js/work_queue_dispatch.cjs +++ b/actions/setup/js/work_queue_dispatch.cjs @@ -218,6 +218,23 @@ function rejectedEvidence(latest, dispatch, assignment, profile, options) { return lifecycleEvidence(assignment, profile, senderRepository, "nonlaunch", "github_api", options.now ?? Date.now(), { receipt, principal: dispatchPrincipal(dispatch) }); } +async function verifyLaunchedRun(options, expected, policy) { + const started = Date.now(); + for (let attempt = 1; ; attempt++) { + try { + return validateNativeRun(await fetchNativeRun(options.dispatchClient || options.githubClient, expected.repository, expected.run_id), expected); + } catch (error) { + log.failure("launch.verification.failed", error, true); + const status = error?.status ?? error?.response?.status; + const transient = status === 404 || (status >= 500 && status <= 599) || ["ETIMEDOUT", "ECONNRESET"].includes(error?.code) || error?.message === "run_correlation_mismatch"; + const remaining = policy.deadline_ms - (Date.now() - started); + if (!transient || attempt >= policy.max_attempts || remaining <= 0) throw error; + await (options.sleepFn || (delay => new Promise(resolve => setTimeout(resolve, delay))))(Math.min(1000, remaining)); + if (Date.now() - started >= policy.deadline_ms) throw error; + } + } +} + async function launchAssignment(options, supplied) { log.debug("launch.start"); assertQueueControlRole(options); @@ -289,20 +306,24 @@ async function launchAssignment(options, supplied) { log.debug("launch.unresolved"); return { state: "launch_unresolved", released: false, dispatched: true }; } + let verifiedEvidence; try { const expected = expectedWorkerRun(assignment, profile, options.context, sender.repository, credential.principal); - const proof = validateNativeRun(await fetchNativeRun(options.dispatchClient || options.githubClient, sender.repository, returned.run_id), { ...expected, run_id: returned.run_id }); + const proof = await verifyLaunchedRun(options, { ...expected, run_id: returned.run_id }, latest.projection.policy.pools[assignment.pool].reconciliation); const binding = bindingForRun(proof, expected); const evidence = lifecycleEvidence(assignment, profile, sender.repository, "reconciliation", "github_api", options.now ?? Date.now(), { run_id: proof.run_id, run_attempt: 1, principal: proof.principal }); + verifiedEvidence = evidence; await publishOperations(options, trustedContext, ["bind", assignment.dispatch_id, proof.run_id], "dispatch", [{ kind: "Dispatch", dispatch_id: assignment.dispatch_id, state: "bound", run: binding, evidence }]); latest = await loadQueue(options); if (canonical(latest.projection.dispatches.get(assignment.dispatch_id)?.run) !== canonical(binding)) throw new Error("run_binding_conflict"); log.debug("launch.binding.persisted"); return { state: "bound", released: false, dispatched: true, run_id: returned.run_id, html_url: returned.html_url }; } catch (error) { - log.failure("launch.binding.failed", error); + log.failure("launch.binding.failed", error, true); try { - await publishOperations(options, trustedContext, ["uncertain", assignment.dispatch_id, returned.run_id], "dispatch", [{ kind: "Dispatch", dispatch_id: assignment.dispatch_id, state: "uncertain", reason: "binding_unconfirmed" }]); + await publishOperations(options, trustedContext, ["uncertain", assignment.dispatch_id, returned.run_id], "dispatch", [ + { kind: "Dispatch", dispatch_id: assignment.dispatch_id, state: "uncertain", reason: "binding_unconfirmed", ...(verifiedEvidence ? { evidence: verifiedEvidence } : {}) }, + ]); } catch (error) { log.failure("launch.binding_uncertain_marker.failed", error); // Authenticated activation can recover binding publication without POST. diff --git a/actions/setup/js/work_queue_dispatch.test.cjs b/actions/setup/js/work_queue_dispatch.test.cjs index 104fefffc24..349fed702d8 100644 --- a/actions/setup/js/work_queue_dispatch.test.cjs +++ b/actions/setup/js/work_queue_dispatch.test.cjs @@ -710,6 +710,91 @@ describe("native queue launch fencing and conservative recovery", () => { expect(assignment).not.toHaveProperty("run"); }); + it.each(["not_found", "unavailable", "timeout", "title_pending"])("verifies an accepted %s launch with bounded reads and exactly one POST", async failure => { + const { fixture, post, options } = setup({ count: 1 }); + const get = fixture.githubClient.rest.actions.getWorkflowRun; + const read = vi.fn(async parameters => { + if (read.mock.calls.length === 1) { + if (failure === "title_pending") return { status: 200, data: { ...fixture.nativeRun(), display_title: "worker" } }; + throw Object.assign(new Error("private response"), failure === "timeout" ? { code: "ETIMEDOUT" } : { status: failure === "not_found" ? 404 : 503 }); + } + return get(parameters); + }); + fixture.githubClient.rest.actions.getWorkflowRun = parameters => (parameters.run_id === "42" ? read(parameters) : get(parameters)); + const sleepFn = vi.fn(async () => {}); + expect(await launchAssignment({ ...options, sleepFn }, fixture.assignment)).toMatchObject({ state: "bound", run_id: "42", dispatched: true }); + expect(read).toHaveBeenCalledTimes(2); + expect(read.mock.calls.every(([parameters]) => parameters.run_id === "42" && parameters.request.retries === 0)).toBe(true); + expect(sleepFn).toHaveBeenCalledExactlyOnceWith(1000); + expect(post).toHaveBeenCalledTimes(1); + expect(fixture.state.dispatches.get(fixture.assignment.dispatch_id)).toMatchObject({ state: "bound", released: false, run: { run_id: "42", run_attempt: 1 } }); + }); + + it("retains the reservation without asserting native evidence after exhausting verification reads", async () => { + const { fixture, post, options } = setup({ count: 1, configurePolicy: policy => (policy.pools.default.reconciliation.max_attempts = 2) }); + const get = fixture.githubClient.rest.actions.getWorkflowRun; + const read = vi.fn().mockRejectedValue({ status: 404 }); + fixture.githubClient.rest.actions.getWorkflowRun = parameters => (parameters.run_id === "42" ? read(parameters) : get(parameters)); + expect(await launchAssignment(options, fixture.assignment)).toMatchObject({ state: "launch_unresolved", run_id: "42", released: false }); + expect(read).toHaveBeenCalledTimes(2); + const dispatch = fixture.state.dispatches.get(fixture.assignment.dispatch_id); + expect(dispatch).toMatchObject({ state: "uncertain", released: false }); + expect(dispatch.run).toBeUndefined(); + expect(fixture.transactions.at(-1).operations[0]).not.toHaveProperty("evidence"); + expect(post).toHaveBeenCalledTimes(1); + expect(fixture.state.claims.size).toBe(1); + }); + + it("retains verified native evidence after a binding publication failure and recovers without redispatch", async () => { + const { fixture, post, options } = setup({ count: 1 }); + const publish = options.publishWorkQueueRequest; + options.publishWorkQueueRequest = async parameters => { + if (parameters.request.parameters.operations?.some(operation => operation.kind === "Dispatch" && operation.state === "bound")) throw new Error("publication unavailable"); + return publish(parameters); + }; + expect(await launchAssignment(options, fixture.assignment)).toMatchObject({ state: "launch_unresolved", run_id: "42", released: false }); + const dispatch = fixture.state.dispatches.get(fixture.assignment.dispatch_id); + expect(dispatch.run).toBeUndefined(); + expect(fixture.transactions.at(-1).operations[0]).toMatchObject({ state: "uncertain", reason: "binding_unconfirmed", evidence: { run_id: "42", run_attempt: 1, principal: "11" } }); + options.publishWorkQueueRequest = publish; + expect(await reconcileDispatch(options)).toMatchObject({ state: "bound", run_id: "42", released: false }); + expect(post).toHaveBeenCalledTimes(1); + expect(fixture.state.claims.size).toBe(1); + }); + + it.each([{ status: 403 }, { head_sha: "b".repeat(40) }, { actor: { id: "99" } }, { run_attempt: 2 }])("does not retry or bind a denied or mismatched returned run: %j", async failure => { + const { fixture, post, options } = setup({ count: 1 }); + const get = fixture.githubClient.rest.actions.getWorkflowRun; + const read = vi.fn(async () => { + if (failure.status) throw failure; + return { status: 200, data: { ...fixture.nativeRun(), ...failure } }; + }); + fixture.githubClient.rest.actions.getWorkflowRun = parameters => (parameters.run_id === "42" ? read() : get(parameters)); + const sleepFn = vi.fn(); + expect(await launchAssignment({ ...options, sleepFn }, fixture.assignment)).toMatchObject({ state: "launch_unresolved", run_id: "42", released: false }); + expect(read).toHaveBeenCalledTimes(1); + expect(sleepFn).not.toHaveBeenCalled(); + expect(post).toHaveBeenCalledTimes(1); + expect(fixture.state.dispatches.get(fixture.assignment.dispatch_id).run).toBeUndefined(); + }); + + it("stops verification reads at the policy deadline without releasing an uncertain launch", async () => { + const { fixture, post, options } = setup({ count: 1, configurePolicy: policy => (policy.pools.default.reconciliation.deadline_ms = 1) }); + const get = fixture.githubClient.rest.actions.getWorkflowRun; + const read = vi.fn().mockRejectedValue({ status: 404 }); + fixture.githubClient.rest.actions.getWorkflowRun = parameters => (parameters.run_id === "42" ? read(parameters) : get(parameters)); + const clock = vi.spyOn(Date, "now").mockReturnValue(0); + const sleepFn = vi.fn(async () => clock.mockReturnValue(1)); + try { + expect(await launchAssignment({ ...options, sleepFn }, fixture.assignment)).toMatchObject({ state: "launch_unresolved", released: false }); + expect(read).toHaveBeenCalledTimes(1); + expect(sleepFn).toHaveBeenCalledExactlyOnceWith(1); + expect(post).toHaveBeenCalledTimes(1); + } finally { + clock.mockRestore(); + } + }); + it("derives native groups from the checked publication state rather than adapter assignment metadata", async () => { const { fixture, options, post } = setup({ granted: false, count: 1 }); const publish = options.publishWorkQueueRequest; diff --git a/docs/src/content/docs/reference/work-queue.md b/docs/src/content/docs/reference/work-queue.md index 154127e219d..c7d50bfad6c 100644 --- a/docs/src/content/docs/reference/work-queue.md +++ b/docs/src/content/docs/reference/work-queue.md @@ -574,6 +574,15 @@ describes validated inputs. Reservation, native launch, verified run binding, Completion, Result, and Release are distinct; successful `dispatch-next` is not proof of launch. +After an accepted worker dispatch, trusted controls retry transient run reads +and pending run-title correlation within the pool's reconciliation attempt and +deadline limits. They never repeat the dispatch POST. If native verification +succeeds but binding publication fails, the uncertainty marker retains the +verified run evidence as a discovery hint, not a durable binding. Unverified +responses never become native evidence. Reconciliation must still verify the exact +repository, workflow, revision, principal, attempt and dispatch token before +binding; uncertainty alone never releases the reservation. + ### Replay an Actions snapshot Reproduce ledger validation and scheduling with the current CLI against a From c0ae424233c87bcdeeabbf83661d04c0562609bf Mon Sep 17 00:00:00 2001 From: pelikhan Date: Thu, 15 Oct 2026 22:02:52 -0700 Subject: [PATCH 2/3] fix: preserve exact launch attempt recovery across compaction Verify immutable attempt one, retain recovery hint visibility across checkpoint history, isolate hint recovery tests, and name transient verification failures. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- actions/setup/js/work_queue_dispatch.cjs | 9 +- actions/setup/js/work_queue_dispatch.test.cjs | 109 +++++++++++++++--- actions/setup/js/work_queue_native.cjs | 6 + actions/setup/js/work_queue_native.test.cjs | 36 ++++++ actions/setup/js/work_queue_reconciler.cjs | 2 +- docs/src/content/docs/reference/work-queue.md | 15 +-- 6 files changed, 150 insertions(+), 27 deletions(-) diff --git a/actions/setup/js/work_queue_dispatch.cjs b/actions/setup/js/work_queue_dispatch.cjs index 3f42a02e342..a7d0a47bf9f 100644 --- a/actions/setup/js/work_queue_dispatch.cjs +++ b/actions/setup/js/work_queue_dispatch.cjs @@ -13,7 +13,7 @@ const store = require("./work_queue_store.cjs"); const { normalizeAssignment, normalizeClaimScope, readClaimScopeContext } = require("./work_queue_claim_scope.cjs"); const { readStagedIntentBatch, requestIdForIntent, requestForIntent, normalizeDispatchParameters, normalizeSubmitParameters, validatePreparedPlanIntentBatch } = require("./work_queue_intents.cjs"); const { assertPolicyProposal, loadQueue, policyProposalFor, publishOperations, validateStoredAssignment, expectedWorkerRun, bindingForRun } = require("./work_queue_binding.cjs"); -const { authenticatePublisher, authenticateIntentPublisher, immutableRef, fetchNativeRun, postQueueDispatch, validateNativeRun } = require("./work_queue_native.cjs"); +const { authenticatePublisher, authenticateIntentPublisher, immutableRef, fetchNativeRunAttempt, isTransientVerificationError, postQueueDispatch, validateNativeRun } = require("./work_queue_native.cjs"); const { dependencyKey, allowedScope, resolveDependencies, resolveExternalEdges } = require("./work_queue_dependency_resolver.cjs"); const { lifecycleEvidence, reconcileDispatch, reconcileQueue, releaseAssignment } = require("./work_queue_reconciler.cjs"); const { DEFAULT_INTENT_PATH } = require("./work_queue_mcp_server.cjs"); @@ -222,13 +222,12 @@ async function verifyLaunchedRun(options, expected, policy) { const started = Date.now(); for (let attempt = 1; ; attempt++) { try { - return validateNativeRun(await fetchNativeRun(options.dispatchClient || options.githubClient, expected.repository, expected.run_id), expected); + return validateNativeRun(await fetchNativeRunAttempt(options.dispatchClient || options.githubClient, expected.repository, expected.run_id, 1), expected); } catch (error) { log.failure("launch.verification.failed", error, true); - const status = error?.status ?? error?.response?.status; - const transient = status === 404 || (status >= 500 && status <= 599) || ["ETIMEDOUT", "ECONNRESET"].includes(error?.code) || error?.message === "run_correlation_mismatch"; const remaining = policy.deadline_ms - (Date.now() - started); - if (!transient || attempt >= policy.max_attempts || remaining <= 0) throw error; + if (!isTransientVerificationError(error) || attempt >= policy.max_attempts || remaining <= 0) throw error; + // Poll at a fixed 1s cadence, never exceeding the remaining reconciliation deadline. await (options.sleepFn || (delay => new Promise(resolve => setTimeout(resolve, delay))))(Math.min(1000, remaining)); if (Date.now() - started >= policy.deadline_ms) throw error; } diff --git a/actions/setup/js/work_queue_dispatch.test.cjs b/actions/setup/js/work_queue_dispatch.test.cjs index 349fed702d8..7b75b0cca68 100644 --- a/actions/setup/js/work_queue_dispatch.test.cjs +++ b/actions/setup/js/work_queue_dispatch.test.cjs @@ -16,7 +16,7 @@ import { main as publishQueueControls, } from "./work_queue_dispatch.cjs"; import { cancelBeforeLaunch, cancellationOperations, reconcileDispatch, reconcileQueue, reconciliationCandidates } from "./work_queue_reconciler.cjs"; -import { appendCommit, newRequest, serializeProjection } from "./work_queue_replay.cjs"; +import { appendCommit, compactTransactions, generateRequestOperations, newRequest, replayTransactions, serializeProjection } from "./work_queue_replay.cjs"; import { assignmentOnly } from "./work_queue_scheduler.cjs"; import { nodeId } from "./work_queue_graph.cjs"; import { validateStoredAssignment } from "./work_queue_binding.cjs"; @@ -89,6 +89,22 @@ function failedFixture(disposition) { return { ...result, member }; } +function compactQueue(options, fixture, transactions = fixture.transactions) { + let ledger = compactTransactions(transactions, "c".repeat(40), fixture.administrator, fixture.at); + options.readWorkQueueLog = async () => ({ sha: "compacted-head", transactions: ledger, state: replayTransactions(ledger) }); + options.publishWorkQueueRequest = async parameters => { + const state = replayTransactions(ledger); + const at = ledger.at(-1).at + 1; + const id = `post-checkpoint-${ledger.length}`; + const decision = (parameters.generateOperations || generateRequestOperations)(state, parameters.request, parameters.actor, at, id); + const commit = { version: 3, id, previous: state.tip, request: parameters.request, actor: parameters.actor, policy_epoch: state.policy_epoch, at, operations: decision.operations }; + const result = appendCommit(ledger, commit); + ledger = result.transactions; + return { ...result, ...decision, persisted: true, publishedNow: true }; + }; + return ledger; +} + describe("native queue launch fencing and conservative recovery", () => { it.each(["cancelled", "failure", "success"])("reclaims never-started singleton reservations after the exact originating attempt ends with %s", async conclusion => { const { fixture, options, post } = setup({ batch: 1 }); @@ -112,7 +128,8 @@ describe("native queue launch fencing and conservative recovery", () => { it("releases an abandoned reservation before the next dispatcher grants and launches retry Work", async () => { const { fixture, options, post } = setup({ count: 1, configurePolicy: policy => (policy.pools.default.retry.backoff_ms = 1) }); - fixture.githubClient.rest.actions.getWorkflowRunAttempt = async () => ({ status: 200, data: { ...fixture.nativeRun(fixture.dispatcherContext), status: "completed", conclusion: "cancelled" } }); + fixture.githubClient.rest.actions.getWorkflowRunAttempt = async parameters => + parameters.run_id === "15" ? { status: 200, data: { ...fixture.nativeRun(fixture.dispatcherContext), status: "completed", conclusion: "cancelled" } } : fixture.githubClient.rest.actions.getWorkflowRun(parameters); const context = { ...fixture.dispatcherContext, runId: "16" }; fixture.githubClient.rest.actions.getWorkflowRun = async ({ run_id }) => ({ status: 200, @@ -712,7 +729,7 @@ describe("native queue launch fencing and conservative recovery", () => { it.each(["not_found", "unavailable", "timeout", "title_pending"])("verifies an accepted %s launch with bounded reads and exactly one POST", async failure => { const { fixture, post, options } = setup({ count: 1 }); - const get = fixture.githubClient.rest.actions.getWorkflowRun; + const get = fixture.githubClient.rest.actions.getWorkflowRunAttempt; const read = vi.fn(async parameters => { if (read.mock.calls.length === 1) { if (failure === "title_pending") return { status: 200, data: { ...fixture.nativeRun(), display_title: "worker" } }; @@ -720,11 +737,11 @@ describe("native queue launch fencing and conservative recovery", () => { } return get(parameters); }); - fixture.githubClient.rest.actions.getWorkflowRun = parameters => (parameters.run_id === "42" ? read(parameters) : get(parameters)); + fixture.githubClient.rest.actions.getWorkflowRunAttempt = read; const sleepFn = vi.fn(async () => {}); expect(await launchAssignment({ ...options, sleepFn }, fixture.assignment)).toMatchObject({ state: "bound", run_id: "42", dispatched: true }); expect(read).toHaveBeenCalledTimes(2); - expect(read.mock.calls.every(([parameters]) => parameters.run_id === "42" && parameters.request.retries === 0)).toBe(true); + expect(read.mock.calls.every(([parameters]) => parameters.run_id === "42" && parameters.attempt_number === 1 && parameters.request.retries === 0)).toBe(true); expect(sleepFn).toHaveBeenCalledExactlyOnceWith(1000); expect(post).toHaveBeenCalledTimes(1); expect(fixture.state.dispatches.get(fixture.assignment.dispatch_id)).toMatchObject({ state: "bound", released: false, run: { run_id: "42", run_attempt: 1 } }); @@ -732,9 +749,8 @@ describe("native queue launch fencing and conservative recovery", () => { it("retains the reservation without asserting native evidence after exhausting verification reads", async () => { const { fixture, post, options } = setup({ count: 1, configurePolicy: policy => (policy.pools.default.reconciliation.max_attempts = 2) }); - const get = fixture.githubClient.rest.actions.getWorkflowRun; const read = vi.fn().mockRejectedValue({ status: 404 }); - fixture.githubClient.rest.actions.getWorkflowRun = parameters => (parameters.run_id === "42" ? read(parameters) : get(parameters)); + fixture.githubClient.rest.actions.getWorkflowRunAttempt = read; expect(await launchAssignment(options, fixture.assignment)).toMatchObject({ state: "launch_unresolved", run_id: "42", released: false }); expect(read).toHaveBeenCalledTimes(2); const dispatch = fixture.state.dispatches.get(fixture.assignment.dispatch_id); @@ -745,7 +761,23 @@ describe("native queue launch fencing and conservative recovery", () => { expect(fixture.state.claims.size).toBe(1); }); - it("retains verified native evidence after a binding publication failure and recovers without redispatch", async () => { + it("verifies the original launch attempt even when the latest native attempt is a rerun", async () => { + const { fixture, post, options } = setup({ count: 1 }); + const get = fixture.githubClient.rest.actions.getWorkflowRun; + fixture.githubClient.rest.actions.getWorkflowRun = vi.fn(async parameters => (parameters.run_id === "42" ? { status: 200, data: { ...fixture.nativeRun(), run_attempt: 2 } } : get(parameters))); + const getAttempt = vi.fn(async parameters => { + expect(parameters).toMatchObject({ run_id: "42", attempt_number: 1, request: { retries: 0 } }); + return { status: 200, data: fixture.nativeRun() }; + }); + fixture.githubClient.rest.actions.getWorkflowRunAttempt = getAttempt; + expect(await launchAssignment(options, fixture.assignment)).toMatchObject({ state: "bound", run_id: "42", dispatched: true }); + expect(fixture.state.dispatches.get(fixture.assignment.dispatch_id)).toMatchObject({ state: "bound", run: { run_id: "42", run_attempt: 1 }, released: false }); + expect(getAttempt).toHaveBeenCalledTimes(1); + expect(fixture.githubClient.rest.actions.getWorkflowRun.mock.calls.every(([parameters]) => parameters.run_id !== "42")).toBe(true); + expect(post).toHaveBeenCalledTimes(1); + }); + + it.each([0, 1, 2])("recovers saved native evidence after a binding publication failure and %s compactions without discovery or redispatch", async compactions => { const { fixture, post, options } = setup({ count: 1 }); const publish = options.publishWorkQueueRequest; options.publishWorkQueueRequest = async parameters => { @@ -757,19 +789,69 @@ describe("native queue launch fencing and conservative recovery", () => { expect(dispatch.run).toBeUndefined(); expect(fixture.transactions.at(-1).operations[0]).toMatchObject({ state: "uncertain", reason: "binding_unconfirmed", evidence: { run_id: "42", run_attempt: 1, principal: "11" } }); options.publishWorkQueueRequest = publish; + let transactions = fixture.transactions; + for (let index = 0; index < compactions; index++) transactions = compactQueue(options, fixture, transactions); + const restored = (await options.readWorkQueueLog()).state; + if (compactions) { + expect(restored.transactions).toHaveLength(1); + expect(restored.historicalTransactions.flatMap(commit => commit.operations)).toContainEqual(expect.objectContaining({ reason: "binding_unconfirmed", evidence: expect.objectContaining({ run_id: "42" }) })); + } + fixture.githubClient.rest.actions.listWorkflowRuns.mockResolvedValue({ status: 200, data: { workflow_runs: [] } }); + const getAttempt = vi.fn(async parameters => { + expect(parameters).toMatchObject({ run_id: "42", attempt_number: 1 }); + return { status: 200, data: fixture.nativeRun() }; + }); + fixture.githubClient.rest.actions.getWorkflowRunAttempt = getAttempt; expect(await reconcileDispatch(options)).toMatchObject({ state: "bound", run_id: "42", released: false }); + expect(getAttempt).toHaveBeenCalledTimes(2); + expect(fixture.githubClient.rest.actions.listWorkflowRuns).toHaveBeenCalledTimes(1); expect(post).toHaveBeenCalledTimes(1); - expect(fixture.state.claims.size).toBe(1); + const recovered = (await options.readWorkQueueLog()).state; + expect(recovered.dispatches.get(fixture.assignment.dispatch_id)).toMatchObject({ state: "bound", run: { run_id: "42", run_attempt: 1 }, released: false }); + expect(recovered.claims.size).toBe(1); + }); + + it.each(["current", "historical", "checkpoint_boundary"])("prefers the newest retained native run hint across %s transactions", async history => { + const { fixture, post, options } = setup({ count: 1, started: true }); + const operation = runId => ({ + kind: "Dispatch", + dispatch_id: fixture.assignment.dispatch_id, + state: "uncertain", + reason: "binding_unconfirmed", + evidence: { kind: "reconciliation", source: "github_api", repository: REPOSITORY, workflow: fixture.binding.workflow, ref: REF, principal: "11", checked_at: fixture.at, run_id: runId, run_attempt: 1 }, + }); + fixture.append("dispatch", { operations: [operation("43")] }, fixture.dispatcher); + if (history === "checkpoint_boundary") { + compactQueue(options, fixture); + await options.publishWorkQueueRequest({ + actor: fixture.dispatcher, + request: newRequest("newer-returned-hint", "dispatch", fixture.dispatcher, { operations: [operation("42")] }), + }); + } else { + fixture.append("dispatch", { operations: [operation("42")] }, fixture.dispatcher); + if (history === "historical") compactQueue(options, fixture); + } + const getAttempt = vi.fn(async parameters => { + expect(parameters).toMatchObject({ run_id: "42", attempt_number: 1 }); + return { status: 200, data: fixture.nativeRun() }; + }); + fixture.githubClient.rest.actions.getWorkflowRunAttempt = getAttempt; + expect(await reconcileDispatch(options)).toMatchObject({ state: "bound", run_id: "42", released: false }); + expect(getAttempt).toHaveBeenCalledTimes(2); + expect(fixture.githubClient.rest.actions.listWorkflowRuns).toHaveBeenCalledExactlyOnceWith(expect.any(Object)); + expect(post).not.toHaveBeenCalled(); + const recovered = (await options.readWorkQueueLog()).state; + expect(recovered.dispatches.get(fixture.assignment.dispatch_id)).toMatchObject({ run: { run_id: "42", run_attempt: 1 }, released: false }); }); it.each([{ status: 403 }, { head_sha: "b".repeat(40) }, { actor: { id: "99" } }, { run_attempt: 2 }])("does not retry or bind a denied or mismatched returned run: %j", async failure => { const { fixture, post, options } = setup({ count: 1 }); - const get = fixture.githubClient.rest.actions.getWorkflowRun; - const read = vi.fn(async () => { + const read = vi.fn(async parameters => { + expect(parameters).toMatchObject({ run_id: "42", attempt_number: 1 }); if (failure.status) throw failure; return { status: 200, data: { ...fixture.nativeRun(), ...failure } }; }); - fixture.githubClient.rest.actions.getWorkflowRun = parameters => (parameters.run_id === "42" ? read() : get(parameters)); + fixture.githubClient.rest.actions.getWorkflowRunAttempt = read; const sleepFn = vi.fn(); expect(await launchAssignment({ ...options, sleepFn }, fixture.assignment)).toMatchObject({ state: "launch_unresolved", run_id: "42", released: false }); expect(read).toHaveBeenCalledTimes(1); @@ -780,9 +862,8 @@ describe("native queue launch fencing and conservative recovery", () => { it("stops verification reads at the policy deadline without releasing an uncertain launch", async () => { const { fixture, post, options } = setup({ count: 1, configurePolicy: policy => (policy.pools.default.reconciliation.deadline_ms = 1) }); - const get = fixture.githubClient.rest.actions.getWorkflowRun; const read = vi.fn().mockRejectedValue({ status: 404 }); - fixture.githubClient.rest.actions.getWorkflowRun = parameters => (parameters.run_id === "42" ? read(parameters) : get(parameters)); + fixture.githubClient.rest.actions.getWorkflowRunAttempt = read; const clock = vi.spyOn(Date, "now").mockReturnValue(0); const sleepFn = vi.fn(async () => clock.mockReturnValue(1)); try { diff --git a/actions/setup/js/work_queue_native.cjs b/actions/setup/js/work_queue_native.cjs index c0d06313150..0821042a212 100644 --- a/actions/setup/js/work_queue_native.cjs +++ b/actions/setup/js/work_queue_native.cjs @@ -107,6 +107,11 @@ async function fetchNativeRunAttempt(githubClient, repository, runId, attempt = return run; } +function isTransientVerificationError(error) { + const status = error?.status ?? error?.response?.status; + return status === 404 || (status >= 500 && status <= 599) || ["ETIMEDOUT", "ECONNRESET"].includes(error?.code) || error?.message === "run_correlation_mismatch"; +} + function validateNativeRun(run, expected) { log.debug("run.validate.start"); const runId = nativeId(run?.id, "run ID"); @@ -279,6 +284,7 @@ module.exports = { dispatchResponse, fetchNativeRun, fetchNativeRunAttempt, + isTransientVerificationError, validateNativeRun, dispatchTag, postQueueDispatch, diff --git a/actions/setup/js/work_queue_native.test.cjs b/actions/setup/js/work_queue_native.test.cjs index 68b4ffd8821..14419716ed6 100644 --- a/actions/setup/js/work_queue_native.test.cjs +++ b/actions/setup/js/work_queue_native.test.cjs @@ -8,6 +8,7 @@ import { hasDispatchToken, dispatchResponse, fetchNativeRunAttempt, + isTransientVerificationError, postQueueDispatch, validateNativeRun, authenticatePublisher, @@ -103,6 +104,41 @@ describe("authenticated native queue runs", () => { expect(getAttempt).toHaveBeenCalledTimes(1); }); + it("verifies immutable attempt one even when the latest run is a rerun", async () => { + const getWorkflowRun = vi.fn().mockResolvedValue({ status: 200, data: { ...run, run_attempt: 2 } }); + const getWorkflowRunAttempt = vi.fn().mockResolvedValue({ status: 200, data: run }); + const githubClient = nativeClient({ getWorkflowRun, getWorkflowRunAttempt }); + expect(validateNativeRun(await fetchNativeRunAttempt(githubClient, expected.repository, run.id, 1), { ...expected, run_id: run.id })).toMatchObject({ run_id: run.id, run_attempt: 1 }); + expect(getWorkflowRun).not.toHaveBeenCalled(); + expect(getWorkflowRunAttempt).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ run_id: run.id, attempt_number: 1, headers: { "X-GitHub-Api-Version": API_VERSION }, request: { retries: 0, timeout: 15000 } })); + }); + + it("classifies only the retryable native verification failures as transient", () => { + for (const status of [404, 500, 502, 599]) { + expect(isTransientVerificationError({ status })).toBe(true); + expect(isTransientVerificationError({ response: { status } })).toBe(true); + } + for (const code of ["ETIMEDOUT", "ECONNRESET"]) expect(isTransientVerificationError({ code })).toBe(true); + expect(isTransientVerificationError(new Error("run_correlation_mismatch"))).toBe(true); + for (const error of [ + undefined, + null, + new Error("rerun_not_authorized"), + new Error("run_binding_conflict"), + new Error("run_repository_mismatch"), + new Error("run_repository_identity_mismatch"), + new Error("run_event_mismatch"), + new Error("run_ref_mismatch"), + new Error("run_workflow_mismatch"), + new Error("run_principal_mismatch"), + { code: "EACCES" }, + { status: 403, response: { status: 503 } }, + ...[200, 400, 401, 403, 422, 429, 499, 600].map(status => ({ status })), + ...[403, 429, 600].map(status => ({ response: { status } })), + ]) + expect(isTransientVerificationError(error)).toBe(false); + }); + it("recovers a protected original intent origin through its exact native attempt without changing request identity", async () => { const context = { repo: { owner: "owner", repo: "repo" }, runId: run.id, runAttempt: 2, actorId: "11", sha: ref, eventName: "workflow_dispatch", payload: { repository: { id: 7 } } }; const origin = { role: "dispatcher", principal: "11", repository: "owner/repo", workflow: expected.workflow, run_id: run.id, run_attempt: 1 }; diff --git a/actions/setup/js/work_queue_reconciler.cjs b/actions/setup/js/work_queue_reconciler.cjs index 0a68810b6f9..1794893f1bd 100644 --- a/actions/setup/js/work_queue_reconciler.cjs +++ b/actions/setup/js/work_queue_reconciler.cjs @@ -103,7 +103,7 @@ async function discoverRuns(options, expected) { } function returnedRunHint(state, dispatchId) { - for (const commit of [...state.transactions].reverse()) { + for (const commit of [...(state.historicalTransactions || []), ...state.transactions].reverse()) { const operation = [...commit.operations].reverse().find(operation => operation.kind === "Dispatch" && operation.dispatch_id === dispatchId && operation.evidence?.run_id); if (operation) return operation.evidence.run_id; } diff --git a/docs/src/content/docs/reference/work-queue.md b/docs/src/content/docs/reference/work-queue.md index c7d50bfad6c..75452afa0b1 100644 --- a/docs/src/content/docs/reference/work-queue.md +++ b/docs/src/content/docs/reference/work-queue.md @@ -574,13 +574,14 @@ describes validated inputs. Reservation, native launch, verified run binding, Completion, Result, and Release are distinct; successful `dispatch-next` is not proof of launch. -After an accepted worker dispatch, trusted controls retry transient run reads -and pending run-title correlation within the pool's reconciliation attempt and -deadline limits. They never repeat the dispatch POST. If native verification -succeeds but binding publication fails, the uncertainty marker retains the -verified run evidence as a discovery hint, not a durable binding. Unverified -responses never become native evidence. Reconciliation must still verify the exact -repository, workflow, revision, principal, attempt and dispatch token before +After an accepted worker dispatch, trusted controls verify the exact first run +attempt, retrying transient reads and pending run-title correlation within the +pool's reconciliation attempt and deadline limits. They never repeat the dispatch +POST. If native verification succeeds but binding publication fails, the +uncertainty marker retains the verified run evidence as a discovery hint, not a +durable binding. The hint remains available after queue compaction. Unverified +responses never become native evidence. Reconciliation must still verify the +exact repository, workflow, revision, principal, attempt and dispatch token before binding; uncertainty alone never releases the reservation. ### Replay an Actions snapshot From f8682d328890eca526a29e59719d6ba3f8249235 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Thu, 15 Oct 2026 22:16:14 -0700 Subject: [PATCH 3/3] test: align credential launch mocks with immutable run attempts Exercise getWorkflowRunAttempt for attempt 1 in both selected-credential launch routes and fail explicitly if verification reads the latest-run endpoint. Preserve the single-POST, durable START, and principal binding assertions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- actions/setup/js/dispatch_workflow.test.cjs | 4 ++++ actions/setup/js/work_queue_control_adapter_checks.cjs | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/actions/setup/js/dispatch_workflow.test.cjs b/actions/setup/js/dispatch_workflow.test.cjs index 69081c051c1..ab0701cb915 100644 --- a/actions/setup/js/dispatch_workflow.test.cjs +++ b/actions/setup/js/dispatch_workflow.test.cjs @@ -135,6 +135,10 @@ describe("dispatch_workflow handler factory", () => { return { status: 200, data: { workflow_run_id: "42", run_url: "https://api.github.com/repos/owner/repo/actions/runs/42", html_url: "https://github.com/owner/repo/actions/runs/42" } }; }, getWorkflowRun: async () => { + throw new Error("Launch verification must read immutable attempt 1, not the latest run"); + }, + getWorkflowRunAttempt: async parameters => { + expect(parameters).toMatchObject({ owner: "owner", repo: "repo", run_id: "42", attempt_number: 1 }); const dispatch = [...replayTransactions(native.log()).dispatches.values()][0]; return { status: 200, data: { ...fixture.nativeRun(), display_title: `gh-aw work-queue ${dispatch.dispatch_id}` } }; }, diff --git a/actions/setup/js/work_queue_control_adapter_checks.cjs b/actions/setup/js/work_queue_control_adapter_checks.cjs index 30b78d42f5a..b6e46749743 100644 --- a/actions/setup/js/work_queue_control_adapter_checks.cjs +++ b/actions/setup/js/work_queue_control_adapter_checks.cjs @@ -150,9 +150,15 @@ function registerTests({ describe, it }) { assert.equal(parameters.ref, `refs/tags/gh-aw/work-queue/${REF}`); return { status: 200, data: { workflow_run_id: "42", run_url: "https://api.github.com/repos/owner/repo/actions/runs/42", html_url: "https://github.com/owner/repo/actions/runs/42" } }; }, - getWorkflowRun: async parameters => { + getWorkflowRun: async () => { + throw new Error("Launch verification must read immutable attempt 1, not the latest run"); + }, + getWorkflowRunAttempt: async parameters => { workerReads++; + assert.equal(parameters.owner, "owner"); + assert.equal(parameters.repo, "repo"); assert.equal(parameters.run_id, "42"); + assert.equal(parameters.attempt_number, 1); const dispatch = [...fixture.state.dispatches.values()][0]; return { status: 200, data: { ...fixture.nativeRun(), display_title: `gh-aw work-queue ${dispatch.dispatch_id}` } }; },