diff --git a/actions/setup/js/handle_agent_failure.cjs b/actions/setup/js/handle_agent_failure.cjs index 7a5d76f69f4..7c9d81bdccd 100644 --- a/actions/setup/js/handle_agent_failure.cjs +++ b/actions/setup/js/handle_agent_failure.cjs @@ -1902,14 +1902,14 @@ function loadReportIncompleteMessages(items) { } /** - * Build report_incomplete context string for display in failure issues/comments. + * Build report_incomplete context for display in failure issues/comments. * This surfaces the agent's structured incompletion signal so maintainers can * distinguish an incomplete task from a confirmed task outcome. * @param {Array} [items] - Optional pre-loaded agent output items. When provided, avoids re-reading the output file. - * @param {{collectorEmptyOutputCause?: string, rootDir?: string}} [options] + * @param {{collectorEmptyOutputCause?: string, rootDir?: string, runId?: number}} [options] * @returns {ReportBlock[]} */ -function buildReportIncompleteContext(items, { collectorEmptyOutputCause, rootDir } = {}) { +function buildReportIncompleteContext(items, { collectorEmptyOutputCause, rootDir, runId } = {}) { const trustedCollectorCause = typeof collectorEmptyOutputCause === "string" && Object.hasOwn(EMPTY_OUTPUT_CAUSES, collectorEmptyOutputCause) ? collectorEmptyOutputCause : undefined; const messages = loadReportIncompleteMessages(items).filter(message => message.reason !== trustedCollectorCause); @@ -1939,13 +1939,36 @@ function buildReportIncompleteContext(items, { collectorEmptyOutputCause, rootDi } context.push( nodes.paragraph([ - nodes.text("A comment or other safe output does not confirm that the requested task was completed. Reported reasons are not verified causes; runtime evidence is shown separately and agent diagnostic details are excluded."), + nodes.text( + "This run needs a follow-up: the agent reported that it could not finish. A successful agent job only confirms a clean process exit, not that the requested work was completed. Treat the agent-declared reason as a lead, not a confirmed root cause; agent diagnostic details are excluded, so compare the note with the runtime evidence below." + ), ]) ); + if (typeof runId === "number" && Number.isSafeInteger(runId) && runId > 0) { + context.push(nodes.paragraph([nodes.text("For a runtime-focused investigation, run "), nodes.code(`gh aw audit ${runId}`), nodes.text(".")])); + } return context; } +/** + * Build safe next-step guidance for an agent-reported incomplete task. + * @param {Array} items + * @param {{runId?: number, secretRedactionFailed?: boolean}} [options] + * @returns {string} + */ +function buildReportIncompleteFailureMessage(items, { runId, secretRedactionFailed = false } = {}) { + const hasUsableReason = loadReportIncompleteMessages(items).length > 0; + const diagnosticNextStep = !hasUsableReason + ? "The report_incomplete signal had no usable reason; inspect agent_output.json in the agent artifact." + : secretRedactionFailed + ? "The conclusion job summary was suppressed because secret redaction failed; inspect agent_output.json in the agent artifact." + : "If available, check the conclusion job summary for the reported reason and inspect agent_output.json in the agent artifact."; + const auditCommand = typeof runId === "number" && Number.isSafeInteger(runId) && runId > 0 ? `gh aw audit ${runId}` : "gh aw audit "; + + return `Agent reported that it could not complete the task (report_incomplete). A successful agent job does not confirm task completion. ${diagnosticNextStep} Run ${auditCommand} for runtime evidence.`; +} + /** * Only fixed redactor diagnostics from the failed step's time window may be * published. Agent logs and arbitrary exception payloads are never excerpts. @@ -4473,6 +4496,7 @@ async function main() { let hasMissingSafeOutputs = false; let hasOnlyNoopOutputs = false; let hasReportIncomplete = false; + let reportIncompleteItems = []; // Tracks the case where agentConclusion is "failure" but the agent completed its work // successfully (terminal_reason: completed) and produced valid non-noop safe outputs. // This is a false-positive failure caused by a transient error after the agent's task @@ -4544,16 +4568,13 @@ async function main() { // classified as a successful review or other completed task. if (agentOutputResult.success && agentOutputResult.items && agentOutputResult.items.length > 0) { // Cancellation can interrupt the agent before it starts or emits a terminal safe output. - const reportIncompleteItems = agentOutputResult.items.filter(item => item.type === "report_incomplete" && !(isCancelledEmptyOutput && item.reason === collectorEmptyOutputCause)); + reportIncompleteItems = agentOutputResult.items.filter(item => item.type === "report_incomplete" && !(isCancelledEmptyOutput && item.reason === collectorEmptyOutputCause)); if (reportIncompleteItems.length > 0) { if (taskCompleteRegistrationIssueOnlyReportIncomplete) { core.info("Ignoring report_incomplete signal(s) caused only by task_complete registration trouble after successful task-level outputs"); } else { hasReportIncomplete = true; core.info(`Agent emitted ${reportIncompleteItems.length} report_incomplete signal(s) - activating failure handling`); - // Continue after marking the step failed so optional issue reporting can still run. - // eslint-disable-next-line gh-aw-custom/require-return-after-core-setfailed - core.setFailed("Agent reported that it could not complete the task"); } } } @@ -4653,6 +4674,11 @@ async function main() { } failureDiagnostics ??= await getFailedAgentDiagnostics({ startupFailure: collectorEmptyOutputCause === "agent_startup_failure" }); + if (hasReportIncomplete) { + // Continue after marking the step failed so optional issue reporting can still run. + // eslint-disable-next-line gh-aw-custom/require-return-after-core-setfailed + core.setFailed(buildReportIncompleteFailureMessage(reportIncompleteItems, { runId: context.runId, secretRedactionFailed: failureDiagnostics.secretRedactionFailed })); + } if (failureDiagnostics.hasNoJobs && !hasCheckoutFailure) { core.info("Skipping failure reporting: no workflow jobs started; the run may not have been approved."); return; @@ -4701,7 +4727,7 @@ async function main() { !(isCancelledEmptyOutput && item?.type === "report_incomplete" && item.reason === collectorEmptyOutputCause) ); const hasReportedIncomplete = hasReportIncomplete && reportItems.some(item => item?.type === "report_incomplete"); - const reportIncompleteContext = buildReportIncompleteContext(reportItems, { collectorEmptyOutputCause: emptyOutputCause, rootDir: path.dirname(getAgentStdioLogPath()) }); + const reportIncompleteContext = buildReportIncompleteContext(reportItems, { collectorEmptyOutputCause: emptyOutputCause, rootDir: path.dirname(getAgentStdioLogPath()), runId: context.runId }); if (hasReportedIncomplete && reportIncompleteContext.length > 0 && !failureDiagnostics.secretRedactionFailed) { try { const summary = renderSizedReport({ type: "report", title: "Incomplete result", children: reportIncompleteContext, agentTainted: false }).body; diff --git a/actions/setup/js/handle_agent_failure.test.cjs b/actions/setup/js/handle_agent_failure.test.cjs index 50735c99b98..8545b7845a5 100644 --- a/actions/setup/js/handle_agent_failure.test.cjs +++ b/actions/setup/js/handle_agent_failure.test.cjs @@ -1004,13 +1004,13 @@ describe("handle_agent_failure", () => { ); }); - it("keeps the incompletion notice concise without repeating diagnostic policy", () => { - const context = buildReportIncompleteContext([{ type: "report_incomplete", reason: "infrastructure_error", details: "agent-generated diagnostic" }]); - expect(context).toContain("completion could not be confirmed"); - expect(context).toContain("A comment or other safe output does not confirm that the requested task was completed."); - expect(context).not.toContain("See the reported reason"); - expect(context).not.toContain("agent-generated"); - expect(context).toContain("infrastructure_error"); + it("explains the incomplete handoff without exposing agent diagnostic details", () => { + const rendered = buildReportIncompleteContext([{ type: "report_incomplete", reason: "infrastructure_error", details: "agent-generated diagnostic" }]); + expect(rendered).toContain("completion could not be confirmed"); + expect(rendered).toContain("Treat the agent-declared reason as a lead, not a confirmed root cause"); + expect(rendered).toContain("agent diagnostic details are excluded"); + expect(rendered).not.toContain("agent-generated"); + expect(rendered).toContain("infrastructure_error"); }); }); @@ -1674,7 +1674,7 @@ describe("handle_agent_failure", () => { const write = vi.spyOn(fs, "writeFileSync"); try { await main(); - expect(global.core.setFailed.mock.calls).toEqual(startup || crashed ? [["Agent reported that it could not complete the task"]] : []); + expect(global.core.setFailed.mock.calls).toEqual(startup || crashed ? [[expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)")]] : []); expect(create).toHaveBeenCalledOnce(); const issue = create.mock.calls[0][0]; expect(issue.title).toBe(startup ? "[aw] Test Workflow: failed before agent execution started" : "[aw] Test Workflow: failed to redact secrets in logs"); @@ -1803,7 +1803,7 @@ describe("handle_agent_failure", () => { }; try { await main(); - expect(global.core.setFailed.mock.calls).toEqual([["Agent reported that it could not complete the task"]]); + expect(global.core.setFailed.mock.calls).toEqual([[expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)")]]); expect(create).toHaveBeenCalledOnce(); const { title, body } = create.mock.calls[0][0]; expect(title).toBe("[aw] Test Workflow: engine driver crashed"); @@ -2292,11 +2292,100 @@ describe("handle_agent_failure", () => { } }); + it("explains clean-exit incompletion and points to diagnostics without logging the agent payload", async () => { + const outputPath = path.join(tmpDir, "agent_output.json"); + fs.writeFileSync( + outputPath, + JSON.stringify({ + items: [{ type: "report_incomplete", reason: "Required compilation and validation cannot run within the configured build-resource permissions.", details: "private-build-diagnostics" }], + }) + ); + vi.stubEnv("GH_AW_AGENT_OUTPUT", outputPath); + vi.stubEnv("GH_AW_AGENT_CONCLUSION", "success"); + vi.stubEnv("GH_AW_FAILURE_REPORT_AS_ISSUE", "false"); + global.context.runId = 123456; + global.github = { + rest: { + search: { issuesAndPullRequests: vi.fn(async () => ({ data: { total_count: 0, items: [] } })) }, + issues: { create: vi.fn(), createComment: vi.fn() }, + }, + graphql: vi.fn(), + }; + + try { + await main(); + expect(global.core.setFailed).toHaveBeenCalledOnce(); + const diagnostic = global.core.setFailed.mock.calls[0][0]; + expect(diagnostic).toContain("report_incomplete"); + expect(diagnostic).toContain("A successful agent job does not confirm task completion"); + expect(diagnostic).toContain("conclusion job summary"); + expect(diagnostic).toContain("agent_output.json in the agent artifact"); + expect(diagnostic).toContain("gh aw audit 123456"); + const logs = JSON.stringify([...global.core.info.mock.calls, ...global.core.warning.mock.calls, ...global.core.setFailed.mock.calls]); + expect(logs).not.toContain("build-resource permissions"); + expect(logs).not.toContain("private-build-diagnostics"); + const summary = JSON.stringify(global.core.summary.addRaw.mock.calls); + expect(summary).toContain("build-resource permissions"); + expect(summary).not.toContain("private-build-diagnostics"); + expect(global.core.summary.write).toHaveBeenCalled(); + expect(global.github.rest.issues.create).not.toHaveBeenCalled(); + } finally { + vi.unstubAllEnvs(); + } + }); + + it("points to the agent artifact when secret redaction suppresses the summary", async () => { + const outputPath = path.join(tmpDir, "agent_output.json"); + fs.writeFileSync(outputPath, JSON.stringify({ items: [{ type: "report_incomplete", reason: "safe but unverified reason", details: "private-redaction-details" }] })); + vi.stubEnv("GH_AW_AGENT_OUTPUT", outputPath); + vi.stubEnv("GH_AW_AGENT_CONCLUSION", "success"); + vi.stubEnv("GH_AW_FAILURE_REPORT_AS_ISSUE", "false"); + global.context.runId = 123456; + global.github = { + rest: { + actions: { + listJobsForWorkflowRun: vi.fn(), + downloadJobLogsForWorkflowRun: vi.fn().mockResolvedValue({ data: "" }), + }, + search: { issuesAndPullRequests: vi.fn(async () => ({ data: { total_count: 0, items: [] } })) }, + issues: { create: vi.fn(), createComment: vi.fn() }, + }, + paginate: vi.fn(async () => [ + { + id: 456, + name: "agent", + conclusion: "failure", + steps: [ + { name: "Run agent", conclusion: "success" }, + { name: "Redact secrets in logs", conclusion: "failure", started_at: "2026-10-10T10:00:02Z", completed_at: "2026-10-10T10:00:04Z" }, + ], + }, + ]), + graphql: vi.fn(), + }; + + try { + await main(); + const diagnostic = global.core.setFailed.mock.calls[0][0]; + expect(diagnostic).toContain("summary was suppressed because secret redaction failed"); + expect(diagnostic).toContain("agent_output.json in the agent artifact"); + expect(diagnostic).not.toContain("check the conclusion job summary"); + expect(diagnostic).not.toContain("safe but unverified reason"); + expect(diagnostic).not.toContain("private-redaction-details"); + expect(global.core.summary.addRaw).not.toHaveBeenCalled(); + expect(global.github.rest.issues.create).not.toHaveBeenCalled(); + } finally { + vi.unstubAllEnvs(); + } + }); + it.each([{}, " "])("does not write an empty incomplete-result summary for invalid reason %s", async reason => { const outputPath = path.join(tmpDir, "agent_output.json"); fs.writeFileSync(outputPath, JSON.stringify({ items: [{ type: "report_incomplete", reason }] })); vi.stubEnv("GH_AW_AGENT_OUTPUT", outputPath); + vi.stubEnv("GH_AW_AGENT_CONCLUSION", "success"); vi.stubEnv("GH_AW_FAILURE_REPORT_AS_ISSUE", "false"); + global.context.runId = 123456; global.github = { rest: { search: { issuesAndPullRequests: vi.fn(async () => ({ data: { total_count: 0, items: [] } })) }, @@ -2309,6 +2398,12 @@ describe("handle_agent_failure", () => { await main(); expect(global.core.summary.addRaw).not.toHaveBeenCalled(); expect(global.core.summary.write).not.toHaveBeenCalled(); + expect(global.core.setFailed).toHaveBeenCalledOnce(); + const diagnostic = global.core.setFailed.mock.calls[0][0]; + expect(diagnostic).toContain("report_incomplete signal had no usable reason"); + expect(diagnostic).toContain("agent_output.json in the agent artifact"); + expect(diagnostic).toContain("gh aw audit 123456"); + expect(diagnostic).not.toContain("conclusion job summary"); } finally { vi.unstubAllEnvs(); } @@ -3582,7 +3677,7 @@ describe("handle_agent_failure", () => { it("renders report_incomplete context in both comment and issue templates", () => { const reportIncompleteItems = [{ type: "report_incomplete", reason: "infrastructure_error", details: reportIncompleteMarker }]; - const reportIncompleteContext = buildReportIncompleteContext(reportIncompleteItems); + const reportIncompleteContext = buildReportIncompleteContext(reportIncompleteItems, { runId: 123456 }); const failureDiagnosticsContext = buildFailureDiagnosticsContext({ failureCategories: ["report_incomplete"], failingStep: "Run agent", @@ -3635,6 +3730,9 @@ describe("handle_agent_failure", () => { expect(rendered).not.toContain(reportIncompleteMarker); expect(rendered).toContain("The workflow recorded a `report_incomplete` signal"); expect(rendered).toContain("completion could not be confirmed"); + expect(rendered).toContain("This run needs a follow-up"); + expect(rendered).toContain("A successful agent job only confirms a clean process exit"); + expect(rendered).toContain("gh aw audit 123456"); expect(rendered).not.toContain("due to an infrastructure or tool failure"); } expect(reportIncompleteContext).toContain("infrastructure_error"); @@ -7674,7 +7772,7 @@ describe("handle_agent_failure", () => { const { main: mainFn } = require("./handle_agent_failure.cjs"); await mainFn(); - expect(global.core.setFailed).toHaveBeenCalledWith("Agent reported that it could not complete the task"); + expect(global.core.setFailed).toHaveBeenCalledWith(expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)")); expect(createIssueMock).toHaveBeenCalled(); }); @@ -7754,7 +7852,7 @@ describe("handle_agent_failure", () => { const { main: mainFn } = require("./handle_agent_failure.cjs"); await mainFn(); - expect(global.core.setFailed).toHaveBeenCalledWith("Agent reported that it could not complete the task"); + expect(global.core.setFailed).toHaveBeenCalledWith(expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)")); expect(createIssueMock).toHaveBeenCalled(); }); @@ -7965,7 +8063,7 @@ describe("handle_agent_failure", () => { // report_incomplete overrides the hasCompletedDespiteJobFailure exemption expect(createIssueMock).toHaveBeenCalled(); - expect(global.core.setFailed).toHaveBeenCalledWith("Agent reported that it could not complete the task"); + expect(global.core.setFailed).toHaveBeenCalledWith(expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)")); expect(global.core.warning).toHaveBeenCalledWith("Failed to create or update failure tracking issue: issue creation failed"); }); @@ -7993,7 +8091,7 @@ describe("handle_agent_failure", () => { const { main: mainFn } = require("./handle_agent_failure.cjs"); await mainFn(); - expect(global.core.setFailed).toHaveBeenCalledWith("Agent reported that it could not complete the task"); + expect(global.core.setFailed).toHaveBeenCalledWith(expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)")); expect(createIssueMock).not.toHaveBeenCalled(); });