Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 35 additions & 9 deletions actions/setup/js/handle_agent_failure.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -1902,14 +1902,14 @@ function loadReportIncompleteMessages(items) {
}

/**
* Build report_incomplete context string for display in failure issues/comments.
* Build report_incomplete context for display in failure issues/comments.
* This surfaces the agent's structured incompletion signal so maintainers can
* distinguish an incomplete task from a confirmed task outcome.
* @param {Array<any>} [items] - Optional pre-loaded agent output items. When provided, avoids re-reading the output file.
* @param {{collectorEmptyOutputCause?: string, rootDir?: string}} [options]
* @param {{collectorEmptyOutputCause?: string, rootDir?: string, runId?: number}} [options]
* @returns {ReportBlock[]}
*/
function buildReportIncompleteContext(items, { collectorEmptyOutputCause, rootDir } = {}) {
function buildReportIncompleteContext(items, { collectorEmptyOutputCause, rootDir, runId } = {}) {
const trustedCollectorCause = typeof collectorEmptyOutputCause === "string" && Object.hasOwn(EMPTY_OUTPUT_CAUSES, collectorEmptyOutputCause) ? collectorEmptyOutputCause : undefined;
const messages = loadReportIncompleteMessages(items).filter(message => message.reason !== trustedCollectorCause);

Expand Down Expand Up @@ -1939,13 +1939,36 @@ function buildReportIncompleteContext(items, { collectorEmptyOutputCause, rootDi
}
context.push(
nodes.paragraph([
nodes.text("A comment or other safe output does not confirm that the requested task was completed. Reported reasons are not verified causes; runtime evidence is shown separately and agent diagnostic details are excluded."),
nodes.text(
"This run needs a follow-up: the agent reported that it could not finish. A successful agent job only confirms a clean process exit, not that the requested work was completed. Treat the agent-declared reason as a lead, not a confirmed root cause; agent diagnostic details are excluded, so compare the note with the runtime evidence below."
),
])
);
if (typeof runId === "number" && Number.isSafeInteger(runId) && runId > 0) {
context.push(nodes.paragraph([nodes.text("For a runtime-focused investigation, run "), nodes.code(`gh aw audit ${runId}`), nodes.text(".")]));
}

return context;
}

/**
* Build safe next-step guidance for an agent-reported incomplete task.
* @param {Array<any>} items
* @param {{runId?: number, secretRedactionFailed?: boolean}} [options]
* @returns {string}
*/
function buildReportIncompleteFailureMessage(items, { runId, secretRedactionFailed = false } = {}) {
const hasUsableReason = loadReportIncompleteMessages(items).length > 0;
const diagnosticNextStep = !hasUsableReason
? "The report_incomplete signal had no usable reason; inspect agent_output.json in the agent artifact."
: secretRedactionFailed
? "The conclusion job summary was suppressed because secret redaction failed; inspect agent_output.json in the agent artifact."
: "If available, check the conclusion job summary for the reported reason and inspect agent_output.json in the agent artifact.";
const auditCommand = typeof runId === "number" && Number.isSafeInteger(runId) && runId > 0 ? `gh aw audit ${runId}` : "gh aw audit <run-id>";

return `Agent reported that it could not complete the task (report_incomplete). A successful agent job does not confirm task completion. ${diagnosticNextStep} Run ${auditCommand} for runtime evidence.`;
}

/**
* Only fixed redactor diagnostics from the failed step's time window may be
* published. Agent logs and arbitrary exception payloads are never excerpts.
Expand Down Expand Up @@ -4473,6 +4496,7 @@ async function main() {
let hasMissingSafeOutputs = false;
let hasOnlyNoopOutputs = false;
let hasReportIncomplete = false;
let reportIncompleteItems = [];
// Tracks the case where agentConclusion is "failure" but the agent completed its work
// successfully (terminal_reason: completed) and produced valid non-noop safe outputs.
// This is a false-positive failure caused by a transient error after the agent's task
Expand Down Expand Up @@ -4544,16 +4568,13 @@ async function main() {
// classified as a successful review or other completed task.
if (agentOutputResult.success && agentOutputResult.items && agentOutputResult.items.length > 0) {
// Cancellation can interrupt the agent before it starts or emits a terminal safe output.
const reportIncompleteItems = agentOutputResult.items.filter(item => item.type === "report_incomplete" && !(isCancelledEmptyOutput && item.reason === collectorEmptyOutputCause));
reportIncompleteItems = agentOutputResult.items.filter(item => item.type === "report_incomplete" && !(isCancelledEmptyOutput && item.reason === collectorEmptyOutputCause));
if (reportIncompleteItems.length > 0) {
if (taskCompleteRegistrationIssueOnlyReportIncomplete) {
core.info("Ignoring report_incomplete signal(s) caused only by task_complete registration trouble after successful task-level outputs");
} else {
hasReportIncomplete = true;
core.info(`Agent emitted ${reportIncompleteItems.length} report_incomplete signal(s) - activating failure handling`);
// Continue after marking the step failed so optional issue reporting can still run.
// eslint-disable-next-line gh-aw-custom/require-return-after-core-setfailed
core.setFailed("Agent reported that it could not complete the task");
}
}
}
Expand Down Expand Up @@ -4653,6 +4674,11 @@ async function main() {
}

failureDiagnostics ??= await getFailedAgentDiagnostics({ startupFailure: collectorEmptyOutputCause === "agent_startup_failure" });
if (hasReportIncomplete) {
// Continue after marking the step failed so optional issue reporting can still run.
// eslint-disable-next-line gh-aw-custom/require-return-after-core-setfailed
core.setFailed(buildReportIncompleteFailureMessage(reportIncompleteItems, { runId: context.runId, secretRedactionFailed: failureDiagnostics.secretRedactionFailed }));
}
if (failureDiagnostics.hasNoJobs && !hasCheckoutFailure) {
core.info("Skipping failure reporting: no workflow jobs started; the run may not have been approved.");
return;
Expand Down Expand Up @@ -4701,7 +4727,7 @@ async function main() {
!(isCancelledEmptyOutput && item?.type === "report_incomplete" && item.reason === collectorEmptyOutputCause)
);
const hasReportedIncomplete = hasReportIncomplete && reportItems.some(item => item?.type === "report_incomplete");
const reportIncompleteContext = buildReportIncompleteContext(reportItems, { collectorEmptyOutputCause: emptyOutputCause, rootDir: path.dirname(getAgentStdioLogPath()) });
const reportIncompleteContext = buildReportIncompleteContext(reportItems, { collectorEmptyOutputCause: emptyOutputCause, rootDir: path.dirname(getAgentStdioLogPath()), runId: context.runId });
if (hasReportedIncomplete && reportIncompleteContext.length > 0 && !failureDiagnostics.secretRedactionFailed) {
try {
const summary = renderSizedReport({ type: "report", title: "Incomplete result", children: reportIncompleteContext, agentTainted: false }).body;
Expand Down
126 changes: 112 additions & 14 deletions actions/setup/js/handle_agent_failure.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -1004,13 +1004,13 @@ describe("handle_agent_failure", () => {
);
});

it("keeps the incompletion notice concise without repeating diagnostic policy", () => {
const context = buildReportIncompleteContext([{ type: "report_incomplete", reason: "infrastructure_error", details: "agent-generated diagnostic" }]);
expect(context).toContain("completion could not be confirmed");
expect(context).toContain("A comment or other safe output does not confirm that the requested task was completed.");
expect(context).not.toContain("See the reported reason");
expect(context).not.toContain("agent-generated");
expect(context).toContain("infrastructure_error");
it("explains the incomplete handoff without exposing agent diagnostic details", () => {
const rendered = buildReportIncompleteContext([{ type: "report_incomplete", reason: "infrastructure_error", details: "agent-generated diagnostic" }]);
expect(rendered).toContain("completion could not be confirmed");
expect(rendered).toContain("Treat the agent-declared reason as a lead, not a confirmed root cause");
expect(rendered).toContain("agent diagnostic details are excluded");
expect(rendered).not.toContain("agent-generated");
expect(rendered).toContain("infrastructure_error");
});
});

Expand Down Expand Up @@ -1674,7 +1674,7 @@ describe("handle_agent_failure", () => {
const write = vi.spyOn(fs, "writeFileSync");
try {
await main();
expect(global.core.setFailed.mock.calls).toEqual(startup || crashed ? [["Agent reported that it could not complete the task"]] : []);
expect(global.core.setFailed.mock.calls).toEqual(startup || crashed ? [[expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)")]] : []);
expect(create).toHaveBeenCalledOnce();
const issue = create.mock.calls[0][0];
expect(issue.title).toBe(startup ? "[aw] Test Workflow: failed before agent execution started" : "[aw] Test Workflow: failed to redact secrets in logs");
Expand Down Expand Up @@ -1803,7 +1803,7 @@ describe("handle_agent_failure", () => {
};
try {
await main();
expect(global.core.setFailed.mock.calls).toEqual([["Agent reported that it could not complete the task"]]);
expect(global.core.setFailed.mock.calls).toEqual([[expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)")]]);
expect(create).toHaveBeenCalledOnce();
const { title, body } = create.mock.calls[0][0];
expect(title).toBe("[aw] Test Workflow: engine driver crashed");
Expand Down Expand Up @@ -2292,11 +2292,100 @@ describe("handle_agent_failure", () => {
}
});

it("explains clean-exit incompletion and points to diagnostics without logging the agent payload", async () => {
const outputPath = path.join(tmpDir, "agent_output.json");
fs.writeFileSync(
outputPath,
JSON.stringify({
items: [{ type: "report_incomplete", reason: "Required compilation and validation cannot run within the configured build-resource permissions.", details: "private-build-diagnostics" }],
})
);
vi.stubEnv("GH_AW_AGENT_OUTPUT", outputPath);
vi.stubEnv("GH_AW_AGENT_CONCLUSION", "success");
vi.stubEnv("GH_AW_FAILURE_REPORT_AS_ISSUE", "false");
global.context.runId = 123456;
global.github = {
rest: {
search: { issuesAndPullRequests: vi.fn(async () => ({ data: { total_count: 0, items: [] } })) },
issues: { create: vi.fn(), createComment: vi.fn() },
},
graphql: vi.fn(),
};

try {
await main();
expect(global.core.setFailed).toHaveBeenCalledOnce();
const diagnostic = global.core.setFailed.mock.calls[0][0];
expect(diagnostic).toContain("report_incomplete");
expect(diagnostic).toContain("A successful agent job does not confirm task completion");
expect(diagnostic).toContain("conclusion job summary");
expect(diagnostic).toContain("agent_output.json in the agent artifact");
expect(diagnostic).toContain("gh aw audit 123456");
const logs = JSON.stringify([...global.core.info.mock.calls, ...global.core.warning.mock.calls, ...global.core.setFailed.mock.calls]);
expect(logs).not.toContain("build-resource permissions");
expect(logs).not.toContain("private-build-diagnostics");
const summary = JSON.stringify(global.core.summary.addRaw.mock.calls);
expect(summary).toContain("build-resource permissions");
expect(summary).not.toContain("private-build-diagnostics");
expect(global.core.summary.write).toHaveBeenCalled();
expect(global.github.rest.issues.create).not.toHaveBeenCalled();
} finally {
vi.unstubAllEnvs();
}
});

it("points to the agent artifact when secret redaction suppresses the summary", async () => {
const outputPath = path.join(tmpDir, "agent_output.json");
fs.writeFileSync(outputPath, JSON.stringify({ items: [{ type: "report_incomplete", reason: "safe but unverified reason", details: "private-redaction-details" }] }));
vi.stubEnv("GH_AW_AGENT_OUTPUT", outputPath);
vi.stubEnv("GH_AW_AGENT_CONCLUSION", "success");
vi.stubEnv("GH_AW_FAILURE_REPORT_AS_ISSUE", "false");
global.context.runId = 123456;
global.github = {
rest: {
actions: {
listJobsForWorkflowRun: vi.fn(),
downloadJobLogsForWorkflowRun: vi.fn().mockResolvedValue({ data: "" }),
},
search: { issuesAndPullRequests: vi.fn(async () => ({ data: { total_count: 0, items: [] } })) },
issues: { create: vi.fn(), createComment: vi.fn() },
},
paginate: vi.fn(async () => [
{
id: 456,
name: "agent",
conclusion: "failure",
steps: [
{ name: "Run agent", conclusion: "success" },
{ name: "Redact secrets in logs", conclusion: "failure", started_at: "2026-10-10T10:00:02Z", completed_at: "2026-10-10T10:00:04Z" },
],
},
]),
graphql: vi.fn(),
};

try {
await main();
const diagnostic = global.core.setFailed.mock.calls[0][0];
expect(diagnostic).toContain("summary was suppressed because secret redaction failed");
expect(diagnostic).toContain("agent_output.json in the agent artifact");
expect(diagnostic).not.toContain("check the conclusion job summary");
expect(diagnostic).not.toContain("safe but unverified reason");
expect(diagnostic).not.toContain("private-redaction-details");
expect(global.core.summary.addRaw).not.toHaveBeenCalled();
expect(global.github.rest.issues.create).not.toHaveBeenCalled();
} finally {
vi.unstubAllEnvs();
}
});

it.each([{}, " "])("does not write an empty incomplete-result summary for invalid reason %s", async reason => {
const outputPath = path.join(tmpDir, "agent_output.json");
fs.writeFileSync(outputPath, JSON.stringify({ items: [{ type: "report_incomplete", reason }] }));
vi.stubEnv("GH_AW_AGENT_OUTPUT", outputPath);
vi.stubEnv("GH_AW_AGENT_CONCLUSION", "success");
vi.stubEnv("GH_AW_FAILURE_REPORT_AS_ISSUE", "false");
global.context.runId = 123456;
global.github = {
rest: {
search: { issuesAndPullRequests: vi.fn(async () => ({ data: { total_count: 0, items: [] } })) },
Expand All @@ -2309,6 +2398,12 @@ describe("handle_agent_failure", () => {
await main();
expect(global.core.summary.addRaw).not.toHaveBeenCalled();
expect(global.core.summary.write).not.toHaveBeenCalled();
expect(global.core.setFailed).toHaveBeenCalledOnce();
const diagnostic = global.core.setFailed.mock.calls[0][0];
expect(diagnostic).toContain("report_incomplete signal had no usable reason");
expect(diagnostic).toContain("agent_output.json in the agent artifact");
expect(diagnostic).toContain("gh aw audit 123456");
expect(diagnostic).not.toContain("conclusion job summary");
} finally {
vi.unstubAllEnvs();
}
Expand Down Expand Up @@ -3582,7 +3677,7 @@ describe("handle_agent_failure", () => {

it("renders report_incomplete context in both comment and issue templates", () => {
const reportIncompleteItems = [{ type: "report_incomplete", reason: "infrastructure_error", details: reportIncompleteMarker }];
const reportIncompleteContext = buildReportIncompleteContext(reportIncompleteItems);
const reportIncompleteContext = buildReportIncompleteContext(reportIncompleteItems, { runId: 123456 });
const failureDiagnosticsContext = buildFailureDiagnosticsContext({
failureCategories: ["report_incomplete"],
failingStep: "Run agent",
Expand Down Expand Up @@ -3635,6 +3730,9 @@ describe("handle_agent_failure", () => {
expect(rendered).not.toContain(reportIncompleteMarker);
expect(rendered).toContain("The workflow recorded a `report_incomplete` signal");
expect(rendered).toContain("completion could not be confirmed");
expect(rendered).toContain("This run needs a follow-up");
expect(rendered).toContain("A successful agent job only confirms a clean process exit");
expect(rendered).toContain("gh aw audit 123456");
expect(rendered).not.toContain("due to an infrastructure or tool failure");
}
expect(reportIncompleteContext).toContain("infrastructure_error");
Expand Down Expand Up @@ -7674,7 +7772,7 @@ describe("handle_agent_failure", () => {
const { main: mainFn } = require("./handle_agent_failure.cjs");
await mainFn();

expect(global.core.setFailed).toHaveBeenCalledWith("Agent reported that it could not complete the task");
expect(global.core.setFailed).toHaveBeenCalledWith(expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)"));
expect(createIssueMock).toHaveBeenCalled();
});

Expand Down Expand Up @@ -7754,7 +7852,7 @@ describe("handle_agent_failure", () => {
const { main: mainFn } = require("./handle_agent_failure.cjs");
await mainFn();

expect(global.core.setFailed).toHaveBeenCalledWith("Agent reported that it could not complete the task");
expect(global.core.setFailed).toHaveBeenCalledWith(expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)"));
expect(createIssueMock).toHaveBeenCalled();
});

Expand Down Expand Up @@ -7965,7 +8063,7 @@ describe("handle_agent_failure", () => {

// report_incomplete overrides the hasCompletedDespiteJobFailure exemption
expect(createIssueMock).toHaveBeenCalled();
expect(global.core.setFailed).toHaveBeenCalledWith("Agent reported that it could not complete the task");
expect(global.core.setFailed).toHaveBeenCalledWith(expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)"));
expect(global.core.warning).toHaveBeenCalledWith("Failed to create or update failure tracking issue: issue creation failed");
});

Expand Down Expand Up @@ -7993,7 +8091,7 @@ describe("handle_agent_failure", () => {
const { main: mainFn } = require("./handle_agent_failure.cjs");
await mainFn();

expect(global.core.setFailed).toHaveBeenCalledWith("Agent reported that it could not complete the task");
expect(global.core.setFailed).toHaveBeenCalledWith(expect.stringContaining("Agent reported that it could not complete the task (report_incomplete)"));
expect(createIssueMock).not.toHaveBeenCalled();
});

Expand Down
Loading