Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/patch-copilot-org-billing-agent-failure.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

62 changes: 58 additions & 4 deletions actions/setup/js/handle_agent_failure.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,14 @@ const ELLIPSIS_LENGTH = ELLIPSIS.length;
const ENGINE_RATE_LIMIT_429_RE =
/(?:\b429\b[\s\S]{0,120}(?:too many requests|rate[\s-]*limit)|\brate_limit_(?:error|exceeded)\b|capierror:\s*429|failed to get response from the ai model[\s\S]{0,120}\b429\b|exceeded your rate limit for utility models)/i;
const ENGINE_MAX_RUNS_EXCEEDED_RE = /(?:\bmax_runs_exceeded\b|\bmaximum\s+llm\s+invocations\s+exceeded\b)/i;
const COPILOT_ORG_BILLING_MODE_RE = /API proxy enabled:[^\n]*Copilot=true \(github-token\)/i;
// Host allowlist kept aligned with isLikelyAWFAPIProxyURL in copilot_harness.cjs:
// awf_reflect rewrites api-proxy to host-bridge addresses for host execution.
const AWF_API_PROXY_HOST_RE_SOURCE = "(?:api-proxy|host\\.docker\\.internal|localhost|127(?:\\.\\d{1,3}){3}|10(?:\\.\\d{1,3}){3}|192\\.168(?:\\.\\d{1,3}){2}|172\\.(?:1[6-9]|2\\d|3[01])(?:\\.\\d{1,3}){2})";
const COPILOT_ORG_BILLING_ERROR_RE = new RegExp(
`(?:awf-reflect: models fetch returned 403\\b|Copilot requests authentication failed through the gh-aw API proxy \\(HTTP 403\\b|Authentication failed with provider at (?:https?:\\/\\/)?${AWF_API_PROXY_HOST_RE_SOURCE}(?::\\d+)?[^\\n]*\\(HTTP 403\\))`,
"i"
);
const ALLOWED_FILES_ERROR_RE = /^(?<summary>.*outside the allowed-files list) \((?<files>.+?)\)\. (?<remediation>Add the files to the allowed-files configuration field or remove them from the (?:patch|bundle)\.)$/;

/**
Expand Down Expand Up @@ -273,6 +281,7 @@ function buildFailureMatchCategories(options) {
if (options.secretVerificationFailed) categories.push("secret_verification_failed");
if (options.hasDockerSbxSecretsFailed) categories.push("docker_sbx_secrets_missing");
if (options.inferenceAccessError) categories.push("inference_access_error");
if (options.copilotOrgBillingError) categories.push("copilot_org_billing_error");
if (options.mcpPolicyError) categories.push("mcp_policy_error");
if (options.modelNotSupportedError) categories.push("model_not_supported_error");
if (options.http400ResponseError) categories.push("http_400_response_error");
Expand Down Expand Up @@ -325,6 +334,7 @@ function buildFailureMatchCategories(options) {
* @param {boolean} options.http400ResponseError
* @param {boolean} options.unknownModelAICredits
* @param {boolean} [options.hasDockerSbxSecretsFailed]
* @param {boolean} [options.copilotOrgBillingError]
* @param {boolean} [options.missingModelPricingError]
* @param {string} [options.missingModelPricingModelName]
* @param {boolean} [options.shellExpansionGuardRejected]
Expand Down Expand Up @@ -355,6 +365,7 @@ function buildFailureIssueTitle(options) {
if (options.hasStaleLockFileFailed) return `[aw] ${workflowName} has stale lock file`;
if (options.shellExpansionGuardRejected) return `[aw] ${workflowName} hit shell expansion guard rejection`;
if (options.hasDockerSbxSecretsFailed) return `[aw] ${workflowName} is missing docker-sbx Docker Hub secrets`;
if (options.copilotOrgBillingError) return `[aw] ${workflowName} hit Copilot organization billing error`;
if (options.isTimedOut) return `[aw] ${workflowName} timed out`;
if (options.hasToolDenialsExceeded) return `[aw] ${workflowName} exceeded tool denial limit`;
if (options.hasCacheMissMisconfiguration) return `[aw] ${workflowName} has cache-memory miss misconfiguration`;
Expand Down Expand Up @@ -1751,6 +1762,19 @@ function buildInferenceAccessErrorContext(hasInferenceAccessError) {
return "\n" + template;
}

/**
* Build remediation for Copilot failures caused by unavailable organization billing.
* @param {boolean} hasCopilotOrgBillingError
* @returns {string}
*/
function buildCopilotOrgBillingErrorContext(hasCopilotOrgBillingError) {
if (!hasCopilotOrgBillingError) {
return "";
}

return "\n" + renderPromptTemplate("copilot_org_billing_error.md");
}

/**
* Build a context string when MCP servers were blocked by enterprise/organization policy.
* This is a persistent configuration error — retrying will not help.
Expand Down Expand Up @@ -2728,6 +2752,27 @@ function hasAgentTerminalReasonCompleted() {
return false;
}

/**
* Detect Copilot authorization failures while the API proxy is using the
* organization-billed GITHUB_TOKEN supplied by copilot-requests: write.
* @param {string} [stdioLogPathOverride]
* @returns {boolean}
*/
function detectCopilotOrgBillingErrorFromLog(stdioLogPathOverride) {
if (process.env.GH_AW_ENGINE_ID !== "copilot") {
return false;
}

const agentOutputFile = process.env.GH_AW_AGENT_OUTPUT;
const stdioLogPath = stdioLogPathOverride || (agentOutputFile ? path.join(path.dirname(agentOutputFile), "agent-stdio.log") : "/tmp/gh-aw/agent-stdio.log");
try {
const logContent = fs.readFileSync(stdioLogPath, "utf8");
return COPILOT_ORG_BILLING_MODE_RE.test(logContent) && COPILOT_ORG_BILLING_ERROR_RE.test(logContent);
} catch {
return false;
}
}

/**
* Detect AWF firewall startup failure signals from log content.
* Uses specific failure patterns to avoid false positives on successful runs
Expand Down Expand Up @@ -3389,6 +3434,7 @@ async function main() {
const timeoutMinutes = process.env.GH_AW_TIMEOUT_MINUTES || "";
const { aiCredits, maxAICredits, aiCreditsRateLimitError, maxAICreditsExceeded } = resolveAICreditsFailureState();
const inferenceAccessError = process.env.GH_AW_INFERENCE_ACCESS_ERROR === "true";
const copilotOrgBillingError = detectCopilotOrgBillingErrorFromLog();

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Threaded in 8dc6865: copilotOrgBillingError now flows into buildFailureMatchCategories as copilot_org_billing_error (added to the schema filter pattern and the safe-outputs docs table) and into buildFailureIssueTitle. Tests assert the category replaces the agent_failure fallback and that the title is emitted.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/codebase-design] copilotOrgBillingError is computed and used to suppress inferenceAccessErrorContext/credentialAuthErrorContext, but it is never added to buildFailureMatchCategories(...) or buildFailureIssueTitle(...) (both called around line 3864-3920). Every other specialized failure signal in this file (inferenceAccessError, mcpPolicyError, modelNotSupportedError, etc.) flows into both the title and the dedup category list.

💡 Impact & suggested fix

Without this wiring:

  • The failure issue title falls back to the generic [aw] {workflow} failed instead of a specific org-billing title.
  • failureCategories won't include an copilot_org_billing_error entry, so the dedup marker can't distinguish this failure from a generic agent_failure, and future runs may not reuse/close the right issue.

Add copilotOrgBillingError as an option to both buildFailureIssueTitle (e.g. a title like [aw] {workflow} hit Copilot organization billing error) and buildFailureMatchCategories (e.g. copilot_org_billing_error category), mirroring how inferenceAccessError is threaded through.

@copilot please address this.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wired in 8dc6865: copilotOrgBillingError is now passed to both buildFailureIssueTitle ([aw] {workflow} hit Copilot organization billing error) and buildFailureMatchCategories (copilot_org_billing_error), mirroring inferenceAccessError. The category is also allowed by the safe-outputs schema pattern and documented.

const mcpPolicyError = process.env.GH_AW_MCP_POLICY_ERROR === "true";
const agenticEngineTimeout = process.env.GH_AW_AGENTIC_ENGINE_TIMEOUT === "true";
const modelNotSupportedError = process.env.GH_AW_MODEL_NOT_SUPPORTED_ERROR === "true";
Expand Down Expand Up @@ -3848,6 +3894,7 @@ async function main() {
missingModelPricingError,
missingModelPricingModelName,
hasDockerSbxSecretsFailed,
copilotOrgBillingError,
});
const failureCategories = buildFailureMatchCategories({
agentConclusion,
Expand All @@ -3868,6 +3915,7 @@ async function main() {
secretVerificationFailed: hasSecretVerificationFailed,
hasDockerSbxSecretsFailed,
inferenceAccessError,
copilotOrgBillingError,
mcpPolicyError,
modelNotSupportedError,
http400ResponseError,
Expand Down Expand Up @@ -4041,7 +4089,8 @@ async function main() {
const timeoutContext = buildTimeoutContext(isTimedOut, timeoutMinutes);

// Build inference access error context
const inferenceAccessErrorContext = buildInferenceAccessErrorContext(inferenceAccessError);
const copilotOrgBillingErrorContext = buildCopilotOrgBillingErrorContext(copilotOrgBillingError);
const inferenceAccessErrorContext = copilotOrgBillingErrorContext ? "" : buildInferenceAccessErrorContext(inferenceAccessError);

// Build MCP policy error context
const mcpPolicyErrorContext = buildMCPPolicyErrorContext(mcpPolicyError);
Expand Down Expand Up @@ -4073,7 +4122,7 @@ async function main() {
const skillInstallFailureContext = buildSkillInstallFailureContext(hasSkillInstallFailures, skillInstallErrors);

// Build credential auth error context (firewall audit.jsonl 401/403 from provider endpoints)
const credentialAuthErrorContext = buildCredentialAuthErrorContext();
const credentialAuthErrorContext = copilotOrgBillingErrorContext ? "" : buildCredentialAuthErrorContext();

// Create template context
const templateContext = {
Expand All @@ -4086,6 +4135,7 @@ async function main() {
secret_verification_context: buildSecretVerificationContext(secretVerificationResult, engineSecretFailureMessage),
docker_sbx_secrets_context: buildDockerSbxSecretsContext(dockerSbxSecretsResult),
credential_auth_error_context: credentialAuthErrorContext,
copilot_org_billing_error_context: copilotOrgBillingErrorContext,
assignment_errors_context: assignmentErrorsContext,
assign_copilot_failure_context: assignCopilotFailureContext,
skill_install_failure_context: skillInstallFailureContext,
Expand Down Expand Up @@ -4271,7 +4321,8 @@ async function main() {
const timeoutContext = buildTimeoutContext(isTimedOut, timeoutMinutes);

// Build inference access error context
const inferenceAccessErrorContext = buildInferenceAccessErrorContext(inferenceAccessError);
const copilotOrgBillingErrorContext = buildCopilotOrgBillingErrorContext(copilotOrgBillingError);
const inferenceAccessErrorContext = copilotOrgBillingErrorContext ? "" : buildInferenceAccessErrorContext(inferenceAccessError);

// Build MCP policy error context
const mcpPolicyErrorContext = buildMCPPolicyErrorContext(mcpPolicyError);
Expand Down Expand Up @@ -4303,7 +4354,7 @@ async function main() {
const skillInstallFailureContext = buildSkillInstallFailureContext(hasSkillInstallFailures, skillInstallErrors);

// Build credential auth error context (firewall audit.jsonl 401/403 from provider endpoints)
const credentialAuthErrorContext = buildCredentialAuthErrorContext();
const credentialAuthErrorContext = copilotOrgBillingErrorContext ? "" : buildCredentialAuthErrorContext();

// Build optimize token consumption context (shown when a guardrail was the failure root cause)
const optimizeTokenConsumptionContext = buildOptimizeTokenConsumptionContext({ maxAICreditsExceeded, hasDailyAICExceeded, hasToolDenialsExceeded, isTimedOut, runUrl });
Expand All @@ -4320,6 +4371,7 @@ async function main() {
secret_verification_context: buildSecretVerificationContext(secretVerificationResult, engineSecretFailureMessage),
docker_sbx_secrets_context: buildDockerSbxSecretsContext(dockerSbxSecretsResult),
credential_auth_error_context: credentialAuthErrorContext,
copilot_org_billing_error_context: copilotOrgBillingErrorContext,
assignment_errors_context: assignmentErrorsContext,
assign_copilot_failure_context: assignCopilotFailureContext,
skill_install_failure_context: skillInstallFailureContext,
Expand Down Expand Up @@ -4463,6 +4515,8 @@ module.exports = {
detectAWFFirewallStartupFailureFromLog,
buildReportIncompleteContext,
buildMCPPolicyErrorContext,
buildCopilotOrgBillingErrorContext,
detectCopilotOrgBillingErrorFromLog,
buildModelNotSupportedErrorContext,
buildHTTP400ResponseErrorContext,
buildMissingDataContext,
Expand Down
80 changes: 80 additions & 0 deletions actions/setup/js/handle_agent_failure.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ describe("handle_agent_failure", () => {
{ flag: "hasMissingTool", expected: "[aw] Test Workflow is missing required tool" },
{ flag: "hasMissingData", expected: "[aw] Test Workflow is missing required data" },
{ flag: "hasAssignmentErrors", expected: "[aw] Test Workflow failed to assign agent" },
{ flag: "copilotOrgBillingError", expected: "[aw] Test Workflow hit Copilot organization billing error" },
];

it.each(cases)("returns expected title for isolated $flag", ({ flag, expected }) => {
Expand Down Expand Up @@ -3562,6 +3563,75 @@ describe("handle_agent_failure", () => {
// buildModelNotSupportedErrorContext
// ──────────────────────────────────────────────────────

describe("buildCopilotOrgBillingErrorContext", () => {
let buildCopilotOrgBillingErrorContext;
let detectCopilotOrgBillingErrorFromLog;
const fs = require("fs");
const path = require("path");
const os = require("os");
let tmpDir;

beforeEach(() => {
vi.resetModules();
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "aw-test-copilot-org-billing-"));
const promptsDir = path.join(tmpDir, "gh-aw", "prompts");
fs.mkdirSync(promptsDir, { recursive: true });
fs.copyFileSync(path.join(runtimePromptsDir, "copilot_org_billing_error.md"), path.join(promptsDir, "copilot_org_billing_error.md"));
process.env.RUNNER_TEMP = tmpDir;
({ buildCopilotOrgBillingErrorContext, detectCopilotOrgBillingErrorFromLog } = require("./handle_agent_failure.cjs"));
});

afterEach(() => {
delete process.env.RUNNER_TEMP;
delete process.env.GH_AW_ENGINE_ID;
fs.rmSync(tmpDir, { recursive: true, force: true });
});

it("returns billing and PAT guidance for a detected organization-billed failure", () => {
const result = buildCopilotOrgBillingErrorContext(true);
expect(result).toContain("Copilot organization billing is unavailable");
expect(result).toContain("Allow use of Copilot CLI billed to the organization");
expect(result).toContain("COPILOT_GITHUB_TOKEN");
expect(result).toContain("https://github.github.com/gh-aw/reference/billing/");
});

it("returns no guidance when the combined error was not detected", () => {
expect(buildCopilotOrgBillingErrorContext(false)).toBe("");
});

it.each([
"[copilot-harness] awf-reflect: models fetch returned 403 for http://api-proxy:10002/models",
"Copilot requests authentication failed through the gh-aw API proxy (HTTP 403, model=auto, stage=listing models).",
"Authentication failed with provider at http://172.30.0.30:10002 (HTTP 403).",
])("detects reported organization-billed Copilot error: %s", errorOutput => {
process.env.GH_AW_ENGINE_ID = "copilot";
const logPath = path.join(tmpDir, "agent-stdio.log");
fs.writeFileSync(logPath, `[INFO] API proxy enabled: OpenAI=false, Copilot=true (github-token)\n${errorOutput}`);
expect(detectCopilotOrgBillingErrorFromLog(logPath)).toBe(true);
});

it("does not classify PAT-based Copilot failures as organization billing errors", () => {
process.env.GH_AW_ENGINE_ID = "copilot";
const logPath = path.join(tmpDir, "agent-stdio.log");
fs.writeFileSync(logPath, "[copilot-harness] awf-reflect: models fetch returned 403 for http://api-proxy:10002/models");
expect(detectCopilotOrgBillingErrorFromLog(logPath)).toBe(false);
});

it.each(["host.docker.internal", "localhost", "127.0.0.1", "10.1.2.3", "192.168.1.4"])("detects provider auth failures against host-bridge proxy host %s", proxyHost => {
process.env.GH_AW_ENGINE_ID = "copilot";
const logPath = path.join(tmpDir, "agent-stdio.log");
fs.writeFileSync(logPath, `[INFO] API proxy enabled: OpenAI=false, Copilot=true (github-token)\nAuthentication failed with provider at http://${proxyHost}:10002 (HTTP 403).`);
expect(detectCopilotOrgBillingErrorFromLog(logPath)).toBe(true);
});

it.each(["Access denied by policy settings", "invalid access to inference"])("does not classify generic inference access rejection %s as an organization billing error", errorOutput => {
process.env.GH_AW_ENGINE_ID = "copilot";
const logPath = path.join(tmpDir, "agent-stdio.log");
fs.writeFileSync(logPath, `[INFO] API proxy enabled: OpenAI=false, Copilot=true (github-token)\n${errorOutput}`);
expect(detectCopilotOrgBillingErrorFromLog(logPath)).toBe(false);
});
});

describe("buildModelNotSupportedErrorContext", () => {
let buildModelNotSupportedErrorContext;
const fs = require("fs");
Expand Down Expand Up @@ -5638,6 +5708,16 @@ describe("handle_agent_failure", () => {
}
});

it("returns copilot_org_billing_error category instead of the agent_failure fallback", () => {
const categories = buildFailureMatchCategories({
agentConclusion: "failure",
isTimedOut: false,
copilotOrgBillingError: true,
});
expect(categories).toContain("copilot_org_billing_error");
expect(categories).not.toContain("agent_failure");
});

it("returns http_400_response_error category", () => {
const categories = buildFailureMatchCategories({
http400ResponseError: true,
Expand Down
2 changes: 1 addition & 1 deletion actions/setup/md/agent_failure_comment.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
Agent job [{run_id}]({run_url}) failed.

{secret_verification_context}{docker_sbx_secrets_context}{credential_auth_error_context}{inference_access_error_context}{mcp_policy_error_context}{model_not_supported_error_context}{http_400_response_error_context}{ai_credits_rate_limit_error_context}{unknown_model_ai_credits_context}{missing_model_pricing_context}{shell_expansion_guard_rejected_context}{app_token_minting_failed_context}{lockdown_check_failed_context}{oauth_token_check_failed_context}{stale_lock_file_failed_context}{daily_ai_credits_exceeded_context}{daily_ai_credits_guardrail_error_context}{assignment_errors_context}{assign_copilot_failure_context}{skill_install_failure_context}{create_discussion_errors_context}{code_push_failure_context}{repo_memory_validation_context}{push_repo_memory_failure_context}{missing_data_context}{missing_tool_context}{permission_denied_context}{tool_denials_exceeded_context}{report_incomplete_context}{missing_safe_outputs_context}{engine_failure_context}{timeout_context}{fork_context}
{secret_verification_context}{docker_sbx_secrets_context}{copilot_org_billing_error_context}{credential_auth_error_context}{inference_access_error_context}{mcp_policy_error_context}{model_not_supported_error_context}{http_400_response_error_context}{ai_credits_rate_limit_error_context}{unknown_model_ai_credits_context}{missing_model_pricing_context}{shell_expansion_guard_rejected_context}{app_token_minting_failed_context}{lockdown_check_failed_context}{oauth_token_check_failed_context}{stale_lock_file_failed_context}{daily_ai_credits_exceeded_context}{daily_ai_credits_guardrail_error_context}{assignment_errors_context}{assign_copilot_failure_context}{skill_install_failure_context}{create_discussion_errors_context}{code_push_failure_context}{repo_memory_validation_context}{push_repo_memory_failure_context}{missing_data_context}{missing_tool_context}{permission_denied_context}{tool_denials_exceeded_context}{report_incomplete_context}{missing_safe_outputs_context}{engine_failure_context}{timeout_context}{fork_context}
2 changes: 1 addition & 1 deletion actions/setup/md/agent_failure_issue.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
**Branch:** {branch}
**Run:** {run_url}{pull_request_info}

{secret_verification_context}{docker_sbx_secrets_context}{credential_auth_error_context}{inference_access_error_context}{mcp_policy_error_context}{model_not_supported_error_context}{http_400_response_error_context}{ai_credits_rate_limit_error_context}{unknown_model_ai_credits_context}{missing_model_pricing_context}{shell_expansion_guard_rejected_context}{app_token_minting_failed_context}{lockdown_check_failed_context}{oauth_token_check_failed_context}{stale_lock_file_failed_context}{daily_ai_credits_exceeded_context}{daily_ai_credits_guardrail_error_context}{assignment_errors_context}{assign_copilot_failure_context}{skill_install_failure_context}{create_discussion_errors_context}{code_push_failure_context}{repo_memory_validation_context}{push_repo_memory_failure_context}{missing_data_context}{missing_tool_context}{permission_denied_context}{tool_denials_exceeded_context}{report_incomplete_context}{missing_safe_outputs_context}{engine_failure_context}{timeout_context}{fork_context}
{secret_verification_context}{docker_sbx_secrets_context}{copilot_org_billing_error_context}{credential_auth_error_context}{inference_access_error_context}{mcp_policy_error_context}{model_not_supported_error_context}{http_400_response_error_context}{ai_credits_rate_limit_error_context}{unknown_model_ai_credits_context}{missing_model_pricing_context}{shell_expansion_guard_rejected_context}{app_token_minting_failed_context}{lockdown_check_failed_context}{oauth_token_check_failed_context}{stale_lock_file_failed_context}{daily_ai_credits_exceeded_context}{daily_ai_credits_guardrail_error_context}{assignment_errors_context}{assign_copilot_failure_context}{skill_install_failure_context}{create_discussion_errors_context}{code_push_failure_context}{repo_memory_validation_context}{push_repo_memory_failure_context}{missing_data_context}{missing_tool_context}{permission_denied_context}{tool_denials_exceeded_context}{report_incomplete_context}{missing_safe_outputs_context}{engine_failure_context}{timeout_context}{fork_context}

### Action Required

Expand Down
Loading
Loading