From e3c87ee5e933547be4907a721609090412bf1f61 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:33:08 +0000 Subject: [PATCH 1/2] Initial plan From 89a0dcc7a8e2db3d0f6b841e2392911a930dc9ae Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:50:09 +0000 Subject: [PATCH 2/2] Handle missing evals usage as zero AIC Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../setup/js/daily_aic_component_coverage.cjs | 31 ++++--------------- .../js/daily_aic_component_coverage.test.cjs | 20 +++++++----- pkg/workflow/evals_steps.go | 6 ---- 3 files changed, 18 insertions(+), 39 deletions(-) diff --git a/actions/setup/js/daily_aic_component_coverage.cjs b/actions/setup/js/daily_aic_component_coverage.cjs index 662e62b4dd0..ee0739703e9 100644 --- a/actions/setup/js/daily_aic_component_coverage.cjs +++ b/actions/setup/js/daily_aic_component_coverage.cjs @@ -65,22 +65,6 @@ function provesExecutionNotStarted(directory, name, runId, runAttempt) { } } -// Step names below must stay in sync with the evals job steps generated by -// buildUploadEvalsArtifactStep in pkg/workflow/evals_steps.go: renaming those -// steps requires updating this list too, or this check silently stops matching. -function provesFailedEvalsHadNoUsage(job) { - if (job.conclusion !== "failure" || !Array.isArray(job.steps)) return false; - const succeeded = name => job.steps.some(step => step.name === name && step.conclusion === "success"); - const hasFailureUpload = job.steps.some(step => step.name === "Upload evals accounting after failure"); - // Legacy workflows had no failure-path accounting upload. Treat a successful - // collector with no published file as zero usage for those workflows. - if (!hasFailureUpload && succeeded("Collect evals token usage")) return true; - // A successful collector only proves the local shell step ran; the eval - // artifact upload (whichever of the two mutually exclusive steps applies) - // must also have succeeded, or a transport failure would be miscounted as zero. - return succeeded("Collect evals token usage") && (succeeded("Upload evals results") || succeeded("Upload evals accounting after failure")); -} - function inspectAccountingFile(directory, file) { const relativeFile = path.relative(directory, file); if (!fs.existsSync(file)) return { file: relativeFile, state: "missing" }; @@ -171,20 +155,17 @@ function sumCoveredComponents(directory, components, artifactCreatedAt, artifact }); continue; } - // The eval collector only publishes its output when it finds non-empty - // firewall accounting. A successful collector with no published file - // therefore proves that the failed eval made no billable model request. - if (!selected && name === "evals" && provesFailedEvalsHadNoUsage(job)) { - logComponentAIC(runId, name, job, 0, "failed_before_accounting", { - source: candidateStates[0].file, - }); - continue; - } if (!selected) { if (provesExecutionNotStarted(directory, name, runId, job.run_attempt)) { logComponentAIC(runId, name, job, 0, "execution_not_started"); continue; } + if (name === "evals" && job.conclusion === "failure") { + logComponentAIC(runId, name, job, 0, "failed_before_accounting", { + source: candidateStates[0].file, + }); + continue; + } throw new Error(`Missing accounting for executed ${name} component in run ${runId} (attempt ${job.run_attempt}, job ${job.id}, conclusion ${job.conclusion}): ${candidateSummary}`); } const componentAIC = sumAICFromUsageJSONLFiles([selected], { strict: true }); diff --git a/actions/setup/js/daily_aic_component_coverage.test.cjs b/actions/setup/js/daily_aic_component_coverage.test.cjs index 0ce71e5a2ea..526dbb1a8ef 100644 --- a/actions/setup/js/daily_aic_component_coverage.test.cjs +++ b/actions/setup/js/daily_aic_component_coverage.test.cjs @@ -189,6 +189,8 @@ it("counts missing evals accounting as zero when the successful-path upload step ] ); await expect(f.result).resolves.toBe(2); + expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"component":"evals"')); + expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"aic":0,"reason":"failed_before_accounting"')); }); it("counts missing evals accounting as zero for a legacy failed evals job", async () => { @@ -211,7 +213,7 @@ it("counts missing evals accounting as zero for a legacy failed evals job", asyn await expect(f.result).resolves.toBe(2); }); -it("still requires evals accounting when collection did not succeed", async () => { +it("counts missing evals accounting as zero when collection did not succeed", async () => { const f = evaluate( { "agent/token_usage.jsonl": '{"aic":2}', @@ -227,10 +229,10 @@ it("still requires evals accounting when collection did not succeed", async () = }), ] ); - await expect(f.result).rejects.toThrow("Missing accounting for executed evals component"); + await expect(f.result).resolves.toBe(2); }); -it("still requires evals accounting when collection succeeded but the eval artifact upload failed", async () => { +it("counts missing evals accounting as zero when the eval artifact upload failed", async () => { const f = evaluate( { "agent/token_usage.jsonl": '{"aic":2}', @@ -246,10 +248,10 @@ it("still requires evals accounting when collection succeeded but the eval artif }), ] ); - await expect(f.result).rejects.toThrow("Missing accounting for executed evals component"); + await expect(f.result).resolves.toBe(2); }); -it("still requires evals accounting when the current failure upload was skipped", async () => { +it("counts missing evals accounting as zero when the current failure upload was skipped", async () => { const f = evaluate( { "agent/token_usage.jsonl": '{"aic":2}', @@ -266,17 +268,19 @@ it("still requires evals accounting when the current failure upload was skipped" }), ] ); - await expect(f.result).rejects.toThrow("Missing accounting for executed evals component"); + await expect(f.result).resolves.toBe(2); + expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"component":"evals"')); + expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"aic":0,"reason":"failed_before_accounting"')); }); -it("still requires evals accounting when the job failed before any step ran", async () => { +it("counts missing evals accounting as zero when the job failed before any step ran", async () => { const f = evaluate( { "agent/token_usage.jsonl": '{"aic":2}', }, [job("agent"), job("evals", { conclusion: "failure", steps: [] })] ); - await expect(f.result).rejects.toThrow("Missing accounting for executed evals component"); + await expect(f.result).resolves.toBe(2); }); it("still requires accounting when a failed agent job has no authoritative source", async () => { diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go index 1672ac0fd36..c6cfd9531ae 100644 --- a/pkg/workflow/evals_steps.go +++ b/pkg/workflow/evals_steps.go @@ -430,12 +430,6 @@ await main();` // buildUploadEvalsArtifactStep creates the step that uploads evals.jsonl as the // evals artifact for downstream consumption. -// -// The step names "Collect evals token usage", "Upload evals results", and -// "Upload evals accounting after failure" are matched by name in -// actions/setup/js/daily_aic_component_coverage.cjs (provesFailedEvalsHadNoUsage) -// to prove a failed evals job had zero billable usage. Renaming these steps -// requires updating that check too. func (c *Compiler) buildUploadEvalsArtifactStep(data *WorkflowData) []string { evalsArtifactName := artifactPrefixExprForDownstreamJob(data) + constants.EvalsArtifactName.String() proxyLogsDir := constants.AWFProxyLogsDir.String()