From df5b5d05fe361861fa194e6b94e22e77b948c995 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 23 Jul 2026 19:54:59 +0800 Subject: [PATCH 1/4] feat(headless): add DeepSWE Pier A/B with a Codex CLI arm - port codex_agent.py to harness_compat (Pier tree selection) and add the Pier-required install_spec/network_allowlist, sharing the proxy-endpoint parsing with kimi_code_agent via provider_proxy.py - extend createPierTaskRunner with a codex arm: pinned toolchain mount, --ak constructor kwargs, proxy wiring, and resolver-credential proxy routing for the Maka arm (Codex OAuth) - freeze the DeepSWE subset-30 task list + task tree fingerprint in harness-ab-manifest and generalize the manifest benchmark field - add run-deepswe-ab.mjs: Maka vs Codex on openai-codex/gpt-5.6-sol via Pier, reusing the harness A/B engine, manifest, and report pipeline Issue #1343. --- packages/headless/harbor/codex_agent.py | 41 +- packages/headless/harbor/harness_compat.py | 1 + packages/headless/harbor/kimi_code_agent.py | 48 +-- packages/headless/harbor/provider_proxy.py | 63 +++ packages/headless/harbor/run-deepswe-ab.mjs | 374 ++++++++++++++++++ packages/headless/harbor/run-harness-ab.mjs | 2 +- .../harbor/tests/test_harness_compat.py | 63 ++- packages/headless/package.json | 1 + .../src/__tests__/pier-task-runner.test.ts | 119 ++++++ packages/headless/src/harness-ab-manifest.ts | 63 ++- packages/headless/src/index.ts | 1 + packages/headless/src/pier-task-runner.ts | 147 ++++--- 12 files changed, 820 insertions(+), 103 deletions(-) create mode 100644 packages/headless/harbor/provider_proxy.py create mode 100644 packages/headless/harbor/run-deepswe-ab.mjs diff --git a/packages/headless/harbor/codex_agent.py b/packages/headless/harbor/codex_agent.py index 41debbc8679..f38a278e0f6 100644 --- a/packages/headless/harbor/codex_agent.py +++ b/packages/headless/harbor/codex_agent.py @@ -13,11 +13,17 @@ from pathlib import Path from typing import Any -from harbor.agents.installed.codex import Codex -from harbor.environments.base import BaseEnvironment -from harbor.models.agent.context import AgentContext - +# harness_compat picks the harbor.* tree under plain Harbor 0.13.2 and the +# pier.* tree under Pier, whose parallel classes are type-incompatible with +# harbor's (Pier's TrialResult only accepts Pier's AgentInfo). +from harness_compat import ( + AgentContext, + BaseEnvironment, + Codex, + NetworkAllowlist as _NetworkAllowlist, +) from process_scope import cleanup_process_scope, scoped_command +from provider_proxy import provider_proxy_endpoint, warn_if_pier_unreachable_proxy_port from trial_pricing import estimate_cost, pricing_from_env _TOOLCHAIN_ROOT = Path("/opt/maka-codex-toolchain") @@ -36,6 +42,33 @@ class MakaCodexAgent(Codex): def get_version_command(self) -> str | None: return f"{shlex.quote(str(_TOOLCHAIN_CODEX))} --version" + def install_spec(self) -> None: + # The pinned Codex toolchain is bind-mounted read-only and only + # verified (sha256 checksums + manifest fingerprint) in install(). + # Pier's inherited spec would instead install Codex from the network + # (npm/nvm), which offline tasks (allow_internet=false) cannot reach + # and which would break the fixed-build comparison. None keeps the + # runtime verify path unchanged (Pier runs install() when no spec is + # preinstalled). + return None + + def network_allowlist(self) -> _NetworkAllowlist | None: + # Called only under Pier; plain Harbor never calls it and + # harness_compat exports NetworkAllowlist = None there. + if _NetworkAllowlist is None: + return None + # The inherited allowlist collects OPENAI_BASE_URL (masked to None by + # this adapter's _get_env) and falls back to api.openai.com — neither + # is what this adapter dials. The container runs the pinned Codex CLI + # against the maka-http provider config, which points at + # MAKA_PROVIDER_PROXY_URL; that proxy host is the only egress the + # container needs. A missing or malformed proxy URL fails here, at + # environment creation — no fallback domain, so a misconfigured trial + # never gets a spurious egress grant. + hostname, port = provider_proxy_endpoint(self._get_env, "Codex") + warn_if_pier_unreachable_proxy_port(port, "Codex") + return _NetworkAllowlist(domains=[hostname]) + async def install(self, environment: BaseEnvironment) -> None: expected_fingerprint = self._get_env("MAKA_CODEX_TOOLCHAIN_FINGERPRINT") if not expected_fingerprint: diff --git a/packages/headless/harbor/harness_compat.py b/packages/headless/harbor/harness_compat.py index 3eaaa0469d3..add628d468c 100644 --- a/packages/headless/harbor/harness_compat.py +++ b/packages/headless/harbor/harness_compat.py @@ -46,6 +46,7 @@ # Symbol -> submodule path, identical in both package trees. _SYMBOL_MODULES = { "BaseInstalledAgent": "agents.installed.base", + "Codex": "agents.installed.codex", "CliFlag": "agents.installed.base", "with_prompt_template": "agents.installed.base", "BaseEnvironment": "environments.base", diff --git a/packages/headless/harbor/kimi_code_agent.py b/packages/headless/harbor/kimi_code_agent.py index 9e83f59006c..8b0f2f61706 100644 --- a/packages/headless/harbor/kimi_code_agent.py +++ b/packages/headless/harbor/kimi_code_agent.py @@ -8,11 +8,9 @@ import re import secrets import shlex -import sys import time from pathlib import Path from typing import Any -from urllib.parse import urlparse # harness_compat picks the harbor.* tree under plain Harbor 0.13.2 and the # pier.* tree under Pier, whose parallel classes are type-incompatible with @@ -25,6 +23,7 @@ with_prompt_template, ) from process_scope import cleanup_process_scope, scoped_command +from provider_proxy import provider_proxy_endpoint, warn_if_pier_unreachable_proxy_port _TOOLCHAIN_ROOT = Path("/opt/maka-kimi-code-toolchain") _TOOLCHAIN_NODE = _TOOLCHAIN_ROOT / "bin" / "node" @@ -69,51 +68,10 @@ def network_allowlist(self) -> _NetworkAllowlist | None: # endpoint. A missing or malformed proxy URL fails here, at environment # creation — no fallback domain, so a misconfigured trial never gets a # spurious egress grant. - hostname, port = self._provider_proxy_endpoint() - if port not in (None, 80, 443): - # Pier's egress proxy for allow_internet=false tasks is Squid with - # `acl Safe_ports port 80 443` + `http_access deny !Safe_ports` - # (pier/environments/agent_setup.py), so any other destination port - # is denied even when the domain is allowlisted. Warn instead of - # raising: the adapter cannot see the task's allow_internet, and on - # internet-enabled tasks the allowlist is ignored and this port is - # legal. - print( - f"WARNING: Kimi Code provider proxy port {port} is unreachable " - "under Pier non-internet tasks: the Squid egress proxy only " - "allows destination ports 80 and 443. Bind the provider proxy " - "to 80/443 or use an internet-enabled task.", - file=sys.stderr, - ) + hostname, port = provider_proxy_endpoint(self._get_env, "Kimi Code") + warn_if_pier_unreachable_proxy_port(port, "Kimi Code") return _NetworkAllowlist(domains=[hostname]) - def _provider_proxy_endpoint(self) -> tuple[str, int | None]: - """Hostname and port of MAKA_PROVIDER_PROXY_URL for the Pier allowlist. - - Pier-only validation, called from network_allowlist() after its plain- - Harbor early return: Pier's NetworkAllowlist domain validator rejects - ':' entries, so an IPv6 literal endpoint can never be allowlisted. - Plain Harbor forwards the URL opaquely in _runtime_env() and must not - have its input domain narrowed by this Pier constraint. - """ - proxy_url = self._get_env("MAKA_PROVIDER_PROXY_URL") - if not proxy_url: - raise ValueError("Kimi Code requires the host provider proxy") - try: - parsed = urlparse(proxy_url if "://" in proxy_url else f"https://{proxy_url}") - hostname = parsed.hostname - port = parsed.port - except ValueError as error: - raise ValueError("Kimi Code requires the host provider proxy") from error - if not hostname: - raise ValueError("Kimi Code requires the host provider proxy") - if ":" in hostname: - raise ValueError( - "Kimi Code provider proxy must use a DNS hostname or IPv4 " - "address; IPv6 literal endpoints are not supported" - ) - return hostname, port - def get_version_command(self) -> str | None: return ( f"{shlex.quote(str(_TOOLCHAIN_NODE))} " diff --git a/packages/headless/harbor/provider_proxy.py b/packages/headless/harbor/provider_proxy.py new file mode 100644 index 00000000000..c994fca358c --- /dev/null +++ b/packages/headless/harbor/provider_proxy.py @@ -0,0 +1,63 @@ +"""Host provider proxy endpoint parsing shared by the proxied CLI adapters. + +The Kimi Code and Codex adapters run a pinned CLI inside the task container +that dials the host provider auth proxy (MAKA_PROVIDER_PROXY_URL). Under Pier +that proxy host is the single egress domain each adapter must allowlist, so +the endpoint-shape rules live here once. +""" + +from __future__ import annotations + +import sys +from typing import Callable +from urllib.parse import urlparse + + +def provider_proxy_endpoint( + get_env: Callable[[str], str | None], agent_label: str +) -> tuple[str, int | None]: + """Hostname and port of MAKA_PROVIDER_PROXY_URL for the Pier allowlist. + + Pier-only validation, called from network_allowlist() after its plain- + Harbor early return: Pier's NetworkAllowlist domain validator rejects + ':' entries, so an IPv6 literal endpoint can never be allowlisted. + Plain Harbor forwards the URL opaquely to the CLI and must not have its + input domain narrowed by this Pier constraint. + """ + proxy_url = get_env("MAKA_PROVIDER_PROXY_URL") + if not proxy_url: + raise ValueError(f"{agent_label} requires the host provider proxy") + try: + parsed = urlparse(proxy_url if "://" in proxy_url else f"https://{proxy_url}") + hostname = parsed.hostname + port = parsed.port + except ValueError as error: + raise ValueError(f"{agent_label} requires the host provider proxy") from error + if not hostname: + raise ValueError(f"{agent_label} requires the host provider proxy") + if ":" in hostname: + raise ValueError( + f"{agent_label} provider proxy must use a DNS hostname or IPv4 " + "address; IPv6 literal endpoints are not supported" + ) + return hostname, port + + +def warn_if_pier_unreachable_proxy_port(port: int | None, agent_label: str) -> None: + """Warn when the proxy port cannot pass Pier's offline-task egress. + + Pier's egress proxy for allow_internet=false tasks is Squid with + `acl Safe_ports port 80 443` + `http_access deny !Safe_ports` + (pier/environments/agent_setup.py), so any other destination port is + denied even when the domain is allowlisted. Warn instead of raising: the + adapter cannot see the task's allow_internet, and on internet-enabled + tasks the allowlist is ignored and this port is legal. + """ + if port not in (None, 80, 443): + print( + f"WARNING: {agent_label} provider proxy port {port} is unreachable " + "under Pier non-internet tasks: the Squid egress proxy only " + "allows destination ports 80 and 443. Bind the provider proxy " + "to 80/443 or use an internet-enabled task.", + file=sys.stderr, + ) diff --git a/packages/headless/harbor/run-deepswe-ab.mjs b/packages/headless/harbor/run-deepswe-ab.mjs new file mode 100644 index 00000000000..8a057537940 --- /dev/null +++ b/packages/headless/harbor/run-deepswe-ab.mjs @@ -0,0 +1,374 @@ +#!/usr/bin/env node + +// DeepSWE (Datacurve, via Pier) harness A/B: Maka vs the pinned Codex CLI, +// both on openai-codex/gpt-5.6-sol through the host provider auth proxy +// (Codex OAuth; the real credential never reaches a task container). +// Issue #1343's Pier leg — the Terminal-Bench 2.1 sibling is run-harness-ab.mjs. + +import { createHash } from 'node:crypto'; +import { homedir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { ensureAbRunManifest } from '#ab-manifest'; +import { + discoverCachedHarborTasks, + fingerprintFixedPromptTaskTree, + resolveFixedPromptRunRoot, + selectTasksByIds, +} from '#fixed-prompt-task-source'; +import { createPierTaskRunner } from '#pier-task-runner'; +import { + CODEX_TOOLCHAIN_FINGERPRINT, + CODEX_TOOLCHAIN_SPEC, + prepareCodexToolchain, +} from '#codex-toolchain'; +import { createCodexOAuthHarnessCredentialBinding } from '#codex-oauth-harness'; +import { + assertDeepSweSubset30TaskTreeFingerprint, + buildHarnessAbResumeFingerprint, + buildHarnessAbRunManifest, + DEEP_SWE_REVISION, + DEEP_SWE_SUBSET_30_TASK_IDS, + HARNESS_MAKA_CONTEXT_BUDGET, +} from '#harness-ab-manifest'; +import { runHarnessAbComparisonUnlocked, withHarnessAbRunLock } from '#harness-ab-run'; +import { DEFAULT_HEADLESS_SYSTEM_PROMPT } from '@maka/headless'; +import { thinkingVariantsForModel } from '@maka/core/model-thinking'; +import { + assertHarnessAbReportCompleted, + buildHarnessAbReport, + renderHarnessAbReportCsv, + renderHarnessAbReportMarkdown, +} from '#harness-ab-report'; +import { envPath as parseEnvPath } from '#headless-run-env'; +import { buildSubjectFingerprint, buildToolchainFingerprint } from '#experiment-fingerprint'; +import { runExperiment } from '#experiment-engine'; +import { + defaultMakaWorkspaceRoot, + harnessMakaContextBudgetEnv, + resolveHarnessCompetitorToolchainPath, +} from './run-harness-ab.mjs'; + +export const DEFAULT_DEEPSWE_AB_RUN_ID = 'gpt-5.6-sol-maka-vs-codex-oauth-deepswe-subset30-v1'; +const PROVIDER = 'openai-codex'; +const MODEL = 'gpt-5.6-sol'; +const REASONING_EFFORT = 'xhigh'; +const BASE_URL = 'https://chatgpt.com/backend-api/codex'; +const ORDER_SEED = 'deep-swe-1.1:gpt-5.6-sol:harness-comparison:v1'; +const BILLING_MODE = 'account-plan'; +const PRICING = { + currency: 'USD', + unit: 'per_1m_tokens', + input: 0, + cachedInput: 0, + output: 0, + source: 'openai-codex-chatgpt-account-plan', +}; +const MAX_PAIR_CONCURRENCY = 2; +const DEFAULT_PAIR_CONCURRENCY = 1; +const DEFAULT_ARM_EXECUTION = 'sequential'; +// Pier's own trial lifecycle watchdog (build/setup/agent/verifier + retries) +// is derived per task inside the runner; this grace only feeds the manifest's +// benchmark record, mirroring the Terminal-Bench sibling. +const PIER_SETUP_TEARDOWN_GRACE_SEC = 15 * 60; + +const CODEX_PROFILE = Object.freeze({ + id: 'codex', + version: CODEX_TOOLCHAIN_SPEC.codex.version, + toolchainFingerprint: CODEX_TOOLCHAIN_FINGERPRINT, + config: Object.freeze({ + adapter: 'codex_agent:MakaCodexAgent', + transport: 'responses-http', + permissions: 'container-full-access', + attemptPolicy: 'single', + billingMode: BILLING_MODE, + }), +}); + +const envPath = (name, fallback) => parseEnvPath(name, process.env[name], fallback); + +export function resolveDeepSweAbTaskSelection(rawTaskId, rawLimit, rawTaskIds) { + const taskId = rawTaskId?.trim(); + const explicitTaskIds = rawTaskIds + ?.split(',') + .map((value) => value.trim()) + .filter(Boolean); + if (rawTaskIds !== undefined && explicitTaskIds?.length === 0) { + throw new Error('MAKA_DEEPSWE_AB_TASK_IDS must contain at least one task id'); + } + if (taskId && explicitTaskIds?.length) { + throw new Error('MAKA_DEEPSWE_AB_TASK_ID and MAKA_DEEPSWE_AB_TASK_IDS are mutually exclusive'); + } + const known = new Set(DEEP_SWE_SUBSET_30_TASK_IDS); + if (explicitTaskIds?.length) { + const unique = [...new Set(explicitTaskIds)]; + if (unique.length !== explicitTaskIds.length) { + throw new Error('MAKA_DEEPSWE_AB_TASK_IDS must not contain duplicate task ids'); + } + const unknown = unique.filter((id) => !known.has(id)); + if (unknown.length > 0) { + throw new Error( + `MAKA_DEEPSWE_AB_TASK_IDS contains tasks outside the DeepSWE subset-30: ${unknown.join(', ')}`, + ); + } + return { taskIds: unique, limit: unique.length }; + } + if (taskId) { + if (!known.has(taskId)) { + throw new Error('MAKA_DEEPSWE_AB_TASK_ID must name a DeepSWE subset-30 task'); + } + return { taskIds: [taskId], limit: 1 }; + } + const limit = rawLimit === undefined ? DEEP_SWE_SUBSET_30_TASK_IDS.length : Number(rawLimit); + if (!Number.isSafeInteger(limit) || limit < 1 || limit > DEEP_SWE_SUBSET_30_TASK_IDS.length) { + throw new Error( + `MAKA_DEEPSWE_AB_LIMIT must be an integer between 1 and ${DEEP_SWE_SUBSET_30_TASK_IDS.length}`, + ); + } + return { taskIds: DEEP_SWE_SUBSET_30_TASK_IDS, limit }; +} + +export function resolveDeepSweAbExecutionPolicy(rawPairConcurrency, rawArmExecution) { + const pairConcurrency = Number(rawPairConcurrency ?? DEFAULT_PAIR_CONCURRENCY); + if ( + !Number.isSafeInteger(pairConcurrency) || + pairConcurrency < 1 || + pairConcurrency > MAX_PAIR_CONCURRENCY + ) { + throw new Error( + `MAKA_DEEPSWE_AB_PAIR_CONCURRENCY must be an integer between 1 and ${MAX_PAIR_CONCURRENCY}`, + ); + } + const armExecution = rawArmExecution?.trim() || DEFAULT_ARM_EXECUTION; + if (armExecution !== 'sequential' && armExecution !== 'parallel') { + throw new Error('MAKA_DEEPSWE_AB_ARM_EXECUTION must be sequential or parallel'); + } + return { pairConcurrency, armExecution }; +} + +export async function main() { + const repoRoot = resolve(fileURLToPath(new URL('../../..', import.meta.url))); + const makaRepoPath = process.env.MAKA_DEEPSWE_AB_MAKA_REPO + ? resolve(process.env.MAKA_DEEPSWE_AB_MAKA_REPO) + : repoRoot; + const outDir = envPath('MAKA_DEEPSWE_AB_OUT_DIR'); + const tasksRoot = envPath( + 'MAKA_DEEPSWE_AB_TASKS_ROOT', + join(homedir(), '.maka/eval/task-sources/deep-swe-6db64a40/tasks'), + ); + const runId = process.env.MAKA_DEEPSWE_AB_RUN_ID || DEFAULT_DEEPSWE_AB_RUN_ID; + const selection = resolveDeepSweAbTaskSelection( + process.env.MAKA_DEEPSWE_AB_TASK_ID, + process.env.MAKA_DEEPSWE_AB_LIMIT, + process.env.MAKA_DEEPSWE_AB_TASK_IDS, + ); + const executionPolicy = resolveDeepSweAbExecutionPolicy( + process.env.MAKA_DEEPSWE_AB_PAIR_CONCURRENCY, + process.env.MAKA_DEEPSWE_AB_ARM_EXECUTION, + ); + const runRoot = resolveFixedPromptRunRoot(outDir, runId, 'MAKA_DEEPSWE_AB_RUN_ID'); + await withHarnessAbRunLock(runRoot, () => + runLocked({ repoRoot, makaRepoPath, tasksRoot, runId, selection, executionPolicy, runRoot }), + ); +} + +async function runLocked({ makaRepoPath, tasksRoot, runId, selection, executionPolicy, runRoot }) { + if (!thinkingVariantsForModel(PROVIDER, MODEL).includes(REASONING_EFFORT)) { + throw new Error(`${PROVIDER}/${MODEL} does not support reasoning effort ${REASONING_EFFORT}`); + } + // The frozen source is always the full subset-30 tree: fingerprint identity + // must not depend on which slice of it a canary run evaluates. + const allTasks = await discoverCachedHarborTasks(tasksRoot); + const subsetTasks = selectTasksByIds(allTasks, DEEP_SWE_SUBSET_30_TASK_IDS, 'DeepSWE subset-30'); + const taskSourceFingerprint = await fingerprintFixedPromptTaskTree(subsetTasks); + assertDeepSweSubset30TaskTreeFingerprint(taskSourceFingerprint); + + if (process.env.MAKA_DEEPSWE_AB_DRY_RUN === '1') { + console.log( + `dry-run: frozen ${subsetTasks.length}-task DeepSWE subset will run ${selection.limit} paired Pass@1 cells via Pier`, + ); + return; + } + + const subjectFingerprint = await buildSubjectFingerprint( + makaRepoPath, + process.env.MAKA_DEEPSWE_AB_EXPLICIT_SUBJECT_FINGERPRINT, + undefined, + 'MAKA_DEEPSWE_AB', + ); + const hostToolchainFingerprint = await buildToolchainFingerprint( + process.env.MAKA_DEEPSWE_AB_TOOLCHAIN_FINGERPRINT, + undefined, + makaRepoPath, + 'MAKA_DEEPSWE_AB', + ); + const credentialsRoot = envPath('MAKA_DEEPSWE_AB_WORKSPACE_ROOT', defaultMakaWorkspaceRoot()); + const credentials = await createCodexOAuthHarnessCredentialBinding({ + credentialsRoot, + connectionSlug: process.env.MAKA_DEEPSWE_AB_OAUTH_CONNECTION_SLUG || 'codex-subscription', + }); + + const toolchainFingerprint = `sha256:${createHash('sha256') + .update( + JSON.stringify({ + hostToolchainFingerprint, + competitor: CODEX_PROFILE.id, + competitorToolchainFingerprint: CODEX_PROFILE.toolchainFingerprint, + }), + ) + .digest('hex')}`; + const manifest = buildHarnessAbRunManifest({ + benchmark: { + dataset: 'deep-swe', + version: '1.1', + revision: DEEP_SWE_REVISION, + timeoutPolicy: 'task-native', + timeoutMultiplier: 1, + outerTimeoutGraceSec: PIER_SETUP_TEARDOWN_GRACE_SEC, + }, + taskIds: selection.taskIds, + orderSeed: ORDER_SEED, + pilotTaskCount: Math.min(2, selection.taskIds.length), + model: { + provider: PROVIDER, + id: MODEL, + reasoningEffort: REASONING_EFFORT, + credentialIdentity: credentials.credentialIdentity, + }, + pricing: PRICING, + arms: [ + { + id: 'maka', + version: subjectFingerprint, + config: { + adapter: 'maka_agent:MakaAgent', + externalSystemPrompt: 'empty', + reasoningEffort: REASONING_EFFORT, + continuation: false, + attemptPolicy: 'single', + billingMode: BILLING_MODE, + contextBudget: HARNESS_MAKA_CONTEXT_BUDGET, + }, + }, + { + id: CODEX_PROFILE.id, + version: CODEX_PROFILE.version, + config: { + ...CODEX_PROFILE.config, + externalSystemPrompt: 'empty', + profile: CODEX_PROFILE.id, + }, + }, + ], + taskBudgetSec: null, + harborTimeoutMs: null, + subjectFingerprint, + taskSourceFingerprint, + toolchainFingerprint, + pairConcurrency: executionPolicy.pairConcurrency, + armExecution: executionPolicy.armExecution, + }); + const manifestPath = join(runRoot, 'harness-ab-manifest.json'); + await ensureAbRunManifest(manifestPath, manifest); + + const tasksById = new Map(subsetTasks.map((task) => [task.id, task])); + const evaluationTasks = manifest.evaluationTaskIds + .slice(0, selection.limit) + .map((taskId) => tasksById.get(taskId)); + if (evaluationTasks.some((task) => !task)) + throw new Error('manifest contains a task absent from the frozen task source'); + + const codexToolchainPath = process.env.MAKA_DEEPSWE_AB_CODEX_TOOLCHAIN + ? resolve(process.env.MAKA_DEEPSWE_AB_CODEX_TOOLCHAIN) + : resolveHarnessCompetitorToolchainPath(runRoot, CODEX_PROFILE); + await prepareCodexToolchain(codexToolchainPath); + + const systemPromptPath = join(runRoot, 'prompts', 'default-system-prompt.txt'); + const report = await runExperiment({ + runRoot, + prompts: () => [{ path: systemPromptPath, content: DEFAULT_HEADLESS_SYSTEM_PROMPT }], + run: async ({ jobsDir, resultsJsonlPath }) => { + const runnerOptions = { + makaRepoPath, + jobsDir, + model: `${PROVIDER}/${MODEL}`, + provider: PROVIDER, + reasoningEffort: REASONING_EFFORT, + baseUrl: BASE_URL, + resolveProviderCredential: credentials.resolveProviderCredential, + pricing: { + inputUsdPer1M: PRICING.input, + cacheReadUsdPer1M: PRICING.cachedInput, + outputUsdPer1M: PRICING.output, + source: PRICING.source, + }, + timeoutMultiplier: 1, + }; + const config = (id) => ({ + id: `deepswe-ab-${id}`, + backend: 'ai-sdk', + llmConnectionSlug: PROVIDER, + model: MODEL, + thinkingLevel: REASONING_EFFORT, + }); + const summary = await runHarnessAbComparisonUnlocked({ + runId, + runRoot, + resultsJsonlPath, + systemPromptPath, + resumeFingerprint: buildHarnessAbResumeFingerprint(manifest), + evaluationTasks, + arms: [ + { + id: 'maka', + config: config('maka'), + expectedPricingProfile: PRICING.source, + billingMode: BILLING_MODE, + harborRunner: createPierTaskRunner({ + ...runnerOptions, + agent: 'maka', + agentEnv: harnessMakaContextBudgetEnv(), + }), + }, + { + id: CODEX_PROFILE.id, + config: config(CODEX_PROFILE.id), + expectedPricingProfile: PRICING.source, + billingMode: BILLING_MODE, + harborRunner: createPierTaskRunner({ + ...runnerOptions, + agent: 'codex', + agentVersion: CODEX_PROFILE.version, + codexToolchainPath, + }), + }, + ], + pairConcurrency: manifest.maxConcurrency, + armExecution: manifest.metadata.execution.armExecution, + }); + return buildHarnessAbReport(summary, { annotations: [], warnings: [] }, BILLING_MODE); + }, + artifacts: (report) => [ + { + path: join(runRoot, 'harness-ab-report.json'), + content: `${JSON.stringify(report, null, 2)}\n`, + }, + { path: join(runRoot, 'harness-ab-report.csv'), content: renderHarnessAbReportCsv(report) }, + { + path: join(runRoot, 'harness-ab-report.md'), + content: renderHarnessAbReportMarkdown(report), + }, + ], + }); + assertHarnessAbReportCompleted(report); + console.log( + `${report.runStatus}: ${report.coverage.attemptedCells}/${report.coverage.scheduledCells} cells attempted; ${report.effectiveness.pairedEvaluated} paired Pass@1 outcomes -> ${runRoot}`, + ); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/packages/headless/harbor/run-harness-ab.mjs b/packages/headless/harbor/run-harness-ab.mjs index 064699d115f..c36c0c90016 100644 --- a/packages/headless/harbor/run-harness-ab.mjs +++ b/packages/headless/harbor/run-harness-ab.mjs @@ -280,7 +280,7 @@ export function resolveHarnessCompetitorToolchain(runRoot, competitorProfile, en const envPath = (name, fallback) => parseEnvPath(name, process.env[name], fallback); const envPathFrom = (env, name, fallback) => parseEnvPath(name, env[name], fallback); -function defaultMakaWorkspaceRoot() { +export function defaultMakaWorkspaceRoot() { if (process.platform === 'darwin') { return join(homedir(), 'Library', 'Application Support', 'Maka', 'workspaces', 'default'); } diff --git a/packages/headless/harbor/tests/test_harness_compat.py b/packages/headless/harbor/tests/test_harness_compat.py index b4889e2e2ce..7696accc42d 100644 --- a/packages/headless/harbor/tests/test_harness_compat.py +++ b/packages/headless/harbor/tests/test_harness_compat.py @@ -42,7 +42,7 @@ if str(_HARBOR_DIR) not in sys.path: sys.path.insert(0, str(_HARBOR_DIR)) -_ADAPTER_MODULES = ("harness_compat", "maka_agent", "kimi_code_agent") +_ADAPTER_MODULES = ("harness_compat", "maka_agent", "kimi_code_agent", "codex_agent") # Sentinel returned by tests that cannot run under this interpreter; the # __main__ runner counts these as skips, not passes. pytest treats the return @@ -108,6 +108,7 @@ def _fresh_adapters(): return ( importlib.import_module("maka_agent"), importlib.import_module("kimi_code_agent"), + importlib.import_module("codex_agent"), ) @@ -157,14 +158,18 @@ def _restore_modules(saved_pier) -> None: def test_pier_tree_selected_and_agent_info_is_pier_type(): if not _pier_gate(): return SKIPPED - maka_mod, kimi_mod = _fresh_adapters() - for cls in (maka_mod.MakaAgent, kimi_mod.MakaKimiCodeAgent): + maka_mod, kimi_mod, codex_mod = _fresh_adapters() + for cls in (maka_mod.MakaAgent, kimi_mod.MakaKimiCodeAgent, codex_mod.MakaCodexAgent): assert issubclass(cls, PierBaseInstalledAgent), cls.__mro__ agent = cls(logs_dir=Path("/tmp"), model_name="k3") # The DeepSWE pilot failed pydantic validation because agent_info was a # harbor-tree AgentInfo; Pier's TrialResult only accepts its own type. info = agent.to_agent_info() assert isinstance(info, PierAgentInfo), type(info) + # Every adapter pins its own toolchain (mounted or host-side); Pier's + # inherited install specs (e.g. Codex's npm/nvm network install) must + # never run, or an offline task would fail its build and the pinned + # build under comparison would drift. assert agent.install_spec() is None @@ -172,7 +177,7 @@ def test_pier_tree_selected_and_agent_info_is_pier_type(): def test_maka_agent_network_policy_under_pier(): if not _pier_gate(): return SKIPPED - maka_mod, _ = _fresh_adapters() + maka_mod, _, _ = _fresh_adapters() # Host-side LLM mode: model calls happen on the host, container is offline. host_side = maka_mod.MakaAgent( @@ -213,7 +218,7 @@ def test_maka_agent_network_policy_under_pier(): def test_kimi_agent_network_shape_under_pier(): if not _pier_gate(): return SKIPPED - _, kimi_mod = _fresh_adapters() + _, kimi_mod, _ = _fresh_adapters() # Portless HTTPS endpoint (default 443): allowlisted without any warning. configured = kimi_mod.MakaKimiCodeAgent( @@ -261,13 +266,56 @@ def test_kimi_agent_network_shape_under_pier(): _raises(ipv6.network_allowlist, ValueError, "IPv6") +@_isolated_maka_env +def test_codex_agent_network_shape_under_pier(): + if not _pier_gate(): + return SKIPPED + _, _, codex_mod = _fresh_adapters() + + # The container runs the pinned Codex CLI against the maka-http provider + # config pointed at the host proxy; that proxy host is the single egress + # grant. Pier's inherited allowlist would fall back to api.openai.com, + # which this adapter never dials. + configured = codex_mod.MakaCodexAgent( + logs_dir=Path("/tmp"), + extra_env={"MAKA_PROVIDER_PROXY_URL": "http://host.docker.internal:443"}, + ) + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + allow = configured.network_allowlist() + assert isinstance(allow, NetworkAllowlist) + assert allow.domains == ["host.docker.internal"] + assert stderr.getvalue() == "" + + # Squid-unreachable port: warn on stderr, still allowlist the host (same + # shared contract as the Kimi adapter). + proxied = codex_mod.MakaCodexAgent( + logs_dir=Path("/tmp"), + extra_env={"MAKA_PROVIDER_PROXY_URL": "https://proxy.internal:8443/v1"}, + ) + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + proxied_allow = proxied.network_allowlist() + assert proxied_allow.domains == ["proxy.internal"] + assert "ports 80 and 443" in stderr.getvalue(), stderr.getvalue() + + # No fallback domain: a missing proxy URL fails at allowlist time instead + # of granting api.openai.com egress the trial would never use. + unset = codex_mod.MakaCodexAgent(logs_dir=Path("/tmp")) + _raises( + unset.network_allowlist, + ValueError, + "Codex requires the host provider proxy", + ) + + @_isolated_maka_env def test_kimi_runtime_env_forwards_ipv6_proxy_url(): # Runs under BOTH trees (no pier gate): _runtime_env keeps main's plain- # Harbor contract and forwards the proxy URL opaquely, IPv6 included; the # IPv6 rejection is a Pier NetworkAllowlist constraint and lives only in # the network_allowlist() path. - _, kimi_mod = _fresh_adapters() + _, kimi_mod, _ = _fresh_adapters() agent = kimi_mod.MakaKimiCodeAgent( logs_dir=Path("/tmp"), extra_env={ @@ -296,7 +344,8 @@ def test_harbor_tree_used_without_pier(): assert compat.NetworkAllowlist is None maka = importlib.import_module("maka_agent") kimi = importlib.import_module("kimi_code_agent") - for cls in (maka.MakaAgent, kimi.MakaKimiCodeAgent): + codex = importlib.import_module("codex_agent") + for cls in (maka.MakaAgent, kimi.MakaKimiCodeAgent, codex.MakaCodexAgent): # The plain-Harbor path must keep subclassing the harbor tree so # Terminal-Bench's Harbor 0.13.2 runner keeps validating its own # AgentInfo type. diff --git a/packages/headless/package.json b/packages/headless/package.json index 93d7150aed7..a5b5390f343 100644 --- a/packages/headless/package.json +++ b/packages/headless/package.json @@ -38,6 +38,7 @@ "#ab-manifest": "./dist/ab-manifest.js", "#prompt-ab-run": "./dist/prompt-ab-run.js", "#harbor-task-runner": "./dist/harbor-task-runner.js", + "#pier-task-runner": "./dist/pier-task-runner.js", "#harbor-smoke-config": "./dist/harbor-smoke-config.js", "#provider-env": "./dist/provider-env.js", "#harbor-cell": "./dist/harbor-cell.js", diff --git a/packages/headless/src/__tests__/pier-task-runner.test.ts b/packages/headless/src/__tests__/pier-task-runner.test.ts index 89da338479c..86e264ddd59 100644 --- a/packages/headless/src/__tests__/pier-task-runner.test.ts +++ b/packages/headless/src/__tests__/pier-task-runner.test.ts @@ -13,6 +13,7 @@ import { type TaskRunInput, type TaskRunOutput, } from '../fixed-prompt-controller.js'; +import { CODEX_TOOLCHAIN_FINGERPRINT, CODEX_TOOLCHAIN_SPEC } from '../codex-toolchain.js'; import { findTrialDir } from '../harbor-task-runner.js'; import { buildPierRunArgs, @@ -1121,6 +1122,124 @@ test('createPierTaskRunner lets Kimi attempts on distinct fixed ports run concur assert.equal(await runConcurrentKimiAttempts([portA!, portB!]), 2); }); +test('buildPierRunArgs targets the Codex adapter and forwards constructor kwargs via --ak', () => { + const args = buildPierRunArgs({ + agent: 'codex', + model: 'gpt-5.6-sol', + taskPath: '/tasks/dasel', + jobsDir: '/jobs', + jobName: 'trial', + environment: 'docker', + timeoutMultiplier: 1, + mounts: [], + agentEnv: {}, + agentKwargs: { version: '0.144.6', reasoning_effort: 'xhigh' }, + }); + assert.match(args.join(' '), /--agent-import-path codex_agent:MakaCodexAgent/); + assert.ok(args.includes('--ak')); + assert.ok(args.includes('version=0.144.6')); + assert.ok(args.includes('reasoning_effort=xhigh')); +}); + +test('createPierTaskRunner rejects a Codex arm whose version does not match the pinned toolchain', () => { + assert.throws( + () => + createPierTaskRunner({ + makaRepoPath: '/repo', + jobsDir: '/jobs', + model: 'gpt-5.6-sol', + agent: 'codex', + agentVersion: '0.0.1', + }), + /Codex adapter version must match toolchain version/, + ); +}); + +test('createPierTaskRunner requires the Codex toolchain mount for the Codex arm', async () => { + await withDirs(async ({ jobsDir, repo }) => { + const runner = createPierTaskRunner( + baseOptions({ + jobsDir, + makaRepoPath: repo, + agent: 'codex', + agentVersion: CODEX_TOOLCHAIN_SPEC.codex.version, + provider: 'openai-codex', + resolveProviderCredential: () => Promise.resolve({ value: 'upstream-key' }), + runPier: fakePier({ reward: 0 }), + }), + ); + await assert.rejects(runner(runInput()), /codexToolchainPath is required/); + }); +}); + +test('createPierTaskRunner wires the Codex arm through the host proxy with the pinned toolchain', async () => { + await withDirs(async ({ jobsDir, repo }) => { + const captured: FakeOptions['captured'] = {}; + const runner = createPierTaskRunner( + baseOptions({ + jobsDir, + makaRepoPath: repo, + agent: 'codex', + agentVersion: CODEX_TOOLCHAIN_SPEC.codex.version, + backend: 'ai-sdk', + provider: 'openai-codex', + model: 'gpt-5.6-sol', + reasoningEffort: 'xhigh', + baseUrl: 'https://chatgpt.com/backend-api/codex', + codexToolchainPath: repo, + resolveProviderCredential: () => Promise.resolve({ value: 'upstream-key' }), + providerProxyPort: 0, + runPier: fakePier({ reward: 0, captured }), + }), + ); + const output = await runner(runInput()); + assert.equal(output.harbor.reward, 0); + // The proxy URL and a minted (non-real) token reach the container via + // env-file, never argv. + assert.match( + captured.envFile?.MAKA_PROVIDER_PROXY_URL ?? '', + /^http:\/\/host\.docker\.internal:\d+$/, + ); + assert.ok((captured.envFile?.MAKA_PROVIDER_PROXY_TOKEN ?? '').length >= 32); + assert.notEqual(captured.envFile?.MAKA_PROVIDER_PROXY_TOKEN, 'upstream-key'); + const args = captured.request?.args ?? []; + // Constructor kwargs ride --ak; the pinned-toolchain fingerprint rides --ae. + assert.ok(args.includes(`version=${CODEX_TOOLCHAIN_SPEC.codex.version}`)); + assert.ok(args.includes('reasoning_effort=xhigh')); + assert.ok(args.includes(`MAKA_CODEX_TOOLCHAIN_FINGERPRINT=${CODEX_TOOLCHAIN_FINGERPRINT}`)); + const mountsFlag = args.indexOf('--mounts-json'); + const mounts = JSON.parse(args[mountsFlag + 1]!) as Array<{ target: string }>; + assert.ok(mounts.some((mount) => mount.target === '/opt/maka-codex-toolchain')); + }); +}); + +test('createPierTaskRunner routes a resolver-backed Maka arm through the host proxy', async () => { + // A resolver credential (e.g. Codex OAuth) is only usable through the proxy, + // so the Maka arm must take the proxy path even without useProviderProxy. + await withDirs(async ({ jobsDir, repo }) => { + const captured: FakeOptions['captured'] = {}; + const runner = createPierTaskRunner( + baseOptions({ + jobsDir, + makaRepoPath: repo, + agent: 'maka', + backend: 'ai-sdk', + provider: 'openai-codex', + model: 'gpt-5.6-sol', + baseUrl: 'https://chatgpt.com/backend-api/codex', + resolveProviderCredential: () => Promise.resolve({ value: 'upstream-key' }), + runPier: fakePier({ reward: 0, captured }), + }), + ); + await runner(runInput()); + // The host cell dials the loopback proxy with a minted token, never the + // upstream credential. + assert.match(captured.envFile?.MAKA_HOST_BASE_URL ?? '', /^http:\/\/127\.0\.0\.1:\d+$/); + assert.ok((captured.envFile?.MAKA_HOST_API_KEY ?? '').length >= 32); + assert.notEqual(captured.envFile?.MAKA_HOST_API_KEY, 'upstream-key'); + }); +}); + test('createPierTaskRunner keeps the real key host-side via a file path for the Maka arm', async () => { await withDirs(async ({ jobsDir, repo }) => { const captured: FakeOptions['captured'] = {}; diff --git a/packages/headless/src/harness-ab-manifest.ts b/packages/headless/src/harness-ab-manifest.ts index 9766115a44b..86c66e599a9 100644 --- a/packages/headless/src/harness-ab-manifest.ts +++ b/packages/headless/src/harness-ab-manifest.ts @@ -120,6 +120,65 @@ export const TERMINAL_BENCH_2_1_TASK_IDS = [ 'write-compressor', ] as const; +// Authoritative snapshot: https://github.com/datacurve-ai/deep-swe (113 scored +// leaderboard tasks; the repo tree at this commit carries 117 task dirs). +export const DEEP_SWE_REVISION = '6db64a40f3318d8659238ff34a8cc4b491c49205'; + +/** 30-task discriminative subset for DeepSWE harness A/Bs (issue #1343). + * Selected from the public v1.1 leaderboard artifacts + * (https://deepswe.datacurve.ai/artifacts/v1.1/{trials,tasks}.json) against the + * `mini_swe_agent_kimi_k3_max` reference config (4 seeds/task): 11 tasks at + * seed pass rate 0.25 and 17 at 0.50 maximize per-task discrimination, plus + * one all-fail and one all-pass anchor to catch harness-level regressions. */ +export const DEEP_SWE_SUBSET_30_TASK_IDS = [ + // K3 mini-swe-agent seed pass rate 0.25: + 'clack-async-autocomplete-options', + 'httpx-streaming-json-iteration', + 'kea-atomic-signal-selectors', + 'kombu-single-active-consumer-priority', + 'koota-query-predicates', + 'langchain-request-coalescing', + 'onedump-dump-encryption-pipeline', + 'pwntools-tube-multiplexing', + 'quill-shared-toolbar-focus', + 'sqlfmt-create-table-ddl-formatting', + 'termenv-preserve-ansi-resets', + // K3 mini-swe-agent seed pass rate 0.50: + 'arktype-json-schema-refs-dependencies', + 'csstree-shorthand-expansion-compression', + 'dateutil-rfc5545-timezone-interop', + 'dynamodb-toolbox-lazy-recursive-schemas', + 'go-critic-doc-link-checker', + 'ink-grid-box-layout', + 'koota-pair-relation-tracking', + 'mashumaro-flattened-dataclass-fields', + 'mobly-grouped-test-barriers', + 'ofetch-per-origin-circuit-breaker', + 'optique-conditional-option-dependencies', + 'oxvg-structural-selector-preservation', + 'participle-grammar-conflict-analysis', + 'psd-tools-blend-range-api', + 'scc-bounded-memory-spilling', + 'testem-bail-on-test-failure', + 'yaegi-go-embed-directives', + // Anchors: K3 all-fail / all-pass. + 'claude-code-by-agents-recursive-delegation', + 'dasel-html-document-format', +] as const; + +/** fingerprintFixedPromptTaskTree over the 30 subset task dirs at + * DEEP_SWE_REVISION — freezes the exact task bytes under comparison. */ +export const DEEP_SWE_SUBSET_30_TASK_TREE_FINGERPRINT = + 'sha256:508aedcbe69ebdf6e9253c2eef9ba2575e9db9a25f899eaf0760d29459e843ba'; + +export function assertDeepSweSubset30TaskTreeFingerprint(actual: string): void { + if (actual !== DEEP_SWE_SUBSET_30_TASK_TREE_FINGERPRINT) { + throw new Error( + `DeepSWE subset-30 task tree fingerprint mismatch; expected ${DEEP_SWE_SUBSET_30_TASK_TREE_FINGERPRINT}, found ${actual}`, + ); + } +} + export function assertTerminalBench21TaskSet(taskIds: readonly string[]): void { const actual = new Set(taskIds); const expected = new Set(TERMINAL_BENCH_2_1_TASK_IDS); @@ -154,8 +213,8 @@ export interface HarnessAbArmInput { export interface HarnessAbRunManifestInput { benchmark: { - dataset: 'terminal-bench'; - version: '2.1'; + dataset: 'terminal-bench' | 'deep-swe'; + version: '2.1' | '1.1'; revision: string; timeoutPolicy: 'task-native'; timeoutMultiplier: 1; diff --git a/packages/headless/src/index.ts b/packages/headless/src/index.ts index a263b5e8c0a..f30fc7cb832 100644 --- a/packages/headless/src/index.ts +++ b/packages/headless/src/index.ts @@ -250,6 +250,7 @@ export { PierInfraError, PIER_PROVIDER_PROXY_DEFAULT_PORT, type BuildPierRunArgsInput, + type PierAgent, type PierProcessRunner, type PierRunRequest, type PierRunResult, diff --git a/packages/headless/src/pier-task-runner.ts b/packages/headless/src/pier-task-runner.ts index e7793195616..12e4a5a6590 100644 --- a/packages/headless/src/pier-task-runner.ts +++ b/packages/headless/src/pier-task-runner.ts @@ -33,6 +33,11 @@ import { type HarborTaskPricing, } from './harbor-task-runner.js'; import { lenientPositiveIntEnv } from './headless-run-env.js'; +import { + CODEX_TOOLCHAIN_CONTAINER_PATH, + CODEX_TOOLCHAIN_FINGERPRINT, + CODEX_TOOLCHAIN_SPEC, +} from './codex-toolchain.js'; import { KIMI_CODE_TOOLCHAIN_CONTAINER_PATH, KIMI_CODE_TOOLCHAIN_FINGERPRINT, @@ -52,23 +57,23 @@ const TRIAL_REWARD_JSON = 'verifier/reward.json'; const TRIAL_RESULT = 'result.json'; const PROVIDER_REQUEST_TELEMETRY = 'provider-request-telemetry.json'; -/** The default port the Kimi arm binds the host provider proxy to. Pier's Squid - * egress for offline (`allow_internet=false`) tasks only permits destination - * ports 80/443 (`acl Safe_ports port 80 443`), so a container reaching the host - * proxy through Squid must present one of those. 443 keeps the model endpoint on - * the conventional TLS port. */ +/** The default port a container-CLI arm (Kimi/Codex) binds the host provider + * proxy to. Pier's Squid egress for offline (`allow_internet=false`) tasks only + * permits destination ports 80/443 (`acl Safe_ports port 80 443`), so a + * container reaching the host proxy through Squid must present one of those. + * 443 keeps the model endpoint on the conventional TLS port. */ export const PIER_PROVIDER_PROXY_DEFAULT_PORT = 443; -/** The Kimi arm binds ONE fixed proxy port per attempt, and Pier's Squid egress - * leaves only two usable destination ports (80/443), so concurrent attempts on - * the same port must hold it one at a time — a second concurrent bind is a - * guaranteed EADDRINUSE. The lock's owner is the shared host PORT, not a runner - * instance: two runners in one process (e.g. an A/B with two Kimi arms) compete - * for the same bind. Hence a module-level per-port queue; cross-PROCESS - * collisions remain the operator's scheduling concern. Serializing the - * proxy-holding section (instead of sharing one proxy) keeps usage/telemetry - * attribution per attempt. A pool over both Squid-legal ports is deferred until - * concurrency actually needs it. */ +/** A container-CLI arm binds ONE fixed proxy port per attempt, and Pier's Squid + * egress leaves only two usable destination ports (80/443), so concurrent + * attempts on the same port must hold it one at a time — a second concurrent + * bind is a guaranteed EADDRINUSE. The lock's owner is the shared host PORT, + * not a runner instance: two runners in one process (e.g. an A/B with two + * container-CLI arms) compete for the same bind. Hence a module-level per-port + * queue; cross-PROCESS collisions remain the operator's scheduling concern. + * Serializing the proxy-holding section (instead of sharing one proxy) keeps + * usage/telemetry attribution per attempt. A pool over both Squid-legal ports + * is deferred until concurrency actually needs it. */ const proxyPortQueues = new Map>(); function withProxyPortLock(port: number, fn: () => Promise): Promise { @@ -110,12 +115,18 @@ export interface PierTaskRunnerOptions { /** Host path to the maka repo, bind-mounted read-only at /opt/maka-agent. */ makaRepoPath: string; /** Pier adapter under test (default: Maka, host-side LLM + offline container). */ - agent?: 'maka' | 'kimi-code'; + agent?: PierAgent; /** In-container/host cell backend. Only the Maka arm reads it. `fake` runs the * inert cell for zero-cost structural checks; `ai-sdk` is the real run. */ backend?: 'ai-sdk' | 'fake'; /** Prepared Kimi Code toolchain bind-mounted read-only into task containers. */ kimiCodeToolchainPath?: string; + /** Prepared Codex toolchain bind-mounted read-only into task containers. */ + codexToolchainPath?: string; + /** Competitor CLI version, forwarded as the adapter's `version` kwarg. The + * Codex arm requires it to match the pinned toolchain spec so the fixed + * build under test is exactly the one fingerprinted into the manifest. */ + agentVersion?: string; /** Base directory under which each task gets an isolated per-task job dir. */ jobsDir: string; /** MAKA_MODEL / pier `-m`, e.g. "k3" or "deepseek/deepseek-v4-flash". */ @@ -132,17 +143,18 @@ export interface PierTaskRunnerOptions { /** Resolves the upstream authority inside the host proxy for every request. */ resolveProviderCredential?: ProviderUpstreamCredentialResolver; /** Route the Maka arm's host cell through the auth proxy instead of reading the - * key file directly. The Kimi arm always uses the proxy. */ + * key file directly. The container-CLI arms (Kimi/Codex) always use the proxy. */ useProviderProxy?: boolean; - /** Explicit host proxy listen port for the Kimi arm (default 443). */ + /** Explicit host proxy listen port for a container-CLI arm (default 443). */ providerProxyPort?: number; - /** Host the Kimi container should dial to reach the host provider proxy. Only - * the Kimi arm honors it; the Maka arm's host cell always uses loopback - * (127.0.0.1) since it runs on the host itself. Unset keeps the default - * host.docker.internal, which Docker Desktop injects but native Linux Docker - * does NOT provide (pier 0.3.0's compose wires no extra_hosts/host-gateway), - * so on native Linux pass the host's docker-bridge-reachable address (e.g. - * 172.17.0.1) or the Kimi container's Squid cannot resolve the proxy. */ + /** Host a container-CLI arm's container should dial to reach the host + * provider proxy. Only those arms honor it; the Maka arm's host cell always + * uses loopback (127.0.0.1) since it runs on the host itself. Unset keeps the + * default host.docker.internal, which Docker Desktop injects but native Linux + * Docker does NOT provide (pier 0.3.0's compose wires no + * extra_hosts/host-gateway), so on native Linux pass the host's + * docker-bridge-reachable address (e.g. 172.17.0.1) or the container's Squid + * cannot resolve the proxy. */ providerProxyAdvertisedHost?: string; /** Per-1M USD pricing forwarded as MAKA_TRIAL_* so the cell emits real costUsd. */ pricing?: PierTaskPricing; @@ -193,6 +205,8 @@ export interface PierRunResult { export type PierProcessRunner = (request: PierRunRequest) => Promise; +export type PierAgent = 'maka' | 'kimi-code' | 'codex'; + interface PierProviderRuntime { /** Proxy-minted secret env delivered via `--env-file` (kept off argv). */ envFile: Record; @@ -204,6 +218,11 @@ interface PierProviderRuntime { } export function createPierTaskRunner(options: PierTaskRunnerOptions): TaskRunner { + if (options.agent === 'codex' && options.agentVersion !== CODEX_TOOLCHAIN_SPEC.codex.version) { + throw new Error( + `Codex adapter version must match toolchain version ${CODEX_TOOLCHAIN_SPEC.codex.version}`, + ); + } const runPier = options.runPier ?? defaultPierProcessRunner; const pierBin = options.pierBin ?? 'pier'; // The bare adapter import path (`maka_agent:MakaAgent`) resolves only when the @@ -282,6 +301,19 @@ export function createPierTaskRunner(options: PierTaskRunnerOptions): TaskRunner mounts, agentEnv: aeEnv, ...(usesEnvFile ? { envFile: envFilePath } : {}), + // Constructor kwargs ride `--ak`; env cannot carry them. The Codex + // adapter pins its CLI build via `version` and forwards the effort + // as the `-c model_reasoning_effort` CLI flag descriptor. + ...(agent === 'codex' + ? { + agentKwargs: { + version: options.agentVersion!, + ...(options.reasoningEffort + ? { reasoning_effort: options.reasoningEffort } + : {}), + }, + } + : {}), }); return await runPier({ pierBin, @@ -331,13 +363,13 @@ export function createPierTaskRunner(options: PierTaskRunnerOptions): TaskRunner ); } }; - // Only the Kimi arm on a FIXED port competes for the bind; the Maka arm - // (ephemeral or no proxy) and an explicit port 0 (OS-assigned, used by - // tests) cannot collide and need no serialization. - const kimiProxyPort = options.providerProxyPort ?? PIER_PROVIDER_PROXY_DEFAULT_PORT; + // Only a container-CLI arm (Kimi/Codex) on a FIXED port competes for the + // bind; the Maka arm (ephemeral or no proxy) and an explicit port 0 + // (OS-assigned, used by tests) cannot collide and need no serialization. + const containerProxyPort = options.providerProxyPort ?? PIER_PROVIDER_PROXY_DEFAULT_PORT; const result = - agent === 'kimi-code' && kimiProxyPort !== 0 - ? await withProxyPortLock(kimiProxyPort, launchAttempt) + agent !== 'maka' && containerProxyPort !== 0 + ? await withProxyPortLock(containerProxyPort, launchAttempt) : await launchAttempt(); try { @@ -472,7 +504,7 @@ export function createPierTaskRunner(options: PierTaskRunnerOptions): TaskRunner } export interface BuildPierRunArgsInput { - agent: 'maka' | 'kimi-code'; + agent: PierAgent; model: string; taskPath: string; jobsDir: string; @@ -482,12 +514,18 @@ export interface BuildPierRunArgsInput { mounts: ReadonlyArray>; agentEnv: Record; envFile?: string; + /** Adapter constructor kwargs forwarded as `--ak key=value`. */ + agentKwargs?: Record; } /** Assemble the `pier run` argv. Exported for deterministic unit tests. */ export function buildPierRunArgs(input: BuildPierRunArgsInput): string[] { const importPath = - input.agent === 'kimi-code' ? 'kimi_code_agent:MakaKimiCodeAgent' : 'maka_agent:MakaAgent'; + input.agent === 'kimi-code' + ? 'kimi_code_agent:MakaKimiCodeAgent' + : input.agent === 'codex' + ? 'codex_agent:MakaCodexAgent' + : 'maka_agent:MakaAgent'; const args = [ 'run', '--agent-import-path', @@ -515,6 +553,9 @@ export function buildPierRunArgs(input: BuildPierRunArgsInput): string[] { '--quiet', ]; if (input.envFile) args.push('--env-file', input.envFile); + for (const [key, value] of Object.entries(input.agentKwargs ?? {})) { + args.push('--ak', `${key}=${value}`); + } for (const [key, value] of Object.entries(input.agentEnv)) { args.push('--ae', `${key}=${value}`); } @@ -523,7 +564,7 @@ export function buildPierRunArgs(input: BuildPierRunArgsInput): string[] { function buildPierMounts( options: PierTaskRunnerOptions, - agent: 'maka' | 'kimi-code', + agent: PierAgent, ): Array> { const mounts: Array> = [ { type: 'bind', source: options.makaRepoPath, target: CONTAINER_MAKA_REPO, read_only: true }, @@ -539,13 +580,24 @@ function buildPierMounts( read_only: true, }); } + if (agent === 'codex') { + if (!options.codexToolchainPath) { + throw new Error('codexToolchainPath is required for the Codex adapter'); + } + mounts.push({ + type: 'bind', + source: options.codexToolchainPath, + target: CODEX_TOOLCHAIN_CONTAINER_PATH, + read_only: true, + }); + } return mounts; } function buildPierAgentEnv( input: TaskRunInput, options: PierTaskRunnerOptions, - agent: 'maka' | 'kimi-code', + agent: PierAgent, providerAgentEnv: Record, ): Record { const provider = options.provider ?? 'deepseek'; @@ -563,6 +615,9 @@ function buildPierAgentEnv( if (agent === 'kimi-code') { env.MAKA_KIMI_CODE_TOOLCHAIN_FINGERPRINT = KIMI_CODE_TOOLCHAIN_FINGERPRINT; } + if (agent === 'codex') { + env.MAKA_CODEX_TOOLCHAIN_FINGERPRINT = CODEX_TOOLCHAIN_FINGERPRINT; + } if (options.pricing) { env.MAKA_TRIAL_INPUT_USD_PER_1M = String(options.pricing.inputUsdPer1M); env.MAKA_TRIAL_OUTPUT_USD_PER_1M = String(options.pricing.outputUsdPer1M); @@ -595,11 +650,17 @@ function buildPierAgentEnv( async function pierProviderRuntime( options: PierTaskRunnerOptions, - agent: 'maka' | 'kimi-code', + agent: PierAgent, ): Promise { const provider = options.provider ?? 'deepseek'; const baseUrl = options.baseUrl ?? providerDefaultBaseUrl(provider); - const usesProxy = agent === 'kimi-code' || options.useProviderProxy === true; + // A resolver-backed credential (e.g. Codex OAuth) is only usable through the + // proxy — same contract as the Harbor runner — so it forces the proxy path + // even when the caller forgot useProviderProxy. + const usesProxy = + agent !== 'maka' || + options.useProviderProxy === true || + options.resolveProviderCredential !== undefined; if (!usesProxy) { if (agent !== 'maka') return null; @@ -644,13 +705,11 @@ async function pierProviderRuntime( if (!baseUrl) throw new Error(`Pier ${agent} provider ${provider} requires a base URL`); const proxyPort = - agent === 'kimi-code' - ? (options.providerProxyPort ?? PIER_PROVIDER_PROXY_DEFAULT_PORT) - : undefined; - // The Maka host cell runs on the host and reaches the proxy on loopback; the - // Kimi container reaches it through Docker's host gateway on a Squid-legal - // port, defaulting to host.docker.internal unless an explicit advertised host - // is supplied (the native-Linux escape hatch, e.g. 172.17.0.1). + agent !== 'maka' ? (options.providerProxyPort ?? PIER_PROVIDER_PROXY_DEFAULT_PORT) : undefined; + // The Maka host cell runs on the host and reaches the proxy on loopback; a + // container-CLI arm (Kimi/Codex) reaches it through Docker's host gateway on + // a Squid-legal port, defaulting to host.docker.internal unless an explicit + // advertised host is supplied (the native-Linux escape hatch, e.g. 172.17.0.1). const advertisedHost = agent === 'maka' ? '127.0.0.1' : options.providerProxyAdvertisedHost; const proxy = await startProviderAuthProxy({ upstreamBaseUrl: baseUrl, From 6f8646d31650d62d2cfd90251b08e85591a0dc20 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 23 Jul 2026 20:31:17 +0800 Subject: [PATCH 2/4] refactor(headless): fold the DeepSWE driver into the benchmark axis of run-harness-ab A benchmark is a bound pair of frozen task source and executor (Terminal-Bench 2.1 <-> Harbor, DeepSWE <-> Pier), so it becomes HARNESS_BENCHMARK_PROFILES + MAKA_HARNESS_AB_BENCHMARK on the existing driver, orthogonal to the competitor axis, instead of a parallel run script. Task selection, run-id/order-seed derivation, the oracle-evidence hook, and the runner factory dispatch on the profile; the Terminal-Bench default behavior (env names, run ids, order seed, resume fingerprints) is unchanged. Deletes run-deepswe-ab.mjs. --- packages/headless/harbor/run-deepswe-ab.mjs | 374 ------------------ .../harbor/run-harness-ab-detached.mjs | 10 +- packages/headless/harbor/run-harness-ab.mjs | 233 ++++++++--- .../src/__tests__/harness-ab-cli.test.ts | 77 ++++ 4 files changed, 256 insertions(+), 438 deletions(-) delete mode 100644 packages/headless/harbor/run-deepswe-ab.mjs diff --git a/packages/headless/harbor/run-deepswe-ab.mjs b/packages/headless/harbor/run-deepswe-ab.mjs deleted file mode 100644 index 8a057537940..00000000000 --- a/packages/headless/harbor/run-deepswe-ab.mjs +++ /dev/null @@ -1,374 +0,0 @@ -#!/usr/bin/env node - -// DeepSWE (Datacurve, via Pier) harness A/B: Maka vs the pinned Codex CLI, -// both on openai-codex/gpt-5.6-sol through the host provider auth proxy -// (Codex OAuth; the real credential never reaches a task container). -// Issue #1343's Pier leg — the Terminal-Bench 2.1 sibling is run-harness-ab.mjs. - -import { createHash } from 'node:crypto'; -import { homedir } from 'node:os'; -import { join, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { ensureAbRunManifest } from '#ab-manifest'; -import { - discoverCachedHarborTasks, - fingerprintFixedPromptTaskTree, - resolveFixedPromptRunRoot, - selectTasksByIds, -} from '#fixed-prompt-task-source'; -import { createPierTaskRunner } from '#pier-task-runner'; -import { - CODEX_TOOLCHAIN_FINGERPRINT, - CODEX_TOOLCHAIN_SPEC, - prepareCodexToolchain, -} from '#codex-toolchain'; -import { createCodexOAuthHarnessCredentialBinding } from '#codex-oauth-harness'; -import { - assertDeepSweSubset30TaskTreeFingerprint, - buildHarnessAbResumeFingerprint, - buildHarnessAbRunManifest, - DEEP_SWE_REVISION, - DEEP_SWE_SUBSET_30_TASK_IDS, - HARNESS_MAKA_CONTEXT_BUDGET, -} from '#harness-ab-manifest'; -import { runHarnessAbComparisonUnlocked, withHarnessAbRunLock } from '#harness-ab-run'; -import { DEFAULT_HEADLESS_SYSTEM_PROMPT } from '@maka/headless'; -import { thinkingVariantsForModel } from '@maka/core/model-thinking'; -import { - assertHarnessAbReportCompleted, - buildHarnessAbReport, - renderHarnessAbReportCsv, - renderHarnessAbReportMarkdown, -} from '#harness-ab-report'; -import { envPath as parseEnvPath } from '#headless-run-env'; -import { buildSubjectFingerprint, buildToolchainFingerprint } from '#experiment-fingerprint'; -import { runExperiment } from '#experiment-engine'; -import { - defaultMakaWorkspaceRoot, - harnessMakaContextBudgetEnv, - resolveHarnessCompetitorToolchainPath, -} from './run-harness-ab.mjs'; - -export const DEFAULT_DEEPSWE_AB_RUN_ID = 'gpt-5.6-sol-maka-vs-codex-oauth-deepswe-subset30-v1'; -const PROVIDER = 'openai-codex'; -const MODEL = 'gpt-5.6-sol'; -const REASONING_EFFORT = 'xhigh'; -const BASE_URL = 'https://chatgpt.com/backend-api/codex'; -const ORDER_SEED = 'deep-swe-1.1:gpt-5.6-sol:harness-comparison:v1'; -const BILLING_MODE = 'account-plan'; -const PRICING = { - currency: 'USD', - unit: 'per_1m_tokens', - input: 0, - cachedInput: 0, - output: 0, - source: 'openai-codex-chatgpt-account-plan', -}; -const MAX_PAIR_CONCURRENCY = 2; -const DEFAULT_PAIR_CONCURRENCY = 1; -const DEFAULT_ARM_EXECUTION = 'sequential'; -// Pier's own trial lifecycle watchdog (build/setup/agent/verifier + retries) -// is derived per task inside the runner; this grace only feeds the manifest's -// benchmark record, mirroring the Terminal-Bench sibling. -const PIER_SETUP_TEARDOWN_GRACE_SEC = 15 * 60; - -const CODEX_PROFILE = Object.freeze({ - id: 'codex', - version: CODEX_TOOLCHAIN_SPEC.codex.version, - toolchainFingerprint: CODEX_TOOLCHAIN_FINGERPRINT, - config: Object.freeze({ - adapter: 'codex_agent:MakaCodexAgent', - transport: 'responses-http', - permissions: 'container-full-access', - attemptPolicy: 'single', - billingMode: BILLING_MODE, - }), -}); - -const envPath = (name, fallback) => parseEnvPath(name, process.env[name], fallback); - -export function resolveDeepSweAbTaskSelection(rawTaskId, rawLimit, rawTaskIds) { - const taskId = rawTaskId?.trim(); - const explicitTaskIds = rawTaskIds - ?.split(',') - .map((value) => value.trim()) - .filter(Boolean); - if (rawTaskIds !== undefined && explicitTaskIds?.length === 0) { - throw new Error('MAKA_DEEPSWE_AB_TASK_IDS must contain at least one task id'); - } - if (taskId && explicitTaskIds?.length) { - throw new Error('MAKA_DEEPSWE_AB_TASK_ID and MAKA_DEEPSWE_AB_TASK_IDS are mutually exclusive'); - } - const known = new Set(DEEP_SWE_SUBSET_30_TASK_IDS); - if (explicitTaskIds?.length) { - const unique = [...new Set(explicitTaskIds)]; - if (unique.length !== explicitTaskIds.length) { - throw new Error('MAKA_DEEPSWE_AB_TASK_IDS must not contain duplicate task ids'); - } - const unknown = unique.filter((id) => !known.has(id)); - if (unknown.length > 0) { - throw new Error( - `MAKA_DEEPSWE_AB_TASK_IDS contains tasks outside the DeepSWE subset-30: ${unknown.join(', ')}`, - ); - } - return { taskIds: unique, limit: unique.length }; - } - if (taskId) { - if (!known.has(taskId)) { - throw new Error('MAKA_DEEPSWE_AB_TASK_ID must name a DeepSWE subset-30 task'); - } - return { taskIds: [taskId], limit: 1 }; - } - const limit = rawLimit === undefined ? DEEP_SWE_SUBSET_30_TASK_IDS.length : Number(rawLimit); - if (!Number.isSafeInteger(limit) || limit < 1 || limit > DEEP_SWE_SUBSET_30_TASK_IDS.length) { - throw new Error( - `MAKA_DEEPSWE_AB_LIMIT must be an integer between 1 and ${DEEP_SWE_SUBSET_30_TASK_IDS.length}`, - ); - } - return { taskIds: DEEP_SWE_SUBSET_30_TASK_IDS, limit }; -} - -export function resolveDeepSweAbExecutionPolicy(rawPairConcurrency, rawArmExecution) { - const pairConcurrency = Number(rawPairConcurrency ?? DEFAULT_PAIR_CONCURRENCY); - if ( - !Number.isSafeInteger(pairConcurrency) || - pairConcurrency < 1 || - pairConcurrency > MAX_PAIR_CONCURRENCY - ) { - throw new Error( - `MAKA_DEEPSWE_AB_PAIR_CONCURRENCY must be an integer between 1 and ${MAX_PAIR_CONCURRENCY}`, - ); - } - const armExecution = rawArmExecution?.trim() || DEFAULT_ARM_EXECUTION; - if (armExecution !== 'sequential' && armExecution !== 'parallel') { - throw new Error('MAKA_DEEPSWE_AB_ARM_EXECUTION must be sequential or parallel'); - } - return { pairConcurrency, armExecution }; -} - -export async function main() { - const repoRoot = resolve(fileURLToPath(new URL('../../..', import.meta.url))); - const makaRepoPath = process.env.MAKA_DEEPSWE_AB_MAKA_REPO - ? resolve(process.env.MAKA_DEEPSWE_AB_MAKA_REPO) - : repoRoot; - const outDir = envPath('MAKA_DEEPSWE_AB_OUT_DIR'); - const tasksRoot = envPath( - 'MAKA_DEEPSWE_AB_TASKS_ROOT', - join(homedir(), '.maka/eval/task-sources/deep-swe-6db64a40/tasks'), - ); - const runId = process.env.MAKA_DEEPSWE_AB_RUN_ID || DEFAULT_DEEPSWE_AB_RUN_ID; - const selection = resolveDeepSweAbTaskSelection( - process.env.MAKA_DEEPSWE_AB_TASK_ID, - process.env.MAKA_DEEPSWE_AB_LIMIT, - process.env.MAKA_DEEPSWE_AB_TASK_IDS, - ); - const executionPolicy = resolveDeepSweAbExecutionPolicy( - process.env.MAKA_DEEPSWE_AB_PAIR_CONCURRENCY, - process.env.MAKA_DEEPSWE_AB_ARM_EXECUTION, - ); - const runRoot = resolveFixedPromptRunRoot(outDir, runId, 'MAKA_DEEPSWE_AB_RUN_ID'); - await withHarnessAbRunLock(runRoot, () => - runLocked({ repoRoot, makaRepoPath, tasksRoot, runId, selection, executionPolicy, runRoot }), - ); -} - -async function runLocked({ makaRepoPath, tasksRoot, runId, selection, executionPolicy, runRoot }) { - if (!thinkingVariantsForModel(PROVIDER, MODEL).includes(REASONING_EFFORT)) { - throw new Error(`${PROVIDER}/${MODEL} does not support reasoning effort ${REASONING_EFFORT}`); - } - // The frozen source is always the full subset-30 tree: fingerprint identity - // must not depend on which slice of it a canary run evaluates. - const allTasks = await discoverCachedHarborTasks(tasksRoot); - const subsetTasks = selectTasksByIds(allTasks, DEEP_SWE_SUBSET_30_TASK_IDS, 'DeepSWE subset-30'); - const taskSourceFingerprint = await fingerprintFixedPromptTaskTree(subsetTasks); - assertDeepSweSubset30TaskTreeFingerprint(taskSourceFingerprint); - - if (process.env.MAKA_DEEPSWE_AB_DRY_RUN === '1') { - console.log( - `dry-run: frozen ${subsetTasks.length}-task DeepSWE subset will run ${selection.limit} paired Pass@1 cells via Pier`, - ); - return; - } - - const subjectFingerprint = await buildSubjectFingerprint( - makaRepoPath, - process.env.MAKA_DEEPSWE_AB_EXPLICIT_SUBJECT_FINGERPRINT, - undefined, - 'MAKA_DEEPSWE_AB', - ); - const hostToolchainFingerprint = await buildToolchainFingerprint( - process.env.MAKA_DEEPSWE_AB_TOOLCHAIN_FINGERPRINT, - undefined, - makaRepoPath, - 'MAKA_DEEPSWE_AB', - ); - const credentialsRoot = envPath('MAKA_DEEPSWE_AB_WORKSPACE_ROOT', defaultMakaWorkspaceRoot()); - const credentials = await createCodexOAuthHarnessCredentialBinding({ - credentialsRoot, - connectionSlug: process.env.MAKA_DEEPSWE_AB_OAUTH_CONNECTION_SLUG || 'codex-subscription', - }); - - const toolchainFingerprint = `sha256:${createHash('sha256') - .update( - JSON.stringify({ - hostToolchainFingerprint, - competitor: CODEX_PROFILE.id, - competitorToolchainFingerprint: CODEX_PROFILE.toolchainFingerprint, - }), - ) - .digest('hex')}`; - const manifest = buildHarnessAbRunManifest({ - benchmark: { - dataset: 'deep-swe', - version: '1.1', - revision: DEEP_SWE_REVISION, - timeoutPolicy: 'task-native', - timeoutMultiplier: 1, - outerTimeoutGraceSec: PIER_SETUP_TEARDOWN_GRACE_SEC, - }, - taskIds: selection.taskIds, - orderSeed: ORDER_SEED, - pilotTaskCount: Math.min(2, selection.taskIds.length), - model: { - provider: PROVIDER, - id: MODEL, - reasoningEffort: REASONING_EFFORT, - credentialIdentity: credentials.credentialIdentity, - }, - pricing: PRICING, - arms: [ - { - id: 'maka', - version: subjectFingerprint, - config: { - adapter: 'maka_agent:MakaAgent', - externalSystemPrompt: 'empty', - reasoningEffort: REASONING_EFFORT, - continuation: false, - attemptPolicy: 'single', - billingMode: BILLING_MODE, - contextBudget: HARNESS_MAKA_CONTEXT_BUDGET, - }, - }, - { - id: CODEX_PROFILE.id, - version: CODEX_PROFILE.version, - config: { - ...CODEX_PROFILE.config, - externalSystemPrompt: 'empty', - profile: CODEX_PROFILE.id, - }, - }, - ], - taskBudgetSec: null, - harborTimeoutMs: null, - subjectFingerprint, - taskSourceFingerprint, - toolchainFingerprint, - pairConcurrency: executionPolicy.pairConcurrency, - armExecution: executionPolicy.armExecution, - }); - const manifestPath = join(runRoot, 'harness-ab-manifest.json'); - await ensureAbRunManifest(manifestPath, manifest); - - const tasksById = new Map(subsetTasks.map((task) => [task.id, task])); - const evaluationTasks = manifest.evaluationTaskIds - .slice(0, selection.limit) - .map((taskId) => tasksById.get(taskId)); - if (evaluationTasks.some((task) => !task)) - throw new Error('manifest contains a task absent from the frozen task source'); - - const codexToolchainPath = process.env.MAKA_DEEPSWE_AB_CODEX_TOOLCHAIN - ? resolve(process.env.MAKA_DEEPSWE_AB_CODEX_TOOLCHAIN) - : resolveHarnessCompetitorToolchainPath(runRoot, CODEX_PROFILE); - await prepareCodexToolchain(codexToolchainPath); - - const systemPromptPath = join(runRoot, 'prompts', 'default-system-prompt.txt'); - const report = await runExperiment({ - runRoot, - prompts: () => [{ path: systemPromptPath, content: DEFAULT_HEADLESS_SYSTEM_PROMPT }], - run: async ({ jobsDir, resultsJsonlPath }) => { - const runnerOptions = { - makaRepoPath, - jobsDir, - model: `${PROVIDER}/${MODEL}`, - provider: PROVIDER, - reasoningEffort: REASONING_EFFORT, - baseUrl: BASE_URL, - resolveProviderCredential: credentials.resolveProviderCredential, - pricing: { - inputUsdPer1M: PRICING.input, - cacheReadUsdPer1M: PRICING.cachedInput, - outputUsdPer1M: PRICING.output, - source: PRICING.source, - }, - timeoutMultiplier: 1, - }; - const config = (id) => ({ - id: `deepswe-ab-${id}`, - backend: 'ai-sdk', - llmConnectionSlug: PROVIDER, - model: MODEL, - thinkingLevel: REASONING_EFFORT, - }); - const summary = await runHarnessAbComparisonUnlocked({ - runId, - runRoot, - resultsJsonlPath, - systemPromptPath, - resumeFingerprint: buildHarnessAbResumeFingerprint(manifest), - evaluationTasks, - arms: [ - { - id: 'maka', - config: config('maka'), - expectedPricingProfile: PRICING.source, - billingMode: BILLING_MODE, - harborRunner: createPierTaskRunner({ - ...runnerOptions, - agent: 'maka', - agentEnv: harnessMakaContextBudgetEnv(), - }), - }, - { - id: CODEX_PROFILE.id, - config: config(CODEX_PROFILE.id), - expectedPricingProfile: PRICING.source, - billingMode: BILLING_MODE, - harborRunner: createPierTaskRunner({ - ...runnerOptions, - agent: 'codex', - agentVersion: CODEX_PROFILE.version, - codexToolchainPath, - }), - }, - ], - pairConcurrency: manifest.maxConcurrency, - armExecution: manifest.metadata.execution.armExecution, - }); - return buildHarnessAbReport(summary, { annotations: [], warnings: [] }, BILLING_MODE); - }, - artifacts: (report) => [ - { - path: join(runRoot, 'harness-ab-report.json'), - content: `${JSON.stringify(report, null, 2)}\n`, - }, - { path: join(runRoot, 'harness-ab-report.csv'), content: renderHarnessAbReportCsv(report) }, - { - path: join(runRoot, 'harness-ab-report.md'), - content: renderHarnessAbReportMarkdown(report), - }, - ], - }); - assertHarnessAbReportCompleted(report); - console.log( - `${report.runStatus}: ${report.coverage.attemptedCells}/${report.coverage.scheduledCells} cells attempted; ${report.effectiveness.pairedEvaluated} paired Pass@1 outcomes -> ${runRoot}`, - ); -} - -if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - main().catch((error) => { - console.error(error); - process.exitCode = 1; - }); -} diff --git a/packages/headless/harbor/run-harness-ab-detached.mjs b/packages/headless/harbor/run-harness-ab-detached.mjs index 47e148ec836..58a9d375d12 100644 --- a/packages/headless/harbor/run-harness-ab-detached.mjs +++ b/packages/headless/harbor/run-harness-ab-detached.mjs @@ -7,7 +7,11 @@ import { fileURLToPath } from 'node:url'; import { spawn } from 'node:child_process'; import { resolveFixedPromptRunRoot } from '#fixed-prompt-task-source'; import { envPath as parseEnvPath } from '#headless-run-env'; -import { resolveHarnessAbRunId, resolveHarnessCompetitorProfile } from './run-harness-ab.mjs'; +import { + resolveHarnessAbRunId, + resolveHarnessBenchmarkProfile, + resolveHarnessCompetitorProfile, +} from './run-harness-ab.mjs'; const JOURNAL_FILENAME = 'background-run.json'; const LOG_FILENAME = 'background-run.log'; @@ -16,6 +20,9 @@ const envPath = (name) => parseEnvPath(name, process.env[name]); function detachedRunPaths() { const outDir = envPath('MAKA_HARNESS_AB_OUT_DIR'); + const benchmarkProfile = resolveHarnessBenchmarkProfile( + process.env.MAKA_HARNESS_AB_BENCHMARK || 'terminal-bench-2.1', + ); const competitorProfile = resolveHarnessCompetitorProfile( process.env.MAKA_HARNESS_AB_COMPETITOR || 'kimi-code', ); @@ -24,6 +31,7 @@ function detachedRunPaths() { process.env.MAKA_HARNESS_AB_RUN_ID, process.env.MAKA_HARNESS_AB_TASK_ID, process.env.MAKA_HARNESS_AB_TASK_IDS, + benchmarkProfile, ); const runRoot = resolveFixedPromptRunRoot(outDir, runId, 'MAKA_HARNESS_AB_RUN_ID'); return { diff --git a/packages/headless/harbor/run-harness-ab.mjs b/packages/headless/harbor/run-harness-ab.mjs index c36c0c90016..314752c9d92 100644 --- a/packages/headless/harbor/run-harness-ab.mjs +++ b/packages/headless/harbor/run-harness-ab.mjs @@ -12,8 +12,10 @@ import { discoverCachedHarborTasks, fingerprintFixedPromptTaskTree, resolveFixedPromptRunRoot, + selectTasksByIds, } from '#fixed-prompt-task-source'; import { createHarborTaskRunner } from '#harbor-task-runner'; +import { createPierTaskRunner } from '#pier-task-runner'; import { buildHarnessOracleExecutionPolicyFingerprint, HARBOR_ORACLE_DOCKER_PLATFORM, @@ -40,10 +42,13 @@ import { } from '#codex-toolchain'; import { createCodexOAuthHarnessCredentialBinding } from '#codex-oauth-harness'; import { + assertDeepSweSubset30TaskTreeFingerprint, assertTerminalBench21TaskSet, assertTerminalBench21TaskTreeFingerprint, buildHarnessAbResumeFingerprint, buildHarnessAbRunManifest, + DEEP_SWE_REVISION, + DEEP_SWE_SUBSET_30_TASK_IDS, HARNESS_MAKA_CONTEXT_BUDGET, TERMINAL_BENCH_2_1_REVISION, TERMINAL_BENCH_2_1_TASK_IDS, @@ -63,14 +68,12 @@ import { runExperiment } from '#experiment-engine'; const execFileAsync = promisify(execFile); -const EXPECTED_SOURCE_TASKS = TERMINAL_BENCH_2_1_TASK_IDS.length; export const DEFAULT_HARNESS_AB_RUN_ID = 'k3-maka-vs-kimi-code-tbench-2.1-full-v2'; const CANARY_TASKS = 5; const PROVIDER = 'kimi-coding-plan'; const MODEL = 'k3'; const REASONING_EFFORT = 'max'; const BASE_URL = 'https://api.kimi.com/coding/v1'; -const ORDER_SEED = 'terminal-bench-2.1:k3:harness-comparison:v1'; const MAX_PAIR_CONCURRENCY = 4; const DEFAULT_PAIR_CONCURRENCY = 1; const DEFAULT_ARM_EXECUTION = 'sequential'; @@ -90,6 +93,71 @@ const BACKGROUND_STARTED_AT_ENV = 'MAKA_HARNESS_AB_DETACHED_STARTED_AT'; const BACKGROUND_JOURNAL_FILENAME = 'background-run.json'; const BACKGROUND_LOG_FILENAME = 'background-run.log'; +/** The benchmark axis of a harness A/B. A benchmark is a BOUND pair of frozen + * task source and executor — Terminal-Bench 2.1 tasks run under plain Harbor + * 0.13.2, DeepSWE tasks under Pier ≥ 0.3.0 — so one profile carries both. The + * competitor axis (HARNESS_COMPETITOR_PROFILES) and the execution-policy env + * stay orthogonal to it. */ +export const HARNESS_BENCHMARK_PROFILES = Object.freeze({ + 'terminal-bench-2.1': Object.freeze({ + id: 'terminal-bench-2.1', + label: 'Terminal-Bench 2.1', + dataset: 'terminal-bench', + version: '2.1', + revision: TERMINAL_BENCH_2_1_REVISION, + taskIds: TERMINAL_BENCH_2_1_TASK_IDS, + runIdSlug: 'tbench-2.1-full', + // The Harbor oracle registry (advisory task-quality evidence) exists only + // for Terminal-Bench; DeepSWE grading is each task's own verifier. + oracle: true, + }), + 'deep-swe-1.1': Object.freeze({ + id: 'deep-swe-1.1', + label: 'DeepSWE subset-30', + dataset: 'deep-swe', + version: '1.1', + revision: DEEP_SWE_REVISION, + taskIds: DEEP_SWE_SUBSET_30_TASK_IDS, + runIdSlug: 'deepswe-subset30', + oracle: false, + }), +}); + +export function resolveHarnessBenchmarkProfile(raw = 'terminal-bench-2.1') { + const profile = HARNESS_BENCHMARK_PROFILES[raw]; + if (!profile) { + throw new Error( + `MAKA_HARNESS_AB_BENCHMARK must be one of: ${Object.keys(HARNESS_BENCHMARK_PROFILES).join(', ')}`, + ); + } + return profile; +} + +export function defaultHarnessBenchmarkTasksRoot(benchmarkProfile) { + return benchmarkProfile.dataset === 'deep-swe' + ? join(homedir(), '.maka/eval/task-sources/deep-swe-6db64a40/tasks') + : join(homedir(), '.cache/harbor/tasks'); +} + +/** Discover, freeze, and fingerprint the benchmark's task source. The frozen + * set is always the benchmark's full task list — fingerprint identity must not + * depend on which slice of it a canary run evaluates. */ +export async function resolveFrozenBenchmarkTasks(benchmarkProfile, tasksRoot) { + const discovered = await discoverCachedHarborTasks(tasksRoot); + if (benchmarkProfile.dataset === 'deep-swe') { + // The DeepSWE repo tree carries more tasks than the frozen subset; pick + // the subset (loud on any missing id) instead of asserting the whole tree. + const tasks = selectTasksByIds(discovered, benchmarkProfile.taskIds, benchmarkProfile.label); + const taskSourceFingerprint = await fingerprintFixedPromptTaskTree(tasks); + assertDeepSweSubset30TaskTreeFingerprint(taskSourceFingerprint); + return { tasks, taskSourceFingerprint }; + } + assertTerminalBench21TaskSet(discovered.map((task) => task.id)); + const taskSourceFingerprint = await fingerprintFixedPromptTaskTree(discovered); + assertTerminalBench21TaskTreeFingerprint(taskSourceFingerprint); + return { tasks: discovered, taskSourceFingerprint }; +} + export const HARNESS_COMPETITOR_PROFILES = Object.freeze({ 'kimi-code': Object.freeze({ id: 'kimi-code', @@ -221,6 +289,7 @@ export function resolveHarnessAbRunId( explicitRunId, isolatedTaskId, explicitTaskIds, + benchmarkProfile = resolveHarnessBenchmarkProfile(), ) { if (isolatedTaskId?.trim() && !explicitRunId?.trim()) { throw new Error('MAKA_HARNESS_AB_RUN_ID is required with MAKA_HARNESS_AB_TASK_ID'); @@ -228,13 +297,14 @@ export function resolveHarnessAbRunId( if (explicitTaskIds?.trim() && !explicitRunId?.trim()) { throw new Error('MAKA_HARNESS_AB_RUN_ID is required with MAKA_HARNESS_AB_TASK_IDS'); } + if (explicitRunId) return explicitRunId; + // Historical name predating the derived template; kept so existing k3 + // Terminal-Bench runs keep resuming into the same run root. + if (benchmarkProfile.dataset === 'terminal-bench' && competitorProfile.id === 'kimi-code') { + return DEFAULT_HARNESS_AB_RUN_ID; + } const runtime = resolveHarnessRuntimeProfile(competitorProfile); - return ( - explicitRunId || - (competitorProfile.id === 'kimi-code' - ? DEFAULT_HARNESS_AB_RUN_ID - : `${runtime.model}-maka-vs-${competitorProfile.id}${runtime.provider === 'openai-codex' ? '-oauth' : ''}-tbench-2.1-full-v1`) - ); + return `${runtime.model}-maka-vs-${competitorProfile.id}${runtime.provider === 'openai-codex' ? '-oauth' : ''}-${benchmarkProfile.runIdSlug}-v1`; } export function resolveHarnessCompetitorToolchainPath(runRoot, competitorProfile) { @@ -280,7 +350,7 @@ export function resolveHarnessCompetitorToolchain(runRoot, competitorProfile, en const envPath = (name, fallback) => parseEnvPath(name, process.env[name], fallback); const envPathFrom = (env, name, fallback) => parseEnvPath(name, env[name], fallback); -export function defaultMakaWorkspaceRoot() { +function defaultMakaWorkspaceRoot() { if (process.platform === 'darwin') { return join(homedir(), 'Library', 'Application Support', 'Maka', 'workspaces', 'default'); } @@ -300,10 +370,11 @@ export function defaultMakaWorkspaceRoot() { ); } -function runLimit(raw) { +function runLimit(raw, benchmarkProfile) { + const fullCount = benchmarkProfile.taskIds.length; const parsed = Number(raw ?? CANARY_TASKS); - if (parsed !== CANARY_TASKS && parsed !== EXPECTED_SOURCE_TASKS) { - throw new Error(`MAKA_HARNESS_AB_LIMIT must be ${CANARY_TASKS} or ${EXPECTED_SOURCE_TASKS}`); + if (parsed !== CANARY_TASKS && parsed !== fullCount) { + throw new Error(`MAKA_HARNESS_AB_LIMIT must be ${CANARY_TASKS} or ${fullCount}`); } return parsed; } @@ -336,7 +407,12 @@ export function resolveHarnessAbExecutionPolicy(rawPairConcurrency, rawArmExecut }; } -export function resolveHarnessAbTaskSelection(rawTaskId, rawLimit, rawTaskIds) { +export function resolveHarnessAbTaskSelection( + rawTaskId, + rawLimit, + rawTaskIds, + benchmarkProfile = resolveHarnessBenchmarkProfile(), +) { const taskId = rawTaskId?.trim(); const explicitTaskIds = rawTaskIds ?.split(',') @@ -354,11 +430,11 @@ export function resolveHarnessAbTaskSelection(rawTaskId, rawLimit, rawTaskIds) { throw new Error('MAKA_HARNESS_AB_TASK_IDS must not contain duplicate task ids'); } const invalidTaskIds = uniqueTaskIds.filter( - (selectedTaskId) => !TERMINAL_BENCH_2_1_TASK_IDS.includes(selectedTaskId), + (selectedTaskId) => !benchmarkProfile.taskIds.includes(selectedTaskId), ); if (invalidTaskIds.length > 0) { throw new Error( - `MAKA_HARNESS_AB_TASK_IDS contains unknown Terminal-Bench 2.1 tasks: ${invalidTaskIds.join(', ')}`, + `MAKA_HARNESS_AB_TASK_IDS contains unknown ${benchmarkProfile.label} tasks: ${invalidTaskIds.join(', ')}`, ); } return { @@ -368,12 +444,12 @@ export function resolveHarnessAbTaskSelection(rawTaskId, rawLimit, rawTaskIds) { } if (!taskId) { return { - taskIds: TERMINAL_BENCH_2_1_TASK_IDS, - limit: runLimit(rawLimit), + taskIds: benchmarkProfile.taskIds, + limit: runLimit(rawLimit, benchmarkProfile), }; } - if (!TERMINAL_BENCH_2_1_TASK_IDS.includes(taskId)) { - throw new Error('MAKA_HARNESS_AB_TASK_ID must name a Terminal-Bench 2.1 task'); + if (!benchmarkProfile.taskIds.includes(taskId)) { + throw new Error(`MAKA_HARNESS_AB_TASK_ID must name a ${benchmarkProfile.label} task`); } return { taskIds: [taskId], limit: 1 }; } @@ -402,7 +478,8 @@ export function buildHarnessAbManifest({ subjectFingerprint, taskSourceFingerprint, toolchainFingerprint, - taskIds = TERMINAL_BENCH_2_1_TASK_IDS, + benchmarkProfile = resolveHarnessBenchmarkProfile(), + taskIds = benchmarkProfile.taskIds, competitorProfile = resolveHarnessCompetitorProfile(), pairConcurrency = Math.min(DEFAULT_PAIR_CONCURRENCY, taskIds.length), armExecution = DEFAULT_ARM_EXECUTION, @@ -413,18 +490,15 @@ export function buildHarnessAbManifest({ const execution = buildHarnessExecutionProfile(competitorProfile); return buildHarnessAbRunManifest({ benchmark: { - dataset: 'terminal-bench', - version: '2.1', - revision: TERMINAL_BENCH_2_1_REVISION, + dataset: benchmarkProfile.dataset, + version: benchmarkProfile.version, + revision: benchmarkProfile.revision, timeoutPolicy: 'task-native', timeoutMultiplier: 1, outerTimeoutGraceSec: HARBOR_SETUP_TEARDOWN_GRACE_SEC, }, taskIds, - orderSeed: - execution.model === MODEL - ? ORDER_SEED - : `terminal-bench-2.1:${execution.model}:harness-comparison:v1`, + orderSeed: `${benchmarkProfile.id}:${execution.model}:harness-comparison:v1`, pilotTaskCount: Math.min(CANARY_TASKS, taskIds.length), model: { provider: execution.provider, @@ -474,7 +548,13 @@ export async function main() { ? resolve(process.env.MAKA_HARNESS_AB_MAKA_REPO) : repoRoot; const outDir = envPath('MAKA_HARNESS_AB_OUT_DIR'); - const tasksRoot = envPath('MAKA_HARNESS_AB_TASKS_ROOT', join(homedir(), '.cache/harbor/tasks')); + const benchmarkProfile = resolveHarnessBenchmarkProfile( + process.env.MAKA_HARNESS_AB_BENCHMARK || 'terminal-bench-2.1', + ); + const tasksRoot = envPath( + 'MAKA_HARNESS_AB_TASKS_ROOT', + defaultHarnessBenchmarkTasksRoot(benchmarkProfile), + ); const competitorProfile = resolveHarnessCompetitorProfile( process.env.MAKA_HARNESS_AB_COMPETITOR || 'kimi-code', ); @@ -483,11 +563,13 @@ export async function main() { process.env.MAKA_HARNESS_AB_RUN_ID, process.env.MAKA_HARNESS_AB_TASK_ID, process.env.MAKA_HARNESS_AB_TASK_IDS, + benchmarkProfile, ); const selection = resolveHarnessAbTaskSelection( process.env.MAKA_HARNESS_AB_TASK_ID, process.env.MAKA_HARNESS_AB_LIMIT, process.env.MAKA_HARNESS_AB_TASK_IDS, + benchmarkProfile, ); const executionPolicy = resolveHarnessAbExecutionPolicy( process.env.MAKA_HARNESS_AB_PAIR_CONCURRENCY, @@ -508,6 +590,7 @@ export async function main() { selection, executionPolicy, runRoot, + benchmarkProfile, competitorProfile, }); } catch (error) { @@ -534,16 +617,22 @@ async function runLocked({ selection, executionPolicy, runRoot, + benchmarkProfile, competitorProfile, }) { - const allTasks = await discoverCachedHarborTasks(tasksRoot); - assertTerminalBench21TaskSet(allTasks.map((task) => task.id)); - const taskSourceFingerprint = await fingerprintFixedPromptTaskTree(allTasks); - assertTerminalBench21TaskTreeFingerprint(taskSourceFingerprint); + if (benchmarkProfile.dataset === 'deep-swe' && competitorProfile.id === 'opencode') { + throw new Error( + 'the OpenCode adapter has no Pier arm; DeepSWE supports the kimi-code and codex competitors', + ); + } + const { tasks: allTasks, taskSourceFingerprint } = await resolveFrozenBenchmarkTasks( + benchmarkProfile, + tasksRoot, + ); if (process.env.MAKA_HARNESS_AB_DRY_RUN === '1') { console.log( - `dry-run: frozen ${EXPECTED_SOURCE_TASKS}-task source will run ${selection.limit} paired Pass@1 cells; Oracle evidence is advisory`, + `dry-run: frozen ${benchmarkProfile.taskIds.length}-task ${benchmarkProfile.label} source will run ${selection.limit} paired Pass@1 cells${benchmarkProfile.oracle ? '; Oracle evidence is advisory' : ' via Pier'}`, ); return; } @@ -560,24 +649,29 @@ async function runLocked({ makaRepoPath, 'MAKA_HARNESS_AB', ); - const [verifierImplementationSource, composeImplementationSource] = await Promise.all([ - readFile(join(makaRepoPath, 'packages/headless/harbor/maka_verifier.py')), - readFile(join(makaRepoPath, 'packages/headless/harbor/docker-compose-linux-amd64.yaml')), - ]); - const executionPolicyFingerprint = buildHarnessOracleExecutionPolicyFingerprint({ - verifierImplementationSource, - composeImplementationSource, - }); - const tasksById = new Map(allTasks.map((task) => [task.id, task])); const manifestPath = join(runRoot, 'harness-ab-manifest.json'); - const oracleEvidence = await resolveHarnessOracleEvidenceForRun(manifestPath, () => - resolveAdvisoryOracleEvidence({ - allTasks, - executionPolicyFingerprint, - }), - ); - for (const warning of oracleEvidence.warnings) console.warn(`warning: ${warning}`); + // Oracle evidence is a Terminal-Bench/Harbor institution (the Maka oracle + // verifier + advisory registry); DeepSWE grading is each task's own verifier, + // so there is no oracle to consult. + let oracleEvidence = null; + if (benchmarkProfile.oracle) { + const [verifierImplementationSource, composeImplementationSource] = await Promise.all([ + readFile(join(makaRepoPath, 'packages/headless/harbor/maka_verifier.py')), + readFile(join(makaRepoPath, 'packages/headless/harbor/docker-compose-linux-amd64.yaml')), + ]); + const executionPolicyFingerprint = buildHarnessOracleExecutionPolicyFingerprint({ + verifierImplementationSource, + composeImplementationSource, + }); + oracleEvidence = await resolveHarnessOracleEvidenceForRun(manifestPath, () => + resolveAdvisoryOracleEvidence({ + allTasks, + executionPolicyFingerprint, + }), + ); + for (const warning of oracleEvidence.warnings) console.warn(`warning: ${warning}`); + } const credentials = await resolveHarnessRuntimeCredentials({ competitorProfile, @@ -597,10 +691,11 @@ async function runLocked({ subjectFingerprint, taskSourceFingerprint, toolchainFingerprint, + benchmarkProfile, taskIds: selection.taskIds, pairConcurrency: executionPolicy.pairConcurrency, armExecution: executionPolicy.armExecution, - oracleEvidence, + ...(oracleEvidence ? { oracleEvidence } : {}), competitorProfile, credentialIdentity: credentials.credentialIdentity, }); @@ -627,6 +722,12 @@ async function runLocked({ runRoot, prompts: () => [{ path: systemPromptPath, content: DEFAULT_HEADLESS_SYSTEM_PROMPT }], run: async ({ jobsDir, resultsJsonlPath }) => { + // The benchmark owns its executor: Terminal-Bench trials run under plain + // Harbor, DeepSWE trials under Pier. The two runners share the TaskRunner + // contract; only the base-URL channel and Docker-platform pin differ + // (Pier's EnvironmentConfig cannot carry an explicit platform). + const createBenchmarkRunner = + benchmarkProfile.dataset === 'deep-swe' ? createPierTaskRunner : createHarborTaskRunner; const runnerOptions = { makaRepoPath, jobsDir, @@ -635,9 +736,13 @@ async function runLocked({ reasoningEffort: execution.reasoningEffort, ...credentials, pricing: execution.pricing, - agentEnv: { MAKA_BASE_URL: execution.baseUrl }, timeoutMultiplier: 1, - dockerPlatform: 'linux/amd64', + ...(benchmarkProfile.dataset === 'deep-swe' + ? { baseUrl: execution.baseUrl } + : { + agentEnv: { MAKA_BASE_URL: execution.baseUrl }, + dockerPlatform: 'linux/amd64', + }), }; const makaContextBudgetEnv = harnessMakaContextBudgetEnv(); const config = (id) => ({ @@ -660,7 +765,7 @@ async function runLocked({ config: config('maka'), expectedPricingProfile: execution.pricing.source, billingMode: execution.billingMode, - harborRunner: createHarborTaskRunner({ + harborRunner: createBenchmarkRunner({ ...runnerOptions, agent: 'maka', agentEnv: { ...runnerOptions.agentEnv, ...makaContextBudgetEnv }, @@ -671,7 +776,7 @@ async function runLocked({ config: config(competitorProfile.id), expectedPricingProfile: execution.pricing.source, billingMode: execution.billingMode, - harborRunner: createHarborTaskRunner({ + harborRunner: createBenchmarkRunner({ ...runnerOptions, agent: competitorProfile.id, agentVersion: competitorProfile.version, @@ -688,15 +793,17 @@ async function runLocked({ }); return buildHarnessAbReport( summary, - { - ...(oracleEvidence.resolvedSnapshotFingerprint - ? { snapshotFingerprint: oracleEvidence.resolvedSnapshotFingerprint } - : {}), - annotations: oracleEvidence.annotations.filter((annotation) => - evaluatedTaskIds.has(annotation.taskId), - ), - warnings: oracleEvidence.warnings, - }, + oracleEvidence + ? { + ...(oracleEvidence.resolvedSnapshotFingerprint + ? { snapshotFingerprint: oracleEvidence.resolvedSnapshotFingerprint } + : {}), + annotations: oracleEvidence.annotations.filter((annotation) => + evaluatedTaskIds.has(annotation.taskId), + ), + warnings: oracleEvidence.warnings, + } + : { annotations: [], warnings: [] }, execution.billingMode, ); }, diff --git a/packages/headless/src/__tests__/harness-ab-cli.test.ts b/packages/headless/src/__tests__/harness-ab-cli.test.ts index e6ca459a383..1e3ae371079 100644 --- a/packages/headless/src/__tests__/harness-ab-cli.test.ts +++ b/packages/headless/src/__tests__/harness-ab-cli.test.ts @@ -716,6 +716,83 @@ test('harness A/B CLI rejects modified task contents before reading credentials' } }); +test('harness A/B resolves the DeepSWE benchmark axis orthogonally to competitors', async () => { + const { + buildHarnessAbManifest, + resolveHarnessAbRunId, + resolveHarnessAbTaskSelection, + resolveHarnessBenchmarkProfile, + resolveHarnessCompetitorProfile, + } = await import(new URL('../../harbor/run-harness-ab.mjs', import.meta.url).href); + const benchmarkProfile = resolveHarnessBenchmarkProfile('deep-swe-1.1'); + + assert.throws( + () => resolveHarnessBenchmarkProfile('swe-bench'), + /MAKA_HARNESS_AB_BENCHMARK must be one of: terminal-bench-2\.1, deep-swe-1\.1/, + ); + + // Task identity comes from the benchmark: a Terminal-Bench task id is + // unknown here, and the 30-task subset replaces the 89-task full set. + const selection = resolveHarnessAbTaskSelection(undefined, '30', undefined, benchmarkProfile); + assert.equal(selection.taskIds.length, 30); + assert.throws( + () => + resolveHarnessAbTaskSelection( + 'extract-moves-from-video', + undefined, + undefined, + benchmarkProfile, + ), + /MAKA_HARNESS_AB_TASK_ID must name a DeepSWE subset-30 task/, + ); + assert.throws( + () => resolveHarnessAbTaskSelection(undefined, '89', undefined, benchmarkProfile), + /MAKA_HARNESS_AB_LIMIT must be 5 or 30/, + ); + + // The derived run id carries model, competitor, credential mode, benchmark. + assert.equal( + resolveHarnessAbRunId( + resolveHarnessCompetitorProfile('kimi-code'), + undefined, + undefined, + undefined, + benchmarkProfile, + ), + 'k3-maka-vs-kimi-code-deepswe-subset30-v1', + ); + assert.equal( + resolveHarnessAbRunId( + resolveHarnessCompetitorProfile('codex'), + undefined, + undefined, + undefined, + benchmarkProfile, + ), + 'gpt-5.6-sol-maka-vs-codex-oauth-deepswe-subset30-v1', + ); + + const manifest = buildHarnessAbManifest({ + subjectFingerprint: 'subject', + taskSourceFingerprint: 'tasks', + toolchainFingerprint: 'tools', + benchmarkProfile, + competitorProfile: resolveHarnessCompetitorProfile('codex'), + }); + assert.deepEqual(manifest.metadata.benchmark.dataset, 'deep-swe'); + assert.equal(manifest.metadata.benchmark.version, '1.1'); + assert.equal(manifest.metadata.order.seed, 'deep-swe-1.1:gpt-5.6-sol:harness-comparison:v1'); + assert.equal(manifest.evaluationTaskIds.length, 30); + + // The Terminal-Bench default keeps its historical order seed byte-for-byte. + const tbenchManifest = buildHarnessAbManifest({ + subjectFingerprint: 'subject', + taskSourceFingerprint: 'tasks', + toolchainFingerprint: 'tools', + }); + assert.equal(tbenchManifest.metadata.order.seed, 'terminal-bench-2.1:k3:harness-comparison:v1'); +}); + async function waitForJournal( path: string, predicate: (value: Record) => boolean, From f7604095b1338fbf690053ed62b24988e5048fdb Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 23 Jul 2026 21:17:31 +0800 Subject: [PATCH 3/4] fix(headless): hydrate Codex Pier logs and freeze the Pier executor identity Review follow-ups on the DeepSWE benchmark axis: - codex_agent.py hydrates codex.txt and the sessions rollouts in run()'s finally, mirroring the Kimi adapter: Pier replaces the default /logs/agent bind mount under --mounts-json while capabilities.mounted stays true, so without this download a real token-burning run reads zero events and every cell is misclassified as failed. - Benchmark profiles now carry their bound executor explicitly, and a Pier benchmark freezes `pier --version` (stdout only) into the toolchain fingerprint so a resume across a Pier upgrade forks instead of mixing cells; the Harbor payload stays byte-identical. - resolveHarnessBenchmarkProfile's no-argument default reads MAKA_HARNESS_AB_BENCHMARK, so defaulted call sites agree with the production entry points. - resolveFrozenBenchmarkTasks passes selectTasksByIds an options object, so missing-task errors keep their DeepSWE subset-30 scope prefix. --- packages/headless/harbor/codex_agent.py | 34 +++++++++ .../harbor/run-harness-ab-detached.mjs | 4 +- packages/headless/harbor/run-harness-ab.mjs | 72 ++++++++++++++----- .../harbor/tests/test_harness_compat.py | 54 ++++++++++++++ .../src/__tests__/harbor-adapter.test.ts | 3 + .../src/__tests__/harness-ab-cli.test.ts | 63 ++++++++++++++++ 6 files changed, 209 insertions(+), 21 deletions(-) diff --git a/packages/headless/harbor/codex_agent.py b/packages/headless/harbor/codex_agent.py index f38a278e0f6..04e94ba4272 100644 --- a/packages/headless/harbor/codex_agent.py +++ b/packages/headless/harbor/codex_agent.py @@ -34,6 +34,8 @@ _TOOLCHAIN_CHECKSUMS = _TOOLCHAIN_ROOT / "checksums.sha256" _DEFAULT_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" _OUTPUT_FILENAME = "codex.txt" +_REMOTE_OUTPUT_PATH = Path("/logs/agent") / _OUTPUT_FILENAME +_REMOTE_SESSIONS_DIR = Path("/logs/agent/sessions") class MakaCodexAgent(Codex): @@ -150,6 +152,38 @@ async def run( await cleanup_process_scope(self, environment, command_scope) finally: self._finished_at_ms = int(time.time() * 1000) + await self._download_agent_logs(environment) + + async def _download_agent_logs(self, environment: BaseEnvironment) -> None: + # populate_context_post_run and _write_cell_output read codex.txt and + # sessions/**/rollout-*.jsonl from the host log dir. Under Harbor the + # agent log dir is bind-mounted, so both are already host-side and are + # skipped; under Pier a --mounts-json run replaces the default log + # mounts while capabilities.mounted stays true (pier docker.py), so + # pier's own log download never runs — without this hydration + # _events() sees nothing and a real token-burning run is misclassified + # as failed. Runs in run()'s finally so failure paths are hydrated too. + local_output = self.logs_dir / _OUTPUT_FILENAME + if not local_output.exists(): + try: + await environment.download_file(_REMOTE_OUTPUT_PATH.as_posix(), local_output) + except Exception as exc: # noqa: BLE001 - best-effort log hydration. + self.logger.debug( + "Could not download Codex output %s: %s", _REMOTE_OUTPUT_PATH, exc + ) + local_sessions = self.logs_dir / "sessions" + if not local_sessions.exists(): + try: + # docker cp of `dir/.` requires an existing target directory. + local_sessions.mkdir(parents=True, exist_ok=True) + await environment.download_dir( + source_dir=_REMOTE_SESSIONS_DIR.as_posix(), + target_dir=local_sessions, + ) + except Exception as exc: # noqa: BLE001 - best-effort log hydration. + self.logger.debug( + "Could not download Codex sessions %s: %s", _REMOTE_SESSIONS_DIR, exc + ) async def _write_http_provider_config( self, environment: BaseEnvironment, proxy_url: str diff --git a/packages/headless/harbor/run-harness-ab-detached.mjs b/packages/headless/harbor/run-harness-ab-detached.mjs index 58a9d375d12..fbd90ec41ec 100644 --- a/packages/headless/harbor/run-harness-ab-detached.mjs +++ b/packages/headless/harbor/run-harness-ab-detached.mjs @@ -20,9 +20,7 @@ const envPath = (name) => parseEnvPath(name, process.env[name]); function detachedRunPaths() { const outDir = envPath('MAKA_HARNESS_AB_OUT_DIR'); - const benchmarkProfile = resolveHarnessBenchmarkProfile( - process.env.MAKA_HARNESS_AB_BENCHMARK || 'terminal-bench-2.1', - ); + const benchmarkProfile = resolveHarnessBenchmarkProfile(); const competitorProfile = resolveHarnessCompetitorProfile( process.env.MAKA_HARNESS_AB_COMPETITOR || 'kimi-code', ); diff --git a/packages/headless/harbor/run-harness-ab.mjs b/packages/headless/harbor/run-harness-ab.mjs index 314752c9d92..fc9489afd04 100644 --- a/packages/headless/harbor/run-harness-ab.mjs +++ b/packages/headless/harbor/run-harness-ab.mjs @@ -106,6 +106,7 @@ export const HARNESS_BENCHMARK_PROFILES = Object.freeze({ version: '2.1', revision: TERMINAL_BENCH_2_1_REVISION, taskIds: TERMINAL_BENCH_2_1_TASK_IDS, + executor: 'harbor', runIdSlug: 'tbench-2.1-full', // The Harbor oracle registry (advisory task-quality evidence) exists only // for Terminal-Bench; DeepSWE grading is each task's own verifier. @@ -118,12 +119,15 @@ export const HARNESS_BENCHMARK_PROFILES = Object.freeze({ version: '1.1', revision: DEEP_SWE_REVISION, taskIds: DEEP_SWE_SUBSET_30_TASK_IDS, + executor: 'pier', runIdSlug: 'deepswe-subset30', oracle: false, }), }); -export function resolveHarnessBenchmarkProfile(raw = 'terminal-bench-2.1') { +export function resolveHarnessBenchmarkProfile( + raw = process.env.MAKA_HARNESS_AB_BENCHMARK || 'terminal-bench-2.1', +) { const profile = HARNESS_BENCHMARK_PROFILES[raw]; if (!profile) { throw new Error( @@ -147,7 +151,9 @@ export async function resolveFrozenBenchmarkTasks(benchmarkProfile, tasksRoot) { if (benchmarkProfile.dataset === 'deep-swe') { // The DeepSWE repo tree carries more tasks than the frozen subset; pick // the subset (loud on any missing id) instead of asserting the whole tree. - const tasks = selectTasksByIds(discovered, benchmarkProfile.taskIds, benchmarkProfile.label); + const tasks = selectTasksByIds(discovered, benchmarkProfile.taskIds, { + label: benchmarkProfile.label, + }); const taskSourceFingerprint = await fingerprintFixedPromptTaskTree(tasks); assertDeepSweSubset30TaskTreeFingerprint(taskSourceFingerprint); return { tasks, taskSourceFingerprint }; @@ -158,6 +164,42 @@ export async function resolveFrozenBenchmarkTasks(benchmarkProfile, tasksRoot) { return { tasks: discovered, taskSourceFingerprint }; } +/** Compose the run's toolchain identity. The Harbor payload is byte-identical + * to the historical formula; a Pier benchmark additionally freezes the Pier + * executor version, so a resume across a Pier upgrade forks instead of mixing + * cells produced under different execution semantics. */ +export function buildHarnessAbToolchainFingerprint({ + hostToolchainFingerprint, + competitorProfile, + pierVersion = null, +}) { + return `sha256:${createHash('sha256') + .update( + JSON.stringify({ + hostToolchainFingerprint, + competitor: competitorProfile.id, + competitorToolchainFingerprint: competitorProfile.toolchainFingerprint, + ...(pierVersion === null ? {} : { pierVersion }), + }), + ) + .digest('hex')}`; +} + +async function readPierVersion() { + try { + // stdout only: pier leaks environment-dependent LiteLLM warnings to + // stderr, which must not enter the frozen resume identity. + const { stdout } = await execFileAsync('pier', ['--version']); + const version = stdout.trim(); + if (!version) throw new Error('pier --version printed nothing on stdout'); + return version; + } catch (error) { + throw new Error( + `the DeepSWE benchmark freezes its Pier executor version into the resume identity, but pier --version failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + export const HARNESS_COMPETITOR_PROFILES = Object.freeze({ 'kimi-code': Object.freeze({ id: 'kimi-code', @@ -548,9 +590,7 @@ export async function main() { ? resolve(process.env.MAKA_HARNESS_AB_MAKA_REPO) : repoRoot; const outDir = envPath('MAKA_HARNESS_AB_OUT_DIR'); - const benchmarkProfile = resolveHarnessBenchmarkProfile( - process.env.MAKA_HARNESS_AB_BENCHMARK || 'terminal-bench-2.1', - ); + const benchmarkProfile = resolveHarnessBenchmarkProfile(); const tasksRoot = envPath( 'MAKA_HARNESS_AB_TASKS_ROOT', defaultHarnessBenchmarkTasksRoot(benchmarkProfile), @@ -620,7 +660,7 @@ async function runLocked({ benchmarkProfile, competitorProfile, }) { - if (benchmarkProfile.dataset === 'deep-swe' && competitorProfile.id === 'opencode') { + if (benchmarkProfile.executor === 'pier' && competitorProfile.id === 'opencode') { throw new Error( 'the OpenCode adapter has no Pier arm; DeepSWE supports the kimi-code and codex competitors', ); @@ -632,7 +672,7 @@ async function runLocked({ if (process.env.MAKA_HARNESS_AB_DRY_RUN === '1') { console.log( - `dry-run: frozen ${benchmarkProfile.taskIds.length}-task ${benchmarkProfile.label} source will run ${selection.limit} paired Pass@1 cells${benchmarkProfile.oracle ? '; Oracle evidence is advisory' : ' via Pier'}`, + `dry-run: frozen ${benchmarkProfile.taskIds.length}-task ${benchmarkProfile.label} source will run ${selection.limit} paired Pass@1 cells${benchmarkProfile.executor === 'pier' ? ' via Pier' : ''}${benchmarkProfile.oracle ? '; Oracle evidence is advisory' : ''}`, ); return; } @@ -678,15 +718,11 @@ async function runLocked({ env: process.env, }); - const toolchainFingerprint = `sha256:${createHash('sha256') - .update( - JSON.stringify({ - hostToolchainFingerprint, - competitor: competitorProfile.id, - competitorToolchainFingerprint: competitorProfile.toolchainFingerprint, - }), - ) - .digest('hex')}`; + const toolchainFingerprint = buildHarnessAbToolchainFingerprint({ + hostToolchainFingerprint, + competitorProfile, + pierVersion: benchmarkProfile.executor === 'pier' ? await readPierVersion() : null, + }); const manifest = buildHarnessAbManifest({ subjectFingerprint, taskSourceFingerprint, @@ -727,7 +763,7 @@ async function runLocked({ // contract; only the base-URL channel and Docker-platform pin differ // (Pier's EnvironmentConfig cannot carry an explicit platform). const createBenchmarkRunner = - benchmarkProfile.dataset === 'deep-swe' ? createPierTaskRunner : createHarborTaskRunner; + benchmarkProfile.executor === 'pier' ? createPierTaskRunner : createHarborTaskRunner; const runnerOptions = { makaRepoPath, jobsDir, @@ -737,7 +773,7 @@ async function runLocked({ ...credentials, pricing: execution.pricing, timeoutMultiplier: 1, - ...(benchmarkProfile.dataset === 'deep-swe' + ...(benchmarkProfile.executor === 'pier' ? { baseUrl: execution.baseUrl } : { agentEnv: { MAKA_BASE_URL: execution.baseUrl }, diff --git a/packages/headless/harbor/tests/test_harness_compat.py b/packages/headless/harbor/tests/test_harness_compat.py index 7696accc42d..b8efa66ad9e 100644 --- a/packages/headless/harbor/tests/test_harness_compat.py +++ b/packages/headless/harbor/tests/test_harness_compat.py @@ -327,6 +327,60 @@ def test_kimi_runtime_env_forwards_ipv6_proxy_url(): assert agent._runtime_env()["KIMI_MODEL_BASE_URL"] == "http://[::1]:43210/v1" +@_isolated_maka_env +def test_codex_agent_hydrates_agent_logs_best_effort(): + # Runs under BOTH trees. Pier replaces the default /logs/agent bind mount + # when a trial passes --mounts-json while capabilities.mounted stays true + # (pier docker.py), so pier's own log download never runs; the adapter must + # hydrate codex.txt and the sessions rollouts itself or _events() reads + # nothing and a real token-burning run is misclassified as failed. Under + # Harbor the log dir is bind-mounted, so hydration must skip host-side + # files, and a download failure must stay best-effort. + import asyncio + import tempfile + + _, _, codex_mod = _fresh_adapters() + + class _RecordingEnvironment: + def __init__(self) -> None: + self.files: list[tuple[str, Path]] = [] + self.dirs: list[tuple[str, Path]] = [] + + async def download_file(self, source_path, target_path): + self.files.append((source_path, Path(target_path))) + Path(target_path).write_text("", encoding="utf-8") + + async def download_dir(self, source_dir, target_dir): + self.dirs.append((source_dir, Path(target_dir))) + + with tempfile.TemporaryDirectory() as tmp: + logs_dir = Path(tmp) + agent = codex_mod.MakaCodexAgent(logs_dir=logs_dir) + environment = _RecordingEnvironment() + asyncio.run(agent._download_agent_logs(environment)) + assert environment.files == [("/logs/agent/codex.txt", logs_dir / "codex.txt")] + assert environment.dirs == [("/logs/agent/sessions", logs_dir / "sessions")] + # docker cp of `dir/.` requires the target directory to exist. + assert (logs_dir / "sessions").is_dir() + + # Both targets are now host-side (the Harbor bind-mount case): no + # further downloads. + asyncio.run(agent._download_agent_logs(environment)) + assert len(environment.files) == 1 + assert len(environment.dirs) == 1 + + class _FailingEnvironment: + async def download_file(self, source_path, target_path): + raise RuntimeError("container already torn down") + + async def download_dir(self, source_dir, target_dir): + raise RuntimeError("container already torn down") + + with tempfile.TemporaryDirectory() as tmp: + agent = codex_mod.MakaCodexAgent(logs_dir=Path(tmp)) + asyncio.run(agent._download_agent_logs(_FailingEnvironment())) + + @_isolated_maka_env def test_harbor_tree_used_without_pier(): saved_pier = _without_pier_modules() diff --git a/packages/headless/src/__tests__/harbor-adapter.test.ts b/packages/headless/src/__tests__/harbor-adapter.test.ts index c68a5fba4a1..d9756552270 100644 --- a/packages/headless/src/__tests__/harbor-adapter.test.ts +++ b/packages/headless/src/__tests__/harbor-adapter.test.ts @@ -3068,6 +3068,9 @@ class Codex: self._version = version self.model_name = model_name self.parent_calls = [] + # Real harbor/pier bases set self.logger in __init__; the adapter's + # best-effort log hydration logs through it. + self.logger = types.SimpleNamespace(debug=lambda *args, **kwargs: None) def _get_env(self, key): return self._extra_env.get(key) or os.environ.get(key) diff --git a/packages/headless/src/__tests__/harness-ab-cli.test.ts b/packages/headless/src/__tests__/harness-ab-cli.test.ts index 1e3ae371079..a7b3cbd3c93 100644 --- a/packages/headless/src/__tests__/harness-ab-cli.test.ts +++ b/packages/headless/src/__tests__/harness-ab-cli.test.ts @@ -793,6 +793,69 @@ test('harness A/B resolves the DeepSWE benchmark axis orthogonally to competitor assert.equal(tbenchManifest.metadata.order.seed, 'terminal-bench-2.1:k3:harness-comparison:v1'); }); +test('harness A/B benchmark profiles bind their executor and resolve from env', async () => { + const { HARNESS_BENCHMARK_PROFILES, resolveHarnessBenchmarkProfile } = await import( + new URL('../../harbor/run-harness-ab.mjs', import.meta.url).href + ); + + // A benchmark is a bound task-source + executor pair; the executor field is + // what selects the runner and what freezes the Pier version into resume + // identity. + assert.equal(HARNESS_BENCHMARK_PROFILES['terminal-bench-2.1'].executor, 'harbor'); + assert.equal(HARNESS_BENCHMARK_PROFILES['deep-swe-1.1'].executor, 'pier'); + + // The no-argument default reads MAKA_HARNESS_AB_BENCHMARK, so every + // defaulted call site agrees with the production entry points. + const saved = process.env.MAKA_HARNESS_AB_BENCHMARK; + try { + delete process.env.MAKA_HARNESS_AB_BENCHMARK; + assert.equal(resolveHarnessBenchmarkProfile().id, 'terminal-bench-2.1'); + process.env.MAKA_HARNESS_AB_BENCHMARK = 'deep-swe-1.1'; + assert.equal(resolveHarnessBenchmarkProfile().id, 'deep-swe-1.1'); + } finally { + if (saved === undefined) delete process.env.MAKA_HARNESS_AB_BENCHMARK; + else process.env.MAKA_HARNESS_AB_BENCHMARK = saved; + } +}); + +test('harness A/B toolchain identity freezes the Pier version only for Pier benchmarks', async () => { + const { buildHarnessAbToolchainFingerprint, resolveHarnessCompetitorProfile } = await import( + new URL('../../harbor/run-harness-ab.mjs', import.meta.url).href + ); + const { createHash } = await import('node:crypto'); + const competitorProfile = resolveHarnessCompetitorProfile('kimi-code'); + + // Harbor benchmarks (pierVersion null) must reproduce the historical + // payload byte-for-byte, or every existing Terminal-Bench run loses resume. + const legacy = `sha256:${createHash('sha256') + .update( + JSON.stringify({ + hostToolchainFingerprint: 'host', + competitor: competitorProfile.id, + competitorToolchainFingerprint: competitorProfile.toolchainFingerprint, + }), + ) + .digest('hex')}`; + assert.equal( + buildHarnessAbToolchainFingerprint({ hostToolchainFingerprint: 'host', competitorProfile }), + legacy, + ); + + // A Pier executor version change forks the resume identity. + const pier030 = buildHarnessAbToolchainFingerprint({ + hostToolchainFingerprint: 'host', + competitorProfile, + pierVersion: 'pier 0.3.0', + }); + const pier040 = buildHarnessAbToolchainFingerprint({ + hostToolchainFingerprint: 'host', + competitorProfile, + pierVersion: 'pier 0.4.0', + }); + assert.notEqual(pier030, legacy); + assert.notEqual(pier030, pier040); +}); + async function waitForJournal( path: string, predicate: (value: Record) => boolean, From 0108e53e8026673cc6c55ca0f0679b957dcaded2 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Thu, 23 Jul 2026 22:47:45 +0800 Subject: [PATCH 4/4] fix(headless): record the Pier executor in the manifest and close review polish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The manifest's benchmark section carries { executor: { id: 'pier', version } } for Pier benchmarks, so the frozen executor identity is auditable without trial-log archaeology; the key is absent for Harbor benchmarks and a test locks the byte-identical Terminal-Bench payload. - The opencode × Pier guard fires in main() before any run root or lock is created, so a doomed pairing leaves no run state behind. - The headless README documents the benchmark axis (MAKA_HARNESS_AB_BENCHMARK, the DeepSWE subset-30 profile, and the Pier executor pin). --- packages/headless/README.md | 6 ++++-- packages/headless/harbor/run-harness-ab.mjs | 21 +++++++++++++------ .../src/__tests__/harness-ab-cli.test.ts | 15 +++++++++++++ packages/headless/src/harness-ab-manifest.ts | 4 ++++ 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/packages/headless/README.md b/packages/headless/README.md index 3bceb4703bc..1d824f04276 100644 --- a/packages/headless/README.md +++ b/packages/headless/README.md @@ -185,9 +185,11 @@ node packages/headless/harbor/run-terminal-bench-smoke.mjs --profile maka-heavy node packages/headless/harbor/run-terminal-bench-smoke.mjs --compare --task '*sqlite-with-gcov' ``` -## Terminal-Bench 2.1 harness comparison +## Harness A/B comparison -`harbor/run-harness-ab.mjs` compares Maka with one pinned CLI under the same model, reasoning effort, task instructions, task order, account/provider route, and Harbor verifier while each agent retains its native runtime policy. The default is the official Kimi Code 0.26.0 CLI on Kimi K3 Max; set `MAKA_HARNESS_AB_COMPETITOR=opencode` for OpenCode 1.17.18 or `MAKA_HARNESS_AB_COMPETITOR=codex` for the official Codex CLI 0.144.6 on `gpt-5.6-sol` with `xhigh` reasoning. The Codex profile uses ChatGPT/Codex OAuth for both arms and records account-plan billing rather than public API spend. The task root must match the 89 task ids and canonical task-tree fingerprint of the frozen official Terminal-Bench 2.1 revision; a matching export with one task directory per id is accepted directly. The A/B task order is frozen independently of Oracle evidence. Maka keeps active and stale tool-result pruning enabled while semantic compact is explicitly disabled in both the manifest and runtime environment. +`harbor/run-harness-ab.mjs` compares Maka with one pinned CLI under the same model, reasoning effort, task instructions, task order, account/provider route, and benchmark verifier while each agent retains its native runtime policy. The default is the official Kimi Code 0.26.0 CLI on Kimi K3 Max; set `MAKA_HARNESS_AB_COMPETITOR=opencode` for OpenCode 1.17.18 or `MAKA_HARNESS_AB_COMPETITOR=codex` for the official Codex CLI 0.144.6 on `gpt-5.6-sol` with `xhigh` reasoning. The Codex profile uses ChatGPT/Codex OAuth for both arms and records account-plan billing rather than public API spend. The task root must match the 89 task ids and canonical task-tree fingerprint of the frozen official Terminal-Bench 2.1 revision; a matching export with one task directory per id is accepted directly. The A/B task order is frozen independently of Oracle evidence. Maka keeps active and stale tool-result pruning enabled while semantic compact is explicitly disabled in both the manifest and runtime environment. + +`MAKA_HARNESS_AB_BENCHMARK` selects the benchmark axis, a bound pair of frozen task source and executor. The default `terminal-bench-2.1` runs the 89-task Terminal-Bench 2.1 set under plain Harbor as above. `deep-swe-1.1` runs the frozen 30-task DeepSWE subset under the Pier executor (`pier` on `PATH`; the resolved `pier --version` is frozen into the toolchain and resume identity and recorded in the manifest's benchmark section), reads tasks from `~/.maka/eval/task-sources/deep-swe-6db64a40/tasks` by default, grades with each task's own verifier so no Oracle evidence applies, and supports the kimi-code and codex competitors only. `MAKA_HARNESS_AB_LIMIT=5` stays the operational canary for every benchmark; the full DeepSWE profile is `30`. Validate the frozen task source and preview the A/B plan without reading a key or starting Harbor: diff --git a/packages/headless/harbor/run-harness-ab.mjs b/packages/headless/harbor/run-harness-ab.mjs index fc9489afd04..5a121ba6de4 100644 --- a/packages/headless/harbor/run-harness-ab.mjs +++ b/packages/headless/harbor/run-harness-ab.mjs @@ -527,6 +527,7 @@ export function buildHarnessAbManifest({ armExecution = DEFAULT_ARM_EXECUTION, oracleEvidence, credentialIdentity, + pierVersion = null, }) { const runtime = resolveHarnessRuntimeProfile(competitorProfile); const execution = buildHarnessExecutionProfile(competitorProfile); @@ -535,6 +536,10 @@ export function buildHarnessAbManifest({ dataset: benchmarkProfile.dataset, version: benchmarkProfile.version, revision: benchmarkProfile.revision, + // Human-readable executor identity for Pier benchmarks; the same + // version is hashed into the toolchain fingerprint. Absent for Harbor + // benchmarks so Terminal-Bench manifests stay byte-identical. + ...(pierVersion === null ? {} : { executor: { id: 'pier', version: pierVersion } }), timeoutPolicy: 'task-native', timeoutMultiplier: 1, outerTimeoutGraceSec: HARBOR_SETUP_TEARDOWN_GRACE_SEC, @@ -598,6 +603,13 @@ export async function main() { const competitorProfile = resolveHarnessCompetitorProfile( process.env.MAKA_HARNESS_AB_COMPETITOR || 'kimi-code', ); + // Fail before any run root or lock exists: a doomed benchmark × competitor + // pairing must not leave run state behind. + if (benchmarkProfile.executor === 'pier' && competitorProfile.id === 'opencode') { + throw new Error( + 'the OpenCode adapter has no Pier arm; DeepSWE supports the kimi-code and codex competitors', + ); + } const runId = resolveHarnessAbRunId( competitorProfile, process.env.MAKA_HARNESS_AB_RUN_ID, @@ -660,11 +672,6 @@ async function runLocked({ benchmarkProfile, competitorProfile, }) { - if (benchmarkProfile.executor === 'pier' && competitorProfile.id === 'opencode') { - throw new Error( - 'the OpenCode adapter has no Pier arm; DeepSWE supports the kimi-code and codex competitors', - ); - } const { tasks: allTasks, taskSourceFingerprint } = await resolveFrozenBenchmarkTasks( benchmarkProfile, tasksRoot, @@ -718,10 +725,11 @@ async function runLocked({ env: process.env, }); + const pierVersion = benchmarkProfile.executor === 'pier' ? await readPierVersion() : null; const toolchainFingerprint = buildHarnessAbToolchainFingerprint({ hostToolchainFingerprint, competitorProfile, - pierVersion: benchmarkProfile.executor === 'pier' ? await readPierVersion() : null, + pierVersion, }); const manifest = buildHarnessAbManifest({ subjectFingerprint, @@ -734,6 +742,7 @@ async function runLocked({ ...(oracleEvidence ? { oracleEvidence } : {}), competitorProfile, credentialIdentity: credentials.credentialIdentity, + pierVersion, }); await ensureAbRunManifest(manifestPath, manifest); const evaluationTasks = manifest.evaluationTaskIds diff --git a/packages/headless/src/__tests__/harness-ab-cli.test.ts b/packages/headless/src/__tests__/harness-ab-cli.test.ts index a7b3cbd3c93..cc2e0d93e1e 100644 --- a/packages/headless/src/__tests__/harness-ab-cli.test.ts +++ b/packages/headless/src/__tests__/harness-ab-cli.test.ts @@ -784,6 +784,18 @@ test('harness A/B resolves the DeepSWE benchmark axis orthogonally to competitor assert.equal(manifest.metadata.order.seed, 'deep-swe-1.1:gpt-5.6-sol:harness-comparison:v1'); assert.equal(manifest.evaluationTaskIds.length, 30); + // The Pier executor identity is auditable in the manifest, not only hashed + // into the toolchain fingerprint. + const pierManifest = buildHarnessAbManifest({ + subjectFingerprint: 'subject', + taskSourceFingerprint: 'tasks', + toolchainFingerprint: 'tools', + benchmarkProfile, + competitorProfile: resolveHarnessCompetitorProfile('codex'), + pierVersion: '0.3.0', + }); + assert.deepEqual(pierManifest.metadata.benchmark.executor, { id: 'pier', version: '0.3.0' }); + // The Terminal-Bench default keeps its historical order seed byte-for-byte. const tbenchManifest = buildHarnessAbManifest({ subjectFingerprint: 'subject', @@ -791,6 +803,9 @@ test('harness A/B resolves the DeepSWE benchmark axis orthogonally to competitor toolchainFingerprint: 'tools', }); assert.equal(tbenchManifest.metadata.order.seed, 'terminal-bench-2.1:k3:harness-comparison:v1'); + // No executor key for Harbor benchmarks: the manifest payload (and with it + // the resume fingerprint) must stay byte-identical to historical runs. + assert.equal('executor' in tbenchManifest.metadata.benchmark, false); }); test('harness A/B benchmark profiles bind their executor and resolve from env', async () => { diff --git a/packages/headless/src/harness-ab-manifest.ts b/packages/headless/src/harness-ab-manifest.ts index 86c66e599a9..a5c6b17985f 100644 --- a/packages/headless/src/harness-ab-manifest.ts +++ b/packages/headless/src/harness-ab-manifest.ts @@ -216,6 +216,10 @@ export interface HarnessAbRunManifestInput { dataset: 'terminal-bench' | 'deep-swe'; version: '2.1' | '1.1'; revision: string; + /** Present only for Pier benchmarks: the frozen executor identity whose + * version the toolchain fingerprint also hashes. Absent for Harbor + * benchmarks so existing Terminal-Bench manifests stay byte-identical. */ + executor?: { id: 'pier'; version: string }; timeoutPolicy: 'task-native'; timeoutMultiplier: 1; outerTimeoutGraceSec: number;