Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions packages/headless/harbor/opencode_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ def _write_cell_output(self, context: AgentContext) -> None:
started_at = getattr(self, "_started_at_ms", int(time.time() * 1000))
finished_at = getattr(self, "_finished_at_ms", started_at)
execution_identity = self._execution_identity()
prompt_hash = execution_identity["systemPromptHash"]
prompt_hash = self._round_prompt_hash()
events = self._parse_stdout() if hasattr(self, "_parse_stdout") else []
tool_call_counts: dict[str, int] = {}
for event in events or []:
Expand Down Expand Up @@ -421,21 +421,25 @@ def _write_cell_output(self, context: AgentContext) -> None:

def _execution_identity(self) -> dict[str, Any]:
provider, model = self.model_name.split("/", 1)
system_prompt = self._get_env("MAKA_SYSTEM_PROMPT") or ""
prompt_hash = "sha256:" + hashlib.sha256(
json.dumps(system_prompt, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
).hexdigest()
pricing_profile = self._get_env("MAKA_TRIAL_PRICING_SOURCE") or "unconfigured"
reasoning_effort = self._get_env("MAKA_REASONING_EFFORT")
toolchain = self._get_env("MAKA_OPENCODE_TOOLCHAIN_FINGERPRINT") or "unpinned"
return {
"llmConnectionSlug": self._get_env("MAKA_LLM_CONNECTION_SLUG") or provider,
"model": model,
**({"reasoningEffort": reasoning_effort} if reasoning_effort else {}),
"systemPromptHash": prompt_hash,
"systemPromptAuthority": "harness-native",
"systemPromptToolchain": toolchain,
"pricingProfile": pricing_profile,
"agentTools": False,
}

def _round_prompt_hash(self) -> str:
system_prompt = self._get_env("MAKA_SYSTEM_PROMPT") or ""
return "sha256:" + hashlib.sha256(
json.dumps(system_prompt, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
).hexdigest()

def _write_execution_identity(self) -> None:
self.logs_dir.mkdir(parents=True, exist_ok=True)
path = self.logs_dir / "maka-cell-execution-identity.json"
Expand Down
64 changes: 64 additions & 0 deletions packages/headless/src/__tests__/cell-output.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,70 @@ describe('Harbor cell output contract', () => {
);
});

test('records a harness-owned prompt against its pinned build instead of a Maka hash', () => {
assert.deepEqual(
validateHarborCellExecutionIdentity({
llmConnectionSlug: 'zai-coding-plan',
model: 'glm-5.2',
systemPromptAuthority: 'harness-native',
systemPromptToolchain: 'sha256:opencode-toolchain',
pricingProfile: 'zai-public',
agentTools: false,
}),
{
llmConnectionSlug: 'zai-coding-plan',
model: 'glm-5.2',
systemPromptAuthority: 'harness-native',
systemPromptToolchain: 'sha256:opencode-toolchain',
pricingProfile: 'zai-public',
agentTools: false,
},
);
});

test('rejects a harness-owned prompt identity that also claims a Maka prompt hash', () => {
assert.throws(
() =>
validateHarborCellExecutionIdentity({
llmConnectionSlug: 'zai-coding-plan',
model: 'glm-5.2',
systemPromptAuthority: 'harness-native',
systemPromptHash: 'sha256:prompt-a',
systemPromptToolchain: 'sha256:opencode-toolchain',
pricingProfile: 'zai-public',
agentTools: false,
}),
/systemPromptHash must be absent when the harness owns the system prompt/,
);
});

test('rejects a harness-owned prompt identity without a pinned build', () => {
assert.throws(
() =>
validateHarborCellExecutionIdentity({
llmConnectionSlug: 'zai-coding-plan',
model: 'glm-5.2',
systemPromptAuthority: 'harness-native',
pricingProfile: 'zai-public',
agentTools: false,
}),
/systemPromptToolchain must pin the harness build/,
);
});

test('keeps decoding a legacy identity that omits the prompt authority', () => {
const identity = validateHarborCellExecutionIdentity({
llmConnectionSlug: 'deepseek',
model: 'deepseek-v4-flash',
systemPromptHash: 'sha256:prompt-a',
pricingProfile: 'public',
agentTools: false,
});

assert.equal(identity.systemPromptAuthority, undefined);
assert.equal(identity.systemPromptHash, 'sha256:prompt-a');
});

test('summarizes runtime outcome, prompt hash, token cost, and event path', () => {
const events: RuntimeEvent[] = [
runtimeEvent({ id: 'user-event' }),
Expand Down
32 changes: 32 additions & 0 deletions packages/headless/src/__tests__/fixed-prompt-controller.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,38 @@ describe('fixed prompt controller', () => {
});
});

test('accepts an execution identity whose prompt is owned by the competitor harness', async () => {
await withDir(async (dir) => {
const systemPromptPath = join(dir, 'system_prompt.md');
await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8');
const output = harborOutput({
taskId: 'task-a',
executionIdentity: {
llmConnectionSlug: 'fake',
model: 'fake-model',
systemPromptAuthority: 'harness-native',
systemPromptToolchain: 'sha256:opencode-toolchain',
pricingProfile: 'test-profile',
},
});

const result = await runFixedPromptController({
runId: 'run-1',
roundId: 'round-1',
config,
systemPromptPath,
resultsJsonlPath: join(dir, 'results.jsonl'),
resultsTsvPath: join(dir, 'results.tsv'),
tasks: [{ id: 'task-a', path: '/bench/task-a' }],
requireExecutionIdentity: true,
expectedPricingProfile: 'test-profile',
taskRunner: async () => output,
});

assert.equal(result.events[0]?.type, 'task_completed');
});
});

test('resumes from completed task events in the WAL', async () => {
await withDir(async (dir) => {
const systemPromptPath = join(dir, 'system_prompt.md');
Expand Down
14 changes: 14 additions & 0 deletions packages/headless/src/__tests__/harbor-adapter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3927,6 +3927,9 @@ try:
assert cell["executionIdentity"]["model"] == "glm-5.2", cell
assert cell["executionIdentity"]["reasoningEffort"] == "max", cell
assert cell["executionIdentity"]["agentTools"] is False, cell
assert cell["executionIdentity"]["systemPromptAuthority"] == "harness-native", cell
assert cell["executionIdentity"]["systemPromptToolchain"] == "sha256:" + "a" * 64, cell
assert "systemPromptHash" not in cell["executionIdentity"], cell
assert cell["tokenSummary"]["cachedInput"] == 40, cell
assert cell["tokenSummary"]["input"] == 110, cell
assert cell["tokenSummary"]["output"] == 25, cell
Expand All @@ -3937,6 +3940,17 @@ try:
assert cell["toolSummary"]["actualToolCallCounts"] == {"bash": 1}, cell
assert "test-zai-key" not in json.dumps(cell), cell

def prompt_identity(system_prompt):
return MakaOpenCodeAgent(Path(tmp), extra_env={
"MAKA_LLM_CONNECTION_SLUG": "zai-coding-plan",
"MAKA_SYSTEM_PROMPT": system_prompt,
"MAKA_OPENCODE_TOOLCHAIN_FINGERPRINT": "sha256:" + "a" * 64,
}, model_name="zai-coding-plan/glm-5.2")._execution_identity()

first_prompt_identity = prompt_identity("benchmark prompt one")
second_prompt_identity = prompt_identity("benchmark prompt two")
assert first_prompt_identity == second_prompt_identity, first_prompt_identity

missing_usage_agent = MakaOpenCodeAgent(Path(tmp), extra_env={
"MAKA_SYSTEM_PROMPT": "",
"MAKA_TRIAL_PRICING_SOURCE": "xiaomi-env",
Expand Down
57 changes: 55 additions & 2 deletions packages/headless/src/cell-output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,13 +137,21 @@ export interface HarborCellRuntimeRefs {
turnId: string;
}

/** Who owns the model-visible system prompt: Maka, or the competitor harness itself. */
export type HarborCellSystemPromptAuthority = 'maka' | 'harness-native';

export interface HarborCellExecutionIdentity {
llmConnectionSlug: string;
model: string;
reasoningEffort?: ThinkingLevel;
/** Present on artifacts written after prompt-source auditing was introduced. */
systemPromptMode?: HeadlessSystemPromptMode;
systemPromptHash: string;
/** Absent on artifacts written before harness-owned prompts were recorded, which are Maka-injected. */
systemPromptAuthority?: HarborCellSystemPromptAuthority;
/** SHA-256 of the Maka-injected prompt; absent when the harness owns the prompt. */
systemPromptHash?: string;
/** Pinned harness build that owns a native prompt Maka cannot read. */
systemPromptToolchain?: string;
pricingProfile: string;
/** Present on artifacts written after Headless Agent tool gating was introduced. */
agentTools?: boolean;
Expand Down Expand Up @@ -396,7 +404,31 @@ export function validateHarborCellExecutionIdentity(value: unknown): HarborCellE
),
}
: {}),
systemPromptHash: requireString(value.systemPromptHash, 'executionIdentity.systemPromptHash'),
...('systemPromptAuthority' in value
? {
systemPromptAuthority: requireStringUnion(
value.systemPromptAuthority,
'executionIdentity.systemPromptAuthority',
['maka', 'harness-native'] as const,
),
}
: {}),
...('systemPromptHash' in value
? {
systemPromptHash: requireString(
value.systemPromptHash,
'executionIdentity.systemPromptHash',
),
}
: {}),
...('systemPromptToolchain' in value
? {
systemPromptToolchain: requireString(
value.systemPromptToolchain,
'executionIdentity.systemPromptToolchain',
),
}
: {}),
pricingProfile: requireString(value.pricingProfile, 'executionIdentity.pricingProfile'),
...('agentTools' in value
? { agentTools: requireBoolean(value.agentTools, 'executionIdentity.agentTools') }
Expand All @@ -408,6 +440,27 @@ export function validateHarborCellExecutionIdentity(value: unknown): HarborCellE
? { supplementalToolSets: validateSupplementalToolSets(value.supplementalToolSets) }
: {}),
};
if (identity.systemPromptAuthority === 'harness-native') {
if (identity.systemPromptHash !== undefined) {
throw new Error(
'executionIdentity.systemPromptHash must be absent when the harness owns the system prompt',
);
}
if (identity.systemPromptToolchain === undefined) {
throw new Error(
'executionIdentity.systemPromptToolchain must pin the harness build that owns the system prompt',
);
}
} else {
if (identity.systemPromptHash === undefined) {
throw new Error('executionIdentity.systemPromptHash must be a non-empty string');
}
if (identity.systemPromptToolchain !== undefined) {
throw new Error(
'executionIdentity.systemPromptToolchain must be absent when Maka owns the system prompt',
);
}
}
if (
identity.productToolSurface &&
identity.agentTools !== undefined &&
Expand Down
3 changes: 2 additions & 1 deletion packages/headless/src/fixed-prompt-controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -932,7 +932,8 @@ function classifyExecutionIdentityFailure(
? !identity.productToolSurface.policy.disabledSurfaceIds.includes('agent')
: identity.agentTools) !==
(expectedConfig.agentTools === true) ||
identity.systemPromptHash !== expectedPromptHash ||
(identity.systemPromptAuthority !== 'harness-native' &&
identity.systemPromptHash !== expectedPromptHash) ||
(expectedPricingProfile !== undefined && identity.pricingProfile !== expectedPricingProfile)
) {
return {
Expand Down
Loading