From 1cce9d93047a59e10b18768bb0ea1c961092ea3d Mon Sep 17 00:00:00 2001 From: donpetry-bot <{}+donpetry-bot@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:57:48 +0000 Subject: [PATCH 01/29] =?UTF-8?q?feat:=20implement=20issue=20#1973=20?= =?UTF-8?q?=E2=80=94=20[Engine=20config=20=C2=B7=20Phase=201]=20Versioned?= =?UTF-8?q?=20config/ai-engines.json=20as=20the=20single=20source=20of=20t?= =?UTF-8?q?ruth=20for=20providers,=20model=20catalog=20and=20per-task=20ch?= =?UTF-8?q?ains?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/daily-pr-review-health.yml | 5 + .github/workflows/dev-lead-reusable.yml | 4 + .github/workflows/lint.yml | 25 ++ .github/workflows/test-dev-lead.yml | 2 + config/ai-engines.json | 116 +++++++++ config/ai-engines.schema.json | 109 +++++++++ docs/engine-configuration.md | 133 +++++++++-- scripts/engine.sh | 71 +++--- scripts/lib/consumer-manifest.json | 2 + scripts/lib/downstream-impact.sh | 2 +- scripts/lib/engine-chain.sh | 72 ++++-- scripts/lib/engine-models.sh | 225 +++++++++++++----- scripts/lib/validate-consumer-manifest.sh | 18 +- scripts/pr_review_health.sh | 22 ++ scripts/validate-ai-engines.py | 136 +++++++++++ .../dev-lead/unit/test_ai_engines_config.bats | 195 +++++++++++++++ tests/test_batch_fallback.bats | 3 + tests/test_consumer_manifest.bats | 15 ++ tests/test_validate_ai_engines.bats | 110 +++++++++ 19 files changed, 1117 insertions(+), 148 deletions(-) create mode 100644 config/ai-engines.json create mode 100644 config/ai-engines.schema.json create mode 100755 scripts/validate-ai-engines.py create mode 100644 tests/dev-lead/unit/test_ai_engines_config.bats create mode 100644 tests/test_validate_ai_engines.bats diff --git a/.github/workflows/daily-pr-review-health.yml b/.github/workflows/daily-pr-review-health.yml index beddb7137..1407ee0e3 100644 --- a/.github/workflows/daily-pr-review-health.yml +++ b/.github/workflows/daily-pr-review-health.yml @@ -105,6 +105,11 @@ jobs: # Export the model override variable so ai_model_for_family honours # operator-configured model chains (vars.AI_MODELS_CLAUDE). AI_MODELS_CLAUDE: ${{ vars.AI_MODELS_CLAUDE }} + # The other break-glass overrides, so the report lists every one that + # differs from config/ai-engines.json (#1973). + AI_MODELS_GEMINI: ${{ vars.AI_MODELS_GEMINI }} + AI_MODELS_COPILOT: ${{ vars.AI_MODELS_COPILOT }} + AI_ENGINES: ${{ vars.AI_ENGINES }} run: bash scripts/pr_review_health.sh - name: Truncate report if needed diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index 8df23e6ef..35fe31a70 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -268,8 +268,10 @@ jobs: # Cross-repo callers: github.token cannot read a private org repo — ensure # GH_PAT_WORKFLOWS is set as an org or repo secret in the calling repo. token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + # config/ holds ai-engines.json, the engine defaults the scripts read (#1973). sparse-checkout: | scripts + config prompts/dev-lead personas/dev-lead @@ -1028,8 +1030,10 @@ jobs: path: .dev-lead token: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} persist-credentials: false + # config/ holds ai-engines.json, the engine defaults the scripts read (#1973). sparse-checkout: | scripts + config - name: Resolve PR from check_run id: relay diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3a34195b9..efd06b35d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,6 +15,7 @@ on: - 'interaction-contracts/**' - 'evals/**' - 'release/**' + - 'config/**' # config/ai-engines.json + schema (validate-ai-engines) - '.gitleaks.toml' # Post-merge coverage parity with `unit` (#1486, epic #1402): re-run the bats # suite against actual trunk state after every merge so shell-script behavioral @@ -30,6 +31,7 @@ on: - 'prompts/**' - '.github/workflows/**' - '.github/**' # check-model-pins scans all of .github + - 'config/**' workflow_dispatch: permissions: @@ -212,6 +214,7 @@ jobs: tests/test_discover_parity_skills.bats \ tests/test_eval_health_notify.bats \ tests/test_consumer_manifest.bats \ + tests/test_validate_ai_engines.bats \ tests/test_downstream_impact_map.bats \ tests/test_downstream_impact_assemble.bats \ tests/test_downstream_impact_triage_inline.bats \ @@ -537,6 +540,28 @@ jobs: - name: Validate per-case eval schema conformance across the tree run: python3 evals/validate-cases.py --schema-tree evals + validate-ai-engines: + # The engine config gate (#1973). config/ai-engines.json is the source of + # truth for enabled providers, the model catalog and each task's chain, so a + # bad value is caught at PR time instead of in a failing review. Validates + # the file against config/ai-engines.schema.json (draft 2020-12, the same + # jsonschema setup as validate-eval-cases), then fails on a task model that is + # missing from `models` or `retired`, a model in another provider's chain, an + # enabled provider's model with no row in scripts/lib/model-pricing.tsv, and a + # duck or Copilot chain with more than one model. Fully HERMETIC. + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install Python deps + run: pip3 install --quiet 'jsonschema>=4' + + - name: Validate config/ai-engines.json + run: python3 scripts/validate-ai-engines.py + validate-interaction-contracts: # Interaction-contract well-formedness gate (Story 2 / #1404, AC #5). Parses # every machine-readable interaction contract — the co-located persona lenses diff --git a/.github/workflows/test-dev-lead.yml b/.github/workflows/test-dev-lead.yml index 7b9ca6008..23465d883 100644 --- a/.github/workflows/test-dev-lead.yml +++ b/.github/workflows/test-dev-lead.yml @@ -6,6 +6,7 @@ on: - 'scripts/auto-rebase-retry.sh' - 'scripts/engine.sh' - 'scripts/lib/**' + - 'config/**' - '.github/workflows/dev-lead*.yml' - '.github/workflows/test-dev-lead.yml' - '.github/workflows/issue-triage-runner.yml' @@ -28,6 +29,7 @@ on: - 'scripts/auto-rebase-retry.sh' - 'scripts/engine.sh' - 'scripts/lib/**' + - 'config/**' - 'tests/dev-lead/**' - 'prompts/dev-lead/**' - '.github/review-mcp.json' diff --git a/config/ai-engines.json b/config/ai-engines.json new file mode 100644 index 000000000..238edd8f0 --- /dev/null +++ b/config/ai-engines.json @@ -0,0 +1,116 @@ +{ + "$schema": "./ai-engines.schema.json", + "$comment": "Source of truth for the review and dev-lead engines (#1973): which providers are enabled, which models exist, and each task's chain. Read by scripts/lib/engine-models.sh; validated by scripts/validate-ai-engines.py in lint.yml. Ships with the pr-review/* and dev-lead/* channel tags, so a change here canaries on next before ring0 and stable. AI_MODELS_* and AI_ENGINES override it in an emergency — fold the override back in here. See docs/engine-configuration.md.", + "version": 1, + "models": { + "claude-haiku-4-5-20251001": { + "provider": "claude", + "status": "active", + "notes": ["Triage primary; the haiku family's current id (ai_model_for_family haiku)."] + }, + "claude-sonnet-5-5": { + "provider": "claude", + "status": "active", + "notes": [ + "Default sonnet across triage, deep, action and duck (#1978, epic #1095 Phase 3).", + "In deep it is the last hop: deep's known-good hop is opus-4-8, so sonnet 5.5 alone is its sonnet tail (#1978 AC-2)." + ] + }, + "claude-sonnet-5": { + "provider": "claude", + "status": "active", + "notes": [ + "Known-good sonnet safety hop one behind sonnet 5.5 in triage and action.", + "The id is bare claude-sonnet-5: the claude-sonnet-5-0 spelling shipped by #1100 404s, so every hop to it silently fell through (#1957)." + ] + }, + "claude-opus-5-5": { + "provider": "claude", + "status": "active", + "notes": [ + "Deep primary since #1898 (epic #1895 Phase 2); audit and single primary since #1901 (epic #1895 Phase 3), replacing Fable 5.", + "Action's last (cross-family) hop since #1900 (epic #1895 Phase 3), replacing opus-4-8 once the deep-tier canary passed (#1899).", + "CLI/API default reasoning effort is medium (low/medium/high/xhigh/max); engine.sh's run_agentic pins --effort high for deep, audit and single so moving off Fable does not lower their effort." + ] + }, + "claude-opus-4-8": { + "provider": "claude", + "status": "active", + "notes": [ + "Known-good second hop behind opus-5-5 in deep, audit and single (#1957), so a throttled opus-5-5 degrades to the model it replaced.", + "Minimum cacheable prefix 4096 tokens." + ] + }, + "claude-opus-4-7": { + "provider": "claude", + "status": "active", + "notes": ["Last hop of the audit and single chains."] + }, + "claude-fable-5": { + "provider": "claude", + "status": "retired", + "notes": [ + "Deprecated by #1901 (epic #1895 Phase 3): opus-5-5 outperforms it at lower cost, so no default chain names it.", + "Reasoning effort: the deep, audit and single tiers pin --effort high (engine.sh run_agentic) so replacing Fable with opus-5-5, whose default effort is medium, did not lower their effort.", + "A chain set through AI_MODELS_CLAUDE or CLAUDE__MODEL_CHAIN is still honoured as a stop-gap and warns once per run; its price rows stay for historical token records." + ] + }, + "gemini-3.8-flash": { + "provider": "gemini", + "status": "active", + "notes": ["Triage and action primary; the known-working fallback of the quality tier."] + }, + "gemini-3.1-pro-preview": { + "provider": "gemini", + "status": "preview", + "notes": ["Quality tier (deep, audit, single) since gemini-2.5-pro was withdrawn for new keys (#1960)."] + }, + "gemini-2.5-pro": { + "provider": "gemini", + "status": "retired", + "notes": ["\"No longer available to new users … use models/gemini-3.1-pro-preview\": it broke every Gemini deep review while it sat in code (#1960)."] + }, + "openai/o4-mini": { + "provider": "copilot", + "status": "active", + "notes": ["GitHub Models id. Copilot has no in-engine chain, so each task names one model."] + } + }, + "providers": { + "fallback_order": ["claude", "gemini", "copilot"], + "claude": { "enabled": true }, + "gemini": { "enabled": true }, + "copilot": { "enabled": true } + }, + "tasks": { + "triage": { + "claude": ["claude-haiku-4-5-20251001", "claude-sonnet-5-5", "claude-sonnet-5"], + "gemini": ["gemini-3.8-flash", "gemini-3.1-pro-preview"], + "copilot": ["openai/o4-mini"] + }, + "deep": { + "claude": ["claude-opus-5-5", "claude-opus-4-8", "claude-sonnet-5-5"], + "gemini": ["gemini-3.1-pro-preview", "gemini-3.8-flash"], + "copilot": ["openai/o4-mini"] + }, + "audit": { + "claude": ["claude-opus-5-5", "claude-opus-4-8", "claude-opus-4-7"], + "gemini": ["gemini-3.1-pro-preview", "gemini-3.8-flash"], + "copilot": ["openai/o4-mini"] + }, + "action": { + "claude": ["claude-sonnet-5-5", "claude-sonnet-5", "claude-opus-5-5"], + "gemini": ["gemini-3.8-flash", "gemini-3.1-pro-preview"], + "copilot": ["openai/o4-mini"] + }, + "single": { + "claude": ["claude-opus-5-5", "claude-opus-4-8", "claude-opus-4-7"], + "gemini": ["gemini-3.1-pro-preview", "gemini-3.8-flash"], + "copilot": ["openai/o4-mini"] + }, + "duck": { + "$comment": "Gemini and Copilot have no duck chain: their duck is the first model of their triage chain.", + "claude": ["claude-sonnet-5-5"] + } + } +} diff --git a/config/ai-engines.schema.json b/config/ai-engines.schema.json new file mode 100644 index 000000000..a118deab7 --- /dev/null +++ b/config/ai-engines.schema.json @@ -0,0 +1,109 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/petry-projects/.github-private/config/ai-engines.schema.json", + "title": "AI engine configuration", + "description": "Providers, model catalog and per-task model chains for the review and dev-lead engines (#1973). Cross-reference checks (catalog membership, retired models, provider match, price rows, one-model chains) live in scripts/validate-ai-engines.py.", + "type": "object", + "required": ["version", "models", "providers", "tasks"], + "additionalProperties": false, + "properties": { + "$schema": { "type": "string" }, + "$comment": { "type": "string" }, + "version": { "const": 1 }, + "models": { + "type": "object", + "minProperties": 1, + "propertyNames": { "$ref": "#/$defs/modelId" }, + "additionalProperties": { "$ref": "#/$defs/model" } + }, + "providers": { + "type": "object", + "required": ["fallback_order", "claude", "gemini", "copilot"], + "additionalProperties": false, + "properties": { + "$comment": { "type": "string" }, + "fallback_order": { + "description": "Default cross-provider fallback order; the first enabled entry is the primary. AI_ENGINES overrides it.", + "type": "array", + "items": { "$ref": "#/$defs/providerName" }, + "minItems": 3, + "maxItems": 3, + "uniqueItems": true + }, + "claude": { "$ref": "#/$defs/provider" }, + "gemini": { "$ref": "#/$defs/provider" }, + "copilot": { "$ref": "#/$defs/provider" } + } + }, + "tasks": { + "type": "object", + "required": ["triage", "deep", "audit", "action", "single", "duck"], + "additionalProperties": false, + "properties": { + "$comment": { "type": "string" }, + "triage": { "$ref": "#/$defs/task" }, + "deep": { "$ref": "#/$defs/task" }, + "audit": { "$ref": "#/$defs/task" }, + "action": { "$ref": "#/$defs/task" }, + "single": { "$ref": "#/$defs/task" }, + "duck": { "$ref": "#/$defs/task" } + } + } + }, + "$defs": { + "providerName": { "enum": ["claude", "gemini", "copilot"] }, + "modelId": { + "description": "Same id rule as AI_MODELS_* entries (scripts/lib/engine-models.sh).", + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:/@-]*$" + }, + "notes": { + "oneOf": [ + { "type": "string", "minLength": 1 }, + { "type": "array", "items": { "type": "string", "minLength": 1 } } + ] + }, + "model": { + "type": "object", + "required": ["provider", "status"], + "additionalProperties": false, + "properties": { + "provider": { "$ref": "#/$defs/providerName" }, + "status": { "enum": ["active", "preview", "retired"] }, + "notes": { "$ref": "#/$defs/notes" } + } + }, + "provider": { + "type": "object", + "required": ["enabled"], + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" }, + "notes": { "$ref": "#/$defs/notes" } + } + }, + "chain": { + "description": "Ordered model chain: primary first, then fallbacks.", + "type": "array", + "items": { "$ref": "#/$defs/modelId" }, + "minItems": 1, + "uniqueItems": true + }, + "task": { + "type": "object", + "additionalProperties": false, + "properties": { + "$comment": { "type": "string" }, + "claude": { "$ref": "#/$defs/chain" }, + "gemini": { "$ref": "#/$defs/chain" }, + "copilot": { "$ref": "#/$defs/chain" }, + "prefer": { + "description": "Reserved for per-task provider preference (Phase 2); not read yet.", + "type": "array", + "items": { "$ref": "#/$defs/providerName" }, + "uniqueItems": true + } + } + } + } +} diff --git a/docs/engine-configuration.md b/docs/engine-configuration.md index 0aee4f017..a7d097b5c 100644 --- a/docs/engine-configuration.md +++ b/docs/engine-configuration.md @@ -1,24 +1,100 @@ -# Engine configuration (AI_ENGINES) +# Engine configuration (config/ai-engines.json, AI_ENGINES) pr-review and dev-lead can run on three LLM engines: **Claude**, **Gemini** and -**Copilot**. Which engines are used, and in what order, is set with GitHub -Actions **variables** (org or repo level). Switching providers needs no code -change and no release: the next run picks the new value up. - -The parsing lives in [`scripts/lib/engine-chain.sh`](../scripts/lib/engine-chain.sh). +**Copilot**. Which engines are enabled, which models exist and which models each +task uses are set in one versioned file, +[`config/ai-engines.json`](../config/ai-engines.json) (#1973). A small set of +GitHub Actions **variables** stays as a documented emergency override. + +The engine chain is parsed in +[`scripts/lib/engine-chain.sh`](../scripts/lib/engine-chain.sh), the model lists +in [`scripts/lib/engine-models.sh`](../scripts/lib/engine-models.sh). + +## The config file + +`config/ai-engines.json` is the source of truth. It has three sections: + +- **`models`** — the catalog, keyed by model id. Each entry gives its + `provider` (`claude`, `gemini` or `copilot`), its `status` (`active`, + `preview` or `retired`) and optional `notes`, such as why a chain has its + fallbacks, the Sonnet 5 id history and the Fable 5 deprecation. +- **`providers`** — `claude`, `gemini` and `copilot`, each with `enabled`, and + `fallback_order`, the default order across providers (its first enabled + entry is the primary). +- **`tasks`** — `triage`, `deep`, `audit`, `action`, `single` and `duck`. Each + gives, per provider, an ordered chain: primary first, then fallbacks. + `tasks..prefer` (a provider list) is accepted for per-task provider + preference but not read yet. + +The file is read with `jq`, resolved relative to `engine-models.sh` (so it is +found wherever the scripts are checked out). A missing, unreadable or malformed +file is an `::error::` and fails the step: there are no built-in defaults to fall +back to, because stale defaults are what this file replaces. `engine.sh` reads it +once when it is sourced. + +**It ships like code.** The file is released with the `pr-review/*` and +`dev-lead/*` channel tags, so a default-model change canaries on `next` before +`ring0` and `stable`. An Actions variable changes every channel at once, which is +why variables are only the emergency path. + +**It is checked at PR time.** The `validate-ai-engines` job in `lint.yml` runs +[`scripts/validate-ai-engines.py`](../scripts/validate-ai-engines.py). It +validates the file against +[`config/ai-engines.schema.json`](../config/ai-engines.schema.json) (JSON Schema +draft 2020-12), and also fails when: + +- a task names a model that is missing from `models`, or is `retired`; +- a model sits in another provider's chain; +- a model of an enabled provider has no price row in + [`scripts/lib/model-pricing.tsv`](../scripts/lib/model-pricing.tsv) (a + vendor-prefixed id such as `openai/o4-mini` is priced by its bare name); +- `duck`, or any `copilot` chain, lists more than one model. + +Run it locally with `python3 scripts/validate-ai-engines.py` (needs +`pip install 'jsonschema>=4'`). + +## Precedence + +For each task's chain, highest first: + +1. **A specific environment variable** — `CLAUDE__MODEL_CHAIN`, + `GEMINI_FLASH_MODEL(_CHAIN)`, `GEMINI_PRO_MODEL(_CHAIN)`, `COPILOT_API_MODEL`. + These are **internal**: they are kept for tests and the A/B runner. Do not + set them as Actions variables; use `AI_MODELS_*`. +2. **`AI_MODELS_*`** — the break-glass override (below). +3. **`config/ai-engines.json`**. + +For providers, the file decides which are enabled. `AI_ENGINES` is the kill +switch and the fallback order: it can disable or reorder providers the file +enables, but it **cannot enable** one the file disables (that is ignored with a +`::warning::`). + +## Emergency path + +When a model breaks in production (retired, renamed, throttled): + +1. **Now:** set the `AI_MODELS_` variable (or `AI_ENGINES` to turn a + provider off). The next run picks it up on every channel. +2. **Then:** land the same change in `config/ai-engines.json` through a normal + PR and release, and delete the variable once `stable` carries it. + +The daily PR-review health check (`scripts/pr_review_health.sh`) lists, under +**Engine configuration overrides**, every `AI_MODELS_*` key and `AI_ENGINES` +value that is set and differs from the file, so a temporary override is folded +back instead of living on silently. ## Variables | Variable | Default | Meaning | |---|---|---| -| `AI_ENGINES` | `claude,gemini,copilot` | Ordered list of **enabled** engines. An engine that is not listed is never used, not even as a fallback. The order is the fallback order, and the **first** entry is the primary engine. | +| `AI_ENGINES` | the file's `providers.fallback_order` (`claude,gemini,copilot`) | Kill switch and fallback order. An engine that is not listed is never used, not even as a fallback. The order is the fallback order, and the **first** entry is the primary engine. It cannot enable an engine the file disables. | | `AI_DUCK_ENGINE` | automatic | Ordered list of engines for pr-review's rubber-duck second opinion, e.g. `gemini,claude`. Each entry is `claude`, `gemini` or `copilot`. If an engine returns no verdict, the next one runs. `none` alone turns the duck off. | | `AI_DUCK_MODEL` | engine default | Model id for the duck. Used only on the **first** engine in `AI_DUCK_ENGINE`. | -| `AI_MODELS_CLAUDE` | see below | Claude's model list: one chain per task. | -| `AI_MODELS_GEMINI` | see below | Gemini's model list, with the same keys. | -| `AI_MODELS_COPILOT` | see below | Copilot's model list, with the same keys (one model per key). | -| `GEMINI_FLASH_MODEL` | unset | Replaces only the first model of the Gemini `triage` and `action` chains. | -| `GEMINI_PRO_MODEL` | unset | Replaces only the first model of the Gemini `deep`, `audit` and `single` chains. | +| `AI_MODELS_CLAUDE` | unset (the file) | **Break-glass** override of Claude's chains, one per task. | +| `AI_MODELS_GEMINI` | unset (the file) | **Break-glass** override of Gemini's chains, with the same keys. | +| `AI_MODELS_COPILOT` | unset (the file) | **Break-glass** override of Copilot's models, with the same keys (one model per key). | +| `GEMINI_FLASH_MODEL` | unset | **Deprecated** (internal). Replaces only the first model of the Gemini `triage` and `action` chains. | +| `GEMINI_PRO_MODEL` | unset | **Deprecated** (internal). Replaces only the first model of the Gemini `deep`, `audit` and `single` chains. | | `REVIEW_ENGINE` | unset | **Legacy** primary override for pr-review. When set, it wins over the first `AI_ENGINES` entry, as long as `AI_ENGINES` enables it. Delete it to let `AI_ENGINES` decide. | | `DEV_LEAD_ENGINE` | unset | **Legacy** primary override for dev-lead; same rule as `REVIEW_ENGINE`. | | `DEV_LEAD_ENGINES` | unset | **Legacy** name for `AI_ENGINES` (dev-lead, #1546). Read only when `AI_ENGINES` is unset. | @@ -27,15 +103,15 @@ The parsing lives in [`scripts/lib/engine-chain.sh`](../scripts/lib/engine-chain `claude,gemini`. Start the value with an engine name (no leading space): the workflows read the primary from the start of the value. -## Model lists (AI_MODELS_*) +## Break-glass model lists (AI_MODELS_*) -Each provider's models live in one variable, so a retired or renamed model is a -variable edit, not a code change. The parsing and the defaults are in -[`scripts/lib/engine-models.sh`](../scripts/lib/engine-models.sh). +Each provider has one variable that overrides its chains from the file, so a +broken model can be replaced at once, before the file change is released. The +parsing is in [`scripts/lib/engine-models.sh`](../scripts/lib/engine-models.sh). Each entry is `=[,,…]`. Separate entries with `;` or new lines. Keys are case-insensitive and spaces around names are ignored. A key you -leave out keeps its default. A chain is walked left to right on a rate limit, +leave out keeps the file's chain. A chain is walked left to right on a rate limit, before the next engine in `AI_ENGINES` is tried. Every provider takes the same six keys, one per task: @@ -53,9 +129,10 @@ Every provider takes the same six keys, one per task: because its GitHub Models client has no chain. A key limited to one model that is given several warns and uses the first. -Defaults: +Defaults (from `config/ai-engines.json` at the time of writing — the file is +authoritative): -| Key | `AI_MODELS_CLAUDE` | `AI_MODELS_GEMINI` | `AI_MODELS_COPILOT` | +| Key | Claude | Gemini | Copilot | |---|---|---|---| | `triage` | `claude-haiku-4-5-20251001,claude-sonnet-5-5,claude-sonnet-5` | `gemini-3.8-flash,gemini-3.1-pro-preview` | `openai/o4-mini` | | `deep` | `claude-opus-5-5,claude-opus-4-8,claude-sonnet-5-5` | `gemini-3.1-pro-preview,gemini-3.8-flash` | `openai/o4-mini` | @@ -84,7 +161,8 @@ AI_MODELS_COPILOT = deep=openai/gpt-5; duck=openai/gpt-5-mini ``` A more specific variable still wins over `AI_MODELS_*`, so existing overrides -keep working: +keep working. These are **internal / deprecated** — kept for tests and the A/B +runner (`model-ab.yml`), not for operators: - `CLAUDE__MODEL_CHAIN` replaces that Claude chain. - `GEMINI_FLASH_MODEL_CHAIN` replaces the Gemini `triage` and `action` chains. @@ -118,9 +196,9 @@ the chains stay the single source of truth: | `haiku` | `triage` | Overrides are honoured with the same precedence engine.sh uses: the per-tier -`CLAUDE__MODEL_CHAIN` env first, then `AI_MODELS_CLAUDE`, then the built-in -default. An operator who supplies a concrete `claude-*` id still has it honoured -verbatim. In gh-aw front-matter the model family is named in `models:` (e.g. +`CLAUDE__MODEL_CHAIN` env first, then `AI_MODELS_CLAUDE`, then the chain +in `config/ai-engines.json`. An operator who supplies a concrete `claude-*` id +still has it honoured verbatim. In gh-aw front-matter the model family is named in `models:` (e.g. `models: { claude: [sonnet] }`), which accepts both a family and a full id; `engine:` selects the runner engine (`claude`), not the model. Workflow `model` inputs likewise accept a family or a full id. @@ -146,8 +224,10 @@ table are allow-listed. | Duck on Gemini, then Claude if Gemini is throttled | `AI_DUCK_ENGINE=gemini,claude` | | Duck on Gemini only, no automatic fallback | `AI_DUCK_ENGINE=gemini,none` | | Turn the duck off | `AI_DUCK_ENGINE=none` | -| Replace a retired Gemini deep-review model | `AI_MODELS_GEMINI=deep=,gemini-3.8-flash` | -| Pin Claude's deep review to Opus 4.8 | `AI_MODELS_CLAUDE=deep=claude-opus-4-8,claude-sonnet-5` | +| Replace a retired Gemini deep-review model (emergency) | `AI_MODELS_GEMINI=deep=,gemini-3.8-flash`, then the same change in `config/ai-engines.json` | +| Pin Claude's deep review to Opus 4.8 (emergency) | `AI_MODELS_CLAUDE=deep=claude-opus-4-8,claude-sonnet-5`, then the same change in the file | +| Change a default model | Edit `tasks..` in `config/ai-engines.json` (add the model to `models` first) | +| Turn a provider off for good | `providers..enabled=false` in the file | ## Behaviour @@ -190,7 +270,8 @@ table are allow-listed. value. - **Model chains inside a provider** (for example Claude's deep tier `claude-opus-5-5 → claude-opus-4-8 → claude-sonnet-5-5`) are walked before any - cross-provider fallback. Set them with `AI_MODELS_*` (above). + cross-provider fallback. Set them in `config/ai-engines.json` (above), or + with `AI_MODELS_*` in an emergency. ## Model isolation in PR review diff --git a/scripts/engine.sh b/scripts/engine.sh index 4930780a0..73b94e7f1 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -44,6 +44,45 @@ set -euo pipefail # concurrent runs do not read each other's values. DEV_LEAD_SIDECAR_DIR="${DEV_LEAD_SIDECAR_DIR:-/tmp}" +# Each provider's model list: the defaults in config/ai-engines.json, read by +# lib/engine-models.sh, with AI_MODELS_CLAUDE / AI_MODELS_GEMINI / +# AI_MODELS_COPILOT as break-glass overrides. Sourced before the engine chain, +# which takes the enabled providers and their order from the same file. +_ENGINE_MODELS_LIB="$(dirname "${BASH_SOURCE[0]}")/lib/engine-models.sh" +if [ ! -f "$_ENGINE_MODELS_LIB" ]; then + echo "::error::engine.sh: missing $_ENGINE_MODELS_LIB (model lists and defaults)" >&2 + return 1 2>/dev/null || exit 1 +fi +# shellcheck source=lib/engine-models.sh +source "$_ENGINE_MODELS_LIB" +unset _ENGINE_MODELS_LIB +# The defaults come from config/ai-engines.json (#1973). Read it once here: +# the lookups below and in child $(…) shells reuse the table, and a missing or +# malformed file fails the step instead of running on stale defaults. +if ! ai_engines_config_load; then + return 1 2>/dev/null || exit 1 +fi +if [ -z "${AI_MODELS_PROBLEM_REPORTED:-}" ]; then + _models_problems="$(ai_models_problems)" + if [ -n "$_models_problems" ]; then + while IFS= read -r _models_line; do + echo "::warning::$_models_line" >&2 + done <<< "$_models_problems" + export AI_MODELS_PROBLEM_REPORTED=1 + fi + unset _models_problems _models_line +fi +# Fable 5 is deprecated (#1901): warn once per run when a configured chain still +# names a claude-fable-* model, but still honour it (a stop-gap, not a rejection). +if [ -z "${AI_MODELS_FABLE_WARNED:-}" ]; then + _fable_warn="$(ai_models_fable_deprecation)" + if [ -n "$_fable_warn" ]; then + echo "::warning::$_fable_warn" >&2 + export AI_MODELS_FABLE_WARNED=1 + fi + unset _fable_warn +fi + # Configured engine chain (AI_ENGINES): which engines are enabled and in what # order. The primary is REVIEW_ENGINE when that is set AND enabled, otherwise # the first engine in AI_ENGINES — so a disabled engine is never used, even when @@ -77,38 +116,6 @@ else fi export REVIEW_ENGINE -# Each provider's model list (AI_MODELS_CLAUDE / AI_MODELS_GEMINI / -# AI_MODELS_COPILOT) and its defaults live in lib/engine-models.sh, so a retired -# or renamed model is an Actions-variable edit, not a code change. -_ENGINE_MODELS_LIB="$(dirname "${BASH_SOURCE[0]}")/lib/engine-models.sh" -if [ ! -f "$_ENGINE_MODELS_LIB" ]; then - echo "::error::engine.sh: missing $_ENGINE_MODELS_LIB (model lists and defaults)" >&2 - return 1 2>/dev/null || exit 1 -fi -# shellcheck source=lib/engine-models.sh -source "$_ENGINE_MODELS_LIB" -unset _ENGINE_MODELS_LIB -if [ -z "${AI_MODELS_PROBLEM_REPORTED:-}" ]; then - _models_problems="$(ai_models_problems)" - if [ -n "$_models_problems" ]; then - while IFS= read -r _models_line; do - echo "::warning::$_models_line" >&2 - done <<< "$_models_problems" - export AI_MODELS_PROBLEM_REPORTED=1 - fi - unset _models_problems _models_line -fi -# Fable 5 is deprecated (#1901): warn once per run when a configured chain still -# names a claude-fable-* model, but still honour it (a stop-gap, not a rejection). -if [ -z "${AI_MODELS_FABLE_WARNED:-}" ]; then - _fable_warn="$(ai_models_fable_deprecation)" - if [ -n "$_fable_warn" ]; then - echo "::warning::$_fable_warn" >&2 - export AI_MODELS_FABLE_WARNED=1 - fi - unset _fable_warn -fi - # Cross-engine rubber-duck model (issue #773). The duck deliberately routes to # Copilot/o4-mini even when the primary engine is NOT copilot (e.g. the default # claude engine sets DUCK_ENGINE=copilot for adversarial diversity), so diff --git a/scripts/lib/consumer-manifest.json b/scripts/lib/consumer-manifest.json index afc6e1033..06c936d28 100644 --- a/scripts/lib/consumer-manifest.json +++ b/scripts/lib/consumer-manifest.json @@ -140,6 +140,7 @@ ], "surface_sources": { ".github/workflows/dev-lead-reusable.yml": [ + "config/ai-engines.json", "prompts/dev-lead/fix-bot-comment.md", "prompts/dev-lead/fix-ci.md", "prompts/dev-lead/fix-issue.md", @@ -159,6 +160,7 @@ "scripts/lib/reviewer-sources.tsv" ], ".github/workflows/pr-review.yml": [ + "config/ai-engines.json", "prompts/cascade-action.md", "prompts/deep-review.md", "prompts/rubber-duck.md", diff --git a/scripts/lib/downstream-impact.sh b/scripts/lib/downstream-impact.sh index 19541608a..b37c89be1 100644 --- a/scripts/lib/downstream-impact.sh +++ b/scripts/lib/downstream-impact.sh @@ -13,7 +13,7 @@ # source-graph expansion. Two and only two match kinds: # (a) a changed reusable-workflow path that a consumer ref pins exactly # (after normalising the manifest's provider prefix) -> direct surface; -# (b) a changed scripts/lib/*.sh, scripts/lib/*.tsv, or prompts/* path listed in a reusable's +# (b) a changed scripts/lib/*.sh, scripts/lib/*.tsv, prompts/* or config/*.json path listed in a reusable's # surface_sources -> that reusable surface -> the reusable's consumers. # # A consumer ref is provider-prefixed (e.g. ".github-private/.github/workflows/ diff --git a/scripts/lib/engine-chain.sh b/scripts/lib/engine-chain.sh index 70c262da3..b14940b4e 100644 --- a/scripts/lib/engine-chain.sh +++ b/scripts/lib/engine-chain.sh @@ -1,15 +1,18 @@ # shellcheck shell=bash # scripts/lib/engine-chain.sh — the configured AI engine chain (AI_ENGINES). # -# One org/repo Actions variable, AI_ENGINES, turns engines on or off and sets -# their order, so switching providers needs no code change: +# config/ai-engines.json (#1973) enables providers and sets the default order +# (providers.fallback_order). One org/repo Actions variable, AI_ENGINES, is the +# kill switch on top of it: it turns engines off and reorders them with no +# release, but it cannot turn on a provider the file disables: # -# AI_ENGINES="claude,gemini,copilot" the default +# AI_ENGINES="claude,gemini,copilot" the default (the file's order) # AI_ENGINES="claude,gemini" Copilot off (never tried, not even as a fallback) # AI_ENGINES="gemini,claude" Gemini first, Claude as the fallback # AI_ENGINES="claude" Claude only # -# - membership = enabled. An engine that is not listed is never used. +# - membership = enabled. An engine that is not listed is never used, nor is +# one the file disables (reported by ai_engine_chain_problem). # - order = fallback preference, walked left to right on a rate limit. # - first = the primary engine, unless REVIEW_ENGINE / DEV_LEAD_ENGINE is # set explicitly (a legacy override; it is ignored, with a @@ -22,10 +25,35 @@ # reports it so validate_engines can warn once. # # Sourced by engine.sh, validate-engines.sh and review-batch.sh. Defines -# functions only; runs nothing at source time and does not call `set`. +# functions only; at source time it only sources engine-models.sh (the file +# reader) when present, and does not call `set`. +# The built-in order, used only when engine-models.sh is not next to this lib +# (or the file cannot be read; engine.sh fails the step on that first). AI_ENGINES_DEFAULT="claude gemini copilot" +if ! declare -F ai_engines_file_providers >/dev/null 2>&1 \ + && [ -f "$(dirname "${BASH_SOURCE[0]}")/engine-models.sh" ]; then + # shellcheck source=scripts/lib/engine-models.sh + source "$(dirname "${BASH_SOURCE[0]}")/engine-models.sh" +fi + +# _ai_engine_default_chain — the file's enabled providers in fallback_order, +# else AI_ENGINES_DEFAULT. +_ai_engine_default_chain() { + local c="" + if declare -F ai_engines_file_providers >/dev/null 2>&1; then + c="$(ai_engines_file_providers 2>/dev/null)" || c="" + fi + printf '%s' "${c:-$AI_ENGINES_DEFAULT}" +} + +# _ai_engine_file_disabled — the providers the file disables (space-separated). +_ai_engine_file_disabled() { + declare -F ai_engines_file_disabled >/dev/null 2>&1 || return 0 + ai_engines_file_disabled 2>/dev/null || true +} + # _ai_engine_spec — the raw configured value (AI_ENGINES, else DEV_LEAD_ENGINES). _ai_engine_spec() { printf '%s' "${AI_ENGINES:-${DEV_LEAD_ENGINES:-}}" @@ -53,29 +81,43 @@ _ai_engine_parse() { } # ai_engine_chain — the enabled engines in preference order, space-separated. -# Empty or invalid configuration → the default chain. +# Engines the file disables are dropped. Empty or invalid configuration, or one +# that names only disabled engines → the default chain. ai_engine_chain() { - local parsed + local parsed disabled e kept="" if parsed="$(_ai_engine_parse "$(_ai_engine_spec)")" && [ -n "$parsed" ]; then - printf '%s' "$parsed" - else - printf '%s' "$AI_ENGINES_DEFAULT" + disabled="$(_ai_engine_file_disabled)" + for e in $parsed; do + [[ " $disabled " == *" $e "* ]] || kept="${kept:+$kept }$e" + done fi + printf '%s' "${kept:-$(_ai_engine_default_chain)}" } # ai_engine_chain_problem — prints a one-line description when the configured -# value is unusable (unknown engine, or nothing but separators); prints nothing -# when it is fine or unset. +# value is unusable (unknown engine, nothing but separators, or an engine the +# file disables); prints nothing when it is fine or unset. ai_engine_chain_problem() { - local spec parsed + local spec parsed disabled e off="" spec="$(_ai_engine_spec)" [ -n "$spec" ] || return 0 if [ -z "${spec//[[:space:],]/}" ]; then printf "AI_ENGINES='%s' lists no engine — using the default chain '%s'" \ - "$spec" "$AI_ENGINES_DEFAULT" + "$spec" "$(_ai_engine_default_chain)" + return 0 elif ! parsed="$(_ai_engine_parse "$spec")" || [ -z "$parsed" ]; then printf "AI_ENGINES='%s' names an unknown engine (expected claude, gemini, copilot) — using the default chain '%s'" \ - "$spec" "$AI_ENGINES_DEFAULT" + "$spec" "$(_ai_engine_default_chain)" + return 0 + fi + disabled="$(_ai_engine_file_disabled)" + for e in $parsed; do + [[ " $disabled " != *" $e "* ]] || off="${off:+$off, }$e" + done + if [ -n "$off" ]; then + # AI_ENGINES narrows the file; it cannot turn a provider back on. + printf "AI_ENGINES='%s' names %s, which config/ai-engines.json has disabled — ignored (enable a provider in the file, not with AI_ENGINES); using '%s'" \ + "$spec" "$off" "$(ai_engine_chain)" elif [[ "$spec" =~ ^[[:space:],] ]]; then # The workflows derive the primary with startsWith(vars.AI_ENGINES, …), # which cannot skip leading separators. diff --git a/scripts/lib/engine-models.sh b/scripts/lib/engine-models.sh index 21967b500..5e5b5381f 100644 --- a/scripts/lib/engine-models.sh +++ b/scripts/lib/engine-models.sh @@ -1,8 +1,16 @@ # shellcheck shell=bash -# scripts/lib/engine-models.sh — the model list of each provider (AI_MODELS_). +# scripts/lib/engine-models.sh — the model list of each provider. # -# One Actions variable per provider holds every model it uses, so a retired or -# renamed model is a variable edit rather than a code change and a release: +# The defaults live in config/ai-engines.json (#1973), a versioned, +# schema-validated file (scripts/validate-ai-engines.py runs in lint.yml) that +# ships with the pr-review/* and dev-lead/* channel tags, so a default-model +# change canaries on `next` before `ring0` and `stable`. It is read with jq, +# resolved relative to this lib (AI_ENGINES_CONFIG overrides the path, for +# tests). A missing or malformed file is an ::error::, never a silent fall back +# to stale built-in defaults. +# +# One Actions variable per provider is the break-glass override on top of the +# file — set it in an emergency, then fold the change back into the file: # # AI_MODELS_CLAUDE triage=… ; deep=… ; audit=… ; action=… ; single=… ; duck=… # AI_MODELS_GEMINI (the same keys) @@ -31,64 +39,113 @@ # whole chain; GEMINI_FLASH_MODEL / GEMINI_PRO_MODEL replace # only its first model; # Copilot COPILOT_API_MODEL (every key); -# 2. AI_MODELS_; -# 3. ai_models_default. +# 2. AI_MODELS_ (break-glass override); +# 3. config/ai-engines.json (ai_models_default). +# The per-tier variables in 1. are internal (tests, the A/B runner); operators +# use AI_MODELS_* in an emergency and the file otherwise. # An unknown key or a malformed model id drops that entry (its default applies); # ai_models_problems reports it so engine.sh can warn once. # -# Sourced by engine.sh. Defines functions only; runs nothing at source time. +# Sourced by engine.sh and engine-chain.sh. Defines functions only; at source +# time it only resolves the config path. -# ai_models_default — the built-in chain for /. -# -# Claude notes: -# - Sonnet 5.5 (#1978, epic #1095 Phase 3) is the default sonnet across triage, -# deep, action and duck. In triage and action it replaces claude-sonnet-5 as -# the sonnet hop and sonnet-5 stays one hop behind it as the known-good -# sonnet safety hop (its id is bare `claude-sonnet-5`: the `claude-sonnet-5-0` -# spelling shipped by #1100 404s, so every hop to it silently fell through — -# #1957). Deep's known-good hop is opus-4-8 (see below), so its sonnet tail is -# sonnet 5.5 alone, as #1978 AC-2 specifies. audit/single are opus tiers and -# keep sonnet out entirely. -# - Deep swapped opus-4-8 → opus-5-5 (#1898, epic #1895 Phase 2). opus-4-8 is -# the known-good 2nd hop (#1957), so a throttled or unavailable opus-5-5 -# degrades to the model it replaced instead of failing; sonnet 5.5 is last. -# - Audit and single swapped fable-5 → opus-5-5 (#1901, epic #1895 Phase 3), -# deprecating Fable 5: the engine no longer depends on Fable, which opus-5-5 -# outperforms at lower cost, so reviews run one current model family end to -# end. Both keep opus-4-8 as the known-good 2nd hop and opus-4-7 as the last. -# - Action's last (cross-family) hop swapped opus-4-8 → opus-5-5 (#1900, epic -# #1895 Phase 3) once the deep-tier canary passed (#1899): the writer's last -# resort is the cheaper, proven model. -# - opus-5-5's CLI/API default reasoning effort is `medium` (levels -# low/medium/high/xhigh/max); the deep, audit and single tiers pin -# `--effort high` in engine.sh's run_agentic so moving off Fable does not -# silently lower their reasoning effort (triage/action keep the default). -# Min cacheable prefix opus-4-8 = 4096 tok. -# - Fable 5 is deprecated but not removed: a chain that still names a -# claude-fable-* model (AI_MODELS_CLAUDE or CLAUDE__MODEL_CHAIN) is -# honoured as a stop-gap and warns once per run (ai_models_fable_deprecation); -# its price rows stay in model-pricing.tsv for historical token records. -# - The subscription cap is shared across Claude models (#206), so an -# in-Claude chain only helps with per-model RPM/TPM limits. +# The model notes (Sonnet 5 id history, the Fable 5 deprecation, why each chain +# has its fallbacks) live next to each model in config/ai-engines.json. +_AI_ENGINES_CONFIG_DEFAULT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)/config/ai-engines.json" + +# ai_engines_config_path — the engine config file: AI_ENGINES_CONFIG when set, +# else config/ai-engines.json at the root of the checkout holding this lib. +ai_engines_config_path() { + printf '%s' "${AI_ENGINES_CONFIG:-$_AI_ENGINES_CONFIG_DEFAULT}" +} + +# The file flattened to one line per fact, so a lookup is parameter expansion +# (Bash 3.2: no associative arrays): +# :=,,… one per chain the file sets +# providers= +# disabled= +# shellcheck disable=SC2016 # a jq program: $p / $t are jq variables +_AI_ENGINES_JQ=' + if (.tasks | type) != "object" or (.providers | type) != "object" + or (.providers.fallback_order | type) != "array" then + error("expected the tasks and providers sections (providers.fallback_order)") + else . end + | .providers as $p + | ( .tasks | to_entries[] | select(.value | type == "object") | .key as $t + | .value | to_entries[] + | select(.key == "claude" or .key == "gemini" or .key == "copilot") + | "\(.key):\($t)=\(.value | join(","))" ), + "providers=\([$p.fallback_order[] | select($p[.].enabled == true)] | join(" "))", + "disabled=\([("claude", "gemini", "copilot") | select($p[.].enabled != true)] | join(" "))" +' + +# _ai_engines_config_read — prints the flattened table, or an ::error:: +# on stderr and returns 1 when the file is missing, unreadable or malformed. +_ai_engines_config_read() { + local path="$1" out why + if [ ! -f "$path" ] || [ ! -r "$path" ]; then + printf '::error::engine config %s is missing or unreadable — it holds the providers, models and task chains; there are no built-in defaults to fall back to (docs/engine-configuration.md)\n' "$path" >&2 + return 1 + fi + if ! command -v jq >/dev/null 2>&1; then + printf '::error::engine config %s cannot be read: jq is not installed\n' "$path" >&2 + return 1 + fi + if ! out="$(jq -r "$_AI_ENGINES_JQ" "$path" 2>/dev/null)"; then + why="$(jq -r "$_AI_ENGINES_JQ" "$path" 2>&1 >/dev/null | head -n 1)" + printf '::error::engine config %s is malformed (%s) — run scripts/validate-ai-engines.py\n' "$path" "$why" >&2 + return 1 + fi + printf '%s' "$out" +} + +# ai_engines_config_load — reads the file once into this shell, so later +# lookups (and the $(…) subshells that inherit the table) skip jq. engine.sh +# calls it at source time and fails the step when it returns 1. A lookup after +# AI_ENGINES_CONFIG changes rereads the file. +ai_engines_config_load() { + local path table + path="$(ai_engines_config_path)" + table="$(_ai_engines_config_read "$path")" || return 1 + _AI_ENGINES_TABLE="$table" + _AI_ENGINES_TABLE_SRC="$path" +} + +# _ai_engines_lookup — the value of in the table ("" when the file +# does not set it); returns 1 when the file cannot be read. +_ai_engines_lookup() { + local name="$1" path table rest nl=$'\n' + path="$(ai_engines_config_path)" + if [ -n "${_AI_ENGINES_TABLE:-}" ] && [ "${_AI_ENGINES_TABLE_SRC:-}" = "$path" ]; then + table="$_AI_ENGINES_TABLE" + else + table="$(_ai_engines_config_read "$path")" || return 1 + fi + table="$nl$table$nl" + rest="${table#*"$nl$name="}" + [ "$rest" != "$table" ] || return 0 + printf '%s' "${rest%%"$nl"*}" +} + +# ai_models_default — the file's chain for / +# (tasks..). Empty when the file sets none: Gemini and Copilot +# have no duck chain, their duck follows the triage model. Returns 1, with an +# ::error::, when the file cannot be read. ai_models_default() { - case "${1:-}:${2:-}" in - claude:triage) printf '%s' "claude-haiku-4-5-20251001,claude-sonnet-5-5,claude-sonnet-5" ;; - claude:deep) printf '%s' "claude-opus-5-5,claude-opus-4-8,claude-sonnet-5-5" ;; - claude:audit) printf '%s' "claude-opus-5-5,claude-opus-4-8,claude-opus-4-7" ;; - claude:action) printf '%s' "claude-sonnet-5-5,claude-sonnet-5,claude-opus-5-5" ;; - claude:single) printf '%s' "claude-opus-5-5,claude-opus-4-8,claude-opus-4-7" ;; - claude:duck) printf '%s' "claude-sonnet-5-5" ;; - # gemini-2.5-pro is withdrawn for new keys ("no longer available to new - # users … use models/gemini-3.1-pro-preview", #1960), so the quality tier - # uses 3.1-pro-preview and degrades to the flash model that is known to work. - # No gemini/copilot duck default: the duck follows the triage model. - gemini:triage|gemini:action) - printf '%s' "gemini-3.8-flash,gemini-3.1-pro-preview" ;; - gemini:deep|gemini:audit|gemini:single) - printf '%s' "gemini-3.1-pro-preview,gemini-3.8-flash" ;; - copilot:triage|copilot:deep|copilot:audit|copilot:action|copilot:single) - printf '%s' "openai/o4-mini" ;; - esac + case "${1:-}" in claude|gemini|copilot) ;; *) return 0 ;; esac + case " $(_ai_models_keys "$1") " in *" ${2:-} "*) ;; *) return 0 ;; esac + _ai_engines_lookup "$1:$2" +} + +# ai_engines_file_providers — the providers the file enables, in its +# fallback_order (space-separated). AI_ENGINES can only narrow this list. +ai_engines_file_providers() { + _ai_engines_lookup providers +} + +# ai_engines_file_disabled — the providers the file disables (space-separated). +ai_engines_file_disabled() { + _ai_engines_lookup disabled } # _ai_models_keys — the keys AI_MODELS_ accepts (the same @@ -180,7 +237,7 @@ ai_models_configured() { ai_models_chain() { local c c="$(ai_models_configured "$1" "$2")" - [ -n "$c" ] || c="$(ai_models_default "$1" "$2")" + [ -n "$c" ] || c="$(ai_models_default "$1" "$2")" || return 1 printf '%s' "$c" } @@ -210,7 +267,7 @@ ai_models_gemini_chain() { deep|audit|single) group="${GEMINI_PRO_MODEL_CHAIN:-}"; first="${GEMINI_PRO_MODEL:-}" ;; duck) c="$(ai_models_configured gemini duck)" - [ -n "$c" ] || { c="$(ai_models_gemini_chain triage)"; c="${c%%,*}"; } + [ -n "$c" ] || { c="$(ai_models_gemini_chain triage)" || return 1; c="${c%%,*}"; } printf '%s' "$c" return 0 ;; *) return 0 ;; @@ -219,7 +276,7 @@ ai_models_gemini_chain() { printf '%s' "${group//[[:space:]]/}" return 0 fi - c="$(ai_models_chain gemini "$key")" + c="$(ai_models_chain gemini "$key")" || return 1 first="${first//[[:space:]]/}" [ -z "$first" ] || c="$(ai_models_replace_first "$first" "$c")" printf '%s' "$c" @@ -238,7 +295,7 @@ ai_models_copilot_model() { else m="$(ai_models_configured copilot "$key")" [ -n "$m" ] || [ "$key" != duck ] || key=triage - [ -n "$m" ] || m="$(ai_models_chain copilot "$key")" + [ -n "$m" ] || m="$(ai_models_chain copilot "$key")" || return 1 fi m="${m%%,*}" printf '%s' "${m//[[:space:]]/}" @@ -252,6 +309,41 @@ ai_models_problems() { done } +# ai_engines_overrides — one line per break-glass override that is set and +# differs from config/ai-engines.json, so the daily health check shows it until +# it is folded back into the file: +# AI_MODELS_ = (file: ) +# AI_ENGINES= → (file: ) +# A Gemini/Copilot duck has no file chain; it is compared with the triage model +# it follows. AI_ENGINES (else the legacy DEV_LEAD_ENGINES) is compared after +# the same normalisation engine-chain.sh applies (case, separators). Prints +# nothing when no override differs; returns 1 when the file cannot be read. +ai_engines_overrides() { + local p k var c d spec chain="" t file + local -a toks=() + for p in claude gemini copilot; do + var="$(_ai_models_var "$p")" + for k in $(_ai_models_keys "$p"); do + c="$(ai_models_configured "$p" "$k")" + [ -n "$c" ] || continue + d="$(ai_models_default "$p" "$k")" || return 1 + if [ -z "$d" ] && [ "$k" = duck ]; then + d="$(ai_models_default "$p" triage)" || return 1 + d="${d%%,*}" + fi + [ "$c" = "$d" ] || printf '%s %s=%s (file: %s)\n' "$var" "$k" "$c" "${d:-none}" + done + done + spec="${AI_ENGINES:-${DEV_LEAD_ENGINES:-}}" + [ -n "${spec//[[:space:],]/}" ] || return 0 + file="$(ai_engines_file_providers)" || return 1 + IFS=$' \t' read -r -a toks <<< "$(printf '%s' "$spec" | tr ',\n\r\t' ' ' | tr '[:upper:]' '[:lower:]')" + for t in ${toks[@]+"${toks[@]}"}; do + [[ " $chain " == *" $t "* ]] || chain="${chain:+$chain }$t" + done + [ "$chain" = "$file" ] || printf "AI_ENGINES=%s → %s (file: %s)\n" "$spec" "$chain" "$file" +} + # ai_models_fable_deprecation — one deprecation line when a *configured* Claude # chain still names a claude-fable-* model, else nothing. Fable 5 is deprecated # (#1901, epic #1895 Phase 3): audit/single now default to claude-opus-5-5. A @@ -297,8 +389,9 @@ ai_models_fable_deprecation() { # haiku → triage (default primary claude-haiku-4-5-20251001) # Overrides are honoured with the same precedence engine.sh uses: the per-tier # CLAUDE__MODEL_CHAIN env first (ai_models_chain does not read it), then -# AI_MODELS_CLAUDE (via ai_models_chain), then the built-in default. An unknown -# family warns and returns non-zero. Bash 3.2-safe. +# AI_MODELS_CLAUDE (via ai_models_chain), then config/ai-engines.json. An +# unknown family warns and returns non-zero, as does an unreadable engine +# config. Bash 3.2-safe. ai_model_for_family() { local family="${1:-}" tier var chain model case "$family" in @@ -312,16 +405,16 @@ ai_model_for_family() { chain="${!var:-}" # Only an EMPTY per-tier var counts as absent (engine.sh's ${VAR:-…} rule). A # set-but-malformed value (",") must not let AI_MODELS_CLAUDE win; it fails the - # id check below and falls back to the built-in default. - [ -n "$chain" ] || chain="$(ai_models_chain claude "$tier")" + # id check below and falls back to the file's default. + [ -n "$chain" ] || chain="$(ai_models_chain claude "$tier")" || return 1 model="$(_ai_models_trim "${chain%%,*}")" # Validate the resolved id. ai_models_chain paths already validate via # _ai_models_scan, but the per-tier CLAUDE__MODEL_CHAIN override is read # raw above — a malformed value like "bad id" would otherwise reach the caller - # and fail the Claude invocation. Fall back to the built-in default on a bad id. + # and fail the Claude invocation. Fall back to the file's default on a bad id. if [[ ! "$model" =~ ^[A-Za-z0-9][A-Za-z0-9._:/@-]*$ ]]; then printf '::warning::ai_model_for_family: %s has an invalid model id %s — default used\n' "$var" "$model" >&2 - chain="$(ai_models_default claude "$tier")" + chain="$(ai_models_default claude "$tier")" || return 1 model="$(_ai_models_trim "${chain%%,*}")" fi printf '%s' "${model//[[:space:]]/}" diff --git a/scripts/lib/validate-consumer-manifest.sh b/scripts/lib/validate-consumer-manifest.sh index 79d23a651..a06890662 100755 --- a/scripts/lib/validate-consumer-manifest.sh +++ b/scripts/lib/validate-consumer-manifest.sh @@ -5,14 +5,14 @@ set -euo pipefail # # The manifest (consumer-manifest.json, same directory) is the single # machine-checkable source of truth mapping each provider reusable workflow to -# the org repos (consumers) that pin it, plus the in-provider scripts/lib + prompts -# each reusable sources (surface_sources). This validator is the gate that keeps +# the org repos (consumers) that pin it, plus the in-provider scripts/lib, prompts +# and config each reusable sources (surface_sources). This validator is the gate that keeps # the manifest honest. It exits non-zero on: # - invalid JSON or a violated top-level schema shape # - a surface_sources KEY that is not an existing reusable-workflow path in # this repo (a file under .github/workflows/ that declares `workflow_call`) # - a surface_sources VALUE that is not an existing scripts/lib/*.sh, -# scripts/lib/*.tsv, or prompts/* path in this repo +# scripts/lib/*.tsv, prompts/*, or config/*.json path in this repo # # Existence is always checked against --repo-root (the .github-private working # tree), independent of where the manifest file itself lives, so a fixture in a @@ -93,13 +93,14 @@ if [ "$schema_ok" != "true" ]; then fi # 3. surface_sources keys must be existing reusable-workflow paths in this repo; -# their values must be existing scripts/lib/*.sh, scripts/lib/*.tsv, or prompts/* -# paths in this repo. +# their values must be existing scripts/lib/*.sh, scripts/lib/*.tsv, prompts/*, +# or config/*.json paths in this repo (config/ai-engines.json, #1973). # Paths are resolved via realpath to prevent traversal sequences from escaping # their intended root directories. wf_root=$(realpath -m "$REPO_ROOT/.github/workflows") lib_root=$(realpath -m "$REPO_ROOT/scripts/lib") prompts_root=$(realpath -m "$REPO_ROOT/prompts") +config_root=$(realpath -m "$REPO_ROOT/config") while IFS= read -r key; do [ -n "$key" ] || continue @@ -124,13 +125,14 @@ done < <(jq -r '.surface_sources | keys[]' "$MANIFEST") while IFS= read -r value; do [ -n "$value" ] || continue case "$value" in - scripts/lib/*.sh|scripts/lib/*.tsv|prompts/*) : ;; + scripts/lib/*.sh|scripts/lib/*.tsv|prompts/*|config/*.json) : ;; *) - err "surface_sources value is not a scripts/lib/*.sh, scripts/lib/*.tsv, or prompts/* path: $value" + err "surface_sources value is not a scripts/lib/*.sh, scripts/lib/*.tsv, prompts/*, or config/*.json path: $value" continue ;; esac resolved=$(realpath -m "$REPO_ROOT/$value") - if [ "${resolved#"${lib_root}/"}" = "$resolved" ] && [ "${resolved#"${prompts_root}/"}" = "$resolved" ]; then + if [ "${resolved#"${lib_root}/"}" = "$resolved" ] && [ "${resolved#"${prompts_root}/"}" = "$resolved" ] \ + && [ "${resolved#"${config_root}/"}" = "$resolved" ]; then err "surface_sources value contains path traversal: $value" continue fi diff --git a/scripts/pr_review_health.sh b/scripts/pr_review_health.sh index e2a54de28..8cd773d97 100644 --- a/scripts/pr_review_health.sh +++ b/scripts/pr_review_health.sh @@ -34,6 +34,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib/pr-review-sweep-metrics.sh" source "$(dirname "${BASH_SOURCE[0]}")/lib/run-attribution.sh" # Model family → current id resolver (#1979): name a family, never pin a version. +# Also lists break-glass overrides of config/ai-engines.json (ai_engines_overrides). # shellcheck source=scripts/lib/engine-models.sh source "$(dirname "${BASH_SOURCE[0]}")/lib/engine-models.sh" @@ -333,9 +334,30 @@ workflow_source=$(gh api "repos/${WORKFLOW_REPO}/contents/.github/workflows/${WO generate_attribution_report "$attr_tsv" "PR-review run attribution (by role, across collapse)" rm -f "$attr_tsv" "$legacy_attr_jsonl" fi + # Break-glass overrides (#1973): an AI_MODELS_* key or AI_ENGINES that differs + # from config/ai-engines.json. Listed every day until it is folded back into + # the file, so a temporary override cannot live on silently. Deterministic — + # same truncation guarantee. + printf '## Engine configuration overrides\n\n' + if ! engine_overrides="$(ai_engines_overrides)"; then + engine_overrides="" + printf 'Could not read `config/ai-engines.json` — see the run log.\n\n' + elif [ -z "$engine_overrides" ]; then + printf 'None: every engine and model setting comes from `config/ai-engines.json`.\n\n' + else + printf 'These Actions variables override `config/ai-engines.json`. Fold each into the file through a normal release, then delete the variable.\n\n' + printf '%s\n' "$engine_overrides" | sed 's/^/- `/; s/$/`/' + printf '\n' + fi } > "$REPORT_FILE" rm -f "$ingress_attr_jsonl" +if [ -n "${engine_overrides:-}" ]; then + while IFS= read -r line; do + echo "::warning::engine override differs from config/ai-engines.json: $line" + done <<< "$engine_overrides" +fi + echo "Duration percentiles — p50 $(fmt_dur "$dur_p50") / p95 $(fmt_dur "$dur_p95") across $dur_n run(s)" echo "Outcome mix (by event):" pr_review_outcomes_by_event "$runs_json" | sed 's/^/ /' diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py new file mode 100755 index 000000000..d89393ca9 --- /dev/null +++ b/scripts/validate-ai-engines.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Validate config/ai-engines.json (#1973). + +Usage: validate-ai-engines.py [config.json] [schema.json] [pricing.tsv] + +Defaults to the repo's config/ai-engines.json, config/ai-engines.schema.json +and scripts/lib/model-pricing.tsv. Validates the file against the JSON Schema +(draft 2020-12, the same `jsonschema>=4` setup evals/validate-cases.py uses), +then checks what a schema cannot express: + + - every model a task names is in `models` and not `retired`; + - every model a task names belongs to the provider of the chain it is in; + - every non-retired model of an enabled provider has a price row in + model-pricing.tsv (a vendor-prefixed id such as openai/o4-mini is priced by + its bare name, as token records store it); + - `duck`, and every `copilot` chain, holds one model (neither has an + in-engine chain). + +Every problem is reported as a `::error::` line. Exit 0 when valid, 1 otherwise. +""" +import json +import re +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +SINGLE_MODEL_TASKS = ("duck",) +SINGLE_MODEL_PROVIDERS = ("copilot",) + + +def error(msg: str) -> None: + print(f"::error::ai-engines config invalid: {msg}", file=sys.stderr) + + +def load_json(path: Path, what: str): + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + error(f"could not read/parse {what} {path}: {exc}") + sys.exit(1) + + +def price_globs(path: Path) -> list: + """The model_glob column of model-pricing.tsv (comments and short rows skipped, + as scripts/lib/model-pricing.sh does).""" + globs = [] + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError as exc: + error(f"could not read price table {path}: {exc}") + sys.exit(1) + for line in lines: + if re.match(r"^\s*#", line): + continue + cols = line.split("\t") + if len(cols) >= 6 and cols[0]: + globs.append(cols[0]) + return globs + + +def glob_matches(glob: str, model: str) -> bool: + """model-pricing.sh's glob rule: '*' and '?' are wildcards, the rest literal.""" + regex = "".join(".*" if c == "*" else "." if c == "?" else re.escape(c) for c in glob) + return re.fullmatch(regex, model) is not None + + +def check_schema(config, schema) -> list: + try: + import jsonschema + except ImportError: + error("jsonschema not installed (pip install 'jsonschema>=4')") + sys.exit(1) + validator = jsonschema.Draft202012Validator(schema) + problems = [] + for err in sorted(validator.iter_errors(config), key=lambda e: list(e.absolute_path)): + where = "/".join(str(p) for p in err.absolute_path) or "(root)" + problems.append(f"schema: {where}: {err.message}") + return problems + + +def check_references(config, globs) -> list: + problems = [] + models = config["models"] + providers = config["providers"] + for task, chains in config["tasks"].items(): + for provider, chain in chains.items(): + if provider not in ("claude", "gemini", "copilot"): + continue # prefer, $comment + for model in chain: + entry = models.get(model) + if entry is None: + problems.append(f"tasks.{task}.{provider}: '{model}' is not in models") + continue + if entry["status"] == "retired": + problems.append(f"tasks.{task}.{provider}: '{model}' is retired") + if entry["provider"] != provider: + problems.append( + f"tasks.{task}.{provider}: '{model}' belongs to provider " + f"{entry['provider']}, not {provider}") + if len(chain) > 1 and (task in SINGLE_MODEL_TASKS or provider in SINGLE_MODEL_PROVIDERS): + why = "the duck runs one model" if task in SINGLE_MODEL_TASKS else "Copilot has no in-engine chain" + problems.append(f"tasks.{task}.{provider}: takes one model ({why}), got {len(chain)}") + for model, entry in models.items(): + if entry["status"] == "retired" or not providers[entry["provider"]]["enabled"]: + continue + bare = model.split("/", 1)[1] if "/" in model else model + if not any(glob_matches(g, model) or glob_matches(g, bare) for g in globs): + problems.append( + f"models.{model}: provider {entry['provider']} is enabled but the model has " + f"no price row in scripts/lib/model-pricing.tsv") + return problems + + +def main(argv) -> int: + config_path = Path(argv[1]) if len(argv) > 1 else REPO / "config" / "ai-engines.json" + schema_path = Path(argv[2]) if len(argv) > 2 else REPO / "config" / "ai-engines.schema.json" + pricing_path = Path(argv[3]) if len(argv) > 3 else REPO / "scripts" / "lib" / "model-pricing.tsv" + + config = load_json(config_path, "config") + schema = load_json(schema_path, "schema") + problems = check_schema(config, schema) + # The reference checks index into the shape the schema guarantees. + if not problems: + problems = check_references(config, price_globs(pricing_path)) + for p in problems: + error(p) + if problems: + print(f"ai-engines config INVALID: {len(problems)} problem(s) in {config_path}", file=sys.stderr) + return 1 + tasks = [t for t in config["tasks"] if t != "$comment"] + print(f"ai-engines config OK: {len(config['models'])} model(s), {len(tasks)} task(s) ({config_path})") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/dev-lead/unit/test_ai_engines_config.bats b/tests/dev-lead/unit/test_ai_engines_config.bats new file mode 100644 index 000000000..a86579361 --- /dev/null +++ b/tests/dev-lead/unit/test_ai_engines_config.bats @@ -0,0 +1,195 @@ +#!/usr/bin/env bats +# config/ai-engines.json (#1973): the versioned source of truth for providers, +# the model catalog and each task's chain. engine-models.sh reads its defaults +# from the file (jq); AI_MODELS_* and AI_ENGINES are break-glass overrides on +# top of it. test_engine_models.bats proves the defaults did not change; this +# file covers the file itself: a missing or malformed file, precedence, +# provider enablement and the override report. + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" +LIB="$SCRIPT_DIR/scripts/lib/engine-models.sh" +CHAIN_LIB="$SCRIPT_DIR/scripts/lib/engine-chain.sh" +CONFIG="$SCRIPT_DIR/config/ai-engines.json" + +setup() { + unset AI_MODELS_CLAUDE AI_MODELS_GEMINI AI_MODELS_COPILOT AI_MODELS_PROBLEM_REPORTED + unset CLAUDE_TRIAGE_MODEL_CHAIN CLAUDE_DEEP_MODEL_CHAIN CLAUDE_AUDIT_MODEL_CHAIN + unset CLAUDE_ACTION_MODEL_CHAIN CLAUDE_SINGLE_MODEL_CHAIN + unset GEMINI_FLASH_MODEL GEMINI_PRO_MODEL GEMINI_FLASH_MODEL_CHAIN GEMINI_PRO_MODEL_CHAIN + unset COPILOT_API_MODEL COPILOT_API_MODEL_DEFAULTED AI_ENGINES DEV_LEAD_ENGINES + unset AI_DUCK_ENGINE AI_DUCK_MODEL GEMINI_AVAILABLE AI_ENGINES_CONFIG REVIEW_ENGINE + # shellcheck source=../../../scripts/lib/engine-models.sh + source "$LIB" +} + +# Writes a copy of the repo config with applied and points +# AI_ENGINES_CONFIG at it. +_config_with() { + local f="$BATS_TEST_TMPDIR/ai-engines.json" + jq "$1" "$CONFIG" > "$f" + export AI_ENGINES_CONFIG="$f" +} + +# ── The file is the default ────────────────────────────────────────────────── + +@test "file: every default chain is the file's task chain" { + local task provider want + for task in triage deep audit action single duck; do + for provider in claude gemini copilot; do + want="$(jq -r --arg t "$task" --arg p "$provider" '.tasks[$t][$p] // [] | join(",")' "$CONFIG")" + [ "$(ai_models_default "$provider" "$task")" = "$want" ] + done + done +} + +@test "file: a changed file changes the default (no built-in table left)" { + _config_with '.tasks.deep.claude = ["claude-opus-4-8"]' + [ "$(ai_models_default claude deep)" = "claude-opus-4-8" ] + [ "$(ai_models_chain claude deep)" = "claude-opus-4-8" ] +} + +@test "file: resolved relative to the lib, whatever the caller's working directory" { + cd "$BATS_TEST_TMPDIR" + [ "$(ai_engines_config_path)" = "$CONFIG" ] + [ -n "$(ai_models_default claude deep)" ] +} + +# ── A missing or malformed file fails loudly ───────────────────────────────── + +@test "missing file: ::error:: and non-zero, no built-in fallback" { + export AI_ENGINES_CONFIG="$BATS_TEST_TMPDIR/nope.json" + run ai_models_default claude deep + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"*"nope.json"* ]] + [[ "$output" != *"claude-opus"* ]] + run ai_models_chain claude deep + [ "$status" -ne 0 ] + run ai_engines_config_load + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "malformed file: invalid JSON or a file without tasks fails loudly" { + printf '{"tasks": {' > "$BATS_TEST_TMPDIR/bad.json" + export AI_ENGINES_CONFIG="$BATS_TEST_TMPDIR/bad.json" + run ai_models_default claude deep + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] + printf '{"models": {}}' > "$BATS_TEST_TMPDIR/bad.json" + run ai_models_default claude deep + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "missing file: sourcing engine.sh fails the step" { + run bash -c "set -e; export REVIEW_ENGINE=claude AI_ENGINES_CONFIG='$BATS_TEST_TMPDIR/nope.json' + source '$SCRIPT_DIR/scripts/engine.sh' >/dev/null; echo sourced" + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] + [[ "$output" != *"sourced"* ]] +} + +@test "family: a missing file returns non-zero instead of an empty id" { + export AI_ENGINES_CONFIG="$BATS_TEST_TMPDIR/nope.json" + run ai_model_for_family opus + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "load: a loaded table is reused, and reloaded when the path changes" { + ai_engines_config_load + [ "$(ai_models_default claude deep)" = "$(jq -r '.tasks.deep.claude | join(",")' "$CONFIG")" ] + _config_with '.tasks.deep.claude = ["claude-opus-4-7"]' + [ "$(ai_models_default claude deep)" = "claude-opus-4-7" ] +} + +# ── Precedence: specific env var > AI_MODELS_* > file ──────────────────────── + +@test "precedence: AI_MODELS_* beats the file; a per-tier env var beats both" { + _config_with '.tasks.deep.claude = ["claude-opus-4-7"]' + [ "$(ai_models_chain claude deep)" = "claude-opus-4-7" ] + export AI_MODELS_CLAUDE="deep=claude-opus-4-8" + [ "$(ai_models_chain claude deep)" = "claude-opus-4-8" ] + export CLAUDE_DEEP_MODEL_CHAIN="claude-sonnet-5" + [ "$(ai_model_for_family opus)" = "claude-sonnet-5" ] + run bash -c "export REVIEW_ENGINE=claude; source '$SCRIPT_DIR/scripts/engine.sh' >/dev/null 2>&1; echo \"deep=\$ENGINE_DEEP_MODEL\"" + [[ "$output" == *"deep=claude-sonnet-5"* ]] +} + +@test "precedence: Gemini and Copilot specific vars still beat AI_MODELS_* and the file" { + export AI_MODELS_GEMINI="deep=gemini-d1" GEMINI_PRO_MODEL_CHAIN="gemini-c1" + [ "$(ai_models_gemini_chain deep)" = "gemini-c1" ] + export AI_MODELS_COPILOT="deep=openai/d1" COPILOT_API_MODEL="openai/pinned" + [ "$(ai_models_copilot_model deep)" = "openai/pinned" ] +} + +# ── Providers: the file enables, AI_ENGINES can only narrow ────────────────── + +@test "providers: the default chain is the file's fallback order of enabled providers" { + source "$CHAIN_LIB" + [ "$(ai_engine_chain)" = "$(jq -r '.providers.fallback_order | join(" ")' "$CONFIG")" ] + _config_with '.providers.fallback_order = ["gemini","claude","copilot"]' + [ "$(ai_engine_chain)" = "gemini claude copilot" ] +} + +@test "providers: AI_ENGINES can disable a provider the file enables" { + source "$CHAIN_LIB" + export AI_ENGINES="claude,gemini" + [ "$(ai_engine_chain)" = "claude gemini" ] + ! ai_engine_enabled copilot +} + +@test "providers: AI_ENGINES cannot enable a provider the file disables" { + source "$CHAIN_LIB" + _config_with '.providers.copilot.enabled = false' + [ "$(ai_engine_chain)" = "claude gemini" ] + export AI_ENGINES="copilot,claude" + [ "$(ai_engine_chain)" = "claude" ] + ! ai_engine_enabled copilot + [ "$(ai_engine_primary copilot)" = "claude" ] + [[ "$(ai_engine_chain_problem)" == *"copilot"*"disabled"* ]] +} + +# ── Override visibility ────────────────────────────────────────────────────── + +@test "overrides: nothing set, or set to the file's value, reports nothing" { + [ -z "$(ai_engines_overrides)" ] + export AI_MODELS_CLAUDE="deep=$(jq -r '.tasks.deep.claude | join(",")' "$CONFIG")" + export AI_ENGINES="$(jq -r '.providers.fallback_order | join(",")' "$CONFIG")" + [ -z "$(ai_engines_overrides)" ] +} + +@test "overrides: an AI_MODELS_* key or AI_ENGINES that differs from the file is listed" { + export AI_MODELS_GEMINI="deep=gemini-x; triage=$(jq -r '.tasks.triage.gemini | join(",")' "$CONFIG")" + export AI_ENGINES="claude,gemini" + run ai_engines_overrides + [ "$status" -eq 0 ] + [[ "$output" == *"AI_MODELS_GEMINI deep=gemini-x"*"file: "* ]] + [[ "$output" != *"triage"* ]] + [[ "$output" == *"AI_ENGINES"*"claude gemini"*"file: claude gemini copilot"* ]] +} + +@test "overrides: the health check prints the override section" { + grep -q 'ai_engines_overrides' "$SCRIPT_DIR/scripts/pr_review_health.sh" + local wf="$SCRIPT_DIR/.github/workflows/daily-pr-review-health.yml" + grep -q 'AI_MODELS_GEMINI: ${{ vars.AI_MODELS_GEMINI }}' "$wf" + grep -q 'AI_MODELS_COPILOT: ${{ vars.AI_MODELS_COPILOT }}' "$wf" + grep -q 'AI_ENGINES: ${{ vars.AI_ENGINES }}' "$wf" +} + +# ── Shipping the file ──────────────────────────────────────────────────────── + +@test "ship: dev-lead's sparse checkouts include config/ next to scripts/" { + local wf="$SCRIPT_DIR/.github/workflows/dev-lead-reusable.yml" + local scripts_n config_n + scripts_n="$(grep -cE '^ {12}scripts$' "$wf")" + config_n="$(grep -cE '^ {12}config$' "$wf")" + [ "$scripts_n" -ge 1 ] + [ "$config_n" = "$scripts_n" ] +} + +@test "ship: the consumer manifest lists the file for pr-review and dev-lead" { + local m="$SCRIPT_DIR/scripts/lib/consumer-manifest.json" + jq -e '.surface_sources[".github/workflows/pr-review.yml"] | index("config/ai-engines.json")' "$m" + jq -e '.surface_sources[".github/workflows/dev-lead-reusable.yml"] | index("config/ai-engines.json")' "$m" +} diff --git a/tests/test_batch_fallback.bats b/tests/test_batch_fallback.bats index 172ad084d..c0f2ecffd 100644 --- a/tests/test_batch_fallback.bats +++ b/tests/test_batch_fallback.bats @@ -176,6 +176,9 @@ EOF # answering 404 for any model listed in $BAD_MODELS. _copilot_smoke_setup() { cp "$BATS_TEST_DIRNAME/../scripts/lib/engine-models.sh" "scripts/lib/" + # The lib reads its defaults from config/ai-engines.json beside scripts/ (#1973). + mkdir -p config + cp "$BATS_TEST_DIRNAME/../config/ai-engines.json" config/ cat > "scripts/validate-engines.sh" <<'EOS' source "$(dirname "${BASH_SOURCE[0]}")/lib/engine-models.sh" validate_engines() { diff --git a/tests/test_consumer_manifest.bats b/tests/test_consumer_manifest.bats index 93420d494..ddaf93efc 100644 --- a/tests/test_consumer_manifest.bats +++ b/tests/test_consumer_manifest.bats @@ -111,6 +111,21 @@ run_validator() { [ "$status" -ne 0 ] } +@test "surface_sources value under config/ passes when it is an existing .json file (#1973)" { + jq '.surface_sources = {".github/workflows/pr-review.yml": ["config/ai-engines.json"]}' \ + "$MANIFEST" > "$FIXTURE" + run_validator "$FIXTURE" + [ "$status" -eq 0 ] + jq '.surface_sources = {".github/workflows/pr-review.yml": ["config/nope.json"]}' \ + "$MANIFEST" > "$FIXTURE" + run_validator "$FIXTURE" + [ "$status" -ne 0 ] + jq '.surface_sources = {".github/workflows/pr-review.yml": ["config/../scripts/engine.json"]}' \ + "$MANIFEST" > "$FIXTURE" + run_validator "$FIXTURE" + [ "$status" -ne 0 ] +} + @test "a well-formed minimal manifest with real surfaces passes" { cat > "$FIXTURE" <<'JSON' { diff --git a/tests/test_validate_ai_engines.bats b/tests/test_validate_ai_engines.bats new file mode 100644 index 000000000..cc494d1bd --- /dev/null +++ b/tests/test_validate_ai_engines.bats @@ -0,0 +1,110 @@ +#!/usr/bin/env bats +# scripts/validate-ai-engines.py (#1973): the lint.yml gate on +# config/ai-engines.json — JSON Schema (draft 2020-12) plus the checks a schema +# cannot express: every task model is in the catalog, not retired and of the +# chain's provider; every model of an enabled provider has a price row; duck and +# Copilot chains hold one model. + +REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/.." && pwd)" +VALIDATOR="$REPO_ROOT/scripts/validate-ai-engines.py" +CONFIG="$REPO_ROOT/config/ai-engines.json" + +# Writes the repo config with applied and runs the validator on it. +_validate_with() { + local f="$BATS_TEST_TMPDIR/ai-engines.json" + jq "$1" "$CONFIG" > "$f" + run python3 "$VALIDATOR" "$f" +} + +@test "the repo's config/ai-engines.json passes" { + run python3 "$VALIDATOR" + [ "$status" -eq 0 ] + [[ "$output" == *"OK"* ]] +} + +@test "schema: a missing section fails" { + _validate_with 'del(.tasks)' + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "schema: an unknown status fails" { + _validate_with '.models["claude-opus-4-8"].status = "deprecated"' + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "schema: an unknown task fails; tasks..prefer is allowed" { + _validate_with '.tasks.review = {"claude": ["claude-opus-4-8"]}' + [ "$status" -ne 0 ] + _validate_with '.tasks.deep.prefer = ["gemini", "claude"]' + [ "$status" -eq 0 ] +} + +@test "malformed JSON fails" { + printf '{"models": ' > "$BATS_TEST_TMPDIR/bad.json" + run python3 "$VALIDATOR" "$BATS_TEST_TMPDIR/bad.json" + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "gate: a task naming a retired model fails" { + _validate_with '.tasks.deep.gemini = ["gemini-2.5-pro", "gemini-3.8-flash"]' + [ "$status" -ne 0 ] + [[ "$output" == *"gemini-2.5-pro"*"retired"* ]] +} + +@test "gate: a task naming a model missing from the catalog fails" { + _validate_with '.tasks.audit.claude = ["claude-opus-9-9"]' + [ "$status" -ne 0 ] + [[ "$output" == *"claude-opus-9-9"*"not in models"* ]] +} + +@test "gate: a model in another provider's chain fails" { + _validate_with '.tasks.triage.gemini = ["claude-sonnet-5-5"]' + [ "$status" -ne 0 ] + [[ "$output" == *"claude-sonnet-5-5"*"provider claude"* ]] +} + +@test "gate: an enabled provider's model without a price row fails" { + _validate_with '.models["mistral-large"] = {"provider": "copilot", "status": "active"}' + [ "$status" -ne 0 ] + [[ "$output" == *"mistral-large"*"model-pricing.tsv"* ]] + # A disabled provider's models need no price row. + _validate_with '.models["mistral-large"] = {"provider": "copilot", "status": "active"} | .providers.copilot.enabled = false' + [ "$status" -eq 0 ] +} + +@test "gate: a vendor-prefixed Copilot id is priced by its bare name" { + # No row matches "azure/o4-mini" itself; the bare "o4-mini" matches o4-mini*. + _validate_with '.models["azure/o4-mini"] = {"provider": "copilot", "status": "active"}' + [ "$status" -eq 0 ] +} + +@test "gate: duck with more than one model fails" { + _validate_with '.tasks.duck.claude = ["claude-sonnet-5-5", "claude-sonnet-5"]' + [ "$status" -ne 0 ] + [[ "$output" == *"duck"*"one model"* ]] +} + +@test "gate: a Copilot chain with more than one model fails" { + _validate_with '.models["openai/gpt-5"] = {"provider": "copilot", "status": "active"} | .tasks.deep.copilot = ["openai/o4-mini", "openai/gpt-5"]' + [ "$status" -ne 0 ] + [[ "$output" == *"copilot"*"one model"* ]] +} + +@test "gate: fallback_order must name each provider once" { + _validate_with '.providers.fallback_order = ["claude", "claude", "gemini"]' + [ "$status" -ne 0 ] +} + +@test "gate: every problem is reported, not only the first" { + _validate_with '.tasks.audit.claude = ["claude-opus-9-9"] | .tasks.duck.claude = ["claude-sonnet-5-5", "claude-sonnet-5"]' + [ "$status" -ne 0 ] + [[ "$output" == *"claude-opus-9-9"* ]] + [[ "$output" == *"duck"* ]] +} + +@test "lint.yml runs the gate" { + grep -q 'python3 scripts/validate-ai-engines.py' "$REPO_ROOT/.github/workflows/lint.yml" +} From 61a7d2f51d1a8725f69bd8cf6125adaefb420f63 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:00:03 +0000 Subject: [PATCH 02/29] fix(sonar): pin jsonschema in validate-ai-engines job and confine validator CLI paths Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/lint.yml | 2 +- scripts/validate-ai-engines.py | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index efd06b35d..cb31f560e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -557,7 +557,7 @@ jobs: persist-credentials: false - name: Install Python deps - run: pip3 install --quiet 'jsonschema>=4' + run: pip3 install --quiet "jsonschema==4.26.0" --only-binary=":all:" - name: Validate config/ai-engines.json run: python3 scripts/validate-ai-engines.py diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py index d89393ca9..f69e1c4ee 100755 --- a/scripts/validate-ai-engines.py +++ b/scripts/validate-ai-engines.py @@ -19,8 +19,10 @@ Every problem is reported as a `::error::` line. Exit 0 when valid, 1 otherwise. """ import json +import os import re import sys +import tempfile from pathlib import Path REPO = Path(__file__).resolve().parent.parent @@ -32,6 +34,17 @@ def error(msg: str) -> None: print(f"::error::ai-engines config invalid: {msg}", file=sys.stderr) +def safe_path(arg: str, what: str) -> Path: + """Resolve a CLI-supplied path and confine it to the repo or the system temp dir + (the bats tests stage fixtures there), refusing traversal elsewhere.""" + resolved = Path(os.path.realpath(arg)) + roots = (os.path.realpath(REPO), os.path.realpath(tempfile.gettempdir())) + if not any(resolved == Path(r) or Path(r) in resolved.parents for r in roots): + error(f"{what} path {arg} is outside the repository and the temp dir") + sys.exit(1) + return resolved + + def load_json(path: Path, what: str): try: return json.loads(path.read_text(encoding="utf-8")) @@ -112,9 +125,9 @@ def check_references(config, globs) -> list: def main(argv) -> int: - config_path = Path(argv[1]) if len(argv) > 1 else REPO / "config" / "ai-engines.json" - schema_path = Path(argv[2]) if len(argv) > 2 else REPO / "config" / "ai-engines.schema.json" - pricing_path = Path(argv[3]) if len(argv) > 3 else REPO / "scripts" / "lib" / "model-pricing.tsv" + config_path = safe_path(argv[1], "config") if len(argv) > 1 else REPO / "config" / "ai-engines.json" + schema_path = safe_path(argv[2], "schema") if len(argv) > 2 else REPO / "config" / "ai-engines.schema.json" + pricing_path = safe_path(argv[3], "pricing") if len(argv) > 3 else REPO / "scripts" / "lib" / "model-pricing.tsv" config = load_json(config_path, "config") schema = load_json(schema_path, "schema") From 738475f94b32b95519deb3b3d59f7b1e1c3b51f3 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:21:45 +0000 Subject: [PATCH 03/29] fix(reviews): validate chains for enabled providers, fail closed on all-disabled, honour file duck chains, harden manifest path check Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/lint.yml | 1 + AGENTS.md | 5 +++-- scripts/lib/engine-chain.sh | 16 ++++++++++------ scripts/lib/engine-models.sh | 2 ++ scripts/lib/validate-consumer-manifest.sh | 5 +++-- scripts/pr_review_health.sh | 9 ++++++++- scripts/validate-ai-engines.py | 12 +++++++++++- 7 files changed, 38 insertions(+), 12 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cb31f560e..26f18bc6d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -215,6 +215,7 @@ jobs: tests/test_eval_health_notify.bats \ tests/test_consumer_manifest.bats \ tests/test_validate_ai_engines.bats \ + tests/dev-lead/unit/test_ai_engines_config.bats \ tests/test_downstream_impact_map.bats \ tests/test_downstream_impact_assemble.bats \ tests/test_downstream_impact_triage_inline.bats \ diff --git a/AGENTS.md b/AGENTS.md index 8a7b08857..1c96a57f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -804,12 +804,13 @@ current concrete id at run time; do **not** hard-pin `claude--` ids across the tree (#1979, companion [`combsmith/.github#1199`](https://github.com/combsmith/.github/issues/1199)). A model swap should be one edit to the chains in -[`scripts/lib/engine-models.sh`](./scripts/lib/engine-models.sh), not a hunt for +[`config/ai-engines.json`](./config/ai-engines.json) (validated by +`scripts/validate-ai-engines.py`), not a hunt for scattered ids. - **Resolve with `ai_model_for_family `** (`scripts/lib/engine-models.sh`): it returns the current id for the family, honouring `CLAUDE__MODEL_CHAIN` - then `AI_MODELS_CLAUDE` then the built-in default. A shell caller sources the lib + then `AI_MODELS_CLAUDE` then `config/ai-engines.json`. A shell caller sources the lib and calls the function; a workflow adds a resolve step that writes the id to a step output. In gh-aw front-matter, `engine:` names the runner engine (e.g. `claude`) and the model family is named in `models:` (e.g. diff --git a/scripts/lib/engine-chain.sh b/scripts/lib/engine-chain.sh index b14940b4e..7556cc161 100644 --- a/scripts/lib/engine-chain.sh +++ b/scripts/lib/engine-chain.sh @@ -38,14 +38,18 @@ if ! declare -F ai_engines_file_providers >/dev/null 2>&1 \ source "$(dirname "${BASH_SOURCE[0]}")/engine-models.sh" fi -# _ai_engine_default_chain — the file's enabled providers in fallback_order, -# else AI_ENGINES_DEFAULT. +# _ai_engine_default_chain — the file's enabled providers in fallback_order. +# AI_ENGINES_DEFAULT is used only when the file cannot be read; a file that +# enables nothing yields an empty chain (the validator rejects such a file) and +# never re-enables the providers it turned off. _ai_engine_default_chain() { - local c="" - if declare -F ai_engines_file_providers >/dev/null 2>&1; then - c="$(ai_engines_file_providers 2>/dev/null)" || c="" + local c + if declare -F ai_engines_file_providers >/dev/null 2>&1 \ + && c="$(ai_engines_file_providers 2>/dev/null)"; then + printf '%s' "$c" + return 0 fi - printf '%s' "${c:-$AI_ENGINES_DEFAULT}" + printf '%s' "$AI_ENGINES_DEFAULT" } # _ai_engine_file_disabled — the providers the file disables (space-separated). diff --git a/scripts/lib/engine-models.sh b/scripts/lib/engine-models.sh index 5e5b5381f..479a90c8e 100644 --- a/scripts/lib/engine-models.sh +++ b/scripts/lib/engine-models.sh @@ -267,6 +267,7 @@ ai_models_gemini_chain() { deep|audit|single) group="${GEMINI_PRO_MODEL_CHAIN:-}"; first="${GEMINI_PRO_MODEL:-}" ;; duck) c="$(ai_models_configured gemini duck)" + [ -n "$c" ] || c="$(ai_models_default gemini duck)" || return 1 [ -n "$c" ] || { c="$(ai_models_gemini_chain triage)" || return 1; c="${c%%,*}"; } printf '%s' "$c" return 0 ;; @@ -294,6 +295,7 @@ ai_models_copilot_model() { m="$COPILOT_API_MODEL" else m="$(ai_models_configured copilot "$key")" + [ -n "$m" ] || [ "$key" != duck ] || m="$(ai_models_default copilot duck)" || return 1 [ -n "$m" ] || [ "$key" != duck ] || key=triage [ -n "$m" ] || m="$(ai_models_chain copilot "$key")" || return 1 fi diff --git a/scripts/lib/validate-consumer-manifest.sh b/scripts/lib/validate-consumer-manifest.sh index a06890662..4cccf422d 100755 --- a/scripts/lib/validate-consumer-manifest.sh +++ b/scripts/lib/validate-consumer-manifest.sh @@ -131,8 +131,9 @@ while IFS= read -r value; do continue ;; esac resolved=$(realpath -m "$REPO_ROOT/$value") - if [ "${resolved#"${lib_root}/"}" = "$resolved" ] && [ "${resolved#"${prompts_root}/"}" = "$resolved" ] \ - && [ "${resolved#"${config_root}/"}" = "$resolved" ]; then + if [[ "$value" == *..* ]] \ + || { [[ "${resolved#"${lib_root}/"}" == "$resolved" ]] && [[ "${resolved#"${prompts_root}/"}" == "$resolved" ]] \ + && [[ "${resolved#"${config_root}/"}" == "$resolved" ]]; }; then err "surface_sources value contains path traversal: $value" continue fi diff --git a/scripts/pr_review_health.sh b/scripts/pr_review_health.sh index 8cd773d97..35a21eaa6 100644 --- a/scripts/pr_review_health.sh +++ b/scripts/pr_review_health.sh @@ -343,7 +343,14 @@ workflow_source=$(gh api "repos/${WORKFLOW_REPO}/contents/.github/workflows/${WO engine_overrides="" printf 'Could not read `config/ai-engines.json` — see the run log.\n\n' elif [ -z "$engine_overrides" ]; then - printf 'None: every engine and model setting comes from `config/ai-engines.json`.\n\n' + model_problems="$(ai_models_problems)" + if [ -n "$model_problems" ]; then + printf 'Invalid engine model overrides (ignored):\n\n' + printf '%s\n' "$model_problems" | sed 's/^/- `/; s/$/`/' + printf '\n' + else + printf 'None: every engine and model setting comes from `config/ai-engines.json`.\n\n' + fi else printf 'These Actions variables override `config/ai-engines.json`. Fold each into the file through a normal release, then delete the variable.\n\n' printf '%s\n' "$engine_overrides" | sed 's/^/- `/; s/$/`/' diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py index f69e1c4ee..5392f4ce6 100755 --- a/scripts/validate-ai-engines.py +++ b/scripts/validate-ai-engines.py @@ -28,6 +28,7 @@ REPO = Path(__file__).resolve().parent.parent SINGLE_MODEL_TASKS = ("duck",) SINGLE_MODEL_PROVIDERS = ("copilot",) +PROVIDERS = ("claude", "gemini", "copilot") def error(msg: str) -> None: @@ -95,9 +96,18 @@ def check_references(config, globs) -> list: problems = [] models = config["models"] providers = config["providers"] + if not any(entry["enabled"] for name, entry in providers.items() if name in PROVIDERS): + problems.append("providers: every provider is disabled; enable at least one") for task, chains in config["tasks"].items(): + if not isinstance(chains, dict): + continue # $comment + for provider in PROVIDERS: + # The duck runs on Claude only; the other tasks need a chain per enabled provider. + needed = provider == "claude" if task in SINGLE_MODEL_TASKS else True + if needed and providers[provider]["enabled"] and provider not in chains: + problems.append(f"tasks.{task}.{provider}: missing chain for enabled provider {provider}") for provider, chain in chains.items(): - if provider not in ("claude", "gemini", "copilot"): + if provider not in PROVIDERS: continue # prefer, $comment for model in chain: entry = models.get(model) From a7817872aabcf8d7de12c674ecbfced6f3e2c784 Mon Sep 17 00:00:00 2001 From: don-petry <{}+don-petry@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:58:04 +0000 Subject: [PATCH 04/29] fix(reviews): read config before empty overrides, report problems with overrides, drop disabled providers, Copilot-only bare pricing Test-Change-Justification: the health-check workflow now forwards the legacy DEV_LEAD_ENGINES fallback (review thread PRRT_kwDOR9SdIs6q5c5L, #1973) and the invocation grep must not match the explanatory comment (PRRT_kwDOR9SdIs6q5c5a). Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/daily-pr-review-health.yml | 2 +- docs/engine-configuration.md | 4 ++-- scripts/lib/engine-models.sh | 7 ++++++- scripts/pr_review_health.sh | 14 ++++++++------ scripts/validate-ai-engines.py | 3 ++- tests/dev-lead/unit/test_ai_engines_config.bats | 4 ++-- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/daily-pr-review-health.yml b/.github/workflows/daily-pr-review-health.yml index 1407ee0e3..09005ad34 100644 --- a/.github/workflows/daily-pr-review-health.yml +++ b/.github/workflows/daily-pr-review-health.yml @@ -109,7 +109,7 @@ jobs: # differs from config/ai-engines.json (#1973). AI_MODELS_GEMINI: ${{ vars.AI_MODELS_GEMINI }} AI_MODELS_COPILOT: ${{ vars.AI_MODELS_COPILOT }} - AI_ENGINES: ${{ vars.AI_ENGINES }} + AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} run: bash scripts/pr_review_health.sh - name: Truncate report if needed diff --git a/docs/engine-configuration.md b/docs/engine-configuration.md index a7d097b5c..8cafa4aa1 100644 --- a/docs/engine-configuration.md +++ b/docs/engine-configuration.md @@ -45,9 +45,9 @@ draft 2020-12), and also fails when: - a task names a model that is missing from `models`, or is `retired`; - a model sits in another provider's chain; -- a model of an enabled provider has no price row in +- a non-retired model of an enabled provider has no price row in [`scripts/lib/model-pricing.tsv`](../scripts/lib/model-pricing.tsv) (a - vendor-prefixed id such as `openai/o4-mini` is priced by its bare name); + vendor-prefixed Copilot id such as `openai/o4-mini` is priced by its bare name); - `duck`, or any `copilot` chain, lists more than one model. Run it locally with `python3 scripts/validate-ai-engines.py` (needs diff --git a/scripts/lib/engine-models.sh b/scripts/lib/engine-models.sh index 479a90c8e..823dfce71 100644 --- a/scripts/lib/engine-models.sh +++ b/scripts/lib/engine-models.sh @@ -321,8 +321,10 @@ ai_models_problems() { # the same normalisation engine-chain.sh applies (case, separators). Prints # nothing when no override differs; returns 1 when the file cannot be read. ai_engines_overrides() { - local p k var c d spec chain="" t file + local p k var c d spec chain="" t file disabled local -a toks=() + # Read the file first, so an unreadable one is an error even with no override set. + ai_engines_config_load || return 1 for p in claude gemini copilot; do var="$(_ai_models_var "$p")" for k in $(_ai_models_keys "$p"); do @@ -339,8 +341,11 @@ ai_engines_overrides() { spec="${AI_ENGINES:-${DEV_LEAD_ENGINES:-}}" [ -n "${spec//[[:space:],]/}" ] || return 0 file="$(ai_engines_file_providers)" || return 1 + disabled="$(ai_engines_file_disabled)" || return 1 IFS=$' \t' read -r -a toks <<< "$(printf '%s' "$spec" | tr ',\n\r\t' ' ' | tr '[:upper:]' '[:lower:]')" for t in ${toks[@]+"${toks[@]}"}; do + # A provider the file disables never runs, so it is not part of the effective chain. + [[ " $disabled " != *" $t "* ]] || continue [[ " $chain " == *" $t "* ]] || chain="${chain:+$chain }$t" done [ "$chain" = "$file" ] || printf "AI_ENGINES=%s → %s (file: %s)\n" "$spec" "$chain" "$file" diff --git a/scripts/pr_review_health.sh b/scripts/pr_review_health.sh index 35a21eaa6..6528f9def 100644 --- a/scripts/pr_review_health.sh +++ b/scripts/pr_review_health.sh @@ -342,19 +342,21 @@ workflow_source=$(gh api "repos/${WORKFLOW_REPO}/contents/.github/workflows/${WO if ! engine_overrides="$(ai_engines_overrides)"; then engine_overrides="" printf 'Could not read `config/ai-engines.json` — see the run log.\n\n' - elif [ -z "$engine_overrides" ]; then + else model_problems="$(ai_models_problems)" + if [ -n "$engine_overrides" ]; then + printf 'These Actions variables override `config/ai-engines.json`. Fold each into the file through a normal release, then delete the variable.\n\n' + printf '%s\n' "$engine_overrides" | sed 's/^/- `/; s/$/`/' + printf '\n' + fi if [ -n "$model_problems" ]; then printf 'Invalid engine model overrides (ignored):\n\n' printf '%s\n' "$model_problems" | sed 's/^/- `/; s/$/`/' printf '\n' - else + fi + if [ -z "$engine_overrides" ] && [ -z "$model_problems" ]; then printf 'None: every engine and model setting comes from `config/ai-engines.json`.\n\n' fi - else - printf 'These Actions variables override `config/ai-engines.json`. Fold each into the file through a normal release, then delete the variable.\n\n' - printf '%s\n' "$engine_overrides" | sed 's/^/- `/; s/$/`/' - printf '\n' fi } > "$REPORT_FILE" rm -f "$ingress_attr_jsonl" diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py index 5392f4ce6..d18359959 100755 --- a/scripts/validate-ai-engines.py +++ b/scripts/validate-ai-engines.py @@ -126,7 +126,8 @@ def check_references(config, globs) -> list: for model, entry in models.items(): if entry["status"] == "retired" or not providers[entry["provider"]]["enabled"]: continue - bare = model.split("/", 1)[1] if "/" in model else model + # Only the Copilot path strips a vendor prefix before pricing a run. + bare = model.split("/", 1)[1] if entry["provider"] == "copilot" and "/" in model else model if not any(glob_matches(g, model) or glob_matches(g, bare) for g in globs): problems.append( f"models.{model}: provider {entry['provider']} is enabled but the model has " diff --git a/tests/dev-lead/unit/test_ai_engines_config.bats b/tests/dev-lead/unit/test_ai_engines_config.bats index a86579361..b48f5bfa4 100644 --- a/tests/dev-lead/unit/test_ai_engines_config.bats +++ b/tests/dev-lead/unit/test_ai_engines_config.bats @@ -170,11 +170,11 @@ _config_with() { } @test "overrides: the health check prints the override section" { - grep -q 'ai_engines_overrides' "$SCRIPT_DIR/scripts/pr_review_health.sh" + grep -q 'engine_overrides="$(ai_engines_overrides)"' "$SCRIPT_DIR/scripts/pr_review_health.sh" local wf="$SCRIPT_DIR/.github/workflows/daily-pr-review-health.yml" grep -q 'AI_MODELS_GEMINI: ${{ vars.AI_MODELS_GEMINI }}' "$wf" grep -q 'AI_MODELS_COPILOT: ${{ vars.AI_MODELS_COPILOT }}' "$wf" - grep -q 'AI_ENGINES: ${{ vars.AI_ENGINES }}' "$wf" + grep -q 'AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES' "$wf" } # ── Shipping the file ──────────────────────────────────────────────────────── From ad2b9ce54be52ae6a107f1ac92f4b4f5cf9acf71 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:12:52 +0000 Subject: [PATCH 05/29] fix(engine-models): ensure config is loaded before model lookups Direct model-family consumers (ai_model_for_family, ai_models_gemini_chain, ai_models_copilot_model) now call ai_engines_config_load before attempting config lookups. This prevents callers from receiving empty model strings when the config file is missing or malformed, ensuring errors are surfaced immediately rather than at invocation time. Co-Authored-By: Claude Haiku 4.5 --- scripts/lib/engine-models.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/lib/engine-models.sh b/scripts/lib/engine-models.sh index 823dfce71..2072a475c 100644 --- a/scripts/lib/engine-models.sh +++ b/scripts/lib/engine-models.sh @@ -262,6 +262,7 @@ ai_models_replace_first() { # first model. ai_models_gemini_chain() { local key="${1:-}" group first c + ai_engines_config_load || return 1 case "$key" in triage|action) group="${GEMINI_FLASH_MODEL_CHAIN:-}"; first="${GEMINI_FLASH_MODEL:-}" ;; deep|audit|single) group="${GEMINI_PRO_MODEL_CHAIN:-}"; first="${GEMINI_PRO_MODEL:-}" ;; @@ -291,6 +292,7 @@ ai_models_gemini_chain() { # a child shell) is not mistaken for the caller's. ai_models_copilot_model() { local key="${1:-}" m="" + ai_engines_config_load || return 1 if [ -n "${COPILOT_API_MODEL:-}" ] && [ "$COPILOT_API_MODEL" != "${COPILOT_API_MODEL_DEFAULTED:-}" ]; then m="$COPILOT_API_MODEL" else @@ -409,6 +411,7 @@ ai_model_for_family() { printf '::warning::ai_model_for_family: unknown family %s (expected opus|sonnet|haiku)\n' "$family" >&2 return 1 ;; esac + ai_engines_config_load || return 1 chain="${!var:-}" # Only an EMPTY per-tier var counts as absent (engine.sh's ${VAR:-…} rule). A # set-but-malformed value (",") must not let AI_MODELS_CLAUDE win; it fails the From 13e11d4514e65af64223969e6434a82b9c4a2ba1 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:57:47 +0000 Subject: [PATCH 06/29] fix(reviews): require an in-effect price row for enabled models in validate-ai-engines --- scripts/validate-ai-engines.py | 14 ++++++++++---- tests/test_validate_ai_engines.bats | 13 +++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py index d18359959..c9d9f8126 100755 --- a/scripts/validate-ai-engines.py +++ b/scripts/validate-ai-engines.py @@ -10,7 +10,8 @@ - every model a task names is in `models` and not `retired`; - every model a task names belongs to the provider of the chain it is in; - - every non-retired model of an enabled provider has a price row in + - every non-retired model of an enabled provider has a price row (effective + today) in model-pricing.tsv (a vendor-prefixed id such as openai/o4-mini is priced by its bare name, as token records store it); - `duck`, and every `copilot` chain, holds one model (neither has an @@ -18,6 +19,7 @@ Every problem is reported as a `::error::` line. Exit 0 when valid, 1 otherwise. """ +import datetime import json import os import re @@ -55,9 +57,10 @@ def load_json(path: Path, what: str): def price_globs(path: Path) -> list: - """The model_glob column of model-pricing.tsv (comments and short rows skipped, - as scripts/lib/model-pricing.sh does).""" + """The model_glob column of the model-pricing.tsv rows in effect today (comments + and short rows skipped, as scripts/lib/model-pricing.sh does).""" globs = [] + today = datetime.date.today().isoformat() try: lines = path.read_text(encoding="utf-8").splitlines() except OSError as exc: @@ -68,7 +71,10 @@ def price_globs(path: Path) -> list: continue cols = line.split("\t") if len(cols) >= 6 and cols[0]: - globs.append(cols[0]) + # model-pricing.sh's price_for ignores a row until its effective_from + # date, so a future-dated row does not price a model running today. + if cols[1].strip() <= today: + globs.append(cols[0]) return globs diff --git a/tests/test_validate_ai_engines.bats b/tests/test_validate_ai_engines.bats index cc494d1bd..2efccedb3 100644 --- a/tests/test_validate_ai_engines.bats +++ b/tests/test_validate_ai_engines.bats @@ -75,6 +75,19 @@ _validate_with() { [ "$status" -eq 0 ] } +@test "gate: a price row that is only future-dated does not price an enabled model" { + local pt="$BATS_TEST_TMPDIR/pricing.tsv" f="$BATS_TEST_TMPDIR/ai-engines.json" + jq '.models["mistral-large"] = {"provider": "copilot", "status": "active"}' "$CONFIG" > "$f" + grep -v '^#' "$REPO_ROOT/scripts/lib/model-pricing.tsv" > "$pt" + printf 'mistral-large*\t2999-01-01\t1\t1\t1\t1\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -ne 0 ] + [[ "$output" == *"mistral-large"*"model-pricing.tsv"* ]] + printf 'mistral-large*\t2000-01-01\t1\t1\t1\t1\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -eq 0 ] +} + @test "gate: a vendor-prefixed Copilot id is priced by its bare name" { # No row matches "azure/o4-mini" itself; the bare "o4-mini" matches o4-mini*. _validate_with '.models["azure/o4-mini"] = {"provider": "copilot", "status": "active"}' From 2ea40eabd577a5107ae83142b67ab3d75b5b12d2 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:28:42 +0000 Subject: [PATCH 07/29] fix(reviews): derive price cutoff date from UTC in validate-ai-engines --- scripts/validate-ai-engines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py index c9d9f8126..9335e7a3a 100755 --- a/scripts/validate-ai-engines.py +++ b/scripts/validate-ai-engines.py @@ -60,7 +60,7 @@ def price_globs(path: Path) -> list: """The model_glob column of the model-pricing.tsv rows in effect today (comments and short rows skipped, as scripts/lib/model-pricing.sh does).""" globs = [] - today = datetime.date.today().isoformat() + today = datetime.datetime.now(datetime.timezone.utc).date().isoformat() try: lines = path.read_text(encoding="utf-8").splitlines() except OSError as exc: From 0f7ee3b10a8b98ff15972411733f0ec488b40769 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:05:23 +0000 Subject: [PATCH 08/29] fix(bot): address Codex engine-config findings - Preserve explicitly narrowed empty engine chain (issue #4) - ai_engine_chain() no longer falls back to default when AI_ENGINES explicitly sets to only disabled providers; returns empty instead - Load provider disablement before token preflight (issue #1) - Checkout config/ai-engines.json before preflight step - Check file for provider disablement before requiring Claude token - Run engine-config validation in required check (issue #3) - Create new engine-config-validate.yml workflow as required check - Validates config/ai-engines.json against schema and constraints Co-Authored-By: Claude Haiku 4.5 --- .github/workflows/dev-lead-reusable.yml | 73 +++++++++++++------- .github/workflows/engine-config-validate.yml | 44 ++++++++++++ scripts/lib/engine-chain.sh | 22 ++++-- 3 files changed, 108 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/engine-config-validate.yml diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index 35fe31a70..25de0e51e 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -211,35 +211,58 @@ jobs: TOKEN_WORKFLOW: dev-lead steps: + - name: Checkout caller repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + + - name: Checkout dev-lead config + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: combsmith/.github-private + ref: ${{ inputs.agent_ref }} + path: .dev-lead-config + token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + sparse-checkout: config + - name: Engine token preflight (#1525) # With the secret declaration relaxed to required:false, presence is # enforced here instead — one actionable line, not a secrets-evaluation - # stack error on every event. Skipped when AI_ENGINES leaves claude out. + # stack error on every event. Skipped when AI_ENGINES leaves claude out or + # config/ai-engines.json disables Claude (provider disablement takes + # precedence over AI_ENGINES, so check the file before requiring the token). env: AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | - # Is Claude enabled? Only a valid AI_ENGINES list that omits claude turns it - # off; empty or invalid values fall back to the default chain, which includes - # claude (scripts/lib/engine-chain.sh — mirrored inline because this runs - # before the scripts are checked out). + # Is Claude enabled? Check config/ai-engines.json first (if it exists), + # then fall back to the inline AI_ENGINES logic. claude_enabled=true - spec="$(printf '%s' "${AI_ENGINES:-}" | tr '[:upper:],' '[:lower:] ')" - if [ -n "${spec//[[:space:]]/}" ]; then - set -f - known=true; has_claude=false - for e in $spec; do - case "$e" in - claude) has_claude=true ;; - gemini|copilot) ;; - *) known=false ;; - esac - done - set +f - if [ "$known" = true ] && [ "$has_claude" = false ]; then claude_enabled=false; fi + if [ -f .dev-lead-config/config/ai-engines.json ]; then + # config/ai-engines.json exists; check if claude provider is disabled. + if grep -q '"claude".*"enabled"\s*:\s*false' .dev-lead-config/config/ai-engines.json; then + claude_enabled=false + fi + fi + if [ "$claude_enabled" = true ]; then + # Config doesn't disable Claude; check if AI_ENGINES omits it. + spec="$(printf '%s' "${AI_ENGINES:-}" | tr '[:upper:],' '[:lower:] ')" + if [ -n "${spec//[[:space:]]/}" ]; then + set -f + known=true; has_claude=false + for e in $spec; do + case "$e" in + claude) has_claude=true ;; + gemini|copilot) ;; + *) known=false ;; + esac + done + set +f + if [ "$known" = true ] && [ "$has_claude" = false ]; then claude_enabled=false; fi + fi fi if [ "$claude_enabled" = false ]; then - echo "claude not in AI_ENGINES (${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" + echo "claude disabled in config or not in AI_ENGINES (${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" exit 0 fi if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then @@ -252,10 +275,8 @@ jobs: fi echo "engine token present — proceeding" - - name: Checkout caller repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + - name: Clean up dev-lead config checkout + run: rm -rf .dev-lead-config - name: Checkout dev-lead scripts uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -799,7 +820,7 @@ jobs: - name: Checkout caller repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - token: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} - name: Checkout dev-lead scripts uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -807,7 +828,7 @@ jobs: repository: combsmith/.github-private ref: ${{ inputs.agent_ref }} path: .dev-lead - token: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} sparse-checkout: | scripts prompts/dev-lead @@ -1028,7 +1049,7 @@ jobs: # defaults to "main" → unchanged behaviour (#535/#506). ref: ${{ inputs.agent_ref }} path: .dev-lead - token: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} persist-credentials: false # config/ holds ai-engines.json, the engine defaults the scripts read (#1973). sparse-checkout: | diff --git a/.github/workflows/engine-config-validate.yml b/.github/workflows/engine-config-validate.yml new file mode 100644 index 000000000..6df71bb9f --- /dev/null +++ b/.github/workflows/engine-config-validate.yml @@ -0,0 +1,44 @@ +name: Validate Engine Config + +on: + pull_request: + paths: + - 'config/ai-engines.json' + - 'config/ai-engines.schema.json' + - 'scripts/lib/model-pricing.tsv' + - 'scripts/validate-ai-engines.py' + - '.github/workflows/engine-config-validate.yml' + push: + branches: [main] + paths: + - 'config/ai-engines.json' + - 'config/ai-engines.schema.json' + - 'scripts/lib/model-pricing.tsv' + - 'scripts/validate-ai-engines.py' + merge_group: + +permissions: + contents: read + +jobs: + validate-ai-engines: + # The engine config gate (#1973). config/ai-engines.json is the source of + # truth for enabled providers, the model catalog and each task's chain, so a + # bad value is caught at PR time instead of in a failing review. Validates + # the file against config/ai-engines.schema.json (draft 2020-12), then fails + # on a task model that is missing from `models` or `retired`, a model in + # another provider's chain, an enabled provider's model with no row in + # scripts/lib/model-pricing.tsv, and a duck or Copilot chain with more than + # one model. Fully HERMETIC. This is the required check for config changes. + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install Python deps + run: pip3 install --quiet "jsonschema==4.26.0" --only-binary=":all:" + + - name: Validate config/ai-engines.json + run: python3 scripts/validate-ai-engines.py diff --git a/scripts/lib/engine-chain.sh b/scripts/lib/engine-chain.sh index 7556cc161..ab19d52ea 100644 --- a/scripts/lib/engine-chain.sh +++ b/scripts/lib/engine-chain.sh @@ -85,17 +85,29 @@ _ai_engine_parse() { } # ai_engine_chain — the enabled engines in preference order, space-separated. -# Engines the file disables are dropped. Empty or invalid configuration, or one -# that names only disabled engines → the default chain. +# Engines the file disables are dropped. Empty or invalid configuration → the +# default chain. A valid configuration that names only disabled engines → empty +# (fails closed rather than silently re-enabling providers the spec excluded). ai_engine_chain() { - local parsed disabled e kept="" - if parsed="$(_ai_engine_parse "$(_ai_engine_spec)")" && [ -n "$parsed" ]; then + local spec parsed disabled e kept="" + spec="$(_ai_engine_spec)" + if [ -z "$spec" ]; then + printf '%s' "$(_ai_engine_default_chain)" + return 0 + fi + if parsed="$(_ai_engine_parse "$spec")" && [ -n "$parsed" ]; then disabled="$(_ai_engine_file_disabled)" for e in $parsed; do [[ " $disabled " == *" $e "* ]] || kept="${kept:+$kept }$e" done fi - printf '%s' "${kept:-$(_ai_engine_default_chain)}" + if [ -z "$kept" ] && [ -z "${spec//[[:space:],]/}" ]; then + printf '%s' "$(_ai_engine_default_chain)" + elif [ -z "$kept" ] && ! _ai_engine_parse "$spec" >/dev/null 2>&1; then + printf '%s' "$(_ai_engine_default_chain)" + else + printf '%s' "$kept" + fi } # ai_engine_chain_problem — prints a one-line description when the configured From b7234b079e25b80b9327d11bc92a02846e53b5e6 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 10 Oct 2026 05:51:51 +0000 Subject: [PATCH 09/29] fix(reviews): parse Claude enablement with jq, preflight config in both jobs, require Claude chains, document lint exception Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/dev-lead-reusable.yml | 36 +++++++++++++++++++------ AGENTS.md | 4 +++ scripts/validate-ai-engines.py | 7 ++++- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index 25de0e51e..aaece0ecc 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -239,8 +239,10 @@ jobs: # then fall back to the inline AI_ENGINES logic. claude_enabled=true if [ -f .dev-lead-config/config/ai-engines.json ]; then - # config/ai-engines.json exists; check if claude provider is disabled. - if grep -q '"claude".*"enabled"\s*:\s*false' .dev-lead-config/config/ai-engines.json; then + # Parse (never grep) the file; fail closed — keep requiring the + # token — when .providers.claude.enabled cannot be resolved. + cfg_claude="$(jq -r '.providers.claude.enabled' .dev-lead-config/config/ai-engines.json 2>/dev/null || echo unknown)" + if [ "$cfg_claude" = false ]; then claude_enabled=false fi fi @@ -1003,18 +1005,36 @@ jobs: issues: read steps: + - name: Checkout dev-lead config + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: petry-projects/.github-private + ref: ${{ inputs.agent_ref }} + path: .dev-lead-config + token: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + persist-credentials: false + sparse-checkout: config + - name: Engine token preflight (#1525) env: AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | - # Is Claude enabled? Only a valid AI_ENGINES list that omits claude turns it - # off; empty or invalid values fall back to the default chain, which includes - # claude (scripts/lib/engine-chain.sh — mirrored inline because this runs - # before the scripts are checked out). + # Is Claude enabled? config/ai-engines.json disabling Claude takes + # precedence; otherwise only a valid AI_ENGINES list that omits claude turns + # it off; empty or invalid values fall back to the default chain, which + # includes claude (scripts/lib/engine-chain.sh — mirrored inline because this + # runs before the scripts are checked out). claude_enabled=true + if [ -f .dev-lead-config/config/ai-engines.json ]; then + # Parse (never grep) the file; fail closed when unresolvable. + cfg_claude="$(jq -r '.providers.claude.enabled' .dev-lead-config/config/ai-engines.json 2>/dev/null || echo unknown)" + if [ "$cfg_claude" = false ]; then + claude_enabled=false + fi + fi spec="$(printf '%s' "${AI_ENGINES:-}" | tr '[:upper:],' '[:lower:] ')" - if [ -n "${spec//[[:space:]]/}" ]; then + if [ "$claude_enabled" = true ] && [ -n "${spec//[[:space:]]/}" ]; then set -f known=true; has_claude=false for e in $spec; do @@ -1028,7 +1048,7 @@ jobs: if [ "$known" = true ] && [ "$has_claude" = false ]; then claude_enabled=false; fi fi if [ "$claude_enabled" = false ]; then - echo "claude not in AI_ENGINES (${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" + echo "claude disabled in config or not in AI_ENGINES (${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" exit 0 fi if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then diff --git a/AGENTS.md b/AGENTS.md index 1c96a57f4..3e5a2ce20 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -230,6 +230,10 @@ calling it a regression, and treat CI as authoritative.** agentic workflow compilation. It is not covered by the org template and must not be removed by template syncs. If the org template gains a `gh-aw-compile` equivalent, remove this exception and defer to the template instead. +- **Exception:** The `validate-ai-engines` job in `lint.yml` (#1973) is a documented repo-specific addition + that runs `scripts/validate-ai-engines.py` against `config/ai-engines.json`; the `paths:` filters also carry + `config/**` for it. It is not covered by the org template and must not be removed by template syncs. If the + org template gains an equivalent, remove this exception and defer to the template instead. - **Exception:** the `lint.yml` `paths:` filters (pull_request and push) carry the repo-specific entry `docs/initiatives/agent-ingress-collapse-markets.md`, which is read by `tests/agent_ingress_markets_package.bats` (#2038). It must not be removed as template drift on a sync; diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py index 9335e7a3a..c20522256 100755 --- a/scripts/validate-ai-engines.py +++ b/scripts/validate-ai-engines.py @@ -109,8 +109,13 @@ def check_references(config, globs) -> list: continue # $comment for provider in PROVIDERS: # The duck runs on Claude only; the other tasks need a chain per enabled provider. + # Claude chains are always required: idea-triage and persona-runner resolve + # Claude models independently of provider routing, so disabling Claude for + # pr-review/dev-lead must not drop them. needed = provider == "claude" if task in SINGLE_MODEL_TASKS else True - if needed and providers[provider]["enabled"] and provider not in chains: + if provider == "claude" and provider not in chains: + problems.append(f"tasks.{task}.claude: missing chain (required even when Claude is disabled)") + elif needed and providers[provider]["enabled"] and provider not in chains: problems.append(f"tasks.{task}.{provider}: missing chain for enabled provider {provider}") for provider, chain in chains.items(): if provider not in PROVIDERS: From ffe6d52a88c5ac7380d625a7a1468b1d665c3d6a Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 10 Oct 2026 06:11:27 +0000 Subject: [PATCH 10/29] fix(reviews): classify engine-config-validate in interaction model; keep engine token preflight first by fetching config via API --- .github/workflows/dev-lead-reusable.yml | 63 ++++++++++--------------- docs/agentic-interaction-model.md | 1 + 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index aaece0ecc..49a6626cb 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -211,20 +211,6 @@ jobs: TOKEN_WORKFLOW: dev-lead steps: - - name: Checkout caller repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} - - - name: Checkout dev-lead config - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: combsmith/.github-private - ref: ${{ inputs.agent_ref }} - path: .dev-lead-config - token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} - sparse-checkout: config - - name: Engine token preflight (#1525) # With the secret declaration relaxed to required:false, presence is # enforced here instead — one actionable line, not a secrets-evaluation @@ -233,18 +219,21 @@ jobs: # precedence over AI_ENGINES, so check the file before requiring the token). env: AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} + AGENT_REF: ${{ inputs.agent_ref }} + GH_TOKEN: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | # Is Claude enabled? Check config/ai-engines.json first (if it exists), # then fall back to the inline AI_ENGINES logic. claude_enabled=true - if [ -f .dev-lead-config/config/ai-engines.json ]; then - # Parse (never grep) the file; fail closed — keep requiring the - # token — when .providers.claude.enabled cannot be resolved. - cfg_claude="$(jq -r '.providers.claude.enabled' .dev-lead-config/config/ai-engines.json 2>/dev/null || echo unknown)" - if [ "$cfg_claude" = false ]; then - claude_enabled=false - fi + # Fetched through the API, not a checkout, so this stays the first step. + # Parse (never grep) the file; fail closed — keep requiring the token — + # when .providers.claude.enabled cannot be fetched or resolved. + cfg_claude="$(gh api -H 'Accept: application/vnd.github.raw' \ + "repos/petry-projects/.github-private/contents/config/ai-engines.json?ref=${AGENT_REF}" 2>/dev/null \ + | jq -r '.providers.claude.enabled' 2>/dev/null || echo unknown)" + if [ "$cfg_claude" = false ]; then + claude_enabled=false fi if [ "$claude_enabled" = true ]; then # Config doesn't disable Claude; check if AI_ENGINES omits it. @@ -277,8 +266,10 @@ jobs: fi echo "engine token present — proceeding" - - name: Clean up dev-lead config checkout - run: rm -rf .dev-lead-config + - name: Checkout caller repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + token: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} - name: Checkout dev-lead scripts uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -1005,19 +996,11 @@ jobs: issues: read steps: - - name: Checkout dev-lead config - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: petry-projects/.github-private - ref: ${{ inputs.agent_ref }} - path: .dev-lead-config - token: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} - persist-credentials: false - sparse-checkout: config - - name: Engine token preflight (#1525) env: AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} + AGENT_REF: ${{ inputs.agent_ref }} + GH_TOKEN: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | # Is Claude enabled? config/ai-engines.json disabling Claude takes @@ -1026,12 +1009,14 @@ jobs: # includes claude (scripts/lib/engine-chain.sh — mirrored inline because this # runs before the scripts are checked out). claude_enabled=true - if [ -f .dev-lead-config/config/ai-engines.json ]; then - # Parse (never grep) the file; fail closed when unresolvable. - cfg_claude="$(jq -r '.providers.claude.enabled' .dev-lead-config/config/ai-engines.json 2>/dev/null || echo unknown)" - if [ "$cfg_claude" = false ]; then - claude_enabled=false - fi + # Fetched through the API, not a checkout, so this stays the first step. + # Parse (never grep) the file; fail closed — keep requiring the token — + # when .providers.claude.enabled cannot be fetched or resolved. + cfg_claude="$(gh api -H 'Accept: application/vnd.github.raw' \ + "repos/petry-projects/.github-private/contents/config/ai-engines.json?ref=${AGENT_REF}" 2>/dev/null \ + | jq -r '.providers.claude.enabled' 2>/dev/null || echo unknown)" + if [ "$cfg_claude" = false ]; then + claude_enabled=false fi spec="$(printf '%s' "${AI_ENGINES:-}" | tr '[:upper:],' '[:lower:] ')" if [ "$claude_enabled" = true ] && [ -n "${spec//[[:space:]]/}" ]; then diff --git a/docs/agentic-interaction-model.md b/docs/agentic-interaction-model.md index b7734a360..b82d3ebf6 100644 --- a/docs/agentic-interaction-model.md +++ b/docs/agentic-interaction-model.md @@ -240,6 +240,7 @@ keeps its own file) carries **no** job qualifier and remains exactly one row. > | `.github/workflows/test-aw.yml` | CI test workflow, not an agentic role | > | `.github/workflows/duplicate-decl-gate.yml` | CI gate guard — enforces duplicate declaration check, not an agentic role | > | `.github/workflows/holdout-guard.yml` | Gate guard — enforces merge conditions, not an agentic role | +> | `.github/workflows/engine-config-validate.yml` | CI gate guard — validates config/ai-engines.json, not an agentic role | > | `.github/workflows/test-deletion-guard.yml` | Gate guard — enforces test retention, not an agentic role | > | `.github/workflows/sync-scope-guard.yml` | Gate guard — enforces sync-PR scope, not an agentic role | > | `.github/workflows/dependabot-automerge.yml` | Dependabot plumbing — thin caller stub | From 50a94a4a455525708c0d51f5b7b9f412fc529988 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:52:34 +0000 Subject: [PATCH 11/29] fix(engine-chain): fail closed when config file cannot be read engine-chain.sh was silently falling back to the built-in provider defaults if the config file could not be read. This could undo provider disablement when a caller expected the file to be authoritative. Changed all functions that depend on ai_engines_file_providers and ai_engines_file_disabled to propagate read errors instead of silently falling back to defaults. - _ai_engine_default_chain: fails when file cannot be read (returns 1) - _ai_engine_file_disabled: propagates errors instead of silencing with '|| true' - ai_engine_chain: propagates errors from both helper functions - ai_engine_chain_problem: propagates errors so invalid config is caught - ai_engine_enabled, ai_engine_primary, ai_engine_next_available, ai_engine_first_available: all propagate config read errors Callers that source engine-chain.sh directly must now check for config availability before using the provider chain; failures are no longer hidden. Co-Authored-By: Claude Haiku 4.5 --- scripts/lib/engine-chain.sh | 64 +++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/scripts/lib/engine-chain.sh b/scripts/lib/engine-chain.sh index ab19d52ea..735bc9cf7 100644 --- a/scripts/lib/engine-chain.sh +++ b/scripts/lib/engine-chain.sh @@ -39,23 +39,26 @@ if ! declare -F ai_engines_file_providers >/dev/null 2>&1 \ fi # _ai_engine_default_chain — the file's enabled providers in fallback_order. -# AI_ENGINES_DEFAULT is used only when the file cannot be read; a file that -# enables nothing yields an empty chain (the validator rejects such a file) and -# never re-enables the providers it turned off. +# AI_ENGINES_DEFAULT is used only when engine-models.sh is not available; if +# the file cannot be read when the lib is available, fails closed (returns 1) +# so callers never silently re-enable providers the file disables. _ai_engine_default_chain() { local c - if declare -F ai_engines_file_providers >/dev/null 2>&1 \ - && c="$(ai_engines_file_providers 2>/dev/null)"; then + if declare -F ai_engines_file_providers >/dev/null 2>&1; then + # The function exists (engine-models.sh was sourced); must succeed. + c="$(ai_engines_file_providers)" || return 1 printf '%s' "$c" return 0 fi + # Function not available; use the built-in default. printf '%s' "$AI_ENGINES_DEFAULT" } # _ai_engine_file_disabled — the providers the file disables (space-separated). +# Returns 1 when the file cannot be read (if engine-models.sh was sourced). _ai_engine_file_disabled() { declare -F ai_engines_file_disabled >/dev/null 2>&1 || return 0 - ai_engines_file_disabled 2>/dev/null || true + ai_engines_file_disabled } # _ai_engine_spec — the raw configured value (AI_ENGINES, else DEV_LEAD_ENGINES). @@ -88,23 +91,27 @@ _ai_engine_parse() { # Engines the file disables are dropped. Empty or invalid configuration → the # default chain. A valid configuration that names only disabled engines → empty # (fails closed rather than silently re-enabling providers the spec excluded). +# Returns 1 if the config file cannot be read (when engine-models.sh is sourced). ai_engine_chain() { - local spec parsed disabled e kept="" + local spec parsed disabled e kept="" chain spec="$(_ai_engine_spec)" if [ -z "$spec" ]; then - printf '%s' "$(_ai_engine_default_chain)" + chain="$(_ai_engine_default_chain)" || return 1 + printf '%s' "$chain" return 0 fi if parsed="$(_ai_engine_parse "$spec")" && [ -n "$parsed" ]; then - disabled="$(_ai_engine_file_disabled)" + disabled="$(_ai_engine_file_disabled)" || return 1 for e in $parsed; do [[ " $disabled " == *" $e "* ]] || kept="${kept:+$kept }$e" done fi if [ -z "$kept" ] && [ -z "${spec//[[:space:],]/}" ]; then - printf '%s' "$(_ai_engine_default_chain)" + chain="$(_ai_engine_default_chain)" || return 1 + printf '%s' "$chain" elif [ -z "$kept" ] && ! _ai_engine_parse "$spec" >/dev/null 2>&1; then - printf '%s' "$(_ai_engine_default_chain)" + chain="$(_ai_engine_default_chain)" || return 1 + printf '%s' "$chain" else printf '%s' "$kept" fi @@ -112,28 +119,32 @@ ai_engine_chain() { # ai_engine_chain_problem — prints a one-line description when the configured # value is unusable (unknown engine, nothing but separators, or an engine the -# file disables); prints nothing when it is fine or unset. +# file disables); prints nothing when it is fine or unset. Returns 1 if the +# config file cannot be read (when engine-models.sh is sourced). ai_engine_chain_problem() { - local spec parsed disabled e off="" + local spec parsed disabled e off="" chain spec="$(_ai_engine_spec)" [ -n "$spec" ] || return 0 if [ -z "${spec//[[:space:],]/}" ]; then + chain="$(_ai_engine_default_chain)" || return 1 printf "AI_ENGINES='%s' lists no engine — using the default chain '%s'" \ - "$spec" "$(_ai_engine_default_chain)" + "$spec" "$chain" return 0 elif ! parsed="$(_ai_engine_parse "$spec")" || [ -z "$parsed" ]; then + chain="$(_ai_engine_default_chain)" || return 1 printf "AI_ENGINES='%s' names an unknown engine (expected claude, gemini, copilot) — using the default chain '%s'" \ - "$spec" "$(_ai_engine_default_chain)" + "$spec" "$chain" return 0 fi - disabled="$(_ai_engine_file_disabled)" + disabled="$(_ai_engine_file_disabled)" || return 1 for e in $parsed; do [[ " $disabled " != *" $e "* ]] || off="${off:+$off, }$e" done if [ -n "$off" ]; then # AI_ENGINES narrows the file; it cannot turn a provider back on. + chain="$(ai_engine_chain)" || return 1 printf "AI_ENGINES='%s' names %s, which config/ai-engines.json has disabled — ignored (enable a provider in the file, not with AI_ENGINES); using '%s'" \ - "$spec" "$off" "$(ai_engine_chain)" + "$spec" "$off" "$chain" elif [[ "$spec" =~ ^[[:space:],] ]]; then # The workflows derive the primary with startsWith(vars.AI_ENGINES, …), # which cannot skip leading separators. @@ -143,8 +154,11 @@ ai_engine_chain_problem() { } # ai_engine_enabled — 0 when is in the configured chain. +# Returns 1 if the config file cannot be read (when engine-models.sh is sourced). ai_engine_enabled() { - [[ " $(ai_engine_chain) " == *" ${1:-} "* ]] + local chain + chain="$(ai_engine_chain)" || return 1 + [[ " $chain " == *" ${1:-} "* ]] } # ai_engine_primary [preferred] — when it is enabled, otherwise the @@ -152,11 +166,11 @@ ai_engine_enabled() { # An INVALID AI_ENGINES value means full defaults, primary included: the # workflows derive REVIEW_ENGINE / DEV_LEAD_ENGINE from the raw value's first # token, so honouring there would let "copilot,typo" still make -# Copilot primary (#1961 review). +# Copilot primary (#1961 review). Returns 1 if the config file cannot be read. ai_engine_primary() { local pref chain spec pref="$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" - chain="$(ai_engine_chain)" + chain="$(ai_engine_chain)" || return 1 spec="$(_ai_engine_spec)" if [ -n "${spec//[[:space:],]/}" ] && ! _ai_engine_parse "$spec" >/dev/null; then pref="" @@ -190,10 +204,10 @@ ai_engine_available() { # ai_engine_next_available — the first available engine AFTER # in the chain (forward only, so a rate-limit walk cannot cycle). # When is not in the chain, searches the whole chain. Prints nothing -# when there is none. +# when there is none. Returns 1 if the config file cannot be read. ai_engine_next_available() { local current="${1:-}" chain e seen=0 - chain="$(ai_engine_chain)" + chain="$(ai_engine_chain)" || return 1 [[ " $chain " == *" $current "* ]] || seen=1 for e in $chain; do if [ "$seen" -eq 0 ]; then @@ -208,9 +222,11 @@ ai_engine_next_available() { } # ai_engine_first_available — the first available engine in the chain. +# Returns 1 if the config file cannot be read (when engine-models.sh is sourced). ai_engine_first_available() { - local e - for e in $(ai_engine_chain); do + local e chain + chain="$(ai_engine_chain)" || return 1 + for e in $chain; do if ai_engine_available "$e"; then printf '%s' "$e" return 0 From 37297e3bd683d279faed83a8467c88ab14f71f42 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:45:06 +0000 Subject: [PATCH 12/29] fix(callers): fail closed when engine-chain config cannot be read Callers of engine-chain.sh functions (validate-engines.sh, dev-lead-preflight.sh, review-batch.sh) now properly check return codes when config read errors occur. Previously, config read errors were silently treated as disabled engines or ignored entirely, potentially allowing the chain to fall back to all built-in providers and undo provider disablement. Changes: - dev-lead-preflight.sh: check ai_engine_enabled return code and fail on config error - validate-engines.sh: check ai_engine_chain_problem and ai_engine_chain return codes - validate-engines.sh: update _validate_engine_enabled to propagate config errors - review-batch.sh: check ai_engine_primary, ai_engine_chain return codes throughout - All callers now exit with error when config cannot be read Co-Authored-By: Claude Haiku 4.5 --- scripts/dev-lead-preflight.sh | 9 +++- scripts/review-batch.sh | 43 +++++++++++++--- scripts/validate-engines.sh | 97 ++++++++++++++++++++++++----------- 3 files changed, 108 insertions(+), 41 deletions(-) diff --git a/scripts/dev-lead-preflight.sh b/scripts/dev-lead-preflight.sh index b3557cceb..e9acb014d 100755 --- a/scripts/dev-lead-preflight.sh +++ b/scripts/dev-lead-preflight.sh @@ -47,8 +47,13 @@ CLAUDE_REQUIRED=required _PREFLIGHT_CHAIN_LIB="$(dirname "${BASH_SOURCE[0]}")/lib/engine-chain.sh" # shellcheck source=lib/engine-chain.sh [ -f "$_PREFLIGHT_CHAIN_LIB" ] && source "$_PREFLIGHT_CHAIN_LIB" -if declare -F ai_engine_enabled >/dev/null 2>&1 && ! ai_engine_enabled claude; then - CLAUDE_REQUIRED=optional +if declare -F ai_engine_enabled >/dev/null 2>&1; then + if ! ai_engine_enabled claude; then + CLAUDE_REQUIRED=optional + fi || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } fi if [ "$CLAUDE_REQUIRED" = required ]; then diff --git a/scripts/review-batch.sh b/scripts/review-batch.sh index 889223ee5..687d65b90 100644 --- a/scripts/review-batch.sh +++ b/scripts/review-batch.sh @@ -115,10 +115,17 @@ fi # Primary engine: REVIEW_ENGINE when AI_ENGINES enables it, otherwise the first # engine in AI_ENGINES — a disabled engine is never used. _requested_engine="${REVIEW_ENGINE:-}" -REVIEW_ENGINE="$(ai_engine_primary "$_requested_engine")" +REVIEW_ENGINE="$(ai_engine_primary "$_requested_engine")" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 +} export REVIEW_ENGINE if [ -n "$_requested_engine" ] && [ "$_requested_engine" != "$REVIEW_ENGINE" ]; then - echo "::warning::REVIEW_ENGINE=$_requested_engine is not enabled in AI_ENGINES ($(ai_engine_chain)) — using $REVIEW_ENGINE" + _chain_msg="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + echo "::warning::REVIEW_ENGINE=$_requested_engine is not enabled in AI_ENGINES ($_chain_msg) — using $REVIEW_ENGINE" fi unset _requested_engine @@ -132,10 +139,18 @@ if ! ai_engine_available "$REVIEW_ENGINE"; then echo "::warning::Primary $(ai_engine_label "$REVIEW_ENGINE") engine unavailable per pre-flight probe — switching to $(ai_engine_label "$_start_engine") for this batch" export REVIEW_ENGINE="$_start_engine" elif [ "$REVIEW_ENGINE" != "claude" ]; then - echo "::error::Primary $(ai_engine_label "$REVIEW_ENGINE") engine unavailable and no other engine in AI_ENGINES ($(ai_engine_chain)) is available (e.g. Copilot fallback also unavailable: gh copilot not installed, classic PAT, or COPILOT_GITHUB_TOKEN not set) — aborting batch" + _chain_msg="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + echo "::error::Primary $(ai_engine_label "$REVIEW_ENGINE") engine unavailable and no other engine in AI_ENGINES ($_chain_msg) is available (e.g. Copilot fallback also unavailable: gh copilot not installed, classic PAT, or COPILOT_GITHUB_TOKEN not set) — aborting batch" exit 1 else - echo "::warning::Claude unavailable per pre-flight probe and no other engine in AI_ENGINES ($(ai_engine_chain)) is available — continuing on Claude" + _chain_msg="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + echo "::warning::Claude unavailable per pre-flight probe and no other engine in AI_ENGINES ($_chain_msg) is available — continuing on Claude" fi unset _start_engine fi @@ -281,8 +296,12 @@ total_candidates=$(grep -c . "$PRS_FILE" || true) # _chain_has_later_engine — 0 when AI_ENGINES lists any engine after # (available or not). _chain_has_later_engine() { - local e seen=0 - for e in $(ai_engine_chain); do + local e seen=0 _chain + _chain="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" >&2 + exit 1 + } + for e in $_chain; do if [ "$seen" -eq 1 ]; then return 0 fi @@ -350,7 +369,11 @@ while IFS= read -r pr_url; do echo "::notice::$(ai_engine_label "$REVIEW_ENGINE") unavailable or rate-limited earlier in this batch — reviewing on $(ai_engine_label "$_start_engine")" export REVIEW_ENGINE="$_start_engine" elif [ -n "$AI_ENGINES_RATE_LIMITED" ]; then - echo "::error::Every engine in AI_ENGINES ($(ai_engine_chain)) is rate-limited or unavailable in this batch — stopping; the next scheduled run retries" + _chain_msg="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" >&2 + exit 1 + } + echo "::error::Every engine in AI_ENGINES ($_chain_msg) is rate-limited or unavailable in this batch — stopping; the next scheduled run retries" processed=$((processed - 1)) session_aborted=1 abort_pr="$pr_url" @@ -408,7 +431,11 @@ while IFS= read -r pr_url; do # notice and keep the batch going. Only a genuinely rate-limited, exhausted # chain falls through to the session-abort branch below. if [ "$rc" -eq 2 ] && { _chain_has_later_engine "$REVIEW_ENGINE" || [ "$_fallback_unavailable" -eq 1 ]; }; then - echo "::warning::No later engine in AI_ENGINES ($(ai_engine_chain)) is available after $(ai_engine_label "$REVIEW_ENGINE") — skipping $pr_url and continuing batch" + _chain_msg="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" >&2 + exit 1 + } + echo "::warning::No later engine in AI_ENGINES ($_chain_msg) is available after $(ai_engine_label "$REVIEW_ENGINE") — skipping $pr_url and continuing batch" post_engine_unavailable_notice "$pr_url" "$REVIEW_ENGINE" failed=$((failed + 1)) echo "::endgroup::" diff --git a/scripts/validate-engines.sh b/scripts/validate-engines.sh index 3f5957f41..5d3b6ced7 100644 --- a/scripts/validate-engines.sh +++ b/scripts/validate-engines.sh @@ -60,12 +60,12 @@ _gemini_probe_model() { } # _validate_engine_enabled — 0 unless AI_ENGINES leaves out. +# Returns 1 if the config file cannot be read (when engine-models.sh is sourced). _validate_engine_enabled() { if declare -F ai_engine_enabled >/dev/null 2>&1; then - ai_engine_enabled "$1" - else - return 0 + ai_engine_enabled "$1" || return 1 fi + return 0 } # _gemini_probe_key — minimal REST call with one Gemini key. @@ -166,21 +166,38 @@ validate_engines() { fi # ── Engine chain (AI_ENGINES) ─────────────────────────────────────────────── - local _chain_problem="" _disabled="" + local _chain_problem="" _disabled="" _chain="" if declare -F ai_engine_chain_problem >/dev/null 2>&1; then - _chain_problem="$(ai_engine_chain_problem)" + _chain_problem="$(ai_engine_chain_problem)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } if [ -n "$_chain_problem" ] && [ -z "${AI_ENGINES_PROBLEM_REPORTED:-}" ]; then echo "::warning::$_chain_problem" export AI_ENGINES_PROBLEM_REPORTED=1 fi - echo "::notice::Engine chain (AI_ENGINES): $(ai_engine_chain)" + _chain="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + echo "::notice::Engine chain (AI_ENGINES): $_chain" fi # ── Claude ────────────────────────────────────────────────────────────────── - if ! _validate_engine_enabled claude; then + if _validate_engine_enabled claude; then + if command -v claude >/dev/null 2>&1 && [ -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]; then + claude_ok=true + fi + else + if declare -F ai_engine_enabled >/dev/null 2>&1; then + # ai_engine_enabled returned 1; could be disabled or config read error + # Try getting the chain to detect if config is readable + ai_engine_chain >/dev/null 2>&1 || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + fi _disabled="${_disabled:+$_disabled, }claude" - elif command -v claude >/dev/null 2>&1 && [ -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]; then - claude_ok=true fi # ── Gemini ────────────────────────────────────────────────────────────────── @@ -196,9 +213,7 @@ validate_engines() { fi } - if ! _validate_engine_enabled gemini; then - _disabled="${_disabled:+$_disabled, }gemini" - else + if _validate_engine_enabled gemini; then if ! command -v gemini >/dev/null 2>&1; then append_gemini_reason "Gemini CLI not installed (fix: npm install -g @google/gemini-cli)" fi @@ -231,6 +246,16 @@ validate_engines() { if [ -n "$gemini_reasons" ]; then echo "::warning::Gemini fallback unavailable — ${gemini_reasons}. When Claude is rate-limited, runs will fall through to the next engine in AI_ENGINES." fi + else + if declare -F ai_engine_enabled >/dev/null 2>&1; then + # ai_engine_enabled returned 1; could be disabled or config read error + # Try getting the chain to detect if config is readable + ai_engine_chain >/dev/null 2>&1 || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + fi + _disabled="${_disabled:+$_disabled, }gemini" fi # ── Copilot ───────────────────────────────────────────────────────────────── @@ -244,27 +269,37 @@ validate_engines() { # --version` succeeding proves nothing — report it unavailable up front, the # same rule dev-lead's fallback applies (#1495, #1960). local _copilot_tok="${COPILOT_GITHUB_TOKEN:-}" - if ! _validate_engine_enabled copilot; then - _disabled="${_disabled:+$_disabled, }copilot" - elif [ -z "$_copilot_tok" ]; then - echo "::warning::Copilot unavailable — COPILOT_GITHUB_TOKEN is not set. Provide a fine-grained PAT with the Copilot entitlement or remove copilot from AI_ENGINES." - elif [[ "$_copilot_tok" == ghp_* ]]; then - echo "::warning::Copilot unavailable — COPILOT_GITHUB_TOKEN is a classic PAT (ghp_), which Copilot rejects. Use a fine-grained PAT or remove copilot from AI_ENGINES." + if _validate_engine_enabled copilot; then + if [ -z "$_copilot_tok" ]; then + echo "::warning::Copilot unavailable — COPILOT_GITHUB_TOKEN is not set. Provide a fine-grained PAT with the Copilot entitlement or remove copilot from AI_ENGINES." + elif [[ "$_copilot_tok" == ghp_* ]]; then + echo "::warning::Copilot unavailable — COPILOT_GITHUB_TOKEN is a classic PAT (ghp_), which Copilot rejects. Use a fine-grained PAT or remove copilot from AI_ENGINES." + else + # A policy denial (no Copilot subscription/policy for the token's org) or any + # other probe failure is reported once, never left as a silent false (#1960). + # The CLI output is matched, not echoed: only the matched phrase is logged. + local _copilot_out _copilot_rc=0 _denied + _copilot_out="$(env GH_TOKEN="$_copilot_tok" gh copilot --version 2>&1)" || _copilot_rc=$? + _denied="$(printf '%s\n' "$_copilot_out" | grep -oiE "$_VALIDATE_LICENSE_DENIED_RE" | head -n 1 \ + | tr '[:upper:]' '[:lower:]' || true)" + if [ -n "$_denied" ]; then + echo "::warning::Copilot unavailable — ${_denied} (the token's account or org has no usable Copilot entitlement). Fix the Copilot policy/subscription or remove copilot from AI_ENGINES." + elif [ "$_copilot_rc" -ne 0 ]; then + echo "::warning::Copilot unavailable — \`gh copilot --version\` failed (exit ${_copilot_rc}). Check the gh CLI and COPILOT_GITHUB_TOKEN, or remove copilot from AI_ENGINES." + else + copilot_ok=true + fi + fi else - # A policy denial (no Copilot subscription/policy for the token's org) or any - # other probe failure is reported once, never left as a silent false (#1960). - # The CLI output is matched, not echoed: only the matched phrase is logged. - local _copilot_out _copilot_rc=0 _denied - _copilot_out="$(env GH_TOKEN="$_copilot_tok" gh copilot --version 2>&1)" || _copilot_rc=$? - _denied="$(printf '%s\n' "$_copilot_out" | grep -oiE "$_VALIDATE_LICENSE_DENIED_RE" | head -n 1 \ - | tr '[:upper:]' '[:lower:]' || true)" - if [ -n "$_denied" ]; then - echo "::warning::Copilot unavailable — ${_denied} (the token's account or org has no usable Copilot entitlement). Fix the Copilot policy/subscription or remove copilot from AI_ENGINES." - elif [ "$_copilot_rc" -ne 0 ]; then - echo "::warning::Copilot unavailable — \`gh copilot --version\` failed (exit ${_copilot_rc}). Check the gh CLI and COPILOT_GITHUB_TOKEN, or remove copilot from AI_ENGINES." - else - copilot_ok=true + if declare -F ai_engine_enabled >/dev/null 2>&1; then + # ai_engine_enabled returned 1; could be disabled or config read error + # Try getting the chain to detect if config is readable + ai_engine_chain >/dev/null 2>&1 || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } fi + _disabled="${_disabled:+$_disabled, }copilot" fi if [ -n "$_disabled" ]; then From f2884f3ded366697274e17ecab92bf5a2c452a9e Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:11:14 +0000 Subject: [PATCH 13/29] fix(reviews): fail closed in dev-lead preflight when engine config unreadable Co-Authored-By: Claude Sonnet 5.5 --- scripts/dev-lead-preflight.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/dev-lead-preflight.sh b/scripts/dev-lead-preflight.sh index e9acb014d..7503c068f 100755 --- a/scripts/dev-lead-preflight.sh +++ b/scripts/dev-lead-preflight.sh @@ -48,12 +48,11 @@ _PREFLIGHT_CHAIN_LIB="$(dirname "${BASH_SOURCE[0]}")/lib/engine-chain.sh" # shellcheck source=lib/engine-chain.sh [ -f "$_PREFLIGHT_CHAIN_LIB" ] && source "$_PREFLIGHT_CHAIN_LIB" if declare -F ai_engine_enabled >/dev/null 2>&1; then - if ! ai_engine_enabled claude; then - CLAUDE_REQUIRED=optional - fi || { + _chain="$(ai_engine_chain)" || { echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" exit 1 } + [[ " $_chain " == *" claude "* ]] || CLAUDE_REQUIRED=optional fi if [ "$CLAUDE_REQUIRED" = required ]; then From ccf5cad2e309cd5327d0c1b9cec2ca313957bb4c Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:23:58 +0000 Subject: [PATCH 14/29] fix(reviews): document legacy Gemini model vars override AI_MODELS_* Co-Authored-By: Claude Sonnet 5.5 --- docs/engine-configuration.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/engine-configuration.md b/docs/engine-configuration.md index 8cafa4aa1..fbd782b58 100644 --- a/docs/engine-configuration.md +++ b/docs/engine-configuration.md @@ -61,6 +61,9 @@ For each task's chain, highest first: `GEMINI_FLASH_MODEL(_CHAIN)`, `GEMINI_PRO_MODEL(_CHAIN)`, `COPILOT_API_MODEL`. These are **internal**: they are kept for tests and the A/B runner. Do not set them as Actions variables; use `AI_MODELS_*`. + The reusable workflows still forward the legacy `GEMINI_FLASH_MODEL` / + `GEMINI_PRO_MODEL` Actions variables, so a leftover one **silently overrides** + `AI_MODELS_GEMINI`; they are deprecated — delete them. 2. **`AI_MODELS_*`** — the break-glass override (below). 3. **`config/ai-engines.json`**. From 4e26c168a73c6513662ab9c0a167d2726e1a96dd Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:55:09 +0000 Subject: [PATCH 15/29] fix(reviews): validate family primaries; derive PR-review/dev-lead primary from file; pr-review preflight reads engine config Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/dev-lead-reusable.yml | 2 +- .github/workflows/pr-review.yml | 23 ++++++++++++++++++++--- scripts/validate-ai-engines.py | 9 +++++++++ tests/test_validate_ai_engines.bats | 6 ++++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index 49a6626cb..2820dfc7e 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -191,7 +191,7 @@ jobs: # legacy DEV_LEAD_ENGINES name is still honoured); DEV_LEAD_ENGINE is a # legacy primary override — leave it unset to let AI_ENGINES decide. AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} - DEV_LEAD_ENGINE: ${{ vars.DEV_LEAD_ENGINE || (startsWith(vars.AI_ENGINES || vars.DEV_LEAD_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES || vars.DEV_LEAD_ENGINES, 'copilot') && 'copilot') || 'claude' }} + DEV_LEAD_ENGINE: ${{ vars.DEV_LEAD_ENGINE || (startsWith(vars.AI_ENGINES || vars.DEV_LEAD_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES || vars.DEV_LEAD_ENGINES, 'copilot') && 'copilot') || '' }} AI_DUCK_ENGINE: ${{ vars.AI_DUCK_ENGINE || '' }} AI_DUCK_MODEL: ${{ vars.AI_DUCK_MODEL || '' }} GEMINI_FLASH_MODEL: ${{ vars.GEMINI_FLASH_MODEL || '' }} diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index d382c35f3..151be773f 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -217,7 +217,7 @@ jobs: # The primary is derived here (expressions cannot split a list) from the # first AI_ENGINES entry; the scripts enforce the full list. AI_ENGINES: ${{ vars.AI_ENGINES || '' }} - REVIEW_ENGINE: ${{ vars.REVIEW_ENGINE || (startsWith(vars.AI_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES, 'copilot') && 'copilot') || 'claude' }} + REVIEW_ENGINE: ${{ vars.REVIEW_ENGINE || (startsWith(vars.AI_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES, 'copilot') && 'copilot') || '' }} AI_DUCK_ENGINE: ${{ vars.AI_DUCK_ENGINE || '' }} AI_DUCK_MODEL: ${{ vars.AI_DUCK_MODEL || '' }} GEMINI_FLASH_MODEL: ${{ vars.GEMINI_FLASH_MODEL || '' }} @@ -332,9 +332,18 @@ jobs: # and install claude only best-effort, so they legitimately run WITHOUT this # token — failing them here would be a regression, not the fail-loud we want. env: - REVIEW_ENGINE: ${{ vars.REVIEW_ENGINE || (startsWith(vars.AI_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES, 'copilot') && 'copilot') || 'claude' }} + REVIEW_ENGINE: ${{ vars.REVIEW_ENGINE || (startsWith(vars.AI_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES, 'copilot') && 'copilot') || '' }} + AGENT_REF: ${{ inputs.agent_ref || 'main' }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | + # The file's enabled providers in fallback_order (what the runtime's + # default chain is). Fetched through the API (this step precedes + # checkout); when it cannot be read, fall back to claude-first and keep + # requiring the token (fail closed). + file_chain="$(gh api -H 'Accept: application/vnd.github.raw' \ + "repos/petry-projects/.github-private/contents/config/ai-engines.json?ref=${AGENT_REF}" 2>/dev/null \ + | jq -r '. as $c | [$c.providers.fallback_order[] | select($c.providers[.].enabled == true)] | join(" ")' 2>/dev/null || true)" + [ -n "$file_chain" ] || file_chain="claude gemini copilot" effective_primary() { local pref spec e chain="" valid=true first pref="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" @@ -348,7 +357,15 @@ jobs: done set +f if [ "$valid" = false ]; then chain=""; pref=""; fi - [ -n "${chain// /}" ] || chain=" claude gemini copilot" + if [ -n "${chain// /}" ]; then + # AI_ENGINES narrows the file; it cannot re-enable a provider the file disables. + local kept="" + for e in $chain; do case " $file_chain " in *" $e "*) kept="$kept $e" ;; esac; done + chain="$kept" + else + chain=" $file_chain" + fi + [ -n "${chain// /}" ] || { printf '%s' none; return; } if [ -n "$pref" ] && [[ "$chain " == *" $pref "* ]]; then printf '%s' "$pref"; return; fi first="${chain# }" printf '%s' "${first%% *}" diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py index c20522256..f9857809d 100755 --- a/scripts/validate-ai-engines.py +++ b/scripts/validate-ai-engines.py @@ -31,6 +31,9 @@ SINGLE_MODEL_TASKS = ("duck",) SINGLE_MODEL_PROVIDERS = ("copilot",) PROVIDERS = ("claude", "gemini", "copilot") +# ai_model_for_family (scripts/lib/engine-models.sh) returns the first Claude model +# of these tasks, so that model must be of the family the task stands for. +FAMILY_TASKS = {"deep": "opus", "action": "sonnet", "triage": "haiku"} def error(msg: str) -> None: @@ -131,6 +134,12 @@ def check_references(config, globs) -> list: problems.append( f"tasks.{task}.{provider}: '{model}' belongs to provider " f"{entry['provider']}, not {provider}") + if provider == "claude" and task in FAMILY_TASKS and chain: + family = FAMILY_TASKS[task] + if not re.match(rf"claude-{family}-", str(chain[0])): + problems.append( + f"tasks.{task}.claude: first model '{chain[0]}' is not a {family} model " + f"(ai_model_for_family {family} returns it)") if len(chain) > 1 and (task in SINGLE_MODEL_TASKS or provider in SINGLE_MODEL_PROVIDERS): why = "the duck runs one model" if task in SINGLE_MODEL_TASKS else "Copilot has no in-engine chain" problems.append(f"tasks.{task}.{provider}: takes one model ({why}), got {len(chain)}") diff --git a/tests/test_validate_ai_engines.bats b/tests/test_validate_ai_engines.bats index 2efccedb3..6d8e0de6d 100644 --- a/tests/test_validate_ai_engines.bats +++ b/tests/test_validate_ai_engines.bats @@ -121,3 +121,9 @@ _validate_with() { @test "lint.yml runs the gate" { grep -q 'python3 scripts/validate-ai-engines.py' "$REPO_ROOT/.github/workflows/lint.yml" } + +@test "family identity: a non-family primary in deep/action/triage fails" { + _validate_with '.tasks.deep.claude = ["claude-sonnet-5-5"]' + [ "$status" -ne 0 ] + [[ "$output" == *"not a opus model"* ]] +} From a4dd1363e2add41645c386dc7ca21f268ce3c6ed Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:53:39 +0000 Subject: [PATCH 16/29] style(validate-engines): fix indentation after rebase merge Co-Authored-By: Claude Sonnet 5.5 --- scripts/validate-engines.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate-engines.sh b/scripts/validate-engines.sh index 5d3b6ced7..1b57a0ee1 100644 --- a/scripts/validate-engines.sh +++ b/scripts/validate-engines.sh @@ -274,7 +274,7 @@ validate_engines() { echo "::warning::Copilot unavailable — COPILOT_GITHUB_TOKEN is not set. Provide a fine-grained PAT with the Copilot entitlement or remove copilot from AI_ENGINES." elif [[ "$_copilot_tok" == ghp_* ]]; then echo "::warning::Copilot unavailable — COPILOT_GITHUB_TOKEN is a classic PAT (ghp_), which Copilot rejects. Use a fine-grained PAT or remove copilot from AI_ENGINES." - else + else # A policy denial (no Copilot subscription/policy for the token's org) or any # other probe failure is reported once, never left as a silent false (#1960). # The CLI output is matched, not echoed: only the matched phrase is logged. From e50faa79f01c0d701bcd8fea871ed27a977a240e Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:54:31 +0000 Subject: [PATCH 17/29] fix(reviews): resolve primary engine in install steps so an empty value never reaches the case Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/dev-lead-reusable.yml | 17 ++++- .github/workflows/pr-review.yml | 14 ++++ .../unit/test_install_step_engine.bats | 64 +++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 tests/dev-lead/unit/test_install_step_engine.bats diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index 2820dfc7e..952742dd0 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -484,7 +484,22 @@ jobs: # Log the CLI version in use, cached or fresh (#1957). echo "claude-code version: $(claude --version 2>/dev/null || echo unavailable)" - case "$DEV_LEAD_ENGINE" in + # DEV_LEAD_ENGINE is empty when no legacy override is set and AI_ENGINES + # does not start with gemini/copilot. Resolve the primary from the + # checked-out config so the fallback CLIs still install; claude is the + # last resort when the config cannot be read. + engine="${DEV_LEAD_ENGINE:-}" + if [ -z "$engine" ]; then + # shellcheck source=scripts/lib/engine-chain.sh + if source .dev-lead/scripts/lib/engine-chain.sh 2>/dev/null \ + && primary="$(ai_engine_primary "")" && [ -n "$primary" ]; then + engine="$primary" + else + engine=claude + fi + fi + + case "$engine" in claude) npm install -g @google/gemini-cli || true if ! gh copilot --version >/dev/null 2>&1; then diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index 151be773f..73f18f6dc 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -550,6 +550,20 @@ jobs: echo "claude-code installed: $(claude --version 2>/dev/null || echo unavailable)" } + # REVIEW_ENGINE is empty when no legacy override is set and AI_ENGINES + # does not start with gemini/copilot. Resolve the primary from the + # checked-out config so an empty value never reaches the case below; + # claude is the last resort when the config cannot be read. + if [ -z "${REVIEW_ENGINE:-}" ]; then + # shellcheck source=scripts/lib/engine-chain.sh + if source scripts/lib/engine-chain.sh 2>/dev/null \ + && primary="$(ai_engine_primary "")" && [ -n "$primary" ]; then + REVIEW_ENGINE="$primary" + else + REVIEW_ENGINE=claude + fi + fi + case "$REVIEW_ENGINE" in claude) install_claude diff --git a/tests/dev-lead/unit/test_install_step_engine.bats b/tests/dev-lead/unit/test_install_step_engine.bats new file mode 100644 index 000000000..51916bc5f --- /dev/null +++ b/tests/dev-lead/unit/test_install_step_engine.bats @@ -0,0 +1,64 @@ +#!/usr/bin/env bats +# The engine-CLI install steps of pr-review.yml and dev-lead-reusable.yml must +# never receive an empty primary engine (#1973 review): with the live +# AI_ENGINES=claude,gemini and no REVIEW_ENGINE / DEV_LEAD_ENGINE variable the +# env value is empty, and the step has to resolve the primary from the config. + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" + +setup() { + unset AI_ENGINES_CONFIG DEV_LEAD_ENGINES REVIEW_ENGINE DEV_LEAD_ENGINE + export HOME="$BATS_TEST_TMPDIR/home" GITHUB_PATH="$BATS_TEST_TMPDIR/github_path" + mkdir -p "$HOME" "$BATS_TEST_TMPDIR/bin" + : > "$GITHUB_PATH" + export CALLS="$BATS_TEST_TMPDIR/calls" + : > "$CALLS" + for tool in npm gh claude gemini; do + printf '#!/usr/bin/env bash\necho "%s $*" >> "$CALLS"\nexit 0\n' "$tool" \ + > "$BATS_TEST_TMPDIR/bin/$tool" + chmod +x "$BATS_TEST_TMPDIR/bin/$tool" + done + # `gh copilot --version` fails, so the step has to install the extension. + printf '#!/usr/bin/env bash\necho "gh $*" >> "$CALLS"\n[ "${1:-}" = copilot ] && exit 1\nexit 0\n' \ + > "$BATS_TEST_TMPDIR/bin/gh" + export PATH="$BATS_TEST_TMPDIR/bin:$PATH" +} + +# _step — writes the step's run script to a file. +_step() { + yq -r ".jobs[].steps[] | select(.name == \"$2\") | .run" \ + "$SCRIPT_DIR/.github/workflows/$1" > "$BATS_TEST_TMPDIR/step.sh" + [ -s "$BATS_TEST_TMPDIR/step.sh" ] +} + +@test "pr-review install step handles an empty REVIEW_ENGINE under AI_ENGINES=claude,gemini" { + _step pr-review.yml "Install review engine CLIs" + cd "$SCRIPT_DIR" + AI_ENGINES="claude,gemini" REVIEW_ENGINE="" CLAUDE_CODE_VERSION=latest \ + COPILOT_GITHUB_TOKEN=x run bash "$BATS_TEST_TMPDIR/step.sh" + [ "$status" -eq 0 ] + [[ "$output" != *"Unknown REVIEW_ENGINE"* ]] + [[ "$output" == *"claude-code cache hit"* ]] + grep -q "npm install -g @google/gemini-cli" "$CALLS" +} + +@test "pr-review install step falls back to claude when the config is unreadable" { + _step pr-review.yml "Install review engine CLIs" + cd "$SCRIPT_DIR" + AI_ENGINES_CONFIG="$BATS_TEST_TMPDIR/missing.json" REVIEW_ENGINE="" \ + CLAUDE_CODE_VERSION=latest COPILOT_GITHUB_TOKEN=x run bash "$BATS_TEST_TMPDIR/step.sh" + [ "$status" -eq 0 ] + [[ "$output" == *"claude-code cache hit"* ]] +} + +@test "dev-lead install step installs fallback CLIs for an empty DEV_LEAD_ENGINE under AI_ENGINES=claude,gemini" { + _step dev-lead-reusable.yml "Install engine CLIs" + mkdir -p "$BATS_TEST_TMPDIR/work" + ln -s "$SCRIPT_DIR" "$BATS_TEST_TMPDIR/work/.dev-lead" + cd "$BATS_TEST_TMPDIR/work" + AI_ENGINES="claude,gemini" DEV_LEAD_ENGINE="" CLAUDE_CODE_VERSION=latest \ + run bash "$BATS_TEST_TMPDIR/step.sh" + [ "$status" -eq 0 ] + grep -q "npm install -g @google/gemini-cli" "$CALLS" + grep -q "gh extension install github/gh-copilot" "$CALLS" +} From 47b9daff4a009c00915303961b987780b5fac0fc Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:37:32 +0000 Subject: [PATCH 18/29] fix(reviews): resolve enabled primary in pr-review install step even when REVIEW_ENGINE is set; price Copilot models by bare id only Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/pr-review.yml | 24 +++++++++---------- scripts/validate-ai-engines.py | 2 +- .../unit/test_install_step_engine.bats | 10 ++++++++ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index 73f18f6dc..bdf56f8d3 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -550,18 +550,18 @@ jobs: echo "claude-code installed: $(claude --version 2>/dev/null || echo unavailable)" } - # REVIEW_ENGINE is empty when no legacy override is set and AI_ENGINES - # does not start with gemini/copilot. Resolve the primary from the - # checked-out config so an empty value never reaches the case below; - # claude is the last resort when the config cannot be read. - if [ -z "${REVIEW_ENGINE:-}" ]; then - # shellcheck source=scripts/lib/engine-chain.sh - if source scripts/lib/engine-chain.sh 2>/dev/null \ - && primary="$(ai_engine_primary "")" && [ -n "$primary" ]; then - REVIEW_ENGINE="$primary" - else - REVIEW_ENGINE=claude - fi + # REVIEW_ENGINE is only a preference: it is empty when no legacy + # override is set and AI_ENGINES does not start with gemini/copilot, + # and it can name an engine the config disables. Always resolve the + # enabled primary from the checked-out config so the case below sees + # the engine the run actually uses; fall back to the preference, then + # claude, when the config cannot be read. + # shellcheck source=scripts/lib/engine-chain.sh + if source scripts/lib/engine-chain.sh 2>/dev/null \ + && primary="$(ai_engine_primary "${REVIEW_ENGINE:-}")" && [ -n "$primary" ]; then + REVIEW_ENGINE="$primary" + else + REVIEW_ENGINE="${REVIEW_ENGINE:-claude}" fi case "$REVIEW_ENGINE" in diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py index f9857809d..d2eaec29c 100755 --- a/scripts/validate-ai-engines.py +++ b/scripts/validate-ai-engines.py @@ -148,7 +148,7 @@ def check_references(config, globs) -> list: continue # Only the Copilot path strips a vendor prefix before pricing a run. bare = model.split("/", 1)[1] if entry["provider"] == "copilot" and "/" in model else model - if not any(glob_matches(g, model) or glob_matches(g, bare) for g in globs): + if not any(glob_matches(g, bare) for g in globs): problems.append( f"models.{model}: provider {entry['provider']} is enabled but the model has " f"no price row in scripts/lib/model-pricing.tsv") diff --git a/tests/dev-lead/unit/test_install_step_engine.bats b/tests/dev-lead/unit/test_install_step_engine.bats index 51916bc5f..696370452 100644 --- a/tests/dev-lead/unit/test_install_step_engine.bats +++ b/tests/dev-lead/unit/test_install_step_engine.bats @@ -42,6 +42,16 @@ _step() { grep -q "npm install -g @google/gemini-cli" "$CALLS" } +@test "pr-review install step resolves the enabled primary when REVIEW_ENGINE names a disabled engine" { + _step pr-review.yml "Install review engine CLIs" + cd "$SCRIPT_DIR" + AI_ENGINES="claude,gemini" REVIEW_ENGINE="copilot" CLAUDE_CODE_VERSION=latest \ + COPILOT_GITHUB_TOKEN=x run bash "$BATS_TEST_TMPDIR/step.sh" + [ "$status" -eq 0 ] + [[ "$output" == *"claude-code cache hit"* ]] + ! grep -q "gh extension install github/gh-copilot" "$CALLS" +} + @test "pr-review install step falls back to claude when the config is unreadable" { _step pr-review.yml "Install review engine CLIs" cd "$SCRIPT_DIR" From f57d7090f1b60211e3edff3a30f6cf8d226ea1dc Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:56:12 +0000 Subject: [PATCH 19/29] fix(tests): make negated ai_engine_enabled assertion able to fail (bats-negation) --- tests/dev-lead/unit/test_ai_engines_config.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dev-lead/unit/test_ai_engines_config.bats b/tests/dev-lead/unit/test_ai_engines_config.bats index b48f5bfa4..95604fa4a 100644 --- a/tests/dev-lead/unit/test_ai_engines_config.bats +++ b/tests/dev-lead/unit/test_ai_engines_config.bats @@ -145,7 +145,7 @@ _config_with() { [ "$(ai_engine_chain)" = "claude gemini" ] export AI_ENGINES="copilot,claude" [ "$(ai_engine_chain)" = "claude" ] - ! ai_engine_enabled copilot + ! ai_engine_enabled copilot || false [ "$(ai_engine_primary copilot)" = "claude" ] [[ "$(ai_engine_chain_problem)" == *"copilot"*"disabled"* ]] } From 922f6c36cd737afe2bd4dea0b20aefc3370457ad Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:49:29 +0000 Subject: [PATCH 20/29] fix(reviews): register engine-config-validate exception, reuse runtime AI_ENGINES parser, price Claude models when disabled Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 4 ++++ scripts/lib/engine-models.sh | 22 ++++++++++++---------- scripts/validate-ai-engines.py | 5 ++++- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3e5a2ce20..03e1692bd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -234,6 +234,10 @@ calling it a regression, and treat CI as authoritative.** that runs `scripts/validate-ai-engines.py` against `config/ai-engines.json`; the `paths:` filters also carry `config/**` for it. It is not covered by the org template and must not be removed by template syncs. If the org template gains an equivalent, remove this exception and defer to the template instead. +- **Exception:** `.github/workflows/engine-config-validate.yml` (#1973) is a documented repo-specific + workflow that runs `scripts/validate-ai-engines.py` when `config/ai-engines.json`, its schema, or the + pricing table change. It is not covered by the org template and must not be removed by template syncs. + If the org template gains an equivalent, remove this exception and defer to the template instead. - **Exception:** the `lint.yml` `paths:` filters (pull_request and push) carry the repo-specific entry `docs/initiatives/agent-ingress-collapse-markets.md`, which is read by `tests/agent_ingress_markets_package.bats` (#2038). It must not be removed as template drift on a sync; diff --git a/scripts/lib/engine-models.sh b/scripts/lib/engine-models.sh index 2072a475c..20960679d 100644 --- a/scripts/lib/engine-models.sh +++ b/scripts/lib/engine-models.sh @@ -323,8 +323,7 @@ ai_models_problems() { # the same normalisation engine-chain.sh applies (case, separators). Prints # nothing when no override differs; returns 1 when the file cannot be read. ai_engines_overrides() { - local p k var c d spec chain="" t file disabled - local -a toks=() + local p k var c d spec chain problem file # Read the file first, so an unreadable one is an error even with no override set. ai_engines_config_load || return 1 for p in claude gemini copilot; do @@ -343,14 +342,17 @@ ai_engines_overrides() { spec="${AI_ENGINES:-${DEV_LEAD_ENGINES:-}}" [ -n "${spec//[[:space:],]/}" ] || return 0 file="$(ai_engines_file_providers)" || return 1 - disabled="$(ai_engines_file_disabled)" || return 1 - IFS=$' \t' read -r -a toks <<< "$(printf '%s' "$spec" | tr ',\n\r\t' ' ' | tr '[:upper:]' '[:lower:]')" - for t in ${toks[@]+"${toks[@]}"}; do - # A provider the file disables never runs, so it is not part of the effective chain. - [[ " $disabled " != *" $t "* ]] || continue - [[ " $chain " == *" $t "* ]] || chain="${chain:+$chain }$t" - done - [ "$chain" = "$file" ] || printf "AI_ENGINES=%s → %s (file: %s)\n" "$spec" "$chain" "$file" + # Reuse the runtime parser so the reported chain is the one that runs: an + # invalid override is rejected whole there, and its problem is surfaced here. + if ! declare -F ai_engine_chain >/dev/null 2>&1; then + # shellcheck source=scripts/lib/engine-chain.sh + source "$(dirname "${BASH_SOURCE[0]}")/engine-chain.sh" + fi + chain="$(ai_engine_chain)" || return 1 + problem="$(ai_engine_chain_problem)" || return 1 + if [ "$chain" != "$file" ] || [ -n "$problem" ]; then + printf "AI_ENGINES=%s → %s (file: %s)%s\n" "$spec" "$chain" "$file" "${problem:+ — $problem}" + fi } # ai_models_fable_deprecation — one deprecation line when a *configured* Claude diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py index d2eaec29c..dd2a914d0 100755 --- a/scripts/validate-ai-engines.py +++ b/scripts/validate-ai-engines.py @@ -144,7 +144,10 @@ def check_references(config, globs) -> list: why = "the duck runs one model" if task in SINGLE_MODEL_TASKS else "Copilot has no in-engine chain" problems.append(f"tasks.{task}.{provider}: takes one model ({why}), got {len(chain)}") for model, entry in models.items(): - if entry["status"] == "retired" or not providers[entry["provider"]]["enabled"]: + # Claude chains are resolved outside provider routing (idea-triage.yml, + # persona-runner-reusable.yml), so Claude models are priced even when disabled. + if entry["status"] == "retired" or ( + entry["provider"] != "claude" and not providers[entry["provider"]]["enabled"]): continue # Only the Copilot path strips a vendor prefix before pricing a run. bare = model.split("/", 1)[1] if entry["provider"] == "copilot" and "/" in model else model From bce75bcdc9e128116ec1f557b931197a0ae8a790 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 14 Oct 2026 03:21:40 +0000 Subject: [PATCH 21/29] fix(reviews): add config to the push job's dev-lead sparse checkout Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/dev-lead-reusable.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index 952742dd0..b4d943928 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -837,8 +837,10 @@ jobs: ref: ${{ inputs.agent_ref }} path: .dev-lead token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + # config/ holds ai-engines.json, the engine defaults the scripts read (#1973). sparse-checkout: | scripts + config prompts/dev-lead personas/dev-lead From c0d7853fbe58b633084c2149e1eacb834bad3767 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:14:31 +0000 Subject: [PATCH 22/29] fix(reviews): gate dev-lead Claude token preflights on the effective primary engine Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/dev-lead-reusable.yml | 120 ++++++++++++++---------- 1 file changed, 68 insertions(+), 52 deletions(-) diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index b4d943928..6ea48389e 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -219,41 +219,49 @@ jobs: # precedence over AI_ENGINES, so check the file before requiring the token). env: AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} + DEV_LEAD_ENGINE: ${{ vars.DEV_LEAD_ENGINE || '' }} AGENT_REF: ${{ inputs.agent_ref }} GH_TOKEN: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | - # Is Claude enabled? Check config/ai-engines.json first (if it exists), - # then fall back to the inline AI_ENGINES logic. - claude_enabled=true - # Fetched through the API, not a checkout, so this stays the first step. - # Parse (never grep) the file; fail closed — keep requiring the token — - # when .providers.claude.enabled cannot be fetched or resolved. - cfg_claude="$(gh api -H 'Accept: application/vnd.github.raw' \ + # Require the token only when the EFFECTIVE primary engine is claude, as + # scripts/lib/engine-chain.sh's ai_engine_primary resolves it (mirrored + # inline: this runs before checkout). The file's enabled providers in + # fallback_order are the default chain; AI_ENGINES narrows it (a valid list + # cannot re-enable a disabled provider; an invalid one means the default + # chain and primary); DEV_LEAD_ENGINE wins only when the chain enables it. + # Fetched through the API; unreadable config fails closed (claude first). + file_chain="$(gh api -H 'Accept: application/vnd.github.raw' \ "repos/petry-projects/.github-private/contents/config/ai-engines.json?ref=${AGENT_REF}" 2>/dev/null \ - | jq -r '.providers.claude.enabled' 2>/dev/null || echo unknown)" - if [ "$cfg_claude" = false ]; then - claude_enabled=false - fi - if [ "$claude_enabled" = true ]; then - # Config doesn't disable Claude; check if AI_ENGINES omits it. + | jq -r '. as $c | [$c.providers.fallback_order[] | select($c.providers[.].enabled == true)] | join(" ")' 2>/dev/null || true)" + [ -n "$file_chain" ] || file_chain="claude gemini copilot" + effective_primary() { + local pref spec e chain="" valid=true first kept="" + pref="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" spec="$(printf '%s' "${AI_ENGINES:-}" | tr '[:upper:],' '[:lower:] ')" - if [ -n "${spec//[[:space:]]/}" ]; then - set -f - known=true; has_claude=false - for e in $spec; do - case "$e" in - claude) has_claude=true ;; - gemini|copilot) ;; - *) known=false ;; - esac - done - set +f - if [ "$known" = true ] && [ "$has_claude" = false ]; then claude_enabled=false; fi + set -f + for e in $spec; do + case "$e" in + claude|gemini|copilot) case " $chain " in *" $e "*) ;; *) chain="$chain $e" ;; esac ;; + *) valid=false ;; + esac + done + set +f + if [ "$valid" = false ]; then chain=""; pref=""; fi + if [ -n "${chain// /}" ]; then + for e in $chain; do case " $file_chain " in *" $e "*) kept="$kept $e" ;; esac; done + chain="$kept" + else + chain=" $file_chain" fi - fi - if [ "$claude_enabled" = false ]; then - echo "claude disabled in config or not in AI_ENGINES (${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" + [ -n "${chain// /}" ] || { printf '%s' none; return; } + if [ -n "$pref" ] && [[ "$chain " == *" $pref "* ]]; then printf '%s' "$pref"; return; fi + first="${chain# }" + printf '%s' "${first%% *}" + } + primary="$(effective_primary "$DEV_LEAD_ENGINE")" + if [ "$primary" != claude ]; then + echo "effective primary engine is ${primary} (AI_ENGINES=${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" exit 0 fi if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then @@ -1016,41 +1024,49 @@ jobs: - name: Engine token preflight (#1525) env: AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} + DEV_LEAD_ENGINE: ${{ vars.DEV_LEAD_ENGINE || '' }} AGENT_REF: ${{ inputs.agent_ref }} GH_TOKEN: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | - # Is Claude enabled? config/ai-engines.json disabling Claude takes - # precedence; otherwise only a valid AI_ENGINES list that omits claude turns - # it off; empty or invalid values fall back to the default chain, which - # includes claude (scripts/lib/engine-chain.sh — mirrored inline because this - # runs before the scripts are checked out). - claude_enabled=true - # Fetched through the API, not a checkout, so this stays the first step. - # Parse (never grep) the file; fail closed — keep requiring the token — - # when .providers.claude.enabled cannot be fetched or resolved. - cfg_claude="$(gh api -H 'Accept: application/vnd.github.raw' \ + # Require the token only when the EFFECTIVE primary engine is claude, as + # scripts/lib/engine-chain.sh's ai_engine_primary resolves it (mirrored + # inline: this runs before checkout). The file's enabled providers in + # fallback_order are the default chain; AI_ENGINES narrows it (a valid list + # cannot re-enable a disabled provider; an invalid one means the default + # chain and primary); DEV_LEAD_ENGINE wins only when the chain enables it. + # Fetched through the API; unreadable config fails closed (claude first). + file_chain="$(gh api -H 'Accept: application/vnd.github.raw' \ "repos/petry-projects/.github-private/contents/config/ai-engines.json?ref=${AGENT_REF}" 2>/dev/null \ - | jq -r '.providers.claude.enabled' 2>/dev/null || echo unknown)" - if [ "$cfg_claude" = false ]; then - claude_enabled=false - fi - spec="$(printf '%s' "${AI_ENGINES:-}" | tr '[:upper:],' '[:lower:] ')" - if [ "$claude_enabled" = true ] && [ -n "${spec//[[:space:]]/}" ]; then + | jq -r '. as $c | [$c.providers.fallback_order[] | select($c.providers[.].enabled == true)] | join(" ")' 2>/dev/null || true)" + [ -n "$file_chain" ] || file_chain="claude gemini copilot" + effective_primary() { + local pref spec e chain="" valid=true first kept="" + pref="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" + spec="$(printf '%s' "${AI_ENGINES:-}" | tr '[:upper:],' '[:lower:] ')" set -f - known=true; has_claude=false for e in $spec; do case "$e" in - claude) has_claude=true ;; - gemini|copilot) ;; - *) known=false ;; + claude|gemini|copilot) case " $chain " in *" $e "*) ;; *) chain="$chain $e" ;; esac ;; + *) valid=false ;; esac done set +f - if [ "$known" = true ] && [ "$has_claude" = false ]; then claude_enabled=false; fi - fi - if [ "$claude_enabled" = false ]; then - echo "claude disabled in config or not in AI_ENGINES (${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" + if [ "$valid" = false ]; then chain=""; pref=""; fi + if [ -n "${chain// /}" ]; then + for e in $chain; do case " $file_chain " in *" $e "*) kept="$kept $e" ;; esac; done + chain="$kept" + else + chain=" $file_chain" + fi + [ -n "${chain// /}" ] || { printf '%s' none; return; } + if [ -n "$pref" ] && [[ "$chain " == *" $pref "* ]]; then printf '%s' "$pref"; return; fi + first="${chain# }" + printf '%s' "${first%% *}" + } + primary="$(effective_primary "$DEV_LEAD_ENGINE")" + if [ "$primary" != claude ]; then + echo "effective primary engine is ${primary} (AI_ENGINES=${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" exit 0 fi if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then From 09ac9c1a26a46caaa62bdd624805d1a3abcf4457 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:14:49 +0000 Subject: [PATCH 23/29] fix(reviews): give the pr-review Claude preflight its own GH_TOKEN fallback for the config fetch Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/pr-review.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index bdf56f8d3..1ad808780 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -334,6 +334,7 @@ jobs: env: REVIEW_ENGINE: ${{ vars.REVIEW_ENGINE || (startsWith(vars.AI_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES, 'copilot') && 'copilot') || '' }} AGENT_REF: ${{ inputs.agent_ref || 'main' }} + GH_TOKEN: ${{ secrets.DON_PETRY_BOT_GH_PAT_CLASSIC || secrets.DON_PETRY_BOT_GH_PAT || github.token }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | # The file's enabled providers in fallback_order (what the runtime's From caa16850ffe793637670f5f144ed246b419788a9 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 14 Oct 2026 04:51:45 +0000 Subject: [PATCH 24/29] fix(reviews): run install-step bats in lint, exempt engine-config-validate from fleet failure tracking, pin model A/B to claude Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/lint.yml | 1 + AGENTS.md | 6 +++--- scripts/evals/model-ab.sh | 2 ++ scripts/fleet_report.sh | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 26f18bc6d..9c828c565 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -216,6 +216,7 @@ jobs: tests/test_consumer_manifest.bats \ tests/test_validate_ai_engines.bats \ tests/dev-lead/unit/test_ai_engines_config.bats \ + tests/dev-lead/unit/test_install_step_engine.bats \ tests/test_downstream_impact_map.bats \ tests/test_downstream_impact_assemble.bats \ tests/test_downstream_impact_triage_inline.bats \ diff --git a/AGENTS.md b/AGENTS.md index 03e1692bd..55c77ec22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -257,9 +257,9 @@ calling it a regression, and treat CI as authoritative.** Because this is a **gate** workflow whose `failure` conclusion is intentional enforcement (it blocks a bad PR), it is exempt from the Fleet Monitor's high-failure issue tracking via `FLEET_GATE_WORKFLOWS` in `scripts/fleet_report.sh` (#941) — otherwise a guard doing its job produces false-positive trackers. - `holdout-guard.yml` and `sync-scope-guard.yml` (which fails closed on a sync PR with no declared-paths - manifest or an out-of-scope diff, #1700/#2373) are exempt for the same reason. Add new gate workflows of - this class to that list. + `holdout-guard.yml`, `sync-scope-guard.yml` (which fails closed on a sync PR with no declared-paths + manifest or an out-of-scope diff, #1700/#2373) and `engine-config-validate.yml` are exempt for the same + reason. Add new gate workflows of this class to that list. - **Exception:** `pr-review-sweep.yml` (stuck-review sweep, #573/#898) is a documented repo-specific workflow with no corresponding org template in `standards/workflows/`. It re-dispatches reviews for PRs that went green after a ci-pending/ci-failing skip, via two triggers: a scheduled cron (the EVENTUAL diff --git a/scripts/evals/model-ab.sh b/scripts/evals/model-ab.sh index 5d7c04287..0a1bee537 100755 --- a/scripts/evals/model-ab.sh +++ b/scripts/evals/model-ab.sh @@ -114,6 +114,7 @@ JUDGE_SHIM="$(mktemp)" cat >"$JUDGE_SHIM" </dev/null 2>&1; then @@ -136,6 +137,7 @@ _is_scored() { [ "$1" -eq 0 ] || [ "$1" -eq 1 ]; } run_arm() { local skill="$1" model="$2" out rc=0 if out="$( + REVIEW_ENGINE=claude \ CLAUDE_TRIAGE_MODEL_CHAIN="$model" \ EVAL_ENGINE_CMD="$EVAL_ENGINE_CMD" \ EVAL_JUDGE_CMD="$JUDGE_SHIM" \ diff --git a/scripts/fleet_report.sh b/scripts/fleet_report.sh index 64e066770..0cbd7ea9b 100755 --- a/scripts/fleet_report.sh +++ b/scripts/fleet_report.sh @@ -13,7 +13,7 @@ # (#941). Matched by workflow file basename. See AGENTS.md "Exception: # test-deletion-guard.yml". Override via env with the full space-separated # gate list for testing; add permanent gates to the default below. -FLEET_GATE_WORKFLOWS="${FLEET_GATE_WORKFLOWS:-test-deletion-guard.yml holdout-guard.yml sync-scope-guard.yml}" +FLEET_GATE_WORKFLOWS="${FLEET_GATE_WORKFLOWS:-test-deletion-guard.yml holdout-guard.yml sync-scope-guard.yml engine-config-validate.yml}" # Dev-Lead status-marker prefix (#1019, story C of #901). Mirrors # ISSUE_MARKER_PREFIX in dev-lead-fix-issue.sh / dev-lead-retry.sh: every From 0129b9841b463cb7d0cd01d4f85f86ffd3379b23 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:33:51 +0000 Subject: [PATCH 25/29] fix(reviews): skip tokenless claude in fallback chains, validate price row fields Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/dev-lead-reusable.yml | 10 ++++++++++ scripts/engine.sh | 8 ++++++++ scripts/validate-ai-engines.py | 10 +++++++++- tests/dev-lead/unit/test_engine_fallback.bats | 13 +++++++++++++ tests/test_validate_ai_engines.bats | 13 +++++++++++++ 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index 6ea48389e..fb0db7783 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -261,6 +261,11 @@ jobs: } primary="$(effective_primary "$DEV_LEAD_ENGINE")" if [ "$primary" != claude ]; then + if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then + # claude may still sit later in the chain; mark it unavailable so + # run_writer_with_fallback skips it instead of hitting a terminal auth error. + echo "CLAUDE_UNAVAILABLE=1" >> "$GITHUB_ENV" + fi echo "effective primary engine is ${primary} (AI_ENGINES=${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" exit 0 fi @@ -1066,6 +1071,11 @@ jobs: } primary="$(effective_primary "$DEV_LEAD_ENGINE")" if [ "$primary" != claude ]; then + if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then + # claude may still sit later in the chain; mark it unavailable so + # run_writer_with_fallback skips it instead of hitting a terminal auth error. + echo "CLAUDE_UNAVAILABLE=1" >> "$GITHUB_ENV" + fi echo "effective primary engine is ${primary} (AI_ENGINES=${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" exit 0 fi diff --git a/scripts/engine.sh b/scripts/engine.sh index 73b94e7f1..e0ede692a 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -2238,6 +2238,14 @@ run_writer_with_fallback() { any_unconfigured=1 continue fi + # The workflow's token preflight sets CLAUDE_UNAVAILABLE=1 when claude is a later + # fallback but CLAUDE_CODE_OAUTH_TOKEN is absent. A tokenless claude auth error is + # terminal, so skip it as a config-gap and let a later usable provider run. + if [ "$engine" = "claude" ] && [ "${CLAUDE_UNAVAILABLE:-}" = "1" ]; then + echo "::warning::Skipping claude fallback: no CLAUDE_CODE_OAUTH_TOKEN (configuration gap, not a rate limit)" >&2 + any_unconfigured=1 + continue + fi if [ "$engine" = "copilot" ] && [[ "${COPILOT_GITHUB_TOKEN:-}" == ghp_* ]]; then echo "::warning::Skipping copilot fallback: classic PAT in COPILOT_GITHUB_TOKEN is unsupported (configuration gap, not a rate limit)" >&2 any_unconfigured=1 diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py index dd2a914d0..11696b3a3 100755 --- a/scripts/validate-ai-engines.py +++ b/scripts/validate-ai-engines.py @@ -21,6 +21,7 @@ """ import datetime import json +import math import os import re import sys @@ -76,7 +77,14 @@ def price_globs(path: Path) -> list: if len(cols) >= 6 and cols[0]: # model-pricing.sh's price_for ignores a row until its effective_from # date, so a future-dated row does not price a model running today. - if cols[1].strip() <= today: + # A malformed date or non-numeric rate would price the model as zero in + # cost_usd, so only well-formed rows count. + try: + datetime.date.fromisoformat(cols[1].strip()) + rates = [float(c) for c in cols[2:6]] + except ValueError: + continue + if all(math.isfinite(r) and r >= 0 for r in rates) and cols[1].strip() <= today: globs.append(cols[0]) return globs diff --git a/tests/dev-lead/unit/test_engine_fallback.bats b/tests/dev-lead/unit/test_engine_fallback.bats index a5d98d419..5b5d2319a 100644 --- a/tests/dev-lead/unit/test_engine_fallback.bats +++ b/tests/dev-lead/unit/test_engine_fallback.bats @@ -89,6 +89,19 @@ STUBEOF [ "$status" -eq 0 ] } +@test "fallback: tokenless claude (CLAUDE_UNAVAILABLE=1) is skipped, a later engine runs" { + _make_stub "claude" 1 + _make_stub "gemini" 0 + export COPILOT_GITHUB_TOKEN="ghp_stub" + export CLAUDE_UNAVAILABLE=1 + _source_engine "claude" + + run run_writer_with_fallback "$TEST_PROMPT" + + [ "$status" -eq 0 ] + [[ "$output" == *"Skipping claude fallback"* ]] +} + @test "fallback: primary exits 2 → tries next engine" { # claude exits 2 (rate-limited), gemini exits 0. Copilot is forced to skip via a # classic-PAT token so gemini remains the tested fallback. diff --git a/tests/test_validate_ai_engines.bats b/tests/test_validate_ai_engines.bats index 6d8e0de6d..747d175b7 100644 --- a/tests/test_validate_ai_engines.bats +++ b/tests/test_validate_ai_engines.bats @@ -88,6 +88,19 @@ _validate_with() { [ "$status" -eq 0 ] } +@test "gate: a price row with a malformed date or non-numeric rate does not price a model" { + local pt="$BATS_TEST_TMPDIR/pricing.tsv" f="$BATS_TEST_TMPDIR/ai-engines.json" + jq '.models["mistral-large"] = {"provider": "copilot", "status": "active"}' "$CONFIG" > "$f" + grep -v '^#' "$REPO_ROOT/scripts/lib/model-pricing.tsv" > "$pt" + printf 'mistral-large*\t2000-01-01\tnope\t\tbad\tx\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -ne 0 ] + [[ "$output" == *"mistral-large"*"model-pricing.tsv"* ]] + printf 'mistral-large*\tnot-a-date\t1\t1\t1\t1\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -ne 0 ] +} + @test "gate: a vendor-prefixed Copilot id is priced by its bare name" { # No row matches "azure/o4-mini" itself; the bare "o4-mini" matches o4-mini*. _validate_with '.models["azure/o4-mini"] = {"provider": "copilot", "status": "active"}' From 70cd676e297588aa2c9373756ad48472d1a5c6a8 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:24:53 +0000 Subject: [PATCH 26/29] fix(reviews): honor CLAUDE_UNAVAILABLE in the dev-lead preflight script Co-Authored-By: Claude Sonnet 5.5 --- scripts/dev-lead-preflight.sh | 3 +++ tests/dev-lead/unit/test_engine_chain_config.bats | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/scripts/dev-lead-preflight.sh b/scripts/dev-lead-preflight.sh index 7503c068f..6e5bd7eb2 100755 --- a/scripts/dev-lead-preflight.sh +++ b/scripts/dev-lead-preflight.sh @@ -54,6 +54,9 @@ if declare -F ai_engine_enabled >/dev/null 2>&1; then } [[ " $_chain " == *" claude "* ]] || CLAUDE_REQUIRED=optional fi +# The workflow's token preflight sets CLAUDE_UNAVAILABLE=1 when a non-Claude effective +# primary runs without the token; claude is then skipped by the fallback, not required. +[ "${CLAUDE_UNAVAILABLE:-}" = "1" ] && [ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] && CLAUDE_REQUIRED=optional if [ "$CLAUDE_REQUIRED" = required ]; then check_required "CLAUDE_CODE_OAUTH_TOKEN" "Claude Code CLI authentication" || FAILED=1 diff --git a/tests/dev-lead/unit/test_engine_chain_config.bats b/tests/dev-lead/unit/test_engine_chain_config.bats index f20d8b807..3723da618 100644 --- a/tests/dev-lead/unit/test_engine_chain_config.bats +++ b/tests/dev-lead/unit/test_engine_chain_config.bats @@ -719,3 +719,16 @@ EOS grep -q '^gemini rl=\[claude\]$' "$BDIR/engine_calls.txt" rm -rf "$BDIR" } + +@test "preflight: tokenless claude later in the chain passes when CLAUDE_UNAVAILABLE=1" { + export AI_ENGINES="gemini,claude" CLAUDE_UNAVAILABLE=1 + run _preflight + [ "$status" -eq 0 ] +} + +@test "preflight: CLAUDE_UNAVAILABLE does not excuse a missing token without the flag" { + export AI_ENGINES="gemini,claude" + unset CLAUDE_UNAVAILABLE + run _preflight + [ "$status" -eq 1 ] +} From 902ce904a0a11445f8e7eff586811bf341bce1fb Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:06:06 +0000 Subject: [PATCH 27/29] fix(reviews): validate the price row price_for selects, not just any valid match Co-Authored-By: Claude Sonnet 5.5 --- scripts/validate-ai-engines.py | 55 ++++++++++++++++++----------- tests/test_validate_ai_engines.bats | 13 +++++++ 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py index 11696b3a3..68e2d8978 100755 --- a/scripts/validate-ai-engines.py +++ b/scripts/validate-ai-engines.py @@ -60,10 +60,11 @@ def load_json(path: Path, what: str): sys.exit(1) -def price_globs(path: Path) -> list: - """The model_glob column of the model-pricing.tsv rows in effect today (comments - and short rows skipped, as scripts/lib/model-pricing.sh does).""" - globs = [] +def price_rows(path: Path) -> list: + """(model_glob, effective_from, well_formed) for each model-pricing.tsv row in + effect today (comments and short rows skipped, as scripts/lib/model-pricing.sh + does). Malformed rows are kept, flagged, because price_for can still select one.""" + rows = [] today = datetime.datetime.now(datetime.timezone.utc).date().isoformat() try: lines = path.read_text(encoding="utf-8").splitlines() @@ -74,19 +75,33 @@ def price_globs(path: Path) -> list: if re.match(r"^\s*#", line): continue cols = line.split("\t") - if len(cols) >= 6 and cols[0]: - # model-pricing.sh's price_for ignores a row until its effective_from - # date, so a future-dated row does not price a model running today. - # A malformed date or non-numeric rate would price the model as zero in - # cost_usd, so only well-formed rows count. - try: - datetime.date.fromisoformat(cols[1].strip()) - rates = [float(c) for c in cols[2:6]] - except ValueError: - continue - if all(math.isfinite(r) and r >= 0 for r in rates) and cols[1].strip() <= today: - globs.append(cols[0]) - return globs + # price_for ignores a row until its effective_from date, so a future-dated + # row does not price a model running today. + if len(cols) < 6 or cols[1] > today: + continue + # A malformed date or non-numeric rate would price the model as zero in + # cost_usd, so only a well-formed row counts as pricing it. + try: + datetime.date.fromisoformat(cols[1].strip()) + rates = [float(c) for c in cols[2:6]] + ok = all(math.isfinite(r) and r >= 0 for r in rates) + except ValueError: + ok = False + rows.append((cols[0], cols[1], ok)) + return rows + + +def is_priced(rows: list, model: str) -> bool: + """True when the row price_for would select for `model` (most-specific glob, ties + to the latest effective_from, then the first row) is well formed.""" + best = None + for glob, eff, ok in rows: + if not glob_matches(glob, model): + continue + key = (len(re.sub(r"[*?]", "", glob)), eff) + if best is None or key > best[0]: + best = (key, ok) + return best is not None and best[1] def glob_matches(glob: str, model: str) -> bool: @@ -109,7 +124,7 @@ def check_schema(config, schema) -> list: return problems -def check_references(config, globs) -> list: +def check_references(config, rows) -> list: problems = [] models = config["models"] providers = config["providers"] @@ -159,7 +174,7 @@ def check_references(config, globs) -> list: continue # Only the Copilot path strips a vendor prefix before pricing a run. bare = model.split("/", 1)[1] if entry["provider"] == "copilot" and "/" in model else model - if not any(glob_matches(g, bare) for g in globs): + if not is_priced(rows, bare): problems.append( f"models.{model}: provider {entry['provider']} is enabled but the model has " f"no price row in scripts/lib/model-pricing.tsv") @@ -176,7 +191,7 @@ def main(argv) -> int: problems = check_schema(config, schema) # The reference checks index into the shape the schema guarantees. if not problems: - problems = check_references(config, price_globs(pricing_path)) + problems = check_references(config, price_rows(pricing_path)) for p in problems: error(p) if problems: diff --git a/tests/test_validate_ai_engines.bats b/tests/test_validate_ai_engines.bats index 747d175b7..31e82a9e5 100644 --- a/tests/test_validate_ai_engines.bats +++ b/tests/test_validate_ai_engines.bats @@ -101,6 +101,19 @@ _validate_with() { [ "$status" -ne 0 ] } +@test "gate: a malformed more-specific row beats a valid broad glob, as price_for selects it" { + local pt="$BATS_TEST_TMPDIR/pricing.tsv" f="$BATS_TEST_TMPDIR/ai-engines.json" + jq '.models["mistral-large"] = {"provider": "copilot", "status": "active"}' "$CONFIG" > "$f" + grep -v '^#' "$REPO_ROOT/scripts/lib/model-pricing.tsv" > "$pt" + printf 'mistral-*\t2000-01-01\t1\t1\t1\t1\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -eq 0 ] + printf 'mistral-large\t2000-01-01\tbad\t1\t1\t1\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -ne 0 ] + [[ "$output" == *"mistral-large"*"model-pricing.tsv"* ]] +} + @test "gate: a vendor-prefixed Copilot id is priced by its bare name" { # No row matches "azure/o4-mini" itself; the bare "o4-mini" matches o4-mini*. _validate_with '.models["azure/o4-mini"] = {"provider": "copilot", "status": "active"}' From 36836391805aa1f6e3f1dcf00fc36889b8f09d97 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:47:56 +0000 Subject: [PATCH 28/29] fix(reviews): let model A/B pin REVIEW_ENGINE past provider disablement Co-Authored-By: Claude Sonnet 5.5 --- scripts/engine.sh | 6 +++++- scripts/evals/model-ab.sh | 2 ++ tests/dev-lead/unit/test_engine_chain_config.bats | 8 ++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/engine.sh b/scripts/engine.sh index e0ede692a..b6028ed2a 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -94,7 +94,11 @@ unset _ENGINE_CHAIN_LIB # shellcheck source=lib/model-sandbox.sh source "$(dirname "${BASH_SOURCE[0]}")/lib/model-sandbox.sh" -if declare -F ai_engine_primary >/dev/null 2>&1; then +if [ -n "${REVIEW_ENGINE_FORCE:-}" ] && [[ "${REVIEW_ENGINE:-}" =~ ^(claude|gemini|copilot)$ ]]; then + # Model A/B evals pin the engine regardless of provider enablement: scoring a + # Claude model on the configured non-Claude primary would invalidate the run. + : +elif declare -F ai_engine_primary >/dev/null 2>&1; then _requested_engine="${REVIEW_ENGINE:-}" REVIEW_ENGINE="$(ai_engine_primary "$_requested_engine")" if [ -n "$_requested_engine" ] && [ "$_requested_engine" != "$REVIEW_ENGINE" ]; then diff --git a/scripts/evals/model-ab.sh b/scripts/evals/model-ab.sh index 0a1bee537..ed040fb9e 100755 --- a/scripts/evals/model-ab.sh +++ b/scripts/evals/model-ab.sh @@ -115,6 +115,7 @@ cat >"$JUDGE_SHIM" </dev/null 2>&1; then @@ -138,6 +139,7 @@ run_arm() { local skill="$1" model="$2" out rc=0 if out="$( REVIEW_ENGINE=claude \ + REVIEW_ENGINE_FORCE=1 \ CLAUDE_TRIAGE_MODEL_CHAIN="$model" \ EVAL_ENGINE_CMD="$EVAL_ENGINE_CMD" \ EVAL_JUDGE_CMD="$JUDGE_SHIM" \ diff --git a/tests/dev-lead/unit/test_engine_chain_config.bats b/tests/dev-lead/unit/test_engine_chain_config.bats index 3723da618..ea12e6511 100644 --- a/tests/dev-lead/unit/test_engine_chain_config.bats +++ b/tests/dev-lead/unit/test_engine_chain_config.bats @@ -732,3 +732,11 @@ EOS run _preflight [ "$status" -eq 1 ] } + +@test "engine.sh: REVIEW_ENGINE_FORCE pins REVIEW_ENGINE past a disabled provider (model A/B)" { + export AI_ENGINES="gemini" + run bash -c "REVIEW_ENGINE=claude; source '$SCRIPT_DIR/scripts/engine.sh' >/dev/null 2>&1; echo \$REVIEW_ENGINE" + [ "${lines[-1]}" = "gemini" ] + run bash -c "REVIEW_ENGINE=claude REVIEW_ENGINE_FORCE=1; source '$SCRIPT_DIR/scripts/engine.sh' >/dev/null 2>&1; echo \$REVIEW_ENGINE" + [ "${lines[-1]}" = "claude" ] +} From 899eff1dfb584b9640ca7545ebcdc67ca222cf71 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:53:18 +0000 Subject: [PATCH 29/29] fix(tests): clean up Gemini API key env vars in test setup The test_engine_chain_config.bats test suite inherits GOOGLE_API_KEY_* and GEMINI_API_KEY environment variables from the test runner's environment. When tests export their own test values (k1, k2, k3), they override the inherited values within their subprocess, but tests that spawn a fresh bash shell with 'bash -c' may still read inherited variables that weren't explicitly exported. Add unset statements for all Gemini key variables in the setup() function to ensure tests run with a clean environment, fixing the failure of the 'probe-depleted Gemini keys' test. Co-Authored-By: Claude Haiku 4.5 --- tests/dev-lead/unit/test_engine_chain_config.bats | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/dev-lead/unit/test_engine_chain_config.bats b/tests/dev-lead/unit/test_engine_chain_config.bats index ea12e6511..bc72eae9b 100644 --- a/tests/dev-lead/unit/test_engine_chain_config.bats +++ b/tests/dev-lead/unit/test_engine_chain_config.bats @@ -10,6 +10,7 @@ LIB="$SCRIPT_DIR/scripts/lib/engine-chain.sh" setup() { unset AI_ENGINES DEV_LEAD_ENGINES AI_DUCK_ENGINE AI_DUCK_MODEL unset CLAUDE_AVAILABLE GEMINI_AVAILABLE COPILOT_AVAILABLE REVIEW_ENGINE + unset GOOGLE_API_KEY GOOGLE_API_KEY_2 GOOGLE_API_KEY_3 GOOGLE_API_KEY_4 GEMINI_API_KEY # shellcheck source=../../../scripts/lib/engine-chain.sh source "$LIB" }