diff --git a/.github/workflows/daily-pr-review-health.yml b/.github/workflows/daily-pr-review-health.yml index beddb7137..09005ad34 100644 --- a/.github/workflows/daily-pr-review-health.yml +++ b/.github/workflows/daily-pr-review-health.yml @@ -105,6 +105,11 @@ jobs: # Export the model override variable so ai_model_for_family honours # operator-configured model chains (vars.AI_MODELS_CLAUDE). AI_MODELS_CLAUDE: ${{ vars.AI_MODELS_CLAUDE }} + # The other break-glass overrides, so the report lists every one that + # differs from config/ai-engines.json (#1973). + AI_MODELS_GEMINI: ${{ vars.AI_MODELS_GEMINI }} + AI_MODELS_COPILOT: ${{ vars.AI_MODELS_COPILOT }} + AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} run: bash scripts/pr_review_health.sh - name: Truncate report if needed diff --git a/.github/workflows/dev-lead-reusable.yml b/.github/workflows/dev-lead-reusable.yml index 8df23e6ef..fb0db7783 100644 --- a/.github/workflows/dev-lead-reusable.yml +++ b/.github/workflows/dev-lead-reusable.yml @@ -191,7 +191,7 @@ jobs: # legacy DEV_LEAD_ENGINES name is still honoured); DEV_LEAD_ENGINE is a # legacy primary override — leave it unset to let AI_ENGINES decide. AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} - DEV_LEAD_ENGINE: ${{ vars.DEV_LEAD_ENGINE || (startsWith(vars.AI_ENGINES || vars.DEV_LEAD_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES || vars.DEV_LEAD_ENGINES, 'copilot') && 'copilot') || 'claude' }} + DEV_LEAD_ENGINE: ${{ vars.DEV_LEAD_ENGINE || (startsWith(vars.AI_ENGINES || vars.DEV_LEAD_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES || vars.DEV_LEAD_ENGINES, 'copilot') && 'copilot') || '' }} AI_DUCK_ENGINE: ${{ vars.AI_DUCK_ENGINE || '' }} AI_DUCK_MODEL: ${{ vars.AI_DUCK_MODEL || '' }} GEMINI_FLASH_MODEL: ${{ vars.GEMINI_FLASH_MODEL || '' }} @@ -214,32 +214,59 @@ jobs: - name: Engine token preflight (#1525) # With the secret declaration relaxed to required:false, presence is # enforced here instead — one actionable line, not a secrets-evaluation - # stack error on every event. Skipped when AI_ENGINES leaves claude out. + # stack error on every event. Skipped when AI_ENGINES leaves claude out or + # config/ai-engines.json disables Claude (provider disablement takes + # precedence over AI_ENGINES, so check the file before requiring the token). env: AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} + DEV_LEAD_ENGINE: ${{ vars.DEV_LEAD_ENGINE || '' }} + AGENT_REF: ${{ inputs.agent_ref }} + GH_TOKEN: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | - # Is Claude enabled? Only a valid AI_ENGINES list that omits claude turns it - # off; empty or invalid values fall back to the default chain, which includes - # claude (scripts/lib/engine-chain.sh — mirrored inline because this runs - # before the scripts are checked out). - claude_enabled=true - spec="$(printf '%s' "${AI_ENGINES:-}" | tr '[:upper:],' '[:lower:] ')" - if [ -n "${spec//[[:space:]]/}" ]; then + # Require the token only when the EFFECTIVE primary engine is claude, as + # scripts/lib/engine-chain.sh's ai_engine_primary resolves it (mirrored + # inline: this runs before checkout). The file's enabled providers in + # fallback_order are the default chain; AI_ENGINES narrows it (a valid list + # cannot re-enable a disabled provider; an invalid one means the default + # chain and primary); DEV_LEAD_ENGINE wins only when the chain enables it. + # Fetched through the API; unreadable config fails closed (claude first). + file_chain="$(gh api -H 'Accept: application/vnd.github.raw' \ + "repos/petry-projects/.github-private/contents/config/ai-engines.json?ref=${AGENT_REF}" 2>/dev/null \ + | jq -r '. as $c | [$c.providers.fallback_order[] | select($c.providers[.].enabled == true)] | join(" ")' 2>/dev/null || true)" + [ -n "$file_chain" ] || file_chain="claude gemini copilot" + effective_primary() { + local pref spec e chain="" valid=true first kept="" + pref="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" + spec="$(printf '%s' "${AI_ENGINES:-}" | tr '[:upper:],' '[:lower:] ')" set -f - known=true; has_claude=false for e in $spec; do case "$e" in - claude) has_claude=true ;; - gemini|copilot) ;; - *) known=false ;; + claude|gemini|copilot) case " $chain " in *" $e "*) ;; *) chain="$chain $e" ;; esac ;; + *) valid=false ;; esac done set +f - if [ "$known" = true ] && [ "$has_claude" = false ]; then claude_enabled=false; fi - fi - if [ "$claude_enabled" = false ]; then - echo "claude not in AI_ENGINES (${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" + if [ "$valid" = false ]; then chain=""; pref=""; fi + if [ -n "${chain// /}" ]; then + for e in $chain; do case " $file_chain " in *" $e "*) kept="$kept $e" ;; esac; done + chain="$kept" + else + chain=" $file_chain" + fi + [ -n "${chain// /}" ] || { printf '%s' none; return; } + if [ -n "$pref" ] && [[ "$chain " == *" $pref "* ]]; then printf '%s' "$pref"; return; fi + first="${chain# }" + printf '%s' "${first%% *}" + } + primary="$(effective_primary "$DEV_LEAD_ENGINE")" + if [ "$primary" != claude ]; then + if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then + # claude may still sit later in the chain; mark it unavailable so + # run_writer_with_fallback skips it instead of hitting a terminal auth error. + echo "CLAUDE_UNAVAILABLE=1" >> "$GITHUB_ENV" + fi + echo "effective primary engine is ${primary} (AI_ENGINES=${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" exit 0 fi if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then @@ -255,7 +282,7 @@ jobs: - name: Checkout caller repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + token: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} - name: Checkout dev-lead scripts uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -268,8 +295,10 @@ jobs: # Cross-repo callers: github.token cannot read a private org repo — ensure # GH_PAT_WORKFLOWS is set as an org or repo secret in the calling repo. token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + # config/ holds ai-engines.json, the engine defaults the scripts read (#1973). sparse-checkout: | scripts + config prompts/dev-lead personas/dev-lead @@ -468,7 +497,22 @@ jobs: # Log the CLI version in use, cached or fresh (#1957). echo "claude-code version: $(claude --version 2>/dev/null || echo unavailable)" - case "$DEV_LEAD_ENGINE" in + # DEV_LEAD_ENGINE is empty when no legacy override is set and AI_ENGINES + # does not start with gemini/copilot. Resolve the primary from the + # checked-out config so the fallback CLIs still install; claude is the + # last resort when the config cannot be read. + engine="${DEV_LEAD_ENGINE:-}" + if [ -z "$engine" ]; then + # shellcheck source=scripts/lib/engine-chain.sh + if source .dev-lead/scripts/lib/engine-chain.sh 2>/dev/null \ + && primary="$(ai_engine_primary "")" && [ -n "$primary" ]; then + engine="$primary" + else + engine=claude + fi + fi + + case "$engine" in claude) npm install -g @google/gemini-cli || true if ! gh copilot --version >/dev/null 2>&1; then @@ -797,7 +841,7 @@ jobs: - name: Checkout caller repo uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - token: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} - name: Checkout dev-lead scripts uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -805,9 +849,11 @@ jobs: repository: combsmith/.github-private ref: ${{ inputs.agent_ref }} path: .dev-lead - token: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + # config/ holds ai-engines.json, the engine defaults the scripts read (#1973). sparse-checkout: | scripts + config prompts/dev-lead personas/dev-lead @@ -983,29 +1029,54 @@ jobs: - name: Engine token preflight (#1525) env: AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES || '' }} + DEV_LEAD_ENGINE: ${{ vars.DEV_LEAD_ENGINE || '' }} + AGENT_REF: ${{ inputs.agent_ref }} + GH_TOKEN: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | - # Is Claude enabled? Only a valid AI_ENGINES list that omits claude turns it - # off; empty or invalid values fall back to the default chain, which includes - # claude (scripts/lib/engine-chain.sh — mirrored inline because this runs - # before the scripts are checked out). - claude_enabled=true - spec="$(printf '%s' "${AI_ENGINES:-}" | tr '[:upper:],' '[:lower:] ')" - if [ -n "${spec//[[:space:]]/}" ]; then + # Require the token only when the EFFECTIVE primary engine is claude, as + # scripts/lib/engine-chain.sh's ai_engine_primary resolves it (mirrored + # inline: this runs before checkout). The file's enabled providers in + # fallback_order are the default chain; AI_ENGINES narrows it (a valid list + # cannot re-enable a disabled provider; an invalid one means the default + # chain and primary); DEV_LEAD_ENGINE wins only when the chain enables it. + # Fetched through the API; unreadable config fails closed (claude first). + file_chain="$(gh api -H 'Accept: application/vnd.github.raw' \ + "repos/petry-projects/.github-private/contents/config/ai-engines.json?ref=${AGENT_REF}" 2>/dev/null \ + | jq -r '. as $c | [$c.providers.fallback_order[] | select($c.providers[.].enabled == true)] | join(" ")' 2>/dev/null || true)" + [ -n "$file_chain" ] || file_chain="claude gemini copilot" + effective_primary() { + local pref spec e chain="" valid=true first kept="" + pref="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" + spec="$(printf '%s' "${AI_ENGINES:-}" | tr '[:upper:],' '[:lower:] ')" set -f - known=true; has_claude=false for e in $spec; do case "$e" in - claude) has_claude=true ;; - gemini|copilot) ;; - *) known=false ;; + claude|gemini|copilot) case " $chain " in *" $e "*) ;; *) chain="$chain $e" ;; esac ;; + *) valid=false ;; esac done set +f - if [ "$known" = true ] && [ "$has_claude" = false ]; then claude_enabled=false; fi - fi - if [ "$claude_enabled" = false ]; then - echo "claude not in AI_ENGINES (${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" + if [ "$valid" = false ]; then chain=""; pref=""; fi + if [ -n "${chain// /}" ]; then + for e in $chain; do case " $file_chain " in *" $e "*) kept="$kept $e" ;; esac; done + chain="$kept" + else + chain=" $file_chain" + fi + [ -n "${chain// /}" ] || { printf '%s' none; return; } + if [ -n "$pref" ] && [[ "$chain " == *" $pref "* ]]; then printf '%s' "$pref"; return; fi + first="${chain# }" + printf '%s' "${first%% *}" + } + primary="$(effective_primary "$DEV_LEAD_ENGINE")" + if [ "$primary" != claude ]; then + if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then + # claude may still sit later in the chain; mark it unavailable so + # run_writer_with_fallback skips it instead of hitting a terminal auth error. + echo "CLAUDE_UNAVAILABLE=1" >> "$GITHUB_ENV" + fi + echo "effective primary engine is ${primary} (AI_ENGINES=${AI_ENGINES}) — CLAUDE_CODE_OAUTH_TOKEN not required" exit 0 fi if [ -z "$CLAUDE_CODE_OAUTH_TOKEN" ]; then @@ -1026,10 +1097,12 @@ jobs: # defaults to "main" → unchanged behaviour (#535/#506). ref: ${{ inputs.agent_ref }} path: .dev-lead - token: ${{ secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} + token: ${{ secrets.GH_PAT_DON_PETRY_WRITER || secrets.GH_PAT_DON_PETRY || secrets.GH_PAT_WORKFLOWS || github.token }} persist-credentials: false + # config/ holds ai-engines.json, the engine defaults the scripts read (#1973). sparse-checkout: | scripts + config - name: Resolve PR from check_run id: relay diff --git a/.github/workflows/engine-config-validate.yml b/.github/workflows/engine-config-validate.yml new file mode 100644 index 000000000..6df71bb9f --- /dev/null +++ b/.github/workflows/engine-config-validate.yml @@ -0,0 +1,44 @@ +name: Validate Engine Config + +on: + pull_request: + paths: + - 'config/ai-engines.json' + - 'config/ai-engines.schema.json' + - 'scripts/lib/model-pricing.tsv' + - 'scripts/validate-ai-engines.py' + - '.github/workflows/engine-config-validate.yml' + push: + branches: [main] + paths: + - 'config/ai-engines.json' + - 'config/ai-engines.schema.json' + - 'scripts/lib/model-pricing.tsv' + - 'scripts/validate-ai-engines.py' + merge_group: + +permissions: + contents: read + +jobs: + validate-ai-engines: + # The engine config gate (#1973). config/ai-engines.json is the source of + # truth for enabled providers, the model catalog and each task's chain, so a + # bad value is caught at PR time instead of in a failing review. Validates + # the file against config/ai-engines.schema.json (draft 2020-12), then fails + # on a task model that is missing from `models` or `retired`, a model in + # another provider's chain, an enabled provider's model with no row in + # scripts/lib/model-pricing.tsv, and a duck or Copilot chain with more than + # one model. Fully HERMETIC. This is the required check for config changes. + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install Python deps + run: pip3 install --quiet "jsonschema==4.26.0" --only-binary=":all:" + + - name: Validate config/ai-engines.json + run: python3 scripts/validate-ai-engines.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index df63956a2..d09d6fed2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,6 +15,7 @@ on: - 'interaction-contracts/**' - 'evals/**' - 'release/**' + - 'config/**' # config/ai-engines.json + schema (validate-ai-engines) - '.gitleaks.toml' # Hash-pinned pip requirements installed by the jobs below (#1240), so a # Dependabot bump of requirements/ci/ is exercised before it merges. @@ -33,6 +34,7 @@ on: - 'prompts/**' - '.github/workflows/**' - '.github/**' # check-model-pins scans all of .github + - 'config/**' workflow_dispatch: permissions: @@ -217,6 +219,9 @@ jobs: tests/test_discover_parity_skills.bats \ tests/test_eval_health_notify.bats \ tests/test_consumer_manifest.bats \ + tests/test_validate_ai_engines.bats \ + tests/dev-lead/unit/test_ai_engines_config.bats \ + tests/dev-lead/unit/test_install_step_engine.bats \ tests/test_downstream_impact_map.bats \ tests/test_downstream_impact_assemble.bats \ tests/test_downstream_impact_triage_inline.bats \ @@ -542,6 +547,28 @@ jobs: - name: Validate per-case eval schema conformance across the tree run: python3 evals/validate-cases.py --schema-tree evals + validate-ai-engines: + # The engine config gate (#1973). config/ai-engines.json is the source of + # truth for enabled providers, the model catalog and each task's chain, so a + # bad value is caught at PR time instead of in a failing review. Validates + # the file against config/ai-engines.schema.json (draft 2020-12, the same + # jsonschema setup as validate-eval-cases), then fails on a task model that is + # missing from `models` or `retired`, a model in another provider's chain, an + # enabled provider's model with no row in scripts/lib/model-pricing.tsv, and a + # duck or Copilot chain with more than one model. Fully HERMETIC. + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install Python deps + run: pip3 install --quiet "jsonschema==4.26.0" --only-binary=":all:" + + - name: Validate config/ai-engines.json + run: python3 scripts/validate-ai-engines.py + validate-interaction-contracts: # Interaction-contract well-formedness gate (Story 2 / #1404, AC #5). Parses # every machine-readable interaction contract — the co-located persona lenses diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index d382c35f3..1ad808780 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -217,7 +217,7 @@ jobs: # The primary is derived here (expressions cannot split a list) from the # first AI_ENGINES entry; the scripts enforce the full list. AI_ENGINES: ${{ vars.AI_ENGINES || '' }} - REVIEW_ENGINE: ${{ vars.REVIEW_ENGINE || (startsWith(vars.AI_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES, 'copilot') && 'copilot') || 'claude' }} + REVIEW_ENGINE: ${{ vars.REVIEW_ENGINE || (startsWith(vars.AI_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES, 'copilot') && 'copilot') || '' }} AI_DUCK_ENGINE: ${{ vars.AI_DUCK_ENGINE || '' }} AI_DUCK_MODEL: ${{ vars.AI_DUCK_MODEL || '' }} GEMINI_FLASH_MODEL: ${{ vars.GEMINI_FLASH_MODEL || '' }} @@ -332,9 +332,19 @@ jobs: # and install claude only best-effort, so they legitimately run WITHOUT this # token — failing them here would be a regression, not the fail-loud we want. env: - REVIEW_ENGINE: ${{ vars.REVIEW_ENGINE || (startsWith(vars.AI_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES, 'copilot') && 'copilot') || 'claude' }} + REVIEW_ENGINE: ${{ vars.REVIEW_ENGINE || (startsWith(vars.AI_ENGINES, 'gemini') && 'gemini') || (startsWith(vars.AI_ENGINES, 'copilot') && 'copilot') || '' }} + AGENT_REF: ${{ inputs.agent_ref || 'main' }} + GH_TOKEN: ${{ secrets.DON_PETRY_BOT_GH_PAT_CLASSIC || secrets.DON_PETRY_BOT_GH_PAT || github.token }} CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} run: | + # The file's enabled providers in fallback_order (what the runtime's + # default chain is). Fetched through the API (this step precedes + # checkout); when it cannot be read, fall back to claude-first and keep + # requiring the token (fail closed). + file_chain="$(gh api -H 'Accept: application/vnd.github.raw' \ + "repos/petry-projects/.github-private/contents/config/ai-engines.json?ref=${AGENT_REF}" 2>/dev/null \ + | jq -r '. as $c | [$c.providers.fallback_order[] | select($c.providers[.].enabled == true)] | join(" ")' 2>/dev/null || true)" + [ -n "$file_chain" ] || file_chain="claude gemini copilot" effective_primary() { local pref spec e chain="" valid=true first pref="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" @@ -348,7 +358,15 @@ jobs: done set +f if [ "$valid" = false ]; then chain=""; pref=""; fi - [ -n "${chain// /}" ] || chain=" claude gemini copilot" + if [ -n "${chain// /}" ]; then + # AI_ENGINES narrows the file; it cannot re-enable a provider the file disables. + local kept="" + for e in $chain; do case " $file_chain " in *" $e "*) kept="$kept $e" ;; esac; done + chain="$kept" + else + chain=" $file_chain" + fi + [ -n "${chain// /}" ] || { printf '%s' none; return; } if [ -n "$pref" ] && [[ "$chain " == *" $pref "* ]]; then printf '%s' "$pref"; return; fi first="${chain# }" printf '%s' "${first%% *}" @@ -533,6 +551,20 @@ jobs: echo "claude-code installed: $(claude --version 2>/dev/null || echo unavailable)" } + # REVIEW_ENGINE is only a preference: it is empty when no legacy + # override is set and AI_ENGINES does not start with gemini/copilot, + # and it can name an engine the config disables. Always resolve the + # enabled primary from the checked-out config so the case below sees + # the engine the run actually uses; fall back to the preference, then + # claude, when the config cannot be read. + # shellcheck source=scripts/lib/engine-chain.sh + if source scripts/lib/engine-chain.sh 2>/dev/null \ + && primary="$(ai_engine_primary "${REVIEW_ENGINE:-}")" && [ -n "$primary" ]; then + REVIEW_ENGINE="$primary" + else + REVIEW_ENGINE="${REVIEW_ENGINE:-claude}" + fi + case "$REVIEW_ENGINE" in claude) install_claude diff --git a/.github/workflows/test-dev-lead.yml b/.github/workflows/test-dev-lead.yml index 6f8ae0254..2fc9e0316 100644 --- a/.github/workflows/test-dev-lead.yml +++ b/.github/workflows/test-dev-lead.yml @@ -6,6 +6,7 @@ on: - 'scripts/auto-rebase-retry.sh' - 'scripts/engine.sh' - 'scripts/lib/**' + - 'config/**' - '.github/workflows/dev-lead*.yml' - '.github/workflows/test-dev-lead.yml' - '.github/workflows/issue-triage-runner.yml' @@ -28,6 +29,7 @@ on: - 'scripts/auto-rebase-retry.sh' - 'scripts/engine.sh' - 'scripts/lib/**' + - 'config/**' - 'tests/dev-lead/**' - 'prompts/dev-lead/**' - '.github/review-mcp.json' diff --git a/AGENTS.md b/AGENTS.md index 8a7b08857..55c77ec22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -230,6 +230,14 @@ calling it a regression, and treat CI as authoritative.** agentic workflow compilation. It is not covered by the org template and must not be removed by template syncs. If the org template gains a `gh-aw-compile` equivalent, remove this exception and defer to the template instead. +- **Exception:** The `validate-ai-engines` job in `lint.yml` (#1973) is a documented repo-specific addition + that runs `scripts/validate-ai-engines.py` against `config/ai-engines.json`; the `paths:` filters also carry + `config/**` for it. It is not covered by the org template and must not be removed by template syncs. If the + org template gains an equivalent, remove this exception and defer to the template instead. +- **Exception:** `.github/workflows/engine-config-validate.yml` (#1973) is a documented repo-specific + workflow that runs `scripts/validate-ai-engines.py` when `config/ai-engines.json`, its schema, or the + pricing table change. It is not covered by the org template and must not be removed by template syncs. + If the org template gains an equivalent, remove this exception and defer to the template instead. - **Exception:** the `lint.yml` `paths:` filters (pull_request and push) carry the repo-specific entry `docs/initiatives/agent-ingress-collapse-markets.md`, which is read by `tests/agent_ingress_markets_package.bats` (#2038). It must not be removed as template drift on a sync; @@ -249,9 +257,9 @@ calling it a regression, and treat CI as authoritative.** Because this is a **gate** workflow whose `failure` conclusion is intentional enforcement (it blocks a bad PR), it is exempt from the Fleet Monitor's high-failure issue tracking via `FLEET_GATE_WORKFLOWS` in `scripts/fleet_report.sh` (#941) — otherwise a guard doing its job produces false-positive trackers. - `holdout-guard.yml` and `sync-scope-guard.yml` (which fails closed on a sync PR with no declared-paths - manifest or an out-of-scope diff, #1700/#2373) are exempt for the same reason. Add new gate workflows of - this class to that list. + `holdout-guard.yml`, `sync-scope-guard.yml` (which fails closed on a sync PR with no declared-paths + manifest or an out-of-scope diff, #1700/#2373) and `engine-config-validate.yml` are exempt for the same + reason. Add new gate workflows of this class to that list. - **Exception:** `pr-review-sweep.yml` (stuck-review sweep, #573/#898) is a documented repo-specific workflow with no corresponding org template in `standards/workflows/`. It re-dispatches reviews for PRs that went green after a ci-pending/ci-failing skip, via two triggers: a scheduled cron (the EVENTUAL @@ -804,12 +812,13 @@ current concrete id at run time; do **not** hard-pin `claude--` ids across the tree (#1979, companion [`combsmith/.github#1199`](https://github.com/combsmith/.github/issues/1199)). A model swap should be one edit to the chains in -[`scripts/lib/engine-models.sh`](./scripts/lib/engine-models.sh), not a hunt for +[`config/ai-engines.json`](./config/ai-engines.json) (validated by +`scripts/validate-ai-engines.py`), not a hunt for scattered ids. - **Resolve with `ai_model_for_family `** (`scripts/lib/engine-models.sh`): it returns the current id for the family, honouring `CLAUDE__MODEL_CHAIN` - then `AI_MODELS_CLAUDE` then the built-in default. A shell caller sources the lib + then `AI_MODELS_CLAUDE` then `config/ai-engines.json`. A shell caller sources the lib and calls the function; a workflow adds a resolve step that writes the id to a step output. In gh-aw front-matter, `engine:` names the runner engine (e.g. `claude`) and the model family is named in `models:` (e.g. diff --git a/config/ai-engines.json b/config/ai-engines.json new file mode 100644 index 000000000..238edd8f0 --- /dev/null +++ b/config/ai-engines.json @@ -0,0 +1,116 @@ +{ + "$schema": "./ai-engines.schema.json", + "$comment": "Source of truth for the review and dev-lead engines (#1973): which providers are enabled, which models exist, and each task's chain. Read by scripts/lib/engine-models.sh; validated by scripts/validate-ai-engines.py in lint.yml. Ships with the pr-review/* and dev-lead/* channel tags, so a change here canaries on next before ring0 and stable. AI_MODELS_* and AI_ENGINES override it in an emergency — fold the override back in here. See docs/engine-configuration.md.", + "version": 1, + "models": { + "claude-haiku-4-5-20251001": { + "provider": "claude", + "status": "active", + "notes": ["Triage primary; the haiku family's current id (ai_model_for_family haiku)."] + }, + "claude-sonnet-5-5": { + "provider": "claude", + "status": "active", + "notes": [ + "Default sonnet across triage, deep, action and duck (#1978, epic #1095 Phase 3).", + "In deep it is the last hop: deep's known-good hop is opus-4-8, so sonnet 5.5 alone is its sonnet tail (#1978 AC-2)." + ] + }, + "claude-sonnet-5": { + "provider": "claude", + "status": "active", + "notes": [ + "Known-good sonnet safety hop one behind sonnet 5.5 in triage and action.", + "The id is bare claude-sonnet-5: the claude-sonnet-5-0 spelling shipped by #1100 404s, so every hop to it silently fell through (#1957)." + ] + }, + "claude-opus-5-5": { + "provider": "claude", + "status": "active", + "notes": [ + "Deep primary since #1898 (epic #1895 Phase 2); audit and single primary since #1901 (epic #1895 Phase 3), replacing Fable 5.", + "Action's last (cross-family) hop since #1900 (epic #1895 Phase 3), replacing opus-4-8 once the deep-tier canary passed (#1899).", + "CLI/API default reasoning effort is medium (low/medium/high/xhigh/max); engine.sh's run_agentic pins --effort high for deep, audit and single so moving off Fable does not lower their effort." + ] + }, + "claude-opus-4-8": { + "provider": "claude", + "status": "active", + "notes": [ + "Known-good second hop behind opus-5-5 in deep, audit and single (#1957), so a throttled opus-5-5 degrades to the model it replaced.", + "Minimum cacheable prefix 4096 tokens." + ] + }, + "claude-opus-4-7": { + "provider": "claude", + "status": "active", + "notes": ["Last hop of the audit and single chains."] + }, + "claude-fable-5": { + "provider": "claude", + "status": "retired", + "notes": [ + "Deprecated by #1901 (epic #1895 Phase 3): opus-5-5 outperforms it at lower cost, so no default chain names it.", + "Reasoning effort: the deep, audit and single tiers pin --effort high (engine.sh run_agentic) so replacing Fable with opus-5-5, whose default effort is medium, did not lower their effort.", + "A chain set through AI_MODELS_CLAUDE or CLAUDE__MODEL_CHAIN is still honoured as a stop-gap and warns once per run; its price rows stay for historical token records." + ] + }, + "gemini-3.8-flash": { + "provider": "gemini", + "status": "active", + "notes": ["Triage and action primary; the known-working fallback of the quality tier."] + }, + "gemini-3.1-pro-preview": { + "provider": "gemini", + "status": "preview", + "notes": ["Quality tier (deep, audit, single) since gemini-2.5-pro was withdrawn for new keys (#1960)."] + }, + "gemini-2.5-pro": { + "provider": "gemini", + "status": "retired", + "notes": ["\"No longer available to new users … use models/gemini-3.1-pro-preview\": it broke every Gemini deep review while it sat in code (#1960)."] + }, + "openai/o4-mini": { + "provider": "copilot", + "status": "active", + "notes": ["GitHub Models id. Copilot has no in-engine chain, so each task names one model."] + } + }, + "providers": { + "fallback_order": ["claude", "gemini", "copilot"], + "claude": { "enabled": true }, + "gemini": { "enabled": true }, + "copilot": { "enabled": true } + }, + "tasks": { + "triage": { + "claude": ["claude-haiku-4-5-20251001", "claude-sonnet-5-5", "claude-sonnet-5"], + "gemini": ["gemini-3.8-flash", "gemini-3.1-pro-preview"], + "copilot": ["openai/o4-mini"] + }, + "deep": { + "claude": ["claude-opus-5-5", "claude-opus-4-8", "claude-sonnet-5-5"], + "gemini": ["gemini-3.1-pro-preview", "gemini-3.8-flash"], + "copilot": ["openai/o4-mini"] + }, + "audit": { + "claude": ["claude-opus-5-5", "claude-opus-4-8", "claude-opus-4-7"], + "gemini": ["gemini-3.1-pro-preview", "gemini-3.8-flash"], + "copilot": ["openai/o4-mini"] + }, + "action": { + "claude": ["claude-sonnet-5-5", "claude-sonnet-5", "claude-opus-5-5"], + "gemini": ["gemini-3.8-flash", "gemini-3.1-pro-preview"], + "copilot": ["openai/o4-mini"] + }, + "single": { + "claude": ["claude-opus-5-5", "claude-opus-4-8", "claude-opus-4-7"], + "gemini": ["gemini-3.1-pro-preview", "gemini-3.8-flash"], + "copilot": ["openai/o4-mini"] + }, + "duck": { + "$comment": "Gemini and Copilot have no duck chain: their duck is the first model of their triage chain.", + "claude": ["claude-sonnet-5-5"] + } + } +} diff --git a/config/ai-engines.schema.json b/config/ai-engines.schema.json new file mode 100644 index 000000000..a118deab7 --- /dev/null +++ b/config/ai-engines.schema.json @@ -0,0 +1,109 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/petry-projects/.github-private/config/ai-engines.schema.json", + "title": "AI engine configuration", + "description": "Providers, model catalog and per-task model chains for the review and dev-lead engines (#1973). Cross-reference checks (catalog membership, retired models, provider match, price rows, one-model chains) live in scripts/validate-ai-engines.py.", + "type": "object", + "required": ["version", "models", "providers", "tasks"], + "additionalProperties": false, + "properties": { + "$schema": { "type": "string" }, + "$comment": { "type": "string" }, + "version": { "const": 1 }, + "models": { + "type": "object", + "minProperties": 1, + "propertyNames": { "$ref": "#/$defs/modelId" }, + "additionalProperties": { "$ref": "#/$defs/model" } + }, + "providers": { + "type": "object", + "required": ["fallback_order", "claude", "gemini", "copilot"], + "additionalProperties": false, + "properties": { + "$comment": { "type": "string" }, + "fallback_order": { + "description": "Default cross-provider fallback order; the first enabled entry is the primary. AI_ENGINES overrides it.", + "type": "array", + "items": { "$ref": "#/$defs/providerName" }, + "minItems": 3, + "maxItems": 3, + "uniqueItems": true + }, + "claude": { "$ref": "#/$defs/provider" }, + "gemini": { "$ref": "#/$defs/provider" }, + "copilot": { "$ref": "#/$defs/provider" } + } + }, + "tasks": { + "type": "object", + "required": ["triage", "deep", "audit", "action", "single", "duck"], + "additionalProperties": false, + "properties": { + "$comment": { "type": "string" }, + "triage": { "$ref": "#/$defs/task" }, + "deep": { "$ref": "#/$defs/task" }, + "audit": { "$ref": "#/$defs/task" }, + "action": { "$ref": "#/$defs/task" }, + "single": { "$ref": "#/$defs/task" }, + "duck": { "$ref": "#/$defs/task" } + } + } + }, + "$defs": { + "providerName": { "enum": ["claude", "gemini", "copilot"] }, + "modelId": { + "description": "Same id rule as AI_MODELS_* entries (scripts/lib/engine-models.sh).", + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:/@-]*$" + }, + "notes": { + "oneOf": [ + { "type": "string", "minLength": 1 }, + { "type": "array", "items": { "type": "string", "minLength": 1 } } + ] + }, + "model": { + "type": "object", + "required": ["provider", "status"], + "additionalProperties": false, + "properties": { + "provider": { "$ref": "#/$defs/providerName" }, + "status": { "enum": ["active", "preview", "retired"] }, + "notes": { "$ref": "#/$defs/notes" } + } + }, + "provider": { + "type": "object", + "required": ["enabled"], + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" }, + "notes": { "$ref": "#/$defs/notes" } + } + }, + "chain": { + "description": "Ordered model chain: primary first, then fallbacks.", + "type": "array", + "items": { "$ref": "#/$defs/modelId" }, + "minItems": 1, + "uniqueItems": true + }, + "task": { + "type": "object", + "additionalProperties": false, + "properties": { + "$comment": { "type": "string" }, + "claude": { "$ref": "#/$defs/chain" }, + "gemini": { "$ref": "#/$defs/chain" }, + "copilot": { "$ref": "#/$defs/chain" }, + "prefer": { + "description": "Reserved for per-task provider preference (Phase 2); not read yet.", + "type": "array", + "items": { "$ref": "#/$defs/providerName" }, + "uniqueItems": true + } + } + } + } +} diff --git a/docs/agentic-interaction-model.md b/docs/agentic-interaction-model.md index b7734a360..b82d3ebf6 100644 --- a/docs/agentic-interaction-model.md +++ b/docs/agentic-interaction-model.md @@ -240,6 +240,7 @@ keeps its own file) carries **no** job qualifier and remains exactly one row. > | `.github/workflows/test-aw.yml` | CI test workflow, not an agentic role | > | `.github/workflows/duplicate-decl-gate.yml` | CI gate guard — enforces duplicate declaration check, not an agentic role | > | `.github/workflows/holdout-guard.yml` | Gate guard — enforces merge conditions, not an agentic role | +> | `.github/workflows/engine-config-validate.yml` | CI gate guard — validates config/ai-engines.json, not an agentic role | > | `.github/workflows/test-deletion-guard.yml` | Gate guard — enforces test retention, not an agentic role | > | `.github/workflows/sync-scope-guard.yml` | Gate guard — enforces sync-PR scope, not an agentic role | > | `.github/workflows/dependabot-automerge.yml` | Dependabot plumbing — thin caller stub | diff --git a/docs/engine-configuration.md b/docs/engine-configuration.md index 0aee4f017..fbd782b58 100644 --- a/docs/engine-configuration.md +++ b/docs/engine-configuration.md @@ -1,24 +1,103 @@ -# Engine configuration (AI_ENGINES) +# Engine configuration (config/ai-engines.json, AI_ENGINES) pr-review and dev-lead can run on three LLM engines: **Claude**, **Gemini** and -**Copilot**. Which engines are used, and in what order, is set with GitHub -Actions **variables** (org or repo level). Switching providers needs no code -change and no release: the next run picks the new value up. - -The parsing lives in [`scripts/lib/engine-chain.sh`](../scripts/lib/engine-chain.sh). +**Copilot**. Which engines are enabled, which models exist and which models each +task uses are set in one versioned file, +[`config/ai-engines.json`](../config/ai-engines.json) (#1973). A small set of +GitHub Actions **variables** stays as a documented emergency override. + +The engine chain is parsed in +[`scripts/lib/engine-chain.sh`](../scripts/lib/engine-chain.sh), the model lists +in [`scripts/lib/engine-models.sh`](../scripts/lib/engine-models.sh). + +## The config file + +`config/ai-engines.json` is the source of truth. It has three sections: + +- **`models`** — the catalog, keyed by model id. Each entry gives its + `provider` (`claude`, `gemini` or `copilot`), its `status` (`active`, + `preview` or `retired`) and optional `notes`, such as why a chain has its + fallbacks, the Sonnet 5 id history and the Fable 5 deprecation. +- **`providers`** — `claude`, `gemini` and `copilot`, each with `enabled`, and + `fallback_order`, the default order across providers (its first enabled + entry is the primary). +- **`tasks`** — `triage`, `deep`, `audit`, `action`, `single` and `duck`. Each + gives, per provider, an ordered chain: primary first, then fallbacks. + `tasks..prefer` (a provider list) is accepted for per-task provider + preference but not read yet. + +The file is read with `jq`, resolved relative to `engine-models.sh` (so it is +found wherever the scripts are checked out). A missing, unreadable or malformed +file is an `::error::` and fails the step: there are no built-in defaults to fall +back to, because stale defaults are what this file replaces. `engine.sh` reads it +once when it is sourced. + +**It ships like code.** The file is released with the `pr-review/*` and +`dev-lead/*` channel tags, so a default-model change canaries on `next` before +`ring0` and `stable`. An Actions variable changes every channel at once, which is +why variables are only the emergency path. + +**It is checked at PR time.** The `validate-ai-engines` job in `lint.yml` runs +[`scripts/validate-ai-engines.py`](../scripts/validate-ai-engines.py). It +validates the file against +[`config/ai-engines.schema.json`](../config/ai-engines.schema.json) (JSON Schema +draft 2020-12), and also fails when: + +- a task names a model that is missing from `models`, or is `retired`; +- a model sits in another provider's chain; +- a non-retired model of an enabled provider has no price row in + [`scripts/lib/model-pricing.tsv`](../scripts/lib/model-pricing.tsv) (a + vendor-prefixed Copilot id such as `openai/o4-mini` is priced by its bare name); +- `duck`, or any `copilot` chain, lists more than one model. + +Run it locally with `python3 scripts/validate-ai-engines.py` (needs +`pip install 'jsonschema>=4'`). + +## Precedence + +For each task's chain, highest first: + +1. **A specific environment variable** — `CLAUDE__MODEL_CHAIN`, + `GEMINI_FLASH_MODEL(_CHAIN)`, `GEMINI_PRO_MODEL(_CHAIN)`, `COPILOT_API_MODEL`. + These are **internal**: they are kept for tests and the A/B runner. Do not + set them as Actions variables; use `AI_MODELS_*`. + The reusable workflows still forward the legacy `GEMINI_FLASH_MODEL` / + `GEMINI_PRO_MODEL` Actions variables, so a leftover one **silently overrides** + `AI_MODELS_GEMINI`; they are deprecated — delete them. +2. **`AI_MODELS_*`** — the break-glass override (below). +3. **`config/ai-engines.json`**. + +For providers, the file decides which are enabled. `AI_ENGINES` is the kill +switch and the fallback order: it can disable or reorder providers the file +enables, but it **cannot enable** one the file disables (that is ignored with a +`::warning::`). + +## Emergency path + +When a model breaks in production (retired, renamed, throttled): + +1. **Now:** set the `AI_MODELS_` variable (or `AI_ENGINES` to turn a + provider off). The next run picks it up on every channel. +2. **Then:** land the same change in `config/ai-engines.json` through a normal + PR and release, and delete the variable once `stable` carries it. + +The daily PR-review health check (`scripts/pr_review_health.sh`) lists, under +**Engine configuration overrides**, every `AI_MODELS_*` key and `AI_ENGINES` +value that is set and differs from the file, so a temporary override is folded +back instead of living on silently. ## Variables | Variable | Default | Meaning | |---|---|---| -| `AI_ENGINES` | `claude,gemini,copilot` | Ordered list of **enabled** engines. An engine that is not listed is never used, not even as a fallback. The order is the fallback order, and the **first** entry is the primary engine. | +| `AI_ENGINES` | the file's `providers.fallback_order` (`claude,gemini,copilot`) | Kill switch and fallback order. An engine that is not listed is never used, not even as a fallback. The order is the fallback order, and the **first** entry is the primary engine. It cannot enable an engine the file disables. | | `AI_DUCK_ENGINE` | automatic | Ordered list of engines for pr-review's rubber-duck second opinion, e.g. `gemini,claude`. Each entry is `claude`, `gemini` or `copilot`. If an engine returns no verdict, the next one runs. `none` alone turns the duck off. | | `AI_DUCK_MODEL` | engine default | Model id for the duck. Used only on the **first** engine in `AI_DUCK_ENGINE`. | -| `AI_MODELS_CLAUDE` | see below | Claude's model list: one chain per task. | -| `AI_MODELS_GEMINI` | see below | Gemini's model list, with the same keys. | -| `AI_MODELS_COPILOT` | see below | Copilot's model list, with the same keys (one model per key). | -| `GEMINI_FLASH_MODEL` | unset | Replaces only the first model of the Gemini `triage` and `action` chains. | -| `GEMINI_PRO_MODEL` | unset | Replaces only the first model of the Gemini `deep`, `audit` and `single` chains. | +| `AI_MODELS_CLAUDE` | unset (the file) | **Break-glass** override of Claude's chains, one per task. | +| `AI_MODELS_GEMINI` | unset (the file) | **Break-glass** override of Gemini's chains, with the same keys. | +| `AI_MODELS_COPILOT` | unset (the file) | **Break-glass** override of Copilot's models, with the same keys (one model per key). | +| `GEMINI_FLASH_MODEL` | unset | **Deprecated** (internal). Replaces only the first model of the Gemini `triage` and `action` chains. | +| `GEMINI_PRO_MODEL` | unset | **Deprecated** (internal). Replaces only the first model of the Gemini `deep`, `audit` and `single` chains. | | `REVIEW_ENGINE` | unset | **Legacy** primary override for pr-review. When set, it wins over the first `AI_ENGINES` entry, as long as `AI_ENGINES` enables it. Delete it to let `AI_ENGINES` decide. | | `DEV_LEAD_ENGINE` | unset | **Legacy** primary override for dev-lead; same rule as `REVIEW_ENGINE`. | | `DEV_LEAD_ENGINES` | unset | **Legacy** name for `AI_ENGINES` (dev-lead, #1546). Read only when `AI_ENGINES` is unset. | @@ -27,15 +106,15 @@ The parsing lives in [`scripts/lib/engine-chain.sh`](../scripts/lib/engine-chain `claude,gemini`. Start the value with an engine name (no leading space): the workflows read the primary from the start of the value. -## Model lists (AI_MODELS_*) +## Break-glass model lists (AI_MODELS_*) -Each provider's models live in one variable, so a retired or renamed model is a -variable edit, not a code change. The parsing and the defaults are in -[`scripts/lib/engine-models.sh`](../scripts/lib/engine-models.sh). +Each provider has one variable that overrides its chains from the file, so a +broken model can be replaced at once, before the file change is released. The +parsing is in [`scripts/lib/engine-models.sh`](../scripts/lib/engine-models.sh). Each entry is `=[,,…]`. Separate entries with `;` or new lines. Keys are case-insensitive and spaces around names are ignored. A key you -leave out keeps its default. A chain is walked left to right on a rate limit, +leave out keeps the file's chain. A chain is walked left to right on a rate limit, before the next engine in `AI_ENGINES` is tried. Every provider takes the same six keys, one per task: @@ -53,9 +132,10 @@ Every provider takes the same six keys, one per task: because its GitHub Models client has no chain. A key limited to one model that is given several warns and uses the first. -Defaults: +Defaults (from `config/ai-engines.json` at the time of writing — the file is +authoritative): -| Key | `AI_MODELS_CLAUDE` | `AI_MODELS_GEMINI` | `AI_MODELS_COPILOT` | +| Key | Claude | Gemini | Copilot | |---|---|---|---| | `triage` | `claude-haiku-4-5-20251001,claude-sonnet-5-5,claude-sonnet-5` | `gemini-3.8-flash,gemini-3.1-pro-preview` | `openai/o4-mini` | | `deep` | `claude-opus-5-5,claude-opus-4-8,claude-sonnet-5-5` | `gemini-3.1-pro-preview,gemini-3.8-flash` | `openai/o4-mini` | @@ -84,7 +164,8 @@ AI_MODELS_COPILOT = deep=openai/gpt-5; duck=openai/gpt-5-mini ``` A more specific variable still wins over `AI_MODELS_*`, so existing overrides -keep working: +keep working. These are **internal / deprecated** — kept for tests and the A/B +runner (`model-ab.yml`), not for operators: - `CLAUDE__MODEL_CHAIN` replaces that Claude chain. - `GEMINI_FLASH_MODEL_CHAIN` replaces the Gemini `triage` and `action` chains. @@ -118,9 +199,9 @@ the chains stay the single source of truth: | `haiku` | `triage` | Overrides are honoured with the same precedence engine.sh uses: the per-tier -`CLAUDE__MODEL_CHAIN` env first, then `AI_MODELS_CLAUDE`, then the built-in -default. An operator who supplies a concrete `claude-*` id still has it honoured -verbatim. In gh-aw front-matter the model family is named in `models:` (e.g. +`CLAUDE__MODEL_CHAIN` env first, then `AI_MODELS_CLAUDE`, then the chain +in `config/ai-engines.json`. An operator who supplies a concrete `claude-*` id +still has it honoured verbatim. In gh-aw front-matter the model family is named in `models:` (e.g. `models: { claude: [sonnet] }`), which accepts both a family and a full id; `engine:` selects the runner engine (`claude`), not the model. Workflow `model` inputs likewise accept a family or a full id. @@ -146,8 +227,10 @@ table are allow-listed. | Duck on Gemini, then Claude if Gemini is throttled | `AI_DUCK_ENGINE=gemini,claude` | | Duck on Gemini only, no automatic fallback | `AI_DUCK_ENGINE=gemini,none` | | Turn the duck off | `AI_DUCK_ENGINE=none` | -| Replace a retired Gemini deep-review model | `AI_MODELS_GEMINI=deep=,gemini-3.8-flash` | -| Pin Claude's deep review to Opus 4.8 | `AI_MODELS_CLAUDE=deep=claude-opus-4-8,claude-sonnet-5` | +| Replace a retired Gemini deep-review model (emergency) | `AI_MODELS_GEMINI=deep=,gemini-3.8-flash`, then the same change in `config/ai-engines.json` | +| Pin Claude's deep review to Opus 4.8 (emergency) | `AI_MODELS_CLAUDE=deep=claude-opus-4-8,claude-sonnet-5`, then the same change in the file | +| Change a default model | Edit `tasks..` in `config/ai-engines.json` (add the model to `models` first) | +| Turn a provider off for good | `providers..enabled=false` in the file | ## Behaviour @@ -190,7 +273,8 @@ table are allow-listed. value. - **Model chains inside a provider** (for example Claude's deep tier `claude-opus-5-5 → claude-opus-4-8 → claude-sonnet-5-5`) are walked before any - cross-provider fallback. Set them with `AI_MODELS_*` (above). + cross-provider fallback. Set them in `config/ai-engines.json` (above), or + with `AI_MODELS_*` in an emergency. ## Model isolation in PR review diff --git a/scripts/dev-lead-preflight.sh b/scripts/dev-lead-preflight.sh index b3557cceb..6e5bd7eb2 100755 --- a/scripts/dev-lead-preflight.sh +++ b/scripts/dev-lead-preflight.sh @@ -47,9 +47,16 @@ CLAUDE_REQUIRED=required _PREFLIGHT_CHAIN_LIB="$(dirname "${BASH_SOURCE[0]}")/lib/engine-chain.sh" # shellcheck source=lib/engine-chain.sh [ -f "$_PREFLIGHT_CHAIN_LIB" ] && source "$_PREFLIGHT_CHAIN_LIB" -if declare -F ai_engine_enabled >/dev/null 2>&1 && ! ai_engine_enabled claude; then - CLAUDE_REQUIRED=optional +if declare -F ai_engine_enabled >/dev/null 2>&1; then + _chain="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + [[ " $_chain " == *" claude "* ]] || CLAUDE_REQUIRED=optional fi +# The workflow's token preflight sets CLAUDE_UNAVAILABLE=1 when a non-Claude effective +# primary runs without the token; claude is then skipped by the fallback, not required. +[ "${CLAUDE_UNAVAILABLE:-}" = "1" ] && [ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] && CLAUDE_REQUIRED=optional if [ "$CLAUDE_REQUIRED" = required ]; then check_required "CLAUDE_CODE_OAUTH_TOKEN" "Claude Code CLI authentication" || FAILED=1 diff --git a/scripts/engine.sh b/scripts/engine.sh index 4930780a0..b6028ed2a 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -44,6 +44,45 @@ set -euo pipefail # concurrent runs do not read each other's values. DEV_LEAD_SIDECAR_DIR="${DEV_LEAD_SIDECAR_DIR:-/tmp}" +# Each provider's model list: the defaults in config/ai-engines.json, read by +# lib/engine-models.sh, with AI_MODELS_CLAUDE / AI_MODELS_GEMINI / +# AI_MODELS_COPILOT as break-glass overrides. Sourced before the engine chain, +# which takes the enabled providers and their order from the same file. +_ENGINE_MODELS_LIB="$(dirname "${BASH_SOURCE[0]}")/lib/engine-models.sh" +if [ ! -f "$_ENGINE_MODELS_LIB" ]; then + echo "::error::engine.sh: missing $_ENGINE_MODELS_LIB (model lists and defaults)" >&2 + return 1 2>/dev/null || exit 1 +fi +# shellcheck source=lib/engine-models.sh +source "$_ENGINE_MODELS_LIB" +unset _ENGINE_MODELS_LIB +# The defaults come from config/ai-engines.json (#1973). Read it once here: +# the lookups below and in child $(…) shells reuse the table, and a missing or +# malformed file fails the step instead of running on stale defaults. +if ! ai_engines_config_load; then + return 1 2>/dev/null || exit 1 +fi +if [ -z "${AI_MODELS_PROBLEM_REPORTED:-}" ]; then + _models_problems="$(ai_models_problems)" + if [ -n "$_models_problems" ]; then + while IFS= read -r _models_line; do + echo "::warning::$_models_line" >&2 + done <<< "$_models_problems" + export AI_MODELS_PROBLEM_REPORTED=1 + fi + unset _models_problems _models_line +fi +# Fable 5 is deprecated (#1901): warn once per run when a configured chain still +# names a claude-fable-* model, but still honour it (a stop-gap, not a rejection). +if [ -z "${AI_MODELS_FABLE_WARNED:-}" ]; then + _fable_warn="$(ai_models_fable_deprecation)" + if [ -n "$_fable_warn" ]; then + echo "::warning::$_fable_warn" >&2 + export AI_MODELS_FABLE_WARNED=1 + fi + unset _fable_warn +fi + # Configured engine chain (AI_ENGINES): which engines are enabled and in what # order. The primary is REVIEW_ENGINE when that is set AND enabled, otherwise # the first engine in AI_ENGINES — so a disabled engine is never used, even when @@ -55,7 +94,11 @@ unset _ENGINE_CHAIN_LIB # shellcheck source=lib/model-sandbox.sh source "$(dirname "${BASH_SOURCE[0]}")/lib/model-sandbox.sh" -if declare -F ai_engine_primary >/dev/null 2>&1; then +if [ -n "${REVIEW_ENGINE_FORCE:-}" ] && [[ "${REVIEW_ENGINE:-}" =~ ^(claude|gemini|copilot)$ ]]; then + # Model A/B evals pin the engine regardless of provider enablement: scoring a + # Claude model on the configured non-Claude primary would invalidate the run. + : +elif declare -F ai_engine_primary >/dev/null 2>&1; then _requested_engine="${REVIEW_ENGINE:-}" REVIEW_ENGINE="$(ai_engine_primary "$_requested_engine")" if [ -n "$_requested_engine" ] && [ "$_requested_engine" != "$REVIEW_ENGINE" ]; then @@ -77,38 +120,6 @@ else fi export REVIEW_ENGINE -# Each provider's model list (AI_MODELS_CLAUDE / AI_MODELS_GEMINI / -# AI_MODELS_COPILOT) and its defaults live in lib/engine-models.sh, so a retired -# or renamed model is an Actions-variable edit, not a code change. -_ENGINE_MODELS_LIB="$(dirname "${BASH_SOURCE[0]}")/lib/engine-models.sh" -if [ ! -f "$_ENGINE_MODELS_LIB" ]; then - echo "::error::engine.sh: missing $_ENGINE_MODELS_LIB (model lists and defaults)" >&2 - return 1 2>/dev/null || exit 1 -fi -# shellcheck source=lib/engine-models.sh -source "$_ENGINE_MODELS_LIB" -unset _ENGINE_MODELS_LIB -if [ -z "${AI_MODELS_PROBLEM_REPORTED:-}" ]; then - _models_problems="$(ai_models_problems)" - if [ -n "$_models_problems" ]; then - while IFS= read -r _models_line; do - echo "::warning::$_models_line" >&2 - done <<< "$_models_problems" - export AI_MODELS_PROBLEM_REPORTED=1 - fi - unset _models_problems _models_line -fi -# Fable 5 is deprecated (#1901): warn once per run when a configured chain still -# names a claude-fable-* model, but still honour it (a stop-gap, not a rejection). -if [ -z "${AI_MODELS_FABLE_WARNED:-}" ]; then - _fable_warn="$(ai_models_fable_deprecation)" - if [ -n "$_fable_warn" ]; then - echo "::warning::$_fable_warn" >&2 - export AI_MODELS_FABLE_WARNED=1 - fi - unset _fable_warn -fi - # Cross-engine rubber-duck model (issue #773). The duck deliberately routes to # Copilot/o4-mini even when the primary engine is NOT copilot (e.g. the default # claude engine sets DUCK_ENGINE=copilot for adversarial diversity), so @@ -2231,6 +2242,14 @@ run_writer_with_fallback() { any_unconfigured=1 continue fi + # The workflow's token preflight sets CLAUDE_UNAVAILABLE=1 when claude is a later + # fallback but CLAUDE_CODE_OAUTH_TOKEN is absent. A tokenless claude auth error is + # terminal, so skip it as a config-gap and let a later usable provider run. + if [ "$engine" = "claude" ] && [ "${CLAUDE_UNAVAILABLE:-}" = "1" ]; then + echo "::warning::Skipping claude fallback: no CLAUDE_CODE_OAUTH_TOKEN (configuration gap, not a rate limit)" >&2 + any_unconfigured=1 + continue + fi if [ "$engine" = "copilot" ] && [[ "${COPILOT_GITHUB_TOKEN:-}" == ghp_* ]]; then echo "::warning::Skipping copilot fallback: classic PAT in COPILOT_GITHUB_TOKEN is unsupported (configuration gap, not a rate limit)" >&2 any_unconfigured=1 diff --git a/scripts/evals/model-ab.sh b/scripts/evals/model-ab.sh index 5d7c04287..ed040fb9e 100755 --- a/scripts/evals/model-ab.sh +++ b/scripts/evals/model-ab.sh @@ -114,6 +114,8 @@ JUDGE_SHIM="$(mktemp)" cat >"$JUDGE_SHIM" </dev/null 2>&1; then @@ -136,6 +138,8 @@ _is_scored() { [ "$1" -eq 0 ] || [ "$1" -eq 1 ]; } run_arm() { local skill="$1" model="$2" out rc=0 if out="$( + REVIEW_ENGINE=claude \ + REVIEW_ENGINE_FORCE=1 \ CLAUDE_TRIAGE_MODEL_CHAIN="$model" \ EVAL_ENGINE_CMD="$EVAL_ENGINE_CMD" \ EVAL_JUDGE_CMD="$JUDGE_SHIM" \ diff --git a/scripts/fleet_report.sh b/scripts/fleet_report.sh index 64e066770..0cbd7ea9b 100755 --- a/scripts/fleet_report.sh +++ b/scripts/fleet_report.sh @@ -13,7 +13,7 @@ # (#941). Matched by workflow file basename. See AGENTS.md "Exception: # test-deletion-guard.yml". Override via env with the full space-separated # gate list for testing; add permanent gates to the default below. -FLEET_GATE_WORKFLOWS="${FLEET_GATE_WORKFLOWS:-test-deletion-guard.yml holdout-guard.yml sync-scope-guard.yml}" +FLEET_GATE_WORKFLOWS="${FLEET_GATE_WORKFLOWS:-test-deletion-guard.yml holdout-guard.yml sync-scope-guard.yml engine-config-validate.yml}" # Dev-Lead status-marker prefix (#1019, story C of #901). Mirrors # ISSUE_MARKER_PREFIX in dev-lead-fix-issue.sh / dev-lead-retry.sh: every diff --git a/scripts/lib/consumer-manifest.json b/scripts/lib/consumer-manifest.json index afc6e1033..06c936d28 100644 --- a/scripts/lib/consumer-manifest.json +++ b/scripts/lib/consumer-manifest.json @@ -140,6 +140,7 @@ ], "surface_sources": { ".github/workflows/dev-lead-reusable.yml": [ + "config/ai-engines.json", "prompts/dev-lead/fix-bot-comment.md", "prompts/dev-lead/fix-ci.md", "prompts/dev-lead/fix-issue.md", @@ -159,6 +160,7 @@ "scripts/lib/reviewer-sources.tsv" ], ".github/workflows/pr-review.yml": [ + "config/ai-engines.json", "prompts/cascade-action.md", "prompts/deep-review.md", "prompts/rubber-duck.md", diff --git a/scripts/lib/downstream-impact.sh b/scripts/lib/downstream-impact.sh index 19541608a..b37c89be1 100644 --- a/scripts/lib/downstream-impact.sh +++ b/scripts/lib/downstream-impact.sh @@ -13,7 +13,7 @@ # source-graph expansion. Two and only two match kinds: # (a) a changed reusable-workflow path that a consumer ref pins exactly # (after normalising the manifest's provider prefix) -> direct surface; -# (b) a changed scripts/lib/*.sh, scripts/lib/*.tsv, or prompts/* path listed in a reusable's +# (b) a changed scripts/lib/*.sh, scripts/lib/*.tsv, prompts/* or config/*.json path listed in a reusable's # surface_sources -> that reusable surface -> the reusable's consumers. # # A consumer ref is provider-prefixed (e.g. ".github-private/.github/workflows/ diff --git a/scripts/lib/engine-chain.sh b/scripts/lib/engine-chain.sh index 70c262da3..735bc9cf7 100644 --- a/scripts/lib/engine-chain.sh +++ b/scripts/lib/engine-chain.sh @@ -1,15 +1,18 @@ # shellcheck shell=bash # scripts/lib/engine-chain.sh — the configured AI engine chain (AI_ENGINES). # -# One org/repo Actions variable, AI_ENGINES, turns engines on or off and sets -# their order, so switching providers needs no code change: +# config/ai-engines.json (#1973) enables providers and sets the default order +# (providers.fallback_order). One org/repo Actions variable, AI_ENGINES, is the +# kill switch on top of it: it turns engines off and reorders them with no +# release, but it cannot turn on a provider the file disables: # -# AI_ENGINES="claude,gemini,copilot" the default +# AI_ENGINES="claude,gemini,copilot" the default (the file's order) # AI_ENGINES="claude,gemini" Copilot off (never tried, not even as a fallback) # AI_ENGINES="gemini,claude" Gemini first, Claude as the fallback # AI_ENGINES="claude" Claude only # -# - membership = enabled. An engine that is not listed is never used. +# - membership = enabled. An engine that is not listed is never used, nor is +# one the file disables (reported by ai_engine_chain_problem). # - order = fallback preference, walked left to right on a rate limit. # - first = the primary engine, unless REVIEW_ENGINE / DEV_LEAD_ENGINE is # set explicitly (a legacy override; it is ignored, with a @@ -22,10 +25,42 @@ # reports it so validate_engines can warn once. # # Sourced by engine.sh, validate-engines.sh and review-batch.sh. Defines -# functions only; runs nothing at source time and does not call `set`. +# functions only; at source time it only sources engine-models.sh (the file +# reader) when present, and does not call `set`. +# The built-in order, used only when engine-models.sh is not next to this lib +# (or the file cannot be read; engine.sh fails the step on that first). AI_ENGINES_DEFAULT="claude gemini copilot" +if ! declare -F ai_engines_file_providers >/dev/null 2>&1 \ + && [ -f "$(dirname "${BASH_SOURCE[0]}")/engine-models.sh" ]; then + # shellcheck source=scripts/lib/engine-models.sh + source "$(dirname "${BASH_SOURCE[0]}")/engine-models.sh" +fi + +# _ai_engine_default_chain — the file's enabled providers in fallback_order. +# AI_ENGINES_DEFAULT is used only when engine-models.sh is not available; if +# the file cannot be read when the lib is available, fails closed (returns 1) +# so callers never silently re-enable providers the file disables. +_ai_engine_default_chain() { + local c + if declare -F ai_engines_file_providers >/dev/null 2>&1; then + # The function exists (engine-models.sh was sourced); must succeed. + c="$(ai_engines_file_providers)" || return 1 + printf '%s' "$c" + return 0 + fi + # Function not available; use the built-in default. + printf '%s' "$AI_ENGINES_DEFAULT" +} + +# _ai_engine_file_disabled — the providers the file disables (space-separated). +# Returns 1 when the file cannot be read (if engine-models.sh was sourced). +_ai_engine_file_disabled() { + declare -F ai_engines_file_disabled >/dev/null 2>&1 || return 0 + ai_engines_file_disabled +} + # _ai_engine_spec — the raw configured value (AI_ENGINES, else DEV_LEAD_ENGINES). _ai_engine_spec() { printf '%s' "${AI_ENGINES:-${DEV_LEAD_ENGINES:-}}" @@ -53,29 +88,63 @@ _ai_engine_parse() { } # ai_engine_chain — the enabled engines in preference order, space-separated. -# Empty or invalid configuration → the default chain. +# Engines the file disables are dropped. Empty or invalid configuration → the +# default chain. A valid configuration that names only disabled engines → empty +# (fails closed rather than silently re-enabling providers the spec excluded). +# Returns 1 if the config file cannot be read (when engine-models.sh is sourced). ai_engine_chain() { - local parsed - if parsed="$(_ai_engine_parse "$(_ai_engine_spec)")" && [ -n "$parsed" ]; then - printf '%s' "$parsed" + local spec parsed disabled e kept="" chain + spec="$(_ai_engine_spec)" + if [ -z "$spec" ]; then + chain="$(_ai_engine_default_chain)" || return 1 + printf '%s' "$chain" + return 0 + fi + if parsed="$(_ai_engine_parse "$spec")" && [ -n "$parsed" ]; then + disabled="$(_ai_engine_file_disabled)" || return 1 + for e in $parsed; do + [[ " $disabled " == *" $e "* ]] || kept="${kept:+$kept }$e" + done + fi + if [ -z "$kept" ] && [ -z "${spec//[[:space:],]/}" ]; then + chain="$(_ai_engine_default_chain)" || return 1 + printf '%s' "$chain" + elif [ -z "$kept" ] && ! _ai_engine_parse "$spec" >/dev/null 2>&1; then + chain="$(_ai_engine_default_chain)" || return 1 + printf '%s' "$chain" else - printf '%s' "$AI_ENGINES_DEFAULT" + printf '%s' "$kept" fi } # ai_engine_chain_problem — prints a one-line description when the configured -# value is unusable (unknown engine, or nothing but separators); prints nothing -# when it is fine or unset. +# value is unusable (unknown engine, nothing but separators, or an engine the +# file disables); prints nothing when it is fine or unset. Returns 1 if the +# config file cannot be read (when engine-models.sh is sourced). ai_engine_chain_problem() { - local spec parsed + local spec parsed disabled e off="" chain spec="$(_ai_engine_spec)" [ -n "$spec" ] || return 0 if [ -z "${spec//[[:space:],]/}" ]; then + chain="$(_ai_engine_default_chain)" || return 1 printf "AI_ENGINES='%s' lists no engine — using the default chain '%s'" \ - "$spec" "$AI_ENGINES_DEFAULT" + "$spec" "$chain" + return 0 elif ! parsed="$(_ai_engine_parse "$spec")" || [ -z "$parsed" ]; then + chain="$(_ai_engine_default_chain)" || return 1 printf "AI_ENGINES='%s' names an unknown engine (expected claude, gemini, copilot) — using the default chain '%s'" \ - "$spec" "$AI_ENGINES_DEFAULT" + "$spec" "$chain" + return 0 + fi + disabled="$(_ai_engine_file_disabled)" || return 1 + for e in $parsed; do + [[ " $disabled " != *" $e "* ]] || off="${off:+$off, }$e" + done + if [ -n "$off" ]; then + # AI_ENGINES narrows the file; it cannot turn a provider back on. + chain="$(ai_engine_chain)" || return 1 + printf "AI_ENGINES='%s' names %s, which config/ai-engines.json has disabled — ignored (enable a provider in the file, not with AI_ENGINES); using '%s'" \ + "$spec" "$off" "$chain" elif [[ "$spec" =~ ^[[:space:],] ]]; then # The workflows derive the primary with startsWith(vars.AI_ENGINES, …), # which cannot skip leading separators. @@ -85,8 +154,11 @@ ai_engine_chain_problem() { } # ai_engine_enabled — 0 when is in the configured chain. +# Returns 1 if the config file cannot be read (when engine-models.sh is sourced). ai_engine_enabled() { - [[ " $(ai_engine_chain) " == *" ${1:-} "* ]] + local chain + chain="$(ai_engine_chain)" || return 1 + [[ " $chain " == *" ${1:-} "* ]] } # ai_engine_primary [preferred] — when it is enabled, otherwise the @@ -94,11 +166,11 @@ ai_engine_enabled() { # An INVALID AI_ENGINES value means full defaults, primary included: the # workflows derive REVIEW_ENGINE / DEV_LEAD_ENGINE from the raw value's first # token, so honouring there would let "copilot,typo" still make -# Copilot primary (#1961 review). +# Copilot primary (#1961 review). Returns 1 if the config file cannot be read. ai_engine_primary() { local pref chain spec pref="$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" - chain="$(ai_engine_chain)" + chain="$(ai_engine_chain)" || return 1 spec="$(_ai_engine_spec)" if [ -n "${spec//[[:space:],]/}" ] && ! _ai_engine_parse "$spec" >/dev/null; then pref="" @@ -132,10 +204,10 @@ ai_engine_available() { # ai_engine_next_available — the first available engine AFTER # in the chain (forward only, so a rate-limit walk cannot cycle). # When is not in the chain, searches the whole chain. Prints nothing -# when there is none. +# when there is none. Returns 1 if the config file cannot be read. ai_engine_next_available() { local current="${1:-}" chain e seen=0 - chain="$(ai_engine_chain)" + chain="$(ai_engine_chain)" || return 1 [[ " $chain " == *" $current "* ]] || seen=1 for e in $chain; do if [ "$seen" -eq 0 ]; then @@ -150,9 +222,11 @@ ai_engine_next_available() { } # ai_engine_first_available — the first available engine in the chain. +# Returns 1 if the config file cannot be read (when engine-models.sh is sourced). ai_engine_first_available() { - local e - for e in $(ai_engine_chain); do + local e chain + chain="$(ai_engine_chain)" || return 1 + for e in $chain; do if ai_engine_available "$e"; then printf '%s' "$e" return 0 diff --git a/scripts/lib/engine-models.sh b/scripts/lib/engine-models.sh index 21967b500..20960679d 100644 --- a/scripts/lib/engine-models.sh +++ b/scripts/lib/engine-models.sh @@ -1,8 +1,16 @@ # shellcheck shell=bash -# scripts/lib/engine-models.sh — the model list of each provider (AI_MODELS_). +# scripts/lib/engine-models.sh — the model list of each provider. # -# One Actions variable per provider holds every model it uses, so a retired or -# renamed model is a variable edit rather than a code change and a release: +# The defaults live in config/ai-engines.json (#1973), a versioned, +# schema-validated file (scripts/validate-ai-engines.py runs in lint.yml) that +# ships with the pr-review/* and dev-lead/* channel tags, so a default-model +# change canaries on `next` before `ring0` and `stable`. It is read with jq, +# resolved relative to this lib (AI_ENGINES_CONFIG overrides the path, for +# tests). A missing or malformed file is an ::error::, never a silent fall back +# to stale built-in defaults. +# +# One Actions variable per provider is the break-glass override on top of the +# file — set it in an emergency, then fold the change back into the file: # # AI_MODELS_CLAUDE triage=… ; deep=… ; audit=… ; action=… ; single=… ; duck=… # AI_MODELS_GEMINI (the same keys) @@ -31,64 +39,113 @@ # whole chain; GEMINI_FLASH_MODEL / GEMINI_PRO_MODEL replace # only its first model; # Copilot COPILOT_API_MODEL (every key); -# 2. AI_MODELS_; -# 3. ai_models_default. +# 2. AI_MODELS_ (break-glass override); +# 3. config/ai-engines.json (ai_models_default). +# The per-tier variables in 1. are internal (tests, the A/B runner); operators +# use AI_MODELS_* in an emergency and the file otherwise. # An unknown key or a malformed model id drops that entry (its default applies); # ai_models_problems reports it so engine.sh can warn once. # -# Sourced by engine.sh. Defines functions only; runs nothing at source time. +# Sourced by engine.sh and engine-chain.sh. Defines functions only; at source +# time it only resolves the config path. -# ai_models_default — the built-in chain for /. -# -# Claude notes: -# - Sonnet 5.5 (#1978, epic #1095 Phase 3) is the default sonnet across triage, -# deep, action and duck. In triage and action it replaces claude-sonnet-5 as -# the sonnet hop and sonnet-5 stays one hop behind it as the known-good -# sonnet safety hop (its id is bare `claude-sonnet-5`: the `claude-sonnet-5-0` -# spelling shipped by #1100 404s, so every hop to it silently fell through — -# #1957). Deep's known-good hop is opus-4-8 (see below), so its sonnet tail is -# sonnet 5.5 alone, as #1978 AC-2 specifies. audit/single are opus tiers and -# keep sonnet out entirely. -# - Deep swapped opus-4-8 → opus-5-5 (#1898, epic #1895 Phase 2). opus-4-8 is -# the known-good 2nd hop (#1957), so a throttled or unavailable opus-5-5 -# degrades to the model it replaced instead of failing; sonnet 5.5 is last. -# - Audit and single swapped fable-5 → opus-5-5 (#1901, epic #1895 Phase 3), -# deprecating Fable 5: the engine no longer depends on Fable, which opus-5-5 -# outperforms at lower cost, so reviews run one current model family end to -# end. Both keep opus-4-8 as the known-good 2nd hop and opus-4-7 as the last. -# - Action's last (cross-family) hop swapped opus-4-8 → opus-5-5 (#1900, epic -# #1895 Phase 3) once the deep-tier canary passed (#1899): the writer's last -# resort is the cheaper, proven model. -# - opus-5-5's CLI/API default reasoning effort is `medium` (levels -# low/medium/high/xhigh/max); the deep, audit and single tiers pin -# `--effort high` in engine.sh's run_agentic so moving off Fable does not -# silently lower their reasoning effort (triage/action keep the default). -# Min cacheable prefix opus-4-8 = 4096 tok. -# - Fable 5 is deprecated but not removed: a chain that still names a -# claude-fable-* model (AI_MODELS_CLAUDE or CLAUDE__MODEL_CHAIN) is -# honoured as a stop-gap and warns once per run (ai_models_fable_deprecation); -# its price rows stay in model-pricing.tsv for historical token records. -# - The subscription cap is shared across Claude models (#206), so an -# in-Claude chain only helps with per-model RPM/TPM limits. +# The model notes (Sonnet 5 id history, the Fable 5 deprecation, why each chain +# has its fallbacks) live next to each model in config/ai-engines.json. +_AI_ENGINES_CONFIG_DEFAULT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." 2>/dev/null && pwd)/config/ai-engines.json" + +# ai_engines_config_path — the engine config file: AI_ENGINES_CONFIG when set, +# else config/ai-engines.json at the root of the checkout holding this lib. +ai_engines_config_path() { + printf '%s' "${AI_ENGINES_CONFIG:-$_AI_ENGINES_CONFIG_DEFAULT}" +} + +# The file flattened to one line per fact, so a lookup is parameter expansion +# (Bash 3.2: no associative arrays): +# :=,,… one per chain the file sets +# providers= +# disabled= +# shellcheck disable=SC2016 # a jq program: $p / $t are jq variables +_AI_ENGINES_JQ=' + if (.tasks | type) != "object" or (.providers | type) != "object" + or (.providers.fallback_order | type) != "array" then + error("expected the tasks and providers sections (providers.fallback_order)") + else . end + | .providers as $p + | ( .tasks | to_entries[] | select(.value | type == "object") | .key as $t + | .value | to_entries[] + | select(.key == "claude" or .key == "gemini" or .key == "copilot") + | "\(.key):\($t)=\(.value | join(","))" ), + "providers=\([$p.fallback_order[] | select($p[.].enabled == true)] | join(" "))", + "disabled=\([("claude", "gemini", "copilot") | select($p[.].enabled != true)] | join(" "))" +' + +# _ai_engines_config_read — prints the flattened table, or an ::error:: +# on stderr and returns 1 when the file is missing, unreadable or malformed. +_ai_engines_config_read() { + local path="$1" out why + if [ ! -f "$path" ] || [ ! -r "$path" ]; then + printf '::error::engine config %s is missing or unreadable — it holds the providers, models and task chains; there are no built-in defaults to fall back to (docs/engine-configuration.md)\n' "$path" >&2 + return 1 + fi + if ! command -v jq >/dev/null 2>&1; then + printf '::error::engine config %s cannot be read: jq is not installed\n' "$path" >&2 + return 1 + fi + if ! out="$(jq -r "$_AI_ENGINES_JQ" "$path" 2>/dev/null)"; then + why="$(jq -r "$_AI_ENGINES_JQ" "$path" 2>&1 >/dev/null | head -n 1)" + printf '::error::engine config %s is malformed (%s) — run scripts/validate-ai-engines.py\n' "$path" "$why" >&2 + return 1 + fi + printf '%s' "$out" +} + +# ai_engines_config_load — reads the file once into this shell, so later +# lookups (and the $(…) subshells that inherit the table) skip jq. engine.sh +# calls it at source time and fails the step when it returns 1. A lookup after +# AI_ENGINES_CONFIG changes rereads the file. +ai_engines_config_load() { + local path table + path="$(ai_engines_config_path)" + table="$(_ai_engines_config_read "$path")" || return 1 + _AI_ENGINES_TABLE="$table" + _AI_ENGINES_TABLE_SRC="$path" +} + +# _ai_engines_lookup — the value of in the table ("" when the file +# does not set it); returns 1 when the file cannot be read. +_ai_engines_lookup() { + local name="$1" path table rest nl=$'\n' + path="$(ai_engines_config_path)" + if [ -n "${_AI_ENGINES_TABLE:-}" ] && [ "${_AI_ENGINES_TABLE_SRC:-}" = "$path" ]; then + table="$_AI_ENGINES_TABLE" + else + table="$(_ai_engines_config_read "$path")" || return 1 + fi + table="$nl$table$nl" + rest="${table#*"$nl$name="}" + [ "$rest" != "$table" ] || return 0 + printf '%s' "${rest%%"$nl"*}" +} + +# ai_models_default — the file's chain for / +# (tasks..). Empty when the file sets none: Gemini and Copilot +# have no duck chain, their duck follows the triage model. Returns 1, with an +# ::error::, when the file cannot be read. ai_models_default() { - case "${1:-}:${2:-}" in - claude:triage) printf '%s' "claude-haiku-4-5-20251001,claude-sonnet-5-5,claude-sonnet-5" ;; - claude:deep) printf '%s' "claude-opus-5-5,claude-opus-4-8,claude-sonnet-5-5" ;; - claude:audit) printf '%s' "claude-opus-5-5,claude-opus-4-8,claude-opus-4-7" ;; - claude:action) printf '%s' "claude-sonnet-5-5,claude-sonnet-5,claude-opus-5-5" ;; - claude:single) printf '%s' "claude-opus-5-5,claude-opus-4-8,claude-opus-4-7" ;; - claude:duck) printf '%s' "claude-sonnet-5-5" ;; - # gemini-2.5-pro is withdrawn for new keys ("no longer available to new - # users … use models/gemini-3.1-pro-preview", #1960), so the quality tier - # uses 3.1-pro-preview and degrades to the flash model that is known to work. - # No gemini/copilot duck default: the duck follows the triage model. - gemini:triage|gemini:action) - printf '%s' "gemini-3.8-flash,gemini-3.1-pro-preview" ;; - gemini:deep|gemini:audit|gemini:single) - printf '%s' "gemini-3.1-pro-preview,gemini-3.8-flash" ;; - copilot:triage|copilot:deep|copilot:audit|copilot:action|copilot:single) - printf '%s' "openai/o4-mini" ;; - esac + case "${1:-}" in claude|gemini|copilot) ;; *) return 0 ;; esac + case " $(_ai_models_keys "$1") " in *" ${2:-} "*) ;; *) return 0 ;; esac + _ai_engines_lookup "$1:$2" +} + +# ai_engines_file_providers — the providers the file enables, in its +# fallback_order (space-separated). AI_ENGINES can only narrow this list. +ai_engines_file_providers() { + _ai_engines_lookup providers +} + +# ai_engines_file_disabled — the providers the file disables (space-separated). +ai_engines_file_disabled() { + _ai_engines_lookup disabled } # _ai_models_keys — the keys AI_MODELS_ accepts (the same @@ -180,7 +237,7 @@ ai_models_configured() { ai_models_chain() { local c c="$(ai_models_configured "$1" "$2")" - [ -n "$c" ] || c="$(ai_models_default "$1" "$2")" + [ -n "$c" ] || c="$(ai_models_default "$1" "$2")" || return 1 printf '%s' "$c" } @@ -205,12 +262,14 @@ ai_models_replace_first() { # first model. ai_models_gemini_chain() { local key="${1:-}" group first c + ai_engines_config_load || return 1 case "$key" in triage|action) group="${GEMINI_FLASH_MODEL_CHAIN:-}"; first="${GEMINI_FLASH_MODEL:-}" ;; deep|audit|single) group="${GEMINI_PRO_MODEL_CHAIN:-}"; first="${GEMINI_PRO_MODEL:-}" ;; duck) c="$(ai_models_configured gemini duck)" - [ -n "$c" ] || { c="$(ai_models_gemini_chain triage)"; c="${c%%,*}"; } + [ -n "$c" ] || c="$(ai_models_default gemini duck)" || return 1 + [ -n "$c" ] || { c="$(ai_models_gemini_chain triage)" || return 1; c="${c%%,*}"; } printf '%s' "$c" return 0 ;; *) return 0 ;; @@ -219,7 +278,7 @@ ai_models_gemini_chain() { printf '%s' "${group//[[:space:]]/}" return 0 fi - c="$(ai_models_chain gemini "$key")" + c="$(ai_models_chain gemini "$key")" || return 1 first="${first//[[:space:]]/}" [ -z "$first" ] || c="$(ai_models_replace_first "$first" "$c")" printf '%s' "$c" @@ -233,12 +292,14 @@ ai_models_gemini_chain() { # a child shell) is not mistaken for the caller's. ai_models_copilot_model() { local key="${1:-}" m="" + ai_engines_config_load || return 1 if [ -n "${COPILOT_API_MODEL:-}" ] && [ "$COPILOT_API_MODEL" != "${COPILOT_API_MODEL_DEFAULTED:-}" ]; then m="$COPILOT_API_MODEL" else m="$(ai_models_configured copilot "$key")" + [ -n "$m" ] || [ "$key" != duck ] || m="$(ai_models_default copilot duck)" || return 1 [ -n "$m" ] || [ "$key" != duck ] || key=triage - [ -n "$m" ] || m="$(ai_models_chain copilot "$key")" + [ -n "$m" ] || m="$(ai_models_chain copilot "$key")" || return 1 fi m="${m%%,*}" printf '%s' "${m//[[:space:]]/}" @@ -252,6 +313,48 @@ ai_models_problems() { done } +# ai_engines_overrides — one line per break-glass override that is set and +# differs from config/ai-engines.json, so the daily health check shows it until +# it is folded back into the file: +# AI_MODELS_ = (file: ) +# AI_ENGINES= → (file: ) +# A Gemini/Copilot duck has no file chain; it is compared with the triage model +# it follows. AI_ENGINES (else the legacy DEV_LEAD_ENGINES) is compared after +# the same normalisation engine-chain.sh applies (case, separators). Prints +# nothing when no override differs; returns 1 when the file cannot be read. +ai_engines_overrides() { + local p k var c d spec chain problem file + # Read the file first, so an unreadable one is an error even with no override set. + ai_engines_config_load || return 1 + for p in claude gemini copilot; do + var="$(_ai_models_var "$p")" + for k in $(_ai_models_keys "$p"); do + c="$(ai_models_configured "$p" "$k")" + [ -n "$c" ] || continue + d="$(ai_models_default "$p" "$k")" || return 1 + if [ -z "$d" ] && [ "$k" = duck ]; then + d="$(ai_models_default "$p" triage)" || return 1 + d="${d%%,*}" + fi + [ "$c" = "$d" ] || printf '%s %s=%s (file: %s)\n' "$var" "$k" "$c" "${d:-none}" + done + done + spec="${AI_ENGINES:-${DEV_LEAD_ENGINES:-}}" + [ -n "${spec//[[:space:],]/}" ] || return 0 + file="$(ai_engines_file_providers)" || return 1 + # Reuse the runtime parser so the reported chain is the one that runs: an + # invalid override is rejected whole there, and its problem is surfaced here. + if ! declare -F ai_engine_chain >/dev/null 2>&1; then + # shellcheck source=scripts/lib/engine-chain.sh + source "$(dirname "${BASH_SOURCE[0]}")/engine-chain.sh" + fi + chain="$(ai_engine_chain)" || return 1 + problem="$(ai_engine_chain_problem)" || return 1 + if [ "$chain" != "$file" ] || [ -n "$problem" ]; then + printf "AI_ENGINES=%s → %s (file: %s)%s\n" "$spec" "$chain" "$file" "${problem:+ — $problem}" + fi +} + # ai_models_fable_deprecation — one deprecation line when a *configured* Claude # chain still names a claude-fable-* model, else nothing. Fable 5 is deprecated # (#1901, epic #1895 Phase 3): audit/single now default to claude-opus-5-5. A @@ -297,8 +400,9 @@ ai_models_fable_deprecation() { # haiku → triage (default primary claude-haiku-4-5-20251001) # Overrides are honoured with the same precedence engine.sh uses: the per-tier # CLAUDE__MODEL_CHAIN env first (ai_models_chain does not read it), then -# AI_MODELS_CLAUDE (via ai_models_chain), then the built-in default. An unknown -# family warns and returns non-zero. Bash 3.2-safe. +# AI_MODELS_CLAUDE (via ai_models_chain), then config/ai-engines.json. An +# unknown family warns and returns non-zero, as does an unreadable engine +# config. Bash 3.2-safe. ai_model_for_family() { local family="${1:-}" tier var chain model case "$family" in @@ -309,19 +413,20 @@ ai_model_for_family() { printf '::warning::ai_model_for_family: unknown family %s (expected opus|sonnet|haiku)\n' "$family" >&2 return 1 ;; esac + ai_engines_config_load || return 1 chain="${!var:-}" # Only an EMPTY per-tier var counts as absent (engine.sh's ${VAR:-…} rule). A # set-but-malformed value (",") must not let AI_MODELS_CLAUDE win; it fails the - # id check below and falls back to the built-in default. - [ -n "$chain" ] || chain="$(ai_models_chain claude "$tier")" + # id check below and falls back to the file's default. + [ -n "$chain" ] || chain="$(ai_models_chain claude "$tier")" || return 1 model="$(_ai_models_trim "${chain%%,*}")" # Validate the resolved id. ai_models_chain paths already validate via # _ai_models_scan, but the per-tier CLAUDE__MODEL_CHAIN override is read # raw above — a malformed value like "bad id" would otherwise reach the caller - # and fail the Claude invocation. Fall back to the built-in default on a bad id. + # and fail the Claude invocation. Fall back to the file's default on a bad id. if [[ ! "$model" =~ ^[A-Za-z0-9][A-Za-z0-9._:/@-]*$ ]]; then printf '::warning::ai_model_for_family: %s has an invalid model id %s — default used\n' "$var" "$model" >&2 - chain="$(ai_models_default claude "$tier")" + chain="$(ai_models_default claude "$tier")" || return 1 model="$(_ai_models_trim "${chain%%,*}")" fi printf '%s' "${model//[[:space:]]/}" diff --git a/scripts/lib/validate-consumer-manifest.sh b/scripts/lib/validate-consumer-manifest.sh index 79d23a651..4cccf422d 100755 --- a/scripts/lib/validate-consumer-manifest.sh +++ b/scripts/lib/validate-consumer-manifest.sh @@ -5,14 +5,14 @@ set -euo pipefail # # The manifest (consumer-manifest.json, same directory) is the single # machine-checkable source of truth mapping each provider reusable workflow to -# the org repos (consumers) that pin it, plus the in-provider scripts/lib + prompts -# each reusable sources (surface_sources). This validator is the gate that keeps +# the org repos (consumers) that pin it, plus the in-provider scripts/lib, prompts +# and config each reusable sources (surface_sources). This validator is the gate that keeps # the manifest honest. It exits non-zero on: # - invalid JSON or a violated top-level schema shape # - a surface_sources KEY that is not an existing reusable-workflow path in # this repo (a file under .github/workflows/ that declares `workflow_call`) # - a surface_sources VALUE that is not an existing scripts/lib/*.sh, -# scripts/lib/*.tsv, or prompts/* path in this repo +# scripts/lib/*.tsv, prompts/*, or config/*.json path in this repo # # Existence is always checked against --repo-root (the .github-private working # tree), independent of where the manifest file itself lives, so a fixture in a @@ -93,13 +93,14 @@ if [ "$schema_ok" != "true" ]; then fi # 3. surface_sources keys must be existing reusable-workflow paths in this repo; -# their values must be existing scripts/lib/*.sh, scripts/lib/*.tsv, or prompts/* -# paths in this repo. +# their values must be existing scripts/lib/*.sh, scripts/lib/*.tsv, prompts/*, +# or config/*.json paths in this repo (config/ai-engines.json, #1973). # Paths are resolved via realpath to prevent traversal sequences from escaping # their intended root directories. wf_root=$(realpath -m "$REPO_ROOT/.github/workflows") lib_root=$(realpath -m "$REPO_ROOT/scripts/lib") prompts_root=$(realpath -m "$REPO_ROOT/prompts") +config_root=$(realpath -m "$REPO_ROOT/config") while IFS= read -r key; do [ -n "$key" ] || continue @@ -124,13 +125,15 @@ done < <(jq -r '.surface_sources | keys[]' "$MANIFEST") while IFS= read -r value; do [ -n "$value" ] || continue case "$value" in - scripts/lib/*.sh|scripts/lib/*.tsv|prompts/*) : ;; + scripts/lib/*.sh|scripts/lib/*.tsv|prompts/*|config/*.json) : ;; *) - err "surface_sources value is not a scripts/lib/*.sh, scripts/lib/*.tsv, or prompts/* path: $value" + err "surface_sources value is not a scripts/lib/*.sh, scripts/lib/*.tsv, prompts/*, or config/*.json path: $value" continue ;; esac resolved=$(realpath -m "$REPO_ROOT/$value") - if [ "${resolved#"${lib_root}/"}" = "$resolved" ] && [ "${resolved#"${prompts_root}/"}" = "$resolved" ]; then + if [[ "$value" == *..* ]] \ + || { [[ "${resolved#"${lib_root}/"}" == "$resolved" ]] && [[ "${resolved#"${prompts_root}/"}" == "$resolved" ]] \ + && [[ "${resolved#"${config_root}/"}" == "$resolved" ]]; }; then err "surface_sources value contains path traversal: $value" continue fi diff --git a/scripts/pr_review_health.sh b/scripts/pr_review_health.sh index 7f1adf787..9119a6c2d 100644 --- a/scripts/pr_review_health.sh +++ b/scripts/pr_review_health.sh @@ -34,6 +34,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib/pr-review-sweep-metrics.sh" source "$(dirname "${BASH_SOURCE[0]}")/lib/run-attribution.sh" # Model family → current id resolver (#1979): name a family, never pin a version. +# Also lists break-glass overrides of config/ai-engines.json (ai_engines_overrides). # shellcheck source=scripts/lib/engine-models.sh source "$(dirname "${BASH_SOURCE[0]}")/lib/engine-models.sh" @@ -357,9 +358,39 @@ workflow_source=$(gh api "repos/${WORKFLOW_REPO}/contents/.github/workflows/${WO generate_attribution_report "$attr_tsv" "PR-review run attribution (by role, across collapse)" rm -f "$attr_tsv" "$legacy_attr_jsonl" fi + # Break-glass overrides (#1973): an AI_MODELS_* key or AI_ENGINES that differs + # from config/ai-engines.json. Listed every day until it is folded back into + # the file, so a temporary override cannot live on silently. Deterministic — + # same truncation guarantee. + printf '## Engine configuration overrides\n\n' + if ! engine_overrides="$(ai_engines_overrides)"; then + engine_overrides="" + printf 'Could not read `config/ai-engines.json` — see the run log.\n\n' + else + model_problems="$(ai_models_problems)" + if [ -n "$engine_overrides" ]; then + printf 'These Actions variables override `config/ai-engines.json`. Fold each into the file through a normal release, then delete the variable.\n\n' + printf '%s\n' "$engine_overrides" | sed 's/^/- `/; s/$/`/' + printf '\n' + fi + if [ -n "$model_problems" ]; then + printf 'Invalid engine model overrides (ignored):\n\n' + printf '%s\n' "$model_problems" | sed 's/^/- `/; s/$/`/' + printf '\n' + fi + if [ -z "$engine_overrides" ] && [ -z "$model_problems" ]; then + printf 'None: every engine and model setting comes from `config/ai-engines.json`.\n\n' + fi + fi } > "$REPORT_FILE" rm -f "$ingress_attr_jsonl" +if [ -n "${engine_overrides:-}" ]; then + while IFS= read -r line; do + echo "::warning::engine override differs from config/ai-engines.json: $line" + done <<< "$engine_overrides" +fi + echo "Duration percentiles — p50 $(fmt_dur "$dur_p50") / p95 $(fmt_dur "$dur_p95") across $dur_n run(s)" echo "Outcome mix (by event):" pr_review_outcomes_by_event "$runs_json" | sed 's/^/ /' diff --git a/scripts/review-batch.sh b/scripts/review-batch.sh index 889223ee5..687d65b90 100644 --- a/scripts/review-batch.sh +++ b/scripts/review-batch.sh @@ -115,10 +115,17 @@ fi # Primary engine: REVIEW_ENGINE when AI_ENGINES enables it, otherwise the first # engine in AI_ENGINES — a disabled engine is never used. _requested_engine="${REVIEW_ENGINE:-}" -REVIEW_ENGINE="$(ai_engine_primary "$_requested_engine")" +REVIEW_ENGINE="$(ai_engine_primary "$_requested_engine")" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 +} export REVIEW_ENGINE if [ -n "$_requested_engine" ] && [ "$_requested_engine" != "$REVIEW_ENGINE" ]; then - echo "::warning::REVIEW_ENGINE=$_requested_engine is not enabled in AI_ENGINES ($(ai_engine_chain)) — using $REVIEW_ENGINE" + _chain_msg="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + echo "::warning::REVIEW_ENGINE=$_requested_engine is not enabled in AI_ENGINES ($_chain_msg) — using $REVIEW_ENGINE" fi unset _requested_engine @@ -132,10 +139,18 @@ if ! ai_engine_available "$REVIEW_ENGINE"; then echo "::warning::Primary $(ai_engine_label "$REVIEW_ENGINE") engine unavailable per pre-flight probe — switching to $(ai_engine_label "$_start_engine") for this batch" export REVIEW_ENGINE="$_start_engine" elif [ "$REVIEW_ENGINE" != "claude" ]; then - echo "::error::Primary $(ai_engine_label "$REVIEW_ENGINE") engine unavailable and no other engine in AI_ENGINES ($(ai_engine_chain)) is available (e.g. Copilot fallback also unavailable: gh copilot not installed, classic PAT, or COPILOT_GITHUB_TOKEN not set) — aborting batch" + _chain_msg="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + echo "::error::Primary $(ai_engine_label "$REVIEW_ENGINE") engine unavailable and no other engine in AI_ENGINES ($_chain_msg) is available (e.g. Copilot fallback also unavailable: gh copilot not installed, classic PAT, or COPILOT_GITHUB_TOKEN not set) — aborting batch" exit 1 else - echo "::warning::Claude unavailable per pre-flight probe and no other engine in AI_ENGINES ($(ai_engine_chain)) is available — continuing on Claude" + _chain_msg="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + echo "::warning::Claude unavailable per pre-flight probe and no other engine in AI_ENGINES ($_chain_msg) is available — continuing on Claude" fi unset _start_engine fi @@ -281,8 +296,12 @@ total_candidates=$(grep -c . "$PRS_FILE" || true) # _chain_has_later_engine — 0 when AI_ENGINES lists any engine after # (available or not). _chain_has_later_engine() { - local e seen=0 - for e in $(ai_engine_chain); do + local e seen=0 _chain + _chain="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" >&2 + exit 1 + } + for e in $_chain; do if [ "$seen" -eq 1 ]; then return 0 fi @@ -350,7 +369,11 @@ while IFS= read -r pr_url; do echo "::notice::$(ai_engine_label "$REVIEW_ENGINE") unavailable or rate-limited earlier in this batch — reviewing on $(ai_engine_label "$_start_engine")" export REVIEW_ENGINE="$_start_engine" elif [ -n "$AI_ENGINES_RATE_LIMITED" ]; then - echo "::error::Every engine in AI_ENGINES ($(ai_engine_chain)) is rate-limited or unavailable in this batch — stopping; the next scheduled run retries" + _chain_msg="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" >&2 + exit 1 + } + echo "::error::Every engine in AI_ENGINES ($_chain_msg) is rate-limited or unavailable in this batch — stopping; the next scheduled run retries" processed=$((processed - 1)) session_aborted=1 abort_pr="$pr_url" @@ -408,7 +431,11 @@ while IFS= read -r pr_url; do # notice and keep the batch going. Only a genuinely rate-limited, exhausted # chain falls through to the session-abort branch below. if [ "$rc" -eq 2 ] && { _chain_has_later_engine "$REVIEW_ENGINE" || [ "$_fallback_unavailable" -eq 1 ]; }; then - echo "::warning::No later engine in AI_ENGINES ($(ai_engine_chain)) is available after $(ai_engine_label "$REVIEW_ENGINE") — skipping $pr_url and continuing batch" + _chain_msg="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" >&2 + exit 1 + } + echo "::warning::No later engine in AI_ENGINES ($_chain_msg) is available after $(ai_engine_label "$REVIEW_ENGINE") — skipping $pr_url and continuing batch" post_engine_unavailable_notice "$pr_url" "$REVIEW_ENGINE" failed=$((failed + 1)) echo "::endgroup::" diff --git a/scripts/validate-ai-engines.py b/scripts/validate-ai-engines.py new file mode 100755 index 000000000..68e2d8978 --- /dev/null +++ b/scripts/validate-ai-engines.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Validate config/ai-engines.json (#1973). + +Usage: validate-ai-engines.py [config.json] [schema.json] [pricing.tsv] + +Defaults to the repo's config/ai-engines.json, config/ai-engines.schema.json +and scripts/lib/model-pricing.tsv. Validates the file against the JSON Schema +(draft 2020-12, the same `jsonschema>=4` setup evals/validate-cases.py uses), +then checks what a schema cannot express: + + - every model a task names is in `models` and not `retired`; + - every model a task names belongs to the provider of the chain it is in; + - every non-retired model of an enabled provider has a price row (effective + today) in + model-pricing.tsv (a vendor-prefixed id such as openai/o4-mini is priced by + its bare name, as token records store it); + - `duck`, and every `copilot` chain, holds one model (neither has an + in-engine chain). + +Every problem is reported as a `::error::` line. Exit 0 when valid, 1 otherwise. +""" +import datetime +import json +import math +import os +import re +import sys +import tempfile +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +SINGLE_MODEL_TASKS = ("duck",) +SINGLE_MODEL_PROVIDERS = ("copilot",) +PROVIDERS = ("claude", "gemini", "copilot") +# ai_model_for_family (scripts/lib/engine-models.sh) returns the first Claude model +# of these tasks, so that model must be of the family the task stands for. +FAMILY_TASKS = {"deep": "opus", "action": "sonnet", "triage": "haiku"} + + +def error(msg: str) -> None: + print(f"::error::ai-engines config invalid: {msg}", file=sys.stderr) + + +def safe_path(arg: str, what: str) -> Path: + """Resolve a CLI-supplied path and confine it to the repo or the system temp dir + (the bats tests stage fixtures there), refusing traversal elsewhere.""" + resolved = Path(os.path.realpath(arg)) + roots = (os.path.realpath(REPO), os.path.realpath(tempfile.gettempdir())) + if not any(resolved == Path(r) or Path(r) in resolved.parents for r in roots): + error(f"{what} path {arg} is outside the repository and the temp dir") + sys.exit(1) + return resolved + + +def load_json(path: Path, what: str): + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + error(f"could not read/parse {what} {path}: {exc}") + sys.exit(1) + + +def price_rows(path: Path) -> list: + """(model_glob, effective_from, well_formed) for each model-pricing.tsv row in + effect today (comments and short rows skipped, as scripts/lib/model-pricing.sh + does). Malformed rows are kept, flagged, because price_for can still select one.""" + rows = [] + today = datetime.datetime.now(datetime.timezone.utc).date().isoformat() + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError as exc: + error(f"could not read price table {path}: {exc}") + sys.exit(1) + for line in lines: + if re.match(r"^\s*#", line): + continue + cols = line.split("\t") + # price_for ignores a row until its effective_from date, so a future-dated + # row does not price a model running today. + if len(cols) < 6 or cols[1] > today: + continue + # A malformed date or non-numeric rate would price the model as zero in + # cost_usd, so only a well-formed row counts as pricing it. + try: + datetime.date.fromisoformat(cols[1].strip()) + rates = [float(c) for c in cols[2:6]] + ok = all(math.isfinite(r) and r >= 0 for r in rates) + except ValueError: + ok = False + rows.append((cols[0], cols[1], ok)) + return rows + + +def is_priced(rows: list, model: str) -> bool: + """True when the row price_for would select for `model` (most-specific glob, ties + to the latest effective_from, then the first row) is well formed.""" + best = None + for glob, eff, ok in rows: + if not glob_matches(glob, model): + continue + key = (len(re.sub(r"[*?]", "", glob)), eff) + if best is None or key > best[0]: + best = (key, ok) + return best is not None and best[1] + + +def glob_matches(glob: str, model: str) -> bool: + """model-pricing.sh's glob rule: '*' and '?' are wildcards, the rest literal.""" + regex = "".join(".*" if c == "*" else "." if c == "?" else re.escape(c) for c in glob) + return re.fullmatch(regex, model) is not None + + +def check_schema(config, schema) -> list: + try: + import jsonschema + except ImportError: + error("jsonschema not installed (pip install 'jsonschema>=4')") + sys.exit(1) + validator = jsonschema.Draft202012Validator(schema) + problems = [] + for err in sorted(validator.iter_errors(config), key=lambda e: list(e.absolute_path)): + where = "/".join(str(p) for p in err.absolute_path) or "(root)" + problems.append(f"schema: {where}: {err.message}") + return problems + + +def check_references(config, rows) -> list: + problems = [] + models = config["models"] + providers = config["providers"] + if not any(entry["enabled"] for name, entry in providers.items() if name in PROVIDERS): + problems.append("providers: every provider is disabled; enable at least one") + for task, chains in config["tasks"].items(): + if not isinstance(chains, dict): + continue # $comment + for provider in PROVIDERS: + # The duck runs on Claude only; the other tasks need a chain per enabled provider. + # Claude chains are always required: idea-triage and persona-runner resolve + # Claude models independently of provider routing, so disabling Claude for + # pr-review/dev-lead must not drop them. + needed = provider == "claude" if task in SINGLE_MODEL_TASKS else True + if provider == "claude" and provider not in chains: + problems.append(f"tasks.{task}.claude: missing chain (required even when Claude is disabled)") + elif needed and providers[provider]["enabled"] and provider not in chains: + problems.append(f"tasks.{task}.{provider}: missing chain for enabled provider {provider}") + for provider, chain in chains.items(): + if provider not in PROVIDERS: + continue # prefer, $comment + for model in chain: + entry = models.get(model) + if entry is None: + problems.append(f"tasks.{task}.{provider}: '{model}' is not in models") + continue + if entry["status"] == "retired": + problems.append(f"tasks.{task}.{provider}: '{model}' is retired") + if entry["provider"] != provider: + problems.append( + f"tasks.{task}.{provider}: '{model}' belongs to provider " + f"{entry['provider']}, not {provider}") + if provider == "claude" and task in FAMILY_TASKS and chain: + family = FAMILY_TASKS[task] + if not re.match(rf"claude-{family}-", str(chain[0])): + problems.append( + f"tasks.{task}.claude: first model '{chain[0]}' is not a {family} model " + f"(ai_model_for_family {family} returns it)") + if len(chain) > 1 and (task in SINGLE_MODEL_TASKS or provider in SINGLE_MODEL_PROVIDERS): + why = "the duck runs one model" if task in SINGLE_MODEL_TASKS else "Copilot has no in-engine chain" + problems.append(f"tasks.{task}.{provider}: takes one model ({why}), got {len(chain)}") + for model, entry in models.items(): + # Claude chains are resolved outside provider routing (idea-triage.yml, + # persona-runner-reusable.yml), so Claude models are priced even when disabled. + if entry["status"] == "retired" or ( + entry["provider"] != "claude" and not providers[entry["provider"]]["enabled"]): + continue + # Only the Copilot path strips a vendor prefix before pricing a run. + bare = model.split("/", 1)[1] if entry["provider"] == "copilot" and "/" in model else model + if not is_priced(rows, bare): + problems.append( + f"models.{model}: provider {entry['provider']} is enabled but the model has " + f"no price row in scripts/lib/model-pricing.tsv") + return problems + + +def main(argv) -> int: + config_path = safe_path(argv[1], "config") if len(argv) > 1 else REPO / "config" / "ai-engines.json" + schema_path = safe_path(argv[2], "schema") if len(argv) > 2 else REPO / "config" / "ai-engines.schema.json" + pricing_path = safe_path(argv[3], "pricing") if len(argv) > 3 else REPO / "scripts" / "lib" / "model-pricing.tsv" + + config = load_json(config_path, "config") + schema = load_json(schema_path, "schema") + problems = check_schema(config, schema) + # The reference checks index into the shape the schema guarantees. + if not problems: + problems = check_references(config, price_rows(pricing_path)) + for p in problems: + error(p) + if problems: + print(f"ai-engines config INVALID: {len(problems)} problem(s) in {config_path}", file=sys.stderr) + return 1 + tasks = [t for t in config["tasks"] if t != "$comment"] + print(f"ai-engines config OK: {len(config['models'])} model(s), {len(tasks)} task(s) ({config_path})") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/validate-engines.sh b/scripts/validate-engines.sh index 3f5957f41..1b57a0ee1 100644 --- a/scripts/validate-engines.sh +++ b/scripts/validate-engines.sh @@ -60,12 +60,12 @@ _gemini_probe_model() { } # _validate_engine_enabled — 0 unless AI_ENGINES leaves out. +# Returns 1 if the config file cannot be read (when engine-models.sh is sourced). _validate_engine_enabled() { if declare -F ai_engine_enabled >/dev/null 2>&1; then - ai_engine_enabled "$1" - else - return 0 + ai_engine_enabled "$1" || return 1 fi + return 0 } # _gemini_probe_key — minimal REST call with one Gemini key. @@ -166,21 +166,38 @@ validate_engines() { fi # ── Engine chain (AI_ENGINES) ─────────────────────────────────────────────── - local _chain_problem="" _disabled="" + local _chain_problem="" _disabled="" _chain="" if declare -F ai_engine_chain_problem >/dev/null 2>&1; then - _chain_problem="$(ai_engine_chain_problem)" + _chain_problem="$(ai_engine_chain_problem)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } if [ -n "$_chain_problem" ] && [ -z "${AI_ENGINES_PROBLEM_REPORTED:-}" ]; then echo "::warning::$_chain_problem" export AI_ENGINES_PROBLEM_REPORTED=1 fi - echo "::notice::Engine chain (AI_ENGINES): $(ai_engine_chain)" + _chain="$(ai_engine_chain)" || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + echo "::notice::Engine chain (AI_ENGINES): $_chain" fi # ── Claude ────────────────────────────────────────────────────────────────── - if ! _validate_engine_enabled claude; then + if _validate_engine_enabled claude; then + if command -v claude >/dev/null 2>&1 && [ -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]; then + claude_ok=true + fi + else + if declare -F ai_engine_enabled >/dev/null 2>&1; then + # ai_engine_enabled returned 1; could be disabled or config read error + # Try getting the chain to detect if config is readable + ai_engine_chain >/dev/null 2>&1 || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + fi _disabled="${_disabled:+$_disabled, }claude" - elif command -v claude >/dev/null 2>&1 && [ -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]; then - claude_ok=true fi # ── Gemini ────────────────────────────────────────────────────────────────── @@ -196,9 +213,7 @@ validate_engines() { fi } - if ! _validate_engine_enabled gemini; then - _disabled="${_disabled:+$_disabled, }gemini" - else + if _validate_engine_enabled gemini; then if ! command -v gemini >/dev/null 2>&1; then append_gemini_reason "Gemini CLI not installed (fix: npm install -g @google/gemini-cli)" fi @@ -231,6 +246,16 @@ validate_engines() { if [ -n "$gemini_reasons" ]; then echo "::warning::Gemini fallback unavailable — ${gemini_reasons}. When Claude is rate-limited, runs will fall through to the next engine in AI_ENGINES." fi + else + if declare -F ai_engine_enabled >/dev/null 2>&1; then + # ai_engine_enabled returned 1; could be disabled or config read error + # Try getting the chain to detect if config is readable + ai_engine_chain >/dev/null 2>&1 || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } + fi + _disabled="${_disabled:+$_disabled, }gemini" fi # ── Copilot ───────────────────────────────────────────────────────────────── @@ -244,27 +269,37 @@ validate_engines() { # --version` succeeding proves nothing — report it unavailable up front, the # same rule dev-lead's fallback applies (#1495, #1960). local _copilot_tok="${COPILOT_GITHUB_TOKEN:-}" - if ! _validate_engine_enabled copilot; then - _disabled="${_disabled:+$_disabled, }copilot" - elif [ -z "$_copilot_tok" ]; then - echo "::warning::Copilot unavailable — COPILOT_GITHUB_TOKEN is not set. Provide a fine-grained PAT with the Copilot entitlement or remove copilot from AI_ENGINES." - elif [[ "$_copilot_tok" == ghp_* ]]; then - echo "::warning::Copilot unavailable — COPILOT_GITHUB_TOKEN is a classic PAT (ghp_), which Copilot rejects. Use a fine-grained PAT or remove copilot from AI_ENGINES." - else - # A policy denial (no Copilot subscription/policy for the token's org) or any - # other probe failure is reported once, never left as a silent false (#1960). - # The CLI output is matched, not echoed: only the matched phrase is logged. - local _copilot_out _copilot_rc=0 _denied - _copilot_out="$(env GH_TOKEN="$_copilot_tok" gh copilot --version 2>&1)" || _copilot_rc=$? - _denied="$(printf '%s\n' "$_copilot_out" | grep -oiE "$_VALIDATE_LICENSE_DENIED_RE" | head -n 1 \ - | tr '[:upper:]' '[:lower:]' || true)" - if [ -n "$_denied" ]; then - echo "::warning::Copilot unavailable — ${_denied} (the token's account or org has no usable Copilot entitlement). Fix the Copilot policy/subscription or remove copilot from AI_ENGINES." - elif [ "$_copilot_rc" -ne 0 ]; then - echo "::warning::Copilot unavailable — \`gh copilot --version\` failed (exit ${_copilot_rc}). Check the gh CLI and COPILOT_GITHUB_TOKEN, or remove copilot from AI_ENGINES." + if _validate_engine_enabled copilot; then + if [ -z "$_copilot_tok" ]; then + echo "::warning::Copilot unavailable — COPILOT_GITHUB_TOKEN is not set. Provide a fine-grained PAT with the Copilot entitlement or remove copilot from AI_ENGINES." + elif [[ "$_copilot_tok" == ghp_* ]]; then + echo "::warning::Copilot unavailable — COPILOT_GITHUB_TOKEN is a classic PAT (ghp_), which Copilot rejects. Use a fine-grained PAT or remove copilot from AI_ENGINES." else - copilot_ok=true + # A policy denial (no Copilot subscription/policy for the token's org) or any + # other probe failure is reported once, never left as a silent false (#1960). + # The CLI output is matched, not echoed: only the matched phrase is logged. + local _copilot_out _copilot_rc=0 _denied + _copilot_out="$(env GH_TOKEN="$_copilot_tok" gh copilot --version 2>&1)" || _copilot_rc=$? + _denied="$(printf '%s\n' "$_copilot_out" | grep -oiE "$_VALIDATE_LICENSE_DENIED_RE" | head -n 1 \ + | tr '[:upper:]' '[:lower:]' || true)" + if [ -n "$_denied" ]; then + echo "::warning::Copilot unavailable — ${_denied} (the token's account or org has no usable Copilot entitlement). Fix the Copilot policy/subscription or remove copilot from AI_ENGINES." + elif [ "$_copilot_rc" -ne 0 ]; then + echo "::warning::Copilot unavailable — \`gh copilot --version\` failed (exit ${_copilot_rc}). Check the gh CLI and COPILOT_GITHUB_TOKEN, or remove copilot from AI_ENGINES." + else + copilot_ok=true + fi + fi + else + if declare -F ai_engine_enabled >/dev/null 2>&1; then + # ai_engine_enabled returned 1; could be disabled or config read error + # Try getting the chain to detect if config is readable + ai_engine_chain >/dev/null 2>&1 || { + echo "::error::Failed to read engine configuration file — config/ai-engines.json may be missing or unreadable" + exit 1 + } fi + _disabled="${_disabled:+$_disabled, }copilot" fi if [ -n "$_disabled" ]; then diff --git a/tests/dev-lead/unit/test_ai_engines_config.bats b/tests/dev-lead/unit/test_ai_engines_config.bats new file mode 100644 index 000000000..95604fa4a --- /dev/null +++ b/tests/dev-lead/unit/test_ai_engines_config.bats @@ -0,0 +1,195 @@ +#!/usr/bin/env bats +# config/ai-engines.json (#1973): the versioned source of truth for providers, +# the model catalog and each task's chain. engine-models.sh reads its defaults +# from the file (jq); AI_MODELS_* and AI_ENGINES are break-glass overrides on +# top of it. test_engine_models.bats proves the defaults did not change; this +# file covers the file itself: a missing or malformed file, precedence, +# provider enablement and the override report. + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" +LIB="$SCRIPT_DIR/scripts/lib/engine-models.sh" +CHAIN_LIB="$SCRIPT_DIR/scripts/lib/engine-chain.sh" +CONFIG="$SCRIPT_DIR/config/ai-engines.json" + +setup() { + unset AI_MODELS_CLAUDE AI_MODELS_GEMINI AI_MODELS_COPILOT AI_MODELS_PROBLEM_REPORTED + unset CLAUDE_TRIAGE_MODEL_CHAIN CLAUDE_DEEP_MODEL_CHAIN CLAUDE_AUDIT_MODEL_CHAIN + unset CLAUDE_ACTION_MODEL_CHAIN CLAUDE_SINGLE_MODEL_CHAIN + unset GEMINI_FLASH_MODEL GEMINI_PRO_MODEL GEMINI_FLASH_MODEL_CHAIN GEMINI_PRO_MODEL_CHAIN + unset COPILOT_API_MODEL COPILOT_API_MODEL_DEFAULTED AI_ENGINES DEV_LEAD_ENGINES + unset AI_DUCK_ENGINE AI_DUCK_MODEL GEMINI_AVAILABLE AI_ENGINES_CONFIG REVIEW_ENGINE + # shellcheck source=../../../scripts/lib/engine-models.sh + source "$LIB" +} + +# Writes a copy of the repo config with applied and points +# AI_ENGINES_CONFIG at it. +_config_with() { + local f="$BATS_TEST_TMPDIR/ai-engines.json" + jq "$1" "$CONFIG" > "$f" + export AI_ENGINES_CONFIG="$f" +} + +# ── The file is the default ────────────────────────────────────────────────── + +@test "file: every default chain is the file's task chain" { + local task provider want + for task in triage deep audit action single duck; do + for provider in claude gemini copilot; do + want="$(jq -r --arg t "$task" --arg p "$provider" '.tasks[$t][$p] // [] | join(",")' "$CONFIG")" + [ "$(ai_models_default "$provider" "$task")" = "$want" ] + done + done +} + +@test "file: a changed file changes the default (no built-in table left)" { + _config_with '.tasks.deep.claude = ["claude-opus-4-8"]' + [ "$(ai_models_default claude deep)" = "claude-opus-4-8" ] + [ "$(ai_models_chain claude deep)" = "claude-opus-4-8" ] +} + +@test "file: resolved relative to the lib, whatever the caller's working directory" { + cd "$BATS_TEST_TMPDIR" + [ "$(ai_engines_config_path)" = "$CONFIG" ] + [ -n "$(ai_models_default claude deep)" ] +} + +# ── A missing or malformed file fails loudly ───────────────────────────────── + +@test "missing file: ::error:: and non-zero, no built-in fallback" { + export AI_ENGINES_CONFIG="$BATS_TEST_TMPDIR/nope.json" + run ai_models_default claude deep + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"*"nope.json"* ]] + [[ "$output" != *"claude-opus"* ]] + run ai_models_chain claude deep + [ "$status" -ne 0 ] + run ai_engines_config_load + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "malformed file: invalid JSON or a file without tasks fails loudly" { + printf '{"tasks": {' > "$BATS_TEST_TMPDIR/bad.json" + export AI_ENGINES_CONFIG="$BATS_TEST_TMPDIR/bad.json" + run ai_models_default claude deep + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] + printf '{"models": {}}' > "$BATS_TEST_TMPDIR/bad.json" + run ai_models_default claude deep + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "missing file: sourcing engine.sh fails the step" { + run bash -c "set -e; export REVIEW_ENGINE=claude AI_ENGINES_CONFIG='$BATS_TEST_TMPDIR/nope.json' + source '$SCRIPT_DIR/scripts/engine.sh' >/dev/null; echo sourced" + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] + [[ "$output" != *"sourced"* ]] +} + +@test "family: a missing file returns non-zero instead of an empty id" { + export AI_ENGINES_CONFIG="$BATS_TEST_TMPDIR/nope.json" + run ai_model_for_family opus + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "load: a loaded table is reused, and reloaded when the path changes" { + ai_engines_config_load + [ "$(ai_models_default claude deep)" = "$(jq -r '.tasks.deep.claude | join(",")' "$CONFIG")" ] + _config_with '.tasks.deep.claude = ["claude-opus-4-7"]' + [ "$(ai_models_default claude deep)" = "claude-opus-4-7" ] +} + +# ── Precedence: specific env var > AI_MODELS_* > file ──────────────────────── + +@test "precedence: AI_MODELS_* beats the file; a per-tier env var beats both" { + _config_with '.tasks.deep.claude = ["claude-opus-4-7"]' + [ "$(ai_models_chain claude deep)" = "claude-opus-4-7" ] + export AI_MODELS_CLAUDE="deep=claude-opus-4-8" + [ "$(ai_models_chain claude deep)" = "claude-opus-4-8" ] + export CLAUDE_DEEP_MODEL_CHAIN="claude-sonnet-5" + [ "$(ai_model_for_family opus)" = "claude-sonnet-5" ] + run bash -c "export REVIEW_ENGINE=claude; source '$SCRIPT_DIR/scripts/engine.sh' >/dev/null 2>&1; echo \"deep=\$ENGINE_DEEP_MODEL\"" + [[ "$output" == *"deep=claude-sonnet-5"* ]] +} + +@test "precedence: Gemini and Copilot specific vars still beat AI_MODELS_* and the file" { + export AI_MODELS_GEMINI="deep=gemini-d1" GEMINI_PRO_MODEL_CHAIN="gemini-c1" + [ "$(ai_models_gemini_chain deep)" = "gemini-c1" ] + export AI_MODELS_COPILOT="deep=openai/d1" COPILOT_API_MODEL="openai/pinned" + [ "$(ai_models_copilot_model deep)" = "openai/pinned" ] +} + +# ── Providers: the file enables, AI_ENGINES can only narrow ────────────────── + +@test "providers: the default chain is the file's fallback order of enabled providers" { + source "$CHAIN_LIB" + [ "$(ai_engine_chain)" = "$(jq -r '.providers.fallback_order | join(" ")' "$CONFIG")" ] + _config_with '.providers.fallback_order = ["gemini","claude","copilot"]' + [ "$(ai_engine_chain)" = "gemini claude copilot" ] +} + +@test "providers: AI_ENGINES can disable a provider the file enables" { + source "$CHAIN_LIB" + export AI_ENGINES="claude,gemini" + [ "$(ai_engine_chain)" = "claude gemini" ] + ! ai_engine_enabled copilot +} + +@test "providers: AI_ENGINES cannot enable a provider the file disables" { + source "$CHAIN_LIB" + _config_with '.providers.copilot.enabled = false' + [ "$(ai_engine_chain)" = "claude gemini" ] + export AI_ENGINES="copilot,claude" + [ "$(ai_engine_chain)" = "claude" ] + ! ai_engine_enabled copilot || false + [ "$(ai_engine_primary copilot)" = "claude" ] + [[ "$(ai_engine_chain_problem)" == *"copilot"*"disabled"* ]] +} + +# ── Override visibility ────────────────────────────────────────────────────── + +@test "overrides: nothing set, or set to the file's value, reports nothing" { + [ -z "$(ai_engines_overrides)" ] + export AI_MODELS_CLAUDE="deep=$(jq -r '.tasks.deep.claude | join(",")' "$CONFIG")" + export AI_ENGINES="$(jq -r '.providers.fallback_order | join(",")' "$CONFIG")" + [ -z "$(ai_engines_overrides)" ] +} + +@test "overrides: an AI_MODELS_* key or AI_ENGINES that differs from the file is listed" { + export AI_MODELS_GEMINI="deep=gemini-x; triage=$(jq -r '.tasks.triage.gemini | join(",")' "$CONFIG")" + export AI_ENGINES="claude,gemini" + run ai_engines_overrides + [ "$status" -eq 0 ] + [[ "$output" == *"AI_MODELS_GEMINI deep=gemini-x"*"file: "* ]] + [[ "$output" != *"triage"* ]] + [[ "$output" == *"AI_ENGINES"*"claude gemini"*"file: claude gemini copilot"* ]] +} + +@test "overrides: the health check prints the override section" { + grep -q 'engine_overrides="$(ai_engines_overrides)"' "$SCRIPT_DIR/scripts/pr_review_health.sh" + local wf="$SCRIPT_DIR/.github/workflows/daily-pr-review-health.yml" + grep -q 'AI_MODELS_GEMINI: ${{ vars.AI_MODELS_GEMINI }}' "$wf" + grep -q 'AI_MODELS_COPILOT: ${{ vars.AI_MODELS_COPILOT }}' "$wf" + grep -q 'AI_ENGINES: ${{ vars.AI_ENGINES || vars.DEV_LEAD_ENGINES' "$wf" +} + +# ── Shipping the file ──────────────────────────────────────────────────────── + +@test "ship: dev-lead's sparse checkouts include config/ next to scripts/" { + local wf="$SCRIPT_DIR/.github/workflows/dev-lead-reusable.yml" + local scripts_n config_n + scripts_n="$(grep -cE '^ {12}scripts$' "$wf")" + config_n="$(grep -cE '^ {12}config$' "$wf")" + [ "$scripts_n" -ge 1 ] + [ "$config_n" = "$scripts_n" ] +} + +@test "ship: the consumer manifest lists the file for pr-review and dev-lead" { + local m="$SCRIPT_DIR/scripts/lib/consumer-manifest.json" + jq -e '.surface_sources[".github/workflows/pr-review.yml"] | index("config/ai-engines.json")' "$m" + jq -e '.surface_sources[".github/workflows/dev-lead-reusable.yml"] | index("config/ai-engines.json")' "$m" +} diff --git a/tests/dev-lead/unit/test_engine_chain_config.bats b/tests/dev-lead/unit/test_engine_chain_config.bats index f20d8b807..bc72eae9b 100644 --- a/tests/dev-lead/unit/test_engine_chain_config.bats +++ b/tests/dev-lead/unit/test_engine_chain_config.bats @@ -10,6 +10,7 @@ LIB="$SCRIPT_DIR/scripts/lib/engine-chain.sh" setup() { unset AI_ENGINES DEV_LEAD_ENGINES AI_DUCK_ENGINE AI_DUCK_MODEL unset CLAUDE_AVAILABLE GEMINI_AVAILABLE COPILOT_AVAILABLE REVIEW_ENGINE + unset GOOGLE_API_KEY GOOGLE_API_KEY_2 GOOGLE_API_KEY_3 GOOGLE_API_KEY_4 GEMINI_API_KEY # shellcheck source=../../../scripts/lib/engine-chain.sh source "$LIB" } @@ -719,3 +720,24 @@ EOS grep -q '^gemini rl=\[claude\]$' "$BDIR/engine_calls.txt" rm -rf "$BDIR" } + +@test "preflight: tokenless claude later in the chain passes when CLAUDE_UNAVAILABLE=1" { + export AI_ENGINES="gemini,claude" CLAUDE_UNAVAILABLE=1 + run _preflight + [ "$status" -eq 0 ] +} + +@test "preflight: CLAUDE_UNAVAILABLE does not excuse a missing token without the flag" { + export AI_ENGINES="gemini,claude" + unset CLAUDE_UNAVAILABLE + run _preflight + [ "$status" -eq 1 ] +} + +@test "engine.sh: REVIEW_ENGINE_FORCE pins REVIEW_ENGINE past a disabled provider (model A/B)" { + export AI_ENGINES="gemini" + run bash -c "REVIEW_ENGINE=claude; source '$SCRIPT_DIR/scripts/engine.sh' >/dev/null 2>&1; echo \$REVIEW_ENGINE" + [ "${lines[-1]}" = "gemini" ] + run bash -c "REVIEW_ENGINE=claude REVIEW_ENGINE_FORCE=1; source '$SCRIPT_DIR/scripts/engine.sh' >/dev/null 2>&1; echo \$REVIEW_ENGINE" + [ "${lines[-1]}" = "claude" ] +} diff --git a/tests/dev-lead/unit/test_engine_fallback.bats b/tests/dev-lead/unit/test_engine_fallback.bats index a5d98d419..5b5d2319a 100644 --- a/tests/dev-lead/unit/test_engine_fallback.bats +++ b/tests/dev-lead/unit/test_engine_fallback.bats @@ -89,6 +89,19 @@ STUBEOF [ "$status" -eq 0 ] } +@test "fallback: tokenless claude (CLAUDE_UNAVAILABLE=1) is skipped, a later engine runs" { + _make_stub "claude" 1 + _make_stub "gemini" 0 + export COPILOT_GITHUB_TOKEN="ghp_stub" + export CLAUDE_UNAVAILABLE=1 + _source_engine "claude" + + run run_writer_with_fallback "$TEST_PROMPT" + + [ "$status" -eq 0 ] + [[ "$output" == *"Skipping claude fallback"* ]] +} + @test "fallback: primary exits 2 → tries next engine" { # claude exits 2 (rate-limited), gemini exits 0. Copilot is forced to skip via a # classic-PAT token so gemini remains the tested fallback. diff --git a/tests/dev-lead/unit/test_install_step_engine.bats b/tests/dev-lead/unit/test_install_step_engine.bats new file mode 100644 index 000000000..696370452 --- /dev/null +++ b/tests/dev-lead/unit/test_install_step_engine.bats @@ -0,0 +1,74 @@ +#!/usr/bin/env bats +# The engine-CLI install steps of pr-review.yml and dev-lead-reusable.yml must +# never receive an empty primary engine (#1973 review): with the live +# AI_ENGINES=claude,gemini and no REVIEW_ENGINE / DEV_LEAD_ENGINE variable the +# env value is empty, and the step has to resolve the primary from the config. + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" + +setup() { + unset AI_ENGINES_CONFIG DEV_LEAD_ENGINES REVIEW_ENGINE DEV_LEAD_ENGINE + export HOME="$BATS_TEST_TMPDIR/home" GITHUB_PATH="$BATS_TEST_TMPDIR/github_path" + mkdir -p "$HOME" "$BATS_TEST_TMPDIR/bin" + : > "$GITHUB_PATH" + export CALLS="$BATS_TEST_TMPDIR/calls" + : > "$CALLS" + for tool in npm gh claude gemini; do + printf '#!/usr/bin/env bash\necho "%s $*" >> "$CALLS"\nexit 0\n' "$tool" \ + > "$BATS_TEST_TMPDIR/bin/$tool" + chmod +x "$BATS_TEST_TMPDIR/bin/$tool" + done + # `gh copilot --version` fails, so the step has to install the extension. + printf '#!/usr/bin/env bash\necho "gh $*" >> "$CALLS"\n[ "${1:-}" = copilot ] && exit 1\nexit 0\n' \ + > "$BATS_TEST_TMPDIR/bin/gh" + export PATH="$BATS_TEST_TMPDIR/bin:$PATH" +} + +# _step — writes the step's run script to a file. +_step() { + yq -r ".jobs[].steps[] | select(.name == \"$2\") | .run" \ + "$SCRIPT_DIR/.github/workflows/$1" > "$BATS_TEST_TMPDIR/step.sh" + [ -s "$BATS_TEST_TMPDIR/step.sh" ] +} + +@test "pr-review install step handles an empty REVIEW_ENGINE under AI_ENGINES=claude,gemini" { + _step pr-review.yml "Install review engine CLIs" + cd "$SCRIPT_DIR" + AI_ENGINES="claude,gemini" REVIEW_ENGINE="" CLAUDE_CODE_VERSION=latest \ + COPILOT_GITHUB_TOKEN=x run bash "$BATS_TEST_TMPDIR/step.sh" + [ "$status" -eq 0 ] + [[ "$output" != *"Unknown REVIEW_ENGINE"* ]] + [[ "$output" == *"claude-code cache hit"* ]] + grep -q "npm install -g @google/gemini-cli" "$CALLS" +} + +@test "pr-review install step resolves the enabled primary when REVIEW_ENGINE names a disabled engine" { + _step pr-review.yml "Install review engine CLIs" + cd "$SCRIPT_DIR" + AI_ENGINES="claude,gemini" REVIEW_ENGINE="copilot" CLAUDE_CODE_VERSION=latest \ + COPILOT_GITHUB_TOKEN=x run bash "$BATS_TEST_TMPDIR/step.sh" + [ "$status" -eq 0 ] + [[ "$output" == *"claude-code cache hit"* ]] + ! grep -q "gh extension install github/gh-copilot" "$CALLS" +} + +@test "pr-review install step falls back to claude when the config is unreadable" { + _step pr-review.yml "Install review engine CLIs" + cd "$SCRIPT_DIR" + AI_ENGINES_CONFIG="$BATS_TEST_TMPDIR/missing.json" REVIEW_ENGINE="" \ + CLAUDE_CODE_VERSION=latest COPILOT_GITHUB_TOKEN=x run bash "$BATS_TEST_TMPDIR/step.sh" + [ "$status" -eq 0 ] + [[ "$output" == *"claude-code cache hit"* ]] +} + +@test "dev-lead install step installs fallback CLIs for an empty DEV_LEAD_ENGINE under AI_ENGINES=claude,gemini" { + _step dev-lead-reusable.yml "Install engine CLIs" + mkdir -p "$BATS_TEST_TMPDIR/work" + ln -s "$SCRIPT_DIR" "$BATS_TEST_TMPDIR/work/.dev-lead" + cd "$BATS_TEST_TMPDIR/work" + AI_ENGINES="claude,gemini" DEV_LEAD_ENGINE="" CLAUDE_CODE_VERSION=latest \ + run bash "$BATS_TEST_TMPDIR/step.sh" + [ "$status" -eq 0 ] + grep -q "npm install -g @google/gemini-cli" "$CALLS" + grep -q "gh extension install github/gh-copilot" "$CALLS" +} diff --git a/tests/test_batch_fallback.bats b/tests/test_batch_fallback.bats index 172ad084d..c0f2ecffd 100644 --- a/tests/test_batch_fallback.bats +++ b/tests/test_batch_fallback.bats @@ -176,6 +176,9 @@ EOF # answering 404 for any model listed in $BAD_MODELS. _copilot_smoke_setup() { cp "$BATS_TEST_DIRNAME/../scripts/lib/engine-models.sh" "scripts/lib/" + # The lib reads its defaults from config/ai-engines.json beside scripts/ (#1973). + mkdir -p config + cp "$BATS_TEST_DIRNAME/../config/ai-engines.json" config/ cat > "scripts/validate-engines.sh" <<'EOS' source "$(dirname "${BASH_SOURCE[0]}")/lib/engine-models.sh" validate_engines() { diff --git a/tests/test_consumer_manifest.bats b/tests/test_consumer_manifest.bats index 93420d494..ddaf93efc 100644 --- a/tests/test_consumer_manifest.bats +++ b/tests/test_consumer_manifest.bats @@ -111,6 +111,21 @@ run_validator() { [ "$status" -ne 0 ] } +@test "surface_sources value under config/ passes when it is an existing .json file (#1973)" { + jq '.surface_sources = {".github/workflows/pr-review.yml": ["config/ai-engines.json"]}' \ + "$MANIFEST" > "$FIXTURE" + run_validator "$FIXTURE" + [ "$status" -eq 0 ] + jq '.surface_sources = {".github/workflows/pr-review.yml": ["config/nope.json"]}' \ + "$MANIFEST" > "$FIXTURE" + run_validator "$FIXTURE" + [ "$status" -ne 0 ] + jq '.surface_sources = {".github/workflows/pr-review.yml": ["config/../scripts/engine.json"]}' \ + "$MANIFEST" > "$FIXTURE" + run_validator "$FIXTURE" + [ "$status" -ne 0 ] +} + @test "a well-formed minimal manifest with real surfaces passes" { cat > "$FIXTURE" <<'JSON' { diff --git a/tests/test_validate_ai_engines.bats b/tests/test_validate_ai_engines.bats new file mode 100644 index 000000000..31e82a9e5 --- /dev/null +++ b/tests/test_validate_ai_engines.bats @@ -0,0 +1,155 @@ +#!/usr/bin/env bats +# scripts/validate-ai-engines.py (#1973): the lint.yml gate on +# config/ai-engines.json — JSON Schema (draft 2020-12) plus the checks a schema +# cannot express: every task model is in the catalog, not retired and of the +# chain's provider; every model of an enabled provider has a price row; duck and +# Copilot chains hold one model. + +REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/.." && pwd)" +VALIDATOR="$REPO_ROOT/scripts/validate-ai-engines.py" +CONFIG="$REPO_ROOT/config/ai-engines.json" + +# Writes the repo config with applied and runs the validator on it. +_validate_with() { + local f="$BATS_TEST_TMPDIR/ai-engines.json" + jq "$1" "$CONFIG" > "$f" + run python3 "$VALIDATOR" "$f" +} + +@test "the repo's config/ai-engines.json passes" { + run python3 "$VALIDATOR" + [ "$status" -eq 0 ] + [[ "$output" == *"OK"* ]] +} + +@test "schema: a missing section fails" { + _validate_with 'del(.tasks)' + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "schema: an unknown status fails" { + _validate_with '.models["claude-opus-4-8"].status = "deprecated"' + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "schema: an unknown task fails; tasks..prefer is allowed" { + _validate_with '.tasks.review = {"claude": ["claude-opus-4-8"]}' + [ "$status" -ne 0 ] + _validate_with '.tasks.deep.prefer = ["gemini", "claude"]' + [ "$status" -eq 0 ] +} + +@test "malformed JSON fails" { + printf '{"models": ' > "$BATS_TEST_TMPDIR/bad.json" + run python3 "$VALIDATOR" "$BATS_TEST_TMPDIR/bad.json" + [ "$status" -ne 0 ] + [[ "$output" == *"::error::"* ]] +} + +@test "gate: a task naming a retired model fails" { + _validate_with '.tasks.deep.gemini = ["gemini-2.5-pro", "gemini-3.8-flash"]' + [ "$status" -ne 0 ] + [[ "$output" == *"gemini-2.5-pro"*"retired"* ]] +} + +@test "gate: a task naming a model missing from the catalog fails" { + _validate_with '.tasks.audit.claude = ["claude-opus-9-9"]' + [ "$status" -ne 0 ] + [[ "$output" == *"claude-opus-9-9"*"not in models"* ]] +} + +@test "gate: a model in another provider's chain fails" { + _validate_with '.tasks.triage.gemini = ["claude-sonnet-5-5"]' + [ "$status" -ne 0 ] + [[ "$output" == *"claude-sonnet-5-5"*"provider claude"* ]] +} + +@test "gate: an enabled provider's model without a price row fails" { + _validate_with '.models["mistral-large"] = {"provider": "copilot", "status": "active"}' + [ "$status" -ne 0 ] + [[ "$output" == *"mistral-large"*"model-pricing.tsv"* ]] + # A disabled provider's models need no price row. + _validate_with '.models["mistral-large"] = {"provider": "copilot", "status": "active"} | .providers.copilot.enabled = false' + [ "$status" -eq 0 ] +} + +@test "gate: a price row that is only future-dated does not price an enabled model" { + local pt="$BATS_TEST_TMPDIR/pricing.tsv" f="$BATS_TEST_TMPDIR/ai-engines.json" + jq '.models["mistral-large"] = {"provider": "copilot", "status": "active"}' "$CONFIG" > "$f" + grep -v '^#' "$REPO_ROOT/scripts/lib/model-pricing.tsv" > "$pt" + printf 'mistral-large*\t2999-01-01\t1\t1\t1\t1\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -ne 0 ] + [[ "$output" == *"mistral-large"*"model-pricing.tsv"* ]] + printf 'mistral-large*\t2000-01-01\t1\t1\t1\t1\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -eq 0 ] +} + +@test "gate: a price row with a malformed date or non-numeric rate does not price a model" { + local pt="$BATS_TEST_TMPDIR/pricing.tsv" f="$BATS_TEST_TMPDIR/ai-engines.json" + jq '.models["mistral-large"] = {"provider": "copilot", "status": "active"}' "$CONFIG" > "$f" + grep -v '^#' "$REPO_ROOT/scripts/lib/model-pricing.tsv" > "$pt" + printf 'mistral-large*\t2000-01-01\tnope\t\tbad\tx\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -ne 0 ] + [[ "$output" == *"mistral-large"*"model-pricing.tsv"* ]] + printf 'mistral-large*\tnot-a-date\t1\t1\t1\t1\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -ne 0 ] +} + +@test "gate: a malformed more-specific row beats a valid broad glob, as price_for selects it" { + local pt="$BATS_TEST_TMPDIR/pricing.tsv" f="$BATS_TEST_TMPDIR/ai-engines.json" + jq '.models["mistral-large"] = {"provider": "copilot", "status": "active"}' "$CONFIG" > "$f" + grep -v '^#' "$REPO_ROOT/scripts/lib/model-pricing.tsv" > "$pt" + printf 'mistral-*\t2000-01-01\t1\t1\t1\t1\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -eq 0 ] + printf 'mistral-large\t2000-01-01\tbad\t1\t1\t1\n' >> "$pt" + run python3 "$VALIDATOR" "$f" "$REPO_ROOT/config/ai-engines.schema.json" "$pt" + [ "$status" -ne 0 ] + [[ "$output" == *"mistral-large"*"model-pricing.tsv"* ]] +} + +@test "gate: a vendor-prefixed Copilot id is priced by its bare name" { + # No row matches "azure/o4-mini" itself; the bare "o4-mini" matches o4-mini*. + _validate_with '.models["azure/o4-mini"] = {"provider": "copilot", "status": "active"}' + [ "$status" -eq 0 ] +} + +@test "gate: duck with more than one model fails" { + _validate_with '.tasks.duck.claude = ["claude-sonnet-5-5", "claude-sonnet-5"]' + [ "$status" -ne 0 ] + [[ "$output" == *"duck"*"one model"* ]] +} + +@test "gate: a Copilot chain with more than one model fails" { + _validate_with '.models["openai/gpt-5"] = {"provider": "copilot", "status": "active"} | .tasks.deep.copilot = ["openai/o4-mini", "openai/gpt-5"]' + [ "$status" -ne 0 ] + [[ "$output" == *"copilot"*"one model"* ]] +} + +@test "gate: fallback_order must name each provider once" { + _validate_with '.providers.fallback_order = ["claude", "claude", "gemini"]' + [ "$status" -ne 0 ] +} + +@test "gate: every problem is reported, not only the first" { + _validate_with '.tasks.audit.claude = ["claude-opus-9-9"] | .tasks.duck.claude = ["claude-sonnet-5-5", "claude-sonnet-5"]' + [ "$status" -ne 0 ] + [[ "$output" == *"claude-opus-9-9"* ]] + [[ "$output" == *"duck"* ]] +} + +@test "lint.yml runs the gate" { + grep -q 'python3 scripts/validate-ai-engines.py' "$REPO_ROOT/.github/workflows/lint.yml" +} + +@test "family identity: a non-family primary in deep/action/triage fails" { + _validate_with '.tasks.deep.claude = ["claude-sonnet-5-5"]' + [ "$status" -ne 0 ] + [[ "$output" == *"not a opus model"* ]] +}