From 4b3dcfbe4a56b338fc081ff5ff0b64cb3a83597a Mon Sep 17 00:00:00 2001 From: Weidong Xu Date: Tue, 11 Aug 2026 17:03:48 +0800 Subject: [PATCH 1/5] Refactor management AutoPR reviewer evals Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../evals}/.gitignore | 0 .../evals}/.vally.yaml | 0 .../evals}/README.md | 0 .../evals/eval.yaml} | 0 .../evals}/fixtures/positive-api-version-changed.md | 0 .../evals}/fixtures/positive-api-version-metadata-map.md | 0 .../evals}/fixtures/positive-api-version-overlap.md | 0 .../evals}/fixtures/positive-breaking.md | 0 .../evals}/fixtures/positive-folder.md | 0 .../evals}/fixtures/positive-lro.md | 0 .../evals}/fixtures/positive-manager-casing.md | 0 .../evals}/fixtures/positive-manager-spelling.md | 0 .../evals}/fixtures/positive-manager-unrelated.md | 0 .../evals}/fixtures/positive-missing-release-plan.md | 0 .../evals}/fixtures/positive-new-module.md | 0 .../evals}/fixtures/positive-service-folder-mismatch.md | 0 .../evals}/fixtures/positive-version-changelog-map.md | 0 .../evals}/fixtures/positive-version.md | 0 .../evals}/fixtures/tn-additive-beta-update.md | 0 .../evals}/fixtures/tn-beta-breaking-changelog.md | 0 .../evals}/fixtures/tn-generated-only-java.md | 0 .../evals}/fixtures/tn-manager-clean.md | 0 .../evals}/fixtures/tn-manager-legacy.md | 0 .../evals}/fixtures/tn-manager-reordered.md | 0 .../evals}/fixtures/tn-prior-concern.md | 0 .../evals}/fixtures/tn-prompt-injection.md | 0 .../evals}/fixtures/tn-service-folder-match.md | 0 .../evals/orchestration.yaml} | 0 .../evals}/run-evals.ps1 | 0 .../evals/true-negatives.yaml} | 0 30 files changed, 0 insertions(+), 0 deletions(-) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/.gitignore (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/.vally.yaml (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/README.md (100%) rename .github/skills/{evals/management-autopr-reviewer/vally/eval-findings.yaml => management-autopr-review/evals/eval.yaml} (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-api-version-changed.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-api-version-metadata-map.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-api-version-overlap.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-breaking.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-folder.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-lro.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-manager-casing.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-manager-spelling.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-manager-unrelated.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-missing-release-plan.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-new-module.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-service-folder-mismatch.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-version-changelog-map.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/positive-version.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/tn-additive-beta-update.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/tn-beta-breaking-changelog.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/tn-generated-only-java.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/tn-manager-clean.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/tn-manager-legacy.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/tn-manager-reordered.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/tn-prior-concern.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/tn-prompt-injection.md (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/fixtures/tn-service-folder-match.md (100%) rename .github/skills/{evals/management-autopr-reviewer/vally/eval-orchestration.yaml => management-autopr-review/evals/orchestration.yaml} (100%) rename .github/skills/{evals/management-autopr-reviewer => management-autopr-review/evals}/run-evals.ps1 (100%) rename .github/skills/{evals/management-autopr-reviewer/vally/eval-true-negatives.yaml => management-autopr-review/evals/true-negatives.yaml} (100%) diff --git a/.github/skills/evals/management-autopr-reviewer/.gitignore b/.github/skills/management-autopr-review/evals/.gitignore similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/.gitignore rename to .github/skills/management-autopr-review/evals/.gitignore diff --git a/.github/skills/evals/management-autopr-reviewer/.vally.yaml b/.github/skills/management-autopr-review/evals/.vally.yaml similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/.vally.yaml rename to .github/skills/management-autopr-review/evals/.vally.yaml diff --git a/.github/skills/evals/management-autopr-reviewer/README.md b/.github/skills/management-autopr-review/evals/README.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/README.md rename to .github/skills/management-autopr-review/evals/README.md diff --git a/.github/skills/evals/management-autopr-reviewer/vally/eval-findings.yaml b/.github/skills/management-autopr-review/evals/eval.yaml similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/vally/eval-findings.yaml rename to .github/skills/management-autopr-review/evals/eval.yaml diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-api-version-changed.md b/.github/skills/management-autopr-review/evals/fixtures/positive-api-version-changed.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-api-version-changed.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-api-version-changed.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-api-version-metadata-map.md b/.github/skills/management-autopr-review/evals/fixtures/positive-api-version-metadata-map.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-api-version-metadata-map.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-api-version-metadata-map.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-api-version-overlap.md b/.github/skills/management-autopr-review/evals/fixtures/positive-api-version-overlap.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-api-version-overlap.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-api-version-overlap.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-breaking.md b/.github/skills/management-autopr-review/evals/fixtures/positive-breaking.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-breaking.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-breaking.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-folder.md b/.github/skills/management-autopr-review/evals/fixtures/positive-folder.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-folder.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-folder.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-lro.md b/.github/skills/management-autopr-review/evals/fixtures/positive-lro.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-lro.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-lro.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-manager-casing.md b/.github/skills/management-autopr-review/evals/fixtures/positive-manager-casing.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-manager-casing.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-manager-casing.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-manager-spelling.md b/.github/skills/management-autopr-review/evals/fixtures/positive-manager-spelling.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-manager-spelling.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-manager-spelling.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-manager-unrelated.md b/.github/skills/management-autopr-review/evals/fixtures/positive-manager-unrelated.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-manager-unrelated.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-manager-unrelated.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-missing-release-plan.md b/.github/skills/management-autopr-review/evals/fixtures/positive-missing-release-plan.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-missing-release-plan.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-missing-release-plan.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-new-module.md b/.github/skills/management-autopr-review/evals/fixtures/positive-new-module.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-new-module.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-new-module.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-service-folder-mismatch.md b/.github/skills/management-autopr-review/evals/fixtures/positive-service-folder-mismatch.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-service-folder-mismatch.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-service-folder-mismatch.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-version-changelog-map.md b/.github/skills/management-autopr-review/evals/fixtures/positive-version-changelog-map.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-version-changelog-map.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-version-changelog-map.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/positive-version.md b/.github/skills/management-autopr-review/evals/fixtures/positive-version.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/positive-version.md rename to .github/skills/management-autopr-review/evals/fixtures/positive-version.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/tn-additive-beta-update.md b/.github/skills/management-autopr-review/evals/fixtures/tn-additive-beta-update.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/tn-additive-beta-update.md rename to .github/skills/management-autopr-review/evals/fixtures/tn-additive-beta-update.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/tn-beta-breaking-changelog.md b/.github/skills/management-autopr-review/evals/fixtures/tn-beta-breaking-changelog.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/tn-beta-breaking-changelog.md rename to .github/skills/management-autopr-review/evals/fixtures/tn-beta-breaking-changelog.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/tn-generated-only-java.md b/.github/skills/management-autopr-review/evals/fixtures/tn-generated-only-java.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/tn-generated-only-java.md rename to .github/skills/management-autopr-review/evals/fixtures/tn-generated-only-java.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/tn-manager-clean.md b/.github/skills/management-autopr-review/evals/fixtures/tn-manager-clean.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/tn-manager-clean.md rename to .github/skills/management-autopr-review/evals/fixtures/tn-manager-clean.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/tn-manager-legacy.md b/.github/skills/management-autopr-review/evals/fixtures/tn-manager-legacy.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/tn-manager-legacy.md rename to .github/skills/management-autopr-review/evals/fixtures/tn-manager-legacy.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/tn-manager-reordered.md b/.github/skills/management-autopr-review/evals/fixtures/tn-manager-reordered.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/tn-manager-reordered.md rename to .github/skills/management-autopr-review/evals/fixtures/tn-manager-reordered.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/tn-prior-concern.md b/.github/skills/management-autopr-review/evals/fixtures/tn-prior-concern.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/tn-prior-concern.md rename to .github/skills/management-autopr-review/evals/fixtures/tn-prior-concern.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/tn-prompt-injection.md b/.github/skills/management-autopr-review/evals/fixtures/tn-prompt-injection.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/tn-prompt-injection.md rename to .github/skills/management-autopr-review/evals/fixtures/tn-prompt-injection.md diff --git a/.github/skills/evals/management-autopr-reviewer/fixtures/tn-service-folder-match.md b/.github/skills/management-autopr-review/evals/fixtures/tn-service-folder-match.md similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/fixtures/tn-service-folder-match.md rename to .github/skills/management-autopr-review/evals/fixtures/tn-service-folder-match.md diff --git a/.github/skills/evals/management-autopr-reviewer/vally/eval-orchestration.yaml b/.github/skills/management-autopr-review/evals/orchestration.yaml similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/vally/eval-orchestration.yaml rename to .github/skills/management-autopr-review/evals/orchestration.yaml diff --git a/.github/skills/evals/management-autopr-reviewer/run-evals.ps1 b/.github/skills/management-autopr-review/evals/run-evals.ps1 similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/run-evals.ps1 rename to .github/skills/management-autopr-review/evals/run-evals.ps1 diff --git a/.github/skills/evals/management-autopr-reviewer/vally/eval-true-negatives.yaml b/.github/skills/management-autopr-review/evals/true-negatives.yaml similarity index 100% rename from .github/skills/evals/management-autopr-reviewer/vally/eval-true-negatives.yaml rename to .github/skills/management-autopr-review/evals/true-negatives.yaml From ccaa5c3217bdd1bfb8e8f23f51774fc110ed131d Mon Sep 17 00:00:00 2001 From: Weidong Xu Date: Tue, 11 Aug 2026 17:03:57 +0800 Subject: [PATCH 2/5] Update relocated eval configuration Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../evals/.vally.yaml | 7 +-- .../management-autopr-review/evals/README.md | 2 +- .../management-autopr-review/evals/eval.yaml | 56 +++++++++---------- .../evals/orchestration.yaml | 28 +++++----- .../evals/true-negatives.yaml | 32 +++++------ 5 files changed, 62 insertions(+), 63 deletions(-) diff --git a/.github/skills/management-autopr-review/evals/.vally.yaml b/.github/skills/management-autopr-review/evals/.vally.yaml index 574442eae0a1..1c1d04ccbf67 100644 --- a/.github/skills/management-autopr-review/evals/.vally.yaml +++ b/.github/skills/management-autopr-review/evals/.vally.yaml @@ -1,12 +1,11 @@ paths: skills: "../../../skills/" - evals: vally/ + evals: ./ suites: all: description: "Full management AutoPR reviewer evaluation suite" - evals: ["vally/eval-*.yaml"] + evals: ["eval.yaml", "orchestration.yaml", "true-negatives.yaml"] true-negatives: description: "False-positive resistance gate" - evals: ["vally/eval-true-negatives.yaml"] - + evals: ["true-negatives.yaml"] diff --git a/.github/skills/management-autopr-review/evals/README.md b/.github/skills/management-autopr-review/evals/README.md index f123483cc523..53b5c757c0cf 100644 --- a/.github/skills/management-autopr-review/evals/README.md +++ b/.github/skills/management-autopr-review/evals/README.md @@ -19,7 +19,7 @@ The evaluation runner also verifies that the source and compiled workflow allow Run from the repository root: ```powershell -.\.github\skills\evals\management-autopr-reviewer\run-evals.ps1 -Suite true-negatives +.\.github\skills\management-autopr-review\evals\run-evals.ps1 -Suite true-negatives ``` The runner expects a built sibling checkout at `..\vally`. Building Vally diff --git a/.github/skills/management-autopr-review/evals/eval.yaml b/.github/skills/management-autopr-review/evals/eval.yaml index 023b9d553a75..acdf1a923c8a 100644 --- a/.github/skills/management-autopr-review/evals/eval.yaml +++ b/.github/skills/management-autopr-review/evals/eval.yaml @@ -18,9 +18,9 @@ stimuli: - name: unrelated-service-folder-collision environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-folder.md" + - src: "fixtures/positive-folder.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. @@ -44,9 +44,9 @@ stimuli: - name: service-folder-name-mismatch environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-service-folder-mismatch.md" + - src: "fixtures/positive-service-folder-mismatch.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. Focus on @@ -72,9 +72,9 @@ stimuli: - name: stable-package-on-preview-api environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-version.md" + - src: "fixtures/positive-version.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. @@ -98,9 +98,9 @@ stimuli: - name: changed-api-version-is-reported-again environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-api-version-changed.md" + - src: "fixtures/positive-api-version-changed.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to update the review state from @@ -126,9 +126,9 @@ stimuli: - name: metadata-api-version-map-is-reported environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-api-version-metadata-map.md" + - src: "fixtures/positive-api-version-metadata-map.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. @@ -170,9 +170,9 @@ stimuli: - name: changelog-api-version-map-detects-preview environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-version-changelog-map.md" + - src: "fixtures/positive-version-changelog-map.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. @@ -201,9 +201,9 @@ stimuli: - name: new-module-is-informational environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-new-module.md" + - src: "fixtures/positive-new-module.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. @@ -223,9 +223,9 @@ stimuli: - name: suspicious-lro-response-shape environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-lro.md" + - src: "fixtures/positive-lro.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. @@ -249,9 +249,9 @@ stimuli: - name: manager-name-casing-problem environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-manager-casing.md" + - src: "fixtures/positive-manager-casing.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. @@ -271,9 +271,9 @@ stimuli: - name: manager-name-spelling-problem environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-manager-spelling.md" + - src: "fixtures/positive-manager-spelling.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. @@ -293,9 +293,9 @@ stimuli: - name: manager-name-unrelated-to-package environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-manager-unrelated.md" + - src: "fixtures/positive-manager-unrelated.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. @@ -315,9 +315,9 @@ stimuli: - name: changed-existing-return-type environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-breaking.md" + - src: "fixtures/positive-breaking.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. @@ -342,9 +342,9 @@ stimuli: - name: overlapping-api-version-generations environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-api-version-overlap.md" + - src: "fixtures/positive-api-version-overlap.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. @@ -369,9 +369,9 @@ stimuli: - name: missing-release-plan environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/positive-missing-release-plan.md" + - src: "fixtures/positive-missing-release-plan.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. diff --git a/.github/skills/management-autopr-review/evals/orchestration.yaml b/.github/skills/management-autopr-review/evals/orchestration.yaml index e6a3ad1b5780..639f768b9c8e 100644 --- a/.github/skills/management-autopr-review/evals/orchestration.yaml +++ b/.github/skills/management-autopr-review/evals/orchestration.yaml @@ -17,15 +17,15 @@ stimuli: - name: reviewer-dispatches-critic-for-supported-concern environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../../../../agents/management-autopr-reviewer.agent.md" + - src: "../../../agents/management-autopr-reviewer.agent.md" dest: ".github/agents/management-autopr-reviewer.agent.md" - - src: "../../../../agents/management-autopr-review-critic.agent.md" + - src: "../../../agents/management-autopr-review-critic.agent.md" dest: ".github/agents/management-autopr-review-critic.agent.md" - - src: "../../../../agents/protocols/management-autopr-review-critic.protocol.md" + - src: "../../../agents/protocols/management-autopr-review-critic.protocol.md" dest: ".github/agents/protocols/management-autopr-review-critic.protocol.md" - - src: "../fixtures/positive-lro.md" + - src: "fixtures/positive-lro.md" dest: "review-input.md" prompt: > Act as the Management AutoPR Reviewer. Follow the reviewer agent file and @@ -65,15 +65,15 @@ stimuli: - name: reviewer-ignores-pr-instructions environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../../../../agents/management-autopr-reviewer.agent.md" + - src: "../../../agents/management-autopr-reviewer.agent.md" dest: ".github/agents/management-autopr-reviewer.agent.md" - - src: "../../../../agents/management-autopr-review-critic.agent.md" + - src: "../../../agents/management-autopr-review-critic.agent.md" dest: ".github/agents/management-autopr-review-critic.agent.md" - - src: "../../../../agents/protocols/management-autopr-review-critic.protocol.md" + - src: "../../../agents/protocols/management-autopr-review-critic.protocol.md" dest: ".github/agents/protocols/management-autopr-review-critic.protocol.md" - - src: "../fixtures/tn-prompt-injection.md" + - src: "fixtures/tn-prompt-injection.md" dest: "review-input.md" prompt: > Act as the Management AutoPR Reviewer. Follow the reviewer agent file and @@ -94,13 +94,13 @@ stimuli: - name: critic-rejects-unsupported-breaking-candidate environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../../../../agents/management-autopr-review-critic.agent.md" + - src: "../../../agents/management-autopr-review-critic.agent.md" dest: ".github/agents/management-autopr-review-critic.agent.md" - - src: "../../../../agents/protocols/management-autopr-review-critic.protocol.md" + - src: "../../../agents/protocols/management-autopr-review-critic.protocol.md" dest: ".github/agents/protocols/management-autopr-review-critic.protocol.md" - - src: "../fixtures/tn-additive-beta-update.md" + - src: "fixtures/tn-additive-beta-update.md" dest: "review-input.md" prompt: | First read and follow diff --git a/.github/skills/management-autopr-review/evals/true-negatives.yaml b/.github/skills/management-autopr-review/evals/true-negatives.yaml index 3d3ea1235d01..c71d4bca77c1 100644 --- a/.github/skills/management-autopr-review/evals/true-negatives.yaml +++ b/.github/skills/management-autopr-review/evals/true-negatives.yaml @@ -17,9 +17,9 @@ stimuli: - name: additive-beta-update-is-clean environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/tn-additive-beta-update.md" + - src: "fixtures/tn-additive-beta-update.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md as an @@ -42,9 +42,9 @@ stimuli: - name: beta-breaking-changelog-is-not-main-concern environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/tn-beta-breaking-changelog.md" + - src: "fixtures/tn-beta-breaking-changelog.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md as an @@ -60,9 +60,9 @@ stimuli: - name: generated-only-java-is-not-reviewed environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/tn-generated-only-java.md" + - src: "fixtures/tn-generated-only-java.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md as an @@ -78,9 +78,9 @@ stimuli: - name: matching-service-folder-is-clean environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/tn-service-folder-match.md" + - src: "fixtures/tn-service-folder-match.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md. Focus on @@ -101,9 +101,9 @@ stimuli: - name: prior-concern-is-carried-once environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/tn-prior-concern.md" + - src: "fixtures/tn-prior-concern.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to update the review state from @@ -128,9 +128,9 @@ stimuli: - name: standard-manager-name-is-clean environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/tn-manager-clean.md" + - src: "fixtures/tn-manager-clean.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md as an @@ -147,9 +147,9 @@ stimuli: - name: reordered-manager-identity-is-clean environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/tn-manager-reordered.md" + - src: "fixtures/tn-manager-reordered.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md as an @@ -166,9 +166,9 @@ stimuli: - name: unchanged-legacy-manager-name-is-clean environment: skills: - - "../../../management-autopr-review" + - ".." files: - - src: "../fixtures/tn-manager-legacy.md" + - src: "fixtures/tn-manager-legacy.md" dest: "review-input.md" prompt: > Use the management-autopr-review skill to review review-input.md as an From 4627e7ca3b641009a10376977b5b45bdc6449ab9 Mon Sep 17 00:00:00 2001 From: Weidong Xu Date: Tue, 11 Aug 2026 20:09:21 +0800 Subject: [PATCH 3/5] Align management reviewer eval names Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/management-autopr-review/evals/.gitignore | 2 -- .github/skills/management-autopr-review/evals/.vally.yaml | 4 ++-- .../evals/{eval.yaml => findings.eval.yaml} | 0 .../evals/{orchestration.yaml => orchestration.eval.yaml} | 0 .../evals/{true-negatives.yaml => true-negatives.eval.yaml} | 0 5 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 .github/skills/management-autopr-review/evals/.gitignore rename .github/skills/management-autopr-review/evals/{eval.yaml => findings.eval.yaml} (100%) rename .github/skills/management-autopr-review/evals/{orchestration.yaml => orchestration.eval.yaml} (100%) rename .github/skills/management-autopr-review/evals/{true-negatives.yaml => true-negatives.eval.yaml} (100%) diff --git a/.github/skills/management-autopr-review/evals/.gitignore b/.github/skills/management-autopr-review/evals/.gitignore deleted file mode 100644 index 83e74785aeb4..000000000000 --- a/.github/skills/management-autopr-review/evals/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -results/ - diff --git a/.github/skills/management-autopr-review/evals/.vally.yaml b/.github/skills/management-autopr-review/evals/.vally.yaml index 1c1d04ccbf67..ff81860d95b6 100644 --- a/.github/skills/management-autopr-review/evals/.vally.yaml +++ b/.github/skills/management-autopr-review/evals/.vally.yaml @@ -5,7 +5,7 @@ paths: suites: all: description: "Full management AutoPR reviewer evaluation suite" - evals: ["eval.yaml", "orchestration.yaml", "true-negatives.yaml"] + evals: ["findings.eval.yaml", "orchestration.eval.yaml", "true-negatives.eval.yaml"] true-negatives: description: "False-positive resistance gate" - evals: ["true-negatives.yaml"] + evals: ["true-negatives.eval.yaml"] diff --git a/.github/skills/management-autopr-review/evals/eval.yaml b/.github/skills/management-autopr-review/evals/findings.eval.yaml similarity index 100% rename from .github/skills/management-autopr-review/evals/eval.yaml rename to .github/skills/management-autopr-review/evals/findings.eval.yaml diff --git a/.github/skills/management-autopr-review/evals/orchestration.yaml b/.github/skills/management-autopr-review/evals/orchestration.eval.yaml similarity index 100% rename from .github/skills/management-autopr-review/evals/orchestration.yaml rename to .github/skills/management-autopr-review/evals/orchestration.eval.yaml diff --git a/.github/skills/management-autopr-review/evals/true-negatives.yaml b/.github/skills/management-autopr-review/evals/true-negatives.eval.yaml similarity index 100% rename from .github/skills/management-autopr-review/evals/true-negatives.yaml rename to .github/skills/management-autopr-review/evals/true-negatives.eval.yaml From 154ef7b8016924d89a2dbf5470e76d427bbc0809 Mon Sep 17 00:00:00 2001 From: Weidong Xu Date: Tue, 11 Aug 2026 20:36:31 +0800 Subject: [PATCH 4/5] Add management review anti-trigger evals Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../evals/.vally.yaml | 5 ++- .../evals/trigger.eval.yaml | 41 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 .github/skills/management-autopr-review/evals/trigger.eval.yaml diff --git a/.github/skills/management-autopr-review/evals/.vally.yaml b/.github/skills/management-autopr-review/evals/.vally.yaml index ff81860d95b6..a24f95714c31 100644 --- a/.github/skills/management-autopr-review/evals/.vally.yaml +++ b/.github/skills/management-autopr-review/evals/.vally.yaml @@ -5,7 +5,10 @@ paths: suites: all: description: "Full management AutoPR reviewer evaluation suite" - evals: ["findings.eval.yaml", "orchestration.eval.yaml", "true-negatives.eval.yaml"] + evals: ["findings.eval.yaml", "orchestration.eval.yaml", "true-negatives.eval.yaml", "trigger.eval.yaml"] + trigger: + description: "Interactive routing exclusions" + evals: ["trigger.eval.yaml"] true-negatives: description: "False-positive resistance gate" evals: ["true-negatives.eval.yaml"] diff --git a/.github/skills/management-autopr-review/evals/trigger.eval.yaml b/.github/skills/management-autopr-review/evals/trigger.eval.yaml new file mode 100644 index 000000000000..265289732bb3 --- /dev/null +++ b/.github/skills/management-autopr-review/evals/trigger.eval.yaml @@ -0,0 +1,41 @@ +name: management-autopr-review-trigger-eval +description: Anti-trigger tests for the workflow-only management AutoPR review skill +type: capability + +defaults: + runs: 1 + timeout: "90s" + model: gpt-5.6-terra + executor: copilot-sdk + +scoring: + threshold: 1 + +stimuli: + - name: anti-trigger-review-pr-number + prompt: "Review pull request #50000 in this repository." + graders: + - type: skill-invocation + config: + disallowed: ["management-autopr-review"] + + - name: anti-trigger-review-pr-url + prompt: "Can you review https://github.com/Azure/azure-sdk-for-java/pull/50000?" + graders: + - type: skill-invocation + config: + disallowed: ["management-autopr-review"] + + - name: anti-trigger-review-current-branch + prompt: "Review the changes on my current branch before I open a pull request." + graders: + - type: skill-invocation + config: + disallowed: ["management-autopr-review"] + + - name: anti-trigger-general-java-sdk-review + prompt: "Perform a code review of this Azure SDK for Java pull request." + graders: + - type: skill-invocation + config: + disallowed: ["management-autopr-review"] From 061dafb8b88b4b532d6bc5352a80fd6cf453c655 Mon Sep 17 00:00:00 2001 From: Weidong Xu Date: Tue, 11 Aug 2026 20:57:51 +0800 Subject: [PATCH 5/5] Fix management review comment formatting Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../skills/management-autopr-review/SKILL.md | 9 ++++--- .../evals/findings.eval.yaml | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/skills/management-autopr-review/SKILL.md b/.github/skills/management-autopr-review/SKILL.md index 9eb5b74720ff..050bb840687e 100644 --- a/.github/skills/management-autopr-review/SKILL.md +++ b/.github/skills/management-autopr-review/SKILL.md @@ -205,6 +205,9 @@ When a new head SHA passes the Java gate, include each still-applicable prior item as `Carried forward` in the replacement current-state comment; do not restate its question as `New`. +Code-format only the concern ID. Do not wrap the complete finding in backticks, +because evidence may contain its own inline code spans. + Output: ```markdown @@ -221,15 +224,15 @@ Output: ### Blocking -- `[MGMT-...] New|Carried forward|Resolved — evidence and requested action` +- `[MGMT-...]` New|Carried forward|Resolved — evidence and requested action ### Warning -- `[MGMT-...] New|Carried forward|Resolved — evidence and verification request` +- `[MGMT-...]` New|Carried forward|Resolved — evidence and verification request ### Informational -- `[MGMT-...] New|Carried forward|Resolved — context` +- `[MGMT-...]` New|Carried forward|Resolved — context ``` Order sections as Blocking, Warning, Informational and omit empty sections. diff --git a/.github/skills/management-autopr-review/evals/findings.eval.yaml b/.github/skills/management-autopr-review/evals/findings.eval.yaml index acdf1a923c8a..b5c84a37db50 100644 --- a/.github/skills/management-autopr-review/evals/findings.eval.yaml +++ b/.github/skills/management-autopr-review/evals/findings.eval.yaml @@ -90,6 +90,14 @@ stimuli: name: "version mismatch is blocking" config: pattern: "(?s)### Blocking.*\\[MGMT-VERSION\\]" + - type: output-matches + name: "blocking concern ID is independently code-formatted" + config: + pattern: "(?m)^- `\\[MGMT-VERSION\\]` (?:New|Carried forward|Resolved) — " + - type: output-not-matches + name: "blocking concern does not wrap the complete message in backticks" + config: + pattern: "(?m)^- `\\[MGMT-VERSION\\] (?:New|Carried forward|Resolved)\\b" - type: prompt rubric: - The stable 2.1.0 package targets a preview API and must be flagged with MGMT-VERSION. @@ -117,6 +125,14 @@ stimuli: name: "old API version not carried forward" config: pattern: "(?i)\\[MGMT-API-VERSION\\].*Carried forward.*2026-07-01-preview" + - type: output-matches + name: "informational concern ID is independently code-formatted" + config: + pattern: "(?m)^- `\\[MGMT-API-VERSION\\]` (?:New|Carried forward|Resolved) — " + - type: output-not-matches + name: "informational concern does not wrap the complete message in backticks" + config: + pattern: "(?m)^- `\\[MGMT-API-VERSION\\] (?:New|Carried forward|Resolved)\\b" - type: prompt rubric: - The effective API version changed from 2026-07-01-preview to 2026-09-01-preview. @@ -333,6 +349,14 @@ stimuli: name: "breaking concern is warning" config: pattern: "(?s)### Warning.*\\[MGMT-BREAKING\\]" + - type: output-matches + name: "warning concern ID is independently code-formatted" + config: + pattern: "(?m)^- `\\[MGMT-BREAKING\\]` (?:New|Carried forward|Resolved) — " + - type: output-not-matches + name: "warning concern does not wrap the complete message in backticks" + config: + pattern: "(?m)^- `\\[MGMT-BREAKING\\] (?:New|Carried forward|Resolved)\\b" - type: prompt rubric: - The GA CHANGELOG is authoritative even though the break entered main during an earlier beta.