Refactor management AutoPR reviewer evals

.github/skills/management-autopr-review/SKILL.md

@@ -205,6 +205,9 @@ When a new head SHA passes the Java gate, include each still-applicable prior
 item as `Carried forward` in the replacement current-state comment; do not
 restate its question as `New`.
 
+Code-format only the concern ID. Do not wrap the complete finding in backticks,
+because evidence may contain its own inline code spans.
+
 Output:
 
 ```markdown
@@ -221,15 +224,15 @@ Output:
 
 ### Blocking
 
-- `[MGMT-...] New|Carried forward|Resolved — evidence and requested action`
+- `[MGMT-...]` New|Carried forward|Resolved — evidence and requested action
 
 ### Warning
 
-- `[MGMT-...] New|Carried forward|Resolved — evidence and verification request`
+- `[MGMT-...]` New|Carried forward|Resolved — evidence and verification request
 
 ### Informational
 
-- `[MGMT-...] New|Carried forward|Resolved — context`
+- `[MGMT-...]` New|Carried forward|Resolved — context
 ```
 
 Order sections as Blocking, Warning, Informational and omit empty sections.

.github/skills/management-autopr-review/evals/.vally.yaml

@@ -0,0 +1,14 @@
+paths:
+  skills: "../../../skills/"
+  evals: ./
+
+suites:
+  all:
+    description: "Full management AutoPR reviewer evaluation suite"
+    evals: ["findings.eval.yaml", "orchestration.eval.yaml", "true-negatives.eval.yaml", "trigger.eval.yaml"]
+  trigger:
+    description: "Interactive routing exclusions"
+    evals: ["trigger.eval.yaml"]
+  true-negatives:
+    description: "False-positive resistance gate"
+    evals: ["true-negatives.eval.yaml"]

.github/skills/evals/management-autopr-reviewer/README.md → .github/skills/management-autopr-review/evals/README.md

@@ -19,7 +19,7 @@ The evaluation runner also verifies that the source and compiled workflow allow
 Run from the repository root:
 
 ```powershell
-.\.github\skills\evals\management-autopr-reviewer\run-evals.ps1 -Suite true-negatives
+.\.github\skills\management-autopr-review\evals\run-evals.ps1 -Suite true-negatives
 ```
 
 The runner expects a built sibling checkout at `..\vally`. Building Vally

.github/skills/evals/management-autopr-reviewer/vally/eval-findings.yaml → .github/skills/management-autopr-review/evals/findings.eval.yaml

@@ -18,9 +18,9 @@ stimuli:
   - name: unrelated-service-folder-collision
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-folder.md"
+        - src: "fixtures/positive-folder.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.
@@ -44,9 +44,9 @@ stimuli:
   - name: service-folder-name-mismatch
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-service-folder-mismatch.md"
+        - src: "fixtures/positive-service-folder-mismatch.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md. Focus on
@@ -72,9 +72,9 @@ stimuli:
   - name: stable-package-on-preview-api
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-version.md"
+        - src: "fixtures/positive-version.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.
@@ -90,6 +90,14 @@ stimuli:
         name: "version mismatch is blocking"
         config:
           pattern: "(?s)### Blocking.*\\[MGMT-VERSION\\]"
+      - type: output-matches
+        name: "blocking concern ID is independently code-formatted"
+        config:
+          pattern: "(?m)^- `\\[MGMT-VERSION\\]` (?:New|Carried forward|Resolved) — "
+      - type: output-not-matches
+        name: "blocking concern does not wrap the complete message in backticks"
+        config:
+          pattern: "(?m)^- `\\[MGMT-VERSION\\] (?:New|Carried forward|Resolved)\\b"
       - type: prompt
     rubric:
       - The stable 2.1.0 package targets a preview API and must be flagged with MGMT-VERSION.
@@ -98,9 +106,9 @@ stimuli:
   - name: changed-api-version-is-reported-again
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-api-version-changed.md"
+        - src: "fixtures/positive-api-version-changed.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to update the review state from
@@ -117,6 +125,14 @@ stimuli:
         name: "old API version not carried forward"
         config:
           pattern: "(?i)\\[MGMT-API-VERSION\\].*Carried forward.*2026-07-01-preview"
+      - type: output-matches
+        name: "informational concern ID is independently code-formatted"
+        config:
+          pattern: "(?m)^- `\\[MGMT-API-VERSION\\]` (?:New|Carried forward|Resolved) — "
+      - type: output-not-matches
+        name: "informational concern does not wrap the complete message in backticks"
+        config:
+          pattern: "(?m)^- `\\[MGMT-API-VERSION\\] (?:New|Carried forward|Resolved)\\b"
       - type: prompt
     rubric:
       - The effective API version changed from 2026-07-01-preview to 2026-09-01-preview.
@@ -126,9 +142,9 @@ stimuli:
   - name: metadata-api-version-map-is-reported
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-api-version-metadata-map.md"
+        - src: "fixtures/positive-api-version-metadata-map.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.
@@ -170,9 +186,9 @@ stimuli:
   - name: changelog-api-version-map-detects-preview
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-version-changelog-map.md"
+        - src: "fixtures/positive-version-changelog-map.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.
@@ -201,9 +217,9 @@ stimuli:
   - name: new-module-is-informational
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-new-module.md"
+        - src: "fixtures/positive-new-module.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.
@@ -223,9 +239,9 @@ stimuli:
   - name: suspicious-lro-response-shape
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-lro.md"
+        - src: "fixtures/positive-lro.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.
@@ -249,9 +265,9 @@ stimuli:
   - name: manager-name-casing-problem
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-manager-casing.md"
+        - src: "fixtures/positive-manager-casing.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.
@@ -271,9 +287,9 @@ stimuli:
   - name: manager-name-spelling-problem
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-manager-spelling.md"
+        - src: "fixtures/positive-manager-spelling.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.
@@ -293,9 +309,9 @@ stimuli:
   - name: manager-name-unrelated-to-package
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-manager-unrelated.md"
+        - src: "fixtures/positive-manager-unrelated.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.
@@ -315,9 +331,9 @@ stimuli:
   - name: changed-existing-return-type
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-breaking.md"
+        - src: "fixtures/positive-breaking.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.
@@ -333,6 +349,14 @@ stimuli:
         name: "breaking concern is warning"
         config:
           pattern: "(?s)### Warning.*\\[MGMT-BREAKING\\]"
+      - type: output-matches
+        name: "warning concern ID is independently code-formatted"
+        config:
+          pattern: "(?m)^- `\\[MGMT-BREAKING\\]` (?:New|Carried forward|Resolved) — "
+      - type: output-not-matches
+        name: "warning concern does not wrap the complete message in backticks"
+        config:
+          pattern: "(?m)^- `\\[MGMT-BREAKING\\] (?:New|Carried forward|Resolved)\\b"
       - type: prompt
     rubric:
       - The GA CHANGELOG is authoritative even though the break entered main during an earlier beta.
@@ -342,9 +366,9 @@ stimuli:
   - name: overlapping-api-version-generations
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-api-version-overlap.md"
+        - src: "fixtures/positive-api-version-overlap.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.
@@ -369,9 +393,9 @@ stimuli:
   - name: missing-release-plan
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../fixtures/positive-missing-release-plan.md"
+        - src: "fixtures/positive-missing-release-plan.md"
           dest: "review-input.md"
     prompt: >
       Use the management-autopr-review skill to review review-input.md.

.github/skills/evals/management-autopr-reviewer/vally/eval-orchestration.yaml → .github/skills/management-autopr-review/evals/orchestration.eval.yaml

@@ -17,15 +17,15 @@ stimuli:
   - name: reviewer-dispatches-critic-for-supported-concern
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../../../../agents/management-autopr-reviewer.agent.md"
+        - src: "../../../agents/management-autopr-reviewer.agent.md"
           dest: ".github/agents/management-autopr-reviewer.agent.md"
-        - src: "../../../../agents/management-autopr-review-critic.agent.md"
+        - src: "../../../agents/management-autopr-review-critic.agent.md"
           dest: ".github/agents/management-autopr-review-critic.agent.md"
-        - src: "../../../../agents/protocols/management-autopr-review-critic.protocol.md"
+        - src: "../../../agents/protocols/management-autopr-review-critic.protocol.md"
           dest: ".github/agents/protocols/management-autopr-review-critic.protocol.md"
-        - src: "../fixtures/positive-lro.md"
+        - src: "fixtures/positive-lro.md"
           dest: "review-input.md"
     prompt: >
       Act as the Management AutoPR Reviewer. Follow the reviewer agent file and
@@ -65,15 +65,15 @@ stimuli:
   - name: reviewer-ignores-pr-instructions
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../../../../agents/management-autopr-reviewer.agent.md"
+        - src: "../../../agents/management-autopr-reviewer.agent.md"
           dest: ".github/agents/management-autopr-reviewer.agent.md"
-        - src: "../../../../agents/management-autopr-review-critic.agent.md"
+        - src: "../../../agents/management-autopr-review-critic.agent.md"
           dest: ".github/agents/management-autopr-review-critic.agent.md"
-        - src: "../../../../agents/protocols/management-autopr-review-critic.protocol.md"
+        - src: "../../../agents/protocols/management-autopr-review-critic.protocol.md"
           dest: ".github/agents/protocols/management-autopr-review-critic.protocol.md"
-        - src: "../fixtures/tn-prompt-injection.md"
+        - src: "fixtures/tn-prompt-injection.md"
           dest: "review-input.md"
     prompt: >
       Act as the Management AutoPR Reviewer. Follow the reviewer agent file and
@@ -94,13 +94,13 @@ stimuli:
   - name: critic-rejects-unsupported-breaking-candidate
     environment:
       skills:
-        - "../../../management-autopr-review"
+        - ".."
       files:
-        - src: "../../../../agents/management-autopr-review-critic.agent.md"
+        - src: "../../../agents/management-autopr-review-critic.agent.md"
           dest: ".github/agents/management-autopr-review-critic.agent.md"
-        - src: "../../../../agents/protocols/management-autopr-review-critic.protocol.md"
+        - src: "../../../agents/protocols/management-autopr-review-critic.protocol.md"
           dest: ".github/agents/protocols/management-autopr-review-critic.protocol.md"
-        - src: "../fixtures/tn-additive-beta-update.md"
+        - src: "fixtures/tn-additive-beta-update.md"
           dest: "review-input.md"
     prompt: |
       First read and follow

.github/skills/management-autopr-review/evals/trigger.eval.yaml

@@ -0,0 +1,41 @@
+name: management-autopr-review-trigger-eval
+description: Anti-trigger tests for the workflow-only management AutoPR review skill
+type: capability
+
+defaults:
+  runs: 1
+  timeout: "90s"
+  model: gpt-5.6-terra
+  executor: copilot-sdk
+
+scoring:
+  threshold: 1
+
+stimuli:
+  - name: anti-trigger-review-pr-number
+    prompt: "Review pull request #50000 in this repository."
+    graders:
+      - type: skill-invocation
+        config:
+          disallowed: ["management-autopr-review"]
+
+  - name: anti-trigger-review-pr-url
+    prompt: "Can you review https://github.com/Azure/azure-sdk-for-java/pull/50000?"
+    graders:
+      - type: skill-invocation
+        config:
+          disallowed: ["management-autopr-review"]
+
+  - name: anti-trigger-review-current-branch
+    prompt: "Review the changes on my current branch before I open a pull request."
+    graders:
+      - type: skill-invocation
+        config:
+          disallowed: ["management-autopr-review"]
+
+  - name: anti-trigger-general-java-sdk-review
+    prompt: "Perform a code review of this Azure SDK for Java pull request."
+    graders:
+      - type: skill-invocation
+        config:
+          disallowed: ["management-autopr-review"]