diff --git a/src/google/adk/evaluation/llm_as_judge.py b/src/google/adk/evaluation/llm_as_judge.py index c344490f7a..978a1ccb21 100644 --- a/src/google/adk/evaluation/llm_as_judge.py +++ b/src/google/adk/evaluation/llm_as_judge.py @@ -40,6 +40,7 @@ from .eval_metrics import RubricsBasedCriterion from .eval_metrics import RubricScore from .evaluator import _validate_invocation_lengths +from .evaluator import EvalStatus from .evaluator import EvaluationResult from .evaluator import Evaluator from .evaluator import PerInvocationResult @@ -191,6 +192,23 @@ async def evaluate_invocations( ) ) if not invocation_result_samples: + # The auto-rater produced no samples for this invocation (e.g. the + # judge model's stream ended without emitting a response). Record it + # as not evaluated instead of silently dropping it from the results, + # which would shrink the denominator downstream with no trace. + # Spelled out explicitly (matching hallucinations_v1's equivalent + # empty-row case) rather than relying on PerInvocationResult's + # defaults, so both sites stay grep-matchable if those defaults ever + # move. + per_invocation_results.append( + PerInvocationResult( + actual_invocation=actual, + expected_invocation=expected, + score=None, + eval_status=EvalStatus.NOT_EVALUATED, + rubric_scores=[], + ) + ) continue per_invocation_results.append( self.aggregate_per_invocation_samples(invocation_result_samples) diff --git a/tests/unittests/evaluation/test_llm_as_judge.py b/tests/unittests/evaluation/test_llm_as_judge.py index 61e0625958..f55a0d3209 100644 --- a/tests/unittests/evaluation/test_llm_as_judge.py +++ b/tests/unittests/evaluation/test_llm_as_judge.py @@ -289,3 +289,64 @@ async def test_evaluate_invocations_grades_criterion_only_metric( assert [r.eval_status for r in result.per_invocation_results] == [ EvalStatus.PASSED ] + + +@pytest.mark.asyncio +async def test_evaluate_invocations_records_not_evaluated_when_no_samples_produced( + mocker, +): + """An invocation whose auto-rater call produces zero samples (e.g. the judge + + model's stream ends without emitting a response) must still show up in + per_invocation_results as NOT_EVALUATED, not vanish from the results + entirely. Silently dropping it shrinks the denominator downstream with no + trace that anything went wrong. + """ + judge = PerInvocationReportingLlmAsJudge( + eval_metric=EvalMetric( + metric_name="test_metric", + criterion=LlmAsAJudgeCriterion( + threshold=0.5, + judge_model_options=JudgeModelOptions( + judge_model="gemini-2.5-flash", + judge_model_config=genai_types.GenerateContentConfig(), + num_samples=1, + ), + ), + ), + criterion_type=LlmAsAJudgeCriterion, + ) + + empty_judge_model = mocker.MagicMock() + + async def mock_generate_content_async_no_response(llm_request): + del llm_request + return + yield # pragma: no cover -- makes this an async generator. + + empty_judge_model.generate_content_async = ( + mock_generate_content_async_no_response + ) + judge._judge_model = empty_judge_model + + actual_invocations = [ + Invocation( + invocation_id="id1", + user_content=genai_types.Content( + parts=[genai_types.Part(text="user content 1")], + role="user", + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text="final response 1")], + role="model", + ), + ) + ] + + result = await judge.evaluate_invocations(actual_invocations) + + assert len(result.per_invocation_results) == 1 + assert ( + result.per_invocation_results[0].eval_status == EvalStatus.NOT_EVALUATED + ) + assert result.per_invocation_results[0].score is None