feat(eval): capture the detail of every failing run, not just the winning one

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -24,6 +24,7 @@
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
 from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
 from gooddata_eval.core.config import ReasoningEffort
+from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
 from gooddata_eval.core.evaluators.visualization import requires_execution_of
 from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
@@ -356,6 +357,20 @@ def _apply_run_counts(item_report: ItemReport, source: Any) -> None:
         item_report.runs_ungraded = unscored
 
 
+def _apply_failed_runs(item_report: ItemReport, source: Any) -> None:
+    """Copy the per-run failure records off an outcome or a failure, if the kind built any.
+
+    Read the same way from both, like the run counts above: a partially passing item raises,
+    so an item's failing runs reach the report through the exception at least as often as
+    through the outcome. A kind that builds none keeps the empty default, which reads as
+    "not instrumented" rather than "nothing failed" -- ``runs_passed``/``runs`` already say
+    how many failed.
+    """
+    failed_runs = getattr(source, "failed_runs", None)
+    if failed_runs:
+        item_report.failed_runs = list(failed_runs)
+
+
 def _apply_timings(item_report: ItemReport, timings: Any) -> None:
     """Copy an outcome's phase breakdown onto the item report, if the kind recorded one.
 
@@ -455,6 +470,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
             item_report.best_detail = detail or {}
             _apply_timings(item_report, getattr(outcome, "timings", None))
             _apply_run_counts(item_report, outcome)
+            _apply_failed_runs(item_report, outcome)
         except AssertionError as exc:
             item_report.gate_passed = False if gated else None
             item_report.runs = k
@@ -464,11 +480,32 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
             item_report.best_detail = getattr(exc, "detail", None) or {}
             _apply_timings(item_report, getattr(exc, "timings", None))
             _apply_run_counts(item_report, exc)
+            _apply_failed_runs(item_report, exc)
             # Read off the counts, not off the gate: pass^K fails items where runs did pass,
             # and reporting those as pass_at_k False would contradict the Langfuse score of
             # the same name. Kinds that report no count read as 0, i.e. a clean failure.
             item_report.pass_at_k = item_report.runs_passed > 0
             print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
+        except JudgeResponseError as exc:
+            # Errored, like the branch below -- there is no verdict for any run, so this is
+            # not K failures -- but NOT diagnostically empty. This is the one failure mode
+            # where the per-run records are most worth having: the judge broke, so what the
+            # agent actually said is still there to read, and the conversation ids are how
+            # anyone gets to it. Reported with the runs it really drove rather than 0.
+            item_report.error = f"{type(exc).__name__}: {exc}"
+            item_report.runs = getattr(exc, "runs_effective", None) or k
+            item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
+            item_report.conversation_id = getattr(exc, "conversation_id", None)
+            item_report.response_id = getattr(exc, "response_id", None)
+            item_report.best_detail = getattr(exc, "detail", None) or {}
+            _apply_timings(item_report, getattr(exc, "timings", None))
+            _apply_run_counts(item_report, exc)
+            _apply_failed_runs(item_report, exc)
+            # Every run ungraded, by definition of this error -- said explicitly rather than
+            # left to _apply_run_counts, whose source is detail["unscored_runs"] and which a
+            # kind that attaches no diagnostics would leave at 0.
+            item_report.runs_ungraded = item_report.runs
+            print(f"[agentic] {item.id} UNGRADED: {exc}", flush=True)
         except Exception as exc:
             item_report.error = f"{type(exc).__name__}: {exc}"
             item_report.runs = 0

packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py

@@ -0,0 +1,62 @@
+# (C) 2026 GoodData Corporation. All rights reserved.
+"""Per-run failure records for the agentic evaluators.
+
+``core/runner.py`` records one of these for every failing run of a single-shot item. The
+agentic kinds cannot reuse it: each of them drives its own K-loop inside the evaluator and
+hands ``cli/agentic_runner`` a single aggregate per item, so by the time the runner sees
+the result the individual attempts are already collapsed into ``best``. The runs themselves
+are not lost -- every kind keeps its ``run_results`` list -- they simply never leave the
+evaluator. This turns that list into the same records the single-shot path writes.
+
+Keys mirror ``core.runner._failed_run_record`` so one consumer can read ``failed_runs`` from
+either path without branching on test kind. ``tool_call_count``/``tool_names`` are the
+addition: the agentic kinds capture tool calls per run, and an agent that produced a final
+answer having made no tool call at all answered from the model's own knowledge rather than
+from the workspace -- a distinction no other recorded field exposes.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from typing import Any
+
+
+def build_failed_runs(
+    run_results: Sequence[Any],
+    *,
+    passed: Callable[[Any], bool],
+    detail: Callable[[Any], dict],
+) -> list[dict]:
+    """One record per run that did not pass, in the order the runs happened.
+
+    ``passed`` and ``detail`` are supplied by the caller because neither is uniform across
+    kinds: guardrail reads ``run.passed`` while visualization reads
+    ``run.eval_result.strict_pass``, and each kind's diagnostic dict is its own. Everything
+    read here directly goes through ``getattr`` with a default, so a kind whose run result
+    lacks a field (no response id, no tool capture) records a null instead of raising.
+
+    A run the judge could not grade is recorded like any other non-passing run, with its
+    ``error`` set. That matches ``core/runner.py``, which also records ungraded runs; the
+    verdict-level accounting of ungraded runs stays with the caller's ``unscored_runs``.
+    """
+    records: list[dict] = []
+    for run_index, run in enumerate(run_results, start=1):
+        if passed(run):
+            continue
+        tool_calls = list(getattr(run, "tool_call_events", None) or [])
+        reasoning_steps = list(getattr(run, "reasoning_steps", None) or [])
+        records.append(
+            {
+                "run_index": run_index,
+                "passed": False,
+                "error": getattr(run, "judge_error", None),
+                "detail": detail(run),
+                "conversation_id": getattr(run, "conversation_id", None),
+                "response_id": getattr(run, "response_id", None),
+                "reasoning_step_count": len(reasoning_steps),
+                "reasoning_steps": reasoning_steps,
+                "tool_call_count": len(tool_calls),
+                "tool_names": [getattr(e, "function_name", None) for e in tool_calls],
+            }
+        )
+    return records

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -12,6 +12,7 @@
 from gooddata_sdk import GoodDataSdk
 
 from gooddata_eval.core.agentic._catalog import AnomalyDetectionGranularity, CatalogMetricAlert
+from gooddata_eval.core.agentic._failed_runs import build_failed_runs
 from gooddata_eval.core.agentic._gate import (
     DEFAULT_GATE,
     EvalGate,
@@ -846,6 +847,33 @@ class AlertSkillAssertionError(AgenticAssertionError):
     """Raised when an alert-skill evaluation fails."""
 
 
+def _run_detail(run: AlertRunResult) -> dict:
+    """The diagnostic fields for ONE run, shared by the best run and every failing one.
+
+    Extracted so a failing run is described by exactly the same keys as the winning run --
+    for this kind that means the full per-field check breakdown and the arguments the run sent.
+    """
+    ev = run.eval
+    return {
+        "alert_created": ev.alert_created,
+        "operator_correct": ev.operator_correct,
+        "threshold_correct": ev.threshold_correct,
+        "trigger_correct": ev.trigger_correct,
+        "filters_correct": ev.filters_correct,
+        "metric_correct": ev.metric_correct,
+        "recipients_correct": ev.recipients_correct,
+        "attributes_correct": ev.attributes_correct,
+        "granularity_correct": ev.granularity_correct,
+        "actual_alert_arguments": run.actual_alert_arguments,
+        # Why the loop stopped. alert_created=False alone cannot tell a refusal from a run
+        # that hit max_iterations while still on track -- see LoopExit. Per-run rather than
+        # best-only: on a failing run it is the first thing worth reading.
+        "exit_reason": run.exit_reason.value,
+        "turns_used": run.turns_used,
+        **timeline_detail(run.tool_call_events, run.reasoning_step_events),
+    }
+
+
 def evaluate_agentic_alert_skill(
     host: str,
     token: str,
@@ -953,24 +981,16 @@ def _write_scores(ctx: RunTraceContext) -> None:
 
     best = summary.best
     ev = best.eval
-    detail = {
-        "alert_created": ev.alert_created,
-        "operator_correct": ev.operator_correct,
-        "threshold_correct": ev.threshold_correct,
-        "trigger_correct": ev.trigger_correct,
-        "filters_correct": ev.filters_correct,
-        "metric_correct": ev.metric_correct,
-        "recipients_correct": ev.recipients_correct,
-        "attributes_correct": ev.attributes_correct,
-        "granularity_correct": ev.granularity_correct,
-        "actual_alert_arguments": best.actual_alert_arguments,
-        # Why the loop stopped. alert_created=False alone cannot tell a refusal from a run
-        # that hit max_iterations while still on track -- see LoopExit.
-        "exit_reason": best.exit_reason.value,
-        "turns_used": best.turns_used,
-        "max_iterations": max_iterations,
-        **timeline_detail(best.tool_call_events, best.reasoning_step_events),
-    }
+    # max_iterations is the same for every run, so it stays at the item level rather than
+    # being repeated into each failing run's detail.
+    detail = {**_run_detail(best), "max_iterations": max_iterations}
+    # Same predicate runs_passed is taken over, so an item's failed_runs and its counts
+    # cannot disagree about which runs failed.
+    failed_runs = build_failed_runs(
+        summary.run_results,
+        passed=lambda r: r.eval.strict_pass,
+        detail=_run_detail,
+    )
 
     if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
         gate_note = gate_failure_note(gate, runs_passed, runs_effective)
@@ -990,6 +1010,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
+        exc.failed_runs = failed_runs
         raise exc
     return AgenticEvalOutcome(
         runs_passed=runs_passed,
@@ -998,4 +1019,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         conversation_id=best.conversation_id,
         response_id=best.response_id,
         detail=detail,
+        # Also on the success path: pass@K clears the gate with one passing run, so a
+        # 1/3 item reports success while two of its runs failed for reasons worth reading.
+        failed_runs=failed_runs,
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py

@@ -31,6 +31,7 @@
 from dataclasses import dataclass, field
 from typing import Any
 
+from gooddata_eval.core.agentic._failed_runs import build_failed_runs
 from gooddata_eval.core.agentic._gate import (
     DEFAULT_GATE,
     EvalGate,
@@ -619,6 +620,13 @@ def _write_scores(ctx: RunTraceContext) -> None:
     best = summary.best
     ev = best.evaluation
     detail = _detail(best)
+    # Same predicate runs_passed is taken over, so an item's failed_runs and its counts
+    # cannot disagree about which runs failed.
+    failed_runs = build_failed_runs(
+        summary.run_results,
+        passed=lambda r: r.evaluation.strict_pass,
+        detail=_detail,
+    )
     runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass)
 
     if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
@@ -638,6 +646,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = len(summary.run_results)
+        exc.failed_runs = failed_runs
         raise exc
 
     return AgenticEvalOutcome(
@@ -647,4 +656,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         conversation_id=best.conversation_id,
         response_id=best.response_id,
         detail=detail,
+        failed_runs=failed_runs,
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py

@@ -9,6 +9,7 @@
 
 import jsonpatch
 
+from gooddata_eval.core.agentic._failed_runs import build_failed_runs
 from gooddata_eval.core.agentic._gate import (
     DEFAULT_GATE,
     EvalGate,
@@ -944,6 +945,22 @@ def _execute_single_dashboard_run(
     )
 
 
+def _run_detail(run: DashboardRunResult) -> dict[str, Any]:
+    """The diagnostic fields for ONE run, shared by the best run and every failing one.
+
+    Extracted so a failing run is described by exactly the same keys as the winning run --
+    for this kind that means the per-check breakdown and the failures it reported, which is
+    the whole diagnosis.
+    """
+    return {
+        **run.evaluation.strict_checks,
+        **run.evaluation.diagnostics,
+        "failures": run.evaluation.failures,
+        "notes": run.evaluation.notes,
+        "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events),
+    }
+
+
 def run_agentic_dashboard_skill(
     host: str,
     token: str,
@@ -1108,13 +1125,14 @@ def _write_scores(ctx: RunTraceContext) -> None:
     runs_effective = len(summary.run_results)
 
     best = summary.best
-    detail: dict[str, Any] = {
-        **best.evaluation.strict_checks,
-        **best.evaluation.diagnostics,
-        "failures": best.evaluation.failures,
-        "notes": best.evaluation.notes,
-        "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
-    }
+    detail: dict[str, Any] = _run_detail(best)
+    # Same predicate runs_passed is taken over, so an item's failed_runs and its counts
+    # cannot disagree about which runs failed.
+    failed_runs = build_failed_runs(
+        summary.run_results,
+        passed=lambda r: r.evaluation.strict_pass,
+        detail=_run_detail,
+    )
 
     if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
         gate_note = gate_failure_note(gate, runs_passed, runs_effective)
@@ -1148,6 +1166,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         exc.detail = detail
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
+        exc.failed_runs = failed_runs
         raise exc
     return AgenticEvalOutcome(
         runs_passed=runs_passed,
@@ -1157,4 +1176,5 @@ def _write_scores(ctx: RunTraceContext) -> None:
         response_id=best.response_id,
         detail=detail,
         timings=item_timings,
+        failed_runs=failed_runs,
     )