Keep agent self-reports out of benchmark judge inputs

packages/flow/benchmark/verifier_baselines/JUDGE_INPUTS.md

@@ -0,0 +1,32 @@
+# Benchmark judge inputs
+
+The current judge evaluates task intent, structured actions, screenshots and
+observations recorded by the harness. Agent final replies and per-turn narration
+are omitted before prompt construction, including action scoring, screenshot
+analysis and fallback calls, rubric rescoring and outcome verification.
+
+Each step can include `observations`, a list of entries with `source` and
+`content`. Supported sources are `application`, `system_of_record`, `http` and
+`tool`. The adapter must establish the source from the recorder or reader; it
+must never relabel agent text as an observation. Other sources and legacy
+`state_description`, `error` and `tool_output` fields have no established
+observation provenance and are omitted. Structured action arguments and observed
+record values retain their contents, including words such as “claim” or “saved.”
+
+The Medplum adapter's pinned recorder writes model response text to each turn's
+`text` field. That text is excluded. The effect-E2E adapter records the measured
+HTTP status as a transport observation. A successful HTTP response does not by
+itself establish a durable record change.
+
+The original [protocol](PROTOCOL.md) and historical results describe the earlier
+judge. This source change does not rescore runs or change the population selected
+by that protocol. The historical Medplum UV-class protocol supplied final replies
+and per-step narration to the judge. Committed aggregates do not pin the judge
+source revision for each call, so exact execution provenance is unknown; those
+results cannot establish claim-blind performance. New runs need a new
+preregistration and separate results. Old views must not be treated as if they
+contained these new observation fields.
+
+The offline regression tests capture actual model messages with a fake client.
+They cover the normal batched screenshot path and its single-pair fallback,
+checking both narrative exclusion and preservation of observation evidence.

packages/flow/benchmark/verifier_baselines/effect_e2e_arms.py

@@ -149,47 +149,31 @@ def steps(status: Optional[int]) -> list[dict[str, Any]]:
     out: list[dict[str, Any]] = [
         {
             "url": url,
-            "state_description": f"The new-encounter form for patient {TARGET_PATIENT} is open.",
             "action_name": "input_text",
             "action_args": {"field": "Type", "text_value": TARGET_TYPE},
         },
         {
             "url": url,
-            "state_description": "Type is filled in; the note is empty.",
             "action_name": "input_text",
             "action_args": {"field": "Note", "text_value": NOTE},
         },
         {
             "url": url,
-            "state_description": "Type and note are filled in.",
             "action_name": "click",
             "action_args": {"target": "Save button"},
         },
     ]
-    if status is None:
-        out[-1]["error"] = f"no reply from the server within {ACTUATOR_TIMEOUT_S} s"
-    elif not 200 <= status < 300:
-        out[-1]["error"] = f"the server replied HTTP {status}"
-    out.append(
+    # The response status is measured by the HTTP harness. It is transport
+    # evidence, not a claim that the intended record was durably saved.
+    out[-1]["observations"] = [
         {
-            "url": url,
-            "state_description": "The page after Save (see the last screenshot).",
-            "action_name": "terminate",
-            "action_args": {
-                "status": "success" if status and 200 <= status < 300 else "failure"
-            },
+            "source": "http",
+            "content": {"status_code": status, "timeout_s": ACTUATOR_TIMEOUT_S},
         }
-    )
+    ]
     return out
 
 
-def predicted(status: Optional[int]) -> str:
-    if status is not None and 200 <= status < 300:
-        return f"Saved the {TARGET_TYPE} encounter for patient {TARGET_PATIENT} with the requested note."
-    why = "no reply from the server" if status is None else f"HTTP {status}"
-    return f"The save did not complete ({why})."
-
-
 def chart_html(base: str) -> str:
     s = requests.Session()
     s.post(
@@ -284,7 +268,6 @@ def run_one(scenario: Any, rep: int, views: Path) -> dict[str, Any]:
                 "task": TASK,
                 "init_url": "",
                 "steps": steps(status),
-                "predicted_output": predicted(status),
             }
             after_page = after_save_page(status, banner)
             j1 = _save_view(views, view, [after_page])
@@ -294,7 +277,6 @@ def run_one(scenario: Any, rep: int, views: Path) -> dict[str, Any]:
                 + [
                     {
                         "url": f"http://clinic.example/chart/patient/{TARGET_PATIENT}",
-                        "state_description": "Opened the patient's chart to look at the saved encounter.",
                         "action_name": "visit_url",
                         "action_args": {"url": f"/chart/patient/{TARGET_PATIENT}"},
                     }

packages/flow/benchmark/verifier_baselines/medplum_views.py

@@ -3,8 +3,9 @@
 Reads the scored rows listed in ``inputs/medplum_rows.manifest`` from the
 leaderboard results at commit 04e76283a (``git show``, so no checkout of that
 branch is needed) and yields, per run: the task text the agent got, one step
-per agent turn, the start and end screenshots, and the agent's final reply.
-See PROTOCOL.md.
+per agent turn and the start and end screenshots. Agent replies are omitted.
+See JUDGE_INPUTS.md for the current input boundary and PROTOCOL.md for
+the historical population.
 """
 
 from __future__ import annotations
@@ -55,6 +56,8 @@ def _action_name(action: dict[str, Any]) -> str:
 
 
 def steps(transcript: list[dict[str, Any]], end_url: str) -> list[dict[str, Any]]:
+    # At COMMIT, claude_cu records model response text blocks as turn.text;
+    # openai_cu records output_text messages there. Neither is observed state.
     out = []
     for turn in transcript:
         actions = [a for a in turn.get("actions") or [] if a]
@@ -66,7 +69,6 @@ def steps(transcript: list[dict[str, Any]], end_url: str) -> list[dict[str, Any]
         out.append(
             {
                 "url": end_url or "",
-                "state_description": (turn.get("text") or "").strip(),
                 "action_name": name,
                 "action_args": {"actions": actions} if actions else {},
             }
@@ -90,7 +92,6 @@ def runs() -> Iterator[dict[str, Any]]:
                 "task": tasks[row["id"]],
                 "init_url": "",
                 "steps": steps(transcript, ""),
-                "predicted_output": claim.get("final_text") or claim.get("note") or "",
                 "screenshots": [
                     _show(f"{folder}/start.png"),
                     _show(f"{folder}/end.png"),

packages/flow/benchmark/verifier_baselines/run_judge.py

@@ -107,7 +107,6 @@ async def judge_one(
                 view["task"],
                 view.get("init_url", ""),
                 view["steps"],
-                view["predicted_output"],
                 shots,
                 client,
                 rubric=rubric,

packages/flow/benchmark/verifier_baselines/smoke_uv_practice.py

@@ -74,7 +74,6 @@ def transcript_to_steps(transcript: list, url: str) -> list:
         steps.append(
             {
                 "url": url,
-                "state_description": turn.get("text", ""),
                 "action_name": name,
                 "action_args": args,
             }
@@ -105,7 +104,7 @@ async def main(run_dir: Path, run_json: Path, out_path: Path) -> None:
     # (404 "deprecated" on 2026-11-03), so gpt-5 plays both roles here.
     try:
         res = await uv_judge.judge(
-            task, init_url, steps, claim.get("final_text", ""), shots, gpt5
+            task, init_url, steps, shots, gpt5
         )
     finally:
         await gpt5.close()

packages/flow/benchmark/verifier_baselines/test_uv_judge.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+import copy
 import io
 import json
 import re
@@ -87,7 +88,7 @@ async def create(self, messages, json_output=False):
         assert json_output is True
         text = self._text(messages)
         step = next(k for k, h in STEP_HEADS.items() if h in text)
-        self.log.append((step, messages))
+        self.log.append((step, copy.deepcopy(messages)))
         if step == "0a":
             out = RUBRIC
         elif step == "0b":
@@ -124,6 +125,13 @@ async def create(self, messages, json_output=False):
                     for i in range(2)
                 ]
             }
+        elif step == "4single":
+            out = {
+                "screenshot_evidence": "e",
+                "criterion_analysis": "a",
+                "discrepancies": "none",
+                "environment_issues_confirmed": False,
+            }
         elif step == "6":
             out = {
                 "items": [
@@ -179,16 +187,18 @@ async def create(self, messages, json_output=False):
 STEPS = [
     {
         "url": "http://localhost:9801/Patient?x=1",
-        "state_description": "list",
+        "observations": [{"source": "application", "content": "list"}],
         "action_name": "left_click",
         "action_args": {"coordinate": [1, 2]},
     },
     {
         "url": "http://localhost:9801/Patient/new",
-        "state_description": "form",
+        "observations": [
+            {"source": "application", "content": "form"},
+            {"source": "tool", "content": {"error": "boom"}},
+        ],
         "action_name": "type",
         "action_args": {"text": "Feld"},
-        "error": "boom",
     },
 ]
 
@@ -199,7 +209,6 @@ def _run(client, **kw):
             "Create patient Noah Feld and save it.",
             "http://localhost:9801/",
             STEPS,
-            "Done.",
             [_png(), _png((0, 255, 0))],
             client,
             **kw,
@@ -254,12 +263,13 @@ def test_precomputed_rubric_skips_generation():
     assert [s for s, _ in c.log][:2] == ["0c", "4"]
 
 
-def test_format_action_history_matches_uv():
+def test_format_action_history_retains_observations():
     h = uv_judge.format_action_history(STEPS)
     assert h.startswith(
-        "State 1: http://localhost:9801/Patient, state_description: list\nAction 1: left_click({"
+        "State 1: http://localhost:9801/Patient\nAction 1: left_click({"
     )
-    assert "Error! The above Action 2 encountered an Error: boom" in h
+    assert 'Observed application: "list"' in h
+    assert 'Observed tool: {"error": "boom"}' in h
 
 
 def test_retry_nudge_on_bad_json():