diff --git a/packages/flow/benchmark/verifier_baselines/JUDGE_INPUTS.md b/packages/flow/benchmark/verifier_baselines/JUDGE_INPUTS.md new file mode 100644 index 0000000000..0348fee804 --- /dev/null +++ b/packages/flow/benchmark/verifier_baselines/JUDGE_INPUTS.md @@ -0,0 +1,32 @@ +# Benchmark judge inputs + +The current judge evaluates task intent, structured actions, screenshots and +observations recorded by the harness. Agent final replies and per-turn narration +are omitted before prompt construction, including action scoring, screenshot +analysis and fallback calls, rubric rescoring and outcome verification. + +Each step can include `observations`, a list of entries with `source` and +`content`. Supported sources are `application`, `system_of_record`, `http` and +`tool`. The adapter must establish the source from the recorder or reader; it +must never relabel agent text as an observation. Other sources and legacy +`state_description`, `error` and `tool_output` fields have no established +observation provenance and are omitted. Structured action arguments and observed +record values retain their contents, including words such as “claim” or “saved.” + +The Medplum adapter's pinned recorder writes model response text to each turn's +`text` field. That text is excluded. The effect-E2E adapter records the measured +HTTP status as a transport observation. A successful HTTP response does not by +itself establish a durable record change. + +The original [protocol](PROTOCOL.md) and historical results describe the earlier +judge. This source change does not rescore runs or change the population selected +by that protocol. The historical Medplum UV-class protocol supplied final replies +and per-step narration to the judge. Committed aggregates do not pin the judge +source revision for each call, so exact execution provenance is unknown; those +results cannot establish claim-blind performance. New runs need a new +preregistration and separate results. Old views must not be treated as if they +contained these new observation fields. + +The offline regression tests capture actual model messages with a fake client. +They cover the normal batched screenshot path and its single-pair fallback, +checking both narrative exclusion and preservation of observation evidence. diff --git a/packages/flow/benchmark/verifier_baselines/effect_e2e_arms.py b/packages/flow/benchmark/verifier_baselines/effect_e2e_arms.py index 4f30aa3281..f37eb90a4e 100644 --- a/packages/flow/benchmark/verifier_baselines/effect_e2e_arms.py +++ b/packages/flow/benchmark/verifier_baselines/effect_e2e_arms.py @@ -149,47 +149,31 @@ def steps(status: Optional[int]) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [ { "url": url, - "state_description": f"The new-encounter form for patient {TARGET_PATIENT} is open.", "action_name": "input_text", "action_args": {"field": "Type", "text_value": TARGET_TYPE}, }, { "url": url, - "state_description": "Type is filled in; the note is empty.", "action_name": "input_text", "action_args": {"field": "Note", "text_value": NOTE}, }, { "url": url, - "state_description": "Type and note are filled in.", "action_name": "click", "action_args": {"target": "Save button"}, }, ] - if status is None: - out[-1]["error"] = f"no reply from the server within {ACTUATOR_TIMEOUT_S} s" - elif not 200 <= status < 300: - out[-1]["error"] = f"the server replied HTTP {status}" - out.append( + # The response status is measured by the HTTP harness. It is transport + # evidence, not a claim that the intended record was durably saved. + out[-1]["observations"] = [ { - "url": url, - "state_description": "The page after Save (see the last screenshot).", - "action_name": "terminate", - "action_args": { - "status": "success" if status and 200 <= status < 300 else "failure" - }, + "source": "http", + "content": {"status_code": status, "timeout_s": ACTUATOR_TIMEOUT_S}, } - ) + ] return out -def predicted(status: Optional[int]) -> str: - if status is not None and 200 <= status < 300: - return f"Saved the {TARGET_TYPE} encounter for patient {TARGET_PATIENT} with the requested note." - why = "no reply from the server" if status is None else f"HTTP {status}" - return f"The save did not complete ({why})." - - def chart_html(base: str) -> str: s = requests.Session() s.post( @@ -284,7 +268,6 @@ def run_one(scenario: Any, rep: int, views: Path) -> dict[str, Any]: "task": TASK, "init_url": "", "steps": steps(status), - "predicted_output": predicted(status), } after_page = after_save_page(status, banner) j1 = _save_view(views, view, [after_page]) @@ -294,7 +277,6 @@ def run_one(scenario: Any, rep: int, views: Path) -> dict[str, Any]: + [ { "url": f"http://clinic.example/chart/patient/{TARGET_PATIENT}", - "state_description": "Opened the patient's chart to look at the saved encounter.", "action_name": "visit_url", "action_args": {"url": f"/chart/patient/{TARGET_PATIENT}"}, } diff --git a/packages/flow/benchmark/verifier_baselines/medplum_views.py b/packages/flow/benchmark/verifier_baselines/medplum_views.py index e10be26b35..8e56839104 100644 --- a/packages/flow/benchmark/verifier_baselines/medplum_views.py +++ b/packages/flow/benchmark/verifier_baselines/medplum_views.py @@ -3,8 +3,9 @@ Reads the scored rows listed in ``inputs/medplum_rows.manifest`` from the leaderboard results at commit 04e76283a (``git show``, so no checkout of that branch is needed) and yields, per run: the task text the agent got, one step -per agent turn, the start and end screenshots, and the agent's final reply. -See PROTOCOL.md. +per agent turn and the start and end screenshots. Agent replies are omitted. +See JUDGE_INPUTS.md for the current input boundary and PROTOCOL.md for +the historical population. """ from __future__ import annotations @@ -55,6 +56,8 @@ def _action_name(action: dict[str, Any]) -> str: def steps(transcript: list[dict[str, Any]], end_url: str) -> list[dict[str, Any]]: + # At COMMIT, claude_cu records model response text blocks as turn.text; + # openai_cu records output_text messages there. Neither is observed state. out = [] for turn in transcript: actions = [a for a in turn.get("actions") or [] if a] @@ -66,7 +69,6 @@ def steps(transcript: list[dict[str, Any]], end_url: str) -> list[dict[str, Any] out.append( { "url": end_url or "", - "state_description": (turn.get("text") or "").strip(), "action_name": name, "action_args": {"actions": actions} if actions else {}, } @@ -90,7 +92,6 @@ def runs() -> Iterator[dict[str, Any]]: "task": tasks[row["id"]], "init_url": "", "steps": steps(transcript, ""), - "predicted_output": claim.get("final_text") or claim.get("note") or "", "screenshots": [ _show(f"{folder}/start.png"), _show(f"{folder}/end.png"), diff --git a/packages/flow/benchmark/verifier_baselines/run_judge.py b/packages/flow/benchmark/verifier_baselines/run_judge.py index c904f892df..9031fd602e 100644 --- a/packages/flow/benchmark/verifier_baselines/run_judge.py +++ b/packages/flow/benchmark/verifier_baselines/run_judge.py @@ -107,7 +107,6 @@ async def judge_one( view["task"], view.get("init_url", ""), view["steps"], - view["predicted_output"], shots, client, rubric=rubric, diff --git a/packages/flow/benchmark/verifier_baselines/smoke_uv_practice.py b/packages/flow/benchmark/verifier_baselines/smoke_uv_practice.py index c52c1abef9..1a0ca0997c 100644 --- a/packages/flow/benchmark/verifier_baselines/smoke_uv_practice.py +++ b/packages/flow/benchmark/verifier_baselines/smoke_uv_practice.py @@ -74,7 +74,6 @@ def transcript_to_steps(transcript: list, url: str) -> list: steps.append( { "url": url, - "state_description": turn.get("text", ""), "action_name": name, "action_args": args, } @@ -105,7 +104,7 @@ async def main(run_dir: Path, run_json: Path, out_path: Path) -> None: # (404 "deprecated" on 2026-11-03), so gpt-5 plays both roles here. try: res = await uv_judge.judge( - task, init_url, steps, claim.get("final_text", ""), shots, gpt5 + task, init_url, steps, shots, gpt5 ) finally: await gpt5.close() diff --git a/packages/flow/benchmark/verifier_baselines/test_uv_judge.py b/packages/flow/benchmark/verifier_baselines/test_uv_judge.py index 6563c797c4..87013438e5 100644 --- a/packages/flow/benchmark/verifier_baselines/test_uv_judge.py +++ b/packages/flow/benchmark/verifier_baselines/test_uv_judge.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import copy import io import json import re @@ -87,7 +88,7 @@ async def create(self, messages, json_output=False): assert json_output is True text = self._text(messages) step = next(k for k, h in STEP_HEADS.items() if h in text) - self.log.append((step, messages)) + self.log.append((step, copy.deepcopy(messages))) if step == "0a": out = RUBRIC elif step == "0b": @@ -124,6 +125,13 @@ async def create(self, messages, json_output=False): for i in range(2) ] } + elif step == "4single": + out = { + "screenshot_evidence": "e", + "criterion_analysis": "a", + "discrepancies": "none", + "environment_issues_confirmed": False, + } elif step == "6": out = { "items": [ @@ -179,16 +187,18 @@ async def create(self, messages, json_output=False): STEPS = [ { "url": "http://localhost:9801/Patient?x=1", - "state_description": "list", + "observations": [{"source": "application", "content": "list"}], "action_name": "left_click", "action_args": {"coordinate": [1, 2]}, }, { "url": "http://localhost:9801/Patient/new", - "state_description": "form", + "observations": [ + {"source": "application", "content": "form"}, + {"source": "tool", "content": {"error": "boom"}}, + ], "action_name": "type", "action_args": {"text": "Feld"}, - "error": "boom", }, ] @@ -199,7 +209,6 @@ def _run(client, **kw): "Create patient Noah Feld and save it.", "http://localhost:9801/", STEPS, - "Done.", [_png(), _png((0, 255, 0))], client, **kw, @@ -254,12 +263,13 @@ def test_precomputed_rubric_skips_generation(): assert [s for s, _ in c.log][:2] == ["0c", "4"] -def test_format_action_history_matches_uv(): +def test_format_action_history_retains_observations(): h = uv_judge.format_action_history(STEPS) assert h.startswith( - "State 1: http://localhost:9801/Patient, state_description: list\nAction 1: left_click({" + "State 1: http://localhost:9801/Patient\nAction 1: left_click({" ) - assert "Error! The above Action 2 encountered an Error: boom" in h + assert 'Observed application: "list"' in h + assert 'Observed tool: {"error": "boom"}' in h def test_retry_nudge_on_bad_json(): diff --git a/packages/flow/benchmark/verifier_baselines/uv_judge.py b/packages/flow/benchmark/verifier_baselines/uv_judge.py index 719685a524..ed3f215874 100644 --- a/packages/flow/benchmark/verifier_baselines/uv_judge.py +++ b/packages/flow/benchmark/verifier_baselines/uv_judge.py @@ -6,8 +6,9 @@ the top of ``uv_prompts.py``). This module re-implements the control flow of ``webeval/src/webeval/rubric_agent/mm_rubric_agent.py`` (``MMRubricAgent``) and the helpers it uses from ``formatting.py`` and ``utils.py``. It does not -import or execute the fara package; the prompts in ``uv_prompts.py`` are -verbatim copies. +import or execute the fara package; the prompts in ``uv_prompts.py`` retain +the upstream license and provenance. +This adaptation omits agent self-reports from every scoring stage. Pipeline, in UV's order (UV step numbers in brackets): @@ -225,12 +226,13 @@ def truncate_observation(text: str, max_chars: int = 1000) -> str: def format_action_history(steps: List[Dict[str, Any]], max_url_chars: int = 150) -> str: - """UV's ``format_action_history`` over plain step dicts. + """Render structured actions and observations supplied by the harness. - Step keys: ``url``, ``state_description``, ``action_name``, - ``action_args``, optional ``error`` (UV ``previous_error``), optional - ``tool_output`` (shown for ``run_command`` as in UV). Steps are numbered - from 1, as UV's ``get_step_summaries`` does. + ``observations`` entries have a ``source`` (application, system_of_record, + http or tool) and ``content``. Adapters must establish that provenance; + agent-authored or ambiguous text is never promoted to an observation. + Legacy ``state_description``, ``error`` and ``tool_output`` have no + provenance and are omitted. Action arguments remain verbatim task data. """ lines: List[str] = [] for idx, s in enumerate(steps, start=1): @@ -238,17 +240,15 @@ def format_action_history(steps: List[Dict[str, Any]], max_url_chars: int = 150) url_shortened = url.split("?")[0].split("#")[0] if len(url_shortened) > max_url_chars: url_shortened = url_shortened[:max_url_chars] + "..." - state_str = ( - f"{url_shortened}, state_description: {s.get('state_description', '')}" - ) action_str = f"{s.get('action_name', '')}({json.dumps(s.get('action_args') or {}, indent=4)})" - entry = f"State {idx}: {state_str}\nAction {idx}: {action_str}" - if s.get("error"): - entry += ( - f"\nError! The above Action {idx} encountered an Error: {s['error']}" - ) - if s.get("action_name") == "run_command" and s.get("tool_output"): - entry += f"\nCommand Output: {truncate_observation(s['tool_output'])}" + entry = f"State {idx}: {url_shortened}\nAction {idx}: {action_str}" + for observation in s.get("observations") or []: + if observation.get("source") not in { + "application", "system_of_record", "http", "tool" + }: + continue + content = json.dumps(observation.get("content"), ensure_ascii=False) + entry += f"\nObserved {observation['source']}: {content}" lines.append(entry) return "\n".join(lines) @@ -710,7 +710,6 @@ async def _action_only_scoring( rubric_dict, task, action_history, - predicted_output, init_url_context, blocks, max_iters, @@ -721,7 +720,6 @@ async def _action_only_scoring( task_definition=task, rubric=rubric_dict, action_history=action_history, - predicted_target=predicted_output, init_url_context=init_url_context, critical_point_context=blocks["critical_point_context"], user_simulator_policy=blocks["user_simulator_policy"], @@ -846,7 +844,6 @@ async def _analyze_screenshot_evidence_batched( task, init_url_context, action_history, - predicted_output, max_iters, ) -> Dict[int, List[Dict]]: num_criteria = len(rubric["items"]) @@ -874,7 +871,6 @@ async def analyze_single_pair(criterion_idx: int, screenshot_idx: int): task_definition=task, init_url_context=init_url_context, action_history=action_history, - agent_predicted_output=predicted_output, criterion_info=criterion_info, conditional_check=conditional_check, conditional_output=conditional_output, @@ -938,7 +934,6 @@ async def analyze_multi(screenshot_idx: int, criterion_indices: List[int]): task_definition=task, init_url_context=init_url_context, action_history=action_history, - agent_predicted_output=predicted_output, criteria_info_block=criteria_info_block, ) messages = _image_message(images_b64[screenshot_idx], prompt) @@ -1020,7 +1015,6 @@ async def _rescore_rubric_with_screenshots( task, init_url_context, action_history, - predicted_output, total_screenshots, max_iters, ) -> dict: @@ -1041,7 +1035,6 @@ async def _rescore_rubric_with_screenshots( task_definition=task, init_url_context=init_url_context, action_history=action_history, - agent_predicted_output=predicted_output, full_rubric_with_baselines=build_full_rubric_with_baselines(rubric), all_screenshot_evidence=build_all_screenshot_evidence_text( rubric, evidence_by_criterion, total_screenshots @@ -1275,7 +1268,6 @@ async def _outcome_verification( task, init_url_context, action_history, - predicted_output, total_screenshots, blocks, max_iters, @@ -1288,7 +1280,6 @@ async def _outcome_verification( rubric, evidence_by_criterion, total_screenshots ), action_history=action_history, - predicted_output=predicted_output or "N/A", critical_point_context=blocks["critical_point_context"], user_simulator_policy=blocks["user_simulator_policy_outcome"], cp_decision_rules=blocks["cp_decision_rules"], @@ -1341,7 +1332,6 @@ async def judge( task: str, init_url: str, steps: List[Dict[str, Any]], - predicted_output: str, screenshots: List[bytes], client: Any, *, @@ -1350,7 +1340,7 @@ async def judge( cp_profile: Optional[Dict[str, Any]] = None, max_iters: int = MAX_ITERS, ) -> Dict[str, Any]: - """Run the UV pipeline on one trajectory. + """Run the UV pipeline on observed evidence, without agent self-reports. ``client`` plays UV's ``gpt5_client`` role; ``action_only_client`` plays the ``o4mini_client`` role (defaults to ``client``). ``rubric`` is an @@ -1392,7 +1382,6 @@ async def judge( rubric_dict, task, action_history, - predicted_output, init_url_context, blocks, max_iters, @@ -1413,7 +1402,6 @@ async def judge( task, init_url_context, action_history, - predicted_output, max_iters, ) conditional_indices = [ @@ -1439,7 +1427,6 @@ async def judge( task, init_url_context, action_history, - predicted_output, len(screenshots), max_iters, ) @@ -1473,7 +1460,6 @@ async def judge( task, init_url_context, action_history, - predicted_output, len(screenshots), blocks, max_iters, diff --git a/packages/flow/benchmark/verifier_baselines/uv_prompts.py b/packages/flow/benchmark/verifier_baselines/uv_prompts.py index 971853dad9..fa31492821 100644 --- a/packages/flow/benchmark/verifier_baselines/uv_prompts.py +++ b/packages/flow/benchmark/verifier_baselines/uv_prompts.py @@ -1,4 +1,4 @@ -"""Prompt templates of Microsoft's Universal Verifier (UV), copied verbatim. +"""Prompt templates adapted from Microsoft's Universal Verifier (UV). Source: https://github.com/microsoft/fara webeval/src/webeval/rubric_agent/prompts.py @@ -6,10 +6,11 @@ webeval/src/webeval/rubric_agent/critical_point_types.yaml (one CP type, quoted below) Commit: a675d6d61c41c47ae87bacefeab22caad18e3e84 -Each template below is the exact source text of the module-level assignment -in prompts.py at that commit (extracted with ``ast.get_source_segment`` and -checked equal to ``ast.literal_eval`` of the original). Templates are filled -with ``string.Template.substitute`` exactly as UV does. +The original templates were extracted from module-level assignments at that +commit. This adaptation removes the agent final-answer input and instructions +that require it. Only task intent, structured actions, provenance-labelled +observations and screenshots are supplied. Historical results used the original +templates; they are not rescored by this change. The original license of microsoft/fara follows. @@ -696,7 +697,7 @@ ACTION_ONLY_RUBRIC_SCORER_PROMPT = """ Instructions: Help evaluate an Agent's Action History with respect to a give Rubric to grade how well an Agent completed a Task. -The Action History is a log of all actions taken by the Agent while attempting to complete the Task. The Agent Output is the final result produced by the Agent after completing all actions, which may or may not be consistent with the Action History. +The Action History contains structured actions and provenance-labelled observations. Agent final answers and intermediate narration are withheld. Evaluate the observed task effects; do not infer completion from an attempted action or an HTTP success status alone. Task: $task_definition$init_url_context @@ -708,10 +709,6 @@ $action_history <<< -Agent Output: >>> -$predicted_target -<<< - $critical_point_context $user_simulator_policy @@ -905,7 +902,7 @@ b. If uncontrollable blocker prevented this criterion: Award full credit and explain why in justification c. If controllable error prevented this criterion: Award appropriate partial/zero credit and explain why d. If criterion was attempted: Score based on the quality of the attempt and outcome -4. Trust the Action History over the Agent Output. Penalize hallucinations where the Output claims something not supported by the Action History. +4. Evaluate the observed evidence. Do not infer a completed task from an action attempt or an HTTP success status alone. 5. Be consistent: If you determine an uncontrollable blocker exists, apply "do not cascade penalties" to ALL downstream dependent criteria. **IMPORTANT**: Do NOT output "total_max_points" or "total_earned_points" fields. The totals will be computed programmatically. Only output the "items" list with each criterion scored. @@ -919,9 +916,6 @@ **Action History:** $action_history -**Agent's Predicted Output (Final Answer):** -$agent_predicted_output - **Criterion Being Evaluated:** $criterion_info @@ -929,19 +923,15 @@ Analyze the provided screenshot to extract visual evidence relevant to evaluating this specific criterion. **CRITICAL — Ground Your Analysis in the ACTUAL Screenshot Pixels:** -You MUST describe ONLY what is LITERALLY VISIBLE in this specific screenshot. Do NOT assume, infer, or fill in content based on the Action History or Predicted Output. +You MUST describe ONLY what is LITERALLY VISIBLE in this specific screenshot. Do NOT assume, infer, or fill in content based on the Action History. - READ the actual text rendered in the screenshot: dropdown/filter labels, table headers, column values, date ranges, page titles. - If a dropdown says "Regular Season", do NOT describe it as "Postseason". If dates only go up to March, do NOT claim April dates are visible. - If the screenshot does not show information relevant to this criterion, say so explicitly — do NOT fabricate evidence to match the agent's claims. -- The Action History and Predicted Output are provided for COMPARISON purposes only — to help you identify discrepancies between what the agent claimed and what the screenshot actually shows. They are NOT a description of the screenshot content. +- The Action History provides context for comparison. It is NOT a description of the screenshot content; independently observed records and transport receipts retain their stated source. - **`run_command` actions**: Command Output in the Action History is ground truth from the system — it will NOT appear in screenshots. An unchanged desktop after `run_command` is expected, not evidence of failure. Note this in your analysis so the rescoring step treats Command Output as authoritative. -**IMPORTANT — Criteria About the Agent's Output:** -Some criteria evaluate the quality, correctness, or completeness of the agent's final output (e.g., "Provide a step-by-step summary," "Report the price," "List the results"). For these criteria: -- The agent's output IS provided above in "Agent's Predicted Output." This is also typically the message associated with the agent's last action. -- Use the screenshots to VERIFY whether the output is correct, accurate, and supported by what is visible on screen — NOT to determine whether an output exists. -- If the criterion is about the agent's output, check whether the predicted output matches, contradicts, or is unsupported by the visual evidence in the screenshot. -- Only penalize if the output is factually wrong, hallucinated, or contradicted by the screenshots — NOT because the output is "not visible in the screenshot" (outputs are delivered as text, not rendered on-screen). +**Evidence Scope:** +Agent replies and intermediate narration are withheld. Evaluate observed task effects. If a criterion requires an unavailable final reply, identify that evidence gap rather than inventing an answer. An action or HTTP status alone does not prove a durable record change. **Analysis Requirements:** @@ -950,20 +940,13 @@ - What state is the page/interface in? (What do the dropdowns, filters, tabs actually say?) - What data or information is displayed? (What are the actual date ranges, values, labels shown?) -2. **criterion_analysis**: Based on the screenshot, the agent's intermediate thoughts/actions, and especially the agent's predicted output, analyze how the evidence indicates: - - Success: Does the screenshot confirm the criterion was fully satisfied? Does the agent's predicted output correctly reflect what is shown? +2. **criterion_analysis**: Based on the screenshot and the structured actions and observations, analyze how the evidence indicates: + - Success: Does the screenshot confirm the criterion was fully satisfied? - Partial success: Does it show partial progress or partial satisfaction? - Failure: Does it show the criterion was not satisfied, or that the agent's output is incorrect/hallucinated? - Provide specific reasoning based on visible evidence - - **For output-quality criteria**: If the agent delivered a predicted output that addresses the criterion, evaluate whether that output is accurate and consistent with what the screenshots show. Do NOT give zero credit simply because the output text is not rendered on screen. -3. **discrepancies**: Compare what the agent CLAIMED to do (from Action History and Predicted Output) versus what the screenshot ACTUALLY SHOWS. Identify any mismatches: - - Did the agent claim something that isn't visible in the screenshot? - - Did the agent miss information that IS visible in the screenshot? - - Does the agent's predicted output contain information that is contradicted by the screenshot? - - Does the agent's predicted output contain hallucinated information not supported by any screenshot? - - Example: Agent says "no videos over 20 mins found" but screenshot shows video thumbnails with "1:36:00" duration visible - - Example: Agent's predicted output lists steps from a guide, and the screenshot confirms those steps are on the page — this is CONSISTENT, not a discrepancy +3. **discrepancies**: Compare the task, structured actions and provenance-labelled observations with what the screenshot ACTUALLY SHOWS. Identify mismatches and missing evidence without inventing an agent report. 4. **environment_issues_confirmed**: Does the screenshot show environmental blockers that prevented task completion? Check for: - CAPTCHAs or bot detection pages @@ -992,27 +975,20 @@ **Action History:** $action_history -**Agent's Predicted Output (Final Answer):** -$agent_predicted_output - **You are given a SINGLE screenshot (the image attached to this message). You must analyze this ONE screenshot against MULTIPLE rubric criteria listed below. Produce one analysis entry per criterion, all based on the SAME screenshot image.** **Criteria to evaluate against this screenshot:** $criteria_info_block **CRITICAL — Ground Your Analysis in the ACTUAL Screenshot Pixels:** -You MUST describe ONLY what is LITERALLY VISIBLE in the attached screenshot image. Do NOT assume, infer, or fill in content based on the Action History or Predicted Output. +You MUST describe ONLY what is LITERALLY VISIBLE in the attached screenshot image. Do NOT assume, infer, or fill in content based on the Action History. - READ the actual text rendered in the screenshot: dropdown/filter labels, table headers, column values, date ranges, page titles. - If a dropdown says "Regular Season", do NOT describe it as "Postseason". If dates only go up to March, do NOT claim April dates are visible. - If the screenshot does not show information relevant to a criterion, say so explicitly — do NOT fabricate evidence to match the agent's claims. -- The Action History and Predicted Output are provided for COMPARISON purposes only — to help you identify discrepancies between what the agent claimed and what the screenshot actually shows. They are NOT a description of the screenshot content. +- The Action History provides context for comparison. It is NOT a description of the screenshot content; independently observed records and transport receipts retain their stated source. -**IMPORTANT — Criteria About the Agent's Output:** -Some criteria evaluate the quality, correctness, or completeness of the agent's final output (e.g., "Provide a step-by-step summary," "Report the price," "List the results"). For these criteria: -- The agent's output IS provided above in "Agent's Predicted Output." This is also typically the message associated with the agent's last action. -- Use the screenshots to VERIFY whether the output is correct, accurate, and supported by what is visible on screen — NOT to determine whether an output exists. -- If the criterion is about the agent's output, check whether the predicted output matches, contradicts, or is unsupported by the visual evidence in the screenshot. -- Only penalize if the output is factually wrong, hallucinated, or contradicted by the screenshots — NOT because the output is "not visible in the screenshot" (outputs are delivered as text, not rendered on-screen). +**Evidence Scope:** +Agent replies and intermediate narration are withheld. Evaluate observed task effects. If a criterion requires an unavailable final reply, identify that evidence gap rather than inventing an answer. An action or HTTP status alone does not prove a durable record change. **Analysis Requirements (produce these for EACH criterion listed above, all from the SAME attached screenshot):** @@ -1021,20 +997,13 @@ - What state is the page/interface in? (What do the dropdowns, filters, tabs actually say?) - What data or information is displayed? (What are the actual date ranges, values, labels shown?) -2. **criterion_analysis**: Based on the screenshot, the agent's intermediate thoughts/actions, and especially the agent's predicted output, analyze how the evidence indicates: - - Success: Does the screenshot confirm the criterion was fully satisfied? Does the agent's predicted output correctly reflect what is shown? +2. **criterion_analysis**: Based on the screenshot and the structured actions and observations, analyze how the evidence indicates: + - Success: Does the screenshot confirm the criterion was fully satisfied? - Partial success: Does it show partial progress or partial satisfaction? - Failure: Does it show the criterion was not satisfied, or that the agent's output is incorrect/hallucinated? - Provide specific reasoning based on visible evidence - - **For output-quality criteria**: If the agent delivered a predicted output that addresses the criterion, evaluate whether that output is accurate and consistent with what the screenshots show. Do NOT give zero credit simply because the output text is not rendered on screen. -3. **discrepancies**: Compare what the agent CLAIMED to do (from Action History and Predicted Output) versus what the screenshot ACTUALLY SHOWS. Identify any mismatches: - - Did the agent claim something that isn't visible in the screenshot? - - Did the agent miss information that IS visible in the screenshot? - - Does the agent's predicted output contain information that is contradicted by the screenshot? - - Does the agent's predicted output contain hallucinated information not supported by any screenshot? - - Example: Agent says "no videos over 20 mins found" but screenshot shows video thumbnails with "1:36:00" duration visible - - Example: Agent's predicted output lists steps from a guide, and the screenshot confirms those steps are on the page — this is CONSISTENT, not a discrepancy +3. **discrepancies**: Compare the task, structured actions and provenance-labelled observations with what the screenshot ACTUALLY SHOWS. Identify mismatches and missing evidence without inventing an agent report. 4. **environment_issues_confirmed**: Does the screenshot show environmental blockers that prevented task completion? Check for: - CAPTCHAs or bot detection pages @@ -1080,7 +1049,7 @@ {{ "criterion_idx": 1, "screenshot_evidence": "The same screenshot also shows a price label reading $$24.99...", - "criterion_analysis": "The price matches the agent's predicted output...", + "criterion_analysis": "The price shown matches the task constraint...", "discrepancies": "None.", "environment_issues_confirmed": false }}, @@ -1103,9 +1072,6 @@ **Action History:** $action_history -**Agent's Predicted Output (Final Answer):** -$agent_predicted_output - **Full Rubric with Baseline (Action-Only) Scores:** $full_rubric_with_baselines @@ -1113,13 +1079,13 @@ $all_screenshot_evidence **Your Task:** -You are rescoring the ENTIRE rubric in a single pass based on the screenshot evidence, the agent's action history, and the agent's predicted output. For each criterion, determine whether the baseline (action-only) score should be adjusted based on the visual evidence. +You are rescoring the ENTIRE rubric in a single pass based on the screenshot evidence and the structured action/observation history. For each criterion, determine whether the baseline (action-only) score should be adjusted based on the visual evidence. **Reality Notes:** - Some criteria may include a "Reality Notes" annotation just below their Description. These notes were generated by comparing the rubric's original assumptions against what the screenshots actually show. They provide interpretive context — clarifying, nuancing, or correcting the Description's **factual claims** in light of reality. When Reality Notes are present, they take precedence over the Description **only for conflicting factual claims** (e.g., what products exist, what a website actually shows, what options are available). However, Reality Notes do NOT override the Core Evaluation Principles below (cascading dependencies, uncontrollable blockers, best-effort evaluation, etc.). - Reality Notes limitations: Reality Notes were generated from screenshot evidence ONLY — they did not have access to the Action History. For URL and navigation information, the **Action History is authoritative**. -**Agent Output Format:** The agent's predicted output above may be a structured dictionary (e.g., `{"target_name": "...", "target_id": "...", "url": "..."}`) rather than free-text prose. This is a valid and complete answer format. Information present in these fields counts as the agent having identified and presented that information. Do NOT penalize the agent for using a structured output format instead of prose. +**Evidence Scope:** Agent replies are withheld. A criterion requiring an unavailable reply has an evidence gap. Do not infer completion from an attempted action or HTTP success status alone. **Core Evaluation Principles:** @@ -1322,16 +1288,7 @@ 1. **Trust visual evidence over action history**: If screenshots contradict what the agent claimed, the visual evidence takes precedence. But remember: always use the LATEST screenshot when multiple screenshots show the same element in different states. -2. **For criteria evaluating the agent's output**: The agent's predicted output is provided above. When a criterion asks whether the agent provided a summary, reported a finding, listed results, etc.: - - Check if the predicted output addresses the criterion's requirements - - Use screenshots to verify whether the output is accurate and consistent with what the agent actually found - - Do NOT give zero credit simply because the output text is not visible on screen — the output is delivered as text, not rendered in a browser - - The standard is simple: **either the agent's answer is consistent with the visual evidence, or it is not.** Do not invent subjective dimensions like "overconfidence" or "tone certainty" — these are not scoring criteria. Evaluate the agent's claims using these five categories: - * **Contradiction** (penalize): Screenshots show X, but the agent claims not-X. Example: screenshot shows a booking calendar exists, but the agent says "no booking system available." The visual evidence directly contradicts the claim. - * **Fabrication** (penalize): The agent claims X with zero evidentiary basis — nothing in the screenshots or action history supports the claim. Example: agent states a specific price that appears nowhere in any screenshot. - * **Omission** (penalize): The agent didn't view everything it needed to. Screenshots show no evidence of X, and the agent concludes X doesn't exist or ignores it — BUT X is commonly known to exist and the agent should have looked for it. Example: Task asks for "highest ranked NHL team in the Western Conference," but the agent only checked the Central Division and never viewed the Pacific Division. This is incomplete exploration, not a supported inference. - * **Supported inference from absence** (do NOT penalize): Screenshots consistently show NO evidence of X (e.g., no booking UI, no date picker, no ticket purchasing interface across all relevant pages visited), and the agent concludes "X does not exist," AND X is not commonly known to exist. This is a reasonable inference consistent with the visual evidence — not a hallucination. Only penalize if screenshots actually CONTRADICT the claim by showing X does exist. Absence of explicit textual confirmation (e.g., no banner saying "we don't offer online booking") is NOT the same as contradiction. - * **Visual confirmation without explicit statement** (do NOT penalize): If the agent's output omits an explicit justification for a claim but the screenshots visually confirm the correct result (e.g., the agent selected female cardiologists but didn't explicitly state "female" — yet their photos in the screenshots confirm they are female-presenting, or the agent chose a "flexible" course and the screenshot shows a "Flexible schedule" label), the visual evidence is sufficient and the agent should not be penalized for the omission. +2. **For criteria evaluating the agent's output**: Agent final replies are withheld. Identify the evidence gap; do not invent an answer. Assess record changes using the available observations and their source. 3. **Verify environment blockers**: If the agent claimed a blocker (CAPTCHA, out-of-stock, login wall), the screenshots must CONFIRM this blocker is actually visible. If not visible, penalize appropriately. - IMPORTANT — Reality-Notes-revealed platform limitations: When the Reality Notes for a criterion indicate that the criterion's assumed entity, interface, or feature DOES NOT EXIST on the site (e.g., "no admissions booking calendar/date-picker is visible," "no online ticketing UI exists"), this is an **uncontrollable platform limitation** — the agent cannot navigate to or interact with something that does not exist. In this case, the scorer MUST apply the criterion's own full-credit escape clause (most criteria include one, e.g., "OR clearly reports platform limitations such as site lacks date-based booking"). Check whether the agent's output reports this limitation; if so, award full credit per the criterion's own description. Do NOT score 0 for "did not navigate to X" when Reality Notes confirm X does not exist on the site. @@ -1475,7 +1432,7 @@ """ -OUTCOME_VERIFICATION_PROMPT = """You are to evaluate the performance of a web navigation agent. The agent is designed to help a human user navigate a website to complete a task. You are given the user's task, the agent's action history, the agent's predicted output, a scored rubric with screenshot evidence grouped by criterion, and the rubric's scores from a separate scoring system. +OUTCOME_VERIFICATION_PROMPT = """You are to evaluate the performance of a web navigation agent. The agent is designed to help a human user navigate a website to complete a task. You are given the user's task, structured action/observation history, a scored rubric with screenshot evidence grouped by criterion, and the rubric's scores from a separate scoring system. **Your role is to make an INDEPENDENT assessment.** The scored rubric is provided as reference context — it shows how a separate system scored individual criteria, and what screenshot evidence was gathered. However, you must form your OWN conclusion about whether the task was successfully completed. The rubric system can sometimes be too strict (penalizing nitpicks, over-reading requirements, cascading penalties) or too lenient (awarding credit for effort without results). Use the rubric's evidence and reasoning as input, but do NOT blindly adopt its verdict. A high rubric score does not guarantee success, and a low rubric score does not guarantee failure. @@ -1697,10 +1654,6 @@ $action_history <<< -Predicted Output: >>> -$predicted_output -<<< - *IMPORTANT* Please output an answer in pure JSON format according to the following schema. The JSON object must be parsable as-is. DO NOT OUTPUT ANYTHING OTHER THAN JSON, AND DO NOT DEVIATE FROM THIS SCHEMA: diff --git a/packages/flow/tests/test_claim_blind_judge.py b/packages/flow/tests/test_claim_blind_judge.py new file mode 100644 index 0000000000..ffc2493872 --- /dev/null +++ b/packages/flow/tests/test_claim_blind_judge.py @@ -0,0 +1,164 @@ +"""Judge information-flow checks over synthetic evidence; no model calls.""" + +from __future__ import annotations + +import asyncio +import copy +import json +from types import SimpleNamespace + +import pytest + +from benchmark.verifier_baselines import medplum_views, run_judge, uv_judge +from benchmark.verifier_baselines.smoke_uv_practice import transcript_to_steps +from benchmark.verifier_baselines.test_uv_judge import FakeClient, _png + +FINAL = "AGENT_FINAL_NARRATIVE_1982" +NARRATION = "AGENT_TURN_NARRATIVE_1982" +INTENT = "USER_INTENT_1982" +ACTION = "STRUCTURED_ACTION_1982" +RECORD = "OBSERVED_RECORD_1982 claim saved success" +HTTP = "MEASURED_HTTP_1982" + + +def _view(tmp_path, monkeypatch): + task_file = tmp_path / "tasks.json" + task_file.write_text(json.dumps({"synthetic-task": INTENT})) + manifest = tmp_path / "manifest" + row_path = "synthetic/agent/rows/run.json" + manifest.write_text(f"unused-hash {row_path}\n") + transcript = [{"text": NARRATION, "actions": [{"text": ACTION}]}] + blobs = { + row_path: json.dumps( + { + "id": "synthetic-task", + "claim": {"status": "done", "final_text": FINAL, "note": FINAL}, + } + ).encode(), + "synthetic/agent/rows/run/transcript.json": json.dumps( + { + "transcript": transcript, + } + ).encode(), + "synthetic/agent/rows/run/start.png": _png(), + "synthetic/agent/rows/run/end.png": _png((0, 255, 0)), + } + monkeypatch.setattr(medplum_views, "TASKS", task_file) + monkeypatch.setattr(medplum_views, "MANIFEST", manifest) + # No historical cases, files or git objects are read by this test. + monkeypatch.setattr(medplum_views, "_show", blobs.__getitem__) + view = next(medplum_views.runs())["view"] + assert "predicted_output" not in view + assert FINAL not in json.dumps(view, default=str) + assert NARRATION not in json.dumps(view, default=str) + smoke_steps = transcript_to_steps(transcript, "") + assert NARRATION not in json.dumps(smoke_steps) + assert ACTION in json.dumps(smoke_steps) + return view + + +class CapturingClient(FakeClient): + def __init__(self, fallback): + super().__init__() + self.fallback = fallback + + async def create(self, messages, json_output=False): + from benchmark.verifier_baselines.test_uv_judge import STEP_HEADS + + if self.fallback and STEP_HEADS["4"] in self._text(messages): + self.log.append(("4", copy.deepcopy(messages))) + return SimpleNamespace(content="{}", usage={}) + return await super().create(messages, json_output) + + +@pytest.mark.parametrize("fallback", [False, True]) +def test_every_model_stage_is_claim_blind(tmp_path, monkeypatch, fallback): + view = _view(tmp_path, monkeypatch) + # A legacy cached view must not reintroduce either narrative channel. + view["predicted_output"] = FINAL + step = view["steps"][0] + step["state_description"] = NARRATION + step["error"] = NARRATION + step["tool_output"] = NARRATION + # Only the harness supplies these observations; agent text is never relabelled. + step["observations"] = [ + {"source": "system_of_record", "content": {"note": RECORD}}, + {"source": "application", "content": {"display": RECORD}}, + {"source": "http", "content": {"status_code": 201, "receipt": HTTP}}, + {"source": "agent", "content": NARRATION}, + {"content": NARRATION}, + ] + monkeypatch.setattr(run_judge, "RES", tmp_path / "results") + monkeypatch.setattr(run_judge, "DATA", tmp_path / "data") + client = CapturingClient(fallback) + output = tmp_path / "judge.json" + asyncio.run( + run_judge.judge_one( + view, + view.pop("screenshots"), + output, + "synthetic", + client, + asyncio.Semaphore(1), + ) + ) + result = json.loads(output.read_text()) + assert "error" not in result + assert result["output_success"] is True + stages = {stage for stage, _ in client.log} + expected = {"0a", "0b", "0c", "4", "6", "7", "8"} + if fallback: + expected.add("4single") + assert stages == expected + for stage, messages in client.log: + serialized = json.dumps(messages) + assert FINAL not in serialized, stage + assert NARRATION not in serialized, stage + assert INTENT in serialized, stage + if stage not in {"0a", "0b"}: + assert ACTION in serialized, stage + assert RECORD in serialized, stage + assert HTTP in serialized, stage + if stage in {"4", "4single"}: + images = [ + part + for message in messages + if isinstance(message["content"], list) + for part in message["content"] + if part["type"] == "image_url" + ] + assert images and images[0]["image_url"]["url"].startswith( + "data:image/jpeg;base64," + ) + + +def test_judge_rejects_final_answer_argument(): + client = FakeClient() + with pytest.raises(TypeError, match="predicted_output"): + asyncio.run( + uv_judge.judge( + INTENT, + "", + [], + [], + client, + predicted_output=FINAL, + ) + ) + assert client.log == [] + + +@pytest.mark.parametrize("status", [201, 500, None]) +def test_transport_receipt_stays_observed(status): + from benchmark.verifier_baselines.effect_e2e_arms import ACTUATOR_TIMEOUT_S, steps + + observed_steps = steps(status) + assert observed_steps[-1]["observations"] == [ + { + "source": "http", + "content": {"status_code": status, "timeout_s": ACTUATOR_TIMEOUT_S}, + } + ] + history = uv_judge.format_action_history(observed_steps) + assert f'"status_code": {json.dumps(status)}' in history + assert '"target": "Save button"' in history