diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fd4d15..a522730 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,24 @@ What changed in PACT, newest first. Each entry says what an author can now write, or what now holds that did not; the commits are listed under it for whoever wants the detail. +## 2026-11 — a metric a model is held to is taken on every case, and `prefer:` says what "best" means (16 November) + +### What holds now + +- **A metric an `accept-when:` line reads is taken on every case.** A case stops at its first failed + rule, so `refund-tone.score >= 0.8` was held to the mean over only the cases a model got past their + first failure — an easier subset, which favoured the model that failed more. The reference now + takes each metric an `accept-when:` line names on every case, a failed one included (the case still + fails, for the reason it failed before); `.score`, `.pass-rate` and + `.cases` are over the whole suite. Any other metric still stops at the first failure, so a + suite nobody holds a model to costs no more grading than before. +- **`best-on-the-evals` is the largest share of the suite's cases passed**, the cheaper of two that + passed the same share. The help said "the highest score", which could be read as a metric's mean. +- **`prefer:` chooses only when `model:` is left out.** A named `model:` is used as named; the help + now says so. +- **`cost` is what the agent's own calls spent per case**, not what grading them cost; the help now + says so. + ## 2026-11 — a benchmark named in any script, a figure recorded under an alias, and DeepEval's ARC challenge set (16 November) ### What holds now diff --git a/README.md b/README.md index 36213dd..4592b26 100644 --- a/README.md +++ b/README.md @@ -1001,11 +1001,11 @@ framework is an extra. |---|---| | **Design** | Thesis, 29 binding decisions, FRD (120 requirements), 60-row refusal ledger — complete | | **Research** | 14 source-grounded studies, ~15,750 lines, over 140 repos (~15 GB) + 57 papers | -| **Code** | Loader, diagnostics, schema engine, CLI, harness, resolver, evals, SLO — **5335 tests (1722 Rust + 3613 adapter), clippy clean, TypeScript type-checked** | +| **Code** | Loader, diagnostics, schema engine, CLI, harness, resolver, evals, SLO — **5336 tests (1722 Rust + 3614 adapter), clippy clean, TypeScript type-checked** | | **Adapters** | **All 7 named targets**, proven against one shared conformance suite | ```bash -./scripts/test-all.sh # 5335 tests, Rust + 7 adapters, fully offline +./scripts/test-all.sh # 5336 tests, Rust + 7 adapters, fully offline ``` **What is still owed** is not hidden — it lives in diff --git a/adapters/python/src/pact_adapters/evals.py b/adapters/python/src/pact_adapters/evals.py index 5a6e2cb..0b78969 100644 --- a/adapters/python/src/pact_adapters/evals.py +++ b/adapters/python/src/pact_adapters/evals.py @@ -632,6 +632,7 @@ def check( judge: "Judge | None" = None, metrics: "list[MetricSpec] | None" = None, run_program: "Callable[[str, dict[str, Any]], str] | None" = None, + taken_on_every_case: "frozenset[str] | set[str]" = frozenset(), ) -> CaseOutcome: """Grade one case. Deterministic first, always, and a judge only if needed. @@ -642,6 +643,13 @@ def check( nothing to do in it. That is what makes "a suite decidable without a model must never invoke one" a property of the code rather than a promise. + `taken_on_every_case` names the metrics an `accept-when:` line holds a + model to (`refund-tone.score >= 0.8`). Each of them is taken on this case + even after an earlier rule failed it, and the outcome still fails: a mean + taken only over the cases a model got past the first failure is a mean over + an easier subset, and it would favour the model that failed more. Every + other metric still stops at the first failure. + `judge` is how weights are reached, supplied by whoever runs the suite in the same way the transport is for a run — `judge.judge_of(doc)` builds it from the author's `graded-by:` line. With none, a `judged:` rule is neither passed @@ -669,27 +677,34 @@ def check( reader who was never shown them. """ text = result.output.lower() + scores = list(metrics or []) + unenforced: list[str] = [] + scored: dict[str, float] = {} + + def failed(why: str) -> CaseOutcome: + # The metrics `accept-when:` reads are taken on every case; see above. + for spec in scores: + if spec.name in taken_on_every_case and spec.name not in scored: + _scores_below(spec, case, result, unenforced, judge, run_program, scored=scored) + return CaseOutcome(case.key, False, why, tuple(unenforced), dict(scored)) for key, want in case.expect.items(): if not isinstance(want, str): continue needle = want.strip().lower() if needle and not _states(text, needle): - return CaseOutcome(case.key, False, f"expected {key} {want!r}, got {result.output!r}") + return failed(f"expected {key} {want!r}, got {result.output!r}") # The case's own extra rules, then the suite's. Case first, because a case # rule is about this situation and reads as the more specific failure. written = list(case.must_also) + list(rules) - scores = list(metrics or []) - unenforced: list[str] = [] - scored: dict[str, float] = {} for rule in written: if rule.kind == JUDGED: continue broken = _breaks(rule, result, text, unenforced=unenforced, run_program=run_program) if broken: - return CaseOutcome(case.key, False, broken, tuple(unenforced)) + return failed(broken) for spec in scores: if not spec.deterministic: @@ -699,21 +714,21 @@ def check( # pass, and only for a case nothing cheaper already settled. missed = _scores_below(spec, case, result, unenforced, scored=scored) if missed: - return CaseOutcome(case.key, False, missed, tuple(unenforced), dict(scored)) + return failed(missed) for rule in written: if rule.kind != JUDGED: continue broken = _breaks(rule, result, text, judge, unenforced, case) if broken: - return CaseOutcome(case.key, False, broken, tuple(unenforced)) + return failed(broken) for spec in scores: if spec.deterministic: continue missed = _scores_below(spec, case, result, unenforced, judge, run_program, scored=scored) if missed: - return CaseOutcome(case.key, False, missed, tuple(unenforced), dict(scored)) + return failed(missed) return CaseOutcome(case.key, True, "", tuple(unenforced), dict(scored)) diff --git a/adapters/python/src/pact_adapters/resolve.py b/adapters/python/src/pact_adapters/resolve.py index 0c4cc81..2b08b12 100644 --- a/adapters/python/src/pact_adapters/resolve.py +++ b/adapters/python/src/pact_adapters/resolve.py @@ -1502,9 +1502,14 @@ def evaluate( min_cases: int = 5, judge: Any = None, metrics: Any = None, + taken_on_every_case: "frozenset[str]" = frozenset(), ) -> Verdict: """Score one model on one strategy against the author's own suite. + `taken_on_every_case` names the metrics the agent's `accept-when:` lines + read (`_selection.metrics_read`); `evals.check` takes each of them on every + case, so the mean a line is held to is over the whole suite. + `document` is REQUIRED and keyword-only, because the one thing it is for is the thing that is silently wrong when it is missing: an agent with a `team:` is only runnable here if `run` is given an `ask_member`, and building one @@ -1602,7 +1607,8 @@ def evaluate( ), ) break - results.append(check(case, result, rules, judge=judge, metrics=metrics)) + results.append(check(case, result, rules, judge=judge, metrics=metrics, + taken_on_every_case=taken_on_every_case)) spends.append(None if "cost-per-request-under" in (result.unmetered or ()) else result.spent) if silence is not None: # UNDECIDED with no RESULTS kept, where `_run_every_case` keeps what it @@ -1799,7 +1805,8 @@ def resolve( candidate = transform(spec) v = evaluate(candidate, cases, rules, bar, transport_for, requested, strategy_name, tools, document=document, judge=judge, - metrics=scores) + metrics=scores, + taken_on_every_case=_selection.metrics_read(needs.get("accept-when") or ())) last = v if v.outcome == "PASS": # 96 §B4: passing the suite is necessary, not sufficient — what the @@ -1969,7 +1976,8 @@ def _cheapest_passing( for strategy_name, transform in strategies.items(): v = evaluate(transform(spec), cases, rules, bar, transport_for, entry.name, strategy_name, tools, document=document, - judge=judge, metrics=scores) + judge=judge, metrics=scores, + taken_on_every_case=_selection.metrics_read(needs.get("accept-when") or ())) if v.outcome == "PASS" and (short := _short_of(needs, v, document)): ruled_out.append(f"{entry.name} passed the suite and {short}") scored_it = True diff --git a/adapters/python/src/pact_adapters/selection.py b/adapters/python/src/pact_adapters/selection.py index dbc042d..4835e86 100644 --- a/adapters/python/src/pact_adapters/selection.py +++ b/adapters/python/src/pact_adapters/selection.py @@ -64,6 +64,7 @@ "held_before", "measured", "measurement_of", + "metrics_read", "parse", "parse_all", "reaches", @@ -402,13 +403,24 @@ def held_before(line: Line, entry: Any) -> "tuple[bool, str]": return False, f"has {line.measure} {have:g}, so `{line.written}` does not hold" +def metrics_read(lines: Sequence[Line]) -> frozenset[str]: + """The suite metrics these lines hold a model to, by name. + + Each is taken on every case of the suite, a failed one included + (`evals.check`), so `.score` and `.pass-rate` are over every + case the suite has and not over the easier ones a model got through. + """ + return frozenset(line.measure for line in lines if line.kind == "metric") + + def measured(outcomes: Sequence[Any], spent: "Sequence[float | None]" = ()) -> dict[str, float]: """What a candidate scored and spent on the suite: the figures `AFTER` lines read. `pass-rate` over every case; each metric's `score` (its mean) and - `pass-rate` over the cases it was taken on — a case that failed earlier is - not scored again, so its count is `.cases`; `cost` the mean over the - cases whose spend was metered, absent when none was. + `pass-rate` over the cases it was taken on, counted as `.cases` — every + case for a metric an `accept-when:` line reads (`metrics_read`), and for any + other the cases no earlier rule failed; `cost` the mean over the cases whose + spend was metered, absent when none was. """ out: dict[str, float] = {} if outcomes: diff --git a/adapters/python/tests/test_an_accept_line_means_one_thing.py b/adapters/python/tests/test_an_accept_line_means_one_thing.py index c13a856..5816b89 100644 --- a/adapters/python/tests/test_an_accept_line_means_one_thing.py +++ b/adapters/python/tests/test_an_accept_line_means_one_thing.py @@ -19,7 +19,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from pact_adapters import benchmarks, resolve, selection # noqa: E402 -from pact_adapters.evals import CaseOutcome, Verdict # noqa: E402 +from pact_adapters.evals import Case, CaseOutcome, Verdict # noqa: E402 from trees import REPO, shown # noqa: E402 TABLE = yaml.safe_load((REPO / "spec/accept-when.yaml").read_text(encoding="utf-8")) @@ -194,6 +194,35 @@ def test_what_a_model_scored_and_spent_on_the_suite_is_held_after_it_passes() -> assert "never metered" in resolve._short_of({"accept-when": (_line("cost < 0.05 USD"),)}, unmetered, doc) +def test_a_metric_an_accept_line_reads_is_taken_on_every_case() -> None: + """A mean over only the cases a model got past their first failure is a mean over an easier + subset, and favours the model that failed more; so a metric `accept-when:` reads is taken on + every case, and any other still stops at the first failure.""" + from pact_adapters.evals import check + from pact_adapters.harness import RunResult + from pact_adapters.providers import MetricSpec + + brevity = MetricSpec.parse({"uri": "pact:at_most_words", "name": "brevity", "with": {"words": 4}}) + cases = [Case("easy", "a refund?", {"answer": "refunded"}), Case("hard", "a refund?", {"answer": "refunded"})] + answers = [RunResult("refunded"), RunResult("I cannot say anything about that refund at all today")] + read = selection.metrics_read( + [selection.parse("brevity.score >= 0.8", benchmarks=BENCH, metrics={"brevity"}), _line("mmlu > 50")] + ) + assert read == {"brevity"} + early = [check(c, r, [], metrics=[brevity]) for c, r in zip(cases, answers)] + every = [check(c, r, [], metrics=[brevity], taken_on_every_case=read) for c, r in zip(cases, answers)] + assert [o.passed for o in early] == [o.passed for o in every] == [True, False] + assert every[1].why == early[1].why, "the case fails for the reason it failed before" + assert selection.measured(early) == {"pass-rate": 0.5, "brevity.score": 1.0, "brevity.cases": 1.0} + taken = selection.measured(every) + assert taken["brevity.cases"] == 2.0 and taken["brevity.score"] < 1.0 + held = lambda outcomes: selection.held_after( # noqa: E731 + selection.parse("brevity.score >= 0.8", benchmarks=BENCH, metrics={"brevity"}), + selection.measured(outcomes), + ) + assert held(early)[0] and not held(every)[0] + + def test_the_desk_reads_its_own_lines_and_its_named_metric() -> None: doc = shown(TREE) needs = resolve.needs_of(doc, "front-desk") diff --git a/docs/90-REVIEW.md b/docs/90-REVIEW.md index 39accec..e9d4e0b 100644 --- a/docs/90-REVIEW.md +++ b/docs/90-REVIEW.md @@ -81,7 +81,7 @@ them and changes nothing. |---|---| | Schema | **43 groups, 262 field entries, 207 distinct field names** (core 123, expert 69, both 15), 40 closed enums, 27 cross-reference constraints, **0 fields missing `tier:` or `surface:`** | | Rust | 5 crates, **1722 tests**, `unsafe_code = forbid`, clippy-deny, edition 2024 | -| Python | **47 modules** (`find adapters/python/src -name "*.py" \| wc -l`), **3613 tests**, **nine transports** — seven model-bound (`anthropic`, `autogen`, `langchain`, `langgraph`, `ollama`, `openai_agents`, `pydantic_ai`), one bound to a remote *agent* (`a2a`), one deterministic mock; `_metering.py`, `_summarise.py` and `_tool_choice.py` are shared code the transports import, which is what the `_` says — offline by default | +| Python | **47 modules** (`find adapters/python/src -name "*.py" \| wc -l`), **3614 tests**, **nine transports** — seven model-bound (`anthropic`, `autogen`, `langchain`, `langgraph`, `ollama`, `openai_agents`, `pydantic_ai`), one bound to a remote *agent* (`a2a`), one deterministic mock; `_metering.py`, `_summarise.py` and `_tool_choice.py` are shared code the transports import, which is what the `_` says — offline by default | | TypeScript | 5 modules, 2,305 lines, a second port of **the stepping, the ceilings and the stage path** (`harness.ts:15-21`: *"AC-5.3's bar is the stage path"*) — **not** of the loop entire: no interceptor chain, no durable suspension record, no ledger (`notDoneHere`; a park returns `halted: "suspended"` and writes nothing down). This row is load-bearing for the Phase 3 pricing below (E4) | | CLI | **6 verbs**: `check`, `show`, `waits`, `discover`, `card`, `help` | | Worked example | 42 files, 1128 lines, **132 distinct authored keys**, `pact check` → OK (498 settings) | diff --git a/site-docs/index.md b/site-docs/index.md index e262516..d1dbc53 100644 --- a/site-docs/index.md +++ b/site-docs/index.md @@ -47,7 +47,7 @@ so plainly rather than leaving it to be discovered. | | | |---|---| -| Tests | **5,335** — 1722 Rust, 3613 adapter | +| Tests | **5,336** — 1722 Rust, 3614 adapter | | Schema kinds | **70** | | Framework targets | **7**, over 2 runtimes | | Eve capabilities accounted for | **101 of 101** | diff --git a/site-docs/status/verified.md b/site-docs/status/verified.md index 4cdcb91..7414411 100644 --- a/site-docs/status/verified.md +++ b/site-docs/status/verified.md @@ -13,8 +13,8 @@ from an earlier document. | Claim | Value | How to check | |---|---|---| | Rust tests | **1722** | `cargo test --workspace` | -| Adapter tests | **3613** | `cd adapters/python && uv run pytest tests/ -q` | -| Total | **5,335** | both of the above | +| Adapter tests | **3614** | `cd adapters/python && uv run pytest tests/ -q` | +| Total | **5,336** | both of the above | | TypeScript type-checks | clean | `cd adapters/typescript && npx tsc --noEmit` | | Lints | clean | `cargo clippy --all-targets` | | Worked example loads | 498 settings | `cargo run -p pact-cli -- check examples/refund-desk` | diff --git a/spec/schema.yaml b/spec/schema.yaml index d49a183..6474dd4 100644 --- a/spec/schema.yaml +++ b/spec/schema.yaml @@ -2222,9 +2222,11 @@ groups: among the models that pass, which one PACT picks. `cheapest` is the usual answer and what you get if you leave this out. `fastest` picks the one whose first words come back soonest — what a voice - agent wants. `best-on-the-evals` picks the highest score whatever it - costs. A model that fails the checks is never picked, whichever you - write. + agent wants. `best-on-the-evals` picks the one that passed the + largest share of your evals' cases, whatever it costs, the cheaper + of two that passed the same share. A model that fails the checks is + never picked, whichever you write. A `model:` you name is used as + named: this only chooses when `model:` is left out. accept-when: # 96 §B4. A model is ACCEPTED for this agent when every line holds — # `mmlu.score > 50`, `refund-tone.score >= 0.8`, `cost < 0.05 USD` — @@ -2256,11 +2258,14 @@ groups: `terminal-bench.score >= 40`, `refund-tone.score >= 0.8`, `pass-rate >= 90%`, `cost < 0.05 USD`. A benchmark is named as the model catalogue names it under `benchmarks:`; a check of your own by the - `name:` it has in your suite's `metrics:`. `pass-rate` and `cost` are - what a model scored and spent on your own cases, on average, and - `input-price`, `output-price` (per million tokens), `context-window` - and `first-words-within` are what the catalogue records. Leave it out - and every model that passes your checks is acceptable. + `name:` it has in your suite's `metrics:`, and that check is taken on + every case, so its score is over all of them. `pass-rate` and `cost` + are what a model scored and spent on your own cases, on average — + `cost` is what the agent's own calls spent, not what grading them + cost — and `input-price`, `output-price` (per million tokens), + `context-window` and `first-words-within` are what the catalogue + records. Leave it out and every model that passes your checks is + acceptable. context-at-least: # Was `type: text`, so `context-at-least: banana` loaded clean and # reached the resolver verbatim, where it was compared against a real