Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,28 @@ memory it took, so a local row the machine could not load was bound and failed o
- **The `deepeval` extra installs `datasets` and `pandas`.** DeepEval's benchmarks import them when
they are built and its package declares neither, so a benchmark run failed on its first line.

## 2026-11 — a copied catalogue row says which row it copied, and check says when that row moves on (17 November)

Recording one benchmark figure for a shipped model copies the distribution's whole row into the
workspace's `models/catalog.yaml` first, because a workspace row replaces the shipped one whole. The
copy then kept the price, the window and the serving it was copied with, and when the distribution
corrected that row the workspace went on binding against the old facts with nothing said.

### What an author can now write

- **`copied-from: {digest}`** on a row of the workspace's own catalogue: `sha256:` and the hex
digest of the distribution's row as it was copied (its JSON as `pact show` prints it, keys sorted,
no spaces). `benchmarks.record` writes it whenever it copies a row.

### What holds now

- **`pact check` warns `loader/a-copied-row-behind-the-distribution`** when the distribution's row
of that name (or alias) no longer has that digest, naming the fields where the copy and the
shipped row now differ and giving the digest to write once they are reconciled; and when the
distribution no longer has the row at all.
- `spec/copied-rows.yaml` holds the Rust checker and the Python writer to the same digests, for a row
with whole numbers, decimals, a share, a date and non-ASCII text.

## 2026-11 — a measured figure says how it was measured, and a line is held to its interval (16 November)

A figure's `provenance:` said where a number came from but not how much of the benchmark it rests on
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1001,11 +1001,11 @@ framework is an extra.
|---|---|
| **Design** | Thesis, 29 binding decisions, FRD (120 requirements), 60-row refusal ledger — complete |
| **Research** | 14 source-grounded studies, ~15,750 lines, over 140 repos (~15 GB) + 57 papers |
| **Code** | Loader, diagnostics, schema engine, CLI, harness, resolver, evals, SLO — **5375 tests (1725 Rust + 3650 adapter), clippy clean, TypeScript type-checked** |
| **Code** | Loader, diagnostics, schema engine, CLI, harness, resolver, evals, SLO — **5383 tests (1728 Rust + 3655 adapter), clippy clean, TypeScript type-checked** |
| **Adapters** | **All 7 named targets**, proven against one shared conformance suite |

```bash
./scripts/test-all.sh # 5375 tests, Rust + 7 adapters, fully offline
./scripts/test-all.sh # 5383 tests, Rust + 7 adapters, fully offline
```

**What is still owed** is not hidden — it lives in
Expand Down
51 changes: 50 additions & 1 deletion adapters/python/src/pact_adapters/benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@

import datetime as _dt
import enum
import hashlib
import json
import subprocess
import tempfile
import typing
from collections.abc import Callable, Mapping
from dataclasses import dataclass
Expand All @@ -31,6 +35,7 @@

import yaml

from .loader import pact_binary
from .providers import AskModel
from .selection import benchmarks_of

Expand Down Expand Up @@ -162,8 +167,14 @@ def record(
# A workspace row REPLACES the distribution's row of the same name — the catalogue is
# layered row by row (`resolve._layer`) — so a row holding only this figure would lose
# where the model is served, its price and its window. The distribution's row is copied
# first, and the reviewer reads all of it in the diff.
# first, and the reviewer reads all of it in the diff. `copied-from:`
# says which version of that row it was, so `pact check` can say when
# the distribution's row moves on without the copy
# (`loader/a-copied-row-behind-the-distribution`).
rows[model] = _shipped_row(model)
shown = _shipped_as_shown(model)
if rows[model] and shown is not None:
rows[model]["copied-from"] = {"digest": row_digest(shown)}
row = rows[model]
row.setdefault("benchmarks", {})[key] = figure
path.parent.mkdir(parents=True, exist_ok=True)
Expand Down Expand Up @@ -200,6 +211,44 @@ def _shipped_row(model: str) -> dict[str, Any]:
return {}


def row_digest(row: Mapping[str, Any]) -> str:
"""The digest `copied-from.digest` records: `sha256:` and the hex SHA-256 of
`row`'s JSON with every mapping's keys sorted and no spaces.

`row` is the JSON `pact show` printed for the row, so its numbers are the
ones the loader read; digesting a YAML reading of its own would let a date
or a `1e3` read differently here than in the checker (`spec/copied-rows.yaml`).
"""
text = json.dumps(row, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()


def _shipped_as_shown(model: str) -> "dict[str, Any] | None":
"""The distribution's row for `model` as `pact show` prints it, or `None` with no loader here.

`pact show` loads a workspace, so the distribution's catalogue is shown
from a scratch workspace that holds nothing else.
"""
from .resolve import BUILTIN_CATALOGUE

binary = pact_binary()
if binary is None:
return None
with tempfile.TemporaryDirectory() as scratch:
root = Path(scratch)
(root / "models").mkdir()
(root / "models" / "catalog.yaml").write_text(Path(BUILTIN_CATALOGUE).read_text(encoding="utf-8"), encoding="utf-8")
(root / "workspace.yaml").write_text("name: shipped-catalogue\n", encoding="utf-8")
shown = subprocess.run([str(binary), "show", str(root)], capture_output=True, text=True)
if shown.returncode != 0:
return None
rows = ((json.loads(shown.stdout) or {}).get("models") or {}).get("models") or {}
for name, row in rows.items():
if name == model or model in (row or {}).get("also-known-as", []):
return row
return None


def run_deepeval(
benchmark: Benchmark, model: str, ask: Callable[[str, str], str]
) -> float: # pragma: no cover - downloads a dataset and asks a live model
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
"""A copied catalogue row records which version of the distribution's row it copied.

`benchmarks.record` copies the distribution's row before adding a figure, and
writes `copied-from.digest` beside it. The digest is taken over the JSON
`pact show` prints, so it is the one the checker computes
(`crates/pact-loader/src/copies.rs`): `spec/copied-rows.yaml` holds both
runtimes to the same digests, and `pact check` stays quiet on a fresh copy.
"""

from __future__ import annotations

import json
import subprocess
from pathlib import Path

import pytest
import yaml

from pact_adapters import benchmarks
from pact_adapters.loader import pact_binary

REPO = Path(__file__).resolve().parents[3]
TABLE = yaml.safe_load((REPO / "spec" / "copied-rows.yaml").read_text(encoding="utf-8"))
BINARY = pact_binary()
needs_binary = pytest.mark.skipif(BINARY is None, reason="no `pact` binary is built here")


def _shown_row(tmp_path: Path, row: dict) -> dict:
(tmp_path / "models").mkdir()
(tmp_path / "workspace.yaml").write_text("name: probe\n", encoding="utf-8")
(tmp_path / "models" / "catalog.yaml").write_text(
yaml.safe_dump({"version": 1, "models": {"probe": row}}, allow_unicode=True, sort_keys=False),
encoding="utf-8",
)
shown = subprocess.run([str(BINARY), "show", str(tmp_path)], capture_output=True, text=True, check=True)
return json.loads(shown.stdout)["models"]["models"]["probe"]


@needs_binary
@pytest.mark.parametrize("case", TABLE["digests"], ids=lambda c: c["because"])
def test_a_row_digests_as_the_shared_table_says(case, tmp_path):
assert benchmarks.row_digest(_shown_row(tmp_path, case["row"])) == case["digest"]


@needs_binary
def test_recording_a_figure_writes_which_row_was_copied_and_check_is_quiet(tmp_path):
path = benchmarks.record(tmp_path, "qwen2.5-7b-instruct", "mmlu", 70.0, source="a run on this machine")
row = yaml.safe_load(path.read_text(encoding="utf-8"))["models"]["qwen2.5-7b-instruct"]
shipped = benchmarks._shipped_as_shown("qwen2.5-7b-instruct")
assert shipped is not None
assert row["copied-from"] == {"digest": benchmarks.row_digest(shipped)}
(tmp_path / "workspace.yaml").write_text("name: copied\n", encoding="utf-8")
checked = subprocess.run([str(BINARY), "check", str(tmp_path)], capture_output=True, text=True)
assert "a-copied-row-behind-the-distribution" not in checked.stdout + checked.stderr


@needs_binary
def test_a_copy_whose_digest_is_not_the_shipped_rows_is_reported(tmp_path):
path = benchmarks.record(tmp_path, "qwen2.5-7b-instruct", "mmlu", 70.0, source="a run on this machine")
data = yaml.safe_load(path.read_text(encoding="utf-8"))
data["models"]["qwen2.5-7b-instruct"]["copied-from"] = {"digest": "sha256:" + "0" * 64}
path.write_text(yaml.safe_dump(data, sort_keys=False, allow_unicode=True), encoding="utf-8")
(tmp_path / "workspace.yaml").write_text("name: copied\n", encoding="utf-8")
checked = subprocess.run([str(BINARY), "check", str(tmp_path)], capture_output=True, text=True)
assert "loader/a-copied-row-behind-the-distribution" in checked.stdout + checked.stderr
assert "`benchmarks`" in checked.stdout + checked.stderr


def test_a_second_figure_does_not_copy_the_row_again(tmp_path):
first = benchmarks.record(tmp_path, "qwen2.5-7b-instruct", "mmlu", 70.0, source="a run on this machine")
before = yaml.safe_load(first.read_text(encoding="utf-8"))["models"]["qwen2.5-7b-instruct"].get("copied-from")
again = benchmarks.record(tmp_path, "qwen2.5-7b-instruct", "gsm8k", 80.0, source="a run on this machine")
assert yaml.safe_load(again.read_text(encoding="utf-8"))["models"]["qwen2.5-7b-instruct"].get("copied-from") == before
3 changes: 3 additions & 0 deletions crates/pact-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2078,6 +2078,9 @@ fn validate_leaving_out(
// What a model needs of the machine that runs it, read the way a runtime
// holding a machine to it will read it (`spec/hardware-fit.yaml`).
pact_loader::hardware::check(root, &mut diags);
// A row of the workspace's own catalogue copied from the distribution's,
// which the distribution has corrected since.
pact_loader::copies::check(root, BUILTIN_CATALOGUE, &mut diags);
// 96 §B5: a `settings:` line the pinned model would throw away.
pact_loader::settings::check(root, BUILTIN_CATALOGUE, &mut diags);
diags.sort();
Expand Down
Loading
Loading