diff --git a/examples/request-routing/README.md b/examples/request-routing/README.md index 3203b1174..5d6f6eaa9 100644 --- a/examples/request-routing/README.md +++ b/examples/request-routing/README.md @@ -1,4 +1,7 @@ -# Route requests without a generation call on every message +# Route requests to handlers + +This example chooses a handler for a request. To train a new decision function +from reviewed examples, use [Train a router](../train-a-router/README.md). For a single native classification call with your own handler labels, see the [direct routing example](./direct). The recorded cascade below uses a diff --git a/examples/request-routing/cost/README.md b/examples/request-routing/cost/README.md index d5e3c574e..c35383897 100644 --- a/examples/request-routing/cost/README.md +++ b/examples/request-routing/cost/README.md @@ -1,4 +1,9 @@ -# Send each request to a cheap or a strong LLM +# Historical cost-routing study + +This is a separately installable study of cheap/strong LLM routing, with its own +locked environment and archived answer costs. It is independent of the parent +handler example and the new [Train a router](../../train-a-router/README.md) +learning example. The figures below describe the dated recorded study. ## What this shows diff --git a/examples/request-routing/score.py b/examples/request-routing/score.py index c73f1ace0..787580ba4 100755 --- a/examples/request-routing/score.py +++ b/examples/request-routing/score.py @@ -329,7 +329,10 @@ def recorded_inputs(directory): require( type(recorded["conf"]) in (int, float) and math.isfinite(recorded["conf"]) - and math.isclose(confidence, recorded["conf"], rel_tol=1e-6, abs_tol=1e-7), + # Float32 BLAS reductions differ across CPUs (e.g. Apple Accelerate + # versus OpenBLAS). Observed max absolute drift on the pinned 3,000 + # cases is 3.8e-6. Route order and the threshold branch stay exact. + and math.isclose(confidence, recorded["conf"], rel_tol=1e-5, abs_tol=1e-6), "recorded_front_confidence", ) require((confidence >= THRESHOLD) == (recorded["conf"] >= THRESHOLD), "front_threshold") diff --git a/examples/train-a-router/.gitignore b/examples/train-a-router/.gitignore new file mode 100644 index 000000000..6f138040d --- /dev/null +++ b/examples/train-a-router/.gitignore @@ -0,0 +1,21 @@ +# Local environments and generated experiment outputs. +.venv/ +__pycache__/ +.pytest_cache/ +.ruff_cache/ +*.pyc +data/ +my-router/ +published-results/ +my-results/ +answers/ +fresh-encoding/ +local-features/ +fresh-answers/ +fresh-grades/ +fresh-cpu-router/ +bbeh-source/ +calibration.html +runs/ +reports/ +outputs/ diff --git a/examples/train-a-router/LABELS.md b/examples/train-a-router/LABELS.md new file mode 100644 index 000000000..9a8532d79 --- /dev/null +++ b/examples/train-a-router/LABELS.md @@ -0,0 +1,85 @@ +# Where the router’s labels come from + +This example labels **model outcomes**, not whether a question looks “easy” or “hard.” For every training question, Luna and Sol each produce an answer independently. A deterministic grader checks each answer against the BBEH reference. The resulting labels are: + +| Luna correct? | Sol correct? | What the router learns | +|---|---|---| +| Yes | Yes | The cheaper answer can be enough | +| No | Yes | Paying for Sol can recover accuracy | +| Yes | No | The stronger model is not always the better choice | +| No | No | Paying more does not necessarily solve this question | + +Both correctness labels are retained. The model also learns expected costs from actual token usage. Neither router sees a reference, an answer, a correctness label, or the test cost when processing a new question. + +## If you have reference answers + +Use them to check real outputs from the two models you intend to serve. This is the validated route in this example. It avoids assuming that a teacher can predict which model will solve a question without seeing their answers. + +BBEH provides reference answers. The supplied recordings were collected with `gpt-6-luna` and `gpt-6.1-sol`, medium reasoning, a 16,384-output-token limit and the same question-only JSON answer contract. A format or quality failure is retained as a failure; it is not regenerated until it looks correct. Transport attempts are tracked separately, including unresolved cost reservations. + +The grader normalizes source-validated equivalent answer formats and preserves the raw upstream score too. For example, a unique option letter plus its matching text can be equivalent to the letter alone. It does not use the expected answer to repair a model’s response. The primary and source-format sensitivity results are both supplied. + +## If you do not have reference answers + +A teacher model can **judge the answers produced by both candidate models**. A useful protocol is: + +1. Collect representative questions and both candidate answers. Keep related questions together when splitting the data. +2. Give a teacher the question, any authoritative context and each anonymized answer. Use a concrete correctness rubric and allow “uncertain.” Randomize answer order if asking for a comparison. +3. Retain its two judgments, confidence/uncertainty and explanation. Avoid reducing every pair to a forced winner: both answers can be right or wrong. +4. Check a separate sample with people, executable tests or authoritative references. Measure teacher errors and disagreements before treating its labels as truth. +5. Train using the accepted labels; choose settings on development data and policies on calibration data. Keep an independently checked test set for the money–quality comparison. + +A frontier API model or a model served through SIE can be the teacher. Its usefulness depends on the task and the quality of its judgments, not only parameter count. If Sol itself judges Sol versus Luna, correlated errors and self-preference can distort the result; an independent judge and an audited sample are preferable. + +**Teacher-only labeling is not a measured result of this BBEH example.** We have not established that Qwen 27B, GLM Flash or another frontier model produces equivalent labels here. The current results should not be presented as evidence of that. Teacher labeling is a way to extend the workflow, with its own validation cost. + +## Collect fresh Sol/Luna answers + +The published recordings are enough for the complete training exercise. The following optional commands make a new paid collection with the same questions and settings. They keep the old recordings intact. Run them from this example folder. + +```bash +# Enter your OpenAI key and inspect the training/development collection plan without making calls. +export OPENAI_API_KEY='your-openai-api-key' +uv run --frozen --group collection python router.py collect \ + --data data --phase learn --output fresh-answers/learn --budget-usd 50 +``` + +```bash +# Make the planned training/development calls, retaining physical attempts and their costs. +uv run --frozen --group collection python router.py collect \ + --data data --phase learn --output fresh-answers/learn --budget-usd 50 --execute +``` + +```bash +# Fetch the exact public benchmark source and grade the new answers against its references. +uv run --frozen python -X utf8 source/fetch.py --output bbeh-source +uv run --frozen --group collection python -X utf8 grade_v2.py \ + --prepared data/source --collection fresh-answers/learn \ + --official bbeh-source/bbeh/evaluate.py --pair pair.json --output fresh-grades/learn +``` + +The budget is a ceiling, not the expected bill. The collector reserves room for in-flight requests and unresolved executions, and stops rather than silently exceeding it. An identical plan can resume in the same output directory: completed responses are reused and unresolved attempts retain their reservations. Inspect the receipts first. A different plan cannot overwrite that run. + +Use `--phase calibration` for a separate calibration collection. Final collection additionally requires `--frozen PATH`, pointing to a policy bundle made before those final answers. When repeating the published experiment, `data/reference/frozen-frontiers.json` is the supplied unchanged bundle: + +```bash +# Plan a fresh final-answer repeat for the already frozen reference policies; add --execute to dispatch it. +uv run --frozen --group collection python router.py collect \ + --data data --phase final --output fresh-answers/final --budget-usd 25 \ + --frozen data/reference/frozen-frontiers.json +``` + +These commands create new answer and grade directories; they do not replace the downloaded evidence or silently change a prepared training run. For example, train a separate CPU embedding router directly from the newly graded answers: + +```bash +# Prepare a new CPU study and fit neighbors/classifiers using the fresh training and development labels. +uv run --frozen python -X utf8 embedding/recipes.py prepare --source data/source --pair pair.json --run fresh-cpu-router +uv run --frozen --group outcomes python -X utf8 embedding/recipes.py fit \ + --run fresh-cpu-router --features data/features \ + --train-grades fresh-grades/learn/grades-train.json \ + --development-grades fresh-grades/learn/grades-development.json +``` + +This lower-level study is separate from the README’s published-data reproduction. Direct scripts use `python -X utf8` so multilingual inputs and JSON files use UTF-8 regardless of the host locale; `router.py` sets this mode automatically, including for its child processes. `native/bbeh_native.py` also accepts explicit grade paths; its prepared plan must bind the new acquisition protocol. Preserve source, answer-pair, grading and acquisition bindings together; changing models or prompts is a new experiment. + +For production feedback, collect outcomes on a representative sample of **both** routes. Labeling only the arm chosen by the current router hides the alternative’s performance and can reinforce its mistakes. New labels can update the neighbor collection, refit the small embedding heads or support another LoRA adaptation. Re-evaluate before replacing a serving policy. diff --git a/examples/train-a-router/QUICKSTART.md b/examples/train-a-router/QUICKSTART.md new file mode 100644 index 000000000..d5d9bdc06 --- /dev/null +++ b/examples/train-a-router/QUICKSTART.md @@ -0,0 +1,64 @@ +# Train a cost-aware router + +**Goal:** send requests to Luna when it is sufficient and pay for Sol when its extra accuracy is worthwhile. The supplied BBEH experiment compares unchanged Laya → LoRA adaptation → embedding neighbors → a small embedding classifier, with an optional learned cost estimate. On 975 held-out questions, an embedding classifier with learned cost estimates saved 16.7% with a 0.9-point accuracy loss; another preselected policy saved 39.7% with a 6.1-point loss. Both savings figures include the Cloud encoder fee. + +**You need:** Python 3.12 and `uv`; an NVIDIA GPU for the full LoRA exercise (validated on H100 80 GB). Replay and classifier training run on CPU. SIE Cloud can supply Qwen3-Embedding-4B vectors; OpenAI supplies live Sol/Luna answers. Cached vectors and answers are included, so replay needs neither API key. + +**1. Set up and inspect the evidence.** Training/development/calibration/test contain 1,969/589/385/975 questions. Related question families stay in one split. Accuracy measures the chosen answer; savings compare its cost with always-Sol. An accuracy change from 80% to 78% is a 2-point loss. Step numbers match the [full walkthrough](README.md). + +```bash +# From your repository clone, install dependencies and download the pinned data and decision-model weights. +cd examples/train-a-router +uv sync --frozen --group training --group outcomes +uv run --frozen python router.py fetch --output data --group core --group weights +uv run --frozen python router.py replay --data data --output published-results +uv run --frozen python router.py prepare --data data --run my-router +``` + +Open `published-results/explorer/index.html` for the full cost–quality curves. + +For the **CPU embedding exercise**, install only `--group outcomes`, fetch only `--group core`, and run the following after `prepare`. Then continue at steps 4–5. This uses recorded Laya comparisons; it does not perform LoRA training. + +```bash +# Supply the saved Laya predictions needed for comparison during an embedding-only refit. +uv run --frozen python router.py reuse-native --data data --run my-router +``` + +**2–3. Try and adapt Laya on GPU.** Start with unchanged weights, then let development data choose the LoRA training duration. On H100, comparing eight training passes took about **51 minutes**; the selected three-pass refit and checkpoint export took another **26 minutes**. Allow **77 minutes total**, plus setup and scoring time. + +```bash +# Score unchanged Laya, train LoRA, and save input-only predictions for both versions. +uv run --frozen --group training python router.py base --data data --run my-router --device cuda +uv run --frozen --group training python router.py fit-native --data data --run my-router +uv run --frozen --group training python router.py score --data data --run my-router --method native --device cuda +``` + +**4–5. Fit embedding routers on CPU.** The large Qwen encoder stays fixed. Neighbors average outcomes of similar questions; two small classifiers learn whether Luna and Sol will answer correctly. + +```bash +# Fit neighbors, classifiers and simple controls, then score held-out question embeddings. +uv run --frozen --group outcomes python router.py fit-embedding --data data --run my-router +uv run --frozen --group outcomes python router.py score --data data --run my-router --method embedding +``` + +**6. Estimate cost and choose your tradeoff.** Cost varies by question, so retain both the original and learned cost estimates. + +```bash +# Fit cost predictors, freeze the calibration choices, and open the calibration explorer. +uv run --frozen --group outcomes python router.py fit-cost --data data --run my-router +uv run --frozen python router.py calibrate --data data --run my-router +uv run --frozen python router.py frontier --run my-router --output calibration.html +``` + +Open `calibration.html` and record a **method and its candidate ID together** before viewing test results. For the headline approach, choose `embedding__textcost`; candidate IDs are specific to a method and run. There is no required quality-loss threshold. + +```bash +# After selecting your calibration policy, evaluate the frozen choices against the published test answers. +uv run --frozen python router.py evaluate-refit --data data --run my-router --output my-results +``` + +Open `my-results/explorer/index.html` to assess your choice. This is a reproduction using published test answers, not a new blind evaluation. + +**7. Try a live request.** Add a SIE key for an embedding route and an OpenAI key for the selected answer. Run `router.py answer` with both the method and candidate copied from calibration for the same run. [The live command](README.md#7-route-a-new-question) uses `embedding__textcost`. Laya routes need no embedding call; after the CPU shortcut, use the supplied `data/reference` run and weights for live Laya inference. + +To regenerate vectors, use [SIE Cloud](README.md#use-sie-cloud-or-your-own-encoder) or [your own SIE server](self-host/README.md). See [labels and collection](LABELS.md) for a new paired answer run. The [README](README.md) includes measured latency, hardware requirements, uncertainty and the effect of router fees. diff --git a/examples/train-a-router/README.md b/examples/train-a-router/README.md new file mode 100644 index 000000000..e4161ad69 --- /dev/null +++ b/examples/train-a-router/README.md @@ -0,0 +1,258 @@ +# Train a router to spend less on reasoning + +A router decides when a cheaper model is enough and when a stronger model is worth its extra cost. This example trains that decision on difficult reasoning questions, then measures **money saved and answer accuracy** on questions kept out of training. + +**On 975 held-out questions, an embedding classifier with learned cost estimates saved 16.7% with a 0.9-point accuracy loss; another preselected policy saved 39.7% with a 6.1-point loss. Both savings figures include the Cloud encoder fee.** The complete cost–quality curves are included: there is no single required accuracy-loss threshold. + +You will: + +1. **Establish the baseline:** compare always using Sol with always using Luna. +2. **Try a decision model:** use Laya to estimate whether each answer model will succeed. +3. **Adapt Laya with LoRA:** teach it from the answer models’ successes and failures. +4. **Use embedding neighbors:** estimate success from similar questions, without adapting an encoder. +5. **Train a small embedding classifier:** learn two correctness predictions on top of fixed embeddings. +6. **Compare cost estimates:** predict each request’s answer cost, then choose a policy on the full curve. +7. **Route a new question:** call only the selected answer model. + +[One-page instructions](QUICKSTART.md) · [Labels and new answer collection](LABELS.md) · [Serve the encoder yourself](self-host/README.md) + +## 1. Understand the data and baseline [Sol 80.8% accuracy; Luna 55.6%] + +We use **Big-Bench Extra Hard (BBEH)**: reasoning problems involving logic, counting, state tracking, language rules and other tasks. We retain **3,918 questions across 23 tasks** after source-only duplicate and ambiguity checks. Each question is answered independently by **`gpt-6-luna`** and **`gpt-6.1-sol`**, using medium reasoning and the same answer format. The supplied answer recordings let you train and evaluate without paying to collect them again. + +| Split | Questions | Purpose | +|---|---:|---| +| Training | 1,969 | Learn from model successes, failures and token costs | +| Development | 589 | Choose training duration and model settings | +| Calibration | 385 | Choose routing policies at different spending levels | +| Final test | 975 | Measure the frozen policies once | + +Repeated questions and identified source/context families stay together. Training uses training labels; model settings use development labels. After those choices, each learned router is fitted afresh on training + development. Calibration chooses the routing tradeoffs; final-test answers are collected only after all models and policies are frozen. + +The main measurements are: + +- **Answer accuracy:** the percentage of chosen model answers judged correct against the benchmark reference. This measures the answers people receive, rather than whether a router matches an arbitrary “best model” label. +- **Cost per 10,000 requests:** the measured input/output token cost of the selected answers, scaled to a common workload size. +- **Savings:** the percentage decrease from always calling Sol. +- **Accuracy loss:** always-Sol accuracy minus routed accuracy. For example, 80% becoming 78% is a **2-point loss**; a negative loss means the router did better on these questions. + +Some questions are answered correctly only by Luna. A good router can therefore occasionally beat always-Sol. The gray random-routing line shows what happens if we choose models without reading the question; a useful router should improve on that tradeoff. + +| Method | Answer accuracy | Accuracy loss | Answer cost / 10k | Answer-cost savings | +|---|---:|---:|---:|---:| +| Always Sol | 80.8% | 0.0 points | $70.77 | 0.0% | +| Always Luna | 55.6% | 25.2 points | $12.23 | 82.7% | +| Unchanged Laya | 68.6% | 12.2 points | $41.82 | 40.9% | +| Laya + LoRA | 78.3% | 2.6 points | $56.80 | 19.7% | +| Embedding neighbors | 76.3% | 4.5 points | $53.80 | 24.0% | +| Embedding classifier | 76.8% | 4.0 points | $54.41 | 23.1% | +| LoRA + learned cost | 79.0% | 1.8 points | $57.63 | 18.6% | +| Embedding classifier + learned cost | 79.9% | 0.9 points | $58.16 | 17.8% | +| Text/length + learned cost | 79.0% | 1.8 points | $55.28 | 21.9% | +| Text-predicted task + learned cost | 80.1% | 0.7 points | $58.31 | 17.6% | + +All router rows use a rule fixed before final collection: select the highest calibration accuracy under an **80% calibration answer-cost cap** relative to always-Sol. It is an illustrative spending level, not a required quality threshold or a guarantee of 20% test savings. The table reports final-test answer costs; add **$0.81 per 10,000 requests** for the Cloud encoder on embedding routes. For embedding + learned cost, that changes $58.16 to **$58.97**, or **16.7% savings** versus always-Sol at $70.77. Native and application hosting are separate. Every method, both cost variants and all preselected spending levels remain in the full report. + +The steps compare different ways to learn the decision; they do not promise that every added model improves every tradeoff. LoRA improves on unchanged Laya. Neighbors provide a simple embedding baseline. Learning request costs improves the embedding classifier at the illustrated spending levels. The inexpensive text controls remain competitive and belong in the comparison. + +### Set up + +For the **complete training sequence**, use a Linux machine with an NVIDIA GPU. The supplied LoRA recipe was validated on one **H100 80 GB**; its longest batches used approximately 51 GB of reserved GPU memory. Other GPU sizes and Apple Silicon LoRA training have not been validated. A Linux or macOS laptop can replay the results, train the embedding classifiers and run Laya inference; full-population Laya scoring is much faster on a GPU. + +Install Python 3.12 and [uv](https://docs.astral.sh/uv/getting-started/installation/). From a clone of this repository: + +```bash +# Enter the self-contained example and install the locked training and CPU dependencies. +cd examples/train-a-router +uv sync --frozen --group training --group outcomes +``` + +```bash +# Download the verified dataset, saved models and Laya weights from Hugging Face. +uv run --frozen python router.py fetch --output data --group core --group weights +``` + +```bash +# Recompute the published test results on CPU, without model or API calls. +uv run --frozen python router.py replay --data data --output published-results +``` + +Open `published-results/explorer/index.html` in your browser. It shows all saved policies, uncertainty intervals, simple controls and costs with or without the measured Cloud encoder fee. For a smaller CPU-only download, omit `--group weights`; weights are needed for Laya training or inference, not result replay. + +```bash +# Create fresh working directories for your own training run; keep downloaded evidence intact. +uv run --frozen python router.py prepare --data data --run my-router +``` + +For an **embedding-only exercise on CPU**, install with `uv sync --frozen --group outcomes` and fetch only `--group core`. After `prepare`, run the following command and continue at step 4: + +```bash +# Reuse the supplied Laya predictions for comparison while you train the embedding routers on CPU. +uv run --frozen python router.py reuse-native --data data --run my-router +``` + +This path uses the recorded Laya and LoRA comparisons; it does not train or copy a Laya checkpoint into `my-router`. The complete GPU path below performs the adaptation itself. + +## 2. Try unchanged Laya [68.6% accuracy; 40.9% answer-cost savings] + +**Laya is a decision model:** a roughly 0.4-billion-parameter ModernBERT encoder with a small decision head. For each request, we ask two questions: will Luna answer correctly, and will Sol answer correctly? Those scores, together with estimated answer costs, determine whether paying for Sol looks worthwhile. The request is the only input; the router never sees the reference answer. + +```bash +# Score the held-out question inputs with unchanged Laya; use --device cpu if needed. +uv run --frozen --group training python router.py base --data data --run my-router --device cuda +``` + +This establishes how much a general decision model knows before seeing this workload. The supplied calibration explorer already includes this baseline, so you can inspect its tradeoffs while your own scoring runs. + +## 3. Adapt Laya with LoRA [78.3% accuracy; 19.7% answer-cost savings] + +**LoRA learns small updates inside an existing model.** We keep Laya’s original encoder weights fixed, learn rank-16 attention updates, and train its existing decision head. The labels are two bits per question: whether Luna’s answer was correct and whether Sol’s was correct. They come from real model answers checked against the benchmark references. + +The complete `fit-native` command took **77 minutes on H100**: about **51 minutes** to compare eight training passes using development data, then **26 minutes** for the selected three-pass refit and checkpoint export. Setup and scoring add time. The saved checkpoint and reports let you inspect the result before your own training finishes. + +```bash +# Train on training data, select duration on development, then refit from the original model on both. +uv run --frozen --group training python router.py fit-native --data data --run my-router +``` + +```bash +# Apply both unchanged and adapted Laya to calibration/test inputs, without reading their answer labels. +uv run --frozen --group training python router.py score --data data --run my-router --method native --device cuda +``` + +The recipe considers the unchanged model and eight training passes. **Three passes gave the best development loss** in the supplied run; later passes worsened it. We then restarted from the original model and trained for those three passes on training + development. This check is why the example does not simply train for as long as possible. Small differences across hardware can change the selected pass; your run records its own selection. + +## 4. Estimate success from similar questions [76.3% accuracy; 22.8% savings incl. encoder] + +An **embedding** represents a question as a list of numbers that captures similarities in its content. We use **Qwen3-Embedding-4B**, fixed throughout the experiment. The neighbor router finds similar training questions and averages their two correctness labels. It selected **32 neighbors** on development data. + +This gives a different way to improve with more examples: add their embeddings and observed outcomes to the reference collection. You do not retrain the large encoder. + +```bash +# Fit neighbors and the small classifiers introduced next, using the supplied embeddings on CPU. +uv run --frozen --group outcomes python router.py fit-embedding --data data --run my-router +``` + +The command fits both embedding approaches and inexpensive text controls in one pass. Compare the neighbor method (`knn`) with the classifier (`embedding`) in the calibration explorer in step 6. + +## 5. Train a small classifier on embeddings [76.8% accuracy; 22.0% savings incl. encoder] + +Instead of looking up neighbors, we can learn a compact mapping from each embedding to two probabilities. The classifier is **two regularized logistic regressions**—one for Luna’s correctness and one for Sol’s—not another large language model. The frozen encoder produces 2,560 numbers; each predictor learns a weight for those numbers plus an offset. + +The previous command trained these classifiers alongside the neighbors. Now generate predictions from both approaches: + +```bash +# Generate their calibration/test predictions from question embeddings, without reading answer labels. +uv run --frozen --group outcomes python router.py score --data data --run my-router --method embedding +``` + +The text and task controls fitted in step 4 matter: if question length or vocabulary explains most of the benefit, a larger model may be unnecessary. The control given the benchmark's true task name is labeled **metadata-assisted** and cannot be selected for arbitrary live requests. + +The encoder is about ten times larger than Laya. This comparison measures useful systems, not proof that one architecture is intrinsically better. Encoder quality, size, training data and task fit all differ. Its practical advantage here is that the expensive encoder stays fixed while the task-specific training is fast and small. + +### Use SIE Cloud or your own encoder + +The supplied vectors make the training exercise reproducible on CPU. To generate fresh ones, create an account in the [Superlinked Console](https://console.superlinked.com), obtain an API key, and use the available Cloud credits. **SIE credits pay for embeddings; Sol/Luna answers use OpenAI credits.** You need no OpenAI key to train using the supplied answer recordings. + +```bash +# Enter your SIE Cloud key without writing it into this repository. +export SIE_API_KEY='your-sie-api-key' +``` + +```bash +# Regenerate all question embeddings and validate singleton encoder fees through SIE Cloud. +uv run --frozen --group encode python router.py encode --data data --output fresh-encoding +``` + +For a fresh run, add `--features fresh-encoding/features` to both embedding fitting and scoring commands. The [self-host instructions](self-host/README.md) run the same pinned Qwen encoder behind your own SIE endpoint. Self-hosting the encoder does not make Sol or Luna locally available; changing the answer models requires collecting their outcomes and training for that pair. + +## 6. Compare cost estimates and choose the tradeoff [79.9% accuracy; 16.7% savings incl. encoder] + +A request’s cost varies with how much a model generates. A router that predicts correctness well can still spend poorly if it assumes the wrong answer cost. We therefore also fit two small cost predictors using question words and length. They learn from the token costs in training + development; the saved final answers are not available to them. + +```bash +# Fit request-cost predictors on CPU while preserving each router’s original cost-estimate variant. +uv run --frozen --group outcomes python router.py fit-cost --data data --run my-router +``` + +```bash +# Use calibration answers to save the complete cost–quality curve and freeze every test policy. +uv run --frozen python router.py calibrate --data data --run my-router +``` + +```bash +# Explore your calibration curve and copy a method plus candidate ID before opening test results. +uv run --frozen python router.py frontier --run my-router --output calibration.html +``` + +Open `calibration.html`. A **Pareto frontier** keeps choices where you cannot lower cost without losing accuracy, or improve accuracy without paying more. Choose a point that fits your application and record its **method and candidate ID together before opening the test report**. Candidate IDs belong to a particular method and run; the same number can mean a different policy elsewhere. There is no built-in one-point accuracy-loss requirement. Increasing the price penalty generally sends more requests to Luna; the saved policy also handles cases where a supposedly stronger answer is predicted to cost less. + +The cost predictor helps some spending levels and hurts others, so the explorer keeps both variants. `embedding` uses the original cost estimate; `embedding__textcost` uses the learned request-cost estimate. The same choice is available for Laya, LoRA, neighbors and simple controls. + +For the embedding classifier with learned costs, the preselected policies give: + +| Calibration answer-cost cap | Final accuracy | Accuracy loss | Answers + encoder / 10k | Savings incl. encoder | +|---|---:|---:|---:|---:| +| 20% of always-Sol | 56.1% | 24.7 points | $13.52 | 80.9% | +| 40% of always-Sol | 66.5% | 14.4 points | $26.51 | 62.5% | +| 60% of always-Sol | 74.8% | 6.1 points | $42.70 | 39.7% | +| 80% of always-Sol | 79.9% | 0.9 points | $58.97 | 16.7% | + +These are four examples from the curve, all chosen on calibration. They expose the price of additional savings; none is designated as the right quality loss. + +```bash +# Evaluate your frozen policies on the published test answers and open the generated report. +uv run --frozen python router.py evaluate-refit --data data --run my-router --output my-results +``` + +Open `my-results/explorer/index.html`. Because these test answers are already public, this is a reproduction exercise, not a new blind evaluation. For a new dataset, preserve separate development, calibration and final-test populations before iterating. + +## 7. Route a new question + +Use the method and candidate ID copied from **calibration.html** for the same run. The command below uses `embedding__textcost`, the classifier with learned costs behind the headline result. Choose that method in the explorer and copy one of its frontier candidate IDs, or replace both values with another selected pair. The command predicts the route first, saves that decision, then calls exactly one answer model. + +```bash +# Enter the OpenAI key used to pay for the selected Sol or Luna answer. +export OPENAI_API_KEY='your-openai-api-key' +``` + +```bash +# Route one question using a method and candidate ID selected together from calibration. +METHOD=embedding__textcost +CANDIDATE_ID=YOUR_CALIBRATION_CANDIDATE_ID +uv run --frozen --group collection --group encode --group outcomes python router.py answer \ + --run my-router --method "$METHOD" --candidate-id "$CANDIDATE_ID" \ + --question 'A robot starts at (0,0), facing north. It turns right, moves 3, turns left, moves 2, then turns left and moves 1. What is its final coordinate? Answer as (x,y).' \ + --output answers/robot +``` + +For Laya or LoRA, select `base` or `lora` (`base__textcost` or `lora__textcost` for learned costs), use `--group collection --group training` (add `--group outcomes` for learned costs), and choose `--device cpu` or `cuda`. These routes need no embedding API call. + +To try the supplied models directly, fetch `--group weights`, use `--run data/reference` and select a policy from the supplied reference calibration explorer. This also provides live Laya inference after the CPU-only exercise, which reused its predictions without copying its checkpoint. Each request needs a new output directory so a failed call is not silently repeated. + +## Serving speed and costs + +| Router deployment | Mean request time | 95th-percentile time | Router cost / 10k requests | +|---|---:|---:|---:| +| Unchanged Laya, resident H100 | 80 ms | 243 ms | $1.08 busy-resource estimate | +| Adapted Laya, resident H100 | 78 ms | 235 ms | $1.05 busy-resource estimate | +| Adapted Laya, 8-core CPU / 32 GB | 3.98 s | 13.70 s | $6.99 busy-resource estimate | +| SIE Cloud Qwen encoder | 204 ms | 226 ms | $0.81 validated singleton-fee projection | +| Resident embedding classifier | 0.049 ms | 0.065 ms | Application hosting extra | +| Classifier + text cost predictors | 2.40 ms | 4.00 ms | Application hosting extra | + +Each model measurement uses the same 90 development questions after warmup. The Cloud encoder plus classifier averages approximately **204 ms**, or **207 ms** with the text cost predictors; these sums combine separately measured means, not an end-to-end latency benchmark. The encoder's longest observed request affects the mean more than its 95th percentile. + +The Cloud encoder measurement includes network time and saving the raw response to disk; native model timings are resident-model measurements on the stated Modal hardware. The $0.81 fee projects the final-test question mix; the native busy-resource estimates use the 90-question development timing sample. Model loading and CLI startup are separate. These are sequential single-request measurements, not a throughput benchmark. The learned head adds very little work; deploying the 4B encoder is the larger serving requirement. + +The default money comparison uses selected answer token costs. The explorer can add per-request Cloud encoder fees, projected from the ratebook and validated against actual batch and singleton calls. Native hosting, application hosting, idle GPU capacity and one-time training are separate: unpriced costs are never entered as zero. The optional Modal cost estimates price busy resource time only; they are not a production hosting bill. Token prices and workloads can change, so the pinned prices and exact usage are included with the data. + +## What the results establish + +The final curves replay policies chosen on calibration. Their 95% uncertainty intervals resample identified question families together within each task; they are pointwise intervals, not a guarantee over every possible threshold. The illustrated embedding + cost policy's 0.9-point accuracy loss has a 95% interval from −0.3 to 2.1 points: it does not establish a guaranteed loss below one point. The test covers new questions from this 23-task mix, not unseen kinds of reasoning. Public benchmark questions may also have appeared in model pretraining. + +Answer scoring preserves meaningful punctuation and normalizes only source-validated equivalent forms. The primary test includes all 975 reserved questions. A supplementary 973-question check excludes two source questions whose answer formats were flagged as ambiguous before final answers were collected; it keeps every routing policy unchanged. + +The example is supervised by observed outcomes, not by an LLM guessing which model is better. When reference answers are unavailable, a teacher can judge candidate answers, but those are noisy labels and need a separately checked evaluation set. [Labels and new answer collection](LABELS.md) explains that distinction and how to collect another paired run. + +All code lives in this example; datasets, saved weights, answer recordings and measurements live in the pinned public Hugging Face release. The `source/`, `native/`, `embedding/` and `evaluation/` folders are parts of this one workflow. `frozen-source-manifest.json` identifies the exact calculation code used for the published results. diff --git a/examples/train-a-router/calibration_view.py b/examples/train-a-router/calibration_view.py new file mode 100644 index 000000000..72a00fd2a --- /dev/null +++ b/examples/train-a-router/calibration_view.py @@ -0,0 +1,88 @@ +"""Inspect calibration policies before opening any final-test results.""" + +import html +import json +from pathlib import Path + +from embedding import recipes as core + +LABELS = { + "base": "Laya", "lora": "Laya + LoRA", "knn": "Embedding neighbors", + "embedding": "Embedding classifier", "text_length": "Text + length control", + "text_topic": "Predicted task control", "task_metadata": "Known task (metadata-assisted control)", +} + + +def label(name): + key = name.removesuffix("__textcost") + return LABELS.get(key, key) + (" · predicted cost" if name.endswith("__textcost") else " · average cost") + + +def render(calibration_path, output, *, reproduction=False): + document = core.read(calibration_path) + core.require(document["format"] == "bbeh-calibration-frontiers-v1" + and document["final_outcomes_accessed"] is False, "Expected calibration-only results") + core.require(not Path(output).exists(), "Preserve previous frontier views") + methods = [] + for name, method in document["methods"].items(): + core.require(method["no_preferred_quality_target"] is True, "Expected the full frontier") + candidates = {row["candidate_id"]: row for row in method["candidates"]} + rows = [] + selected = {row["candidate_id"]: row for row in method["frontier"]} + selected.update({cid: row for cid, row in candidates.items() if row["policy"]["mode"].startswith("always_")}) + for row in sorted(selected.values(), key=lambda r: r["calibration"]["answer_cost_usd_per10000"]): + core.require(candidates[row["candidate_id"]] == row, "Frontier policy changed") + value = row["calibration"] + rows.append({"id": row["candidate_id"], "policy": row["policy"], + "accuracy": value["answer_accuracy_percent"], "gap": value["gap_vs_strong_pp"], + "answer": value["answer_cost_usd_per10000"], "total": value["total_cost_usd_per10000"], + "share": 1 - value["cheap_share"], "baseline": row["policy"]["mode"].startswith("always_")}) + methods.append({"name": name, "label": label(name), "assisted": method["metadata_assisted"], "points": rows}) + data = json.dumps(methods, ensure_ascii=False, allow_nan=False).replace("<", "\\u003c") + mode = "Published-data reproduction. " if reproduction else "" + text = TEMPLATE.replace("__DATA__", data).replace("__COUNT__", str(len(document["ids"]))) + text = text.replace("__MODE__", html.escape(mode)).replace("__SHA__", core.sha(calibration_path)) + Path(output).write_text(text, encoding="utf-8") + return {"open": str(output), "calibration_questions": len(document["ids"]), "final_answers_read": False} + + +TEMPLATE = r""" + + +Choose a routing policy + +

Choose a routing policy

+

__MODE__Compare cost and answer accuracy on __COUNT__ calibration questions. This page never reads final-test answers. Choose the tradeoff that fits your application, then evaluate the same policy on the test set.

+
+
+

+

Select a point or table row to copy its method and candidate ID.


+
CandidateCost / 10,000AccuracyAccuracy lossSent to Sol
+

The blue curve contains policies for which no other calibration policy is both cheaper and more accurate. The dashed line is the expected result of choosing Luna or Sol at random, at different spending levels. A negative accuracy loss means this router beat always-Sol on these questions. A one-point loss means, for example, 80% accuracy becomes 79%. Training, idle capacity and application hosting are separate costs.

+
Source

Calibration artifact SHA-256: __SHA__

+""" diff --git a/examples/train-a-router/collect.py b/examples/train-a-router/collect.py new file mode 100644 index 000000000..7447334e2 --- /dev/null +++ b/examples/train-a-router/collect.py @@ -0,0 +1,73 @@ +"""BBEH-specific OpenAI request adapter; no reference answers or SIE credentials.""" + +import json +from decimal import Decimal + +ARMS = {"luna": {"provider": "openai", "model": "gpt-6-luna"}, + "sol": {"provider": "openai", "model": "gpt-6.1-sol"}} +PRICES = { + "gpt-6.1-sol": {"input": "2", "cached_input": "0.10", "cache_write": "2.50", "output": "10"}, + "gpt-6-luna": {"input": "0.10", "cached_input": "0.01", "cache_write": "0.125", "output": "0.50"}, +} +SYSTEM = ( + "Solve the question independently, using the full rules and options supplied. " + "Do not use tools or external sources. Return the requested JSON object. " + "The answer field must contain only the complete final answer in the format " + "requested by the source question. Preserve semantic brackets, quotes, " + "punctuation, nesting, case and internal spaces. JSON is only the transport " + "container; it does not replace or remove any part of the requested answer. " +) +OPENAI_SETTINGS = {"reasoning": {"effort": "medium"}, "max_output_tokens": 16384, + "store": False, "service_tier": "default"} + + +def body_for(item, arm): + if set(item) != {"id", "request", "response_contract"}: + raise ValueError("Only question ID, request and public answer contract may reach collector") + if not isinstance(item["request"], str) or not item["request"].strip(): + raise ValueError("Empty question") + if item["response_contract"] != {"type": "exact_string", "max_length": 2048}: + raise ValueError("BBEH uses one prospective complete-string answer contract") + return {"model": ARMS[arm]["model"], "instructions": SYSTEM, + "input": item["request"] + "\n\nFinal-answer type: exact_string.", + **OPENAI_SETTINGS, "text": {"format": {"type": "json_schema", "name": "answer", "strict": True, + "schema": {"type": "object", "properties": {"answer": {"type": "string"}}, + "required": ["answer"], "additionalProperties": False}}}} + + +def parse_answer(content, contract): + try: + value = json.loads(content) + except (ValueError, TypeError): + return None + if not isinstance(value, dict) or set(value) != {"answer"} or not isinstance(value["answer"], str): + return None + answer = value["answer"].strip() + return answer if answer and len(answer) <= contract["max_length"] else None + + +def openai_cost(usage, body): + incoming, outgoing = usage.get("input_tokens"), usage.get("output_tokens") + details = usage.get("input_tokens_details") or {} + cached, written = details.get("cached_tokens", 0), details.get("cache_write_tokens", 0) + reasoning = (usage.get("output_tokens_details") or {}).get("reasoning_tokens", 0) + values = (incoming, outgoing, cached, written, reasoning) + if any(type(value) is not int or value < 0 for value in values): + raise ValueError("Invalid token usage") + if cached + written > incoming or reasoning > outgoing or outgoing > body["max_output_tokens"]: + raise ValueError("Inconsistent usage") + p = {key: Decimal(value) for key, value in PRICES[body["model"]].items()} + cost = ((incoming - cached - written) * p["input"] + cached * p["cached_input"] + + written * p["cache_write"] + outgoing * p["output"]) / 1_000_000 + budget = ((incoming - cached) * p["cache_write"] + cached * p["cached_input"] + + outgoing * p["output"]) / 1_000_000 + return {"usd_estimate": str(cost), "budget_usd": str(budget), + "basis": "token-accounted standard list prices; output includes reasoning", + "cache_write_detail_present": "cache_write_tokens" in details} + + +def reservation(body, arm): + p = PRICES[body["model"]] + input_bound = len(json.dumps(body, ensure_ascii=False).encode()) + 2048 + return (Decimal(input_bound) * Decimal(p["cache_write"]) + + Decimal(body["max_output_tokens"]) * Decimal(p["output"])) / 1_000_000 diff --git a/examples/train-a-router/collect_answers.py b/examples/train-a-router/collect_answers.py new file mode 100644 index 000000000..65e2f1402 --- /dev/null +++ b/examples/train-a-router/collect_answers.py @@ -0,0 +1,123 @@ +"""Collect a new paired reproduction with the unchanged question-only recipe.""" + +import argparse +from decimal import Decimal +import hashlib +import json +import math +import os +from pathlib import Path + +import collect +import openai_population as acquisition +from reader_io import durable_writes, exclusive_run, read, require, sha + + +PHASES = {"learn": ("train", "development"), "calibration": ("calibration",), "final": ("final",)} + + +def checked_final(path, inputs, source_hash, pair_hash): + require(path is not None, "Final acquisition requires the raw frozen policy bundle") + frozen = read(path) + require( + frozen["format"] == "bbeh-frozen-frontiers-v1" + and frozen["source_plan_sha256"] == source_hash + and frozen["pair_sha256"] == pair_hash + and frozen["ids"] == [row["id"] for row in inputs] + and frozen["final_outcomes_accessed"] is False + and frozen["frozen_before_final_outcomes"] is True, + "Frozen policies do not match this final population and pair", + ) + methods = frozen["methods"] + require({"base", "lora"} <= set(methods), "Freeze both native variants before final collection") + kinds = [method["scores"]["router_kind"] for method in methods.values()] + require(kinds.count("embedding") >= 2 and "cpu" in kinds, "Freeze embedding methods and controls") + for method in methods.values(): + scores = method["scores"] + require(method["candidates"] and method["calibration_frontier_candidate_ids"], "Missing frozen policies") + for name in ("probabilities", "expected_usd"): + values = scores[name] + require(len(values) == len(inputs), "Missing final input scores") + require(all(isinstance(row, list) and len(row) == 2 for row in values), "Invalid paired scores") + require(all(type(v) in (int, float) and math.isfinite(v) and v >= 0 + and (name != "probabilities" or v <= 1) for row in values for v in row), "Invalid score value") + return {"sha256": sha(path), "methods": sorted(methods), "n": len(inputs)} + + +def prepare(source, pair_path, phase, budget, dispatch_seconds, frozen=None): + source, pair_path = Path(source), Path(pair_path) + cap = Decimal(str(budget)) + require(cap.is_finite() and cap > 0, "Budget must be a positive finite USD amount") + require(type(dispatch_seconds) is int and dispatch_seconds > 0, "Invalid dispatch deadline") + pair = read(pair_path) + require(pair["models"] == {"cheap": collect.ARMS["luna"]["model"], "strong": collect.ARMS["sol"]["model"]} + and pair["settings"] == collect.OPENAI_SETTINGS and pair["system_prompt"] == collect.SYSTEM + and pair["prices_usd_per_million"] == collect.PRICES, "Pair settings differ from the collection recipe") + source_plan = read(source / "source-plan.json") + require(sha(source / "meta.json") == source_plan["files_sha256"]["meta.json"], "Source metadata changed") + metadata = read(source / "meta.json") + by_id = {row["id"]: row for row in metadata} + require(len(by_id) == len(metadata), "Duplicate source ID") + items, files = {}, {} + for role in PHASES[phase]: + path = source / f"question-only/{role}-inputs.json" + require(sha(path) == source_plan["files_sha256"][f"question-only/{role}-inputs.json"], "Question inputs changed") + rows = read(path) + expected = [row["id"] for row in metadata if row["retained"] and row["role"] == role] + require(rows and [row["id"] for row in rows] == expected, "Role population/order changed") + for row in rows: + require(hashlib.sha256(row["request"].encode()).hexdigest() == by_id[row["id"]]["request_sha256"], "Request changed") + for arm in collect.ARMS: + collect.body_for(row, arm) # Rejects references and every extra payload field. + items[role], files[f"{role}-inputs.json"] = rows, sha(path) + here = Path(__file__).parent + files.update({name: sha(here / name) for name in ("collect.py", "openai_population.py", "collect_answers.py", "reader_io.py")}) + plan = { + "format": "bbeh-public-acquisition-v1", "reproduction": True, + "phase": phase, "roles": list(items), "counts": {role: len(rows) for role, rows in items.items()}, + "source_plan_sha256": sha(source / "source-plan.json"), "pair_sha256": sha(pair_path), + "files_sha256": files, "budget_usd": str(cap), "dispatch_seconds": dispatch_seconds, + "workers_per_arm": acquisition.WORKERS_PER_ARM, "max_physical_attempts": acquisition.MAX_ATTEMPTS, + "selection": "first terminal provider response; no content/format/quality retries", + "budget_scope": "All known physical-attempt charges plus unresolved execution reservations and in-flight reservations", + "collector_inputs_question_only": True, "references_loaded_by_collector": False, + "final_gate": None, + } + if phase == "final": + plan["final_gate"] = checked_final(frozen, items["final"], plan["source_plan_sha256"], plan["pair_sha256"]) + else: + require(frozen is None, "A final gate is only applicable to final collection") + return items, plan + + +def execute(items, plan, output, *, call=None): + output = Path(output).resolve() + require(call is not None or bool(os.environ.get("OPENAI_API_KEY")), "Set OPENAI_API_KEY for the OpenAI collector") + with exclusive_run(output), durable_writes(acquisition, output) as commit: + return acquisition.run(items, output, plan, commit, call=call or acquisition.acquire, + dispatch_seconds=plan["dispatch_seconds"]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--pair", type=Path, default=Path(__file__).with_name("pair.json")) + parser.add_argument("--phase", choices=PHASES, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--budget-usd", required=True) + parser.add_argument("--dispatch-seconds", type=int, default=20900) + parser.add_argument("--frozen", type=Path) + parser.add_argument("--execute", action="store_true", help="Dispatch paid calls; otherwise print the immutable plan") + args = parser.parse_args() + items, plan = prepare(args.source, args.pair, args.phase, args.budget_usd, args.dispatch_seconds, args.frozen) + if not args.execute: + print(json.dumps(plan, indent=2, sort_keys=True)) + return + result = execute(items, plan, args.output) + print(json.dumps({key: result[key] for key in ( + "status", "physical_attempts", "selected_responses", "known_list_usd", "unknown_reservation_usd", "stop" + )}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/cost_reader.py b/examples/train-a-router/cost_reader.py new file mode 100644 index 000000000..5b63f3367 --- /dev/null +++ b/examples/train-a-router/cost_reader.py @@ -0,0 +1,60 @@ +"""Repeat the selected cost recipe without binding new fits to original timing bytes.""" + +from pathlib import Path +import time + +import numpy as np + +from embedding import recipes as core +from embedding.cost_iteration import refit, study + + +def fit_score(run, train_path, development_path, output): + run, output = Path(run), Path(output) + core.require(not output.exists(), "Preserve the previous cost fit") + plan = core.checked_plan(run) + quality = core.read(run / "model.json") + core.require(quality["plan_sha256"] == core.sha(run / "plan.json") + and quality["train_grades_sha256"] == core.sha(train_path) + and quality["development_grades_sha256"] == core.sha(development_path) + and quality["calibration_or_final_outcomes_accessed"] is False, "Quality fit/input lineage changed") + rows, dollars = [], [] + families = set() + for role, path in (("train", train_path), ("development", development_path)): + selected = core.role_rows(run, plan, role) + current = {row["family"] for row in selected} + core.require(not families.intersection(current), "Training/development families overlap") + families.update(current) + _, usd, _ = core.grades(plan, selected, path, role) + rows.extend(selected) + dollars.append(usd) + promotion = Path(refit.__file__).with_name("promotion.json") + core.require(core.sha(promotion) == refit.PROMOTION_SHA, "Published cost-recipe decision changed") + alpha = core.read(promotion)["alpha"] + started = time.perf_counter() + _, matrix, text = core.text_matrix(rows) + model = {"format": "bbeh-shared-input-dollar-model-v1", "fit_role": "train+development", + "text_features": text, "models": {"text_length_log_ridge": study.fit_ridge(matrix, np.concatenate(dollars), alpha)}, + "source_plan_sha256": plan["source_plan_sha256"], "pair_sha256": plan["pair_sha256"], + "promotion_sha256": refit.PROMOTION_SHA, "protocol_sha256": study.PROTOCOL_SHA, + "train_grades_sha256": core.sha(train_path), "development_grades_sha256": core.sha(development_path), + "refit_ids": [row["id"] for row in rows], "quality_model_sha256": core.sha(run / "model.json"), + "study_source_sha256": core.sha(study.__file__), "adapter_sha256": core.sha(__file__), + "calibration_or_final_outcomes_accessed": False, "fit_seconds": time.perf_counter() - started, + "reproduction": True, "new_blind_evaluation": False} + output.mkdir(parents=True) + core.write(output / "cost-model.json", model) + for role in ("calibration", "final"): + held = core.role_rows(run, plan, role) + expected = study.predict_rows(model, held, "text_length_log_ridge") + prediction = {"format": "bbeh-shared-input-cost-scores-v1", "role": role, + "ids": [row["id"] for row in held], "arms": ["cheap", "strong"], + "source_plan_sha256": plan["source_plan_sha256"], "pair_sha256": plan["pair_sha256"], + "promotion_sha256": refit.PROMOTION_SHA, "cost_model_sha256": core.sha(output / "cost-model.json"), + "expected_usd": expected.tolist(), "outcomes_accessed": False} + original = core.read(run / f"scores-{role}.json") + core.require(original["model_sha256"] == core.sha(run / "model.json"), "Quality scores use another fit") + core.write(output / f"costs-{role}.json", prediction) + refit.augment_files(run / f"scores-{role}.json", output / f"costs-{role}.json", + output / f"scores-{role}.json", output / "combined-model.json") + return {"alpha": alpha, "fit_n": len(rows), "model_sha256": core.sha(output / "cost-model.json")} diff --git a/examples/train-a-router/embedding/__init__.py b/examples/train-a-router/embedding/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/train-a-router/embedding/collect_features.py b/examples/train-a-router/embedding/collect_features.py new file mode 100644 index 000000000..cdc52a626 --- /dev/null +++ b/examples/train-a-router/embedding/collect_features.py @@ -0,0 +1,323 @@ +"""Question-only feature acquisition on SSD with compact durable checkpoints.""" + +from __future__ import annotations + +from decimal import Decimal +import hashlib +from pathlib import Path +import time + +import numpy as np + +from embedding import recipes as core, token_audit, wire + +RATE = {"rate_numerator": 3, "rate_denominator": 500, "unit": "input_tokens"} + + +def credits(tokens): + return (tokens * 3 + 499) // 500 + + +def source_rows(source, source_sha): + source = Path(source) + core.require( + core.sha(source / "source-plan.json") == source_sha, "Frozen source changed" + ) + plan = core.read(source / "source-plan.json") + metadata = source / "meta.json" + core.require( + core.sha(metadata) == plan["files_sha256"]["meta.json"], + "Source metadata changed", + ) + by_id = {row["id"]: row for row in core.read(metadata)} + rows, role_inputs = [], {} + families = {} + for role in core.ROLES: + relative = f"question-only/{role}-inputs.json" + path = source / relative + core.require( + core.sha(path) == plan["files_sha256"][relative], "Input hash changed" + ) + role_inputs[role] = core.sha(path) + for item in core.read(path): + core.require( + set(item) == {"id", "request", "response_contract"}, + "Question-only payload contains extra fields", + ) + meta = by_id[item["id"]] + core.require( + meta["retained"] is True and meta["role"] == role and item["request"], + "Source eligibility mismatch", + ) + core.require( + families.setdefault(meta["family"], role) == role, + "Family crosses source roles", + ) + rows.append({**item, "role": role, "task": meta["task"]}) + core.require( + len(rows) + == len({r["id"] for r in rows}) + == sum(r["retained"] for r in by_id.values()), + "Active source population mismatch", + ) + return rows, role_inputs + + +def singleton_rows(rows): + development = [row for row in rows if row["role"] == "development"] + order = sorted( + development, + key=lambda row: hashlib.sha256( + ("bbeh-native-timing89:" + row["id"]).encode() + ).hexdigest(), + ) + core.require(len(order) >= 2, "Need a separate warmup and timed request") + count = min(90, len(order) - 1) + return [order[count], *order[:count]] + + +class TimedClient: + def __init__(self, client): + self.client, self.timings = client, [] + + def __getattr__(self, name): + return getattr(self.client, name) + + def encode(self, model, items, **kwargs): + start = time.perf_counter() + result = self.client.encode(model, items, **kwargs) + self.timings.append( + { + "ids": [item["id"] for item in items], + "seconds": time.perf_counter() - start, + } + ) + return result + + +def check_tokens(root, counts): + collection = core.read(Path(root) / "collection.json") + core.require( + collection["status"] == "complete" and not collection["unknown_charge_batches"], + "Complete settled features required", + ) + pricebook = None + for binding in collection["batch_receipts"]: + path = Path(root) / binding["path"] + core.require(core.sha(path) == binding["sha256"], "Batch binding changed") + receipt, intent = core.read(path), core.read(path.parent / "intent.json") + quote = intent["quote"] + core.require( + quote["applied_rates"] == [RATE] + and quote["rounding_rule"] == "ceil-once-per-terminal-event" + and quote["minimum_billed_units"] is None, + "Encoder pricing basis changed", + ) + binding = {key: quote[key] for key in ("rate_book_version", "rate_book_sha256")} + core.require( + pricebook is None or pricebook == binding, + "Pricebook changed across batches", + ) + pricebook = binding + tokens = sum(counts[item] for item in receipt["ids"]) + core.require( + receipt["usage"]["input_tokens"] == tokens + and Decimal(receipt["credits_decimal"]) == credits(tokens), + "Exact token/fee projection differs from observed charge", + ) + return pricebook + + +def run( + source, + output, + plan, + tokenizer, + tokenizer_binding, + checkpoint, + *, + client_factory=wire.make_client, +): + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + source = Path(source) + rows, input_hashes = source_rows(source, plan["source_plan_sha256"]) + core.require(len(rows) == plan["n"], "Launch/source count changed") + counts = {} + for role in core.ROLES: + selected = [row for row in rows if row["role"] == role] + lengths = token_audit.audit_rows(selected, tokenizer) + audit = { + "format": "bbeh-encoder-token-audit-v1", + "role": role, + "model": core.MODEL, + "hf_revision": core.REVISION, + "instruction": core.INSTRUCTION, + "template": token_audit.TEMPLATE, + "max_length": token_audit.MAX_LENGTH, + "truncation": False, + "source_plan_sha256": plan["source_plan_sha256"], + "inputs_sha256": input_hashes[role], + "tokenizer": tokenizer_binding, + "rows": lengths, + "truncated_ids": [], + "overflow_ids": [ + r["id"] for r in lengths if r["input_tokens"] > token_audit.MAX_LENGTH + ], + } + core.write(output / "features" / role / "token-audit.json", audit) + core.require( + not audit["overflow_ids"], + "Encoder context overflow; stop before model calls, never truncate", + ) + counts.update({row["id"]: row["input_tokens"] for row in lengths}) + core.write(output / "plan.json", plan) + core.write( + output / "question-inputs.json", + [ + {k: r[k] for k in ("id", "request", "response_contract", "role", "task")} + for r in rows + ], + ) + core.write(output / "tokenizer.json", tokenizer_binding) + checkpoint(output) + common = { + "instruction": core.INSTRUCTION, + "model": core.MODEL, + "hf_revision": core.REVISION, + "dimensions": 2560, + "max_seconds": plan["collection_seconds"], + "budget_usd": plan["budget_usd"], + } + capture = wire.WireCapture() + with client_factory(capture) as client: + batch = wire.collect_batches( + rows, + output / "batches", + {**common, "n": len(rows), "batch_size": 16}, + client, + capture, + commit=lambda: checkpoint(output), + ) + core.require( + batch["status"] == "complete", + "Batch collection stopped; retained intents must never be automatically repeated", + ) + ids, matrix, _ = wire.load_vectors(output / "batches", rows) + pricebook = check_tokens(output / "batches", counts) + for role in core.ROLES: + indices = [i for i, row in enumerate(rows) if row["role"] == role] + directory = output / "features" / role + with (directory / "vectors.npy").open("xb") as stream: + np.save(stream, matrix[indices], allow_pickle=False) + core.write(directory / "ids.json", [ids[i] for i in indices]) + core.write( + directory / "features.json", + { + "format": "routerbench-fixed-embeddings-v1", + "model": core.MODEL, + "hf_revision": core.REVISION, + "instruction": core.INSTRUCTION, + "dimensions": 2560, + "n": len(indices), + "inputs_sha256": input_hashes[role], + "source_plan_sha256": plan["source_plan_sha256"], + "ids_sha256": core.sha(directory / "ids.json"), + "vectors_sha256": core.sha(directory / "vectors.npy"), + "token_audit_sha256": core.sha(directory / "token-audit.json"), + "collection_sha256": core.sha(output / "batches/collection.json"), + "dtype": "float32", + "normalization": "float64 L2 then float32", + "outcomes_accessed": False, + }, + ) + checkpoint(output) + sample = singleton_rows(rows) + core.write( + output / "singleton-sample.json", + { + "warmup_id": sample[0]["id"], + "timed_ids": [r["id"] for r in sample[1:]], + "selection": "First90 development IDs by SHA256('bbeh-native-timing89:'+id); nextID is separate warmup executed first; all if fewer than91", + }, + ) + remaining = Decimal(plan["budget_usd"]) - Decimal( + batch["known_usd_at_declared_credit_price"] + ) + timed = TimedClient(client) + singles = wire.collect_batches( + sample, + output / "singletons", + {**common, "n": len(sample), "batch_size": 1, "budget_usd": str(remaining)}, + timed, + capture, + commit=lambda: checkpoint(output), + ) + core.require( + singles["status"] == "complete", + "Singleton validation incomplete; preserve all completed features", + ) + core.require( + check_tokens(output / "singletons", counts) == pricebook, + "Singleton pricebook differs from batches", + ) + durations = [row["seconds"] for row in timed.timings[1:]] + core.write( + output / "singleton-timing.json", + { + "model": core.MODEL, + "hf_revision": core.REVISION, + "sample_sha256": core.sha(output / "singleton-sample.json"), + "timings": timed.timings, + "mean_ms": float(np.mean(durations) * 1000), + "p50_ms": float(np.median(durations) * 1000), + "median_ms": float(np.median(durations) * 1000), + "p95_ms": float(np.quantile(durations, 0.95) * 1000), + "scope": "One SDK encode request, including synchronous raw-response SSD persistence; excludes quote/catalog/Volume commits and downstream head scoring. One warmup excluded.", + "batch_scoring_or_solver_calls": False, + }, + ) + projection = { + "format": "bbeh-singleton-embedding-fees-v1", + "source_plan_sha256": plan["source_plan_sha256"], + "model": core.MODEL, + "hf_revision": core.REVISION, + "instruction": core.INSTRUCTION, + "cost_basis": "validated_singleton_projection", + "pricebook": pricebook, + "batch_collection_sha256": core.sha(output / "batches/collection.json"), + "singleton_collection_sha256": core.sha(output / "singletons/collection.json"), + "all_observed_batch_and_singleton_tokens_and_charges_reproduced": True, + "rows": [ + { + "id": row["id"], + "role": row["role"], + "input_tokens": counts[row["id"]], + "projected_singleton_credits": credits(counts[row["id"]]), + "projected_singleton_usd": str( + wire.CREDIT_USD * credits(counts[row["id"]]) + ), + } + for row in rows + ], + } + core.write(output / "fee-projection.json", projection) + total = Decimal(batch["known_usd_at_declared_credit_price"]) + Decimal( + singles["known_usd_at_declared_credit_price"] + ) + terminal = { + "status": "complete", + "n": len(rows), + "known_api_usd": str(total), + "unknown_charge_batches": [], + "batch_rows": len(rows), + "singleton_requests_including_warmup": len(sample), + "fee_projection_sha256": core.sha(output / "fee-projection.json"), + "features": { + role: core.sha(output / "features" / role / "features.json") + for role in core.ROLES + }, + } + core.write(output / "terminal.json", terminal) + checkpoint(output) + return terminal diff --git a/examples/train-a-router/embedding/cost_iteration/__init__.py b/examples/train-a-router/embedding/cost_iteration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/train-a-router/embedding/cost_iteration/promotion.json b/examples/train-a-router/embedding/cost_iteration/promotion.json new file mode 100644 index 000000000..42885ab3b --- /dev/null +++ b/examples/train-a-router/embedding/cost_iteration/promotion.json @@ -0,0 +1,35 @@ +{ + "alpha": 0.1, + "baseline_artifacts_unchanged": true, + "calibration_or_final_outcomes_accessed": false, + "chosen_model": "text_length_log_ridge", + "cost_protocol_sha256": "be1a9641d79474b70e7013c7ffaf9a390c66627ca98b6c5cf46adee46a15f872", + "decision": "Retain original cost estimates plus one shared text+length cost option for every frozen quality method; no mean/length-only primary variants.", + "development_summary_sha256": "4d0552340d1238a0552142676e5e8af7ae0337b5893f1f4e82cbbbb18239046b", + "development_terminal_sha256": "db8ef97a6c29373ab320b1b9165b57ce2be08eaa38755f30eaff6c6029504d2c", + "fees": "Encoder singleton fees and CPU/native hosting remain separately accounted; cost model is not assumed free.", + "format": "bbeh-shared-cost-promotion-v1", + "input_only_score_roles": [ + "calibration", + "final" + ], + "method_naming": "Original method unchanged; added __textcost, with cost and quality identities separate.", + "native_extension": [ + "base", + "lora" + ], + "quality_heads_refitted": false, + "quality_methods": [ + "knn", + "embedding", + "text_length", + "text_topic", + "task_metadata" + ], + "refit": "Fresh TRAIN+development TFIDF vocabulary/IDF/length scaling; same fixed log-Ridge objective, tolerances and residual smearing, no new selection.", + "refit_roles": [ + "train", + "development" + ], + "selection_basis": "Declared development normalized dollar MSE selects alpha0.1. Development frontier shows middle-budget gains and high-budget tradeoffs; no claim of universal dominance." +} diff --git a/examples/train-a-router/embedding/cost_iteration/protocol.json b/examples/train-a-router/embedding/cost_iteration/protocol.json new file mode 100644 index 000000000..cb6b864c9 --- /dev/null +++ b/examples/train-a-router/embedding/cost_iteration/protocol.json @@ -0,0 +1,24 @@ +{ + "format": "bbeh-shared-input-cost-iteration-v1", + "purpose": "Test whether a shared input-only dollar predictor improves routing beyond method-specific global conditional cost means; preserve all original quality heads and baseline costs.", + "source_plan_sha256": "b15e9c2d9db284fe170e0460550f02fc2cb48f2b9234b697d181716264984a53", + "quality_model_sha256": "2db976ac0db140eae7b608716ccd9ece372f2bcf4dc4b12a55f7ff31a1e0221b", + "train_grades_sha256": "fc97681161956f9743a4e25fa1133c4a6683f47b8f1cda708d39df1c939531bf", + "development_grades_sha256": "7d8b1e289bde88fc4623088a0432d35ccdb80169829f19bdffb9abc18697b827", + "roles": {"fit": "train", "selection_and_diagnostic": "development", "calibration": "not opened", "final": "not opened"}, + "features": "The existing recipes.text_matrix TRAIN-only word TF-IDF unigrams/bigrams, min_df2, max_features20000, plus the existing five TRAIN-scaled request length/structure features. No explicit task metadata, embeddings or output tokens at prediction time.", + "models": { + "text_length_log_ridge": "Two independent Ridge models predict natural log actual selected-answer USD, with a shared alpha. Intercept is unpenalized; sklearn solver lsqr, tol1e-8,max_iter10000. Positive dollar prediction is exp(predicted log USD) times the TRAIN mean exp(log USD minus TRAIN prediction), separately per arm.", + "length_log_ridge": "Same model and alpha grid using only the existing five TRAIN-scaled length/structure features.", + "mean": "Each arm's scalar TRAIN mean actual USD. No tuning." + }, + "alphas": [0.1, 1.0, 10.0, 100.0], + "selection": "For each ridge feature set choose one shared alpha minimizing mean squared dollar error across both arms and development rows, after each arm's dollar error is divided by that arm's TRAIN mean USD. Exact ties choose larger alpha. No quality-based alpha selection.", + "cost_domain": "All selected costs must be finite and strictly positive. No zero or missing-cost imputation; failures stop the diagnostic.", + "quality": "Use only the already selected k32/embedding-lambda1 and fixed text/task controls' stored TRAIN-fitted development probabilities. No new correctness fit or quality hyperparameter choice. Native can consume the same cost predictions later without changing its correctness probabilities.", + "comparison": "For every fixed quality method retain original cost predictions and each of the three common cost controls. Retain complete signed-utility development frontiers and descriptive best accuracy under 20/40/60/80 percent of always-Sol answer cost, with analytic random comparators. No calibration/final policy selection here.", + "uncertainty": "Pointwise paired whole-family bootstrap within task,2000draws,seed89,fixed observed task weights. These development-selected curves do not provide unbiased or simultaneous guarantees.", + "fees": "Answer-model costs only until singleton fee validation. All CPU/encoder hosting fees remain unpriced, not free.", + "refit": "This initial study does not refit on development. If selected after review, cost-only train+development refit and input-only scoring will be separately bound; original archives remain immutable.", + "compute": "Modal CPU only; no model API, GPU, calibration or final outcomes. Fixed numpy2.4.6/scipy1.17.0/sklearn1.6.1 runtime." +} diff --git a/examples/train-a-router/embedding/cost_iteration/refit.py b/examples/train-a-router/embedding/cost_iteration/refit.py new file mode 100644 index 000000000..6098b722b --- /dev/null +++ b/examples/train-a-router/embedding/cost_iteration/refit.py @@ -0,0 +1,236 @@ +"""Promoted fixed dollar model: train+development refit, input-only future scoring.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +from pathlib import Path +import time + +import numpy as np + +from embedding import recipes as core +from embedding.cost_iteration import study +from evaluation import utility_router as utility + +PROMOTION_SHA = "13a500178d56ed1f08b17b8ad15be5f6a5f1644ea768e7aacf878e44519d48da" + + +def file_bytes(value): + return ( + json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + + "\n" + ).encode() + + +def augment(score, prediction): + """Return augmented scores + combined-model binding; works for native or CPU heads.""" + core.require( + score["format"] == "bbeh-input-only-scores-v1" + and prediction["format"] == "bbeh-shared-input-cost-scores-v1" + and score["role"] in {"calibration", "final"} + and score["outcomes_accessed"] is False + and prediction["outcomes_accessed"] is False, + "Input-only future scores required", + ) + for key in ("role", "ids", "source_plan_sha256", "pair_sha256"): + core.require( + score[key] == prediction[key], "Cost/quality source-role-order mismatch" + ) + core.require( + len(set(score["ids"])) == len(score["ids"]) > 0, "Duplicate/empty score IDs" + ) + for key in ("model_sha256",): + core.require_hash(score[key]) + core.require_hash(prediction["cost_model_sha256"]) + core.require( + prediction["promotion_sha256"] == PROMOTION_SHA, "Cost decision changed" + ) + expected = study.positive_costs(prediction["expected_usd"], len(score["ids"])) + binding = { + "format": "bbeh-quality-and-shared-cost-binding-v1", + "quality_model_sha256": score["model_sha256"], + "cost_model_sha256": prediction["cost_model_sha256"], + "source_plan_sha256": score["source_plan_sha256"], + "pair_sha256": score["pair_sha256"], + "promotion_sha256": PROMOTION_SHA, + "methods": sorted(score["methods"]), + "original_methods_preserved": True, + } + out = copy.deepcopy(score) + for name, value in score["methods"].items(): + core.require( + not name.endswith("__textcost") + and name + "__textcost" not in score["methods"], + "Scores already augmented or method-name collision", + ) + p = utility._matrix( + value["probabilities"], "quality probabilities", probability=True + ) + core.require(len(p) == len(expected), "Quality probability population changed") + core.require( + value["router_kind"] in {"native", "embedding", "cpu"} + and type(value["metadata_assisted"]) is bool, + "Invalid quality method metadata", + ) + out["methods"][name + "__textcost"] = { + **copy.deepcopy(value), + "expected_usd": expected.tolist(), + "quality_method": name, + "cost_variant": "shared_text_length", + "cost_model_sha256": prediction["cost_model_sha256"], + "cost_prediction_inputs": "Question TFIDF and length/structure only", + "additional_cpu_cost_predictor_hosting": "unpriced; not assumed zero", + } + out.update( + { + "model_sha256": hashlib.sha256(file_bytes(binding)).hexdigest(), + "quality_model_sha256": score["model_sha256"], + "cost_model_sha256": prediction["cost_model_sha256"], + "promotion_sha256": PROMOTION_SHA, + "variant_scope": "Original quality and cost methods retained; added shared text-cost variants only", + } + ) + return out, binding + + +def augment_files(score_path, cost_path, output, binding_path): + score, binding = augment(core.read(score_path), core.read(cost_path)) + score["parent_scores_sha256"] = core.sha(score_path) + score["cost_scores_sha256"] = core.sha(cost_path) + if Path(binding_path).exists(): + core.require( + Path(binding_path).read_bytes() == file_bytes(binding), + "Combined model binding changed", + ) + else: + core.write(binding_path, binding) + core.require( + core.sha(binding_path) == score["model_sha256"], "Binding encoding differs" + ) + core.write(output, score) + return score + + +def fit_score(run, train_path, dev_path, scores_root, output, *, promotion_path=None): + run, scores_root, output = Path(run), Path(scores_root), Path(output) + promotion_path = Path(promotion_path or Path(__file__).with_name("promotion.json")) + core.require( + core.sha(promotion_path) == PROMOTION_SHA, "Prospective refit decision changed" + ) + core.require(not output.exists(), "Preserve cost refit") + promotion = core.read(promotion_path) + plan, quality, rows, grades = study.load_inputs( + run, train_path, dev_path, Path(__file__).with_name("protocol.json") + ) + # No fitting function receives calibration/final labels or their file paths. + fit_rows = rows["train"] + rows["development"] + fit_costs = np.concatenate((grades["train"][1], grades["development"][1])) + started = time.perf_counter() + _, matrix, text = core.text_matrix(fit_rows) + cost_model = { + "format": "bbeh-shared-input-dollar-model-v1", + "fit_role": "train+development", + "tuning_role": "development_previous_fixed_choice", + "refit_on_development": True, + "text_features": text, + "models": { + "text_length_log_ridge": study.fit_ridge( + matrix, fit_costs, promotion["alpha"] + ) + }, + "train_mean_usd": fit_costs.mean(axis=0).tolist(), + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "promotion_sha256": PROMOTION_SHA, + "protocol_sha256": study.PROTOCOL_SHA, + "train_grades_sha256": core.sha(train_path), + "development_grades_sha256": core.sha(dev_path), + "refit_ids": [row["id"] for row in fit_rows], + "fit_seconds": time.perf_counter() - started, + "quality_model_sha256": core.sha(run / "model.json"), + "study_source_sha256": core.sha(study.__file__), + "refit_source_sha256": core.sha(__file__), + "calibration_or_final_outcomes_accessed": False, + } + output.mkdir(parents=True) + core.write(output / "cost-model.json", cost_model) + results = {} + for role in ("calibration", "final"): + role_rows = core.role_rows(run, plan, role) + expected = study.predict_rows(cost_model, role_rows, "text_length_log_ridge") + prediction = { + "format": "bbeh-shared-input-cost-scores-v1", + "role": role, + "ids": [row["id"] for row in role_rows], + "arms": ["cheap", "strong"], + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "promotion_sha256": PROMOTION_SHA, + "cost_model_sha256": core.sha(output / "cost-model.json"), + "expected_usd": expected.tolist(), + "outcomes_accessed": False, + } + cost_path = output / f"costs-{role}.json" + core.write(cost_path, prediction) + old_score_path = scores_root / f"scores-{role}.json" + old = core.read(old_score_path) + core.require( + old["model_sha256"] == core.sha(run / "model.json") + and set(old["methods"]) == study.QUALITY_METHODS, + "Original quality score/model changed", + ) + combined = augment_files( + old_score_path, + cost_path, + output / f"scores-{role}.json", + output / "combined-model.json", + ) + for name in old["methods"]: + core.require( + combined["methods"][name] == old["methods"][name], + "Original method mutated", + ) + results[role] = { + "n": len(role_rows), + "cost_scores_sha256": core.sha(cost_path), + "scores_sha256": core.sha(output / f"scores-{role}.json"), + "parent_scores_sha256": core.sha(old_score_path), + "methods": list(combined["methods"]), + } + receipt = { + "format": "bbeh-shared-cost-refit-receipt-v1", + "promotion_sha256": PROMOTION_SHA, + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "cost_model_sha256": core.sha(output / "cost-model.json"), + "combined_model_sha256": core.sha(output / "combined-model.json"), + "roles": results, + "fit_n": len(fit_rows), + "alpha": promotion["alpha"], + "quality_selected": quality["selected"], + "quality_heads_refitted": False, + "calibration_or_final_outcomes_accessed": False, + } + core.write(output / "refit.json", receipt) + return receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser( + "augment", help="Add the same refit cost arrays to native or CPU quality scores" + ) + p.add_argument("--scores", required=True) + p.add_argument("--costs", required=True) + p.add_argument("--output", required=True) + p.add_argument("--binding", required=True) + args = parser.parse_args() + augment_files(args.scores, args.costs, args.output, args.binding) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/embedding/cost_iteration/study.py b/examples/train-a-router/embedding/cost_iteration/study.py new file mode 100644 index 000000000..ac445e4a6 --- /dev/null +++ b/examples/train-a-router/embedding/cost_iteration/study.py @@ -0,0 +1,438 @@ +"""Train-only shared dollar prediction; saved quality probabilities never change.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import time + +import numpy as np + +from embedding import development_budget, recipes as core +from evaluation import frontier, utility_router as utility + +PROTOCOL_SHA = "be1a9641d79474b70e7013c7ffaf9a390c66627ca98b6c5cf46adee46a15f872" +ALPHAS = (0.1, 1.0, 10.0, 100.0) +FEATURE_MODELS = ("length_log_ridge", "text_length_log_ridge") +QUALITY_METHODS = {"knn", "embedding", "text_length", "text_topic", "task_metadata"} + + +def positive_costs(values, n=None): + value = np.asarray(values, dtype=float) + core.require( + value.ndim == 2 + and value.shape[1] == 2 + and (n is None or len(value) == n) + and np.isfinite(value).all() + and (value > 0).all(), + "Expected complete finite strictly positive cheap/strong USD", + ) + return value + + +def error_metrics(actual, predicted, train_means): + actual, predicted = positive_costs(actual), positive_costs(predicted) + means = np.asarray(train_means, dtype=float) + core.require( + actual.shape == predicted.shape + and means.shape == (2,) + and np.isfinite(means).all() + and (means > 0).all(), + "Cost metric alignment", + ) + residual = predicted - actual + normalized = np.mean((residual / means) ** 2, axis=0) + return { + "mean_normalized_squared_dollar_error": float(normalized.mean()), + "per_arm_normalized_squared_dollar_error": normalized.tolist(), + "per_arm_mae_usd": np.mean(abs(residual), axis=0).tolist(), + "per_arm_bias_usd": residual.mean(axis=0).tolist(), + "per_arm_actual_mean_usd": actual.mean(axis=0).tolist(), + "per_arm_predicted_mean_usd": predicted.mean(axis=0).tolist(), + "incremental_cost_rmse_usd": float( + np.sqrt(np.mean((residual[:, 1] - residual[:, 0]) ** 2)) + ), + "train_mean_usd_normalizer": means.tolist(), + } + + +def fit_ridge(matrix, costs, alpha): + from sklearn.linear_model import Ridge + + costs = positive_costs(costs, matrix.shape[0]) + log_cost = np.log(costs) + model = Ridge( + alpha=alpha, fit_intercept=True, solver="lsqr", tol=1e-8, max_iter=10000 + ) + model.fit(matrix, log_cost) + core.require( + np.all(np.asarray(model.n_iter_) < 10000), "Ridge iteration limit reached" + ) + residual = log_cost - model.predict(matrix) + smearing = np.exp(residual).mean(axis=0) + core.require( + np.isfinite(smearing).all() and (smearing > 0).all(), "Nonfinite TRAIN smearing" + ) + saved = { + "alpha": alpha, + "weights": model.coef_.tolist(), + "bias": model.intercept_.tolist(), + "train_residual_smearing": smearing.tolist(), + "iterations": np.asarray(model.n_iter_).tolist(), + } + positive_costs(predict_ridge(matrix, saved), len(costs)) + return saved + + +def predict_ridge(matrix, saved): + log_cost = np.asarray(matrix @ np.asarray(saved["weights"]).T) + np.asarray( + saved["bias"] + ) + result = np.exp(log_cost) * np.asarray(saved["train_residual_smearing"]) + return positive_costs(result, matrix.shape[0]) + + +def select_candidate(candidates): + core.require(bool(candidates), "No cost candidates") + return min( + candidates, + key=lambda row: ( + row["development_error"]["mean_normalized_squared_dollar_error"], + -row["alpha"], + ), + ) + + +def fit_costs(train_rows, train_costs, dev_rows, dev_costs): + core.require( + all(r["role"] == "train" for r in train_rows) + and all(r["role"] == "development" for r in dev_rows), + "Wrong fit/tuning roles", + ) + core.require( + not ({r["family"] for r in train_rows} & {r["family"] for r in dev_rows}), + "Family crosses cost fitting and tuning", + ) + train_costs, dev_costs = ( + positive_costs(train_costs, len(train_rows)), + positive_costs(dev_costs, len(dev_rows)), + ) + started = time.perf_counter() + _, train_text, text = core.text_matrix(train_rows) + _, dev_text, _ = core.text_matrix(dev_rows, text) + scale = np.asarray(text["length_scale"]) + matrices = { + "length_log_ridge": ( + core.lengths(train_rows) / scale, + core.lengths(dev_rows) / scale, + ), + "text_length_log_ridge": (train_text, dev_text), + } + means = train_costs.mean(axis=0) + predictions = {"mean": np.tile(means, (len(dev_rows), 1))} + models = {"mean": {"train_mean_usd": means.tolist()}} + errors = {"mean": error_metrics(dev_costs, predictions["mean"], means)} + tuning = {} + for name, (train_matrix, dev_matrix) in matrices.items(): + candidates = [] + for alpha in ALPHAS: + before = time.perf_counter() + fitted = fit_ridge(train_matrix, train_costs, alpha) + predicted = predict_ridge(dev_matrix, fitted) + candidates.append( + { + "alpha": alpha, + "model": fitted, + "development_error": error_metrics(dev_costs, predicted, means), + "fit_and_development_prediction_seconds": time.perf_counter() + - before, + } + ) + chosen = select_candidate(candidates) + models[name] = chosen["model"] + predictions[name] = predict_ridge(dev_matrix, chosen["model"]) + errors[name] = chosen["development_error"] + tuning[name] = [ + {k: v for k, v in row.items() if k != "model"} for row in candidates + ] + return { + "format": "bbeh-shared-input-dollar-model-v1", + "fit_role": "train", + "tuning_role": "development", + "refit_on_development": False, + "text_features": text, + "models": models, + "tuning": tuning, + "train_mean_usd": means.tolist(), + "development_errors": errors, + "training_and_tuning_seconds": time.perf_counter() - started, + "inference_dependencies": "Request text only; no embeddings, topic metadata, labels or generated tokens", + }, predictions + + +def predict_rows(model, rows, name): + """Reusable input-only cost scorer, also suitable for native correctness heads.""" + core.require(name in {"mean", *FEATURE_MODELS}, "Unknown cost model") + if name == "mean": + return np.tile(np.asarray(model["train_mean_usd"]), (len(rows), 1)) + if name == "length_log_ridge": + matrix = core.lengths(rows) / np.asarray(model["text_features"]["length_scale"]) + else: + _, matrix, _ = core.text_matrix(rows, model["text_features"]) + return predict_ridge(matrix, model["models"][name]) + + +def selected_quality(model, y): + selected = {} + for candidate in model["tuning"]: + name = candidate["method"] + if candidate["setting"] != model["selected"].get(name, "fixed"): + continue + core.require( + name in QUALITY_METHODS and name not in selected, + "Unexpected/duplicate quality method", + ) + p = utility._matrix( + candidate["probabilities"], + "saved development probabilities", + probability=True, + ) + positive_costs(candidate["expected_usd"], len(y)) + core.require( + len(p) == len(y) and abs(core.nll(y, p) - candidate["nll"]) <= 1e-12, + "Stored quality NLL/population changed", + ) + selected[name] = candidate + core.require(set(selected) == QUALITY_METHODS, "Missing frozen quality method") + return selected + + +def make_curve(candidate, expected, rows, y, costs, bootstrap): + p = np.asarray(candidate["probabilities"]) + curve = frontier.enumerate_frontier( + {"probabilities": p, "expected_usd": expected}, y, costs, None + ) + frontier_ids = {node["candidate_id"] for node in curve["frontier"]} + points = [] + for point in curve["candidates"]: + node = { + "candidate_id": point["candidate_id"], + "rule": point["policy"], + "development": point["calibration"], + "development_matched_random_strong_probability": point[ + "calibration_matched_random_strong_probability" + ], + } + if point["candidate_id"] in frontier_ids: + routes = utility.replay(point["policy"], p, expected) + fees = ( + np.zeros(len(rows)) + if point["policy"]["mode"].startswith("always_") + else None + ) + intervals = frontier.uncertainty( + bootstrap, + routes, + y, + costs, + fees, + point["calibration_matched_random_strong_probability"], + ) + node["paired_ci95"] = { + k.replace("calibration_budget", "development_budget"): v + for k, v in intervals.items() + } + points.append(node) + return { + "setting": candidate["setting"], + "nll": candidate["nll"], + "selected_by_development_nll": True, + "probabilities": candidate["probabilities"], + "expected_usd": np.asarray(expected).tolist(), + "points": points, + "frontier_candidate_ids": sorted(frontier_ids), + "randomized_hull_candidate_ids": curve["randomized_hull_candidate_ids"], + } + + +def compare(quality, predictions, rows, y, costs, train_means, *, draws=2000): + bootstrap = frontier.Bootstrap(rows, draws) + methods, original_errors = {}, {} + for name, candidate in quality.items(): + original_errors[name] = error_metrics( + costs, candidate["expected_usd"], train_means + ) + for cost_name, expected in { + "original": np.asarray(candidate["expected_usd"]), + **predictions, + }.items(): + methods[f"{name}__{cost_name}"] = { + "quality_method": name, + "cost_method": cost_name, + "router_kind": "embedding" if name in {"knn", "embedding"} else "cpu", + "metadata_assisted": name == "task_metadata", + "settings": [ + make_curve(candidate, expected, rows, y, costs, bootstrap) + ], + } + return methods, original_errors + + +def load_inputs(run, train_path, dev_path, protocol_path): + run = Path(run) + core.require(core.sha(protocol_path) == PROTOCOL_SHA, "Cost protocol changed") + protocol, plan = core.read(protocol_path), core.checked_plan(run) + core.require( + plan["source_plan_sha256"] == protocol["source_plan_sha256"], + "Source plan changed", + ) + core.require( + core.sha(run / "model.json") == protocol["quality_model_sha256"], + "Quality model changed", + ) + model = core.read(run / "model.json") + core.require( + model["plan_sha256"] == core.sha(run / "plan.json") + and model["source_plan_sha256"] == plan["source_plan_sha256"] + and model["pair_sha256"] == plan["pair_sha256"] + and model["calibration_or_final_outcomes_accessed"] is False, + "Quality lineage changed", + ) + rows, data = {}, {} + for role, path in (("train", train_path), ("development", dev_path)): + expected = protocol[f"{role}_grades_sha256"] + core.require( + core.sha(path) == expected == model[f"{role}_grades_sha256"], + "Grades changed", + ) + rows[role] = core.role_rows(run, plan, role) + data[role] = core.grades(plan, rows[role], path, role) + core.require( + data["train"][2]["grader_sha256"] == data["development"][2]["grader_sha256"], + "Grader changed between roles", + ) + return plan, model, rows, data + + +def build(run, train_path, dev_path, output, *, protocol_path=None, draws=2000): + protocol_path = Path(protocol_path or Path(__file__).with_name("protocol.json")) + output = Path(output) + core.require(not output.exists(), "Preserve prior cost iteration") + plan, quality_model, rows, grades = load_inputs( + run, train_path, dev_path, protocol_path + ) + train_costs = grades["train"][1] + y, costs = grades["development"][:2] + quality = selected_quality(quality_model, y) + quality_hashes = { + name: core.digest(value["probabilities"]) for name, value in quality.items() + } + cost_model, predictions = fit_costs( + rows["train"], train_costs, rows["development"], costs + ) + output.mkdir(parents=True) + lineage = { + "protocol_sha256": core.sha(protocol_path), + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "quality_model_sha256": core.sha(Path(run) / "model.json"), + "train_grades_sha256": core.sha(train_path), + "development_grades_sha256": core.sha(dev_path), + "quality_probabilities_sha256": quality_hashes, + "study_source_sha256": core.sha(__file__), + "calibration_or_final_outcomes_accessed": False, + } + cost_model.update(lineage) + core.write(output / "cost-model.json", cost_model) + core.write( + output / "development-cost-predictions.json", + { + "format": "bbeh-shared-development-cost-predictions-v1", + "role": "development", + **lineage, + "cost_model_sha256": core.sha(output / "cost-model.json"), + "ids": [r["id"] for r in rows["development"]], + "expected_usd": { + name: value.tolist() for name, value in predictions.items() + }, + "arms": ["cheap", "strong"], + "actual_outcomes_used_as_prediction_inputs": False, + }, + ) + before = time.perf_counter() + methods, original_errors = compare( + quality, + predictions, + rows["development"], + y, + costs, + train_costs.mean(axis=0), + draws=draws, + ) + core.require( + quality_hashes + == { + name: core.digest(value["probabilities"]) for name, value in quality.items() + }, + "Quality probabilities mutated", + ) + report = { + "format": "bbeh-development-candidate-frontiers-v1", + "role": "development", + **lineage, + "plan_sha256": core.sha(Path(run) / "plan.json"), + "model_sha256": core.sha(Path(run) / "model.json"), + "grades_sha256": core.sha(dev_path), + "cost_model_sha256": core.sha(output / "cost-model.json"), + "ids": [r["id"] for r in rows["development"]], + "n": len(y), + "families": len({r["family"] for r in rows["development"]}), + "methods": methods, + "random": { + "analytic_expectation": True, + "strong_probability_range": [0, 1], + "accuracy_endpoints_percent": (100 * y.mean(axis=0)).tolist(), + "answer_usd_per_10000_endpoints": (10000 * costs.mean(axis=0)).tolist(), + }, + "cost_scope": grades["development"][2]["cost_scope"], + "fees_sha256": None, + "router_cost_scope": "Answer-model costs only; encoder and CPU/native hosting are unpriced, not free.", + "refit_used_for_development_predictions": False, + "bootstrap": { + "draws": draws, + "seed": 89, + "unit": "whole families within task, fixed observed task weights", + }, + "interpretation": "Development diagnostic after fixed cost-NMSE selection. Quality models/NLL choices unchanged. Best budget points and pointwise intervals are neither unbiased final estimates nor simultaneous curve guarantees.", + } + core.write(output / "development-frontiers.json", report) + summary = development_budget.summarize(report) + summary.update( + { + "cost_iteration": lineage, + "selected_alpha": { + name: cost_model["models"][name]["alpha"] for name in FEATURE_MODELS + }, + "shared_cost_errors": cost_model["development_errors"], + "original_cost_errors": original_errors, + "cost_tuning": cost_model["tuning"], + "cost_fit_seconds": cost_model["training_and_tuning_seconds"], + "frontier_seconds": time.perf_counter() - before, + } + ) + core.write(output / "budget-summary.json", summary) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", required=True) + parser.add_argument("--train-grades", required=True) + parser.add_argument("--development-grades", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + build(args.run, args.train_grades, args.development_grades, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/embedding/development_budget.py b/examples/train-a-router/embedding/development_budget.py new file mode 100644 index 000000000..9c9427abd --- /dev/null +++ b/examples/train-a-router/embedding/development_budget.py @@ -0,0 +1,112 @@ +"""Compact descriptive development comparisons at fixed answer-cost budgets.""" + +from __future__ import annotations + +from embedding import recipes as core + +FRACTIONS = (0.2, 0.4, 0.6, 0.8) + + +def summarize(report): + core.require( + report["role"] == "development" + and report["refit_used_for_development_predictions"] is False + and report["calibration_or_final_outcomes_accessed"] is False, + "Only train-fitted development diagnostics are allowed", + ) + low_cost, high_cost = report["random"]["answer_usd_per_10000_endpoints"] + low_quality, high_quality = report["random"]["accuracy_endpoints_percent"] + core.require( + high_cost > low_cost, "Budget interpretation needs higher mean Sol cost" + ) + methods = {} + for name, method in report["methods"].items(): + selected = [s for s in method["settings"] if s["selected_by_development_nll"]] + core.require( + len(selected) == 1, "Exactly one NLL-selected or fixed recipe required" + ) + setting = selected[0] + points = [] + for fraction in FRACTIONS: + budget = fraction * high_cost + eligible = [ + p + for p in setting["points"] + if p["development"]["answer_cost_usd_per10000"] <= budget + ] + random_p = (budget - low_cost) / (high_cost - low_cost) + random_quality = ( + low_quality + random_p * (high_quality - low_quality) + if 0 <= random_p <= 1 + else None + ) + row = { + "fraction_of_always_sol_answer_cost": fraction, + "answer_budget_usd_per_10000": budget, + "analytic_random_accuracy_at_budget_percent": random_quality, + "analytic_random_strong_probability": random_p + if 0 <= random_p <= 1 + else None, + "available": bool(eligible), + } + if eligible: + best = min( + eligible, + key=lambda p: ( + -p["development"]["answer_accuracy_percent"], + p["development"]["answer_cost_usd_per10000"], + p["candidate_id"], + ), + ) + metric = best["development"] + spend = metric["answer_cost_usd_per10000"] + matched_p = (spend - low_cost) / (high_cost - low_cost) + matched_quality = ( + low_quality + matched_p * (high_quality - low_quality) + if 0 <= matched_p <= 1 + else None + ) + row.update( + candidate_id=best["candidate_id"], + rule=best["rule"], + metrics=metric, + paired_ci95=best.get("paired_ci95"), + accuracy_gain_over_budget_random_pp=( + metric["answer_accuracy_percent"] - random_quality + if random_quality is not None + else None + ), + accuracy_gain_over_actual_spend_random_pp=( + metric["answer_accuracy_percent"] - matched_quality + if matched_quality is not None + else None + ), + ) + points.append(row) + methods[name] = { + "metadata_assisted": method["metadata_assisted"], + "selected_setting": setting["setting"], + "selected_nll": setting["nll"], + "candidate_nlls": [ + {"setting": s["setting"], "nll": s["nll"]} for s in method["settings"] + ], + "budget_points": points, + } + return { + "format": "bbeh-development-answer-budget-diagnostic-v1", + "role": "development", + "n": report["n"], + "families": report["families"], + "model_sha256": report["model_sha256"], + "grades_sha256": report["grades_sha256"], + "baselines": report["random"], + "methods": methods, + "interpretation": ( + "Exploratory development-only accuracy maxima under four declared answer-cost budgets; " + "the selected k/penalty remains the NLL choice. These operating points are not frozen " + "calibration/final choices. Pointwise intervals ignore this curve/threshold selection. " + "Random comparators are analytic mixtures, not observed runs. Encoder and hosting " + "costs are unpriced here; no net-savings claim. Metadata-assisted task control is diagnostic." + ), + "calibration_or_final_outcomes_accessed": False, + } diff --git a/examples/train-a-router/embedding/development_report.py b/examples/train-a-router/embedding/development_report.py new file mode 100644 index 000000000..9a3d0d682 --- /dev/null +++ b/examples/train-a-router/embedding/development_report.py @@ -0,0 +1,171 @@ +"""Inspect fixed train-only candidates on development; never recalibrate or refit.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np + +from embedding import recipes as core +from evaluation import frontier, utility_router as utility + + +def build(run, grades_path, fees_path, *, draws=2000): + run = Path(run) + plan, model = core.checked_plan(run), core.read(run / "model.json") + core.require( + model["plan_sha256"] == core.sha(run / "plan.json") + and model["source_plan_sha256"] == plan["source_plan_sha256"] + and model["pair_sha256"] == plan["pair_sha256"] + and model["development_grades_sha256"] == core.sha(grades_path) + and model["calibration_or_final_outcomes_accessed"] is False, + "Development model/grade lineage changed", + ) + rows = core.role_rows(run, plan, "development") + y, costs, grade = core.grades(plan, rows, grades_path, "development") + fees = frontier.fee_rows(plan, rows, fees_path) + bootstrap = frontier.Bootstrap(rows, draws) + methods, seen = {}, set() + for candidate in model["tuning"]: + name, setting = candidate["method"], candidate["setting"] + key = (name, setting) + core.require(key not in seen, "Duplicate development candidate") + seen.add(key) + p = utility._matrix( + candidate["probabilities"], "development probabilities", probability=True + ) + expected = utility._matrix( + candidate["expected_usd"], "development expected cost" + ) + core.require( + len(p) == len(expected) == len(rows), "Development score population changed" + ) + nll = core.nll(y, p) + core.require( + abs(nll - candidate["nll"]) <= 1e-12, + "Development NLL differs from fitted receipt", + ) + priced = name in {"knn", "embedding"} + overhead = fees if priced else None + curve = frontier.enumerate_frontier(candidate, y, costs, overhead) + frontier_ids = {node["candidate_id"] for node in curve["frontier"]} + points = [] + for point in curve["candidates"]: + node = { + "candidate_id": point["candidate_id"], + "rule": point["policy"], + "development": point["calibration"], + "development_matched_random_strong_probability": point[ + "calibration_matched_random_strong_probability" + ], + } + if point["candidate_id"] in frontier_ids: + routes = utility.replay(point["policy"], p, expected) + charged = ( + np.zeros(len(rows)) + if point["policy"]["mode"].startswith("always_") + else overhead + ) + intervals = frontier.uncertainty( + bootstrap, + routes, + y, + costs, + charged, + point["calibration_matched_random_strong_probability"], + ) + node["paired_ci95"] = { + key.replace("calibration_budget", "development_budget"): value + for key, value in intervals.items() + } + points.append(node) + method = methods.setdefault( + name, + { + "router_kind": "embedding" if priced else "cpu", + "metadata_assisted": name == "task_metadata", + "settings": [], + }, + ) + method["settings"].append( + { + "setting": setting, + "nll": nll, + "selected_by_development_nll": model["selected"].get(name, "fixed") + == setting, + "probabilities": candidate["probabilities"], + "expected_usd": candidate["expected_usd"], + "points": points, + "frontier_candidate_ids": sorted(frontier_ids), + "randomized_hull_candidate_ids": curve["randomized_hull_candidate_ids"], + } + ) + expected_settings = {("knn", value) for value in core.PROTOCOL["neighbors"]} + expected_settings |= {("embedding", value) for value in core.PROTOCOL["penalties"]} + expected_settings |= { + (name, "fixed") for name in ("text_length", "text_topic", "task_metadata") + } + core.require(seen == expected_settings, "Fixed development candidate set changed") + task_summary = [] + for task in sorted({row["task"] for row in rows}): + indices = [i for i, row in enumerate(rows) if row["task"] == task] + task_summary.append( + { + "task": task, + "n": len(indices), + "families": len({rows[i]["family"] for i in indices}), + "accuracy_percent": (100 * y[indices].mean(axis=0)).tolist(), + "answer_usd_per_10000": (10000 * costs[indices].mean(axis=0)).tolist(), + } + ) + return { + "format": "bbeh-development-candidate-frontiers-v1", + "role": "development", + "plan_sha256": core.sha(run / "plan.json"), + "model_sha256": core.sha(run / "model.json"), + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "grades_sha256": core.sha(grades_path), + "fees_sha256": core.sha(fees_path), + "ids": [row["id"] for row in rows], + "n": len(rows), + "families": len({row["family"] for row in rows}), + "tasks": task_summary, + "methods": methods, + "selected_settings": model["selected"], + "random": { + "analytic_expectation": True, + "strong_probability_range": [0, 1], + "accuracy_endpoints_percent": (100 * y.mean(axis=0)).tolist(), + "answer_usd_per_10000_endpoints": (10000 * costs.mean(axis=0)).tolist(), + }, + "cost_scope": grade["cost_scope"], + "router_cost_scope": "Encoder singleton API fee for kNN/embedding; head/CPU hosting unpriced. Always-arm bypasses incur no router fee.", + "interpretation": "Train-only predictions evaluated on the tuning population. Full curves and pointwise family intervals are development diagnostics, not unbiased held-out estimates or simultaneous guarantees. NLL alone chooses the declared k/penalty; no frontier point is selected here.", + "refit_used_for_development_predictions": False, + "calibration_or_final_outcomes_accessed": False, + "bootstrap": { + "draws": draws, + "seed": 89, + "unit": "paired whole families within task; fixed observed task weights", + }, + "implementation_sha256": { + "development_report": core.sha(__file__), + "frontier": core.sha(frontier.__file__), + }, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", required=True) + parser.add_argument("--grades", required=True) + parser.add_argument("--fees", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + core.write(args.output, build(args.run, args.grades, args.fees)) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/embedding/encode.py b/examples/train-a-router/embedding/encode.py new file mode 100644 index 000000000..23703dec0 --- /dev/null +++ b/examples/train-a-router/embedding/encode.py @@ -0,0 +1,71 @@ +"""Acquire a fresh question-only Qwen feature set and singleton fee validation.""" + +import argparse +from decimal import Decimal +import json +import os +from pathlib import Path + +from embedding import collect_features, recipes as core, selfhost, token_audit, wire +from reader_io import ChangedFileSync, require, sha + + +def prepare(source, budget, seconds): + cap = Decimal(str(budget)) + require(cap.is_finite() and cap > 0 and type(seconds) is int and seconds > 0, "Invalid budget/deadline") + source = Path(source) + source_hash = sha(source / "source-plan.json") + rows, input_hashes = collect_features.source_rows(source, source_hash) + return { + "format": "bbeh-public-feature-acquisition-v1", "reproduction": True, + "source_plan_sha256": source_hash, "n": len(rows), "role_inputs_sha256": input_hashes, + "model": core.MODEL, "hf_revision": core.REVISION, "instruction": core.INSTRUCTION, + "source_sha256": {"encode": sha(__file__), "collector": sha(collect_features.__file__), + "wire": sha(wire.__file__), "token_audit": sha(token_audit.__file__)}, + "budget_usd": str(cap), "collection_seconds": seconds, + "singleton_validation_including_warmup": len(collect_features.singleton_rows(rows)), + "source_only": True, "automatic_redispatch": False, + } + + +def execute(source, output, plan): + from huggingface_hub import snapshot_download + from transformers import AutoTokenizer + + require(not Path(output).exists(), "Preserve earlier feature receipts; do not repeat unresolved calls") + local = plan["format"] == "bbeh-public-selfhost-feature-plan-v1" + require(local or bool(os.environ.get("SIE_API_KEY")), "Set SIE_API_KEY for the SIE Cloud encoder") + root = Path(snapshot_download(core.MODEL, revision=core.REVISION, token=False, + allow_patterns=["*.json", "*.txt", "*.model"])) + binding = {"model": core.MODEL, "hf_revision": core.REVISION, + "files_sha256": {p.relative_to(root).as_posix(): sha(p) for p in root.rglob("*") if p.is_file()}} + tokenizer = AutoTokenizer.from_pretrained(root, local_files_only=True, trust_remote_code=False) + if local: + return selfhost.run(source, output, plan, tokenizer, binding) + return collect_features.run(source, output, plan, tokenizer, binding, ChangedFileSync()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--budget-usd", help="Required for Cloud; self-hosted execution remains unpriced") + parser.add_argument("--self-hosted", action="store_true") + parser.add_argument("--sie-base-url", help="Explicit self-hosted server origin") + parser.add_argument("--collection-seconds", type=int, default=4500) + parser.add_argument("--execute", action="store_true", help="Dispatch SIE requests; default only prepares the plan") + args = parser.parse_args() + if args.self_hosted: + require(args.sie_base_url is not None and args.budget_usd is None, + "Self-hosted encoding requires --sie-base-url; a Cloud USD cap does not price local hosting") + plan = selfhost.prepare(args.source, args.sie_base_url, args.collection_seconds) + else: + require(args.sie_base_url is None and args.budget_usd is not None, + "Cloud encoding requires --budget-usd; use --self-hosted for a custom SIE origin") + plan = prepare(args.source, args.budget_usd, args.collection_seconds) + result = execute(args.source, args.output, plan) if args.execute else plan + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/embedding/recipes.py b/examples/train-a-router/embedding/recipes.py new file mode 100644 index 000000000..3d3b499c3 --- /dev/null +++ b/examples/train-a-router/embedding/recipes.py @@ -0,0 +1,591 @@ +"""Small frozen-encoder recipes; role-separated fitting, tuning and scoring.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path +import time + +import numpy as np + +ARMS = ("cheap", "strong") +ROLES = ("train", "development", "calibration", "final") +MODEL = "Qwen/Qwen3-Embedding-4B" +REVISION = "5cf2132abc99cad020ac570b19d031efec650f2b" +INSTRUCTION = "Represent the knowledge and reasoning required to answer this question correctly." +PROTOCOL = { + "format": "bbeh-embedding-recipes-v1", + "seed": 89, + "neighbors": [8, 32, 128], + "neighbor_prior_rows": 2, + "penalties": [0.01, 0.1, 1.0, 10.0], + "selection": "Minimum summed development Bernoulli NLL; ties larger k/penalty", + "refit": "One selected refit on train+development; calibration/final never fit", + "cost": "Four fit-only arm/correctness means; absent class uses arm mean", + "task_prior_rows": 20, + "text": {"ngram_range": [1, 2], "min_df": 2, "max_features": 20000}, + "text_penalty": 1.0, + "optimizer": {"maxiter": 2000, "ftol": 1e-12, "gtol": 1e-7, "maxls": 40}, + "versions": {"numpy": "2.4.6", "scipy": "1.17.0", "sklearn": "1.6.1"}, +} + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sha(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def read(path): + return json.loads(Path(path).read_text()) + + +def write(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x") as stream: + json.dump(value, stream, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + stream.write("\n") + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False).encode()).hexdigest() + + +def require_hash(value): + require(isinstance(value, str) and len(value) == 64 and set(value) <= set("0123456789abcdef"), "Invalid SHA256") + + +def prepare(source, pair_path, run): + source, run = Path(source), Path(run) + require(not run.exists(), "Preserve prior prepared runs") + source_plan, pair = read(source / "source-plan.json"), read(pair_path) + require(pair["models"] == {"cheap": "gpt-6-luna", "strong": "gpt-6.1-sol"}, "Wrong selected pair") + require(bool(pair["cost_scope"]), "Explicit selected-response cost scope required") + paths = ["meta.json", *[f"question-only/{role}-inputs.json" for role in ROLES]] + for name in paths: + require(sha(source / name) == source_plan["files_sha256"][name], "Source file hash changed: " + name) + metadata = read(source / "meta.json") + by_id = {row["id"]: row for row in metadata} + require(len(by_id) == len(metadata), "Duplicate source ID") + families, roles, seen = {}, {}, set() + for role in ROLES: + items = read(source / f"question-only/{role}-inputs.json") + require(isinstance(items, list) and items, "Empty fixed role") + rows = [] + for item in items: + require(set(item) == {"id", "request", "response_contract"}, "Unexpected question-only fields") + meta = by_id[item["id"]] + require(meta["retained"] is True and meta["role"] == role, "Source role/eligibility changed") + require( + item["id"] not in seen and isinstance(item["request"], str) and item["request"], "Duplicate/empty input" + ) + seen.add(item["id"]) + require(families.setdefault(meta["family"], role) == role, "Family crosses source roles") + rows.append({**item, "role": role, "task": meta["task"], "family": meta["family"], "dataset": "bbeh"}) + roles[role] = rows + require(seen == {row["id"] for row in metadata if row["retained"]}, "Source retained population mismatch") + run.mkdir(parents=True) + for role, rows in roles.items(): + write(run / f"rows-{role}.json", rows) + (run / "pair.json").write_bytes(Path(pair_path).read_bytes()) + plan = { + "format": "bbeh-recipe-plan-v1", + "protocol": PROTOCOL, + "source_plan_sha256": sha(source / "source-plan.json"), + "pair_sha256": sha(pair_path), + "pair": pair, + "recipe_source_sha256": sha(__file__), + "roles": { + role: { + "n": len(rows), + "rows_sha256": sha(run / f"rows-{role}.json"), + "inputs_sha256": sha(source / f"question-only/{role}-inputs.json"), + } + for role, rows in roles.items() + }, + } + write(run / "plan.json", plan) + return plan + + +def checked_plan(run): + run = Path(run) + plan = read(run / "plan.json") + require(plan["format"] == "bbeh-recipe-plan-v1" and plan["protocol"] == PROTOCOL, "Recipe protocol changed") + require(plan["recipe_source_sha256"] == sha(__file__), "Recipe source changed") + require(plan["pair_sha256"] == sha(run / "pair.json") and plan["pair"] == read(run / "pair.json"), "Pair changed") + return plan + + +def role_rows(run, plan, role): + require(role in ROLES, "Unknown role") + path = Path(run) / f"rows-{role}.json" + require(sha(path) == plan["roles"][role]["rows_sha256"], "Role metadata hash changed") + rows = read(path) + require(len(rows) == plan["roles"][role]["n"] and all(row["role"] == role for row in rows), "Wrong role metadata") + return rows + + +def grades(plan, rows, path, role): + data = read(path) + require(data["format"] == "bbeh-paired-grades-v1" and data["role"] == role, "Wrong grade role/format") + require( + data["source_plan_sha256"] == plan["source_plan_sha256"] and data["pair_sha256"] == plan["pair_sha256"], + "Grade source/pair changed", + ) + require( + data["models"] == plan["pair"]["models"] and data["cost_scope"] == plan["pair"]["cost_scope"], + "Grade models/pricing changed", + ) + for key in ("grader_sha256", "acquisition_protocol_sha256"): + require_hash(data[key]) + require([row["id"] for row in data["rows"]] == [row["id"] for row in rows], "Exact role grade population required") + for row in data["rows"]: + for arm in ARMS: + require(type(row[f"{arm}_correct"]) in (int, bool) and row[f"{arm}_correct"] in (0, 1), "Nonbinary label") + require_hash(row[f"{arm}_record_sha256"]) + value = row[f"{arm}_cost_usd"] + require( + value is not None and not isinstance(value, bool) and math.isfinite(float(value)) and float(value) >= 0, + "Unknown costs cannot be dropped or imputed free", + ) + y = np.array([[row[f"{arm}_correct"] for arm in ARMS] for row in data["rows"]], dtype=float) + c = np.array([[float(row[f"{arm}_cost_usd"]) for arm in ARMS] for row in data["rows"]]) + return y, c, data + + +def features(plan, rows, root, role): + path = Path(root) / role + info = read(path / "features.json") + require(info["inputs_sha256"] == plan["roles"][role]["inputs_sha256"], "Features use different role inputs") + require( + (info["model"], info["hf_revision"], info["instruction"], info["dimensions"]) + == (MODEL, REVISION, INSTRUCTION, 2560), + "Encoder identity changed", + ) + require( + sha(path / "ids.json") == info["ids_sha256"] and sha(path / "vectors.npy") == info["vectors_sha256"], + "Feature bytes changed", + ) + require(read(path / "ids.json") == [row["id"] for row in rows], "Feature order changed") + audit = read(path / "token-audit.json") + require( + sha(path / "token-audit.json") == info["token_audit_sha256"] + and audit["inputs_sha256"] == info["inputs_sha256"] + and audit["model"] == MODEL + and audit["hf_revision"] == REVISION + and audit["instruction"] == INSTRUCTION + and audit["truncated_ids"] == [] + and audit["overflow_ids"] == [], + "Exact no-truncation encoder audit required", + ) + matrix = np.load(path / "vectors.npy", mmap_mode="r", allow_pickle=False) + require( + matrix.dtype == np.float32 + and matrix.shape == (len(rows), 2560) + and np.isfinite(matrix).all() + and np.allclose(np.linalg.norm(matrix, axis=1), 1, atol=2e-6, rtol=0), + "Invalid normalized features", + ) + return matrix, sha(path / "features.json") + + +def conditional_costs(y, costs): + means, counts = [], [] + for a in range(2): + means.append( + [ + float(costs[y[:, a] == label, a].mean()) if (y[:, a] == label).any() else float(costs[:, a].mean()) + for label in (0, 1) + ] + ) + counts.append([int((y[:, a] == label).sum()) for label in (0, 1)]) + return {"means": means, "counts": counts, "missing_class": "arm-wide mean", "unit": "USD"} + + +def expected_cost(probabilities, stats): + means = np.array(stats["means"]) + return (1 - probabilities) * means[:, 0] + probabilities * means[:, 1] + + +def nll(y, p): + p = np.clip(p, 1e-15, 1 - 1e-15) + return float(np.mean(-y * np.log(p) - (1 - y) * np.log1p(-p), axis=0).sum()) + + +def fit_logistic(matrix, target, penalty): + from scipy.optimize import minimize + from scipy.special import expit + from scipy import sparse + + matrix = sparse.csr_matrix(matrix, dtype=float) if sparse.issparse(matrix) else np.asarray(matrix, dtype=float) + target = np.asarray(target, dtype=float) + require(matrix.ndim == 2 and target.shape == (matrix.shape[0],) and np.isin(target, (0, 1)).all(), "Misaligned fit") + require(np.isfinite(matrix.data if sparse.issparse(matrix) else matrix).all(), "Nonfinite features") + mean = np.asarray(matrix.mean(axis=0)).reshape(-1) + frequency = float(target.mean()) + if frequency in (0, 1): + return { + "weights": [0.0] * matrix.shape[1], + "bias": float(np.log((target.sum() + 0.5) / (len(target) - target.sum() + 0.5))), + "constant_class": True, + } + initial = np.zeros(matrix.shape[1] + 1) + initial[-1] = np.log(frequency / (1 - frequency)) + + def loss(theta): + w, b = theta[:-1], theta[-1] + logits = np.asarray(matrix @ w).reshape(-1) - mean @ w + b + residual = expit(logits) - target + value = np.logaddexp(0, (1 - 2 * target) * logits).sum() + 0.5 * penalty * (w @ w) + gradient = np.asarray(matrix.T @ residual).reshape(-1) - mean * residual.sum() + penalty * w + return float(value), np.r_[gradient, residual.sum()] + + result = minimize(loss, initial, method="L-BFGS-B", jac=True, options=PROTOCOL["optimizer"]) + require(result.success and np.isfinite(result.x).all(), "Optimizer failed: " + str(result.message)) + return { + "weights": result.x[:-1].tolist(), + "bias": float(result.x[-1] - mean @ result.x[:-1]), + "iterations": int(result.nit), + } + + +def logistic_probability(matrix, head): + logits = np.asarray(matrix @ np.asarray(head["weights"])).reshape(-1) + head["bias"] + return np.exp(-np.logaddexp(0, -logits)) + + +def two_heads(matrix, y, penalty): + return [fit_logistic(matrix, y[:, arm], penalty) for arm in range(2)] + + +def predict_heads(matrix, heads): + return np.column_stack([logistic_probability(matrix, head) for head in heads]) + + +def nearest(matrix, reference, labels, k, *, batch=128): + require(type(k) is int and 0 < k <= len(reference), "Invalid fixed neighbor count") + prior = np.asarray(labels, dtype=float).mean(axis=0) + result = [] + reference = np.asarray(reference, dtype=float) + for start in range(0, len(matrix), batch): + similarity = np.asarray(matrix[start : start + batch], dtype=float) @ reference.T + indices = np.argsort(-similarity, axis=1, kind="stable")[:, :k] + result.append((labels[indices].sum(axis=1) + 2 * prior) / (k + 2)) + return np.concatenate(result) + + +def lengths(rows): + values = [] + for row in rows: + text = row["request"] + values.append( + [ + np.log1p(len(text)), + np.log1p(len(text.split())), + np.log1p(text.count("\n") + 1), + sum(c.isdigit() for c in text) / max(len(text), 1), + sum(c in "[](){}" for c in text) / max(len(text), 1), + ] + ) + return np.array(values) + + +def text_matrix(rows, saved=None): + from sklearn.feature_extraction.text import TfidfVectorizer + from scipy import sparse + + spec = {**PROTOCOL["text"], "ngram_range": tuple(PROTOCOL["text"]["ngram_range"])} + vectorizer = TfidfVectorizer(**spec, vocabulary=None if saved is None else saved["vocabulary"]) + if saved is None: + matrix = vectorizer.fit_transform([row["request"] for row in rows]) + raw = lengths(rows) + scale = np.maximum(raw.std(axis=0), 1e-6) + saved = { + "vocabulary": {k: int(v) for k, v in vectorizer.vocabulary_.items()}, + "idf": vectorizer.idf_.tolist(), + "length_scale": scale.tolist(), + } + else: + vectorizer.idf_ = np.array(saved["idf"]) + matrix = vectorizer.transform([row["request"] for row in rows]) + return matrix, sparse.hstack((matrix, lengths(rows) / np.array(saved["length_scale"])), format="csr"), saved + + +def task_statistics(rows, y, costs): + names = sorted({row["task"] for row in rows}) + return { + "names": names, + "probabilities": [ + ( + (y[[r["task"] == name for r in rows]].sum(axis=0) + 20 * y.mean(axis=0)) + / (sum(r["task"] == name for r in rows) + 20) + ).tolist() + for name in names + ], + "costs": [ + ( + (costs[[r["task"] == name for r in rows]].sum(axis=0) + 20 * costs.mean(axis=0)) + / (sum(r["task"] == name for r in rows) + 20) + ).tolist() + for name in names + ], + "global_probabilities": y.mean(axis=0).tolist(), + "global_costs": costs.mean(axis=0).tolist(), + } + + +def fit_controls(rows, y, costs): + from sklearn.naive_bayes import MultinomialNB + + word, text, text_info = text_matrix(rows) + task = task_statistics(rows, y, costs) + nb = MultinomialNB(alpha=1).fit(word, [row["task"] for row in rows]) + require(nb.classes_.tolist() == task["names"], "Task class alignment changed") + return { + "text": text_info, + "text_heads": two_heads(text, y, PROTOCOL["text_penalty"]), + "task": task, + "topic_nb": { + "feature_log_prob": nb.feature_log_prob_.tolist(), + "class_log_prior": nb.class_log_prior_.tolist(), + }, + } + + +def predict_controls(model, rows): + from scipy.special import softmax + + word, text, _ = text_matrix(rows, model["text"]) + nb, task = model["topic_nb"], model["task"] + weights = softmax(word @ np.array(nb["feature_log_prob"]).T + np.array(nb["class_log_prior"]), axis=1) + by_task = {name: i for i, name in enumerate(task["names"])} + return { + "text_length": {"probabilities": predict_heads(text, model["text_heads"])}, + "text_topic": { + "probabilities": weights @ np.array(task["probabilities"]), + "expected_usd": weights @ np.array(task["costs"]), + }, + "task_metadata": { + "probabilities": np.array( + [ + task["probabilities"][by_task[r["task"]]] if r["task"] in by_task else task["global_probabilities"] + for r in rows + ] + ), + "expected_usd": np.array( + [task["costs"][by_task[r["task"]]] if r["task"] in by_task else task["global_costs"] for r in rows] + ), + }, + } + + +def fit_arrays(train_x, train_rows, train_y, train_cost, dev_x, dev_rows, dev_y, dev_cost): + require( + all(r["role"] == "train" for r in train_rows) and all(r["role"] == "development" for r in dev_rows), + "Only train/development may fit or tune", + ) + require(not ({r["family"] for r in train_rows} & {r["family"] for r in dev_rows}), "Families cross fitting/tuning") + require( + len(train_x) == len(train_rows) == len(train_y) == len(train_cost) + and len(dev_x) == len(dev_rows) == len(dev_y) == len(dev_cost), + "Fit arrays misaligned", + ) + tuning, train_means = [], conditional_costs(train_y, train_cost) + started = time.perf_counter() + for k in PROTOCOL["neighbors"]: + p = nearest(dev_x, train_x, train_y, k) + tuning.append( + { + "method": "knn", + "setting": k, + "nll": nll(dev_y, p), + "probabilities": p.tolist(), + "expected_usd": expected_cost(p, train_means).tolist(), + } + ) + for penalty in PROTOCOL["penalties"]: + heads = two_heads(train_x, train_y, penalty) + p = predict_heads(dev_x, heads) + tuning.append( + { + "method": "embedding", + "setting": penalty, + "nll": nll(dev_y, p), + "probabilities": p.tolist(), + "expected_usd": expected_cost(p, train_means).tolist(), + } + ) + train_controls = fit_controls(train_rows, train_y, train_cost) + for name, values in predict_controls(train_controls, dev_rows).items(): + p = values["probabilities"] + c = values.get("expected_usd", expected_cost(p, train_means)) + tuning.append( + { + "method": name, + "setting": "fixed", + "nll": nll(dev_y, p), + "probabilities": p.tolist(), + "expected_usd": c.tolist(), + "metadata_assisted": name == "task_metadata", + } + ) + selected = { + name: min([r for r in tuning if r["method"] == name], key=lambda r: (r["nll"], -r["setting"]))["setting"] + for name in ("knn", "embedding") + } + tuning_seconds = time.perf_counter() - started + x, y, cost = ( + np.concatenate((train_x, dev_x)), + np.concatenate((train_y, dev_y)), + np.concatenate((train_cost, dev_cost)), + ) + rows = train_rows + dev_rows + started = time.perf_counter() + heads = two_heads(x, y, selected["embedding"]) + controls = fit_controls(rows, y, cost) + result = { + "selected": selected, + "tuning": tuning, + "conditional_cost_usd": conditional_costs(y, cost), + "heads": heads, + "labels": y.tolist(), + "reference_ids": [r["id"] for r in rows], + **controls, + "timing": {"tuning_seconds": tuning_seconds, "refit_seconds": time.perf_counter() - started}, + "knn_storage": { + "rows": len(x), + "dimensions": x.shape[1], + "float32_bytes": len(x) * x.shape[1] * 4, + "similarity_multiply_adds_per_request": len(x) * x.shape[1], + }, + } + return result, x.astype(np.float32) + + +def predict_arrays(model, reference, matrix, rows): + p = { + "knn": nearest(matrix, reference, np.array(model["labels"]), model["selected"]["knn"]), + "embedding": predict_heads(matrix, model["heads"]), + } + out = { + name: { + "probabilities": value.tolist(), + "expected_usd": expected_cost(value, model["conditional_cost_usd"]).tolist(), + "router_kind": "embedding" if name in {"knn", "embedding"} else "cpu", + "metadata_assisted": False, + } + for name, value in p.items() + } + for name, values in predict_controls(model, rows).items(): + out[name] = { + "probabilities": values["probabilities"].tolist(), + "expected_usd": values.get( + "expected_usd", expected_cost(values["probabilities"], model["conditional_cost_usd"]) + ).tolist(), + "router_kind": "cpu", + "metadata_assisted": name == "task_metadata", + } + return out + + +def fit(run, train_grades, development_grades, feature_root): + run = Path(run) + require(not (run / "model.json").exists(), "Preserve fitted models") + plan = checked_plan(run) + rows = [role_rows(run, plan, role) for role in ("train", "development")] + g = [ + grades(plan, r, path, role) + for r, path, role in zip(rows, (train_grades, development_grades), ("train", "development"), strict=True) + ] + require(g[0][2]["grader_sha256"] == g[1][2]["grader_sha256"], "Tuning grading semantics changed") + x = [features(plan, r, feature_root, role) for r, role in zip(rows, ("train", "development"), strict=True)] + model, reference = fit_arrays(x[0][0], rows[0], g[0][0], g[0][1], x[1][0], rows[1], g[1][0], g[1][1]) + with (run / "reference.npy").open("xb") as stream: + np.save(stream, reference, allow_pickle=False) + model.update( + { + "format": "bbeh-embedding-model-v1", + "plan_sha256": sha(run / "plan.json"), + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "train_grades_sha256": sha(train_grades), + "development_grades_sha256": sha(development_grades), + "features_sha256": {"train": x[0][1], "development": x[1][1]}, + "reference_sha256": sha(run / "reference.npy"), + "calibration_or_final_outcomes_accessed": False, + } + ) + write(run / "model.json", model) + return model + + +def score(run, feature_root, role, output): + run = Path(run) + require( + role in {"calibration", "final"}, + "Selected refit scores only calibration/final; tuning predictions are saved separately", + ) + plan, model = checked_plan(run), read(run / "model.json") + require( + model["plan_sha256"] == sha(run / "plan.json") and model["reference_sha256"] == sha(run / "reference.npy"), + "Fitted model/reference changed", + ) + rows = role_rows(run, plan, role) + matrix, feature_sha = features(plan, rows, feature_root, role) + reference = np.load(run / "reference.npy", mmap_mode="r", allow_pickle=False) + started = time.perf_counter() + methods = predict_arrays(model, reference, matrix, rows) + result = { + "format": "bbeh-input-only-scores-v1", + "role": role, + "ids": [r["id"] for r in rows], + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "model_sha256": sha(run / "model.json"), + "features_sha256": feature_sha, + "methods": methods, + "outcomes_accessed": False, + "batch_scoring_seconds": time.perf_counter() - started, + "latency_scope": "Batch scoring throughput; not a single-request serving measurement", + } + write(output, result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + p = commands.add_parser("prepare") + p.add_argument("--source", required=True) + p.add_argument("--pair", required=True) + p.add_argument("--run", required=True) + p = commands.add_parser("fit") + p.add_argument("--run", required=True) + p.add_argument("--train-grades", required=True) + p.add_argument("--development-grades", required=True) + p.add_argument("--features", required=True) + p = commands.add_parser("score") + p.add_argument("--run", required=True) + p.add_argument("--split", choices=("calibration", "final"), required=True) + p.add_argument("--features", required=True) + p.add_argument("--output", required=True) + args = parser.parse_args() + if args.command == "prepare": + prepare(args.source, args.pair, args.run) + elif args.command == "fit": + fit(args.run, args.train_grades, args.development_grades, args.features) + else: + score(args.run, args.features, args.split, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/embedding/selfhost.py b/examples/train-a-router/embedding/selfhost.py new file mode 100644 index 000000000..dac314a7d --- /dev/null +++ b/examples/train-a-router/embedding/selfhost.py @@ -0,0 +1,178 @@ +"""Pinned self-hosted SIE features; hosting charges remain explicitly unknown.""" + +import hashlib +import os +from pathlib import Path +import re +import time +from urllib.parse import urlsplit + +import httpx +import numpy as np + +from embedding import collect_features, recipes as core, token_audit, wire +from reader_io import ChangedFileSync + + +def checked_url(value): + url = urlsplit(value) + core.require(url.scheme in {"http", "https"} and url.hostname + and url.username is None and url.password is None + and not url.query and not url.fragment and url.path in {"", "/"}, + "Use an explicit HTTP(S) server origin without credentials or query parameters") + core.require(url.hostname != "api.superlinked.com", "Use the Cloud path for the SIE Cloud origin") + return value.rstrip("/") + + +def make_client(base_url, capture, *, transport=None): + base_url = checked_url(base_url) + session = httpx.Client(base_url=base_url, trust_env=False, follow_redirects=False, + timeout=httpx.Timeout(300, connect=10), transport=transport, + event_hooks={"response": [capture]}) + # Empty explicitly suppresses the SDK's SIE_API_KEY environment fallback. + # A separately named credential is scoped only to this explicit server. + return wire.SingleSendClient(base_url, api_key=os.environ.get("SIE_SELF_HOSTED_API_KEY", ""), + remote="forbid", http_client=session, timeout_s=300) + + +def prepare(source, base_url, seconds, *, batch_size=8): + core.require(type(seconds) is int and seconds > 0, "Invalid collection deadline") + core.require(type(batch_size) is int and 1 <= batch_size <= 16, "Invalid batch size") + source = Path(source) + rows, hashes = collect_features.source_rows(source, core.sha(source / "source-plan.json")) + return {"format": "bbeh-public-selfhost-feature-plan-v1", "reproduction": True, + "source_plan_sha256": core.sha(source / "source-plan.json"), "role_inputs_sha256": hashes, + "n": len(rows), "model": core.MODEL, "hf_revision": core.REVISION, + "instruction": core.INSTRUCTION, "dimensions": 2560, "base_url": checked_url(base_url), + "collection_seconds": seconds, "batch_size": batch_size, "automatic_redispatch": False, + "encoder_cost_usd": None, "cost_scope": "Self-hosted execution; hosting and encoder fees are unpriced", + "source_sha256": {"selfhost": core.sha(__file__), "source_reader": core.sha(collect_features.__file__), + "wire": core.sha(wire.__file__), "token_audit": core.sha(token_audit.__file__)}, + "cloud_credentials_used": False, "source_only": True} + + +def validate(reply, rows, capture, client, body): + receipt = capture.receipt + core.require(receipt is not None and receipt["http_status"] == 200, "Missing terminal encoder response") + expected_wire = hashlib.sha256(wire.packb(body, use_bin_type=True)).hexdigest() + core.require(receipt["request_wire_sha256"] == expected_wire, "SDK request settings changed") + core.require(isinstance(reply, list) and len(reply) == len(rows), "Wrong encoder result count") + by_id = {value.get("id"): value for value in reply} + core.require(len(by_id) == len(rows) and set(by_id) == {row["id"] for row in rows}, "Encoder IDs changed") + core.require(receipt["response_model"] == core.MODEL + and all(value.get("model") == core.MODEL for value in reply), "Response model changed") + execution = client.last_model_revision + core.require(isinstance(execution, str) and re.fullmatch(r"[0-9a-f]{64}", execution) + and receipt["headers"].get("x-sie-model-revision") == execution, + "Missing or inconsistent execution revision") + core.require(client.last_retry_count == 0, "Unexpected SDK retry") + vectors = np.asarray([by_id[row["id"]]["dense"] for row in rows], dtype=np.float64) + core.require(vectors.shape == (len(rows), 2560) and np.isfinite(vectors).all(), "Invalid encoder vectors") + norms = np.linalg.norm(vectors, axis=1, keepdims=True) + core.require(np.isfinite(norms).all() and (norms > 0).all(), "Invalid vector norms") + return (vectors / norms).astype(np.float32), execution + + +def run(source, output, plan, tokenizer, tokenizer_binding, *, client_factory=make_client): + source, output = Path(source), Path(output) + core.require(not output.exists(), "Preserve previous intents; self-host requests are not automatically repeated") + rows, input_hashes = collect_features.source_rows(source, plan["source_plan_sha256"]) + core.require(len(rows) == plan["n"] and input_hashes == plan["role_inputs_sha256"], "Source changed") + core.require(plan == prepare(source, plan["base_url"], plan["collection_seconds"], batch_size=plan["batch_size"]), + "Self-host collection plan changed") + core.require(tokenizer_binding["model"] == core.MODEL and tokenizer_binding["hf_revision"] == core.REVISION, + "Wrong tokenizer identity") + output.mkdir(parents=True) + checkpoint = ChangedFileSync() + core.write(output / "plan.json", plan) + for role in core.ROLES: + selected = [row for row in rows if row["role"] == role] + lengths = token_audit.audit_rows(selected, tokenizer) + audit = {"format": "bbeh-encoder-token-audit-v1", "role": role, "model": core.MODEL, + "hf_revision": core.REVISION, "instruction": core.INSTRUCTION, + "source_plan_sha256": plan["source_plan_sha256"], "inputs_sha256": input_hashes[role], + "template": token_audit.TEMPLATE, "max_length": token_audit.MAX_LENGTH, + "tokenizer": tokenizer_binding, "rows": lengths, "truncation": False, "truncated_ids": [], + "overflow_ids": [r["id"] for r in lengths if r["input_tokens"] > token_audit.MAX_LENGTH]} + core.write(output / "features" / role / "token-audit.json", audit) + core.require(not audit["overflow_ids"], "Encoder context overflow; no requests dispatched or rows truncated") + checkpoint(output) + capture, batches, vectors, current, execution = wire.WireCapture(), [], [], None, None + started = time.monotonic() + terminal = {"status": "incomplete", "n": len(rows), "complete_rows": 0, + "encoder_cost_usd": None, "cost_scope": plan["cost_scope"], "self_hosted": True} + try: + with client_factory(plan["base_url"], capture) as client: + catalog = client.get_model(core.MODEL) + core.require(catalog.get("name") == core.MODEL and catalog.get("revision") == core.REVISION + and catalog.get("dims", {}).get("dense") == 2560 and catalog.get("loaded") is True, + "Self-hosted encoder is not the ready pinned model") + core.write(output / "catalog.json", catalog) + for number, offset in enumerate(range(0, len(rows), plan["batch_size"])): + core.require(time.monotonic() - started < plan["collection_seconds"], "Collection deadline reached") + batch = rows[offset:offset + plan["batch_size"]] + current = output / "batches" / f"{number:06d}" + body = wire.body_for(batch, core.INSTRUCTION) + core.write(current / "intent.json", {"request": body, "plan_sha256": core.sha(output / "plan.json"), + "ids": [row["id"] for row in batch], "physical_attempt": 1}) + checkpoint(output) + capture.arm(current) + before = time.monotonic() + reply = client.encode(core.MODEL, body["items"], output_types=["dense"], + instruction=core.INSTRUCTION, output_dtype="float32", is_query=True, + wait_for_capacity=False, max_oom_retries=0, provision_timeout_s=150) + matrix, revision = validate(reply, batch, capture, client, body) + core.require(execution is None or execution == revision, "Execution changed during collection") + execution = revision + with (current / "vectors.npy").open("xb") as stream: + np.save(stream, matrix, allow_pickle=False) + receipt = {"ids": [row["id"] for row in batch], "status": "complete", + "intent_sha256": core.sha(current / "intent.json"), + "http_sha256": core.sha(current / "http.json"), + "vectors_sha256": core.sha(current / "vectors.npy"), + "execution_bundle_sha256": execution, "catalog_hf_revision": core.REVISION, + "elapsed_seconds": time.monotonic() - before, + "usage": capture.receipt["usage"], "encoder_cost_usd": None} + core.write(current / "receipt.json", receipt) + batches.append({"path": (current / "receipt.json").relative_to(output).as_posix(), + "sha256": core.sha(current / "receipt.json")}) + vectors.append(matrix) + terminal["complete_rows"] += len(batch) + current = None + checkpoint(output) + collection = {"format": "bbeh-selfhost-feature-collection-v1", "status": "complete", "n": len(rows), + "plan_sha256": core.sha(output / "plan.json"), "batches": batches, + "encoder_cost_usd": None, "cost_scope": plan["cost_scope"]} + core.write(output / "collection.json", collection) + matrix = np.concatenate(vectors) + for role in core.ROLES: + indices = [i for i, row in enumerate(rows) if row["role"] == role] + directory = output / "features" / role + core.write(directory / "ids.json", [rows[i]["id"] for i in indices]) + with (directory / "vectors.npy").open("xb") as stream: + np.save(stream, matrix[indices], allow_pickle=False) + core.write(directory / "features.json", { + "format": "routerbench-fixed-embeddings-v1", "model": core.MODEL, + "hf_revision": core.REVISION, "instruction": core.INSTRUCTION, "dimensions": 2560, + "n": len(indices), "source_plan_sha256": plan["source_plan_sha256"], + "inputs_sha256": input_hashes[role], "ids_sha256": core.sha(directory / "ids.json"), + "vectors_sha256": core.sha(directory / "vectors.npy"), + "token_audit_sha256": core.sha(directory / "token-audit.json"), + "collection_sha256": core.sha(output / "collection.json"), "dtype": "float32", + "normalization": "float64 L2 then float32", "outcomes_accessed": False, "self_hosted": True}) + core.write(output / "unpriced-encoder.json", { + "format": "bbeh-unpriced-selfhost-encoder-v1", "cost_basis": "unpriced", + "source_plan_sha256": plan["source_plan_sha256"], "model": core.MODEL, + "hf_revision": core.REVISION, "instruction": core.INSTRUCTION, + "rows": [{"id": row["id"], "projected_singleton_usd": None} for row in rows], + "hosting_cost_usd": None, "cloud_fee_projection_applicable": False}) + terminal.update(status="complete", collection_sha256=core.sha(output / "collection.json")) + return terminal + except Exception as error: + terminal.update(error_type=type(error).__name__, + unresolved_batch=None if current is None else current.relative_to(output).as_posix()) + raise + finally: + terminal["elapsed_seconds"] = time.monotonic() - started + core.write(output / "terminal.json", terminal) + checkpoint(output) diff --git a/examples/train-a-router/embedding/token_audit.py b/examples/train-a-router/embedding/token_audit.py new file mode 100644 index 000000000..1c0488ca2 --- /dev/null +++ b/examples/train-a-router/embedding/token_audit.py @@ -0,0 +1,77 @@ +"""Count exact encoder inputs on a remote CPU; never truncate or call a model.""" + +import argparse +from pathlib import Path + +from embedding import recipes as core + +TEMPLATE = "Instruct: {instruction}\nQuery: {text}" +MAX_LENGTH = 32768 + + +def audit_rows(rows, tokenizer): + result = [] + for start in range(0, len(rows), 128): + batch = rows[start : start + 128] + encoded = tokenizer( + [TEMPLATE.format(instruction=core.INSTRUCTION, text=r["request"]) for r in batch], truncation=False + )["input_ids"] + core.require(len(encoded) == len(batch), "Tokenizer population changed") + for row, tokens in zip(batch, encoded, strict=True): + core.require(len(tokens) > 0, "Empty encoded request") + result.append({"id": row["id"], "input_tokens": len(tokens)}) + return result + + +def main(): + from transformers import AutoTokenizer + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", required=True, type=Path) + parser.add_argument("--tokenizer", required=True, type=Path) + parser.add_argument("--tokenizer-receipt", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + binding = core.read(args.tokenizer_receipt) + core.require( + binding["model"] == core.MODEL and binding["hf_revision"] == core.REVISION, "Tokenizer revision changed" + ) + for name, expected in binding["files_sha256"].items(): + relative = Path(name) + core.require( + not relative.is_absolute() + and ".." not in relative.parts + and core.sha(args.tokenizer / relative) == expected, + "Tokenizer file binding changed", + ) + tokenizer = AutoTokenizer.from_pretrained(args.tokenizer, local_files_only=True, trust_remote_code=False) + plan = core.read(args.source / "source-plan.json") + for role in core.ROLES: + source = args.source / f"question-only/{role}-inputs.json" + core.require( + core.sha(source) == plan["files_sha256"][source.relative_to(args.source).as_posix()], + "Question inputs changed", + ) + counts = audit_rows(core.read(source), tokenizer) + result = { + "format": "bbeh-encoder-token-audit-v1", + "role": role, + "source_plan_sha256": core.sha(args.source / "source-plan.json"), + "inputs_sha256": core.sha(source), + "model": core.MODEL, + "hf_revision": core.REVISION, + "instruction": core.INSTRUCTION, + "template": TEMPLATE, + "max_length": MAX_LENGTH, + "truncation": False, + "tokenizer_receipt_sha256": core.sha(args.tokenizer_receipt), + "rows": counts, + "overflow_ids": [r["id"] for r in counts if r["input_tokens"] > MAX_LENGTH], + "truncated_ids": [], + "source_sha256": core.sha(__file__), + } + core.write(args.output / role / "token-audit.json", result) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/embedding/wire.py b/examples/train-a-router/embedding/wire.py new file mode 100644 index 000000000..bc318b21c --- /dev/null +++ b/examples/train-a-router/embedding/wire.py @@ -0,0 +1,429 @@ +"""Durable question-only SIE embedding batches; no solver or grading calls.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import time +from decimal import Decimal, InvalidOperation +from pathlib import Path + +import httpx +import numpy as np +from sie_sdk import SIEClient +from sie_sdk._msgpack import packb, unpackb + +MODEL = "Qwen/Qwen3-Embedding-4B" +REVISION = "5cf2132abc99cad020ac570b19d031efec650f2b" +DIMENSIONS = 2560 +INSTRUCTION = "Represent the knowledge and reasoning required to answer this question correctly." +BASE_URL = "https://api.superlinked.com" +CREDIT_USD = Decimal("0.00001") +HEADER_NAMES = ( + "content-type", + "x-sie-request-id", + "x-sie-model-revision", + "x-sie-credits-debited", + "x-sie-credits-debited-decimal", + "x-sie-execution-identity-sha256", + "x-sie-execution-binding-sha256", + "x-sie-runtime-instance", + "x-sie-server-version", +) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sha(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def encoded(value): + return ( + json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + "\n" + ).encode() + + +def write(path, value, *, replace=False): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + require(replace or not path.exists(), f"Preserve existing artifact: {path.name}") + temporary = path.with_name(path.name + ".pending") + with temporary.open("xb") as stream: + stream.write(value if isinstance(value, bytes) else encoded(value)) + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + + +def decimal(value): + require(not isinstance(value, bool) and isinstance(value, (str, int, float)), "Invalid charge type") + try: + result = Decimal(str(value)) + except InvalidOperation as error: + raise ValueError("Invalid charge") from error + require(result.is_finite() and result >= 0, "Nonfinite or negative charge") + return result + + +def body_for(rows, instruction): + require(isinstance(instruction, str) and bool(instruction.strip()), "Explicit embedding instruction required") + return { + "items": [{"id": row["id"], "text": row["request"]} for row in rows], + "params": { + "output_types": ["dense"], + "instruction": instruction, + "output_dtype": "float32", + "options": {"is_query": True}, + }, + } + + +class SingleSendClient(SIEClient): + def _record_retry(self): + raise RuntimeError("Automatic retries disabled; retain the first physical attempt") + + +class WireCapture: + """A public httpx response hook: preserve received bytes before SDK parsing.""" + + def __init__(self): + self.directory = None + self.receipt = None + + def arm(self, directory): + self.directory, self.receipt = Path(directory), None + + def __call__(self, response): + if response.request.url.path != f"/v1/encode/{MODEL}": + return + require(self.directory is not None and self.receipt is None, "Unexpected extra embedding dispatch") + raw = response.read() + write(self.directory / "response.bin", raw) + receipt = { + "http_status": response.status_code, + "request_wire_sha256": hashlib.sha256(response.request.content).hexdigest(), + "response_sha256": hashlib.sha256(raw).hexdigest(), + "headers": {key: response.headers[key] for key in HEADER_NAMES if key in response.headers}, + "usage": None, + "response_model": None, + } + try: + if "json" in response.headers.get("content-type", ""): + envelope = response.json() + else: + envelope = unpackb(raw, numeric_arrays=True) + if isinstance(envelope, dict): + receipt["usage"] = envelope.get("usage") + receipt["response_model"] = envelope.get("model") + except Exception: + receipt["body_parse_status"] = "unparsed; exact raw bytes preserved" + write(self.directory / "http.json", receipt) + self.receipt = receipt + require( + response.status_code != 202, "Asynchronous continuation unsupported; preserve intent without redispatch" + ) + + +def charge_from_wire(receipt): + if receipt is None: + return None + usage, headers = receipt.get("usage") or {}, receipt["headers"] + for value in ( + usage.get("credits_charged_decimal"), + headers.get("x-sie-credits-debited-decimal"), + usage.get("credits_charged"), + headers.get("x-sie-credits-debited"), + ): + if value is not None: + return decimal(value) + return None + + +def validate_reply(reply, rows, wire, execution, quote, expected_wire_sha): + require(isinstance(reply, list) and len(reply) == len(rows), "Wrong batch result count") + ids = [item.get("id") for item in reply] + require(len(set(ids)) == len(ids) and set(ids) == {row["id"] for row in rows}, "Wrong or duplicate response IDs") + require( + all(item.get("model") == MODEL for item in reply) and wire["response_model"] == MODEL, + "Actual response model changed", + ) + require(wire["request_wire_sha256"] == expected_wire_sha, "SDK wire request differs from the quoted request") + request_ids = {item.get("request", {}).get("id") for item in reply} + require( + len(request_ids) == 1 and None not in request_ids and "" not in request_ids, "Missing/inconsistent request ID" + ) + request_id = next(iter(request_ids)) + require(request_id == wire["headers"].get("x-sie-request-id"), "Body/header request IDs differ") + require( + isinstance(execution, str) and re.fullmatch(r"[0-9a-f]{64}", execution), "Missing execution bundle revision" + ) + require(execution == wire["headers"].get("x-sie-model-revision"), "Execution revision differs") + # Execution bundle SHA64 is deliberately NOT compared with the HF weights SHA40. + charge = charge_from_wire(wire) + require(charge is not None, "Response has no settled charge; it is not free") + estimate = decimal(quote.get("estimated_credits_decimal", quote.get("estimated_credits"))) + require(charge <= estimate, "Settled charge exceeds the quoted budget ceiling") + rate_book = (wire.get("usage") or {}).get("rate_book_version") + require(rate_book == quote.get("rate_book_version"), "Rate book changed between quote and dispatch") + by_id = {item["id"]: item for item in reply} + vectors = np.asarray([by_id[row["id"]]["dense"] for row in rows], dtype=np.float64) + require(vectors.shape == (len(rows), DIMENSIONS) and np.isfinite(vectors).all(), "Bad vector shape or values") + norms = np.linalg.norm(vectors, axis=1, keepdims=True) + require(np.isfinite(norms).all() and (norms > 0).all(), "Zero/nonfinite vector norm") + return (vectors / norms).astype(np.float32), { + "request_id": request_id, + "model": MODEL, + "execution_bundle_sha256": execution, + "catalog_hf_revision": REVISION, + "credits_decimal": str(charge), + "usd_at_declared_credit_price": str(charge * CREDIT_USD), + "rate_book_version": rate_book, + "usage": wire["usage"], + "request_metadata": [item["request"] for item in reply], + } + + +def collect_batches(rows, output, plan, client, capture, *, commit=lambda: None): + """One sequential attempt. Any failure stops; this never resumes intents.""" + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + require(len(rows) == len({row["id"] for row in rows}) == plan["n"], "Wrong input IDs/count") + require(all(isinstance(row["id"], str) and row["id"] for row in rows), "Invalid input ID") + require(all(isinstance(row["request"], str) and row["request"].strip() for row in rows), "Empty request") + require(type(plan["batch_size"]) is int and 1 <= plan["batch_size"] <= 16, "Invalid batch size") + require(decimal(plan["budget_usd"]) > 0 and plan["max_seconds"] > 180, "Invalid budget/deadline") + require(plan["instruction"] == INSTRUCTION, "Frozen task-neutral instruction changed") + require( + plan.get("model", MODEL) == MODEL + and plan.get("hf_revision", REVISION) == REVISION + and plan.get("dimensions", DIMENSIONS) == DIMENSIONS, + "Planned encoder identity changed", + ) + write(output / "plan.json", plan) + commit() + started, known, completed, execution = time.monotonic(), Decimal(0), [], None + terminal = { + "status": "incomplete", + "expected_rows": len(rows), + "complete_rows": 0, + "known_credits_decimal": "0", + "unknown_charge_batches": [], + "batch_receipts": [], + } + current, quote, intent, dispatched = None, None, None, False + try: + metadata = client.get_model(MODEL) + require( + metadata.get("revision") == REVISION and metadata.get("dims", {}).get("dense") == DIMENSIONS, + "Catalog model differs from the pinned encoder", + ) + require(metadata.get("name", metadata.get("id", MODEL)) == MODEL, "Catalog model ID differs") + write(output / "catalog.json", metadata) + commit() + for number, offset in enumerate(range(0, len(rows), plan["batch_size"])): + require(time.monotonic() - started < plan["max_seconds"] - 180, "Internal collection deadline reached") + batch = rows[offset : offset + plan["batch_size"]] + current, quote, intent, dispatched = output / "batches" / f"{number:06d}", None, None, False + capture.receipt = None + body = body_for(batch, plan["instruction"]) + body_bytes = packb(body, use_bin_type=True) + quote = client.estimate(f"/v1/encode/{MODEL}", body, timeout=30) + require( + quote.get("identity", {}).get("model") == MODEL and quote["identity"].get("operation") == "encode", + "Quote identity changed", + ) + require( + isinstance(quote.get("rate_book_version"), str) + and re.fullmatch(r"[0-9a-f]{64}", quote.get("rate_book_sha256", "")), + "Missing quoted price-book binding", + ) + ceiling = decimal(quote.get("estimated_credits_decimal", quote.get("estimated_credits"))) + require(known + ceiling <= decimal(plan["budget_usd"]) / CREDIT_USD, "Next batch exceeds remaining budget") + prepared = { + "batch": number, + "ids": [row["id"] for row in batch], + "request": body, + "request_sha256": hashlib.sha256(encoded(body)).hexdigest(), + "expected_wire_sha256": hashlib.sha256(body_bytes).hexdigest(), + "quote": quote, + "physical_attempt": "one; never automatically repeated", + } + write(current / "intent.json", prepared) + intent = prepared + commit() # A durable intent must precede any chargeable request. + capture.arm(current) + before = time.monotonic() + dispatched = True + reply = client.encode( + MODEL, + body["items"], + output_types=["dense"], + instruction=plan["instruction"], + output_dtype="float32", + is_query=True, + wait_for_capacity=False, + max_oom_retries=0, + provision_timeout_s=150, + ) + vectors, receipt = validate_reply( + reply, batch, capture.receipt, client.last_model_revision, quote, intent["expected_wire_sha256"] + ) + require(client.last_retry_count == 0, "Unexpected SDK retry") + require( + execution is None or execution == receipt["execution_bundle_sha256"], + "Execution bundle changed mid-collection", + ) + execution = receipt["execution_bundle_sha256"] + with (current / "vectors.npy.pending").open("xb") as stream: + np.save(stream, vectors, allow_pickle=False) + stream.flush() + os.fsync(stream.fileno()) + (current / "vectors.npy.pending").replace(current / "vectors.npy") + receipt.update( + status="complete", + batch=number, + ids=intent["ids"], + elapsed_seconds=time.monotonic() - before, + intent_sha256=sha(current / "intent.json"), + response_sha256=sha(current / "response.bin"), + http_sha256=sha(current / "http.json"), + vectors_sha256=sha(current / "vectors.npy"), + ) + write(current / "receipt.json", receipt) + known += decimal(receipt["credits_decimal"]) + completed.append(receipt) + terminal.update( + complete_rows=sum(len(item["ids"]) for item in completed), + known_credits_decimal=str(known), + batch_receipts=[ + { + "path": f"batches/{item['batch']:06d}/receipt.json", + "sha256": sha(output / "batches" / f"{item['batch']:06d}" / "receipt.json"), + } + for item in completed + ], + ) + write(output / "progress.json", terminal, replace=True) + commit() # Vectors AND charge are durable before the next batch starts. + print(f"Embedded {terminal['complete_rows']}/{len(rows)}; settled credits {known}", flush=True) + current, quote, intent, dispatched = None, None, None, False + terminal["status"] = "complete" + except Exception as error: + terminal.update(error_type=type(error).__name__, error=str(error)) + if current is not None and intent is not None and dispatched: + try: + charge = charge_from_wire(capture.receipt) + except ValueError: + charge = None + failed = { + "status": "failed_or_unknown", + "ids": intent["ids"], + "error_type": type(error).__name__, + "intent_sha256": sha(current / "intent.json"), + "credits_decimal": None if charge is None else str(charge), + "quoted_credit_ceiling": str( + decimal(quote.get("estimated_credits_decimal", quote.get("estimated_credits"))) + ), + "response_received": capture.receipt is not None, + "retry_permitted": False, + } + if not (current / "receipt.json").exists(): + write(current / "failure.json", failed) + if charge is None: + terminal["unknown_charge_batches"].append(failed) + else: + known += charge + terminal["known_credits_decimal"] = str(known) + finally: + terminal.update( + elapsed_seconds=time.monotonic() - started, + execution_bundle_sha256=execution, + plan_sha256=sha(output / "plan.json"), + ordered_ids_sha256=hashlib.sha256(encoded([row["id"] for row in rows])).hexdigest(), + catalog_hf_revision=REVISION, + price_usd_per_credit=str(CREDIT_USD), + known_usd_at_declared_credit_price=str(known * CREDIT_USD), + estimated_unknown_usd_upper_bound=str( + sum((decimal(x["quoted_credit_ceiling"]) for x in terminal["unknown_charge_batches"]), Decimal(0)) + * CREDIT_USD + ), + cost_scope="Observed batch charges, not singleton deployment fees. Unknown charges are not zero.", + runtime_scope="Question-only input packet; the shared artifact Volume is technically accessible.", + ) + write(output / "collection.json", terminal) + commit() + return terminal + + +def load_vectors(output, rows): + """Verify a complete collection and return aligned IDs, normalized vectors, metadata.""" + output = Path(output) + collection = json.loads((output / "collection.json").read_text()) + require(collection["status"] == "complete" and not collection["unknown_charge_batches"], "Collection incomplete") + require(sha(output / "plan.json") == collection["plan_sha256"], "Collection plan changed") + ids = [row["id"] for row in rows] + require(hashlib.sha256(encoded(ids)).hexdigest() == collection["ordered_ids_sha256"], "Input order changed") + accumulated, vectors, charges = [], [], Decimal(0) + for number, item in enumerate(collection["batch_receipts"]): + require(item["path"] == f"batches/{number:06d}/receipt.json", "Noncanonical batch path") + receipt_path = output / item["path"] + require(sha(receipt_path) == item["sha256"], "Batch receipt changed") + receipt = json.loads(receipt_path.read_text()) + require( + receipt["status"] == "complete" + and receipt["model"] == MODEL + and receipt["catalog_hf_revision"] == REVISION + and receipt["execution_bundle_sha256"] == collection["execution_bundle_sha256"], + "Batch identity changed", + ) + for name, key in ( + ("intent.json", "intent_sha256"), + ("response.bin", "response_sha256"), + ("http.json", "http_sha256"), + ("vectors.npy", "vectors_sha256"), + ): + require(sha(receipt_path.parent / name) == receipt[key], f"Batch {name} changed") + intent = json.loads((receipt_path.parent / "intent.json").read_text()) + count = len(receipt["ids"]) + expected = rows[len(accumulated) : len(accumulated) + count] + require( + intent["request"] == body_for(expected, INSTRUCTION) and intent["ids"] == receipt["ids"], + "Vector request differs from exact input text", + ) + matrix = np.load(receipt_path.parent / "vectors.npy", allow_pickle=False) + require( + matrix.dtype == np.float32 and matrix.shape == (count, DIMENSIONS) and np.isfinite(matrix).all(), + "Invalid stored vectors", + ) + require( + np.allclose(np.linalg.norm(matrix.astype(np.float64), axis=1), 1, rtol=1e-6, atol=1e-7), + "Stored vectors are not normalized", + ) + accumulated.extend(receipt["ids"]) + vectors.append(matrix) + charges += decimal(receipt["credits_decimal"]) + require( + accumulated == ids and len(ids) == collection["complete_rows"] == collection["expected_rows"], + "Incomplete vector ID alignment", + ) + require(charges == decimal(collection["known_credits_decimal"]), "Collection charge sum differs") + return ids, np.concatenate(vectors), collection + + +def make_client(capture): + http = httpx.Client(base_url=BASE_URL, follow_redirects=False, timeout=150, event_hooks={"response": [capture]}) + return SingleSendClient( + base_url=BASE_URL, + api_key=os.environ["SIE_API_KEY"], + timeout_s=150, + read_timeout_s=150, + connect_timeout_s=15, + http_client=http, + ) diff --git a/examples/train-a-router/evaluation/__init__.py b/examples/train-a-router/evaluation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/train-a-router/evaluation/frontier.py b/examples/train-a-router/evaluation/frontier.py new file mode 100644 index 000000000..c97a9d215 --- /dev/null +++ b/examples/train-a-router/evaluation/frontier.py @@ -0,0 +1,392 @@ +"""Whole calibration frontier and unchanged final replay; no model fitting.""" + +from __future__ import annotations + +import argparse +from fractions import Fraction +from pathlib import Path + +import numpy as np + +from embedding import recipes as core +from evaluation import sweep, utility_router as utility + +PROTOCOL = { + "format": "bbeh-frontier-evaluation-v1", + "population": "Uniform per retained question; exact original source roles/families", + "thresholds": "All calibration crossings/intervals, cost limit and always-arm bypasses; no preferred gap target", + "final": "Replay saved calibration candidates only; never select on final quality", + "bootstrap": {"draws": 2000, "seed": 89, "unit": "paired whole family within task, fixed observed task weights"}, + "interval_scope": ( + "Pointwise for calibration-frontier rules, conditional on observed tasks; " + "no simultaneous curve guarantee" + ), + "random": "Analytic expected mixtures; p for each matched calibration-budget comparator is frozen on calibration", + "cost": ( + "Selected answer USD plus validated singleton encoder fee when applicable; " + "other hosting unpriced, no zero imputation" + ), +} + + +def checked_scores(plan, rows, paths, role): + methods, bindings = {}, {} + for path in paths: + doc = core.read(path) + core.require( + doc["format"] == "bbeh-input-only-scores-v1" + and doc["role"] == role + and doc["source_plan_sha256"] == plan["source_plan_sha256"] + and doc["pair_sha256"] == plan["pair_sha256"] + and doc["outcomes_accessed"] is False + and doc["ids"] == [r["id"] for r in rows], + "Score source/role/order mismatch", + ) + core.require_hash(doc["model_sha256"]) + for name, value in doc["methods"].items(): + core.require( + name not in methods + and value["router_kind"] in {"embedding", "cpu", "native"} + and type(value["metadata_assisted"]) is bool, + "Duplicate/invalid score method", + ) + p = utility._matrix(value["probabilities"], "probabilities", probability=True) + c = utility._matrix(value["expected_usd"], "expected USD") + core.require(len(p) == len(c) == len(rows), "Score count mismatch") + methods[name] = value + bindings[name] = {"score_sha256": core.sha(path), "model_sha256": doc["model_sha256"]} + core.require(bool(methods), "No methods supplied") + return methods, bindings + + +def fee_rows(plan, rows, path): + info = core.read(path) + core.require( + info["source_plan_sha256"] == plan["source_plan_sha256"] + and info["model"] == core.MODEL + and info["hf_revision"] == core.REVISION + and info["instruction"] == core.INSTRUCTION + and info["cost_basis"] in {"observed_singleton", "validated_singleton_projection"}, + "Need a verified per-request encoder fee basis", + ) + by_id = {row["id"]: row for row in info["rows"]} + core.require(len(by_id) == len(info["rows"]), "Duplicate fee ID") + values = [utility._number(float(by_id[row["id"]]["projected_singleton_usd"]), "fee USD") for row in rows] + return values + + +def random_probability(spend, cheap, strong): + if strong == cheap: + return None + value = (spend - cheap) / (strong - cheap) + return float(value) if 0 <= value <= 1 else None + + +def randomized_hull(candidates): + """Upper quality hull over already nondominated cost-ordered candidates.""" + hull = [] + for row in candidates: + metric = row["calibration"] + x = Fraction.from_float(metric["selection_cost_usd_per10000"]) + y = Fraction.from_float(metric["answer_accuracy_percent"]) + while len(hull) >= 2: + _, ax, ay = hull[-2] + _, bx, by = hull[-1] + if (by - ay) * (x - bx) > (y - by) * (bx - ax): + break + hull.pop() + hull.append((row["candidate_id"], x, y)) + return [row[0] for row in hull] + + +def enumerate_frontier(value, y, costs, fees): + # The reused implementation requires a nonempty target list. Only its + # target-independent enumeration is retained; no target choice is exposed. + result = utility.calibrate( + value["probabilities"], value["expected_usd"], y, costs, router_usd=fees, targets_pp=(0,) + ) + result.pop("selected") + result.pop("targets_pp") + result["no_preferred_quality_target"] = True + result["randomized_hull_candidate_ids"] = randomized_hull(result["frontier"]) + result["randomized_hull_interpretation"] = "Mixtures between saved rules; not additional deterministic policies" + cheap, strong = costs.mean(axis=0) + for row in result["candidates"]: + spend = row["calibration"]["selection_cost_usd_per10000"] / 10000 + row["calibration_matched_random_strong_probability"] = random_probability(spend, cheap, strong) + return result + + +def calibrate(run, grade_path, score_paths, fees_path, output): + run = Path(run) + plan = core.checked_plan(run) + rows = core.role_rows(run, plan, "calibration") + y, costs, grade = core.grades(plan, rows, grade_path, "calibration") + scores, bindings = checked_scores(plan, rows, score_paths, "calibration") + fees = fee_rows(plan, rows, fees_path) + methods = {} + for name, value in scores.items(): + result = enumerate_frontier(value, y, costs, fees if value["router_kind"] == "embedding" else None) + methods[name] = { + **result, + "metadata_assisted": value["metadata_assisted"], + "router_kind": value["router_kind"], + **bindings[name], + } + result = { + "format": "bbeh-calibration-frontiers-v1", + "protocol": PROTOCOL, + "plan_sha256": core.sha(run / "plan.json"), + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "grades_sha256": core.sha(grade_path), + "grader_sha256": grade["grader_sha256"], + "cost_scope": grade["cost_scope"], + "fees_sha256": core.sha(fees_path), + "ids": [r["id"] for r in rows], + "methods": methods, + "random": {"strong_probability_range": [0, 1], "analytic_expectation": True}, + "implementation_sha256": { + "frontier": core.sha(__file__), + "utility": core.sha(utility.__file__), + "sweep": core.sha(sweep.__file__), + }, + "final_outcomes_accessed": False, + } + core.write(output, result) + return result + + +def freeze(run, calibration_path, score_paths, fees_path, output): + run = Path(run) + plan, calibration = core.checked_plan(run), core.read(calibration_path) + core.require( + calibration["format"] == "bbeh-calibration-frontiers-v1" + and calibration["protocol"] == PROTOCOL + and calibration["plan_sha256"] == core.sha(run / "plan.json") + and calibration["fees_sha256"] == core.sha(fees_path) + and calibration["implementation_sha256"]["frontier"] == core.sha(__file__), + "Calibration source/protocol changed", + ) + rows = core.role_rows(run, plan, "final") + methods, bindings = checked_scores(plan, rows, score_paths, "final") + core.require(set(methods) == set(calibration["methods"]), "All calibrated methods need final input scores") + fees = fee_rows(plan, rows, fees_path) + saved = {} + for name, scores in methods.items(): + prior = calibration["methods"][name] + core.require( + prior["model_sha256"] == bindings[name]["model_sha256"] + and prior["router_kind"] == scores["router_kind"] + and prior["metadata_assisted"] == scores["metadata_assisted"], + "Model changed after calibration", + ) + saved[name] = { + "scores": scores, + **bindings[name], + "candidates": prior["candidates"], + "calibration_frontier_candidate_ids": [r["candidate_id"] for r in prior["frontier"]], + "calibration_randomized_hull_candidate_ids": prior["randomized_hull_candidate_ids"], + } + result = { + "format": "bbeh-frozen-frontiers-v1", + "protocol": PROTOCOL, + "plan_sha256": core.sha(run / "plan.json"), + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "calibration_sha256": core.sha(calibration_path), + "calibration_grader_sha256": calibration["grader_sha256"], + "cost_scope": calibration["cost_scope"], + "source_rows": [{k: r[k] for k in ("id", "dataset", "task", "family", "role")} for r in rows], + "ids": [r["id"] for r in rows], + "methods": saved, + "encoder_fees_usd": fees, + "fees_sha256": core.sha(fees_path), + "final_outcomes_accessed": False, + "frozen_before_final_outcomes": True, + "implementation_sha256": calibration["implementation_sha256"], + } + core.write(output, result) + return result + + +class Bootstrap: + def __init__(self, rows, draws=2000): + self.parts, self.draws = [], draws + families = {} + for row in rows: + core.require( + families.setdefault(row["family"], row["task"]) == row["task"], + "A family spans tasks; requires a joint resampling design", + ) + rng = np.random.default_rng(89) + for task in sorted({r["task"] for r in rows}): + ids = [i for i, r in enumerate(rows) if r["task"] == task] + families = sorted({rows[i]["family"] for i in ids}) + group = np.array([families.index(rows[i]["family"]) for i in ids]) + weights = rng.multinomial(len(families), [1 / len(families)] * len(families), size=draws) + denominator = weights @ np.bincount(group, minlength=len(families)) + self.parts.append((ids, group, weights, denominator, len(ids) / len(rows))) + + def means(self, values): + values = np.asarray(values) + result = np.zeros((self.draws, values.shape[1])) + for ids, group, weights, denominator, proportion in self.parts: + sums = np.array( + [ + np.bincount(group, weights=values[ids, col], minlength=weights.shape[1]) + for col in range(values.shape[1]) + ] + ).T + result += proportion * (weights @ sums) / denominator[:, None] + return result + + +def interval(values): + values = np.asarray(values) + finite = values[np.isfinite(values)] + return { + "lower": float(np.quantile(finite, 0.025)) if len(finite) else None, + "upper": float(np.quantile(finite, 0.975)) if len(finite) else None, + "defined_draws": len(finite), + "total_draws": len(values), + } + + +def uncertainty(bootstrap, routes, y, costs, fees, random_p): + index = np.arange(len(y)) + chosen_y, chosen_cost = y[index, routes.astype(int)], costs[index, routes.astype(int)] + fee_array = np.zeros(len(y)) if fees is None else np.asarray(fees) + matrix = np.column_stack((y[:, 0], y[:, 1], costs[:, 0], costs[:, 1], chosen_y, chosen_cost, fee_array)) + means = bootstrap.means(matrix) + baseline = means[:, 3] + with np.errstate(divide="ignore", invalid="ignore"): + values = { + "accuracy_percent": 100 * means[:, 4], + "quality_gap_pp": 100 * (means[:, 1] - means[:, 4]), + "accuracy_gain_over_luna_pp": 100 * (means[:, 4] - means[:, 0]), + "answer_usd_per_10000": 10000 * means[:, 5], + "answer_savings_percent": 100 * (1 - means[:, 5] / baseline), + } + if fees is not None: + values["total_usd_per_10000"] = 10000 * (means[:, 5] + means[:, 6]) + values["net_savings_percent"] = 100 * (1 - (means[:, 5] + means[:, 6]) / baseline) + if random_p is not None: + random_y = (1 - random_p) * means[:, 0] + random_p * means[:, 1] + random_cost = (1 - random_p) * means[:, 2] + random_p * means[:, 3] + values["accuracy_gain_over_calibration_budget_random_pp"] = 100 * (means[:, 4] - random_y) + values["cost_difference_from_calibration_budget_random_usd_per_10000"] = 10000 * ( + means[:, 5] + means[:, 6] - random_cost + ) + return {name: interval(value) for name, value in values.items()} + + +def evaluate(run, frozen_path, grades_path, output, *, draws=2000): + run = Path(run) + plan, frozen = core.checked_plan(run), core.read(frozen_path) + core.require( + frozen["format"] == "bbeh-frozen-frontiers-v1" + and frozen["protocol"] == PROTOCOL + and frozen["plan_sha256"] == core.sha(run / "plan.json") + and frozen["implementation_sha256"]["frontier"] == core.sha(__file__) + and frozen["frozen_before_final_outcomes"] is True, + "Missing original frozen lineage", + ) + rows = core.role_rows(run, plan, "final") + metadata = [{k: r[k] for k in ("id", "dataset", "task", "family", "role")} for r in rows] + core.require( + frozen["source_rows"] == metadata and frozen["ids"] == [r["id"] for r in rows], + "Final source population changed", + ) + y, costs, grade = core.grades(plan, rows, grades_path, "final") + core.require( + grade["grader_sha256"] == frozen["calibration_grader_sha256"] + or grade.get("parent_grader_sha256") == frozen["calibration_grader_sha256"], + "Final grading semantics changed", + ) + bootstrap = Bootstrap(rows, draws) + methods = {} + for name, saved in frozen["methods"].items(): + scores = saved["scores"] + fees = frozen["encoder_fees_usd"] if scores["router_kind"] == "embedding" else None + candidates = saved["candidates"] + points = sweep.sweep_metrics( + [r["policy"] for r in candidates], scores["probabilities"], scores["expected_usd"], y, costs, fees + ) + result = [] + for candidate, metrics in zip(candidates, points, strict=True): + node = {"candidate_id": candidate["candidate_id"], "rule": candidate["policy"], **metrics} + if candidate["candidate_id"] in saved["calibration_frontier_candidate_ids"]: + routes = utility.replay(candidate["policy"], scores["probabilities"], scores["expected_usd"]) + charged = np.zeros(len(rows)) if candidate["policy"]["mode"].startswith("always_") else fees + p = candidate["calibration_matched_random_strong_probability"] + node["calibration_matched_random_strong_probability"] = p + node["paired_ci95"] = uncertainty(bootstrap, routes, y, costs, charged, p) + result.append(node) + methods[name] = { + "points": result, + "metadata_assisted": scores["metadata_assisted"], + "router_kind": scores["router_kind"], + "calibration_frontier_candidate_ids": saved["calibration_frontier_candidate_ids"], + "native_or_cpu_hosting_priced": False, + } + result = { + "format": "bbeh-frontier-final-report-v1", + "protocol": PROTOCOL, + "frozen_sha256": core.sha(frozen_path), + "grades_sha256": core.sha(grades_path), + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "n": len(rows), + "families": len({r["family"] for r in rows}), + "methods": methods, + "random": { + "analytic_expectation": True, + "strong_probability_range": [0, 1], + "accuracy_endpoints_percent": (100 * y.mean(axis=0)).tolist(), + "answer_usd_per_10000_endpoints": (10000 * costs.mean(axis=0)).tolist(), + }, + "bootstrap_draws": draws, + "no_final_selection": True, + "cost_scope": grade["cost_scope"], + "router_cost_scope": ( + "Embedding answer+encoder API fees; local head hosting unpriced. " + "Native/text/task hosting unpriced: answer-only curves." + ), + "latency": None, + "latency_note": ( + "Attach independently measured resident inference receipts; " + "batch-fit timing is not serving latency." + ), + } + core.write(output, result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + for name in ("calibrate", "freeze", "evaluate"): + p = sub.add_parser(name) + p.add_argument("--run", required=True) + p.add_argument("--output", required=True) + if name in {"calibrate", "evaluate"}: + p.add_argument("--grades", required=True) + if name in {"calibrate", "freeze"}: + p.add_argument("--scores", action="append", required=True) + p.add_argument("--fees", required=True) + if name == "freeze": + p.add_argument("--calibration", required=True) + if name == "evaluate": + p.add_argument("--frozen", required=True) + args = parser.parse_args() + if args.command == "calibrate": + calibrate(args.run, args.grades, args.scores, args.fees, args.output) + elif args.command == "freeze": + freeze(args.run, args.calibration, args.scores, args.fees, args.output) + else: + evaluate(args.run, args.frozen, args.grades, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/evaluation/report.py b/examples/train-a-router/evaluation/report.py new file mode 100644 index 000000000..848319a2f --- /dev/null +++ b/examples/train-a-router/evaluation/report.py @@ -0,0 +1,618 @@ +"""Render frozen calibration policies on final data as an offline HTML explorer. + +This module validates report lineage and formats saved metrics. It does not fit, +grade, select policies, recompute a frontier, or require third-party packages. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path + + +METHOD_LABELS = { + "base": "Laya (unchanged weights)", + "lora": "Laya + LoRA", + "embedding": "Embedding + logistic heads", + "knn": "Embedding nearest neighbors", + "text_length": "Text + length", + "text_topic": "Predicted task from text", + "task_metadata": "True task metadata (assisted control)", +} + + +def method_label(name): + original = name.removesuffix("__textcost") + label = METHOD_LABELS.get(original, original) + return label + " · predicted cost" if original != name else label + + +METRICS = ( + "answer_accuracy_percent", + "quality_gap_pp", + "answer_usd_per_10000", + "router_usd_per_10000", + "total_usd_per_10000", + "strong_share", + "answer_savings_percent", + "net_savings_percent", +) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def number(value, name, *, nullable=False): + if value is None and nullable: + return + require( + type(value) in (int, float) and math.isfinite(value), + f"Invalid finite metric: {name}", + ) + + +def unique_map(rows, key, context): + require(isinstance(rows, list), f"Expected a list: {context}") + result = {} + for row in rows: + value = row[key] + require( + type(value) is int and value >= 0 and value not in result, + f"Invalid/duplicate ID: {context}", + ) + result[value] = row + return result + + +def checked_data(calibration_path, frozen_path, final_path): + """Return presentation data after verifying the three saved artifact links.""" + paths = { + "calibration": Path(calibration_path), + "frozen": Path(frozen_path), + "final_report": Path(final_path), + } + docs = {key: json.loads(path.read_text()) for key, path in paths.items()} + hashes = {key: sha(path) for key, path in paths.items()} + cal, frozen, final = ( + docs[key] for key in ("calibration", "frozen", "final_report") + ) + for doc, expected in ( + (cal, "bbeh-calibration-frontiers-v1"), + (frozen, "bbeh-frozen-frontiers-v1"), + (final, "bbeh-frontier-final-report-v1"), + ): + require(doc["format"] == expected, "Unsupported report format") + require( + cal["protocol"] == frozen["protocol"] == final["protocol"], "Protocol mismatch" + ) + require( + cal["final_outcomes_accessed"] is False + and frozen["final_outcomes_accessed"] is False + and frozen["frozen_before_final_outcomes"] is True + and final["no_final_selection"] is True, + "Missing calibration-before-final selection boundary", + ) + require( + frozen["calibration_sha256"] == hashes["calibration"], + "Calibration file hash mismatch", + ) + require(final["frozen_sha256"] == hashes["frozen"], "Frozen file hash mismatch") + for key in ("source_plan_sha256", "pair_sha256", "cost_scope"): + require( + cal[key] == frozen[key] == final[key], f"Artifact binding mismatch: {key}" + ) + for key in ("plan_sha256", "fees_sha256", "implementation_sha256"): + require(cal[key] == frozen[key], f"Calibration/freeze mismatch: {key}") + require( + cal["grader_sha256"] == frozen["calibration_grader_sha256"], + "Calibration grader mismatch", + ) + source = frozen["source_rows"] + require( + len(source) == final["n"] > 0 + and len({row["family"] for row in source}) == final["families"] + and len(set(frozen["ids"])) == len(source) + and frozen["ids"] == [row["id"] for row in source] + and all(row["role"] == "final" for row in source), + "Final population mismatch", + ) + require( + set(cal["methods"]) == set(frozen["methods"]) == set(final["methods"]), + "Method set mismatch", + ) + random = final["random"] + require( + random["analytic_expectation"] is True + and random["strong_probability_range"] == [0, 1], + "Expected the analytic always-arm mixture", + ) + for key in ("accuracy_endpoints_percent", "answer_usd_per_10000_endpoints"): + require(len(random[key]) == 2, "Invalid random baseline endpoints") + for value in random[key]: + number(value, key) + require( + all(0 <= v <= 100 for v in random["accuracy_endpoints_percent"]), + "Invalid baseline accuracy", + ) + require( + all(v >= 0 for v in random["answer_usd_per_10000_endpoints"]), + "Negative baseline cost", + ) + + points, methods, baselines = [], [], {} + order = {name: index for index, name in enumerate(METHOD_LABELS)} + for name in sorted( + cal["methods"], key=lambda item: (order.get(item.removesuffix("__textcost"), len(order)), item) + ): + prior, saved, result = ( + cal["methods"][name], + frozen["methods"][name], + final["methods"][name], + ) + candidates = unique_map(prior["candidates"], "candidate_id", name) + final_points = unique_map(result["points"], "candidate_id", name) + frontier = unique_map(prior["frontier"], "candidate_id", name + " frontier") + require( + list(frontier) + == saved["calibration_frontier_candidate_ids"] + == result["calibration_frontier_candidate_ids"] + and prior["candidates"] == saved["candidates"] + and set(candidates) == set(final_points) + and all(candidates[cid] == row for cid, row in frontier.items()), + f"Saved candidate/frontier mismatch: {name}", + ) + require( + prior["no_preferred_quality_target"] is True, + "Expected target-independent calibration", + ) + require( + prior["model_sha256"] == saved["model_sha256"], + "Model changed after calibration", + ) + kind, assisted = result["router_kind"], result["metadata_assisted"] + require( + kind in {"embedding", "native", "cpu"} + and type(assisted) is bool + and prior["router_kind"] == saved["scores"]["router_kind"] == kind + and prior["metadata_assisted"] + == saved["scores"]["metadata_assisted"] + == assisted, + f"Method identity changed: {name}", + ) + require( + result["native_or_cpu_hosting_priced"] is False, + "This report schema has no bound native/CPU hosting-price receipt", + ) + label = method_label(name) + if assisted and name.removesuffix("__textcost") != "task_metadata": + label += " (true task metadata; assisted control)" + methods.append( + {"name": name, "label": label, "kind": kind, "assisted": assisted} + ) + for cid, point in final_points.items(): + candidate = candidates[cid] + require( + point["rule"] == candidate["policy"], + f"Final rule differs from calibration: {name}/{cid}", + ) + for metric in METRICS: + number( + point[metric], + metric, + nullable=metric + in { + "router_usd_per_10000", + "total_usd_per_10000", + "answer_savings_percent", + "net_savings_percent", + }, + ) + require( + 0 <= point["answer_accuracy_percent"] <= 100 + and 0 <= point["strong_share"] <= 1, + "Accuracy/share outside its range", + ) + require(point["answer_usd_per_10000"] >= 0, "Negative answer cost") + bypass = point["rule"]["mode"] in {"always_cheap", "always_strong"} + if kind != "embedding" and not bypass: + require( + point["router_usd_per_10000"] is None + and point["total_usd_per_10000"] is None + and point["net_savings_percent"] is None, + "Unpriced hosting must remain unknown", + ) + else: + require( + point["router_usd_per_10000"] is not None + and point["total_usd_per_10000"] is not None, + "Missing known fee/bypass cost", + ) + require( + point["router_usd_per_10000"] >= 0 + and math.isclose( + point["total_usd_per_10000"], + point["answer_usd_per_10000"] + point["router_usd_per_10000"], + rel_tol=1e-10, + abs_tol=1e-10, + ), + "Answer plus router cost mismatch", + ) + require( + not bypass or point["router_usd_per_10000"] == 0, + "Static baseline charged a router fee", + ) + if cid in frontier: + require(point.get("paired_ci95"), "Missing frontier uncertainty") + require( + point.get("calibration_matched_random_strong_probability") + == candidate["calibration_matched_random_strong_probability"], + "Random comparator changed", + ) + probability = point["calibration_matched_random_strong_probability"] + number(probability, "random strong probability", nullable=True) + require( + probability is None or 0 <= probability <= 1, + "Random probability outside [0, 1]", + ) + for metric, interval in point.get("paired_ci95", {}).items(): + lower, upper = interval["lower"], interval["upper"] + number(lower, metric, nullable=True) + number(upper, metric, nullable=True) + defined, total = interval["defined_draws"], interval["total_draws"] + require( + type(defined) is int + and type(total) is int + and 0 <= defined <= total + and total == final["bootstrap_draws"], + "Invalid interval draw counts", + ) + require( + (defined == 0 and lower is None and upper is None) + or ( + defined > 0 + and lower is not None + and upper is not None + and lower <= upper + ), + "Invalid uncertainty bounds", + ) + item = { + **point, + "key": f"{name}:{cid}", + "method": name, + "label": label, + "kind": kind, + "assisted": assisted, + "calibration": candidate["calibration"], + "on_calibration_frontier": cid in frontier, + "baseline": False, + } + if bypass: + arm = int(point["rule"]["mode"] == "always_strong") + require( + math.isclose( + point["answer_accuracy_percent"], + random["accuracy_endpoints_percent"][arm], + abs_tol=1e-10, + ) + and math.isclose( + point["answer_usd_per_10000"], + random["answer_usd_per_10000_endpoints"][arm], + abs_tol=1e-10, + ), + "Baseline disagrees with analytic endpoints", + ) + previous = baselines.get(arm) + if previous is None or ( + not previous.get("paired_ci95") and point.get("paired_ci95") + ): + baselines[arm] = { + **item, + "key": f"baseline:{arm}", + "method": "baseline", + "baseline": True, + "label": "Always Sol" if arm else "Always Luna", + "assisted": False, + "candidate_id": None, + } + elif cid in frontier: + points.append(item) + require(set(baselines) == {0, 1}, "Missing always-arm baselines") + # Order within each method is the saved calibration frontier order. Never + # reorder or remove policies using final accuracy, final cost, or a new hull. + rank = { + (name, cid): i + for name in cal["methods"] + for i, cid in enumerate( + frozen["methods"][name]["calibration_frontier_candidate_ids"] + ) + } + method_order = {method["name"]: i for i, method in enumerate(methods)} + points.sort( + key=lambda row: ( + method_order[row["method"]], + rank[row["method"], row["candidate_id"]], + ) + ) + return { + "format": "bbeh-cost-quality-explorer-v1", + "n": final["n"], + "families": final["families"], + "tasks": len({row["task"] for row in source}), + "calibration_n": len(cal["ids"]), + "methods": methods, + "points": [baselines[0], baselines[1], *points], + "random": random, + "bootstrap_draws": final["bootstrap_draws"], + "protocol": final["protocol"], + "cost_scope": final["cost_scope"], + "router_cost_scope": final["router_cost_scope"], + "latency_note": final["latency_note"], + "input_sha256": hashes, + "source_plan_sha256": final["source_plan_sha256"], + "pair_sha256": final["pair_sha256"], + } + + +CSS = r""" +:root{color-scheme:light;--ink:#17263b;--muted:#46586c;--line:#d8e0e8;--paper:#fff;--accent:#1551a8} +*{box-sizing:border-box}body{margin:0;background:#f4f7fa;color:var(--ink);font:16px/1.5 system-ui,sans-serif} +main{max-width:1420px;margin:auto;padding:28px}h1{font-size:clamp(1.8rem,4vw,2.5rem);line-height:1.2;margin:0 0 12px} +h2{font-size:1.18rem;margin:0 0 12px}h3{font-size:1rem}p{max-width:92ch;margin:10px 0}.muted,caption{color:var(--muted)} +.card{background:var(--paper);border:1px solid var(--line);border-radius:12px;padding:20px;margin:18px 0} +.controls,.legend,.pager,.stats{display:flex;gap:12px;flex-wrap:wrap;align-items:center}.controls{align-items:flex-start} +.controls>*{flex:1;min-width:240px}fieldset{border:1px solid var(--line);border-radius:8px;padding:10px 14px} +legend{font-weight:650}label{cursor:pointer}fieldset label{display:block;margin:5px 0}input{accent-color:var(--accent)} +input[type=search]{font:inherit;padding:8px;width:100%;border:1px solid #72849a;border-radius:6px} +button{font:inherit;background:white;border:1px solid #72849a;border-radius:6px;padding:6px 12px;color:var(--ink);cursor:pointer} +button:hover{background:#edf3fb}button:disabled{opacity:.45;cursor:default}:focus-visible{outline:3px solid #d26c00;outline-offset:3px} +.legend label{font-size:.9rem;display:flex;gap:6px;align-items:center}.mark{font-weight:800;font-size:1.2rem} +.plot{width:100%;height:auto;display:block;min-height:240px}svg text{font:13px system-ui,sans-serif;fill:var(--ink)} +.grid{stroke:#e4e9ef;stroke-width:1}.axis{stroke:#6c7c8c;stroke-width:1.2}.point{cursor:pointer}.point:hover{stroke-width:3} +.point:focus{outline:none;stroke:#111;stroke-width:4}.selected{stroke:#111;stroke-width:3}.random-line{stroke:#4d5965;stroke-width:2;stroke-dasharray:7 5} +.split{display:grid;grid-template-columns:1fr 1fr;gap:18px}.stats div{background:#f4f7fa;padding:8px 12px;border-radius:6px} +.stats strong{font-size:1.25rem;display:block}.scroll{overflow:auto}table{width:100%;border-collapse:collapse;font-size:.9rem} +caption{text-align:left;margin:5px 0 10px}th,td{text-align:left;padding:9px;border-bottom:1px solid var(--line);vertical-align:top} +th{background:#f4f7fa}td.num,th.num{text-align:right;font-variant-numeric:tabular-nums}tr.active{background:#edf3fb} +.pager{justify-content:flex-end;margin-top:12px}code,pre{font:13px/1.4 ui-monospace,monospace;overflow-wrap:anywhere} +pre{white-space:pre-wrap;background:#f4f7fa;padding:12px;border-radius:6px}details{margin:12px 0}summary{cursor:pointer;font-weight:650} +.badge{font-size:.8rem;border:1px solid #916400;border-radius:4px;padding:1px 5px;color:#624200}.notice{border-left:4px solid #bd7a00;padding-left:12px} +.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0} +@media(max-width:800px){main{padding:16px}.card{padding:14px}.split{grid-template-columns:1fr}.plot{min-height:200px}} +@media print{body{background:white}main{max-width:none}.controls,.pager,.legend input,button{display:none}.card{break-inside:avoid}.scroll{overflow:visible}} +""" + + +JS = r""" +'use strict'; +const data=JSON.parse(document.getElementById('report-data').textContent); +const $=id=>document.getElementById(id), NS='http://www.w3.org/2000/svg'; +const colors=['#1358a4','#a83459','#007970','#8b5300','#7250a3','#586a16','#ab4500']; +const symbols=['●','■','▲','◆','▼','⬟','✚']; +const enabled=new Set(data.methods.map(m=>m.name)); +let mode='gross',page=0,selected=data.points[0].key; +const perPage=20; +const money=v=>v===null||v===undefined?'Not priced':'$'+v.toLocaleString('en-US',{minimumFractionDigits:2,maximumFractionDigits:4}); +const num=(v,d=2)=>v===null||v===undefined?'Unavailable':v.toLocaleString('en-US',{minimumFractionDigits:d,maximumFractionDigits:d}); +const pct=v=>num(v)+'%'; +const cost=p=>mode==='gross'?p.answer_usd_per_10000:p.total_usd_per_10000; +const methodIndex=p=>data.methods.findIndex(m=>m.name===p.method); +const color=p=>p.baseline?'#17263b':colors[methodIndex(p)%colors.length]; +function node(tag,text,attrs={}){const n=document.createElement(tag);if(text!==null)n.textContent=text;for(const[k,v]of Object.entries(attrs))n.setAttribute(k,v);return n;} +function svgNode(tag,attrs,text){const n=document.createElementNS(NS,tag);for(const[k,v]of Object.entries(attrs))n.setAttribute(k,v);if(text!==undefined)n.textContent=text;return n;} +function visible(){const q=$('search').value.trim().toLowerCase();return data.points.filter(p=>(p.baseline||enabled.has(p.method))&&cost(p)!==null&&(!q||(p.label+' '+p.candidate_id+' '+p.rule.mode).toLowerCase().includes(q)));} +function ruleName(p){if(p.baseline)return p.label;return 'Policy '+p.candidate_id+' · '+p.rule.mode.replaceAll('_',' ');} +function choose(key){const focus=document.activeElement,chartFocus=focus?.closest('#chart'),tableFocus=focus?.closest('#policies');selected=key;render();$('selection-status').textContent='Selected '+describe(data.points.find(p=>p.key===key));if(chartFocus)$('chart').querySelector('.selected')?.focus();else if(tableFocus)$('policies').querySelector('button[aria-pressed="true"]')?.focus();} +function describe(p){return p.label+', '+ruleName(p)+', '+money(cost(p))+' per 10,000 requests, '+pct(p.answer_accuracy_percent)+' answer accuracy';} +function marker(p,x,y){const index=p.baseline?-1:methodIndex(p)%7;let n; + if(index===1)n=svgNode('rect',{x:x-5,y:y-5,width:10,height:10}); + else if(index===2||index===4){const sign=index===2?1:-1;n=svgNode('polygon',{points:`${x},${y-6*sign} ${x-6},${y+5*sign} ${x+6},${y+5*sign}`});} + else if(index===3)n=svgNode('polygon',{points:`${x},${y-7} ${x-6},${y} ${x},${y+7} ${x+6},${y}`}); + else if(index===5)n=svgNode('polygon',{points:[0,1,2,3,4,5].map(i=>`${x+6*Math.cos(i*Math.PI/3)},${y+6*Math.sin(i*Math.PI/3)}`).join(' ')}); + else if(index===6)n=svgNode('path',{d:`M ${x-6} ${y-2} H ${x-2} V ${y-6} H ${x+2} V ${y-2} H ${x+6} V ${y+2} H ${x+2} V ${y+6} H ${x-2} V ${y+2} H ${x-6} Z`}); + else n=svgNode('circle',{cx:x,cy:y,r:p.baseline?6:5}); + n.setAttribute('fill',color(p));n.setAttribute('stroke',p.key===selected?'#111':'white'); + n.setAttribute('class','point'+(p.key===selected?' selected':''));n.setAttribute('tabindex','0');n.setAttribute('role','button'); + n.setAttribute('aria-label',describe(p));n.setAttribute('aria-pressed',String(p.key===selected));n.append(svgNode('title',{},describe(p))); + n.addEventListener('click',()=>choose(p.key));n.addEventListener('keydown',e=>{if(e.key==='Enter'||e.key===' '){e.preventDefault();choose(p.key);}});return n; +} +function draw(points){const svg=$('chart');svg.replaceChildren(svgNode('title',{id:'chart-title'},'Final answer accuracy and cost of policies chosen on calibration'),svgNode('desc',{id:'chart-desc'},'Each symbol is one saved policy. The dashed segment is analytic random routing between always Luna and always Sol. Use the table below for the same values.')); + const chosen=points.find(p=>p.key===selected),ci=chosen?.paired_ci95||{},costKey=mode==='gross'?'answer_usd_per_10000':'total_usd_per_10000'; + const xs=[...points.map(cost),...data.random.answer_usd_per_10000_endpoints]; + const ys=[...points.map(p=>p.answer_accuracy_percent),...data.random.accuracy_endpoints_percent]; + if(ci[costKey]?.lower!==null&&ci[costKey]?.lower!==undefined)xs.push(ci[costKey].lower,ci[costKey].upper); + if(ci.accuracy_percent?.lower!==null&&ci.accuracy_percent?.lower!==undefined)ys.push(ci.accuracy_percent.lower,ci.accuracy_percent.upper); + const xmin=0,xmax=Math.max(...xs,0.01)*1.08,spread=Math.max(4,Math.max(...ys)-Math.min(...ys)); + const ymin=Math.max(0,Math.min(...ys)-spread*.12),ymax=Math.min(100,Math.max(...ys)+spread*.12); + const x=v=>84+v/xmax*886,y=v=>440-(v-ymin)/(ymax-ymin||1)*386; + for(let i=0;i<=5;i++){const xv=xmax*i/5,yv=ymin+(ymax-ymin)*i/5; + svg.append(svgNode('line',{x1:x(xv),x2:x(xv),y1:54,y2:440,class:'grid'}),svgNode('text',{x:x(xv),y:464,'text-anchor':'middle'},money(xv))); + svg.append(svgNode('line',{x1:84,x2:970,y1:y(yv),y2:y(yv),class:'grid'}),svgNode('text',{x:72,y:y(yv)+4,'text-anchor':'end'},num(yv,1)+'%')); + } + svg.append(svgNode('line',{x1:84,x2:970,y1:440,y2:440,class:'axis'}),svgNode('line',{x1:84,x2:84,y1:54,y2:440,class:'axis'})); + svg.append(svgNode('text',{x:527,y:505,'text-anchor':'middle'},(mode==='gross'?'Answer model cost':'Answer + known router fees')+' · USD per 10,000 requests')); + svg.append(svgNode('text',{transform:'translate(22 247) rotate(-90)','text-anchor':'middle'},'Answer accuracy (%)')); + const[cheap,strong]=data.random.answer_usd_per_10000_endpoints,[qa,qb]=data.random.accuracy_endpoints_percent; + svg.append(svgNode('line',{x1:x(cheap),y1:y(qa),x2:x(strong),y2:y(qb),class:'random-line'})); + if(chosen){const c=ci[costKey],a=ci.accuracy_percent,attrs={stroke:color(chosen),'stroke-width':2}; + if(c?.lower!==null&&c?.lower!==undefined){const yy=y(chosen.answer_accuracy_percent);svg.append(svgNode('line',{x1:x(c.lower),x2:x(c.upper),y1:yy,y2:yy,...attrs}));for(const v of[c.lower,c.upper])svg.append(svgNode('line',{x1:x(v),x2:x(v),y1:yy-5,y2:yy+5,...attrs}));} + if(a?.lower!==null&&a?.lower!==undefined){const xx=x(cost(chosen));svg.append(svgNode('line',{x1:xx,x2:xx,y1:y(a.lower),y2:y(a.upper),...attrs}));for(const v of[a.lower,a.upper])svg.append(svgNode('line',{x1:xx-5,x2:xx+5,y1:y(v),y2:y(v),...attrs}));} + } + for(const p of points.filter(p=>p.key!==selected))svg.append(marker(p,x(cost(p)),y(p.answer_accuracy_percent))); + if(chosen)svg.append(marker(chosen,x(cost(chosen)),y(chosen.answer_accuracy_percent))); +} +const ciNames={accuracy_percent:'Answer accuracy (%)',quality_gap_pp:'Accuracy loss versus always Sol (pp)',accuracy_gain_over_luna_pp:'Accuracy gain versus always Luna (pp)',answer_usd_per_10000:'Answer cost ($/10,000)',answer_savings_percent:'Answer cost savings versus Sol (%)',total_usd_per_10000:'Answer + known router fees ($/10,000)',net_savings_percent:'Savings after known router fees (%)',accuracy_gain_over_calibration_budget_random_pp:'Accuracy gain versus calibration-budget random (pp)',cost_difference_from_calibration_budget_random_usd_per_10000:'Cost minus calibration-budget random ($/10,000)'}; +function estimate(p,key){if(key==='accuracy_percent')return p.answer_accuracy_percent;if(key==='accuracy_gain_over_luna_pp')return p.answer_accuracy_percent-data.random.accuracy_endpoints_percent[0]; + const rp=p.calibration_matched_random_strong_probability; + if(key==='accuracy_gain_over_calibration_budget_random_pp')return p.answer_accuracy_percent-((1-rp)*data.random.accuracy_endpoints_percent[0]+rp*data.random.accuracy_endpoints_percent[1]); + if(key==='cost_difference_from_calibration_budget_random_usd_per_10000')return(p.total_usd_per_10000??p.answer_usd_per_10000)-((1-rp)*data.random.answer_usd_per_10000_endpoints[0]+rp*data.random.answer_usd_per_10000_endpoints[1]); + return p[key]; +} +function details(p){const box=$('detail');box.replaceChildren();if(!p){box.append(node('p','No policy matches these filters.'));return;} + box.append(node('h3',p.label+' · '+ruleName(p)));if(p.assisted)box.append(node('p','Assisted control: uses the true source task label. It is not an ordinary question-only router.',{class:'notice'})); + const stats=node('div',null,{class:'stats'});for(const[label,value]of[['Final accuracy',pct(p.answer_accuracy_percent)],['Final cost / 10,000',money(cost(p))],['Requests sent to Sol',pct(100*p.strong_share)]]){const d=node('div',label);d.append(node('strong',value));stats.append(d);}box.append(stats); + box.append(node('p','Answer model cost: '+money(p.answer_usd_per_10000)+' / 10,000. Known router fee: '+money(p.router_usd_per_10000)+' / 10,000. '+(p.baseline?'Static baseline bypasses the router.':p.kind==='embedding'?'Encoder API fee included in the priced view; local head hosting remains unpriced.':'Router hosting is unpriced; this policy appears in the answer-cost view only.'))); + box.append(node('p','Calibration: '+pct(p.calibration.answer_accuracy_percent)+' accuracy, '+money(p.calibration.selection_cost_usd_per10000)+' / 10,000 under its declared cost basis. '+(p.baseline?'Always-arm comparator.':'Policy belongs to the calibration frontier; its final result did not determine inclusion.'))); + const intervals=p.paired_ci95||{},table=node('table');table.append(node('caption','95% pointwise paired bootstrap intervals; '+data.bootstrap_draws.toLocaleString()+' draws. Negative accuracy loss means higher accuracy than always Sol.')); + const head=node('tr');for(const label of['Measure','Estimate','95% interval','Defined draws'])head.append(node('th',label,{scope:'col'}));const thead=node('thead');thead.append(head);table.append(thead);const body=node('tbody'); + for(const[key,value]of Object.entries(intervals)){const row=node('tr');row.append(node('th',ciNames[key]||key,{scope:'row'}),node('td',num(estimate(p,key),4),{class:'num'}),node('td',value.lower===null?'Unavailable':num(value.lower,4)+' to '+num(value.upper,4),{class:'num'}),node('td',value.defined_draws+'/'+value.total_draws,{class:'num'}));body.append(row);}table.append(body); + if(Object.keys(intervals).length){const scroll=node('div',null,{class:'scroll'});scroll.append(table);box.append(scroll);}else box.append(node('p','No uncertainty interval was supplied for this comparator; none is fabricated.')); + const rp=p.calibration_matched_random_strong_probability;if(rp!==null&&rp!==undefined)box.append(node('p','The random comparator sends '+pct(100*rp)+' of requests to Sol. This probability matched the policy’s calibration budget and was frozen there; its final cost may differ. The cost difference uses '+(p.total_usd_per_10000===null?'answer cost only.':'answer cost plus known router fees.'))); + const raw=node('details');raw.append(node('summary','Exact saved rule'),node('pre',JSON.stringify(p.rule,null,2)));box.append(raw); +} +function table(points){const body=$('policies');body.replaceChildren();const pages=Math.max(1,Math.ceil(points.length/perPage));page=Math.min(page,pages-1); + for(const p of points.slice(page*perPage,(page+1)*perPage)){const tr=node('tr',null,{class:p.key===selected?'active':''});const pick=node('button',p.baseline?'Select baseline':'Select '+p.candidate_id,{type:'button','aria-label':'Select '+describe(p),'aria-pressed':String(p.key===selected)});pick.addEventListener('click',()=>choose(p.key));const first=node('td');first.append(pick);tr.append(first,node('th',p.label,{scope:'row'}),node('td',money(cost(p)),{class:'num'}),node('td',pct(p.answer_accuracy_percent),{class:'num'}),node('td',pct(100*p.strong_share),{class:'num'}),node('td',pct(p.calibration.answer_accuracy_percent),{class:'num'}),node('td',money(p.calibration.selection_cost_usd_per10000),{class:'num'}));body.append(tr);} + $('page-info').textContent='Page '+(page+1)+' of '+pages+' · '+points.length+' policies and baselines';$('previous').disabled=page===0;$('next').disabled=page===pages-1; +} +function render(){const points=visible();if(!points.some(p=>p.key===selected))selected=points[0]?.key??null; + const unpriced=data.points.filter(p=>!p.baseline&&enabled.has(p.method)&&p.total_usd_per_10000===null).length; + $('cost-note').textContent=mode==='gross'?'All methods use answer model cost only. Router serving, training, and acquisition overhead are excluded.':'Embedding policies include the known encoder API fee. '+unpriced+' policies with unpriced native/CPU hosting are omitted. Local head hosting, training, and acquisition overhead remain excluded; this is not a complete operating-cost estimate.'; + $('count').textContent=points.length+' policies and baselines shown. The policy order is fixed by calibration, not final performance.'; + draw(points);details(points.find(p=>p.key===selected));table(points); +} +for(const[index,m]of data.methods.entries()){const label=node('label'),input=node('input',null,{type:'checkbox',checked:'','aria-label':'Show '+m.label});input.addEventListener('change',()=>{input.checked?enabled.add(m.name):enabled.delete(m.name);page=0;render();});label.append(input,node('span',symbols[index%symbols.length],{class:'mark',style:'color:'+colors[index%colors.length],'aria-hidden':'true'}),node('span',m.label));$('legend').append(label);} +document.querySelectorAll('input[name=cost]').forEach(input=>input.addEventListener('change',()=>{mode=input.value;page=0;render();})); +$('search').addEventListener('input',()=>{page=0;render();});$('previous').addEventListener('click',()=>{page--;table(visible());});$('next').addEventListener('click',()=>{page++;table(visible());}); +$('population').textContent=data.n.toLocaleString()+' final questions · '+data.families.toLocaleString()+' families · '+data.tasks+' tasks. Calibration used '+data.calibration_n.toLocaleString()+' questions.'; +$('scope').textContent=data.cost_scope;$('latency').textContent=data.latency_note; +for(const[key,value]of Object.entries(data.input_sha256)){const p=node('p',key+': ');p.append(node('code',value));$('hashes').append(p);} +render(); +""" + + +def render_html(data): + banner = ( + '

Replay on published test answers; not a new blind evaluation.

' + if data.get("reproduction") else "" + ) + # JSON is inert text, but escape HTML delimiters even inside script[type=json]. + payload = json.dumps( + data, ensure_ascii=False, separators=(",", ":"), allow_nan=False + ) + for char, escaped in ( + ("&", "\\u0026"), + ("<", "\\u003c"), + (">", "\\u003e"), + ("\u2028", "\\u2028"), + ("\u2029", "\\u2029"), + ): + payload = payload.replace(char, escaped) + return ( + """ + + +BBEH routing: cost and answer quality +

BBEH routing: cost and answer quality

+""" + + banner + + """ +

Explore the final performance of policies chosen on calibration. Each symbol is a distinct saved policy; there is no new selection or frontier fitted to final answers.

+

+ +

Cost and accuracy

+
Cost basis + +
+
+

+

Black circles: always Luna / always Sol. Dashed line: analytic expected random routing between those arms, with zero router inference fee.

+ +

+

Select a symbol or a table row for its uncertainty. Axis bounds may zoom to the data. Error bars are pointwise 95% intervals for the selected policy, not a confidence band for the curve.

+

Selected policy

+

Intervals resample whole families within each task, retaining the observed task weights. They are conditional on these tasks and do not provide simultaneous coverage across policies or a guarantee on future requests.

+
+

All calibration-frontier policies

+
+ +
Final measurements alongside the calibration values used to include each policy. Always-arm baselines are shown once. Calibration cost includes encoder fees for embedding methods and excludes unpriced hosting for native/CPU methods.
ChooseMethodFinal $ / 10,000Final accuracySol shareCalibration accuracyCalibration $ / 10,000
+

Interpretation and provenance

+

True task metadata is an assisted control: it uses the source task label, which an ordinary question-only router does not receive. Predicted task from text is a separate method.

+

The random line is an expected mixture, not an observed random run. A policy’s calibration-budget random comparator keeps its calibration mixing probability; its final spend need not match the policy.

+

Exact input file hashes
+

This file is self-contained and makes no network requests. It only formats the supplied reports; it performs no grading, training, or policy selection.

+
+""" + ) + + +def render(calibration_path, frozen_path, final_path, output, *, reproduction=False): + data = checked_data(calibration_path, frozen_path, final_path) + if reproduction: + data["reproduction"] = True + data["new_blind_evaluation"] = False + output = Path(output) + require( + not output.exists(), "Output directory already exists; use a fresh directory" + ) + document = render_html(data) + hashes = "\n".join( + f"- {name}: `{value}`" for name, value in data["input_sha256"].items() + ) + disclosure = "Replay on published test answers; not a new blind evaluation.\n\n" if reproduction else "" + replay_flag = " --reproduction" if reproduction else "" + readme = f"""# BBEH cost–quality explorer + +{disclosure}Open [index.html](index.html) in a browser. It is self-contained and works offline. + +Each point is a policy from the calibration frontier, replayed unchanged on {data["n"]:,} final questions ({data["families"]:,} families). Always-Luna and always-Sol baselines are included. No final winner or new final frontier is selected. + +The default view compares answer-model dollars per 10,000 requests with answer accuracy. The second view adds known encoder API fees and omits native/CPU policies whose hosting is unpriced. Local head hosting, training, and data acquisition are not included. The true-task-metadata method is an assisted control. + +Select a point or table row for every supplied 95% paired family-bootstrap interval. These are pointwise, conditional on the observed tasks, and not a guarantee for future requests. The random line is an analytic expectation. A calibration-budget random comparator may have different spend on final data. + +Regenerate from the three exact saved artifacts, using a new output directory: + +```bash +# Render saved calibration policies and their frozen final evaluation. +python -m evaluation.report --calibration calibration.json --frozen frozen.json --final-report final-report.json --output explorer{replay_flag} +``` + +The renderer verifies artifact hashes, policy identities, population metadata, and the calibration-frontier membership before writing either output. No model, grading, or network call occurs. + +{hashes} +""" + output.mkdir(parents=True) + (output / "index.html").write_text(document, encoding="utf-8") + (output / "README.md").write_text(readme, encoding="utf-8") + return { + "format": data["format"], + "policies_and_baselines": len(data["points"]), + "input_sha256": data["input_sha256"], + "html_sha256": sha(output / "index.html"), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--calibration", type=Path, required=True) + parser.add_argument("--frozen", type=Path, required=True) + parser.add_argument("--final-report", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--reproduction", action="store_true", help="Label replay on already published test answers") + args = parser.parse_args() + print( + json.dumps( + render(args.calibration, args.frozen, args.final_report, args.output, reproduction=args.reproduction), + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/evaluation/source_sensitivity.py b/examples/train-a-router/evaluation/source_sensitivity.py new file mode 100644 index 000000000..bcb80f59d --- /dev/null +++ b/examples/train-a-router/evaluation/source_sensitivity.py @@ -0,0 +1,94 @@ +"""Replay unchanged frozen policies after a source-only, prespecified exclusion.""" + +import argparse +import hashlib +from pathlib import Path + +import numpy as np + +from embedding import recipes as core +from evaluation import frontier, sweep, utility_router as utility + + +def evaluate(run, frozen_path, grades_path, primary_path, specification_path, output): + run = Path(run) + plan = core.checked_plan(run) + frozen, primary, specification = map(core.read, (frozen_path, primary_path, specification_path)) + core.require( + primary["frozen_sha256"] == core.sha(frozen_path) + and primary["grades_sha256"] == core.sha(grades_path) + and primary["no_final_selection"] is True + and frozen["plan_sha256"] == core.sha(run / "plan.json") + and frozen["frozen_before_final_outcomes"] is True + and specification["fixed_before_final_outcomes"] is True + and specification["source_plan_sha256"] == plan["source_plan_sha256"], + "Sensitivity requires the verified primary replay and original source-only specification", + ) + rows = core.role_rows(run, plan, "final") + y, costs, grade = core.grades(plan, rows, grades_path, "final") + setting = specification["final_sensitivity"] + excluded = set(setting["excluded_ids"]) + core.require(len(excluded) == len(setting["excluded_ids"]), "Duplicate sensitivity ID") + declared = {r["id"]: r for r in specification["source_ambiguous_rows"] if r["role"] == "final"} + core.require(excluded == set(declared), "Sensitivity differs from predeclared source ambiguities") + for row in rows: + if row["id"] in excluded: + evidence = declared[row["id"]] + core.require( + row["family"] == evidence["family"] + and hashlib.sha256(row["request"].encode()).hexdigest() == evidence["request_sha256"], + "Excluded source input changed", + ) + keep = np.array([row["id"] not in excluded for row in rows], dtype=bool) + core.require(len(rows) == setting["n_primary"] and int(keep.sum()) == setting["n_sensitivity"], + "Sensitivity population changed") + subset = [row for row, retained in zip(rows, keep, strict=True) if retained] + y, costs = y[keep], costs[keep] + bootstrap = frontier.Bootstrap(subset, primary["bootstrap_draws"]) + methods = {} + for name, saved in frozen["methods"].items(): + score = saved["scores"] + probabilities = np.asarray(score["probabilities"])[keep] + expected = np.asarray(score["expected_usd"])[keep] + fees = np.asarray(frozen["encoder_fees_usd"])[keep] if score["router_kind"] == "embedding" else None + frontier_ids = set(saved["calibration_frontier_candidate_ids"]) + candidates = [row for row in saved["candidates"] if row["candidate_id"] in frontier_ids] + metrics = sweep.sweep_metrics([row["policy"] for row in candidates], probabilities, expected, y, costs, fees) + points = [] + for candidate, metric in zip(candidates, metrics, strict=True): + policy = candidate["policy"] + charged = np.zeros(len(subset)) if policy["mode"].startswith("always_") else fees + routes = utility.replay(policy, probabilities, expected) + points.append({ + "candidate_id": candidate["candidate_id"], "rule": policy, **metric, + "paired_ci95": frontier.uncertainty(bootstrap, routes, y, costs, charged, + candidate["calibration_matched_random_strong_probability"]), + }) + methods[name] = {"points": points, "router_kind": score["router_kind"], + "metadata_assisted": score["metadata_assisted"]} + result = { + "format": "bbeh-source-format-sensitivity-result-v1", + "specification_sha256": core.sha(specification_path), "primary_report_sha256": core.sha(primary_path), + "frozen_sha256": core.sha(frozen_path), "grades_sha256": core.sha(grades_path), + "source_plan_sha256": plan["source_plan_sha256"], "excluded_ids": sorted(excluded), + "n_primary": len(rows), "n": len(subset), "families": len({row["family"] for row in subset}), + "random": {"accuracy_endpoints_percent": (100 * y.mean(axis=0)).tolist(), + "answer_usd_per_10000_endpoints": (10000 * costs.mean(axis=0)).tolist()}, + "methods": methods, "policies_unchanged": True, "no_refit_recalibration_or_selection": True, + "cost_scope": grade["cost_scope"], "implementation_sha256": core.sha(__file__), + "interpretation": "Supplementary source-quality sensitivity. Primary fixed-population results remain authoritative; no threshold or method is selected using this subset.", + } + core.write(output, result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for flag in ("run", "frozen", "grades", "primary", "specification", "output"): + parser.add_argument("--" + flag, required=True) + args = parser.parse_args() + evaluate(args.run, args.frozen, args.grades, args.primary, args.specification, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/evaluation/sweep.py b/examples/train-a-router/evaluation/sweep.py new file mode 100644 index 000000000..588a4cb2e --- /dev/null +++ b/examples/train-a-router/evaluation/sweep.py @@ -0,0 +1,130 @@ +"""Exact metric sweep copied from the validated public frontier implementation.""" + +from fractions import Fraction +from embedding import recipes as core +from evaluation import utility_router as utility + +def _fraction(value): + return Fraction.from_float(float(value)) + + +def _metrics( + correct_count, + answer_cost, + strong_count, + n, + baseline_correct, + baseline_cost, + router_cost, +): + total = None if router_cost is None else answer_cost + router_cost + return { + "answer_accuracy_percent": float(100 * correct_count / n), + "quality_gap_pp": float(100 * (baseline_correct - correct_count) / n), + "answer_usd_per_10000": float(10000 * answer_cost / n), + "router_usd_per_10000": None + if router_cost is None + else float(10000 * router_cost / n), + "total_usd_per_10000": None if total is None else float(10000 * total / n), + "strong_share": float(strong_count / n), + "answer_savings_percent": None + if not baseline_cost + else float(100 * (1 - answer_cost / baseline_cost)), + "net_savings_percent": None + if total is None or not baseline_cost + else float(100 * (1 - total / baseline_cost)), + } + + +def sweep_metrics( + policies, probabilities, expected_usd, correct, observed_usd, router_usd=None +): + """Evaluate K saved rules in O((N+K) log(N+K)) with exact crossing decisions. + + Observed costs only contribute to reporting. The rule uses the frozen + predicted probabilities/costs; it never sees a selected answer's length. + """ + deltas = utility._deltas(probabilities, expected_usd) + labels = utility._matrix(correct, "correct", binary=True) + costs = [ + [_fraction(v) for v in row] + for row in utility._matrix(observed_usd, "observed_usd") + ] + n = len(deltas) + core.require(len(labels) == len(costs) == n, "Sweep rows do not align") + if router_usd is None: + overhead = None + else: + core.require(len(router_usd) == n, "Sweep router fees do not align") + overhead = sum( + (_fraction(utility._number(v, "router_usd")) for v in router_usd), + Fraction(), + ) + parsed = [utility._read_policy(policy) for policy in policies] + queries = sorted({penalty for mode, penalty in parsed if mode == "utility"}) + baseline_correct = sum(row[1] for row in labels) + baseline_cost = sum((row[1] for row in costs), Fraction()) + decisions = [quality >= 0 for quality, _ in deltas] + + def counts(routes): + return ( + sum(row[int(route)] for row, route in zip(labels, routes, strict=True)), + sum( + (row[int(route)] for row, route in zip(costs, routes, strict=True)), + Fraction(), + ), + sum(routes), + ) + + state = list(counts(decisions)) + + def flip(index, use_strong): + if decisions[index] != use_strong: + sign = 1 if use_strong else -1 + state[0] += sign * (labels[index][1] - labels[index][0]) + state[1] += sign * (costs[index][1] - costs[index][0]) + state[2] += sign + decisions[index] = use_strong + + events = {} + for index, (quality, cost) in enumerate(deltas): + if cost and quality / cost >= 0: + events.setdefault(quality / cost, []).append(index) + events = sorted(events.items()) + position, snapshots = 0, {} + for query in queries: + while position < len(events) and events[position][0] < query: + for index in events[position][1]: + flip(index, deltas[index][1] < 0) + position += 1 + equal = position < len(events) and events[position][0] == query + if equal: + # Both directions choose strong exactly on their shared crossing. + for index in events[position][1]: + if deltas[index][1] < 0: + flip(index, True) + snapshots[query] = tuple(state) + if equal: + for index in events[position][1]: + if deltas[index][1] > 0: + flip(index, False) + position += 1 + limit = counts( + [cost < 0 or (cost == 0 and quality >= 0) for quality, cost in deltas] + ) + bypass = { + "always_strong": (baseline_correct, baseline_cost, n), + "always_cheap": counts([False] * n), + } + result = [] + for mode, penalty in parsed: + values = ( + bypass[mode] + if mode in bypass + else limit + if mode == "cost_limit" + else snapshots[penalty] + ) + fee = Fraction() if mode in bypass else overhead + result.append(_metrics(*values, n, baseline_correct, baseline_cost, fee)) + return result diff --git a/examples/train-a-router/evaluation/utility_router.py b/examples/train-a-router/evaluation/utility_router.py new file mode 100644 index 000000000..0e2c17aa9 --- /dev/null +++ b/examples/train-a-router/evaluation/utility_router.py @@ -0,0 +1,271 @@ +"""Calibrate a frozen two-arm expected-utility rule; no model fitting. + +Every matrix is N x 2 in cheap/strong order. Those arm names do not imply a +per-request cost or quality ordering. Costs use the caller's declared USD +basis; they may be measured charges or an explicitly labeled projection. + +The rule selects strong when delta_probability - lambda * delta_USD >= 0. +Nonnegative lambda is calibrated against paired answers, never at dispatch. +Saved rational lambdas preserve exact crossings and intervals, including when +two crossings have no representable float between them. Arithmetic treats the +finite float64 inputs as exact values; no tolerance changes tie decisions. +""" + +from __future__ import annotations + +import math +from fractions import Fraction +from numbers import Real + +import numpy as np + +FORMAT = "expected-utility-policy-v1" +PROTOCOL = { + "rule": "strong iff (p_strong-p_cheap) - lambda*(expected_USD_strong-expected_USD_cheap) >= 0", + "lambda": "nonnegative; exact crossings, intervening intervals and the cost-minimizing limit", + "tie": "strong on equal utility; at the cost limit, lower expected cost then higher probability then strong", + "selection": "least calibration cost, then smaller signed quality loss, router bypass, smaller lambda, always-strong bypass tie", + "cost": "Supplied USD basis; router overhead applies to every utility-routed request and to neither always-arm bypass.", + "unknown_router_cost": "None means unpriced: select by answer costs and report no native net-cost claim.", + "quality": "Empirical signed gap versus always strong; rescues offset regressions. Not a future-risk guarantee.", + "arithmetic": "Float64 inputs are exact for utility/cost comparison; gap targets use their decimal spelling. No rounding tolerance changes ties.", +} + + +def _require(condition, message): + if not condition: + raise ValueError(message) + + +def _number(value, name, *, binary=False): + if binary: + _require(isinstance(value, (Real, bool, np.bool_)) and value in (0, 1), f"{name} must be binary") + return int(value) + _require( + isinstance(value, Real) and not isinstance(value, (bool, np.bool_)), f"{name} must be numeric, not boolean" + ) + value = float(value) + _require(math.isfinite(value) and value >= 0, f"{name} must be finite and nonnegative") + return value + + +def _matrix(value, name, *, probability=False, binary=False): + try: + array = np.asarray(value, dtype=object) + except (TypeError, ValueError) as error: + raise ValueError(f"{name} must be a nonempty N x 2 matrix") from error + _require(array.ndim == 2 and array.shape[0] > 0 and array.shape[1] == 2, f"{name} must be a nonempty N x 2 matrix") + rows = [[_number(item, name, binary=binary) for item in row] for row in array.tolist()] + _require(not probability or all(item <= 1 for row in rows for item in row), "Probabilities must lie in [0,1]") + return rows + + +def _fraction(value): + return Fraction.from_float(float(value)) + + +def _deltas(probabilities, expected_usd): + probabilities = _matrix(probabilities, "probabilities", probability=True) + expected_usd = _matrix(expected_usd, "expected_usd") + _require(len(probabilities) == len(expected_usd), "Probability and expected-cost rows differ") + return [ + (_fraction(p[1]) - _fraction(p[0]), _fraction(c[1]) - _fraction(c[0])) + for p, c in zip(probabilities, expected_usd, strict=True) + ] + + +def _policy(mode, penalty=None): + result = {"format": FORMAT, "mode": mode} + if mode == "utility": + result["lambda"] = {"numerator": str(penalty.numerator), "denominator": str(penalty.denominator)} + return result + + +def _read_policy(policy): + _require(isinstance(policy, dict) and policy.get("format") == FORMAT, "Unknown policy format") + mode = policy.get("mode") + _require( + isinstance(mode, str) and mode in {"always_strong", "always_cheap", "utility", "cost_limit"}, + "Unknown policy mode", + ) + expected = {"format", "mode", "lambda"} if mode == "utility" else {"format", "mode"} + _require(set(policy) == expected, "Unexpected policy fields") + if mode != "utility": + return mode, None + ratio = policy["lambda"] + _require(isinstance(ratio, dict) and set(ratio) == {"numerator", "denominator"}, "Invalid saved lambda") + try: + n, d = int(ratio["numerator"]), int(ratio["denominator"]) + _require(n >= 0 and d > 0, "Lambda must be nonnegative") + penalty = Fraction(n, d) + except (TypeError, ValueError, ZeroDivisionError) as error: + raise ValueError("Invalid saved lambda") from error + _require( + ratio == {"numerator": str(penalty.numerator), "denominator": str(penalty.denominator)}, + "Saved lambda must be a canonical exact fraction", + ) + return mode, penalty + + +def replay(policy, probabilities, expected_usd): + """Return strong-arm booleans, using only input-derived predictions. + + A serving caller must inspect an always-arm mode before running its router: + that saved mode is a direct bypass and needs no probabilities or embedding. + This array replay interface still validates the supplied evaluation inputs. + """ + mode, penalty = _read_policy(policy) + deltas = _deltas(probabilities, expected_usd) + if mode.startswith("always_"): + return np.full(len(deltas), mode == "always_strong", dtype=bool) + if mode == "cost_limit": + return np.array([cost < 0 or (cost == 0 and quality >= 0) for quality, cost in deltas], dtype=bool) + return np.array([quality - penalty * cost >= 0 for quality, cost in deltas], dtype=bool) + + +def _float(value): + try: + result = float(value) + except OverflowError as error: + raise ValueError("Metric exceeds the finite reporting range") from error + _require(math.isfinite(result), "Metric exceeds the finite reporting range") + return result + + +def calibrate(probabilities, expected_usd, correct, observed_usd, *, targets_pp=(1, 2, 0, 5), router_usd=None): + """Enumerate all utility decisions and choose cheapest feasible policies. + + ``correct`` contains both paired binary outcomes. ``observed_usd`` contains + their costs under the declared accounting basis, including a common price + projection when appropriate. ``router_usd`` is a scalar or N-vector charged + on every utility-routed request. None leaves router pricing unknown. + + The sweep is O(N log N), apart from output size. It records crossings AND + open intervals: simultaneous opposite-direction crossings can produce a + boundary routing pattern different from both neighboring patterns. + Full candidates and the nondominated calibration frontier are returned. + Freeze the selected policies before accessing final outcomes. + """ + deltas = _deltas(probabilities, expected_usd) + y = _matrix(correct, "correct", binary=True) + costs = _matrix(observed_usd, "observed_usd") + n = len(deltas) + _require(len(y) == len(costs) == n, "Calibration rows must align exactly") + costs = [[_fraction(item) for item in row] for row in costs] + targets = [_number(value, "targets_pp") for value in targets_pp] + _require( + bool(targets) and len(set(targets)) == len(targets) and max(targets) <= 100, "Invalid or duplicate gap targets" + ) + if router_usd is None: + overhead = None + elif isinstance(router_usd, Real): + overhead = n * _fraction(_number(router_usd, "router_usd")) + else: + values = np.asarray(router_usd, dtype=object) + _require(values.shape == (n,), "Router costs must be a scalar or aligned N-vector") + overhead = sum((_fraction(_number(value, "router_usd")) for value in values.tolist()), Fraction()) + + strong_correct = sum(row[1] for row in y) + strong_cost = sum((row[1] for row in costs), Fraction()) + candidates, internals = [], [] + + def counts(decisions): + return { + "correct": sum(row[int(route)] for row, route in zip(y, decisions, strict=True)), + "cost": sum((row[int(route)] for row, route in zip(costs, decisions, strict=True)), Fraction()), + "cheap": sum(not route for route in decisions), + "regressions": sum(not route and row[1] > row[0] for row, route in zip(y, decisions, strict=True)), + "rescues": sum(not route and row[0] > row[1] for row, route in zip(y, decisions, strict=True)), + } + + def add(policy, state): + mode, penalty = _read_policy(policy) + bypass = mode.startswith("always_") + router = Fraction() if bypass else overhead + selection_cost = state["cost"] + (router if router is not None else 0) + loss = strong_correct - state["correct"] + metric = { + "answer_accuracy_percent": 100 * state["correct"] / n, + "gap_vs_strong_pp": 100 * loss / n, + "cheap_share": state["cheap"] / n, + "regressions": int(state["regressions"]), + "rescues": int(state["rescues"]), + "answer_cost_usd_per10000": _float(10000 * state["cost"] / n), + "router_cost_usd_per10000": None if router is None else _float(10000 * router / n), + "total_cost_usd_per10000": None if router is None else _float(10000 * selection_cost / n), + "selection_cost_usd_per10000": _float(10000 * selection_cost / n), + "gross_answer_savings_percent": None + if not strong_cost + else _float(100 * (1 - state["cost"] / strong_cost)), + "net_savings_percent": None + if router is None or not strong_cost + else _float(100 * (1 - selection_cost / strong_cost)), + } + record = {"candidate_id": len(candidates), "policy": policy, "calibration": metric} + candidates.append(record) + # Infinity follows every finite penalty; exact bypass ties prefer strong. + tie = (not bypass, mode == "cost_limit", penalty or Fraction(), mode != "always_strong") + internals.append((selection_cost, loss, tie, record)) + + add(_policy("always_strong"), counts([True] * n)) + add(_policy("always_cheap"), counts([False] * n)) + decisions = [quality >= 0 for quality, _ in deltas] + state = counts(decisions) + add(_policy("utility", Fraction()), state) + + def flip(index, use_strong): + if decisions[index] == use_strong: + return + sign = 1 if use_strong else -1 + state["correct"] += sign * (y[index][1] - y[index][0]) + state["cost"] += sign * (costs[index][1] - costs[index][0]) + state["cheap"] -= sign + state["regressions"] -= sign * (y[index][1] > y[index][0]) + state["rescues"] -= sign * (y[index][0] > y[index][1]) + decisions[index] = use_strong + + crossings = {} + for index, (quality, cost) in enumerate(deltas): + if cost and quality / cost >= 0: + crossings.setdefault(quality / cost, []).append(index) + previous = Fraction() + for crossing in sorted(crossings): + if crossing > previous: + add(_policy("utility", (previous + crossing) / 2), state) + for index in crossings[crossing]: + if deltas[index][1] < 0: + flip(index, True) + if crossing: + add(_policy("utility", crossing), state) + for index in crossings[crossing]: + if deltas[index][1] > 0: + flip(index, False) + previous = crossing + if crossings: + add(_policy("utility", previous + 1), state) + add(_policy("cost_limit"), state) + + ordered = sorted(internals, key=lambda item: (item[0], item[1], item[2], item[3]["candidate_id"])) + frontier, best_loss = [], None + for _, loss, _, record in ordered: + if best_loss is None or loss < best_loss: + frontier.append(record) + best_loss = loss + selected = {} + for target in targets: + winner = next(record for _, loss, _, record in ordered if 100 * loss <= n * Fraction(str(target))) + key = str(int(target)) if target.is_integer() else repr(target) + selected[key] = {"target_gap_pp": target, **winner} + return { + "format": "expected-utility-calibration-v1", + "protocol": PROTOCOL, + "n": n, + "arm_order": ["cheap", "strong"], + "selection_cost_basis": "answer USD plus router USD" + if overhead is not None + else "answer USD only; router unpriced", + "targets_pp": targets, + "candidates": candidates, + "frontier": frontier, + "selected": selected, + } diff --git a/examples/train-a-router/evidence.py b/examples/train-a-router/evidence.py new file mode 100644 index 000000000..f0fe25d84 --- /dev/null +++ b/examples/train-a-router/evidence.py @@ -0,0 +1,141 @@ +"""Download the example's public artifacts and verify their published hashes.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path, PurePosixPath +import re +import tempfile +from urllib.parse import quote, urljoin, urlsplit +from urllib.request import urlopen + +FORMAT = "train-a-router-artifacts-v1" +GROUPS = frozenset({"core", "weights", "recordings"}) +MANIFEST_LIMIT = 2 * 1024 * 1024 + + +def sha(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def checked_manifest(body, expected_sha256): + if not isinstance(expected_sha256, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_sha256): + raise ValueError("Supply the 64-character SHA-256 published with the artifact manifest") + if len(body) > MANIFEST_LIMIT or hashlib.sha256(body).hexdigest() != expected_sha256: + raise ValueError("The downloaded manifest does not match the published SHA-256") + manifest = json.loads(body) + if manifest.get("format") != FORMAT or not isinstance(manifest.get("files"), list) or not manifest["files"]: + raise ValueError("Unknown or empty artifact manifest") + names = set() + for entry in manifest["files"]: + name = entry.get("path") + if not isinstance(name, str) or not name or "\\" in name: + raise ValueError("Artifact paths must be relative POSIX paths") + path = PurePosixPath(name) + if path.is_absolute() or ".." in path.parts or path.as_posix() != name or name in (".", "manifest.json"): + raise ValueError("Artifact path can escape or overwrite the manifest") + if name in names or any(parent.as_posix() in names for parent in path.parents): + raise ValueError("Duplicate or conflicting artifact path") + if any(PurePosixPath(other).is_relative_to(path) for other in names): + raise ValueError("Artifact path conflicts with an earlier directory") + names.add(name) + if ( + entry.get("group") not in GROUPS + or type(entry.get("size")) is not int + or entry["size"] < 0 + or not isinstance(entry.get("sha256"), str) + or not re.fullmatch(r"[0-9a-f]{64}", entry["sha256"]) + ): + raise ValueError("Artifact is missing its group, byte count, or SHA-256") + return manifest + + +def artifact_path(root, relative): + root = Path(root).resolve() + target = root / relative + if target.is_symlink() or not target.resolve().is_relative_to(root): + raise ValueError("Artifact destination escapes the download directory through a symlink") + return target + + +def verify(directory, *, manifest_sha256, groups=("core",)): + directory = Path(directory) + if not groups or not set(groups).issubset(GROUPS): + raise ValueError("Choose core, weights, or recordings artifact groups") + manifest = checked_manifest((directory / "manifest.json").read_bytes(), manifest_sha256) + for entry in manifest["files"]: + if entry["group"] not in groups: + continue + path = artifact_path(directory, entry["path"]) + if not path.is_file() or path.stat().st_size != entry["size"] or sha(path) != entry["sha256"]: + raise ValueError(f"Missing or changed artifact: {entry['path']}; repeat fetch to repair the download") + return manifest + + +def download(url, target, entry): + target.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile( + dir=target.parent, prefix=target.name + ".", suffix=".partial", delete=False + ) as stream: + temporary = Path(stream.name) + received, digest = 0, hashlib.sha256() + with urlopen(url, timeout=120) as response: + while chunk := response.read(1024 * 1024): + received += len(chunk) + if received > entry["size"]: + raise ValueError("Artifact is larger than its published byte count") + digest.update(chunk) + stream.write(chunk) + if received != entry["size"] or digest.hexdigest() != entry["sha256"]: + raise ValueError(f"Downloaded artifact failed verification: {entry['path']}") + temporary.replace(target) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def fetch(manifest_url, manifest_sha256, output, groups=("core",)): + """Reuse verified downloads; optional weights are unnecessary for CPU replay.""" + if not groups or not set(groups).issubset(GROUPS): + raise ValueError("Choose core, weights, or recordings artifact groups") + url = urlsplit(manifest_url) + if url.scheme != "https" or not url.hostname or url.username or url.password or url.query or url.fragment: + raise ValueError("Use the public HTTPS manifest URL without credentials, query parameters, or fragments") + with urlopen(manifest_url, timeout=60) as response: + body = response.read(MANIFEST_LIMIT + 1) + manifest = checked_manifest(body, manifest_sha256) + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + manifest_path = output / "manifest.json" + if manifest_path.is_symlink(): + raise ValueError("The manifest destination must not be a symlink") + if manifest_path.exists(): + if manifest_path.read_bytes() != body: + raise ValueError("This directory contains another study; choose a new output directory") + else: + if any(output.iterdir()): + raise ValueError("Use an empty directory for the first download") + manifest_path.write_bytes(body) + downloaded, reused, total = 0, 0, 0 + for entry in manifest["files"]: + if entry["group"] not in groups: + continue + target = artifact_path(output, entry["path"]) + total += entry["size"] + if target.is_file() and target.stat().st_size == entry["size"] and sha(target) == entry["sha256"]: + reused += 1 + continue + download(urljoin(manifest_url, quote(entry["path"], safe="/")), target, entry) + downloaded += 1 + return { + "directory": str(output), + "manifest_sha256": manifest_sha256, + "downloaded_files": downloaded, + "reused_files": reused, + "selected_bytes": total, + "groups": list(groups), + } diff --git a/examples/train-a-router/frozen-source-manifest.json b/examples/train-a-router/frozen-source-manifest.json new file mode 100644 index 000000000..580cc9f58 --- /dev/null +++ b/examples/train-a-router/frozen-source-manifest.json @@ -0,0 +1,134 @@ +{ + "files": { + "collect.py": { + "bytes": 3949, + "sha256": "4d7188d826551f4f277009c98ab95af93e38071d240d2815f8174142dea428f1" + }, + "embedding/__init__.py": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "embedding/collect_features.py": { + "bytes": 12307, + "sha256": "49abf01bc86c5670b7775613dc8ddb448fe507ac71da83e44dee9db7be7b4a3d" + }, + "embedding/cost_iteration/__init__.py": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "embedding/cost_iteration/promotion.json": { + "bytes": 1562, + "sha256": "13a500178d56ed1f08b17b8ad15be5f6a5f1644ea768e7aacf878e44519d48da" + }, + "embedding/cost_iteration/protocol.json": { + "bytes": 3327, + "sha256": "be1a9641d79474b70e7013c7ffaf9a390c66627ca98b6c5cf46adee46a15f872" + }, + "embedding/cost_iteration/refit.py": { + "bytes": 9481, + "sha256": "4aee599952289237c3f76b2629f312698912b6a91bf1953db4a23ffdd913dc32" + }, + "embedding/cost_iteration/study.py": { + "bytes": 16772, + "sha256": "902d4b69bba411165727df3c8620bd1bc2379b69976760d1c312bbefb7541898" + }, + "embedding/development_budget.py": { + "bytes": 4762, + "sha256": "e05d0bd202f214f24de4a317be2847616efa1196abbaed01c4f1b6947b141352" + }, + "embedding/development_report.py": { + "bytes": 7281, + "sha256": "a413f926d0fec5f2843259f1711c05f2f1d8cddb5fcf076795a9c68487419b0d" + }, + "embedding/recipes.py": { + "bytes": 23825, + "sha256": "f8fc0a3ccf8008f08a342314d4ac816b3649e80b94fe526664607b7edd85d7a8" + }, + "embedding/token_audit.py": { + "bytes": 3070, + "sha256": "d1e0f90ff42dff18eec660eecc6f6417fadf328083b70042ba330c10e905a4b0" + }, + "embedding/wire.py": { + "bytes": 19282, + "sha256": "0e3ac46beb16ebf4c909efc54ee7c9fc9839398d8c9c21a3342e7f2a0326f4d2" + }, + "evaluation/__init__.py": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "evaluation/frontier.py": { + "bytes": 17390, + "sha256": "1e1f730292193a3faa3424a445ea806c59b4c87bcb1dc27ad08bfc4c79f3959c" + }, + "evaluation/report.py": { + "bytes": 39432, + "sha256": "c59f4c34a424b91097a06d60898beeb9b03211a2eee48cab20d71ad691b214a4" + }, + "evaluation/source_sensitivity.py": { + "bytes": 5146, + "sha256": "19dc23b1d6677fc813429167d1f5549717b8fa2cd4729e1b6500a02f98e7ec12" + }, + "evaluation/sweep.py": { + "bytes": 4754, + "sha256": "4046b1e2ebf60ea15e49da4785f5c5c908795a1ef9684e0f3ea564a5d1ea2afa" + }, + "evaluation/utility_router.py": { + "bytes": 12538, + "sha256": "8f79c84e15cbbe8b174b08d5e4b6af5c66a545f429eda6cf2536b3f086f4d441" + }, + "grade_v2.py": { + "bytes": 15668, + "sha256": "cd5d53a22b024406eecf18de92d24175a287e482dc63d58be6ec01b7a23cbd08" + }, + "native/bbeh_native.py": { + "bytes": 27440, + "sha256": "c379f86f55bfccbfcd6c0e613d54adf9f8f155cc1c654a0e30e45998181b9e28" + }, + "native/benchmark.py": { + "bytes": 8295, + "sha256": "f4d07643c2ffff4e1d47c46dfbfc5f56982f2ce1023f3a64ba9d81e25419f2b9" + }, + "native/development_report.py": { + "bytes": 8576, + "sha256": "f9d42e4909fe6922c0c010213ab499f97b9a7e320fb74630ec40ae66dfefe8a9" + }, + "native/recipe.py": { + "bytes": 15868, + "sha256": "c500d1278ae7fe5c726f9accd277ff94c8b09b86c78cfbd19c1db1b035ba21e1" + }, + "normalize.py": { + "bytes": 4558, + "sha256": "258a60874b11b5bbeefe2a4fe304dfcc2da341eba995258262ce1e5931ef5772" + }, + "openai_population.py": { + "bytes": 25064, + "sha256": "b94d77b28686230d6bdb3469e0a22577998b0a1181b152d02a82489790d87e0c" + }, + "pair.json": { + "bytes": 1511, + "sha256": "79995e7f2274c97eee847d46276e1b0a48d396533bba1c4b768667d209d30ead" + }, + "source-format-sensitivity.json": { + "bytes": 3023, + "sha256": "75c3dfece6da4a543868530b24fa040c481c08ea4cbb219c844a4e8818440362" + }, + "source/grouping.py": { + "bytes": 8096, + "sha256": "dae9d6f61a441771a9ab34a36ad5591c6d246b2de2c3115b485e62cc841a864b" + }, + "source/prepare.py": { + "bytes": 14306, + "sha256": "6ac6f5f8bad4fd6e433caa9f631880d2080df346933f014abcc7f731b2412150" + }, + "source/test_source.py": { + "bytes": 5418, + "sha256": "3c679fabf7430b4917210bc67485aac8f362da58beb51b54c0fc8c1f570e73c9" + }, + "test_normalization.py": { + "bytes": 6044, + "sha256": "990b80187ed1126c42ec72e5d0be665fae95e3527710ca901572ffb018b3ca6e" + } + }, + "format": "train-a-router-scientific-source-v1", + "purpose": "Exact implementations used for the published experiment. Reader commands wrap these files without changing their calculations." +} diff --git a/examples/train-a-router/grade_v2.py b/examples/train-a-router/grade_v2.py new file mode 100644 index 000000000..8e2a6b8e2 --- /dev/null +++ b/examples/train-a-router/grade_v2.py @@ -0,0 +1,241 @@ +"""Verify all physical attempts and grade every prospectively retained BBEH row.""" + +import argparse +import contextlib +import hashlib +import importlib.util +import io +import json +import math +from decimal import Decimal, InvalidOperation +from pathlib import Path + +import collect +import normalize +import openai_population as acquisition + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def read(path): + return json.loads(Path(path).read_bytes()) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def bound(root, relative, expected): + path = root / relative + require(not Path(relative).is_absolute() and root.resolve() in path.resolve().parents, "Unsafe receipt path") + require(path.is_file() and not path.is_symlink() and sha(path) == expected, "Receipt hash mismatch") + return path + + +def load_official(path, expected): + require(sha(path) == expected, "Official BBEH grader changed") + spec = importlib.util.spec_from_file_location("pinned_bbeh_evaluate", path) + module = importlib.util.module_from_spec(spec) + # The audited upstream file has eight demonstration prints, but no other side effects. + with contextlib.redirect_stdout(io.StringIO()): + spec.loader.exec_module(module) + return module + + +def decimal_match(module, answer, target): + prediction = module.preprocess_sample(answer) + reference = module.preprocess_reference(target) + if prediction == reference: + return True + if len(prediction) == 3 and prediction[0] == "(" and prediction[-1] == ")": + return prediction[1] == reference + if len(reference) == 3 and reference[0] == "(" and reference[-1] == ")": + return reference[1] == prediction + try: + a, b = Decimal(prediction), Decimal(reference) + if a.is_finite() and b.is_finite() and a == b: + return True + except InvalidOperation: + pass + return (prediction.replace("'", "") == reference.replace("'", "") + or f"[{reference}]" == prediction or f"[{prediction}]" == reference + or (prediction.endswith("?") and prediction[:-1] == reference)) + + +def checked_arm(root, plan, collection, inputs, arm, role): + binding = collection["selections"][f"{role}/{arm}"] + selection_path = bound(root, binding["path"], binding["sha256"]) + selection = read(selection_path) + ids = [item["id"] for item in inputs] + plan_sha = sha(root / "plan.json") + require(selection["ordered_ids"] == ids and [entry["id"] for entry in selection["entries"]] == ids, + "Selection population/order changed") + require(selection["plan_sha256"] == plan_sha and selection["model"] == collect.ARMS[arm]["model"] + and selection["arm"] == arm and selection["split"] == role + and selection["first_terminal_response_only"] and selection["no_quality_retries"], "Selection contract changed") + rows, ledger, declared = {}, [], set() + for item, entry in zip(inputs, selection["entries"], strict=True): + body = collect.body_for(item, arm) + body_sha = acquisition.digest(json.dumps(body, sort_keys=True).encode()) + first = None + require(len(entry["attempts"]) <= acquisition.MAX_ATTEMPTS, "Attempt cap exceeded") + for number, attempt in enumerate(entry["attempts"], 1): + require(first is None, "A terminal answer was retried") + require(attempt["attempt"] == number, "Attempt order changed") + prefix = f"{role}/{arm}/attempts/{acquisition.digest(item['id'].encode())}/{number:03d}" + require(attempt["intent_path"] == prefix + "-intent.json", "Intent ID/path mismatch") + intent_path = bound(root, attempt["intent_path"], attempt["intent_sha256"]) + declared.add(intent_path.relative_to(root).as_posix()) + intent = read(intent_path) + expected = {"id": item["id"], "arm": arm, "split": role, "attempt": number, + "body_sha256": body_sha, "plan_sha256": plan_sha} + require(all(intent.get(key) == value for key, value in expected.items()), "Intent binding changed") + reserve = collect.reservation(body, arm) + require(Decimal(intent["reserved_usd"]) == reserve, "Reservation changed") + if attempt["record_path"] is None: + require(not (root / (prefix + "-response.json")).exists(), "Undeclared physical response") + response = {"status": "interrupted_unknown", "terminal_response": False, + "error_type": "ProcessInterrupted", "created_unix": intent["created_unix"]} + else: + require(attempt["record_path"] == prefix + "-response.json", "Response ID/path mismatch") + path = bound(root, attempt["record_path"], attempt["record_sha256"]) + declared.add(path.relative_to(root).as_posix()) + response = read(path) + require(all(response.get(key) == value for key, value in { + "id": item["id"], "arm": arm, "model": body["model"], "body_sha256": body_sha, + "intent_sha256": attempt["intent_sha256"], "attempt": number}.items()), "Response binding changed") + if response.get("terminal_response"): + require(response.get("resolved_model") == body["model"], + "Returned model differs from the requested model") + require(response.get("service_tier") == body["service_tier"] == "default", + "Returned service tier differs from standard pricing") + action = acquisition.decision(response, number) + require(attempt["action"] == action and attempt["status"] == response["status"], "Attempt decision changed") + cost = response.get("cost") + if cost is not None: + require(cost == collect.openai_cost(response["usage"], body), "Usage cost changed") + require(attempt["known_list_usd"] == cost["usd_estimate"] + and Decimal(attempt["known_budget_usd"]) == Decimal(cost["budget_usd"]) + and Decimal(attempt["unknown_reservation_usd"]) == 0, "Known attempt ledger changed") + else: + require(attempt["known_list_usd"] is None and attempt["known_budget_usd"] is None + and Decimal(attempt["unknown_reservation_usd"]) == reserve, "Unknown attempt ledger changed") + ledger.append({"id": item["id"], "arm": arm, "role": role, **attempt}) + if action == "terminal": + require(response.get("http_status") == 200 and response.get("binding_valid") is True, + "Invalid terminal identity receipt") + first = (number, response, attempt["record_sha256"]) + require(entry["selected_attempt"] == (first[0] if first else None), "Selected response is not first terminal") + if first: + response = first[1] + text = "".join(part.get("text") or "" for part in response.get("messages", []) if part["type"] == "output_text") + answer = collect.parse_answer(text, item["response_contract"]) if response.get("provider_status") == "completed" else None + if response.get("status") == "complete": + require(response.get("answer") == answer and response.get("valid_answer") == (answer is not None), + "Saved final-answer parse changed") + rows[item["id"]] = {"response": response, "record_sha256": first[2], "answer": answer} + else: + rows[item["id"]] = {"response": {"status": entry["status"]}, "record_sha256": None, "answer": None} + actual = {path.relative_to(root).as_posix() for path in (root / role / arm / "attempts").rglob("*.json")} + require(actual == declared, "Undeclared or missing physical attempt files") + return rows, ledger + + +def grade(prepared, root, official, pair_path, output): + prepared, root = Path(prepared), Path(root) + source, pair = read(prepared / "source-plan.json"), read(pair_path) + for name in ("meta.json", "gold.json"): + bound(prepared, name, source["files_sha256"][name]) + metadata, gold = (read(prepared / name) for name in ("meta.json", "gold.json")) + meta_by_id, gold_by_id = ({row["id"]: row for row in rows} for rows in (metadata, gold)) + require(len(meta_by_id) == len(metadata) == len(gold_by_id) == len(gold), "Duplicate or inconsistent source IDs") + plan, collection = read(root / "plan.json"), read(root / "collection.json") + require(plan["source_plan_sha256"] == sha(prepared / "source-plan.json") + and collection["plan_sha256"] == sha(root / "plan.json"), "Source/acquisition plan mismatch") + require(plan["pair_sha256"] == sha(pair_path), "Pair specification changed") + require(pair["models"] == {"cheap": "gpt-6-luna", "strong": "gpt-6.1-sol"}, "Answer models changed") + for name in ("collect.py", "openai_population.py"): + require(plan["files_sha256"][name] == sha(Path(__file__).parent / name), "Executed collection source changed") + require(set(plan["roles"]) <= {"train", "development", "calibration", "final"}, "Unknown phase role") + module = load_official(Path(official), "4b4f06e5babb015de2ba639bae995a5526182ffb5b5890af54dfcf038580eb34") + ledger, outputs = [], {} + output = Path(output) + require(not output.exists(), "Preserve prior grade outputs") + output.mkdir(parents=True) + for role in plan["roles"]: + input_path = prepared / "question-only" / f"{role}-inputs.json" + require(sha(input_path) == plan["files_sha256"][f"{role}-inputs.json"], "Collected inputs changed") + bound(prepared, f"question-only/{role}-inputs.json", source["files_sha256"][f"question-only/{role}-inputs.json"]) + inputs = read(input_path) + require(len(inputs) == plan["counts"][role] and len({row["id"] for row in inputs}) == len(inputs), "Role population changed") + arms, role_ledger = {}, [] + for arm in ("luna", "sol"): + arms[arm], physical = checked_arm(root, plan, collection, inputs, arm, role) + role_ledger.extend(physical) + ledger.extend(role_ledger) + result_rows = [] + for item in inputs: + meta, reference = meta_by_id[item["id"]], gold_by_id[item["id"]] + require(meta["role"] == role and meta["retained"] is True, "Inactive or cross-role row in acquisition") + require(meta["request_sha256"] == hashlib.sha256(item["request"].encode()).hexdigest(), "Source request changed") + row = {"id": item["id"], "task": meta["task"], "family": meta["family"], "role": role, "dataset": "bbeh", + "source_format_flags": normalize.source_flags(item["request"], meta["task"])} + for arm, label in (("luna", "cheap"), ("sol", "strong")): + selected = arms[arm][item["id"]] + response, answer = selected["response"], selected["answer"] + valid = answer is not None and response.get("status") == "complete" + cost = response.get("cost") + normalized, normalization_rule = normalize.canonicalize(answer, item["request"], meta["task"]) + elapsed = response.get("elapsed_seconds") + require(elapsed is None or (type(elapsed) in (int, float) and math.isfinite(elapsed) and elapsed >= 0), + "Invalid selected-response elapsed time") + values = {"correct": int(valid and normalized is not None and decimal_match(module, normalized, reference["target"])), + "official_raw_correct": int(valid and module.evaluate_correctness(answer, reference["target"])), + "literal_correct": int(valid and answer.strip() == reference["target"].strip()), + "decimal_correct": int(valid and normalized is not None and decimal_match(module, normalized, reference["target"])), + "normalized_answer": normalized, "normalization_rule": normalization_rule, + "valid": valid, "status": response["status"], "provider_status": response.get("provider_status"), + "truncated": response.get("provider_status") == "incomplete", "usage": response.get("usage"), + "cost_usd": float(Decimal(cost["usd_estimate"])) if cost else None, + "elapsed_seconds": elapsed, + "record_sha256": selected["record_sha256"]} + row.update({label + "_" + key: value for key, value in values.items()}) + result_rows.append(row) + result = {"format": "bbeh-paired-grades-v1", "role": role, "n": len(result_rows), + "source_plan_sha256": sha(prepared / "source-plan.json"), "pair_sha256": sha(pair_path), + "collection_sha256": sha(root / "collection.json"), "acquisition_protocol_sha256": sha(root / "plan.json"), + "grader_sha256": sha(Path(__file__)), "official_grader_sha256": sha(official), + "normalizer_sha256": sha(normalize.__file__), "normalization_version": normalize.VERSION, + "grading_semantics": "Source-only option/tuple normalization, then upstream string normalization with exact Decimal numeric equivalence. Original raw official score retained separately. No answer regeneration or target-aware repair.", + "cost_scope": pair["cost_scope"], "models": pair["models"], + "latency_scope": "Selected physical attempt wall time inside acquire(): client construction, HTTP request and response processing; excludes queueing, retry delays and checkpoint commits. Missing measurements remain null.", + "prices_usd_per_million": {"cheap": collect.PRICES["gpt-6-luna"], "strong": collect.PRICES["gpt-6.1-sol"]}, + "rows": result_rows, "acquisition_ledger": {"physical_attempts": len(role_ledger), "attempts": role_ledger}} + acquisition.dump(output / f"grades-{role}.json", result) + outputs[role] = {"path": f"grades-{role}.json", "sha256": sha(output / f"grades-{role}.json"), "n": len(result_rows)} + sums = {key: sum((Decimal(row[key] or "0") for row in ledger), Decimal(0)) + for key in ("known_list_usd", "known_budget_usd", "unknown_reservation_usd")} + require(len(ledger) == collection["physical_attempts"] + and all(value == Decimal(collection[key]) for key, value in sums.items()), "Physical acquisition totals changed") + require(sums["known_budget_usd"] + sums["unknown_reservation_usd"] <= Decimal(plan["budget_usd"]), "Budget cap exceeded") + index = {"format": "bbeh-phase-grading-v1", "status": collection["status"], "roles": outputs, + "physical_attempts": len(ledger), **{key: str(value) for key, value in sums.items()}, + "cap_usd": plan["budget_usd"], "source_plan_sha256": sha(prepared / "source-plan.json"), + "acquisition_protocol_sha256": sha(root / "plan.json"), "pair_sha256": sha(pair_path)} + acquisition.dump(output / "index.json", index) + return index + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--prepared", required=True, type=Path) + parser.add_argument("--collection", required=True, type=Path) + parser.add_argument("--official", required=True, type=Path) + parser.add_argument("--pair", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + result = grade(args.prepared, args.collection, args.official, args.pair, args.output) + print(json.dumps({"roles": result["roles"], "status": result["status"]})) diff --git a/examples/train-a-router/grading-policy.json b/examples/train-a-router/grading-policy.json new file mode 100644 index 000000000..8ca886865 --- /dev/null +++ b/examples/train-a-router/grading-policy.json @@ -0,0 +1,13 @@ +{ + "diagnostics": [ + "raw official correctness", + "literal exact final-answer correctness" + ], + "format": "bbeh-public-grading-policy-v2", + "grader_sha256": "cd5d53a22b024406eecf18de92d24175a287e482dc63d58be6ec01b7a23cbd08", + "normalizer_sha256": "258a60874b11b5bbeefe2a4fe304dfcc2da341eba995258262ce1e5931ef5772", + "official_grader_sha256": "4b4f06e5babb015de2ba639bae995a5526182ffb5b5890af54dfcf038580eb34", + "primary": "Source-aware canonicalization followed by upstream normalization with finite Decimal equality", + "source_plan_bytes_unchanged": true, + "source_review_decision_sha256": "7f07e78805d20a0a8d6c781a1e8d0b3f4267ff513107a043e1cacb97c798a580" +} diff --git a/examples/train-a-router/live.py b/examples/train-a-router/live.py new file mode 100644 index 000000000..02e1b8d5c --- /dev/null +++ b/examples/train-a-router/live.py @@ -0,0 +1,188 @@ +"""Apply one calibration-selected router and call exactly one answer model.""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import time +from urllib.parse import urlsplit + +import numpy as np + +import collect +from embedding import recipes as core +from embedding.cost_iteration import study as costs +from evaluation import utility_router as utility +from reader_io import sync_files +import workflow + + +def selected_policy(run, method, candidate_id): + run = Path(run).resolve() + if (run / "reproduction.json").exists(): + provenance = core.read(run / "reproduction.json") + workflow.checked_run(provenance["artifact_directory"], run) + calibration_path = run / "internal/calibration-frontiers.json" + frozen_path = run / "internal/frozen-frontiers.json" + envelope = core.read(run / "calibration.json") + core.require(envelope["new_blind_evaluation"] is False + and envelope["reproduction_sha256"] == core.sha(run / "reproduction.json") + and envelope["calibration_sha256"] == core.sha(calibration_path) + and envelope["frozen_sha256"] == core.sha(frozen_path), "Reproduction calibration changed") + else: + data = workflow.checked_data(run.parent) + core.require(run == data / "reference", "Use a prepared run or the verified reference directory") + calibration_path = run / "calibration-frontiers.json" + frozen_path = run / "frozen-frontiers.json" + calibration = core.read(calibration_path) + frozen, plan = core.read(frozen_path), core.checked_plan(run / "embedding") + core.require(calibration["format"] == "bbeh-calibration-frontiers-v1" + and calibration["final_outcomes_accessed"] is False, "Expected calibration-only policy selection") + core.require(frozen["calibration_sha256"] == core.sha(calibration_path) + and calibration["source_plan_sha256"] == frozen["source_plan_sha256"] == plan["source_plan_sha256"] + and calibration["pair_sha256"] == frozen["pair_sha256"] == plan["pair_sha256"] + and calibration["plan_sha256"] == frozen["plan_sha256"] == core.sha(run / "embedding/plan.json"), + "Policy source/pair/plan lineage changed") + value = calibration["methods"][method] + core.require(value["model_sha256"] == frozen["methods"][method]["model_sha256"] + and value["candidates"] == frozen["methods"][method]["candidates"], "Frozen policy binding changed") + core.require(not value["metadata_assisted"], "This control requires benchmark metadata unavailable in live requests") + selected = [r for r in value["candidates"] if r["candidate_id"] == candidate_id] + core.require(len(selected) == 1, "Unknown candidate ID; choose one from calibration") + allowed = {r["candidate_id"] for r in value["frontier"]} + core.require(candidate_id in allowed or selected[0]["policy"]["mode"].startswith("always_"), + "Choose a calibration-frontier candidate or always-arm baseline") + return selected[0]["policy"], value["model_sha256"], core.sha(calibration_path) + + +def binding(run, method, expected): + quality = method.removesuffix("__textcost") + native = quality in {"base", "lora"} + path = run / "cost" / ("native-combined-model.json" if native else "combined-model.json") + saved = core.read(path) + core.require(core.sha(path) == expected and quality in saved["methods"], "Quality/cost model binding changed") + model_path = run / ("native/native/trained.json" if native else "embedding/model.json") + core.require(core.sha(model_path) == saved["quality_model_sha256"] + and core.sha(run / "cost/cost-model.json") == saved["cost_model_sha256"], "Fitted model bytes changed") + return quality, core.read(model_path) + + +def embed(item, output, *, base_url=None, self_hosted=False): + from embedding import wire # Optional SIE/httpx dependencies. + from transformers import AutoTokenizer + from embedding import token_audit + + tokenizer = AutoTokenizer.from_pretrained(core.MODEL, revision=core.REVISION, trust_remote_code=False) + count = token_audit.audit_rows([item], tokenizer)[0]["input_tokens"] + core.require(count <= token_audit.MAX_LENGTH, "Question exceeds the encoder context; shorten it rather than truncate") + + endpoint = base_url or wire.BASE_URL + parsed = urlsplit(endpoint) + core.require(parsed.scheme in {"https", "http"} and parsed.hostname and not parsed.username + and not parsed.password and not parsed.query and not parsed.fragment, "Use a plain SIE endpoint URL") + if not self_hosted: + core.require(endpoint.rstrip("/") == wire.BASE_URL, "Use --self-hosted for another endpoint") + core.require(bool(os.environ.get("SIE_API_KEY")), "Set SIE_API_KEY for Cloud embeddings") + capture = wire.WireCapture() + with wire.make_client(capture) as client: + receipt = wire.collect_batches([item], output, { + "n": 1, "batch_size": 1, "instruction": core.INSTRUCTION, "max_seconds": 600, + "budget_usd": "1.00", "model": core.MODEL, "hf_revision": core.REVISION, "dimensions": 2560, + }, client, capture) + core.require(receipt["status"] == "complete", "Embedding failed; physical attempt evidence is saved") + _, matrix, _ = wire.load_vectors(output, [item]) + return matrix, {"encoder_usd": str(wire.decimal(receipt["known_credits_decimal"]) * wire.CREDIT_USD), + "deployment": "SIE Cloud", "hosting_usd": None} + core.require(base_url is not None, "Supply your SIE server with --sie-base-url") + from embedding import selfhost + + # Use the same single-attempt SDK/receipt adapter as dataset encoding. + capture = wire.WireCapture() + with selfhost.make_client(endpoint, capture) as client: + metadata = client.get_model(core.MODEL) + core.require(metadata.get("name") == core.MODEL and metadata.get("revision") == core.REVISION + and metadata.get("loaded") is True and metadata.get("dims", {}).get("dense") == 2560, + "Self-hosted encoder differs from the pinned model") + body = wire.body_for([item], core.INSTRUCTION) + core.write(output / "catalog.json", metadata) + core.write(output / "intent.json", {"request": body, "physical_attempt": 1}) + sync_files(list(output.glob("*.json")), output) + capture.arm(output) + result = client.encode(core.MODEL, body["items"], + output_types=["dense"], instruction=core.INSTRUCTION, output_dtype="float32", is_query=True, + wait_for_capacity=False, max_oom_retries=0, provision_timeout_s=150) + matrix, revision = selfhost.validate(result, [item], capture, client, body) + core.write(output / "receipt.json", {"execution_bundle_sha256": revision, + "http_sha256": core.sha(output / "http.json"), "encoder_usd": None, "hosting_usd": None}) + sync_files(list(output.glob("*.json")), output) + return matrix, {"encoder_usd": None, "deployment": "self-hosted SIE", "hosting_usd": None} + + +def predict(run, item, output, method, quality, model, *, device, base_url, self_hosted): + row = {**item, "task": "live"} + info = {"encoder_usd": None, "hosting_usd": None} + if quality in {"base", "lora"}: + workflow.invoke("native/reader.py", "--run", run / "native", "--device", device, + "--question-file", output / "question.json", "--variant", quality, + "--output", output / "native-prediction.json") + result = core.read(output / "native-prediction.json") + probabilities, expected = np.asarray(result["probabilities"]), np.asarray(result["expected_usd"]) + elif quality in {"embedding", "knn"}: + vector, info = embed(item, output / "encoder", base_url=base_url, self_hosted=self_hosted) + if quality == "embedding": + probabilities = core.predict_heads(vector, model["heads"]) + else: + core.require(core.sha(run / "embedding/reference.npy") == model["reference_sha256"], "Neighbors changed") + reference = np.load(run / "embedding/reference.npy", mmap_mode="r", allow_pickle=False) + probabilities = core.nearest(vector, reference, np.asarray(model["labels"]), model["selected"]["knn"]) + expected = core.expected_cost(probabilities, model["conditional_cost_usd"]) + else: + core.require(quality in {"text_length", "text_topic"}, "Unsupported live method") + result = core.predict_controls(model, [row])[quality] + probabilities = result["probabilities"] + expected = result.get("expected_usd", core.expected_cost(probabilities, model["conditional_cost_usd"])) + if method.endswith("__textcost"): + expected = costs.predict_rows(core.read(run / "cost/cost-model.json"), [row], "text_length_log_ridge") + return probabilities, expected, info + + +def answer(run, question, output, *, method, candidate_id, device="cpu", base_url=None, self_hosted=False): + import openai_population # Optional HTTP dependency; no provider call until the route is saved. + + run, output = Path(run).resolve(), Path(output).resolve() + core.require(isinstance(question, str) and bool(question.strip()), "Provide the full question and answer format") + core.require(not output.exists(), "Use a new output directory to avoid repeating unresolved calls") + core.require(bool(os.environ.get("OPENAI_API_KEY")), "Set OPENAI_API_KEY for Sol or Luna answers") + policy, model_hash, calibration_hash = selected_policy(run, method, candidate_id) + output.mkdir(parents=True) + item = {"id": "live:" + hashlib.sha256(question.encode()).hexdigest()[:20], "request": question, + "response_contract": {"type": "exact_string", "max_length": 2048}} + core.write(output / "question.json", item) + started = time.perf_counter() + if policy["mode"].startswith("always_"): + strong = policy["mode"] == "always_strong" + probabilities, expected, info = None, None, {"router_bypassed": True, "encoder_usd": "0"} + else: + quality, model = binding(run, method, model_hash) + probabilities, expected, info = predict(run, item, output, method, quality, model, + device=device, base_url=base_url, self_hosted=self_hosted) + strong = bool(utility.replay(policy, probabilities, expected)[0]) + arm = "sol" if strong else "luna" + route = {"method": method, "candidate_id": candidate_id, "policy": policy, + "calibration_sha256": calibration_hash, "model": collect.ARMS[arm]["model"], + "probabilities": None if probabilities is None else probabilities[0].tolist(), + "expected_answer_usd": None if expected is None else expected[0].tolist(), + "router_seconds_including_load": time.perf_counter() - started, **info} + core.write(output / "route.json", route) + core.write(output / "answer-intent.json", {"request": collect.body_for(item, arm), "physical_attempts": 1}) + sync_files(list(output.glob("*.json")), output) + result = openai_population.acquire(item, arm) + core.write(output / "answer.json", result) + sync_files([output / "answer.json"], output) + disposition = openai_population.decision(result, 1) + core.require(disposition == "terminal" and result.get("status") == "complete" + and result.get("valid_answer") is True, + "Answer call did not produce a valid response under the requested model/settings; receipt saved, no retry") + return {"model": route["model"], "answer": result.get("answer"), "status": result["status"], + "answer_cost": result.get("cost"), "encoder_usd": info.get("encoder_usd"), "output": str(output)} diff --git a/examples/train-a-router/native/base.py b/examples/train-a-router/native/base.py new file mode 100644 index 000000000..d4a752ee9 --- /dev/null +++ b/examples/train-a-router/native/base.py @@ -0,0 +1,98 @@ +"""Unchanged-weight Laya baseline with matched train/development cost estimates.""" + +import argparse +import json +from pathlib import Path + +import numpy as np + +import bbeh_native as native +import recipe as core + + +def prepare(run, training_grades, development_grades, initial_checkpoint, output): + run, output = Path(run), Path(output) + plan, rows = core.checked_plan(run) + core.require(not output.exists(), "Preserve the previous base export") + y, costs, hashes = [], [], {} + for role, path in (("train", training_grades), ("development", development_grades)): + _, labels, usd = core.grade_arrays(plan, rows, core.read(path), role) + y.append(labels) + costs.append(usd) + hashes[role] = core.sha(path) + stats = core.conditional_usd(np.concatenate(y), np.concatenate(costs)) + output.mkdir(parents=True) + export = native.copy_base_checkpoint(initial_checkpoint, output / "checkpoint", plan["native"]) + result = { + "format": "bbeh-unchanged-base-v1", "plan_sha256": core.sha(run / "plan.json"), + "source_plan_sha256": plan["source_plan_sha256"], "pair_sha256": plan["pair_sha256"], + "adapter_sha256": core.sha(__file__), "grade_files_sha256": hashes, + "native_source_sha256": core.sources(), "checkpoint": export, + "conditional_cost_usd": stats, + "weights_fitted": False, "calibration_or_final_outcomes_accessed": False, + "cost_scope": "The same train+development conditional USD means as the native refit; base weights are unchanged.", + } + core.write(output / "base.json", result) + return result + + +def checked_base(run, base): + run, base = Path(run), Path(base) + plan, rows = core.checked_plan(run) + model = core.read(base / "base.json") + core.require(model["format"] == "bbeh-unchanged-base-v1" and model["plan_sha256"] == core.sha(run / "plan.json") + and model["source_plan_sha256"] == plan["source_plan_sha256"] and model["pair_sha256"] == plan["pair_sha256"] + and model["adapter_sha256"] == core.sha(__file__) and model["native_source_sha256"] == core.sources() + and model["weights_fitted"] is False and model["calibration_or_final_outcomes_accessed"] is False, + "Base source/plan/weight contract changed") + actual = native.checkpoint_files(base / "checkpoint") + core.require(actual == model["checkpoint"]["files_sha256"] and model["checkpoint"]["weights_byte_identical"] is True + and actual.get("model.safetensors") == native.INITIAL_WEIGHTS_SHA256, "Base checkpoint bytes changed") + return plan, rows, model + + +def score(run, base, role, output, device): + core.require(role in ("calibration", "final") and device in ("cpu", "cuda"), "Invalid input-only scoring role/device") + core.require(not Path(output).exists(), "Preserve previous scores") + plan, rows, model = checked_base(run, base) + selected = [row for row in rows if row["role"] == role] + probabilities = native.checkpoint_probabilities(Path(base) / "checkpoint", selected, plan["pair"], + plan["native"], device=device) + expected = core.expected_usd(probabilities, model["conditional_cost_usd"]) + result = { + "format": "bbeh-input-only-scores-v1", "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], "model_sha256": core.sha(Path(base) / "base.json"), + "outcomes_accessed": False, "role": role, "ids": [row["id"] for row in selected], + "methods": {"base": {"probabilities": probabilities.tolist(), "expected_usd": expected.tolist(), + "router_kind": "native", "metadata_assisted": False}}, + "device": device, "router_hosting_usd": None, + } + core.write(output, result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("prepare") + p.add_argument("--run", required=True, type=Path) + p.add_argument("--training-grades", required=True, type=Path) + p.add_argument("--development-grades", required=True, type=Path) + p.add_argument("--initial-checkpoint", required=True, type=Path) + p.add_argument("--output", required=True, type=Path) + p = sub.add_parser("score") + p.add_argument("--run", required=True, type=Path) + p.add_argument("--base", required=True, type=Path) + p.add_argument("--split", choices=("calibration", "final"), required=True) + p.add_argument("--device", choices=("cpu", "cuda"), default="cpu") + p.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + if args.command == "prepare": + result = prepare(args.run, args.training_grades, args.development_grades, args.initial_checkpoint, args.output) + else: + result = score(args.run, args.base, args.split, args.output, args.device) + print(json.dumps({"format": result["format"], "output": str(args.output)})) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/native/bbeh_native.py b/examples/train-a-router/native/bbeh_native.py new file mode 100644 index 000000000..7f58db022 --- /dev/null +++ b/examples/train-a-router/native/bbeh_native.py @@ -0,0 +1,631 @@ +"""Pinned Laya recipe with explicit BBEH train/development roles and audited full context.""" + +from __future__ import annotations + +import argparse +import gc +import hashlib +import importlib.metadata +import json +import os +import random +import shutil +import tempfile +import time +from dataclasses import asdict +from pathlib import Path + +import numpy as np +import recipe as core + +INITIAL_WEIGHTS_SHA256 = "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c" +OPTION_ORDER = ("incorrect", "correct") + + +def checkpoint_files(root): + root = Path(root) + return {path.relative_to(root).as_posix(): core.sha(path) for path in sorted(root.rglob("*")) if path.is_file()} + + +def copy_base_checkpoint(source, destination, native, *, expected_weights_sha256=INITIAL_WEIGHTS_SHA256): + """Preserve original weights/tokenizer/encoder; change only the context limits.""" + source, destination = Path(source), Path(destination) + max_len, head_len = native["max_len"], native["head_max_len"] + original = checkpoint_files(source) + core.require(original.get("model.safetensors") == expected_weights_sha256, "Initial weights changed") + core.require("rl_agent_config.json" in original, "Missing original Laya configuration") + shutil.copytree(source, destination, symlinks=False) + config = core.read(source / "rl_agent_config.json") + config.update(max_len=max_len, head_max_len=head_len) + config_path = destination / "rl_agent_config.json" + config_path.unlink() + core.write(config_path, config) + exported = checkpoint_files(destination) + core.require( + set(original) == set(exported) + and all(original[name] == exported[name] for name in original if name != "rl_agent_config.json"), + "Unchanged base export altered non-context checkpoint bytes", + ) + return { + "original_files_sha256": original, + "files_sha256": exported, + "weights_byte_identical": True, + "amended_config_fields": {"max_len": max_len, "head_max_len": head_len}, + } + + +def load_checkpoint_preserving(lt, checkpoint): + """Laya normalizes tokenizer metadata on load; isolate that from immutable files.""" + checkpoint = Path(checkpoint).resolve() + with tempfile.TemporaryDirectory(prefix="router-laya-load-") as temporary: + local = Path(temporary) + for path in sorted(checkpoint.rglob("*")): + if not path.is_file(): + continue + target = local / path.relative_to(checkpoint) + target.parent.mkdir(parents=True, exist_ok=True) + if path.name == "model.safetensors": + target.symlink_to(path.resolve()) # The loader only reads weights; avoid a multi-GB copy. + else: + shutil.copy2(path, target) + return lt.load_checkpoint(str(local)) + + +def targets(values, count): + result = np.asarray(values, dtype=float) + core.require( + result.shape == (count, 2) and count > 0 and np.isin(result, (0, 1)).all(), "Wrong paired binary targets" + ) + return result + + +def question_rows(rows, pair, labels=None): + y = np.zeros((len(rows), 2)) if labels is None else targets(labels, len(rows)) + questions = core.questions(pair) + return [ + { + "state": row["request"], + "questions": questions, + "gold": { + f"{arm}_correct": {"probabilities": {"incorrect": float(1 - values[i]), "correct": float(values[i])}} + for i, arm in enumerate(core.ARMS) + }, + } + for row, values in zip(rows, y, strict=True) + ] + + +def encode_rows(lt, tokenizer, rows, pair, native, labels=None): + max_len, head_len = native["max_len"], native["head_max_len"] + y = np.zeros((len(rows), 2)) if labels is None else targets(labels, len(rows)) + questions = core.questions(pair) + items, skipped = lt.items_from_rows(tokenizer, question_rows(rows, pair, y), max_len, head_len, label_smoothing=0.0) + core.require(not skipped and len(items) == 2 * len(rows), f"No native item may be skipped: {skipped}") + for index, row in enumerate(rows): + uncapped = lt.encode_state(tokenizer, row["request"], 1_000_000) + core.require(len(uncapped) <= max_len, "State exceeds fixed context; do not silently truncate") + for arm, name in enumerate(core.ARMS): + item = items[2 * index + arm] + core.require( + item["state_ids"] == uncapped + and item["q"]["ins"] == questions[f"{name}_correct"]["instructions"] + and list(item["q"]["crit"]) == list(OPTION_ORDER) + and item["k"] == 2 + and item["target"] == [float(1 - y[index, arm]), float(y[index, arm])], + "Native request/question/option/label order changed", + ) + head, _, _ = lt.build_head(tokenizer, item["q"], head_len) + uncut_head, _, _ = lt.build_head(tokenizer, item["q"], max_len) + core.require(head == uncut_head, "Decision instructions/options exceed head cap") + _, markers, truncation = lt.build_sequence( + tokenizer, None, item["q"], max_len, head_len, state_ids=uncapped, return_truncation_stats=True + ) + core.require( + len(markers) == 2 and not truncation["truncated"] and truncation["state_tokens_dropped"] == 0, + "Combined native request would be truncated", + ) + return items + + +def margins_from_records(records, y): + y = targets(y, len(y)) + logits = np.asarray([row[1] for row in records], dtype=float) + actual = np.asarray([row[2] for row in records], dtype=float) + flat = y.reshape(-1) + core.require( + logits.shape == actual.shape == (len(y) * 2, 2) + and np.isfinite(logits).all() + and all(row[3] == 2 for row in records) + and np.array_equal(actual, np.column_stack([1 - flat, flat])), + "Scoring order or binary targets changed", + ) + return (logits[:, 1] - logits[:, 0]).reshape(len(y), 2) + + +def summed_nll(margins, y): + values = np.asarray(margins, dtype=float) + y = targets(y, len(y)) + core.require(values.shape == y.shape and np.isfinite(values).all(), "Invalid logits") + return float(np.logaddexp(0, (1 - 2 * y) * values).sum(axis=1).mean()) + + +def select_epoch(history): + core.require([row["epoch"] for row in history] == list(range(9)), "Retain all epoch0..8 candidates") + core.require(all(np.isfinite(row["summed_binary_nll"]) for row in history), "Invalid held-family loss") + return min(range(9), key=lambda epoch: (history[epoch]["summed_binary_nll"], epoch)) + + +def config_values(epochs, native): + max_len, head_len = native["max_len"], native["head_max_len"] + return { + "epochs": epochs, + "micro_batch": native["micro_batch"], + "grad_accum": native["grad_accum"], + "encoder_lr": native["encoder_lr"], + "head_lr": native["head_lr"], + "weight_decay": native["weight_decay"], + "seed": 89, + "max_len": max_len, + "head_max_len": head_len, + "loss": "soft-ce", + "shuffle_options": ("choice",), + "gradient_checkpointing": native["gradient_checkpointing"], + "label_smoothing": 0.0, + "log_every": 64, + } + + +def memory_probe(run, initial_checkpoint, lengths_path, output): + """One discarded synthetic update at the longest source-only request.""" + run, initial = Path(run), Path(initial_checkpoint) + plan, metadata = core.checked_plan(run) + native = plan["native"] + max_len, head_len = native["max_len"], native["head_max_len"] + audit = core.read(lengths_path) + core.require( + audit["source_only"] + and audit["source_plan_sha256"] == plan["source_plan_sha256"] + and audit["pair_sha256"] == plan["pair_sha256"], + "Memory probe must use the pinned source-only length audit", + ) + core.require(core.sha(initial / "model.safetensors") == INITIAL_WEIGHTS_SHA256, "Initial weights changed") + longest = max(audit["rows"], key=lambda row: (row["sequence_tokens"], row["id"])) + by_id = {row["id"]: row for row in metadata} + core.require(set(by_id) == {row["id"] for row in audit["rows"]}, "Length audit source population differs") + row = by_id[longest["id"]] + core.require( + hashlib.sha256(row["request"].encode()).hexdigest() == longest["request_sha256"], + "Longest source request changed", + ) + os.environ.update(TORCH_COMPILE_DISABLE="1", TORCHDYNAMO_DISABLE="1") + import laya.train as lt + import torch + from peft import LoraConfig, get_peft_model + + versions = {name: importlib.metadata.version(name).split("+", 1)[0] for name in native["versions"]} + core.require(versions == native["versions"] and torch.cuda.is_available(), "Pinned CUDA runtime required") + torch.manual_seed(89) + np.random.seed(89) + random.seed(89) + model, tokenizer, _ = load_checkpoint_preserving(lt, initial) + core.require(model.encoder.config.max_position_embeddings >= max_len, "Audited context exceeds the pinned encoder") + model.encoder.config.reference_compile = False + model.encoder = get_peft_model( + model.encoder, + LoraConfig( + r=native["rank"], + lora_alpha=native["alpha"], + lora_dropout=native["dropout"], + target_modules=r".*attn\.(Wqkv|Wo)", + bias="none", + ), + ) + device = torch.device("cuda") + model.to(device) + count = max(native["micro_batch"] * native["grad_accum"], native["evaluation_batch"]) // 2 + y = np.array([[i % 2, 1 - i % 2] for i in range(count)]) + items = encode_rows(lt, tokenizer, [row] * count, plan["pair"], native, y) + torch.cuda.reset_peak_memory_stats() + started = time.monotonic() + losses = lt.train_model( + model, tokenizer, items, lt.TrainConfig(**config_values(1, native)), device, max_len, head_len + ) + records = lt.calibration_records( + model, tokenizer, items, device, max_len, head_len, batch_size=native["evaluation_batch"] + ) + margins = margins_from_records(records, y) + torch.cuda.synchronize() + core.require(np.isfinite(losses).all() and np.isfinite(margins).all(), "Synthetic memory step was nonfinite") + receipt = { + "format": "bbeh-native-memory-probe-v1", + "plan_sha256": core.sha(run / "plan.json"), + "length_audit_sha256": core.sha(lengths_path), + "request_id": row["id"], + "request_sha256": longest["request_sha256"], + "state_tokens": longest["state_tokens"], + "max_len": max_len, + "head_max_len": head_len, + "micro_batch": native["micro_batch"], + "grad_accum": native["grad_accum"], + "evaluation_batch": native["evaluation_batch"], + "gpu": torch.cuda.get_device_name(0), + "total_gpu_bytes": torch.cuda.get_device_properties(0).total_memory, + "peak_allocated_bytes": torch.cuda.max_memory_allocated(), + "peak_reserved_bytes": torch.cuda.max_memory_reserved(), + "elapsed_seconds": time.monotonic() - started, + "versions": versions, + "labels": "Synthetic alternating labels only; all updated parameters discarded", + "actual_outcomes_accessed": False, + "passed": True, + } + del model, tokenizer, items + gc.collect() + torch.cuda.empty_cache() + core.write(output, receipt) + return receipt + + +def train(run, training_grades, development_grades, initial_checkpoint, *, commit=lambda: None): + run, initial = Path(run), Path(initial_checkpoint) + core.require(not (run / "final-access.json").exists(), "Final outcomes already accessed") + plan, metadata = core.checked_plan(run) + native = plan["native"] + max_len, head_len = native["max_len"], native["head_max_len"] + training, development = core.read(training_grades), core.read(development_grades) + fit_rows, fit_y, fit_costs = core.grade_arrays(plan, metadata, training, "train") + held_rows, held_y, held_costs = core.grade_arrays(plan, metadata, development, "development") + core.require(training["cost_scope"] == development["cost_scope"], "Mixed answer-cost scopes") + rows = fit_rows + held_rows + y, costs = np.concatenate([fit_y, held_y]), np.concatenate([fit_costs, held_costs]) + held = np.array([False] * len(fit_rows) + [True] * len(held_rows)) + cost_statistics = core.conditional_usd(y, costs) + core.require(core.sha(initial / "model.safetensors") == INITIAL_WEIGHTS_SHA256, "Pinned initial weights changed") + core.require(held.any() and (~held).any(), "Empty fixed train/development role") + out = run / "native" + out.mkdir(parents=True, exist_ok=False) + os.environ.update(TORCH_COMPILE_DISABLE="1", TORCHDYNAMO_DISABLE="1") + import laya.train as lt + import torch + from peft import LoraConfig, get_peft_model + from safetensors.torch import save_file + + versions = {name: importlib.metadata.version(name) for name in native["versions"]} + core.require( + {name: value.split("+", 1)[0] for name, value in versions.items()} == native["versions"] + and torch.cuda.is_available(), + "Pinned Laya training dependencies and CUDA required", + ) + device = torch.device("cuda") + core.write( + out / "training-start.json", + { + "plan_sha256": core.sha(run / "plan.json"), + "training_grades_sha256": core.sha(training_grades), + "development_grades_sha256": core.sha(development_grades), + "versions": versions, + "gpu": torch.cuda.get_device_name(0), + "config": native, + "initial_files_sha256": { + path.relative_to(initial).as_posix(): core.sha(path) + for path in sorted(initial.rglob("*")) + if path.is_file() + }, + "training_rows": len(rows), + "training_items": len(rows) * 2, + "selection_fit_rows": int((~held).sum()), + "selection_validation_rows": int(held.sum()), + }, + ) + commit() + started, history = time.monotonic(), [] + + def release(): + gc.collect() + torch.cuda.synchronize() + torch.cuda.empty_cache() + + def load(lora): + torch.manual_seed(89) + np.random.seed(89) + random.seed(89) + model, tokenizer, configuration = load_checkpoint_preserving(lt, initial) + core.require( + model.encoder.config.max_position_embeddings >= max_len, "Audited context exceeds the pinned encoder" + ) + configuration.update(max_len=max_len, head_max_len=head_len) + model.encoder.config.reference_compile = False + if lora: + model.encoder = get_peft_model( + model.encoder, + LoraConfig( + r=native["rank"], + lora_alpha=native["alpha"], + lora_dropout=native["dropout"], + target_modules=r".*attn\.(Wqkv|Wo)", + bias="none", + ), + ) + return model.to(device), tokenizer, configuration + + def score(model, tokenizer, items, labels): + records = lt.calibration_records( + model, tokenizer, items, device, max_len, head_len, batch_size=native["evaluation_batch"] + ) + return margins_from_records(records, labels) + + fit_rows, held_rows = ( + [row for row, flag in zip(rows, held, strict=True) if not flag], + [row for row, flag in zip(rows, held, strict=True) if flag], + ) + model, tokenizer, configuration = load(True) + trainable = {name: parameter.numel() for name, parameter in model.named_parameters() if parameter.requires_grad} + core.require( + all("lora_" in name for name in trainable if name.startswith("encoder.")), + "Only LoRA parameters may train inside the encoder", + ) + core.write( + out / "trainable-parameters.json", + { + "total_parameters": sum(parameter.numel() for parameter in model.parameters()), + "trainable_parameters": sum(trainable.values()), + "names_and_sizes": trainable, + "encoder_original_parameters_frozen": True, + }, + ) + fitting = encode_rows(lt, tokenizer, fit_rows, plan["pair"], native, y[~held]) + validation = encode_rows(lt, tokenizer, held_rows, plan["pair"], native, y[held]) + config = lt.TrainConfig(**config_values(8, native)) + core.write(out / "selection-config.json", asdict(config)) + + def observe(epoch, train_loss, model=model, tokenizer=tokenizer, validation=validation): + values = score(model, tokenizer, validation, y[held]) + record = { + "epoch": epoch + 1, + "train_loss": train_loss, + "summed_binary_nll": summed_nll(values, y[held]), + "margins": values.tolist(), + "elapsed_seconds": time.monotonic() - started, + } + history.append(record) + core.write(out / f"epoch-{epoch + 1}.json", record) + commit() + print( + json.dumps({key: record[key] for key in ("epoch", "train_loss", "summed_binary_nll", "elapsed_seconds")}), + flush=True, + ) + model.train() + + observe(-1, None) + lt.train_model(model, tokenizer, fitting, config, device, max_len, head_len, on_epoch_end=observe) + epoch = select_epoch(history) + core.write( + out / "selected-epoch.json", + { + "epoch": epoch, + "validation_ids": [row["id"] for row in held_rows], + "criterion": native["selection"], + "history": history, + }, + ) + observe = None + del model, tokenizer, fitting, validation + release() + checkpoints, refit_losses = {}, [] + for variant in ("base", "lora"): + model, tokenizer, configuration = load(variant == "lora") + validation = encode_rows(lt, tokenizer, held_rows, plan["pair"], native, y[held]) + if variant == "lora": + config = lt.TrainConfig(**config_values(epoch, native)) + core.write(out / "refit-config.json", asdict(config)) + if epoch: + fitting = encode_rows(lt, tokenizer, rows, plan["pair"], native, y) + refit_losses = lt.train_model( + model, + tokenizer, + fitting, + config, + device, + max_len, + head_len, + on_epoch_end=lambda e, loss: ( + core.write(out / f"refit-epoch-{e + 1}.json", {"epoch": e + 1, "loss": loss}), + commit(), + ), + ) + del fitting + before = score(model, tokenizer, validation, y[held]) + if variant == "lora": + model.encoder = model.encoder.merge_and_unload() + merged = score(model, tokenizer, validation, y[held]) + merge_error = float(np.abs(merged - before).max()) + core.require(merge_error <= 0.001, "LoRA merge numerical check failed") + destination = out / variant + base_copy = None + if variant == "base": + base_copy = copy_base_checkpoint(initial, destination, native) + else: + lt.save_checkpoint(model, tokenizer, configuration, str(destination)) + save_file( + {name: tensor.detach().float().cpu().contiguous() for name, tensor in model.state_dict().items()}, + str(destination / "model.safetensors"), + ) + del model + release() + model, _, saved = load_checkpoint_preserving(lt, destination) + core.require( + saved["max_len"] == max_len and saved["head_max_len"] == head_len, "Export reverted to512/context defaults" + ) + model.encoder.config.reference_compile = False + model.to(device).eval() + exported = score(model, tokenizer, validation, y[held]) + export_error = float(np.abs(exported - before).max()) + core.require(export_error <= 0.03, "Checkpoint reload numerical check failed") + checkpoints[variant] = { + "files_sha256": checkpoint_files(destination), + "base_copy": base_copy, + "max_merge_margin_error": merge_error, + "max_export_margin_error": export_error, + "reference_margins": before.tolist(), + "exported_margins": exported.tolist(), + } + core.write(out / f"{variant}-export.json", checkpoints[variant]) + commit() + del model, tokenizer, validation + release() + trained = { + "format": "bbeh-paired-native-v1", + "plan_sha256": core.sha(run / "plan.json"), + "training_grades_sha256": core.sha(training_grades), + "development_grades_sha256": core.sha(development_grades), + "grader_sha256": {"train": training["grader_sha256"], "development": development["grader_sha256"]}, + "cost_scope": training["cost_scope"], + "selection_train_ids": [r["id"] for r in fit_rows], + "selection_development_ids": [r["id"] for r in held_rows], + "training_ids": [row["id"] for row in rows], + "selected_epoch": epoch, + "refit_losses": refit_losses, + "conditional_cost_usd": cost_statistics, + "checkpoints": checkpoints, + "training_start_sha256": core.sha(out / "training-start.json"), + "elapsed_seconds": time.monotonic() - started, + "hosting": "Unpriced native inference; report gross answer-model savings until hosting is measured.", + "boundary": "Development selects epoch; fresh train+development refit; no calibration/final outcomes read.", + } + core.write(out / "trained.json", trained) + commit() + return trained + + +def checkpoint_probabilities(checkpoint, rows, pair, native, *, device="cuda"): + """Input-only score helper; canonical cheap/strong column order, no token truncation.""" + max_len, head_len = native["max_len"], native["head_max_len"] + os.environ.update(TORCH_COMPILE_DISABLE="1", TORCHDYNAMO_DISABLE="1") + import laya.train as lt + import torch + + model, tokenizer, config = load_checkpoint_preserving(lt, checkpoint) + core.require( + config["max_len"] == max_len and config["head_max_len"] == head_len, "Wrong native context configuration" + ) + model.encoder.config.reference_compile = False + model.to(torch.device(device)).eval() + items = encode_rows(lt, tokenizer, rows, pair, native) + records = lt.calibration_records( + model, + tokenizer, + items, + torch.device(device), + max_len, + head_len, + batch_size=native["evaluation_batch"] if device == "cuda" else 1, + ) + margins = margins_from_records(records, np.zeros((len(rows), 2))) + probabilities = np.exp(-np.logaddexp(0, -margins)) + del model, tokenizer, items, records + gc.collect() + if str(device).startswith("cuda"): + torch.cuda.synchronize() + torch.cuda.empty_cache() + return probabilities + + +def score_native(run, output, *, device="cuda", commit=lambda: None): + run = Path(run) + plan, rows = core.checked_plan(run) + native = plan["native"] + root = run / "native" + trained = core.read(root / "trained.json") + core.require( + trained["plan_sha256"] == core.sha(run / "plan.json") + and trained["training_ids"] == plan["populations"]["train"] + plan["populations"]["development"], + "Different trained population/plan", + ) + selected = [row for row in rows if row["role"] in ("calibration", "final")] + result = { + "format": "bbeh-input-only-native-v1", + "plan_sha256": core.sha(run / "plan.json"), + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "trained_sha256": core.sha(root / "trained.json"), + "outcomes_accessed": False, + "variants": {}, + } + for variant in ("base", "lora"): + checkpoint = root / variant + actual = { + path.relative_to(checkpoint).as_posix(): core.sha(path) + for path in sorted(checkpoint.rglob("*")) + if path.is_file() + } + core.require(actual == trained["checkpoints"][variant]["files_sha256"], "Checkpoint bytes changed") + probabilities = checkpoint_probabilities(checkpoint, selected, plan["pair"], native, device=device) + expected = core.expected_usd(probabilities, trained["conditional_cost_usd"]) + scored = { + "method": variant, + "conditional_cost_usd": trained["conditional_cost_usd"], + "router_hosting_usd": None, + "roles": {}, + } + for role in ("calibration", "final"): + indices = [i for i, row in enumerate(selected) if row["role"] == role] + scored["roles"][role] = { + "ids": [selected[i]["id"] for i in indices], + "probabilities": probabilities[indices].tolist(), + "expected_usd": expected[indices].tolist(), + } + result["variants"][variant] = scored + core.write(root / f"{variant}-input-scores.json", scored) + commit() + for role in ("calibration", "final"): + core.write( + root / f"scores-{role}.json", + { + "format": "bbeh-input-only-scores-v1", + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "model_sha256": core.sha(root / "trained.json"), + "outcomes_accessed": False, + "role": role, + "ids": plan["populations"][role], + "methods": { + variant: { + "probabilities": result["variants"][variant]["roles"][role]["probabilities"], + "expected_usd": result["variants"][variant]["roles"][role]["expected_usd"], + "router_kind": "native", + "metadata_assisted": False, + } + for variant in ("base", "lora") + }, + }, + ) + core.write(output, result) + commit() + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", type=Path, required=True) + parser.add_argument("--phase", choices=("memory", "train", "score"), required=True) + parser.add_argument("--training-grades", type=Path) + parser.add_argument("--development-grades", type=Path) + parser.add_argument("--initial-checkpoint", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + if args.phase == "memory": + core.require(args.initial_checkpoint and args.output, "Memory probe needs checkpoint/output") + result = memory_probe(args.run, args.initial_checkpoint, args.run / "lengths.json", args.output) + elif args.phase == "train": + core.require( + args.training_grades and args.development_grades and args.initial_checkpoint, + "Train needs both allowed grade roles and the checkpoint", + ) + result = train(args.run, args.training_grades, args.development_grades, args.initial_checkpoint) + else: + core.require(args.output, "Score needs output") + result = score_native(args.run, args.output) + print(json.dumps({key: result[key] for key in ("format", "selected_epoch", "elapsed_seconds") if key in result})) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/native/benchmark.py b/examples/train-a-router/native/benchmark.py new file mode 100644 index 000000000..7fc132cc5 --- /dev/null +++ b/examples/train-a-router/native/benchmark.py @@ -0,0 +1,188 @@ +"""Measure resident two-question inference on a source-only development sample.""" + +from __future__ import annotations + +import argparse +import gc +import importlib.metadata +import math +import os +import statistics +import time +from pathlib import Path + +import recipe as core + + +def select(rows, count=90): + candidates = [row for row in rows if row["role"] == "development"] + ordered = sorted(candidates, key=lambda row: core.text_sha("bbeh-native-timing89:" + row["id"])) + core.require(len(ordered) >= 2, "Need a separate warmup and timed request") + count = min(count, len(ordered) - 1) + return ordered[:count], ordered[count] + + +def summarize(records): + values = sorted(row["elapsed_seconds"] for row in records) + core.require(values and all(math.isfinite(v) and v > 0 for v in values), "Invalid timed durations") + + def quantile(q): + index = q * (len(values) - 1) + lo, hi = math.floor(index), math.ceil(index) + return values[lo] + (values[hi] - values[lo]) * (index - lo) + + return { + "requests": len(values), + "mean_seconds": statistics.mean(values), + "median_seconds": statistics.median(values), + "p90_seconds": quantile(0.90), + "p95_seconds": quantile(0.95), + "reciprocal_mean_service_seconds": len(values) / sum(values), + "by_task": { + task: { + "n": sum(r["task"] == task for r in records), + "mean_seconds": statistics.mean(r["elapsed_seconds"] for r in records if r["task"] == task), + } + for task in sorted({r["task"] for r in records}) + }, + } + + +def measure(study, output, *, variant, device, resource_rate_usd_per_second=None, resource_rate_source=None): + study, output = Path(study), Path(output) + core.require(variant in ("base", "lora") and device in ("cuda", "cpu"), "Unsupported timing variant/device") + if resource_rate_usd_per_second is not None: + core.require( + math.isfinite(resource_rate_usd_per_second) and resource_rate_usd_per_second > 0 and resource_rate_source, + "Need an explicit priced resource and source", + ) + output.mkdir(parents=True, exist_ok=False) + os.environ.update(TORCH_COMPILE_DISABLE="1", TORCHDYNAMO_DISABLE="1", TOKENIZERS_PARALLELISM="false") + import laya.train as lt + import numpy as np + import torch + + import bbeh_native as native + + torch.set_num_threads(8) + plan, rows = core.checked_plan(study) + trained = core.read(study / "native/trained.json") + core.require(trained["plan_sha256"] == core.sha(study / "plan.json"), "Training plan changed") + sample, warmup = select(rows) + protocol = { + "rule": "First90 development IDs by SHA256('bbeh-native-timing89:'+id); nextID is separate warmup; all if fewer than91", + "ids": [row["id"] for row in sample], + "warmup_id": warmup["id"], + "request_sha256": {row["id"]: core.text_sha(row["request"]) for row in sample + [warmup]}, + "source_plan_sha256": plan["source_plan_sha256"], + "outcomes_read": False, + "policy_computation": "Signed quality advantage and expected USD; lambda0 is timing-only, not a selected policy", + } + core.write(output / "sample-protocol.json", protocol) + checkpoint = study / "native" / variant + verification_start = time.monotonic() + core.require( + native.checkpoint_files(checkpoint) == trained["checkpoints"][variant]["files_sha256"], "Checkpoint changed" + ) + verification_seconds = time.monotonic() - verification_start + load_start = time.monotonic() + model, tokenizer, saved = native.load_checkpoint_preserving(lt, checkpoint) + settings = plan["native"] + maximum, head = settings["max_len"], settings["head_max_len"] + core.require(saved["max_len"] == maximum and saved["head_max_len"] == head, "Export context differs") + model.encoder.config.reference_compile = False + execution_device = torch.device(device) + model.to(execution_device).eval() + core.require({str(p.dtype) for p in model.parameters()} == {"torch.float32"}, "Expected FP32 resident inference") + if device == "cuda": + torch.cuda.synchronize() + load_seconds = time.monotonic() - load_start + + def timed(row): + if device == "cuda": + torch.cuda.synchronize() + started = time.perf_counter() + items = native.encode_rows(lt, tokenizer, [row], plan["pair"], settings) + values = lt.calibration_records(model, tokenizer, items, execution_device, maximum, head, batch_size=2) + margins = native.margins_from_records(values, np.zeros((1, 2))) + probabilities = np.exp(-np.logaddexp(0, -margins)) + expected = core.expected_usd(probabilities, trained["conditional_cost_usd"]) + decision = bool(probabilities[0, 1] - probabilities[0, 0] >= 0) + if device == "cuda": + torch.cuda.synchronize() + return { + "id": row["id"], + "task": row["task"], + "elapsed_seconds": time.perf_counter() - started, + "probabilities": probabilities[0].tolist(), + "expected_usd": expected[0].tolist(), + "timing_only_strong": decision, + "state_tokens": len(items[0]["state_ids"]), + } + + first = timed(warmup) + if device == "cuda": + torch.cuda.reset_peak_memory_stats() + records = [timed(row) for row in sample] + summary = summarize(records) + report = { + "format": "bbeh-resident-native-timing-v1", + "variant": variant, + "device": device, + "plan_sha256": core.sha(study / "plan.json"), + "trained_sha256": core.sha(study / "native/trained.json"), + "benchmark_sha256": core.sha(__file__), + "sample_protocol_sha256": core.sha(output / "sample-protocol.json"), + "checkpoint_verification_seconds": verification_seconds, + "model_load_seconds": load_seconds, + "warmup": first, + "records": records, + "summary": summary, + "native_context": settings, + "runtime_versions": {name: importlib.metadata.version(name) for name in settings["versions"]}, + "cpu_threads": torch.get_num_threads(), + "gpu": torch.cuda.get_device_name(0) if device == "cuda" else None, + "timing_includes": "Tokenization/full-context validation, both resident model questions, sigmoid probabilities and expected-cost/decision arithmetic", + "timing_excludes": "Checkpoint verification/loading, startup, warmup, artifact I/O, HTTP/network, queueing and idle capacity", + "limitations": "Sequential resident service measurements; reciprocal mean is not measured concurrent throughput", + "resource_rate_usd_per_second": resource_rate_usd_per_second, + "resource_rate_source": resource_rate_source, + "busy_resource_usd_per_10000": None + if resource_rate_usd_per_second is None + else summary["mean_seconds"] * 10000 * resource_rate_usd_per_second, + "cost_scope": "Timing-sample-based busy-resource projection only when a priced rate is supplied; not observed billing or full deployment cost", + } + if device == "cuda": + report.update( + peak_allocated_bytes=torch.cuda.max_memory_allocated(), peak_reserved_bytes=torch.cuda.max_memory_reserved() + ) + core.write(output / "report.json", report) + model, tokenizer = None, None + gc.collect() + if device == "cuda": + torch.cuda.empty_cache() + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--study", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--variant", choices=("base", "lora"), required=True) + parser.add_argument("--device", choices=("cpu", "cuda"), required=True) + parser.add_argument("--resource-rate-usd-per-second", type=float) + parser.add_argument("--resource-rate-source") + args = parser.parse_args() + report = measure( + args.study, + args.output, + variant=args.variant, + device=args.device, + resource_rate_usd_per_second=args.resource_rate_usd_per_second, + resource_rate_source=args.resource_rate_source, + ) + print(report["summary"]) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/native/development_report.py b/examples/train-a-router/native/development_report.py new file mode 100644 index 000000000..ce0e9c678 --- /dev/null +++ b/examples/train-a-router/native/development_report.py @@ -0,0 +1,183 @@ +"""Development utility curves from saved train-only epoch margins; no inference.""" + +from __future__ import annotations + +import argparse +import math +from pathlib import Path + +import numpy as np + +import bbeh_native as native +import recipe as core +from evaluation import frontier, utility_router as utility + + +def epoch_history(run, plan, train_path, development_path, development_rows): + run = Path(run) + start = core.read(run / "native/training-start.json") + core.require( + start["plan_sha256"] == core.sha(run / "plan.json") + and start["training_grades_sha256"] == core.sha(train_path) + and start["development_grades_sha256"] == core.sha(development_path) + and start["config"] == plan["native"] + and start["selection_fit_rows"] == len(plan["populations"]["train"]) + and start["selection_validation_rows"] == len(development_rows) + and start["initial_files_sha256"]["model.safetensors"] == core.INITIAL_WEIGHTS_SHA256, + "Selection source/grade/configuration binding changed", + ) + selected = core.read(run / "native/selected-epoch.json") + history = [core.read(run / f"native/epoch-{epoch}.json") for epoch in range(9)] + core.require( + selected["history"] == history + and selected["validation_ids"] == [row["id"] for row in development_rows] + and selected["criterion"] == plan["native"]["selection"] + and selected["epoch"] == native.select_epoch(history), + "Selection history/order/selected epoch changed", + ) + return history, selected["epoch"] + + +def curve_for_epoch(record, rows, labels, observed_costs, train_costs, bootstrap): + margins = np.asarray(record["margins"], dtype=float) + nll = native.summed_nll(margins, labels) + core.require( + math.isclose(nll, record["summed_binary_nll"], rel_tol=0, abs_tol=1e-12), + "Saved development NLL does not match its epoch margins", + ) + probabilities = np.exp(-np.logaddexp(0, -margins)) + expected = core.expected_usd(probabilities, train_costs) + curve = frontier.enumerate_frontier( + {"probabilities": probabilities, "expected_usd": expected}, labels, observed_costs, None + ) + frontier_ids = {node["candidate_id"] for node in curve["frontier"]} + points = [] + for point in curve["candidates"]: + node = { + "candidate_id": point["candidate_id"], + "rule": point["policy"], + "development": point["calibration"], + "development_matched_random_strong_probability": point["calibration_matched_random_strong_probability"], + } + if point["candidate_id"] in frontier_ids: + routes = utility.replay(point["policy"], probabilities, expected) + fees = np.zeros(len(rows)) if point["policy"]["mode"].startswith("always_") else None + intervals = frontier.uncertainty( + bootstrap, + routes, + labels, + observed_costs, + fees, + point["calibration_matched_random_strong_probability"], + ) + node["paired_ci95"] = { + name.replace("calibration_budget", "development_budget"): value for name, value in intervals.items() + } + points.append(node) + return { + "setting": record["epoch"], + "label": "Unadapted Laya (epoch0)" if record["epoch"] == 0 else f"LoRA epoch{record['epoch']}", + "nll": nll, + "train_loss": record["train_loss"], + "probabilities": probabilities.tolist(), + "expected_usd": expected.tolist(), + "points": points, + "frontier_candidate_ids": sorted(frontier_ids), + "randomized_hull_candidate_ids": curve["randomized_hull_candidate_ids"], + "randomized_hull_interpretation": curve["randomized_hull_interpretation"], + } + + +def build(run, training_grades, development_grades, *, draws=2000): + run = Path(run) + plan, metadata = core.checked_plan(run) + training, development = core.read(training_grades), core.read(development_grades) + train_rows, train_y, train_cost = core.grade_arrays(plan, metadata, training, "train") + rows, y, costs = core.grade_arrays(plan, metadata, development, "development") + core.require(training["cost_scope"] == development["cost_scope"], "Mixed answer-cost accounting") + history, selected = epoch_history(run, plan, training_grades, development_grades, rows) + cost_model = core.conditional_usd(train_y, train_cost) + bootstrap = frontier.Bootstrap(rows, draws) + settings = [] + for record in history: + curve = curve_for_epoch(record, rows, y, costs, cost_model, bootstrap) + curve["selected_by_development_nll"] = record["epoch"] == selected + settings.append(curve) + task_summary = [] + for task in sorted({row["task"] for row in rows}): + indices = [i for i, row in enumerate(rows) if row["task"] == task] + task_summary.append( + { + "task": task, + "n": len(indices), + "families": len({rows[i]["family"] for i in indices}), + "accuracy_percent": (100 * y[indices].mean(axis=0)).tolist(), + "answer_usd_per_10000": (10000 * costs[indices].mean(axis=0)).tolist(), + } + ) + return { + "format": "bbeh-development-candidate-frontiers-v1", + "role": "development", + "plan_sha256": core.sha(run / "plan.json"), + "model_sha256": core.sha(run / "native/selected-epoch.json"), + "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], + "grades_sha256": core.sha(development_grades), + "training_grades_sha256": core.sha(training_grades), + "selection_receipts_sha256": { + name: core.sha(run / "native" / name) + for name in ("training-start.json", "selected-epoch.json", *(f"epoch-{i}.json" for i in range(9))) + }, + "ids": [row["id"] for row in rows], + "n": len(rows), + "families": len({row["family"] for row in rows}), + "tasks": task_summary, + "methods": {"lora": {"router_kind": "native", "metadata_assisted": False, "settings": settings}}, + "unadapted_base_setting": 0, + "selected_settings": {"lora": selected}, + "conditional_cost_usd": cost_model, + "cost_estimation_ids": [row["id"] for row in train_rows], + "cost_estimation_role": "train", + "random": { + "analytic_expectation": True, + "strong_probability_range": [0, 1], + "accuracy_endpoints_percent": (100 * y.mean(axis=0)).tolist(), + "answer_usd_per_10000_endpoints": (10000 * costs.mean(axis=0)).tolist(), + }, + "cost_scope": development["cost_scope"], + "router_cost_scope": "Native hosting unpriced in these curves; answer costs only. Always-arm bypasses have no router cost.", + "interpretation": "Train-only selection models scored on development. This tuning population selects the declared epoch by NLL; curves and pointwise family intervals are diagnostics, not unbiased held-out estimates or simultaneous guarantees. No frontier point is selected here.", + "refit_used_for_development_predictions": False, + "additional_model_inference": False, + "calibration_or_final_outcomes_accessed": False, + "no_preferred_quality_target": True, + "bootstrap": { + "draws": draws, + "seed": 89, + "unit": "paired whole families within task; fixed observed task weights", + }, + "implementation_sha256": { + "development_report": core.sha(__file__), + "native": core.sha(native.__file__), + "recipe": core.sha(core.__file__), + "frontier": core.sha(frontier.__file__), + "utility": core.sha(utility.__file__), + }, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", type=Path, required=True) + parser.add_argument("--training-grades", type=Path, required=True) + parser.add_argument("--development-grades", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--draws", type=int, default=2000) + args = parser.parse_args() + result = build(args.run, args.training_grades, args.development_grades, draws=args.draws) + core.write(args.output, result) + print({"output": str(args.output), "selected_epoch": result["selected_settings"]["lora"], "n": result["n"]}) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/native/reader.py b/examples/train-a-router/native/reader.py new file mode 100644 index 000000000..d315f176d --- /dev/null +++ b/examples/train-a-router/native/reader.py @@ -0,0 +1,42 @@ +"""Apply both exported Laya checkpoints on CPU or CUDA without reading labels.""" + +import argparse +import json +from pathlib import Path +import time + +import bbeh_native + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", type=Path, required=True) + parser.add_argument("--device", choices=("cpu", "cuda"), default="cuda") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--question-file", type=Path) + parser.add_argument("--variant", choices=("base", "lora")) + args = parser.parse_args() + if args.question_file: + core = bbeh_native.core + core.require(args.variant is not None, "Select base or lora") + plan, _ = core.checked_plan(args.run) + trained = core.read(args.run / "native/trained.json") + core.require(trained["plan_sha256"] == core.sha(args.run / "plan.json") + and trained["training_ids"] == plan["populations"]["train"] + plan["populations"]["development"], + "Native checkpoint belongs to another training plan") + checkpoint = args.run / "native" / args.variant + core.require(bbeh_native.checkpoint_files(checkpoint) == trained["checkpoints"][args.variant]["files_sha256"], + "Native checkpoint changed") + item = core.read(args.question_file) + started = time.perf_counter() + p = bbeh_native.checkpoint_probabilities(checkpoint, [item], plan["pair"], plan["native"], device=args.device) + result = {"probabilities": p.tolist(), "expected_usd": core.expected_usd(p, trained["conditional_cost_usd"]).tolist(), + "elapsed_seconds_including_load": time.perf_counter() - started, "outcomes_accessed": False} + core.write(args.output, result) + else: + result = bbeh_native.score_native(args.run, args.output, device=args.device) + print(json.dumps({"output": str(args.output), "outcomes_accessed": result["outcomes_accessed"]})) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/native/recipe.py b/examples/train-a-router/native/recipe.py new file mode 100644 index 000000000..f518de301 --- /dev/null +++ b/examples/train-a-router/native/recipe.py @@ -0,0 +1,378 @@ +"""Small BBEH adapter: fixed source roles, paired correctness and USD costs.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import math +import re +from pathlib import Path + +ARMS = ("cheap", "strong") +ROLES = ("train", "development", "calibration", "final") +CONTEXTS = (512, 1024, 2048, 4096, 8192) +BASE = "convaiinnovations/laya" +REVISION = "aa8c91ca088ec597df95a0d1c76b3063cb2ae5e8" +INITIAL_WEIGHTS_SHA256 = "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c" +NATIVE = { + "base": BASE, + "revision": REVISION, + "head_max_len": 256, + "rank": 16, + "alpha": 32, + "dropout": 0.05, + "encoder_lr": 0.0002, + "head_lr": 0.0001, + "weight_decay": 0.01, + "micro_batch": 4, + "grad_accum": 4, + "evaluation_batch": 4, + "gradient_checkpointing": True, + "selection_epochs": 8, + "seed": 89, + "selection": "Minimum development summed binary NLL over epochs0..8; exact ties choose earlier epoch", + "refit": "Fresh pinned base/seed; train+development for selected epoch count; no calibration/final labels", + "versions": {"laya": "0.3.29", "torch": "2.9.1", "transformers": "4.57.6", "peft": "0.18.1"}, +} +PROTOCOL = { + "objective": "Two binary correctness questions, unweighted summed Bernoulli NLL; cheap-only wins retained", + "policy": "p_strong-p_cheap-lambda*(expected_USD_strong-expected_USD_cheap)>=0; lambda>=0", + "cost_model": "Four train+development arm-by-correctness USD means, shared by base and LoRA; an absent class uses that arm's unconditional mean; no cost floor", + "selection": "Development selects epoch; refit train+development; full utility frontier on calibration", + "final": "Input-only predictions may be saved; final labels never enter fit, selection, timing or calibration", + "native": NATIVE, +} + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sha(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def text_sha(value): + return hashlib.sha256(value.encode()).hexdigest() + + +def read(path): + return json.loads(Path(path).read_text()) + + +def write(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x") as stream: + json.dump(value, stream, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + stream.write("\n") + + +def sources(): + return {name: sha(Path(__file__).parent / name) for name in ("recipe.py", "bbeh_native.py")} + + +def questions(pair): + models = pair.get("models") or {arm: pair["arms"][arm]["model"] for arm in ARMS} + require(set(models) == set(ARMS) and all(isinstance(models[a], str) and models[a] for a in ARMS), "Bad model pair") + return { + f"{arm}_correct": { + "type": "choice", + "instructions": f"Predict whether {models[arm]} will answer this request correctly. Judge its likely answer quality; do not answer the request.", + "criteria": { + "incorrect": "The solver will give an incorrect answer.", + "correct": "The solver will give the correct answer.", + }, + } + for arm in ARMS + } + + +def source_rows(source): + source = Path(source) + plan = read(source / "source-plan.json") + names = ["meta.json", *(f"question-only/{role}-inputs.json" for role in ROLES)] + for name in names: + require(sha(source / name) == plan["files_sha256"][name], f"Source binding differs: {name}") + metadata = read(source / "meta.json") + require(isinstance(metadata, list), "meta.json must be the source owner's flat list") + by_id = {row["id"]: row for row in metadata} + require(len(by_id) == len(metadata), "Duplicate source metadata ID") + rows, families, ids = [], {}, set() + for role in ROLES: + inputs = read(source / f"question-only/{role}-inputs.json") + require(isinstance(inputs, list) and inputs, f"Empty source role: {role}") + expected = [row["id"] for row in metadata if row["role"] == role and row["retained"]] + require([row["id"] for row in inputs] == expected, f"Source role/order differs: {role}") + for item in inputs: + require(set(item) == {"id", "request", "response_contract"}, "Unexpected question-only fields") + require( + item["id"] not in ids and isinstance(item["request"], str) and item["request"].strip(), + "Bad source input", + ) + row = by_id[item["id"]] + require(text_sha(item["request"]) == row["request_sha256"], "Full request hash differs") + family = row["family"] + require(isinstance(family, str) and family, "Missing family") + require(family not in families or families[family] == role, "A family crosses source roles") + families[family] = role + ids.add(item["id"]) + rows.append({**item, "role": role, "family": family, "task": row["task"]}) + return plan, rows + + +def audit(source, pair_path, output, checkpoint=None): + """Remote CPU only: exact full sequence lengths, with no weights or outcomes.""" + require(not Path(output).exists(), "Preserve earlier length audit") + import laya.train as lt + from transformers import AutoTokenizer + + require(importlib.metadata.version("laya") == "0.3.29", "Use pinned Laya tokenizer serialization") + require(importlib.metadata.version("transformers") == "4.57.6", "Use pinned Transformers") + _, rows = source_rows(source) + pair = read(pair_path) + tokenizer = ( + AutoTokenizer.from_pretrained(Path(checkpoint) / "tokenizer", local_files_only=True, trust_remote_code=False) + if checkpoint + else AutoTokenizer.from_pretrained(BASE, revision=REVISION, subfolder="tokenizer", trust_remote_code=False) + ) + heads = {} + for key, question in questions(pair).items(): + internal = lt.to_internal(key, question) + head, _, _ = lt.build_head(tokenizer, internal, NATIVE["head_max_len"]) + uncut, _, _ = lt.build_head(tokenizer, internal, 8192) + require(head == uncut, "Decision head exceeds the fixed256-token head cap") + heads[key] = {"question": internal, "tokens": len(head)} + lengths = [] + for row in rows: + state = lt.encode_state(tokenizer, row["request"], 1_000_000) + lengths.append( + { + "id": row["id"], + "role": row["role"], + "task": row["task"], + "request_sha256": text_sha(row["request"]), + "state_tokens": len(state), + "sequence_tokens": len(state) + max(head["tokens"] for head in heads.values()) + 1, + } + ) + maximum = max(row["sequence_tokens"] for row in lengths) + chosen = next((length for length in CONTEXTS if maximum <= length), None) + result = { + "format": "bbeh-native-length-audit-v1", + "source_only": True, + "model_weights_read": False, + "source_plan_sha256": sha(Path(source) / "source-plan.json"), + "pair_sha256": sha(pair_path), + "audit_source_sha256": sha(__file__), + "model": BASE, + "revision": REVISION, + "questions": questions(pair), + "head_tokens": {key: value["tokens"] for key, value in heads.items()}, + "count": len(rows), + "max_sequence_tokens": maximum, + "selected_max_len": chosen, + "overflow_counts": { + str(length): sum(row["sequence_tokens"] > length for row in lengths) for length in CONTEXTS + }, + "unsupported_ids": [row["id"] for row in lengths if row["sequence_tokens"] > 8192], + "rows": lengths, + "context_rule": "Smallest supported power of two covering both actual decision heads and every full request; never truncate or remove an overflow row", + } + write(output, result) + return result + + +def prepare(source, pair_path, lengths_path, output, acquisition_protocol_sha256): + output = Path(output) + require(not output.exists(), "Preserve existing prepared study") + _, rows = source_rows(source) + audit, pair = read(lengths_path), read(pair_path) + source_hash, pair_hash = sha(Path(source) / "source-plan.json"), sha(pair_path) + require( + audit["source_only"] and audit["source_plan_sha256"] == source_hash and audit["pair_sha256"] == pair_hash, + "Length audit uses different source/pair", + ) + require(audit["audit_source_sha256"] == sha(__file__), "Length auditor changed") + require( + audit["selected_max_len"] in CONTEXTS and not audit["unsupported_ids"], + "Some full inputs exceed supported native context; report them, do not drop them", + ) + require( + [row["id"] for row in audit["rows"]] == [row["id"] for row in rows], "Length audit population/order differs" + ) + require( + all(a["request_sha256"] == text_sha(r["request"]) for a, r in zip(audit["rows"], rows, strict=True)), + "Length audit request changed", + ) + require(re.fullmatch(r"[0-9a-f]{64}", acquisition_protocol_sha256), "Missing acquisition protocol hash") + output.mkdir(parents=True) + write(output / "source-rows.json", rows) + (output / "pair.json").write_bytes(Path(pair_path).read_bytes()) + (output / "lengths.json").write_bytes(Path(lengths_path).read_bytes()) + plan = { + "format": "bbeh-native-plan-v1", + "protocol": PROTOCOL, + "native": {**NATIVE, "max_len": audit["selected_max_len"]}, + "source_plan_sha256": source_hash, + "source_rows_sha256": sha(output / "source-rows.json"), + "pair_sha256": pair_hash, + "pair": pair, + "questions": questions(pair), + "acquisition_protocol_sha256": acquisition_protocol_sha256, + "length_audit_sha256": sha(output / "lengths.json"), + "source_code_sha256": sources(), + "populations": {role: [row["id"] for row in rows if row["role"] == role] for role in ROLES}, + "initial_weights_sha256": INITIAL_WEIGHTS_SHA256, + } + write(output / "plan.json", plan) + return plan + + +def checked_plan(run): + run = Path(run) + plan = read(run / "plan.json") + require( + plan["format"] == "bbeh-native-plan-v1" + and plan["protocol"] == PROTOCOL + and plan["source_code_sha256"] == sources(), + "Native recipe changed", + ) + require( + plan["native"] == {**NATIVE, "max_len": plan["native"]["max_len"]} and plan["native"]["max_len"] in CONTEXTS, + "Native settings changed", + ) + for name, key in ( + ("source-rows.json", "source_rows_sha256"), + ("pair.json", "pair_sha256"), + ("lengths.json", "length_audit_sha256"), + ): + require(sha(run / name) == plan[key], f"Prepared bytes changed: {name}") + rows = read(run / "source-rows.json") + require( + {role: [r["id"] for r in rows if r["role"] == role] for role in ROLES} == plan["populations"], + "Native populations changed", + ) + require( + plan["pair"] == read(run / "pair.json") and plan["questions"] == questions(plan["pair"]), + "Pair/criteria changed", + ) + require(read(run / "lengths.json")["selected_max_len"] == plan["native"]["max_len"], "Context changed after audit") + return plan, rows + + +def grade_arrays(plan, metadata, grades, role): + import numpy as np + + require(role in ("train", "development"), "Native fitting cannot read calibration/final grades") + require(grades["format"] == "bbeh-paired-grades-v1" and grades["role"] == role, "Wrong grade format/role") + require( + all(grades[key] == plan[key] for key in ("source_plan_sha256", "pair_sha256", "acquisition_protocol_sha256")), + "Grade source/pair/acquisition differs", + ) + models = plan["pair"].get("models") or {a: plan["pair"]["arms"][a]["model"] for a in ARMS} + require( + grades["models"] == models and grades["cost_scope"] == plan["pair"]["cost_scope"], "Wrong models/cost scope" + ) + require(re.fullmatch(r"[0-9a-f]{64}", grades["grader_sha256"]), "Missing grader provenance") + rows = [row for row in metadata if row["role"] == role] + require( + [row["id"] for row in grades["rows"]] == [row["id"] for row in rows], + "Require complete grades in fixed source order", + ) + labels, costs = [], [] + for grade in grades["rows"]: + require( + all(type(grade[f"{a}_correct"]) in (bool, int) and grade[f"{a}_correct"] in (0, 1) for a in ARMS), + "Expected two binary labels", + ) + values = [grade[f"{a}_cost_usd"] for a in ARMS] + require( + all(type(v) in (int, float) and math.isfinite(v) and v >= 0 for v in values), + "Missing costs cannot become zero", + ) + require( + all(re.fullmatch(r"[0-9a-f]{64}", grade[f"{a}_record_sha256"]) for a in ARMS), "Missing response provenance" + ) + labels.append([int(grade[f"{a}_correct"]) for a in ARMS]) + costs.append(values) + return rows, np.asarray(labels, dtype=float), np.asarray(costs, dtype=float) + + +def conditional_usd(targets, costs): + import numpy as np + + targets, costs = np.asarray(targets), np.asarray(costs, dtype=float) + require( + targets.ndim == 2 + and targets.shape[1] == 2 + and costs.shape == targets.shape + and np.isin(targets, (0, 1)).all() + and np.isfinite(costs).all() + and (costs >= 0).all(), + "Bad paired labels/costs", + ) + means, counts = np.empty((2, 2)), np.empty((2, 2), dtype=int) + for arm in range(2): + for label in range(2): + mask = targets[:, arm] == label + means[arm, label] = costs[mask, arm].mean() if mask.any() else costs[:, arm].mean() + counts[arm, label] = mask.sum() + return { + "unit": "USD", + "arm_order": list(ARMS), + "class_order": [0, 1], + "means": means.tolist(), + "counts": counts.tolist(), + } + + +def expected_usd(probabilities, statistics): + import numpy as np + + p, means = np.asarray(probabilities, dtype=float), np.asarray(statistics["means"], dtype=float) + require( + statistics["unit"] == "USD" and statistics["arm_order"] == list(ARMS) and statistics["class_order"] == [0, 1], + "Wrong cost units/order", + ) + require( + p.ndim == 2 + and p.shape[1] == 2 + and means.shape == (2, 2) + and np.isfinite(p).all() + and ((p >= 0) & (p <= 1)).all() + and np.isfinite(means).all() + and (means >= 0).all(), + "Invalid cost predictions", + ) + return (1 - p) * means[:, 0] + p * means[:, 1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + for name in ("audit", "prepare"): + command = commands.add_parser(name) + command.add_argument("--source", type=Path, required=True) + command.add_argument("--pair", type=Path, required=True) + command.add_argument("--output", type=Path, required=True) + if name == "audit": + command.add_argument("--checkpoint", type=Path) + else: + command.add_argument("--lengths", type=Path, required=True) + command.add_argument("--acquisition-protocol-sha256", required=True) + args = parser.parse_args() + if args.command == "audit": + result = audit(args.source, args.pair, args.output, args.checkpoint) + print(json.dumps({key: result[key] for key in ("count", "selected_max_len", "unsupported_ids")})) + else: + result = prepare(args.source, args.pair, args.lengths, args.output, args.acquisition_protocol_sha256) + print(json.dumps({"plan": str(args.output / "plan.json"), "max_len": result["native"]["max_len"]})) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/normalize.py b/examples/train-a-router/normalize.py new file mode 100644 index 000000000..e9a5460e9 --- /dev/null +++ b/examples/train-a-router/normalize.py @@ -0,0 +1,95 @@ +"""Source-aware answer formats; never inspect the target or the answering model.""" + +import re + +VERSION = "bbeh-source-formats-v2" +SINGLE_CHOICE_TASKS = frozenset({ + "boolean_expressions", "disambiguation_qa", "geometric_shapes", + "movie_recommendation", "nycc", "shuffled_objects", +}) +CHOICE = re.compile(r"^\(([A-Z])\)[ \t]+", re.MULTILINE) +GEOMETRY_SUFFIX = "\nCoordinates have been rounded to 5 decimal places so ignore slight differences." + + +def text_key(value): + # Whitespace is presentational for these choice texts, including multiline + # captions. Do not remove words, quotes, numbers, brackets or negations. + return " ".join(value.split()).casefold().removesuffix(".") + + +def options_from_question(request, task): + matches = list(CHOICE.finditer(request)) + options = {} + for index, match in enumerate(matches): + end = matches[index + 1].start() if index + 1 < len(matches) else len(request) + text = request[match.end():end].strip() + if task == "geometric_shapes" and index == len(matches) - 1: + text = text.removesuffix(GEOMETRY_SUFFIX).strip() + label = match.group(1) + if not text: + raise ValueError("Single-choice source has an empty option") + # One public source question repeats a label. Preserve the benchmark's + # label semantics; never guess a repaired label or reference answer. + options.setdefault(label, []).append(text) + if len(options) < 2: + raise ValueError("Single-choice source has no unambiguous option block") + return options + + +def source_flags(request, task): + if task not in SINGLE_CHOICE_TASKS: + return [] + options = options_from_question(request, task) + flags = [] + if any(len(texts) != 1 for texts in options.values()): + flags.append("duplicate_source_option_label") + texts_to_labels = {} + for label, texts in options.items(): + for text in texts: + texts_to_labels.setdefault(text_key(text), set()).add(label) + if any(len(labels) > 1 for labels in texts_to_labels.values()): + flags.append("identical_source_option_text_under_distinct_labels") + return flags + + +def canonicalize(answer, request, task): + """Return (canonical answer or None, rule), without access to gold labels. + + A label followed by text is accepted only if the entire text agrees with + that source option. Text alone must identify exactly one option. Unknown, + contradictory or multiple-choice responses fail closed for single-choice + tasks. All other task-specific punctuation is preserved. + """ + if answer is None: + return None, "missing" + value = answer.strip() + if task in SINGLE_CHOICE_TASKS: + options = options_from_question(request, task) + bare = re.fullmatch(r"(?:\(([A-Za-z])\)|([A-Za-z]))", value) + if bare: + label = (bare.group(1) or bare.group(2)).upper() + return (f"({label})", "choice_label") if label in options else (None, "invalid_choice_label") + labelled = re.fullmatch(r"\(([A-Za-z])\)\s+(.+)", value, flags=re.DOTALL) + if labelled: + label, text = labelled.group(1).upper(), labelled.group(2) + if label in options and text_key(text) in {text_key(option) for option in options[label]}: + return f"({label})", "choice_label_and_exact_text" + return None, "choice_label_text_disagreement" + matching = [label for label, texts in options.items() if text_key(value) in {text_key(text) for text in texts}] + if len(matching) == 1: + return f"({matching[0]})", "unique_exact_choice_text" + return None, "unrecognized_or_ambiguous_choice" + if task == "web_of_lies": + # The public task itself demonstrates etc. These + # delimiters mark the whole three-answer tuple, not reasoning syntax. + marker = "Your answer should be a list of three words" + if marker not in request: + raise ValueError("Unrecognized source truth-tuple contract") + instruction = request.rsplit(marker, 1)[1] + terms = "yes|no|unknown" if "unknown" in instruction else "yes|no" + inner = value[1:-1].strip() if value.startswith("<") and value.endswith(">") else value + matched = re.fullmatch(rf"({terms})\s*,\s*({terms})\s*,\s*({terms})", inner, flags=re.IGNORECASE) + if matched: + return ", ".join(part.lower() for part in matched.groups()), "source_truth_tuple" + return None, "invalid_truth_tuple" + return value, "unchanged" diff --git a/examples/train-a-router/openai_population.py b/examples/train-a-router/openai_population.py new file mode 100644 index 000000000..9671dfac2 --- /dev/null +++ b/examples/train-a-router/openai_population.py @@ -0,0 +1,420 @@ +"""Resumable question-only acquisition; frozen bodies, explicit physical attempts.""" + +from collections import Counter, deque +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from datetime import datetime, timezone +from decimal import Decimal +from email.utils import parsedate_to_datetime +import hashlib +import json +import os +from pathlib import Path +import re +import time + +import httpx + +from collect import ARMS, body_for, openai_cost, parse_answer, reservation + +MAX_ATTEMPTS = 3 +WORKERS_PER_ARM = 16 +TRANSIENT_HTTP = {408, 429, 500, 502, 503, 504} +TRANSIENT_ERRORS = {"ReadTimeout", "ConnectTimeout", "WriteTimeout", "PoolTimeout", "ConnectError", "ReadError", "WriteError", "RemoteProtocolError"} +FATAL_CODES = {"insufficient_quota", "credit_balance_exhausted", "billing_hard_limit_reached", "invalid_api_key", "account_deactivated"} + + +def encoded(value): + return (json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n").encode() + + +def digest(value): + return hashlib.sha256(value).hexdigest() + + +def file_hash(path): + return digest(Path(path).read_bytes()) + + +def dump(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_bytes(encoded(value)) + temp.replace(path) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def safe_code(value): + return value if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9_.:-]{1,128}", value) else None + + +def retry_after(value, now=None): + if not isinstance(value, str) or len(value) > 128: + return None + try: + seconds = float(value) + except ValueError: + try: + when = parsedate_to_datetime(value) + if when.tzinfo is None: + when = when.replace(tzinfo=timezone.utc) + seconds = when.timestamp() - (time.time() if now is None else now) + except (ValueError, TypeError, OverflowError): + return None + return seconds if 0 <= seconds < float("inf") else None + + +def numeric_usage(value): + if isinstance(value, dict): + return all(isinstance(key, str) and numeric_usage(item) for key, item in value.items()) + return type(value) is int and value >= 0 + + +def acquire(item, arm, client_factory=None): + """Equivalent request body/endpoint; retain Retry-After and readable responses.""" + body = body_for(item, arm) + row = {"id": item["id"], "arm": arm, "model": body["model"], + "body_sha256": digest(json.dumps(body, sort_keys=True).encode()), "created_unix": time.time(), + "terminal_response": False, "execution_unknown": True} + started = time.monotonic() + try: + factory = client_factory or (lambda: httpx.Client(timeout=httpx.Timeout(600, connect=30), + follow_redirects=False, transport=httpx.HTTPTransport(retries=0), trust_env=False)) + with factory() as client: + response = client.post("https://api.openai.com/v1/responses", json=body, + headers={"Authorization": "Bearer " + os.environ["OPENAI_API_KEY"], "Content-Type": "application/json"}) + row["http_status"] = response.status_code + row["retry_after_seconds"] = retry_after(response.headers.get("retry-after")) + try: + payload = response.json() + except ValueError: + payload = None + if response.status_code != 200: + error = payload.get("error") if isinstance(payload, dict) else None + error = error if isinstance(error, dict) else {} + row.update(status="provider_error", error_code=safe_code(error.get("code")), error_type=safe_code(error.get("type"))) + return row + # Any HTTP200 response is retained. Parsing/format/truncation never triggers another answer. + row["terminal_response"] = True + require(isinstance(payload, dict), "Invalid provider JSON object") + row.update(provider_status=payload.get("status"), resolved_model=payload.get("model"), service_tier=payload.get("service_tier")) + messages = [{"type": part["type"], **({"text": part.get("text")} if part["type"] == "output_text" else {"refusal": part.get("refusal")})} + for output in payload.get("output", []) if output.get("type") == "message" + for part in output.get("content", []) if part.get("type") in {"output_text", "refusal"}] + row["messages"] = messages + usage = payload.get("usage") + require(isinstance(usage, dict) and numeric_usage(usage), "Usage must be a readable numeric object") + row["usage"] = usage + row["cost"] = openai_cost(usage, body) + row["execution_unknown"] = False + details = payload.get("incomplete_details") or {} + row["incomplete_reason"] = safe_code(details.get("reason")) if isinstance(details, dict) else None + text = "".join(part.get("text") or "" for part in messages if part["type"] == "output_text") + row["answer"] = parse_answer(text, item["response_contract"]) if row["provider_status"] == "completed" else None + row.update(status="complete", valid_answer=row["answer"] is not None) + if Decimal(row["cost"]["budget_usd"]) > reservation(body, arm): + row["status"] = "charge_exceeds_reservation" + except Exception as error: + row.update(status="receipt_incomplete" if row["terminal_response"] else "failed_or_unknown", error_type=type(error).__name__) + finally: + row["elapsed_seconds"] = time.monotonic() - started + return row + + +def decision(row, attempt): + if row.get("http_status") in {401, 403} or row.get("error_code") in FATAL_CODES or row.get("error_type") in FATAL_CODES: + return "fatal" + if row.get("status") == "charge_exceeds_reservation" or row.get("binding_valid") is False: + return "fatal" + if row.get("terminal_response"): + if row.get("resolved_model") != row["model"] or row.get("service_tier") != "default": + return "fatal" + return "terminal" + transient = row.get("http_status") in TRANSIENT_HTTP or row.get("error_type") in TRANSIENT_ERRORS or row.get("status") == "interrupted_unknown" + if not transient: + return "incomplete" + if attempt >= MAX_ATTEMPTS: + return "exhausted" + if (row.get("retry_after_seconds") or 0) > 60: + return "defer_server_delay" + return "retry" + + +def retry_delay(row, attempt, identity): + base = (5, 15)[min(attempt - 1, 1)] + jitter = int(digest(f"{identity}:{attempt}".encode())[:4], 16) / 65535 + return max(base, row.get("retry_after_seconds") or 0) + jitter + + +def fits_budget(known, unknown, pending, reserve, cap): + return known + unknown + pending + reserve <= cap + + +def run(items_by_split, root, plan, commit, *, call=acquire, dispatch_seconds=20900, emit=True): + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + plan_path = root / "plan.json" + plan_bytes = encoded(plan) + if plan_path.exists(): + require(plan_path.read_bytes() == plan_bytes, "Cannot change a resumed collection plan") + else: + plan_path.write_bytes(plan_bytes) + plan_sha = digest(plan_bytes) + states, items, queues = {}, {}, {arm: deque() for arm in ("luna", "sol")} + retries = {arm: deque() for arm in queues} + known, unknown, known_list = Decimal(0), Decimal(0), Decimal(0) + fatal, pause_until = None, 0.0 + for split, split_items in items_by_split.items(): + for item in split_items: + for arm in queues: + key = (split, arm, item["id"]) + items[key] = item + body = body_for(item, arm) + body_sha = digest(json.dumps(body, sort_keys=True).encode()) + directory = root / split / arm / "attempts" / digest(item["id"].encode()) + state = {"id": item["id"], "arm": arm, "split": split, "attempts": [], "selected_attempt": None, + "status": "pending", "not_before_unix": 0.0} + states[key] = state + paths = sorted(directory.glob("*-intent.json")) if directory.exists() else [] + require(len(paths) <= MAX_ATTEMPTS, "Attempt cap violated") + for number, intent_path in enumerate(paths, 1): + require(intent_path.name == f"{number:03d}-intent.json", "Attempt sequence changed") + intent = json.loads(intent_path.read_bytes()) + expected = {"id": item["id"], "arm": arm, "split": split, "attempt": number, + "body_sha256": body_sha, "plan_sha256": plan_sha} + require(all(intent.get(k) == v for k, v in expected.items()), "Prior intent binding changed") + reserve = Decimal(intent["reserved_usd"]) + require(reserve == reservation(body, arm), "Prior reservation changed") + response_path = directory / f"{number:03d}-response.json" + row = json.loads(response_path.read_bytes()) if response_path.exists() else { + "id": item["id"], "arm": arm, "model": body["model"], "body_sha256": body_sha, + "status": "interrupted_unknown", "terminal_response": False, + "created_unix": intent["created_unix"], "error_type": "ProcessInterrupted"} + if response_path.exists(): + require(row.get("intent_sha256") == file_hash(intent_path) + and all(row.get(k) == v for k, v in {"id": item["id"], "arm": arm, "model": body["model"], "body_sha256": body_sha}.items()), + "Prior response binding changed") + amount = row.get("cost", {}).get("budget_usd") + if amount is None: + unknown += reserve + else: + amount = Decimal(amount) + require(amount.is_finite() and amount >= 0, "Invalid prior charge") + known += amount + known_list += Decimal(row["cost"]["usd_estimate"]) + action = decision(row, number) + attempt = {"attempt": number, "intent_path": intent_path.relative_to(root).as_posix(), "intent_sha256": file_hash(intent_path), + "record_path": response_path.relative_to(root).as_posix() if response_path.exists() else None, + "record_sha256": file_hash(response_path) if response_path.exists() else None, "status": row["status"], + "action": action, "known_list_usd": row.get("cost", {}).get("usd_estimate"), + "known_budget_usd": None if amount is None else str(amount), "unknown_reservation_usd": str(reserve) if amount is None else "0"} + state["attempts"].append(attempt) + if action == "terminal": + require(state["selected_attempt"] is None, "A terminal response was followed by an unauthorized retry") + state.update(selected_attempt=number, status="complete" if row.get("status") == "complete" else "incomplete_receipt") + elif state["selected_attempt"] is not None: + raise ValueError("Attempt after terminal provider response") + elif action == "fatal": + fatal = "preserved_fatal_provider_or_identity_error" + else: + state["status"] = action + delay = retry_delay(row, number, item["id"]) if action == "retry" else (row.get("retry_after_seconds") or 0) + finished = row.get("created_unix", intent["created_unix"]) + (row.get("elapsed_seconds") or 0) + if row["status"] == "interrupted_unknown": + finished = time.time() + state["not_before_unix"] = max(state["not_before_unix"], finished + delay) + if state["selected_attempt"] is None and (not state["attempts"] or state["attempts"][-1]["action"] in {"retry", "defer_server_delay"}): + (retries if state["attempts"] else queues)[arm].append(key) + checkpoint_count, checkpoint_seconds, dirty_responses = 0, 0.0, 0 + last_checkpoint = time.monotonic() + + def checkpoint(): + nonlocal checkpoint_count, checkpoint_seconds, dirty_responses, last_checkpoint + checkpoint_start = time.monotonic() + commit() + checkpoint_seconds += time.monotonic() - checkpoint_start + checkpoint_count += 1 + dirty_responses = 0 + last_checkpoint = time.monotonic() + + checkpoint() + cap = Decimal(plan["budget_usd"]) + require(known + unknown <= cap, "Preserved acquisition already exceeds its cap") + pending = {} + physical_count = sum(len(state["attempts"]) for state in states.values()) + selected_count = sum(state["selected_attempt"] is not None for state in states.values()) + selected_by_split = Counter(state["split"] for state in states.values() if state["selected_attempt"] is not None) + selections, written_splits = {}, set() + + def write_split(split): + split_items = items_by_split[split] + for arm in queues: + rows = [states[(split, arm, item["id"])] for item in split_items] + path = root / split / arm / "selection.json" + dump(path, {"format": "routerbench-openai-attempt-selection-v1", "split": split, "arm": arm, + "model": ARMS[arm]["model"], "plan_sha256": plan_sha, "ordered_ids": [item["id"] for item in split_items], + "entries": rows, "first_terminal_response_only": True, "no_quality_retries": True}) + selections[f"{split}/{arm}"] = {"path": path.relative_to(root).as_posix(), "sha256": file_hash(path), + "expected": len(rows), "selected": sum(row["selected_attempt"] is not None for row in rows), + "complete_receipts": sum(row["status"] == "complete" for row in rows)} + split_selections = {arm: selections[f"{split}/{arm}"] for arm in queues} + dump(root / split / "collection.json", {"format": "routerbench-openai-split-collection-v1", "split": split, + "status": "complete" if all(row["complete_receipts"] == len(split_items) for row in split_selections.values()) else "incomplete", + "expected_pairs": len(split_items), "plan_sha256": plan_sha, "selections": split_selections, + "pair_spec_sha256": plan.get("pair_spec_sha256"), "acquisition_protocol_sha256": plan_sha, + "source_plan_sha256": plan.get("source_plan_sha256"), "correctness_inspected": False}) + written_splits.add(split) + + start, last_progress, last_refill = time.monotonic(), 0, 0.0 + stop = fatal + with ThreadPoolExecutor(max_workers=WORKERS_PER_ARM * 2) as pool: + while any(queues.values()) or any(retries.values()) or pending: + if time.monotonic() - start >= dispatch_seconds: + stop = stop or "dispatch_deadline" + candidates = [] + reserved_pending = sum((value[2] for value in pending.values()), Decimal(0)) + counts = Counter(key[1] for key, _, _ in pending.values()) + vacancies = WORKERS_PER_ARM * 2 - len(pending) + refill_ready = not pending or vacancies >= 8 or time.monotonic() - last_refill >= 0.5 + if stop is None and time.time() >= pause_until and refill_ready: + for arm, queue in queues.items(): + candidates_for_arm = retries[arm] + examined = len(candidates_for_arm) + retry_keys = [] + while candidates_for_arm and examined: + key = candidates_for_arm.popleft() + examined -= 1 + if time.time() < states[key]["not_before_unix"]: + candidates_for_arm.append(key) + else: + retry_keys.append(key) + queue.extendleft(reversed(retry_keys)) + while queue and counts[arm] < WORKERS_PER_ARM: + key = queue.popleft() + state, item = states[key], items[key] + if time.time() < state["not_before_unix"]: + retries[arm].append(key) + continue + body = body_for(item, arm) + reserve = reservation(body, arm) + if not fits_budget(known, unknown, reserved_pending, reserve, cap): + queue.appendleft(key) + break + number = len(state["attempts"]) + 1 + require(number <= MAX_ATTEMPTS and state["selected_attempt"] is None, "Ineligible new attempt") + directory = root / key[0] / arm / "attempts" / digest(item["id"].encode()) + intent_path = directory / f"{number:03d}-intent.json" + response_path = directory / f"{number:03d}-response.json" + require(not intent_path.exists() and not response_path.exists(), "Never overwrite a physical attempt") + dump(intent_path, {"id": item["id"], "arm": arm, "split": key[0], "attempt": number, + "body_sha256": digest(json.dumps(body, sort_keys=True).encode()), "plan_sha256": plan_sha, + "created_unix": time.time(), "reserved_usd": str(reserve)}) + candidates.append((key, (intent_path, response_path, number), reserve)) + reserved_pending += reserve + counts[arm] += 1 + if candidates: + checkpoint() # Every intent is durable before its physical request starts. + last_refill = time.monotonic() + for key, paths, reserve in candidates: + pending[pool.submit(call, items[key], key[1])] = (key, paths, reserve) + if not pending: + if stop or not (any(queues.values()) or any(retries.values())): + break + remaining = [key for queue in list(queues.values()) + list(retries.values()) for key in queue] + if not any(fits_budget(known, unknown, Decimal(0), reservation(body_for(items[key], key[1]), key[1]), cap) for key in remaining): + stop = "budget_bound" + break + time.sleep(min(1, max(0.01, pause_until-time.time()))) + continue + done, _ = wait(pending, timeout=0.5, return_when=FIRST_COMPLETED) + if not done: + if dirty_responses and time.monotonic() - last_checkpoint >= 2: + checkpoint() + continue + for future in done: + key, (intent_path, response_path, number), reserve = pending.pop(future) + try: + row = future.result() + except Exception as error: + body = body_for(items[key], key[1]) + row = {"id": key[2], "arm": key[1], "model": body["model"], "body_sha256": digest(json.dumps(body, sort_keys=True).encode()), + "created_unix": time.time(), "status": "failed_or_unknown", "terminal_response": False, "error_type": type(error).__name__} + expected_body = body_for(items[key], key[1]) + row["binding_valid"] = all(row.get(k) == v for k, v in { + "id": key[2], "arm": key[1], "model": expected_body["model"], + "body_sha256": digest(json.dumps(expected_body, sort_keys=True).encode())}.items()) + row["intent_sha256"] = file_hash(intent_path) + row["attempt"] = number + dump(response_path, row) + dirty_responses += 1 + amount = row.get("cost", {}).get("budget_usd") + if amount is None: + unknown += reserve + else: + amount = Decimal(amount) + require(amount.is_finite() and amount >= 0, "Invalid charge") + known += amount + known_list += Decimal(row["cost"]["usd_estimate"]) + if amount > reserve: + stop = "charge_exceeds_reservation" + action = decision(row, number) + state = states[key] + state["attempts"].append({"attempt": number, "intent_path": intent_path.relative_to(root).as_posix(), + "intent_sha256": file_hash(intent_path), "record_path": response_path.relative_to(root).as_posix(), + "record_sha256": file_hash(response_path), "status": row["status"], "action": action, + "known_list_usd": row.get("cost", {}).get("usd_estimate"), "known_budget_usd": None if amount is None else str(amount), + "unknown_reservation_usd": str(reserve) if amount is None else "0"}) + state["status"] = action + physical_count += 1 + if action == "terminal": + state.update(selected_attempt=number, status="complete" if row.get("status") == "complete" else "incomplete_receipt") + selected_count += 1 + selected_by_split[key[0]] += 1 + elif action == "fatal": + stop = "provider_auth_quota_or_identity_error" + elif action == "defer_server_delay": + state["not_before_unix"] = time.time() + row["retry_after_seconds"] + stop = "server_delay_exceeds_supported_cap" + elif action == "retry": + state["not_before_unix"] = time.time() + retry_delay(row, number, key[2]) + retries[key[1]].append(key) + if row.get("http_status") in {429, 503}: + pause_until = max(pause_until, state["not_before_unix"]) + if dirty_responses >= 16 or time.monotonic() - last_checkpoint >= 2 or not pending: + checkpoint() + for split, split_items in items_by_split.items(): + if split not in written_splits and selected_by_split[split] == len(split_items) * 2: + write_split(split) + checkpoint() + if emit: + print(json.dumps({"split_ready": split, "pairs": len(split_items), + "collection_sha256": file_hash(root / split / "collection.json")}), flush=True) + if emit and physical_count-last_progress >= 32: + last_progress = physical_count + progress = {"physical_attempts": physical_count, "selected_responses": selected_count, + "expected_responses": len(states), "known_list_usd": str(known_list), "known_budget_usd": str(known), + "unknown_reservation_usd": str(unknown), "inflight": len(pending), "elapsed_seconds": time.monotonic()-start, + "checkpoints": checkpoint_count, "checkpoint_seconds": checkpoint_seconds, "stop": stop} + dump(root / "progress.json", progress) + print(json.dumps(progress), flush=True) + for split, split_items in items_by_split.items(): + if split not in written_splits: + for row in (states[(split, arm, item["id"])] for arm in queues for item in split_items): + if row["selected_attempt"] is None and row["status"] in {"pending", "retry"}: + row["status"] = "incomplete_" + (stop or "unavailable") + write_split(split) + summary = {"format": "routerbench-openai-population-collection-v1", "status": "complete" if all(state["status"] == "complete" for state in states.values()) else "incomplete", + "expected_responses": len(states), "selected_responses": sum(state["selected_attempt"] is not None for state in states.values()), + "physical_attempts": sum(len(state["attempts"]) for state in states.values()), "known_list_usd": str(known_list), + "known_budget_usd": str(known), "unknown_reservation_usd": str(unknown), "stop": stop, + "selections": selections, "plan_sha256": plan_sha, "elapsed_seconds": time.monotonic()-start, + "final_dispatched": "final" in items_by_split, "correctness_inspected": False, + "checkpoint_metrics_before_terminal_commit": {"count": checkpoint_count, "seconds": checkpoint_seconds}, + "checkpoint_policy": {"intent_before_dispatch": True, "refill_coalescing_seconds": 0.5, + "refill_immediate_vacancies": 8, "completion_count": 16, "completion_seconds": 2}} + dump(root / "collection.json", summary) + checkpoint() + return summary diff --git a/examples/train-a-router/pair.json b/examples/train-a-router/pair.json new file mode 100644 index 000000000..356ae92d6 --- /dev/null +++ b/examples/train-a-router/pair.json @@ -0,0 +1,44 @@ +{ + "arms": { + "cheap": { + "model": "gpt-6-luna", + "provider": "openai" + }, + "strong": { + "model": "gpt-6.1-sol", + "provider": "openai" + } + }, + "cost_scope": "Observed selected-response standard token/list-price USD, including reasoning once; physical acquisition retries and router overhead reported separately.", + "format": "bbeh-selected-answer-pair-v1", + "models": { + "cheap": "gpt-6-luna", + "strong": "gpt-6.1-sol" + }, + "pair": "sol_luna", + "prices_usd_per_million": { + "gpt-6-luna": { + "cache_write": "0.125", + "cached_input": "0.01", + "input": "0.10", + "output": "0.50" + }, + "gpt-6.1-sol": { + "cache_write": "2.50", + "cached_input": "0.10", + "input": "2", + "output": "10" + } + }, + "pricing_checked": "2026-10-21", + "pricing_source": "https://developers.openai.com/api/docs/pricing", + "settings": { + "max_output_tokens": 16384, + "reasoning": { + "effort": "medium" + }, + "service_tier": "default", + "store": false + }, + "system_prompt": "Solve the question independently, using the full rules and options supplied. Do not use tools or external sources. Return the requested JSON object. The answer field must contain only the complete final answer in the format requested by the source question. Preserve semantic brackets, quotes, punctuation, nesting, case and internal spaces. JSON is only the transport container; it does not replace or remove any part of the requested answer. " +} diff --git a/examples/train-a-router/pyproject.toml b/examples/train-a-router/pyproject.toml new file mode 100644 index 000000000..adaa302a7 --- /dev/null +++ b/examples/train-a-router/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "train-a-router" +version = "0.1.0" +description = "Outcome-supervised routing on family-separated BBEH problems" +requires-python = ">=3.12,<3.13" +dependencies = ["numpy==2.4.6"] + +[tool.uv] +package = false + +[tool.uv.sources] +sie-sdk = { git = "https://github.com/superlinked/sie.git", rev = "975d5309ba3af9eaeb13725b20dcf10d906c3e2a", subdirectory = "packages/sie_sdk" } + +[dependency-groups] +outcomes = ["scipy==1.17.0", "scikit-learn==1.6.1"] +collection = ["httpx==0.28.1"] +encode = ["sie-sdk==0.10.1", "httpx==0.28.1", "transformers==4.57.6", "huggingface-hub==0.36.0"] +training = ["laya==0.3.29", "peft==0.18.1", "torch==2.9.1", "transformers==4.57.6"] +dev = ["ruff>=0.14"] + +[tool.ruff] +line-length = 120 +target-version = "py312" + +[tool.ruff.lint.per-file-ignores] +# Preserve the exact scheduler bytes bound by acquisition receipts. +"openai_population.py" = ["F401"] diff --git a/examples/train-a-router/reader_io.py b/examples/train-a-router/reader_io.py new file mode 100644 index 000000000..7340b437d --- /dev/null +++ b/examples/train-a-router/reader_io.py @@ -0,0 +1,100 @@ +"""Local durability and provenance helpers; no service or credential dependency.""" + +from contextlib import contextmanager +import fcntl +import hashlib +import json +import os +from pathlib import Path + + +def sha(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def read(path): + return json.loads(Path(path).read_bytes()) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sync_files(paths, root): + directories = {Path(root)} + for path in paths: + path = Path(path) + require(path.is_file() and not path.is_symlink(), "Expected a regular output file") + with path.open("rb") as stream: + os.fsync(stream.fileno()) + directories.update(parent for parent in path.parents if parent == root or root in parent.parents) + for directory in sorted(directories, key=lambda p: len(p.parts), reverse=True): + descriptor = os.open(directory, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +@contextmanager +def exclusive_run(root): + root = Path(root).resolve() + root.mkdir(parents=True, exist_ok=True) + with (root / ".collector.lock").open("a") as stream: + try: + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + raise ValueError("Another collector owns this output directory") from error + yield + + +@contextmanager +def durable_writes(module, root): + """Wrap file persistence only; request, retry and selection code is unchanged.""" + root, original, dirty = Path(root).resolve(), module.dump, set() + + def write(path, value): + original(path, value) + dirty.add(Path(path).resolve()) + + def commit(): + # run() writes the immutable plan directly, before its first commit. + plan = root / "plan.json" + if plan.exists() and not commit.plan_synced: + dirty.add(plan) + commit.plan_synced = True + if dirty: + sync_files(dirty, root) + dirty.clear() + + commit.plan_synced = False + module.dump = write + try: + yield commit + finally: + try: + commit() + finally: + module.dump = original + + +class ChangedFileSync: + """Sync changed local feature files without re-reading or hashing prior bytes.""" + + def __init__(self): + self.seen = {} + + def __call__(self, root): + root, changed = Path(root).resolve(), [] + for path in root.rglob("*"): + if not path.is_file(): + continue + stat = path.stat() + signature = (stat.st_ino, stat.st_size, stat.st_mtime_ns) + if self.seen.get(path) != signature: + changed.append(path) + self.seen[path] = signature + if changed: + sync_files(changed, root) diff --git a/examples/train-a-router/reader_runtime.py b/examples/train-a-router/reader_runtime.py new file mode 100644 index 000000000..2ff7165d2 --- /dev/null +++ b/examples/train-a-router/reader_runtime.py @@ -0,0 +1,30 @@ +"""Interpreter contract for readers of the immutable UTF-8 experiment files.""" + +import os +import sys + + +def ensure_utf8(): + """Restart before artifact I/O; retain interpreter flags, CLI arguments and exit behavior.""" + if sys.flags.utf8_mode: + return + arguments = list(sys.orig_argv) + arguments[0] = sys.executable + # The final len(sys.argv) entries are the program and its arguments. Keep + # -m/-c/-- with that program, after interpreter options such as -W and -X. + position = len(arguments) - len(sys.argv) + if position > 1 and arguments[position - 1] in {"-m", "-c", "--"}: + position -= 1 + # UTF-8 mode is chosen during preinitialization: an earlier -X utf8=0 + # must be replaced, not merely followed by another -X utf8. + options = iter(arguments[1:position]) + flags = [] + for option in options: + if option == "-X": + value = next(options) + flags.extend((option, "utf8" if value.partition("=")[0] == "utf8" else value)) + elif option in {"-W", "--check-hash-based-pycs"}: + flags.extend((option, next(options))) + else: + flags.append("-Xutf8" if option.startswith("-Xutf8=") else option) + os.execv(sys.executable, [sys.executable, "-X", "utf8", *flags, *arguments[position:]]) diff --git a/examples/train-a-router/release.json b/examples/train-a-router/release.json new file mode 100644 index 000000000..f739c8617 --- /dev/null +++ b/examples/train-a-router/release.json @@ -0,0 +1,7 @@ +{ + "format": "train-a-router-release-v1", + "manifest_url": "https://huggingface.co/datasets/superlinked/train-a-router-bbeh/resolve/86dd7494a4c2396bc9ffa890f994b9722610b357/manifest.json", + "manifest_sha256": "4fd8a49d760384ad745c1bd49b3ee61cb58fd8d98c71cef6552ff6dbfda66b55", + "dataset": "superlinked/train-a-router-bbeh", + "revision": "86dd7494a4c2396bc9ffa890f994b9722610b357" +} diff --git a/examples/train-a-router/router.py b/examples/train-a-router/router.py new file mode 100644 index 000000000..599e9540e --- /dev/null +++ b/examples/train-a-router/router.py @@ -0,0 +1,131 @@ +"""Train routers and compare money saved with answer quality.""" + +import argparse +import json +from pathlib import Path + +import calibration_view +import workflow +from embedding import recipes as core +from reader_runtime import ensure_utf8 + + +def parser(): + p = argparse.ArgumentParser(description=__doc__) + commands = p.add_subparsers(dest="command", required=True) + c = commands.add_parser("fetch", help="Download and verify the pinned public data; add weights for Laya") + c.add_argument("--output", type=Path, required=True) + c.add_argument("--group", choices=("core", "weights", "recordings"), action="append") + for name, help_text in ( + ("prepare", "Create a fresh reproduction run from the supplied dataset"), + ("base", "Apply unchanged Laya before training; CPU or CUDA"), + ("fit-native", "Select and refit LoRA on an NVIDIA GPU"), + ("fit-embedding", "Fit neighbors, small linear heads and simple controls on CPU"), + ("score", "Predict correctness on calibration/test inputs without their answers"), + ("fit-cost", "Fit the small request-cost predictor and add its routing variants"), + ("reuse-native", "Reuse supplied Laya comparisons during a CPU-only embedding exercise"), + ("calibrate", "Save the full calibration frontier and freeze input-only test decisions"), + ("evaluate-refit", "Check your frozen policies against published test answers"), + ): + c = commands.add_parser(name, help=help_text) + c.add_argument("--data", type=Path, required=True) + c.add_argument("--run", type=Path, required=True) + if name in {"base", "score"}: + c.add_argument("--device", choices=("cpu", "cuda"), default="cpu" if name == "base" else "cuda") + if name in {"fit-embedding", "score"}: + c.add_argument("--features", type=Path, help="Fresh features; defaults to the verified supplied vectors") + if name == "score": + c.add_argument("--method", choices=("embedding", "native"), required=True) + if name == "evaluate-refit": + c.add_argument("--output", type=Path, required=True) + c = commands.add_parser("replay", help="Recompute and verify the published final results without model calls") + c.add_argument("--data", type=Path, required=True) + c.add_argument("--output", type=Path, required=True) + c = commands.add_parser("frontier", help="Explore calibration data before looking at test results") + c.add_argument("--run", type=Path, required=True, help="Your run, or data/reference for supplied routers") + c.add_argument("--output", type=Path, required=True) + c = commands.add_parser("answer", help="Route one new question using a saved calibration candidate") + c.add_argument("--run", type=Path, required=True, help="Your run, or data/reference for supplied routers") + c.add_argument("--method", required=True) + c.add_argument("--candidate-id", type=int, required=True) + c.add_argument("--question", required=True) + c.add_argument("--output", type=Path, required=True) + c.add_argument("--device", choices=("cpu", "cuda"), default="cpu") + c.add_argument("--sie-base-url") + c.add_argument("--self-hosted", action="store_true") + c = commands.add_parser("encode", help="Regenerate question embeddings through Cloud or your own SIE server") + c.add_argument("--data", type=Path, required=True) + c.add_argument("--output", type=Path, required=True) + c.add_argument("--budget-usd", default="2.00", help="Ceiling for Cloud calls, including unresolved attempts") + c.add_argument("--sie-base-url") + c.add_argument("--self-hosted", action="store_true") + c = commands.add_parser("collect", help="Optional new Sol/Luna paired answer collection") + c.add_argument("--data", type=Path, required=True) + c.add_argument("--phase", choices=("learn", "calibration", "final"), required=True) + c.add_argument("--output", type=Path, required=True) + c.add_argument("--budget-usd", required=True) + c.add_argument("--frozen", type=Path) + c.add_argument("--execute", action="store_true", help="Make paid calls; otherwise show the plan") + return p + + +def execute(a): + if a.command == "fetch": + return workflow.fetch(a.output, a.group or ["core"]) + if a.command == "prepare": + return workflow.prepare(a.data, a.run) + if a.command == "base": + return workflow.base(a.data, a.run, a.device) + if a.command == "fit-native": + return workflow.fit_native(a.data, a.run) + if a.command == "fit-embedding": + return workflow.fit_embedding(a.data, a.run, a.features) + if a.command == "score": + return workflow.score(a.data, a.run, a.method, a.device, a.features) + if a.command == "fit-cost": + return workflow.fit_cost(a.data, a.run) + if a.command == "reuse-native": + return workflow.reuse_native(a.data, a.run) + if a.command == "calibrate": + return workflow.calibrate(a.data, a.run) + if a.command == "evaluate-refit": + return workflow.evaluate_refit(a.data, a.run, a.output) + if a.command == "replay": + return workflow.replay(a.data, a.output) + if a.command == "frontier": + path = a.run / "internal/calibration-frontiers.json" + if not path.exists(): + path = a.run / "calibration-frontiers.json" + return calibration_view.render(path, a.output, reproduction=(a.run / "reproduction.json").exists()) + if a.command == "answer": + import live # Optional provider dependencies. + + return live.answer(a.run, a.question, a.output, method=a.method, candidate_id=a.candidate_id, + device=a.device, base_url=a.sie_base_url, self_hosted=a.self_hosted) + if a.command == "encode": + data = workflow.checked_data(a.data) + args = ["--source", data / "source", "--output", a.output, "--execute"] + if a.self_hosted: + core.require(a.sie_base_url, "Supply --sie-base-url for your server") + args.extend(["--self-hosted", "--sie-base-url", a.sie_base_url]) + else: + core.require(a.sie_base_url is None, "Use --self-hosted with a custom endpoint") + args.extend(["--budget-usd", a.budget_usd]) + workflow.invoke("embedding/encode.py", *args) + return {"output": str(a.output), "features": str(a.output / "features")} + if a.command == "collect": + data = workflow.checked_data(a.data) + args = ["--source", data / "source", "--pair", workflow.ROOT / "pair.json", "--phase", a.phase, + "--output", a.output, "--budget-usd", a.budget_usd] + if a.frozen: + args.extend(["--frozen", a.frozen]) + if a.execute: + args.append("--execute") + workflow.invoke("collect_answers.py", *args) + return {"output": str(a.output), "paid_calls_enabled": a.execute} + raise ValueError("Unknown command") + + +if __name__ == "__main__": + ensure_utf8() + print(json.dumps(execute(parser().parse_args()), indent=2, allow_nan=False)) diff --git a/examples/train-a-router/self-host/README.md b/examples/train-a-router/self-host/README.md new file mode 100644 index 000000000..b2f123e5e --- /dev/null +++ b/examples/train-a-router/self-host/README.md @@ -0,0 +1,109 @@ +# Serve the Qwen encoder on your own GPU + +This folder starts **one SIE embedding server** for the router. The two small +correctness predictors run in the example's Python process. **Sol and Luna +remain OpenAI API answer models**; this setup does not serve them locally. + +Use a Linux x86-64 GPU machine, Docker Engine, Docker Compose v2 +and the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). +The target machine for this example is an H100 80 GB, which also supports the +LoRA training exercise. Stop the encoder before training on the same GPU. + +Compose pulls the published **SIE v0.10.1 CUDA default image**, pinned by its +content digest. The model configuration in this folder selects the Flash +adapter for the same Qwen3-Embedding-4B checkpoint. No SIE source build is +needed. The first pull downloads the image; the first server start downloads +approximately 8.04 GB of encoder weights. A named volume retains the weights +and kernel caches. + +This choice uses an available release image. SGLang was faster in most of +our [matched 4B measurements](https://huggingface.co/datasets/superlinked/sie-task-evidence/blob/193a7dc36da168693a64e6add67d1f57dd69ba45/serving-perf-encoders/2026-10-22/qwen3-embedding-4b-adapters/README.md); +it remains the main model catalog's default. Flash and +SGLang produce close, but not identical, vectors. Keep the supplied vectors +for exact result replay; fit and calibrate with your fresh vectors when +building your own router. + +Run these commands from +the example folder on the GPU machine: + +```bash +# Check GPU visibility and the Compose configuration without starting a model. +nvidia-smi +docker compose -f self-host/compose.yaml config --quiet +``` + +```bash +# Download the pinned public SIE image; no local image build is needed. +docker compose -f self-host/compose.yaml pull encoder +``` + +```bash +# Start the encoder and wait for the exact model revision to be loaded. +docker compose -f self-host/compose.yaml up --detach --wait --wait-timeout 1200 encoder +``` + +Set `ENCODER_GPU` to a device ID before starting if GPU 0 is not the intended +device. The health check reads model metadata: it requires the pinned revision, +`loaded=true` and 2,560 output dimensions. It does not make inference requests. +Cold download/loading can take several minutes. Inspect a failed startup with +`docker compose -f self-host/compose.yaml logs --tail 100 encoder`. + +```bash +# Make one synthetic embedding request through the SIE SDK; no answer model is called. +docker compose -f self-host/compose.yaml exec -T encoder \ + python /opt/check_encoder.py --smoke > encoder-smoke.json +``` + +The smoke validates the returned model, ID, vector size, finiteness and +near-unit norm at the model's BF16 precision. Feature collection then +normalizes in float64 before storing float32 vectors, just as the Cloud +feature path does. The smoke uses the same instruction and `is_query=true` +as the example. No SIE API key is needed; GPU hosting is charged by your +provider rather than by SIE Cloud. + +```bash +# Regenerate question features locally while keeping the published vectors unchanged. +uv run --frozen --group encode python router.py encode \ + --data data --output local-features \ + --sie-base-url http://127.0.0.1:8000 --self-hosted +``` + +Use `--features local-features/features` for both classifier fitting and embedding +scoring in a new run, then repeat the calibration step in the main walkthrough. + +```bash +# Use your freshly calibrated policy with the local encoder; OpenAI still supplies the selected answer. +CANDIDATE_ID=YOUR_CALIBRATION_CANDIDATE_ID +uv run --frozen --group encode --group collection --group outcomes python router.py answer \ + --run my-router --method embedding --candidate-id "$CANDIDATE_ID" \ + --question 'Which planet is closest to the Sun? A. Mercury B. Venus. Answer A or B.' \ + --output answers/local-encoder-planet \ + --sie-base-url http://127.0.0.1:8000 --self-hosted +``` + +With `--self-hosted`, the example explicitly suppresses SIE Cloud credentials +for the encoder. Your shell environment does not need to change. Use a new +output directory for each live request. + +```bash +# Stop the encoder while preserving the downloaded model and kernel caches. +docker compose -f self-host/compose.yaml down +``` + +For a GPU VM, run Compose on that machine. Run the example there too, or +forward its loopback endpoint over SSH: + +```bash +# From your client machine, keep a tunnel to the GPU VM open while using the local endpoint. +ssh -N -L 8000:127.0.0.1:8000 your-user@your-gpu-vm +``` + +Only `127.0.0.1:8000` is published. Authentication is disabled for this trusted +local endpoint; do not expose the port publicly. The server runs as UID 1000 +with a writable named cache. + +The exact image, profile, server arguments, health check and smoke test passed +on an H100 80 GB as UID/GID 1000 with an empty model cache. The GPU check ran +on Modal; Docker Compose orchestration and host port mapping were checked +statically rather than run there. The [validation receipt](https://huggingface.co/datasets/superlinked/sie-task-evidence/blob/193a7dc36da168693a64e6add67d1f57dd69ba45/serving-perf-encoders/2026-10-22/qwen3-embedding-4b-adapters/released-image-validation.json) +records the image digest and hashes of these files. diff --git a/examples/train-a-router/self-host/check_encoder.py b/examples/train-a-router/self-host/check_encoder.py new file mode 100644 index 000000000..b662a0fbb --- /dev/null +++ b/examples/train-a-router/self-host/check_encoder.py @@ -0,0 +1,104 @@ +"""Check the pinned encoder; --smoke additionally makes one synthetic SDK call.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import time + +import numpy as np +from sie_sdk import SIEClient + +MODEL = "Qwen/Qwen3-Embedding-4B" +REVISION = "5cf2132abc99cad020ac570b19d031efec650f2b" +DIMENSIONS = 2560 +INSTRUCTION = "Represent the knowledge and reasoning required to answer this question correctly." + + +class SingleSendClient(SIEClient): + def _record_retry(self): + raise RuntimeError("No automatic retry: inspect the server before making another request") + + +def check_metadata(metadata): + if ( + metadata.get("name") != MODEL + or metadata.get("revision") != REVISION + or metadata.get("loaded") is not True + or metadata.get("dims", {}).get("dense") != DIMENSIONS + ): + raise RuntimeError("The exact encoder is not loaded with 2,560 output dimensions") + + +def check_vector(reply): + if not isinstance(reply, list) or len(reply) != 1: + raise RuntimeError("Expected exactly one embedding") + item = reply[0] + if item.get("id") != "encoder-smoke" or item.get("model") != MODEL: + raise RuntimeError("The response does not match the request") + vector = item.get("dense") + if isinstance(vector, np.ndarray): + vector = vector.tolist() + if not isinstance(vector, list) or len(vector) != DIMENSIONS: + raise RuntimeError("Wrong vector dimensions") + if not all(type(value) in (int, float) and math.isfinite(value) for value in vector): + raise RuntimeError("The vector contains nonfinite or nonnumeric values") + norm = math.sqrt(math.fsum(value * value for value in vector)) + # BF16 normalization precedes the float32 output cast; feature collection + # subsequently renormalizes in float64 before saving float32 vectors. + if not math.isclose(norm, 1.0, rel_tol=0, abs_tol=1e-2): + raise RuntimeError("Expected a normalized, nonzero vector") + return norm, hashlib.sha256(json.dumps(vector, separators=(",", ":")).encode()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--smoke", action="store_true") + args = parser.parse_args() + timeout = 300 if args.smoke else 3 + # An explicit empty key prevents inheriting any Cloud credentials. + with SingleSendClient( + "http://127.0.0.1:8080", api_key="", timeout_s=timeout, read_timeout_s=timeout + ) as client: + metadata = client.get_model(MODEL) + check_metadata(metadata) + if not args.smoke: + return + started = time.perf_counter() + reply = client.encode( + MODEL, + [{"id": "encoder-smoke", "text": "Which is an even integer? A. 3 B. 4 C. 5"}], + output_types=["dense"], + output_dtype="float32", + instruction=INSTRUCTION, + is_query=True, + wait_for_capacity=False, + max_oom_retries=0, + provision_timeout_s=150, + ) + elapsed = time.perf_counter() - started + norm, digest = check_vector(reply) + if client.last_retry_count != 0: + raise RuntimeError("The SDK unexpectedly retried the request") + print(json.dumps({ + "status": "complete", + "model": MODEL, + "revision": REVISION, + "dimensions": DIMENSIONS, + "instruction": INSTRUCTION, + "is_query": True, + "vector_l2_norm": norm, + "vector_sha256": digest, + "execution_bundle_sha256": client.last_model_revision, + "elapsed_seconds": elapsed, + "physical_encoding_calls": 1, + "answer_model_calls": 0, + "observed_cloud_usd": None, + "hosting_cost": "unpriced", + }, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/self-host/compose.yaml b/examples/train-a-router/self-host/compose.yaml new file mode 100644 index 000000000..ca9bd830c --- /dev/null +++ b/examples/train-a-router/self-host/compose.yaml @@ -0,0 +1,47 @@ +name: train-a-router-encoder + +services: + encoder: + image: ghcr.io/superlinked/sie-server:v0.10.1-cuda12-default@sha256:eff8cb767a7c7c7318c0feca7f677738f27e654fd774eaebab72feb4f61f84b4 + platform: linux/amd64 + restart: "no" + shm_size: 8gb + environment: + SIE_AUTH_MODE: none + HF_HOME: /app/.cache/huggingface + volumes: + - ./models:/app/models:ro + - ./check_encoder.py:/opt/check_encoder.py:ro + - encoder-cache:/app/.cache + ports: + - "127.0.0.1:8000:8080" + command: + - serve + - --host + - 0.0.0.0 + - --port + - "8080" + - --models-dir + - /app/models + - --models + - Qwen/Qwen3-Embedding-4B + - --preload + - Qwen/Qwen3-Embedding-4B + - --device + - cuda + healthcheck: + test: [CMD, python, /opt/check_encoder.py] + interval: 10s + timeout: 10s + start_period: 10m + retries: 3 + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ["${ENCODER_GPU:-0}"] + capabilities: [gpu] + +volumes: + encoder-cache: diff --git a/examples/train-a-router/self-host/models/Qwen__Qwen3-Embedding-4B.yaml b/examples/train-a-router/self-host/models/Qwen__Qwen3-Embedding-4B.yaml new file mode 100644 index 000000000..3eb0b27d8 --- /dev/null +++ b/examples/train-a-router/self-host/models/Qwen__Qwen3-Embedding-4B.yaml @@ -0,0 +1,33 @@ +sie_id: Qwen/Qwen3-Embedding-4B +hf_id: Qwen/Qwen3-Embedding-4B +hf_revision: 5cf2132abc99cad020ac570b19d031efec650f2b +inputs: + text: true + image: false + audio: false + video: false +tasks: + encode: + dense: + dim: 2560 + sparse: null + multivector: null + score: null + extract: null +max_sequence_length: 32768 +profiles: + default: + max_batch_tokens: 16384 + compute_precision: bfloat16 + adapter_path: sie_server.adapters.qwen2_flash:Qwen2FlashAdapter + adapter_options: + loadtime: + causal: true + fused_kernels: true + runtime: + pooling: last + normalize: true + query_template: |- + Instruct: {instruction} + Query: {text} + default_instruction: Given a query, retrieve relevant passages that answer the query diff --git a/examples/train-a-router/source-format-sensitivity.json b/examples/train-a-router/source-format-sensitivity.json new file mode 100644 index 000000000..aa2bc9d91 --- /dev/null +++ b/examples/train-a-router/source-format-sensitivity.json @@ -0,0 +1,74 @@ +{ + "basis": "Public question text only; no acquired response, gold label, model identity or correctness used to identify ambiguity.", + "final_sensitivity": { + "excluded_ids": [ + "bbeh:disambiguation_qa:051", + "bbeh:disambiguation_qa:052" + ], + "n_primary": 975, + "n_sensitivity": 973, + "name": "exclude_source_ambiguous_options", + "rule": "Evaluate the same frozen policies and per-row costs on final rows except these fixed source-ambiguous IDs. Recompute the same aggregate metrics and uncertainty protocol; report both denominators. No refitting, recalibration, threshold selection or answer regeneration." + }, + "fixed_before_final_outcomes": true, + "format": "bbeh-source-format-sensitivity-v1", + "limitations": "A duplicated label can encode two different meanings and identical text can occupy two labels. Upstream label-only references cannot resolve these source defects. Primary scores retain that limitation; the separate sensitivity exposes its final-evaluation effect.", + "normalizer_sha256": "258a60874b11b5bbeefe2a4fe304dfcc2da341eba995258262ce1e5931ef5772", + "primary_population_and_reference_bytes_unchanged": true, + "primary_role_counts": { + "calibration": 385, + "development": 589, + "final": 975, + "train": 1969 + }, + "source_ambiguous_rows": [ + { + "duplicate_labels": [], + "duplicate_text_labels": [ + [ + "B", + "D" + ] + ], + "family": "2d6679ee5418599e7e79d8eada436d3251c513fd874df67786f41f63e014ab88", + "id": "bbeh:disambiguation_qa:018", + "index": 18, + "request_sha256": "d43a5ad79fd64b28f935dfdd5e308d504200060111f51e7b0bbdfd730927f5be", + "retained": true, + "role": "train", + "task": "disambiguation_qa" + }, + { + "duplicate_labels": [ + "E" + ], + "duplicate_text_labels": [], + "family": "70b9e67438ab0f8bd2e92b6196f9fdb4b82d2011167faddd7115fa9c09132fd8", + "id": "bbeh:disambiguation_qa:051", + "index": 51, + "request_sha256": "3434899c00f908b6f27290e90df888a21c23b739e79357b4d4cad0ecd4046324", + "retained": true, + "role": "final", + "task": "disambiguation_qa" + }, + { + "duplicate_labels": [], + "duplicate_text_labels": [ + [ + "C", + "D" + ] + ], + "family": "fd4491ee83e2c6a26a27f2d38f4056681bbf0564742eae2e3a6e8f13694948e2", + "id": "bbeh:disambiguation_qa:052", + "index": 52, + "request_sha256": "f443e7d31db4fabcd10704f8bd58bdf354a5b847c6803fcad013b5f16bfd7b47", + "retained": true, + "role": "final", + "task": "disambiguation_qa" + } + ], + "source_audit_sha256": "bf618b8be32a6391094b984919f802c8f0dd5c5c98417e8475764c56ee579cdd", + "source_plan_sha256": "b15e9c2d9db284fe170e0460550f02fc2cb48f2b9234b697d181716264984a53", + "training_ambiguity_policy": "Retain disambiguation_qa:018 in the fixed training population with the original source label semantics; do not repair its label or choose an option from model outputs." +} diff --git a/examples/train-a-router/source/fetch.py b/examples/train-a-router/source/fetch.py new file mode 100644 index 000000000..bd953c2ca --- /dev/null +++ b/examples/train-a-router/source/fetch.py @@ -0,0 +1,60 @@ +"""Download the pinned public BBEH source; verify bytes before any source use.""" + +import argparse +import hashlib +import json +import shutil +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path, PurePosixPath +from urllib.request import urlopen + +REVISION = "80d12ca916b7158f22293fcf3144f4d3d854d4be" +PREFIX = f"https://raw.githubusercontent.com/google-deepmind/bbeh/{REVISION}/" + + +def fetch(output): + output = Path(output) + if output.exists(): + raise ValueError("Preserve existing source snapshots; use a new output directory") + pins = Path(__file__).parent / "pins" + manifests, files = {}, {} + for name in ("source-download.json", "full-source-download.json"): + data = (pins / name).read_bytes() + manifest = json.loads(data) + if manifest["revision"] != REVISION: + raise ValueError("Source revision changed") + manifests[name] = data + for relative, receipt in manifest["files"].items(): + path = PurePosixPath(relative) + if path.is_absolute() or ".." in path.parts or receipt["url"] != PREFIX + relative: + raise ValueError("Unpinned source URL or unsafe output path") + if relative in files and files[relative] != receipt: + raise ValueError("Conflicting source inventory") + files[relative] = receipt + output.mkdir(parents=True) + + def download(item): + relative, receipt = item + with urlopen(receipt["url"], timeout=60) as response: + data = response.read(receipt["bytes"] + 1) + if len(data) != receipt["bytes"] or hashlib.sha256(data).hexdigest() != receipt["sha256"]: + raise ValueError("Source download failed exact size/hash validation: " + relative) + destination = output / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(data) + + try: + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(download, files.items())) + for name, data in manifests.items(): + (output / name).write_bytes(data) + except BaseException: + shutil.rmtree(output, ignore_errors=True) + raise + return {"revision": REVISION, "files": len(files), "bytes": sum(r["bytes"] for r in files.values())} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + print(json.dumps(fetch(parser.parse_args().output))) diff --git a/examples/train-a-router/source/grouping.py b/examples/train-a-router/source/grouping.py new file mode 100644 index 000000000..3575ec1fa --- /dev/null +++ b/examples/train-a-router/source/grouping.py @@ -0,0 +1,180 @@ +"""Prospective source-text families; never reads references or model outcomes.""" + +import hashlib +import json +import re +import unicodedata + +VERSION = "bbeh-source-context-families-v1" + + +def normalize(text): + return " ".join(unicodedata.normalize("NFKC", text).split()).casefold() + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False).encode()).hexdigest() + + +def first_sentence(text): + # Titles such as Prof. and Mr. are not complete story stems. + parts = re.split(r"(?<=[.!?])\s+", text.strip()) + result = "" + for part in parts: + result = (result + " " + part).strip() + if len(result) >= 35: + return result + return result + + +def context_keys(task, text): + keys = [("normalized-full-input", normalize(text))] + covered = False + + def add(kind, value, minimum=30): + nonlocal covered + value = normalize(value) + if len(value) < minimum: + return + keys.append((task + ":" + kind, value)) + covered = True + + def match(kind, expression, minimum=30): + found = re.search(expression, text, re.S | re.I) + if found: + add(kind, found.group(1), minimum) + + if task == "linguini": + # Full prompts keep the original language-problem introduction while + # varying the requested blank/translation. Tables are an additional + # link when a setup's paragraph boundary differs. + add("language-problem-introduction", re.split(r"\n\s*\n", text.strip())[0]) + for block in re.split(r"\n\s*\n", text): + if "|" in block and len(block) >= 120: + add("shared-language-table", block) + elif task == "causal_understanding": + story = re.sub(r"^\s*Question:\s*", "", text) + story = re.split(r"\s+Reply Yes or No", story, flags=re.I)[0] + add("story-opening-sentence", first_sentence(story)) + words = re.findall(r"\w+", normalize(story)) + words = [word for word in words if word not in {"a", "an", "the"}] + add("story-opening-12-content-tokens", " ".join(words[:12]), 35) + for sentence in re.split(r"(?<=[.!?])\s+", story): + if not re.match(r"(?:Reply|If you think people|Did |Was |Is |Would |Does )", sentence, re.I): + add("shared-story-sentence", sentence, 90) + elif task == "nycc": + match("cartoon-description", r"Description:\s*(.*?)\s*Which of the following") + elif task == "sarc_triples": + parts = re.findall(r"POST\s+\d+:\s*(.*?)\s*REPLY\s+\d+:\s*(.*?)(?=\s*POST\s+\d+:|\Z)", text, re.S | re.I) + if len(parts) != 3: + raise ValueError("SARC source does not contain exactly three post/reply pairs") + for post, reply in parts: + add("post-reply-component", post + "\nREPLY:\n" + reply) + elif task == "word_sorting": + found = re.search(r"Q:\s*Sort the following words alphabetically:\s*List:\s*(.*?)\s*Thought\s+1:", text, re.S) + if found: + add("base-word-multiset", " ".join(sorted(normalize(found.group(1)).split()))) + else: + found = re.search( + r"\A(Consider a new alphabet.*?)Sort the following words with the new alphabet and separate them with comma:\s*(.*)\Z", + text, + re.S, + ) + if found: + add( + "custom-alphabet-and-word-multiset", + found.group(1) + "\n" + "\n".join(sorted(normalize(found.group(2)).split(","))), + ) + elif task == "dyck_languages": + found = re.search(r"\bInput:\s*(.*?)\s*Thought\s+1:", text, re.S) + if found: + brackets = re.sub(r"\s+", "", found.group(1)) + if not re.fullmatch(r"[\[\]{}()<>]+", brackets): + raise ValueError("Unexpected Dyck base-input grammar") + add("base-bracket-sequence", brackets, 1) + elif task == "disambiguation_qa": + match("source-sentence", r"Sentence:\s*(.*?)\s*Which of the following") + elif task == "sportqa": + match("main-sport-scenario", r"Main Question:\s*(.*?)(?=\s+A[).])", 40) + elif task == "time_arithmetic": + question_numbers = re.findall(r"Question([123]):", text) + if question_numbers in (["1", "2"], ["1", "2", "3"]): + # The intervening numerical transformations are part of the + # composed problem; sharing just one date primitive is not a link. + add("combined-time-program", text) + elif task == "buggy_tables": + match("observed-table", r"\A(.*?)\n\s*\nHowever,", 100) + elif task == "zebra_puzzles": + context = re.split(r"\n\s*Question:", text, maxsplit=1)[0] + lines = [re.sub(r"^Clue\s+\d+:\s*", "", line) for line in context.splitlines() if line.strip()] + add("unordered-puzzle-clues-and-attributes", "\n".join(sorted(normalize(line) for line in lines))) + elif task == "hyperbaton": + add("adjective-order-examples", re.split(r"\n\s*\n", text, maxsplit=1)[0]) + elif task == "multistep_arithmetic": + # Shared operator definitions alone are a reused primitive, not a problem. + add("definitions-and-expressions", text) + elif task == "object_counting": + match("inventory-before-query", r"\A(.*?)\s+What\s+(?:is|are)\s+") + elif task == "object_properties": + match("object-history-before-query", r"\A(.*?)\s+In my current collection,") + elif task == "temporal_sequence": + match("calendar-and-constraints", r"\A(.*?)\s+Let X be the length") + elif task == "web_of_lies": + match("truth-assignment-context", r"\A(.*?)\b(?:Do|Does)\s+") + elif task == "geometric_shapes": + match("svg-path", r"SVG path element:\s*(.*?)(?=\s+(?:Which|What|How|Among|The following))", 50) + elif task == "spatial_reasoning": + match("spatial-world-and-path", r"\A(.*?)\s+(?:What (?:will|do|object|is)|Which object)\b", 50) + elif task == "boardgame_qa": + match("board-state-rules", r"\A(.*?)\s+Based on the facts, rules, and preferences,") + elif task == "boolean_expressions": + options = re.findall(r"(?:\A|\n)\([A-E]\)\s*(.*?)(?=\n\([A-E]\)|\Z)", text, re.S) + if len(options) == 5: + add("unordered-expression-options", "\n".join(sorted(normalize(option) for option in options))) + elif task == "movie_recommendation": + options = re.findall(r"(?:\A|\n)\([A-J]\)\s*(.*?)(?=\n\([A-J]\)|\Z)", text, re.S) + if len(options) == 10: + add("movie-pool", "\n".join(sorted({normalize(part) for option in options for part in option.split(",")}))) + elif task == "shuffled_objects": + # Literal scenario prefix, excluding the final interrogative clause. + match("trade-history", r"\A(.*)\s+At the end of\s+") + else: + raise ValueError("Unknown BBEH task") + return keys, covered + + +def families(rows): + parent = list(range(len(rows))) + + def find(index): + while parent[index] != index: + parent[index] = parent[parent[index]] + index = parent[index] + return index + + def join(left, right): + left, right = find(left), find(right) + if left != right: + parent[max(left, right)] = min(left, right) + + first, edges, coverage = {}, [], {} + for index, row in enumerate(rows): + keys, covered = context_keys(row["task"], row["request"]) + coverage[row["id"]] = covered + for kind, value in keys: + key = (kind, digest(value)) + if key in first: + previous = first[key] + join(previous, index) + edges.append({"left": rows[previous]["id"], "right": row["id"], "rule": kind, "key_sha256": key[1]}) + else: + first[key] = index + components = {} + for index, row in enumerate(rows): + components.setdefault(find(index), []).append(row["id"]) + identity = { + member: digest({"version": VERSION, "members": sorted(members)}) + for members in components.values() + for member in members + } + return identity, edges, coverage diff --git a/examples/train-a-router/source/pins/full-source-download.json b/examples/train-a-router/source/pins/full-source-download.json new file mode 100644 index 000000000..3bc31823f --- /dev/null +++ b/examples/train-a-router/source/pins/full-source-download.json @@ -0,0 +1,172 @@ +{ + "files": { + "bbeh/benchmark_tasks/bbeh_boardgame_qa/task.json": { + "bytes": 1358526, + "git_blob_sha1": "59bfa63f5b98bd93d13137df28297d69fc85ceab", + "rows": 200, + "sha256": "3d47dc41eb825b2ee77a696e79859d8d2043b1bf04077e120caec4ad324f32a8", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_boardgame_qa/task.json" + }, + "bbeh/benchmark_tasks/bbeh_boolean_expressions/task.json": { + "bytes": 543587, + "git_blob_sha1": "3b1b165e0f026d61c1ba8a7112cc5289d978262a", + "rows": 200, + "sha256": "49e763f10dbea048be56036c8cbf2a6d0bb735235e3806b50809a1addb57eee2", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_boolean_expressions/task.json" + }, + "bbeh/benchmark_tasks/bbeh_buggy_tables/task.json": { + "bytes": 1012820, + "git_blob_sha1": "e2eeae579ff119ffd424c7eb269fc4dfb68a8da3", + "rows": 200, + "sha256": "c27b76200bf1b0d4ada3729f5a40aa1f0062267e42c4f40820b3e40891c3fe85", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_buggy_tables/task.json" + }, + "bbeh/benchmark_tasks/bbeh_causal_understanding/task.json": { + "bytes": 188324, + "git_blob_sha1": "1222dc4953e0147aa954c76d8d33910c05ba460d", + "rows": 200, + "sha256": "b10bbf43f7f5cdde927b0ad03f99a7a78b3525de4bfa90008b36b12b010c9a81", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_causal_understanding/task.json" + }, + "bbeh/benchmark_tasks/bbeh_disambiguation_qa/task.json": { + "bytes": 95806, + "git_blob_sha1": "04a553ccada1641884d6b07a3d2aff73ea25b2f5", + "rows": 120, + "sha256": "6d34e0c89c3b1e1386df5919159b8d67b4668308d1225bd1ee54eb1e946472bd", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_disambiguation_qa/task.json" + }, + "bbeh/benchmark_tasks/bbeh_dyck_languages/task.json": { + "bytes": 586949, + "git_blob_sha1": "9c73487fbb6c5e453ad5bbf1b372f79b9a18f668", + "rows": 200, + "sha256": "a089612502bdfb98a447c8cba84f972724308ea0d059e8561e3cc9785de585cc", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_dyck_languages/task.json" + }, + "bbeh/benchmark_tasks/bbeh_geometric_shapes/task.json": { + "bytes": 491290, + "git_blob_sha1": "5ff75c179f5f406967f8b02ec57db2ab7c5817eb", + "rows": 200, + "sha256": "7ff506d026450cb878a0963ce478724ad1451bc56469c813c791b9345ac1bea7", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_geometric_shapes/task.json" + }, + "bbeh/benchmark_tasks/bbeh_hyperbaton/task.json": { + "bytes": 921276, + "git_blob_sha1": "9e276ca2c4b57a7e506f2d7128f7a4cacd55c960", + "rows": 200, + "sha256": "0fbacc94f4c740c3050021f4c57089f2011d7bd77fcff3346a7f4b9dbef842e2", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_hyperbaton/task.json" + }, + "bbeh/benchmark_tasks/bbeh_linguini/task.json": { + "bytes": 239942, + "git_blob_sha1": "e8edd908fb96511855d29c0b128f3a89f163c7b0", + "rows": 200, + "sha256": "ba387d2aa1bde122e6c9937fca4e075b9cc7d64e6047b170cfcffd6304919ea8", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_linguini/task.json" + }, + "bbeh/benchmark_tasks/bbeh_movie_recommendation/task.json": { + "bytes": 221785, + "git_blob_sha1": "c07c9f17ca3def29062934bf79b53cbee99b9ede", + "rows": 200, + "sha256": "808882647b84c116a57e0975e3d357749525e12d215c4a5dae83de83ae8c4772", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_movie_recommendation/task.json" + }, + "bbeh/benchmark_tasks/bbeh_multistep_arithmetic/task.json": { + "bytes": 205121, + "git_blob_sha1": "ac431dc3911aa5640f001c6a4de74588713e8506", + "rows": 200, + "sha256": "47207086cd2afe74df9968172d24134c82af8c666144092f04e4c1ebad8cf3d2", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_multistep_arithmetic/task.json" + }, + "bbeh/benchmark_tasks/bbeh_nycc/task.json": { + "bytes": 182646, + "git_blob_sha1": "09666db9e95cf7b65a0d2d33e7acc3a84104e5c6", + "rows": 200, + "sha256": "67c9a9dca40e6ec820a279da19a019cedd5427fbaaae3fd175cdb74965474b08", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_nycc/task.json" + }, + "bbeh/benchmark_tasks/bbeh_object_counting/task.json": { + "bytes": 1759338, + "git_blob_sha1": "757780e10d5ea8d40c4d2cc9a8444a9c20a0d603", + "rows": 200, + "sha256": "a0a8b10ca1aa06eae611c7c7e854fbec4f9622c8f74a74ffa00436a9a8609b67", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_object_counting/task.json" + }, + "bbeh/benchmark_tasks/bbeh_object_properties/task.json": { + "bytes": 1061333, + "git_blob_sha1": "e8bc36ad02c89d763e4981967ec5235f64a107ad", + "rows": 200, + "sha256": "d606b3aa0c35d5397afc5937207174593c82ad5dfcaf7e5dfb40a99278aef0e4", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_object_properties/task.json" + }, + "bbeh/benchmark_tasks/bbeh_sarc_triples/task.json": { + "bytes": 267005, + "git_blob_sha1": "fcb7b152123953c768f7fb415c29d37df0aa6b62", + "rows": 200, + "sha256": "4c7b8eb4b542d326152394456c0a1afc8ef5447ee06dd4187b7eb91027a6b821", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_sarc_triples/task.json" + }, + "bbeh/benchmark_tasks/bbeh_shuffled_objects/task.json": { + "bytes": 2687569, + "git_blob_sha1": "8099cd6873756c47cbba0538fc11767e5fe2bfc6", + "rows": 200, + "sha256": "43cda28220c66e8d63e667d3091cd18b0b17aa55c005bb24bbb63144afdba9c1", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_shuffled_objects/task.json" + }, + "bbeh/benchmark_tasks/bbeh_spatial_reasoning/task.json": { + "bytes": 1275134, + "git_blob_sha1": "dc362278dac49294d0225c0249e6bf9e7c8e31d4", + "rows": 200, + "sha256": "e77986b73e0ed15b5f950978b7758f55311b4d50e7215961084b126e4d960cb7", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_spatial_reasoning/task.json" + }, + "bbeh/benchmark_tasks/bbeh_sportqa/task.json": { + "bytes": 355509, + "git_blob_sha1": "c7407a7bd287cda7719fb7b7cbdca840a50a79ad", + "rows": 200, + "sha256": "69b3f164d9617eab9185830eacf4250badb56c2d718ddc75012597ed1a4bc9bf", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_sportqa/task.json" + }, + "bbeh/benchmark_tasks/bbeh_temporal_sequence/task.json": { + "bytes": 758382, + "git_blob_sha1": "e0ffd804b234344a967ecf9f6b2dad2b5f960279", + "rows": 200, + "sha256": "55f6711d3d76925acb5d2fffdc1947e0e7186c686f0a343598394a57df6c7c6e", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_temporal_sequence/task.json" + }, + "bbeh/benchmark_tasks/bbeh_time_arithmetic/task.json": { + "bytes": 238051, + "git_blob_sha1": "e1a8fcfe7cb58c0eb12700969904e16f3d5bff51", + "rows": 200, + "sha256": "2510515a6f12024a742743e29a389f07beacb3bf4c71e0b59be12f3e364cf31b", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_time_arithmetic/task.json" + }, + "bbeh/benchmark_tasks/bbeh_web_of_lies/task.json": { + "bytes": 512076, + "git_blob_sha1": "2bccd9eae0f1f10559a0d1e830aa0ccd4ac63c5a", + "rows": 200, + "sha256": "15f23d32579285701f68dae7947e2191eff21ee97440dfa73cb3565a66c77a6d", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_web_of_lies/task.json" + }, + "bbeh/benchmark_tasks/bbeh_word_sorting/task.json": { + "bytes": 456902, + "git_blob_sha1": "2995a286e30f32ce0eac590a52461243d3d383ff", + "rows": 200, + "sha256": "f49674b7b85774d6190c2844e1682c2ee8ff591ca1eeee4e785a45ef1e20024d", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_word_sorting/task.json" + }, + "bbeh/benchmark_tasks/bbeh_zebra_puzzles/task.json": { + "bytes": 2303863, + "git_blob_sha1": "4a42b417ae48dda20f34f607ab3f4b2bbcb58066", + "rows": 200, + "sha256": "0acd979060ee49f063f241dd1a298bd80b63d6b7fbfb08057ff480bff98a6198", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/benchmark_tasks/bbeh_zebra_puzzles/task.json" + } + }, + "repository": "https://github.com/google-deepmind/bbeh", + "revision": "80d12ca916b7158f22293fcf3144f4d3d854d4be", + "total_bytes": 17723234, + "total_rows": 4520, + "tree_sha256": "cd4e7caa7ac45e18a0bb1a8bf1fbed19f1c9a8de8d05a6f615aeefc3c81d4eea", + "tree_url": "https://api.github.com/repos/google-deepmind/bbeh/git/trees/80d12ca916b7158f22293fcf3144f4d3d854d4be?recursive=1", + "upstream_code_executed": false +} diff --git a/examples/train-a-router/source/pins/source-download.json b/examples/train-a-router/source/pins/source-download.json new file mode 100644 index 000000000..b702e5bc0 --- /dev/null +++ b/examples/train-a-router/source/pins/source-download.json @@ -0,0 +1,25 @@ +{ + "revision": "80d12ca916b7158f22293fcf3144f4d3d854d4be", + "files": { + "bbeh/mini/data.json": { + "bytes": 1798615, + "sha256": "14e77b3d6be68faa008d268abf53b1f8d2420ffdd762504a304dafc3f8d43026", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/mini/data.json" + }, + "bbeh/evaluate.py": { + "bytes": 3701, + "sha256": "4b4f06e5babb015de2ba639bae995a5526182ffb5b5890af54dfcf038580eb34", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/bbeh/evaluate.py" + }, + "README.md": { + "bytes": 7541, + "sha256": "86d16ad213c64a4777a5368ae1a76664d1152b9316c04dede597b8e8cbedf56c", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/README.md" + }, + "LICENSE": { + "bytes": 30393, + "sha256": "ed714919b09bcaee887dfed4bca3972041cac53453c3089f57a367e05a50704d", + "url": "https://raw.githubusercontent.com/google-deepmind/bbeh/80d12ca916b7158f22293fcf3144f4d3d854d4be/LICENSE" + } + } +} diff --git a/examples/train-a-router/source/prepare.py b/examples/train-a-router/source/prepare.py new file mode 100644 index 000000000..041d795e0 --- /dev/null +++ b/examples/train-a-router/source/prepare.py @@ -0,0 +1,297 @@ +"""Prepare a prospective BBEH family split from pinned public source only.""" + +import argparse +from collections import Counter, defaultdict +import hashlib +import json +from pathlib import Path +import shutil +import unicodedata + +from grouping import VERSION, digest, families, normalize + +REVISION = "80d12ca916b7158f22293fcf3144f4d3d854d4be" +ROLES = ("train", "development", "calibration", "final") +WEIGHTS = (50, 15, 10, 25) +SEED = "bbeh-recipes-20261021-source-only-v1" +CONTRACT = {"type": "exact_string", "max_length": 2048} + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n") + + +def duplicate_key(text): + # Case is semantic in some tasks. Case folding is used conservatively for + # family links, never to delete a source problem as a duplicate. + return " ".join(unicodedata.normalize("NFKC", text).split()) + + +def allocate(task, family_ids): + ordered = sorted(set(family_ids), key=lambda family: digest([SEED, task, family])) + n = len(ordered) + if n < len(ROLES): + raise ValueError(f"Task {task} lacks four independent retained families") + counts = [n * weight // 100 for weight in WEIGHTS] + residual = sorted(range(4), key=lambda i: (-(n * WEIGHTS[i] % 100), i)) + for index in residual[: n - sum(counts)]: + counts[index] += 1 + if not all(counts): + raise ValueError(f"Task {task} cannot populate every role under this allocation") + result, start = {}, 0 + for role, count in zip(ROLES, counts, strict=True): + result.update({family: role for family in ordered[start : start + count]}) + start += count + return result + + +def load(source): + manifests = [ + json.loads((source / name).read_bytes()) for name in ("source-download.json", "full-source-download.json") + ] + inventory = {} + for manifest in manifests: + if manifest["revision"] != REVISION: + raise ValueError("Unexpected BBEH source revision") + for name, item in manifest["files"].items(): + if sha(source / name) != item["sha256"]: + raise ValueError(f"Source hash mismatch: {name}") + inventory[name] = item + rows = [] + for name in sorted(inventory): + if not name.endswith("/task.json"): + continue + task = Path(name).parent.name.removeprefix("bbeh_") + for index, example in enumerate(json.loads((source / name).read_bytes())["examples"]): + if not isinstance(example["input"], str) or not isinstance(example["target"], str): + raise ValueError("Non-string source input/reference") + rows.append( + { + "id": f"bbeh:{task}:{index:03d}", + "task": task, + "index": index, + "source_id": f"{name}#examples/{index}", + "source_path": name, + "request": example["input"], + "target": example["target"], + "source_metadata": {k: v for k, v in example.items() if k not in {"input", "target"}}, + } + ) + mini = json.loads((source / "bbeh/mini/data.json").read_bytes())["examples"] + if len(rows) != 4520 or len({row["task"] for row in rows}) != 23 or len(mini) != 460: + raise ValueError("Unexpected pinned source population") + return rows, mini, inventory + + +def partition(rows, mini): + # Grouping sees no reference labels, model responses, costs, or scores. + identity, edges, coverage = families([{k: row[k] for k in ("id", "task", "request")} for row in rows]) + if not all(coverage.values()): + raise ValueError("An input did not match its task's source-context grammar") + mini_by_prompt = defaultdict(list) + references = defaultdict(set) + for index, row in enumerate(mini): + mini_by_prompt[normalize(row["input"])].append(index) + references[duplicate_key(row["input"])].add(row["target"]) + duplicates = defaultdict(list) + for row in rows: + key = duplicate_key(row["request"]) + duplicates[key].append(row) + references[key].add(row["target"]) + conflicting = {key for key, targets in references.items() if len(targets) > 1} + conflict_families = {identity[row["id"]] for row in rows if duplicate_key(row["request"]) in conflicting} + mini_families = {identity[row["id"]] for row in rows if normalize(row["request"]) in mini_by_prompt} + inactive = conflict_families | mini_families + canonical = {key: min(group, key=lambda row: row["id"])["id"] for key, group in duplicates.items()} + active = [ + row + for row in rows + if identity[row["id"]] not in inactive and canonical[duplicate_key(row["request"])] == row["id"] + ] + family_tasks = defaultdict(set) + for row in active: + family_tasks[identity[row["id"]]].add(row["task"]) + if any(len(tasks) != 1 for tasks in family_tasks.values()): + raise ValueError("Cross-task family needs an explicit allocation rule") + assignment = {} + for task in sorted({row["task"] for row in rows}): + assignment.update(allocate(task, [identity[row["id"]] for row in active if row["task"] == task])) + meta = [] + for row in rows: + family = identity[row["id"]] + reasons = [] + if family in mini_families: + reasons.append("prior_official_mini_problem_family") + if family in conflict_families: + reasons.append("reference_conflict_problem_family") + original = canonical[duplicate_key(row["request"])] + duplicate_of = original if original != row["id"] else None + if duplicate_of: + reasons.append("duplicate_source_input") + retained = not reasons + role = ( + assignment[family] + if retained + else ("excluded" if family in conflict_families or duplicate_of else "reserved") + ) + meta.append( + {k: row[k] for k in ("id", "task", "index", "source_id", "source_path", "source_metadata")} + | { + "dataset": "bbeh", + "source": "bbeh", + "family": family, + "role": role, + "retained": retained, + "reservation_reasons": reasons, + "duplicate_of": duplicate_of, + "mini_indices": mini_by_prompt.get(normalize(row["request"]), []), + "request_sha256": hashlib.sha256(row["request"].encode()).hexdigest(), + "normalized_request_sha256": hashlib.sha256(duplicate_key(row["request"]).encode()).hexdigest(), + } + ) + conflicts = [ + { + "normalized_input_sha256": hashlib.sha256(key.encode()).hexdigest(), + "full_source_ids": [row["id"] for row in duplicates.get(key, [])], + "target_sha256": sorted(hashlib.sha256(target.encode()).hexdigest() for target in references[key]), + "affected_family_ids": sorted({identity[row["id"]] for row in duplicates.get(key, [])}), + } + for key in sorted(conflicting) + ] + return meta, edges, conflicts + + +def prepare(source, output): + if output.exists(): + raise FileExistsError("Preserve the existing prepared snapshot; use a new output directory") + rows, mini, inventory = load(source) + meta, edges, conflicts = partition(rows, mini) + output.mkdir(parents=True) + write(output / "meta.json", meta) + write(output / "gold.json", [{"id": row["id"], "target": row["target"]} for row in rows]) + inputs = [ + {"id": row["id"], "request": row["request"], "response_contract": CONTRACT} + for row, item in zip(rows, meta, strict=True) + if item["retained"] + ] + role_by_id = {item["id"]: item["role"] for item in meta} + for role in ROLES: + write( + output / "question-only" / f"{role}-inputs.json", [row for row in inputs if role_by_id[row["id"]] == role] + ) + write(output / "question-only/all-inputs.json", inputs) + write(output / "grouping-edges.json", edges) + write(output / "reference-conflicts.json", conflicts) + tasks = {} + for task in sorted({item["task"] for item in meta}): + items = [item for item in meta if item["task"] == task] + counts = Counter(item["family"] for item in items) + tasks[task] = { + "source_rows": len(items), + "source_families": len(counts), + "largest_source_family": max(counts.values()), + "rows_by_role": dict(Counter(item["role"] for item in items)), + "families_by_role": { + role: len({item["family"] for item in items if item["role"] == role}) for role in ROLES + }, + "retained_rows": sum(item["retained"] for item in items), + "reserved_mini_family_rows": sum( + "prior_official_mini_problem_family" in item["reservation_reasons"] for item in items + ), + } + audit = { + "format": "bbeh-source-split-audit-v1", + "source_only": True, + "model_outcomes_accessed": False, + "source_rows": len(rows), + "retained_rows": len(inputs), + "tasks_retained": len(tasks), + "rows_by_role": dict(Counter(item["role"] for item in meta)), + "families_by_role": {role: len({item["family"] for item in meta if item["role"] == role}) for role in ROLES}, + "reservation_reason_counts": dict(Counter(reason for item in meta for reason in item["reservation_reasons"])), + "tasks": tasks, + "context_rule_coverage": "all 4520 source rows", + "grouping_edges_by_rule": dict(Counter(edge["rule"] for edge in edges)), + "overgrouping_controls": [ + "A reused primitive, common task instruction, or generic template is not itself a family link.", + "Time arithmetic groups the complete composed program, including numerical transformations; single subquestions do not link rows.", + "SARC requires a shared post-and-reply pair, not a shared post alone; SportQA uses its main scenario, not a generic subquestion.", + "Linguini shares original language-problem introductions/tables; causal variants share specific story text.", + ], + "limitations": [ + "Upstream provides no complete original-problem/template IDs; text-derived families do not prove absence of paraphrased or structurally isomorphic contexts.", + "Within-task family splits measure performance on new problems from these tasks, not generalization to unseen tasks.", + "Base-model pretraining overlap is unknown; this grouping constrains router training leakage only.", + "Unequal family sizes make row proportions approximate. Final paired comparisons should bootstrap whole families and remain descriptive.", + ], + } + write(output / "split-audit.json", audit) + for name in ("source-download.json", "full-source-download.json", "LICENSE", "README.md", "bbeh/evaluate.py"): + target = output / "provenance" / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source / name, target) + plan = { + "format": "bbeh-recipes-source-plan-v1", + "repository": "https://github.com/google-deepmind/bbeh", + "revision": REVISION, + "source_only": True, + "model_outcomes_accessed": False, + "split_seed": SEED, + "role_proportions": dict(zip(ROLES, [weight / 100 for weight in WEIGHTS], strict=True)), + "allocation": "Within each task: SHA256(seed,task,family)-ordered families; Hamilton integer apportionment in declared role order; keep whole families.", + "row_order": "Sorted pinned task path, then original examples[] index; roles preserve this order.", + "grouping_version": VERSION, + "family_normalization": "Unicode NFKC + whitespace collapse + casefold for conservative context links", + "duplicate_normalization": "Unicode NFKC + whitespace collapse, case preserved; retain smallest source ID unless whole family reserved/excluded", + "prior_mini_policy": "Reserve the complete input-derived family of every official Mini row, including prior excluded/conflicting Mini rows.", + "reference_conflict_policy": "Exclude the entire original-problem family when an identical normalized input has conflicting original references; no model outcomes used.", + "response_contract": CONTRACT, + "models": {"cheap": "gpt-6-luna", "strong": "gpt-6.1-sol"}, + "grading": { + "official_source_sha256": inventory["bbeh/evaluate.py"]["sha256"], + "primary": "Unchanged official evaluate_correctness on the decoded answer string and original target", + "sensitivities": [ + "literal answer.strip() == target.strip()", + "official normalization with finite Decimal equality replacing binary float equality", + ], + "invalid": "Unsuccessful outcome with original usage retained; no response-based exclusions or quality retries", + }, + "counts": {role: audit["rows_by_role"].get(role, 0) for role in ROLES}, + "family_counts": audit["families_by_role"], + "tasks": list(tasks), + "source_inventory": inventory, + "preparation_sources_sha256": { + name: sha(Path(__file__).parent / name) for name in ("prepare.py", "grouping.py", "test_source.py") + }, + "files_sha256": { + path.relative_to(output).as_posix(): sha(path) for path in sorted(output.rglob("*")) if path.is_file() + }, + } + write(output / "source-plan.json", plan) + return plan + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--source", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + plan = prepare(args.source, args.output) + print( + json.dumps( + { + "source_plan_sha256": sha(args.output / "source-plan.json"), + "counts": plan["counts"], + "family_counts": plan["family_counts"], + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/train-a-router/source/test_fetch.py b/examples/train-a-router/source/test_fetch.py new file mode 100644 index 000000000..48fa184db --- /dev/null +++ b/examples/train-a-router/source/test_fetch.py @@ -0,0 +1,104 @@ +"""Source-download recovery tests with synthetic bytes and no network access.""" + +import hashlib +import json +import unittest +from io import BytesIO +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +import fetch + + +class FetchTests(unittest.TestCase): + def write_pins(self, root): + payloads = {"tasks/first.json": b'{"input":"first"}', "tasks/second.json": b'{"input":"second"}'} + pins = root / "pins" + pins.mkdir() + for name, (relative, data) in zip( + ("source-download.json", "full-source-download.json"), payloads.items(), strict=True + ): + (pins / name).write_text( + json.dumps( + { + "revision": fetch.REVISION, + "files": { + relative: { + "url": fetch.PREFIX + relative, + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + }, + } + ) + ) + return payloads + + def test_interrupted_download_cleans_partial_snapshot_and_allows_retry(self): + for failure in (TimeoutError("download interrupted"), KeyboardInterrupt()): + with self.subTest(failure=type(failure).__name__), TemporaryDirectory() as directory: + root = Path(directory) + output = root / "snapshot" + payloads = self.write_pins(root) + + def interrupted(url, timeout, *, payloads=payloads, failure=failure): + relative = url.removeprefix(fetch.PREFIX) + if relative == "tasks/second.json": + raise failure + return BytesIO(payloads[relative]) + + def complete(url, timeout, *, payloads=payloads): + return BytesIO(payloads[url.removeprefix(fetch.PREFIX)]) + + with patch.object(fetch, "__file__", str(root / "fetch.py")): + with ( + patch.object(fetch, "urlopen", side_effect=interrupted), + self.assertRaises(type(failure)), + ): + fetch.fetch(output) + self.assertFalse(output.exists()) + with patch.object(fetch, "urlopen", side_effect=complete): + result = fetch.fetch(output) + self.assertEqual(result["files"], 2) + self.assertEqual(result["bytes"], sum(map(len, payloads.values()))) + for relative, data in payloads.items(): + self.assertEqual((output / relative).read_bytes(), data) + for name in ("source-download.json", "full-source-download.json"): + self.assertEqual((output / name).read_bytes(), (root / "pins" / name).read_bytes()) + + def test_hash_mismatch_cleans_new_output(self): + with TemporaryDirectory() as directory: + root = Path(directory) + output = root / "snapshot" + payloads = self.write_pins(root) + + def corrupted(url, timeout): + data = payloads[url.removeprefix(fetch.PREFIX)] + return BytesIO(b"x" * len(data)) + + with ( + patch.object(fetch, "__file__", str(root / "fetch.py")), + patch.object(fetch, "urlopen", side_effect=corrupted), + self.assertRaisesRegex(ValueError, "size/hash validation"), + ): + fetch.fetch(output) + self.assertFalse(output.exists()) + + def test_preexisting_snapshot_is_preserved_without_network_access(self): + with TemporaryDirectory() as directory: + output = Path(directory) / "snapshot" + output.mkdir() + existing = output / "existing.txt" + existing.write_bytes(b"preserve this data") + with ( + patch.object(fetch, "urlopen") as download, + self.assertRaisesRegex(ValueError, "Preserve existing source snapshots"), + ): + fetch.fetch(output) + self.assertEqual(existing.read_bytes(), b"preserve this data") + download.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/train-a-router/source/test_source.py b/examples/train-a-router/source/test_source.py new file mode 100644 index 000000000..7c819dda6 --- /dev/null +++ b/examples/train-a-router/source/test_source.py @@ -0,0 +1,125 @@ +"""Small prospective source-family and allocation tests; no model outcomes.""" + +from collections import Counter +import unittest + +from grouping import families +from prepare import ROLES, allocate, duplicate_key, partition + + +def row(identifier, task, request, target="unused"): + return { + "id": identifier, + "task": task, + "index": 0, + "source_id": identifier, + "source_path": "synthetic/task.json", + "source_metadata": {}, + "request": request, + "target": target, + } + + +class SourceTests(unittest.TestCase): + def test_linguini_shared_problem_but_not_common_format(self): + rows = [ + row( + "a", + "linguini", + "These are translations in the Kewa language of an original problem.\n\nTranslate apple.", + ), + row( + "b", + "linguini", + "These are translations in the Kewa language of an original problem.\n\nTranslate pear.", + ), + row("c", "linguini", "These are translations in the Tonga language of another problem.\n\nTranslate pear."), + ] + identity, _, _ = families(rows) + self.assertEqual(identity["a"], identity["b"]) + self.assertNotEqual(identity["a"], identity["c"]) + + def test_causal_specific_story_variants_link(self): + rows = [ + row( + "a", + "causal_understanding", + "Question: Janet works in the factory that assembles complex electronic circuit boards for large cars. She presses a switch. Did it fail? Reply Yes or No.", + ), + row( + "b", + "causal_understanding", + "Question: Janet works in a factory that assembles complex electronic circuit boards for large cars. She does not press a switch. Did it fail? Reply Yes or No.", + ), + ] + identity, _, _ = families(rows) + self.assertEqual(identity["a"], identity["b"]) + + def test_reused_time_primitive_does_not_link_compositions(self): + common = "Question1: What date follows 2024-01-01?\n\n" + rows = [ + row("a", "time_arithmetic", common + "Let X = X + 2.\n\nQuestion2: What is X o'clock in Paris?"), + row("b", "time_arithmetic", common + "Let X = X + 3.\n\nQuestion2: What is X o'clock in Paris?"), + ] + identity, _, coverage = families(rows) + self.assertNotEqual(identity["a"], identity["b"]) + self.assertTrue(all(coverage.values())) + + def test_same_sarc_post_with_different_replies_is_not_a_family(self): + def request(suffix): + return "\n".join( + f"POST {i}: This is a lengthy common post number {i}. REPLY {i}: This is different response {suffix} {i}." + for i in range(1, 4) + ) + + identity, _, _ = families( + [row("a", "sarc_triples", request("first")), row("b", "sarc_triples", request("second"))] + ) + self.assertNotEqual(identity["a"], identity["b"]) + + def test_family_identity_does_not_depend_on_reference(self): + rows = [ + row("a", "multistep_arithmetic", "Consider this complete composed arithmetic problem with two operators."), + row( + "b", "multistep_arithmetic", "Consider this different composed arithmetic problem with three operators." + ), + ] + before = families(rows)[0] + for item in rows: + item["target"] = "changed reference that grouping must not read" + self.assertEqual(before, families(rows)[0]) + + def test_allocation_is_stable_whole_family_and_proportional(self): + groups = [f"family-{index}" for index in range(180)] + assigned = allocate("task", groups) + self.assertEqual(assigned, allocate("task", list(reversed(groups)) + groups[:4])) + self.assertEqual(Counter(assigned.values()), {"train": 90, "development": 27, "calibration": 18, "final": 45}) + + def test_mini_reserves_context_family_and_conflict_excludes_whole_family(self): + rows = [ + row( + str(i), + "linguini", + f"Original independent language problem introduction number {i} is here.\n\nTranslate one word.", + ) + for i in range(30) + ] + rows.append(row("variant", "linguini", rows[0]["request"].split("\n\n")[0] + "\n\nTranslate another word.")) + mini = [{"input": rows[0]["request"], "target": "unused"}] + metadata, _, _ = partition(rows, mini) + by_id = {item["id"]: item for item in metadata} + self.assertEqual(by_id["0"]["role"], "reserved") + self.assertEqual(by_id["variant"]["role"], "reserved") + rows.append(row("conflicting", "linguini", rows[1]["request"], "different")) + metadata, _, conflicts = partition(rows, mini) + self.assertEqual(len(conflicts), 1) + self.assertTrue(all(item["role"] == "excluded" for item in metadata if item["id"] in {"1", "conflicting"})) + self.assertEqual({item["role"] for item in metadata if item["retained"]}, set(ROLES)) + + def test_duplicate_normalization_preserves_semantic_case(self): + self.assertEqual(duplicate_key("A\n B"), duplicate_key("A B")) + self.assertNotEqual(duplicate_key("A B"), duplicate_key("a b")) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/train-a-router/test_evidence.py b/examples/train-a-router/test_evidence.py new file mode 100644 index 000000000..54025ad26 --- /dev/null +++ b/examples/train-a-router/test_evidence.py @@ -0,0 +1,94 @@ +"""Small download-boundary tests using invented local artifacts.""" + +from copy import deepcopy +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from io import BytesIO + +import evidence + + +class EvidenceTests(unittest.TestCase): + def fixture(self): + content = b"invented data\n" + manifest = { + "format": evidence.FORMAT, + "files": [ + { + "path": "source/rows.json", + "group": "core", + "size": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + }, + { + "path": "reference/weights.bin", + "group": "weights", + "size": 0, + "sha256": hashlib.sha256(b"").hexdigest(), + }, + ], + } + body = json.dumps(manifest).encode() + return manifest, body, hashlib.sha256(body).hexdigest(), content + + def test_manifest_and_path_tampering_rejected(self): + manifest, body, digest, _ = self.fixture() + with self.assertRaises(ValueError): + evidence.checked_manifest(body + b" ", digest) + for name in ("../outside", "/absolute", "manifest.json", "source/../outside", "source//rows.json"): + changed = deepcopy(manifest) + changed["files"][0]["path"] = name + value = json.dumps(changed).encode() + with self.subTest(name=name), self.assertRaises(ValueError): + evidence.checked_manifest(value, hashlib.sha256(value).hexdigest()) + + def test_fetch_reuses_valid_core_and_repairs_corruption_without_optional_weights(self): + _, body, digest, content = self.fixture() + urls = [] + + def opening(url, timeout): + urls.append(url) + return BytesIO(body if url.endswith("manifest.json") else content) + + with tempfile.TemporaryDirectory() as temporary, patch.object(evidence, "urlopen", side_effect=opening): + path = Path(temporary) / "data" + first = evidence.fetch("https://example.test/pinned/manifest.json", digest, path) + self.assertEqual(first["downloaded_files"], 1) + self.assertFalse((path / "reference/weights.bin").exists()) + again = evidence.fetch("https://example.test/pinned/manifest.json", digest, path) + self.assertEqual(again["reused_files"], 1) + (path / "source/rows.json").write_bytes(b"corrupt") + with self.assertRaises(ValueError): + evidence.verify(path, manifest_sha256=digest) + evidence.fetch("https://example.test/pinned/manifest.json", digest, path) + evidence.verify(path, manifest_sha256=digest) + self.assertEqual(urls.count("https://example.test/pinned/source/rows.json"), 2) + + def test_bad_download_does_not_replace_existing_file(self): + manifest, _, _, _ = self.fixture() + with ( + tempfile.TemporaryDirectory() as temporary, + patch.object(evidence, "urlopen", return_value=BytesIO(b"wrong")), + ): + target = Path(temporary) / "rows.json" + target.write_bytes(b"preserved") + with self.assertRaises(ValueError): + evidence.download("https://example.test/rows.json", target, manifest["files"][0]) + self.assertEqual(target.read_bytes(), b"preserved") + self.assertEqual(list(Path(temporary).glob("*.partial")), []) + + def test_destination_symlink_cannot_escape(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "download" + root.mkdir() + (root / "source").symlink_to(Path(temporary), target_is_directory=True) + with self.assertRaises(ValueError): + evidence.artifact_path(root, "source/outside") + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/train-a-router/test_normalization.py b/examples/train-a-router/test_normalization.py new file mode 100644 index 000000000..af500f4fe --- /dev/null +++ b/examples/train-a-router/test_normalization.py @@ -0,0 +1,114 @@ +"""Independent, answer-model-free tests of the source-derived answer contract. + +Run with the reviewed collection directory on PYTHONPATH. No recorded answer, +reference label, model identity, or correctness result is loaded by these tests. +""" + +import unittest + +from normalize import canonicalize + + +YES_NO = ( + "Your answer should be a list of three words, yes or no " + "(for example, yes, no, yes)." +) +THREE_WAY = ( + "Your answer should be a list of three words separated by commas. " + "Each of them must be either yes (if the person tells the truth), " + "no (if the person lies), or unknown (if the person's truth cannot be " + "determined). For example, or ." +) + + +class SourceCanonicalizationTests(unittest.TestCase): + def test_both_choice_meanings_survive_without_a_reference(self): + prompt = "Pick one.\n(A) orange ball\n(B) purple ball" + for task in ( + "boolean_expressions", "disambiguation_qa", "geometric_shapes", + "movie_recommendation", "nycc", "shuffled_objects", + ): + for label, option in (("A", "orange ball"), ("B", "purple ball")): + for answer in (label, f"({label})", f"({label}) {option}", option): + with self.subTest(task=task, answer=answer): + self.assertEqual(canonicalize(answer, prompt, task)[0], f"({label})") + + def test_label_text_contradictions_and_explanations_are_not_extracted(self): + prompt = "Pick one.\n(A) orange ball\n(B) purple ball" + for answer in ( + "(A) purple ball", "(B) orange ball", "(A) or (B)", "AB", + "(A) orange ball; alternatively (B) purple ball", "orange ball because A wins", + "(A) orange ball\nActually, choose B.", "The answer is (A)", + "(A) orange ball\nThe final answer is (B)", "(Z) orange ball", + ): + with self.subTest(answer=answer): + self.assertIsNone(canonicalize(answer, prompt, "nycc")[0]) + + def test_whole_option_matching_preserves_negation_and_internal_punctuation(self): + prompt = "Pick one.\n(A) We aren't leaving.\n(B) We are leaving." + self.assertEqual(canonicalize("(A) We aren't leaving", prompt, "nycc")[0], "(A)") + for answer in ("(A) We are leaving.", "We arent leaving.", "We are not leaving."): + with self.subTest(answer=answer): + self.assertIsNone(canonicalize(answer, prompt, "nycc")[0]) + + def test_repeated_text_under_different_labels_is_ambiguous(self): + prompt = "Pick one.\n(A) same meaning\n(B) different meaning\n(C) same meaning" + self.assertIsNone(canonicalize("same meaning", prompt, "disambiguation_qa")[0]) + self.assertEqual(canonicalize("(C) same meaning", prompt, "disambiguation_qa")[0], "(C)") + + def test_duplicate_label_retains_upstream_label_semantics_only(self): + prompt = "Pick one.\n(A) first meaning\n(B) second meaning\n(B) third meaning" + for answer in ("B", "(B) second meaning", "(B) third meaning", "third meaning"): + with self.subTest(answer=answer): + self.assertEqual(canonicalize(answer, prompt, "disambiguation_qa")[0], "(B)") + self.assertIsNone(canonicalize("(A) third meaning", prompt, "disambiguation_qa")[0]) + + def test_geometry_footer_is_not_an_option_or_acceptable_explanation(self): + footer = "Coordinates have been rounded to 5 decimal places so ignore slight differences." + prompt = f"Options:\n(A) only 1\n(B) 2 and 3\n{footer}" + self.assertEqual(canonicalize("(B) 2 and 3", prompt, "geometric_shapes")[0], "(B)") + self.assertIsNone(canonicalize("(B) 2 and 3 " + footer, prompt, "geometric_shapes")[0]) + + def test_truth_tuples_preserve_all_three_positions(self): + for prompt, answer, expected in ( + (YES_NO, "", "yes, no, yes"), + (YES_NO, "no, yes, no", "no, yes, no"), + (THREE_WAY, "", "unknown, no, yes"), + (THREE_WAY, "", "yes, unknown, no"), + ): + with self.subTest(answer=answer): + value = canonicalize(answer, prompt, "web_of_lies")[0] + self.assertEqual(value.casefold().replace(" ", ""), expected.replace(" ", "")) + + def test_truth_vocabulary_is_taken_from_the_question(self): + for answer in ("", "unknown, no, yes"): + with self.subTest(answer=answer): + self.assertIsNone(canonicalize(answer, YES_NO, "web_of_lies")[0]) + self.assertIsNotNone(canonicalize(answer, THREE_WAY, "web_of_lies")[0]) + + def test_malformed_truth_tuples_cannot_reach_permissive_upstream_extraction(self): + for answer in ( + "", "", "<>", "", + "yes, no, yes\nActually, no, no, no", " because the facts imply it", + "\nThe final answer is no, no, no", "yes,no,yes or no,no,no", + "yes, no, yes>", ""), + ("time_arithmetic", "[Emma, Lily]"), ("temporal_sequence", "45, 3"), + ("multistep_arithmetic", "9007199254740993"), ("boardgame_qa", "disproved"), + ("causal_understanding", "Ambiguous"), ("object_properties", "unknown"), + ("spatial_reasoning", "Granny Smith"), + ): + with self.subTest(task=task, answer=answer): + self.assertEqual(canonicalize(answer, "source request", task), (answer, "unchanged")) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/train-a-router/test_public_adapters.py b/examples/train-a-router/test_public_adapters.py new file mode 100644 index 000000000..51d934b4c --- /dev/null +++ b/examples/train-a-router/test_public_adapters.py @@ -0,0 +1,184 @@ +"""Tiny mocked lifecycle checks; no API calls, downloads, training, or weights.""" + +import hashlib +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import collect +import collect_answers as answers +import grade_v2 as grading +import openai_population as acquisition +from reader_io import sha + + +def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, sort_keys=True)) + + +def source_fixture(root): + source = root / "source" + meta, files = [], {} + for role in ("train", "development", "calibration", "final"): + item = {"id": role + "-q", "request": "Select the orange ball.", + "response_contract": {"type": "exact_string", "max_length": 2048}} + meta.append({"id": item["id"], "role": role, "retained": True, + "task": "dyck_languages", "family": role + "-family", + "request_sha256": hashlib.sha256(item["request"].encode()).hexdigest()}) + path = source / f"question-only/{role}-inputs.json" + write(path, [item]) + files[f"question-only/{role}-inputs.json"] = sha(path) + write(source / "meta.json", meta) + files["meta.json"] = sha(source / "meta.json") + write(source / "source-plan.json", {"files_sha256": files}) + # A reference file exists, but neither preparation nor acquisition may open it. + (source / "gold.json").write_text("DO NOT READ THIS FILE") + return source + + +def response(item, arm, *, valid=True): + body = collect.body_for(item, arm) + usage = {"input_tokens": 3, "output_tokens": 1, "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}} + return {"id": item["id"], "arm": arm, "model": body["model"], "resolved_model": body["model"], + "body_sha256": acquisition.digest(json.dumps(body, sort_keys=True).encode()), + "status": "complete", "terminal_response": True, "execution_unknown": False, + "created_unix": 0, "elapsed_seconds": 0.1, "http_status": 200, "service_tier": "default", + "provider_status": "completed", "usage": usage, "cost": collect.openai_cost(usage, body), + "answer": "orange ball" if valid else None, "valid_answer": valid, + "messages": [{"type": "output_text", "text": '{"answer":"orange ball"}' if valid else "invalid JSON"}]} + + +class AnswerAdapterTests(unittest.TestCase): + def test_question_only_plan_excludes_calibration_final_and_gold(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = source_fixture(root) + items, plan = answers.prepare(source, Path(__file__).with_name("pair.json"), "learn", "1", 20) + self.assertEqual(list(items), ["train", "development"]) + self.assertEqual(plan["roles"], ["train", "development"]) + self.assertFalse(plan["references_loaded_by_collector"]) + self.assertEqual(collect.body_for(items["train"][0], "sol")["max_output_tokens"], 16384) + + def test_resume_never_repeats_a_terminal_bad_format_answer(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = source_fixture(root) + items, plan = answers.prepare(source, Path(__file__).with_name("pair.json"), "calibration", "1", 20) + calls = [] + + def call(item, arm): + calls.append((item["id"], arm)) + return response(item, arm, valid=False) + + output = root / "records" + first = answers.execute(items, plan, output, call=call) + original = {p.relative_to(output).as_posix(): sha(p) for p in output.rglob("*-response.json")} + second = answers.execute(items, plan, output, call=lambda *_: self.fail("Paid terminal response repeated")) + self.assertEqual(len(calls), 2) + self.assertEqual(first["physical_attempts"], second["physical_attempts"]) + self.assertEqual(original, {p.relative_to(output).as_posix(): sha(p) for p in output.rglob("*-response.json")}) + self.assertGreater(float(second["known_list_usd"]), 0) + with self.assertRaisesRegex(ValueError, "resumed collection plan"): + answers.execute(items, {**plan, "budget_usd": "2"}, output, call=call) + + def test_final_is_blocked_before_dispatch_without_exact_frozen_population(self): + with tempfile.TemporaryDirectory() as directory: + source = source_fixture(Path(directory)) + with self.assertRaisesRegex(ValueError, "raw frozen policy bundle"): + answers.prepare(source, Path(__file__).with_name("pair.json"), "final", "1", 20) + plan_path = Path(directory) / "wrong-freeze.json" + write(plan_path, {"format": "public-retrospective-envelope"}) + with self.assertRaises(ValueError): + answers.prepare(source, Path(__file__).with_name("pair.json"), "final", "1", 20, plan_path) + + def test_changed_source_or_pair_cannot_dispatch(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = source_fixture(root) + path = source / "question-only/train-inputs.json" + path.write_text("[]") + with self.assertRaisesRegex(ValueError, "inputs changed"): + answers.prepare(source, Path(__file__).with_name("pair.json"), "learn", "1", 20) + + def test_new_receipts_grade_and_enter_existing_learner_without_rebinding(self): + from embedding import recipes + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = source_fixture(root) + metadata = json.loads((source / "meta.json").read_bytes()) + write(source / "gold.json", [{"id": row["id"], "target": "orange ball"} for row in metadata]) + source_plan = json.loads((source / "source-plan.json").read_bytes()) + source_plan["files_sha256"]["gold.json"] = sha(source / "gold.json") + write(source / "source-plan.json", source_plan) + pair = Path(__file__).with_name("pair.json") + items, plan = answers.prepare(source, pair, "learn", "1", 20) + records = root / "records" + answers.execute(items, plan, records, call=response) + official = root / "official.py" + official.write_text("# Synthetic official adapter; no upstream execution.\n") + fixture = SimpleNamespace(preprocess_sample=lambda value: value.strip(), + preprocess_reference=lambda value: value.strip(), + evaluate_correctness=lambda answer, target: answer.strip() == target.strip()) + with patch.object(grading, "load_official", return_value=fixture): + index = grading.grade(source, records, official, pair, root / "grades") + run = root / "run" + learner_plan = recipes.prepare(source, pair, run) + for role in ("train", "development"): + rows = recipes.role_rows(run, learner_plan, role) + y, costs, document = recipes.grades(learner_plan, rows, root / "grades" / f"grades-{role}.json", role) + self.assertEqual(y.tolist(), [[1, 1]]) + self.assertTrue((costs > 0).all()) + self.assertEqual(document["acquisition_protocol_sha256"], sha(records / "plan.json")) + self.assertEqual(document["grader_sha256"], sha(Path(grading.__file__))) + self.assertEqual(set(index["roles"]), {"train", "development"}) + + +class NativeBaseAdapterTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Direct native scripts resolve their preserved sibling imports this way too. + cls.native_path = str(Path(__file__).parent / "native") + sys.path.insert(0, cls.native_path) + spec = importlib.util.spec_from_file_location("public_base_adapter", Path(cls.native_path) / "base.py") + cls.base = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.base) + + @classmethod + def tearDownClass(cls): + sys.path.remove(cls.native_path) + + def test_cpu_score_uses_only_requested_input_role_and_same_cost_rule(self): + import numpy as np + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write(root / "base.json", {"synthetic": True}) + plan = {"source_plan_sha256": "a" * 64, "pair_sha256": "b" * 64, + "pair": {"models": {"cheap": "gpt-6-luna", "strong": "gpt-6.1-sol"}}, + "native": {"max_len": 8192, "head_max_len": 256}} + rows = [{"id": "c", "role": "calibration", "request": "calibration question"}, + {"id": "f", "role": "final", "request": "final question"}] + stats = {"unit": "USD", "arm_order": ["cheap", "strong"], "class_order": [0, 1], + "means": [[1, 2], [10, 20]], "counts": [[1, 1], [1, 1]]} + with patch.object(self.base, "checked_base", return_value=(plan, rows, {"conditional_cost_usd": stats})), \ + patch.object(self.base.native, "checkpoint_probabilities", return_value=np.array([[0.25, 0.75]])) as scorer: + result = self.base.score(root, root, "calibration", root / "scores.json", "cpu") + self.assertEqual(scorer.call_args.kwargs["device"], "cpu") + self.assertEqual(scorer.call_args.args[1], [rows[0]]) + self.assertEqual(result["ids"], ["c"]) + self.assertEqual(result["methods"]["base"]["expected_usd"], [[1.25, 17.5]]) + self.assertFalse(result["outcomes_accessed"]) + with self.assertRaisesRegex(ValueError, "role/device"): + self.base.score(root, root, "train", root / "bad.json", "cpu") + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/train-a-router/test_reader_selfhost.py b/examples/train-a-router/test_reader_selfhost.py new file mode 100644 index 000000000..1e0d08f4b --- /dev/null +++ b/examples/train-a-router/test_reader_selfhost.py @@ -0,0 +1,62 @@ +"""Reader wiring and one-call local-encoder checks using the actual SDK.""" + +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import httpx +import numpy as np +from transformers import AutoTokenizer + +from embedding import selfhost +import live +import router +from test_selfhost_encode import FakeServer, Tokenizer +import workflow + + +class ReaderSelfHostTests(unittest.TestCase): + def test_reader_encoder_omits_cloud_budget_and_returns_feature_directory(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + args = router.parser().parse_args(["encode", "--data", str(root / "data"), + "--output", str(root / "encoded"), "--self-hosted", "--sie-base-url", "http://127.0.0.1:8000"]) + with patch.object(workflow, "checked_data", return_value=root / "data"), \ + patch.object(workflow, "invoke") as invoke: + result = router.execute(args) + argv = list(map(str, invoke.call_args.args)) + self.assertNotIn("--budget-usd", argv) + self.assertIn("--self-hosted", argv) + self.assertEqual(result["features"], str(root / "encoded/features")) + + def test_live_local_encoder_preserves_one_sdk_receipt_without_cloud_key(self): + original_client = selfhost.make_client + for fail in (False, True): + server = FakeServer(fail=fail) + with self.subTest(failure=fail), tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "encoder" + item = {"id": "synthetic-live", "request": "Synthetic question"} + def factory(base_url, capture): + return original_client(base_url, capture, transport=httpx.MockTransport(server)) + with patch.object(AutoTokenizer, "from_pretrained", return_value=Tokenizer()), \ + patch.object(selfhost, "make_client", side_effect=factory), \ + patch.dict(os.environ, {"SIE_API_KEY": "synthetic-never-forwarded", "SIE_SELF_HOSTED_API_KEY": ""}): + if fail: + with self.assertRaisesRegex(RuntimeError, "Automatic retries disabled"): + live.embed(item, output, base_url="http://127.0.0.1:8000", self_hosted=True) + else: + matrix, fee = live.embed(item, output, base_url="http://127.0.0.1:8000", self_hosted=True) + np.testing.assert_allclose(matrix[0, :2], [.6, .8], atol=1e-7) + self.assertIsNone(fee["encoder_usd"]) + self.assertIsNone(fee["hosting_usd"]) + self.assertTrue((output / "receipt.json").exists()) + self.assertEqual(sum(request.method == "POST" for request in server.requests), 1) + self.assertTrue(all("authorization" not in request.headers for request in server.requests)) + for name in ("intent.json", "response.bin", "http.json"): + self.assertTrue((output / name).exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/train-a-router/test_selfhost_encode.py b/examples/train-a-router/test_selfhost_encode.py new file mode 100644 index 000000000..51cda14dc --- /dev/null +++ b/examples/train-a-router/test_selfhost_encode.py @@ -0,0 +1,114 @@ +"""Exercise the real pinned SDK against an in-memory fake HTTP server.""" + +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import httpx +import numpy as np + +from embedding import recipes, selfhost, wire +from test_public_adapters import source_fixture + + +class Tokenizer: + def __init__(self, n=12): + self.n = n + + def __call__(self, texts, *, truncation): + assert truncation is False + return {"input_ids": [[1] * self.n for _ in texts]} + + +class FakeServer: + def __init__(self, *, wrong_revision=False, fail=False): + self.requests = [] + self.wrong_revision = wrong_revision + self.fail = fail + + def __call__(self, request): + self.requests.append(request) + if request.method == "GET": + return httpx.Response(200, json={"name": recipes.MODEL, "revision": "0" * 40 if self.wrong_revision else recipes.REVISION, + "dims": {"dense": 2560}, "loaded": True}) + if self.fail: + return httpx.Response(503, json={"error": {"code": "MODEL_LOADING", "message": "synthetic loading"}}) + body = wire.unpackb(request.content, numeric_arrays=True) + vectors = [{"id": row["id"], "dense": {"dims": 2560, "values": np.asarray([3.0, 4.0] + [0.0] * 2558, dtype=np.float32)}} + for row in body["items"]] + content = wire.packb({"model": recipes.MODEL, "items": vectors, "usage": {"input_tokens": 12}}, use_bin_type=True) + return httpx.Response(200, content=content, headers={"content-type": "application/msgpack", + "x-sie-request-id": "synthetic-request", "x-sie-model-revision": "a" * 64}) + + def factory(self, base_url, capture): + return selfhost.make_client(base_url, capture, transport=httpx.MockTransport(self)) + + +class SelfHostTests(unittest.TestCase): + def run_fixture(self, directory, server, tokenizer=None): + root = Path(directory) + source = source_fixture(root) + plan = selfhost.prepare(source, "http://127.0.0.1:8080", 60) + binding = {"model": recipes.MODEL, "hf_revision": recipes.REVISION, + "files_sha256": {"synthetic-tokenizer.json": "b" * 64}} + result = selfhost.run(source, root / "encoded", plan, tokenizer or Tokenizer(), binding, + client_factory=server.factory) + return source, root / "encoded", result + + def test_sdk_vectors_enter_existing_learner_and_cloud_key_never_leaves(self): + server = FakeServer() + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {"SIE_API_KEY": "unused-cloud-key", + "SIE_SELF_HOSTED_API_KEY": ""}): + source, output, result = self.run_fixture(directory, server) + self.assertEqual(result["status"], "complete") + self.assertIsNone(result["encoder_cost_usd"]) + self.assertTrue(all("authorization" not in request.headers for request in server.requests)) + posts = [request for request in server.requests if request.method == "POST"] + self.assertEqual(len(posts), 1) + body = wire.unpackb(posts[0].content, numeric_arrays=True) + self.assertEqual(body["params"], {"output_types": ["dense"], "instruction": recipes.INSTRUCTION, + "output_dtype": "float32", "options": {"is_query": True}}) + run = Path(directory) / "run" + plan = recipes.prepare(source, Path(__file__).with_name("pair.json"), run) + for role in recipes.ROLES: + rows = recipes.role_rows(run, plan, role) + matrix, _ = recipes.features(plan, rows, output / "features", role) + self.assertEqual(matrix.shape, (1, 2560)) + self.assertAlmostEqual(float(matrix[0, 0]), 0.6, places=6) + fees = json.loads((output / "unpriced-encoder.json").read_bytes()) + self.assertTrue(all(row["projected_singleton_usd"] is None for row in fees["rows"])) + self.assertFalse((output / "fee-projection.json").exists()) + + def test_wrong_revision_stops_before_any_encode_call(self): + server = FakeServer(wrong_revision=True) + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, "ready pinned model"): + self.run_fixture(directory, server) + self.assertFalse(any(request.method == "POST" for request in server.requests)) + + def test_failed_physical_attempt_is_preserved_and_not_retried(self): + server = FakeServer(fail=True) + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(Exception): + self.run_fixture(directory, server) + self.assertEqual(sum(request.method == "POST" for request in server.requests), 1) + output = Path(directory) / "encoded" + self.assertTrue((output / "batches/000000/intent.json").is_file()) + self.assertTrue((output / "batches/000000/response.bin").is_file()) + terminal = json.loads((output / "terminal.json").read_bytes()) + self.assertEqual(terminal["status"], "incomplete") + self.assertEqual(terminal["unresolved_batch"], "batches/000000") + + def test_context_overflow_precedes_even_catalog_request(self): + server = FakeServer() + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, "context overflow"): + self.run_fixture(directory, server, Tokenizer(32769)) + self.assertEqual(server.requests, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/train-a-router/test_utf8.py b/examples/train-a-router/test_utf8.py new file mode 100644 index 000000000..80a8363da --- /dev/null +++ b/examples/train-a-router/test_utf8.py @@ -0,0 +1,182 @@ +"""Offline Unicode regressions for the reader boundary around frozen recipes.""" + +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +SAMPLE = "Linguini: ə ʃ ŋ — 日本語" +ASCII_ENV = { + **os.environ, + "LC_ALL": "C", + "LANG": "C", + "PYTHONCOERCECLOCALE": "0", + "PYTHONUTF8": "0", + "PYTHONPATH": str(ROOT), +} + + +def run_python(*arguments): + return subprocess.run( + [sys.executable, "-X", "utf8=0", *map(str, arguments)], + cwd=ROOT, + env=ASCII_ENV, + capture_output=True, + timeout=30, + check=False, + ) + + +def write_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf-8")) + + +class Utf8ReaderTests(unittest.TestCase): + def assert_success(self, result): + self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", errors="replace")) + + def probe(self, root, *, bootstrap): + path = root / "probe.py" + path.write_text( + "import json, sys\n" + "from pathlib import Path\n" + "from reader_runtime import ensure_utf8\n" + "from embedding import recipes\n" + "from native import recipe\n" + + ("ensure_utf8()\n" if bootstrap else "") + + "output, request = Path(sys.argv[1]), sys.argv[2]\n" + "value = {'request': request}\n" + "for name, module in [('embedding', recipes), ('native', recipe)]:\n" + " path = output / (name + '.json')\n" + " module.write(path, value)\n" + " assert module.read(path) == value\n" + "print(json.dumps({'utf8': sys.flags.utf8_mode, 'options': sys._xoptions,\n" + " 'warnings': sys.warnoptions, 'args': sys.argv[1:],\n" + " 'unbuffered': sys.stdout.write_through}))\n", + encoding="utf-8", + ) + return path + + def assert_probe_bytes(self, output): + expected = (json.dumps({"request": SAMPLE}, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode() + for name in ("embedding", "native"): + self.assertEqual((output / f"{name}.json").read_bytes(), expected) + + def test_ascii_parent_reproduces_frozen_reader_and_writer_failure(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "sample.json" + write_json(path, {"request": SAMPLE}) + result = run_python( + "-c", + "import codecs, locale, sys\n" + "from native import recipe\n" + "assert sys.flags.utf8_mode == 0\n" + "assert codecs.lookup(locale.getpreferredencoding(False)).name == 'ascii'\n" + "try:\n" + " recipe.read(sys.argv[1])\n" + "except UnicodeDecodeError:\n" + " pass\n" + "else:\n" + " raise AssertionError('Expected an ASCII decode failure')\n" + "try:\n" + " recipe.write(sys.argv[2], {'request': '\\u0259 \\u0283 \\u014b'})\n" + "except UnicodeEncodeError:\n" + " pass\n" + "else:\n" + " raise AssertionError('Expected an ASCII encode failure')\n", + path, + path.with_name("failed-write.json"), + ) + self.assert_success(result) + + def test_startup_preserves_flags_unicode_arguments_and_utf8_bytes(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + output = root / "ə output" + result = run_python( + "-u", "-W", "error", "-X", "reader_test=retained", self.probe(root, bootstrap=True), output, SAMPLE + ) + self.assert_success(result) + receipt = json.loads(result.stdout) + self.assertEqual(receipt["utf8"], 1) + self.assertEqual(receipt["options"]["reader_test"], "retained") + self.assertIn("error", receipt["warnings"]) + self.assertTrue(receipt["unbuffered"]) + self.assertEqual(receipt["args"], [str(output), SAMPLE]) + self.assert_probe_bytes(output) + + def test_workflow_child_uses_utf8_even_when_parent_does_not(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + output = root / "result" + result = run_python( + "-c", + "import sys, workflow\nassert sys.flags.utf8_mode == 0\nworkflow.invoke(*sys.argv[1:])\n", + self.probe(root, bootstrap=False), + output, + SAMPLE, + ) + self.assert_success(result) + self.assertEqual(json.loads(result.stdout)["utf8"], 1) + self.assert_probe_bytes(output) + + def test_actual_cli_reads_unicode_with_file_module_and_separator_invocations(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + candidate = { + "candidate_id": 0, + "policy": {"mode": "always_strong"}, + "calibration": { + "answer_accuracy_percent": 80, + "gap_vs_strong_pp": 0, + "answer_cost_usd_per10000": 1, + "total_cost_usd_per10000": 1, + "cheap_share": 0, + }, + } + write_json( + root / "calibration-frontiers.json", + { + "format": "bbeh-calibration-frontiers-v1", + "final_outcomes_accessed": False, + "ids": ["synthetic"], + "methods": { + SAMPLE: { + "no_preferred_quality_target": True, + "metadata_assisted": False, + "candidates": [candidate], + "frontier": [candidate], + } + }, + }, + ) + for index, command in enumerate(((ROOT / "router.py",), ("-m", "router"), ("--", ROOT / "router.py"))): + with self.subTest(command=command): + output = root / f"ə view {index}.html" + result = run_python(*command, "frontier", "--run", root, "--output", output) + self.assert_success(result) + self.assertEqual(json.loads(result.stdout)["open"], str(output)) + self.assertIn(SAMPLE.encode("utf-8"), output.read_bytes()) + + def test_cli_retains_argument_error_exit_status(self): + result = run_python(ROOT / "router.py", "--invalid-reader-option") + self.assertEqual(result.returncode, 2) + self.assertIn(b"usage:", result.stderr) + + def test_frozen_source_bytes_still_match_the_manifest(self): + manifest = json.loads((ROOT / "frozen-source-manifest.json").read_bytes()) + for name, expected in manifest["files"].items(): + with self.subTest(path=name): + data = (ROOT / name).read_bytes() + self.assertEqual(len(data), expected["bytes"]) + self.assertEqual(hashlib.sha256(data).hexdigest(), expected["sha256"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/train-a-router/test_workflow.py b/examples/train-a-router/test_workflow.py new file mode 100644 index 000000000..b552d4a30 --- /dev/null +++ b/examples/train-a-router/test_workflow.py @@ -0,0 +1,319 @@ +"""Tiny independent public-reader checks; no provider, model or real outcome calls.""" + +import ast +import copy +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np + +import calibration_view +import cost_reader +from embedding import recipes as core +from embedding.cost_iteration import study +import live +import openai_population +import workflow + + +def replace_json(path, value): + if path.exists(): + path.unlink() + core.write(path, value) + + +def make_run(root): + """Construct genuine hash-bound tiny recipe/grade files, not published outcomes.""" + run = root / "run" + run.mkdir() + pair = {"models": {"cheap": "gpt-6-luna", "strong": "gpt-6.1-sol"}, + "cost_scope": "Synthetic known selected-response USD"} + core.write(run / "pair.json", pair) + plan = {"format": "bbeh-recipe-plan-v1", "protocol": core.PROTOCOL, + "source_plan_sha256": "a" * 64, "pair_sha256": core.sha(run / "pair.json"), + "pair": pair, "recipe_source_sha256": core.sha(core.__file__), "roles": {}} + rows, paths, dollars = {}, {}, {} + for role, n in (("train", 4), ("development", 3), ("calibration", 2), ("final", 2)): + rows[role] = [{"id": f"{role}-{i}", "role": role, "task": "synthetic", + "family": f"family-{role}-{i}", "dataset": "bbeh", + "request": f"Shared arithmetic question repeat {i} plus {i + 1}", + "response_contract": {"type": "exact_string", "max_length": 2048}} + for i in range(n)] + path = run / f"rows-{role}.json" + core.write(path, rows[role]) + plan["roles"][role] = {"n": n, "rows_sha256": core.sha(path), "inputs_sha256": "c" * 64} + if role in {"train", "development"}: + dollars[role] = np.array([[0.01 + i * 0.003, 0.08 + i * 0.01] for i in range(n)]) + paths[role] = root / f"grades-{role}.json" + grade = {"format": "bbeh-paired-grades-v1", "role": role, + "source_plan_sha256": plan["source_plan_sha256"], "pair_sha256": plan["pair_sha256"], + "models": pair["models"], "cost_scope": pair["cost_scope"], + "grader_sha256": "d" * 64, "acquisition_protocol_sha256": "e" * 64, + "rows": [{"id": row["id"], "cheap_correct": i % 2, + "strong_correct": (i + 1) % 2, "cheap_record_sha256": "f" * 64, + "strong_record_sha256": "0" * 64, + "cheap_cost_usd": str(dollars[role][i, 0]), + "strong_cost_usd": str(dollars[role][i, 1])} + for i, row in enumerate(rows[role])]} + core.write(paths[role], grade) + core.write(run / "plan.json", plan) + quality = {"plan_sha256": core.sha(run / "plan.json"), + "train_grades_sha256": core.sha(paths["train"]), + "development_grades_sha256": core.sha(paths["development"]), + "calibration_or_final_outcomes_accessed": False, + "different_student_fit_metadata": "This intentionally is not the original model hash"} + core.write(run / "model.json", quality) + for role in ("calibration", "final"): + core.write(run / f"scores-{role}.json", { + "format": "bbeh-input-only-scores-v1", "role": role, + "ids": [r["id"] for r in rows[role]], "source_plan_sha256": plan["source_plan_sha256"], + "pair_sha256": plan["pair_sha256"], "model_sha256": core.sha(run / "model.json"), + "outcomes_accessed": False, + "methods": {"embedding": {"probabilities": [[0.2, 0.8], [0.8, 0.4]], + "expected_usd": [[0.01, 0.08], [0.02, 0.09]], + "router_kind": "embedding", "metadata_assisted": False}}}) + return run, plan, rows, paths, dollars + + +class CostReaderTests(unittest.TestCase): + def test_student_model_hash_and_math_parity_without_future_grades(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + run, _, rows, paths, dollars = make_run(root) + old = {role: core.read(run / f"scores-{role}.json") for role in ("calibration", "final")} + actual_read, reads = core.read, [] + + def recorded_read(path): + reads.append(str(path)) + self.assertNotIn("grades-calibration", str(path)) + self.assertNotIn("grades-final", str(path)) + return actual_read(path) + + with patch.object(core, "read", side_effect=recorded_read): + receipt = cost_reader.fit_score(run, paths["train"], paths["development"], root / "cost") + self.assertEqual(receipt["fit_n"], 7) + self.assertEqual(receipt["alpha"], 0.1) + model = actual_read(root / "cost/cost-model.json") + self.assertTrue(model["reproduction"]) + self.assertFalse(model["new_blind_evaluation"]) + self.assertEqual(model["quality_model_sha256"], core.sha(run / "model.json")) + _, matrix, text = core.text_matrix(rows["train"] + rows["development"]) + expected = study.fit_ridge(matrix, np.concatenate((dollars["train"], dollars["development"])), 0.1) + self.assertEqual(text, model["text_features"]) + for name in ("weights", "bias", "train_residual_smearing"): + np.testing.assert_allclose(model["models"]["text_length_log_ridge"][name], expected[name], + rtol=1e-12, atol=1e-12) + for role in ("calibration", "final"): + prediction = actual_read(root / f"cost/costs-{role}.json") + augmented = actual_read(root / f"cost/scores-{role}.json") + self.assertEqual(augmented["methods"]["embedding"], old[role]["methods"]["embedding"]) + np.testing.assert_allclose(prediction["expected_usd"], + study.predict_rows(model, rows[role], "text_length_log_ridge"), rtol=1e-12, atol=1e-12) + self.assertEqual(augmented["methods"]["embedding__textcost"]["expected_usd"], prediction["expected_usd"]) + self.assertTrue(any("rows-final.json" in path for path in reads)) + + def test_changed_grade_binding_is_rejected_before_fit(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + run, _, _, paths, _ = make_run(root) + grade = core.read(paths["train"]) + grade["rows"][0]["cheap_cost_usd"] = "0.012" + replace_json(paths["train"], grade) + with patch.object(study, "fit_ridge") as fitted, self.assertRaisesRegex(ValueError, "lineage"): + cost_reader.fit_score(run, paths["train"], paths["development"], root / "cost") + fitted.assert_not_called() + + def test_future_score_wrong_role_is_rejected(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + run, _, _, paths, _ = make_run(root) + score = core.read(run / "scores-calibration.json") + score["role"] = "final" + replace_json(run / "scores-calibration.json", score) + with self.assertRaisesRegex(ValueError, "source-role-order"): + cost_reader.fit_score(run, paths["train"], paths["development"], root / "cost") + + +class ReaderUiAndCallTests(unittest.TestCase): + @unittest.skipUnless(shutil.which("node"), "Node is needed for actual generated-script parsing") + def test_calibration_html_emits_valid_javascript(self): + path = Path(calibration_view.__file__) + tree = ast.parse(path.read_text()) + template = next(ast.literal_eval(node.value) for node in tree.body + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "TEMPLATE" for target in node.targets)) + script = template.split("", 1)[0].replace("__DATA__", "[]") + result = subprocess.run(["node", "--check", "-"], input=script, text=True, capture_output=True, check=False) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_wrong_returned_model_or_tier_not_presented_as_success_and_never_retried(self): + for resolved_model, tier in (("not-the-selected-model", "default"), ("gpt-6-luna", "flex")): + with self.subTest(model=resolved_model, tier=tier), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + response = {"status": "complete", "terminal_response": True, "model": "gpt-6-luna", + "resolved_model": resolved_model, "service_tier": tier, "answer": "synthetic", + "valid_answer": True} + with patch.dict(os.environ, {"OPENAI_API_KEY": "synthetic-unused"}), \ + patch.object(live, "selected_policy", return_value=({"mode": "always_cheap"}, "a" * 64, "b" * 64)), \ + patch.object(openai_population, "acquire", return_value=response) as acquire: + try: + result = live.answer(root / "run", "Synthetic question", root / "answer", + method="embedding", candidate_id=0) + except ValueError: + pass + else: + self.assertNotEqual(result["status"], "complete") + acquire.assert_called_once() + self.assertEqual(core.read(root / "answer/answer.json"), response) + + def test_replay_numeric_tolerance_keeps_structure_and_large_drift_strict(self): + expected = {"quality": [0.123456789], "count": 7, "nested": {"ok": True}} + rounded = copy.deepcopy(expected) + rounded["quality"][0] += 1e-12 + self.assertTrue(workflow.same_values(expected, rounded)) + rounded["quality"][0] += 1e-5 + self.assertFalse(workflow.same_values(expected, rounded)) + self.assertFalse(workflow.same_values(expected, {**expected, "extra": 1})) + self.assertFalse(workflow.same_values(expected, {**expected, "count": 7.0})) + + +class CalibrationSelectionTests(unittest.TestCase): + def fixture(self, root): + embedding, plan, _, _, _ = make_run(root) + run = root / "student" + run.mkdir() + shutil.move(embedding, run / "embedding") + candidate = {"candidate_id": 2, "policy": {"mode": "always_cheap"}, "calibration": {}} + cal = {"format": "bbeh-calibration-frontiers-v1", "final_outcomes_accessed": False, + "plan_sha256": core.sha(run / "embedding/plan.json"), + "source_plan_sha256": plan["source_plan_sha256"], "pair_sha256": plan["pair_sha256"], + "methods": {"embedding": {"model_sha256": "b" * 64, "metadata_assisted": False, + "candidates": [candidate], "frontier": [candidate]}}} + core.write(run / "internal/calibration-frontiers.json", cal) + frozen = {key: cal[key] for key in ("plan_sha256", "source_plan_sha256", "pair_sha256")} + frozen.update(calibration_sha256=core.sha(run / "internal/calibration-frontiers.json"), + methods={"embedding": {"model_sha256": "b" * 64, "candidates": [candidate]}}) + core.write(run / "internal/frozen-frontiers.json", frozen) + core.write(run / "reproduction.json", {"artifact_directory": str(root / "data")}) + self.bind_envelope(run) + return run + + def bind_envelope(self, run): + replace_json(run / "calibration.json", { + "new_blind_evaluation": False, + "reproduction_sha256": core.sha(run / "reproduction.json"), + "calibration_sha256": core.sha(run / "internal/calibration-frontiers.json"), + "frozen_sha256": core.sha(run / "internal/frozen-frontiers.json")}) + + def test_own_calibration_and_stale_envelope(self): + with tempfile.TemporaryDirectory() as temporary: + run = self.fixture(Path(temporary)) + with patch.object(workflow, "checked_run") as checked: + policy, model_sha, _ = live.selected_policy(run, "embedding", 2) + self.assertEqual(policy, {"mode": "always_cheap"}) + self.assertEqual(model_sha, "b" * 64) + checked.assert_called_once_with(str(Path(temporary) / "data"), run.resolve()) + cal = core.read(run / "internal/calibration-frontiers.json") + cal["pair_sha256"] = "f" * 64 + replace_json(run / "internal/calibration-frontiers.json", cal) + with self.assertRaisesRegex(ValueError, "Reproduction calibration changed"): + live.selected_policy(run, "embedding", 2) + + def test_rebound_envelope_cannot_hide_pair_mismatch(self): + with tempfile.TemporaryDirectory() as temporary: + run = self.fixture(Path(temporary)) + frozen = core.read(run / "internal/frozen-frontiers.json") + frozen["pair_sha256"] = "f" * 64 + replace_json(run / "internal/frozen-frontiers.json", frozen) + self.bind_envelope(run) + with patch.object(workflow, "checked_run"), self.assertRaisesRegex(ValueError, "source/pair/plan"): + live.selected_policy(run, "embedding", 2) + + def test_unknown_or_metadata_only_choice_is_rejected(self): + with tempfile.TemporaryDirectory() as temporary: + run = self.fixture(Path(temporary)) + with patch.object(workflow, "checked_run"), self.assertRaisesRegex(ValueError, "Unknown candidate"): + live.selected_policy(run, "embedding", 99) + cal = core.read(run / "internal/calibration-frontiers.json") + cal["methods"]["embedding"]["metadata_assisted"] = True + replace_json(run / "internal/calibration-frontiers.json", cal) + frozen = core.read(run / "internal/frozen-frontiers.json") + frozen["calibration_sha256"] = core.sha(run / "internal/calibration-frontiers.json") + replace_json(run / "internal/frozen-frontiers.json", frozen) + self.bind_envelope(run) + with patch.object(workflow, "checked_run"), self.assertRaisesRegex(ValueError, "benchmark metadata"): + live.selected_policy(run, "embedding", 2) + + +class WorkflowIntegrationTests(unittest.TestCase): + def test_student_cost_calibration_and_live_binding_use_own_model_and_no_final_answers(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + embedding, plan, rows, grade_paths, _ = make_run(root) + run, data = root / "student", root / "data" + run.mkdir() + shutil.move(embedding, run / "embedding") + data.mkdir() + core.write(data / "manifest.json", {"synthetic": True}) + core.write(data / "reference/frozen-frontiers.json", {"synthetic_reference": True}) + for role, path in grade_paths.items(): + (data / "grades").mkdir(exist_ok=True) + shutil.copyfile(path, data / f"grades/grades-{role}.json") + calibration_grade = core.read(grade_paths["train"]) + calibration_grade["role"] = "calibration" + calibration_grade["rows"] = [ + {**calibration_grade["rows"][i], "id": row["id"]} + for i, row in enumerate(rows["calibration"])] + core.write(data / "grades/grades-calibration.json", calibration_grade) + core.write(data / "fees.json", { + "source_plan_sha256": plan["source_plan_sha256"], "model": core.MODEL, + "hf_revision": core.REVISION, "instruction": core.INSTRUCTION, + "cost_basis": "validated_singleton_projection", + "rows": [{"id": row["id"], "projected_singleton_usd": "0.00001"} + for role in ("calibration", "final") for row in rows[role]]}) + core.write(run / "reproduction.json", { + "format": "bbeh-public-reproduction-v1", "artifact_directory": str(data), + "manifest_sha256": core.sha(data / "manifest.json"), + "reference_frozen_sha256": core.sha(data / "reference/frozen-frontiers.json"), + "new_blind_evaluation": False}) + for role in ("calibration", "final"): + native = core.read(run / f"embedding/scores-{role}.json") + native["model_sha256"] = "f" * 64 + native["methods"] = {name: {**copy.deepcopy(native["methods"]["embedding"]), + "router_kind": "native"} for name in ("base", "lora")} + core.write(run / f"scores/native-{role}.json", native) + actual_read = core.read + + def no_final_grades(path): + self.assertNotIn("grades-final", str(path)) + return actual_read(path) + + with patch.object(workflow, "checked_data", side_effect=lambda path: Path(path).resolve()), \ + patch.object(core, "read", side_effect=no_final_grades): + cost = workflow.fit_cost(data, run) + self.assertEqual(cost["alpha"], 0.1) + receipt = workflow.calibrate(data, run) + self.assertEqual(receipt["new_model_calls"], 0) + calibration = actual_read(run / "internal/calibration-frontiers.json") + method = calibration["methods"]["embedding__textcost"] + candidate = method["frontier"][0] + chosen, model_sha, _ = live.selected_policy(run, "embedding__textcost", candidate["candidate_id"]) + self.assertEqual(chosen, candidate["policy"]) + quality, model = live.binding(run, "embedding__textcost", model_sha) + self.assertEqual(quality, "embedding") + self.assertEqual(model, actual_read(run / "embedding/model.json")) + self.assertFalse(receipt["new_model_calls"]) + self.assertFalse(actual_read(run / "calibration.json")["new_blind_evaluation"]) + self.assertFalse((data / "grades/grades-final.json").exists()) + frozen = actual_read(run / "internal/frozen-frontiers.json") + self.assertEqual(set(frozen["methods"]), + {"embedding", "embedding__textcost", "base", "lora", "base__textcost", "lora__textcost"}) + self.assertEqual(frozen["ids"], [row["id"] for row in rows["final"]]) + +if __name__ == "__main__": + unittest.main() diff --git a/examples/train-a-router/uv.lock b/examples/train-a-router/uv.lock new file mode 100644 index 000000000..e353d2382 --- /dev/null +++ b/examples/train-a-router/uv.lock @@ -0,0 +1,1101 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" + +[[package]] +name = "accelerate" +version = "1.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/b5/1d3ed029ac71d3f2961346829a268da923698e9fd63f218f78841f216bfd/accelerate-1.15.0.tar.gz", hash = "sha256:5654f8c5eaa0d4fa68b33e287a97765da6849bf6d51dcac874e73fbbddfb6134", size = 422615, upload-time = "2026-09-09T13:04:49.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/4c/34f0450479d01195027260da68d8a3880683f1640c3ca5adf64acb3185f1/accelerate-1.15.0-py3-none-any.whl", hash = "sha256:97eacca0b73e45cb867dbf8c5d5d4dc32219544300e0c8992c7334dc2ef33cec", size = 394295, upload-time = "2026-09-09T13:04:47.331Z" }, +] + +[[package]] +name = "aiohappyeyeballs" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.14.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "typing-extensions" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/4c/bdccd81e9ee225b69c60e7766c9a5b05364f118f4d383713b89a682d772d/aiohttp-3.14.5.tar.gz", hash = "sha256:5558a7f5a05af9ecf744af91e5baefc436f93c9333e656c27ec253f9a6bbe178", size = 8078052, upload-time = "2026-10-11T01:05:12.408Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/94/6ba86efddcb616c811b40e6a0dfdd862738f647e4e3860a961075d5e9ed8/aiohttp-3.14.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df37b620684e19b5e25724412518ccafc3b1a49cdac706fdbd2f983fad943450", size = 817499, upload-time = "2026-10-11T01:00:18.901Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e1/7bca6d84dabd228aa8eb4b7f9feac2586aaa9be5505d7d65bf287a615c43/aiohttp-3.14.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ef60869969180ec2464f1349aff07138ae35ca2200f0946cb3552e49e8f301a8", size = 549467, upload-time = "2026-10-11T01:00:20.854Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/11b89f45ca486252dd67dd5f3231fec04bf5518da39a95cb3997619f17fb/aiohttp-3.14.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d079c0a0135c36e7beb6f1c88087c8f108dc5891cdd0b5eafa778421bda70ed2", size = 548067, upload-time = "2026-10-11T01:00:22.659Z" }, + { url = "https://files.pythonhosted.org/packages/22/ff/c6615806c14aab34f82b9424ccde8ce6e417315fd57ce1c5b4d4747888e1/aiohttp-3.14.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abfda5cb094a829f7bc25216a32f7db2e85cc65bd59910f8e7b40b3d9b224764", size = 1896245, upload-time = "2026-10-11T01:00:24.638Z" }, + { url = "https://files.pythonhosted.org/packages/39/b2/25a8c971ae6a92c8394d77e42422d7f38989e05cf41a5ceb92d73d67ab7e/aiohttp-3.14.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9cc882cf8619109583c906b4d4a85d6a111a98afa34b7a450d1e08118d016820", size = 1872180, upload-time = "2026-10-11T01:00:26.838Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d5/99f93ea36cc5205e47c1e5a803e087f2ad21b5430b5db2e942cb6e988a37/aiohttp-3.14.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7457580535e019e1247ea35d6a02bf081ad30c26d0cbc210c93f6c3ab67a0835", size = 1937261, upload-time = "2026-10-11T01:00:28.74Z" }, + { url = "https://files.pythonhosted.org/packages/40/a6/9ac9c9e6695040bd73d2584a1b59a9388f6433c76d5294a7bf591e21ffe5/aiohttp-3.14.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c5ed596aedb9c42afd3fe0aae3117725378ac73d2cc5ddc735056fbdb96c5d02", size = 2064994, upload-time = "2026-10-11T01:00:30.623Z" }, + { url = "https://files.pythonhosted.org/packages/da/e4/aa172eb534b7f02f1f8ff1c3213347eaf3cf218db91a727c6863c22f1035/aiohttp-3.14.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20f085697d7e911f1f73c43ed03fafbed1e7121797e2eb5428efa80398060584", size = 1919630, upload-time = "2026-10-11T01:00:32.548Z" }, + { url = "https://files.pythonhosted.org/packages/06/7d/4eedafc5bababa8932636c141e346806966eade12c0b7e5946d43bf8218b/aiohttp-3.14.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:74b0a9c8270f9b0a11410e124ff8d4f18bfc1f1837440ec84da5ae7b50927b5d", size = 1710016, upload-time = "2026-10-11T01:00:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/b2/94/eee018537ba19da0ceb2ac79cab83faed4ac49568e08376e2799043f2538/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:19e2ba471507c34f8252402ab50f5ab512398b9ea8c8f1cb26beb3f75793ba30", size = 1858112, upload-time = "2026-10-11T01:00:36.581Z" }, + { url = "https://files.pythonhosted.org/packages/68/76/354653a306547238f3427283905972d796ba7c292ba9977abec9f2b6f260/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d418ce2af40c6bb685b3f663e9e8de27cb0a22431d8e88a167348d7f01878073", size = 1896371, upload-time = "2026-10-11T01:00:38.478Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ec/63e8c7136b570e356345ad3174e3820fdc973ea10712cb6c649bf875755a/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:70cb4008ac2ed1e0ca9e824deb4b53d3aa0d939109698ebf1e723a84337bd794", size = 1905836, upload-time = "2026-10-11T01:00:40.527Z" }, + { url = "https://files.pythonhosted.org/packages/57/d8/11365bda144b127928cd42533d0eff78a55613c9e28f81941bd6630ea887/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a23fe35d776bc03cb495938b9594450d047e3bc08c5255315a82323e9cb7d2dd", size = 1696534, upload-time = "2026-10-11T01:00:42.686Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3d/82df0461b18e00b2998f205c03e0d3010222478c43640aceb8e03dcd7e8f/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3e0eb43bed3c6801a6cee315195377789e90b2a72c2277a475b578535312488d", size = 1933970, upload-time = "2026-10-11T01:00:44.71Z" }, + { url = "https://files.pythonhosted.org/packages/03/ad/6ddfe0aacd931c17b53533336d97e9d11a98b96d6ae815a9da0b19f82ccf/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be7dd397d64ca3e1869626fa9318aaebb54b7bf93bc72d7a205448d83e4f748", size = 1899501, upload-time = "2026-10-11T01:00:46.629Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8e/189bdd9ae4793059bb09f6dc880f211a6c6c7859cebcbf33912dbfab7dd7/aiohttp-3.14.5-cp312-cp312-win32.whl", hash = "sha256:eb324e2009fb54db30a071dad7caf6998ee2879c4704007efb244514dad1fec1", size = 480370, upload-time = "2026-10-11T01:00:48.468Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ce/1f08114679d49655b30a6e0a29858375c94b82c1de1a0bd0a20c2fee8b02/aiohttp-3.14.5-cp312-cp312-win_amd64.whl", hash = "sha256:2cc38a4f2b516bef1714e690df87a0e043faf1a7693c82d860091684453d5111", size = 506141, upload-time = "2026-10-11T01:00:50.272Z" }, + { url = "https://files.pythonhosted.org/packages/79/d4/c7b4f60b16a1b7e43249fa9031ae05e7e8c341ba4b7d1866f914dafeaa0e/aiohttp-3.14.5-cp312-cp312-win_arm64.whl", hash = "sha256:a63afd1f757de949028387e65a7127b61ad0f775432dbb0e62816ae619fe69ac", size = 488108, upload-time = "2026-10-11T01:00:52.321Z" }, + { url = "https://files.pythonhosted.org/packages/68/30/173960c42b05a6c59f7558e4b12a4b0d9ba376cf6aa9bde7f9e08a30ca8d/aiohttp-3.14.5-py3-none-any.whl", hash = "sha256:efc21a454892828368b11c2c780de0ff8bc991f73f6b99c6b66e56205470929b", size = 279517, upload-time = "2026-10-11T01:05:08.523Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + +[[package]] +name = "anyio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "certifi" +version = "2026.10.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "filelock" +version = "4.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/28/3893324bcadf8f89b1616637468d766af3f625008c0735bf704fdfa86617/filelock-4.2.4.tar.gz", hash = "sha256:c6d8663535e1650ee430507b4ffbac7b464dd64b7647ea0112a9f406499d163d", size = 652444, upload-time = "2026-10-19T01:57:51.001Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/1b/5e74289a1878e666bb4b65fe5805a04cafcac2aa5f519ef1ead67ef91c56/filelock-4.2.4-py3-none-any.whl", hash = "sha256:301241438c57936ef93f4243a94f2559022268bfcf9b375a9a72eb0f8db8358e", size = 162475, upload-time = "2026-10-19T01:57:49.419Z" }, +] + +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/cd/9be253869fc42e764de7f3dedd6969af7d44ff9c3375214a3442a6f3fc08/fsspec-2026.9.0.tar.gz", hash = "sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe", size = 333545, upload-time = "2026-09-18T17:50:42.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/c0/a98505f18594f1bce828bb159cec0fcf9860562f1a2c85913409fc8f3d9e/fsspec-2026.9.0-py3-none-any.whl", hash = "sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f", size = 221738, upload-time = "2026-09-18T17:50:41.341Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/27/06d899ea7bd721d272f84aac98bdb238de98af4cc767a69056d967d68c71/hf_xet-1.7.0.tar.gz", hash = "sha256:d406ec79053c0871817f700c2ac8c36ba0d87f9c34b7458b0f0063bb218b0466", size = 985689, upload-time = "2026-10-06T20:18:43.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/0b/b03be21ffaada749ba0d3197d8aefbf1aa698bac149580421c15239b299e/hf_xet-1.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e3e88a7a75d7d95cbee1f37dc31341d6201124cf21c6c4b1dfab8ccba9b09e0f", size = 3796096, upload-time = "2026-10-06T20:18:28.43Z" }, + { url = "https://files.pythonhosted.org/packages/c3/47/a26ebdce7056a61e931f228439bc0ab08cbec239d1690f965e5e637cba79/hf_xet-1.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:59fba37039233c7fcbe196817d6cdcf1b40dfb17b410f229d85b0cf0a1848da4", size = 3560352, upload-time = "2026-10-06T20:18:30.365Z" }, + { url = "https://files.pythonhosted.org/packages/a3/4c/2bf3b66c215d409655f28de1622393dde04c9461280d48c7924bb3b2decd/hf_xet-1.7.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2814a6e999d13464c4d679b788cc5d784eb5a4edfc638a31f10e9a11ab531ef8", size = 4212180, upload-time = "2026-10-06T20:18:32.292Z" }, + { url = "https://files.pythonhosted.org/packages/49/0c/a2f703a5a78267556e89e03316fa0805c86b72b50829bc67665746e8ebf0/hf_xet-1.7.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:fcfd6c22418e57dd5b3aea649e813b2e2cfb2aebf317b210d90f1fe4b3018b52", size = 3990011, upload-time = "2026-10-06T20:18:34.21Z" }, + { url = "https://files.pythonhosted.org/packages/a4/77/e52e4201b1cbf571530a61cc57f70182045a39a230089ee5f1df182a4de2/hf_xet-1.7.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80f79dae613ce9e0ea1fd1ae15616ca9ac74aed4c770aabc199c4f03ebecc863", size = 4190628, upload-time = "2026-10-06T20:18:36.062Z" }, + { url = "https://files.pythonhosted.org/packages/6c/dc/03a21b89f118664a0926ff25b0f8e44a519bf22724a6a8fc7a9abbc188b6/hf_xet-1.7.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0a9e802f33bf50c851abe45fc5380e61f959e2d369647d6742b79ad9d6c27cab", size = 4418814, upload-time = "2026-10-06T20:18:37.888Z" }, + { url = "https://files.pythonhosted.org/packages/4d/59/b35106dfa71b6eef605dc88bd038fe99c7f86fb132a15b60d0bf2f235b2c/hf_xet-1.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:2b7bb5727889b0f2436dbaaad8fc4c3e66b8240d992716989e0c086b4278b1bc", size = 3822644, upload-time = "2026-10-06T20:18:40.052Z" }, + { url = "https://files.pythonhosted.org/packages/48/cd/072313585f74fe9d441e2eb5e0a4703c30586cd709810ea369675f61b74e/hf_xet-1.7.0-cp38-abi3-win_arm64.whl", hash = "sha256:acc3851cf2576a8fb2ae926da863f4efabe21303cf292e9a44332802ab0dcc6a", size = 3662436, upload-time = "2026-10-06T20:18:42.205Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "0.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/63/4910c5fa9128fdadf6a9c5ac138e8b1b6cee4ca44bf7915bbfbce4e355ee/huggingface_hub-0.36.0.tar.gz", hash = "sha256:47b3f0e2539c39bf5cde015d63b72ec49baff67b6931c3d97f3f84532e2b8d25", size = 463358, upload-time = "2025-10-23T12:12:01.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/bd/1a875e0d592d447cbc02805fd3fe0f497714d6a2583f59d14fa9ebad96eb/huggingface_hub-0.36.0-py3-none-any.whl", hash = "sha256:7bcc9ad17d5b3f07b57c78e79d527102d08313caa278a641993acddcb894548d", size = 566094, upload-time = "2025-10-23T12:11:59.557Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "joblib" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cloudpickle" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/1d/537ab090f302b838943a1b56497dd53059b9a9b46a074936470173a2e207/joblib-1.6.0.tar.gz", hash = "sha256:2ccc96785b12046c08fd6d55839c12857831b54a3c1673ffadd2f04bfc4eda03", size = 327903, upload-time = "2026-08-31T09:39:04.122Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/53/84099323c2ec4be98d935f63c033ac4151ee83836ca1050ede3b3aadf155/joblib-1.6.0-py3-none-any.whl", hash = "sha256:3dbbf9f6e4b592a2357b854608e980fe6390d131d7a82f011a377ef2ebef7aba", size = 306115, upload-time = "2026-08-31T09:39:02.298Z" }, +] + +[[package]] +name = "laya" +version = "0.3.29" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "safetensors" }, + { name = "torch" }, + { name = "transformers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/2e/9dcc0e424ea8f9e6c0cbcc91d6dbf4a0e30aba33021517eebf2a5dda286d/laya-0.3.29.tar.gz", hash = "sha256:7a3c64163be195c8ba6ca65c9cf6abb439de3d70d890900f80f3d0e923e6d0b4", size = 1082279, upload-time = "2026-10-07T10:14:53.095Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/56/c82db3e34fe507ab6cda40ee5b8357ad5e30c1196e887ea7351896c96cde/laya-0.3.29-py3-none-any.whl", hash = "sha256:41a51775eae4bd624688f13425d71ce66aa0f6b9a49a922a775a92dc4a85c885", size = 377251, upload-time = "2026-10-07T10:14:51.103Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/09/4c59d56b8461ae8eb0d8ba34bb25b7e618547044679d58a82ef9b2479fc1/markupsafe-3.0.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6", size = 11658, upload-time = "2026-10-02T23:04:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f0/d6613774d86fbf6d145751d43c59875e47a6f9f17daee0aef173bd36d90e/markupsafe-3.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f", size = 12050, upload-time = "2026-10-02T23:04:52.931Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f2/8f18e0b806eb13c1f8d07d917a720831ead54253a6dec011fbc78098a6f8/markupsafe-3.0.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b", size = 24363, upload-time = "2026-10-02T23:04:53.895Z" }, + { url = "https://files.pythonhosted.org/packages/60/ce/fa07dbe8a5675558fa36dea033e19995bc783de2dec5f540ccb9030b06aa/markupsafe-3.0.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df", size = 28525, upload-time = "2026-10-02T23:04:54.905Z" }, + { url = "https://files.pythonhosted.org/packages/85/40/be87c01f3868ec217f8a2015089d71c22c8c5a75324822e5ed1cdd87210d/markupsafe-3.0.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c", size = 24733, upload-time = "2026-10-02T23:04:56.229Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a7/aeedb5140afa41fc74c225e9184ab96723a6e873b6ee1c9fede7283456d8/markupsafe-3.0.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581", size = 22985, upload-time = "2026-10-02T23:04:57.521Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fc/e91352bb08c6a59da3ef0909d457bf95a5f5908fbf151b30a06d9dbcfbb4/markupsafe-3.0.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77", size = 22001, upload-time = "2026-10-02T23:04:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f8/bffee5e7d2a3deb59748a797650a48af7e672025cf641a79344a771ad106/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c", size = 23793, upload-time = "2026-10-02T23:04:59.686Z" }, + { url = "https://files.pythonhosted.org/packages/ed/59/b853d6628ecb4d658e1d637224846d5e9bb4adf4f8df97f3be9f29dce2ec/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749", size = 22640, upload-time = "2026-10-02T23:05:00.768Z" }, + { url = "https://files.pythonhosted.org/packages/09/b2/1506df394f0f075797c418d0301498f49e43be194e3ffcb49e6fe6ccf022/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed", size = 24074, upload-time = "2026-10-02T23:05:01.813Z" }, + { url = "https://files.pythonhosted.org/packages/c7/81/5ed69cda630ac69ef60d06c09ba5a7f84ff66a2e28cf986fd5614ab3c6e6/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786", size = 21563, upload-time = "2026-10-02T23:05:03.239Z" }, + { url = "https://files.pythonhosted.org/packages/0c/fe/fb1e79be0fea60aa32602ebefc9c35a82bb42b4df157285ab7dfec12341a/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e", size = 23048, upload-time = "2026-10-02T23:05:04.479Z" }, + { url = "https://files.pythonhosted.org/packages/c8/52/7632a53360671a9b750cdbabaf9cdd89f18b42248b8e4cb42c0b0296e459/markupsafe-3.0.4-cp312-cp312-win32.whl", hash = "sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237", size = 14108, upload-time = "2026-10-02T23:05:05.513Z" }, + { url = "https://files.pythonhosted.org/packages/3f/bf/62495e180b7000aaf30000fff849e933f74264638057176cf46852500adc/markupsafe-3.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7", size = 14303, upload-time = "2026-10-02T23:05:06.538Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8e/4c24208776a65878d656996945aacfbfe010d3720d1a98fc0eb8491fc03b/markupsafe-3.0.4-cp312-cp312-win_arm64.whl", hash = "sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9", size = 14174, upload-time = "2026-10-02T23:05:07.617Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "msgpack" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/e7/bb605a7bab2d8425a64b3fa762b39dc1bf1c7e3f11ba6fb5413d6db0ff8c/msgpack-1.2.3.tar.gz", hash = "sha256:32edb81a2b5eb7cd7c9d941b2bfbbb082fd2cd09e0e725930316af6b708db186", size = 196517, upload-time = "2026-09-29T02:33:52.276Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/12/4d7c6d6203416d9fbf0f59ebaa805e70fb929b93a41b611bc821ec5964a0/msgpack-1.2.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:89c930aece4e972b208ba589c8410b4167b05e411a5ea2cb25fd96f8bc47ee43", size = 91577, upload-time = "2026-09-29T02:32:02.141Z" }, + { url = "https://files.pythonhosted.org/packages/eb/c7/8576ad39f4ca42ddad26f68eb8621d2d0a60501193d480f504bd9d7f36c4/msgpack-1.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:905a189853d6bdb204c7ae5f4ab77fb857448abfff574d3d93c62e2815b24b4f", size = 90027, upload-time = "2026-09-29T02:32:03.508Z" }, + { url = "https://files.pythonhosted.org/packages/0a/3a/aa9c580aea1314529a0f3562461479780b0d254b064f0880956bfbcc74a8/msgpack-1.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3d7b3d0018746b5997dd6b14a1870b07cc4c327d9101145d94a1fc264a51a06", size = 460343, upload-time = "2026-09-29T02:32:04.906Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cf/9c2e4d6c179529d5bf4a64cff76fa581486569e9fbdd35bd98f51cb624bf/msgpack-1.2.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede33b2892ceb976283e009ad12fa1834cfdf1f9c43ee9c97849fc588d00a618", size = 472998, upload-time = "2026-09-29T02:32:06.69Z" }, + { url = "https://files.pythonhosted.org/packages/7b/41/915c81fe6df2d3cbdb0dece4f1a5cd313e1cd2abd9f501d0f50c0582517e/msgpack-1.2.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:666ef5601ab0e6e345e47febc96aa81143cc932201543480cbb9499164f05ffb", size = 423216, upload-time = "2026-09-29T02:32:08.739Z" }, + { url = "https://files.pythonhosted.org/packages/a2/e7/7dda8b1039abfd9bba4c5068172c67135c9e33089f503512db9226f23c24/msgpack-1.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87cf2ef05ff2f2493ba29fcdaef27e960ca64dacfd13460ae29e6f92e0ed05bb", size = 451218, upload-time = "2026-09-29T02:32:10.517Z" }, + { url = "https://files.pythonhosted.org/packages/16/5b/ce995c1ed4a0522b7f2d034bc2034fd63005f240b945961b70fb56fbaf3d/msgpack-1.2.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:b774ff994d844e541439ac5d2d49a14def4104830c3465e9394c153f86200ffb", size = 422453, upload-time = "2026-09-29T02:32:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/ce191fb87e2650d0166b34c437e499ee4a7f9db9c1eb164f41725eb6160e/msgpack-1.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eaf7e82249837e3aa97297b34a0bb9ff562027381631e057cea6e1367f10b438", size = 469003, upload-time = "2026-09-29T02:32:13.663Z" }, + { url = "https://files.pythonhosted.org/packages/42/35/539123407fe200fb16609c835675496fbeb6017ace9fc93909f0613223ae/msgpack-1.2.3-cp312-cp312-win32.whl", hash = "sha256:7c047250096f9fc19dba26e3d1639b5e7a84114003605c94def667149a70ced1", size = 68303, upload-time = "2026-09-29T02:32:15.02Z" }, + { url = "https://files.pythonhosted.org/packages/6f/4c/331b45f9b86fbda6b9e103244d189068e51f726d8c40021ed66e1f2c415e/msgpack-1.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:3ec409b0d6aa8e9eec6eaf881b893caa215dbe68c5319ca96e8a271d81bb111d", size = 76744, upload-time = "2026-09-29T02:32:16.344Z" }, + { url = "https://files.pythonhosted.org/packages/13/9f/fb572dc42b9fac06c7ea848aaee6e140d84469743bd1402bc07089fc4566/msgpack-1.2.3-cp312-cp312-win_arm64.whl", hash = "sha256:59612b4ed48a04cf024584218e813562f3b30a3bafa5f55abe300b15da314751", size = 71580, upload-time = "2026-09-29T02:32:17.617Z" }, +] + +[[package]] +name = "msgpack-numpy" +version = "0.4.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msgpack" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/94/61e8aee142733ebfdc400a05bdac6e1763c4514bba3b42743d223f388450/msgpack-numpy-0.4.8.tar.gz", hash = "sha256:c667d3180513422f9c7545be5eec5d296dcbb357e06f72ed39cc683797556e69", size = 10923, upload-time = "2022-06-09T03:43:08.739Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/5d/f25ac7d4fb77cbd53ddc6d05d833c6bf52b12770a44fa9a447eed470ca9a/msgpack_numpy-0.4.8-py2.py3-none-any.whl", hash = "sha256:773c19d4dfbae1b3c7b791083e2caf66983bb19b40901646f61d8731554ae3da", size = 6919, upload-time = "2022-06-09T03:43:06.82Z" }, +] + +[[package]] +name = "multidict" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/79/84ddb5ba16c4eb2c69c71db76ae3c579fe546e511f7170c7e27eedbab7c1/multidict-7.1.0.tar.gz", hash = "sha256:61a4e5d81b8d4e4ad61964b230129e7a2b914793d96289029078fc9009f074ec", size = 362341, upload-time = "2026-10-09T20:31:38.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e1/215e7df354e2907f3af9d910c8136653cfec94796012a776359bc802a326/multidict-7.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ccfb950359a80de0fcd2030ad60ac1b1a861462de3e2ef746697c9256659af21", size = 165426, upload-time = "2026-10-09T13:59:50.564Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ec/461ba588b308ada2cd16907d6ea425ca4d417596b88c12814ad9a0bb7325/multidict-7.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:939d8cd2d8c35e3956f6bc858390b6ccb611e6152b4920d64ab5e98f3fcf39e4", size = 95943, upload-time = "2026-10-09T13:59:52.381Z" }, + { url = "https://files.pythonhosted.org/packages/c9/84/31444ef07ec13a33c42c2772d986129e137e69c4beb89ca64f2138988145/multidict-7.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f8e95c95039eab6a2dad8c83c38ab87fc5431d28849e0c8a7e2a4e70ba38710d", size = 98294, upload-time = "2026-10-09T13:59:54.066Z" }, + { url = "https://files.pythonhosted.org/packages/dc/10/aca13806d73d88b5b01e35e828e23a346cb1abad710a49dff0507702efc9/multidict-7.1.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:05d12b4bac53abe0c65f3163af2b45894e2e1c0cc55493ac784d52a350047d88", size = 453301, upload-time = "2026-10-09T13:59:55.397Z" }, + { url = "https://files.pythonhosted.org/packages/96/8c/382d771bfb3a9282d98332d0e1f27fd1f8b4ae0e7175bd7e008ebf934900/multidict-7.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e79ed92b1dece6bb57e9b46effd74d7a5d3d00187c85466d880ed184239a698", size = 478570, upload-time = "2026-10-09T13:59:56.899Z" }, + { url = "https://files.pythonhosted.org/packages/52/9c/e81b0c92449da3a1950a575c520d957d7be618777d170717a71e00558d7f/multidict-7.1.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fab380fcff8b3555eb2bd04304fa4330909a771a9a9b0dc07666cfc23148a711", size = 443699, upload-time = "2026-10-09T13:59:58.465Z" }, + { url = "https://files.pythonhosted.org/packages/63/72/f8f5fee6960d1b580a7b046c7c5abccf67aa26fa5647927a91c9c835c2f8/multidict-7.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4b5c41e44da74383c924cc5d75ef0a268f301d69305b3c42bd17af685d55e412", size = 497332, upload-time = "2026-10-09T14:08:01.88Z" }, + { url = "https://files.pythonhosted.org/packages/e2/92/a25d7db3ca451b588e743e067ee80f2b475edf6467a56908454faf6a714c/multidict-7.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3dbaa7f7c2f0ca8578895fc61fb8c8e50ebb405dad8982f92f4343285c7a3fda", size = 512457, upload-time = "2026-10-09T14:08:25.465Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f3/374c0ab122bb98b1a62a8742b1e3f59563e4d609942005a4041861a0df67/multidict-7.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed6b7705d49dd07e5e0dd5f5c873fc44047e92d714299b13245b5fecac49d01", size = 483000, upload-time = "2026-10-09T14:08:27.132Z" }, + { url = "https://files.pythonhosted.org/packages/46/1f/01c8522859771dc3cee840d5852233fe84c91aa982a1cd8ad594306d25cc/multidict-7.1.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:53daa47dd176db64bb35170e3d5d0ae2388c060121201883696278f055a0e70c", size = 430746, upload-time = "2026-10-09T14:08:28.715Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f3/af90affc8cca6b4ee59b2ec354a20839005e829d14d155451009810ef1ce/multidict-7.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:542429c796430de924d03b68a6173bb6d79d5c4967d4e9a18de3e501cad55593", size = 466859, upload-time = "2026-10-09T20:27:24.139Z" }, + { url = "https://files.pythonhosted.org/packages/f8/97/1b6762f37f6331449e164af9d5500f0abb0f23670e81ba543441e0a6be1d/multidict-7.1.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c44ca6d3cdf4cfcbcd4f928fdcbe87af5fd7319f6ad4169617b7fd6b4527c33c", size = 441952, upload-time = "2026-10-09T20:27:26.559Z" }, + { url = "https://files.pythonhosted.org/packages/bf/d2/4908177fbf22438799c04ba11a2853a99c69d028fccefe61f19e68caba0c/multidict-7.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:eb0228c809b2e7eb47921876050af0bc4214b351bad8d8112f70b6ed4288763c", size = 452810, upload-time = "2026-10-09T20:27:28.347Z" }, + { url = "https://files.pythonhosted.org/packages/8d/05/5031f44f680ec54fc182c4d71c9ab7c07216983946aa64ce6fdd56a52692/multidict-7.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6ad60de1f4c702448fc8f1449f05e810f6b7957c08a5b3950c8a792dfb13b50a", size = 483934, upload-time = "2026-10-09T20:27:30.14Z" }, + { url = "https://files.pythonhosted.org/packages/da/3b/9b21d107dbe96fa7e6966ff9e5e10b9ac0d2d1c2cf1633098e4c700f865e/multidict-7.1.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:f79def86aee67b5ba01b2565f1610f262bf88ae53c379f93e5fa29c50fe793be", size = 427042, upload-time = "2026-10-09T20:27:32.433Z" }, + { url = "https://files.pythonhosted.org/packages/53/ce/5b01b1041580072866b30e39c6380bff269e72fb5ca41a7f2fb828ede943/multidict-7.1.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:248dabb89b5aa90b2f7e43e045f048f7e5392ec77b6446d80853ba7117d7bbdf", size = 492543, upload-time = "2026-10-09T20:27:34.459Z" }, + { url = "https://files.pythonhosted.org/packages/1e/6f/6508a23fcc7b1122e4f18409d7900ffeb3cd020cd080fe98aba3eabf9486/multidict-7.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0747a83e7ae617793181a4763ee8b84863cec5c0bbbde70c4394e4c0276c36de", size = 475796, upload-time = "2026-10-09T20:27:36.289Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b6/ebb6433f4aa55ac1fea4bf11e860e99611640d07cc92d21cb3f35608de22/multidict-7.1.0-cp312-cp312-win32.whl", hash = "sha256:1df055e51fe7491120cc84f3362bd43db186be78d0e4c476acad45e435af9ffb", size = 81250, upload-time = "2026-10-09T20:27:38.229Z" }, + { url = "https://files.pythonhosted.org/packages/61/0a/11240e5e7d2e986e288f4a6090f569ad00225b7c2c513d4096b36643f9ab/multidict-7.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:10202ba98cfb3f7eb60da7ca87a2c458a69b7d0d6e4d4388cd6773ebbce89085", size = 93817, upload-time = "2026-10-09T20:27:39.739Z" }, + { url = "https://files.pythonhosted.org/packages/e9/9b/a04ffd76db7cca2a60d347e839315e1ada63017c6c343fa90a23a5a55433/multidict-7.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:0aa1ba3ff7cdda05a1242490612976b2ae1c90fc6200903ef8f53815dcb35c5d", size = 92312, upload-time = "2026-10-09T20:27:41.285Z" }, + { url = "https://files.pythonhosted.org/packages/d0/86/a3de309c5e28ee85b314d0e3ba0e0dea6fd361c313322a05e67be4656e1e/multidict-7.1.0-py3-none-any.whl", hash = "sha256:d9ef29cfd98e17085b4f91bba8fa1570bec6787d5c52ce653ed33a58785585d0", size = 28087, upload-time = "2026-10-09T20:31:35.945Z" }, +] + +[[package]] +name = "networkx" +version = "3.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/76/3af777226b63a5e64a6b36b1ec5855c14e2b94a37096d4760e595fc43511/networkx-3.7.tar.gz", hash = "sha256:fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a", size = 1866482, upload-time = "2026-09-21T16:45:16.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/cd/fe58041e9011f307c490e3e17dd48cc516448f7c698a3f2d9d9d65d7e6a8/networkx-3.7-py3-none-any.whl", hash = "sha256:e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0", size = 2142205, upload-time = "2026-09-21T16:45:14.609Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, + { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, + { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.8.4.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.10.2.21" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.3.83" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.1.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.9.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.3.90" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cusparse-cu12" }, + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.8.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.27.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457", size = 322348229, upload-time = "2025-06-26T04:11:28.385Z" }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu12" +version = "3.3.20" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/6c/99acb2f9eb85c29fc6f3a7ac4dccfd992e22666dd08a642b303311326a97/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d00f26d3f9b2e3c3065be895e3059d6479ea5c638a3f38c9fec49b1b9dd7c1e5", size = 124657145, upload-time = "2025-08-04T20:25:19.995Z" }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "peft" +version = "0.18.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "accelerate" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "transformers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/48/147b3ea999560b40a34fd78724c7777aa9d18409c2250bdcaf9c4f2db7fc/peft-0.18.1.tar.gz", hash = "sha256:2dd0d6bfce936d1850e48aaddbd250941c5c02fc8ef3237cd8fd5aac35e0bae2", size = 635030, upload-time = "2026-01-09T13:08:01.136Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/14/b4e3f574acf349ae6f61f9c000a77f97a3b315b4bb6ad03791e79ae4a568/peft-0.18.1-py3-none-any.whl", hash = "sha256:0bf06847a3551e3019fc58c440cffc9a6b73e6e2962c95b52e224f77bbdb50f1", size = 556960, upload-time = "2026-01-09T13:07:55.865Z" }, +] + +[[package]] +name = "pillow" +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/b8/58d79332229482eb0d27c82973850808c19d37e5bdc14443d3417bd373f5/pillow-13.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79", size = 5568643, upload-time = "2026-10-16T22:15:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/39/4b/f2ed53d83aa44ac19599ac0a54897f867fafb95f406980d1e24083f8cbe3/pillow-13.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52", size = 4852329, upload-time = "2026-10-16T22:15:46.284Z" }, + { url = "https://files.pythonhosted.org/packages/3b/bb/9e30e8ed18c20800ee327a12107d5249ad8dc444867b3adfcc21654e3c71/pillow-13.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d", size = 6519309, upload-time = "2026-10-16T22:15:48.14Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/fdf65cc539e266ab93546c261bdf98411fe165c6e4e13f9dbf597097192b/pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca", size = 7144542, upload-time = "2026-10-16T22:15:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/26/9b/f38b7c4d8f599537e2d1a6d546cec9beef7536ff8f5c49a3f9c78179d8ac/pillow-13.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620", size = 6618042, upload-time = "2026-10-16T22:15:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7c/45ee28531f89ee4582967d8a17f7a26d71ca64cd902b93c269590cd60a00/pillow-13.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40", size = 7228137, upload-time = "2026-10-16T22:15:54.009Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/9c2a969cb1b70201d04fa17fd5aedf02a732870b8da86bceaaa30b0ca385/pillow-13.0.0-cp312-cp312-win32.whl", hash = "sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27", size = 6515423, upload-time = "2026-10-16T22:15:56.25Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/73a79ecdc822076002eb24b5e1e4a530b0bb5310fcb6d239abbec877110b/pillow-13.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b", size = 7272166, upload-time = "2026-10-16T22:15:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a3/be5a1236efd085b3984e4305da8f179ae9dab80c991d2ad78f2bd5c4c843/pillow-13.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51", size = 2698943, upload-time = "2026-10-16T22:16:00.637Z" }, + { url = "https://files.pythonhosted.org/packages/60/da/f6d644e631c5b370c5d0b899f3ccadd2780aab070b29a110527cf4e1fd2c/pillow-13.0.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463", size = 5523188, upload-time = "2026-10-16T22:17:59.334Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b0/2594f202d743a35d4fe80682bfd96f277e5ca7aecb27cf64ef5bfc1d000b/pillow-13.0.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e", size = 4806808, upload-time = "2026-10-16T22:18:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ed/d504165ef79af6fafe25c76726cf30f544cc236002f7ed054f1d51adc4ff/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63", size = 5464358, upload-time = "2026-10-16T22:18:02.779Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fe/a10aba5503ca13e9e009b8bd0a1a4ce5584bdce246245fd6f3a83701369c/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61", size = 6043514, upload-time = "2026-10-16T22:18:04.538Z" }, + { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, +] + +[[package]] +name = "propcache" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/9a/9fbf4e4ec0c2d7f1c32519fff782ef467859b8faa9fbc5331a96f6395d43/propcache-0.5.4.tar.gz", hash = "sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558", size = 61545, upload-time = "2026-09-16T00:17:14.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/cd/348d58f142aebc4873345c6b31087629182ca6e0f2b3caeaa528cf882eba/propcache-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b28f41fa3b8c6900457f858ec5b03998f3a6d535fbc1bb2edec5961ea05ec429", size = 87285, upload-time = "2026-09-16T00:14:29.362Z" }, + { url = "https://files.pythonhosted.org/packages/df/f4/f3ffaee281b276da854ac1d7a6a506d26cbc62ea2e623756f1d0a4a1ba1a/propcache-0.5.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dcbf346a318a5e30063f547630b02bb787ce2f45b6368d5da143660b6a3835d8", size = 50984, upload-time = "2026-09-16T00:14:30.473Z" }, + { url = "https://files.pythonhosted.org/packages/25/88/1d7df7201750b37765ef2b23bc1c526c028dadde80afa0f57a118fc01182/propcache-0.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87a3caecf8095e48dc72f84bfa42e23a848cf410cc9cc13031fba4869b706a21", size = 52460, upload-time = "2026-09-16T00:14:31.692Z" }, + { url = "https://files.pythonhosted.org/packages/83/4f/48865bd02a16ee5236bc46166b2946f37b93e07b0eae355dac0be0b216ca/propcache-0.5.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60a64cbccaa11b7760ce705a14ada17ba459e7ca9f23ba587eb013821032d7ef", size = 251768, upload-time = "2026-09-16T00:14:32.908Z" }, + { url = "https://files.pythonhosted.org/packages/b0/19/3742a5eed62317b03b4002ee865dc9fd720308bdd0da1f29a5786c630311/propcache-0.5.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a74bfa37147cc08fb29df10bd9c16f40fa7f860cd3a6d2fff853323a94f6e17f", size = 257723, upload-time = "2026-09-16T00:14:34.267Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d5/ee6350fb0be9122bb6c67082a876d34b90d980d100c106af4b81023e04f4/propcache-0.5.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a4d7a54719b67338a305dca2ce6aafe366817df94ddfd4b5514374356f5ca546", size = 265597, upload-time = "2026-09-16T00:14:35.56Z" }, + { url = "https://files.pythonhosted.org/packages/85/9f/83a07b6ec0e043c050cfdd35fb0cf1b7897b91d554d6eea293740309afe7/propcache-0.5.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2814ecd8e818f487bee4b0f921bc4d1c176cc5fc71ac0f072d0fa67eda4ac14b", size = 250424, upload-time = "2026-09-16T00:14:36.894Z" }, + { url = "https://files.pythonhosted.org/packages/33/2c/a763a8251f50fba042af0fb1f02bfec4b31381e40aff760db2be7b2e1f84/propcache-0.5.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6af4693716bfb03f1752ef1b30faa593db2c01d5272e9b8564a1549452a979ab", size = 216748, upload-time = "2026-09-16T00:14:38.369Z" }, + { url = "https://files.pythonhosted.org/packages/6a/e2/4d11bea8fd6a777149c6c20645f873952eab5de3a2497aa11648ec9ab6ab/propcache-0.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4fbc1a15dc8cd1689508758d626b372b1f09d28d9577667feaf9e6bfcd8efcbc", size = 246533, upload-time = "2026-09-16T00:14:39.82Z" }, + { url = "https://files.pythonhosted.org/packages/9f/36/6683597de4907e70c717e3588c541202c66086a72ff3db58be49de66e72c/propcache-0.5.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cdee8205a44d0be91bbac4c41b95d86641b72dfc7aef1279400e4fda3f26a937", size = 238173, upload-time = "2026-09-16T00:14:41.259Z" }, + { url = "https://files.pythonhosted.org/packages/85/84/cb08d79f1762daafeb2b030c470cd0c725c97b8ad67412457c6f35c53e9d/propcache-0.5.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9a2a8a50a93dee0268a860a07fa3b4bd968f8ce4dbd794957da772f395368526", size = 251128, upload-time = "2026-09-16T00:14:42.652Z" }, + { url = "https://files.pythonhosted.org/packages/c2/0d/41b848036db6621370c1f2e5471a7da8149c730f8552a5257567721f4576/propcache-0.5.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7ffafcbfc7b549ab940047e505c831eabac5e67de53e1bc174adbc5285c55944", size = 214821, upload-time = "2026-09-16T00:14:44.112Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/adfae4bf9c63bccf12e2d9690a175c6579047a6eec3b5a6a5f51428c15e2/propcache-0.5.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d1f5a500bfcbb2c0ab85e98a0dcd70f5899d34efe365a0187700369a79603031", size = 254793, upload-time = "2026-09-16T00:14:45.429Z" }, + { url = "https://files.pythonhosted.org/packages/51/6f/eeca9647245d5f92e87d53e5f14335bb42fce1a7e6842c8045b364eded8b/propcache-0.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8a235f73d6e020855dc29dff012d920c02ee0feab8d73a24185a7569f4be1161", size = 247134, upload-time = "2026-09-16T00:14:46.976Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a9/424e38838793d37160b4379c702f61c74c598fc6cd17204adbe3c554f7a8/propcache-0.5.4-cp312-cp312-win32.whl", hash = "sha256:b3083bfe87f95c756e610bd8025f26cbd1cd4aaa03a422f2d65efb7a97cd53d8", size = 43073, upload-time = "2026-09-16T00:14:48.338Z" }, + { url = "https://files.pythonhosted.org/packages/58/7b/6e8ef26f6d510a7916064fec68d55fcbfbdf7eb01e377480d66a122152d8/propcache-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:98914de2c4d7f0f9f4a8c6ea4bf05841f4175796941e3ef7d47eb718f22311fb", size = 46190, upload-time = "2026-09-16T00:14:49.99Z" }, + { url = "https://files.pythonhosted.org/packages/08/b9/72028c5b56ced97f456de6aefa79435ca64d7f77af78ea8cf3c76fc5195f/propcache-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:8876b39961e33d912afe3c1bee18ee564fdad0206f873cc15d522756b7f50737", size = 43075, upload-time = "2026-09-16T00:14:51.155Z" }, + { url = "https://files.pythonhosted.org/packages/f5/cd/785c64ed382f3f04201870267b02783f63b4678c2acfddc177a3ebcc2727/propcache-0.5.4-py3-none-any.whl", hash = "sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468", size = 16338, upload-time = "2026-09-16T00:17:13.106Z" }, +] + +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "regex" +version = "2026.9.29" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f2/af1da9d3ceed77bfcdce40427d49ba0be94e4fe84245e3bfef68c10e75b6/regex-2026.9.29.tar.gz", hash = "sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb", size = 419199, upload-time = "2026-09-29T00:49:58.298Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/48/3fdcde9a0baa84d7d25571223265d6e434e114763b438601d54a8028bf3e/regex-2026.9.29-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:dc79d36d0618752265f0d575915bdc5c5130ecb9c9f6b3bcefeae32e4bdfafcf", size = 497903, upload-time = "2026-09-29T00:46:38.938Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1c/4ee3e97c76f53940488dfe7a7e18705e78daac8cd7fb161d246b9e328449/regex-2026.9.29-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3a21a9509d0ee88e7a70e1ad228cd2f0e0fd1e187458db132e8a8d18c97daf9d", size = 296416, upload-time = "2026-09-29T00:46:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/37/14/f3f0ba083d2094392d5eabf56db5ea6ba469fd6e927afd187042054ea68a/regex-2026.9.29-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f57dc6b8fef170f105d2cf5cdce254f47b137d7755086cf7050f47e16582abba", size = 293633, upload-time = "2026-09-29T00:46:41.959Z" }, + { url = "https://files.pythonhosted.org/packages/c9/72/67e7a8ce17f1aea49df215564048efb49cc8c2b31a0e0fc30f36838f8516/regex-2026.9.29-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f93bc1c3486ef3747e07c9d7c1d0a147b8fbaab975f80e348aed6f71309dfaca", size = 805885, upload-time = "2026-09-29T00:46:43.373Z" }, + { url = "https://files.pythonhosted.org/packages/f6/78/25436bcfd4d2260b4b4090094d55d7ab53ec8a1ab4865a0b8bcb33c7d5c0/regex-2026.9.29-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e1d3a4cb7993b708f0ada8d0c84590efd853f169e7147d2202c9da503180242", size = 878344, upload-time = "2026-09-29T00:46:45.328Z" }, + { url = "https://files.pythonhosted.org/packages/97/e6/a09ec3a23ae41d6179880e67f0aace9284b2d95f2d7b326eff203f8eec5e/regex-2026.9.29-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dabee8f4935e731fb46b2a3091bdda0d3d94b3bbfb907d2b4f12eefce4009619", size = 919181, upload-time = "2026-09-29T00:46:47.041Z" }, + { url = "https://files.pythonhosted.org/packages/26/83/d2fbd2e4e3afb1167daa825187d196f313cbaa1a4768f311fb041bb0e3d2/regex-2026.9.29-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0", size = 807783, upload-time = "2026-09-29T00:46:48.894Z" }, + { url = "https://files.pythonhosted.org/packages/46/0b/eb429a7016610d44fc89a597163f8c9127505f0d7dc724dc9effbb6a3ac0/regex-2026.9.29-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c1a9a6651197fbed6f0212591418b9def774fc3f8324f78d1bf0e6a63e5f8aa1", size = 783465, upload-time = "2026-09-29T00:46:50.64Z" }, + { url = "https://files.pythonhosted.org/packages/1b/07/58a3c0153c7476898430f6a7cf3d9062a1d17fbea4f43399ecaf411c7b4c/regex-2026.9.29-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87fb80cbe3557e27e7b28b995c2b2eedf689b8886f941ab93e0e288f0976518a", size = 793519, upload-time = "2026-09-29T00:46:52.396Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e8/161b94d39164520e21a7befe0245569bf7fda4c7cf1fc4e2df2b5def49da/regex-2026.9.29-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3c5c2ef13797466aa64170cbb66ad98a32351dd4127694cea7199f80f213750d", size = 869293, upload-time = "2026-09-29T00:46:54.128Z" }, + { url = "https://files.pythonhosted.org/packages/8f/07/3b02ed829aa2decdc1955d222bd1e2f99d1c8bb4873bbb9a66b2f0a36bff/regex-2026.9.29-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:59b49507f47479e299a9e1bc41b5cb83a7afda0540625f1dbae886615978acbf", size = 770239, upload-time = "2026-09-29T00:46:56.106Z" }, + { url = "https://files.pythonhosted.org/packages/42/5b/ba61f6fe062eb8562e742367d177bb75370434138ef6c9d2a27114f8d613/regex-2026.9.29-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0dd8af32e9f7b56b7f95cc1fd79b23054c3bdc172392ae560acc24d57b7ffe71", size = 861973, upload-time = "2026-09-29T00:46:57.665Z" }, + { url = "https://files.pythonhosted.org/packages/cc/27/767259b20e8a842948990f5e99138d6c077248fd42f8b5468b1d9ca4b814/regex-2026.9.29-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db5e82ba15c142425b8406690032df89e39cca4a2e8afbbb9a3d84edc2373ac3", size = 796470, upload-time = "2026-09-29T00:46:59.236Z" }, + { url = "https://files.pythonhosted.org/packages/a0/05/2566c4ba849b68a8ab81a6bf428fa79d20aae7ddee83979103c0381df254/regex-2026.9.29-cp312-cp312-win32.whl", hash = "sha256:d0c3082bf79bcd6a614d55916590ad4b8f93200e10b97f463ea5d9d07c9b5f23", size = 269327, upload-time = "2026-09-29T00:47:01.135Z" }, + { url = "https://files.pythonhosted.org/packages/93/19/489bc8db91196381c935752df01ba3f607140daece33b78d88573f028e64/regex-2026.9.29-cp312-cp312-win_amd64.whl", hash = "sha256:fdd88ed5e20b1bcdd234421e454962c971aa44b653bdb7f1ea9ef683e90fb649", size = 280334, upload-time = "2026-09-29T00:47:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/0b/47/fb88ba779d0e5e7d4b0ec1aceeb13845948a2cb876bd572a2d1dfdba090b/regex-2026.9.29-cp312-cp312-win_arm64.whl", hash = "sha256:4fe97894d1b306c919b4e50def1e6f6c522f4d03a7283811f4d108f1ce5d3ac2", size = 279614, upload-time = "2026-09-29T00:47:06.541Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "ruff" +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e7/e6d5749983b020e6cb1055abec0a6f361a8e94f0184cdcf56d4760995f02/ruff-0.17.1.tar.gz", hash = "sha256:5bb796c5112e9fb9527f2ef2faa0a130d6312d19de92bcfaddd70dbb2d41020f", size = 5323349, upload-time = "2026-10-15T15:23:18.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/78/90c46ca6250709f7c37e397b60d06061adee8d0f7d13959ed6a8e49b14fd/ruff-0.17.1-py3-none-linux_armv6l.whl", hash = "sha256:1796c41eafec66f4635125dd1e074a7aa27b8d20c187d484360acf11d247fc4a", size = 10147528, upload-time = "2026-10-15T15:22:33.667Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d2/a2e28bc35b083ae51fa07dfba5280a667c8b01ecaee7802fc8849dcd5c46/ruff-0.17.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:8eadb09a788010be29c895418bdfa513e4e1c65e64b94e5b19e4cd381f6cc7fb", size = 10225974, upload-time = "2026-10-15T15:22:36.328Z" }, + { url = "https://files.pythonhosted.org/packages/8e/71/7a9029a81c8206ee01a98f6cfeeb56e6c095af86857581a37045c46d7e94/ruff-0.17.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:29b454f18ca3eeaf37fbf140841dcf27d36109e29dee9aae8acecb68c26c6f74", size = 9270361, upload-time = "2026-10-15T15:22:38.651Z" }, + { url = "https://files.pythonhosted.org/packages/c0/8c/e075e9c917cb0575297880c96768304e881240171c870cbd8d9b7b54bec2/ruff-0.17.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872e03dd231744d6c5ec5c8af6be52b71323fc6ee096e4c8b34d9f844ee3b978", size = 9938300, upload-time = "2026-10-15T15:22:41.284Z" }, + { url = "https://files.pythonhosted.org/packages/87/27/0b4bb926a49de2383f161272b4d863571a4c7ecdbf595a6bc58813a13367/ruff-0.17.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4c0d6cb5f8fbac77eec9af49f4fc0eaf856b05545e372b0694d24b4e76b1c640", size = 9948464, upload-time = "2026-10-15T15:22:43.681Z" }, + { url = "https://files.pythonhosted.org/packages/14/27/518d57e0e02aecb8ea4acac6b57b51a53173d25fa734952bfdd7d531e88e/ruff-0.17.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78beced4a3c94dfedbee4b999b95b6a27b35820fdc0afbacc6d37a8da26c1cd1", size = 10638639, upload-time = "2026-10-15T15:22:46.081Z" }, + { url = "https://files.pythonhosted.org/packages/01/82/d89a4e498ade59e6cee52d67f8da569019372212b87468c1fdeae884601c/ruff-0.17.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7feefc63921202894f0e158b0e32f45a5950d70a7fb693ccb9264816c25b1c05", size = 11337238, upload-time = "2026-10-15T15:22:48.894Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7c/3b0fdef9352656317c977086fe1d802902406e4f2a0d8578da90c0e6d1c5/ruff-0.17.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684154861a819bf1f433669d2a0ce9c3e54677c65b8ebb866cc2a7ec86345785", size = 10896327, upload-time = "2026-10-15T15:22:51.45Z" }, + { url = "https://files.pythonhosted.org/packages/62/86/5566290219708ce2928f3a29c4f843a1185b341ec6111c4631a33c5ceced/ruff-0.17.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6ef98e51de8ef38082f90653ceeb84944bd7673783eea13e1647c5b9b064189", size = 10350811, upload-time = "2026-10-15T15:22:54.401Z" }, + { url = "https://files.pythonhosted.org/packages/ad/d2/ca8462dabb3c413990d49703ba306c30c862195a75448d080fa8fcce9775/ruff-0.17.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8f9aa5b5bd5fa1937803ea609c8f153b8de39bf3e0e149897ae34b0c5b52f467", size = 10569994, upload-time = "2026-10-15T15:22:57.142Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ad/dd1864c1f1db17c8cfb297ce70b352388e6977620f66f7182c52855407df/ruff-0.17.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:80e53b0e5faa3c5667f035daaa032893ae42b7cfe708c886d39ed131e54688ab", size = 10059672, upload-time = "2026-10-15T15:23:00.084Z" }, + { url = "https://files.pythonhosted.org/packages/65/45/3f56cf1d2168c2165cdd5ac6f9668aee66a4cfdd4caf2b2cc5ccb2b1d568/ruff-0.17.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:056c5d5fa0ac790051ae165d7035c7bf956d02de2db557f2e9b31d877a505fca", size = 9947803, upload-time = "2026-10-15T15:23:03.132Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4b/5457cb2be4c1d1ea5e919558131a0f197a4a454a0530d9e30820e8206e4c/ruff-0.17.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bd8f5a55b10f371642eacd0e8665a42a8b97da15c9a7187c79e697b77b5e0ce4", size = 10351807, upload-time = "2026-10-15T15:23:05.628Z" }, + { url = "https://files.pythonhosted.org/packages/49/fb/db15b4f8fa24a1e36b99e1401f42ef6df16ee299a48670d5d262b771e7d7/ruff-0.17.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ce05e6f3973611d9550a600e04bb531061e368baba31610eb57c1509a8f21a0a", size = 10765716, upload-time = "2026-10-15T15:23:08.092Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e8/1e186578076aff7a80735f625005fde678412ab609cd323ed182eb7d6b87/ruff-0.17.1-py3-none-win32.whl", hash = "sha256:1de0030a8aa78c65c5fecb84b2e5c390ac684c0452875a02afc7c083d49a496a", size = 9895093, upload-time = "2026-10-15T15:23:10.582Z" }, + { url = "https://files.pythonhosted.org/packages/6a/2e/da3d9589bae8e8b58f3a48bd5bc0b980232fb14a83bf02c3be7660f87c1b/ruff-0.17.1-py3-none-win_amd64.whl", hash = "sha256:5df76f16580fab58d0f6c98a3ac254a4abceb7b2489b8b678335869cff77b3d7", size = 10097955, upload-time = "2026-10-15T15:23:13.309Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/0f2a4dea06a5e12c1191ee3d96b1e1233797d0d5e18f97a2b909a95a9836/ruff-0.17.1-py3-none-win_arm64.whl", hash = "sha256:42400b9a6ff8515ff27796f07f92e0195be011c86e81e381606f58ea5502fef4", size = 9853542, upload-time = "2026-10-15T15:23:16.05Z" }, +] + +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, +] + +[[package]] +name = "scikit-learn" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/a5/4ae3b3a0755f7b35a280ac90b28817d1f380318973cff14075ab41ef50d9/scikit_learn-1.6.1.tar.gz", hash = "sha256:b4fc2525eca2c69a59260f583c56a7557c6ccdf8deafdba6e060f94c1c59738e", size = 7068312, upload-time = "2025-01-10T08:07:55.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/18/c797c9b8c10380d05616db3bfb48e2a3358c767affd0857d56c2eb501caa/scikit_learn-1.6.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:926f207c804104677af4857b2c609940b743d04c4c35ce0ddc8ff4f053cddc1b", size = 12104516, upload-time = "2025-01-10T08:06:40.009Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b7/2e35f8e289ab70108f8cbb2e7a2208f0575dc704749721286519dcf35f6f/scikit_learn-1.6.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2c2cae262064e6a9b77eee1c8e768fc46aa0b8338c6a8297b9b6759720ec0ff2", size = 11167837, upload-time = "2025-01-10T08:06:43.305Z" }, + { url = "https://files.pythonhosted.org/packages/a4/f6/ff7beaeb644bcad72bcfd5a03ff36d32ee4e53a8b29a639f11bcb65d06cd/scikit_learn-1.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1061b7c028a8663fb9a1a1baf9317b64a257fcb036dae5c8752b2abef31d136f", size = 12253728, upload-time = "2025-01-10T08:06:47.618Z" }, + { url = "https://files.pythonhosted.org/packages/29/7a/8bce8968883e9465de20be15542f4c7e221952441727c4dad24d534c6d99/scikit_learn-1.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e69fab4ebfc9c9b580a7a80111b43d214ab06250f8a7ef590a4edf72464dd86", size = 13147700, upload-time = "2025-01-10T08:06:50.888Z" }, + { url = "https://files.pythonhosted.org/packages/62/27/585859e72e117fe861c2079bcba35591a84f801e21bc1ab85bce6ce60305/scikit_learn-1.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:70b1d7e85b1c96383f872a519b3375f92f14731e279a7b4c6cfd650cf5dffc52", size = 11110613, upload-time = "2025-01-10T08:06:54.115Z" }, +] + +[[package]] +name = "scipy" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/56/3e/9cca699f3486ce6bc12ff46dc2031f1ec8eb9ccc9a320fdaf925f1417426/scipy-1.17.0.tar.gz", hash = "sha256:2591060c8e648d8b96439e111ac41fd8342fdeff1876be2e19dea3fe8930454e", size = 30396830, upload-time = "2026-01-10T21:34:23.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/11/7241a63e73ba5a516f1930ac8d5b44cbbfabd35ac73a2d08ca206df007c4/scipy-1.17.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:0d5018a57c24cb1dd828bcf51d7b10e65986d549f52ef5adb6b4d1ded3e32a57", size = 31364580, upload-time = "2026-01-10T21:25:25.717Z" }, + { url = "https://files.pythonhosted.org/packages/ed/1d/5057f812d4f6adc91a20a2d6f2ebcdb517fdbc87ae3acc5633c9b97c8ba5/scipy-1.17.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:88c22af9e5d5a4f9e027e26772cc7b5922fab8bcc839edb3ae33de404feebd9e", size = 27969012, upload-time = "2026-01-10T21:25:30.921Z" }, + { url = "https://files.pythonhosted.org/packages/e3/21/f6ec556c1e3b6ec4e088da667d9987bb77cc3ab3026511f427dc8451187d/scipy-1.17.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f3cd947f20fe17013d401b64e857c6b2da83cae567adbb75b9dcba865abc66d8", size = 20140691, upload-time = "2026-01-10T21:25:34.802Z" }, + { url = "https://files.pythonhosted.org/packages/7a/fe/5e5ad04784964ba964a96f16c8d4676aa1b51357199014dce58ab7ec5670/scipy-1.17.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:e8c0b331c2c1f531eb51f1b4fc9ba709521a712cce58f1aa627bc007421a5306", size = 22463015, upload-time = "2026-01-10T21:25:39.277Z" }, + { url = "https://files.pythonhosted.org/packages/4a/69/7c347e857224fcaf32a34a05183b9d8a7aca25f8f2d10b8a698b8388561a/scipy-1.17.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5194c445d0a1c7a6c1a4a4681b6b7c71baad98ff66d96b949097e7513c9d6742", size = 32724197, upload-time = "2026-01-10T21:25:44.084Z" }, + { url = "https://files.pythonhosted.org/packages/d1/fe/66d73b76d378ba8cc2fe605920c0c75092e3a65ae746e1e767d9d020a75a/scipy-1.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9eeb9b5f5997f75507814ed9d298ab23f62cf79f5a3ef90031b1ee2506abdb5b", size = 35009148, upload-time = "2026-01-10T21:25:50.591Z" }, + { url = "https://files.pythonhosted.org/packages/af/07/07dec27d9dc41c18d8c43c69e9e413431d20c53a0339c388bcf72f353c4b/scipy-1.17.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:40052543f7bbe921df4408f46003d6f01c6af109b9e2c8a66dd1cf6cf57f7d5d", size = 34798766, upload-time = "2026-01-10T21:25:59.41Z" }, + { url = "https://files.pythonhosted.org/packages/81/61/0470810c8a093cdacd4ba7504b8a218fd49ca070d79eca23a615f5d9a0b0/scipy-1.17.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0cf46c8013fec9d3694dc572f0b54100c28405d55d3e2cb15e2895b25057996e", size = 37405953, upload-time = "2026-01-10T21:26:07.75Z" }, + { url = "https://files.pythonhosted.org/packages/92/ce/672ed546f96d5d41ae78c4b9b02006cedd0b3d6f2bf5bb76ea455c320c28/scipy-1.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:0937a0b0d8d593a198cededd4c439a0ea216a3f36653901ea1f3e4be949056f8", size = 36328121, upload-time = "2026-01-10T21:26:16.509Z" }, + { url = "https://files.pythonhosted.org/packages/9d/21/38165845392cae67b61843a52c6455d47d0cc2a40dd495c89f4362944654/scipy-1.17.0-cp312-cp312-win_arm64.whl", hash = "sha256:f603d8a5518c7426414d1d8f82e253e454471de682ce5e39c29adb0df1efb86b", size = 24314368, upload-time = "2026-01-10T21:26:23.087Z" }, +] + +[[package]] +name = "setuptools" +version = "84.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/b3/c19426f783614d625699a4b247faf628ea9ab1db228674491767b417f9e1/setuptools-84.2.1.tar.gz", hash = "sha256:3ff8b3b5492a0aa66960f4419365a86b84e013070d4313e4d4a6894f5d64d6bc", size = 1230289, upload-time = "2026-10-20T19:01:37.742Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/55/6dc6d6bb953bdfc52519a0059592c52b6e3b8ad51cf8256dffceb55b0610/setuptools-84.2.1-py3-none-any.whl", hash = "sha256:ca140b45d5325a9478f880e351484f62fef15f336462e449ca0c03c873d589c2", size = 876263, upload-time = "2026-10-20T19:01:35.678Z" }, +] + +[[package]] +name = "sie-sdk" +version = "0.10.1" +source = { git = "https://github.com/superlinked/sie.git?subdirectory=packages%2Fsie_sdk&rev=975d5309ba3af9eaeb13725b20dcf10d906c3e2a#975d5309ba3af9eaeb13725b20dcf10d906c3e2a" } +dependencies = [ + { name = "aiohttp" }, + { name = "httpx" }, + { name = "msgpack" }, + { name = "msgpack-numpy" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "pyyaml" }, + { name = "websockets" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "threadpoolctl" +version = "3.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/dc/6c58154c1c65f758ea979e7139cb76993a9cfc662d14e9be3c4a667cfb77/threadpoolctl-3.7.0.tar.gz", hash = "sha256:61348cfb77d53b9242e0017029244b559b810c142ced65b4e21eeca1843959a7", size = 31961, upload-time = "2026-09-15T15:46:20.263Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/3f/f88a53f60a472b46f4023f56d204dd7de33d34c5d2acbfa0d70a674e639e/threadpoolctl-3.7.0-py3-none-any.whl", hash = "sha256:cd8b60b5641b45c67bbf73c64c843235fc2d8a480c87389f52f5dbee893b86be", size = 26362, upload-time = "2026-09-15T15:46:19.168Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.22.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, + { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, + { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, + { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, +] + +[[package]] +name = "torch" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/27/07c645c7673e73e53ded71705045d6cb5bae94c4b021b03aa8d03eee90ab/torch-2.9.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:da5f6f4d7f4940a173e5572791af238cb0b9e21b1aab592bd8b26da4c99f1cd6", size = 104126592, upload-time = "2025-11-12T15:20:41.62Z" }, + { url = "https://files.pythonhosted.org/packages/19/17/e377a460603132b00760511299fceba4102bd95db1a0ee788da21298ccff/torch-2.9.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:27331cd902fb4322252657f3902adf1c4f6acad9dcad81d8df3ae14c7c4f07c4", size = 899742281, upload-time = "2025-11-12T15:22:17.602Z" }, + { url = "https://files.pythonhosted.org/packages/b1/1a/64f5769025db846a82567fa5b7d21dba4558a7234ee631712ee4771c436c/torch-2.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:81a285002d7b8cfd3fdf1b98aa8df138d41f1a8334fd9ea37511517cedf43083", size = 110940568, upload-time = "2025-11-12T15:21:18.689Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ab/07739fd776618e5882661d04c43f5b5586323e2f6a2d7d84aac20d8f20bd/torch-2.9.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:c0d25d1d8e531b8343bea0ed811d5d528958f1dcbd37e7245bc686273177ad7e", size = 74479191, upload-time = "2025-11-12T15:21:25.816Z" }, +] + +[[package]] +name = "tqdm" +version = "4.70.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/ea/b2a5bd54b28a324dae8211928b2d730b6547500342c7e6c6dea08bd0a485/tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4", size = 171846, upload-time = "2026-09-11T07:25:16.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73", size = 80199, upload-time = "2026-09-11T07:25:14.599Z" }, +] + +[[package]] +name = "train-a-router" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "numpy" }, +] + +[package.dev-dependencies] +collection = [ + { name = "httpx" }, +] +dev = [ + { name = "ruff" }, +] +encode = [ + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "sie-sdk" }, + { name = "transformers" }, +] +outcomes = [ + { name = "scikit-learn" }, + { name = "scipy" }, +] +training = [ + { name = "laya" }, + { name = "peft" }, + { name = "torch" }, + { name = "transformers" }, +] + +[package.metadata] +requires-dist = [{ name = "numpy", specifier = "==2.4.6" }] + +[package.metadata.requires-dev] +collection = [{ name = "httpx", specifier = "==0.28.1" }] +dev = [{ name = "ruff", specifier = ">=0.14" }] +encode = [ + { name = "httpx", specifier = "==0.28.1" }, + { name = "huggingface-hub", specifier = "==0.36.0" }, + { name = "sie-sdk", git = "https://github.com/superlinked/sie.git?subdirectory=packages%2Fsie_sdk&rev=975d5309ba3af9eaeb13725b20dcf10d906c3e2a" }, + { name = "transformers", specifier = "==4.57.6" }, +] +outcomes = [ + { name = "scikit-learn", specifier = "==1.6.1" }, + { name = "scipy", specifier = "==1.17.0" }, +] +training = [ + { name = "laya", specifier = "==0.3.29" }, + { name = "peft", specifier = "==0.18.1" }, + { name = "torch", specifier = "==2.9.1" }, + { name = "transformers", specifier = "==4.57.6" }, +] + +[[package]] +name = "transformers" +version = "4.57.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "requests" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/35/67252acc1b929dc88b6602e8c4a982e64f31e733b804c14bc24b47da35e6/transformers-4.57.6.tar.gz", hash = "sha256:55e44126ece9dc0a291521b7e5492b572e6ef2766338a610b9ab5afbb70689d3", size = 10134912, upload-time = "2026-01-16T10:38:39.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/b8/e484ef633af3887baeeb4b6ad12743363af7cce68ae51e938e00aaa0529d/transformers-4.57.6-py3-none-any.whl", hash = "sha256:4c9e9de11333ddfe5114bc872c9f370509198acf0b87a832a0ab9458e2bd0550", size = 11993498, upload-time = "2026-01-16T10:38:31.289Z" }, +] + +[[package]] +name = "triton" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/50/9a8358d3ef58162c0a415d173cfb45b67de60176e1024f71fbc4d24c0b6d/triton-3.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d2c6b915a03888ab931a9fd3e55ba36785e1fe70cbea0b40c6ef93b20fc85232", size = 170470207, upload-time = "2025-11-11T17:41:00.253Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + +[[package]] +name = "websockets" +version = "14.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/54/8359678c726243d19fae38ca14a334e740782336c9f19700858c4eb64a1e/websockets-14.2.tar.gz", hash = "sha256:5059ed9c54945efb321f097084b4c7e52c246f2c869815876a69d1efc4ad6eb5", size = 164394, upload-time = "2025-01-19T21:00:56.431Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/81/04f7a397653dc8bec94ddc071f34833e8b99b13ef1a3804c149d59f92c18/websockets-14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1f20522e624d7ffbdbe259c6b6a65d73c895045f76a93719aa10cd93b3de100c", size = 163096, upload-time = "2025-01-19T20:59:29.763Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c5/de30e88557e4d70988ed4d2eabd73fd3e1e52456b9f3a4e9564d86353b6d/websockets-14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:647b573f7d3ada919fd60e64d533409a79dcf1ea21daeb4542d1d996519ca967", size = 160758, upload-time = "2025-01-19T20:59:32.095Z" }, + { url = "https://files.pythonhosted.org/packages/e5/8c/d130d668781f2c77d106c007b6c6c1d9db68239107c41ba109f09e6c218a/websockets-14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6af99a38e49f66be5a64b1e890208ad026cda49355661549c507152113049990", size = 160995, upload-time = "2025-01-19T20:59:33.527Z" }, + { url = "https://files.pythonhosted.org/packages/a6/bc/f6678a0ff17246df4f06765e22fc9d98d1b11a258cc50c5968b33d6742a1/websockets-14.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:091ab63dfc8cea748cc22c1db2814eadb77ccbf82829bac6b2fbe3401d548eda", size = 170815, upload-time = "2025-01-19T20:59:35.837Z" }, + { url = "https://files.pythonhosted.org/packages/d8/b2/8070cb970c2e4122a6ef38bc5b203415fd46460e025652e1ee3f2f43a9a3/websockets-14.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b374e8953ad477d17e4851cdc66d83fdc2db88d9e73abf755c94510ebddceb95", size = 169759, upload-time = "2025-01-19T20:59:38.216Z" }, + { url = "https://files.pythonhosted.org/packages/81/da/72f7caabd94652e6eb7e92ed2d3da818626e70b4f2b15a854ef60bf501ec/websockets-14.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a39d7eceeea35db85b85e1169011bb4321c32e673920ae9c1b6e0978590012a3", size = 170178, upload-time = "2025-01-19T20:59:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/31/e0/812725b6deca8afd3a08a2e81b3c4c120c17f68c9b84522a520b816cda58/websockets-14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0a6f3efd47ffd0d12080594f434faf1cd2549b31e54870b8470b28cc1d3817d9", size = 170453, upload-time = "2025-01-19T20:59:41.996Z" }, + { url = "https://files.pythonhosted.org/packages/66/d3/8275dbc231e5ba9bb0c4f93144394b4194402a7a0c8ffaca5307a58ab5e3/websockets-14.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:065ce275e7c4ffb42cb738dd6b20726ac26ac9ad0a2a48e33ca632351a737267", size = 169830, upload-time = "2025-01-19T20:59:44.669Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ae/e7d1a56755ae15ad5a94e80dd490ad09e345365199600b2629b18ee37bc7/websockets-14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e9d0e53530ba7b8b5e389c02282f9d2aa47581514bd6049d3a7cffe1385cf5fe", size = 169824, upload-time = "2025-01-19T20:59:46.932Z" }, + { url = "https://files.pythonhosted.org/packages/b6/32/88ccdd63cb261e77b882e706108d072e4f1c839ed723bf91a3e1f216bf60/websockets-14.2-cp312-cp312-win32.whl", hash = "sha256:20e6dd0984d7ca3037afcb4494e48c74ffb51e8013cac71cf607fffe11df7205", size = 163981, upload-time = "2025-01-19T20:59:49.228Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7d/32cdb77990b3bdc34a306e0a0f73a1275221e9a66d869f6ff833c95b56ef/websockets-14.2-cp312-cp312-win_amd64.whl", hash = "sha256:44bba1a956c2c9d268bdcdf234d5e5ff4c9b6dc3e300545cbe99af59dda9dcce", size = 164421, upload-time = "2025-01-19T20:59:50.674Z" }, + { url = "https://files.pythonhosted.org/packages/7b/c8/d529f8a32ce40d98309f4470780631e971a5a842b60aec864833b3615786/websockets-14.2-py3-none-any.whl", hash = "sha256:7a6ceec4ea84469f15cf15807a747e9efe57e369c384fa86e022b3bea679b79b", size = 157416, upload-time = "2025-01-19T21:00:54.843Z" }, +] + +[[package]] +name = "yarl" +version = "1.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/16/e8be8e2fb175bbf41a0680381a319f1199fae256588241a2ac8677eafb49/yarl-1.25.1.tar.gz", hash = "sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7", size = 246245, upload-time = "2026-09-15T19:35:02.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/b3/cd32ac66ae622b854c2df0ac52106dda220d361b65a64fde7d5b3684aa3f/yarl-1.25.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4", size = 144798, upload-time = "2026-09-15T19:31:01.821Z" }, + { url = "https://files.pythonhosted.org/packages/61/fb/a2c52a8007c2051ba74662afb112ecf3d00346af4c25e33df9d80fd14fb8/yarl-1.25.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d", size = 104583, upload-time = "2026-09-15T19:31:04.05Z" }, + { url = "https://files.pythonhosted.org/packages/be/dd/ee38aec8e09fdf957e50d4085453fbe202f56c6c3b4cf07b81cdb4f09ee9/yarl-1.25.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6", size = 104325, upload-time = "2026-09-15T19:31:06.338Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b3/058dbfb1857b484c9cf9cc135659f50b85ce66e03c99e44dc2f7b6161f55/yarl-1.25.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58", size = 115358, upload-time = "2026-09-15T19:31:08.593Z" }, + { url = "https://files.pythonhosted.org/packages/db/39/29693446cf0cf6b15a0e2f75a5d40f93c56819b05b0622196f45e95b5cc0/yarl-1.25.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad", size = 107658, upload-time = "2026-09-15T19:31:10.802Z" }, + { url = "https://files.pythonhosted.org/packages/86/b3/3c4dd7e1af43b931fba95e0a722737f2ea94a6d199c802585282831d7abd/yarl-1.25.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40", size = 122660, upload-time = "2026-09-15T19:31:12.84Z" }, + { url = "https://files.pythonhosted.org/packages/bd/b5/1b60dbc3cfc9c5712b15148c206748f2bc93953ffdbe25ea75b63dfc89c9/yarl-1.25.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c", size = 126506, upload-time = "2026-09-15T19:31:15.088Z" }, + { url = "https://files.pythonhosted.org/packages/bc/7b/ca212cbe170ac8b96e45317ecbcf9c3c3ecf0cdec98d5b088a9c4088929b/yarl-1.25.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d", size = 117050, upload-time = "2026-09-15T19:31:17.241Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c3/72b4938cdbe619ad71ac156182faef4908846b84dc3ca4dbb4c4e6f84014/yarl-1.25.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54", size = 114174, upload-time = "2026-09-15T19:31:19.294Z" }, + { url = "https://files.pythonhosted.org/packages/e8/43/268717870f9ba0cc9701a95181587f6dc8c5f387aab4aeecc83158f38a79/yarl-1.25.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f", size = 114944, upload-time = "2026-09-15T19:31:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/da/84/baa5bf504d51fe062c4bcaf62936da97fffb43285978d0b39984824231fd/yarl-1.25.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc", size = 108263, upload-time = "2026-09-15T19:31:23.388Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/779a2ed9e0152a601a27039bed9aead3f0b79797a67e2c44bfa444622dd8/yarl-1.25.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d", size = 122184, upload-time = "2026-09-15T19:31:25.343Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1f/118e9e5b8f07694d63fd3222e801d7782270003f1a222aa798df3f8d5933/yarl-1.25.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1", size = 114001, upload-time = "2026-09-15T19:31:27.465Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/a4cf1cf372313734b17996d4007f9f73596e7a178b9485802e5494ecf484/yarl-1.25.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68", size = 120565, upload-time = "2026-09-15T19:31:29.47Z" }, + { url = "https://files.pythonhosted.org/packages/05/79/ad94f93ca731bc9e44d321833ab96b82a4f9f5f63cf773f81a4aeea5ecc1/yarl-1.25.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707", size = 117060, upload-time = "2026-09-15T19:31:31.367Z" }, + { url = "https://files.pythonhosted.org/packages/bb/cc/51a7b4abf4ac593b8e7eb3794b28e5a35ae26eed8bc04787628d215af82f/yarl-1.25.1-cp312-cp312-win_amd64.whl", hash = "sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03", size = 102593, upload-time = "2026-09-15T19:31:33.329Z" }, + { url = "https://files.pythonhosted.org/packages/9d/21/0941a6b93a58b59a1ec75e5333bf06929b671309c43c0cd201c172d9c39f/yarl-1.25.1-cp312-cp312-win_arm64.whl", hash = "sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85", size = 97697, upload-time = "2026-09-15T19:31:35.855Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/318c7980066769c6bcd9221ed2248294f5698811da099013098c670565ed/yarl-1.25.1-py3-none-any.whl", hash = "sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3", size = 63617, upload-time = "2026-09-15T19:34:59.616Z" }, +] diff --git a/examples/train-a-router/workflow.py b/examples/train-a-router/workflow.py new file mode 100644 index 000000000..61a977daa --- /dev/null +++ b/examples/train-a-router/workflow.py @@ -0,0 +1,236 @@ +"""Short reader commands around the pinned scientific recipe implementations.""" + +from __future__ import annotations + +import math +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import cost_reader +import evidence +from embedding import recipes as core +from embedding.cost_iteration import refit as cost_refit +from evaluation import frontier + +ROOT = Path(__file__).resolve().parent +ROLES = ("calibration", "final") + + +def invoke(script, *arguments): + """Run an optional component with UTF-8 defaults and its original sibling import layout.""" + script = ROOT / script + environment = {**os.environ, "PYTHONPATH": os.pathsep.join((str(ROOT), str(script.parent)))} + subprocess.run([sys.executable, "-X", "utf8", str(script), *map(str, arguments)], env=environment, check=True) + + +def fetch(output, groups, release_path=None): + release = core.read(release_path or ROOT / "release.json") + core.require(release["format"] == "train-a-router-release-v1", "Unknown artifact release") + return evidence.fetch(release["manifest_url"], release["manifest_sha256"], output, groups=groups) + + +def checked_data(data): + data = Path(data).resolve() + release = core.read(ROOT / "release.json") + evidence.verify(data, manifest_sha256=release["manifest_sha256"], groups=("core",)) + return data + + +def prepare(data, run): + data, run = checked_data(data), Path(run).resolve() + core.require(not run.exists(), "Use a new run directory") + run.mkdir(parents=True) + core.prepare(data / "source", ROOT / "pair.json", run / "embedding") + original = core.read(data / "reference/native/plan.json") + invoke("native/recipe.py", "prepare", "--source", data / "source", "--pair", ROOT / "pair.json", + "--lengths", data / "reference/native/lengths.json", "--output", run / "native", + "--acquisition-protocol-sha256", original["acquisition_protocol_sha256"]) + (run / "scores").mkdir() + core.write(run / "reproduction.json", { + "format": "bbeh-public-reproduction-v1", + "evaluation_mode": "post_publication_reproduction", + "test_answers_already_published": True, + "new_blind_evaluation": False, + "manifest_sha256": core.sha(data / "manifest.json"), + "artifact_directory": str(data), + "reference_frozen_sha256": core.sha(data / "reference/frozen-frontiers.json"), + "source_plan_sha256": original["source_plan_sha256"], + "purpose": "Repeat the recipe using the published training and held-out answers.", + }) + return {"run": str(run), "prepared": True, "model_calls": False} + + +def checked_run(data, run): + data, run = checked_data(data), Path(run).resolve() + record = core.read(run / "reproduction.json") + core.require(record["format"] == "bbeh-public-reproduction-v1" + and record["manifest_sha256"] == core.sha(data / "manifest.json") + and record["reference_frozen_sha256"] == core.sha(data / "reference/frozen-frontiers.json") + and record["new_blind_evaluation"] is False, "Run belongs to another published dataset") + core.checked_plan(run / "embedding") + return data, run + + +def fit_embedding(data, run, features=None): + data, run = checked_run(data, run) + model = core.fit(run / "embedding", data / "grades/grades-train.json", + data / "grades/grades-development.json", features or data / "features") + return {"selected": model["selected"], "timing": model["timing"], "run": str(run)} + + +def fit_native(data, run): + data, run = checked_run(data, run) + invoke("native/bbeh_native.py", "--phase", "train", "--run", run / "native", + "--training-grades", data / "grades/grades-train.json", + "--development-grades", data / "grades/grades-development.json", + "--initial-checkpoint", data / "reference/native/native/base") + trained = core.read(run / "native/native/trained.json") + return {"selected_epoch": trained["selected_epoch"], "elapsed_seconds": trained["elapsed_seconds"]} + + +def base(data, run, device): + data, run = checked_run(data, run) + output = run / "unchanged-base" + if not output.exists(): + invoke("native/base.py", "prepare", "--run", run / "native", + "--training-grades", data / "grades/grades-train.json", + "--development-grades", data / "grades/grades-development.json", + "--initial-checkpoint", data / "reference/native/native/base", "--output", output) + for role in ROLES: + invoke("native/base.py", "score", "--run", run / "native", "--base", output, + "--split", role, "--device", device, "--output", run / f"scores/base-{role}.json") + return {"run": str(run), "method": "base", "device": device, "held_out_outcomes_accessed": False} + + +def score(data, run, method, device="cuda", features=None): + data, run = checked_run(data, run) + if method == "embedding": + for role in ROLES: + target = run / f"scores/embedding-{role}.json" + core.require(not target.exists(), "Preserve previous scores") + core.score(run / "embedding", features or data / "features", role, + run / f"embedding/scores-{role}.json") + shutil.copyfile(run / f"embedding/scores-{role}.json", target) + else: + invoke("native/reader.py", "--run", run / "native", "--device", device, + "--output", run / "native/native/input-scores.json") + for role in ROLES: + target = run / f"scores/native-{role}.json" + core.require(not target.exists(), "Preserve previous scores") + shutil.copyfile(run / f"native/native/scores-{role}.json", target) + return {"run": str(run), "method": method, "held_out_outcomes_accessed": False} + + +def reuse_native(data, run): + data, run = checked_run(data, run) + reference = data / "reference" + core.require(core.sha(run / "native/plan.json") == core.sha(reference / "native/plan.json"), + "Reference native scores require the identical recipe plan") + for role in ROLES: + target = run / f"scores/native-{role}.json" + core.require(not target.exists(), "Existing native scores; preserve them") + shutil.copyfile(reference / f"scores/native-{role}.json", target) + core.write(run / "native-comparison.json", { + "fitting_performed": False, + "scores_sha256": {role: core.sha(run / f"scores/native-{role}.json") for role in ROLES}, + "purpose": "Reuse supplied Laya comparisons during an embedding-only refit.", + }) + return {"run": str(run), "native_predictions_reused": True} + + +def fit_cost(data, run): + data, run = checked_run(data, run) + receipt = cost_reader.fit_score(run / "embedding", data / "grades/grades-train.json", + data / "grades/grades-development.json", run / "cost") + for role in ROLES: + cost_refit.augment_files(run / f"scores/native-{role}.json", run / f"cost/costs-{role}.json", + run / f"cost/native-scores-{role}.json", run / "cost/native-combined-model.json") + return {"run": str(run), "cost_model": str(run / "cost/cost-model.json"), "alpha": receipt["alpha"]} + + +def calibrate(data, run): + """Keep post-publication artifacts explicitly marked as a reproduction.""" + data, run = checked_run(data, run) + internal = run / "internal" + core.require(not internal.exists(), "Preserve the existing calibration; use a fresh run") + internal.mkdir() + frontier.calibrate(run / "embedding", data / "grades/grades-calibration.json", + [run / "cost/scores-calibration.json", run / "cost/native-scores-calibration.json"], + data / "fees.json", internal / "calibration-frontiers.json") + frontier.freeze(run / "embedding", internal / "calibration-frontiers.json", + [run / "cost/scores-final.json", run / "cost/native-scores-final.json"], + data / "fees.json", internal / "frozen-frontiers.json") + core.write(run / "calibration.json", { + "format": "bbeh-public-calibration-v1", "evaluation_mode": "post_publication_reproduction", + "new_blind_evaluation": False, "final_outcomes_read_by_calibrator": False, + "calibration_sha256": core.sha(internal / "calibration-frontiers.json"), + "frozen_sha256": core.sha(internal / "frozen-frontiers.json"), + "reproduction_sha256": core.sha(run / "reproduction.json"), + "router_fee_scenario": "Published SIE Cloud singleton fees; self-hosting is unpriced.", + }) + return {"run": str(run), "calibration": str(run / "calibration.json"), "new_model_calls": 0} + + +def evaluate_refit(data, run, output): + data, run = checked_run(data, run) + output, internal = Path(output).resolve(), run / "internal" + core.require(not output.exists(), "Use a new report directory") + envelope = core.read(run / "calibration.json") + core.require(envelope["new_blind_evaluation"] is False + and envelope["reproduction_sha256"] == core.sha(run / "reproduction.json") + and envelope["frozen_sha256"] == core.sha(internal / "frozen-frontiers.json") + and envelope["calibration_sha256"] == core.sha(internal / "calibration-frontiers.json"), + "Public reproduction lineage changed") + output.mkdir(parents=True) + actual = frontier.evaluate(run / "embedding", internal / "frozen-frontiers.json", + data / "grades/grades-final.json", output / "final-report.json") + invoke("evaluation/source_sensitivity.py", "--run", run / "embedding", + "--frozen", internal / "frozen-frontiers.json", "--grades", data / "grades/grades-final.json", + "--primary", output / "final-report.json", "--specification", ROOT / "source-format-sensitivity.json", + "--output", output / "source-format-sensitivity.json") + core.write(output / "reproduction.json", { + **envelope, "final_report_sha256": core.sha(output / "final-report.json"), + "interpretation": "A refit evaluated on already published test answers, not a new blind test.", + }) + invoke("evaluation/report.py", "--calibration", internal / "calibration-frontiers.json", + "--frozen", internal / "frozen-frontiers.json", "--final-report", output / "final-report.json", + "--output", output / "explorer", "--reproduction") + return {"report": str(output / "final-report.json"), "open": str(output / "explorer/index.html"), + "questions": actual["n"], "new_blind_evaluation": False, "new_model_calls": 0} + + +def same_values(expected, actual): + """Allow floating-point roundoff across CPUs without ignoring altered results.""" + if type(expected) is not type(actual): + return False + if isinstance(expected, dict): + return expected.keys() == actual.keys() and all(same_values(expected[k], actual[k]) for k in expected) + if isinstance(expected, list): + return len(expected) == len(actual) and all(same_values(a, b) for a, b in zip(expected, actual, strict=True)) + if isinstance(expected, float): + return math.isfinite(expected) and math.isfinite(actual) and math.isclose( + expected, actual, rel_tol=1e-10, abs_tol=1e-10) + return expected == actual + + +def replay(data, output): + data, output = checked_data(data), Path(output).resolve() + core.require(not output.exists(), "Use a new report directory") + output.mkdir(parents=True) + reference = data / "reference" + actual = frontier.evaluate(reference / "embedding", reference / "frozen-frontiers.json", + data / "grades/grades-final.json", output / "final-report.json") + core.require(same_values(core.read(reference / "final-report.json"), actual), + "Replay differs from the published report; keep the output for inspection") + invoke("evaluation/source_sensitivity.py", "--run", reference / "embedding", + "--frozen", reference / "frozen-frontiers.json", "--grades", data / "grades/grades-final.json", + "--primary", output / "final-report.json", "--specification", ROOT / "source-format-sensitivity.json", + "--output", output / "source-format-sensitivity.json") + invoke("evaluation/report.py", "--calibration", reference / "calibration-frontiers.json", + "--frozen", reference / "frozen-frontiers.json", "--final-report", output / "final-report.json", + "--output", output / "explorer") + return {"report": str(output / "final-report.json"), "open": str(output / "explorer/index.html"), + "questions": actual["n"], "matches_published_result": True, "new_model_calls": 0}