From c3eb42fc7cbee75a06026341616e1e1aea33e518 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 5 Oct 2026 12:24:29 -0600 Subject: [PATCH 1/2] docs(models): add workflow routing and single-node serving skills Signed-off-by: Doug Holt --- AGENTS.md | 13 +++- scripts/validation/tests/test_model_skills.py | 57 +++++++++++++++ skills/README.md | 2 + skills/model-workflows/SKILL.md | 49 +++++++++++++ skills/serve-model/SKILL.md | 71 +++++++++++++++++++ 5 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 scripts/validation/tests/test_model_skills.py create mode 100644 skills/model-workflows/SKILL.md create mode 100644 skills/serve-model/SKILL.md diff --git a/AGENTS.md b/AGENTS.md index a84a5fdcf..258b73d2a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,6 +58,15 @@ python3 scripts/validation/validate_k8s.py --json --cuda-smoke The validator must report `"ok": true` with `cuda_smoke_ok: true`. See `skills/deploy-k8s-gpu-cluster/` for the full procedure and failure branches. +## Choose a model workflow + +Start with `skills/model-workflows/` to distinguish cached inputs, single-node +inference and training. Use only the documented cache and serving paths below. +Fine-tuning has no supported recipe or training-result validator here yet; stop +and obtain a reviewed recipe and success criteria rather than reporting cache +readiness as training success. Multi-node workflows are not supported by these +skills. + ## Golden path: shared model cache Use `playbooks/model-cache.yml` only on an approved, existing NFS mount and one @@ -73,8 +82,8 @@ model coverage. See `skills/manage-model-cache/` and ## Golden path: single-node model serving -Follow `docs/model-workflows/vllm-quickstart.md` inside an approved, isolated GPU -allocation. Use the pinned local snapshot, one GPU, and a loopback-only endpoint. +Use `skills/serve-model/` and follow `docs/model-workflows/vllm-quickstart.md` +inside an approved, isolated GPU allocation. Use the pinned local snapshot, one GPU, and a loopback-only endpoint. After startup, run `scripts/validation/validate_vllm.py --json` with the expected served model name and the cache validator's explicit inputs. This submits one small completion request; it does not start a server or download models. Require diff --git a/scripts/validation/tests/test_model_skills.py b/scripts/validation/tests/test_model_skills.py new file mode 100644 index 000000000..ec33f8957 --- /dev/null +++ b/scripts/validation/tests/test_model_skills.py @@ -0,0 +1,57 @@ +"""Offline documentation contracts; these do not test agent behavior or GPUs.""" +import pathlib +import re +import shlex +import subprocess +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[3] +SKILLS = ("model-workflows", "serve-model") + + +class ModelSkillDocsTest(unittest.TestCase): + def skill(self, name): + path = ROOT / "skills" / name / "SKILL.md" + self.assertTrue(path.is_file(), f"Missing skill: {name}") + return path, path.read_text() + + def test_discovery(self): + for name in SKILLS: + with self.subTest(skill=name): + _, text = self.skill(name) + self.assertIn(f"name: {name}\n", text) + self.assertRegex(text, r"(?m)^description: Use when .+") + for index in ("AGENTS.md", "skills/README.md"): + self.assertIn(f"{name}/", (ROOT / index).read_text()) + + def test_relative_links_exist(self): + for name in SKILLS: + path, text = self.skill(name) + links = re.findall(r"\[[^\]]+\]\(([^)]+)\)", text) + self.assertTrue(links) + for link in links: + with self.subTest(skill=name, link=link): + self.assertNotIn("://", link) + self.assertTrue((path.parent / link.split("#")[0]).is_file()) + + def test_serving_command_matches_quickstart(self): + _, text = self.skill("serve-model") + quickstart = (ROOT / "docs/model-workflows/vllm-quickstart.md").read_text() + commands = re.findall(r"```bash\n(.*?)\n```", text, re.S) + self.assertEqual(len(commands), 1) + command = commands[0] + self.assertIn(command, quickstart) + args = shlex.split(command.replace("\\\n", " ")) + self.assertEqual(args[:2], ["python3", "scripts/validation/validate_vllm.py"]) + # --help parses the real CLI without touching the cache or endpoint. + result = subprocess.run(args + ["--help"], cwd=ROOT, + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + for flag in args[2:]: + if flag.startswith("--"): + self.assertIn(flag, result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/README.md b/skills/README.md index 0d16e8656..a981d30d2 100644 --- a/skills/README.md +++ b/skills/README.md @@ -15,7 +15,9 @@ agent (or person) can simply read the relevant `SKILL.md` before acting. | [deploy-k8s-gpu-cluster](deploy-k8s-gpu-cluster/SKILL.md) | Deploying or rebuilding a Kubernetes GPU cluster. | | [provision-with-maas](provision-with-maas/SKILL.md) | Provisioning, tagging, validating, or releasing machines through MAAS. | | [deploy-airgapped](deploy-airgapped/SKILL.md) | Preparing mirrors and deploying Slurm or Kubernetes without Internet access. | +| [model-workflows](model-workflows/SKILL.md) | Choosing cache or single-node serving procedures; identifying unsupported training or multi-node requests. | | [manage-model-cache](manage-model-cache/SKILL.md) | Preparing shared model files, pinning revisions, or diagnosing offline cache access. | +| [serve-model](serve-model/SKILL.md) | Serving a cached snapshot on one GPU and validating a local completion endpoint. | | [validate-gpu-cluster](validate-gpu-cluster/SKILL.md) | Health checks and post-deploy verification. | | [diagnose-driver-install](diagnose-driver-install/SKILL.md) | NVIDIA driver failures, `nvidia-smi` errors, GPU pods crash-looping. | diff --git a/skills/model-workflows/SKILL.md b/skills/model-workflows/SKILL.md new file mode 100644 index 000000000..de8fdc1c5 --- /dev/null +++ b/skills/model-workflows/SKILL.md @@ -0,0 +1,49 @@ +--- +name: model-workflows +description: Use when choosing a DeepOps model workflow, combining shared model caching with single-node serving, or determining whether a fine-tuning or multi-node request has a supported procedure. +--- + +# Choose a model workflow + +This is the umbrella entry point, not a launcher. Start from the repository +root. Identify the requested outcome before running anything: cached inputs, +working inference, or training are different outcomes with different evidence. + +## Select the supported path + +| Requested outcome | Procedure | Evidence | +|---|---|---| +| Prepare or repair shared model inputs | [Manage model cache](../manage-model-cache/SKILL.md) | Cache validator exit `0` and `ok: true`, as the real consumer. Only named-file readability is proven. | +| Serve a pinned snapshot on one GPU | [Serve model](../serve-model/SKILL.md) | Cache gate first, then serving validator exit `0` and `ok: true`. One bounded completion, not training or performance validation. | +| Fine-tune a model | Stop: no supported training recipe or training-result validator is provided here yet. | Cache readiness is not training success. Obtain a reviewed single-node recipe and explicit success criteria before proceeding. | +| Multi-node serving or training | Stop: unsupported by these skills. | Do not infer distributed support from a successful single-node check. | + +## Common preflight + +1. Confirm the operator's goal, model license, repository ID, immutable full + revision, complete required-file list, cache path and real reader UID/group. + Do not accept a directory or `config.json` alone as complete model coverage. +2. Confirm the target and authorization for each action. Cache preparation, + downloads, package installation, GPU allocation, service startup and inference + requests are separate actions. A skill or validator grants none of them. +3. Reuse the existing cache interface: share only `HF_HUB_CACHE`, use a designated + writer and pass the local snapshot path to consumers. Mount the whole cache + read-only; keep authentication and framework caches private. Never put training + outputs or checkpoints in the shared input cache. +4. Follow the selected skill's failure branches. Stop on absent authorization, + incomplete files, space pressure, unknown endpoint ownership or required site + authentication. Never remove controls, change infrastructure or delete other + users' files to turn a failure into a pass. + +## Report only the evidence obtained + +Record which validator ran, its exit code, schema version, explicit inputs and +checks. The [cache guide](../../docs/model-workflows/shared-cache.md) and +[serving guide](../../docs/model-workflows/vllm-quickstart.md) define their JSON +contracts; this umbrella does not invent a combined success field. + +A readable cache is preparation only. A completion is inference only. Neither +proves that a particular revision was loaded, a GPU was used, or training +succeeded. A checkpoint filename by itself is not training validation. Keep +startup logs private; do not include credentials or sensitive prompts in reports. +No multi-node hooks or implicit fallback to a different workflow are provided. diff --git a/skills/serve-model/SKILL.md b/skills/serve-model/SKILL.md new file mode 100644 index 000000000..7a750b7aa --- /dev/null +++ b/skills/serve-model/SKILL.md @@ -0,0 +1,71 @@ +--- +name: serve-model +description: Use when serving a cached model on one GPU with vLLM, checking a local completion endpoint, or diagnosing startup and inference failures in the single-node quickstart. +--- + +# Serve a model on one node + +Run from the repository root on the allocated serving node. Read the +[vLLM quickstart](../../docs/model-workflows/vllm-quickstart.md) for the exact +installation and foreground launch commands. This skill does not allocate a GPU +or authorize a workload. + +## Preconditions + +- Confirm permission for the node, one isolated GPU, package installation, + server startup and the validator's inference request. Record the allocation + and the process you own. Never use a login shell outside that allocation or + override scheduler device isolation. +- Confirm model license, immutable revision, complete file list, reader identity, + cache mount and served alias. Use [manage-model-cache](../manage-model-cache/SKILL.md) + for missing files; do not make consumers download them. +- Require Python 3.12 and a fresh private serving environment using + `examples/vllm/requirements.txt`. Check wheel/driver compatibility against the + quickstart; do not change drivers or the cache writer's environment. +- Keep the whole Hub cache read-only, including blobs. Keep authentication, + `HF_HOME`, Xet and framework caches private. Pause snapshot writes during use. + +## Procedure and success gate + +1. Follow quickstart steps 1–2. Require cache validator exit `0` **and** `ok: true` + as the actual reader before startup. Config alone is not a complete model; + name every required weight shard and tokenizer file. +2. Use quickstart step 3 unchanged for the public example: local snapshot and + tokenizer, offline loading, one GPU, foreground process, loopback only. Never + enable remote model code. `HF_HUB_OFFLINE=1` is not a firewall and belongs only + on the loading process, not globally on clients. +3. Wait for startup; use another shell in the same allocation and namespace to + validate. This submits inference, not just a read-only health check: + +```bash +python3 scripts/validation/validate_vllm.py --json \ + --base-url http://127.0.0.1:8000 --model deepops-gpt2 \ + --cache-dir /shared/hf-cache/hub \ + --repo-id openai-community/gpt2 \ + --revision 607a30d783dfa663caf39e06633721c8d4cfcd7e \ + --require-file config.json --require-file tokenizer.json --require-file model.safetensors +``` + +4. Require exit `0` **and** `ok: true`. Schema version 1 checks cache, health, + advertised model and bounded completion in order. Exit `1` fails validation; + exit `2` is a command syntax error without guaranteed JSON. A false check can + mean skipped; inspect the first error stage. Preserve the invocation, package + versions and private startup log. Success does not prove loaded revision, + GPU use, quality or throughput; an alias alone is not revision attestation. +5. Stop only your own foreground server with Ctrl-C. Confirm exit before releasing + the allocation. Do not kill another listener or prune the shared cache. + +## Stop and recovery + +| Observation | Action | +|---|---| +| Missing files or unreadable cache | Stop; designated writer repairs the pinned snapshot, then reader revalidates. Never loosen permissions globally. | +| Full cache | Stop downloads; request operator-reviewed retention, not automatic deletion. | +| Port occupied or endpoint ownership unclear | Stop; identify the owner. Do not validate or kill it without permission. | +| Startup/health failure | Inspect private logs for compatibility, memory and files; rerun validation only after startup is confirmed. | +| Authentication required / HTTP 401 | Stop; use the site's approved authenticated workflow. Never disable controls for this example. | +| Model/completion failure | Compare alias, full file list and server log. Health alone is not success. | +| Multi-node, remote access or production service requested | Stop; unsupported here. Do not widen the bind address or improvise distributed flags. | + +Offline fixtures test the validator, not real cache reuse, GPU inference or +agent decision-making. Those require separate verification. From 22acd3e5e9090f4efb75a101fd7db69d1648cd85 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Tue, 6 Oct 2026 11:52:07 -0600 Subject: [PATCH 2/2] feat(models): add readiness-only fine-tuning validation Check single-node scheduler, container and cached model prerequisites without starting a workload by default. Keep the explicit one-GPU container probe separate from training success and cover failure paths offline. Co-Authored-By: Paperclip --- .github/workflows/model-workflows.yml | 19 ++ AGENTS.md | 22 +- docs/model-workflows/finetune-readiness.md | 118 ++++++++ .../tests/test_finetune_readiness.py | 255 ++++++++++++++++++ scripts/validation/tests/test_model_skills.py | 2 +- scripts/validation/validate_finetune.py | 209 ++++++++++++++ skills/README.md | 3 +- skills/finetune-model/SKILL.md | 62 +++++ skills/model-workflows/SKILL.md | 7 +- 9 files changed, 688 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/model-workflows.yml create mode 100644 docs/model-workflows/finetune-readiness.md create mode 100644 scripts/validation/tests/test_finetune_readiness.py create mode 100644 scripts/validation/validate_finetune.py create mode 100644 skills/finetune-model/SKILL.md diff --git a/.github/workflows/model-workflows.yml b/.github/workflows/model-workflows.yml new file mode 100644 index 000000000..afac283bc --- /dev/null +++ b/.github/workflows/model-workflows.yml @@ -0,0 +1,19 @@ +--- +name: offline model workflow validation +on: + - push + - pull_request +permissions: + contents: read +jobs: + offline-validation: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install local fixture dependencies + run: python3 -m pip install ansible==10.7.0 PyYAML==6.0.2 + - name: Run offline validator contracts + run: python3 -m unittest discover -s scripts/validation/tests -v diff --git a/AGENTS.md b/AGENTS.md index 258b73d2a..16cd772b8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,11 +61,11 @@ The validator must report `"ok": true` with `cuda_smoke_ok: true`. See ## Choose a model workflow Start with `skills/model-workflows/` to distinguish cached inputs, single-node -inference and training. Use only the documented cache and serving paths below. -Fine-tuning has no supported recipe or training-result validator here yet; stop -and obtain a reviewed recipe and success criteria rather than reporting cache -readiness as training success. Multi-node workflows are not supported by these -skills. +inference and training. Use the documented cache, serving and fine-tuning +readiness paths below. Fine-tuning has no training recipe or training-result +validator here yet; obtain a reviewed recipe and success criteria rather than +reporting readiness as training success. Multi-node workflows are not supported +by these skills. ## Golden path: shared model cache @@ -91,6 +91,18 @@ exit `0` and `ok: true`, but do not treat this as revision attestation, GPU-use proof, or a performance result. Never run it against someone else's service without permission. No live testing is implied by offline fixtures. +## Golden path: fine-tuning readiness (not training) + +Use `skills/finetune-model/` for the readiness-only scope. Run +`scripts/validation/validate_finetune.py --json` as the consumer on the selected +Slurm compute host, with explicit node, partition, digest-pinned image and cache +inputs. It checks scheduler availability, idle GPU capacity, local container +support and the supported model file layout. Exit `0` and `status: ready` mean +preparation only; `training_validated` is always false. No job runs by default. +The optional `--gpu-smoke` requires separate authorization and proves only a +single-GPU container probe. See `docs/model-workflows/finetune-readiness.md` for +limits, failure reasons and the distinction from training success. + ## Rules for operating this repository 1. **Validate, don't assume.** Run the doctor before deploying and the diff --git a/docs/model-workflows/finetune-readiness.md b/docs/model-workflows/finetune-readiness.md new file mode 100644 index 000000000..9c923bdbd --- /dev/null +++ b/docs/model-workflows/finetune-readiness.md @@ -0,0 +1,118 @@ +# Single-node fine-tuning readiness + +This is a **preparation check, not a training recipe**. It never reports training +success. Run `scripts/validation/validate_finetune.py` as the intended consumer +on the selected Slurm compute host, with the shared cache mounted there. It +uses Python's standard library and the existing local cache validator. + +## Supported boundary + +- One explicitly named Slurm node and partition. `scontrol show node` must + report the local hostname as `NodeHostName`; aliases not matching the local + hostname or its short form fail closed. The partition must be UP and include + the node. Node names cannot be ranges, lists or command options. +- The node must be exactly IDLE, with at least one configured GPU and zero + allocated GPUs. Mixed, drained, powered-down and flagged states do not pass. + This is a point-in-time check, not a reservation or promise that a job will + schedule; accounts, reservations, memory, licenses and job policy can still + prevent an allocation. +- Local `enroot version` and Pyxis options in `srun --help` must be available. + These prove installation/advertisement only, not that a container executes. + The optional probe below tests that narrower runtime outcome. +- The image reference must be `registry#repository@sha256:<64 lowercase hex + digits>`. A version tag alone is not immutable. The validator does not fetch + an image or verify its framework compatibility, contents, trust or registry + accessibility. The operator must choose a reviewed fine-tuning image. +- Supported cache layout: `config.json`, `tokenizer_config.json`, + `tokenizer.json`, and either `model.safetensors` or + `model.safetensors.index.json` with a nonempty `weight_map`. Every indexed + shard must exist, be nonempty, and resolve within the cache. Weight indexes + larger than 8 MiB fail closed. The explicit required-file list must include + all other assets required by the chosen model (for example custom config or + tokenizer assets). Other weight/tokenizer layouts need a reviewed recipe; + do not claim unsupported models are ready by dropping required files. + +File checks prove readable named inputs and complete indexed shards, not hashes, +valid model contents, license rights or successful loading. The required-file +list is supplied by the operator, not inferred as a guarantee of every model's +requirements. No model code is imported. Keep cache credentials private and +outputs separate from the read-only input cache. + +## Default: no workload submitted + +Set these variables to site-approved values. Do not paste credentials into +commands. Choose the full immutable model commit, the full image digest and the +complete model file list before running: + +```bash +python3 scripts/validation/validate_finetune.py --json \ + --node "$NODE" --partition "$PARTITION" \ + --container-image "$TRAINING_IMAGE" \ + --cache-dir "$HF_HUB_CACHE" --repo-id "$MODEL_REPO" --revision "$MODEL_REVISION" \ + --require-file config.json --require-file tokenizer.json \ + --require-file tokenizer_config.json --require-file model.safetensors +``` + +For sharded models replace `--require-file model.safetensors` with +`--require-file model.safetensors.index.json`; shard paths are read from the +validated local index. Repeat `--require-file` for additional assets. The +validator does not warm the cache, install anything, start a container or submit +a job by default. It only runs the read-only scheduler and local capability +queries described above. Every external command has a per-command timeout +(default 30 seconds, maximum 120); this is not a whole-invocation time budget. + +## Optional GPU container probe + +With separate authorization for the exact target and reviewed image, append +`--gpu-smoke`. Only after **all** readiness checks pass, it runs: + +```text +srun --nodes=1 --ntasks=1 --gpus=1 --immediate=5 --time=1 \ + --nodelist= --partition= --container-image= \ + nvidia-smi --query-gpu=index,name --format=csv,noheader +``` + +This may pull the container image. The allocation has a one-minute scheduler +time limit and a five-second immediate-allocation limit; the local command has +the configured timeout. A slow image pull can make the probe fail. Confirm job +termination through the site's normal procedure after a timeout; do not blindly +retry. Require a zero exit and exactly one GPU row. No training, dataset loading, +model loading or shared-cache mounting is performed by this probe. In particular, +it cannot attest cache permissions inside a future training container. + +## JSON contract (schema version 1) + +| Field | Meaning | +|---|---| +| `status` | `ready`, `not_ready`, or `bad_input`. | +| `ok` | True only when all requested readiness/probe checks pass. | +| `training_validated` | Always false, including after the optional probe. | +| `checks` | Booleans for `scheduler`, `local_node`, `gpu_capacity`, `container_support`, `cache`, `container_pin`. | +| `errors` | One object per failed check: stable `stage` plus safe `message`. Multiple independent failures are retained. | +| `gpu_smoke_ran` | Whether the optional allocation command was attempted. False by default and after any failed preflight. | +| `gpu_smoke_ok` | Whether the requested container GPU probe passed. False if skipped. | + +Exit `0` means ready; `1` means not ready; `2` means bad input. Missing and +unknown CLI arguments also produce `bad_input` JSON when `--json` is supplied. +Bad inputs run no scheduler or container commands. Multi-node options are not +supported and fail as bad input. `--help` prints ordinary usage text. +Raw scheduler/runtime output, image credentials and generated text are not +included in results. Before sharing results externally, still check them against +your site's data policy. + +A successful fine-tuning recipe would require separate evidence of completed +training, improving loss, checkpoint reload and inference from the tuned model. +None is supplied or asserted by this readiness tool. + +## Offline tests + +```bash +python3 -m unittest discover -s scripts/validation/tests -v +``` + +The fixtures use temporary model files and inert command boundaries/executables. +They cover successful readiness, each failed prerequisite, aggregated failures, +malformed inputs, incomplete or escaping cache paths, indexed weights, JSON/exit +codes, and optional probe bounds. They do not run on or certify live hardware. +See [the fine-tuning skill](../../skills/finetune-model/SKILL.md) for agent routing +and [the cache guide](shared-cache.md) for cache setup and permissions. diff --git a/scripts/validation/tests/test_finetune_readiness.py b/scripts/validation/tests/test_finetune_readiness.py new file mode 100644 index 000000000..c6cc0ff58 --- /dev/null +++ b/scripts/validation/tests/test_finetune_readiness.py @@ -0,0 +1,255 @@ +"""Offline readiness contracts: real cache files, inert scheduler/process boundary.""" +import argparse +import importlib.util +import json +from pathlib import Path +import socket +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +SCRIPTS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPTS)) +SCRIPT = SCRIPTS / "validate_finetune.py" + + +class ReadinessTests(unittest.TestCase): + def setUp(self): + self.assertTrue(SCRIPT.is_file(), "fine-tuning readiness validator is missing") + spec = importlib.util.spec_from_file_location("readiness", SCRIPT) + self.validator = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.validator) + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.cache = Path(self.tmp.name) + self.snapshot = self.cache / "models--example--model/snapshots" / ("a" * 40) + self.snapshot.mkdir(parents=True) + for name in ("config.json", "tokenizer.json", "tokenizer_config.json", "model.safetensors"): + (self.snapshot / name).write_text("{}") + self.args = argparse.Namespace(cache_dir=str(self.cache), repo_id="example/model", + revision="a" * 40, require_file=["config.json", "tokenizer.json", "tokenizer_config.json", "model.safetensors"], + node="gpu-node", partition="gpu", container_image="registry.example#team/train@sha256:" + "b" * 64, + timeout=5, gpu_smoke=False) + self.node = ("NodeName=gpu-node NodeHostName=" + socket.gethostname() + + " State=IDLE Partitions=gpu CfgTRES=cpu=8,gres/gpu=2 AllocTRES=cpu=0,gres/gpu=0") + self.partition = "PartitionName=gpu State=UP" + self.commands = [] + self.fail_command = None + self.smoke_output = "0, Example GPU" + + def run_command(self, command, timeout): + self.commands.append(command) + if command[0] == self.fail_command: + return 1, "private output", "private failure" + if command[:4] == ["scontrol", "show", "node", "--oneliner"]: + return 0, self.node, "" + if command[:4] == ["scontrol", "show", "partition", "--oneliner"]: + return 0, self.partition, "" + if command == ["enroot", "version"]: + return 0, "3.5.0", "" + if command == ["srun", "--help"]: + return 0, "--container-image --container-mounts", "" + if command[0] == "srun" and "--gpus=1" in command: + return 0, self.smoke_output, "" + self.fail("unexpected command: " + repr(command)) + + def validate(self): + with patch.object(self.validator, "run", side_effect=self.run_command): + return self.validator.validate(self.args) + + def assert_not_ready(self, stage): + result = self.validate() + self.assertEqual(result["status"], "not_ready") + self.assertFalse(result["ok"]) + self.assertFalse(result["training_validated"]) + self.assertIn(stage, [e["stage"] for e in result["errors"]]) + self.assertNotIn("private", json.dumps(result)) + return result + + def test_ready_is_not_training_and_never_submits_by_default(self): + result = self.validate() + self.assertEqual(result["status"], "ready") + self.assertTrue(result["ok"]) + self.assertEqual(result["schema_version"], 1) + self.assertFalse(result["training_validated"]) + self.assertFalse(result["gpu_smoke_ran"]) + self.assertEqual([c for c in self.commands if c[0] == "srun"], [["srun", "--help"]]) + + def test_scheduler_failure(self): + self.fail_command = "scontrol" + self.assert_not_ready("scheduler") + + def test_empty_or_malformed_scheduler_output_fails_closed(self): + for output in ("", "garbage", "NodeName=gpu-node", self.node + "\n" + self.node): + with self.subTest(output=output): + self.node = output + self.assert_not_ready("scheduler") + + def test_wrong_host_cannot_attest_compute_runtime_or_cache(self): + self.node = self.node.replace(socket.gethostname(), "other-node.invalid") + self.assert_not_ready("local_node") + + def test_partition_must_be_up_and_include_the_selected_node(self): + self.partition = "PartitionName=gpu State=DOWN" + self.assert_not_ready("scheduler") + self.partition = "PartitionName=gpu State=UP" + self.node = self.node.replace("Partitions=gpu", "Partitions=other") + self.assert_not_ready("scheduler") + + def test_no_free_gpu_and_unavailable_nodes_fail(self): + original = self.node + for state in ("ALLOCATED", "MIXED", "IDLE+DRAIN", "DOWN", "IDLE*"): + with self.subTest(state=state): + self.node = original.replace("State=IDLE", "State=" + state) + self.assert_not_ready("gpu_capacity") + self.node = original.replace("gres/gpu=2", "gres/gpu=0") + self.assert_not_ready("gpu_capacity") + self.node = original.replace("AllocTRES=cpu=0,gres/gpu=0", "AllocTRES=cpu=0,gres/gpu=2") + self.assert_not_ready("gpu_capacity") + + def test_missing_runtime_or_container_plugin_fails(self): + self.fail_command = "enroot" + self.assert_not_ready("container_support") + self.fail_command = "srun" + self.assert_not_ready("container_support") + + def test_help_without_pyxis_is_not_container_support(self): + original = self.run_command + def plain_help(command, timeout): + if command == ["srun", "--help"]: + return 0, "ordinary srun help", "" + return original(command, timeout) + with patch.object(self.validator, "run", side_effect=plain_help): + result = self.validator.validate(self.args) + self.assertFalse(result["checks"]["container_support"]) + + def test_multiple_failure_reasons_are_preserved(self): + (self.snapshot / "tokenizer.json").unlink() + self.fail_command = "enroot" + self.node = self.node.replace("State=IDLE", "State=DOWN") + result = self.assert_not_ready("cache") + self.assertTrue({"cache", "container_support", "gpu_capacity"}.issubset({e["stage"] for e in result["errors"]})) + + def test_cache_missing_empty_or_escaping_file_fails(self): + target = self.snapshot / "model.safetensors" + target.unlink() + self.assert_not_ready("cache") + target.touch() + self.assert_not_ready("cache") + target.unlink() + target.symlink_to("/etc/passwd") + self.assert_not_ready("cache") + + def test_config_only_cannot_pass_as_a_complete_model(self): + self.args.require_file = ["config.json"] + self.assert_not_ready("cache") + + def test_shard_index_requires_every_referenced_weight(self): + self.args.require_file[-1] = "model.safetensors.index.json" + index = self.snapshot / "model.safetensors.index.json" + index.write_text(json.dumps({"weight_map": {"first": "part-1.safetensors", "second": "part-2.safetensors"}})) + (self.snapshot / "part-1.safetensors").write_bytes(b"first") + self.assert_not_ready("cache") + (self.snapshot / "part-2.safetensors").write_bytes(b"second") + self.assertEqual(self.validate()["status"], "ready") + for value in ({}, {"weight_map": {}}, {"weight_map": {"x": "../../outside"}}, {"weight_map": {"x": 1}}): + index.write_text(json.dumps(value)) + self.assert_not_ready("cache") + index.write_text("invalid json") + self.assert_not_ready("cache") + + def test_bad_input_is_distinct_and_runs_no_commands(self): + for field, value in (("container_image", "image:latest"), ("container_image", "image:v1"), + ("container_image", "--evil@sha256:" + "b" * 64), ("timeout", 0), + ("timeout", float("nan")), ("timeout", 121), ("node", "n[1-2]"), + ("node", "--all"), ("partition", "a,b"), ("revision", "main"), + ("repo_id", "../outside"), ("require_file", []), ("require_file", ["../outside"])): + with self.subTest(field=field, value=value): + old = getattr(self.args, field) + setattr(self.args, field, value) + self.commands.clear() + result = self.validate() + self.assertEqual(result["status"], "bad_input") + self.assertFalse(result["ok"]) + self.assertEqual(self.commands, []) + setattr(self.args, field, old) + + def test_optional_probe_is_single_node_single_gpu_pinned_and_bounded(self): + self.args.gpu_smoke = True + result = self.validate() + self.assertTrue(result["gpu_smoke_ran"]) + self.assertTrue(result["gpu_smoke_ok"]) + self.assertFalse(result["training_validated"]) + job = self.commands[-1] + for arg in ("--nodes=1", "--ntasks=1", "--gpus=1", "--nodelist=gpu-node", "--partition=gpu", "--immediate=5", "--time=1", "--container-image=" + self.args.container_image): + self.assertIn(arg, job) + self.assertEqual(job[-3:], ["nvidia-smi", "--query-gpu=index,name", "--format=csv,noheader"]) + + def test_optional_probe_is_not_run_after_preflight_failure(self): + self.args.gpu_smoke = True + self.fail_command = "enroot" + result = self.assert_not_ready("container_support") + self.assertFalse(result["gpu_smoke_ran"]) + self.assertFalse(any("--gpus=1" in c for c in self.commands)) + + def test_optional_probe_empty_output_or_multiple_gpus_is_failure(self): + self.args.gpu_smoke = True + for output in ("", "0, GPU\n1, GPU", "garbage"): + self.smoke_output = output + self.assert_not_ready("gpu_smoke") + + def test_command_failures_are_safe(self): + for exception in (FileNotFoundError(), PermissionError(), subprocess.TimeoutExpired("private", 5)): + with patch.object(self.validator.subprocess, "run", side_effect=exception): + rc, out, err = self.validator.run(["missing"], timeout=1) + self.assertNotEqual(rc, 0) + self.assertNotIn("private", out + err) + + def test_cli_ready_and_not_ready_exit_codes_with_inert_commands(self): + # Run the actual CLI without any path to real Slurm/Enroot executables. + tools = self.cache / "bin" + tools.mkdir() + outputs = {"scontrol": self.node, "enroot": "3.5.0", "srun": "--container-image --container-mounts"} + for name, output in outputs.items(): + path = tools / name + path.write_text("#!" + sys.executable + "\nimport sys\n" + + "print(" + repr(self.partition) + " if 'partition' in sys.argv else " + repr(output) + ")\n") + path.chmod(0o755) + command = [sys.executable, str(SCRIPT), "--json"] + for field in ("cache_dir", "repo_id", "revision", "node", "partition", "container_image"): + command += ["--" + field.replace("_", "-"), getattr(self.args, field)] + for name in self.args.require_file: + command += ["--require-file", name] + for expected in (0, 1): + proc = subprocess.run(command, env={"PATH": str(tools)}, capture_output=True, text=True, timeout=10) + self.assertEqual(proc.returncode, expected, proc.stderr) + result = json.loads(proc.stdout) + self.assertEqual(result["status"], "ready" if expected == 0 else "not_ready") + self.assertFalse(result["training_validated"]) + (self.snapshot / "model.safetensors").write_bytes(b"") + + def test_optional_probe_nonzero_status_is_failure(self): + self.args.gpu_smoke = True + original = self.run_command + def failed_job(command, timeout): + if "--gpus=1" in command: + return 1, "private output", "private failure" + return original(command, timeout) + with patch.object(self.validator, "run", side_effect=failed_job): + result = self.validator.validate(self.args) + self.assertEqual(result["status"], "not_ready") + self.assertTrue(result["gpu_smoke_ran"]) + self.assertFalse(result["gpu_smoke_ok"]) + self.assertNotIn("private", json.dumps(result)) + + def test_cli_usage_error_returns_json_and_exit_two(self): + proc = subprocess.run([sys.executable, str(SCRIPT), "--json", "--nodes", "2"], + capture_output=True, text=True, timeout=10) + self.assertEqual(proc.returncode, 2) + self.assertEqual(json.loads(proc.stdout)["status"], "bad_input") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validation/tests/test_model_skills.py b/scripts/validation/tests/test_model_skills.py index ec33f8957..eb7ec2f82 100644 --- a/scripts/validation/tests/test_model_skills.py +++ b/scripts/validation/tests/test_model_skills.py @@ -7,7 +7,7 @@ ROOT = pathlib.Path(__file__).resolve().parents[3] -SKILLS = ("model-workflows", "serve-model") +SKILLS = ("model-workflows", "serve-model", "finetune-model") class ModelSkillDocsTest(unittest.TestCase): diff --git a/scripts/validation/validate_finetune.py b/scripts/validation/validate_finetune.py new file mode 100644 index 000000000..cbc23347c --- /dev/null +++ b/scripts/validation/validate_finetune.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Read-only single-node fine-tuning readiness; never train or download by default. + +Run as the model consumer on the selected Slurm compute host. Optional +--gpu-smoke submits one bounded Pyxis container job and may pull its image. +Exit codes: 0 ready, 1 not ready, 2 bad input. Training is never validated. +""" +import argparse +import json +import math +from pathlib import Path, PurePosixPath +import re +import socket +import subprocess +import sys + +from validate_model_cache import validate as validate_cache + + +class BadInput(ValueError): + """Invalid request, reported without raw arguments.""" + + +def run(command, timeout): + """Bound local probes; never invoke a shell or expose command output on failure.""" + try: + process = subprocess.run(command, capture_output=True, text=True, timeout=timeout) + return process.returncode, process.stdout.strip(), "" + except (OSError, subprocess.TimeoutExpired, UnicodeError): + return 1, "", "command unavailable or timed out" + + +def new_result(): + return { + "schema_version": 1, "status": "not_ready", "ok": False, + "training_validated": False, "gpu_smoke_ran": False, "gpu_smoke_ok": False, + "checks": {name: False for name in ("scheduler", "local_node", "gpu_capacity", "container_support", "cache", "container_pin")}, + "errors": [], + } + + +def safe_relative(value): + return (isinstance(value, str) and bool(value) and value != "." + and not PurePosixPath(value).is_absolute() and ".." not in PurePosixPath(value).parts + and str(PurePosixPath(value)) == value and "\\" not in value) + + +def check_arguments(args): + if not math.isfinite(args.timeout) or not 0 < args.timeout <= 120: + raise BadInput("timeout must be greater than zero and at most 120 seconds") + for name in ("node", "partition"): + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", getattr(args, name)): + raise BadInput("node and partition must each name one explicit target, not a range or list") + if not re.fullmatch(r"[0-9a-f]{40}", args.revision): + raise BadInput("revision must be a full lowercase 40-character commit hash") + parts = args.repo_id.split("/") + if (len(parts) not in (1, 2) or len(args.repo_id) > 96 or any( + not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]*", part) + or part.endswith((".", "-")) or ".." in part or "--" in part for part in parts)): + raise BadInput("repo-id must be a model name or namespace/model") + if not args.require_file or not all(safe_relative(name) for name in args.require_file): + raise BadInput("require-file must contain normalized relative model paths without traversal") + # Pyxis/Enroot registry#repository syntax. Digest-only: mutable tags are not pins. + if not re.fullmatch(r"[a-z0-9][a-z0-9.-]*(?::[0-9]{1,5})?#[a-z0-9][a-z0-9._/-]*@sha256:[0-9a-f]{64}", args.container_image): + raise BadInput("container-image must use registry#repository@sha256:<64 lowercase hex digits>, without credentials") + + +def parse_record(output): + """Require one unambiguous scontrol --oneliner record.""" + if len(output.splitlines()) != 1: + return {} + record = {} + for item in output.split(): + if "=" in item: + key, value = item.split("=", 1) + if key in record: + return {} + record[key] = value + return record + + +def gpu_count(tres): + matches = re.findall(r"(?:^|,)gres/gpu=(\d+)(?:,|$)", tres) + return int(matches[0]) if len(matches) == 1 else None + + +def cache_ready(args): + """Verify explicit files plus all shards of the supported safetensors layout. + + This checks completeness of this layout, not model semantics or integrity. + Other tokenizers/weight formats require a separately reviewed recipe. + """ + required = set(args.require_file) + if not {"config.json", "tokenizer.json", "tokenizer_config.json"}.issubset(required): + return False + index_name = "model.safetensors.index.json" + if not ({"model.safetensors", index_name} & required): + return False + result = validate_cache(args.cache_dir, args.repo_id, args.revision, args.require_file) + if not result["ok"]: + return False + if index_name in required: + try: + index_path = Path(result["snapshot_path"]) / index_name + if index_path.stat().st_size > 8 * 1024 * 1024: + return False + with index_path.open("rb") as stream: + raw = stream.read(8 * 1024 * 1024 + 1) + if len(raw) > 8 * 1024 * 1024: + return False + index = json.loads(raw) + weights = index.get("weight_map") if isinstance(index, dict) else None + if (not isinstance(weights, dict) or not weights + or not all(safe_relative(name) and name.endswith(".safetensors") for name in weights.values())): + return False + return validate_cache(args.cache_dir, args.repo_id, args.revision, sorted(required | set(weights.values())))["ok"] + except (OSError, ValueError, UnicodeError, RecursionError): + return False + return True + + +def validate(args): + result = new_result() + + def check(stage, passed, message): + result["checks"][stage] = bool(passed) + if not passed: + result["errors"].append({"stage": stage, "message": message}) + + try: + check_arguments(args) + except BadInput as exc: + result["status"] = "bad_input" + result["errors"].append({"stage": "arguments", "message": str(exc)}) + return result + result["checks"]["container_pin"] = True + rc, output, _ = run(["scontrol", "show", "node", "--oneliner", args.node], args.timeout) + node = parse_record(output) if rc == 0 else {} + rc, output, _ = run(["scontrol", "show", "partition", "--oneliner", args.partition], args.timeout) + partition = parse_record(output) if rc == 0 else {} + check("scheduler", node.get("NodeName") == args.node and bool(node.get("State")) + and partition.get("PartitionName") == args.partition and partition.get("State") == "UP" + and args.partition in node.get("Partitions", "").split(","), + "scheduler must return the selected node in the selected UP partition") + local_names = {socket.gethostname(), socket.gethostname().split(".")[0]} + check("local_node", node.get("NodeHostName") in local_names, + "run as the consumer on the selected compute host; remote runtime and cache are not verified") + configured = gpu_count(node.get("CfgTRES", "")) + # Slurm emits an empty AllocTRES when nothing is allocated. + allocated = gpu_count(node.get("AllocTRES", "")) + if allocated is None and "AllocTRES" in node and not node["AllocTRES"]: + allocated = 0 + check("gpu_capacity", node.get("State") == "IDLE" and configured is not None and configured >= 1 + and allocated == 0, + "selected node must be IDLE with at least one configured GPU and no allocated GPUs; mixed or flagged nodes are not accepted") + runtime_rc, runtime, _ = run(["enroot", "version"], args.timeout) + plugin_rc, help_text, _ = run(["srun", "--help"], args.timeout) + check("container_support", runtime_rc == 0 and bool(runtime) and plugin_rc == 0 + and "--container-image" in help_text and "--container-mounts" in help_text, + "local Enroot and Slurm Pyxis container options must be available; execution requires the optional probe") + check("cache", cache_ready(args), + "cache must contain readable nonempty config, tokenizer, and safetensors weights, including all indexed shards and explicitly required files") + if args.gpu_smoke and not result["errors"]: + result["gpu_smoke_ran"] = True + command = ["srun", "--nodes=1", "--ntasks=1", "--gpus=1", "--immediate=5", "--time=1", + "--nodelist=" + args.node, "--partition=" + args.partition, + "--container-image=" + args.container_image, + "nvidia-smi", "--query-gpu=index,name", "--format=csv,noheader"] + rc, output, _ = run(command, args.timeout) + result["gpu_smoke_ok"] = rc == 0 and bool(re.fullmatch(r"\d+,\s*[^\n,]+", output)) + if not result["gpu_smoke_ok"]: + result["errors"].append({"stage": "gpu_smoke", "message": "optional container job failed or did not report exactly one GPU"}) + result["ok"] = not result["errors"] + result["status"] = "ready" if result["ok"] else "not_ready" + return result + + +class Parser(argparse.ArgumentParser): + def error(self, message): + raise BadInput("missing, invalid, or unsupported arguments; use --help") + + +def main(argv=None): + argv = sys.argv[1:] if argv is None else argv + parser = Parser(description=__doc__, allow_abbrev=False) + for name in ("cache-dir", "repo-id", "revision", "node", "partition", "container-image"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--require-file", required=True, action="append") + parser.add_argument("--timeout", type=float, default=30, help="Per-command bound, 0 < seconds <= 120") + parser.add_argument("--gpu-smoke", action="store_true", help="Opt in to one GPU container job; may pull the image, never trains") + parser.add_argument("--json", action="store_true") + try: + args = parser.parse_args(argv) + result = validate(args) + except BadInput as exc: + result = new_result() + result["status"] = "bad_input" + result["errors"].append({"stage": "arguments", "message": str(exc)}) + if "--json" in argv: + print(json.dumps(result, sort_keys=True)) + else: + print(result["status"] + ": readiness only; training_validated=false") + for error in result["errors"]: + print(error["stage"] + ": " + error["message"]) + return {"ready": 0, "not_ready": 1, "bad_input": 2}[result["status"]] + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/README.md b/skills/README.md index a981d30d2..d127a7ef3 100644 --- a/skills/README.md +++ b/skills/README.md @@ -15,9 +15,10 @@ agent (or person) can simply read the relevant `SKILL.md` before acting. | [deploy-k8s-gpu-cluster](deploy-k8s-gpu-cluster/SKILL.md) | Deploying or rebuilding a Kubernetes GPU cluster. | | [provision-with-maas](provision-with-maas/SKILL.md) | Provisioning, tagging, validating, or releasing machines through MAAS. | | [deploy-airgapped](deploy-airgapped/SKILL.md) | Preparing mirrors and deploying Slurm or Kubernetes without Internet access. | -| [model-workflows](model-workflows/SKILL.md) | Choosing cache or single-node serving procedures; identifying unsupported training or multi-node requests. | +| [model-workflows](model-workflows/SKILL.md) | Choosing cache, single-node serving or fine-tuning readiness; identifying unsupported training or multi-node requests. | | [manage-model-cache](manage-model-cache/SKILL.md) | Preparing shared model files, pinning revisions, or diagnosing offline cache access. | | [serve-model](serve-model/SKILL.md) | Serving a cached snapshot on one GPU and validating a local completion endpoint. | +| [finetune-model](finetune-model/SKILL.md) | Checking single-node training prerequisites without claiming training success. | | [validate-gpu-cluster](validate-gpu-cluster/SKILL.md) | Health checks and post-deploy verification. | | [diagnose-driver-install](diagnose-driver-install/SKILL.md) | NVIDIA driver failures, `nvidia-smi` errors, GPU pods crash-looping. | diff --git a/skills/finetune-model/SKILL.md b/skills/finetune-model/SKILL.md new file mode 100644 index 000000000..da43944d1 --- /dev/null +++ b/skills/finetune-model/SKILL.md @@ -0,0 +1,62 @@ +--- +name: finetune-model +description: Use when checking whether one DeepOps Slurm GPU node is prepared for fine-tuning, diagnosing missing cached training inputs or container prerequisites, or distinguishing readiness from actual training success. +--- + +# Check fine-tuning readiness + +This skill checks preparation only. It provides **no training recipe**. A +`ready` result always has `training_validated: false`. Multi-node training is +unsupported here; do not invent a distributed command or report a successful +fine-tune from this check. + +## Preconditions + +- Read the [readiness guide](../../docs/model-workflows/finetune-readiness.md). + Run from the repository root as the intended model consumer on the selected + compute host, not the login node. The validator checks local host identity + against the scheduler record; do not bypass a mismatch. +- Obtain authorization for the target and read-only scheduler queries. Specify + one node and one partition. An IDLE node with an unallocated GPU is required; + this conservative check does not accept mixed, draining or flagged nodes. +- Follow [manage-model-cache](../manage-model-cache/SKILL.md) for an approved + shared cache, immutable revision and complete reviewed required-file list. + Supported inputs are Hugging Face `tokenizer.json` plus safetensors weights; + list every extra model-specific asset. Indexed weights require every shard. +- Select a site-reviewed fine-tuning image pinned by digest, using Pyxis syntax + `registry#repository@sha256:`. Do not use mutable tags, credentials in + the reference, or substitute an unreviewed image to get a pass. + +## Readiness check + +Set the variables to reviewed values. For sharded weights, replace the final +file argument with `model.safetensors.index.json`; the validator follows its +weight map. Add every additional file needed by the selected model. + +```bash +python3 scripts/validation/validate_finetune.py --json \ + --node "$NODE" --partition "$PARTITION" \ + --container-image "$TRAINING_IMAGE" \ + --cache-dir "$HF_HUB_CACHE" --repo-id "$MODEL_REPO" --revision "$MODEL_REVISION" \ + --require-file config.json --require-file tokenizer.json \ + --require-file tokenizer_config.json --require-file model.safetensors +``` + +| Result | Next action | +|---|---| +| Exit `0`, `status: ready`, `ok: true` | Report readiness only, including that container execution and training remain unproven. | +| Exit `1`, `status: not_ready` | Report every `errors[]` stage. Repair the named prerequisite through its approved procedure; do not drain nodes, install software or download models implicitly. | +| Exit `2`, `status: bad_input` | Correct explicit inputs or unsupported flags using `--help`. No scheduler commands run for bad input. | + +## Optional one-GPU probe + +Off by default. Only with explicit authorization to allocate a GPU and run the +reviewed image, repeat the command with `--gpu-smoke`. This can download the +container image and submits one bounded, single-node, single-GPU `nvidia-smi` +job after all readiness gates pass. It does not load the model or train. +`gpu_smoke_ok: true` proves only the container GPU probe, not cache access +inside the container, framework compatibility, learning or checkpoint recovery. + +Keep credentials private, mount shared inputs read-only in any future reviewed +recipe, and keep outputs/checkpoints outside the input cache. A readable cache, +a pinned image string, and a GPU probe cannot certify a training outcome. diff --git a/skills/model-workflows/SKILL.md b/skills/model-workflows/SKILL.md index de8fdc1c5..cd8535f90 100644 --- a/skills/model-workflows/SKILL.md +++ b/skills/model-workflows/SKILL.md @@ -15,7 +15,8 @@ working inference, or training are different outcomes with different evidence. |---|---|---| | Prepare or repair shared model inputs | [Manage model cache](../manage-model-cache/SKILL.md) | Cache validator exit `0` and `ok: true`, as the real consumer. Only named-file readability is proven. | | Serve a pinned snapshot on one GPU | [Serve model](../serve-model/SKILL.md) | Cache gate first, then serving validator exit `0` and `ok: true`. One bounded completion, not training or performance validation. | -| Fine-tune a model | Stop: no supported training recipe or training-result validator is provided here yet. | Cache readiness is not training success. Obtain a reviewed single-node recipe and explicit success criteria before proceeding. | +| Check fine-tuning prerequisites | [Fine-tuning readiness](../finetune-model/SKILL.md) | Readiness validator exit `0`, `status: ready`, `training_validated: false`. Optional one-GPU container probe is off by default. | +| Actually fine-tune a model | Stop: no supported training recipe or training-result validator is provided here yet. | Readiness is not training success. Obtain a reviewed recipe and explicit success criteria before proceeding. | | Multi-node serving or training | Stop: unsupported by these skills. | Do not infer distributed support from a successful single-node check. | ## Common preflight @@ -40,7 +41,9 @@ working inference, or training are different outcomes with different evidence. Record which validator ran, its exit code, schema version, explicit inputs and checks. The [cache guide](../../docs/model-workflows/shared-cache.md) and [serving guide](../../docs/model-workflows/vllm-quickstart.md) define their JSON -contracts; this umbrella does not invent a combined success field. +contracts; the [fine-tuning readiness guide](../../docs/model-workflows/finetune-readiness.md) +separates preparation from training. This umbrella does not invent a combined +success field. A readable cache is preparation only. A completion is inference only. Neither proves that a particular revision was loaded, a GPU was used, or training