diff --git a/alembic/versions/a8d3f1c6e2b9_add_image_facts_and_job_image_columns.py b/alembic/versions/a8d3f1c6e2b9_add_image_facts_and_job_image_columns.py new file mode 100644 index 0000000..f57ad5f --- /dev/null +++ b/alembic/versions/a8d3f1c6e2b9_add_image_facts_and_job_image_columns.py @@ -0,0 +1,33 @@ +"""add image_facts, compute_jobs.min_cuda_major and compute_jobs.image_bytes: the checks on a job's image + +Revision ID: a8d3f1c6e2b9 +Revises: e2b7c4d9a1f6 +Create Date: 2026-11-19 23:30:00 +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +revision = "a8d3f1c6e2b9" +down_revision = "e2b7c4d9a1f6" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "image_facts", + sa.Column("digest", sa.String(length=80), primary_key=True), + sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("checked_at", sa.DateTime(timezone=True), nullable=False), + ) + op.add_column("compute_jobs", sa.Column("min_cuda_major", sa.SmallInteger(), nullable=True)) + op.add_column("compute_jobs", sa.Column("image_bytes", sa.BigInteger(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("compute_jobs", "image_bytes") + op.drop_column("compute_jobs", "min_cuda_major") + op.drop_table("image_facts") diff --git a/cli/pyproject.toml b/cli/pyproject.toml index 573376c..e7b98e5 100644 --- a/cli/pyproject.toml +++ b/cli/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "krater-cli" -version = "0.7.0" +version = "0.8.0" description = "The krater command: run jobs and boxes on Krater Compute (Project Ganymede, Patchwork Labs)" requires-python = ">=3.10" dependencies = ["httpx>=0.27", "tomli>=2; python_version < \"3.11\""] diff --git a/cli/src/krater_cli/__init__.py b/cli/src/krater_cli/__init__.py index 247f2b2..b35d82a 100644 --- a/cli/src/krater_cli/__init__.py +++ b/cli/src/krater_cli/__init__.py @@ -1,3 +1,3 @@ """The `krater` command: Krater Compute from a terminal (docs/compute/cli.md).""" -__version__ = "0.7.0" +__version__ = "0.8.0" diff --git a/cli/src/krater_cli/api.py b/cli/src/krater_cli/api.py index 5ae37ac..2cddb91 100644 --- a/cli/src/krater_cli/api.py +++ b/cli/src/krater_cli/api.py @@ -11,7 +11,7 @@ import httpx -from krater_cli import auth +from krater_cli import auth, newer class KraterError(Exception): @@ -78,6 +78,7 @@ def request(self, method: str, path: str, *, json: Any = None) -> dict: if response.status_code == 401: self.creds = auth.refresh(self.creds, http=self.http) response = self._send_patiently(method, path, json) + newer.notice(response.headers.get(newer.HEADER)) try: data = response.json() except ValueError: diff --git a/cli/src/krater_cli/code.py b/cli/src/krater_cli/code.py index 6eb4b13..ce94e6e 100644 --- a/cli/src/krater_cli/code.py +++ b/cli/src/krater_cli/code.py @@ -14,6 +14,7 @@ import hashlib import os +import re from dataclasses import dataclass #: The image a `--code` job runs on when it names none: Krater's form default, PyTorch with CUDA 12.8 (runs on every @@ -69,12 +70,55 @@ def has_requirements(self) -> bool: return any(rel == "requirements.txt" for _, rel in self.files) def command(self, command: str) -> str: - """The job's command: into the folder, the requirements if there are any, then the member's own.""" + """The job's command: into the folder, a plain error when the image has no Python, the requirements if there + are any, then the member's own. PIP_BREAK_SYSTEM_PACKAGES lets pip install into an image whose Python is the + system's own (Ubuntu's, in the newer pytorch/pytorch images), which it otherwise refuses; elsewhere it does + nothing.""" parts = [f'cd "$KRATER_INPUT_DIR/{INPUTS_PREFIX}/{self.digest}"'] + if self.has_requirements or re.search(r"\bpython3?\b|\bpip3?\b", command): + parts.append(NEEDS_PYTHON) if self.has_requirements: - parts.append("pip install --quiet --no-cache-dir -r requirements.txt") + parts.append("PIP_BREAK_SYSTEM_PACKAGES=1 pip install --quiet --no-cache-dir -r requirements.txt") return " && ".join(parts) + " && " + command + def builds_cuda(self) -> list[str]: + """What in the folder compiles CUDA code on the machine, which needs an image with the CUDA toolkit (a + -devel one): `load_inline`, `cpp_extension`, `torch.cuda._compile_kernel` or NVRTC in its Python files, or + packages in its requirements.txt that build CUDA extensions when installed. Triton doesn't need it.""" + found: list[str] = [] + for path, rel in self.files: + if rel == "requirements.txt": + text = _read(path) + found += [ + f"{name} in requirements.txt" + for name in _BUILDS_PACKAGES + if re.search(rf"^\s*{re.escape(name)}\b", text, re.IGNORECASE | re.MULTILINE) + ] + elif rel.endswith((".py", ".cu", ".cpp")) or rel == "setup.py": + text = _read(path) + if rel.endswith(".cu"): + found.append(rel) + elif hit := _BUILDS_CODE.search(text): + found.append(f"{hit[0]} in {rel}") + return found + + +#: Stops with a plain error when the image has no Python, instead of "python: not found" deep in a log. +NEEDS_PYTHON = ( + "{ command -v python >/dev/null 2>&1 || command -v python3 >/dev/null 2>&1 || " + '{ echo "krater: this image has no Python; use one that does, such as the default" >&2; exit 127; }; }' +) +_BUILDS_CODE = re.compile(r"load_inline|cpp_extension|_compile_kernel|\bnvrtc\b|CUDAExtension") +_BUILDS_PACKAGES = ("flash-attn", "flash_attn", "deepspeed", "apex", "mamba-ssm", "causal-conv1d", "xformers") + + +def _read(path: str) -> str: + try: + with open(path, encoding="utf-8", errors="replace") as handle: + return handle.read(2_000_000) + except OSError: + return "" + def collect(folder: str) -> Code: """The folder's files, in a stable order, with their hash. Raises `CodeError` when it isn't a usable code folder.""" diff --git a/cli/src/krater_cli/main.py b/cli/src/krater_cli/main.py index ae64444..756500f 100644 --- a/cli/src/krater_cli/main.py +++ b/cli/src/krater_cli/main.py @@ -334,8 +334,20 @@ def _parser() -> argparse.ArgumentParser: wait.add_argument("job", help=job_help) wait.add_argument("--timeout", type=_positive, help="give up after this many minutes (exit code 124)") wait.add_argument("--interval", type=_positive, default=5, help="seconds between checks (default 5)") - compare = commands.add_parser("compare", help="jobs' outputs/metrics.json side by side") - compare.add_argument("jobs", nargs="+", metavar="job", help=job_help) + compare = commands.add_parser("compare", help="jobs' outputs/metrics.json side by side, with cost and runtime") + compare.add_argument("jobs", nargs="*", metavar="job", help=job_help) + compare.add_argument( + "--sweep", + metavar="NAME", + help="every job of a sweep (or --count): the jobs named NAME-0, NAME-1, ... (with --project)", + ) + compare.add_argument("--project", help="the project the sweep is in") + limits = commands.add_parser("limits", help="change a queued or running job's max runtime or spend cap") + limits.add_argument("job", help=job_help) + limits.add_argument("--max-runtime", type=int, metavar="MINUTES", help="minutes of running (1 to 10080)") + limits.add_argument( + "--max-spend", metavar="DOLLARS", help="stop it once it has cost this much; `none` removes the cap" + ) download = commands.add_parser("download", help="download a job's outputs, checkpoints and logs") download.add_argument("job", help=job_help) download.add_argument("folder", nargs="?", help="the folder to put them in (default: here)") @@ -833,6 +845,13 @@ def _apply_code(args: argparse.Namespace, from_command_line: str | None) -> code folder = os.path.join(os.path.dirname(os.path.abspath(args.file)), folder) found = code_mod.collect(folder) args.image = args.image or code_mod.DEFAULT_IMAGE + builds = found.builds_cuda() + if builds and "devel" not in args.image.rsplit("/", 1)[-1]: + args.code_warning = ( + f"This code builds CUDA code on the machine ({', '.join(builds[:3])}), which needs the CUDA toolkit, and " + f"{args.image} looks like it has none. Use a -devel image (such as " + f"{code_mod.DEFAULT_IMAGE.replace('-runtime', '-devel')}) with --image, or a prebuilt wheel." + ) args.job_command = found.command(args.job_command) args.input = [*args.input, found.input_name] return found @@ -1176,6 +1195,8 @@ def render_projects() -> None: runs = runs[:1] args.smoke_of_sweep = True project_id = resolve.resolve_project(krater, args.project) + if getattr(args, "code_warning", None): + say(args.code_warning) if found is not None: _upload_code(krater, http, project_id, found, say) path = f"/projects/{project_id}/jobs" @@ -1318,7 +1339,47 @@ def render_logs() -> None: elif args.command == "download": return _download(krater, http, resolve.resolve_job(krater, args.job), args, say) elif args.command == "compare": - return _compare(krater, http, args.jobs, say, out) + refs = list(args.jobs) + if args.sweep: + if not args.project: + say("Give --project with --sweep: the project the sweep is in.") + return 2 + project_id = resolve.resolve_project(krater, args.project) + listed = krater.get(f"/projects/{project_id}/jobs")["jobs"] + pattern = re.compile(rf"^{re.escape(args.sweep)}-\d+$") + found = [j["id"] for j in reversed(listed) if pattern.match(str(j.get("name", "")))] + if not found: + say(f"No jobs named {args.sweep}-0, {args.sweep}-1, ... in that project.") + return 1 + refs += found + if not refs: + say("Give the jobs to compare, or --sweep NAME --project PROJECT.") + return 2 + return _compare(krater, http, refs, say, out) + elif args.command == "limits": + if args.max_runtime is None and args.max_spend is None: + say("Give --max-runtime, --max-spend, or both.") + return 2 + body: dict = {} + if args.max_runtime is not None: + body["max_runtime_minutes"] = args.max_runtime + if args.max_spend is not None: + if args.max_spend.strip().lower() in ("none", "off", "0"): + body["max_spend_cents"] = None + else: + try: + body["max_spend_cents"] = _spend(args.max_spend) + except argparse.ArgumentTypeError as exc: + say(str(exc)) + return 2 + job_id = resolve.resolve_job(krater, args.job) + changed = krater.post(f"/jobs/{job_id}/limits", body) + out( + changed, + lambda: say( + ("Changed: " + "; ".join(changed["changed"]) + ".") if changed.get("changed") else "Nothing to change." + ), + ) elif args.command in ("cancel", "end", "stop", "resume", "move"): job_id = resolve.resolve_job(krater, args.job) try: @@ -1860,6 +1921,7 @@ def _follow_then_download( def _compare(krater: Krater, http: httpx.Client, refs: list[str], say, out) -> int: """`krater compare`: each job's outputs/metrics.json (a JSON object), one column per job.""" columns: list[tuple[str, dict]] = [] + facts: list[dict] = [] for ref in refs: job_id = resolve.resolve_job(krater, ref) job = krater.get(f"/jobs/{job_id}") @@ -1881,20 +1943,51 @@ def _compare(krater: Krater, http: httpx.Client, refs: list[str], say, out) -> i else: say(f"{label}: its metrics.json isn't a JSON object.") columns.append((label, metrics)) + facts.append(_job_facts(job)) keys = sorted({key for _, metrics in columns for key in metrics}) def render() -> None: - if not keys: - say("No metrics to compare. A job writes them to $KRATER_OUTPUT_DIR/metrics.json, as a JSON object.") - return rows = [["", *(label for label, _ in columns)]] + rows += [[name, *(f[name] for f in facts)] for name in ("state", "cost", "runtime", "gpu")] rows += [[key, *(_cell(metrics.get(key)) for _, metrics in columns)] for key in keys] _table(rows, say) + if not keys: + say("No metrics to compare. A job writes them to $KRATER_OUTPUT_DIR/metrics.json, as a JSON object.") - out({"jobs": [{"job": label, "metrics": metrics} for label, metrics in columns]}, render) + out( + { + "jobs": [ + {"job": label, **fact, "metrics": metrics} + for (label, metrics), fact in zip(columns, facts, strict=True) + ] + }, + render, + ) return 0 if keys else 1 +def _job_facts(job: dict) -> dict: + """What `compare` shows above the metrics: the job's state, what it cost, how long it ran, and its GPU.""" + started, ended = job.get("started_at"), job.get("ended_at") + runtime = "-" + if started: + from datetime import UTC, datetime + + begin = datetime.fromisoformat(started) + finish = datetime.fromisoformat(ended) if ended else datetime.now(UTC) + minutes = max(0, int((finish - begin).total_seconds() // 60)) + runtime = f"{minutes // 60}h{minutes % 60:02d}m" if minutes >= 60 else f"{minutes}m" + machine = job.get("machine") or {} + gpu = machine.get("gpu") or "/".join(job.get("gpu_names") or []) or "-" + cents = job.get("spend_cents") + return { + "state": job.get("state", "-"), + "cost": "-" if cents is None else f"${cents / 100:,.2f}", + "runtime": runtime, + "gpu": gpu, + } + + def _flat(data: dict, prefix: str = "") -> dict: """Nested objects as dotted keys (`eval.loss`); lists are shown as they are.""" flat: dict = {} diff --git a/cli/src/krater_cli/newer.py b/cli/src/krater_cli/newer.py new file mode 100644 index 0000000..fb6cc68 --- /dev/null +++ b/cli/src/krater_cli/newer.py @@ -0,0 +1,53 @@ +"""Saying, at most once a day, that a newer `krater` is out. Krater sends the newest version it knows in a header on +every API answer (`X-Krater-CLI-Latest`), so this asks nothing of its own. Printed to stderr, so `--json` output +stays clean; `KRATER_NO_UPDATE_NOTICE=1` turns it off.""" + +from __future__ import annotations + +import json +import os +import re +import sys +import time + +from krater_cli import __version__, store + +HEADER = "X-Krater-CLI-Latest" +EVERY_SECONDS = 24 * 3600 +UPGRADE = 'uv tool install --force "git+https://github.com/patchworklabsorg/krater#subdirectory=cli"' +_said = False + + +def _parts(version: str) -> tuple[int, ...] | None: + if not re.fullmatch(r"\d+(?:\.\d+)*", version or ""): + return None + return tuple(int(part) for part in version.split(".")) + + +def is_newer(latest: str | None, current: str = __version__) -> bool: + theirs, ours = _parts(latest or ""), _parts(current) + return theirs is not None and ours is not None and theirs > ours + + +def notice(latest: str | None, *, now: float | None = None) -> None: + global _said + if _said or os.environ.get("KRATER_NO_UPDATE_NOTICE") or not is_newer(latest): + return + _said = True + now = time.time() if now is None else now + path = store.config_dir() / "update-notice.json" + try: + last = json.loads(path.read_text(encoding="utf-8")).get("at", 0) + except (OSError, ValueError, AttributeError): + last = 0 + if now - last < EVERY_SECONDS: + return + print( + f"krater: version {latest} is out (you have {__version__}). Update with: {UPGRADE}", + file=sys.stderr, + ) + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"at": now, "version": latest}), encoding="utf-8") + except OSError: + pass diff --git a/cli/tests/test_cli.py b/cli/tests/test_cli.py index 2a12455..9ac611d 100644 --- a/cli/tests/test_cli.py +++ b/cli/tests/test_cli.py @@ -2438,7 +2438,9 @@ def test_submit_with_code_uploads_the_folder_and_runs_the_command_inside_it(http body = fake.jobs[0]["body"] assert body["image"] == main.code_mod.DEFAULT_IMAGE assert body["inputs"] == [prefix] - assert body["command"] == f'cd "$KRATER_INPUT_DIR/code/{found.digest}" && python train.py' + assert body["command"] == ( + f'cd "$KRATER_INPUT_DIR/code/{found.digest}" && {main.code_mod.NEEDS_PYTHON} && python train.py' + ) assert f"Code: uploaded {folder} (2 files, 0 KB) as the input {prefix}" in said @@ -2470,7 +2472,10 @@ def test_code_with_requirements_installs_them_first_and_keeps_a_given_image(http body = fake.jobs[0]["body"] assert body["image"] == "python:3.12" - assert " && pip install --quiet --no-cache-dir -r requirements.txt && python train.py" in body["command"] + assert ( + " && PIP_BREAK_SYSTEM_PACKAGES=1 pip install --quiet --no-cache-dir -r requirements.txt && python train.py" + in body["command"] + ) def test_code_in_a_job_file_is_relative_to_the_file(http, fake, tmp_path) -> None: @@ -2527,7 +2532,7 @@ def test_run_with_code_mounts_the_folder_where_krater_puts_it(http, fake, tmp_pa digest = main.code_mod.collect(folder).digest assert spec.image == main.code_mod.DEFAULT_IMAGE assert spec.mounts == [(folder, f"/krater/inputs/code/{digest}")] - assert spec.command == f'cd "$KRATER_INPUT_DIR/code/{digest}" && python train.py' + assert spec.command == f'cd "$KRATER_INPUT_DIR/code/{digest}" && {main.code_mod.NEEDS_PYTHON} && python train.py' assert f"{os.path.abspath(folder)}:/krater/inputs/code/{digest}:ro" in spec.docker_args("c") diff --git a/cli/tests/test_polish_cli.py b/cli/tests/test_polish_cli.py index 856d89f..024f024 100644 --- a/cli/tests/test_polish_cli.py +++ b/cli/tests/test_polish_cli.py @@ -17,14 +17,26 @@ def __init__(self) -> None: super().__init__() self.metrics: dict[str, object] = {} self.latest_attempt: int | None = None + self.limits: list[dict] = [] + self.cli_latest: str | None = None def handler(self, request: httpx.Request) -> httpx.Response: url = str(request.url) if url.startswith("https://store.test/metrics/"): return httpx.Response(200, content=json.dumps(self.metrics[url.rsplit("/", 1)[1]]).encode()) - return super().handler(request) + response = super().handler(request) + if self.cli_latest and str(request.url).startswith("https://krater.test/api/v1/"): + response.headers["X-Krater-CLI-Latest"] = self.cli_latest + return response def api(self, request: httpx.Request, path: str) -> httpx.Response: + if path == f"/jobs/{JOB}/limits" and request.method == "POST": + body = json.loads(request.content) + self.limits.append(body) + changed = ( + [f"max runtime 480 → {body['max_runtime_minutes']} minutes"] if "max_runtime_minutes" in body else [] + ) + return httpx.Response(200, json={**self.details[JOB], "changed": changed}) for job_id in self.metrics: if path == f"/jobs/{job_id}/files": file = {"area": "outputs", "name": "metrics.json", "size_bytes": 9} @@ -115,8 +127,8 @@ def test_compare_shows_each_jobs_metrics_side_by_side(http, fake) -> None: header, *rows = said assert "nb (22222222)" in header and "sing (9f9f9f9f)" in header - assert [row.split()[0] for row in rows] == ["eval.acc", "loss", "steps"] - assert rows[1].split()[1:] == ["0.123457", "0.2"] and rows[2].split()[1:] == ["-", "100"] + assert [row.split()[0] for row in rows] == ["state", "cost", "runtime", "gpu", "eval.acc", "loss", "steps"] + assert rows[5].split()[1:] == ["0.123457", "0.2"] and rows[6].split()[1:] == ["-", "100"] def test_compare_says_when_there_is_nothing_to_compare(http, fake) -> None: @@ -143,3 +155,91 @@ def test_run_without_signing_in_says_so_plainly(http, fake, tmp_path) -> None: assert ( "Running without the project's secrets: couldn't read their names, as you're not signed in (krater login)." ) in said + + +def test_compare_shows_cost_runtime_and_gpu(http, fake) -> None: + _login(http, []) + fake.details[JOB].update( + spend_cents=1234, + started_at="2026-11-19T10:00:00+00:00", + ended_at="2026-11-19T11:30:00+00:00", + state="succeeded", + gpu_names=["RTX 3090"], + machine=None, + ) + fake.metrics = {JOB: {"loss": 0.5}} + said: list[str] = [] + + run(["compare", JOB], http=http, say=said.append) + + rows = {row.split()[0]: row.split()[1:] for row in said[1:]} + assert rows["state"] == ["succeeded"] and rows["cost"] == ["$12.34"] + assert rows["runtime"] == ["1h30m"] and rows["gpu"] == ["RTX", "3090"] + + +def test_compare_takes_a_whole_sweep(http, fake) -> None: + _login(http, []) + fake.project_jobs[PROJECT] = [ + {**fake.project_jobs[PROJECT][0], "id": JOB, "name": "lr-1"}, + {**fake.project_jobs[PROJECT][0], "id": JOB_C, "name": "lr-0"}, + {**fake.project_jobs[PROJECT][0], "id": "77777777-7777-7777-7777-777777777777", "name": "lr-final"}, + ] + fake.details[JOB_C] = {**fake.details[JOB], "id": JOB_C, "name": "lr-0"} + fake.metrics = {JOB: {"loss": 0.1}, JOB_C: {"loss": 0.2}} + said: list[str] = [] + + assert run(["compare", "--sweep", "lr", "--project", PROJECT], http=http, say=said.append) == 0 + + assert said[0].split()[0:2] == ["sing", "(9f9f9f9f)"] or "(9f9f9f9f)" in said[0] + assert "lr-final" not in said[0] + + +def test_limits_sends_the_changes(http, fake) -> None: + _login(http, []) + said: list[str] = [] + + assert run(["limits", JOB, "--max-runtime", "600", "--max-spend", "none"], http=http, say=said.append) == 0 + + assert fake.limits == [{"max_runtime_minutes": 600, "max_spend_cents": None}] + assert said == ["Changed: max runtime 480 → 600 minutes."] + + +def test_limits_needs_something_to_change(http, fake) -> None: + _login(http, []) + said: list[str] = [] + assert run(["limits", JOB], http=http, say=said.append) == 2 + + +def test_a_newer_cli_is_mentioned_once_a_day(http, fake, capsys, monkeypatch) -> None: + from krater_cli import newer + + monkeypatch.setattr(newer, "_said", False) + _login(http, []) + fake.cli_latest = "99.0.0" + + run(["whoami"], http=http, say=lambda _: None) + assert "version 99.0.0 is out" in capsys.readouterr().err + + monkeypatch.setattr(newer, "_said", False) + run(["whoami"], http=http, say=lambda _: None) + assert capsys.readouterr().err == "" # not again within the day + + assert not newer.is_newer("0.1.0") and not newer.is_newer("garbage") and newer.is_newer("100.0") + + +def test_code_that_builds_cuda_is_warned_of_on_a_runtime_image(http, fake, tmp_path) -> None: + from test_cli import SUBMIT_CODE, _code_folder + + _login(http, []) + files = { + "train.py": "from torch.utils.cpp_extension import load_inline\n", + "requirements.txt": "flash-attn==2.6.3\nnumpy\n", + } + folder = _code_folder(tmp_path, files) + said: list[str] = [] + + assert run([*SUBMIT_CODE, "--code", folder, "--limit", "0.4"], http=http, say=said.append) == 0 + + (warning,) = [line for line in said if "builds CUDA code" in line] + assert "flash-attn in requirements.txt" in warning and "cpp_extension in train.py" in warning + assert "-devel" in warning diff --git a/cli/uv.lock b/cli/uv.lock index f445c67..47b68c8 100644 --- a/cli/uv.lock +++ b/cli/uv.lock @@ -103,7 +103,7 @@ wheels = [ [[package]] name = "krater-cli" -version = "0.7.0" +version = "0.8.0" source = { editable = "." } dependencies = [ { name = "httpx" }, diff --git a/docs/compute/cli.md b/docs/compute/cli.md index 78b10f2..7ee879e 100644 --- a/docs/compute/cli.md +++ b/docs/compute/cli.md @@ -189,7 +189,24 @@ Ctrl-C stops following and leaves the job running. `krater logs` (without `--att the logs API and says when that attempt has no output yet and an earlier one's is shown. `krater compare ...` reads each job's `outputs/metrics.json` (a JSON object; nested objects become dotted keys -such as `eval.acc`) and prints one column per job. `--json` prints them as read. Exit code 1 when none had metrics. +such as `eval.acc`) and prints one column per job, under rows for its state, cost (`spend_cents`), runtime +(started to ended, or now) and GPU. `--sweep NAME --project P` adds every job in the project named `NAME-`, +oldest first. `--json` prints them as read. Exit code 1 when none had metrics. + +`krater limits --max-runtime MINUTES --max-spend DOLLARS` posts to `POST /api/v1/jobs/{id}/limits` (either +or both; `--max-spend none` removes the cap) and prints what changed. The server refuses a cap at or below what the +job has cost, a runtime outside 1 to 10,080 minutes, and a smoke test's runtime (`compute_jobs.change_limits`). + +`submit` and `--estimate` print the image warnings Krater returns (`warnings`, `docs/compute/images.md`). +`--code` adds its own when the folder builds CUDA code (`code.builds_cuda`: `load_inline`, `cpp_extension`, `.cu` +files, or flash-attn, deepspeed and friends in requirements.txt) and the image isn't a `-devel` one. Its command +starts with a check that `python` exists (exit 127 with a message otherwise) and installs requirements with +`PIP_BREAK_SYSTEM_PACKAGES=1`, which a venv or conda Python ignores. + +Every API answer carries `X-Krater-CLI-Latest`, the version in `cli/src/krater_cli/__init__.py` of the deployed +code. When it's newer than the CLI's own, `krater_cli.newer` prints how to update, to stderr and at most once a day +(`update-notice.json` in the config folder). `KRATER_NO_UPDATE_NOTICE=1` turns it off. Bump the version in +`__init__.py`, `pyproject.toml` and `uv.lock` together when the CLI changes. `--max-spend DOLLARS` (job files: `max-spend = 25`) sends `max_spend_cents`: the scheduler fails the job (or stops the box) once its spend reaches it (`docs/compute/scheduler.md`). diff --git a/docs/compute/images.md b/docs/compute/images.md index 6dbf87c..715b480 100644 --- a/docs/compute/images.md +++ b/docs/compute/images.md @@ -19,6 +19,33 @@ The form, the API's submit and its estimate (so the CLI too) check the image bef so an expired anonymous token is replaced, not taken for a refusal. The shell submitter (`krater.compute.submit`) doesn't ask. +- **The platform.** An image with no `linux/amd64` variant (an arm64 build from a Mac) is refused + (`registry.WrongPlatform`), naming the platforms it has and the `docker buildx --platform` fix. +- **The size.** An image whose compressed size is larger than the job's disk is refused (on `disk_gb`). + +Then `krater.services.image_checks` reads the image's config (env, user, history) and returns warnings, which the +form, the submit and estimate answers (`warnings`) and the CLI show, and which are kept on the job as +`image_warning` events. None of them refuses the job: + +- an RTX 50-series (Blackwell) GPU among the job's GPUs, with an image older than CUDA 12.8; +- an image that may not fit unpacked (`UNPACKED_FACTOR` 2.5 times its compressed size) in the disk; +- an image over 10 GB (`BIG_IMAGE_BYTES`): slow starts. The scheduler's "didn't start in time" message names the + size too; +- `pip install` in the command with a system Python marked externally managed (PEP 668: Debian/Ubuntu's apt Python, + no conda or venv on `PATH`) and no `PIP_BREAK_SYSTEM_PACKAGES`; +- a command that builds CUDA code (flash-attn, `nvcc`, `load_inline` ...) on an image without the toolkit + (`-runtime`, or no `nvcc` and no `-devel`); +- `python` in the command and no sign of Python in the image; +- `apt-get` in the command on an image whose `USER` isn't root. + +The CUDA version comes from the image's `CUDA_VERSION` variable, else its tag (`cuda12.8`, `cu128`). Its major +version is stored on the job (`min_cuda_major`) and the scheduler and the live quote only take offers whose +`cuda_max_good` is at least that (CUDA minor-version compatibility lets a 12.x image run on any 12.x driver). + +An inspection costs a few registry calls and Docker Hub counts manifest GETs against its pull limit, so the result +is kept by digest in `image_facts`: the next job on the same digest asks only for the digest (a HEAD, which Docker +Hub doesn't count). A registry that can't say still gets the tag-based checks. + The form starts with `compute_jobs.DEFAULT_IMAGE` (`pytorch/pytorch:2.7.1-cuda12.8-cudnn9-runtime`), a public image with a shell and Python, so the example on the form works as it is. diff --git a/docs/compute/runner.md b/docs/compute/runner.md index c0a49ed..fca6e65 100644 --- a/docs/compute/runner.md +++ b/docs/compute/runner.md @@ -292,6 +292,9 @@ page instead. | `KRATER_SMOKE` | `1` in a smoke test (`krater submit --smoke`): the command is stopped after its minutes, so take few steps | | the project's secrets | Such as `HF_TOKEN`, set on the project's Compute page | +The runner also sets `PYTHONUNBUFFERED=1` unless the image's environment or a project secret sets it, so a +Python job's prints reach the log as they happen rather than when a buffer fills. + Write a checkpoint under a temporary name ending in `.tmp` or `.partial` and rename it when it's complete: the runner skips those names, so it never uploads a half-written file. The image needs a POSIX `sh` and one of curl, wget or Python; its own ENTRYPOINT is not run, so put any setup in the command. An image whose `USER` isn't root can't diff --git a/docs/compute/scheduler.md b/docs/compute/scheduler.md index 3254451..110b8a3 100644 --- a/docs/compute/scheduler.md +++ b/docs/compute/scheduler.md @@ -136,6 +136,29 @@ spent half a budget on it renting directly. So: Every destroy is confirmed with `show_instance(...) is None`. Until it is, the row keeps `destroy_requested_at` and the next pass tries again. +### Limits while it runs + +`compute_jobs.change_limits` (the job page's **Change limits**, `POST /api/v1/jobs/{id}/limits`, `krater limits`) +changes a queued or running job's max runtime (1 to 10,080 minutes; not a smoke test's) and spend cap. A cap must be +above what the job has cost. A higher cap raises the job's budget hold to the smaller of the cap and what the project +has free plus what the job has spent, so a cap the budget can't cover is still taken and the hold covers what it can. +It writes a `limits_changed` event, and an audit event when an admin makes the change. `resume_box` and +`move_box` refuse a box whose spend has reached its cap (it would stop again at once): raise the cap first. + +### CUDA + +A job whose image says its CUDA version (`docs/compute/images.md`) only takes offers with `cuda_max_good` at least +its major version (`offers.shape_query(min_cuda_major=...)`), so a CUDA 13 image isn't put on a host whose driver +only runs 12. + +### Checkpoints overdue + +Every 15 minutes the worker (`compute_checkpoint_watch`, `krater.compute.checkpoint_watch`) lists the checkpoints of +each running interruptible job that isn't a smoke test. When none has been stored for two upload intervals since the +newest one or since the machine started running, whichever is later (plus 5 minutes before the first), it writes a +`checkpoint_overdue` event once per attempt and an audit event that `slack_notify` posts to the project's channel, +mentioning the submitter. The job page uses the same rule (`compute_files.checkpoint_freshness`). + ### Credit reserve Off by default. With the organization limit "Credit reserve" above 0 hours, a job doesn't rent while the provider diff --git a/docs/guides/compute-jobs.md b/docs/guides/compute-jobs.md index 62415d3..730c35d 100644 --- a/docs/guides/compute-jobs.md +++ b/docs/guides/compute-jobs.md @@ -133,7 +133,17 @@ Things to know: - **Use a CUDA build that matches your framework.** Hosts provide recent NVIDIA drivers. Newer GPUs (the RTX 50 series) need CUDA 12.8 or later, so `pytorch/pytorch:2.7.1-cuda12.8-cudnn9-runtime` is a safe start: it runs on every GPU Krater rents (Krater's own tests ran it on an RTX 5060 Ti). Older CUDA builds fail on those cards with - *no kernel image is available*. + *no kernel image is available*. Krater reads your image's CUDA version and only rents hosts whose driver supports + it. +- **Build it for `linux/amd64`.** Rented machines are x86. An image built on an Apple Silicon Mac is `linux/arm64` + unless you say otherwise, and Krater refuses it: `docker buildx build --platform linux/amd64 ...`. + +When you queue a job (or estimate it), Krater looks at the image and warns you, on the form and in `krater submit`, +about things that usually make a job fail: an RTX 50-series GPU with a CUDA build older than 12.8, an image too big +for the disk you asked for (refused if it can't fit at all), a very big image (every new machine downloads it, so +starts are slow), `pip install` into a system Python that refuses it (add `PIP_BREAK_SYSTEM_PACKAGES=1` or use a +venv), building CUDA code (flash-attn, `load_inline`) on a `-runtime` image with no compiler (use a `-devel` image), +no Python in the image, and `apt-get` in an image that doesn't run as root. Warnings don't stop the job. A minimal training image (the [starter](first-job.md) is a complete one to copy): @@ -240,12 +250,19 @@ torch.save(model.state_dict(), os.path.join(out_dir, "model.pt")) With Hugging Face `Trainer`, point `output_dir` at `$KRATER_CHECKPOINT_DIR`, set `save_total_limit=1` or `2`, and pass `resume_from_checkpoint=True` when `KRATER_RESUMED` is `1`. Save the final model to `$KRATER_OUTPUT_DIR`. +If a running job on an interruptible machine hasn't stored a checkpoint for two upload intervals (plus five minutes +before the first one), Krater posts once per attempt in the project's Slack channel, mentioning whoever submitted +it: a move would start it over. + ### Output and logs Everything your command prints (stdout and stderr) goes to the job's log, which Krater uploads every minute and shows on the job page. It keeps up to 16 MB per attempt. Print progress lines (`step 1200/50000, loss 0.84`) so you can see the job is alive. The command's exit code decides the outcome: `0` is *succeeded*, anything else *failed*. +Python's output reaches the log as it's printed: Krater sets `PYTHONUNBUFFERED=1` unless your image or a secret +sets it. + When an outbid machine comes back and your command starts again on it, the log carries on in the same attempt: a line `[krater: the runner started again on this machine at …]` marks where, and the lines above it are from before the pause. A move to another machine is a new attempt with its own log (`krater logs --attempt N` shows an @@ -313,6 +330,11 @@ See [Choosing a price](#choosing-a-price). In short, choose an order type and fi | **Stop after (minutes of running)** | The max runtime. The job fails and its machine is destroyed when it has run this long. 480 (8 hours) by default, up to 7 days | | **Stop once it has cost ($)** | Optional. A spend cap for this job: once its machine and data together have cost this much, the job fails with "Stopped at its spend cap" and its machine is destroyed (a box is stopped instead, keeping its disk). Krater checks it every minute or so, so it can go a few cents over. `--max-spend 25` from a terminal | +Both limits can be changed while the job is queued or running: **Change limits** on the job page, or `krater limits + --max-runtime 720 --max-spend 40` (`--max-spend none` removes the cap). A higher cap holds more of the +project's budget, so it can't be raised past what the project has free. A box stopped at its spend cap can't be +resumed or moved until you raise its cap. + Prices are the **full hourly cost**: the GPU plus your disk. No price can be above your project's hourly cap, shown at the top of the form ($5 an hour unless an admin set another). diff --git a/docs/guides/compute-runbook.md b/docs/guides/compute-runbook.md index 3e86972..75b6f41 100644 --- a/docs/guides/compute-runbook.md +++ b/docs/guides/compute-runbook.md @@ -137,8 +137,10 @@ Each alert says what failed and why: the error's root cause and what it usually | "The host refused the connection" | Vast's address answered but refused: likely Vast | | "took too long to answer" (`ReadTimeout`), "Vast answered 5xx" | Vast is slow or erroring | -When two or more alerts open at once and each says the DNS lookup failed or the network is unreachable, Krater posts -them as one, "this server's network or DNS looks down", listing them. +When two or more alerts are open at once and each says the DNS lookup failed or the network is unreachable, Krater +opens one alert for them, `network_outage` ("this server's network or DNS looks down"), listing them, and keeps +theirs quiet. It's reminded like any other while they stay open, and posts one "cleared" when the last of them +clears, instead of one per alert. When Weave's syncs or Quilt and Stash deliveries fail at the same minutes as the scheduler, the server's own network is down, not Vast. A blip of a few minutes is harmless: the scheduler doesn't fail a machine for heartbeats it missed @@ -160,6 +162,10 @@ All of this is only where `KRATER_EDGE_PING_EXPECTED` is on: turn it on once ala Off, pings are recorded but ignored. The endpoint is open, so a setting, not the first ping that arrives, turns the check on; a request from anyone else can't start holds or this alert. On and no ping yet counts as stale. +alastor posts to Slack itself after 3 minutes without an answer from a Krater URL, hourly while it lasts, and +once when it's back (infra#112, on once `secrets/krater-edge-alert.age` holds a webhook URL): that's the alert +that still arrives when Krater, and so its own alerts, are down. + To check: `systemctl status krater-edge-ping` and its journal on alastor; `curl -fsS https://krater.patchworklabs.org/healthz/edge` from anywhere answers `{"status":"ok"}` (harmless: it can only make the check pass). diff --git a/krater/compute/checkpoint_watch.py b/krater/compute/checkpoint_watch.py new file mode 100644 index 0000000..7b6230f --- /dev/null +++ b/krater/compute/checkpoint_watch.py @@ -0,0 +1,88 @@ +"""Warning a job's team when its checkpoints have stopped reaching storage. + +A job on a bid machine can be outbid and moved at any time, and then carries on from its last stored checkpoint. A +job that never writes to `$KRATER_CHECKPOINT_DIR`, or stopped writing, would start over: the job page marks that +"overdue" (`compute_files.checkpoint_freshness`), and this tells the project's channel once per attempt, mentioning +whoever submitted it. Run every 15 minutes by the worker, since it lists storage for each running job. Smoke tests +and on-demand machines (which can't be outbid) are left out. Framework-free. +""" + +from __future__ import annotations + +import logging +from datetime import datetime + +import sqlalchemy as sa +from sqlalchemy.orm import Session + +from krater.models import ComputeEvent, ComputeJob +from krater.models.enums import ComputeJobKind, ComputeJobState +from krater.services import audit, compute_files, compute_jobs +from krater.storage import ObjectStore, StorageError + +logger = logging.getLogger(__name__) + +EVENT = "checkpoint_overdue" + + +def _warned(session: Session, job: ComputeJob) -> bool: + payloads = session.scalars( + sa.select(ComputeEvent.payload).where(ComputeEvent.job_id == job.id, ComputeEvent.kind == EVENT) + ).all() + return any((payload or {}).get("attempt") == job.attempts for payload in payloads) + + +def warn_overdue(session: Session, store: ObjectStore, *, now: datetime) -> int: + """Warn, once per attempt, about each running job whose checkpoints are overdue. Returns how many. Flushes.""" + jobs = session.scalars( + sa.select(ComputeJob).where( + ComputeJob.kind == ComputeJobKind.JOB, + ComputeJob.state == ComputeJobState.RUNNING, + ComputeJob.interruptible.is_(True), + sa.or_(ComputeJob.smoke_minutes.is_(None), ComputeJob.smoke_minutes == 0), + ) + ).all() + warned = 0 + for job in jobs: + live = [row for row in job.instances if row.is_live] + running_since = live[-1].entered_running_at if live else None + if running_since is None or _warned(session, job): + continue + every = job.checkpoint_upload_minutes or compute_jobs.DEFAULT_CHECKPOINT_UPLOAD_MINUTES + prefix = f"projects/{job.project_id}/jobs/{job.id}/checkpoints/" + try: + files = [ + compute_files.JobFile("checkpoints", item.key[len(prefix) :], item.size_bytes, item.last_modified) + for item in store.list_prefix(prefix) + ] + except StorageError: + logger.warning("krater: couldn't list %s's checkpoints; trying again next run", job.id) + continue + fresh = compute_files.checkpoint_freshness(files, every_minutes=every, running_since=running_since, now=now) + if not fresh["overdue"]: + continue + message = ( + f"No checkpoint stored for {fresh['minutes_ago']} minutes" + if fresh["newest"] is not None + else "No checkpoint stored yet" + ) + ( + f" (it uploads every {every} minutes). If the machine is outbid and the job moves, it starts again from " + "its last stored checkpoint: save them to $KRATER_CHECKPOINT_DIR." + ) + session.add(ComputeEvent(job_id=job.id, kind=EVENT, message=message, payload={"attempt": job.attempts}, at=now)) + audit.record( + session, + None, + compute_jobs.AUDIT_CHECKPOINT_OVERDUE, + project=job.project, + payload={ + "job_id": str(job.id), + "job_name": job.name, + "owner_id": str(job.submitter_id), + "minutes_ago": fresh["minutes_ago"], + "every_minutes": every, + }, + ) + warned += 1 + session.flush() + return warned diff --git a/krater/compute/offers.py b/krater/compute/offers.py index bceaee5..660417b 100644 --- a/krater/compute/offers.py +++ b/krater/compute/offers.py @@ -114,6 +114,7 @@ def shape_query( limit: int, max_price_microusd: int | None = None, order_by_price: bool = False, + min_cuda_major: int | None = None, ) -> OfferQuery: """The search for a job's shape: what the scheduler rents from, and what a live price check asks Vast for, so the two can't drift. A box needs a direct port for SSH.""" @@ -129,6 +130,8 @@ def shape_query( max_price_microusd=max_price_microusd, max_transfer_microusd_per_gb=transfer_cap_microusd_per_gb, order_by_price=order_by_price, + # Only hosts whose driver supports the image's CUDA major version (`krater.services.image_checks`). + min_cuda=float(min_cuda_major) if min_cuda_major else None, ) diff --git a/krater/compute/scheduler.py b/krater/compute/scheduler.py index a8fd3a3..b9f2dc2 100644 --- a/krater/compute/scheduler.py +++ b/krater/compute/scheduler.py @@ -75,6 +75,7 @@ compute_settings, compute_watch, funding_holds, + image_checks, job_costs, network_watch, smoke, @@ -729,7 +730,13 @@ def _watch_one(self, row: ComputeInstance, seen: Instance | None) -> None: _finish( job, ComputeJobState.FAILED, - "The machine didn't start within 30 minutes. Check that the image exists and can be pulled.", + "The machine didn't start within 30 minutes. " + + ( + f"Its image is {job.image_bytes / 1e9:.1f} GB, which this host may have been too slow to download; " + "a smaller image starts faster." + if (job.image_bytes or 0) > image_checks.BIG_IMAGE_BYTES + else "Check that the image exists and can be pulled." + ), now=self.now, ) self._destroy(row, "never started") @@ -1938,6 +1945,7 @@ def _offers_for( transfer_cap_microusd_per_gb=policy.transfer_cap_microusd_per_gb, limit=SEARCH_LIMIT, max_price_microusd=limit, + min_cuda_major=job.min_cuda_major, ) key = json.dumps(query.to_body(), sort_keys=True) # The price is part of what's paced: an escalated job (a higher limit) searches at once. diff --git a/krater/models/__init__.py b/krater/models/__init__.py index 837e877..ad46dbc 100644 --- a/krater/models/__init__.py +++ b/krater/models/__init__.py @@ -32,6 +32,7 @@ from krater.models.gpu_price import GpuPrice from krater.models.gpu_price_sample import GpuPriceSample from krater.models.held_grant import HeldGrant +from krater.models.image_fact import ImageFact from krater.models.image_inspection import ImageInspection from krater.models.live_price import LivePrice, LivePriceUsage, LiveQuote from krater.models.ops_alert import EdgePing, OpsAlert, OpsAlertFailure, WorkerHeartbeat @@ -74,6 +75,7 @@ "GpuPrice", "GpuPriceSample", "HeldGrant", + "ImageFact", "ImageInspection", "EdgePing", "OpsAlert", diff --git a/krater/models/compute_job.py b/krater/models/compute_job.py index a691220..2388061 100644 --- a/krater/models/compute_job.py +++ b/krater/models/compute_job.py @@ -87,6 +87,11 @@ class ComputeJob(UUIDPrimaryKeyMixin, TimestampMixin, Base): # Stop the job (a box: stop it) once it has cost this much, in cents; None for no cap of its own. Checked # while it runs, against `spend_cents`. max_spend_cents: Mapped[int | None] = mapped_column(sa.Integer, nullable=True) + # The CUDA major version the image was built for (`krater.services.image_checks`): only machines whose driver + # supports it are rented. None when the image doesn't say. + min_cuda_major: Mapped[int | None] = mapped_column(sa.SmallInteger, nullable=True) + # The image's compressed size when it was queued, if its registry said: a machine that never starts names it. + image_bytes: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True) state: Mapped[ComputeJobState] = mapped_column(pg_enum(ComputeJobState, name="compute_job_state"), nullable=False) # Why the job is where it is, for the member: "waiting for an RTX 4090 bid at or under $0.20/h", ... diff --git a/krater/models/image_fact.py b/krater/models/image_fact.py new file mode 100644 index 0000000..59ac91e --- /dev/null +++ b/krater/models/image_fact.py @@ -0,0 +1,24 @@ +"""ImageFact: what an image's registry said about one digest (`krater.registry.inspect`), kept for the checks Krater +runs on a job's image when it's queued (`krater.services.image_checks`). A digest's content never changes, so a row +is never updated; it only saves asking the registry again (Docker Hub counts manifest GETs against its pull limit). +""" + +from __future__ import annotations + +from datetime import datetime + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from krater.db import Base + + +class ImageFact(Base): + __tablename__ = "image_facts" + + # The manifest digest the image's tag resolved to (the multi-platform index's for one that has several). + digest: Mapped[str] = mapped_column(sa.String(80), primary_key=True) + # `krater.registry.ImageDetails.to_json()`, for the linux/amd64 image. + details: Mapped[dict] = mapped_column(JSONB, nullable=False) + checked_at: Mapped[datetime] = mapped_column(sa.DateTime(timezone=True), nullable=False) diff --git a/krater/registry/__init__.py b/krater/registry/__init__.py index 3c40789..66c4411 100644 --- a/krater/registry/__init__.py +++ b/krater/registry/__init__.py @@ -67,6 +67,10 @@ class ImageUnavailable(RegistryError): reach, rate limits, odd answers) stays a plain RegistryError: it says nothing about the image.""" +class WrongPlatform(ImageUnavailable): + """The image has no linux/amd64 version, so no machine Vast rents could run it.""" + + class UnsafeHostError(RegistryError): """A request would go to a host Krater must not reach (not HTTPS, or a private, loopback or link-local address).""" @@ -350,7 +354,15 @@ def _platform_manifest(self, ref: ImageRef) -> tuple[str, str | None, dict]: None, ) if wanted is None: - raise RegistryError("This image has no linux/amd64 version, which is what Vast's machines run.") + found = sorted( + f"{each.get('platform', {}).get('os')}/{each.get('platform', {}).get('architecture')}" + for each in manifest.get("manifests", []) + if each.get("platform", {}).get("os") not in (None, "unknown") + ) + raise WrongPlatform( + f"This image is built only for {', '.join(found) or 'other platforms'}, and Vast's machines are " + "linux/amd64. Build it with docker buildx build --platform linux/amd64, then push it again." + ) platform_digest, manifest, _ = self._manifest(ref, wanted["digest"]) return digest, platform_digest, manifest diff --git a/krater/services/compute_files.py b/krater/services/compute_files.py index 75c19f1..bd36ea9 100644 --- a/krater/services/compute_files.py +++ b/krater/services/compute_files.py @@ -113,3 +113,28 @@ def log_tail( if truncated and "\n" in text: text = text.split("\n", 1)[1] # start at a whole line return LogTail(attempt=chosen, text=text, size_bytes=item.size_bytes, truncated=truncated) + + +#: A checkpoint is overdue once nothing new has been stored for this many upload intervals... +OVERDUE_INTERVALS = 2 +#: ...and, before the first one, this much longer: the command has to start and write it first. +FIRST_CHECKPOINT_GRACE_SECONDS = 300 + + +def checkpoint_freshness( + files: list[JobFile], *, every_minutes: int, running_since: datetime | None, now: datetime +) -> dict: + """When a checkpoint last reached storage, and whether that's overdue: nothing stored for `OVERDUE_INTERVALS` + upload intervals since the newest checkpoint, or since the machine started running (`running_since`) when that's + later, as after a move (checkpoints restored from an earlier machine keep their old times).""" + times = [f.modified for f in files if f.area == "checkpoints" and f.modified is not None] + newest = max(times) if times else None + since = max((t for t in (newest, running_since) if t is not None), default=None) + grace = ( + 0 + if newest is not None and (running_since is None or newest >= running_since) + else (FIRST_CHECKPOINT_GRACE_SECONDS) + ) + overdue = since is not None and (now - since).total_seconds() > OVERDUE_INTERVALS * every_minutes * 60 + grace + minutes = int((now - newest).total_seconds() // 60) if newest else None + return {"newest": newest, "minutes_ago": minutes, "overdue": overdue} diff --git a/krater/services/compute_jobs.py b/krater/services/compute_jobs.py index 25b6b1b..161db0a 100644 --- a/krater/services/compute_jobs.py +++ b/krater/services/compute_jobs.py @@ -13,8 +13,8 @@ import math import re import uuid -from dataclasses import dataclass -from datetime import datetime +from dataclasses import dataclass, replace +from datetime import UTC, datetime import sqlalchemy as sa from sqlalchemy.orm import Session @@ -32,15 +32,17 @@ ComputeOutbid, Project, ) -from krater.models.compute_job import FINISHED_JOB_STATES -from krater.registry import ImageUnavailable, Registry, RegistryError, parse_reference +from krater.models.compute_job import ACTIVE_JOB_STATES, FINISHED_JOB_STATES +from krater.registry import ImageUnavailable, Registry, RegistryError, WrongPlatform, parse_reference from krater.services import ( + audit, budget, clearance, compute_halt, compute_inputs, compute_settings, funding_holds, + image_checks, smoke, teams, workspaces, @@ -67,6 +69,8 @@ MAX_SPEND_CAP_CENTS = 100_000_00 #: A batch job's GPU sat nearly unused for the policy's idle warning time. AUDIT_JOB_IDLE = "compute_job_idle" +#: A running job's checkpoints stopped reaching storage (`krater.compute.checkpoint_watch`). +AUDIT_CHECKPOINT_OVERDUE = "compute_checkpoint_overdue" #: How a member's own cancel reads (`cancel_job`): no channel notice for it, they know. MEMBER_CANCEL_REASON = "Cancelled by a team member." #: Machines that hold a place under a project's "machines at once" limit (`max_machines`): all but a stopped box. @@ -125,16 +129,25 @@ def image_error(image: str) -> str | None: return None -def check_image(registry: Registry | None, image: str) -> None: +def check_image( + registry: Registry | None, + image: str, + *, + spec: JobSpec | None = None, + session: Session | None = None, + now: datetime | None = None, +) -> image_checks.ImageReport: """Raise `ValidationFailed` when `image`'s registry says no one can pull it without signing in (it doesn't exist, the tag is wrong, or it's private): a machine would be rented only to be refused the same. In production (2026-11-12) a box named the guide's example image with a full stop after it. When the registry can't be asked (down, rate-limiting, a host Krater won't reach) the job goes ahead: the scheduler still fails it as soon as a machine's pull is refused (`krater.compute.scheduler.pull_refused`).""" if registry is None: - return + return image_checks.ImageReport() try: - registry.resolve_digest(image) + details = image_checks.describe(registry, image, session=session, now=now or datetime.now(UTC)) + except WrongPlatform as exc: + raise ValidationFailed({"image": f"{image} can't run on Krater's machines: {exc}"}) from None except ImageUnavailable as exc: raise ValidationFailed( { @@ -143,8 +156,34 @@ def check_image(registry: Registry | None, image: str) -> None: } ) from None except RegistryError: - return + details = None + # Certain to fail: refused. Likely to: warnings (`krater.services.image_checks`). + if details is not None and (problem := image_checks.platform_problem(details)) is not None: + raise ValidationFailed({"image": f"{image} can't run on Krater's machines: {problem}"}) + if spec is not None and details is not None and details.size_bytes > spec.disk_gb * image_checks.BYTES_PER_GB: + raise ValidationFailed( + { + "disk_gb": f"The image alone is {details.size_bytes / image_checks.BYTES_PER_GB:.1f} GB compressed, " + f"more than {spec.disk_gb} GB of disk. Ask for at least " + f"{math.ceil(details.size_bytes * image_checks.UNPACKED_FACTOR / image_checks.BYTES_PER_GB) + 10} GB." + } + ) + warnings = ( + image_checks.warnings_for( + details, image, command=spec.command, gpu_names=tuple(spec.gpu_names), disk_gb=spec.disk_gb + ) + if spec is not None + else [] + ) + return image_checks.ImageReport( + warnings=warnings, + cuda=image_checks.cuda_version(details, image), + size_bytes=details.size_bytes if details is not None else None, + ) + +#: The event a job's image checks write for each warning (`krater.services.image_checks`). +IMAGE_WARNING = "image_warning" #: The most data a job may say it moves each way, in GB (100 TB). MAX_DATA_GB = 100_000 @@ -191,6 +230,8 @@ class JobSpec: workspace_id: uuid.UUID | None = None workspace_name: str | None = None end_holder: bool = False + # Set from the image's checks, not by the member: the CUDA major version a machine's driver must support. + min_cuda_major: int | None = None @dataclass(frozen=True) @@ -266,11 +307,13 @@ def _can_launch_for(actor: Actor, project: Project) -> bool: return teams.on_team(project, actor.user.id) -def validate_spec(spec: JobSpec, *, cap_cents: int, registry: Registry | None = None) -> None: +def validate_spec( + spec: JobSpec, *, cap_cents: int, registry: Registry | None = None, session: Session | None = None +) -> image_checks.ImageReport: """Raise `ValidationFailed` for what `submit_job` would refuse in `spec` itself (for estimating before queueing), - and, given a `registry`, for an image it refuses (`check_image`).""" + and, given a `registry`, for an image it refuses (`check_image`). Returns what the image's checks warn of.""" _validate(spec, cap_microusd=cap_cents * MICROUSD_PER_CENT) - check_image(registry, spec.image) + return check_image(registry, spec.image, spec=spec, session=session) def _validate(spec: JobSpec, *, cap_microusd: int) -> None: @@ -384,7 +427,8 @@ def submit_job( if project.status not in LAUNCHABLE_STATUSES: raise InvalidState(f"A {project.status.value} project can't run jobs.") _validate(spec, cap_microusd=cap_cents * MICROUSD_PER_CENT) - check_image(registry, spec.image) + report = check_image(registry, spec.image, spec=spec, session=session, now=now) + spec = replace(spec, min_cuda_major=report.min_cuda_major) funding_holds.require_not_held(project) clearance.require_team_cleared(session, weave_client, project) inputs_bytes = inputs_size(store, project, spec) @@ -416,6 +460,8 @@ def submit_job( smoke_minutes=spec.smoke_minutes, checkpoint_upload_minutes=spec.checkpoint_upload_minutes, max_spend_cents=spec.max_spend_cents, + min_cuda_major=spec.min_cuda_major, + image_bytes=report.size_bytes, state=ComputeJobState.QUEUED, # While compute is stopped, say so from the start, as the scheduler's next pass would. state_reason=f"Waiting: {halted}." @@ -432,6 +478,10 @@ def submit_job( session.add(job) session.flush() session.add(ComputeEvent(job_id=job.id, kind="queued", message="Queued.", payload={}, at=now)) + for warning in report.warnings: + session.add(ComputeEvent(job_id=job.id, kind=IMAGE_WARNING, message=warning, payload={}, at=now)) + # Not stored on the job: what the API's submit answers with (the events above keep them). + job.image_warnings = report.warnings if spec.kind is ComputeJobKind.BOX: if spec.workspace_id is not None: workspaces.attach( @@ -515,6 +565,15 @@ def stop_box(session: Session, actor: Actor, *, job: ComputeJob, now: datetime) session.flush() +def _require_under_spend_cap(job: ComputeJob) -> None: + """A box at its spend cap would be stopped again at once: say so instead, and how to go on.""" + if job.max_spend_cents is not None and (job.spend_cents or 0) >= job.max_spend_cents: + raise InvalidState( + f"This box has reached its spend cap of ${job.max_spend_cents / 100:,.2f}. Raise it under Change limits " + "(or krater limits) first." + ) + + def resume_box(session: Session, actor: Actor, *, job: ComputeJob, now: datetime, weave_client: WeaveClient) -> None: """Ask the scheduler to start a stopped box on its machine again (only works while the machine's GPU is free). Starting it pays for its GPU again, so the team must be cleared in Weave, as for a new job.""" @@ -522,6 +581,7 @@ def resume_box(session: Session, actor: Actor, *, job: ComputeJob, now: datetime row = _live_instance(job) if job.state is not ComputeJobState.STOPPED or row is None: raise InvalidState("Only a stopped box can be resumed.") + _require_under_spend_cap(job) if job.project.status not in LAUNCHABLE_STATUSES: raise InvalidState(f"A {job.project.status.value} project can't run boxes.") funding_holds.require_not_held(job.project) @@ -585,6 +645,7 @@ def move_box(session: Session, actor: Actor, *, job: ComputeJob, now: datetime) row = _live_instance(job) if job.state is not ComputeJobState.STOPPED or row is None: raise InvalidState("Only a stopped box can be moved.") + _require_under_spend_cap(job) row.destroy_requested_at = now row.destroy_reason = MOVED_BY_MEMBER job.state_reason = "Moving: starting elsewhere from the last saved workspace." @@ -594,3 +655,100 @@ def move_box(session: Session, actor: Actor, *, job: ComputeJob, now: datetime) ) ) session.flush() + + +#: The audit action for an admin changing a job's limits (a team member's change is in the job's history only). +AUDIT_JOB_LIMITS = "compute_job_limits" +MAX_RUNTIME_MINUTES = 7 * 24 * 60 +#: `change_limits`'s `max_spend_cents` when the request leaves the cap as it is. +UNCHANGED = object() + + +def _rate_microusd(job: ComputeJob) -> int | None: + """What the job's current machine costs an hour, if it has one.""" + live = [ + i for i in job.instances if i.state not in (ComputeInstanceState.DESTROYED, ComputeInstanceState.CREATE_FAILED) + ] + return live[-1].full_microusd if live else None + + +def _others_held(session: Session, job: ComputeJob) -> int: + rows = session.execute( + sa.select(ComputeJob.hold_cents, ComputeJob.spend_cents).where( + ComputeJob.project_id == job.project_id, + ComputeJob.state.in_(ACTIVE_JOB_STATES), + ComputeJob.id != job.id, + ) + ) + return sum(max(0, hold - (spent or 0)) for hold, spent in rows) + + +def change_limits( + session: Session, + actor: Actor, + *, + job: ComputeJob, + now: datetime, + max_runtime_minutes: int | None = None, + max_spend_cents=UNCHANGED, +) -> list[str]: + """Change an unfinished job's max runtime and spend cap (None removes the cap), by its team or an admin. Raising + the runtime of a job with a machine raises its budget hold as far as the project's free budget allows, as when + it was rented. A box stopped at its cap can be resumed once the cap is raised. Returns what changed, in words.""" + if not (actor.is_admin or _can_launch_for(actor, job.project)): + raise NotAllowed("Only the project's team or an admin can change this job.") + if job.state in FINISHED_JOB_STATES: + raise InvalidState(f"The job is already {job.state.value}.") + errors: dict[str, str] = {} + changed: list[str] = [] + if max_runtime_minutes is not None and max_runtime_minutes != job.max_runtime_minutes: + if job.smoke_minutes: + errors["max_runtime_minutes"] = "A smoke test stops after its own minutes; rerun it without --smoke." + elif not 1 <= max_runtime_minutes <= MAX_RUNTIME_MINUTES: + errors["max_runtime_minutes"] = "Choose a max runtime of 1 minute to 7 days." + else: + changed.append(f"max runtime {job.max_runtime_minutes} → {max_runtime_minutes} minutes") + if max_spend_cents is not UNCHANGED and max_spend_cents != job.max_spend_cents: + spent = job.spend_cents or 0 + if max_spend_cents is not None and not 1 <= max_spend_cents <= MAX_SPEND_CAP_CENTS: + errors["max_spend"] = "Give a spend cap of $0.01 to $100,000, or none." + elif max_spend_cents is not None and max_spend_cents <= spent: + errors["max_spend"] = f"It has already cost ${spent / 100:,.2f}: give a cap above that, or cancel it." + else: + before = "none" if job.max_spend_cents is None else f"${job.max_spend_cents / 100:,.2f}" + after = "none" if max_spend_cents is None else f"${max_spend_cents / 100:,.2f}" + changed.append(f"spend cap {before} → {after}") + if errors: + raise ValidationFailed(errors) + if not changed: + return [] + if max_runtime_minutes is not None: + job.max_runtime_minutes = max_runtime_minutes + if max_spend_cents is not UNCHANGED: + job.max_spend_cents = max_spend_cents + note = "" + rate = _rate_microusd(job) + if rate is not None and max_runtime_minutes is not None: + wanted = math.ceil(rate * max_runtime_minutes / 60 / MICROUSD_PER_CENT) + free = budget.remaining_cents(session, job.project) - _others_held(session, job) + job.hold_cents = max(job.hold_cents, min(wanted, free + (job.spend_cents or 0))) + if job.hold_cents < wanted: + note = ( + f" The project's free budget covers about ${job.hold_cents / 100:,.2f} of it; it stops when the " + "budget runs out." + ) + who = _who(actor, job) + message = f"Limits changed by {who}: {'; '.join(changed)}.{note}" + session.add( + ComputeEvent(job_id=job.id, kind="limits_changed", message=message, payload={"by": str(actor.user.id)}, at=now) + ) + if who == "an admin": + audit.record( + session, + actor, + AUDIT_JOB_LIMITS, + project=job.project, + payload={"job_id": str(job.id), "changed": changed}, + ) + session.flush() + return changed diff --git a/krater/services/image_checks.py b/krater/services/image_checks.py new file mode 100644 index 0000000..ec4e8b7 --- /dev/null +++ b/krater/services/image_checks.py @@ -0,0 +1,230 @@ +"""What a job's image tells Krater before it rents a machine: problems certain to fail it (refused at submit), and +problems likely to (warnings the member sees on the form's estimate, in the CLI and in the job's history). + +Read from the image's config (`krater.registry.inspect`): its platform, size, environment, user and build history, +plus the job's own command, GPUs and disk. Only what the image certainly says refuses a job: a platform Vast's +machines can't run, or layers bigger than the disk. Everything else is a heuristic, so it only warns. An image the +registry can't describe is checked no further, as before. Framework-free. + +The inspection is cached by digest (`ImageFact`): the registry's HEAD for the digest is free on Docker Hub, the +manifest and config GETs aren't, and a sweep submits the same image many times. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from datetime import datetime + +from sqlalchemy.orm import Session + +from krater.models import ImageFact +from krater.registry import ImageDetails, ImageUnavailable, Registry, RegistryError, WrongPlatform + +logger = logging.getLogger(__name__) + +BYTES_PER_GB = 1_000_000_000 +#: A compressed image this big takes long to pull on a slow host (and every move pulls it again). +BIG_IMAGE_BYTES = 10 * BYTES_PER_GB +#: Unpacked, an image takes about this many times its compressed size on disk. +UNPACKED_FACTOR = 2.5 +#: GPUs that need CUDA 12.8 or later (Blackwell): an older build has no kernels for them. +BLACKWELL = re.compile(r"\bRTX 50\d0\b|\bB[12]00\b|\bGB200\b|RTX PRO 6000|\bRTX 5000 Blackwell\b", re.IGNORECASE) +BLACKWELL_CUDA = (12, 8) + +_CUDA_TAG = re.compile(r"cuda[-_]?(\d{2})\.(\d)", re.IGNORECASE) +_CU_TAG = re.compile(r"(? int | None: + """What a machine's driver must support: the CUDA major version. Within one, CUDA's minor version + compatibility lets an image built for 12.8 run on a 12.4 driver, so asking for more would only shrink the + offers.""" + return self.cuda[0] if self.cuda else None + + +def _env(details: ImageDetails) -> dict[str, str]: + pairs = (item.split("=", 1) for item in details.env if "=" in item) + return {key: value for key, value in pairs} + + +def cuda_version(details: ImageDetails | None, image: str) -> tuple[int, int] | None: + if details is not None: + found = re.match(r"(\d+)\.(\d+)", _env(details).get("CUDA_VERSION", "")) + if found: + return int(found[1]), int(found[2]) + tag = image.rsplit("/", 1)[-1] + if found := _CUDA_TAG.search(tag): + return int(found[1]), int(found[2]) + if found := _CU_TAG.search(tag): + return int(found[1]), int(found[2]) + return None + + +def _history(details: ImageDetails) -> str: + return "\n".join(step.created_by for step in details.history) + + +def _path(details: ImageDetails) -> str: + return _env(details).get("PATH", "") + + +def has_cuda_toolkit(details: ImageDetails, image: str) -> bool: + env, history = _env(details), _history(details) + return ( + "devel" in image.rsplit("/", 1)[-1] + or "CUDA_HOME" in env + or bool(re.search(r"cuda-nvcc|cuda-toolkit|cuda-compiler|cuda-libraries-dev|\bnvcc\b", history)) + ) + + +def has_python(details: ImageDetails) -> bool: + env, path, history = _env(details), _path(details), _history(details) + return ( + any(key.startswith("PYTHON") for key in env) + or bool(re.search(r"conda|venv|python|mamba", path, re.IGNORECASE)) + or bool(re.search(r"\bpython3?\b|conda|uv\s+python", history)) + ) + + +def externally_managed(details: ImageDetails) -> bool: + """Whether the image's Python is the distribution's own (Debian or Ubuntu's apt `python3`, as in the newer + `pytorch/pytorch` images), where `pip install` is refused (PEP 668) without a virtual environment or + `--break-system-packages`. A conda, venv or python.org Python on the PATH takes precedence and isn't.""" + env, path = _env(details), _path(details) + if "PIP_BREAK_SYSTEM_PACKAGES" in env or "PYTHON_VERSION" in env: + return False + if re.search(r"conda|venv|mamba|/opt/python", path, re.IGNORECASE): + return False + return bool(_APT_PYTHON.search(_history(details))) + + +def warnings_for( + details: ImageDetails | None, + image: str, + *, + command: str | None, + gpu_names: tuple[str, ...], + disk_gb: int, +) -> list[str]: + """What's likely to go wrong, in words for the member. Empty when nothing is known to.""" + said: list[str] = [] + command = command or "" + cuda = cuda_version(details, image) + blackwell = sorted(name for name in gpu_names if BLACKWELL.search(name)) + if blackwell and cuda is not None and cuda < BLACKWELL_CUDA: + said.append( + f"{', '.join(blackwell)} need{'s' if len(blackwell) == 1 else ''} an image built for CUDA 12.8 or later, " + f'and this one is CUDA {cuda[0]}.{cuda[1]}: the job would fail with "no kernel image is available" on ' + "them. Use a newer image (pytorch/pytorch:2.7.1-cuda12.8-cudnn9-runtime) or leave those GPUs out." + ) + if details is None: + return said + size_gb = details.size_bytes / BYTES_PER_GB + if details.size_bytes * UNPACKED_FACTOR > disk_gb * BYTES_PER_GB: + said.append( + f"The image is {size_gb:.1f} GB compressed, about {size_gb * UNPACKED_FACTOR:.0f} GB unpacked, which may " + f"not fit in {disk_gb} GB of disk with your data and checkpoints. Ask for more disk." + ) + if details.size_bytes > BIG_IMAGE_BYTES: + said.append( + f"The image is {size_gb:.1f} GB compressed. Every new machine downloads it first, which can take many " + "minutes on a slow host (a machine that hasn't started within 30 minutes is given up on). A smaller " + "image starts faster." + ) + python = has_python(details) + if _USES_PYTHON.search(command) and not python: + said.append( + "The command runs Python, but this image doesn't look like it has Python. Use an image that does, such " + "as pytorch/pytorch or python:3.12." + ) + if ( + python + and _PIP_INSTALL.search(command) + and "break-system-packages" not in command + and "PIP_BREAK_SYSTEM_PACKAGES" not in command + and "venv" not in command + and externally_managed(details) + ): + said.append( + "This image's Python is the system's own, and pip refuses to install into it (\"externally-managed-" + 'environment"). Put PIP_BREAK_SYSTEM_PACKAGES=1 before pip install in the command, or use a virtual ' + "environment." + ) + if _COMPILES.search(command) and not has_cuda_toolkit(details, image): + said.append( + "The command looks like it builds CUDA code (nvcc, a CUDA extension, or a package such as flash-attn or " + "deepspeed that compiles one), and this image has no CUDA toolkit. Use a -devel image, or a package's " + "prebuilt wheel." + ) + user = (details.user or "").split(":")[0] + if user not in ("", "root", "0") and _APT.search(command) and "sudo" not in command: + said.append( + f"The image runs as the user {user!r}, so apt-get install in the command will be refused. Install " + "packages in the image instead." + ) + return said + + +def platform_problem(details: ImageDetails) -> str | None: + if details.platform == "linux/amd64": + return None + arm = "arm" in details.platform + return ( + f"This image is built for {details.platform}, and Vast's machines are linux/amd64" + + (" (an image built on an Apple Silicon Mac is arm64)" if arm else "") + + ". Build it with docker buildx build --platform linux/amd64, then push it again." + ) + + +def _cached(session: Session | None, digest: str) -> ImageDetails | None: + if session is None: + return None + row = session.get(ImageFact, digest) + return None if row is None else ImageDetails.from_json(row.details) + + +def _remember(session: Session | None, digest: str, details: ImageDetails, now: datetime) -> None: + if session is None or session.get(ImageFact, digest) is not None: + return + session.add(ImageFact(digest=digest, details=details.to_json(), checked_at=now)) + session.flush() + + +def describe(registry: Registry, image: str, *, session: Session | None, now: datetime) -> ImageDetails | None: + """The image's details: cached by digest, else inspected (and cached). `None` when the registry can't say. + Raises `ImageUnavailable` (`WrongPlatform` among them) when it says no machine could pull it.""" + digest = registry.resolve_digest(image) # ImageUnavailable propagates + if (known := _cached(session, digest)) is not None: + return known + try: + details = registry.inspect(image) + except ImageUnavailable: + raise + except RegistryError as exc: + logger.info("krater: couldn't inspect %s for its checks: %s", image, exc) + return None + _remember(session, digest, details, now) + return details diff --git a/krater/services/live_prices.py b/krater/services/live_prices.py index 8a044a2..f004552 100644 --- a/krater/services/live_prices.py +++ b/krater/services/live_prices.py @@ -75,6 +75,8 @@ class Shape: reliability_floor: float transfer_cap_microusd_per_gb: int | None bid_margin_percent: int + # The image's CUDA major version (`krater.services.image_checks`), so a check sees the machines a job would. + min_cuda_major: int | None = None @property def key(self) -> str: @@ -97,6 +99,7 @@ def shape_of(spec, policy: ComputePolicy) -> Shape: reliability_floor=policy.reliability_floor, transfer_cap_microusd_per_gb=policy.transfer_cap_microusd_per_gb, bid_margin_percent=policy.bid_margin_percent, + min_cuda_major=getattr(spec, "min_cuda_major", None), ) @@ -174,6 +177,7 @@ def check(session: Session, client: ComputeProvider, key: str, *, now: datetime) transfer_cap_microusd_per_gb=shape.transfer_cap_microusd_per_gb, limit=QUOTE_ROWS, order_by_price=True, + min_cuda_major=shape.min_cuda_major, ) count_searches(session, PURPOSE_QUOTE, 1, now=now) try: diff --git a/krater/services/ops_alerts.py b/krater/services/ops_alerts.py index 5163fe8..11abdcc 100644 --- a/krater/services/ops_alerts.py +++ b/krater/services/ops_alerts.py @@ -112,7 +112,11 @@ class FailureRule: "credit_out": "The Vast account is out of credit", "store_full": "Krater's storage is nearly full", "edge_ping_stale": "alastor's ping stopped reaching Krater", + "network_outage": "This server's network or DNS looks down", } +#: The alert that stands for several whose failures all say this server's own network or DNS is down +#: (`net_errors.OWN_NETWORK_DOWN`): it opens and clears once, and the alerts it covers stay quiet meanwhile. +OUTAGE = "network_outage" def _row(session: Session, key: str) -> OpsAlert: @@ -475,19 +479,11 @@ def deliver(session: Session, slack_client: SlackClient | None, channel_id: str .with_for_update() ).all() posted = [] - # Several alerts that all say this server's own network is down are one problem: post them as one. - outage = [ - row - for row in rows - if row.opened_at is not None and any(hint in row.message for hint in net_errors.OWN_NETWORK_DOWN) - ] - if len(outage) >= 2: - if _post(slack_client, channel_id, _outage_text(outage), cleared=False, key="network_outage"): - for row in outage: - row.notified_at = now - posted.append(row.key) - rows = [row for row in rows if row not in outage] + rows, covering = _deliver_outage(session, slack_client, channel_id, rows, posted, now=now) for row in rows: + if covering and row.opened_at is None and _says_outage(row): + row.notified_at = None # cleared under the outage alert, which says so once + continue cleared = row.opened_at is None text = _cleared_text(row) if cleared else _open_text(row, again=row.notified_at is not None) if not _post(slack_client, channel_id, text, cleared=cleared, key=row.key): @@ -513,8 +509,58 @@ def _post(slack_client: SlackClient | None, channel_id: str | None, text: str, * return True -def _outage_text(rows: list[OpsAlert]) -> str: - again = all(row.notified_at is not None for row in rows) +def _says_outage(row: OpsAlert) -> bool: + return any(hint in (row.message or "") for hint in net_errors.OWN_NETWORK_DOWN) + + +def _deliver_outage( + session: Session, + slack_client: SlackClient | None, + channel_id: str | None, + rows: list[OpsAlert], + posted: list[str], + *, + now: datetime, +) -> tuple[list[OpsAlert], bool]: + """Open, remind or clear the `OUTAGE` alert. Returns the rows still to post one by one, and whether the outage + alert covers this run (its alerts' own posts are then left out).""" + net = session.get(OpsAlert, OUTAGE, with_for_update=True) + members = [ + row + for row in session.scalars( + sa.select(OpsAlert).where(OpsAlert.opened_at.is_not(None), OpsAlert.key != OUTAGE).with_for_update() + ).all() + if _says_outage(row) + ] + open_before = net is not None and net.opened_at is not None + rows = [row for row in rows if row.key != OUTAGE] + if len(members) >= 2 or (open_before and members): + net = net or _row(session, OUTAGE) + if net.opened_at is None: + net.opened_at, net.first_failed_at, net.notified_at = now, now, None + net.last_failed_at, net.message = now, members[0].message + if net.notified_at is None or net.notified_at <= now - RENOTIFY: + text = _outage_text(members, again=net.notified_at is not None) + if _post(slack_client, channel_id, text, cleared=False, key=OUTAGE): + net.notified_at = now + posted.append(OUTAGE) + for row in members: + row.notified_at = now # quiet while the outage alert covers it + covered = {row.key for row in members} + return [row for row in rows if row.key not in covered], True + if open_before: + # Every alert it covered has cleared (or none is left open): one cleared post for all of them. + net.opened_at = None + if net.notified_at is not None and _post( + slack_client, channel_id, _cleared_text(net), cleared=True, key=OUTAGE + ): + net.notified_at = None + posted.append(OUTAGE) + return rows, True + return rows, False + + +def _outage_text(rows: list[OpsAlert], *, again: bool = False) -> str: lines = [ ":rotating_light: *Krater: this server's network or DNS looks down*" + (" (still)" if again else ""), f"{len(rows)} alerts failed the same way, so the services are probably fine:", diff --git a/krater/services/slack_notify.py b/krater/services/slack_notify.py index ec79262..2ecd237 100644 --- a/krater/services/slack_notify.py +++ b/krater/services/slack_notify.py @@ -46,6 +46,7 @@ AUDIT_BOX_DESTROY_SOON, AUDIT_BOX_DESTROYED, AUDIT_BOX_STOPPED, + AUDIT_CHECKPOINT_OVERDUE, AUDIT_JOB_ENDED, AUDIT_JOB_IDLE, AUDIT_TRANSFER_WARNING, @@ -763,6 +764,7 @@ def sync_budget_notifications(session: Session, slack_client: SlackClient) -> No AUDIT_TRANSFER_WARNING, AUDIT_JOB_ENDED, AUDIT_JOB_IDLE, + AUDIT_CHECKPOINT_OVERDUE, ) #: How each ended state reads in a job-ended notice. _ENDED = { @@ -813,6 +815,14 @@ def _box_message(session: Session, slack_client: SlackClient, event: AuditEvent, f"${hourly / 1_000_000:.2f}/h. It may not be using the GPU, or it may be stuck. <{link}|Check its log>, " "and cancel it if it isn't doing what you expect." ) + elif event.action == AUDIT_CHECKPOINT_OVERDUE: + ago = payload.get("minutes_ago") + when = f"for {ago} minutes" if ago is not None else "yet" + text = ( + f":floppy_disk: *Checkpoints not saving*: {who}, *{name}* hasn't stored a checkpoint {when} (it uploads " + f"every {payload.get('every_minutes', 10)} minutes). If its machine is outbid and it moves, it starts " + f"again from its last stored checkpoint. <{link}|Check the job>: it should save to $KRATER_CHECKPOINT_DIR." + ) elif event.action == AUDIT_TRANSFER_WARNING: text = ( f":warning: *Data transfer cost*: {who}, data in and out of *{name}*'s machine has cost " diff --git a/krater/web/app.py b/krater/web/app.py index be35f36..b7e7e77 100644 --- a/krater/web/app.py +++ b/krater/web/app.py @@ -6,6 +6,7 @@ from __future__ import annotations import logging +import re from collections.abc import AsyncIterator from contextlib import asynccontextmanager from pathlib import Path @@ -85,6 +86,19 @@ async def _lifespan(app: FastAPI) -> AsyncIterator[None]: procrastinate_app.close() +def _cli_version() -> str | None: + """The CLI version in this checkout (`cli/src/krater_cli/__init__.py`, copied into the image), if it's there.""" + path = Path(__file__).resolve().parents[2] / "cli" / "src" / "krater_cli" / "__init__.py" + try: + found = re.search(r'__version__ = "([0-9.]+)"', path.read_text(encoding="utf-8")) + except OSError: + return None + return found[1] if found else None + + +CLI_LATEST = _cli_version() + + def create_app() -> FastAPI: settings = get_settings() configure_logging(settings) @@ -142,6 +156,14 @@ def create_app() -> FastAPI: app.include_router(slack_interactions.router) app.include_router(api.router) + @app.middleware("http") + async def _cli_latest(request: Request, call_next): + # The newest `krater` this Krater knows, so an older CLI can say it's out of date (`krater_cli.newer`). + response = await call_next(request) + if request.url.path.startswith("/api/v1/") and CLI_LATEST: + response.headers["X-Krater-CLI-Latest"] = CLI_LATEST + return response + @app.exception_handler(NotAllowed) def _handle_not_allowed(request: Request, exc: NotAllowed): if _is_api(request): diff --git a/krater/web/routers/api.py b/krater/web/routers/api.py index ba3982f..8045e73 100644 --- a/krater/web/routers/api.py +++ b/krater/web/routers/api.py @@ -13,6 +13,7 @@ import enum import uuid +from dataclasses import replace from datetime import UTC, datetime from decimal import Decimal, InvalidOperation from typing import Annotated, Any @@ -494,7 +495,10 @@ async def submit( db_session.rollback() return _error(409, exc.message) db_session.commit() - return {**_job_json(job), "estimate": _estimate_json(db_session, project, job)} + shown = {**_job_json(job), "estimate": _estimate_json(db_session, project, job)} + # What the image's checks warn of (`krater.services.image_checks`); the job's history keeps them too. + shown["warnings"] = list(getattr(job, "image_warnings", [])) + return shown @router.post("/projects/{project_id}/jobs/estimate") @@ -514,13 +518,17 @@ async def estimate( return _error(400, "Send a JSON object.") try: spec = _spec(body, session=db_session, project=project) - compute_jobs.validate_spec(spec, cap_cents=hourly_cap_cents(project, get_settings()), registry=registry) + report = compute_jobs.validate_spec( + spec, cap_cents=hourly_cap_cents(project, get_settings()), registry=registry, session=db_session + ) + spec = replace(spec, min_cuda_major=report.min_cuda_major) inputs_bytes = compute_jobs.inputs_size(get_compute_store() if spec.inputs else None, project, spec) except ValidationFailed as exc: return _error(422, "Check the job's fields.", fields=exc.errors) except StorageError: return _error(503, "Krater's storage didn't answer; try again in a minute.") shown = _estimate_json(db_session, project, spec, inputs_bytes=inputs_bytes) + shown["warnings"] = report.warnings quote = live_quote.start(db_session, project, spec) if quote is not None: # `krater submit --estimate` polls `poll` while `status` is "checking" (krater.services.live_prices). @@ -861,6 +869,33 @@ async def post_update(project_id: uuid.UUID, request: Request, actor: Me, db_ses } +@router.post("/jobs/{job_id}/limits") +async def job_limits(job_id: uuid.UUID, request: Request, actor: Me, db_session: Db): + """Change an unfinished job's `max_runtime_minutes` and `max_spend_cents` (null removes the cap); a field left + out stays as it is.""" + job = _job(db_session, actor, job_id) + body = await _body(request) + if body is None: + return _error(400, "Send a JSON object.") + try: + runtime = _int(body, "max_runtime_minutes", None) + spend = _int(body, "max_spend_cents", None) if "max_spend_cents" in body else compute_jobs.UNCHANGED + changed = compute_jobs.change_limits( + db_session, actor, job=job, now=datetime.now(UTC), max_runtime_minutes=runtime, max_spend_cents=spend + ) + except ValidationFailed as exc: + db_session.rollback() + return _error(422, "Check the limits.", fields=exc.errors) + except NotAllowed as exc: + db_session.rollback() + return _error(403, exc.message) + except (InvalidState, DomainError) as exc: + db_session.rollback() + return _error(409, exc.message) + db_session.commit() + return {**_job_json(job), "changed": changed} + + @router.post("/jobs/{job_id}/{action}") def job_action( job_id: uuid.UUID, diff --git a/krater/web/routers/compute.py b/krater/web/routers/compute.py index 4d02b17..fc75440 100644 --- a/krater/web/routers/compute.py +++ b/krater/web/routers/compute.py @@ -7,6 +7,7 @@ from __future__ import annotations import uuid +from dataclasses import replace from datetime import UTC, datetime, timedelta from decimal import Decimal, InvalidOperation from typing import Annotated @@ -130,13 +131,14 @@ def _checkpoint_freshness(files: list[compute_files.JobFile], job: ComputeJob, * nothing stored, or the newest stored more than twice the interval ago. None for a box or a job not running.""" if job.kind is not ComputeJobKind.JOB or job.state is not ComputeJobState.RUNNING: return None - every = (job.checkpoint_upload_minutes or compute_jobs.DEFAULT_CHECKPOINT_UPLOAD_MINUTES) * 60 - times = [f.modified for f in files if f.area == "checkpoints" and f.modified is not None] - newest = max(times) if times else None - since = newest or job.started_at - overdue = since is not None and (now - since).total_seconds() > 2 * every + (0 if newest else 300) - minutes = int((now - newest).total_seconds() // 60) if newest else None - return {"newest": newest, "minutes_ago": minutes, "overdue": overdue} + live = [row for row in getattr(job, "instances", None) or () if getattr(row, "is_live", False)] + running_since = live[-1].entered_running_at if live else None + return compute_files.checkpoint_freshness( + files, + every_minutes=job.checkpoint_upload_minutes or compute_jobs.DEFAULT_CHECKPOINT_UPLOAD_MINUTES, + running_since=running_since or job.started_at, + now=now, + ) def _spend_cap(raw: str) -> int: @@ -289,6 +291,7 @@ def _form_page( quote: dict | None = None, live_table: dict | None = None, starter: bool = False, + image_warnings: list[str] | None = None, ): settings = get_settings() policy = compute_settings.effective_policy(db_session, project) @@ -328,6 +331,7 @@ def _form_page( "live_table": live_table, "source": source, "starter": starter, + "image_warnings": image_warnings or [], "escalations": ESCALATION_LABELS, "errors": errors or {}, "posted": posted or {}, @@ -491,7 +495,10 @@ async def submit_job( store = get_compute_store() if spec.inputs else None if posted.get("action") == "estimate": # Only the price: nothing is queued, and the form comes back as it was. - compute_jobs.validate_spec(spec, cap_cents=hourly_cap_cents(project, get_settings()), registry=registry) + report = compute_jobs.validate_spec( + spec, cap_cents=hourly_cap_cents(project, get_settings()), registry=registry, session=db_session + ) + spec = replace(spec, min_cuda_major=report.min_cuda_major) policy = compute_settings.effective_policy(db_session, project) estimate = job_costs.estimate( db_session, @@ -511,6 +518,7 @@ async def submit_job( estimate=estimate, quote=quote, live_table=live_table, + image_warnings=report.warnings, ) job = compute_jobs.submit_job( db_session, @@ -858,6 +866,44 @@ def delete_workspace( } +@router.post("/projects/{project_id}/compute/{job_id}/limits", dependencies=[Depends(verify_csrf_token)]) +async def job_limits( + request: Request, + project_id: uuid.UUID, + job_id: uuid.UUID, + db_session: Annotated[Session, Depends(get_session)], + actor: Annotated[Actor, Depends(fresh_actor)], +): + """The job page's "Change limits": its max runtime and spend cap (blank: no cap).""" + _require_compute() + project = _project(db_session, actor, project_id) + job = _job(db_session, project, job_id) + form = await request.form() + try: + raw_runtime = str(form.get("max_runtime_minutes", "")).strip() + try: + runtime = int(raw_runtime) if raw_runtime else None + except ValueError: + raise ValidationFailed({"max_runtime_minutes": "Give the max runtime in whole minutes."}) from None + raw_cap = str(form.get("max_spend", "")).strip() + changed = compute_jobs.change_limits( + db_session, + actor, + job=job, + now=datetime.now(UTC), + max_runtime_minutes=runtime, + max_spend_cents=_spend_cap(raw_cap) if raw_cap else None, + ) + except (ValidationFailed, InvalidState) as exc: + db_session.rollback() + text = "; ".join(exc.errors.values()) if isinstance(exc, ValidationFailed) else exc.message + flash(request, text, "error") + else: + db_session.commit() + flash(request, ("Changed: " + "; ".join(changed) + ".") if changed else "Nothing to change.", "success") + return RedirectResponse(f"/projects/{project.id}/compute/{job.id}", status_code=303) + + @router.post("/projects/{project_id}/compute/{job_id}/{action}", dependencies=[Depends(verify_csrf_token)]) def job_action( request: Request, diff --git a/krater/web/templates/compute/job.html b/krater/web/templates/compute/job.html index 629a860..fc3c0ff 100644 --- a/krater/web/templates/compute/job.html +++ b/krater/web/templates/compute/job.html @@ -313,6 +313,24 @@

Actions

{% endif %} + {% if can_cancel and not job.smoke_minutes %} +
+ Change limits +
+ + + +

A longer runtime holds more of the project's budget while it runs. A box stopped at its + spend cap can be resumed once the cap is above what it has cost.

+ +
+
+ {% endif %} {% if box and box.ending %}

Ending: the box is saving /workspace, then its machine is destroyed (within 10 minutes). End now destroys it at once, without waiting for the save.

diff --git a/krater/web/templates/compute/new.html b/krater/web/templates/compute/new.html index 5961bb7..771c7ea 100644 --- a/krater/web/templates/compute/new.html +++ b/krater/web/templates/compute/new.html @@ -264,6 +264,12 @@

New job or box

{% elif posted.gpu %}

No recent prices for these GPU models: see the price history.

{% endif %} + {% if image_warnings %} +

Check the image first.

+
    + {% for warning in image_warnings %}
  • {{ warning }}
  • {% endfor %} +
+ {% endif %} {% if quote %}

{{ quote.text }}

diff --git a/krater/worker/app.py b/krater/worker/app.py index 83d2b70..66964ad 100644 --- a/krater/worker/app.py +++ b/krater/worker/app.py @@ -13,6 +13,7 @@ import procrastinate from krater import net_errors, providers +from krater.compute import checkpoint_watch from krater.compute import launch as compute_launch from krater.compute import reconcile as compute_reconciler from krater.compute import scheduler as compute_scheduler @@ -385,6 +386,27 @@ def compute_storage_cleanup(timestamp: int) -> None: session.close() +@app.periodic(cron="*/15 * * * *") +@app.task(name="compute_checkpoint_watch") +def compute_checkpoint_watch(timestamp: int) -> None: + """Every 15 minutes, with compute on: tell a running job's team once per attempt when its checkpoints have + stopped reaching storage (`krater.compute.checkpoint_watch`).""" + if providers.active(get_settings()) is None: + return + now = datetime.fromtimestamp(timestamp, tz=UTC) + session = get_sessionmaker()() + try: + warned = checkpoint_watch.warn_overdue(session, get_compute_store(), now=now) + session.commit() + if warned: + logger.info("krater compute_checkpoint_watch: warned about %d jobs", warned) + except Exception: + logger.exception("krater.compute_checkpoint_watch failed") + session.rollback() + finally: + session.close() + + # -------------------------------------------------------------------------------------------------- # Budget grants held for the team's verification in Weave (`krater.services.grants`): written to the # ledger, and so sent to Quilt, once Weave clears every team member. Runs whether or not compute is on. diff --git a/runner/main.go b/runner/main.go index 9bca027..cad3ab0 100644 --- a/runner/main.go +++ b/runner/main.go @@ -313,6 +313,11 @@ func Run(ctx context.Context, cfg Config, apiHTTP, storageHTTP *http.Client, gpu if info.SmokeSeconds > 0 { cmd.Env = append(cmd.Env, "KRATER_SMOKE=1") } + // Python buffers stdout when it isn't a terminal, so a job's log would stay empty for minutes and then arrive in + // a block. Unbuffered unless the job or image says otherwise. + if _, set := os.LookupEnv("PYTHONUNBUFFERED"); !set && !hasEnv(secrets, "PYTHONUNBUFFERED") { + cmd.Env = append(cmd.Env, "PYTHONUNBUFFERED=1") + } cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} if err := cmd.Start(); err != nil { _ = api.Exit(ctx, 127) @@ -591,3 +596,13 @@ func restoreInputs(ctx context.Context, api *API, cfg *Config, storage *http.Cli _ = api.Exit(ctx, SetupFailed) return SetupFailed, fmt.Errorf("copying inputs: %w", err) } + +// hasEnv says whether `env` (NAME=value entries) sets `name`. +func hasEnv(env []string, name string) bool { + for _, entry := range env { + if strings.HasPrefix(entry, name+"=") { + return true + } + } + return false +} diff --git a/runner/runner_test.go b/runner/runner_test.go index 7960b3f..7e71d37 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -436,6 +436,21 @@ func TestAnOrdinaryJobHasNoSmokeVariable(t *testing.T) { } } +func TestPythonOutputIsUnbufferedUnlessTheImageSaysOtherwise(t *testing.T) { + f := newFakeKrater(t) + f.command = `test "$PYTHONUNBUFFERED" = 1` + if code, err := Run(context.Background(), f.cfg(t), http.DefaultClient, http.DefaultClient, noGPU); err != nil || code != 0 { + t.Fatalf("Run = %d, %v; want 0: PYTHONUNBUFFERED wasn't 1", code, err) + } + + t.Setenv("PYTHONUNBUFFERED", "") + g := newFakeKrater(t) + g.command = `test -z "$PYTHONUNBUFFERED"` + if code, err := Run(context.Background(), g.cfg(t), http.DefaultClient, http.DefaultClient, noGPU); err != nil || code != 0 { + t.Fatalf("Run = %d, %v; want 0: the image's own PYTHONUNBUFFERED was replaced", code, err) + } +} + func TestADeniedTokenGivesUpAtOnce(t *testing.T) { f := newFakeKrater(t) cfg := f.cfg(t) diff --git a/tests/compute/test_scheduler.py b/tests/compute/test_scheduler.py index 9fb45f3..e326ab7 100644 --- a/tests/compute/test_scheduler.py +++ b/tests/compute/test_scheduler.py @@ -1641,3 +1641,77 @@ def test_a_job_that_reaches_its_spend_cap_is_stopped(db_session, member, project assert "spend cap" in job.state_reason and "of its $0.03" in job.state_reason assert row.state is ComputeInstanceState.DESTROYED assert fake.instances == {} + + +def test_raising_a_running_jobs_runtime_raises_its_hold(db_session, member, project, client, fake) -> None: # noqa: F811 + job, _ = _running_job(db_session, member, project, client, fake, max_runtime_minutes=60) + before = job.hold_cents + + changed = compute_jobs.change_limits( + db_session, member, job=job, now=NOW + timedelta(minutes=6), max_runtime_minutes=120, max_spend_cents=500 + ) + + assert changed == ["max runtime 60 → 120 minutes", "spend cap none → $5.00"] + assert job.max_runtime_minutes == 120 and job.max_spend_cents == 500 + assert job.hold_cents > before + assert "limits_changed" in _events(db_session, job) + + +def test_limits_refuse_a_cap_under_what_it_cost_and_a_smoke_tests_runtime( + db_session, # noqa: F811 + member, # noqa: F811 + project, # noqa: F811 + client, # noqa: F811 + fake, # noqa: F811 +) -> None: + job, _ = _running_job(db_session, member, project, client, fake) + job.spend_cents = 50 + with pytest.raises(ValidationFailed) as refused: + compute_jobs.change_limits(db_session, member, job=job, now=NOW, max_spend_cents=40) + assert "already cost $0.50" in refused.value.errors["max_spend"] + + smoke = _submit(db_session, member, project, smoke_minutes=5) + with pytest.raises(ValidationFailed) as refused: + compute_jobs.change_limits(db_session, member, job=smoke, now=NOW, max_runtime_minutes=90) + assert "smoke test" in refused.value.errors["max_runtime_minutes"] + + +def test_a_box_at_its_spend_cap_is_not_resumed_until_the_cap_is_raised() -> None: + from types import SimpleNamespace + + with pytest.raises(InvalidState, match=r"reached its spend cap of \$1.00"): + compute_jobs._require_under_spend_cap(SimpleNamespace(max_spend_cents=100, spend_cents=100)) + compute_jobs._require_under_spend_cap(SimpleNamespace(max_spend_cents=200, spend_cents=100)) + + +def test_overdue_checkpoints_are_warned_of_once_per_attempt(db_session, member, project, client, fake) -> None: # noqa: F811 + from krater.compute import checkpoint_watch + from krater.storage import FakeObjectStore + + job, row = _running_job(db_session, member, project, client, fake) + store = FakeObjectStore() + started = row.entered_running_at + + assert checkpoint_watch.warn_overdue(db_session, store, now=started + timedelta(minutes=20)) == 0 + assert checkpoint_watch.warn_overdue(db_session, store, now=started + timedelta(minutes=26)) == 1 + assert checkpoint_watch.warn_overdue(db_session, store, now=started + timedelta(minutes=41)) == 0 + assert "checkpoint_overdue" in _events(db_session, job) + assert db_session.scalars( + sa.select(AuditEvent).where(AuditEvent.action == compute_jobs.AUDIT_CHECKPOINT_OVERDUE) + ).one().payload["job_id"] == str(job.id) + + +def test_fresh_checkpoints_are_not_warned_of(db_session, member, project, client, fake) -> None: # noqa: F811 + from krater.compute import checkpoint_watch + from krater.storage import FakeObjectStore + + job, row = _running_job(db_session, member, project, client, fake) + store = FakeObjectStore() + started = row.entered_running_at + store.put( + f"projects/{job.project_id}/jobs/{job.id}/checkpoints/model.pt", + content_type="", + size_bytes=9, + modified=started + timedelta(minutes=30), + ) + assert checkpoint_watch.warn_overdue(db_session, store, now=started + timedelta(minutes=45)) == 0 diff --git a/tests/services/test_image_checks.py b/tests/services/test_image_checks.py new file mode 100644 index 0000000..8972add --- /dev/null +++ b/tests/services/test_image_checks.py @@ -0,0 +1,160 @@ +"""The checks on a job's image when it's queued (`krater.services.image_checks`): what's refused, what's warned of, +the CUDA floor for offers, and the inspection cached by digest.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from krater.compute import offers +from krater.models import ImageFact +from krater.registry import FakeRegistry, HistoryStep, ImageDetails +from krater.services import compute_jobs, image_checks +from krater.services.compute_jobs import JobSpec +from krater.services.errors import ValidationFailed + +NOW = datetime(2026, 11, 19, 12, 0, tzinfo=UTC) +GB = 1_000_000_000 + + +def _details(*, platform="linux/amd64", size=3 * GB, env=(), history=(), user=None, digest="sha256:" + "1" * 64): + return ImageDetails( + digest=digest, + platform_digest=None, + platform=platform, + created=None, + size_bytes=size, + layers=5, + env=list(env), + user=user, + history=[HistoryStep(created=None, created_by=step, empty_layer=False) for step in history], + ) + + +CONDA = ("PATH=/opt/conda/bin:/usr/local/sbin:/usr/bin", "PYTORCH_VERSION=2.7.1") +UBUNTU_PY = ("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin",) +APT_PYTHON = ("/bin/sh -c apt-get update && apt-get install -y python3 python3-pip",) + + +def _warn(details, image="ghcr.io/me/train:v1", *, command="python train.py", gpus=("RTX 3090",), disk=32): + return image_checks.warnings_for(details, image, command=command, gpu_names=gpus, disk_gb=disk) + + +@pytest.mark.parametrize( + ("image", "env", "expected"), + [ + ("pytorch/pytorch:2.7.1-cuda12.8-cudnn9-runtime", (), (12, 8)), + ("pytorch/pytorch:2.4.0-cuda11.8-cudnn9-runtime", (), (11, 8)), + ("ghcr.io/me/train:torch2.5-cu124", (), (12, 4)), + ("nvidia/cuda:13.0.1-runtime-ubuntu24.04", ("CUDA_VERSION=13.0.1",), (13, 0)), + ("ubuntu:24.04", (), None), + ("ghcr.io/me/cupcake:v1", (), None), + ], +) +def test_the_images_cuda_version(image, env, expected) -> None: + assert image_checks.cuda_version(_details(env=env), image) == expected + + +def test_an_image_without_problems_has_no_warnings() -> None: + assert _warn(_details(env=CONDA)) == [] + + +def test_blackwell_gpus_with_an_older_cuda_are_warned_of() -> None: + (said,) = _warn(_details(env=CONDA), "pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime", gpus=("RTX 5090", "RTX 4090")) + assert said.startswith("RTX 5090 needs an image built for CUDA 12.8 or later, and this one is CUDA 12.4") + assert _warn(_details(env=CONDA), "pytorch/pytorch:2.7.1-cuda12.8-cudnn9-runtime", gpus=("RTX 5090",)) == [] + + +def test_a_big_image_and_one_that_may_not_fit_are_warned_of() -> None: + said = _warn(_details(env=CONDA, size=14 * GB), disk=32) + assert any("may not fit in 32 GB" in s for s in said) + assert any("Every new machine downloads it first" in s for s in said) + + +def test_pip_into_a_system_python_is_warned_of() -> None: + details = _details(env=UBUNTU_PY, history=APT_PYTHON) + (said,) = _warn(details, command="pip install -r requirements.txt && python train.py") + assert "PIP_BREAK_SYSTEM_PACKAGES=1" in said + assert _warn(details, command="PIP_BREAK_SYSTEM_PACKAGES=1 pip install x && python t.py") == [] + assert _warn(_details(env=CONDA), command="pip install x && python t.py") == [] + + +def test_building_cuda_code_without_a_toolkit_is_warned_of() -> None: + command = "pip install flash-attn --no-build-isolation && python train.py" + (said,) = _warn(_details(env=CONDA), "pytorch/pytorch:2.7.1-cuda12.8-cudnn9-runtime", command=command) + assert "no CUDA toolkit" in said + assert _warn(_details(env=CONDA), "pytorch/pytorch:2.7.1-cuda12.8-cudnn9-devel", command=command) == [] + + +def test_python_in_an_image_without_it_is_warned_of() -> None: + (said,) = _warn(_details(env=UBUNTU_PY), "ubuntu:24.04", command="python train.py") + assert "doesn't look like it has Python" in said + + +def test_apt_as_a_non_root_user_is_warned_of() -> None: + (said,) = _warn(_details(env=CONDA, user="app"), command="apt-get install -y ffmpeg && python t.py") + assert "runs as the user 'app'" in said + + +def _spec(**fields) -> JobSpec: + base = { + "name": "t", + "image": "ghcr.io/me/train:v1", + "command": "python train.py", + "gpu_names": ("RTX 3090",), + "target_microusd": 70_000, + "max_microusd": 100_000, + } + return JobSpec(**{**base, **fields}) + + +def test_an_arm64_image_is_refused(db_session) -> None: + registry = FakeRegistry() + registry.add("ghcr.io/me/train:v1", _details(platform="linux/arm64", env=CONDA)) + with pytest.raises(ValidationFailed) as refused: + compute_jobs.check_image(registry, "ghcr.io/me/train:v1", spec=_spec(), session=db_session, now=NOW) + assert "linux/arm64" in refused.value.errors["image"] and "--platform linux/amd64" in refused.value.errors["image"] + + +def test_an_image_bigger_than_the_disk_is_refused(db_session) -> None: + registry = FakeRegistry() + registry.add("ghcr.io/me/train:v1", _details(size=40 * GB, env=CONDA)) + with pytest.raises(ValidationFailed) as refused: + compute_jobs.check_image(registry, "ghcr.io/me/train:v1", spec=_spec(disk_gb=32), session=db_session, now=NOW) + assert "more than 32 GB of disk" in refused.value.errors["disk_gb"] + + +def test_the_inspection_is_cached_by_digest(db_session) -> None: + registry = FakeRegistry() + registry.add("ghcr.io/me/train:v1", _details(env=("CUDA_VERSION=12.8.1", *CONDA))) + first = compute_jobs.check_image(registry, "ghcr.io/me/train:v1", spec=_spec(), session=db_session, now=NOW) + again = compute_jobs.check_image(registry, "ghcr.io/me/train:v1", spec=_spec(), session=db_session, now=NOW) + + assert first.min_cuda_major == again.min_cuda_major == 12 + assert [call for call in registry.calls if call[0] == "inspect"] == [("inspect", "ghcr.io/me/train:v1")] + assert db_session.get(ImageFact, "sha256:" + "1" * 64) is not None + + +def test_a_registry_that_cant_say_still_reads_the_tag(db_session) -> None: + report = compute_jobs.check_image( + FakeRegistry(), "pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime", spec=_spec(gpu_names=("RTX 5080",)) + ) + assert report.min_cuda_major == 12 and report.size_bytes is None + assert any("CUDA 12.8" in s for s in report.warnings) + + +def test_offers_need_a_driver_for_the_images_cuda_major_version() -> None: + query = offers.shape_query( + rental_type="bid", + gpu_names=("RTX 3090",), + min_gpu_ram_gb=None, + num_gpus=1, + disk_gb=32, + box=False, + reliability_floor=0.9, + transfer_cap_microusd_per_gb=None, + limit=10, + min_cuda_major=13, + ) + assert query.to_body()["cuda_max_good"] == {"gte": 13.0} diff --git a/tests/services/test_ops_alerts.py b/tests/services/test_ops_alerts.py index 1829f99..5ffb8e5 100644 --- a/tests/services/test_ops_alerts.py +++ b/tests/services/test_ops_alerts.py @@ -375,8 +375,15 @@ def test_alerts_that_all_say_the_network_is_down_are_posted_as_one(db_session: S for n in range(ops_alerts.FAILURE_RULES[key].threshold): ops_alerts.record_failure(db_session, key, message, now=NOW + timedelta(minutes=n)) - assert sorted(_watch(db_session, slack, NOW + timedelta(minutes=6))) == ["quilt", "stash"] + assert _watch(db_session, slack, NOW + timedelta(minutes=6)) == ["network_outage"] (text,) = _texts(slack) assert "network or DNS looks down" in text assert "Sending events to Quilt" in text and "Archiving submission links" in text assert _watch(db_session, slack, NOW + timedelta(minutes=11)) == [] + + # Both windows pass without a failure: one cleared post, not one per alert. + later = NOW + timedelta(minutes=40) + assert _watch(db_session, slack, later) == ["network_outage"] + assert len(_texts(slack)) == 2 and "cleared" in _texts(slack)[-1] and "network or DNS" in _texts(slack)[-1] + assert _watch(db_session, slack, later + timedelta(minutes=5)) == [] + assert len(_texts(slack)) == 2 diff --git a/tests/web/test_polish_batch_web.py b/tests/web/test_polish_batch_web.py index 91d4cb6..55105e0 100644 --- a/tests/web/test_polish_batch_web.py +++ b/tests/web/test_polish_batch_web.py @@ -142,3 +142,67 @@ def test_checkpoint_freshness() -> None: assert _checkpoint_freshness([], _running(started_at=NOW - timedelta(minutes=14)), now=NOW)["overdue"] is False assert _checkpoint_freshness([], _running(), now=NOW)["overdue"] is True assert _checkpoint_freshness([], SimpleNamespace(kind=ComputeJobKind.BOX, state=None), now=NOW) is None + + +def test_the_api_changes_a_queued_jobs_limits(client: TestClient, project) -> None: + queued = client.post(f"/api/v1/projects/{project.id}/jobs", json=BODY, headers=_as(MEMBER_SUB)).json() + + changed = client.post( + f"/api/v1/jobs/{queued['id']}/limits", + json={"max_runtime_minutes": 300, "max_spend_cents": 900}, + headers=_as(MEMBER_SUB), + ) + assert changed.status_code == 200, changed.text + assert changed.json()["changed"] == ["max runtime 240 → 300 minutes", "spend cap none → $9.00"] + + removed = client.post( + f"/api/v1/jobs/{queued['id']}/limits", json={"max_spend_cents": None}, headers=_as(MEMBER_SUB) + ) + assert removed.json()["changed"] == ["spend cap $9.00 → none"] and removed.json()["max_spend_cents"] is None + + bad = client.post(f"/api/v1/jobs/{queued['id']}/limits", json={"max_runtime_minutes": 0}, headers=_as(MEMBER_SUB)) + assert bad.status_code == 422 + + +def test_a_reviewer_cant_change_a_jobs_limits(client: TestClient, project) -> None: + queued = client.post(f"/api/v1/projects/{project.id}/jobs", json=BODY, headers=_as(MEMBER_SUB)).json() + response = client.post( + f"/api/v1/jobs/{queued['id']}/limits", json={"max_runtime_minutes": 300}, headers=_as(REVIEWER_SUB) + ) + assert response.status_code == 403 + + +def test_the_job_page_changes_its_limits(client: TestClient, login_as, project, db_session) -> None: + queued = client.post(f"/api/v1/projects/{project.id}/jobs", json=BODY, headers=_as(MEMBER_SUB)).json() + login_as(MEMBER_SUB) + page = client.get(f"/projects/{project.id}/compute/{queued['id']}") + assert "Change limits" in page.text + + response = client.post( + f"/projects/{project.id}/compute/{queued['id']}/limits", + data={"max_runtime_minutes": "360", "max_spend": "15", "csrf_token": get_csrf_token(page.text)}, + follow_redirects=False, + ) + assert response.status_code == 303 + job = db_session.get(ComputeJob, queued["id"]) + db_session.refresh(job) + assert job.max_runtime_minutes == 360 and job.max_spend_cents == 1500 + + +def test_submit_says_what_the_image_checks_warn_of(client: TestClient, project, db_session) -> None: + body = {**BODY, "image": "pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime", "gpu_names": ["RTX 5090"]} + estimate = client.post(f"/api/v1/projects/{project.id}/jobs/estimate", json=body, headers=_as(MEMBER_SUB)) + assert any("CUDA 12.8" in w for w in estimate.json()["warnings"]) + + queued = client.post(f"/api/v1/projects/{project.id}/jobs", json=body, headers=_as(MEMBER_SUB)) + assert queued.status_code == 201, queued.text + assert any("CUDA 12.8" in w for w in queued.json()["warnings"]) + job = db_session.get(ComputeJob, queued.json()["id"]) + assert job.min_cuda_major == 12 + + +def test_api_answers_name_the_newest_cli(client: TestClient, project) -> None: + from krater.web.app import CLI_LATEST + + response = client.get("/api/v1/me", headers=_as(MEMBER_SUB)) + assert CLI_LATEST and response.headers["X-Krater-CLI-Latest"] == CLI_LATEST