Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
"""add image_facts, compute_jobs.min_cuda_major and compute_jobs.image_bytes: the checks on a job's image

Revision ID: a8d3f1c6e2b9
Revises: e2b7c4d9a1f6
Create Date: 2026-11-19 23:30:00
"""

import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

from alembic import op

revision = "a8d3f1c6e2b9"
down_revision = "e2b7c4d9a1f6"
branch_labels = None
depends_on = None


def upgrade() -> None:
op.create_table(
"image_facts",
sa.Column("digest", sa.String(length=80), primary_key=True),
sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.Column("checked_at", sa.DateTime(timezone=True), nullable=False),
)
op.add_column("compute_jobs", sa.Column("min_cuda_major", sa.SmallInteger(), nullable=True))
op.add_column("compute_jobs", sa.Column("image_bytes", sa.BigInteger(), nullable=True))


def downgrade() -> None:
op.drop_column("compute_jobs", "image_bytes")
op.drop_column("compute_jobs", "min_cuda_major")
op.drop_table("image_facts")
2 changes: 1 addition & 1 deletion cli/pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "krater-cli"
version = "0.7.0"
version = "0.8.0"
description = "The krater command: run jobs and boxes on Krater Compute (Project Ganymede, Patchwork Labs)"
requires-python = ">=3.10"
dependencies = ["httpx>=0.27", "tomli>=2; python_version < \"3.11\""]
Expand Down
2 changes: 1 addition & 1 deletion cli/src/krater_cli/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
"""The `krater` command: Krater Compute from a terminal (docs/compute/cli.md)."""

__version__ = "0.7.0"
__version__ = "0.8.0"
3 changes: 2 additions & 1 deletion cli/src/krater_cli/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

import httpx

from krater_cli import auth
from krater_cli import auth, newer


class KraterError(Exception):
Expand Down Expand Up @@ -78,6 +78,7 @@ def request(self, method: str, path: str, *, json: Any = None) -> dict:
if response.status_code == 401:
self.creds = auth.refresh(self.creds, http=self.http)
response = self._send_patiently(method, path, json)
newer.notice(response.headers.get(newer.HEADER))
try:
data = response.json()
except ValueError:
Expand Down
48 changes: 46 additions & 2 deletions cli/src/krater_cli/code.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import hashlib
import os
import re
from dataclasses import dataclass

#: The image a `--code` job runs on when it names none: Krater's form default, PyTorch with CUDA 12.8 (runs on every
Expand Down Expand Up @@ -69,12 +70,55 @@ def has_requirements(self) -> bool:
return any(rel == "requirements.txt" for _, rel in self.files)

def command(self, command: str) -> str:
"""The job's command: into the folder, the requirements if there are any, then the member's own."""
"""The job's command: into the folder, a plain error when the image has no Python, the requirements if there
are any, then the member's own. PIP_BREAK_SYSTEM_PACKAGES lets pip install into an image whose Python is the
system's own (Ubuntu's, in the newer pytorch/pytorch images), which it otherwise refuses; elsewhere it does
nothing."""
parts = [f'cd "$KRATER_INPUT_DIR/{INPUTS_PREFIX}/{self.digest}"']
if self.has_requirements or re.search(r"\bpython3?\b|\bpip3?\b", command):
parts.append(NEEDS_PYTHON)
if self.has_requirements:
parts.append("pip install --quiet --no-cache-dir -r requirements.txt")
parts.append("PIP_BREAK_SYSTEM_PACKAGES=1 pip install --quiet --no-cache-dir -r requirements.txt")
return " && ".join(parts) + " && " + command

def builds_cuda(self) -> list[str]:
"""What in the folder compiles CUDA code on the machine, which needs an image with the CUDA toolkit (a
-devel one): `load_inline`, `cpp_extension`, `torch.cuda._compile_kernel` or NVRTC in its Python files, or
packages in its requirements.txt that build CUDA extensions when installed. Triton doesn't need it."""
found: list[str] = []
for path, rel in self.files:
if rel == "requirements.txt":
text = _read(path)
found += [
f"{name} in requirements.txt"
for name in _BUILDS_PACKAGES
if re.search(rf"^\s*{re.escape(name)}\b", text, re.IGNORECASE | re.MULTILINE)
]
elif rel.endswith((".py", ".cu", ".cpp")) or rel == "setup.py":
text = _read(path)
if rel.endswith(".cu"):
found.append(rel)
elif hit := _BUILDS_CODE.search(text):
found.append(f"{hit[0]} in {rel}")
return found


#: Stops with a plain error when the image has no Python, instead of "python: not found" deep in a log.
NEEDS_PYTHON = (
"{ command -v python >/dev/null 2>&1 || command -v python3 >/dev/null 2>&1 || "
'{ echo "krater: this image has no Python; use one that does, such as the default" >&2; exit 127; }; }'
)
_BUILDS_CODE = re.compile(r"load_inline|cpp_extension|_compile_kernel|\bnvrtc\b|CUDAExtension")
_BUILDS_PACKAGES = ("flash-attn", "flash_attn", "deepspeed", "apex", "mamba-ssm", "causal-conv1d", "xformers")


def _read(path: str) -> str:
try:
with open(path, encoding="utf-8", errors="replace") as handle:
return handle.read(2_000_000)
except OSError:
return ""


def collect(folder: str) -> Code:
"""The folder's files, in a stable order, with their hash. Raises `CodeError` when it isn't a usable code folder."""
Expand Down
107 changes: 100 additions & 7 deletions cli/src/krater_cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,8 +334,20 @@ def _parser() -> argparse.ArgumentParser:
wait.add_argument("job", help=job_help)
wait.add_argument("--timeout", type=_positive, help="give up after this many minutes (exit code 124)")
wait.add_argument("--interval", type=_positive, default=5, help="seconds between checks (default 5)")
compare = commands.add_parser("compare", help="jobs' outputs/metrics.json side by side")
compare.add_argument("jobs", nargs="+", metavar="job", help=job_help)
compare = commands.add_parser("compare", help="jobs' outputs/metrics.json side by side, with cost and runtime")
compare.add_argument("jobs", nargs="*", metavar="job", help=job_help)
compare.add_argument(
"--sweep",
metavar="NAME",
help="every job of a sweep (or --count): the jobs named NAME-0, NAME-1, ... (with --project)",
)
compare.add_argument("--project", help="the project the sweep is in")
limits = commands.add_parser("limits", help="change a queued or running job's max runtime or spend cap")
limits.add_argument("job", help=job_help)
limits.add_argument("--max-runtime", type=int, metavar="MINUTES", help="minutes of running (1 to 10080)")
limits.add_argument(
"--max-spend", metavar="DOLLARS", help="stop it once it has cost this much; `none` removes the cap"
)
download = commands.add_parser("download", help="download a job's outputs, checkpoints and logs")
download.add_argument("job", help=job_help)
download.add_argument("folder", nargs="?", help="the folder to put them in (default: here)")
Expand Down Expand Up @@ -833,6 +845,13 @@ def _apply_code(args: argparse.Namespace, from_command_line: str | None) -> code
folder = os.path.join(os.path.dirname(os.path.abspath(args.file)), folder)
found = code_mod.collect(folder)
args.image = args.image or code_mod.DEFAULT_IMAGE
builds = found.builds_cuda()
if builds and "devel" not in args.image.rsplit("/", 1)[-1]:
args.code_warning = (
f"This code builds CUDA code on the machine ({', '.join(builds[:3])}), which needs the CUDA toolkit, and "
f"{args.image} looks like it has none. Use a -devel image (such as "
f"{code_mod.DEFAULT_IMAGE.replace('-runtime', '-devel')}) with --image, or a prebuilt wheel."
)
args.job_command = found.command(args.job_command)
args.input = [*args.input, found.input_name]
return found
Expand Down Expand Up @@ -1176,6 +1195,8 @@ def render_projects() -> None:
runs = runs[:1]
args.smoke_of_sweep = True
project_id = resolve.resolve_project(krater, args.project)
if getattr(args, "code_warning", None):
say(args.code_warning)
if found is not None:
_upload_code(krater, http, project_id, found, say)
path = f"/projects/{project_id}/jobs"
Expand Down Expand Up @@ -1318,7 +1339,47 @@ def render_logs() -> None:
elif args.command == "download":
return _download(krater, http, resolve.resolve_job(krater, args.job), args, say)
elif args.command == "compare":
return _compare(krater, http, args.jobs, say, out)
refs = list(args.jobs)
if args.sweep:
if not args.project:
say("Give --project with --sweep: the project the sweep is in.")
return 2
project_id = resolve.resolve_project(krater, args.project)
listed = krater.get(f"/projects/{project_id}/jobs")["jobs"]
pattern = re.compile(rf"^{re.escape(args.sweep)}-\d+$")
found = [j["id"] for j in reversed(listed) if pattern.match(str(j.get("name", "")))]
if not found:
say(f"No jobs named {args.sweep}-0, {args.sweep}-1, ... in that project.")
return 1
refs += found
if not refs:
say("Give the jobs to compare, or --sweep NAME --project PROJECT.")
return 2
return _compare(krater, http, refs, say, out)
elif args.command == "limits":
if args.max_runtime is None and args.max_spend is None:
say("Give --max-runtime, --max-spend, or both.")
return 2
body: dict = {}
if args.max_runtime is not None:
body["max_runtime_minutes"] = args.max_runtime
if args.max_spend is not None:
if args.max_spend.strip().lower() in ("none", "off", "0"):
body["max_spend_cents"] = None
else:
try:
body["max_spend_cents"] = _spend(args.max_spend)
except argparse.ArgumentTypeError as exc:
say(str(exc))
return 2
job_id = resolve.resolve_job(krater, args.job)
changed = krater.post(f"/jobs/{job_id}/limits", body)
out(
changed,
lambda: say(
("Changed: " + "; ".join(changed["changed"]) + ".") if changed.get("changed") else "Nothing to change."
),
)
elif args.command in ("cancel", "end", "stop", "resume", "move"):
job_id = resolve.resolve_job(krater, args.job)
try:
Expand Down Expand Up @@ -1860,6 +1921,7 @@ def _follow_then_download(
def _compare(krater: Krater, http: httpx.Client, refs: list[str], say, out) -> int:
"""`krater compare`: each job's outputs/metrics.json (a JSON object), one column per job."""
columns: list[tuple[str, dict]] = []
facts: list[dict] = []
for ref in refs:
job_id = resolve.resolve_job(krater, ref)
job = krater.get(f"/jobs/{job_id}")
Expand All @@ -1881,20 +1943,51 @@ def _compare(krater: Krater, http: httpx.Client, refs: list[str], say, out) -> i
else:
say(f"{label}: its metrics.json isn't a JSON object.")
columns.append((label, metrics))
facts.append(_job_facts(job))
keys = sorted({key for _, metrics in columns for key in metrics})

def render() -> None:
if not keys:
say("No metrics to compare. A job writes them to $KRATER_OUTPUT_DIR/metrics.json, as a JSON object.")
return
rows = [["", *(label for label, _ in columns)]]
rows += [[name, *(f[name] for f in facts)] for name in ("state", "cost", "runtime", "gpu")]
rows += [[key, *(_cell(metrics.get(key)) for _, metrics in columns)] for key in keys]
_table(rows, say)
if not keys:
say("No metrics to compare. A job writes them to $KRATER_OUTPUT_DIR/metrics.json, as a JSON object.")

out({"jobs": [{"job": label, "metrics": metrics} for label, metrics in columns]}, render)
out(
{
"jobs": [
{"job": label, **fact, "metrics": metrics}
for (label, metrics), fact in zip(columns, facts, strict=True)
]
},
render,
)
return 0 if keys else 1


def _job_facts(job: dict) -> dict:
"""What `compare` shows above the metrics: the job's state, what it cost, how long it ran, and its GPU."""
started, ended = job.get("started_at"), job.get("ended_at")
runtime = "-"
if started:
from datetime import UTC, datetime

begin = datetime.fromisoformat(started)
finish = datetime.fromisoformat(ended) if ended else datetime.now(UTC)
minutes = max(0, int((finish - begin).total_seconds() // 60))
runtime = f"{minutes // 60}h{minutes % 60:02d}m" if minutes >= 60 else f"{minutes}m"
machine = job.get("machine") or {}
gpu = machine.get("gpu") or "/".join(job.get("gpu_names") or []) or "-"
cents = job.get("spend_cents")
return {
"state": job.get("state", "-"),
"cost": "-" if cents is None else f"${cents / 100:,.2f}",
"runtime": runtime,
"gpu": gpu,
}


def _flat(data: dict, prefix: str = "") -> dict:
"""Nested objects as dotted keys (`eval.loss`); lists are shown as they are."""
flat: dict = {}
Expand Down
53 changes: 53 additions & 0 deletions cli/src/krater_cli/newer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
"""Saying, at most once a day, that a newer `krater` is out. Krater sends the newest version it knows in a header on
every API answer (`X-Krater-CLI-Latest`), so this asks nothing of its own. Printed to stderr, so `--json` output
stays clean; `KRATER_NO_UPDATE_NOTICE=1` turns it off."""

from __future__ import annotations

import json
import os
import re
import sys
import time

from krater_cli import __version__, store

HEADER = "X-Krater-CLI-Latest"
EVERY_SECONDS = 24 * 3600
UPGRADE = 'uv tool install --force "git+https://github.com/patchworklabsorg/krater#subdirectory=cli"'
_said = False


def _parts(version: str) -> tuple[int, ...] | None:
if not re.fullmatch(r"\d+(?:\.\d+)*", version or ""):
return None
return tuple(int(part) for part in version.split("."))


def is_newer(latest: str | None, current: str = __version__) -> bool:
theirs, ours = _parts(latest or ""), _parts(current)
return theirs is not None and ours is not None and theirs > ours


def notice(latest: str | None, *, now: float | None = None) -> None:
global _said
if _said or os.environ.get("KRATER_NO_UPDATE_NOTICE") or not is_newer(latest):
return
_said = True
now = time.time() if now is None else now
path = store.config_dir() / "update-notice.json"
try:
last = json.loads(path.read_text(encoding="utf-8")).get("at", 0)
except (OSError, ValueError, AttributeError):
last = 0
if now - last < EVERY_SECONDS:
return
print(
f"krater: version {latest} is out (you have {__version__}). Update with: {UPGRADE}",
file=sys.stderr,
)
try:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"at": now, "version": latest}), encoding="utf-8")
except OSError:
pass
11 changes: 8 additions & 3 deletions cli/tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2438,7 +2438,9 @@ def test_submit_with_code_uploads_the_folder_and_runs_the_command_inside_it(http
body = fake.jobs[0]["body"]
assert body["image"] == main.code_mod.DEFAULT_IMAGE
assert body["inputs"] == [prefix]
assert body["command"] == f'cd "$KRATER_INPUT_DIR/code/{found.digest}" && python train.py'
assert body["command"] == (
f'cd "$KRATER_INPUT_DIR/code/{found.digest}" && {main.code_mod.NEEDS_PYTHON} && python train.py'
)
assert f"Code: uploaded {folder} (2 files, 0 KB) as the input {prefix}" in said


Expand Down Expand Up @@ -2470,7 +2472,10 @@ def test_code_with_requirements_installs_them_first_and_keeps_a_given_image(http

body = fake.jobs[0]["body"]
assert body["image"] == "python:3.12"
assert " && pip install --quiet --no-cache-dir -r requirements.txt && python train.py" in body["command"]
assert (
" && PIP_BREAK_SYSTEM_PACKAGES=1 pip install --quiet --no-cache-dir -r requirements.txt && python train.py"
in body["command"]
)


def test_code_in_a_job_file_is_relative_to_the_file(http, fake, tmp_path) -> None:
Expand Down Expand Up @@ -2527,7 +2532,7 @@ def test_run_with_code_mounts_the_folder_where_krater_puts_it(http, fake, tmp_pa
digest = main.code_mod.collect(folder).digest
assert spec.image == main.code_mod.DEFAULT_IMAGE
assert spec.mounts == [(folder, f"/krater/inputs/code/{digest}")]
assert spec.command == f'cd "$KRATER_INPUT_DIR/code/{digest}" && python train.py'
assert spec.command == f'cd "$KRATER_INPUT_DIR/code/{digest}" && {main.code_mod.NEEDS_PYTHON} && python train.py'
assert f"{os.path.abspath(folder)}:/krater/inputs/code/{digest}:ro" in spec.docker_args("c")


Expand Down
Loading
Loading