Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ jobs:
platforms: linux/amd64
push: true
tags: ghcr.io/${{ github.repository }}:${{ github.sha }}
# /healthz reports it, so `curl <site>/healthz` says which commit is live.
build-args: |
KRATER_REVISION=${{ github.sha }}
# krater-build on weft checks this label against the commit it asked for.
labels: |
org.patchworklabs.krater.rev=${{ github.sha }}
Expand Down
43 changes: 26 additions & 17 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,24 +40,19 @@ env:
URL: https://krater-staging.patchworklabs.org

jobs:
deploy:
# Its own job, so a run for a commit that is no longer the tip of main shows its deploy job as *skipped*,
# not as a green "success" that deployed nothing.
pick:
# A CI run on a push to main that passed, or a manual run from main.
if: >-
(github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') ||
(github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push')
runs-on: ubuntu-latest
# At job level: a run whose job the `if` skips does not take the group, so it cannot replace a
# pending deploy.
concurrency:
group: deploy-krater-staging
cancel-in-progress: false
# The environment is also the OIDC subject the Tailscale federated identity trusts.
environment:
name: staging
url: https://krater-staging.patchworklabs.org
permissions:
contents: read
id-token: write
outputs:
deploy: ${{ steps.sha.outputs.deploy }}
sha: ${{ steps.sha.outputs.sha }}
steps:
- name: Pick the commit
id: sha
Expand All @@ -69,27 +64,42 @@ jobs:
# replaces a newer one.
tip=$(gh api "repos/${{ github.repository }}/commits/main" --jq .sha)
if [ "$SHA" != "$tip" ]; then
echo "::notice::$SHA is no longer the tip of main ($tip). Skipping."
echo "::notice::$SHA is no longer the tip of main ($tip). Skipping; the tip's own run deploys it."
echo "deploy=false" >> "$GITHUB_OUTPUT"
else
echo "deploy=true" >> "$GITHUB_OUTPUT"
fi
echo "sha=$SHA" >> "$GITHUB_OUTPUT"

deploy:
needs: pick
if: needs.pick.outputs.deploy == 'true'
runs-on: ubuntu-latest
# At job level: a run whose job the `if` skips does not take the group, so it cannot replace a
# pending deploy.
concurrency:
group: deploy-krater-staging
cancel-in-progress: false
# The environment is also the OIDC subject the Tailscale federated identity trusts.
environment:
name: staging
url: https://krater-staging.patchworklabs.org
permissions:
contents: read
id-token: write
steps:
- name: Join the tailnet
if: steps.sha.outputs.deploy == 'true'
uses: tailscale/github-action@064b10558093f38325e838ddea35486bdc6b1d5c # v4.4.0
with:
oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
audience: ${{ secrets.TS_AUDIENCE }}
tags: tag:ci-krater
version: latest

- name: Deploy ${{ steps.sha.outputs.sha }}
if: steps.sha.outputs.deploy == 'true'
- name: Deploy ${{ needs.pick.outputs.sha }}
env:
KEY: ${{ secrets.KRATER_DEPLOY_SSH_KEY }}
SHA: ${{ steps.sha.outputs.sha }}
SHA: ${{ needs.pick.outputs.sha }}
# The first build of the Python dependencies takes several minutes on weft. The build may take
# 45 minutes and the start 10.
timeout-minutes: 60
Expand All @@ -114,7 +124,6 @@ jobs:
ssh_weft "$SHA"

- name: Check the site
if: steps.sha.outputs.deploy == 'true'
run: |
for _ in $(seq 1 12); do
if curl -fsS --max-time 10 "$URL/healthz" && curl -fsS --max-time 10 "$URL/healthz/worker"; then
Expand Down
4 changes: 4 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ RUN --mount=type=cache,target=/home/krater/.cache/uv,uid=999,gid=999 \
uv sync --frozen --no-dev
COPY --chown=krater:krater --from=runner /out/krater-runner-linux-amd64 /app/runner/dist/krater-runner-linux-amd64

# The commit this image was built from, for /healthz (CI passes it; a local build leaves it empty).
ARG KRATER_REVISION=""
ENV KRATER_REVISION=$KRATER_REVISION

EXPOSE 8000

# The portal is the default; docker-compose.yml overrides `command` for `migrate` and `worker`.
Expand Down
23 changes: 23 additions & 0 deletions alembic/versions/e2b7c4d9a1f6_add_compute_jobs_max_spend_cents.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
"""add compute_jobs.max_spend_cents: a job's own spend cap (`--max-spend`)

Revision ID: e2b7c4d9a1f6
Revises: c5d1e8a3f7b4
Create Date: 2026-11-19 22:30:00
"""

import sqlalchemy as sa

from alembic import op

revision = "e2b7c4d9a1f6"
down_revision = "c5d1e8a3f7b4"
branch_labels = None
depends_on = None


def upgrade() -> None:
op.add_column("compute_jobs", sa.Column("max_spend_cents", sa.Integer(), nullable=True))


def downgrade() -> None:
op.drop_column("compute_jobs", "max_spend_cents")
2 changes: 1 addition & 1 deletion cli/pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "krater-cli"
version = "0.6.0"
version = "0.7.0"
description = "The krater command: run jobs and boxes on Krater Compute (Project Ganymede, Patchwork Labs)"
requires-python = ">=3.10"
dependencies = ["httpx>=0.27", "tomli>=2; python_version < \"3.11\""]
Expand Down
2 changes: 1 addition & 1 deletion cli/src/krater_cli/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
"""The `krater` command: Krater Compute from a terminal (docs/compute/cli.md)."""

__version__ = "0.6.0"
__version__ = "0.7.0"
1 change: 1 addition & 0 deletions cli/src/krater_cli/jobfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
"json",
"estimate",
"confirm_transfer_cost",
"follow",
# `krater run`'s own: about this computer, not the job.
"local",
"input_dir",
Expand Down
139 changes: 135 additions & 4 deletions cli/src/krater_cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,18 @@ def _price(text: str) -> str:
return cleaned


def _spend(text: str) -> int:
"""A spend cap in dollars, `$25` included, as whole cents."""
cleaned = text.strip().lstrip("$").strip()
try:
value = Decimal(cleaned)
except InvalidOperation:
value = Decimal("nan")
if not value.is_finite() or value <= 0:
raise argparse.ArgumentTypeError(f"{text!r} isn't an amount in dollars, such as 25")
return max(1, int((value * 100).to_integral_value()))


def _positive(text: str) -> float:
try:
value = float(text)
Expand Down Expand Up @@ -153,6 +165,12 @@ def _add_job_options(parser: argparse.ArgumentParser) -> None:
help="when outbid for a while: move to another machine, or hold this one (default: hold with --limit)",
)
parser.add_argument("--max-runtime", type=int, help="minutes (default 480)")
parser.add_argument(
"--max-spend",
type=_spend,
metavar="DOLLARS",
help="stop it once it has cost this much, machine and data together (default: no cap but the budget)",
)
parser.add_argument(
"--data-in", type=int, help="GB the job moves into its machine (image, datasets); Vast bills it per GB"
)
Expand Down Expand Up @@ -234,6 +252,12 @@ def _parser() -> argparse.ArgumentParser:
metavar="N",
help=f"how long the smoke test runs the command (default {DEFAULT_SMOKE_MINUTES}, at most 30)",
)
submit.add_argument(
"--follow",
action="store_true",
help="then print its output until it's finished, and download its outputs here if it succeeded "
"(exit code like krater wait)",
)
submit_workspace = submit.add_mutually_exclusive_group()
submit_workspace.add_argument(
"--workspace",
Expand Down Expand Up @@ -310,6 +334,8 @@ def _parser() -> argparse.ArgumentParser:
wait.add_argument("job", help=job_help)
wait.add_argument("--timeout", type=_positive, help="give up after this many minutes (exit code 124)")
wait.add_argument("--interval", type=_positive, default=5, help="seconds between checks (default 5)")
compare = commands.add_parser("compare", help="jobs' outputs/metrics.json side by side")
compare.add_argument("jobs", nargs="+", metavar="job", help=job_help)
download = commands.add_parser("download", help="download a job's outputs, checkpoints and logs")
download.add_argument("job", help=job_help)
download.add_argument("folder", nargs="?", help="the folder to put them in (default: here)")
Expand Down Expand Up @@ -684,10 +710,19 @@ def say_live() -> None:
"started. Try `krater run` to test it on this computer.",
status=1,
)
if body.get("max_spend_cents") and "max_spend_cents" not in job:
# A Krater from before spend caps ignores the field: the job would run uncapped, so take it back at once.
krater.post(f"/jobs/{job['id']}/cancel")
raise KraterError(
"This Krater doesn't take --max-spend yet, so it queued the job without a spend cap; it was cancelled "
"before it started. Leave out --max-spend, or set a shorter --max-runtime to limit what it can cost.",
status=1,
)
if body.get("checkpoint_upload_minutes") and "checkpoint_upload_minutes" not in job:
# A Krater from before per-job intervals ignores the field: the job runs, uploading on the default timer.
say("This Krater doesn't take --checkpoint-upload yet: the job uploads its checkpoints every 10 minutes.")
queued.append((job, variables))
args.queued_jobs = [job for job, _ in queued]

if len(queued) == 1:
job = queued[0][0]
Expand Down Expand Up @@ -777,6 +812,8 @@ def _job_body(args: argparse.Namespace, prices: dict) -> dict:
body["data_out_gb"] = args.data_out
if getattr(args, "checkpoint_upload", None) is not None:
body["checkpoint_upload_minutes"] = args.checkpoint_upload
if getattr(args, "max_spend", None) is not None:
body["max_spend_cents"] = args.max_spend
if args.input:
body["inputs"] = args.input
return body
Expand Down Expand Up @@ -840,9 +877,17 @@ def _project_env_names(http: httpx.Client, project: str, say: Callable[[str], No
try:
krater = Krater(http)
data = krater.get(f"/projects/{resolve.resolve_project(krater, project)}/secrets")
except (auth.AuthError, ResolveError, KraterError, httpx.HTTPError) as exc:
say(f"Couldn't read the project's secret names ({exc}); running without them.")
return []
except auth.AuthError:
why = "you're not signed in (krater login)"
except httpx.HTTPError as exc:
why = _unreachable(exc)
except (ResolveError, KraterError) as exc:
why = str(exc)
else:
return [s["name"] for s in data["secrets"]]
say(f"Running without the project's secrets: couldn't read their names, as {why.rstrip('.')}.")
say("A job that needs one reads it from your environment here: set it before krater run.")
return []
return [s["name"] for s in data["secrets"]]


Expand Down Expand Up @@ -1122,6 +1167,9 @@ def render_projects() -> None:
except jobfile.JobFileError as exc:
say(str(exc))
return 2
if args.follow and (args.box or args.estimate or args.json or len(runs) > 1):
say("--follow is for one job: not a box, a sweep, --estimate or --json.")
return 2
if smoke is not None and len(runs) > 1:
# A smoke test checks that the job starts and runs; one of the sweep's jobs shows that for all of them.
say(f"A smoke test runs only the first of the sweep's {len(runs)} jobs ({runs[0][0]['name']}).")
Expand All @@ -1131,7 +1179,10 @@ def render_projects() -> None:
if found is not None:
_upload_code(krater, http, project_id, found, say)
path = f"/projects/{project_id}/jobs"
return _queue(krater, path, runs, args, say=say, prompt=prompt, out=out, sleep=sleep)
queued = _queue(krater, path, runs, args, say=say, prompt=prompt, out=out, sleep=sleep)
if queued != 0 or not args.follow:
return queued
return _follow_then_download(krater, http, args.queued_jobs[0]["id"], args, say, sleep)
elif args.command == "rerun":
source = krater.get(f"/jobs/{resolve.resolve_job(krater, args.job)}")
if not source.get("spec"):
Expand Down Expand Up @@ -1232,6 +1283,9 @@ def render() -> None:
data = krater.get(f"/jobs/{job_id}/logs{query}")

def render_logs() -> None:
latest = data.get("latest_attempt")
if args.attempt is None and latest and data["attempt"] is not None and latest > data["attempt"]:
say(f"[attempt {latest} has no output yet; showing attempt {data['attempt']}]")
if not data["text"]:
say("No output yet.")
return
Expand Down Expand Up @@ -1263,6 +1317,8 @@ def render_logs() -> None:
return follow.exit_code_for(job)
elif args.command == "download":
return _download(krater, http, resolve.resolve_job(krater, args.job), args, say)
elif args.command == "compare":
return _compare(krater, http, args.jobs, say, out)
elif args.command in ("cancel", "end", "stop", "resume", "move"):
job_id = resolve.resolve_job(krater, args.job)
try:
Expand Down Expand Up @@ -1784,6 +1840,81 @@ def _download(krater: Krater, http: httpx.Client, job_id: str, args: argparse.Na
return 1 if failed else 0


def _follow_then_download(
krater: Krater, http: httpx.Client, job_id: str, args: argparse.Namespace, say, sleep: Callable[[float], None]
) -> int:
"""`krater submit --follow`: the job's output until it's finished, then its outputs here if it succeeded."""
say(f"Following {short(job_id)}; Ctrl+C stops following (the job keeps running).")
try:
job = follow.follow_logs(krater, job_id, tail_bytes=65536, interval=5, say=say, sleep=sleep)
except KeyboardInterrupt:
say(f"Stopped following. krater logs -f {short(job_id)} picks it up again.")
return INTERRUPTED
if job["state"] == "succeeded":
_download(
krater, http, job_id, argparse.Namespace(only="outputs", folder=None, to=None, skip_existing=False), say
)
return follow.exit_code_for(job)


def _compare(krater: Krater, http: httpx.Client, refs: list[str], say, out) -> int:
"""`krater compare`: each job's outputs/metrics.json (a JSON object), one column per job."""
columns: list[tuple[str, dict]] = []
for ref in refs:
job_id = resolve.resolve_job(krater, ref)
job = krater.get(f"/jobs/{job_id}")
label = f"{job['name']} ({short(job_id)})"
found = next((f for f in _file_list(krater, job_id, "outputs") if f["name"] == "metrics.json"), None)
metrics: dict = {}
if found is None:
say(f"{label}: no outputs/metrics.json.")
else:
try:
response = http.get(found["url"])
response.raise_for_status()
data = response.json()
except (httpx.HTTPError, ValueError) as exc:
say(f"{label}: couldn't read its metrics.json ({exc}).")
else:
if isinstance(data, dict):
metrics = _flat(data)
else:
say(f"{label}: its metrics.json isn't a JSON object.")
columns.append((label, metrics))
keys = sorted({key for _, metrics in columns for key in metrics})

def render() -> None:
if not keys:
say("No metrics to compare. A job writes them to $KRATER_OUTPUT_DIR/metrics.json, as a JSON object.")
return
rows = [["", *(label for label, _ in columns)]]
rows += [[key, *(_cell(metrics.get(key)) for _, metrics in columns)] for key in keys]
_table(rows, say)

out({"jobs": [{"job": label, "metrics": metrics} for label, metrics in columns]}, render)
return 0 if keys else 1


def _flat(data: dict, prefix: str = "") -> dict:
"""Nested objects as dotted keys (`eval.loss`); lists are shown as they are."""
flat: dict = {}
for key, value in data.items():
if isinstance(value, dict):
flat.update(_flat(value, f"{prefix}{key}."))
else:
flat[f"{prefix}{key}"] = value
return flat


def _cell(value: object) -> str:
if value is None:
return "-"
if isinstance(value, float):
return f"{value:.6g}"
text = value if isinstance(value, str) else json.dumps(value)
return follow.clean(text)[:40]


def _unreachable(exc: httpx.HTTPError) -> str:
try:
host = exc.request.url.host
Expand Down
10 changes: 10 additions & 0 deletions cli/tests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
from __future__ import annotations

import pytest


@pytest.fixture(autouse=True)
def config_dir(tmp_path, monkeypatch):
"""Every test signs in to a folder of its own, never the real one (`%APPDATA%/krater`, `~/.config/krater`)."""
monkeypatch.setenv("KRATER_CONFIG_DIR", str(tmp_path / "krater"))
return tmp_path / "krater"
Loading
Loading