From 0fb6a1ae40123dde89ddc4a5b6b4527b6ec41285 Mon Sep 17 00:00:00 2001 From: Adambomb210 Date: Thu, 19 Nov 2026 17:15:46 -0500 Subject: [PATCH 1/2] Polish batch: spend cap, starter form, compare, --follow, grouped outage alerts - Per-job spend cap (max_spend_cents; form "Stop once it has cost", API, CLI --max-spend): the scheduler fails the job (or stops a box) once its accrued spend reaches it. - Web "Try the starter job": puts examples/starter/train.py in the project's inputs and fills in the form. - krater submit --follow (logs, then outputs downloaded on success, wait's exit code) and krater compare (outputs/metrics.json side by side). CLI 0.7.0. - krater logs says when the newest attempt has no output yet (logs API: latest_attempt); krater run explains plainly when it can't read secret names. - Job page: when a checkpoint last reached storage, and an overdue badge. - Old code// input folders no unfinished job uses are pruned after 7 days. - Ops alerts that all say this server's DNS/network is down are posted as one. - /healthz reports the deployed commit (KRATER_REVISION from CI); the Deploy workflow skips its deploy job instead of reporting success for a commit it didn't deploy. - Pause wording no longer assumes outbid outright. - CLI tests: a shared conftest keeps every test's sign-in out of the real config folder. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 3 + .github/workflows/deploy.yml | 43 +++--- Dockerfile | 4 + ...d9a1f6_add_compute_jobs_max_spend_cents.py | 23 +++ cli/pyproject.toml | 2 +- cli/src/krater_cli/__init__.py | 2 +- cli/src/krater_cli/jobfile.py | 1 + cli/src/krater_cli/main.py | 131 +++++++++++++++- cli/tests/conftest.py | 10 ++ cli/tests/test_cli.py | 6 - cli/tests/test_polish_cli.py | 135 ++++++++++++++++ cli/uv.lock | 2 +- docs/compute/cli.md | 14 ++ docs/compute/scheduler.md | 2 + docs/dev/production.md | 3 +- docs/guides/compute-jobs.md | 21 ++- docs/guides/compute-runbook.md | 3 + krater/compute/scheduler.py | 25 ++- krater/compute/storage_cleanup.py | 55 +++++++ krater/models/compute_job.py | 3 + krater/net_errors.py | 16 +- krater/services/compute_files.py | 7 +- krater/services/compute_inputs.py | 18 +++ krater/services/compute_jobs.py | 9 ++ krater/services/ops_alerts.py | 51 +++++-- krater/storage/client.py | 4 + krater/storage/fake.py | 21 ++- krater/storage/live.py | 8 +- krater/storage/types.py | 2 + krater/web/routers/api.py | 16 +- krater/web/routers/compute.py | 64 ++++++++ krater/web/routers/pages.py | 6 +- krater/web/templates/compute/job.html | 1 + krater/web/templates/compute/new.html | 19 +++ krater/worker/app.py | 3 + tests/compute/test_runner_api.py | 23 +++ tests/compute/test_scheduler.py | 17 ++- tests/services/test_ops_alerts.py | 17 +++ tests/web/test_pages.py | 5 + tests/web/test_polish_batch_web.py | 144 ++++++++++++++++++ 40 files changed, 878 insertions(+), 61 deletions(-) create mode 100644 alembic/versions/e2b7c4d9a1f6_add_compute_jobs_max_spend_cents.py create mode 100644 cli/tests/conftest.py create mode 100644 cli/tests/test_polish_cli.py create mode 100644 tests/web/test_polish_batch_web.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f07ad0..d386e2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,9 @@ jobs: platforms: linux/amd64 push: true tags: ghcr.io/${{ github.repository }}:${{ github.sha }} + # /healthz reports it, so `curl /healthz` says which commit is live. + build-args: | + KRATER_REVISION=${{ github.sha }} # krater-build on weft checks this label against the commit it asked for. labels: | org.patchworklabs.krater.rev=${{ github.sha }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a268462..1a9b89e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -40,24 +40,19 @@ env: URL: https://krater-staging.patchworklabs.org jobs: - deploy: + # Its own job, so a run for a commit that is no longer the tip of main shows its deploy job as *skipped*, + # not as a green "success" that deployed nothing. + pick: # A CI run on a push to main that passed, or a manual run from main. if: >- (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push') runs-on: ubuntu-latest - # At job level: a run whose job the `if` skips does not take the group, so it cannot replace a - # pending deploy. - concurrency: - group: deploy-krater-staging - cancel-in-progress: false - # The environment is also the OIDC subject the Tailscale federated identity trusts. - environment: - name: staging - url: https://krater-staging.patchworklabs.org permissions: contents: read - id-token: write + outputs: + deploy: ${{ steps.sha.outputs.deploy }} + sha: ${{ steps.sha.outputs.sha }} steps: - name: Pick the commit id: sha @@ -69,15 +64,31 @@ jobs: # replaces a newer one. tip=$(gh api "repos/${{ github.repository }}/commits/main" --jq .sha) if [ "$SHA" != "$tip" ]; then - echo "::notice::$SHA is no longer the tip of main ($tip). Skipping." + echo "::notice::$SHA is no longer the tip of main ($tip). Skipping; the tip's own run deploys it." echo "deploy=false" >> "$GITHUB_OUTPUT" else echo "deploy=true" >> "$GITHUB_OUTPUT" fi echo "sha=$SHA" >> "$GITHUB_OUTPUT" + deploy: + needs: pick + if: needs.pick.outputs.deploy == 'true' + runs-on: ubuntu-latest + # At job level: a run whose job the `if` skips does not take the group, so it cannot replace a + # pending deploy. + concurrency: + group: deploy-krater-staging + cancel-in-progress: false + # The environment is also the OIDC subject the Tailscale federated identity trusts. + environment: + name: staging + url: https://krater-staging.patchworklabs.org + permissions: + contents: read + id-token: write + steps: - name: Join the tailnet - if: steps.sha.outputs.deploy == 'true' uses: tailscale/github-action@064b10558093f38325e838ddea35486bdc6b1d5c # v4.4.0 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} @@ -85,11 +96,10 @@ jobs: tags: tag:ci-krater version: latest - - name: Deploy ${{ steps.sha.outputs.sha }} - if: steps.sha.outputs.deploy == 'true' + - name: Deploy ${{ needs.pick.outputs.sha }} env: KEY: ${{ secrets.KRATER_DEPLOY_SSH_KEY }} - SHA: ${{ steps.sha.outputs.sha }} + SHA: ${{ needs.pick.outputs.sha }} # The first build of the Python dependencies takes several minutes on weft. The build may take # 45 minutes and the start 10. timeout-minutes: 60 @@ -114,7 +124,6 @@ jobs: ssh_weft "$SHA" - name: Check the site - if: steps.sha.outputs.deploy == 'true' run: | for _ in $(seq 1 12); do if curl -fsS --max-time 10 "$URL/healthz" && curl -fsS --max-time 10 "$URL/healthz/worker"; then diff --git a/Dockerfile b/Dockerfile index e8cbf5d..e614b42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,6 +42,10 @@ RUN --mount=type=cache,target=/home/krater/.cache/uv,uid=999,gid=999 \ uv sync --frozen --no-dev COPY --chown=krater:krater --from=runner /out/krater-runner-linux-amd64 /app/runner/dist/krater-runner-linux-amd64 +# The commit this image was built from, for /healthz (CI passes it; a local build leaves it empty). +ARG KRATER_REVISION="" +ENV KRATER_REVISION=$KRATER_REVISION + EXPOSE 8000 # The portal is the default; docker-compose.yml overrides `command` for `migrate` and `worker`. diff --git a/alembic/versions/e2b7c4d9a1f6_add_compute_jobs_max_spend_cents.py b/alembic/versions/e2b7c4d9a1f6_add_compute_jobs_max_spend_cents.py new file mode 100644 index 0000000..ba77689 --- /dev/null +++ b/alembic/versions/e2b7c4d9a1f6_add_compute_jobs_max_spend_cents.py @@ -0,0 +1,23 @@ +"""add compute_jobs.max_spend_cents: a job's own spend cap (`--max-spend`) + +Revision ID: e2b7c4d9a1f6 +Revises: c5d1e8a3f7b4 +Create Date: 2026-11-19 22:30:00 +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "e2b7c4d9a1f6" +down_revision = "c5d1e8a3f7b4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("compute_jobs", sa.Column("max_spend_cents", sa.Integer(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("compute_jobs", "max_spend_cents") diff --git a/cli/pyproject.toml b/cli/pyproject.toml index 0a321eb..573376c 100644 --- a/cli/pyproject.toml +++ b/cli/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "krater-cli" -version = "0.6.0" +version = "0.7.0" description = "The krater command: run jobs and boxes on Krater Compute (Project Ganymede, Patchwork Labs)" requires-python = ">=3.10" dependencies = ["httpx>=0.27", "tomli>=2; python_version < \"3.11\""] diff --git a/cli/src/krater_cli/__init__.py b/cli/src/krater_cli/__init__.py index 08d61be..247f2b2 100644 --- a/cli/src/krater_cli/__init__.py +++ b/cli/src/krater_cli/__init__.py @@ -1,3 +1,3 @@ """The `krater` command: Krater Compute from a terminal (docs/compute/cli.md).""" -__version__ = "0.6.0" +__version__ = "0.7.0" diff --git a/cli/src/krater_cli/jobfile.py b/cli/src/krater_cli/jobfile.py index 311b5c8..35fe859 100644 --- a/cli/src/krater_cli/jobfile.py +++ b/cli/src/krater_cli/jobfile.py @@ -36,6 +36,7 @@ "json", "estimate", "confirm_transfer_cost", + "follow", # `krater run`'s own: about this computer, not the job. "local", "input_dir", diff --git a/cli/src/krater_cli/main.py b/cli/src/krater_cli/main.py index d2823b0..33d885b 100644 --- a/cli/src/krater_cli/main.py +++ b/cli/src/krater_cli/main.py @@ -98,6 +98,18 @@ def _price(text: str) -> str: return cleaned +def _spend(text: str) -> int: + """A spend cap in dollars, `$25` included, as whole cents.""" + cleaned = text.strip().lstrip("$").strip() + try: + value = Decimal(cleaned) + except InvalidOperation: + value = Decimal("nan") + if not value.is_finite() or value <= 0: + raise argparse.ArgumentTypeError(f"{text!r} isn't an amount in dollars, such as 25") + return max(1, int((value * 100).to_integral_value())) + + def _positive(text: str) -> float: try: value = float(text) @@ -153,6 +165,12 @@ def _add_job_options(parser: argparse.ArgumentParser) -> None: help="when outbid for a while: move to another machine, or hold this one (default: hold with --limit)", ) parser.add_argument("--max-runtime", type=int, help="minutes (default 480)") + parser.add_argument( + "--max-spend", + type=_spend, + metavar="DOLLARS", + help="stop it once it has cost this much, machine and data together (default: no cap but the budget)", + ) parser.add_argument( "--data-in", type=int, help="GB the job moves into its machine (image, datasets); Vast bills it per GB" ) @@ -234,6 +252,12 @@ def _parser() -> argparse.ArgumentParser: metavar="N", help=f"how long the smoke test runs the command (default {DEFAULT_SMOKE_MINUTES}, at most 30)", ) + submit.add_argument( + "--follow", + action="store_true", + help="then print its output until it's finished, and download its outputs here if it succeeded " + "(exit code like krater wait)", + ) submit_workspace = submit.add_mutually_exclusive_group() submit_workspace.add_argument( "--workspace", @@ -310,6 +334,8 @@ def _parser() -> argparse.ArgumentParser: wait.add_argument("job", help=job_help) wait.add_argument("--timeout", type=_positive, help="give up after this many minutes (exit code 124)") wait.add_argument("--interval", type=_positive, default=5, help="seconds between checks (default 5)") + compare = commands.add_parser("compare", help="jobs' outputs/metrics.json side by side") + compare.add_argument("jobs", nargs="+", metavar="job", help=job_help) download = commands.add_parser("download", help="download a job's outputs, checkpoints and logs") download.add_argument("job", help=job_help) download.add_argument("folder", nargs="?", help="the folder to put them in (default: here)") @@ -688,6 +714,7 @@ def say_live() -> None: # A Krater from before per-job intervals ignores the field: the job runs, uploading on the default timer. say("This Krater doesn't take --checkpoint-upload yet: the job uploads its checkpoints every 10 minutes.") queued.append((job, variables)) + args.queued_jobs = [job for job, _ in queued] if len(queued) == 1: job = queued[0][0] @@ -777,6 +804,8 @@ def _job_body(args: argparse.Namespace, prices: dict) -> dict: body["data_out_gb"] = args.data_out if getattr(args, "checkpoint_upload", None) is not None: body["checkpoint_upload_minutes"] = args.checkpoint_upload + if getattr(args, "max_spend", None) is not None: + body["max_spend_cents"] = args.max_spend if args.input: body["inputs"] = args.input return body @@ -840,9 +869,17 @@ def _project_env_names(http: httpx.Client, project: str, say: Callable[[str], No try: krater = Krater(http) data = krater.get(f"/projects/{resolve.resolve_project(krater, project)}/secrets") - except (auth.AuthError, ResolveError, KraterError, httpx.HTTPError) as exc: - say(f"Couldn't read the project's secret names ({exc}); running without them.") - return [] + except auth.AuthError: + why = "you're not signed in (krater login)" + except httpx.HTTPError as exc: + why = _unreachable(exc) + except (ResolveError, KraterError) as exc: + why = str(exc) + else: + return [s["name"] for s in data["secrets"]] + say(f"Running without the project's secrets: couldn't read their names, as {why.rstrip('.')}.") + say("A job that needs one reads it from your environment here: set it before krater run.") + return [] return [s["name"] for s in data["secrets"]] @@ -1122,6 +1159,9 @@ def render_projects() -> None: except jobfile.JobFileError as exc: say(str(exc)) return 2 + if args.follow and (args.box or args.estimate or args.json or len(runs) > 1): + say("--follow is for one job: not a box, a sweep, --estimate or --json.") + return 2 if smoke is not None and len(runs) > 1: # A smoke test checks that the job starts and runs; one of the sweep's jobs shows that for all of them. say(f"A smoke test runs only the first of the sweep's {len(runs)} jobs ({runs[0][0]['name']}).") @@ -1131,7 +1171,10 @@ def render_projects() -> None: if found is not None: _upload_code(krater, http, project_id, found, say) path = f"/projects/{project_id}/jobs" - return _queue(krater, path, runs, args, say=say, prompt=prompt, out=out, sleep=sleep) + queued = _queue(krater, path, runs, args, say=say, prompt=prompt, out=out, sleep=sleep) + if queued != 0 or not args.follow: + return queued + return _follow_then_download(krater, http, args.queued_jobs[0]["id"], args, say, sleep) elif args.command == "rerun": source = krater.get(f"/jobs/{resolve.resolve_job(krater, args.job)}") if not source.get("spec"): @@ -1232,6 +1275,9 @@ def render() -> None: data = krater.get(f"/jobs/{job_id}/logs{query}") def render_logs() -> None: + latest = data.get("latest_attempt") + if args.attempt is None and latest and data["attempt"] is not None and latest > data["attempt"]: + say(f"[attempt {latest} has no output yet; showing attempt {data['attempt']}]") if not data["text"]: say("No output yet.") return @@ -1263,6 +1309,8 @@ def render_logs() -> None: return follow.exit_code_for(job) elif args.command == "download": return _download(krater, http, resolve.resolve_job(krater, args.job), args, say) + elif args.command == "compare": + return _compare(krater, http, args.jobs, say, out) elif args.command in ("cancel", "end", "stop", "resume", "move"): job_id = resolve.resolve_job(krater, args.job) try: @@ -1784,6 +1832,81 @@ def _download(krater: Krater, http: httpx.Client, job_id: str, args: argparse.Na return 1 if failed else 0 +def _follow_then_download( + krater: Krater, http: httpx.Client, job_id: str, args: argparse.Namespace, say, sleep: Callable[[float], None] +) -> int: + """`krater submit --follow`: the job's output until it's finished, then its outputs here if it succeeded.""" + say(f"Following {short(job_id)}; Ctrl+C stops following (the job keeps running).") + try: + job = follow.follow_logs(krater, job_id, tail_bytes=65536, interval=5, say=say, sleep=sleep) + except KeyboardInterrupt: + say(f"Stopped following. krater logs -f {short(job_id)} picks it up again.") + return INTERRUPTED + if job["state"] == "succeeded": + _download( + krater, http, job_id, argparse.Namespace(only="outputs", folder=None, to=None, skip_existing=False), say + ) + return follow.exit_code_for(job) + + +def _compare(krater: Krater, http: httpx.Client, refs: list[str], say, out) -> int: + """`krater compare`: each job's outputs/metrics.json (a JSON object), one column per job.""" + columns: list[tuple[str, dict]] = [] + for ref in refs: + job_id = resolve.resolve_job(krater, ref) + job = krater.get(f"/jobs/{job_id}") + label = f"{job['name']} ({short(job_id)})" + found = next((f for f in _file_list(krater, job_id, "outputs") if f["name"] == "metrics.json"), None) + metrics: dict = {} + if found is None: + say(f"{label}: no outputs/metrics.json.") + else: + try: + response = http.get(found["url"]) + response.raise_for_status() + data = response.json() + except (httpx.HTTPError, ValueError) as exc: + say(f"{label}: couldn't read its metrics.json ({exc}).") + else: + if isinstance(data, dict): + metrics = _flat(data) + else: + say(f"{label}: its metrics.json isn't a JSON object.") + columns.append((label, metrics)) + keys = sorted({key for _, metrics in columns for key in metrics}) + + def render() -> None: + if not keys: + say("No metrics to compare. A job writes them to $KRATER_OUTPUT_DIR/metrics.json, as a JSON object.") + return + rows = [["", *(label for label, _ in columns)]] + rows += [[key, *(_cell(metrics.get(key)) for _, metrics in columns)] for key in keys] + _table(rows, say) + + out({"jobs": [{"job": label, "metrics": metrics} for label, metrics in columns]}, render) + return 0 if keys else 1 + + +def _flat(data: dict, prefix: str = "") -> dict: + """Nested objects as dotted keys (`eval.loss`); lists are shown as they are.""" + flat: dict = {} + for key, value in data.items(): + if isinstance(value, dict): + flat.update(_flat(value, f"{prefix}{key}.")) + else: + flat[f"{prefix}{key}"] = value + return flat + + +def _cell(value: object) -> str: + if value is None: + return "-" + if isinstance(value, float): + return f"{value:.6g}" + text = value if isinstance(value, str) else json.dumps(value) + return follow.clean(text)[:40] + + def _unreachable(exc: httpx.HTTPError) -> str: try: host = exc.request.url.host diff --git a/cli/tests/conftest.py b/cli/tests/conftest.py new file mode 100644 index 0000000..16052c4 --- /dev/null +++ b/cli/tests/conftest.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +import pytest + + +@pytest.fixture(autouse=True) +def config_dir(tmp_path, monkeypatch): + """Every test signs in to a folder of its own, never the real one (`%APPDATA%/krater`, `~/.config/krater`).""" + monkeypatch.setenv("KRATER_CONFIG_DIR", str(tmp_path / "krater")) + return tmp_path / "krater" diff --git a/cli/tests/test_cli.py b/cli/tests/test_cli.py index 9c1229e..bfc8dec 100644 --- a/cli/tests/test_cli.py +++ b/cli/tests/test_cli.py @@ -305,12 +305,6 @@ def log_tail(self, path: str) -> httpx.Response: ) -@pytest.fixture(autouse=True) -def config_dir(tmp_path, monkeypatch): - monkeypatch.setenv("KRATER_CONFIG_DIR", str(tmp_path / "krater")) - return tmp_path / "krater" - - @pytest.fixture def fake() -> Fake: return Fake() diff --git a/cli/tests/test_polish_cli.py b/cli/tests/test_polish_cli.py new file mode 100644 index 0000000..d880e04 --- /dev/null +++ b/cli/tests/test_polish_cli.py @@ -0,0 +1,135 @@ +"""`--max-spend`, `submit --follow`, `compare`, `logs` naming an attempt with no output yet, and `run` without Krater.""" + +from __future__ import annotations + +import json + +import httpx +import pytest +from test_cli import JOB, JOB_C, PROJECT, SUBMIT, Docker, Fake, _login, _run_local + +from krater_cli.main import run + + +class PolishFake(Fake): + def __init__(self) -> None: + super().__init__() + self.metrics: dict[str, object] = {} + self.latest_attempt: int | None = None + + def handler(self, request: httpx.Request) -> httpx.Response: + url = str(request.url) + if url.startswith("https://store.test/metrics/"): + return httpx.Response(200, content=json.dumps(self.metrics[url.rsplit("/", 1)[1]]).encode()) + return super().handler(request) + + def api(self, request: httpx.Request, path: str) -> httpx.Response: + for job_id in self.metrics: + if path == f"/jobs/{job_id}/files": + file = {"area": "outputs", "name": "metrics.json", "size_bytes": 9} + return httpx.Response(200, json={"files": [{**file, "url": f"https://store.test/metrics/{job_id}"}]}) + response = super().api(request, path) + if path.startswith(f"/jobs/{JOB}/logs") and self.latest_attempt is not None: + return httpx.Response(200, json={**response.json(), "latest_attempt": self.latest_attempt}) + return response + + +@pytest.fixture +def fake() -> PolishFake: + return PolishFake() + + +@pytest.fixture +def http(fake: PolishFake) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(fake.handler)) + + +def test_max_spend_is_sent_in_cents(http, fake) -> None: + _login(http, []) + assert run([*SUBMIT, "--limit", "0.1", "--max-spend", "$12.5"], http=http, say=lambda _: None) == 0 + assert fake.jobs[-1]["body"]["max_spend_cents"] == 1250 + + +def test_max_spend_must_be_an_amount(http, fake, capsys) -> None: + with pytest.raises(SystemExit): + run([*SUBMIT, "--limit", "0.1", "--max-spend", "0"], http=http, say=lambda _: None) + assert "isn't an amount in dollars" in capsys.readouterr().err + + +def test_submit_follow_prints_the_output_then_downloads_the_outputs(http, fake, tmp_path, monkeypatch) -> None: + _login(http, []) + monkeypatch.chdir(tmp_path) + fake.details[JOB].update(kind="job", state="running") + fake.log_files = {1: b"epoch 1\n"} + + def finish(_seconds: float) -> None: + fake.log_files[1] += b"done\n" + fake.details[JOB].update(state="succeeded", exit_code=0) + + said: list[str] = [] + code = run([*SUBMIT, "--limit", "0.1", "--follow"], http=http, say=said.append, sleep=finish) + + assert code == 0 + assert "epoch 1" in said and "done" in said + assert (tmp_path / "outputs" / "eval" / "score.txt").read_bytes() == b"0.93\n" + assert not (tmp_path / "logs").exists() # only the outputs + + +def test_submit_follow_exits_with_a_failed_jobs_code_and_downloads_nothing(http, fake, tmp_path, monkeypatch) -> None: + _login(http, []) + monkeypatch.chdir(tmp_path) + fake.details[JOB].update(kind="job", state="failed", exit_code=3) + fake.log_files = {1: b"Traceback\n"} + + assert run([*SUBMIT, "--limit", "0.1", "--follow"], http=http, say=lambda _: None) == 3 + assert not (tmp_path / "outputs").exists() + + +def test_submit_follow_is_for_one_job(http, fake) -> None: + _login(http, []) + said: list[str] = [] + assert ( + run([*SUBMIT, "--limit", "0.1", "--follow", "--count", "2", "--command", "go"], http=http, say=said.append) == 2 + ) + assert said == ["--follow is for one job: not a box, a sweep, --estimate or --json."] + assert fake.jobs == [] + + +def test_compare_shows_each_jobs_metrics_side_by_side(http, fake) -> None: + _login(http, []) + fake.details[JOB_C] = {**fake.details[JOB], "id": JOB_C, "name": "sing"} + fake.metrics = {JOB: {"loss": 0.1234567, "eval": {"acc": 0.9}}, JOB_C: {"loss": 0.2, "steps": 100}} + said: list[str] = [] + + assert run(["compare", JOB, JOB_C], http=http, say=said.append) == 0 + + header, *rows = said + assert "nb (22222222)" in header and "sing (9f9f9f9f)" in header + assert [row.split()[0] for row in rows] == ["eval.acc", "loss", "steps"] + assert rows[1].split()[1:] == ["0.123457", "0.2"] and rows[2].split()[1:] == ["-", "100"] + + +def test_compare_says_when_there_is_nothing_to_compare(http, fake) -> None: + _login(http, []) + said: list[str] = [] + assert run(["compare", JOB], http=http, say=said.append) == 1 + assert said[0].startswith("nb (22222222): no outputs/metrics.json") + + +def test_logs_says_when_the_newest_attempt_has_no_output_yet(http, fake) -> None: + _login(http, []) + fake.log_files = {1: b"epoch 9\n"} + fake.latest_attempt = 2 + said: list[str] = [] + + assert run(["logs", JOB], http=http, say=said.append) == 0 + + assert said == ["[attempt 2 has no output yet; showing attempt 1]", "epoch 9"] + + +def test_run_without_signing_in_says_so_plainly(http, fake, tmp_path) -> None: + said: list[str] = [] + _run_local([PROJECT, "--image", "x", "--command", "go", "--dir", str(tmp_path)], http, Docker(), said) + assert ( + "Running without the project's secrets: couldn't read their names, as you're not signed in (krater login)." + ) in said diff --git a/cli/uv.lock b/cli/uv.lock index dc41ef7..f445c67 100644 --- a/cli/uv.lock +++ b/cli/uv.lock @@ -103,7 +103,7 @@ wheels = [ [[package]] name = "krater-cli" -version = "0.6.0" +version = "0.7.0" source = { editable = "." } dependencies = [ { name = "httpx" }, diff --git a/docs/compute/cli.md b/docs/compute/cli.md index 85894cb..78b10f2 100644 --- a/docs/compute/cli.md +++ b/docs/compute/cli.md @@ -183,6 +183,20 @@ Krater's proxy, and any request that couldn't connect at all, is tried again aft another change that gets a 502 is never sent again, because Krater may already have done it: check with `krater jobs` before submitting again. +`krater submit ... --follow` is `submit`, then `logs -f`, then `download --only outputs` into the current folder +if the job succeeded, with `wait`'s exit code. It's for one job: not a box, a sweep, `--estimate` or `--json`. +Ctrl-C stops following and leaves the job running. `krater logs` (without `--attempt`) reads `latest_attempt` from +the logs API and says when that attempt has no output yet and an earlier one's is shown. + +`krater compare ...` reads each job's `outputs/metrics.json` (a JSON object; nested objects become dotted keys +such as `eval.acc`) and prints one column per job. `--json` prints them as read. Exit code 1 when none had metrics. + +`--max-spend DOLLARS` (job files: `max-spend = 25`) sends `max_spend_cents`: the scheduler fails the job (or stops +the box) once its spend reaches it (`docs/compute/scheduler.md`). + +`krater run` reads the project's secret names when it can. Signed out or offline, it says why in one line and runs +without them. + Output is printed with terminal control characters and escape sequences removed (a job's output can come from any teammate who can launch on the project); `krater logs --raw` prints it as it is. diff --git a/docs/compute/scheduler.md b/docs/compute/scheduler.md index 3bc3475..3254451 100644 --- a/docs/compute/scheduler.md +++ b/docs/compute/scheduler.md @@ -29,6 +29,8 @@ workspace sync, and step 5 starts and searches for nothing: queued jobs say why. holds when outbid (`on_outbid = hold`, which limit orders use) instead keeps the paused machine, with its disk, and moves only once a search finds another machine free at or under its target (paced like any search); - running past the job's max runtime → job failed, machine destroyed; + - spend (`spend_cents`, from the accrual) at or over the job's `max_spend_cents` → job failed ("Stopped at its + spend cap"), machine destroyed; a box is stopped instead, like the max runtime; - its image refused by the registry (the provider's status message is Docker's `Error response from daemon` with `denied`, `unauthorized`, `manifest unknown`, `pull access denied`, `name unknown` or `invalid reference format`: `PULL_REFUSALS`) before its container ever ran → job failed at once, quoting the message, and the machine diff --git a/docs/dev/production.md b/docs/dev/production.md index 7b28be6..6e3e6df 100644 --- a/docs/dev/production.md +++ b/docs/dev/production.md @@ -228,7 +228,8 @@ long it lasted. Each failure names its root cause (`krater/net_errors.py`) and i It can't report its own death, so set up one check outside Krater (an uptime service, or the lab's monitoring) on `https:///healthz/worker`, every 5 minutes, alerting on anything but `200`. It answers `503` once the worker's heartbeat is 5 minutes old. Point a second check at `/healthz`, which fails when the portal or its database -is down. +is down. `/healthz` also says which commit is live (`"revision"`, the image's `KRATER_REVISION` build +argument, set by CI), so `curl -s https:///healthz` confirms a deploy. ## 4. Backups diff --git a/docs/guides/compute-jobs.md b/docs/guides/compute-jobs.md index 0673fc9..62415d3 100644 --- a/docs/guides/compute-jobs.md +++ b/docs/guides/compute-jobs.md @@ -100,8 +100,9 @@ krater submit vision --name train --code . --command "python train.py" --gpu "RT try runs the same command. `krater init` writes a starter job set up this way. - It's for jobs; a box keeps its own `/workspace`. On the web form, use an image. -Old versions stay in the project's inputs and count toward its storage quota: `krater inputs rm code/` -clears them all once no queued job needs them. +Old versions stay in the project's inputs for a while and count toward its storage quota. Krater deletes a +`code//` folder once it's a week old and no unfinished job uses it; `krater inputs rm code/` clears +them all sooner. Build your own image when installing your packages takes long enough to matter on every start, or needs more than `pip`. @@ -273,6 +274,10 @@ Two ways to find that out cheaply: On your project's page, open **Compute**, then **New job or box**. The page shows current prices beside the form. +**Try the starter job** (a link at the top of the form) fills the form in with the [starter job](first-job.md): a +small PyTorch training run of about 20 minutes on a cheap GPU, as a smoke test first. Krater puts its script in the +project's inputs as `starter/train.py`. Queue it as it is to see the whole flow, or change it. + ### What to run | Field | What to enter | @@ -306,6 +311,7 @@ See [Choosing a price](#choosing-a-price). In short, choose an order type and fi | **Wait at most (minutes)** | How long to wait for a machine at your price. 720 (12 hours) by default, up to 7 days | | **Then** | What happens after that wait: keep waiting, accept offers up to your max price, rent on demand up to your max price, or cancel the job | | **Stop after (minutes of running)** | The max runtime. The job fails and its machine is destroyed when it has run this long. 480 (8 hours) by default, up to 7 days | +| **Stop once it has cost ($)** | Optional. A spend cap for this job: once its machine and data together have cost this much, the job fails with "Stopped at its spend cap" and its machine is destroyed (a box is stopped instead, keeping its disk). Krater checks it every minute or so, so it can go a few cents over. `--max-spend 25` from a terminal | Prices are the **full hourly cost**: the GPU plus your disk. No price can be above your project's hourly cap, shown at the top of the form ($5 an hour unless an admin set another). @@ -462,6 +468,9 @@ Open the job from your project's **Compute** page. It shows: from before 12 November 2026 show only the total; - **Output**: the end of what the command printed, updated every minute, with a link to the whole log; - **Files**: checkpoints, outputs and logs, to download; +- **Last checkpoint stored** (a running job): when a checkpoint last reached Krater's storage, which is how much + work a move to another machine would lose. It's marked **overdue** when nothing has arrived for twice the upload + interval: check that your code writes to `$KRATER_CHECKPOINT_DIR`; - **What happened**: every step, with times: queued, rented, started, outbid, bid raised, moved, finished; - **Machine**: the GPU and price it was rented at, and for a box, how to connect, when `/workspace` was last saved, and, once it's stopped, what it costs while stopped and when it will be destroyed; @@ -504,7 +513,10 @@ want its last outputs. For a box, see [Ending a box](#ending-a-box). inputs, workspaces and every job's files, with how much of the storage quota they use. - **From a terminal:** `krater download results/` puts them under `results/outputs/`, `results/checkpoints/` and `results/logs/`. `--only outputs` fetches just one kind, and `--skip-existing` leaves alone files you already have - (handy for logs, which only grow). + (handy for logs, which only grow). `krater submit ... --follow` queues the job, prints its output as it runs, and + downloads its outputs into the current folder if it succeeds. +- **Comparing runs:** if your job writes a JSON object to `$KRATER_OUTPUT_DIR/metrics.json` (such as + `{"loss": 0.41, "eval": {"acc": 0.88}}`), `krater compare ...` shows each job's metrics side by side. Who can see them: everyone on the team, and admins. Reviewers see the job but not its files or output. @@ -747,7 +759,8 @@ queues nothing. | `krater rerun ` | Queues it again with the same settings (`--name` for a new name) | | `krater init [FOLDER]` | Writes the [starter job](first-job.md) into a folder (`--project` fills in `job.toml`) | | `krater inputs list\|upload\|rm ` | The project's [inputs](#inputs) | -| `krater logs ` | The end of the output. `-f` follows it; `--attempt 1` shows an earlier attempt | +| `krater logs ` | The end of the output. `-f` follows it; `--attempt 1` shows an earlier attempt. When the newest attempt hasn't printed anything yet, it says so and shows the one before | +| `krater compare ...` | Each job's `outputs/metrics.json`, side by side | | `krater wait ` | Blocks until the job is finished. Exits 0 if it succeeded, the command's own code if it failed with one, 124 on `--timeout` | | `krater cancel ` | Cancels it; the machine is destroyed (a box: at once, without a last save) | | `krater end ` | Saves a running box's `/workspace`, then destroys its machine; the workspace stays | diff --git a/docs/guides/compute-runbook.md b/docs/guides/compute-runbook.md index 823f263..3e86972 100644 --- a/docs/guides/compute-runbook.md +++ b/docs/guides/compute-runbook.md @@ -137,6 +137,9 @@ Each alert says what failed and why: the error's root cause and what it usually | "The host refused the connection" | Vast's address answered but refused: likely Vast | | "took too long to answer" (`ReadTimeout`), "Vast answered 5xx" | Vast is slow or erroring | +When two or more alerts open at once and each says the DNS lookup failed or the network is unreachable, Krater posts +them as one, "this server's network or DNS looks down", listing them. + When Weave's syncs or Quilt and Stash deliveries fail at the same minutes as the scheduler, the server's own network is down, not Vast. A blip of a few minutes is harmless: the scheduler doesn't fail a machine for heartbeats it missed while it couldn't reach out (`WATCH_GAP`). diff --git a/krater/compute/scheduler.py b/krater/compute/scheduler.py index b16320e..a8fd3a3 100644 --- a/krater/compute/scheduler.py +++ b/krater/compute/scheduler.py @@ -234,6 +234,10 @@ def _stopped(row: ComputeInstance) -> dict: return {"disk_microusd": row.disk_microusd, "stopped_at": row.stopped_at.isoformat() if row.stopped_at else None} +def _cents(cents: int) -> str: + return f"${cents / 100:,.2f}" + + def _dollars(microusd: int) -> str: return f"${microusd_to_usd(microusd).quantize(Decimal('0.0001'))}/h" @@ -756,6 +760,22 @@ def _watch_one(self, row: ComputeInstance, seen: Instance | None) -> None: self.request_stop(row, job, f"max runtime of {job.max_runtime_minutes} minutes reached") self._finish_stopping(row, job, policy) return + over_spend = job.max_spend_cents is not None and (job.spend_cents or 0) >= job.max_spend_cents + if over_spend and job.kind is ComputeJobKind.BOX: + if row.stop_requested_at is None: + self.request_stop(row, job, f"spend cap of {_cents(job.max_spend_cents)} reached") + self._finish_stopping(row, job, policy) + return + if over_spend: + _finish( + job, + ComputeJobState.FAILED, + f"Stopped at its spend cap: it had cost {_cents(job.spend_cents or 0)} of its " + f"{_cents(job.max_spend_cents)}.", + now=self.now, + ) + self._destroy(row, "spend cap reached") + return if over_runtime: _finish( job, @@ -798,7 +818,7 @@ def _watch_one(self, row: ComputeInstance, seen: Instance | None) -> None: self._hold_for_credit(row, job) why, cause = "the account ran out of credit", "credit_out" elif row.interruptible: - job.state_reason = "Outbid: the machine is paused." + ( + job.state_reason = "Paused: Vast stopped the machine, almost always because it was outbid." + ( f" Krater raises the bid up to {_dollars(job.max_microusd)} and keeps the machine, with the job's " "state on its disk, until it runs again." if job.on_outbid is ComputeOutbid.HOLD @@ -1594,7 +1614,8 @@ def _move_if_free_elsewhere(self, row: ComputeInstance, job: ComputeJob, policy: paused_minutes = int((self.now - row.paused_since).total_seconds() // 60) if choice is None: job.state_reason = ( - f"Outbid: paused for {paused_minutes} minutes. Keeping this machine, with the job's state on its disk, " + f"Paused for {paused_minutes} minutes (almost always outbid). Keeping this machine, with the job's " + f"state on its disk, " f"until its bid of up to {_dollars(job.max_microusd)} wins again or another machine is free at or " f"under {_dollars(limit)}." ) diff --git a/krater/compute/storage_cleanup.py b/krater/compute/storage_cleanup.py index f8f531b..e68ea55 100644 --- a/krater/compute/storage_cleanup.py +++ b/krater/compute/storage_cleanup.py @@ -1,4 +1,5 @@ """Abort multipart uploads that machines started and never finished: their parts take space on Krater's storage. +Also delete old `--code` folders no unfinished job uses (`prune_code_inputs`). SeaweedFS (4.47) doesn't say when an upload started, so age alone can't decide. An upload is abandoned once its job has ended (with a grace for a final upload still finishing), or its key belongs to no job Krater knows. Where the @@ -12,8 +13,10 @@ import uuid from datetime import datetime, timedelta +import sqlalchemy as sa from sqlalchemy.orm import Session +from krater.compute import runner_api from krater.models import ComputeJob from krater.models.compute_job import FINISHED_JOB_STATES from krater.storage import ObjectStore @@ -45,3 +48,55 @@ def abort_abandoned_uploads(session: Session, store: ObjectStore, *, now: dateti store.abort_multipart_upload(upload.key, upload_id=upload.upload_id) aborted += 1 return aborted + + +#: `krater submit --code` uploads each version of a folder as the input `code//` (`cli/src/krater_cli/code.py`). +CODE_PREFIX = "code/" +#: A code folder no unfinished job uses is deleted once its newest file is this old: long enough to `krater rerun` +#: a recent job, short enough that a project's storage doesn't fill with every edit. +CODE_KEEP = timedelta(days=7) +#: Only projects with a job this recent can have code folders worth looking at. +CODE_LOOKBACK = timedelta(days=60) + + +def prune_code_inputs(session: Session, store: ObjectStore, *, now: datetime) -> int: + """Delete the `code//` input folders that no unfinished job asks for and whose newest file is older than + `CODE_KEEP`. Returns how many folders went. A folder whose files carry no time is kept.""" + # Every unfinished job counts, however old; recent jobs only say which projects to look at. + jobs = session.scalars( + sa.select(ComputeJob).where( + sa.or_(ComputeJob.queued_at >= now - CODE_LOOKBACK, ComputeJob.state.not_in(FINISHED_JOB_STATES)) + ) + ).all() + projects: dict[uuid.UUID, set[str]] = {} + for job in jobs: + code = {name for name in job.input_names or () if name.startswith(CODE_PREFIX)} + if not code: + continue + used = projects.setdefault(job.project_id, set()) + if job.state not in FINISHED_JOB_STATES: + used.update(name.rstrip("/") + "/" for name in code) + gone = 0 + for project_id, used in projects.items(): + prefix = runner_api.inputs_prefix(project_id) + folders: dict[str, list] = {} + for item in store.list_prefix(prefix + CODE_PREFIX): + rest = item.key[len(prefix) :] + parts = rest.split("/") + if len(parts) < 3: + continue + folders.setdefault(f"{parts[0]}/{parts[1]}/", []).append(item) + for folder, items in folders.items(): + times = [i.last_modified for i in items] + if folder in used or any(t is None for t in times) or now - max(times) < CODE_KEEP: + continue + for item in items: + store.delete(item.key) + gone += 1 + logger.info( + "krater.compute: deleted code folder %s of project %s (unused, %s old)", + folder, + project_id, + now - max(times), + ) + return gone diff --git a/krater/models/compute_job.py b/krater/models/compute_job.py index 80eff69..a691220 100644 --- a/krater/models/compute_job.py +++ b/krater/models/compute_job.py @@ -84,6 +84,9 @@ class ComputeJob(UUIDPrimaryKeyMixin, TimestampMixin, Base): # How often the runner uploads changed checkpoint files to storage, in minutes (`--checkpoint-upload`). Null: the # default (`krater.compute.runner_api.CHECKPOINT_SYNC_SECONDS`). Jobs only: a box saves /workspace on its own timer. checkpoint_upload_minutes: Mapped[int | None] = mapped_column(sa.Integer, nullable=True) + # Stop the job (a box: stop it) once it has cost this much, in cents; None for no cap of its own. Checked + # while it runs, against `spend_cents`. + max_spend_cents: Mapped[int | None] = mapped_column(sa.Integer, nullable=True) state: Mapped[ComputeJobState] = mapped_column(pg_enum(ComputeJobState, name="compute_job_state"), nullable=False) # Why the job is where it is, for the member: "waiting for an RTX 4090 bid at or under $0.20/h", ... diff --git a/krater/net_errors.py b/krater/net_errors.py index 3dffadc..a6030bf 100644 --- a/krater/net_errors.py +++ b/krater/net_errors.py @@ -20,6 +20,15 @@ #: How far down the `__cause__` / `__context__` chain to look. _MAX_DEPTH = 8 +#: The `hint`s that mean this server's own network or DNS is down rather than one service: when several alerts open +#: with one of these at once, `krater.services.ops_alerts.deliver` posts them as one. +DNS_FAILED = ( + "The DNS lookup failed: this server's DNS servers didn't answer, or its own internet connection is " + "down. The service itself may be fine." +) +NO_ROUTE = "This server has no route to the internet (network unreachable): its own connection is down." +OWN_NETWORK_DOWN = (DNS_FAILED, NO_ROUTE) + def _chain(exc: BaseException) -> list[BaseException]: seen: list[BaseException] = [] @@ -41,10 +50,7 @@ def hint(exc: BaseException) -> str | None: if isinstance(link, socket.gaierror): if link.errno is not None and link.errno == _eai("EAI_NONAME"): return "DNS says the host name doesn't exist (or this server's DNS servers can't find it)." - return ( - "The DNS lookup failed: this server's DNS servers didn't answer, or its own internet connection is " - "down. The service itself may be fine." - ) + return DNS_FAILED if isinstance(link, ConnectionRefusedError): return "The host refused the connection: it is up, but nothing answered on that port." if isinstance(link, ConnectionResetError | ConnectionAbortedError | BrokenPipeError): @@ -52,7 +58,7 @@ def hint(exc: BaseException) -> str | None: if isinstance(link, ssl.SSLError): return "The TLS handshake failed: a certificate problem, or something in between answering instead." if isinstance(link, OSError) and link.errno in (errno.ENETUNREACH, errno.ENETDOWN): - return "This server has no route to the internet (network unreachable): its own connection is down." + return NO_ROUTE if isinstance(link, OSError) and link.errno == errno.EHOSTUNREACH: return "No route to the host: the network between this server and the service is down." for link in chain: diff --git a/krater/services/compute_files.py b/krater/services/compute_files.py index 4042cfd..75c19f1 100644 --- a/krater/services/compute_files.py +++ b/krater/services/compute_files.py @@ -13,6 +13,7 @@ import re from dataclasses import dataclass +from datetime import datetime from krater.models import ComputeJob, Project from krater.services import teams @@ -34,6 +35,7 @@ class JobFile: area: str name: str size_bytes: int + modified: datetime | None = None @dataclass(frozen=True) @@ -64,7 +66,10 @@ def list_files(store: ObjectStore, actor: Actor, *, job: ComputeJob) -> list[Job files = [] for area in AREAS: prefix = _prefix(job, area) - files += [JobFile(area, item.key[len(prefix) :], item.size_bytes) for item in store.list_prefix(prefix)] + files += [ + JobFile(area, item.key[len(prefix) :], item.size_bytes, item.last_modified) + for item in store.list_prefix(prefix) + ] return sorted(files, key=lambda f: (AREAS.index(f.area), f.name)) diff --git a/krater/services/compute_inputs.py b/krater/services/compute_inputs.py index b98c9d8..cdacf2b 100644 --- a/krater/services/compute_inputs.py +++ b/krater/services/compute_inputs.py @@ -12,6 +12,7 @@ import uuid from dataclasses import dataclass from datetime import datetime +from pathlib import Path from typing import Any from sqlalchemy.orm import Session @@ -93,6 +94,23 @@ def complete_upload( runner_api.finish_upload(store, key, upload_id, etags) +#: The starter job (`examples/starter`, also what `krater init` writes): its script, put in a project's inputs +#: under `STARTER_INPUT` for the web form's "Try the starter job", since the web form takes an image, not a folder. +STARTER_SCRIPT = Path(__file__).resolve().parents[2] / "examples" / "starter" / "train.py" +STARTER_INPUT = "starter/" + + +def ensure_starter(store: ObjectStore, actor: Actor, *, project: Project) -> str: + """Put the starter's train.py in the project's inputs (again if it changed). Returns the input folder name.""" + _team_only(actor, project) + content = STARTER_SCRIPT.read_bytes() + key = runner_api.inputs_prefix(project.id) + STARTER_INPUT + "train.py" + stored = store.head(key) + if stored is None or stored.size_bytes != len(content): + store.put_bytes(key, content, content_type="text/x-python") + return STARTER_INPUT + + def delete_input(store: ObjectStore, actor: Actor, *, project: Project, name: str) -> int: """Delete input `name`, or everything under it when it ends in /. Returns how many files went. Jobs already queued that asked for it fail to start.""" diff --git a/krater/services/compute_jobs.py b/krater/services/compute_jobs.py index a3e654d..25b6b1b 100644 --- a/krater/services/compute_jobs.py +++ b/krater/services/compute_jobs.py @@ -63,6 +63,8 @@ AUDIT_TRANSFER_WARNING = "compute_transfer_warning" #: A batch job ended (succeeded, failed, or was stopped by Krater or an admin): a notice in its project's channel. AUDIT_JOB_ENDED = "compute_job_ended" +#: The highest spend cap a job can set: far above any real job, low enough to catch a typo's extra zeros. +MAX_SPEND_CAP_CENTS = 100_000_00 #: A batch job's GPU sat nearly unused for the policy's idle warning time. AUDIT_JOB_IDLE = "compute_job_idle" #: How a member's own cancel reads (`cancel_job`): no channel notice for it, they know. @@ -181,6 +183,8 @@ class JobSpec: # Upload changed checkpoint files every this many minutes; None for the default. More often loses less work when # the job has to move to another machine, and costs more data out (Vast bills it per GB). checkpoint_upload_minutes: int | None = None + # Stop once the job has cost this many cents (a box: stop it); None for no cap of its own. + max_spend_cents: int | None = None # A box's /workspace (`krater.services.workspaces`): continue this workspace of the project, or (None) start a new # one, named `workspace_name` (the box's name if blank). `end_holder`: end the stopped box that still holds the # workspace, so this one can continue it. @@ -327,6 +331,10 @@ def _validate(spec: JobSpec, *, cap_microusd: int) -> None: f"Choose a checkpoint upload every 1 to {MAX_CHECKPOINT_UPLOAD_MINUTES} minutes, or leave it blank for " f"every {DEFAULT_CHECKPOINT_UPLOAD_MINUTES}." ) + if spec.max_spend_cents is not None and not 1 <= spec.max_spend_cents <= MAX_SPEND_CAP_CENTS: + errors["max_spend"] = ( + f"Choose a spend cap of $0.01 to ${MAX_SPEND_CAP_CENTS // 100:,}, or leave it blank for none." + ) _check_provider(spec, errors) if errors: raise ValidationFailed(errors) @@ -407,6 +415,7 @@ def submit_job( inputs_bytes=inputs_bytes, smoke_minutes=spec.smoke_minutes, checkpoint_upload_minutes=spec.checkpoint_upload_minutes, + max_spend_cents=spec.max_spend_cents, state=ComputeJobState.QUEUED, # While compute is stopped, say so from the start, as the scheduler's next pass would. state_reason=f"Waiting: {halted}." diff --git a/krater/services/ops_alerts.py b/krater/services/ops_alerts.py index db6d1b4..5163fe8 100644 --- a/krater/services/ops_alerts.py +++ b/krater/services/ops_alerts.py @@ -30,6 +30,7 @@ import sqlalchemy as sa from sqlalchemy.orm import Session +from krater import net_errors from krater.config import Settings from krater.models import ( AuditEvent, @@ -474,25 +475,55 @@ def deliver(session: Session, slack_client: SlackClient | None, channel_id: str .with_for_update() ).all() posted = [] + # Several alerts that all say this server's own network is down are one problem: post them as one. + outage = [ + row + for row in rows + if row.opened_at is not None and any(hint in row.message for hint in net_errors.OWN_NETWORK_DOWN) + ] + if len(outage) >= 2: + if _post(slack_client, channel_id, _outage_text(outage), cleared=False, key="network_outage"): + for row in outage: + row.notified_at = now + posted.append(row.key) + rows = [row for row in rows if row not in outage] for row in rows: cleared = row.opened_at is None text = _cleared_text(row) if cleared else _open_text(row, again=row.notified_at is not None) - if slack_client is None or not channel_id: - (logger.info if cleared else logger.error)("krater ops alert: %s", text) - else: - try: - slack_client.post_message( - channel_id, blocks=[{"type": "section", "text": {"type": "mrkdwn", "text": text}}], text=text - ) - except SlackError: - logger.exception("krater: could not post the ops alert %s; trying again next run", row.key) - continue + if not _post(slack_client, channel_id, text, cleared=cleared, key=row.key): + continue row.notified_at = None if cleared else now posted.append(row.key) session.flush() return posted +def _post(slack_client: SlackClient | None, channel_id: str | None, text: str, *, cleared: bool, key: str) -> bool: + """Post one alert's text (to the log without a channel). False when Slack refused it: try again next run.""" + if slack_client is None or not channel_id: + (logger.info if cleared else logger.error)("krater ops alert: %s", text) + return True + try: + slack_client.post_message( + channel_id, blocks=[{"type": "section", "text": {"type": "mrkdwn", "text": text}}], text=text + ) + except SlackError: + logger.exception("krater: could not post the ops alert %s; trying again next run", key) + return False + return True + + +def _outage_text(rows: list[OpsAlert]) -> str: + again = all(row.notified_at is not None for row in rows) + lines = [ + ":rotating_light: *Krater: this server's network or DNS looks down*" + (" (still)" if again else ""), + f"{len(rows)} alerts failed the same way, so the services are probably fine:", + ] + lines += [f"• {_title(row.key)}" for row in rows] + lines.append(f"> {rows[0].message}") + return "\n".join(lines) + + def beat(session: Session, *, now: datetime) -> None: """Record the worker's heartbeat. The caller commits.""" row = session.get(WorkerHeartbeat, 1) diff --git a/krater/storage/client.py b/krater/storage/client.py index fdd6e75..c564fda 100644 --- a/krater/storage/client.py +++ b/krater/storage/client.py @@ -50,6 +50,10 @@ def read_tail(self, key: str, n: int) -> bytes | None: end of a job's log without fetching all of it.""" ... + def put_bytes(self, key: str, content: bytes, *, content_type: str = "application/octet-stream") -> None: + """Write a small object from Krater itself (the starter job's script). Raises `StorageError`.""" + ... + def delete(self, key: str) -> None: """Delete `key`. Safe to call on a key that's already gone (S3's own `DeleteObject` semantics).""" ... diff --git a/krater/storage/fake.py b/krater/storage/fake.py index b0b1207..b2ab392 100644 --- a/krater/storage/fake.py +++ b/krater/storage/fake.py @@ -10,6 +10,7 @@ from __future__ import annotations import itertools +from datetime import datetime from krater.storage.types import ListedObject, MultipartUpload, ObjectMeta, PresignedPost @@ -18,6 +19,8 @@ class FakeObjectStore: def __init__(self) -> None: self._objects: dict[str, ObjectMeta] = {} self._content: dict[str, bytes] = {} + #: When a test says an object was written (`put(modified=...)`), for `list_prefix`. + self.modified: dict[str, datetime] = {} #: upload id -> key, for multipart uploads started and not yet completed. self.multipart_uploads: dict[str, str] = {} self._upload_ids = itertools.count(1) @@ -52,6 +55,10 @@ def read_tail(self, key: str, n: int) -> bytes | None: return None return content[-n:] if n else b"" + def put_bytes(self, key: str, content: bytes, *, content_type: str = "application/octet-stream") -> None: + self._objects[key] = ObjectMeta(size_bytes=len(content), content_type=content_type) + self._content[key] = content + def delete(self, key: str) -> None: self._objects.pop(key, None) self._content.pop(key, None) @@ -61,7 +68,7 @@ def presign_put(self, key: str, *, content_length: int, expires: int = 300) -> s def list_prefix(self, prefix: str) -> list[ListedObject]: return [ - ListedObject(key=k, size_bytes=m.size_bytes) + ListedObject(key=k, size_bytes=m.size_bytes, last_modified=self.modified.get(k)) for k, m in sorted(self._objects.items()) if k.startswith(prefix) ] @@ -97,13 +104,23 @@ def complete_multipart_upload(self, key: str, *, upload_id: str, parts: list[tup # -- Test helpers ---------------------------------------------------------------------------------- - def put(self, key: str, *, content_type: str, size_bytes: int, content: bytes = b"") -> None: + def put( + self, + key: str, + *, + content_type: str, + size_bytes: int, + content: bytes = b"", + modified: datetime | None = None, + ) -> None: """Simulate a browser having uploaded `key` through the presigned POST above. `content` is the object's actual bytes (or just enough of a prefix for a signature check) -- tests that need `confirm_screenshot` to accept the upload must pass real magic bytes here; the default (empty) deliberately fails any signature check, the same way an empty/garbage upload would for real.""" self._objects[key] = ObjectMeta(size_bytes=size_bytes, content_type=content_type) self._content[key] = content + if modified is not None: + self.modified[key] = modified __all__ = ["FakeObjectStore"] diff --git a/krater/storage/live.py b/krater/storage/live.py index bbae5a3..cec125e 100644 --- a/krater/storage/live.py +++ b/krater/storage/live.py @@ -174,6 +174,12 @@ def read_tail(self, key: str, n: int) -> bytes | None: raise StorageUnavailableError(f"could not read {key!r}: {exc}") from exc return response["Body"].read() + def put_bytes(self, key: str, content: bytes, *, content_type: str = "application/octet-stream") -> None: + try: + self._internal.put_object(Bucket=self._bucket, Key=key, Body=content, ContentType=content_type) + except ClientError as exc: + raise StorageUnavailableError(f"could not write {key!r}: {exc}") from exc + def delete(self, key: str) -> None: try: self._internal.delete_object(Bucket=self._bucket, Key=key) @@ -191,7 +197,7 @@ def list_prefix(self, prefix: str) -> list[ListedObject]: try: pages = self._internal.get_paginator("list_objects_v2").paginate(Bucket=self._bucket, Prefix=prefix) return [ - ListedObject(key=item["Key"], size_bytes=int(item["Size"])) + ListedObject(key=item["Key"], size_bytes=int(item["Size"]), last_modified=item.get("LastModified")) for page in pages for item in page.get("Contents", []) ] diff --git a/krater/storage/types.py b/krater/storage/types.py index 8265b5b..f8fe9b3 100644 --- a/krater/storage/types.py +++ b/krater/storage/types.py @@ -34,6 +34,8 @@ class ListedObject: key: str size_bytes: int + # When it was last written (S3's LastModified), if the store says. + last_modified: datetime | None = None @dataclass(frozen=True) diff --git a/krater/web/routers/api.py b/krater/web/routers/api.py index 24abf97..ba3982f 100644 --- a/krater/web/routers/api.py +++ b/krater/web/routers/api.py @@ -177,6 +177,7 @@ def _job_json(job: ComputeJob, *, detail: bool = False, session: Session | None # Null: the default (`checkpoint_upload_minutes_effective` says what that is). "checkpoint_upload_minutes": job.checkpoint_upload_minutes, "checkpoint_upload_minutes_effective": runner_api.checkpoint_sync_seconds(job) // 60, + "max_spend_cents": job.max_spend_cents, "workspace": _workspace_ref(job.workspace) if job.workspace is not None else None, "end_requested": job.end_requested_at is not None, "queued_at": _iso(job.queued_at), @@ -269,6 +270,8 @@ def submit_body(job: ComputeJob) -> dict[str, Any]: body["smoke_minutes"] = job.smoke_minutes if job.checkpoint_upload_minutes: body["checkpoint_upload_minutes"] = job.checkpoint_upload_minutes + if job.max_spend_cents: + body["max_spend_cents"] = job.max_spend_cents if job.workspace_id is not None: # Queued again (`krater rerun`), a box continues its workspace. body["workspace"] = str(job.workspace_id) @@ -335,6 +338,7 @@ def _spec(body: dict, *, session: Session | None = None, project: Project | None inputs=tuple(compute_inputs.check_names(inputs)), smoke_minutes=_int(body, "smoke_minutes", None), checkpoint_upload_minutes=_int(body, "checkpoint_upload_minutes", None), + max_spend_cents=_int(body, "max_spend_cents", None), **_workspace_spec(body, session, project), ) @@ -635,9 +639,17 @@ def job_logs(job_id: uuid.UUID, actor: Me, db_session: Db, attempt: int | None = return _error(403, exc.message) except StorageError: return _error(503, "Krater's storage didn't answer; try again in a minute.") + # The newest attempt, so a client can say when it has no output yet and an earlier one's is shown. + latest = job.attempts or None if tail is None: - return {"attempt": attempt, "text": "", "size_bytes": 0, "truncated": False} - return {"attempt": tail.attempt, "text": tail.text, "size_bytes": tail.size_bytes, "truncated": tail.truncated} + return {"attempt": attempt, "text": "", "size_bytes": 0, "truncated": False, "latest_attempt": latest} + return { + "attempt": tail.attempt, + "text": tail.text, + "size_bytes": tail.size_bytes, + "truncated": tail.truncated, + "latest_attempt": latest, + } @router.get("/jobs/{job_id}/files") diff --git a/krater/web/routers/compute.py b/krater/web/routers/compute.py index 3f583d6..4d02b17 100644 --- a/krater/web/routers/compute.py +++ b/krater/web/routers/compute.py @@ -107,6 +107,49 @@ def _job(session: Session, project: Project, job_id: uuid.UUID) -> ComputeJob: return job +def _starter_posted(folder: str) -> dict: + """The new-job form, filled in for the starter job (`examples/starter`, the same as `krater init`'s job.toml).""" + return { + "kind": "job", + "name": "starter", + "image": compute_jobs.DEFAULT_IMAGE, + "command": f'cd "$KRATER_INPUT_DIR/{folder.rstrip("/")}" && python train.py --minutes 20', + "gpu": ["RTX 3060", "RTX 4060 Ti", "RTX 3090"], + "order": "limit", + "limit": "0.15", + "max_runtime_minutes": "45", + "checkpoint_upload_minutes": "5", + "inputs": folder, + "smoke": "on", + } + + +def _checkpoint_freshness(files: list[compute_files.JobFile], job: ComputeJob, *, now: datetime) -> dict | None: + """For a running job's page: when a checkpoint last reached storage, and whether that's overdue (what a move + would lose). Overdue: running longer than twice the upload interval (and 5 minutes for the first upload) with + nothing stored, or the newest stored more than twice the interval ago. None for a box or a job not running.""" + if job.kind is not ComputeJobKind.JOB or job.state is not ComputeJobState.RUNNING: + return None + every = (job.checkpoint_upload_minutes or compute_jobs.DEFAULT_CHECKPOINT_UPLOAD_MINUTES) * 60 + times = [f.modified for f in files if f.area == "checkpoints" and f.modified is not None] + newest = max(times) if times else None + since = newest or job.started_at + overdue = since is not None and (now - since).total_seconds() > 2 * every + (0 if newest else 300) + minutes = int((now - newest).total_seconds() // 60) if newest else None + return {"newest": newest, "minutes_ago": minutes, "overdue": overdue} + + +def _spend_cap(raw: str) -> int: + """The form's "Stop once it has cost ($)": dollars, to cents (rounded down, so the cap never grows).""" + try: + value = Decimal(raw.strip().lstrip("$").replace(",", "")) + except InvalidOperation: + raise ValidationFailed({"max_spend": "Give the spend cap in dollars, such as 5 or 12.50."}) from None + if not value.is_finite(): + raise ValidationFailed({"max_spend": "Give the spend cap in dollars, such as 5 or 12.50."}) + return int(value * 100) + + def _rate(raw: str, field: str) -> int: try: value = Decimal(raw.strip().lstrip("$")) @@ -245,6 +288,7 @@ def _form_page( source: ComputeJob | None = None, quote: dict | None = None, live_table: dict | None = None, + starter: bool = False, ): settings = get_settings() policy = compute_settings.effective_policy(db_session, project) @@ -283,6 +327,7 @@ def _form_page( "quote": quote, "live_table": live_table, "source": source, + "starter": starter, "escalations": ESCALATION_LABELS, "errors": errors or {}, "posted": posted or {}, @@ -309,11 +354,25 @@ def new_job( weave_client: Annotated[WeaveClient, Depends(get_weave_client)], from_job: Annotated[uuid.UUID | None, Query(alias="from")] = None, workspace: uuid.UUID | None = None, + starter: bool = False, ): _require_compute() project = _project(db_session, actor, project_id) if not _is_team(actor, project): raise NotAllowed("Only the project's team can run jobs on its budget.") + if starter: + # "Try the starter job": its script goes in the project's inputs, and the form is filled in to run it. + folder = compute_inputs.ensure_starter(get_compute_store(), actor, project=project) + live_table = live_quote.start_table(db_session) + return _form_page( + request, + db_session, + project, + weave_client, + posted=_starter_posted(folder), + live_table=live_table, + starter=True, + ) source = None if from_job is None else _job(db_session, project, from_job) # Opening the form searches Vast again for the "Cheapest now" column, if it's stale (`live_prices.TABLE_FRESH`). live_table = live_quote.start_table(db_session) @@ -361,6 +420,8 @@ def _posted_from(job: ComputeJob) -> dict: posted["smoke"] = "on" if job.checkpoint_upload_minutes: posted["checkpoint_upload_minutes"] = str(job.checkpoint_upload_minutes) + if job.max_spend_cents: + posted["max_spend"] = f"{job.max_spend_cents / 100:.2f}" if job.workspace_id is not None: # "Run again" on a box continues its workspace. posted["workspace"] = str(job.workspace_id) @@ -392,6 +453,7 @@ async def submit_job( min_vram = posted.get("min_vram", "").strip() data_in, data_out = posted.get("data_in_gb", "").strip(), posted.get("data_out_gb", "").strip() upload_every = posted.get("checkpoint_upload_minutes", "").strip() + spend_cap = posted.get("max_spend", "").strip() inputs = [line.strip() for line in posted.get("inputs", "").splitlines() if line.strip()] if posted.get("order") == "limit": target = high = _rate(posted.get("limit", ""), "limit") @@ -423,6 +485,7 @@ async def submit_job( checkpoint_upload_minutes=_int(upload_every, "checkpoint_upload_minutes", "checkpoint upload interval") if upload_every and kind is ComputeJobKind.JOB else None, + max_spend_cents=_spend_cap(spend_cap) if spend_cap else None, **_workspace_fields(posted, kind), ) store = get_compute_store() if spec.inputs else None @@ -588,6 +651,7 @@ def job_page( "estimate": estimate, "costs": job_costs.breakdown(job), "checkpoint_upload_default": compute_jobs.DEFAULT_CHECKPOINT_UPLOAD_MINUTES, + "checkpoints": _checkpoint_freshness(files, job, now=datetime.now(UTC)), "can_cancel": (is_team or actor.is_admin) and (job.is_active or job.state is ComputeJobState.QUEUED), # A running box can end after a save ("End box"); "End now" (cancel) skips it. "can_end": (is_team or actor.is_admin) diff --git a/krater/web/routers/pages.py b/krater/web/routers/pages.py index 7ff25c5..c14db39 100644 --- a/krater/web/routers/pages.py +++ b/krater/web/routers/pages.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os from datetime import UTC, datetime from typing import Annotated @@ -67,8 +68,11 @@ def home( @router.get("/healthz") def healthz(session: Annotated[Session, Depends(get_session)]) -> dict[str, str]: + """200 once the database answers, with the commit this image was built from (`KRATER_REVISION`, set by CI's + image build), so `curl /healthz` says which commit is live. Open: a commit id is public in the repo.""" session.execute(text("SELECT 1")) - return {"status": "ok"} + revision = os.environ.get("KRATER_REVISION", "").strip() + return {"status": "ok", **({"revision": revision} if revision else {})} @router.get("/healthz/edge") diff --git a/krater/web/templates/compute/job.html b/krater/web/templates/compute/job.html index 92c0c82..629a860 100644 --- a/krater/web/templates/compute/job.html +++ b/krater/web/templates/compute/job.html @@ -37,6 +37,7 @@

{{ "Box" if job.kind.value == "box" else "Job" }}

{% if job.command %}
Command
{{ job.command }}
{% endif %} {% if job.smoke_minutes %}
Smoke test
the command is stopped after {{ job.smoke_minutes }} minutes of running, with KRATER_SMOKE=1 set; it passes if it hasn't failed by then. Run it again without the smoke test for the full run.
{% endif %} {% if job.kind.value == "job" %}
Checkpoint uploads
changed files in $KRATER_CHECKPOINT_DIR go up every {{ job.checkpoint_upload_minutes or checkpoint_upload_default }} minutes{% if not job.checkpoint_upload_minutes %} (the default){% endif %}, and when the command ends
{% endif %} + {% if checkpoints %}
Last checkpoint stored
{% if checkpoints.newest %}{{ checkpoints.newest.strftime("%H:%M UTC") }} ({{ checkpoints.minutes_ago }} min ago){% else %}none yet{% endif %}{% if checkpoints.overdue %} overdue
If the machine moved now, the work since then would be lost. Is your program saving into $KRATER_CHECKPOINT_DIR?{% endif %}
{% endif %} {% if job.input_names %}
Inputs
{% for name in job.input_names %}{{ name }}{% if not loop.last %}, {% endif %}{% endfor %} ({{ job.inputs_bytes | filesize }}), in $KRATER_INPUT_DIR
{% endif %}
GPU
diff --git a/krater/web/templates/compute/new.html b/krater/web/templates/compute/new.html index 39626b9..5961bb7 100644 --- a/krater/web/templates/compute/new.html +++ b/krater/web/templates/compute/new.html @@ -16,6 +16,19 @@

New job or box

+{% if starter %} +
+

The starter job: a small training run that saves checkpoints, resumes after an outbid pause, + and writes a model, its metrics and a sample of what it learned to write. Its script is now in this project's + inputs (starter/train.py). Smoke test first is ticked, so it runs for a few + minutes for about a cent; once that passes, choose Run again on its page and untick it for the + full 20 minutes. Your first job does the same from a terminal.

+
+{% elif not source and not posted %} +

New to Krater compute? Try the starter + job: everything filled in, a few cents.

+{% endif %} + {% if source %}

Filled in from {{ source.name }}. Change anything you like, then queue it.

@@ -217,6 +230,12 @@

New job or box

{% if errors.max_runtime_minutes %}

{{ errors.max_runtime_minutes }}

{% endif %}
+
+ + +

Everything it costs counts: GPU, disk and data. A job stops as failed; a box stops and keeps its disk.

+ {% if errors.max_spend %}

{{ errors.max_spend }}

{% endif %} +
diff --git a/krater/worker/app.py b/krater/worker/app.py index d477d0a..83d2b70 100644 --- a/krater/worker/app.py +++ b/krater/worker/app.py @@ -372,6 +372,9 @@ def compute_storage_cleanup(timestamp: int) -> None: aborted = compute_storage.abort_abandoned_uploads(session, store, now=now) if aborted: logger.info("krater compute_storage_cleanup: aborted %d unfinished uploads", aborted) + pruned = compute_storage.prune_code_inputs(session, store, now=now) + if pruned: + logger.info("krater compute_storage_cleanup: deleted %d unused code folders", pruned) compute_watch.record_storage(session, store, get_object_store(), now=now) session.commit() except StorageError as exc: diff --git a/tests/compute/test_runner_api.py b/tests/compute/test_runner_api.py index dd23147..a16b944 100644 --- a/tests/compute/test_runner_api.py +++ b/tests/compute/test_runner_api.py @@ -501,3 +501,26 @@ def test_a_job_without_inputs_gets_none(http, machine, store) -> None: store.put(f"projects/{job.project_id}/inputs/a.csv", content_type="", size_bytes=4) assert http.post("/runner/v1/start", headers=_auth(token)).json()["inputs"] is False assert http.post("/runner/v1/files/inputs/list", headers=_auth(token)).json() == {"files": []} + + +def test_old_code_folders_no_unfinished_job_uses_are_deleted(machine, store, db_session) -> None: + from krater.compute import storage_cleanup + + job, _, _ = machine + job.input_names = ["code/aaaa/"] + db_session.flush() + base = f"projects/{job.project_id}/inputs/" + old, recent = NOW - timedelta(days=10), NOW - timedelta(days=1) + for name, when in ( + ("code/aaaa/train.py", old), # the running job's + ("code/bbbb/train.py", old), # unused and old: goes + ("code/bbbb/lib/util.py", old), + ("code/cccc/train.py", recent), # unused, but recent + ("code/dddd/train.py", None), # no time: kept + ("data/old.csv", old), # not code + ): + store.put(base + name, content_type="", size_bytes=1, modified=when) + + assert storage_cleanup.prune_code_inputs(db_session, store, now=NOW) == 1 + left = sorted(item.key[len(base) :] for item in store.list_prefix(base)) + assert left == ["code/aaaa/train.py", "code/cccc/train.py", "code/dddd/train.py", "data/old.csv"] diff --git a/tests/compute/test_scheduler.py b/tests/compute/test_scheduler.py index a73428b..9fb45f3 100644 --- a/tests/compute/test_scheduler.py +++ b/tests/compute/test_scheduler.py @@ -1333,7 +1333,7 @@ def test_an_outbid_machine_is_still_a_pause(db_session, member, project, client, _pass(db_session, client, NOW + timedelta(minutes=7)) - assert job.state is ComputeJobState.PAUSED and "Outbid" in job.state_reason + assert job.state is ComputeJobState.PAUSED and "almost always because it was outbid" in job.state_reason assert fake.instances[row.provider_instance_id]["intended_status"] == "stopped" @@ -1462,7 +1462,7 @@ def test_an_outbid_interruptible_machine_is_still_outbid_with_credit(db_session, _pass(db_session, client, NOW + timedelta(minutes=7)) - assert "Outbid" in job.state_reason and row.outbid_count == 1 and row.start_requested_at is None + assert "outbid" in job.state_reason and row.outbid_count == 1 and row.start_requested_at is None def test_after_a_pause_the_runner_must_start_again(db_session, member, project, client, fake) -> None: # noqa: F811 @@ -1628,3 +1628,16 @@ def test_a_job_whose_machine_only_looked_stopped_carries_on(db_session, member, assert job.state is ComputeJobState.RUNNING and vast_id in fake.instances assert "runner_kept_running" in _events(db_session, job) + + +def test_a_job_that_reaches_its_spend_cap_is_stopped(db_session, member, project, client, fake) -> None: # noqa: F811 + job, row = _running_job(db_session, member, project, client, fake, max_spend_cents=3) + assert job.state is ComputeJobState.RUNNING + + _pass(db_session, client, NOW + timedelta(hours=1)) # the machine's accrued spend passes 3 cents + + assert job.state is ComputeJobState.FAILED + assert job.spend_cents >= 3 + assert "spend cap" in job.state_reason and "of its $0.03" in job.state_reason + assert row.state is ComputeInstanceState.DESTROYED + assert fake.instances == {} diff --git a/tests/services/test_ops_alerts.py b/tests/services/test_ops_alerts.py index e69534a..1829f99 100644 --- a/tests/services/test_ops_alerts.py +++ b/tests/services/test_ops_alerts.py @@ -363,3 +363,20 @@ def test_alastors_ping_alerts_only_where_it_is_expected(db_session: Session) -> db_session.execute(sa.delete(EdgePing)) assert _watch(db_session, slack, NOW + timedelta(hours=2), Settings(edge_ping_expected=True)) == ["edge_ping_stale"] assert "None has arrived yet" in _texts(slack)[-1] + + +def test_alerts_that_all_say_the_network_is_down_are_posted_as_one(db_session: Session) -> None: + from krater import net_errors + + _fresh_prices(db_session) + slack = FakeSlackClient() + message = f"ConnectError: [Errno -3] Temporary failure in name resolution. {net_errors.DNS_FAILED}" + for key in ("quilt", "stash"): + for n in range(ops_alerts.FAILURE_RULES[key].threshold): + ops_alerts.record_failure(db_session, key, message, now=NOW + timedelta(minutes=n)) + + assert sorted(_watch(db_session, slack, NOW + timedelta(minutes=6))) == ["quilt", "stash"] + (text,) = _texts(slack) + assert "network or DNS looks down" in text + assert "Sending events to Quilt" in text and "Archiving submission links" in text + assert _watch(db_session, slack, NOW + timedelta(minutes=11)) == [] diff --git a/tests/web/test_pages.py b/tests/web/test_pages.py index 4851405..5a33009 100644 --- a/tests/web/test_pages.py +++ b/tests/web/test_pages.py @@ -14,6 +14,11 @@ def test_healthz_runs_a_query_and_reports_ok(client: TestClient) -> None: assert response.json() == {"status": "ok"} +def test_healthz_says_which_commit_is_live(client: TestClient, monkeypatch) -> None: + monkeypatch.setenv("KRATER_REVISION", "e5caa36" + "0" * 33) + assert client.get("/healthz").json() == {"status": "ok", "revision": "e5caa36" + "0" * 33} + + def test_home_page_renders_signed_out(client: TestClient) -> None: response = client.get("/") diff --git a/tests/web/test_polish_batch_web.py b/tests/web/test_polish_batch_web.py new file mode 100644 index 0000000..91d4cb6 --- /dev/null +++ b/tests/web/test_polish_batch_web.py @@ -0,0 +1,144 @@ +"""The spend cap (form, API, rerun), the web form's starter job, a running job's checkpoint freshness, and the logs +API naming the newest attempt.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient + +from krater.config import get_settings +from krater.models import ComputeJob +from krater.models.compute_job import ComputeJobKind, ComputeJobState +from krater.services import compute_inputs +from krater.services.compute_files import JobFile +from krater.storage import FakeObjectStore +from krater.weave import StubWeaveClient +from krater.web.routers.compute import _checkpoint_freshness +from tests.conftest import MEMBER_SUB, REVIEWER_SUB, get_csrf_token +from tests.web.test_compute_pages import FORM + + +@pytest.fixture(autouse=True) +def compute_on(monkeypatch) -> None: + monkeypatch.setattr(get_settings(), "compute_mode", "vast") + monkeypatch.setattr(get_settings(), "weave_cli_client_id", "krater-cli-app") + + +@pytest.fixture +def store(monkeypatch) -> FakeObjectStore: + store = FakeObjectStore() + for router in ("compute", "api"): + monkeypatch.setattr(f"krater.web.routers.{router}.get_compute_store", lambda: store) + return store + + +@pytest.fixture +def project(login_as, approved_project): + member = login_as(MEMBER_SUB) + reviewer = login_as(REVIEWER_SUB) + return approved_project(member, reviewer) + + +def _as(sub: str) -> dict[str, str]: + return {"Authorization": f"Bearer {StubWeaveClient.cli_token_for(sub)}"} + + +BODY = { + "name": "train", + "image": "ghcr.io/alice/train:v3", + "command": "python train.py", + "gpu_names": ["RTX 3060"], + "limit_usd_per_hour": "0.09", + "max_runtime_minutes": 240, +} + + +def test_the_api_takes_a_spend_cap_and_a_rerun_keeps_it(client: TestClient, project) -> None: + queued = client.post( + f"/api/v1/projects/{project.id}/jobs", json={**BODY, "max_spend_cents": 2500}, headers=_as(MEMBER_SUB) + ) + assert queued.status_code == 201, queued.text + assert queued.json()["max_spend_cents"] == 2500 + detail = client.get(f"/api/v1/jobs/{queued.json()['id']}", headers=_as(MEMBER_SUB)).json() + assert detail["spec"]["max_spend_cents"] == 2500 + + +def test_the_api_refuses_a_spend_cap_of_nothing(client: TestClient, project) -> None: + response = client.post( + f"/api/v1/projects/{project.id}/jobs", json={**BODY, "max_spend_cents": 0}, headers=_as(MEMBER_SUB) + ) + assert response.status_code == 422 + + +def test_the_form_queues_a_job_with_a_spend_cap(client: TestClient, login_as, project, db_session) -> None: + login_as(MEMBER_SUB) + page = client.get(f"/projects/{project.id}/compute/new") + assert 'name="max_spend"' in page.text + response = client.post( + f"/projects/{project.id}/compute", + data={**FORM, "max_spend": "$12.50", "csrf_token": get_csrf_token(page.text)}, + follow_redirects=False, + ) + assert response.status_code == 303, response.text + assert db_session.query(ComputeJob).filter_by(project_id=project.id).one().max_spend_cents == 1250 + + +def test_the_form_says_what_is_wrong_with_a_spend_cap(client: TestClient, login_as, project) -> None: + login_as(MEMBER_SUB) + page = client.get(f"/projects/{project.id}/compute/new") + response = client.post( + f"/projects/{project.id}/compute", + data={**FORM, "max_spend": "lots", "csrf_token": get_csrf_token(page.text)}, + follow_redirects=False, + ) + assert response.status_code == 422 + assert "Give the spend cap in dollars" in response.text + + +def test_the_starter_puts_its_script_in_the_inputs_and_fills_in_the_form( + client: TestClient, login_as, project, store +) -> None: + login_as(MEMBER_SUB) + page = client.get(f"/projects/{project.id}/compute/new") + assert "?starter=1" in page.text + + page = client.get(f"/projects/{project.id}/compute/new?starter=1") + assert page.status_code == 200 + key = f"projects/{project.id}/inputs/starter/train.py" + assert store.read_prefix(key, 1 << 20) == compute_inputs.STARTER_SCRIPT.read_bytes() + assert 'value="starter"' in page.text and "python train.py --minutes 20" in page.text + + # Opening it again leaves the stored script alone. + client.get(f"/projects/{project.id}/compute/new?starter=1") + assert len(store.list_prefix(f"projects/{project.id}/inputs/")) == 1 + + +def test_only_the_team_gets_the_starter(client: TestClient, login_as, project, store) -> None: + login_as(REVIEWER_SUB) + assert client.get(f"/projects/{project.id}/compute/new?starter=1").status_code == 403 + assert store.list_prefix("projects/") == [] + + +NOW = datetime(2026, 11, 19, 12, 0, tzinfo=UTC) + + +def _running(**fields) -> SimpleNamespace: + defaults = {"checkpoint_upload_minutes": 5, "started_at": NOW - timedelta(minutes=30)} + return SimpleNamespace(kind=ComputeJobKind.JOB, state=ComputeJobState.RUNNING, **{**defaults, **fields}) + + +def _ckpt(minutes_ago: int) -> JobFile: + return JobFile("checkpoints", "model.pt", 10, NOW - timedelta(minutes=minutes_ago)) + + +def test_checkpoint_freshness() -> None: + fresh = _checkpoint_freshness([_ckpt(4), _ckpt(20)], _running(), now=NOW) + assert fresh == {"newest": NOW - timedelta(minutes=4), "minutes_ago": 4, "overdue": False} + assert _checkpoint_freshness([_ckpt(11)], _running(), now=NOW)["overdue"] is True + # None stored: overdue only past twice the interval plus 5 minutes for the first upload. + assert _checkpoint_freshness([], _running(started_at=NOW - timedelta(minutes=14)), now=NOW)["overdue"] is False + assert _checkpoint_freshness([], _running(), now=NOW)["overdue"] is True + assert _checkpoint_freshness([], SimpleNamespace(kind=ComputeJobKind.BOX, state=None), now=NOW) is None From 2ea2baebd77f5c73a50da64f07ac25a82fce39e4 Mon Sep 17 00:00:00 2001 From: Adambomb210 Date: Thu, 19 Nov 2026 17:19:15 -0500 Subject: [PATCH 2/2] CLI cancels a job a Krater without spend caps queued uncapped; logs API test sees latest_attempt Co-Authored-By: Claude Opus 5.5 --- cli/src/krater_cli/main.py | 8 ++++++++ cli/tests/test_cli.py | 3 +++ cli/tests/test_polish_cli.py | 10 ++++++++++ tests/web/test_api.py | 2 +- 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cli/src/krater_cli/main.py b/cli/src/krater_cli/main.py index 33d885b..ae64444 100644 --- a/cli/src/krater_cli/main.py +++ b/cli/src/krater_cli/main.py @@ -710,6 +710,14 @@ def say_live() -> None: "started. Try `krater run` to test it on this computer.", status=1, ) + if body.get("max_spend_cents") and "max_spend_cents" not in job: + # A Krater from before spend caps ignores the field: the job would run uncapped, so take it back at once. + krater.post(f"/jobs/{job['id']}/cancel") + raise KraterError( + "This Krater doesn't take --max-spend yet, so it queued the job without a spend cap; it was cancelled " + "before it started. Leave out --max-spend, or set a shorter --max-runtime to limit what it can cost.", + status=1, + ) if body.get("checkpoint_upload_minutes") and "checkpoint_upload_minutes" not in job: # A Krater from before per-job intervals ignores the field: the job runs, uploading on the default timer. say("This Krater doesn't take --checkpoint-upload yet: the job uploads its checkpoints every 10 minutes.") diff --git a/cli/tests/test_cli.py b/cli/tests/test_cli.py index bfc8dec..2a12455 100644 --- a/cli/tests/test_cli.py +++ b/cli/tests/test_cli.py @@ -58,6 +58,7 @@ def __init__(self) -> None: self.knows_smoke = True # An older Krater ignores `checkpoint_upload_minutes` and doesn't send it back. self.knows_checkpoint_upload = True + self.knows_max_spend = True self.jobs_422 = False # Krater asks before queuing a job whose data transfer could cost a lot (409 with `confirm`). self.transfer_confirm = False @@ -227,6 +228,8 @@ def api(self, request: httpx.Request, path: str) -> httpx.Response: job["smoke_minutes"] = body.get("smoke_minutes") if self.knows_checkpoint_upload: job["checkpoint_upload_minutes"] = body.get("checkpoint_upload_minutes") + if self.knows_max_spend: + job["max_spend_cents"] = body.get("max_spend_cents") self.jobs.append(job) return httpx.Response(201, json=job) acted = re.fullmatch(r"/jobs/([0-9a-f-]{36})/(cancel|end|stop|resume|move)", path) diff --git a/cli/tests/test_polish_cli.py b/cli/tests/test_polish_cli.py index d880e04..856d89f 100644 --- a/cli/tests/test_polish_cli.py +++ b/cli/tests/test_polish_cli.py @@ -8,6 +8,7 @@ import pytest from test_cli import JOB, JOB_C, PROJECT, SUBMIT, Docker, Fake, _login, _run_local +from krater_cli.api import KraterError from krater_cli.main import run @@ -50,6 +51,15 @@ def test_max_spend_is_sent_in_cents(http, fake) -> None: assert fake.jobs[-1]["body"]["max_spend_cents"] == 1250 +def test_max_spend_on_a_krater_without_spend_caps_cancels_the_job(http, fake) -> None: + _login(http, []) + fake.knows_max_spend = False + with pytest.raises(KraterError, match="doesn't take --max-spend yet"): + run([*SUBMIT, "--limit", "0.1", "--max-spend", "5"], http=http, say=lambda _: None) + + assert any(r.url.path == f"/api/v1/jobs/{JOB}/cancel" for r in fake.requests) + + def test_max_spend_must_be_an_amount(http, fake, capsys) -> None: with pytest.raises(SystemExit): run([*SUBMIT, "--limit", "0.1", "--max-spend", "0"], http=http, say=lambda _: None) diff --git a/tests/web/test_api.py b/tests/web/test_api.py index 48e6855..15fa585 100644 --- a/tests/web/test_api.py +++ b/tests/web/test_api.py @@ -323,7 +323,7 @@ def test_a_member_reads_a_jobs_output_and_lists_its_files(client: TestClient, pr files = client.get(f"/api/v1/jobs/{job['id']}/files", headers=_as(MEMBER_SUB)).json()["files"] refused = client.get(f"/api/v1/jobs/{job['id']}/files", headers=_as(REVIEWER_SUB)) - assert logs == {"attempt": 1, "text": "hello\n", "size_bytes": 6, "truncated": False} + assert logs == {"attempt": 1, "text": "hello\n", "size_bytes": 6, "truncated": False, "latest_attempt": None} assert [(f["area"], f["name"], f["size_bytes"]) for f in files] == [ ("outputs", "score.txt", 5), ("logs", "attempt-1.log", 6),