diff --git a/central/erg_central/processor.py b/central/erg_central/processor.py index 46ea958..0f42bd9 100644 --- a/central/erg_central/processor.py +++ b/central/erg_central/processor.py @@ -595,7 +595,7 @@ def flush(): indexed_staged_batches=sum(len(v) for v in completed["scopes"].values()), elapsed_seconds=time.monotonic() - started, outcomes=results, - zerobus_requests=dict(zerobus.metrics), + zerobus_requests=dict(zerobus.metrics) if zerobus is not None else {}, zerobus_metered_bytes=None, databricks_dbu=None, reported_cloud_charge=None, diff --git a/central/erg_central/run_gate.py b/central/erg_central/run_gate.py new file mode 100644 index 0000000..b56a34f --- /dev/null +++ b/central/erg_central/run_gate.py @@ -0,0 +1,48 @@ +"""Decide whether a durable journal has unprocessed work before starting compute.""" + +from erg import delivery_protocol as wire + +from .journal import PAGE_SIZE, decode_page, page_key +from .receiver import Conflict + +CURSOR = "processor-journal-cursor.json" + + +def pending(store, root): + """Read bounded journal metadata and receipts. Missing state requires a run.""" + raw = store.read_control(CURSOR) + if raw is None: + return True + state = wire.decode(raw) + if ( + not isinstance(state, dict) + or state.get("version") != 1 + or state.get("root") != root + or type(state.get("page")) is not int + or state["page"] < 0 + or type(state.get("offset")) is not int + or not 0 <= state["offset"] < PAGE_SIZE + or not isinstance(state.get("retry"), list) + or len(state["retry"]) > 4096 + ): + raise Conflict("invalid gate cursor") + rows = decode_page(store.get(page_key(state["page"]))) + if len(rows) > state["offset"]: + return True + if len(rows) < state["offset"]: + raise Conflict("journal page regressed") + for marker in state["retry"]: + from .journal import valid_marker + + if not valid_marker(marker): + raise Conflict("invalid gate marker") + space, batch = marker.rsplit("/batches/", 1) + batch_id = batch.split("/")[0] + receipt = store.read_control(space + "/batches/" + batch_id + ".json") + if receipt is not None: + value = wire.decode(receipt) + if value.get("items_staged") is True: + continue + if store.read_control(space + "/quarantine/" + batch_id + ".json") is None: + return True + return False diff --git a/central/receiver/function_app.py b/central/receiver/function_app.py index 6ce231d..ada6666 100644 --- a/central/receiver/function_app.py +++ b/central/receiver/function_app.py @@ -107,3 +107,46 @@ def status(request): return respond(dict(error="invalid status request"), 400) except Exception: return respond(dict(error="status unavailable"), 503) + + +@app.timer_trigger( + schedule="0 0 0,4,8,12,16,20 * * *", arg_name="timer", run_on_startup=False, use_monitor=True +) +def process_pending(timer): + if os.environ.get("ERG_PROCESSOR_GATE_ENABLED") != "true": + return + import datetime + import json + import urllib.request + + from azure.identity import DefaultAzureCredential + from erg_central.run_gate import pending + + store, _ = configured( + os.environ["ERG_STORAGE_ACCOUNT_URL"], + os.environ["ERG_SPOOL_CONTAINER"], + os.environ["ERG_CONTROL_CONTAINER"], + os.environ.get("ERG_SPOOL_PREFIX", "v1"), + os.environ["ERG_TENANT_ID"], + os.environ["ERG_INGEST_AUDIENCE"], + os.environ.get("ERG_REQUIRED_ROLE", "Erg.Collector"), + ) + if not pending(store, os.environ["ERG_PROCESSOR_SPOOL_ROOT"]): + logging.warning("erg_processor_gate empty_skip") + return + credential = DefaultAzureCredential() + token = credential.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + request = urllib.request.Request( + os.environ["ERG_PROCESSOR_HOST"] + "/api/2.1/jobs/run-now", + data=json.dumps( + { + "job_id": int(os.environ["ERG_PROCESSOR_JOB_ID"]), + "idempotency_token": "erg-gate-" + + datetime.datetime.now(datetime.timezone.utc).isoformat()[:13], + } + ).encode(), + headers={"Authorization": "Bearer " + token, "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + run = json.load(response) + logging.warning("erg_processor_gate submitted run_id=%s", run["run_id"]) diff --git a/scripts/package-central-receiver.py b/scripts/package-central-receiver.py index 48b28f3..7a27157 100644 --- a/scripts/package-central-receiver.py +++ b/scripts/package-central-receiver.py @@ -15,7 +15,15 @@ } files["erg/__init__.py"] = None files["erg/delivery_protocol.py"] = root / "erg" / "delivery_protocol.py" -for name in ("__init__.py", "auth.py", "blob.py", "journal.py", "receiver.py", "status.py"): +for name in ( + "__init__.py", + "auth.py", + "blob.py", + "journal.py", + "run_gate.py", + "receiver.py", + "status.py", +): files["erg_central/" + name] = root / "central" / "erg_central" / name with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive: for name, path in sorted(files.items()): diff --git a/tests/test_processor_run_gate.py b/tests/test_processor_run_gate.py new file mode 100644 index 0000000..44dfd8e --- /dev/null +++ b/tests/test_processor_run_gate.py @@ -0,0 +1,39 @@ +"""Compute starts only for unprocessed journal entries or unresolved receipts.""" + +import tempfile +import unittest +from pathlib import Path + +from central.erg_central import journal, run_gate +from central.erg_central.receiver import Conflict, FileStore +from erg import delivery_protocol as wire + + +class RunGate(unittest.TestCase): + def test_empty_pending_completion_and_new_batch(self): + with tempfile.TemporaryDirectory() as tmp: + store = FileStore(Path(tmp) / "spool") + root = "/Volumes/erg/ingest/spool" + self.assertTrue(run_gate.pending(store, root)) + marker = ( + "v1/" + "a" * 64 + "/batches/00000000-0000-0000-0000-000000000001/accepted.json" + ) + state = dict(version=1, root=root, page=0, offset=0, retry=[]) + cursor = "processor-journal-cursor.json" + store.put_control_once(cursor, wire.encode(state)) + self.assertFalse(run_gate.pending(store, root)) + journal.append(store, marker) + self.assertTrue(run_gate.pending(store, root)) + state.update(offset=1, retry=[marker]) + Path(tmp, "spool/control", cursor).write_bytes(wire.encode(state)) + self.assertTrue(run_gate.pending(store, root)) + scope, tail = marker.rsplit("/batches/", 1) + store.put_control_once( + scope + "/batches/" + tail.split("/")[0] + ".json", + wire.encode(dict(items_staged=True)), + ) + self.assertFalse(run_gate.pending(store, root)) + state["offset"] = 2 + Path(tmp, "spool/control", cursor).write_bytes(wire.encode(state)) + with self.assertRaises(Conflict): + run_gate.pending(store, root)