Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion central/erg_central/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -595,7 +595,7 @@ def flush():
indexed_staged_batches=sum(len(v) for v in completed["scopes"].values()),
elapsed_seconds=time.monotonic() - started,
outcomes=results,
zerobus_requests=dict(zerobus.metrics),
zerobus_requests=dict(zerobus.metrics) if zerobus is not None else {},
zerobus_metered_bytes=None,
databricks_dbu=None,
reported_cloud_charge=None,
Expand Down
48 changes: 48 additions & 0 deletions central/erg_central/run_gate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""Decide whether a durable journal has unprocessed work before starting compute."""

from erg import delivery_protocol as wire

from .journal import PAGE_SIZE, decode_page, page_key
from .receiver import Conflict

CURSOR = "processor-journal-cursor.json"


def pending(store, root):
"""Read bounded journal metadata and receipts. Missing state requires a run."""
raw = store.read_control(CURSOR)
if raw is None:
return True
state = wire.decode(raw)
if (
not isinstance(state, dict)
or state.get("version") != 1
or state.get("root") != root
or type(state.get("page")) is not int
or state["page"] < 0
or type(state.get("offset")) is not int
or not 0 <= state["offset"] < PAGE_SIZE
or not isinstance(state.get("retry"), list)
or len(state["retry"]) > 4096
):
raise Conflict("invalid gate cursor")
rows = decode_page(store.get(page_key(state["page"])))
if len(rows) > state["offset"]:
return True
if len(rows) < state["offset"]:
raise Conflict("journal page regressed")
for marker in state["retry"]:
from .journal import valid_marker

if not valid_marker(marker):
raise Conflict("invalid gate marker")
space, batch = marker.rsplit("/batches/", 1)
batch_id = batch.split("/")[0]
receipt = store.read_control(space + "/batches/" + batch_id + ".json")
if receipt is not None:
value = wire.decode(receipt)
if value.get("items_staged") is True:
continue
if store.read_control(space + "/quarantine/" + batch_id + ".json") is None:
return True
return False
43 changes: 43 additions & 0 deletions central/receiver/function_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,3 +107,46 @@ def status(request):
return respond(dict(error="invalid status request"), 400)
except Exception:
return respond(dict(error="status unavailable"), 503)


@app.timer_trigger(
schedule="0 0 0,4,8,12,16,20 * * *", arg_name="timer", run_on_startup=False, use_monitor=True
)
def process_pending(timer):
if os.environ.get("ERG_PROCESSOR_GATE_ENABLED") != "true":
return
import datetime
import json
import urllib.request

from azure.identity import DefaultAzureCredential
from erg_central.run_gate import pending

store, _ = configured(
os.environ["ERG_STORAGE_ACCOUNT_URL"],
os.environ["ERG_SPOOL_CONTAINER"],
os.environ["ERG_CONTROL_CONTAINER"],
os.environ.get("ERG_SPOOL_PREFIX", "v1"),
os.environ["ERG_TENANT_ID"],
os.environ["ERG_INGEST_AUDIENCE"],
os.environ.get("ERG_REQUIRED_ROLE", "Erg.Collector"),
)
if not pending(store, os.environ["ERG_PROCESSOR_SPOOL_ROOT"]):
logging.warning("erg_processor_gate empty_skip")
return
credential = DefaultAzureCredential()
token = credential.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token
request = urllib.request.Request(
os.environ["ERG_PROCESSOR_HOST"] + "/api/2.1/jobs/run-now",
data=json.dumps(
{
"job_id": int(os.environ["ERG_PROCESSOR_JOB_ID"]),
"idempotency_token": "erg-gate-"
+ datetime.datetime.now(datetime.timezone.utc).isoformat()[:13],
}
).encode(),
headers={"Authorization": "Bearer " + token, "Content-Type": "application/json"},
)
with urllib.request.urlopen(request, timeout=30) as response:
run = json.load(response)
logging.warning("erg_processor_gate submitted run_id=%s", run["run_id"])
10 changes: 9 additions & 1 deletion scripts/package-central-receiver.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,15 @@
}
files["erg/__init__.py"] = None
files["erg/delivery_protocol.py"] = root / "erg" / "delivery_protocol.py"
for name in ("__init__.py", "auth.py", "blob.py", "journal.py", "receiver.py", "status.py"):
for name in (
"__init__.py",
"auth.py",
"blob.py",
"journal.py",
"run_gate.py",
"receiver.py",
"status.py",
):
files["erg_central/" + name] = root / "central" / "erg_central" / name
with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive:
for name, path in sorted(files.items()):
Expand Down
39 changes: 39 additions & 0 deletions tests/test_processor_run_gate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
"""Compute starts only for unprocessed journal entries or unresolved receipts."""

import tempfile
import unittest
from pathlib import Path

from central.erg_central import journal, run_gate
from central.erg_central.receiver import Conflict, FileStore
from erg import delivery_protocol as wire


class RunGate(unittest.TestCase):
def test_empty_pending_completion_and_new_batch(self):
with tempfile.TemporaryDirectory() as tmp:
store = FileStore(Path(tmp) / "spool")
root = "/Volumes/erg/ingest/spool"
self.assertTrue(run_gate.pending(store, root))
marker = (
"v1/" + "a" * 64 + "/batches/00000000-0000-0000-0000-000000000001/accepted.json"
)
state = dict(version=1, root=root, page=0, offset=0, retry=[])
cursor = "processor-journal-cursor.json"
store.put_control_once(cursor, wire.encode(state))
self.assertFalse(run_gate.pending(store, root))
journal.append(store, marker)
self.assertTrue(run_gate.pending(store, root))
state.update(offset=1, retry=[marker])
Path(tmp, "spool/control", cursor).write_bytes(wire.encode(state))
self.assertTrue(run_gate.pending(store, root))
scope, tail = marker.rsplit("/batches/", 1)
store.put_control_once(
scope + "/batches/" + tail.split("/")[0] + ".json",
wire.encode(dict(items_staged=True)),
)
self.assertFalse(run_gate.pending(store, root))
state["offset"] = 2
Path(tmp, "spool/control", cursor).write_bytes(wire.encode(state))
with self.assertRaises(Conflict):
run_gate.pending(store, root)
Loading