diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29f6fc9fd..e31a24406 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,6 +91,12 @@ jobs: run: | python3 interactive/bench/ldbc/authored/check.py --server target/release/ddir_server \ --fixture --workers 4 --reps 1 --warmup 0 --output /tmp/ldbc-authored-check + # Programs installed late read every import as of the time they start. A late install + # that reads an import at its stored times can hang (no progress for 120 s) or disagree. + - name: Late-install difftest sweep + run: | + DIFFTEST_SEED=40000000 DIFFTEST_CASES=500 DIFFTEST_ORACLES=split,bind,reinstall \ + cargo test --release -p interactive --test difftest -- --ignored --nocapture # The explanation suite's heavier sweeps (soundness regression, fuzz) # are #[ignore]d for local debug runs but must gate merges. The metric # report is skipped (it prints, it does not assert). diff --git a/interactive/server/src/loop_.rs b/interactive/server/src/loop_.rs index 12f39cbf1..fdca2291c 100644 --- a/interactive/server/src/loop_.rs +++ b/interactive/server/src/loop_.rs @@ -477,7 +477,7 @@ fn start_tail( let tag = reqid.to_string(); let mut probe = ProbeHandle::new(); let shutdown = interactive::server::host_dataflow(worker, |_, scope| { - let (containers, shutdown) = trace.import_containers(scope.clone(), "TailImport"); + let (containers, shutdown) = trace.import_containers(scope, "TailImport", server.import_as_of()); let pending = Arc::new(AtomicUsize::new(0)); let (mut upper, mut dropped) = (None::, false); containers diff --git a/interactive/src/backend/corgi.rs b/interactive/src/backend/corgi.rs index 0d797aa44..854f2206e 100644 --- a/interactive/src/backend/corgi.rs +++ b/interactive/src/backend/corgi.rs @@ -10,7 +10,7 @@ use timely::dataflow::Scope; use timely::dataflow::channels::pact::Pipeline; use timely::dataflow::operators::generic::Operator; -use timely::progress::Antichain; +use timely::progress::{Antichain, Timestamp}; use differential_dataflow::AsCollection; use differential_dataflow::Collection; @@ -34,7 +34,7 @@ use crate::runtime::half_join::{CorgiHalfJoinBackend, HalfJoinBatcher}; use crate::runtime::join::CorgiJoinBackend; use differential_dogs3::operators::half_join::half_join_with_tactic; use crate::runtime::reduce::CorgiReduceBackend; -use differential_dataflow::operators::int_proxy::{ProxyJoinTactic, ProxyReduceTactic}; +use differential_dataflow::operators::int_proxy::{BackendFrontier, ProxyJoinTactic, ProxyReduceTactic}; use differential_dogs3::operators::half_join::proxy::ProxyHalfJoinTactic; use crate::lambda::{compile_flatmap, compile_join_function, compile_predicate, compile_projection, compile_scalar}; use corgi::{Graph, NumOp, Shape}; @@ -230,7 +230,29 @@ fn apply_ops(mut c: CC, ops: &[LinearOp], level: usize, plans: &[Plan]) -> CC { /// The corgi backend's arrangement: one a program built, or a published export it imported. /// Both are the same trace type at the same time, so every operator reads either in place. -pub(crate) type Arr<'scope> = Arranged<'scope, TraceAgent>; +/// +/// An import is read as of `since`, the time its reader started: a published trace has compacted +/// to that time, from which on its accumulations are correct, and before which its batches can +/// hold counts no time had. Every reader advances the times it draws to `since`: a collection as +/// it flattens the chunks, a join or reduce through DD's `BackendFrontier`. An arrangement the +/// program built is read as it is, which `since` at the minimum says. +/// +/// One time, not a frontier: the server compacts every published trace by the one-element +/// frontier `{(epoch - 1, [])}` and a reader starts at that same element, so the frontier a +/// reader reads as of is `{since}`, which is beyond the frontier its trace compacted by, as DD's +/// `BackendFrontier` requires. +#[derive(Clone)] +pub(crate) struct Arr<'scope> { + pub(crate) arranged: Arranged<'scope, TraceAgent>, + pub(crate) since: Time, +} + +impl<'scope> Arr<'scope> { + /// An arrangement the program built, read as it is. + fn built(arranged: Arranged<'scope, TraceAgent>) -> Self { + Arr { arranged, since: Time::minimum() } + } +} /// The time each coordinate of `t` less one, at least zero: the half join's compaction of its /// arrangement. Every later stream time is at or beyond some such frontier element, and advancing @@ -241,16 +263,20 @@ fn step_back(t: &Time, out: &mut Antichain