From 89a2a0401a6bfc601ba286c963ca74b21344f097 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 21 Oct 2026 02:13:37 +0000 Subject: [PATCH] feat(plan-selection): a setting to accept outputs without accuracy guarantees PlanSelectionSettings::ignore_accuracy_guarantees selects the built-in IgnoreAccuracyGuarantees model in place of the caller's accuracy model. Each accepted output records the cause the guarantee check would have rejected it with, and the installable plan's report lists those queries. Executability checks still apply. stage_pipeline gains --ignore-accuracy-guarantees. --- crates/devtools/src/bin/stage_pipeline.rs | 16 +- crates/devtools/tests/stage_pipeline.rs | 39 +++ crates/plan-selection/src/accuracy.rs | 35 +- crates/plan-selection/src/lib.rs | 326 ++++++++++++------ crates/plan-selection/src/pass/mod.rs | 20 +- crates/planner/src/installable.rs | 17 + crates/planner/tests/runtime_install.rs | 198 +++++++++++ .../examples/planner-layering-example1.json | 38 ++ 8 files changed, 586 insertions(+), 103 deletions(-) diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 7cce00c29..fa2bb91c6 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -39,6 +39,9 @@ // metric or null when exact or accepted regardless of guarantee, // whether the deployment's accuracy model accepted it directly, // whatever its guarantee says (`accepted_regardless_of_guarantee`), +// the cause the guarantee check would have rejected it with when it was +// accepted only so (`guarantee_check_rejection`, e.g. +// `no_accuracy_guarantee`; null otherwise), // the sketches read and a one-line // summary), and every other shown candidate under `rejected` with // its `cause`: a failed check (`failure_probability_resize_failed`, @@ -120,6 +123,11 @@ // `non_repeating_query_horizon_seconds` and `measured_operation_costs`; // an omitted field takes its default). It replaces `--cost-model`, the // `--cluster-*` flags and `--measured-operation-costs`. +// `--ignore-accuracy-guarantees` sets the settings' +// `ignore_accuracy_guarantees` (also with `--plan-selection-settings`): +// Stage 3 accepts every query output regardless of its accuracy guarantee, +// so candidates are compared on cost alone; executability checks still +// apply. // `--sample-interval-ms` is the sample interval declared for every metric the // PromQL queries read (default 15000); the data declares no other facts. // `--table ''` (repeatable) declares a table the SQL queries read: `{"name": ..., @@ -163,7 +171,7 @@ const USAGE: &str = [--name ]... [--table ...] [--max-error --failure-probability --accuracy-metric ] [--sample-interval-ms ])) \ [--cost-model usage-over-time|aws-fargate-us-east-1|provisioned-capacity|cluster-capacity [--cluster-memory-bytes \ --cluster-disk-bytes [--cluster-cpu-cores ] --cluster-objective lowest-latency|least-memory|pareto-front]] \ -[--plan-selection-settings ] [--deployment-capabilities ] [--max-combinations ] [--max-displayed ] --out "; +[--plan-selection-settings ] [--ignore-accuracy-guarantees] [--deployment-capabilities ] [--max-combinations ] [--max-displayed ] --out "; /// The default `--max-combinations`: the most Stage 1 candidates (Pass 2's /// included) a run enumerates. @@ -255,6 +263,7 @@ fn run(args: Vec) -> Result<(), Failure> { let mut out = None; let mut measured_operation_costs_path: Option = None; let mut plan_selection_settings_path: Option = None; + let mut ignore_accuracy_guarantees = false; let mut args = args.into_iter(); while let Some(flag) = args.next() { let mut value = || args.next().ok_or(format!("{flag} needs a value")); @@ -310,6 +319,7 @@ fn run(args: Vec) -> Result<(), Failure> { "--out" => out = Some(value()?), "--measured-operation-costs" => measured_operation_costs_path = Some(value()?), "--plan-selection-settings" => plan_selection_settings_path = Some(value()?), + "--ignore-accuracy-guarantees" => ignore_accuracy_guarantees = true, other => return Err(format!("unknown argument {other}").into()), } } @@ -384,7 +394,7 @@ fn run(args: Vec) -> Result<(), Failure> { } None => MeasuredOperationCosts::default(), }; - let plan_selection = match &plan_selection_settings_path { + let mut plan_selection: PlanSelectionSettings = match &plan_selection_settings_path { Some(path) => { let text = std::fs::read_to_string(path) .map_err(|e| format!("--plan-selection-settings {path}: {e}"))?; @@ -397,6 +407,7 @@ fn run(args: Vec) -> Result<(), Failure> { ..default_settings }, }; + plan_selection.ignore_accuracy_guarantees |= ignore_accuracy_guarantees; let from_file = match &workload_path { Some(path) if example.is_none() && queries.is_empty() => { let text = @@ -715,6 +726,7 @@ fn plan_selection_json( "target_metric": accepted.target_metric, "accepted_metric": accepted.accepted_metric, "accepted_regardless_of_guarantee": accepted.accepted_regardless_of_guarantee, + "guarantee_check_rejection": accepted.guarantee_check_rejection, "sketches": accepted.sketches, "summary": accepted.to_string(), }) diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index 7059a4553..782689870 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -988,3 +988,42 @@ fn plan_selection_settings_flag_loads_the_settings_from_json() { ); std::fs::remove_file(path).unwrap(); } + +/// `--ignore-accuracy-guarantees` (issue #774): the built-in model rejects +/// every sketch candidate of an average of p99s as having no guarantee; +/// with the flag each is valid, accepted regardless of its guarantee, and +/// records why the guarantee check would have rejected it. +#[test] +fn ignore_accuracy_guarantees_flag_accepts_outputs_without_a_guarantee() { + let args = [ + "--promql", + "avg by (g) (quantile_over_time(0.99, x[100s]))", + "--max-error", + "0.01", + "--failure-probability", + "0.01", + "--accuracy-metric", + "rank", + ]; + let builtin = document_of(&args); + let unguaranteed: Vec = builtin["stage3_plan_selection"]["rejected"] + .as_array() + .unwrap() + .iter() + .filter(|r| r["cause"] == "no_accuracy_guarantee") + .map(|r| r["id"].as_str().unwrap().to_string()) + .collect(); + assert!(!unguaranteed.is_empty(), "{builtin}"); + let mut ignoring = args.to_vec(); + ignoring.push("--ignore-accuracy-guarantees"); + let document = document_of(&ignoring); + let accepted = &document["stage3_plan_selection"]["accepted"]; + for id in &unguaranteed { + let query = &accepted[id][0]; + assert_eq!(query["accepted_regardless_of_guarantee"], true, "{id}"); + assert_eq!( + query["guarantee_check_rejection"], "no_accuracy_guarantee", + "{id}: {query}" + ); + } +} diff --git a/crates/plan-selection/src/accuracy.rs b/crates/plan-selection/src/accuracy.rs index 192a05d7c..58983068e 100644 --- a/crates/plan-selection/src/accuracy.rs +++ b/crates/plan-selection/src/accuracy.rs @@ -37,8 +37,9 @@ pub trait AccuracyModel { /// Whether the deployment accepts `output`, one query's output in a /// candidate, for `target` directly, whatever its guarantee says: /// missing (Stage 1 had no analytical rule), met or missed. Stage 3 asks - /// this for every query output before any guarantee check; when it - /// returns `true`, no guarantee or evidence check runs for that output. + /// this for every query output; when it returns `true`, the output is + /// accepted, and the guarantee and evidence check only records whether + /// it would have rejected it (`AcceptedAccuracy::guarantee_check_rejection`). /// It does not override the executability check before it: a plan that /// reads Count-Min (plain, CmsWithHeap or Hydra Count-Min) over update /// weights not proven non-negative is rejected @@ -85,6 +86,36 @@ impl AccuracyModel for BuiltinAccuracyModel { } } +/// Accepts every query output regardless of its guarantee, for comparing +/// candidates on cost alone (e.g. a benchmark of sketch plans that have no +/// analytical guarantee); [`Self::satisfies`] is the built-in rule. Selected +/// by [`PlanSelectionSettings::ignore_accuracy_guarantees`](crate::PlanSelectionSettings::ignore_accuracy_guarantees). +/// Executability checks still run: Count-Min over update weights not proven +/// non-negative, and exact per-series aggregates the executor has no kernel +/// for, are rejected. +#[derive(Debug, Default, Clone, Copy)] +pub struct IgnoreAccuracyGuarantees; + +impl AccuracyModel for IgnoreAccuracyGuarantees { + fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { + BuiltinAccuracyModel.satisfies(guarantee, target) + } + + fn accepts_output_regardless_of_guarantee( + &self, + _output: &QueryRoot, + _target: &AccuracyTarget, + ) -> bool { + true + } + + /// It accepts every output, so no candidate is discarded early for + /// its accuracy. + fn may_accept_regardless_of_guarantee(&self) -> bool { + true + } +} + /// The deployment's planning-time accuracy evidence, read only by plan /// selection: enforced operand domains the propagation rules need, and /// empirical guarantees for whole query outputs. Passed in diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index e252da672..b2c0af52b 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -7,9 +7,12 @@ //! from the data workload and the deployment's operand-domain contracts), //! or when the deployment's empirical evidence shows this //! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, -//! read only here). Before either, the deployment's accuracy model may accept -//! the output directly, whatever its guarantee says -//! (`AccuracyModel::accepts_output_regardless_of_guarantee`). Before that, for +//! read only here). The deployment's accuracy model may instead accept the +//! output directly, whatever its guarantee says +//! (`AccuracyModel::accepts_output_regardless_of_guarantee`); the settings' +//! `ignore_accuracy_guarantees` accepts every output so +//! ([`IgnoreAccuracyGuarantees`]), and the report records each one the +//! guarantee check would have rejected. Before that, for //! a query with a target, Count-Min is admitted only over weights proven //! non-negative, which the executor needs to run it; a query without one //! constrains nothing. @@ -257,7 +260,8 @@ use asap_physical_asap_optimization::PhysicalCandidate; pub mod accuracy; pub use accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence, + AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, IgnoreAccuracyGuarantees, + NoAccuracyEvidence, }; pub mod cost; /// Costed reuse of compatible physical pane producers. @@ -513,6 +517,16 @@ pub struct PlanSelectionSettings { /// costs"). Also loads the earlier array of [`MeasuredOperationCost`]. /// Default empty: every node is priced from the formulas. pub measured_operation_costs: MeasuredOperationCosts, + /// Accept every query output regardless of its accuracy guarantee: + /// planning uses [`IgnoreAccuracyGuarantees`] in place of + /// [`PlanningModels::accuracy`] ([`PlanningModels::effective_accuracy`]). + /// For comparing candidates on cost alone, e.g. a benchmark of sketch + /// plans with no analytical guarantee. Executability checks still apply. + /// Each output accepted only because of it, i.e. whose guarantee check + /// fails, is marked in the report + /// ([`AcceptedAccuracy::guarantee_check_rejection`]). JSON: + /// `"ignore_accuracy_guarantees": true`. Default `false`. + pub ignore_accuracy_guarantees: bool, } impl Default for PlanSelectionSettings { @@ -522,6 +536,7 @@ impl Default for PlanSelectionSettings { resource_speed: ResourceSpeed::illustrative(), non_repeating_query_horizon_seconds: DEFAULT_NON_REPEATING_QUERY_HORIZON_SECONDS, measured_operation_costs: MeasuredOperationCosts::default(), + ignore_accuracy_guarantees: false, } } } @@ -803,6 +818,14 @@ pub struct AcceptedAccuracy { /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]); `false` /// when a guarantee met the target. pub accepted_regardless_of_guarantee: bool, + /// When accepted regardless of its guarantee, the cause the guarantee + /// check would have rejected it with + /// ([`RejectionCause::NoAccuracyGuarantee`] or + /// [`RejectionCause::AccuracyTargetMissed`]): the output was accepted + /// only because of the accuracy model (e.g. + /// [`PlanSelectionSettings::ignore_accuracy_guarantees`]). `None` when + /// its guarantee meets the target. + pub guarantee_check_rejection: Option, /// The sketch algorithms the query's output reads, in plan order /// (e.g. `["Kll"]`); empty for an exact plan. pub sketches: Vec, @@ -824,10 +847,15 @@ impl std::fmt::Display for AcceptedAccuracy { Some(metric) => write!(f, "metric {}; ", name(metric))?, } match self.accepted_metric { - _ if self.accepted_regardless_of_guarantee => write!( - f, - "accepted by the deployment's accuracy model regardless of guarantee" - )?, + _ if self.accepted_regardless_of_guarantee => { + write!( + f, + "accepted by the deployment's accuracy model regardless of guarantee" + )?; + if let Some(cause) = self.guarantee_check_rejection { + write!(f, ", ignoring a failed guarantee check ({cause:?})")?; + } + } None => write!(f, "accepted exact")?, Some(metric) => write!(f, "accepted {}", name(metric))?, } @@ -1860,9 +1888,11 @@ pub fn resize_for_failure_probability( /// The Count-Min weight check runs first and rejects as /// [`RejectionCause::CountMinWeightsNotProvenNonNegative`]: the executor /// cannot run such a plan. Then the deployment's accuracy model is asked -/// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]): an output it -/// accepts is accepted whatever its guarantee says, and no guarantee or -/// evidence check runs for it. Otherwise, an output with no guarantee from either +/// ([`AccuracyModel::accepts_output_regardless_of_guarantee`], through +/// [`PlanningModels::effective_accuracy`]): an output it accepts is accepted +/// whatever its guarantee says, and the guarantee check's rejection, if any, +/// is only recorded ([`AcceptedAccuracy::guarantee_check_rejection`]). +/// Otherwise, an output with no guarantee from either /// source (Stage 1 had no rule, and the evidence has none) is rejected as /// [`RejectionCause::NoAccuracyGuarantee`], and any other miss as /// [`RejectionCause::AccuracyTargetMissed`]. @@ -1925,104 +1955,30 @@ fn accuracy_check( }); } } + // The guarantee check: the theoretical guarantee, then the + // deployment's evidence. + let checked = guarantee_check(query, root, target, data, models); // The deployment's model may accept the output directly, whatever - // its guarantee says; no guarantee or evidence check runs then. + // its guarantee says; the check's result is then only recorded. if models - .accuracy + .effective_accuracy() .accepts_output_regardless_of_guarantee(root, target) { accepted.push(AcceptedAccuracy { target_metric: target.accuracy_metric, accepted_metric: None, accepted_regardless_of_guarantee: true, + guarantee_check_rejection: checked.err().map(|failure| failure.cause), sketches, }); continue; } - // A guarantee per value is stated over the whole result's `G` - // values for a whole-result target. - let series = series_read(root, data); - let values = values_per_result(root, series); - let over_scope = |g: AccuracyGuarantee| match target.accuracy_scope { - AccuracyScope::WholeResult => g.over_whole_result(values), - AccuracyScope::PerValue => g, - }; - let guarantee = bound_root_guarantee(root, series, data, models).map(over_scope); - let meets = |g: &AccuracyGuarantee| models.accuracy.satisfies(g, target); - // The deployment's evidence is read only when the theoretical - // guarantee does not meet the target. - let evidence = || models.evidence.output_guarantee(root).map(over_scope); - let (met, evidence) = match guarantee.clone().filter(meets) { - Some(met) => (Some(met), None), - None => { - let evidence = evidence(); - (evidence.clone().filter(meets), evidence) - } - }; - if let Some(met) = met { - accepted.push(AcceptedAccuracy { - target_metric: target.accuracy_metric, - accepted_metric: (!met.is_exact()).then_some(met.spec.accuracy_metric), - accepted_regardless_of_guarantee: false, - sketches, - }); - continue; - } - if guarantee.is_none() && evidence.is_none() { - return Err(CheckFailure { - cause: RejectionCause::NoAccuracyGuarantee, - reason: format!( - "q{query}: the output has no guarantee, and the deployment has no evidence \ - for it, so it cannot meet target {target:?}" - ), - // No rule composes one, whatever the sizes; the deployment's - // model and evidence may still judge another size otherwise. - size_independent: !models.accuracy.may_accept_regardless_of_guarantee() - && !models.evidence.may_provide_evidence(), - }); - } - // A bound or failure probability that stays unknown once bound is - // unknown at every size; a known one may shrink with larger summaries. - let unknown = |g: &AccuracyGuarantee| { - g.spec.error.evaluate().is_none() - || (target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none()) - }; - let size_independent = evidence.is_none() - && guarantee.as_ref().is_some_and(unknown) - && !models.accuracy.may_accept_regardless_of_guarantee() - && !models.evidence.may_provide_evidence(); - return Err(CheckFailure { - cause: RejectionCause::AccuracyTargetMissed, - size_independent, - reason: match guarantee { - Some(g) - if values.is_none() - && target.accuracy_scope == AccuracyScope::WholeResult - && target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none() => - { - format!( - "q{query}: the output's guarantee holds for each value, and the number \ - of values one evaluation returns is not bound (no declared series \ - count for a source a PromQL vector reads; no bound on a SQL result's \ - groups yet), so it does not meet whole-result target {target:?}, and \ - the deployment has no evidence that it meets it" - ) - } - Some(g) => format!( - "q{query}: the output guarantees a {:?} bound {:?}, failure probability \ - {:?}, which misses target {target:?}, and the deployment has no evidence \ - that it meets it", - g.spec.accuracy_metric, - g.spec.error.evaluate(), - g.spec.failure_probability.evaluate(), - ), - None => format!( - "q{query}: the output has no guarantee, and the deployment has no \ - evidence that it meets target {target:?}" - ), - }, + accepted.push(AcceptedAccuracy { + target_metric: target.accuracy_metric, + accepted_metric: checked?, + accepted_regardless_of_guarantee: false, + guarantee_check_rejection: None, + sketches, }); } Ok(accepted) @@ -2041,6 +1997,103 @@ pub(crate) struct CheckFailure { pub(crate) size_independent: bool, } +/// Whether query `query`'s output `root` meets `target` by its theoretical +/// root guarantee or, failing that, the deployment's evidence +/// ([`accuracy_check`]): the metric of the guarantee that met it (`None` +/// for an exact output), or why not, as +/// [`RejectionCause::NoAccuracyGuarantee`] or +/// [`RejectionCause::AccuracyTargetMissed`]. +fn guarantee_check( + query: usize, + root: &QueryRoot, + target: &AccuracyTarget, + data: &DataWorkload, + models: &PlanningModels<'_>, +) -> Result, CheckFailure> { + // A guarantee per value is stated over the whole result's `G` + // values for a whole-result target. + let series = series_read(root, data); + let values = values_per_result(root, series); + let over_scope = |g: AccuracyGuarantee| match target.accuracy_scope { + AccuracyScope::WholeResult => g.over_whole_result(values), + AccuracyScope::PerValue => g, + }; + let guarantee = bound_root_guarantee(root, series, data, models).map(over_scope); + let meets = |g: &AccuracyGuarantee| models.effective_accuracy().satisfies(g, target); + // The deployment's evidence is read only when the theoretical + // guarantee does not meet the target. + let evidence = || models.evidence.output_guarantee(root).map(over_scope); + let (met, evidence) = match guarantee.clone().filter(meets) { + Some(met) => (Some(met), None), + None => { + let evidence = evidence(); + (evidence.clone().filter(meets), evidence) + } + }; + if let Some(met) = met { + return Ok((!met.is_exact()).then_some(met.spec.accuracy_metric)); + } + // Whether the deployment's model or evidence may judge another size of + // the plan's summaries otherwise. + let model_or_evidence_may_judge = models + .effective_accuracy() + .may_accept_regardless_of_guarantee() + || models.evidence.may_provide_evidence(); + if guarantee.is_none() && evidence.is_none() { + return Err(CheckFailure { + cause: RejectionCause::NoAccuracyGuarantee, + reason: format!( + "q{query}: the output has no guarantee, and the deployment has no evidence \ + for it, so it cannot meet target {target:?}" + ), + // No rule composes one, whatever the sizes. + size_independent: !model_or_evidence_may_judge, + }); + } + // A bound or failure probability that stays unknown once bound is + // unknown at every size; a known one may shrink with larger summaries. + let unknown = |g: &AccuracyGuarantee| { + g.spec.error.evaluate().is_none() + || (target.failure_probability.is_some() + && g.spec.failure_probability.evaluate().is_none()) + }; + let size_independent = evidence.is_none() + && guarantee.as_ref().is_some_and(unknown) + && !model_or_evidence_may_judge; + Err(CheckFailure { + cause: RejectionCause::AccuracyTargetMissed, + size_independent, + reason: match guarantee { + Some(g) + if values.is_none() + && target.accuracy_scope == AccuracyScope::WholeResult + && target.failure_probability.is_some() + && g.spec.failure_probability.evaluate().is_none() => + { + format!( + "q{query}: the output's guarantee holds for each value, and the number \ + of values one evaluation returns is not bound (no declared series \ + count for a source a PromQL vector reads; no bound on a SQL result's \ + groups yet), so it does not meet whole-result target {target:?}, and \ + the deployment has no evidence that it meets it" + ) + } + Some(g) => format!( + "q{query}: the output guarantees a {:?} bound {:?}, failure probability \ + {:?}, which misses target {target:?}, and the deployment has no evidence \ + that it meets it", + g.spec.accuracy_metric, + g.spec.error.evaluate(), + g.spec.failure_probability.evaluate(), + ), + None => format!( + "q{query}: the output has no guarantee, and the deployment has no \ + evidence that it meets target {target:?}" + ), + }, + }) +} + /// `domain` with an open [`WeightDomain::SamplesOfMetric`] requirement /// answered from `data`'s declared metric types: proven when the metric is /// declared a counter, unknown otherwise. No type is inferred from a name. @@ -5326,6 +5379,85 @@ mod tests { } } + /// `ignore_accuracy_guarantees` (issue #774) does not override the + /// executor's requirement on Count-Min update weights either: over + /// samples of a gauge, P2's CmsWithHeap is still rejected as + /// `CountMinWeightsNotProvenNonNegative`. + #[test] + fn unproven_count_min_weights_reject_even_when_ignoring_guarantees() { + let target = AccuracyTarget::max_error(0.01) + .failure_probability(0.001) + .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); + let gauge = with_metric_types(data(), &[("m", MetricType::Gauge)]); + let settings = PlanSelectionSettings { + ignore_accuracy_guarantees: true, + ..PlanSelectionSettings::default() + }; + let selection = select_plan( + &candidates(), + &[entry_with(target, repeating(10_000))], + &gauge, + PlanningModels::builtin().with_plan_selection(&settings), + ) + .unwrap(); + let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); + assert_eq!( + p2.cause, + RejectionCause::CountMinWeightsNotProvenNonNegative, + "{}", + p2.reason + ); + } + + /// With `ignore_accuracy_guarantees` (issue #774), each summary + /// candidate the built-in model rejects (no guarantee, or one that + /// misses the target) is valid, and its accepted output records the + /// cause the guarantee check would have rejected it with; an output + /// whose guarantee meets the target records none. + #[test] + fn ignoring_guarantees_accepts_and_marks_outputs_the_check_rejects() { + let target = AccuracyTarget::max_error(0.01) + .failure_probability(0.001) + .accuracy_metric(AccuracyMetric::ABSOLUTE); + let entries = [entry_with(target, repeating(10_000))]; + let settings = PlanSelectionSettings { + ignore_accuracy_guarantees: true, + ..PlanSelectionSettings::default() + }; + let builtin = + select_plan(&candidates(), &entries, &data(), PlanningModels::builtin()).unwrap(); + let ignoring = select_plan( + &candidates(), + &entries, + &data(), + PlanningModels::builtin().with_plan_selection(&settings), + ) + .unwrap(); + let mut marked = 0; + for rejection in builtin.rejected.iter().filter(|r| { + matches!( + r.cause, + RejectionCause::NoAccuracyGuarantee | RejectionCause::AccuracyTargetMissed + ) + }) { + let [accepted] = ignoring.accepted[&rejection.id].as_slice() else { + panic!("{} has one query", rejection.id) + }; + assert!(accepted.accepted_regardless_of_guarantee); + assert_eq!(accepted.guarantee_check_rejection, Some(rejection.cause)); + assert!(accepted.to_string().contains("ignoring"), "{accepted}"); + marked += 1; + } + assert!(marked > 0, "{:?}", builtin.rejected); + for (id, queries) in &builtin.accepted { + assert_eq!( + ignoring.accepted[id][0].guarantee_check_rejection, None, + "{id}" + ); + assert_eq!(queries[0].guarantee_check_rejection, None, "{id}"); + } + } + /// `node` with every CmsWithHeap summary build replaced by a Hydra /// Count-Min over the same update. fn with_hydra_count_min(node: &Rc) -> Rc { diff --git a/crates/plan-selection/src/pass/mod.rs b/crates/plan-selection/src/pass/mod.rs index a6d8b78be..529a14101 100644 --- a/crates/plan-selection/src/pass/mod.rs +++ b/crates/plan-selection/src/pass/mod.rs @@ -28,7 +28,8 @@ use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; use crate::accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence, + AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, IgnoreAccuracyGuarantees, + NoAccuracyEvidence, }; use crate::{PlanSelectionSettings, Selection, SelectionError}; use asap_logical_asap_optimization::deployment_capabilities::DeploymentCapabilities; @@ -41,6 +42,7 @@ pub use stage1_to_stage3::{ }; static BUILTIN_ACCURACY_MODEL: BuiltinAccuracyModel = BuiltinAccuracyModel; +static IGNORE_ACCURACY_GUARANTEES: IgnoreAccuracyGuarantees = IgnoreAccuracyGuarantees; static NO_ACCURACY_EVIDENCE: NoAccuracyEvidence = NoAccuracyEvidence; static DEFAULT_PLAN_SELECTION: std::sync::LazyLock = std::sync::LazyLock::new(PlanSelectionSettings::default); @@ -62,7 +64,9 @@ static DEFAULT_DEPLOYMENT: std::sync::LazyLock = pub struct PlanningModels<'a> { /// Decides whether a candidate's accuracy guarantee meets its query's /// target, and whether the deployment accepts an output regardless of - /// its guarantee; [`BuiltinAccuracyModel`] by default. + /// its guarantee; [`BuiltinAccuracyModel`] by default. Planning reads it + /// through [`Self::effective_accuracy`], which + /// [`PlanSelectionSettings::ignore_accuracy_guarantees`] overrides. pub accuracy: &'a dyn AccuracyModel, /// Deployment facts the guarantees need (operand domains, observed /// guarantees); [`NoAccuracyEvidence`] by default. @@ -103,6 +107,18 @@ impl<'a> PlanningModels<'a> { } } + /// The accuracy model planning uses: [`IgnoreAccuracyGuarantees`] when + /// the plan selection settings set + /// [`PlanSelectionSettings::ignore_accuracy_guarantees`], overriding + /// `accuracy`; otherwise `accuracy`. + pub fn effective_accuracy(&self) -> &'a dyn AccuracyModel { + if self.plan_selection.ignore_accuracy_guarantees { + &IGNORE_ACCURACY_GUARANTEES + } else { + self.accuracy + } + } + pub fn with_accuracy(mut self, accuracy: &'a dyn AccuracyModel) -> Self { self.accuracy = accuracy; self diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index e5e6efed0..b581b6c49 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -192,6 +192,13 @@ pub struct SelectedPlanReport { /// (`Selection::search`); empty when not recorded. #[serde(default)] pub search: SearchReport, + /// The workload queries (entry indices) whose output Stage 3 accepted + /// only because the accuracy model accepts it regardless of its + /// guarantee (e.g. `PlanSelectionSettings::ignore_accuracy_guarantees`): + /// the guarantee check would have rejected it. Empty when every + /// output's guarantee meets its target; omitted from JSON then. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub queries_accepted_without_guarantee: Vec, } /// What Stage 3 predicts the selected plan uses, per second of wall time @@ -1058,7 +1065,17 @@ fn report_of( installed_node_ids: &HashMap>, ) -> SelectedPlanReport { let cost = selection.costs.get(&selection.selected); + let queries_accepted_without_guarantee = selection + .accepted + .get(&selection.selected) + .into_iter() + .flatten() + .enumerate() + .filter(|(_, accepted)| accepted.guarantee_check_rejection.is_some()) + .map(|(query, _)| query) + .collect(); SelectedPlanReport { + queries_accepted_without_guarantee, selected_candidate_id: selection.selected.clone(), cost_per_second: cost.and_then(|cost| cost.cost_per_second), predicted_usage: cost.map(|cost| PredictedUsage { diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index 298881db8..18afa9256 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -1377,3 +1377,201 @@ async fn an_undeclared_gauges_merged_ddsketch_panes_get_no_relative_bound_throug let error = plan_p99_avg(false).await.expect_err("no valid plan"); assert!(error.contains("P5-m1"), "{error}"); } + +/// `avg by (g) (quantile_over_time(0.99, x[100s]))` every 10 s at rank +/// error 0.01 and δ 0.01, over 100 ingesting `x` series sampled every +/// second: ASAPQuery's random mixed workload query. Stage 1 has no +/// analytical rule for an average of quantile estimates, so every sketch +/// candidate has no guarantee. +fn average_of_p99s_workload() -> PlanningWorkload { + serde_json::from_str( + r#"{"query_workload": {"language": "promql", "repeating_queries": [{ + "query": "avg by (g) (quantile_over_time(0.99, x[100s]))", + "demand": {"fixed_interval": 10000}, + "requirements": {"accuracy": {"accuracy_metric": "rank", "max_error": 0.01, + "failure_probability": 0.01, "accuracy_scope": "per_value"}}, + "predictability": {"predictable": {"known_at": null}}}]}, + "data_workload": {"arrival": {"ingesting": {"rows_per_second": 100.0}}, + "sources": {"x": {"series_count": 100, "sample_interval": 1000}}}}"#, + ) + .unwrap() +} + +/// Raw `x` samples of series `a` and `b` in group `g="1"` and `c` in group +/// `g="2"`, one every second from 200 s before `EVALUATION_MS` to it, off +/// the second boundaries: series `i`'s value is `100 × i + 1..=101`, all +/// positive, which DDSketch needs at run time (nothing declares it here). +fn grouped_samples() -> Vec { + let mut samples = Vec::new(); + for (i, (instance, g)) in [("a", "1"), ("b", "1"), ("c", "2")].iter().enumerate() { + for step in 0..200 { + samples.push(Sample { + labels: BTreeMap::from([ + ("__name__".to_string(), "x".to_string()), + ("instance".to_string(), instance.to_string()), + ("g".to_string(), g.to_string()), + ]), + timestamp_ms: EVALUATION_MS - 199_500 + step * 1_000, + value: (100 * i) as f64 + 1.0 + ((step * 37 + 11 * i as i64) % 101) as f64, + }); + } + } + samples +} + +/// The exact answer of `avg by (g) (quantile_over_time(0.99, x[100s]))` at +/// `EVALUATION_MS` over [`grouped_samples`]: PromQL's p99 (linear +/// interpolation at rank `0.99 × (n − 1)`) of each series' samples in +/// `(EVALUATION_MS − 100 s, EVALUATION_MS]`, averaged per group. +fn exact_average_of_p99s(samples: &[Sample]) -> BTreeMap { + let mut per_series: BTreeMap<(String, String), Vec> = BTreeMap::new(); + for s in samples + .iter() + .filter(|s| EVALUATION_MS - 100_000 < s.timestamp_ms && s.timestamp_ms <= EVALUATION_MS) + { + per_series + .entry((s.labels["g"].clone(), s.labels["instance"].clone())) + .or_default() + .push(s.value); + } + let mut per_group: BTreeMap> = BTreeMap::new(); + for ((g, _), mut values) in per_series { + values.sort_by(f64::total_cmp); + let rank = 0.99 * (values.len() - 1) as f64; + let (low, high) = (rank.floor() as usize, rank.ceil() as usize); + let p99 = values[low] + (values[high] - values[low]) * (rank - low as f64); + per_group.entry(g).or_default().push(p99); + } + per_group + .into_iter() + .map(|(g, p99s)| (g, p99s.iter().sum::() / p99s.len() as f64)) + .collect() +} + +/// Issue #774: without raw samples retained, every candidate of an average +/// of p99s fails a check (the exact ones scan raw samples at query time, +/// the sketch ones have no guarantee), so planning fails. With +/// `ignore_accuracy_guarantees` loaded from the settings JSON, the cheapest +/// sketch plan is selected, the report marks its output as accepted without +/// a guarantee, and it runs: equal to its query-time reference and within a +/// loose tolerance of the exact answer. +#[tokio::test] +async fn ignoring_accuracy_guarantees_selects_a_sketch_plan_that_runs() { + let workload = average_of_p99s_workload(); + let capabilities = DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + }; + let plan = |settings: PlanSelectionSettings| { + let workload = workload.clone(); + let capabilities = capabilities.clone(); + async move { + plan_installable(UserInput::new( + &workload, + FrontendInput::Promql { histograms: None }, + PlanningModels::builtin() + .with_deployment_capabilities(&capabilities) + .with_plan_selection(&settings), + )) + .await + } + }; + // Without the setting, no candidate is valid. + let error = plan(PlanSelectionSettings::default()) + .await + .expect_err("no candidate is valid without the setting"); + assert!(error.to_string().contains("no valid candidate"), "{error}"); + // With it, loaded from JSON as a deployment passes it. + let settings: PlanSelectionSettings = + serde_json::from_str(r#"{"ignore_accuracy_guarantees": true}"#).unwrap(); + assert!(settings.ignore_accuracy_guarantees); + let installed = plan(settings).await.expect("a sketch plan is selected"); + // A per-series KLL or DDSketch plan. + let families: Vec = installed + .dag + .dag + .nodes + .iter() + .filter_map(|node| match &node.payload { + asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( + ASAPOp::SummaryAgg { family, .. } | ASAPOp::PaneChain { family, .. }, + ) => Some(format!("{family:?}")), + _ => None, + }) + .collect(); + // Its sketches (beside the average's exact sum and count) are KLL or + // DDSketch. + let sketches: Vec<&String> = families + .iter() + .filter(|f| f.starts_with("Sketch")) + .collect(); + assert!( + !sketches.is_empty() + && sketches + .iter() + .all(|f| f.contains("Kll") || f.contains("DDSketch")), + "{families:?}" + ); + // The report marks the query's output as accepted without a guarantee. + let report = installed.selection.as_ref().expect("Stage 3's report"); + assert_eq!(report.queries_accepted_without_guarantee, vec![0]); + // It runs, equal to its query-time reference ... + let plan_json = serde_json::to_string(&installed).unwrap(); + let samples = grouped_samples(); + let run = run_data_plane( + &plan_json, + &samples, + EVALUATION_MS, + &mut SummaryStore::new(), + ); + assert_eq!(run.answers_per_query, run.reference_answers_per_query); + // ... and within 5 (of values spanning 100 per series) of the exact + // average of p99s, per group. + let exact = exact_average_of_p99s(&samples); + let answered: BTreeMap = run.answers_per_query[0] + .iter() + .map(|(labels, value)| (labels["g"].clone(), *value)) + .collect(); + assert_eq!( + answered.keys().collect::>(), + exact.keys().collect::>() + ); + for (g, value) in &answered { + assert!( + (value - exact[g]).abs() <= 5.0, + "g={g}: {value} vs exact {}", + exact[g] + ); + } +} + +/// The MVP benchmark's p99 query over an undeclared gauge, which has no +/// valid plan with the built-in model, gets one with +/// `ignore_accuracy_guarantees`, and the report marks its output as +/// accepted without a guarantee. +#[tokio::test] +async fn ignoring_accuracy_guarantees_plans_the_undeclared_p99_query() { + let capabilities = DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + }; + let settings = PlanSelectionSettings { + ignore_accuracy_guarantees: true, + ..PlanSelectionSettings::default() + }; + let plan = plan_installable(UserInput::new( + &p99_avg_workload(false), + FrontendInput::Promql { histograms: None }, + PlanningModels::builtin() + .with_deployment_capabilities(&capabilities) + .with_plan_selection(&settings), + )) + .await + .expect("a plan is selected"); + let report = plan.selection.as_ref().expect("Stage 3's report"); + assert_eq!(report.queries_accepted_without_guarantee, vec![0]); + assert!( + plan.query_time_raw_windows.is_empty(), + "no raw samples read" + ); +} diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index e7e36edf9..b9c75123b 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -103821,6 +103821,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103828,6 +103829,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103837,6 +103839,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103844,6 +103847,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103853,6 +103857,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103860,6 +103865,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103869,6 +103875,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103876,6 +103883,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103885,6 +103893,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103892,6 +103901,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103901,6 +103911,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103908,6 +103919,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103917,6 +103929,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103924,6 +103937,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103933,6 +103947,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103940,6 +103955,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103949,6 +103965,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103956,6 +103973,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103965,6 +103983,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103972,6 +103991,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103981,6 +104001,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103988,6 +104009,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103997,6 +104019,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104004,6 +104027,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104013,6 +104037,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104020,6 +104045,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104029,6 +104055,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104036,6 +104063,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104045,6 +104073,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104052,6 +104081,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104061,6 +104091,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104068,6 +104099,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104077,6 +104109,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104084,6 +104117,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104093,6 +104127,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104100,6 +104135,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104109,6 +104145,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104116,6 +104153,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null