diff --git a/crates/devtools/src/bin/stage_pipeline.rs b/crates/devtools/src/bin/stage_pipeline.rs index 7cce00c29..fa2bb91c6 100644 --- a/crates/devtools/src/bin/stage_pipeline.rs +++ b/crates/devtools/src/bin/stage_pipeline.rs @@ -39,6 +39,9 @@ // metric or null when exact or accepted regardless of guarantee, // whether the deployment's accuracy model accepted it directly, // whatever its guarantee says (`accepted_regardless_of_guarantee`), +// the cause the guarantee check would have rejected it with when it was +// accepted only so (`guarantee_check_rejection`, e.g. +// `no_accuracy_guarantee`; null otherwise), // the sketches read and a one-line // summary), and every other shown candidate under `rejected` with // its `cause`: a failed check (`failure_probability_resize_failed`, @@ -120,6 +123,11 @@ // `non_repeating_query_horizon_seconds` and `measured_operation_costs`; // an omitted field takes its default). It replaces `--cost-model`, the // `--cluster-*` flags and `--measured-operation-costs`. +// `--ignore-accuracy-guarantees` sets the settings' +// `ignore_accuracy_guarantees` (also with `--plan-selection-settings`): +// Stage 3 accepts every query output regardless of its accuracy guarantee, +// so candidates are compared on cost alone; executability checks still +// apply. // `--sample-interval-ms` is the sample interval declared for every metric the // PromQL queries read (default 15000); the data declares no other facts. // `--table ''` (repeatable) declares a table the SQL queries read: `{"name": ..., @@ -163,7 +171,7 @@ const USAGE: &str = [--name ]... [--table ...] [--max-error --failure-probability --accuracy-metric ] [--sample-interval-ms ])) \ [--cost-model usage-over-time|aws-fargate-us-east-1|provisioned-capacity|cluster-capacity [--cluster-memory-bytes \ --cluster-disk-bytes [--cluster-cpu-cores ] --cluster-objective lowest-latency|least-memory|pareto-front]] \ -[--plan-selection-settings ] [--deployment-capabilities ] [--max-combinations ] [--max-displayed ] --out "; +[--plan-selection-settings ] [--ignore-accuracy-guarantees] [--deployment-capabilities ] [--max-combinations ] [--max-displayed ] --out "; /// The default `--max-combinations`: the most Stage 1 candidates (Pass 2's /// included) a run enumerates. @@ -255,6 +263,7 @@ fn run(args: Vec) -> Result<(), Failure> { let mut out = None; let mut measured_operation_costs_path: Option = None; let mut plan_selection_settings_path: Option = None; + let mut ignore_accuracy_guarantees = false; let mut args = args.into_iter(); while let Some(flag) = args.next() { let mut value = || args.next().ok_or(format!("{flag} needs a value")); @@ -310,6 +319,7 @@ fn run(args: Vec) -> Result<(), Failure> { "--out" => out = Some(value()?), "--measured-operation-costs" => measured_operation_costs_path = Some(value()?), "--plan-selection-settings" => plan_selection_settings_path = Some(value()?), + "--ignore-accuracy-guarantees" => ignore_accuracy_guarantees = true, other => return Err(format!("unknown argument {other}").into()), } } @@ -384,7 +394,7 @@ fn run(args: Vec) -> Result<(), Failure> { } None => MeasuredOperationCosts::default(), }; - let plan_selection = match &plan_selection_settings_path { + let mut plan_selection: PlanSelectionSettings = match &plan_selection_settings_path { Some(path) => { let text = std::fs::read_to_string(path) .map_err(|e| format!("--plan-selection-settings {path}: {e}"))?; @@ -397,6 +407,7 @@ fn run(args: Vec) -> Result<(), Failure> { ..default_settings }, }; + plan_selection.ignore_accuracy_guarantees |= ignore_accuracy_guarantees; let from_file = match &workload_path { Some(path) if example.is_none() && queries.is_empty() => { let text = @@ -715,6 +726,7 @@ fn plan_selection_json( "target_metric": accepted.target_metric, "accepted_metric": accepted.accepted_metric, "accepted_regardless_of_guarantee": accepted.accepted_regardless_of_guarantee, + "guarantee_check_rejection": accepted.guarantee_check_rejection, "sketches": accepted.sketches, "summary": accepted.to_string(), }) diff --git a/crates/devtools/tests/stage_pipeline.rs b/crates/devtools/tests/stage_pipeline.rs index 7059a4553..782689870 100644 --- a/crates/devtools/tests/stage_pipeline.rs +++ b/crates/devtools/tests/stage_pipeline.rs @@ -988,3 +988,42 @@ fn plan_selection_settings_flag_loads_the_settings_from_json() { ); std::fs::remove_file(path).unwrap(); } + +/// `--ignore-accuracy-guarantees` (issue #774): the built-in model rejects +/// every sketch candidate of an average of p99s as having no guarantee; +/// with the flag each is valid, accepted regardless of its guarantee, and +/// records why the guarantee check would have rejected it. +#[test] +fn ignore_accuracy_guarantees_flag_accepts_outputs_without_a_guarantee() { + let args = [ + "--promql", + "avg by (g) (quantile_over_time(0.99, x[100s]))", + "--max-error", + "0.01", + "--failure-probability", + "0.01", + "--accuracy-metric", + "rank", + ]; + let builtin = document_of(&args); + let unguaranteed: Vec = builtin["stage3_plan_selection"]["rejected"] + .as_array() + .unwrap() + .iter() + .filter(|r| r["cause"] == "no_accuracy_guarantee") + .map(|r| r["id"].as_str().unwrap().to_string()) + .collect(); + assert!(!unguaranteed.is_empty(), "{builtin}"); + let mut ignoring = args.to_vec(); + ignoring.push("--ignore-accuracy-guarantees"); + let document = document_of(&ignoring); + let accepted = &document["stage3_plan_selection"]["accepted"]; + for id in &unguaranteed { + let query = &accepted[id][0]; + assert_eq!(query["accepted_regardless_of_guarantee"], true, "{id}"); + assert_eq!( + query["guarantee_check_rejection"], "no_accuracy_guarantee", + "{id}: {query}" + ); + } +} diff --git a/crates/plan-selection/src/accuracy.rs b/crates/plan-selection/src/accuracy.rs index 192a05d7c..58983068e 100644 --- a/crates/plan-selection/src/accuracy.rs +++ b/crates/plan-selection/src/accuracy.rs @@ -37,8 +37,9 @@ pub trait AccuracyModel { /// Whether the deployment accepts `output`, one query's output in a /// candidate, for `target` directly, whatever its guarantee says: /// missing (Stage 1 had no analytical rule), met or missed. Stage 3 asks - /// this for every query output before any guarantee check; when it - /// returns `true`, no guarantee or evidence check runs for that output. + /// this for every query output; when it returns `true`, the output is + /// accepted, and the guarantee and evidence check only records whether + /// it would have rejected it (`AcceptedAccuracy::guarantee_check_rejection`). /// It does not override the executability check before it: a plan that /// reads Count-Min (plain, CmsWithHeap or Hydra Count-Min) over update /// weights not proven non-negative is rejected @@ -85,6 +86,36 @@ impl AccuracyModel for BuiltinAccuracyModel { } } +/// Accepts every query output regardless of its guarantee, for comparing +/// candidates on cost alone (e.g. a benchmark of sketch plans that have no +/// analytical guarantee); [`Self::satisfies`] is the built-in rule. Selected +/// by [`PlanSelectionSettings::ignore_accuracy_guarantees`](crate::PlanSelectionSettings::ignore_accuracy_guarantees). +/// Executability checks still run: Count-Min over update weights not proven +/// non-negative, and exact per-series aggregates the executor has no kernel +/// for, are rejected. +#[derive(Debug, Default, Clone, Copy)] +pub struct IgnoreAccuracyGuarantees; + +impl AccuracyModel for IgnoreAccuracyGuarantees { + fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { + BuiltinAccuracyModel.satisfies(guarantee, target) + } + + fn accepts_output_regardless_of_guarantee( + &self, + _output: &QueryRoot, + _target: &AccuracyTarget, + ) -> bool { + true + } + + /// It accepts every output, so no candidate is discarded early for + /// its accuracy. + fn may_accept_regardless_of_guarantee(&self) -> bool { + true + } +} + /// The deployment's planning-time accuracy evidence, read only by plan /// selection: enforced operand domains the propagation rules need, and /// empirical guarantees for whole query outputs. Passed in diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index e252da672..b2c0af52b 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -7,9 +7,12 @@ //! from the data workload and the deployment's operand-domain contracts), //! or when the deployment's empirical evidence shows this //! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, -//! read only here). Before either, the deployment's accuracy model may accept -//! the output directly, whatever its guarantee says -//! (`AccuracyModel::accepts_output_regardless_of_guarantee`). Before that, for +//! read only here). The deployment's accuracy model may instead accept the +//! output directly, whatever its guarantee says +//! (`AccuracyModel::accepts_output_regardless_of_guarantee`); the settings' +//! `ignore_accuracy_guarantees` accepts every output so +//! ([`IgnoreAccuracyGuarantees`]), and the report records each one the +//! guarantee check would have rejected. Before that, for //! a query with a target, Count-Min is admitted only over weights proven //! non-negative, which the executor needs to run it; a query without one //! constrains nothing. @@ -257,7 +260,8 @@ use asap_physical_asap_optimization::PhysicalCandidate; pub mod accuracy; pub use accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence, + AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, IgnoreAccuracyGuarantees, + NoAccuracyEvidence, }; pub mod cost; /// Costed reuse of compatible physical pane producers. @@ -513,6 +517,16 @@ pub struct PlanSelectionSettings { /// costs"). Also loads the earlier array of [`MeasuredOperationCost`]. /// Default empty: every node is priced from the formulas. pub measured_operation_costs: MeasuredOperationCosts, + /// Accept every query output regardless of its accuracy guarantee: + /// planning uses [`IgnoreAccuracyGuarantees`] in place of + /// [`PlanningModels::accuracy`] ([`PlanningModels::effective_accuracy`]). + /// For comparing candidates on cost alone, e.g. a benchmark of sketch + /// plans with no analytical guarantee. Executability checks still apply. + /// Each output accepted only because of it, i.e. whose guarantee check + /// fails, is marked in the report + /// ([`AcceptedAccuracy::guarantee_check_rejection`]). JSON: + /// `"ignore_accuracy_guarantees": true`. Default `false`. + pub ignore_accuracy_guarantees: bool, } impl Default for PlanSelectionSettings { @@ -522,6 +536,7 @@ impl Default for PlanSelectionSettings { resource_speed: ResourceSpeed::illustrative(), non_repeating_query_horizon_seconds: DEFAULT_NON_REPEATING_QUERY_HORIZON_SECONDS, measured_operation_costs: MeasuredOperationCosts::default(), + ignore_accuracy_guarantees: false, } } } @@ -803,6 +818,14 @@ pub struct AcceptedAccuracy { /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]); `false` /// when a guarantee met the target. pub accepted_regardless_of_guarantee: bool, + /// When accepted regardless of its guarantee, the cause the guarantee + /// check would have rejected it with + /// ([`RejectionCause::NoAccuracyGuarantee`] or + /// [`RejectionCause::AccuracyTargetMissed`]): the output was accepted + /// only because of the accuracy model (e.g. + /// [`PlanSelectionSettings::ignore_accuracy_guarantees`]). `None` when + /// its guarantee meets the target. + pub guarantee_check_rejection: Option, /// The sketch algorithms the query's output reads, in plan order /// (e.g. `["Kll"]`); empty for an exact plan. pub sketches: Vec, @@ -824,10 +847,15 @@ impl std::fmt::Display for AcceptedAccuracy { Some(metric) => write!(f, "metric {}; ", name(metric))?, } match self.accepted_metric { - _ if self.accepted_regardless_of_guarantee => write!( - f, - "accepted by the deployment's accuracy model regardless of guarantee" - )?, + _ if self.accepted_regardless_of_guarantee => { + write!( + f, + "accepted by the deployment's accuracy model regardless of guarantee" + )?; + if let Some(cause) = self.guarantee_check_rejection { + write!(f, ", ignoring a failed guarantee check ({cause:?})")?; + } + } None => write!(f, "accepted exact")?, Some(metric) => write!(f, "accepted {}", name(metric))?, } @@ -1860,9 +1888,11 @@ pub fn resize_for_failure_probability( /// The Count-Min weight check runs first and rejects as /// [`RejectionCause::CountMinWeightsNotProvenNonNegative`]: the executor /// cannot run such a plan. Then the deployment's accuracy model is asked -/// ([`AccuracyModel::accepts_output_regardless_of_guarantee`]): an output it -/// accepts is accepted whatever its guarantee says, and no guarantee or -/// evidence check runs for it. Otherwise, an output with no guarantee from either +/// ([`AccuracyModel::accepts_output_regardless_of_guarantee`], through +/// [`PlanningModels::effective_accuracy`]): an output it accepts is accepted +/// whatever its guarantee says, and the guarantee check's rejection, if any, +/// is only recorded ([`AcceptedAccuracy::guarantee_check_rejection`]). +/// Otherwise, an output with no guarantee from either /// source (Stage 1 had no rule, and the evidence has none) is rejected as /// [`RejectionCause::NoAccuracyGuarantee`], and any other miss as /// [`RejectionCause::AccuracyTargetMissed`]. @@ -1925,104 +1955,30 @@ fn accuracy_check( }); } } + // The guarantee check: the theoretical guarantee, then the + // deployment's evidence. + let checked = guarantee_check(query, root, target, data, models); // The deployment's model may accept the output directly, whatever - // its guarantee says; no guarantee or evidence check runs then. + // its guarantee says; the check's result is then only recorded. if models - .accuracy + .effective_accuracy() .accepts_output_regardless_of_guarantee(root, target) { accepted.push(AcceptedAccuracy { target_metric: target.accuracy_metric, accepted_metric: None, accepted_regardless_of_guarantee: true, + guarantee_check_rejection: checked.err().map(|failure| failure.cause), sketches, }); continue; } - // A guarantee per value is stated over the whole result's `G` - // values for a whole-result target. - let series = series_read(root, data); - let values = values_per_result(root, series); - let over_scope = |g: AccuracyGuarantee| match target.accuracy_scope { - AccuracyScope::WholeResult => g.over_whole_result(values), - AccuracyScope::PerValue => g, - }; - let guarantee = bound_root_guarantee(root, series, data, models).map(over_scope); - let meets = |g: &AccuracyGuarantee| models.accuracy.satisfies(g, target); - // The deployment's evidence is read only when the theoretical - // guarantee does not meet the target. - let evidence = || models.evidence.output_guarantee(root).map(over_scope); - let (met, evidence) = match guarantee.clone().filter(meets) { - Some(met) => (Some(met), None), - None => { - let evidence = evidence(); - (evidence.clone().filter(meets), evidence) - } - }; - if let Some(met) = met { - accepted.push(AcceptedAccuracy { - target_metric: target.accuracy_metric, - accepted_metric: (!met.is_exact()).then_some(met.spec.accuracy_metric), - accepted_regardless_of_guarantee: false, - sketches, - }); - continue; - } - if guarantee.is_none() && evidence.is_none() { - return Err(CheckFailure { - cause: RejectionCause::NoAccuracyGuarantee, - reason: format!( - "q{query}: the output has no guarantee, and the deployment has no evidence \ - for it, so it cannot meet target {target:?}" - ), - // No rule composes one, whatever the sizes; the deployment's - // model and evidence may still judge another size otherwise. - size_independent: !models.accuracy.may_accept_regardless_of_guarantee() - && !models.evidence.may_provide_evidence(), - }); - } - // A bound or failure probability that stays unknown once bound is - // unknown at every size; a known one may shrink with larger summaries. - let unknown = |g: &AccuracyGuarantee| { - g.spec.error.evaluate().is_none() - || (target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none()) - }; - let size_independent = evidence.is_none() - && guarantee.as_ref().is_some_and(unknown) - && !models.accuracy.may_accept_regardless_of_guarantee() - && !models.evidence.may_provide_evidence(); - return Err(CheckFailure { - cause: RejectionCause::AccuracyTargetMissed, - size_independent, - reason: match guarantee { - Some(g) - if values.is_none() - && target.accuracy_scope == AccuracyScope::WholeResult - && target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none() => - { - format!( - "q{query}: the output's guarantee holds for each value, and the number \ - of values one evaluation returns is not bound (no declared series \ - count for a source a PromQL vector reads; no bound on a SQL result's \ - groups yet), so it does not meet whole-result target {target:?}, and \ - the deployment has no evidence that it meets it" - ) - } - Some(g) => format!( - "q{query}: the output guarantees a {:?} bound {:?}, failure probability \ - {:?}, which misses target {target:?}, and the deployment has no evidence \ - that it meets it", - g.spec.accuracy_metric, - g.spec.error.evaluate(), - g.spec.failure_probability.evaluate(), - ), - None => format!( - "q{query}: the output has no guarantee, and the deployment has no \ - evidence that it meets target {target:?}" - ), - }, + accepted.push(AcceptedAccuracy { + target_metric: target.accuracy_metric, + accepted_metric: checked?, + accepted_regardless_of_guarantee: false, + guarantee_check_rejection: None, + sketches, }); } Ok(accepted) @@ -2041,6 +1997,103 @@ pub(crate) struct CheckFailure { pub(crate) size_independent: bool, } +/// Whether query `query`'s output `root` meets `target` by its theoretical +/// root guarantee or, failing that, the deployment's evidence +/// ([`accuracy_check`]): the metric of the guarantee that met it (`None` +/// for an exact output), or why not, as +/// [`RejectionCause::NoAccuracyGuarantee`] or +/// [`RejectionCause::AccuracyTargetMissed`]. +fn guarantee_check( + query: usize, + root: &QueryRoot, + target: &AccuracyTarget, + data: &DataWorkload, + models: &PlanningModels<'_>, +) -> Result, CheckFailure> { + // A guarantee per value is stated over the whole result's `G` + // values for a whole-result target. + let series = series_read(root, data); + let values = values_per_result(root, series); + let over_scope = |g: AccuracyGuarantee| match target.accuracy_scope { + AccuracyScope::WholeResult => g.over_whole_result(values), + AccuracyScope::PerValue => g, + }; + let guarantee = bound_root_guarantee(root, series, data, models).map(over_scope); + let meets = |g: &AccuracyGuarantee| models.effective_accuracy().satisfies(g, target); + // The deployment's evidence is read only when the theoretical + // guarantee does not meet the target. + let evidence = || models.evidence.output_guarantee(root).map(over_scope); + let (met, evidence) = match guarantee.clone().filter(meets) { + Some(met) => (Some(met), None), + None => { + let evidence = evidence(); + (evidence.clone().filter(meets), evidence) + } + }; + if let Some(met) = met { + return Ok((!met.is_exact()).then_some(met.spec.accuracy_metric)); + } + // Whether the deployment's model or evidence may judge another size of + // the plan's summaries otherwise. + let model_or_evidence_may_judge = models + .effective_accuracy() + .may_accept_regardless_of_guarantee() + || models.evidence.may_provide_evidence(); + if guarantee.is_none() && evidence.is_none() { + return Err(CheckFailure { + cause: RejectionCause::NoAccuracyGuarantee, + reason: format!( + "q{query}: the output has no guarantee, and the deployment has no evidence \ + for it, so it cannot meet target {target:?}" + ), + // No rule composes one, whatever the sizes. + size_independent: !model_or_evidence_may_judge, + }); + } + // A bound or failure probability that stays unknown once bound is + // unknown at every size; a known one may shrink with larger summaries. + let unknown = |g: &AccuracyGuarantee| { + g.spec.error.evaluate().is_none() + || (target.failure_probability.is_some() + && g.spec.failure_probability.evaluate().is_none()) + }; + let size_independent = evidence.is_none() + && guarantee.as_ref().is_some_and(unknown) + && !model_or_evidence_may_judge; + Err(CheckFailure { + cause: RejectionCause::AccuracyTargetMissed, + size_independent, + reason: match guarantee { + Some(g) + if values.is_none() + && target.accuracy_scope == AccuracyScope::WholeResult + && target.failure_probability.is_some() + && g.spec.failure_probability.evaluate().is_none() => + { + format!( + "q{query}: the output's guarantee holds for each value, and the number \ + of values one evaluation returns is not bound (no declared series \ + count for a source a PromQL vector reads; no bound on a SQL result's \ + groups yet), so it does not meet whole-result target {target:?}, and \ + the deployment has no evidence that it meets it" + ) + } + Some(g) => format!( + "q{query}: the output guarantees a {:?} bound {:?}, failure probability \ + {:?}, which misses target {target:?}, and the deployment has no evidence \ + that it meets it", + g.spec.accuracy_metric, + g.spec.error.evaluate(), + g.spec.failure_probability.evaluate(), + ), + None => format!( + "q{query}: the output has no guarantee, and the deployment has no \ + evidence that it meets target {target:?}" + ), + }, + }) +} + /// `domain` with an open [`WeightDomain::SamplesOfMetric`] requirement /// answered from `data`'s declared metric types: proven when the metric is /// declared a counter, unknown otherwise. No type is inferred from a name. @@ -5326,6 +5379,85 @@ mod tests { } } + /// `ignore_accuracy_guarantees` (issue #774) does not override the + /// executor's requirement on Count-Min update weights either: over + /// samples of a gauge, P2's CmsWithHeap is still rejected as + /// `CountMinWeightsNotProvenNonNegative`. + #[test] + fn unproven_count_min_weights_reject_even_when_ignoring_guarantees() { + let target = AccuracyTarget::max_error(0.01) + .failure_probability(0.001) + .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); + let gauge = with_metric_types(data(), &[("m", MetricType::Gauge)]); + let settings = PlanSelectionSettings { + ignore_accuracy_guarantees: true, + ..PlanSelectionSettings::default() + }; + let selection = select_plan( + &candidates(), + &[entry_with(target, repeating(10_000))], + &gauge, + PlanningModels::builtin().with_plan_selection(&settings), + ) + .unwrap(); + let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); + assert_eq!( + p2.cause, + RejectionCause::CountMinWeightsNotProvenNonNegative, + "{}", + p2.reason + ); + } + + /// With `ignore_accuracy_guarantees` (issue #774), each summary + /// candidate the built-in model rejects (no guarantee, or one that + /// misses the target) is valid, and its accepted output records the + /// cause the guarantee check would have rejected it with; an output + /// whose guarantee meets the target records none. + #[test] + fn ignoring_guarantees_accepts_and_marks_outputs_the_check_rejects() { + let target = AccuracyTarget::max_error(0.01) + .failure_probability(0.001) + .accuracy_metric(AccuracyMetric::ABSOLUTE); + let entries = [entry_with(target, repeating(10_000))]; + let settings = PlanSelectionSettings { + ignore_accuracy_guarantees: true, + ..PlanSelectionSettings::default() + }; + let builtin = + select_plan(&candidates(), &entries, &data(), PlanningModels::builtin()).unwrap(); + let ignoring = select_plan( + &candidates(), + &entries, + &data(), + PlanningModels::builtin().with_plan_selection(&settings), + ) + .unwrap(); + let mut marked = 0; + for rejection in builtin.rejected.iter().filter(|r| { + matches!( + r.cause, + RejectionCause::NoAccuracyGuarantee | RejectionCause::AccuracyTargetMissed + ) + }) { + let [accepted] = ignoring.accepted[&rejection.id].as_slice() else { + panic!("{} has one query", rejection.id) + }; + assert!(accepted.accepted_regardless_of_guarantee); + assert_eq!(accepted.guarantee_check_rejection, Some(rejection.cause)); + assert!(accepted.to_string().contains("ignoring"), "{accepted}"); + marked += 1; + } + assert!(marked > 0, "{:?}", builtin.rejected); + for (id, queries) in &builtin.accepted { + assert_eq!( + ignoring.accepted[id][0].guarantee_check_rejection, None, + "{id}" + ); + assert_eq!(queries[0].guarantee_check_rejection, None, "{id}"); + } + } + /// `node` with every CmsWithHeap summary build replaced by a Hydra /// Count-Min over the same update. fn with_hydra_count_min(node: &Rc) -> Rc { diff --git a/crates/plan-selection/src/pass/mod.rs b/crates/plan-selection/src/pass/mod.rs index a6d8b78be..529a14101 100644 --- a/crates/plan-selection/src/pass/mod.rs +++ b/crates/plan-selection/src/pass/mod.rs @@ -28,7 +28,8 @@ use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; use crate::accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence, + AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, IgnoreAccuracyGuarantees, + NoAccuracyEvidence, }; use crate::{PlanSelectionSettings, Selection, SelectionError}; use asap_logical_asap_optimization::deployment_capabilities::DeploymentCapabilities; @@ -41,6 +42,7 @@ pub use stage1_to_stage3::{ }; static BUILTIN_ACCURACY_MODEL: BuiltinAccuracyModel = BuiltinAccuracyModel; +static IGNORE_ACCURACY_GUARANTEES: IgnoreAccuracyGuarantees = IgnoreAccuracyGuarantees; static NO_ACCURACY_EVIDENCE: NoAccuracyEvidence = NoAccuracyEvidence; static DEFAULT_PLAN_SELECTION: std::sync::LazyLock = std::sync::LazyLock::new(PlanSelectionSettings::default); @@ -62,7 +64,9 @@ static DEFAULT_DEPLOYMENT: std::sync::LazyLock = pub struct PlanningModels<'a> { /// Decides whether a candidate's accuracy guarantee meets its query's /// target, and whether the deployment accepts an output regardless of - /// its guarantee; [`BuiltinAccuracyModel`] by default. + /// its guarantee; [`BuiltinAccuracyModel`] by default. Planning reads it + /// through [`Self::effective_accuracy`], which + /// [`PlanSelectionSettings::ignore_accuracy_guarantees`] overrides. pub accuracy: &'a dyn AccuracyModel, /// Deployment facts the guarantees need (operand domains, observed /// guarantees); [`NoAccuracyEvidence`] by default. @@ -103,6 +107,18 @@ impl<'a> PlanningModels<'a> { } } + /// The accuracy model planning uses: [`IgnoreAccuracyGuarantees`] when + /// the plan selection settings set + /// [`PlanSelectionSettings::ignore_accuracy_guarantees`], overriding + /// `accuracy`; otherwise `accuracy`. + pub fn effective_accuracy(&self) -> &'a dyn AccuracyModel { + if self.plan_selection.ignore_accuracy_guarantees { + &IGNORE_ACCURACY_GUARANTEES + } else { + self.accuracy + } + } + pub fn with_accuracy(mut self, accuracy: &'a dyn AccuracyModel) -> Self { self.accuracy = accuracy; self diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index e5e6efed0..b581b6c49 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -192,6 +192,13 @@ pub struct SelectedPlanReport { /// (`Selection::search`); empty when not recorded. #[serde(default)] pub search: SearchReport, + /// The workload queries (entry indices) whose output Stage 3 accepted + /// only because the accuracy model accepts it regardless of its + /// guarantee (e.g. `PlanSelectionSettings::ignore_accuracy_guarantees`): + /// the guarantee check would have rejected it. Empty when every + /// output's guarantee meets its target; omitted from JSON then. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub queries_accepted_without_guarantee: Vec, } /// What Stage 3 predicts the selected plan uses, per second of wall time @@ -1058,7 +1065,17 @@ fn report_of( installed_node_ids: &HashMap>, ) -> SelectedPlanReport { let cost = selection.costs.get(&selection.selected); + let queries_accepted_without_guarantee = selection + .accepted + .get(&selection.selected) + .into_iter() + .flatten() + .enumerate() + .filter(|(_, accepted)| accepted.guarantee_check_rejection.is_some()) + .map(|(query, _)| query) + .collect(); SelectedPlanReport { + queries_accepted_without_guarantee, selected_candidate_id: selection.selected.clone(), cost_per_second: cost.and_then(|cost| cost.cost_per_second), predicted_usage: cost.map(|cost| PredictedUsage { diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index 298881db8..18afa9256 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -1377,3 +1377,201 @@ async fn an_undeclared_gauges_merged_ddsketch_panes_get_no_relative_bound_throug let error = plan_p99_avg(false).await.expect_err("no valid plan"); assert!(error.contains("P5-m1"), "{error}"); } + +/// `avg by (g) (quantile_over_time(0.99, x[100s]))` every 10 s at rank +/// error 0.01 and δ 0.01, over 100 ingesting `x` series sampled every +/// second: ASAPQuery's random mixed workload query. Stage 1 has no +/// analytical rule for an average of quantile estimates, so every sketch +/// candidate has no guarantee. +fn average_of_p99s_workload() -> PlanningWorkload { + serde_json::from_str( + r#"{"query_workload": {"language": "promql", "repeating_queries": [{ + "query": "avg by (g) (quantile_over_time(0.99, x[100s]))", + "demand": {"fixed_interval": 10000}, + "requirements": {"accuracy": {"accuracy_metric": "rank", "max_error": 0.01, + "failure_probability": 0.01, "accuracy_scope": "per_value"}}, + "predictability": {"predictable": {"known_at": null}}}]}, + "data_workload": {"arrival": {"ingesting": {"rows_per_second": 100.0}}, + "sources": {"x": {"series_count": 100, "sample_interval": 1000}}}}"#, + ) + .unwrap() +} + +/// Raw `x` samples of series `a` and `b` in group `g="1"` and `c` in group +/// `g="2"`, one every second from 200 s before `EVALUATION_MS` to it, off +/// the second boundaries: series `i`'s value is `100 × i + 1..=101`, all +/// positive, which DDSketch needs at run time (nothing declares it here). +fn grouped_samples() -> Vec { + let mut samples = Vec::new(); + for (i, (instance, g)) in [("a", "1"), ("b", "1"), ("c", "2")].iter().enumerate() { + for step in 0..200 { + samples.push(Sample { + labels: BTreeMap::from([ + ("__name__".to_string(), "x".to_string()), + ("instance".to_string(), instance.to_string()), + ("g".to_string(), g.to_string()), + ]), + timestamp_ms: EVALUATION_MS - 199_500 + step * 1_000, + value: (100 * i) as f64 + 1.0 + ((step * 37 + 11 * i as i64) % 101) as f64, + }); + } + } + samples +} + +/// The exact answer of `avg by (g) (quantile_over_time(0.99, x[100s]))` at +/// `EVALUATION_MS` over [`grouped_samples`]: PromQL's p99 (linear +/// interpolation at rank `0.99 × (n − 1)`) of each series' samples in +/// `(EVALUATION_MS − 100 s, EVALUATION_MS]`, averaged per group. +fn exact_average_of_p99s(samples: &[Sample]) -> BTreeMap { + let mut per_series: BTreeMap<(String, String), Vec> = BTreeMap::new(); + for s in samples + .iter() + .filter(|s| EVALUATION_MS - 100_000 < s.timestamp_ms && s.timestamp_ms <= EVALUATION_MS) + { + per_series + .entry((s.labels["g"].clone(), s.labels["instance"].clone())) + .or_default() + .push(s.value); + } + let mut per_group: BTreeMap> = BTreeMap::new(); + for ((g, _), mut values) in per_series { + values.sort_by(f64::total_cmp); + let rank = 0.99 * (values.len() - 1) as f64; + let (low, high) = (rank.floor() as usize, rank.ceil() as usize); + let p99 = values[low] + (values[high] - values[low]) * (rank - low as f64); + per_group.entry(g).or_default().push(p99); + } + per_group + .into_iter() + .map(|(g, p99s)| (g, p99s.iter().sum::() / p99s.len() as f64)) + .collect() +} + +/// Issue #774: without raw samples retained, every candidate of an average +/// of p99s fails a check (the exact ones scan raw samples at query time, +/// the sketch ones have no guarantee), so planning fails. With +/// `ignore_accuracy_guarantees` loaded from the settings JSON, the cheapest +/// sketch plan is selected, the report marks its output as accepted without +/// a guarantee, and it runs: equal to its query-time reference and within a +/// loose tolerance of the exact answer. +#[tokio::test] +async fn ignoring_accuracy_guarantees_selects_a_sketch_plan_that_runs() { + let workload = average_of_p99s_workload(); + let capabilities = DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + }; + let plan = |settings: PlanSelectionSettings| { + let workload = workload.clone(); + let capabilities = capabilities.clone(); + async move { + plan_installable(UserInput::new( + &workload, + FrontendInput::Promql { histograms: None }, + PlanningModels::builtin() + .with_deployment_capabilities(&capabilities) + .with_plan_selection(&settings), + )) + .await + } + }; + // Without the setting, no candidate is valid. + let error = plan(PlanSelectionSettings::default()) + .await + .expect_err("no candidate is valid without the setting"); + assert!(error.to_string().contains("no valid candidate"), "{error}"); + // With it, loaded from JSON as a deployment passes it. + let settings: PlanSelectionSettings = + serde_json::from_str(r#"{"ignore_accuracy_guarantees": true}"#).unwrap(); + assert!(settings.ignore_accuracy_guarantees); + let installed = plan(settings).await.expect("a sketch plan is selected"); + // A per-series KLL or DDSketch plan. + let families: Vec = installed + .dag + .dag + .nodes + .iter() + .filter_map(|node| match &node.payload { + asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( + ASAPOp::SummaryAgg { family, .. } | ASAPOp::PaneChain { family, .. }, + ) => Some(format!("{family:?}")), + _ => None, + }) + .collect(); + // Its sketches (beside the average's exact sum and count) are KLL or + // DDSketch. + let sketches: Vec<&String> = families + .iter() + .filter(|f| f.starts_with("Sketch")) + .collect(); + assert!( + !sketches.is_empty() + && sketches + .iter() + .all(|f| f.contains("Kll") || f.contains("DDSketch")), + "{families:?}" + ); + // The report marks the query's output as accepted without a guarantee. + let report = installed.selection.as_ref().expect("Stage 3's report"); + assert_eq!(report.queries_accepted_without_guarantee, vec![0]); + // It runs, equal to its query-time reference ... + let plan_json = serde_json::to_string(&installed).unwrap(); + let samples = grouped_samples(); + let run = run_data_plane( + &plan_json, + &samples, + EVALUATION_MS, + &mut SummaryStore::new(), + ); + assert_eq!(run.answers_per_query, run.reference_answers_per_query); + // ... and within 5 (of values spanning 100 per series) of the exact + // average of p99s, per group. + let exact = exact_average_of_p99s(&samples); + let answered: BTreeMap = run.answers_per_query[0] + .iter() + .map(|(labels, value)| (labels["g"].clone(), *value)) + .collect(); + assert_eq!( + answered.keys().collect::>(), + exact.keys().collect::>() + ); + for (g, value) in &answered { + assert!( + (value - exact[g]).abs() <= 5.0, + "g={g}: {value} vs exact {}", + exact[g] + ); + } +} + +/// The MVP benchmark's p99 query over an undeclared gauge, which has no +/// valid plan with the built-in model, gets one with +/// `ignore_accuracy_guarantees`, and the report marks its output as +/// accepted without a guarantee. +#[tokio::test] +async fn ignoring_accuracy_guarantees_plans_the_undeclared_p99_query() { + let capabilities = DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + }; + let settings = PlanSelectionSettings { + ignore_accuracy_guarantees: true, + ..PlanSelectionSettings::default() + }; + let plan = plan_installable(UserInput::new( + &p99_avg_workload(false), + FrontendInput::Promql { histograms: None }, + PlanningModels::builtin() + .with_deployment_capabilities(&capabilities) + .with_plan_selection(&settings), + )) + .await + .expect("a plan is selected"); + let report = plan.selection.as_ref().expect("Stage 3's report"); + assert_eq!(report.queries_accepted_without_guarantee, vec![0]); + assert!( + plan.query_time_raw_windows.is_empty(), + "no raw samples read" + ); +} diff --git a/tools/dag-viewer/examples/planner-layering-example1.json b/tools/dag-viewer/examples/planner-layering-example1.json index e7e36edf9..b9c75123b 100644 --- a/tools/dag-viewer/examples/planner-layering-example1.json +++ b/tools/dag-viewer/examples/planner-layering-example1.json @@ -103821,6 +103821,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103828,6 +103829,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103837,6 +103839,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103844,6 +103847,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103853,6 +103857,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103860,6 +103865,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103869,6 +103875,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103876,6 +103883,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103885,6 +103893,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103892,6 +103901,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103901,6 +103911,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103908,6 +103919,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103917,6 +103929,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103924,6 +103937,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103933,6 +103947,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103940,6 +103955,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103949,6 +103965,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103956,6 +103973,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103965,6 +103983,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103972,6 +103991,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103981,6 +104001,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103988,6 +104009,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -103997,6 +104019,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104004,6 +104027,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104013,6 +104037,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104020,6 +104045,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104029,6 +104055,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104036,6 +104063,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104045,6 +104073,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104052,6 +104081,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104061,6 +104091,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104068,6 +104099,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104077,6 +104109,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104084,6 +104117,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104093,6 +104127,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104100,6 +104135,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104109,6 +104145,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null @@ -104116,6 +104153,7 @@ { "accepted_metric": null, "accepted_regardless_of_guarantee": false, + "guarantee_check_rejection": null, "sketches": [], "summary": "metric not set; accepted exact", "target_metric": null