From 5b04d6520811ba8118b93d10f62e8ff5bbd75471 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:45:44 +0000 Subject: [PATCH] refactor(plan-selection): delete the unused physical-DAG, storage and handoff cost estimators No planning stage, devtool or downstream repository reads the whole-DAG cost path any more: #667, #686 and #734 removed its last callers. Delete it, the types only it used, and the legacy wire compatibility no committed payload needs. - plan-selection: delete cost/query_physical_lowering.rs, storage_io.rs and physical_handoff_cost.rs; in analytical_cost.rs delete estimate_physical_dag{,_with_cache,_comparison}, the cache profile resolver, PhysicalDAGNode/EvidenceBackedPhysicalDAG and their validators; in physical_operator_statistics.rs delete ComparisonScope, ScanSelection, validate_comparison_scopes and OperatorStatisticsProvider; drop the error variants only they produced. - types: delete resources::{cache, physical_handoff, storage}; read Schema with a plain derive (no `columns` / bare-dtype layouts) and require AggIntent's `col` / `cols` keys. - executor: delete the `dag` compatibility re-export; tests import the real module paths. - docs: delete the storage and handoff develop docs; mark the deleted path in the analytical-cost proposal and physical-plan integration doc. --- crates/executor/README.md | 2 +- crates/executor/src/dag/mod.rs | 8 - crates/executor/src/lib.rs | 2 - crates/executor/tests/blocking_resources.rs | 2 +- crates/executor/tests/current_series_heap.rs | 2 +- crates/executor/tests/hydra_cms_execution.rs | 8 +- .../tests/keyed_summary_row_weight.rs | 9 +- crates/executor/tests/physical_dag.rs | 33 +- crates/executor/tests/raw_scan.rs | 9 +- .../executor/tests/weighted_topk_binding.rs | 6 +- .../tests/physical_common/mod.rs | 6 +- .../src/cost/analytical_cost.rs | 1775 +--------- crates/plan-selection/src/cost/mod.rs | 7 - .../src/cost/physical_handoff_cost.rs | 230 -- .../src/cost/physical_operator_statistics.rs | 204 +- .../src/cost/query_physical_lowering.rs | 3028 ----------------- crates/plan-selection/src/cost/storage_io.rs | 261 -- .../tests/physical_handoff_cost.rs | 447 --- crates/plan-selection/tests/storage_io.rs | 368 -- crates/types/src/ir/operator/agg_intent.rs | 43 - crates/types/src/ir/schema/mod.rs | 114 - crates/types/src/workload/resources.rs | 8 - crates/types/src/workload/resources/cache.rs | 115 - .../workload/resources/physical_handoff.rs | 125 - .../types/src/workload/resources/storage.rs | 94 - crates/types/tests/planner_vocabulary.rs | 23 - .../architecture/physical-plan-integration.md | 9 +- .../analytical-resource-cost.md | 19 +- docs/develop_docs/README.md | 1 - .../asap-aware-mapping-architecture.md | 3 +- docs/develop_docs/physical-handoff-costs.md | 99 - .../planner-vocabulary-migration.md | 11 +- docs/develop_docs/storage-operation-costs.md | 87 - 33 files changed, 71 insertions(+), 7087 deletions(-) delete mode 100644 crates/executor/src/dag/mod.rs delete mode 100644 crates/plan-selection/src/cost/physical_handoff_cost.rs delete mode 100644 crates/plan-selection/src/cost/query_physical_lowering.rs delete mode 100644 crates/plan-selection/src/cost/storage_io.rs delete mode 100644 crates/plan-selection/tests/physical_handoff_cost.rs delete mode 100644 crates/plan-selection/tests/storage_io.rs delete mode 100644 crates/types/src/workload/resources/cache.rs delete mode 100644 crates/types/src/workload/resources/physical_handoff.rs delete mode 100644 crates/types/src/workload/resources/storage.rs delete mode 100644 docs/develop_docs/physical-handoff-costs.md delete mode 100644 docs/develop_docs/storage-operation-costs.md diff --git a/crates/executor/README.md b/crates/executor/README.md index 5e60d1589..6136bdc62 100644 --- a/crates/executor/README.md +++ b/crates/executor/README.md @@ -45,7 +45,7 @@ let run = RunContext::new( let mut output = plan.execute(&[1], run)?.remove(0); let batch = block_on(output.next()).unwrap()?; assert!(matches!(batch.to_rows()[0][0], Value::Int64(-7))); -# Ok::<(), asap_executor::dag::Error>(()) +# Ok::<(), asap_executor::Error>(()) ``` `physical_planner::compile` accepts a Planner `PhysicalASAPDAG` and typed input contracts. diff --git a/crates/executor/src/dag/mod.rs b/crates/executor/src/dag/mod.rs deleted file mode 100644 index 74835cd98..000000000 --- a/crates/executor/src/dag/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Compatibility imports. New code should use plan, runtime, operators, physical_planner and sources directly. -pub use crate::plan::{NodeId, PhysicalDAG, PhysicalOperator}; -pub use crate::runtime::batch_execution; -pub use crate::runtime::{ - Input, Limits, OutputStream, Reservation, RunContext, Scope, SharedValue, -}; -pub use crate::Error; -pub use crate::{expressions, operators, physical_planner as planner, sources as scan, values}; diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs index 52f1799b6..9b7969927 100644 --- a/crates/executor/src/lib.rs +++ b/crates/executor/src/lib.rs @@ -27,8 +27,6 @@ pub use planner_types as planner; /// columns ([`values::Batch::try_from_columns`], [`values::Batch::column`]). pub use arrow; -pub mod dag; - pub mod evaluation; mod error; diff --git a/crates/executor/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs index a0b69edcb..83d2cfcdd 100644 --- a/crates/executor/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -238,7 +238,7 @@ fn weighted_summary_build_yields_within_a_batch() { input, family, vec![0], - asap_executor::dag::operators::RowWeight::Column(1), + asap_executor::operators::RowWeight::Column(1), vec![], ) .unwrap(); diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index 8074b74d1..da482baa1 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -175,7 +175,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { schema(), family, vec![3], - asap_executor::dag::operators::RowWeight::Column(1), + asap_executor::operators::RowWeight::Column(1), vec![2], ) .unwrap(); diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index 4b9c558bb..bdf9f273c 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -1,12 +1,10 @@ //! A Hydra-grouped Count-Min `SummaryAgg` compiles through the physical //! planner and answers each group's item point counts from one shared grid. use asap_executor::{ - dag::{ - operators::{Operator, RowWeight}, - values::{Batch, SchemaRef, Value}, - Limits, RunContext, Scope, - }, + operators::{Operator, RowWeight}, physical_planner::{compile_node, CompiledPhysicalDAG, InputContract, Source}, + runtime::{Limits, RunContext, Scope}, + values::{Batch, SchemaRef, Value}, }; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::{GroupKeys, Reduction}; diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index 9ff711df3..654e802b8 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -2,12 +2,11 @@ //! group) adds each row's `RowWeight` to the row's item: the value of a //! weight column, or a constant such as 1 for a row count. use asap_executor::{ - dag::{ - operators::{Operator, RowWeight}, - values::{Batch, SchemaRef, Value}, - Limits, PhysicalDAG, RunContext, Scope, - }, + operators::{Operator, RowWeight}, physical_planner::compile_node, + plan::PhysicalDAG, + runtime::{Limits, RunContext, Scope}, + values::{Batch, SchemaRef, Value}, }; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::{GroupKeys, Reduction}; diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 970e6f15e..41f68991f 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -1,10 +1,9 @@ //! Acceptance tests use the library directly, without either backend engine. use asap_executor::{ - dag::{ - operators::{Expression, Operator, Reduction, SortKey}, - values::{Batch, SchemaRef, Value}, - Limits, PhysicalDAG, RunContext, Scope, - }, + operators::{Expression, Operator, Reduction, SortKey}, + plan::PhysicalDAG, + runtime::{Limits, RunContext, Scope}, + values::{Batch, SchemaRef, Value}, Statistic, }; use futures::{executor::block_on, StreamExt}; @@ -498,7 +497,7 @@ fn exact_state_and_family_validation() { // Planner binding rejects unknown computation instead of accepting a fallback. #[test] fn bind_physical_asap_dag_before_execution() { - use asap_executor::dag::planner::bind; + use asap_executor::physical_planner::bind; use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::{ArithmeticOpKind, ScalarValue}; use std::collections::BTreeMap; @@ -553,7 +552,7 @@ fn bind_physical_asap_dag_before_execution() { roots: vec![1], scalar_roots: vec![], }; - let sources = || -> BTreeMap> { + let sources = || -> BTreeMap> { BTreeMap::from([( 0, Box::new( @@ -562,7 +561,7 @@ fn bind_physical_asap_dag_before_execution() { vec![Batch::try_new(schema.clone(), vec![vec![Value::Float64(1.)]]).unwrap()], ) .unwrap(), - ) as asap_executor::dag::planner::Source<'static>, + ) as asap_executor::physical_planner::Source<'static>, )]) }; let native = bind(&dag, sources(), &[1]).unwrap(); @@ -612,7 +611,7 @@ fn empty_exact_count_is_an_integer_state_evaluation() { // A deployment source cannot pass a different row shape to bound expressions. #[test] fn source_batches_must_match_the_bound_schema() { - use asap_executor::dag::{self, PhysicalOperator}; + use asap_executor::{physical_planner, plan::PhysicalOperator, runtime}; use planner_types::ir::properties::ExecutionDataState; use std::{cell::Cell, collections::BTreeMap, rc::Rc}; struct WrongSource { @@ -634,9 +633,9 @@ fn source_batches_must_match_the_bound_schema() { } fn start<'a>( &'a self, - _: Vec>, + _: Vec>, _: RunContext, - ) -> Result, dag::Error> { + ) -> Result, asap_executor::Error> { self.starts.set(self.starts.get() + 1); Ok( futures::stream::once(async { Batch::try_new(schema(&[]), vec![vec![]]) }) @@ -667,14 +666,14 @@ fn source_batches_must_match_the_bound_schema() { let source = Box::new(WrongSource { schema: expected, starts: starts.clone(), - }) as dag::planner::Source<'static>; - let native = dag::planner::bind(&plan, BTreeMap::from([(0, source)]), &[0]).unwrap(); + }) as physical_planner::Source<'static>; + let native = physical_planner::bind(&plan, BTreeMap::from([(0, source)]), &[0]).unwrap(); assert_eq!(starts.get(), 0); let context = RunContext::new(query(), Limits::default()).unwrap(); let mut output = native.execute(&[0], context).unwrap().remove(0); assert!(matches!( block_on(output.next()), - Some(Err(dag::Error::AtNode { node: 0, .. })) + Some(Err(asap_executor::Error::AtNode { node: 0, .. })) )); assert_eq!(starts.get(), 1); } @@ -720,7 +719,7 @@ fn extrema_preserve_numeric_values_in_the_presence_of_nan() { // Planner wire nodes, including grouping and edge roles, are executable at either phase. #[test] fn planner_semijoin_sort_limit_contract_at_both_phases() { - use asap_executor::dag::planner::{bind, Source}; + use asap_executor::physical_planner::{bind, Source}; use planner_types::ir::properties::*; use planner_types::ir::schema::*; @@ -888,7 +887,7 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { // Planner scalar signatures, collection access and null predicates share native execution. #[test] fn planner_expressions_preserve_collection_and_nullable_types() { - use asap_executor::dag::expressions::CompiledExpression; + use asap_executor::expressions::CompiledExpression; use planner_types::ir::scalar::{CompareOpKind, ScalarValue}; let input_schema = schema(&[( @@ -1114,7 +1113,7 @@ fn assert_weighted_rate_topk(count_sketch: bool) { rates.schema(), family, vec![0], - asap_executor::dag::operators::RowWeight::Column(3), + asap_executor::operators::RowWeight::Column(3), vec![1], ) .unwrap(); diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 098c80a87..7a0454645 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -1,9 +1,10 @@ //! Scan acceptance uses the public connector contract and Planner physical DAGs. -use asap_executor::dag::{ - planner::bind_with_data_sources, - scan::{DataSources, MemorySource, RawSource}, +use asap_executor::{ + physical_planner::bind_with_data_sources, + runtime::{Limits, OutputStream, RunContext, Scope}, + sources::{DataSources, MemorySource, RawSource}, values::{Batch, SchemaRef, Value}, - Error, Limits, OutputStream, RunContext, Scope, + Error, }; use futures::{executor::block_on, stream, StreamExt}; use planner_types::ir::operator::{GroupKeys, Source}; diff --git a/crates/executor/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs index 43640b007..c4ed2bd30 100644 --- a/crates/executor/tests/weighted_topk_binding.rs +++ b/crates/executor/tests/weighted_topk_binding.rs @@ -1,10 +1,10 @@ //! Planner output binds directly to the shared runtime at a declared rate-value frontier. mod common; -use asap_executor::dag::{ +use asap_executor::{ operators::Operator, - planner::{compile, InputContract, Source}, + physical_planner::{compile, InputContract, Source}, + runtime::{Limits, RunContext, Scope}, values::{Batch, Value}, - Limits, RunContext, Scope, }; use common::{compile_physical_asap_dag, logical_candidates}; use futures::{executor::block_on, StreamExt}; diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index dd807acba..ca84877b7 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -84,9 +84,9 @@ pub fn execute_raw_rows( root: &std::rc::Rc, rows: Vec>, ) -> Vec> { - use asap_executor::dag::{ - planner::bind_with_data_sources, - scan::{DataSources, MemorySource}, + use asap_executor::{ + physical_planner::bind_with_data_sources, + sources::{DataSources, MemorySource}, }; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::NonASAPOp; diff --git a/crates/plan-selection/src/cost/analytical_cost.rs b/crates/plan-selection/src/cost/analytical_cost.rs index 9f484f2b4..addb16058 100644 --- a/crates/plan-selection/src/cost/analytical_cost.rs +++ b/crates/plan-selection/src/cost/analytical_cost.rs @@ -4,151 +4,14 @@ //! separate. Plan selection (Stage 3) turns them into one objective with //! [`crate::ResourceUnitCost`]. -use std::collections::{HashMap, HashSet}; - use asap_types::ir::schema::{SketchAlgorithm, SketchParams}; -pub use asap_types::workload::resources::{ - CacheCapacityEvidence, CacheEvidence, CacheProfile, ModeledCpu, -}; -use asap_types::workload::DataArrival; +pub use asap_types::workload::resources::ModeledCpu; use serde::{Deserialize, Serialize}; use crate::cost::physical_operator_statistics::{ - validate_comparison_scopes, ComparisonScope, EdgeStatistics, OperatorStatistics, - OperatorStatisticsProvider, PromqlEdgeStatistics, PromqlValueKind, ScanSelection, + EdgeStatistics, OperatorStatistics, PromqlEdgeStatistics, PromqlValueKind, }; -/// Version of the analytical formulas applied to evidenced physical plans. -/// -/// This identifies the estimation method, not a physical executor or runtime -/// implementation version. -pub const ANALYTICAL_COST_MODEL_VERSION: &str = "analytical-cost-v2-cache-aware"; - -#[derive(Debug, Clone, Copy)] -struct ResolvedCacheProfile { - integer_executions: Option, - cpu_execution_factor: f64, - scan_execution_factor: f64, - result_hit_ratio: f64, - buffer_hit_ratio: f64, - buffer_miss_bytes: u64, - buffer_working_set_bytes: u64, -} - -fn resolve_cache_profile( - profile: &CacheProfile, - evaluation_count: u64, - data_arrival: DataArrival, -) -> Result { - if profile.version().trim().is_empty() { - return Err(AnalyticalCostError::InvalidCacheEvidence( - "blank cache profile version", - )); - } - let CacheProfile::Evidence(evidence) = profile else { - return Ok(ResolvedCacheProfile { - integer_executions: Some(evaluation_count), - cpu_execution_factor: evaluation_count as f64, - scan_execution_factor: evaluation_count as f64, - result_hit_ratio: 0.0, - buffer_hit_ratio: 0.0, - buffer_miss_bytes: 1, - buffer_working_set_bytes: 1, - }); - }; - let declared = evidence - .distinct_evaluations - .checked_add(evidence.repeated_identical_evaluations) - .ok_or(AnalyticalCostError::Overflow)?; - if declared != evaluation_count || evidence.distinct_evaluations == 0 { - return Err(AnalyticalCostError::InvalidCacheEvidence( - "cache demand requires an initial distinct evaluation and must sum to evaluation count", - )); - } - fn miss_fraction(value: CacheCapacityEvidence) -> Result { - if value.working_set_bytes == 0 { - return Err(AnalyticalCostError::InvalidCacheEvidence( - "cache working set must be non-zero", - )); - } - Ok( - value.working_set_bytes.saturating_sub(value.capacity_bytes) as f64 - / value.working_set_bytes as f64, - ) - } - if evidence - .result_invalidation_ratio - .is_some_and(|ratio| !ratio.is_finite() || !(0.0..=1.0).contains(&ratio)) - { - return Err(AnalyticalCostError::InvalidCacheEvidence( - "result-cache invalidation ratio must be finite and in [0, 1]", - )); - } - let invalidation = - match data_arrival { - DataArrival::AtRest => { - if evidence - .result_invalidation_ratio - .is_some_and(|ratio| ratio != 0.0) - { - return Err(AnalyticalCostError::InvalidCacheEvidence( - "at-rest data cannot invalidate cached results", - )); - } - 0.0 - } - DataArrival::Ingesting { .. } => evidence.result_invalidation_ratio.ok_or( - AnalyticalCostError::InvalidCacheEvidence( - "continuous ingestion requires result-cache invalidation evidence", - ), - )?, - DataArrival::Mixed { .. } | DataArrival::Unknown => { - return Err(AnalyticalCostError::InvalidCacheEvidence( - "cache invalidation cannot be derived for mixed or unknown data arrival", - )); - } - }; - // Subtract integer byte counts before conversion: 1 - residency can - // round a nonempty uncovered working set to zero near full capacity. - let result_miss_ratio = - invalidation + miss_fraction(evidence.result_cache)? * (1.0 - invalidation); - let buffer_miss_ratio = miss_fraction(evidence.buffer_cache)?; - let result_hit_ratio = 1.0 - result_miss_ratio; - let buffer_hit_ratio = 1.0 - buffer_miss_ratio; - let cpu_execution_factor = evidence.distinct_evaluations as f64 - + evidence.repeated_identical_evaluations as f64 * result_miss_ratio; - Ok(ResolvedCacheProfile { - integer_executions: if evidence.result_cache.capacity_bytes == 0 || invalidation == 1.0 { - Some(evaluation_count) - } else if evidence.result_cache.capacity_bytes >= evidence.result_cache.working_set_bytes - && invalidation == 0.0 - { - Some(evidence.distinct_evaluations) - } else { - None - }, - cpu_execution_factor, - scan_execution_factor: cpu_execution_factor * buffer_miss_ratio, - result_hit_ratio, - buffer_hit_ratio, - buffer_miss_bytes: evidence - .buffer_cache - .working_set_bytes - .saturating_sub(evidence.buffer_cache.capacity_bytes), - buffer_working_set_bytes: evidence.buffer_cache.working_set_bytes, - }) -} - -/// Derive cache hit ratios from the shared assumptions for this workload. -pub fn cache_hit_ratios( - profile: &CacheProfile, - evaluation_count: u64, - data_arrival: DataArrival, -) -> Result<(f64, f64), AnalyticalCostError> { - let resolved = resolve_cache_profile(profile, evaluation_count, data_arrival)?; - Ok((resolved.result_hit_ratio, resolved.buffer_hit_ratio)) -} - /// An analytical estimate requires CPU work, peak memory, and scanned bytes. /// The shared resource container keeps other dimensions explicitly unknown. #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] @@ -349,446 +212,6 @@ pub enum HashJoinBuildSide { Right, } -/// One node in an already-selected physical DAG. Logical output cardinality, -/// transient edge buffering, and state retained across the horizon are -/// separate values. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct PhysicalDAGNode { - pub id: String, - pub operator: PhysicalOperator, - pub children: Vec, - /// Exact comparison-scope coverage consumed by a scan. Non-scan nodes - /// leave this empty. Reusing `ScanSelection` prevents a physical plan - /// from naming a source independently of its snapshot and predicates. - pub scan_selection: Option, - /// Maximum transient edge buffer, distinct from logical `output_bytes`. - pub output_buffer_bytes: u64, - /// State that remains live after this node finishes (zero for ordinary - /// streaming operators). - pub retained_bytes: u64, - pub execution: ExecutionMultiplicity, -} - -/// Authoritative statistics and allocation evidence for one physical node. -/// The identity is repeated so a map entry cannot be accidentally rebound to -/// a different node while an estimate is in progress. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct PhysicalNodeEvidence { - pub physical_id: String, - pub statistics: OperatorStatistics, - pub output_buffer_bytes: u64, -} - -/// A complete physical DAG together with the evidence used to cost it. -/// -/// This representation is shared by query lowering and summary-maintenance -/// lowering. Keeping the root and evidence beside the nodes prevents callers -/// from estimating a valid node list with a different entry point or snapshot. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct EvidenceBackedPhysicalDAG { - pub nodes: Vec, - pub root: String, - pub evidence: HashMap, -} - -impl OperatorStatisticsProvider for HashMap { - fn statistics(&self, node_id: &str) -> Result { - let evidence = self - .get(node_id) - .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into()))?; - if evidence.physical_id != node_id { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "evidence map key differs from embedded physical identity", - )); - } - Ok(evidence.statistics.clone()) - } -} - -impl OperatorStatisticsProvider for EvidenceBackedPhysicalDAG { - fn statistics(&self, node_id: &str) -> Result { - let evidence = self - .evidence - .get(node_id) - .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into()))?; - let node = self.nodes.iter().find(|node| node.id == node_id).ok_or( - AnalyticalCostError::InvalidPhysicalDAG("evidence has no matching physical node"), - )?; - if evidence.physical_id != node_id { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "evidence map key differs from embedded physical identity", - )); - } - if node.output_buffer_bytes != evidence.output_buffer_bytes { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "physical node buffer differs from evidence snapshot", - )); - } - Ok(evidence.statistics.clone()) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum ExecutionMultiplicity { - Once, - PerEvaluation, -} - -/// Borrowed inputs for one physical-DAG estimate. This remains available -/// independently for diagnostics; plan selection should use -/// [`estimate_physical_dag_comparison`] so scope equality is mandatory. -pub struct PhysicalDAGEstimateRequest<'a> { - pub nodes: &'a [PhysicalDAGNode], - pub root: &'a str, - pub scope: &'a ComparisonScope, - pub statistics: &'a dyn OperatorStatisticsProvider, - pub cache_profile: &'a CacheProfile, -} - -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -pub struct PhysicalDAGComparisonEstimate { - pub raw: ResourceEstimate, - pub candidate: ResourceEstimate, -} - -/// Estimate two plans only after proving that their source, snapshot, -/// predicate, event-time, recurrence, and horizon scopes are identical. -pub fn estimate_physical_dag_comparison( - raw: PhysicalDAGEstimateRequest<'_>, - candidate: PhysicalDAGEstimateRequest<'_>, -) -> Result { - validate_comparison_scopes(raw.scope, candidate.scope)?; - if raw.cache_profile != candidate.cache_profile { - return Err(AnalyticalCostError::ComparisonScopeMismatch( - "cache profile", - )); - } - Ok(PhysicalDAGComparisonEstimate { - raw: estimate_physical_dag_with_cache( - raw.nodes, - raw.root, - raw.scope, - raw.statistics, - raw.cache_profile, - )?, - candidate: estimate_physical_dag_with_cache( - candidate.nodes, - candidate.root, - candidate.scope, - candidate.statistics, - candidate.cache_profile, - )?, - }) -} - -/// Compose local operator estimates once per physical identity. CPU and disk -/// are additive; peak memory is simulated over a child-before-parent schedule -/// and releases transient child outputs after their last consumer. -pub fn estimate_physical_dag( - nodes: &[PhysicalDAGNode], - root: &str, - scope: &ComparisonScope, - statistics: &(impl OperatorStatisticsProvider + ?Sized), -) -> Result { - estimate_physical_dag_with_cache(nodes, root, scope, statistics, &CacheProfile::no_cache()) -} - -pub fn estimate_physical_dag_with_cache( - nodes: &[PhysicalDAGNode], - root: &str, - scope: &ComparisonScope, - statistics: &(impl OperatorStatisticsProvider + ?Sized), - cache_profile: &CacheProfile, -) -> Result { - let evaluation_count = scope.validate()?; - let cache = resolve_cache_profile(cache_profile, evaluation_count, scope.data_arrival)?; - let by_id: HashMap<&str, &PhysicalDAGNode> = nodes.iter().map(|n| (n.id.as_str(), n)).collect(); - if by_id.len() != nodes.len() { - return Err(AnalyticalCostError::InvalidPhysicalDAG("duplicate node id")); - } - let mut visiting = HashSet::new(); - let mut visited = HashSet::new(); - let mut order = Vec::new(); - fn visit<'a>( - id: &'a str, - nodes: &HashMap<&'a str, &'a PhysicalDAGNode>, - visiting: &mut HashSet<&'a str>, - visited: &mut HashSet<&'a str>, - order: &mut Vec<&'a str>, - ) -> Result<(), AnalyticalCostError> { - if visited.contains(id) { - return Ok(()); - } - if !visiting.insert(id) { - return Err(AnalyticalCostError::InvalidPhysicalDAG("cycle")); - } - let node = nodes - .get(id) - .ok_or(AnalyticalCostError::InvalidPhysicalDAG("missing node"))?; - for child in &node.children { - visit(child, nodes, visiting, visited, order)?; - } - visiting.remove(id); - visited.insert(id); - order.push(id); - Ok(()) - } - visit(root, &by_id, &mut visiting, &mut visited, &mut order)?; - - // Resolve each reachable node exactly once. A provider may be backed by a - // live catalog; one estimate must not mix observations from two refreshes. - let resolved_statistics: HashMap<&str, OperatorStatistics> = order - .iter() - .map(|id| statistics.statistics(id).map(|value| (*id, value))) - .collect::>()?; - - let mut consumed_sources = Vec::new(); - for id in &order { - let node = by_id[id]; - let node_statistics = &resolved_statistics[id]; - match node.operator { - PhysicalOperator::Scan => { - let coverage = node - .scan_selection - .as_ref() - .ok_or_else(|| AnalyticalCostError::MissingScanSelection(node.id.clone()))?; - if !scope.sources.contains(coverage) { - return Err(AnalyticalCostError::ScanOutsideComparisonScope( - node.id.clone(), - )); - } - if !consumed_sources.contains(&coverage) { - consumed_sources.push(coverage); - } - } - _ if node.scan_selection.is_some() => { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "only scan nodes may declare scan selection", - )); - } - _ => {} - } - validate_operator_statistics(node, node_statistics, &by_id, &resolved_statistics)?; - if node.retained_bytes > 0 && matches!(node.execution, ExecutionMultiplicity::PerEvaluation) - { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "per-evaluation node cannot retain state across the horizon", - )); - } - for child in &node.children { - let child = by_id[child.as_str()]; - if matches!(node.execution, ExecutionMultiplicity::Once) - && matches!(child.execution, ExecutionMultiplicity::PerEvaluation) - { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "build-once node cannot consume a per-evaluation child", - )); - } - if matches!(node.execution, ExecutionMultiplicity::PerEvaluation) - && matches!(child.execution, ExecutionMultiplicity::Once) - && child.retained_bytes == 0 - { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "per-evaluation node reads a non-retained build-once child", - )); - } - } - } - if scope - .sources - .iter() - .any(|expected| !consumed_sources.contains(&expected)) - { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "physical scans omit a comparison-scope source", - )); - } - - let mut remaining_consumers: HashMap<&str, usize> = HashMap::new(); - for id in &order { - for child in &by_id[id].children { - *remaining_consumers.entry(child).or_default() += 1; - } - } - let mut cpu_ops = 0.0; - let mut scan_bytes = 0_u64; - let mut live_bytes = 0_u64; - let mut peak_memory_bytes = 0_u64; - let mut live_outputs: HashMap<&str, u64> = HashMap::new(); - for id in order { - let node = by_id[id]; - let local = estimate_operator(node.operator, resolved_statistics[id].clone())?; - let (cpu_executions, scan_executions) = match node.execution { - ExecutionMultiplicity::Once => ( - 1.0, - cache.buffer_miss_bytes as f64 / cache.buffer_working_set_bytes as f64, - ), - ExecutionMultiplicity::PerEvaluation => { - (cache.cpu_execution_factor, cache.scan_execution_factor) - } - }; - cpu_ops += local.cpu_ops() * cpu_executions; - let integer_executions = match node.execution { - ExecutionMultiplicity::Once => Some(1), - ExecutionMultiplicity::PerEvaluation => cache.integer_executions, - }; - let local_scan = if let Some(executions) = integer_executions { - let total = u128::from(local.scan_bytes()) * u128::from(executions); - let denominator = u128::from(cache.buffer_working_set_bytes); - let numerator = u128::from(cache.buffer_miss_bytes); - // Split the product before scaling. Both products fit u128 because - // numerator <= denominator; retain exact bytes even above 2^53. - let adjusted = (total / denominator) * numerator - + ((total % denominator) * numerator).div_ceil(denominator); - u64::try_from(adjusted).map_err(|_| AnalyticalCostError::Overflow)? - } else { - let adjusted = local.scan_bytes() as f64 * scan_executions; - // u64::MAX rounds up to 2^64 in f64. Equality is already outside - // the integer domain; casting it would silently saturate. - if !adjusted.is_finite() || adjusted >= u64::MAX as f64 { - return Err(AnalyticalCostError::Overflow); - } - adjusted.ceil() as u64 - }; - scan_bytes = scan_bytes - .checked_add(local_scan) - .ok_or(AnalyticalCostError::Overflow)?; - peak_memory_bytes = peak_memory_bytes.max( - live_bytes - .checked_add(local.peak_memory_bytes()) - .and_then(|bytes| bytes.checked_add(node.output_buffer_bytes)) - .ok_or(AnalyticalCostError::Overflow)?, - ); - if node.retained_bytes > 0 { - live_bytes = live_bytes - .checked_add(node.retained_bytes) - .ok_or(AnalyticalCostError::Overflow)?; - peak_memory_bytes = peak_memory_bytes.max(live_bytes); - } - let output = node.output_buffer_bytes; - if remaining_consumers.get(id).copied().unwrap_or(0) > 0 || id == root { - live_bytes = live_bytes - .checked_add(output) - .ok_or(AnalyticalCostError::Overflow)?; - live_outputs.insert(id, output); - peak_memory_bytes = peak_memory_bytes.max(live_bytes); - } - for child in &node.children { - let remaining = remaining_consumers.get_mut(child.as_str()).ok_or( - AnalyticalCostError::InvalidPhysicalDAG("invalid consumer count"), - )?; - *remaining -= 1; - if *remaining == 0 { - if let Some(bytes) = live_outputs.remove(child.as_str()) { - live_bytes = live_bytes - .checked_sub(bytes) - .ok_or(AnalyticalCostError::Overflow)?; - } - } - } - } - if !cpu_ops.is_finite() { - return Err(AnalyticalCostError::Overflow); - } - Ok(ResourceEstimate::new( - cpu_ops, - peak_memory_bytes, - scan_bytes, - )) -} - -fn validate_operator_statistics( - node: &PhysicalDAGNode, - node_statistics: &OperatorStatistics, - nodes: &HashMap<&str, &PhysicalDAGNode>, - statistics: &HashMap<&str, OperatorStatistics>, -) -> Result<(), AnalyticalCostError> { - let arity = expected_input_arity(node.operator, node.children.len()); - if node_statistics.input_count() != arity.statistics_inputs { - return Err(AnalyticalCostError::InvalidOperatorStatistics { - node: node.id.clone(), - reason: "operator-statistics input count does not match physical arity", - }); - } - if node.children.len() != arity.dag_children { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "operator child count does not match physical arity", - )); - } - if !statistics_match_operator(node.operator, node_statistics) { - return Err(AnalyticalCostError::InvalidOperatorStatistics { - node: node.id.clone(), - reason: "statistics variant does not match physical operator", - }); - } - for edge in (0..node_statistics.input_count()) - .filter_map(|index| node_statistics.input(index)) - .chain(std::iter::once(node_statistics.output())) - { - if !edge.is_consistent() { - return Err(AnalyticalCostError::InvalidOperatorStatistics { - node: node.id.clone(), - reason: "edge rows and logical bytes are inconsistent", - }); - } - } - for (input_index, child_id) in node.children.iter().enumerate() { - let child = nodes - .get(child_id.as_str()) - .ok_or(AnalyticalCostError::InvalidPhysicalDAG("missing node"))?; - let child_statistics = &statistics[child.id.as_str()]; - if node_statistics.input(input_index) != Some(child_statistics.output()) { - return Err(AnalyticalCostError::ConflictingEdgeStatistics { - parent: node.id.clone(), - child: child.id.clone(), - input_index, - }); - } - } - validate_operator_semantics(node.operator, node_statistics)?; - validate_promql_child_edges(node, node_statistics, statistics)?; - Ok(()) -} - -fn validate_promql_child_edges( - node: &PhysicalDAGNode, - node_statistics: &OperatorStatistics, - statistics: &HashMap<&str, OperatorStatistics>, -) -> Result<(), AnalyticalCostError> { - let output = node_statistics.promql_output(); - let child_has_promql = node - .children - .iter() - .any(|child_id| statistics[child_id.as_str()].promql_output().is_some()); - if output.is_none() && child_has_promql { - return Err(AnalyticalCostError::InvalidOperatorStatistics { - node: node.id.clone(), - reason: "operator drops child PromQL edge statistics", - }); - } - let Some(output) = output else { return Ok(()) }; - validate_promql_edge(output)?; - for (index, child_id) in node.children.iter().enumerate() { - let input = - node_statistics - .promql_input(index) - .ok_or(AnalyticalCostError::MissingOrStale( - "promql_input_edge_statistics", - ))?; - let child = statistics[child_id.as_str()].promql_output().ok_or( - AnalyticalCostError::MissingOrStale("promql_child_output_statistics"), - )?; - validate_promql_edge(input)?; - if input != child { - return Err(AnalyticalCostError::InvalidOperatorStatistics { - node: node.id.clone(), - reason: "PromQL parent input does not match child output", - }); - } - } - Ok(()) -} - fn validate_promql_edge(edge: PromqlEdgeStatistics) -> Result<(), AnalyticalCostError> { if edge.evaluation_steps == 0 { return Err(AnalyticalCostError::MissingOrZero("evaluation_steps")); @@ -873,72 +296,6 @@ fn statistics_match_operator(operator: PhysicalOperator, statistics: &OperatorSt } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct OperatorInputArity { - /// Number of logical input-edge records required in `OperatorStatistics`. - statistics_inputs: usize, - /// Number of upstream physical nodes required in the DAG. - dag_children: usize, -} - -/// Declare both notions of operator input explicitly. A scan has one external -/// source-input statistics record but no upstream DAG node. Every other -/// operator's statistics inputs correspond one-to-one with its DAG children. -/// -/// Keep this match exhaustive: adding a physical operator must also define its -/// statistics and DAG arity instead of silently inheriting unary behavior. -fn expected_input_arity( - operator: PhysicalOperator, - variadic_child_count: usize, -) -> OperatorInputArity { - let unary = OperatorInputArity { - statistics_inputs: 1, - dag_children: 1, - }; - match operator { - PhysicalOperator::Scan => OperatorInputArity { - statistics_inputs: 1, - dag_children: 0, - }, - // The row input, then one input per scalar the expressions read. - PhysicalOperator::Filter { .. } | PhysicalOperator::Project { .. } => OperatorInputArity { - statistics_inputs: variadic_child_count.max(1), - dag_children: variadic_child_count.max(1), - }, - PhysicalOperator::HashAggregate { .. } - | PhysicalOperator::InMemoryComparisonSort { .. } - | PhysicalOperator::TopK { .. } - | PhysicalOperator::HashDeduplicate { .. } - | PhysicalOperator::InMemoryAnalyticWindow { .. } - | PhysicalOperator::Limit { .. } - | PhysicalOperator::PassThrough - | PhysicalOperator::TimeShift - | PhysicalOperator::TimeRange - | PhysicalOperator::PromqlRange { .. } - | PhysicalOperator::PromqlSubquery { .. } - | PhysicalOperator::PromqlRelabel { .. } - | PhysicalOperator::PromqlSeriesSample { .. } - | PhysicalOperator::PromqlScalarToVector - | PhysicalOperator::PromqlVectorToScalar - | PhysicalOperator::PromqlPerSeries { .. } - | PhysicalOperator::PromqlPresence { .. } => unary, - PhysicalOperator::HashJoin { .. } - | PhysicalOperator::PromqlBinary { .. } - | PhysicalOperator::PromqlInfoEnrich { .. } => OperatorInputArity { - statistics_inputs: 2, - dag_children: 2, - }, - PhysicalOperator::PromqlScalarLeaf => OperatorInputArity { - statistics_inputs: 0, - dag_children: 0, - }, - PhysicalOperator::Concat => OperatorInputArity { - statistics_inputs: variadic_child_count, - dag_children: variadic_child_count, - }, - } -} - fn checked_cpu_product(rows: u64, operations_per_row: u64) -> Result { Ok(rows .checked_mul(operations_per_row) @@ -2115,8 +1472,6 @@ fn validate_promql_series_sample( #[derive(Debug, Clone, PartialEq, thiserror::Error)] pub enum AnalyticalCostError { - #[error("data arrival {0:?} is unsupported by the selected analytical adapter")] - UnsupportedDataArrival(DataArrival), #[error("ingestion rate must be finite and non-negative, got {0}")] InvalidIngestionRate(f64), #[error("bootstrap row and byte evidence must either both be zero or both be non-zero")] @@ -2129,14 +1484,10 @@ pub enum AnalyticalCostError { MissingOrStale(&'static str), #[error("query recurrence cannot be resolved over the planning horizon")] InvalidRecurrence, - #[error("query has no evaluations in the planning horizon")] - NoEvaluationsInHorizon, #[error("calibration {0} must be finite and non-negative, got {1}")] InvalidCalibration(&'static str, f64), #[error("at least one calibration coefficient must be positive")] ZeroCalibration, - #[error("invalid or incomplete cache evidence: {0}")] - InvalidCacheEvidence(&'static str), #[error("algorithm {0:?} does not match parameters {1:?}")] ParameterMismatch(SketchAlgorithm, SketchParams), #[error("{0} needs a value-range/bin-count model before it can be estimated")] @@ -2145,34 +1496,12 @@ pub enum AnalyticalCostError { Overflow, #[error("candidate has no supported exact or sketch state")] UnsupportedCandidate, - #[error("query operator has no physical implementation in the analytical model")] - UnsupportedQueryOperator, #[error("inconsistent operator statistics: {0}")] InconsistentOperatorStatistics(&'static str), #[error("summary operation {0} has no cost formula")] UnsupportedSummaryOperation(&'static str), - #[error("required comparison-scope field {0} is missing")] - MissingComparisonScope(&'static str), - #[error("scan node {0} does not declare scan selection")] - MissingScanSelection(String), - #[error("scan node {0} reads scan selection outside the comparison scope")] - ScanOutsideComparisonScope(String), - #[error("raw and candidate comparison scopes differ in {0}")] - ComparisonScopeMismatch(&'static str), - #[error("operator statistics are unavailable for physical node {0}")] - MissingOperatorStatistics(String), #[error("invalid operator statistics for {node}: {reason}")] InvalidOperatorStatistics { node: String, reason: &'static str }, - #[error( - "operator statistics conflict: parent {parent} input {input_index} does not match child {child} output" - )] - ConflictingEdgeStatistics { - parent: String, - child: String, - input_index: usize, - }, - #[error("invalid physical DAG: {0}")] - InvalidPhysicalDAG(&'static str), /// Bad input: plan selection's measured table has two entries for one /// operator, phase and family (named in the message). #[error("measured_operation_costs has two entries for {0}")] @@ -2200,9 +1529,9 @@ fn checked_bytes(parts: &[u64]) -> Result { mod tests { use super::*; use crate::cost::physical_operator_statistics::{ - validate_comparison_scopes, BinaryEdgeStatistics, ComparisonScope, EdgeStatistics, - OperatorStatistics, PartitionStatistics, PromqlBinaryEdgeStatistics, PromqlEdgeStatistics, - PromqlUnaryEdgeStatistics, PromqlValueKind, ScanSelection, UnaryEdgeStatistics, + BinaryEdgeStatistics, EdgeStatistics, OperatorStatistics, PartitionStatistics, + PromqlBinaryEdgeStatistics, PromqlEdgeStatistics, PromqlUnaryEdgeStatistics, + PromqlValueKind, UnaryEdgeStatistics, }; /// Analytical estimates reuse the shared dimensions while preserving exact @@ -2353,44 +1682,6 @@ mod tests { } } - #[test] - fn operator_arity_distinguishes_source_statistics_from_dag_children() { - assert_eq!( - expected_input_arity(PhysicalOperator::Scan, 0), - OperatorInputArity { - statistics_inputs: 1, - dag_children: 0, - } - ); - assert_eq!( - expected_input_arity(filter_operator(), 1), - OperatorInputArity { - statistics_inputs: 1, - dag_children: 1, - } - ); - assert_eq!( - expected_input_arity( - PhysicalOperator::HashJoin { - build_side: HashJoinBuildSide::Left, - equality_key_count: 1, - }, - 2, - ), - OperatorInputArity { - statistics_inputs: 2, - dag_children: 2, - } - ); - assert_eq!( - expected_input_arity(PhysicalOperator::Concat, 3), - OperatorInputArity { - statistics_inputs: 3, - dag_children: 3, - } - ); - } - #[test] fn promql_binary_operation_controls_cardinality_bounds() { let operator = |operation| PhysicalOperator::PromqlBinary { @@ -2469,66 +1760,6 @@ mod tests { assert_eq!(estimate.peak_memory_bytes(), 8); } - #[test] - fn physical_dag_cannot_drop_promql_edge_evidence() { - let edge = EdgeStatistics { - rows: 10, - bytes: 80, - }; - let promql = promql_edge(1, 10, PromqlValueKind::Vector); - let nodes = vec![ - PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(comparison_scope().sources[0].clone()), - output_buffer_bytes: 8, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "filter".into(), - operator: filter_operator(), - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 8, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - ]; - let provided = HashMap::from([ - ( - "scan".into(), - OperatorStatistics::Scan { - edges: UnaryEdgeStatistics { - input: edge, - output: edge, - promql: Some(PromqlUnaryEdgeStatistics { - input: promql, - output: promql, - }), - }, - source_read_bytes: 80, - }, - ), - ( - "filter".into(), - OperatorStatistics::Filter { - edges: unary_edges(edge, edge), - scalar_inputs: vec![], - }, - ), - ]); - - assert!(matches!( - estimate_physical_dag(&nodes, "filter", &comparison_scope(), &provided), - Err(AnalyticalCostError::InvalidOperatorStatistics { - reason: "operator drops child PromQL edge statistics", - .. - }) - )); - } - #[test] fn generic_hash_aggregate_cannot_masquerade_as_promql_aggregation() { let input = EdgeStatistics { @@ -2816,194 +2047,6 @@ mod tests { assert_eq!(offset_limit.cpu_ops(), 900_010.0); } - #[test] - fn physical_dag_counts_shared_scan_once_and_uses_live_memory() { - let coverage = comparison_scope().sources[0].clone(); - let nodes = vec![ - PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(coverage), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "left".into(), - operator: filter_operator(), - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 4, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "right".into(), - operator: filter_operator(), - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 4, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "root".into(), - operator: PhysicalOperator::Concat, - children: vec!["left".into(), "right".into()], - scan_selection: None, - output_buffer_bytes: 8, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - ]; - let scan_edge = EdgeStatistics { - rows: 100, - bytes: 1_000, - }; - let branch_edge = EdgeStatistics { - rows: 40, - bytes: 400, - }; - let provided = HashMap::from([ - ( - "scan".into(), - OperatorStatistics::Scan { - edges: unary_edges(scan_edge, scan_edge), - source_read_bytes: 1_000, - }, - ), - ( - "left".into(), - OperatorStatistics::Filter { - edges: unary_edges(scan_edge, branch_edge), - scalar_inputs: vec![], - }, - ), - ( - "right".into(), - OperatorStatistics::Filter { - edges: unary_edges(scan_edge, branch_edge), - scalar_inputs: vec![], - }, - ), - ( - "root".into(), - OperatorStatistics::Concat { - inputs: vec![branch_edge, branch_edge], - output: EdgeStatistics { - rows: 80, - bytes: 800, - }, - promql: None, - }, - ), - ]); - let mut scope = comparison_scope(); - scope.horizon.0 = 20_000; - let estimate = estimate_physical_dag(&nodes, "root", &scope, &provided).unwrap(); - assert_eq!(estimate.cpu_ops(), 760.0); - assert_eq!(estimate.scan_bytes(), 2_000); - // This is neither the sum of every node's memory nor just the largest - // node: it is the maximum state simultaneously live at the fan-out. - assert_eq!(estimate.peak_memory_bytes(), 28); - } - - #[test] - fn physical_dag_separates_build_once_from_per_evaluation_work() { - let coverage = comparison_scope().sources[0].clone(); - let nodes = vec![ - PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(coverage), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::Once, - }, - PhysicalDAGNode { - id: "state".into(), - operator: aggregate_operator(), - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 16, - retained_bytes: 32, - execution: ExecutionMultiplicity::Once, - }, - PhysicalDAGNode { - id: "read".into(), - operator: PhysicalOperator::Limit { - limit: 1, - offset: 0, - }, - children: vec!["state".into()], - scan_selection: None, - output_buffer_bytes: 16, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - ]; - let scan_edge = EdgeStatistics { - rows: 100, - bytes: 1_000, - }; - let state_edge = EdgeStatistics { rows: 1, bytes: 16 }; - let provided = HashMap::from([ - ( - "scan".into(), - OperatorStatistics::Scan { - edges: unary_edges(scan_edge, scan_edge), - source_read_bytes: 1_000, - }, - ), - ( - "state".into(), - OperatorStatistics::HashAggregate { - edges: unary_edges(scan_edge, state_edge), - group_count: 1, - key_bytes: 8, - accumulator_bytes_per_group: 8, - }, - ), - ( - "read".into(), - OperatorStatistics::Limit { - edges: unary_edges(state_edge, state_edge), - }, - ), - ]); - let mut scope = comparison_scope(); - scope.horizon.0 = 100_000; - let estimate = estimate_physical_dag(&nodes, "read", &scope, &provided).unwrap(); - assert_eq!(estimate.cpu_ops(), 310.0); - assert_eq!(estimate.scan_bytes(), 1_000); - } - - fn comparison_scope() -> ComparisonScope { - use asap_types::ir::operator::operator_properties::Source; - use asap_types::workload::{ - DurationMs, QueryRecurrence, RepeatedDemand, RepetitionInterval, TimestampMs, - }; - - ComparisonScope { - data_arrival: DataArrival::AtRest, - planning_time: TimestampMs(1_000), - horizon: DurationMs(60_000), - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - RepetitionInterval(10_000), - )), - sources: vec![ScanSelection { - source: Source::Table { - table_ref: "metrics".into(), - }, - source_snapshot_id: "catalog-version-42".into(), - predicates: vec![], - info_matchers: vec![], - }], - } - } - fn unary_edges(input: EdgeStatistics, output: EdgeStatistics) -> UnaryEdgeStatistics { UnaryEdgeStatistics { input, @@ -3012,443 +2055,6 @@ mod tests { } } - #[test] - fn comparison_rejects_different_snapshot_predicate_time_or_horizon() { - use asap_types::ir::{Predicate, ScalarExpr}; - use asap_types::workload::DurationMs; - - let raw = comparison_scope(); - assert_eq!(validate_comparison_scopes(&raw, &raw).unwrap(), 6); - - let mut candidate = raw.clone(); - candidate.sources[0].source_snapshot_id = "catalog-version-43".into(); - assert_eq!( - validate_comparison_scopes(&raw, &candidate), - Err(AnalyticalCostError::ComparisonScopeMismatch("sources")) - ); - - candidate = raw.clone(); - candidate.sources[0] - .predicates - .push(Predicate(ScalarExpr::literal_f64(1.0))); - assert_eq!( - validate_comparison_scopes(&raw, &candidate), - Err(AnalyticalCostError::ComparisonScopeMismatch("sources")) - ); - - candidate = raw.clone(); - candidate.horizon = DurationMs(120_000); - assert_eq!( - validate_comparison_scopes(&raw, &candidate), - Err(AnalyticalCostError::ComparisonScopeMismatch("horizon")) - ); - } - - #[test] - fn physical_dag_fails_closed_on_missing_or_conflicting_edge_statistics() { - use std::collections::HashMap; - - let coverage = comparison_scope().sources[0].clone(); - let nodes = vec![ - PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(coverage), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "filter".into(), - operator: filter_operator(), - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 4, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - ]; - let scope = comparison_scope(); - let mut provided = HashMap::from([( - "scan".to_string(), - OperatorStatistics::Scan { - edges: unary_edges( - EdgeStatistics { - rows: 100, - bytes: 1_000, - }, - EdgeStatistics { - rows: 100, - bytes: 1_000, - }, - ), - source_read_bytes: 1_000, - }, - )]); - - assert_eq!( - estimate_physical_dag(&nodes, "filter", &scope, &provided), - Err(AnalyticalCostError::MissingOperatorStatistics( - "filter".into() - )) - ); - - provided.insert( - "filter".into(), - OperatorStatistics::Filter { - edges: unary_edges( - EdgeStatistics { - rows: 99, - bytes: 990, - }, - EdgeStatistics { - rows: 40, - bytes: 400, - }, - ), - scalar_inputs: vec![], - }, - ); - assert_eq!( - estimate_physical_dag(&nodes, "filter", &scope, &provided), - Err(AnalyticalCostError::ConflictingEdgeStatistics { - parent: "filter".into(), - child: "scan".into(), - input_index: 0, - }) - ); - } - - #[test] - fn provider_statistics_drive_a_consistent_physical_dag_estimate() { - use std::collections::HashMap; - - let coverage = comparison_scope().sources[0].clone(); - let nodes = vec![ - PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(coverage), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "filter".into(), - operator: filter_operator(), - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 4, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - ]; - let provided = HashMap::from([ - ( - "scan".to_string(), - OperatorStatistics::Scan { - edges: unary_edges( - EdgeStatistics { - rows: 100, - bytes: 1_000, - }, - EdgeStatistics { - rows: 100, - bytes: 1_000, - }, - ), - source_read_bytes: 1_000, - }, - ), - ( - "filter".to_string(), - OperatorStatistics::Filter { - edges: unary_edges( - EdgeStatistics { - rows: 100, - bytes: 1_000, - }, - EdgeStatistics { - rows: 40, - bytes: 400, - }, - ), - scalar_inputs: vec![], - }, - ), - ]); - - let estimate = - estimate_physical_dag(&nodes, "filter", &comparison_scope(), &provided).unwrap(); - assert_eq!(estimate.cpu_ops(), 1_200.0); - assert_eq!(estimate.scan_bytes(), 6_000); - } - - #[test] - fn physical_dag_accepts_an_empty_operator_output() { - let coverage = comparison_scope().sources[0].clone(); - let nodes = vec![ - PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(coverage), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "filter".into(), - operator: filter_operator(), - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - ]; - let input = EdgeStatistics { - rows: 100, - bytes: 1_000, - }; - let provided = HashMap::from([ - ( - "scan".into(), - OperatorStatistics::Scan { - edges: unary_edges(input, input), - source_read_bytes: 1_000, - }, - ), - ( - "filter".into(), - OperatorStatistics::Filter { - edges: unary_edges(input, EdgeStatistics { rows: 0, bytes: 0 }), - scalar_inputs: vec![], - }, - ), - ]); - - let estimate = - estimate_physical_dag(&nodes, "filter", &comparison_scope(), &provided).unwrap(); - assert_eq!(estimate.cpu_ops(), 1_200.0); - assert_eq!(estimate.peak_memory_bytes(), 20); - assert_eq!(estimate.scan_bytes(), 6_000); - } - - #[test] - fn physical_dag_rejects_a_scan_not_covered_by_its_scope() { - let nodes = vec![PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(ScanSelection { - source: asap_types::ir::operator::operator_properties::Source::Table { - table_ref: "other_metrics".into(), - }, - source_snapshot_id: "catalog-version-42".into(), - predicates: vec![], - info_matchers: vec![], - }), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }]; - let edge = EdgeStatistics { - rows: 100, - bytes: 1_000, - }; - let provided = HashMap::from([( - "scan".into(), - OperatorStatistics::Scan { - edges: unary_edges(edge, edge), - source_read_bytes: 250, - }, - )]); - - assert_eq!( - estimate_physical_dag(&nodes, "scan", &comparison_scope(), &provided), - Err(AnalyticalCostError::ScanOutsideComparisonScope( - "scan".into() - )) - ); - } - - #[test] - fn physical_dag_rejects_a_scan_without_explicit_coverage() { - let nodes = vec![PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: None, - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }]; - let edge = EdgeStatistics { - rows: 100, - bytes: 1_000, - }; - let provided = HashMap::from([( - "scan".into(), - OperatorStatistics::Scan { - edges: unary_edges(edge, edge), - source_read_bytes: 250, - }, - )]); - - assert_eq!( - estimate_physical_dag(&nodes, "scan", &comparison_scope(), &provided), - Err(AnalyticalCostError::MissingScanSelection("scan".into())) - ); - } - - #[test] - fn physical_dag_rejects_an_unconsumed_scope_source() { - let mut scope = comparison_scope(); - let coverage = scope.sources[0].clone(); - scope.sources.push(ScanSelection { - source: asap_types::ir::operator::operator_properties::Source::Table { - table_ref: "auxiliary".into(), - }, - source_snapshot_id: "catalog-version-42".into(), - predicates: vec![], - info_matchers: vec![], - }); - let nodes = vec![PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(coverage), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }]; - let edge = EdgeStatistics { - rows: 100, - bytes: 1_000, - }; - let provided = HashMap::from([( - "scan".into(), - OperatorStatistics::Scan { - edges: unary_edges(edge, edge), - source_read_bytes: 250, - }, - )]); - - assert_eq!( - estimate_physical_dag(&nodes, "scan", &scope, &provided), - Err(AnalyticalCostError::InvalidPhysicalDAG( - "physical scans omit a comparison-scope source" - )) - ); - } - - #[test] - fn build_once_parent_cannot_consume_a_per_evaluation_child() { - let coverage = comparison_scope().sources[0].clone(); - let nodes = vec![ - PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(coverage), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "aggregate".into(), - operator: aggregate_operator(), - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 16, - retained_bytes: 32, - execution: ExecutionMultiplicity::Once, - }, - ]; - let input = EdgeStatistics { - rows: 100, - bytes: 1_000, - }; - let output = EdgeStatistics { rows: 1, bytes: 16 }; - let provided = HashMap::from([ - ( - "scan".into(), - OperatorStatistics::Scan { - edges: unary_edges(input, input), - source_read_bytes: 250, - }, - ), - ( - "aggregate".into(), - OperatorStatistics::HashAggregate { - edges: unary_edges(input, output), - group_count: 1, - key_bytes: 8, - accumulator_bytes_per_group: 8, - }, - ), - ]); - - assert_eq!( - estimate_physical_dag(&nodes, "aggregate", &comparison_scope(), &provided), - Err(AnalyticalCostError::InvalidPhysicalDAG( - "build-once node cannot consume a per-evaluation child" - )) - ); - } - - #[test] - fn scoped_comparison_rejects_different_source_snapshots() { - let coverage = comparison_scope().sources[0].clone(); - let nodes = vec![PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(coverage), - output_buffer_bytes: 10, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }]; - let edge = EdgeStatistics { - rows: 100, - bytes: 1_000, - }; - let provided = HashMap::from([( - "scan".into(), - OperatorStatistics::Scan { - edges: unary_edges(edge, edge), - source_read_bytes: 250, - }, - )]); - let raw_scope = comparison_scope(); - let mut candidate_scope = raw_scope.clone(); - let no_cache = CacheProfile::no_cache(); - candidate_scope.sources[0].source_snapshot_id = "catalog-version-43".into(); - - assert_eq!( - estimate_physical_dag_comparison( - PhysicalDAGEstimateRequest { - nodes: &nodes, - root: "scan", - scope: &raw_scope, - statistics: &provided, - cache_profile: &no_cache, - }, - PhysicalDAGEstimateRequest { - nodes: &nodes, - root: "scan", - scope: &candidate_scope, - statistics: &provided, - cache_profile: &no_cache, - }, - ), - Err(AnalyticalCostError::ComparisonScopeMismatch("sources")) - ); - } - #[test] fn scan_uses_physical_source_bytes_not_decoded_logical_bytes() { let logical = EdgeStatistics { @@ -3629,375 +2235,4 @@ mod tests { .is_err() ); } - - fn cache_test_scan() -> (Vec, HashMap) { - let edge = EdgeStatistics { - rows: 100, - bytes: 1_000, - }; - ( - vec![PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(comparison_scope().sources[0].clone()), - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }], - HashMap::from([( - "scan".into(), - OperatorStatistics::Scan { - edges: unary_edges(edge, edge), - source_read_bytes: 1_000, - }, - )]), - ) - } - - fn cache_profile(result_capacity: u64, buffer_capacity: u64) -> CacheProfile { - CacheProfile::Evidence(CacheEvidence { - version: "cache-evidence-7".into(), - distinct_evaluations: 2, - repeated_identical_evaluations: 4, - result_cache: CacheCapacityEvidence { - working_set_bytes: 100, - capacity_bytes: result_capacity, - }, - buffer_cache: CacheCapacityEvidence { - working_set_bytes: 100, - capacity_bytes: buffer_capacity, - }, - result_invalidation_ratio: None, - }) - } - - // Legacy mapping imports are aliases of the shared schema, not a second type. - #[test] - fn shared_cache_types_work_through_legacy_mapping_imports() { - let central: asap_types::workload::resources::CacheProfile = cache_profile(100, 50); - let legacy: CacheProfile = - serde_json::from_value(serde_json::to_value(¢ral).unwrap()).unwrap(); - fn accepts_shared(_: &asap_types::workload::resources::CacheProfile) {} - accepts_shared(&legacy); - assert_eq!(central, legacy); - assert_eq!( - cache_hit_ratios(¢ral, 6, DataArrival::AtRest).unwrap(), - (1.0, 0.5) - ); - } - - #[test] - fn no_cache_reproduces_legacy_estimate_and_cache_effects_are_distinct() { - let (nodes, evidence) = cache_test_scan(); - let scope = comparison_scope(); - let legacy = estimate_physical_dag(&nodes, "scan", &scope, &evidence).unwrap(); - let explicit_no_cache = estimate_physical_dag_with_cache( - &nodes, - "scan", - &scope, - &evidence, - &CacheProfile::no_cache(), - ) - .unwrap(); - assert_eq!(legacy, explicit_no_cache); - assert_eq!(legacy.cpu_ops(), 600.0); - assert_eq!(legacy.scan_bytes(), 6_000); - - let result_only = estimate_physical_dag_with_cache( - &nodes, - "scan", - &scope, - &evidence, - &cache_profile(100, 0), - ) - .unwrap(); - assert_eq!(result_only.cpu_ops(), 200.0); - assert_eq!(result_only.scan_bytes(), 2_000); - - let result_and_buffer = estimate_physical_dag_with_cache( - &nodes, - "scan", - &scope, - &evidence, - &cache_profile(100, 50), - ) - .unwrap(); - assert_eq!(result_and_buffer.cpu_ops(), result_only.cpu_ops()); - assert_eq!(result_and_buffer.scan_bytes(), 1_000); - } - - #[test] - fn cache_hits_are_monotone_and_reduce_summary_benefit() { - let (nodes, evidence) = cache_test_scan(); - let scope = comparison_scope(); - let low = estimate_physical_dag_with_cache( - &nodes, - "scan", - &scope, - &evidence, - &cache_profile(0, 0), - ) - .unwrap(); - let high = estimate_physical_dag_with_cache( - &nodes, - "scan", - &scope, - &evidence, - &cache_profile(100, 0), - ) - .unwrap(); - assert!(high.cpu_ops() <= low.cpu_ops()); - assert!(high.scan_bytes() <= low.scan_bytes()); - - // A build-once summary has fixed work; as exact result-cache hits - // rise, the raw-minus-summary advantage cannot increase. - let mut summary_nodes = nodes.clone(); - summary_nodes[0].execution = ExecutionMultiplicity::Once; - let summary = estimate_physical_dag_with_cache( - &summary_nodes, - "scan", - &scope, - &evidence, - &cache_profile(0, 0), - ) - .unwrap(); - assert!(high.cpu_ops() - summary.cpu_ops() <= low.cpu_ops() - summary.cpu_ops()); - } - - #[test] - fn cache_evidence_fails_closed_for_bad_demand_and_streaming_invalidation() { - let mut profile = cache_profile(100, 100); - let CacheProfile::Evidence(evidence) = &mut profile else { - unreachable!() - }; - evidence.distinct_evaluations = 1; - assert!(matches!( - cache_hit_ratios(&profile, 6, DataArrival::AtRest), - Err(AnalyticalCostError::InvalidCacheEvidence(_)) - )); - - let profile = cache_profile(100, 100); - assert!(matches!( - cache_hit_ratios( - &profile, - 6, - DataArrival::Ingesting { - rows_per_second: 1_000.0, - } - ), - Err(AnalyticalCostError::InvalidCacheEvidence(_)) - )); - } - - // Cache-enabled I/O must keep exact integers and reject unrepresentable totals. - #[test] - fn cache_io_preserves_large_integer_bytes_and_detects_overflow() { - let (mut nodes, mut evidence) = cache_test_scan(); - let mut scope = comparison_scope(); - scope.horizon.0 = 20_000; - let mut cache = cache_profile(0, 0); - let CacheProfile::Evidence(inputs) = &mut cache else { - unreachable!() - }; - inputs.distinct_evaluations = 1; - inputs.repeated_identical_evaluations = 1; - let OperatorStatistics::Scan { - source_read_bytes, .. - } = evidence.get_mut("scan").unwrap() - else { - unreachable!() - }; - *source_read_bytes = 1_u64 << 63; - assert_eq!( - estimate_physical_dag_with_cache(&nodes, "scan", &scope, &evidence, &cache), - Err(AnalyticalCostError::Overflow) - ); - let OperatorStatistics::Scan { - source_read_bytes, .. - } = evidence.get_mut("scan").unwrap() - else { - unreachable!() - }; - *source_read_bytes = (1_u64 << 53) + 1; - nodes[0].execution = ExecutionMultiplicity::Once; - assert_eq!( - estimate_physical_dag_with_cache(&nodes, "scan", &scope, &evidence, &cache) - .unwrap() - .scan_bytes(), - (1_u64 << 53) + 1 - ); - } - - // Repeated demand needs an initial evaluation; contradictory AtRest evidence is invalid. - #[test] - fn cache_evidence_rejects_impossible_demand_and_invalid_at_rest_invalidation() { - let mut cache = cache_profile(100, 0); - let CacheProfile::Evidence(inputs) = &mut cache else { - unreachable!() - }; - inputs.distinct_evaluations = 0; - inputs.repeated_identical_evaluations = 6; - assert!(cache_hit_ratios(&cache, 6, DataArrival::AtRest).is_err()); - for ratio in [f64::NAN, f64::INFINITY, -1.0, 0.5] { - let mut cache = cache_profile(100, 0); - let CacheProfile::Evidence(inputs) = &mut cache else { - unreachable!() - }; - inputs.result_invalidation_ratio = Some(ratio); - assert!(cache_hit_ratios(&cache, 6, DataArrival::AtRest).is_err()); - } - } - - // Buffer residency affects source reads equally for raw scans and summary builds. - #[test] - fn buffer_cache_applies_to_build_once_scans_without_reducing_cpu() { - let (mut nodes, evidence) = cache_test_scan(); - nodes[0].execution = ExecutionMultiplicity::Once; - let scope = comparison_scope(); - let estimate = estimate_physical_dag_with_cache( - &nodes, - "scan", - &scope, - &evidence, - &cache_profile(100, 50), - ) - .unwrap(); - assert_eq!(estimate.cpu_ops(), 100.0); - assert_eq!(estimate.scan_bytes(), 500); - } - - // A sub-ULP uncovered fraction must not become a fabricated full cache hit. - #[test] - fn partial_cache_preserves_tiny_buffer_misses() { - let (mut nodes, mut evidence) = cache_test_scan(); - nodes[0].execution = ExecutionMultiplicity::Once; - let working_set = 1_u64 << 63; - let OperatorStatistics::Scan { - source_read_bytes, .. - } = evidence.get_mut("scan").unwrap() - else { - unreachable!() - }; - *source_read_bytes = working_set; - let mut profile = cache_profile(0, 0); - let CacheProfile::Evidence(inputs) = &mut profile else { - unreachable!() - }; - inputs.buffer_cache = CacheCapacityEvidence { - working_set_bytes: working_set, - capacity_bytes: working_set - 1, - }; - let estimate = estimate_physical_dag_with_cache( - &nodes, - "scan", - &comparison_scope(), - &evidence, - &profile, - ) - .unwrap(); - assert_eq!(estimate.scan_bytes(), 1); - } - - // A result-cache miss remains chargeable even when the displayed hit rounds to one. - #[test] - fn partial_cache_preserves_tiny_result_misses() { - let working_set = 1_u64 << 63; - let mut profile = cache_profile(0, 0); - let CacheProfile::Evidence(inputs) = &mut profile else { - unreachable!() - }; - inputs.distinct_evaluations = 1; - inputs.repeated_identical_evaluations = working_set; - inputs.result_cache = CacheCapacityEvidence { - working_set_bytes: working_set, - capacity_bytes: working_set - 1, - }; - assert_eq!( - resolve_cache_profile(&profile, working_set + 1, DataArrival::AtRest) - .unwrap() - .cpu_execution_factor, - 2.0 - ); - } - - // Byte accounting remains exact above f64 integer precision after cache scaling. - #[test] - fn partial_buffer_cache_rounds_integer_bytes_without_precision_loss() { - let (mut nodes, mut evidence) = cache_test_scan(); - nodes[0].execution = ExecutionMultiplicity::Once; - let OperatorStatistics::Scan { - source_read_bytes, .. - } = evidence.get_mut("scan").unwrap() - else { - unreachable!() - }; - *source_read_bytes = (1_u64 << 54) + 2; - let estimate = estimate_physical_dag_with_cache( - &nodes, - "scan", - &comparison_scope(), - &evidence, - &cache_profile(0, 50), - ) - .unwrap(); - assert_eq!(estimate.scan_bytes(), (1_u64 << 53) + 1); - } - - // A one-third miss fraction must not round five ordinary bytes up to six. - #[test] - fn partial_buffer_cache_does_not_round_an_exact_small_total_up() { - let (mut nodes, mut evidence) = cache_test_scan(); - nodes[0].execution = ExecutionMultiplicity::Once; - let OperatorStatistics::Scan { - source_read_bytes, .. - } = evidence.get_mut("scan").unwrap() - else { - unreachable!() - }; - *source_read_bytes = 15; - let mut profile = cache_profile(0, 0); - let CacheProfile::Evidence(inputs) = &mut profile else { - unreachable!() - }; - inputs.buffer_cache = CacheCapacityEvidence { - working_set_bytes: 3, - capacity_bytes: 2, - }; - assert_eq!( - estimate_physical_dag_with_cache( - &nodes, - "scan", - &comparison_scope(), - &evidence, - &profile - ) - .unwrap() - .scan_bytes(), - 5 - ); - } - - // Declared invalidations bound otherwise-resident results under ingestion. - #[test] - fn streaming_invalidation_bounds_result_hits_and_changes_physical_work() { - let (nodes, evidence) = cache_test_scan(); - let mut scope = comparison_scope(); - scope.data_arrival = DataArrival::Ingesting { - rows_per_second: 1_000.0, - }; - let mut profile = cache_profile(100, 50); - let CacheProfile::Evidence(inputs) = &mut profile else { - unreachable!() - }; - inputs.result_invalidation_ratio = Some(0.5); - let estimate = - estimate_physical_dag_with_cache(&nodes, "scan", &scope, &evidence, &profile).unwrap(); - assert_eq!( - cache_hit_ratios(&profile, 6, scope.data_arrival).unwrap(), - (0.5, 0.5) - ); - assert_eq!(estimate.cpu_ops(), 400.0); - assert_eq!(estimate.scan_bytes(), 2_000); - } } diff --git a/crates/plan-selection/src/cost/mod.rs b/crates/plan-selection/src/cost/mod.rs index 96877f873..2b80231d7 100644 --- a/crates/plan-selection/src/cost/mod.rs +++ b/crates/plan-selection/src/cost/mod.rs @@ -2,12 +2,5 @@ //! //! - [`analytical_cost`] and [`physical_operator_statistics`] — the //! analytical per-operator resource model. -//! - [`physical_handoff_cost`] — transfer and materialization handoffs. -//! - [`query_physical_lowering`] — the physical operators a query lowers -//! to, as the cost model prices them. -//! - [`storage_io`] — storage request and byte costs. pub mod analytical_cost; -pub mod physical_handoff_cost; pub mod physical_operator_statistics; -pub mod query_physical_lowering; -pub mod storage_io; diff --git a/crates/plan-selection/src/cost/physical_handoff_cost.rs b/crates/plan-selection/src/cost/physical_handoff_cost.rs deleted file mode 100644 index a5a442aa5..000000000 --- a/crates/plan-selection/src/cost/physical_handoff_cost.rs +++ /dev/null @@ -1,230 +0,0 @@ -//! Byte estimates at deployment-declared physical handoffs. - -use crate::cost::analytical_cost::{ - estimate_physical_dag, AnalyticalCostError, EvidenceBackedPhysicalDAG, ExecutionMultiplicity, - PhysicalDAGNode, -}; -use crate::cost::physical_operator_statistics::{ComparisonScope, OperatorStatistics}; -pub use asap_types::workload::resources::{PhysicalHandoffBytes, PhysicalHandoffKind}; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; - -pub const PHYSICAL_HANDOFF_MODEL_VERSION: &str = "physical-boundary-bytes-v1"; - -/// A physical action on a producer output, distinct from a logical DAG edge. -/// With `consumer = None`, one action serves all consumers. With a consumer, -/// it is a separate action per execution of that downstream physical node. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalHandoff { - pub id: String, - pub consumer: Option, - pub kind: PhysicalHandoffKind, - pub logical_bytes: u64, - /// Encoded payload per execution; compression is explicit evidence. - pub encoded_bytes: u64, - /// Copies actually transferred or written (e.g. broadcast replicas). - pub copies: u64, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalHandoffNodeEvidence { - pub node: PhysicalDAGNode, - pub statistics: OperatorStatistics, - #[serde(rename = "boundaries")] - pub handoffs: Vec, -} - -/// Complete supplementary physical binding captured atomically with the -/// planner snapshot. An empty per-node list explicitly asserts no handoff. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalHandoffProfile { - pub evidence_version: String, - pub observed_at_ms: u64, - pub valid_until_ms: u64, - pub calibration: PhysicalHandoffCalibration, - pub plans: Vec, -} - -/// Handoffs belong to a complete alternative: a producer's consumers may -/// differ between plans even when its physical identity is unchanged. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalHandoffPlanEvidence { - pub root: String, - pub nodes: HashMap, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PhysicalHandoffCalibration { - pub version: String, - pub cost_per_network_byte: f64, - pub cost_per_materialization_byte: f64, -} - -impl PhysicalHandoffCalibration { - pub fn cost(&self, value: PhysicalHandoffBytes) -> Result { - if self.version.trim().is_empty() - || [ - self.cost_per_network_byte, - self.cost_per_materialization_byte, - ] - .iter() - .any(|value| !value.is_finite() || *value < 0.0) - { - return Err(invalid("invalid handoff calibration")); - } - let cost = value.network_bytes as f64 * self.cost_per_network_byte - + value.materialization_bytes as f64 * self.cost_per_materialization_byte; - if !cost.is_finite() { - return Err(AnalyticalCostError::Overflow); - } - Ok(cost) - } -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct PhysicalHandoffEstimate { - pub total: PhysicalHandoffBytes, - pub per_node: HashMap, - #[serde(rename = "per_boundary")] - pub per_handoff: HashMap, - pub model_version: String, - pub evidence_version: String, - pub calibration_version: String, - pub cost: f64, -} - -fn invalid(reason: &'static str) -> AnalyticalCostError { - AnalyticalCostError::InvalidPhysicalDAG(reason) -} - -pub fn estimate_physical_handoffs( - dag: &EvidenceBackedPhysicalDAG, - scope: &ComparisonScope, - profile: &PhysicalHandoffProfile, - evidence_version: &str, -) -> Result { - estimate_physical_dag(&dag.nodes, &dag.root, scope, dag)?; - let evaluations = scope.validate()?; - if profile.evidence_version.trim().is_empty() - || profile.evidence_version != evidence_version - || profile.observed_at_ms > scope.planning_time.0 - || profile.valid_until_ms <= scope.planning_time.0 - { - return Err(AnalyticalCostError::MissingOrStale("handoff profile")); - } - let mut matching_plans = profile.plans.iter().filter(|plan| { - plan.root == dag.root - && plan.nodes.len() == dag.nodes.len() - && dag.nodes.iter().all(|node| { - plan.nodes.get(&node.id).is_some_and(|evidence| { - evidence.node == *node - && dag.evidence.get(&node.id).is_some_and(|physical| { - physical.physical_id == node.id - && physical.output_buffer_bytes == node.output_buffer_bytes - && evidence.statistics == physical.statistics - }) - }) - }) - }); - let plan = matching_plans - .next() - .ok_or(AnalyticalCostError::MissingOrStale("handoff physical plan"))?; - if matching_plans.next().is_some() { - return Err(invalid("ambiguous handoff physical plan")); - } - let by_id: HashMap<_, _> = dag - .nodes - .iter() - .map(|node| (node.id.as_str(), node)) - .collect(); - let mut reachable = HashSet::new(); - let mut pending = vec![dag.root.as_str()]; - while let Some(id) = pending.pop() { - if reachable.insert(id) { - pending.extend(by_id[id].children.iter().map(String::as_str)); - } - } - let mut total = PhysicalHandoffBytes::default(); - let mut per_node = HashMap::new(); - let mut per_handoff = HashMap::new(); - for id in &reachable { - let node = by_id[id]; - let evidence = &plan.nodes[*id]; - let mut local = PhysicalHandoffBytes::default(); - for handoff in &evidence.handoffs { - if handoff.id.trim().is_empty() || per_handoff.contains_key(&handoff.id) { - return Err(invalid("duplicate or blank physical handoff identity")); - } - if handoff.logical_bytes != evidence.statistics.output().bytes - || (handoff.logical_bytes == 0) != (handoff.encoded_bytes == 0) - || handoff.copies == 0 - { - return Err(invalid( - "handoff payload is incompatible with producer output", - )); - } - let execution = if let Some(consumer) = &handoff.consumer { - if !reachable.contains(consumer.as_str()) - || !by_id[consumer.as_str()] - .children - .iter() - .any(|child| child == *id) - { - return Err(invalid("handoff consumer is not a reachable physical edge")); - } - by_id[consumer.as_str()].execution - } else { - node.execution - }; - let executions = match execution { - ExecutionMultiplicity::Once => 1, - ExecutionMultiplicity::PerEvaluation => evaluations, - }; - let bytes = handoff - .encoded_bytes - .checked_mul(handoff.copies) - .and_then(|bytes| bytes.checked_mul(executions)) - .ok_or(AnalyticalCostError::Overflow)?; - let mut term = PhysicalHandoffBytes::default(); - match &handoff.kind { - PhysicalHandoffKind::Network { - source_location, - destination_location, - } => { - if source_location.trim().is_empty() - || destination_location.trim().is_empty() - || source_location.trim() == destination_location.trim() - { - return Err(invalid( - "network handoff needs distinct non-empty locations", - )); - } - term.network_bytes = bytes; - } - PhysicalHandoffKind::Materialization => term.materialization_bytes = bytes, - } - local = local - .checked_add(term) - .ok_or(AnalyticalCostError::Overflow)?; - per_handoff.insert(handoff.id.clone(), term); - } - total = total - .checked_add(local) - .ok_or(AnalyticalCostError::Overflow)?; - per_node.insert((*id).into(), local); - } - Ok(PhysicalHandoffEstimate { - total, - per_node, - per_handoff, - model_version: PHYSICAL_HANDOFF_MODEL_VERSION.into(), - evidence_version: profile.evidence_version.clone(), - calibration_version: profile.calibration.version.clone(), - cost: profile.calibration.cost(total)?, - }) -} diff --git a/crates/plan-selection/src/cost/physical_operator_statistics.rs b/crates/plan-selection/src/cost/physical_operator_statistics.rs index 19c1b8427..9ed0b0648 100644 --- a/crates/plan-selection/src/cost/physical_operator_statistics.rs +++ b/crates/plan-selection/src/cost/physical_operator_statistics.rs @@ -1,194 +1,10 @@ -//! Authoritative comparison-scope and physical-statistics contracts. +//! Physical-operator statistics contracts. //! -//! This module does not lower logical query nodes. It defines the evidence a -//! lowering or catalog provider must supply before two physical DAGs can be -//! compared by the analytical resource model. +//! This module does not lower logical query nodes. It defines the per-operator +//! statistics the analytical resource model prices an operator from. -use std::collections::HashMap; - -use asap_types::ir::operator::operator_properties::{InfoMatcher, Source}; -use asap_types::ir::Predicate; - -use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, - TimestampMs, -}; use serde::{Deserialize, Serialize}; -use crate::cost::analytical_cost::AnalyticalCostError; - -/// The semantic and workload boundary within which two resource estimates -/// may be compared. Canonical workload and query-IR types remain authoritative; -/// only the storage snapshot identifier is new because neither IR names a -/// concrete catalog/storage version. -#[derive(Debug, Clone, PartialEq)] -pub struct ComparisonScope { - pub data_arrival: DataArrival, - pub planning_time: TimestampMs, - pub horizon: DurationMs, - pub recurrence: QueryRecurrence, - pub sources: Vec, -} - -/// Exact source selection covered by a physical plan. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct ScanSelection { - pub source: Source, - /// Provider-owned stable identifier for the physical source contents, - /// such as a catalog snapshot, table version, or object generation. - /// This is independent of the query's event-time `as_of` value. - pub source_snapshot_id: String, - /// Canonical predicates copied from the bound/canonicalized query IR. - pub predicates: Vec, - /// Symbolic selectors on an info-metric source. Ordinary query scans leave - /// this empty because their selection is represented by `predicates`. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub info_matchers: Vec, -} - -impl ComparisonScope { - /// Build a comparison scope from canonical workload fields plus the - /// physical snapshot identities supplied by the storage/catalog layer. - pub fn from_workload( - data: &DataWorkload, - query: &QueryWorkloadEntry, - planning_time: TimestampMs, - horizon: DurationMs, - sources: Vec, - ) -> Result { - let scope = Self { - data_arrival: data.arrival, - planning_time, - horizon, - recurrence: query.recurrence.clone(), - sources, - }; - scope.validate()?; - Ok(scope) - } - - /// Validate this scope and return its effective query evaluation count. - pub fn validate(&self) -> Result { - if !matches!( - self.data_arrival, - DataArrival::AtRest | DataArrival::Ingesting { .. } - ) { - return Err(AnalyticalCostError::UnsupportedDataArrival( - self.data_arrival, - )); - } - if self.horizon.0 == 0 { - return Err(AnalyticalCostError::MissingOrZero("horizon")); - } - if self - .sources - .iter() - .enumerate() - .any(|(index, source)| self.sources[..index].contains(source)) - { - return Err(AnalyticalCostError::MissingComparisonScope( - "duplicate scan selection", - )); - } - if self - .sources - .iter() - .any(|source| source.source_snapshot_id.is_empty()) - { - return Err(AnalyticalCostError::MissingComparisonScope( - "source_snapshot_id", - )); - } - evaluations_in_horizon(&self.recurrence, self.planning_time.0, self.horizon.0) - } -} - -/// Require exact scope equality before comparing raw and post-ASAP costs. -/// Exact matching is intentionally conservative: coverage/subsumption needs -/// a separate semantic proof and is not inferred by the resource estimator. -pub fn validate_comparison_scopes( - raw: &ComparisonScope, - candidate: &ComparisonScope, -) -> Result { - let evaluations = raw.validate()?; - candidate.validate()?; - for (name, matches) in [ - ("data_arrival", raw.data_arrival == candidate.data_arrival), - ( - "planning_time", - raw.planning_time == candidate.planning_time, - ), - ("horizon", raw.horizon == candidate.horizon), - ("recurrence", raw.recurrence == candidate.recurrence), - ( - "sources", - raw.sources.len() == candidate.sources.len() - && raw - .sources - .iter() - .all(|source| candidate.sources.contains(source)), - ), - ] { - if !matches { - return Err(AnalyticalCostError::ComparisonScopeMismatch(name)); - } - } - Ok(evaluations) -} - -pub(crate) fn evaluations_in_horizon( - recurrence: &QueryRecurrence, - planning_time_ms: u64, - horizon_ms: u64, -) -> Result { - if horizon_ms == 0 { - return Err(AnalyticalCostError::MissingOrZero("horizon_ms")); - } - let end = planning_time_ms.saturating_add(horizon_ms); - let count = match recurrence { - QueryRecurrence::OneTime { - invocations, - execute_at, - } => { - if execute_at.is_none_or(|at| at.0 >= planning_time_ms && at.0 <= end) { - *invocations - } else { - 0 - } - } - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) - | QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { interval, .. }) => { - if interval.0 == 0 { - return Err(AnalyticalCostError::InvalidRecurrence); - } - horizon_ms / u64::from(interval.0) - } - QueryRecurrence::Repeated(RepeatedDemand::Scheduled(schedule)) => schedule - .iter() - .filter(|at| at.0 >= planning_time_ms && at.0 <= end) - .count() - as u64, - QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { - if !estimate.is_fresh_at(planning_time_ms) - || !estimate.expected_rate.0.is_finite() - || estimate.expected_rate.0 < 0.0 - { - return Err(AnalyticalCostError::InvalidRecurrence); - } - let expected = estimate.expected_rate.0 * horizon_ms as f64 / 1000.0; - if expected > u64::MAX as f64 { - return Err(AnalyticalCostError::Overflow); - } - expected.ceil() as u64 - } - QueryRecurrence::Unknown => return Err(AnalyticalCostError::InvalidRecurrence), - }; - if count == 0 { - return Err(AnalyticalCostError::NoEvaluationsInHorizon); - } - Ok(count) -} - /// Logical cardinality and byte width carried by one physical edge. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct EdgeStatistics { @@ -553,17 +369,3 @@ impl OperatorStatistics { } } } - -/// Resolves physical statistics and owns their catalog/observation freshness. -/// Returning an error makes the entire candidate unavailable. -pub trait OperatorStatisticsProvider { - fn statistics(&self, node_id: &str) -> Result; -} - -impl OperatorStatisticsProvider for HashMap { - fn statistics(&self, node_id: &str) -> Result { - self.get(node_id) - .cloned() - .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(node_id.into())) - } -} diff --git a/crates/plan-selection/src/cost/query_physical_lowering.rs b/crates/plan-selection/src/cost/query_physical_lowering.rs deleted file mode 100644 index f58334949..000000000 --- a/crates/plan-selection/src/cost/query_physical_lowering.rs +++ /dev/null @@ -1,3028 +0,0 @@ -//! Recursive lowering from the operator IR to evidenced physical DAGs. - -use std::rc::Rc; - -use asap_types::ir::{NonASAPOp, OperatorNode, ScalarExpr}; - -use crate::cost::analytical_cost::{ - validate_operator_semantics, AnalyticalCostError, EvidenceBackedPhysicalDAG, - ExecutionMultiplicity, HashJoinBuildSide, PhysicalDAGNode, PhysicalNodeEvidence, - PhysicalOperator, PromqlBinaryOperandMode, PromqlBinaryOperation, PromqlPresenceKind, - PromqlSeriesSampleKind, PromqlVectorCardinality, -}; -use crate::cost::physical_operator_statistics::{ - ComparisonScope, EdgeStatistics, OperatorStatistics, ScanSelection, -}; - -pub struct PhysicalNodeRequest<'a> { - pub logical_node: &'a OperatorNode, - pub operator: PhysicalOperator, - pub occurrence: usize, - pub synthetic: bool, - pub children: &'a [String], - pub scan_selection: Option<&'a ScanSelection>, -} - -pub trait PhysicalNodeEvidenceProvider { - fn evidence( - &self, - request: PhysicalNodeRequest<'_>, - ) -> Result; -} - -impl PhysicalNodeEvidenceProvider for F -where - F: Fn(PhysicalNodeRequest<'_>) -> Result, -{ - fn evidence( - &self, - request: PhysicalNodeRequest<'_>, - ) -> Result { - self(request) - } -} - -/// Lower a non-ASAP operator DAG to the physical operators understood by -/// this cost model. The authoritative provider supplies statistics by the -/// stable physical IDs owned by that provider; missing evidence makes the -/// complete query unavailable. Scalar expressions remain part of their -/// containing operator's local cost. An ASAP node is unsupported here. -pub fn lower_query_physical_dag( - root: &Rc, - scope: &ComparisonScope, - evidence: &dyn PhysicalNodeEvidenceProvider, -) -> Result { - use std::collections::HashMap; - - use asap_types::ir::operator::{GroupKeys, RelationalSetOpKind}; - - scope.validate()?; - - struct Lowerer<'a> { - scope: &'a ComparisonScope, - provider: &'a dyn PhysicalNodeEvidenceProvider, - evidence: HashMap, - next_id: usize, - nodes: Vec, - } - - impl Lowerer<'_> { - fn lower(&mut self, query: &OperatorNode) -> Result { - let occurrence = self.next_id; - self.next_id += 1; - self.lower_new(query, occurrence) - } - - fn resolve( - &self, - query: &OperatorNode, - operator: PhysicalOperator, - occurrence: usize, - synthetic: bool, - children: &[String], - scan_selection: Option<&ScanSelection>, - ) -> Result { - let evidence = self.provider.evidence(PhysicalNodeRequest { - logical_node: query, - operator, - occurrence, - synthetic, - children, - scan_selection, - })?; - if evidence.physical_id.is_empty() { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "provider returned an empty physical identity", - )); - } - Ok(evidence) - } - - fn push( - &mut self, - evidence: PhysicalNodeEvidence, - operator: PhysicalOperator, - children: Vec, - scan_selection: Option, - ) -> Result { - let id = evidence.physical_id.clone(); - let node = PhysicalDAGNode { - id: id.clone(), - operator, - children, - scan_selection, - output_buffer_bytes: evidence.output_buffer_bytes, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }; - if let Some(existing) = self.nodes.iter().find(|existing| existing.id == id) { - if existing != &node || self.evidence.get(&id) != Some(&evidence) { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "provider reused a physical identity for conflicting evidence", - )); - } - return Ok(id); - } - self.nodes.push(node); - self.evidence.insert(id.clone(), evidence); - Ok(id) - } - - fn lower_unary( - &mut self, - query: &OperatorNode, - occurrence: usize, - operator: PhysicalOperator, - child: &OperatorNode, - ) -> Result { - let child_id = self.lower(child)?; - self.push_unary(query, occurrence, operator, child_id) - } - - /// `operator` over an already-lowered child. - fn push_unary( - &mut self, - query: &OperatorNode, - occurrence: usize, - operator: PhysicalOperator, - child_id: String, - ) -> Result { - let children = vec![child_id.clone()]; - let evidence = self.resolve(query, operator, occurrence, false, &children, None)?; - let statistics = &evidence.statistics; - let child_statistics = self.node_statistics(&child_id)?; - require_unary_edge( - &evidence.physical_id, - statistics, - &child_id, - child_statistics, - )?; - require_promql_edge(statistics, 0, child_statistics)?; - require_operator_statistics(operator, statistics)?; - self.push(evidence, operator, children, None) - } - - /// `operator` over its row input `child`, plus one - /// `PromqlVectorToScalar` input per distinct operator reference - /// (`scalar(v)`) its expressions read, in reference order. - fn lower_with_scalar_inputs<'e>( - &mut self, - query: &OperatorNode, - occurrence: usize, - operator: PhysicalOperator, - child: &OperatorNode, - exprs: impl IntoIterator, - ) -> Result { - let mut refs: Vec<&Rc> = Vec::new(); - for expr in exprs { - for node in expr.operator_refs() { - if !refs.iter().any(|r| Rc::ptr_eq(r, node)) { - refs.push(node); - } - } - } - if refs.is_empty() { - return self.lower_unary(query, occurrence, operator, child); - } - let mut children = vec![self.lower(child)?]; - for node in refs { - let scalar = ScalarExpr::PromqlScalarFromVector(Rc::clone(node)); - let scalar_occurrence = self.next_id; - self.next_id += 1; - children.push(self.lower_scalar_operand(query, scalar_occurrence, &scalar)?); - } - let evidence = self.resolve(query, operator, occurrence, false, &children, None)?; - require_statistics_shape(&evidence.physical_id, &evidence.statistics, children.len())?; - for (index, child_id) in children.iter().enumerate() { - let child_statistics = self.node_statistics(child_id)?; - if evidence.statistics.input(index) != Some(child_statistics.output()) { - return Err(AnalyticalCostError::ConflictingEdgeStatistics { - parent: evidence.physical_id.clone(), - child: child_id.clone(), - input_index: index, - }); - } - require_promql_edge(&evidence.statistics, index, child_statistics)?; - } - require_operator_statistics(operator, &evidence.statistics)?; - self.push(evidence, operator, children, None) - } - - fn lower_promql_unary( - &mut self, - query: &OperatorNode, - occurrence: usize, - operator: PhysicalOperator, - child: &OperatorNode, - ) -> Result { - self.lower_unary(query, occurrence, operator, child) - } - - fn lower_promql_scalar_leaf( - &mut self, - query: &OperatorNode, - occurrence: usize, - ) -> Result { - let operator = PhysicalOperator::PromqlScalarLeaf; - let evidence = self.resolve(query, operator, occurrence, false, &[], None)?; - require_statistics_shape(&evidence.physical_id, &evidence.statistics, 0)?; - require_operator_statistics(operator, &evidence.statistics)?; - self.push(evidence, operator, vec![], None) - } - - /// Lower the owned scalar operand of `vector(s)`. A literal or - /// `time()` is a physical scalar leaf; `scalar(v)` reads its vector - /// through `PromqlVectorToScalar`. - fn lower_scalar_operand( - &mut self, - query: &OperatorNode, - occurrence: usize, - expr: &ScalarExpr, - ) -> Result { - match expr { - ScalarExpr::Literal(asap_types::ir::scalar::ScalarValue::Float64(_)) - | ScalarExpr::EvalTimestamp => self.lower_promql_scalar_leaf(query, occurrence), - ScalarExpr::PromqlScalarFromVector(vector) => self.lower_promql_unary( - query, - occurrence, - PhysicalOperator::PromqlVectorToScalar, - vector, - ), - _ => Err(AnalyticalCostError::UnsupportedQueryOperator), - } - } - - fn node_statistics(&self, id: &str) -> Result<&OperatorStatistics, AnalyticalCostError> { - self.evidence - .get(id) - .map(|evidence| &evidence.statistics) - .ok_or(AnalyticalCostError::InvalidPhysicalDAG( - "lowered child statistics are missing", - )) - } - - fn lower_new( - &mut self, - query: &OperatorNode, - occurrence: usize, - ) -> Result { - let Some(op) = query.non_asap() else { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - }; - match op { - NonASAPOp::Scan { - source, predicates, .. - } => { - let coverage = bind_scan_coverage( - &format!("occurrence-{occurrence}"), - source, - predicates, - self.scope, - )?; - if predicates.is_empty() { - let evidence = self.resolve( - query, - PhysicalOperator::Scan, - occurrence, - false, - &[], - Some(&coverage), - )?; - require_statistics_shape(&evidence.physical_id, &evidence.statistics, 1)?; - require_scan_edges_equal(&evidence.statistics)?; - return self.push(evidence, PhysicalOperator::Scan, vec![], Some(coverage)); - } - let scan_evidence = self.resolve( - query, - PhysicalOperator::Scan, - occurrence, - true, - &[], - Some(&coverage), - )?; - require_statistics_shape( - &scan_evidence.physical_id, - &scan_evidence.statistics, - 1, - )?; - require_scan_edges_equal(&scan_evidence.statistics)?; - let scan_statistics = scan_evidence.statistics.clone(); - let scan_id = self.push( - scan_evidence, - PhysicalOperator::Scan, - vec![], - Some(coverage), - )?; - let children = vec![scan_id.clone()]; - let predicate_operations_per_row = predicates - .iter() - .try_fold(0_u64, |total, predicate| { - total - .checked_add(scalar_operation_count(&predicate.0)?) - .ok_or(AnalyticalCostError::Overflow) - })? - .max(1); - let filter_operator = PhysicalOperator::Filter { - predicate_operations_per_row, - }; - let filter_evidence = - self.resolve(query, filter_operator, occurrence, false, &children, None)?; - require_unary_edge( - &filter_evidence.physical_id, - &filter_evidence.statistics, - &scan_id, - &scan_statistics, - )?; - require_operator_statistics(filter_operator, &filter_evidence.statistics)?; - self.push(filter_evidence, filter_operator, children, None) - } - NonASAPOp::Filter { pred, child } => { - let operator = PhysicalOperator::Filter { - predicate_operations_per_row: scalar_operation_count(&pred.0)?.max(1), - }; - self.lower_with_scalar_inputs(query, occurrence, operator, child, [&pred.0]) - } - NonASAPOp::Project { cols, child, .. } => { - let expression_operations_per_row = cols - .iter() - .try_fold(0_u64, |total, item| { - total - .checked_add( - scalar_operation_count(&item.expr)? - .checked_add(1) - .ok_or(AnalyticalCostError::Overflow)?, - ) - .ok_or(AnalyticalCostError::Overflow) - })? - .max(1); - self.lower_with_scalar_inputs( - query, - occurrence, - PhysicalOperator::Project { - expression_operations_per_row, - }, - child, - cols.iter().map(|item| &item.expr), - ) - } - NonASAPOp::Aggregate { - reduction, - measures, - filters, - having, - child, - .. - } => { - if having.is_some() - || asap_types::ir::operator::non_asap::any_measure_filtered(filters) - || measures.is_empty() - { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - } - if matches!(reduction, asap_types::ir::operator::Reduction::PerEntity) { - if measures.len() != 1 { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - } - let accumulator_count = u64::try_from(measures.len()) - .map_err(|_| AnalyticalCostError::Overflow)?; - let operator = if measures.iter().all(presence_intent) { - PhysicalOperator::PromqlPresence { - kind: PromqlPresenceKind::Absent, - operations_per_row: accumulator_count, - } - } else if measures.iter().all(present_over_time_intent) { - PhysicalOperator::PromqlPresence { - kind: PromqlPresenceKind::PresentPerSeries, - operations_per_row: accumulator_count, - } - } else if measures.iter().all(fixed_state_per_series_intent) { - PhysicalOperator::PromqlPerSeries { - operations_per_row: accumulator_count, - accumulator_count, - } - } else { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - }; - return self.lower_promql_unary(query, occurrence, operator, child); - } - let asap_types::ir::operator::Reduction::Reduce(grouping) = reduction else { - unreachable!("per-entity reduction returned above") - }; - if grouping.is_without() || !supports_hash_aggregate(reduction, measures) { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - } - self.lower_unary( - query, - occurrence, - PhysicalOperator::HashAggregate { - grouping_key_count: u64::try_from(grouping.keys().len()) - .map_err(|_| AnalyticalCostError::Overflow)?, - accumulator_count: u64::try_from(measures.len()) - .map_err(|_| AnalyticalCostError::Overflow)?, - }, - child, - ) - } - NonASAPOp::Dedup { cols, child } => { - let key_count = if cols.is_empty() { - child.schema.fields.len() - } else { - cols.len() - }; - if key_count == 0 { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - } - self.lower_unary( - query, - occurrence, - PhysicalOperator::HashDeduplicate { - key_count: u64::try_from(key_count) - .map_err(|_| AnalyticalCostError::Overflow)?, - }, - child, - ) - } - NonASAPOp::Sort { - keys, - partition_by, - child, - .. - } => { - if keys.is_empty() { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - } - self.lower_unary( - query, - occurrence, - PhysicalOperator::InMemoryComparisonSort { - ordering_key_count: u64::try_from(keys.len()) - .map_err(|_| AnalyticalCostError::Overflow)?, - partitioned: partition_by != &GroupKeys::none(), - }, - child, - ) - } - NonASAPOp::Limit { - n, - offset, - partition_by: limit_partition_by, - child, - } => { - // Offset-only and per-group limits have no physical - // operator here. - let Some(n) = n else { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - }; - if limit_partition_by != &GroupKeys::none() { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - } - if let Some(NonASAPOp::Sort { - keys, - partition_by, - child: sorted_child, - .. - }) = child.non_asap() - { - if !keys.is_empty() && partition_by == &GroupKeys::none() { - let child_id = self.lower(sorted_child)?; - let children = vec![child_id.clone()]; - let limit = - u64::try_from(*n).map_err(|_| AnalyticalCostError::Overflow)?; - let offset = u64::try_from(*offset) - .map_err(|_| AnalyticalCostError::Overflow)?; - let operator = PhysicalOperator::TopK { - limit, - offset, - ordering_key_count: u64::try_from(keys.len()) - .map_err(|_| AnalyticalCostError::Overflow)?, - }; - let evidence = - self.resolve(query, operator, occurrence, false, &children, None)?; - let statistics = &evidence.statistics; - let child_statistics = self.node_statistics(&child_id)?; - require_unary_edge( - &evidence.physical_id, - statistics, - &child_id, - child_statistics, - )?; - let bound = limit - .checked_add(offset) - .ok_or(AnalyticalCostError::Overflow)?; - if bound == 0 { - return Err(AnalyticalCostError::MissingOrZero("topk_k")); - } - require_operator_statistics(operator, statistics)?; - return self.push(evidence, operator, children, None); - } - } - let child_id = self.lower(child)?; - let children = vec![child_id.clone()]; - let operator = PhysicalOperator::Limit { - limit: u64::try_from(*n).map_err(|_| AnalyticalCostError::Overflow)?, - offset: u64::try_from(*offset) - .map_err(|_| AnalyticalCostError::Overflow)?, - }; - let evidence = - self.resolve(query, operator, occurrence, false, &children, None)?; - let statistics = &evidence.statistics; - let child_statistics = self.node_statistics(&child_id)?; - require_unary_edge( - &evidence.physical_id, - statistics, - &child_id, - child_statistics, - )?; - require_operator_statistics(operator, statistics)?; - self.push(evidence, operator, children, None) - } - NonASAPOp::SQLWindowFunc { - func, - partition_by, - order_by, - child, - .. - } => { - if order_by.is_empty() - || !matches!( - func, - asap_types::ir::operator::WindowFuncKind::RowNumber - | asap_types::ir::operator::WindowFuncKind::Rank - | asap_types::ir::operator::WindowFuncKind::DenseRank - ) - { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - } - self.lower_unary( - query, - occurrence, - PhysicalOperator::InMemoryAnalyticWindow { - partition_key_count: u64::try_from(partition_by.keys().len()) - .map_err(|_| AnalyticalCostError::Overflow)?, - ordering_key_count: u64::try_from(order_by.len()) - .map_err(|_| AnalyticalCostError::Overflow)?, - function_operations_per_row: 1, - }, - child, - ) - } - NonASAPOp::TimeRange { range, child, .. } => { - let range_millis = duration_millis(*range, "range")?; - self.lower_promql_unary( - query, - occurrence, - PhysicalOperator::PromqlRange { range_millis }, - child, - ) - } - NonASAPOp::PromqlSubquery { - range, - resolution, - child, - } => { - let range_millis = duration_millis(*range, "subquery range")?; - let resolution_millis = resolution - .map(|value| duration_millis(value, "subquery resolution")) - .transpose()?; - let operator = PhysicalOperator::PromqlSubquery { - range_millis, - resolution_millis, - }; - let id = self.lower_promql_unary(query, occurrence, operator, child)?; - let statistics = self.node_statistics(&id)?; - let OperatorStatistics::PromqlSubquery { subquery_steps, .. } = statistics - else { - unreachable!("operator/statistics matching was validated") - }; - if let Some(resolution_millis) = resolution_millis { - let expected = range_millis - .checked_div(resolution_millis) - .and_then(|steps| steps.checked_add(1)) - .ok_or(AnalyticalCostError::Overflow)?; - if *subquery_steps != expected { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "subquery steps disagree with range and resolution", - )); - } - } - Ok(id) - } - NonASAPOp::PromqlRelabel { value, child, .. } => self.lower_promql_unary( - query, - occurrence, - PhysicalOperator::PromqlRelabel { - expression_operations_per_row: scalar_operation_count(value)?.max(1), - }, - child, - ), - NonASAPOp::PromqlSeriesSample { - by, kind, child, .. - } => { - if by.is_without() { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - } - let kind = match kind { - asap_types::ir::operator::SampleKind::LimitK(k) => { - let k = u64::try_from(*k).map_err(|_| AnalyticalCostError::Overflow)?; - if k == 0 { - return Err(AnalyticalCostError::MissingOrZero("limitk")); - } - PromqlSeriesSampleKind::LimitK { k } - } - asap_types::ir::operator::SampleKind::LimitRatio(ratio) - if ratio.is_finite() && (-1.0..=1.0).contains(ratio) => - { - PromqlSeriesSampleKind::LimitRatio { - ratio_bits: ratio.to_bits(), - } - } - asap_types::ir::operator::SampleKind::LimitRatio(_) => { - return Err(AnalyticalCostError::UnsupportedQueryOperator) - } - }; - self.lower_promql_unary( - query, - occurrence, - PhysicalOperator::PromqlSeriesSample { - kind, - grouping_key_count: u64::try_from(by.keys().len()) - .map_err(|_| AnalyticalCostError::Overflow)?, - }, - child, - ) - } - NonASAPOp::PromqlInfoEnrich { selector, child } => { - let left_id = self.lower(child)?; - let coverage = bind_info_coverage( - &format!("occurrence-{occurrence}-info"), - selector, - self.scope, - )?; - let info_evidence = self.resolve( - query, - PhysicalOperator::Scan, - occurrence, - true, - &[], - Some(&coverage), - )?; - require_statistics_shape( - &info_evidence.physical_id, - &info_evidence.statistics, - 1, - )?; - require_scan_edges_equal(&info_evidence.statistics)?; - let info_statistics = info_evidence.statistics.clone(); - let info_id = self.push( - info_evidence, - PhysicalOperator::Scan, - vec![], - Some(coverage), - )?; - let children = vec![left_id.clone(), info_id.clone()]; - let operator = PhysicalOperator::PromqlInfoEnrich { - matcher_operations_per_info_row: u64::try_from( - selector - .iter() - .filter(|matcher| matcher.label != "__name__") - .count() - .max(1), - ) - .map_err(|_| AnalyticalCostError::Overflow)?, - }; - let evidence = - self.resolve(query, operator, occurrence, false, &children, None)?; - require_binary_edges( - &evidence.physical_id, - &evidence.statistics, - &left_id, - self.node_statistics(&left_id)?, - &info_id, - &info_statistics, - )?; - require_operator_statistics(operator, &evidence.statistics)?; - self.push(evidence, operator, children, None) - } - NonASAPOp::BinaryOp { - operator, lhs, rhs, .. - } => { - let op = &operator.kind; - let vector_match = &operator.vector_match; - let operation = promql_binary_operation(op); - let operand_mode = PromqlBinaryOperandMode::VectorVector; - let cardinality = promql_vector_cardinality(vector_match.as_ref()); - let left_id = self.lower(lhs)?; - let right_id = self.lower(rhs)?; - let children = vec![left_id.clone(), right_id.clone()]; - let left_statistics = self.node_statistics(&left_id)?; - let right_statistics = self.node_statistics(&right_id)?; - let build_side = (operand_mode == PromqlBinaryOperandMode::VectorVector) - .then_some( - if left_statistics.output().bytes <= right_statistics.output().bytes { - HashJoinBuildSide::Left - } else { - HashJoinBuildSide::Right - }, - ); - let operator = PhysicalOperator::PromqlBinary { - operation, - operand_mode, - cardinality, - build_side, - }; - let evidence = - self.resolve(query, operator, occurrence, false, &children, None)?; - require_binary_edges( - &evidence.physical_id, - &evidence.statistics, - &left_id, - left_statistics, - &right_id, - right_statistics, - )?; - require_operator_statistics(operator, &evidence.statistics)?; - self.push(evidence, operator, children, None) - } - NonASAPOp::PromqlVectorFromScalar(scalar) => { - let scalar_occurrence = self.next_id; - self.next_id += 1; - let child_id = self.lower_scalar_operand(query, scalar_occurrence, scalar)?; - self.push_unary( - query, - occurrence, - PhysicalOperator::PromqlScalarToVector, - child_id, - ) - } - NonASAPOp::TimeShift { shift, child } => { - if !shift.is_identity() { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - } - self.lower_unary(query, occurrence, PhysicalOperator::PassThrough, child) - } - NonASAPOp::Concat { children, .. } => { - let child_ids = children - .iter() - .map(|child| self.lower(child)) - .collect::, _>>()?; - self.lower_concat(query, occurrence, child_ids) - } - NonASAPOp::SetOp { - kind: RelationalSetOpKind::Union, - all: true, - left, - right, - } => { - let left_id = self.lower(left)?; - let right_id = self.lower(right)?; - self.lower_concat(query, occurrence, vec![left_id, right_id]) - } - NonASAPOp::Join { - kind, - pred, - left, - right, - } => { - let equality_key_count = - if matches!(kind, asap_types::ir::operator::JoinKind::Cross) { - None - } else { - hash_join_key_count(&pred.0, left, right) - }; - let Some(equality_key_count) = equality_key_count else { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - }; - let left_id = self.lower(left)?; - let right_id = self.lower(right)?; - let children = vec![left_id.clone(), right_id.clone()]; - let left_statistics = self.node_statistics(&left_id)?; - let right_statistics = self.node_statistics(&right_id)?; - let build_side = - if left_statistics.output().bytes <= right_statistics.output().bytes { - HashJoinBuildSide::Left - } else { - HashJoinBuildSide::Right - }; - let operator = PhysicalOperator::HashJoin { - build_side, - equality_key_count, - }; - let evidence = - self.resolve(query, operator, occurrence, false, &children, None)?; - let statistics = &evidence.statistics; - require_binary_edges( - &evidence.physical_id, - statistics, - &left_id, - left_statistics, - &right_id, - right_statistics, - )?; - require_operator_statistics(operator, statistics)?; - self.push(evidence, operator, children, None) - } - _ => Err(AnalyticalCostError::UnsupportedQueryOperator), - } - } - - fn lower_concat( - &mut self, - query: &OperatorNode, - occurrence: usize, - child_ids: Vec, - ) -> Result { - if child_ids.is_empty() { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "concat has no children", - )); - } - let evidence = self.resolve( - query, - PhysicalOperator::Concat, - occurrence, - false, - &child_ids, - None, - )?; - let statistics = &evidence.statistics; - require_statistics_shape(&evidence.physical_id, statistics, child_ids.len())?; - let (rows, bytes) = child_ids.iter().enumerate().try_fold( - (0_u64, 0_u64), - |(rows, bytes), (index, child)| { - let child_statistics = self.node_statistics(child)?; - if statistics.input(index) != Some(child_statistics.output()) { - return Err(AnalyticalCostError::ConflictingEdgeStatistics { - parent: evidence.physical_id.clone(), - child: child.clone(), - input_index: index, - }); - } - require_promql_edge(statistics, index, child_statistics)?; - Ok::<_, AnalyticalCostError>(( - rows.checked_add(child_statistics.output().rows) - .ok_or(AnalyticalCostError::Overflow)?, - bytes - .checked_add(child_statistics.output().bytes) - .ok_or(AnalyticalCostError::Overflow)?, - )) - }, - )?; - if statistics.output() != (EdgeStatistics { rows, bytes }) { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "concat statistics do not equal the sum of child outputs", - )); - } - require_operator_statistics(PhysicalOperator::Concat, statistics)?; - self.push(evidence, PhysicalOperator::Concat, child_ids, None) - } - } - - let mut lowerer = Lowerer { - scope, - provider: evidence, - evidence: HashMap::new(), - next_id: 0, - nodes: Vec::new(), - }; - let root = lowerer.lower(root)?; - validate_source_consumption(&lowerer.nodes, scope)?; - Ok(EvidenceBackedPhysicalDAG { - nodes: lowerer.nodes, - root, - evidence: lowerer.evidence, - }) -} - -fn validate_source_consumption( - nodes: &[PhysicalDAGNode], - scope: &ComparisonScope, -) -> Result<(), AnalyticalCostError> { - let consumed = nodes - .iter() - .filter(|node| matches!(node.operator, PhysicalOperator::Scan)) - .filter_map(|node| node.scan_selection.as_ref()) - .collect::>(); - for coverage in &consumed { - if !scope.sources.contains(coverage) { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "physical scan consumes a source outside the comparison scope", - )); - } - } - if scope - .sources - .iter() - .any(|expected| !consumed.contains(&expected)) - { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "physical scans omit a comparison-scope source", - )); - } - Ok(()) -} - -fn require_statistics_shape( - node: &str, - statistics: &OperatorStatistics, - input_count: usize, -) -> Result<(), AnalyticalCostError> { - if statistics.input_count() != input_count { - return Err(AnalyticalCostError::InvalidOperatorStatistics { - node: node.into(), - reason: "wrong input-edge count", - }); - } - if (0..statistics.input_count()) - .filter_map(|index| statistics.input(index)) - .chain(std::iter::once(statistics.output())) - .any(|edge| !edge.is_consistent()) - { - return Err(AnalyticalCostError::InvalidOperatorStatistics { - node: node.into(), - reason: "edge rows and logical bytes are inconsistent", - }); - } - Ok(()) -} - -fn require_scan_edges_equal(statistics: &OperatorStatistics) -> Result<(), AnalyticalCostError> { - if statistics.input(0) != Some(statistics.output()) { - return Err(AnalyticalCostError::InconsistentOperatorStatistics( - "Scan external input edge does not match its output edge", - )); - } - Ok(()) -} - -fn require_unary_edge( - node: &str, - statistics: &OperatorStatistics, - child_id: &str, - child: &OperatorStatistics, -) -> Result<(), AnalyticalCostError> { - require_statistics_shape(node, statistics, 1)?; - if statistics.input(0) != Some(child.output()) { - return Err(AnalyticalCostError::ConflictingEdgeStatistics { - parent: node.into(), - child: child_id.into(), - input_index: 0, - }); - } - Ok(()) -} - -fn require_binary_edges( - node: &str, - statistics: &OperatorStatistics, - left_id: &str, - left: &OperatorStatistics, - right_id: &str, - right: &OperatorStatistics, -) -> Result<(), AnalyticalCostError> { - require_statistics_shape(node, statistics, 2)?; - for (index, (child_id, child)) in [(left_id, left), (right_id, right)].into_iter().enumerate() { - if statistics.input(index) != Some(child.output()) { - return Err(AnalyticalCostError::ConflictingEdgeStatistics { - parent: node.into(), - child: child_id.into(), - input_index: index, - }); - } - require_promql_edge(statistics, index, child)?; - } - Ok(()) -} - -fn require_promql_edge( - parent: &OperatorStatistics, - input_index: usize, - child: &OperatorStatistics, -) -> Result<(), AnalyticalCostError> { - match (parent.promql_input(input_index), child.promql_output()) { - (None, None) => Ok(()), - (Some(input), Some(output)) if input == output => Ok(()), - (Some(_), Some(_)) => Err(AnalyticalCostError::InconsistentOperatorStatistics( - "PromQL parent input does not match child output", - )), - _ => Err(AnalyticalCostError::MissingOrStale( - "promql_edge_statistics", - )), - } -} - -fn require_operator_statistics( - operator: PhysicalOperator, - statistics: &OperatorStatistics, -) -> Result<(), AnalyticalCostError> { - validate_operator_semantics(operator, statistics) -} - -fn bind_scan_coverage( - node_id: &str, - source: &asap_types::ir::operator::Source, - predicates: &[asap_types::ir::Predicate], - scope: &ComparisonScope, -) -> Result { - let mut matches = scope.sources.iter().filter(|coverage| { - coverage.source == *source - && coverage.predicates == predicates - && coverage.info_matchers.is_empty() - }); - let coverage = matches - .next() - .cloned() - .ok_or_else(|| AnalyticalCostError::ScanOutsideComparisonScope(node_id.into()))?; - if matches.any(|candidate| candidate != &coverage) { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "scan selection is ambiguous", - )); - } - Ok(coverage) -} - -fn bind_info_coverage( - node_id: &str, - selector: &[asap_types::ir::operator::InfoMatcher], - scope: &ComparisonScope, -) -> Result { - use asap_types::ir::operator::Source; - use asap_types::ir::scalar::CompareOpKind; - - let mut metric: Option<&str> = None; - for matcher in selector - .iter() - .filter(|matcher| matcher.label == "__name__") - { - if matcher.op != CompareOpKind::Eq || metric.is_some_and(|current| current != matcher.value) - { - return Err(AnalyticalCostError::UnsupportedQueryOperator); - } - metric = Some(&matcher.value); - } - let source = Source::TimeSeries { - metric: metric.unwrap_or("target_info").into(), - }; - let mut matches = scope.sources.iter().filter(|coverage| { - coverage.source == source - && coverage.predicates.is_empty() - && coverage.info_matchers == selector - }); - let coverage = matches - .next() - .cloned() - .ok_or_else(|| AnalyticalCostError::ScanOutsideComparisonScope(node_id.into()))?; - if matches.next().is_some() { - return Err(AnalyticalCostError::InvalidPhysicalDAG( - "info scan selection is ambiguous", - )); - } - Ok(coverage) -} - -fn duration_millis( - duration: std::time::Duration, - field: &'static str, -) -> Result { - if duration.is_zero() { - return Err(AnalyticalCostError::MissingOrZero(field)); - } - u64::try_from(duration.as_millis()).map_err(|_| AnalyticalCostError::Overflow) -} - -fn promql_binary_operation( - operation: &asap_types::ir::operator::BinaryOpKind, -) -> PromqlBinaryOperation { - use asap_types::ir::operator::{BinaryOpKind, PromQLVectorSetOpKind}; - match operation { - BinaryOpKind::Set(PromQLVectorSetOpKind::And) => PromqlBinaryOperation::And, - BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => PromqlBinaryOperation::Or, - BinaryOpKind::Set(PromQLVectorSetOpKind::Unless) => PromqlBinaryOperation::Unless, - BinaryOpKind::Arithmetic(_) | BinaryOpKind::Compare(_) | BinaryOpKind::CompareBool(_) => { - PromqlBinaryOperation::ArithmeticOrComparison - } - } -} - -fn promql_vector_cardinality( - vector_match: Option<&asap_types::ir::operator::VectorMatch>, -) -> PromqlVectorCardinality { - use asap_types::ir::operator::GroupSide; - match vector_match.and_then(|matching| matching.grouping.as_ref()) { - Some(grouping) if grouping.side == GroupSide::Left => PromqlVectorCardinality::ManyToOne, - Some(_) => PromqlVectorCardinality::OneToMany, - None => PromqlVectorCardinality::OneToOne, - } -} - -fn hash_join_key_count( - expr: &ScalarExpr, - left: &OperatorNode, - right: &OperatorNode, -) -> Option { - use asap_types::ir::scalar::CompareOpKind; - - let left_width = left.schema.fields.len(); - let total_width = left_width.saturating_add(right.schema.fields.len()); - - fn column_side(column: usize, left_width: usize, total_width: usize) -> Option { - if column < left_width { - Some(false) - } else if column < total_width { - Some(true) - } else { - None - } - } - - fn predicate(expr: &ScalarExpr, left_width: usize, total_width: usize) -> Option { - match expr { - ScalarExpr::Compare { - left, - op: CompareOpKind::Eq, - right, - .. - } => match (left.as_ref(), right.as_ref()) { - (ScalarExpr::Column(left), ScalarExpr::Column(right)) => match ( - column_side(*left, left_width, total_width), - column_side(*right, left_width, total_width), - ) { - (Some(false), Some(true)) | (Some(true), Some(false)) => Some(1), - _ => None, - }, - _ => None, - }, - ScalarExpr::BoolAnd(parts) if !parts.is_empty() => { - parts.iter().try_fold(0_u64, |count, part| { - count.checked_add(predicate(part, left_width, total_width)?) - }) - } - _ => None, - } - } - - predicate(expr, left_width, total_width) -} - -fn scalar_operation_count(expr: &ScalarExpr) -> Result { - let add = |parts: &[&ScalarExpr]| { - parts.iter().try_fold(0_u64, |total, part| { - total - .checked_add(scalar_operation_count(part)?) - .ok_or(AnalyticalCostError::Overflow) - }) - }; - let with_local = |children| { - add(children)? - .checked_add(1) - .ok_or(AnalyticalCostError::Overflow) - }; - match expr { - ScalarExpr::Column(_) - | ScalarExpr::Literal(_) - | ScalarExpr::EvalTimestamp - | ScalarExpr::CurrentTimestamp => Ok(0), - // Read from a scalar input of the containing operator. - ScalarExpr::PromqlScalarFromVector(_) => Ok(0), - ScalarExpr::Compare { left, right, .. } | ScalarExpr::Arithmetic { left, right, .. } => { - with_local(&[left, right]) - } - ScalarExpr::BoolAnd(parts) | ScalarExpr::BoolOr(parts) => { - let children = parts.iter().collect::>(); - add(&children)? - .checked_add( - u64::try_from(parts.len().saturating_sub(1)) - .map_err(|_| AnalyticalCostError::Overflow)?, - ) - .ok_or(AnalyticalCostError::Overflow) - } - ScalarExpr::Not(child) | ScalarExpr::IsNull(child) | ScalarExpr::IsNotNull(child) => { - with_local(&[child]) - } - ScalarExpr::Cast { expr, .. } => with_local(&[expr]), - ScalarExpr::InList { expr, list, .. } => { - let mut children = Vec::with_capacity(list.len() + 1); - children.push(expr.as_ref()); - children.extend(list.iter()); - add(&children)? - .checked_add(u64::try_from(list.len()).map_err(|_| AnalyticalCostError::Overflow)?) - .ok_or(AnalyticalCostError::Overflow) - } - ScalarExpr::FunctionCall { args, .. } => { - let children = args.iter().collect::>(); - with_local(&children) - } - ScalarExpr::Case { - operand, - branches, - else_expr, - } => { - let mut children = Vec::new(); - if let Some(operand) = operand { - children.push(operand.as_ref()); - } - for (when, then) in branches { - children.extend([when, then]); - } - if let Some(else_expr) = else_expr { - children.push(else_expr.as_ref()); - } - with_local(&children) - } - _ => Err(AnalyticalCostError::UnsupportedQueryOperator), - } -} - -fn supports_hash_aggregate( - reduction: &asap_types::ir::operator::Reduction, - measures: &[asap_types::ir::operator::AggIntent], -) -> bool { - use asap_types::ir::operator::{AggIntent, Reduction}; - - matches!(reduction, Reduction::Reduce(_)) - && !measures.is_empty() - && measures.iter().all(|intent| { - matches!( - intent, - AggIntent::Count { .. } - | AggIntent::Sum { .. } - | AggIntent::Min { .. } - | AggIntent::Max { .. } - | AggIntent::Avg { .. } - | AggIntent::StdDev { .. } - | AggIntent::Variance { .. } - | AggIntent::PearsonCorr { .. } - | AggIntent::Group - | AggIntent::CountValues { .. } - | AggIntent::Cardinality { .. } - | AggIntent::FrequencyL2 { .. } - | AggIntent::FrequencyEntropy { .. } - ) - }) -} - -fn presence_intent(intent: &asap_types::ir::operator::AggIntent) -> bool { - matches!( - intent, - asap_types::ir::operator::AggIntent::Absent - | asap_types::ir::operator::AggIntent::AbsentOverTime - ) -} - -fn present_over_time_intent(intent: &asap_types::ir::operator::AggIntent) -> bool { - matches!(intent, asap_types::ir::operator::AggIntent::PresentOverTime) -} - -fn fixed_state_per_series_intent(intent: &asap_types::ir::operator::AggIntent) -> bool { - use asap_types::ir::operator::AggIntent; - matches!( - intent, - AggIntent::Rate - | AggIntent::Count { .. } - | AggIntent::Sum { .. } - | AggIntent::Min { .. } - | AggIntent::Max { .. } - | AggIntent::Avg { .. } - | AggIntent::StdDev { .. } - | AggIntent::Variance { .. } - | AggIntent::Increase - | AggIntent::Changes - | AggIntent::Delta - | AggIntent::IDelta - | AggIntent::Deriv - | AggIntent::Resets - | AggIntent::PredictLinear { .. } - | AggIntent::DoubleExpSmoothing { .. } - | AggIntent::HistogramCount - | AggIntent::HistogramSum - | AggIntent::HistogramAvg - | AggIntent::HistogramStdDev - | AggIntent::HistogramStdVar - | AggIntent::HistogramFraction { .. } - | AggIntent::Math(_) - | AggIntent::TimeFn(_) - | AggIntent::LastOverTime - | AggIntent::FirstOverTime - | AggIntent::TsOfMinOverTime - | AggIntent::TsOfMaxOverTime - | AggIntent::TsOfFirstOverTime - | AggIntent::TsOfLastOverTime - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::cost::analytical_cost::{ - estimate_physical_dag, estimate_physical_dag_comparison, PhysicalDAGEstimateRequest, - }; - use crate::cost::physical_operator_statistics::{ - validate_comparison_scopes, BinaryEdgeStatistics, PartitionStatistics, - PromqlEdgeStatistics, PromqlUnaryEdgeStatistics, PromqlValueKind, UnaryEdgeStatistics, - }; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::{BinaryOperator, ExprSemantics, Predicate, SortKey, TimeRangeKind}; - use asap_types::workload::{DataArrival, DurationMs, QueryRecurrence, TimestampMs}; - use std::collections::HashMap; - - fn edge(rows: u64, bytes: u64) -> EdgeStatistics { - EdgeStatistics { rows, bytes } - } - - fn unary_edges(input: EdgeStatistics, output: EdgeStatistics) -> UnaryEdgeStatistics { - UnaryEdgeStatistics { - input, - output, - promql: None, - } - } - - fn scan_stats(edge: EdgeStatistics, source_read_bytes: u64) -> OperatorStatistics { - OperatorStatistics::Scan { - edges: unary_edges(edge, edge), - source_read_bytes, - } - } - - fn promql_edge( - series: u64, - evaluation_steps: u64, - value_kind: PromqlValueKind, - ) -> PromqlEdgeStatistics { - PromqlEdgeStatistics { - series, - evaluation_steps, - value_kind, - } - } - - fn promql_unary_edges( - input: EdgeStatistics, - output: EdgeStatistics, - promql_input: PromqlEdgeStatistics, - promql_output: PromqlEdgeStatistics, - ) -> UnaryEdgeStatistics { - UnaryEdgeStatistics { - input, - output, - promql: Some(PromqlUnaryEdgeStatistics { - input: promql_input, - output: promql_output, - }), - } - } - - fn unary_statistics( - operator: PhysicalOperator, - input: EdgeStatistics, - output: EdgeStatistics, - ) -> OperatorStatistics { - let edges = unary_edges(input, output); - match operator { - PhysicalOperator::Filter { .. } => OperatorStatistics::Filter { - edges, - scalar_inputs: vec![], - }, - PhysicalOperator::Project { .. } => OperatorStatistics::Project { - edges, - scalar_inputs: vec![], - }, - PhysicalOperator::InMemoryComparisonSort { .. } => { - OperatorStatistics::InMemoryComparisonSort { - edges, - input_partitioning: PartitionStatistics { - partitions: (!input.eq(&edge(0, 0))) - .then_some(input) - .into_iter() - .collect(), - }, - } - } - PhysicalOperator::TopK { .. } => OperatorStatistics::TopK { edges }, - PhysicalOperator::InMemoryAnalyticWindow { .. } => { - OperatorStatistics::InMemoryAnalyticWindow { - edges, - input_partitioning: PartitionStatistics { - partitions: (!input.eq(&edge(0, 0))) - .then_some(input) - .into_iter() - .collect(), - }, - } - } - PhysicalOperator::Limit { .. } => OperatorStatistics::Limit { edges }, - PhysicalOperator::PassThrough => OperatorStatistics::PassThrough { edges }, - _ => panic!("test helper requires a stateless unary operator"), - } - } - - fn evidence(statistics: OperatorStatistics) -> PhysicalNodeEvidence { - PhysicalNodeEvidence { - physical_id: String::new(), - output_buffer_bytes: statistics.output().bytes.min(1_024), - statistics, - } - } - - fn scripted<'a>( - provided: &'a HashMap, - ) -> impl Fn(PhysicalNodeRequest<'_>) -> Result + 'a - { - move |request| { - let key = if request.synthetic { - format!("query-{}-scan", request.occurrence) - } else { - format!("query-{}", request.occurrence) - }; - let mut evidence = provided - .get(&key) - .cloned() - .ok_or_else(|| AnalyticalCostError::MissingOperatorStatistics(key.clone()))?; - evidence.physical_id = key; - Ok(evidence) - } - } - - fn scope(sources: Vec) -> ComparisonScope { - ComparisonScope { - data_arrival: DataArrival::AtRest, - planning_time: TimestampMs(1_000), - horizon: DurationMs(1_000), - recurrence: QueryRecurrence::OneTime { - invocations: 1, - execute_at: None, - }, - sources, - } - } - - fn coverage( - source: asap_types::ir::operator::Source, - predicates: Vec, - ) -> ScanSelection { - ScanSelection { - source, - source_snapshot_id: "snapshot-1".into(), - predicates, - info_matchers: vec![], - } - } - - #[test] - fn info_scan_selection_includes_symbolic_selector_matchers() { - use asap_types::ir::operator::{InfoMatcher, Source}; - use asap_types::ir::scalar::CompareOpKind; - - let selector = vec![InfoMatcher { - label: "cluster".into(), - op: CompareOpKind::Eq, - value: "prod".into(), - }]; - let info_coverage = ScanSelection { - source: Source::TimeSeries { - metric: "target_info".into(), - }, - source_snapshot_id: "snapshot-1".into(), - predicates: vec![], - info_matchers: selector.clone(), - }; - let matching_scope = scope(vec![info_coverage.clone()]); - assert_eq!( - bind_info_coverage("info", &selector, &matching_scope), - Ok(info_coverage) - ); - - let wrong_scope = scope(vec![coverage( - Source::TimeSeries { - metric: "target_info".into(), - }, - vec![], - )]); - assert!(matches!( - bind_info_coverage("info", &selector, &wrong_scope), - Err(AnalyticalCostError::ScanOutsideComparisonScope(_)) - )); - } - - // Correlation can be costed as an exact hash aggregate using provider-supplied state size. - #[test] - fn correlation_lowers_to_physical_hash_aggregate() { - use asap_types::ir::operator::{AggIntent, Reduction, Source}; - use asap_types::ir::schema::{DataType, Field, Schema}; - let source = Source::Table { - table_ref: "pairs".into(), - }; - let root = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::PearsonCorr { left: 0, right: 1 }], - output_names: vec!["r".into()], - filters: vec![], - having: None, - child: OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::Scan { - source: source.clone(), - predicates: vec![], - schema: Schema::new(vec![ - Field::plain("x", DataType::Float64, true), - Field::plain("y", DataType::Float64, true), - ]), - }, - )) - .unwrap(), - })) - .unwrap(); - let scope = scope(vec![coverage(source, vec![])]); - let provided = HashMap::from([ - ( - "query-1".into(), - evidence(scan_stats(edge(100, 1600), 1600)), - ), - ( - "query-0".into(), - evidence(OperatorStatistics::HashAggregate { - edges: unary_edges(edge(100, 1600), edge(1, 8)), - group_count: 1, - key_bytes: 0, - accumulator_bytes_per_group: 48, - }), - ), - ]); - let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); - assert!(matches!( - dag.nodes.last().unwrap().operator, - PhysicalOperator::HashAggregate { - grouping_key_count: 0, - accumulator_count: 1, - } - )); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); - } - - // The exact distinct count, L2 norm and entropy are hash aggregates: the - // executor runs them as native reducers over one identity column. - #[test] - fn exact_frequency_intents_lower_to_physical_hash_aggregates() { - use asap_types::ir::operator::{AggIntent, Reduction, Source}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::schema::{DataType, Field, Schema}; - let source = Source::Table { - table_ref: "flows".into(), - }; - for measure in [ - AggIntent::Cardinality { - cols: vec![0], - accuracy: AccuracyTarget::exact(), - }, - AggIntent::FrequencyL2 { - col: Some(0), - accuracy: AccuracyTarget::exact(), - }, - AggIntent::FrequencyEntropy { - col: Some(0), - accuracy: AccuracyTarget::exact(), - }, - ] { - let root = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![measure.clone()], - output_names: vec!["v".into()], - filters: vec![], - having: None, - child: OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::Scan { - source: source.clone(), - predicates: vec![], - schema: Schema::new(vec![Field::plain( - "src_ip", - DataType::Utf8, - false, - )]), - }, - )) - .unwrap(), - })) - .unwrap(); - let scope = scope(vec![coverage(source.clone(), vec![])]); - let provided = HashMap::from([ - ( - "query-1".into(), - evidence(scan_stats(edge(100, 1600), 1600)), - ), - ( - "query-0".into(), - evidence(OperatorStatistics::HashAggregate { - edges: unary_edges(edge(100, 1600), edge(1, 8)), - group_count: 1, - key_bytes: 0, - accumulator_bytes_per_group: 48, - }), - ), - ]); - let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)) - .unwrap_or_else(|e| panic!("{measure:?}: {e:?}")); - assert!(matches!( - dag.nodes.last().unwrap().operator, - PhysicalOperator::HashAggregate { - grouping_key_count: 0, - accumulator_count: 1, - } - )); - } - } - - #[test] - fn query_lowering_recurses_and_fuses_global_sort_limit() { - use asap_types::ir::operator::{AggIntent, GroupKeys, Reduction, Source}; - use asap_types::ir::schema::{DataType, Field, Schema}; - use std::rc::Rc; - - let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "events".into(), - }, - predicates: vec![Predicate(ScalarExpr::Literal( - asap_types::ir::scalar::ScalarValue::Boolean(true), - ))], - schema: Schema::new(vec![ - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - ]), - })) - .unwrap(); - let aggregate = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![0]), - measures: vec![AggIntent::Sum { col: Some(1) }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::clone(&scan), - })) - .unwrap(); - let sort = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(0), - ascending: false, - nulls_first: false, - }], - partition_by: GroupKeys::none(), - child: aggregate, - })) - .unwrap(); - let root = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Limit { - n: Some(10), - offset: 5, - partition_by: GroupKeys::none(), - child: sort, - })) - .unwrap(); - - let scan_coverage = coverage( - Source::Table { - table_ref: "events".into(), - }, - vec![Predicate(ScalarExpr::Literal( - asap_types::ir::scalar::ScalarValue::Boolean(true), - ))], - ); - let scope = scope(vec![scan_coverage]); - let aggregate_statistics = OperatorStatistics::HashAggregate { - edges: unary_edges(edge(400, 25_600), edge(100, 4_000)), - group_count: 100, - key_bytes: 16, - accumulator_bytes_per_group: 8, - }; - let topk_statistics = unary_statistics( - PhysicalOperator::TopK { - limit: 10, - offset: 5, - ordering_key_count: 1, - }, - edge(100, 4_000), - edge(10, 400), - ); - let raw_scan = scan_stats(edge(1_000, 64_000), 64_000); - let provided = HashMap::from([ - ("query-2-scan".into(), evidence(raw_scan)), - ( - "query-2".into(), - evidence(unary_statistics( - PhysicalOperator::Filter { - predicate_operations_per_row: 1, - }, - edge(1_000, 64_000), - edge(400, 25_600), - )), - ), - ("query-1".into(), evidence(aggregate_statistics)), - ("query-0".into(), evidence(topk_statistics)), - ]); - let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); - - assert_eq!( - dag.nodes - .iter() - .map(|node| node.operator) - .collect::>(), - vec![ - PhysicalOperator::Scan, - PhysicalOperator::Filter { - predicate_operations_per_row: 1, - }, - PhysicalOperator::HashAggregate { - grouping_key_count: 1, - accumulator_count: 1, - }, - PhysicalOperator::TopK { - limit: 10, - offset: 5, - ordering_key_count: 1, - }, - ] - ); - let topk = dag.nodes.last().unwrap(); - assert_eq!(topk.children, vec![dag.nodes[2].id.clone()]); - assert!(matches!( - provided[&topk.id].statistics, - OperatorStatistics::TopK { .. } - )); - let physical_scan = &dag.nodes[0]; - assert_eq!(physical_scan.id, "query-2-scan"); - assert_eq!(physical_scan.scan_selection, Some(scope.sources[0].clone())); - assert_eq!(physical_scan.output_buffer_bytes, 1_024); - assert_ne!( - physical_scan.output_buffer_bytes, - provided[&physical_scan.id].statistics.output().bytes - ); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); - - let mut inconsistent_scan = provided.clone(); - inconsistent_scan - .get_mut("query-2-scan") - .unwrap() - .statistics = OperatorStatistics::Scan { - edges: unary_edges(edge(1_000, 64_000), edge(999, 63_936)), - source_read_bytes: 64_000, - }; - assert_eq!( - lower_query_physical_dag(&root, &scope, &scripted(&inconsistent_scan)), - Err(AnalyticalCostError::InconsistentOperatorStatistics( - "Scan external input edge does not match its output edge" - )) - ); - } - - #[test] - fn query_lowering_shares_only_provider_identified_physical_nodes() { - use asap_types::ir::operator::{JoinKind, Source}; - use asap_types::ir::scalar::CompareOpKind; - use asap_types::ir::schema::{DataType, Field, Schema}; - use std::rc::Rc; - - let shared = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "dimensions".into(), - }, - predicates: vec![], - schema: Schema::new(vec![Field::plain("id", DataType::Int64, false)]), - })) - .unwrap(); - let root = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Join { - kind: JoinKind::Inner, - pred: Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(0)), - op: CompareOpKind::Eq, - right: Box::new(ScalarExpr::Column(1)), - semantics: ExprSemantics::Sql, - }), - left: Rc::clone(&shared), - right: Rc::clone(&shared), - })) - .unwrap(); - let scan_selection = coverage( - Source::Table { - table_ref: "dimensions".into(), - }, - vec![], - ); - let independent_scope = scope(vec![scan_selection.clone()]); - let scan_statistics = scan_stats(edge(100, 800), 800); - let join_statistics = OperatorStatistics::HashJoin { - edges: BinaryEdgeStatistics { - inputs: [edge(100, 800), edge(100, 800)], - output: edge(25, 400), - promql: None, - }, - }; - let provided = HashMap::from([ - ("query-1".into(), evidence(scan_statistics.clone())), - ("query-2".into(), evidence(scan_statistics.clone())), - ("query-0".into(), evidence(join_statistics.clone())), - ]); - let dag = - lower_query_physical_dag(&root, &independent_scope, &scripted(&provided)).unwrap(); - - assert_eq!(dag.nodes.len(), 3); - assert_ne!(dag.nodes[2].children[0], dag.nodes[2].children[1]); - let estimate = - estimate_physical_dag(&dag.nodes, &dag.root, &independent_scope, &dag.evidence) - .unwrap(); - assert_eq!(estimate.scan_bytes(), 1_600); - - let shared_provider = |request: PhysicalNodeRequest<'_>| { - let (physical_id, statistics) = match request.operator { - PhysicalOperator::Scan => ("shared-scan", scan_statistics.clone()), - PhysicalOperator::HashJoin { .. } => ("join", join_statistics.clone()), - _ => return Err(AnalyticalCostError::UnsupportedQueryOperator), - }; - Ok(PhysicalNodeEvidence { - physical_id: physical_id.into(), - output_buffer_bytes: statistics.output().bytes.min(1_024), - statistics, - }) - }; - let shared_scope = scope(vec![scan_selection]); - let no_cache = crate::cost::analytical_cost::CacheProfile::no_cache(); - let shared_dag = lower_query_physical_dag(&root, &shared_scope, &shared_provider).unwrap(); - assert_eq!(shared_dag.nodes.len(), 2); - assert_eq!( - shared_dag.nodes[1].children, - vec!["shared-scan".to_owned(); 2] - ); - let comparison = estimate_physical_dag_comparison( - PhysicalDAGEstimateRequest { - nodes: &dag.nodes, - root: &dag.root, - scope: &independent_scope, - statistics: &dag, - cache_profile: &no_cache, - }, - PhysicalDAGEstimateRequest { - nodes: &shared_dag.nodes, - root: &shared_dag.root, - scope: &shared_scope, - statistics: &shared_dag, - cache_profile: &no_cache, - }, - ) - .unwrap(); - assert_eq!(comparison.raw.scan_bytes(), 1_600); - assert_eq!(comparison.candidate.scan_bytes(), 800); - - let mut drifted_buffer = shared_dag.clone(); - drifted_buffer.nodes[0].output_buffer_bytes += 1; - assert_eq!( - estimate_physical_dag( - &drifted_buffer.nodes, - &drifted_buffer.root, - &shared_scope, - &drifted_buffer, - ), - Err(AnalyticalCostError::InvalidPhysicalDAG( - "physical node buffer differs from evidence snapshot" - )) - ); - let mut drifted_identity = shared_dag.clone(); - drifted_identity - .evidence - .get_mut("shared-scan") - .unwrap() - .physical_id = "different".into(); - assert_eq!( - estimate_physical_dag( - &drifted_identity.nodes, - &drifted_identity.root, - &shared_scope, - &drifted_identity, - ), - Err(AnalyticalCostError::InvalidPhysicalDAG( - "evidence map key differs from embedded physical identity" - )) - ); - - let conflicting_identity = |request: PhysicalNodeRequest<'_>| { - let (physical_id, mut statistics) = match request.operator { - PhysicalOperator::Scan => ("shared-scan", scan_statistics.clone()), - PhysicalOperator::HashJoin { .. } => ("join", join_statistics.clone()), - _ => return Err(AnalyticalCostError::UnsupportedQueryOperator), - }; - if request.operator == PhysicalOperator::Scan && request.occurrence == 2 { - statistics = scan_stats(edge(100, 800), 801); - } - Ok(PhysicalNodeEvidence { - physical_id: physical_id.into(), - output_buffer_bytes: statistics.output().bytes.min(1_024), - statistics, - }) - }; - assert_eq!( - lower_query_physical_dag(&root, &shared_scope, &conflicting_identity), - Err(AnalyticalCostError::InvalidPhysicalDAG( - "provider reused a physical identity for conflicting evidence" - )) - ); - - let invalid = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Join { - kind: JoinKind::Inner, - pred: Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(0)), - op: CompareOpKind::Eq, - right: Box::new(ScalarExpr::Column(0)), - semantics: ExprSemantics::Sql, - }), - left: Rc::clone(&shared), - right: Rc::clone(&shared), - })) - .unwrap(); - assert_eq!( - lower_query_physical_dag(&invalid, &shared_scope, &shared_provider), - Err(AnalyticalCostError::UnsupportedQueryOperator) - ); - } - - #[test] - fn query_lowering_covers_relational_unary_operators() { - use asap_types::ir::operator::{GroupKeys, Source, TimeShift, WindowFuncKind}; - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::schema::{DataType, Field, Schema}; - - let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "events".into(), - }, - predicates: vec![], - schema: Schema::new(vec![Field::plain("id", DataType::Int64, false)]), - })) - .unwrap(); - let filter = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), - child: scan, - })) - .unwrap(); - let project = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - cols: vec![], - qualifier: None, - child: filter, - })) - .unwrap(); - let dedup = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Dedup { - cols: vec![0], - child: project, - })) - .unwrap(); - let window = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::SQLWindowFunc { - func: WindowFuncKind::RowNumber, - args: vec![], - partition_by: GroupKeys::none(), - order_by: vec![SortKey { - expr: ScalarExpr::Column(0), - ascending: true, - nulls_first: false, - }], - frame: None, - output_name: "rn".into(), - child: dedup, - }, - )) - .unwrap(); - let sort = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(0), - ascending: true, - nulls_first: false, - }], - partition_by: GroupKeys::by(vec![0]), - child: window, - })) - .unwrap(); - let limit = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Limit { - n: Some(20), - offset: 0, - partition_by: GroupKeys::none(), - child: sort, - })) - .unwrap(); - let root = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift::default(), - child: limit, - })) - .unwrap(); - - let scan_selection = coverage( - Source::Table { - table_ref: "events".into(), - }, - vec![], - ); - let scope = scope(vec![scan_selection]); - let scan_statistics = scan_stats(edge(1_000, 8_000), 8_000); - let dedup_statistics = OperatorStatistics::HashDeduplicate { - edges: unary_edges(edge(800, 3_200), edge(500, 2_000)), - distinct_key_count: 500, - key_bytes: 8, - }; - let provided = HashMap::from([ - ("query-7".into(), evidence(scan_statistics)), - ( - "query-6".into(), - evidence(unary_statistics( - PhysicalOperator::Filter { - predicate_operations_per_row: 1, - }, - edge(1_000, 8_000), - edge(800, 6_400), - )), - ), - ( - "query-5".into(), - evidence(unary_statistics( - PhysicalOperator::Project { - expression_operations_per_row: 1, - }, - edge(800, 6_400), - edge(800, 3_200), - )), - ), - ("query-4".into(), evidence(dedup_statistics)), - ( - "query-3".into(), - evidence(unary_statistics( - PhysicalOperator::InMemoryAnalyticWindow { - partition_key_count: 0, - ordering_key_count: 1, - function_operations_per_row: 1, - }, - edge(500, 2_000), - edge(500, 6_000), - )), - ), - ( - "query-2".into(), - evidence(unary_statistics( - PhysicalOperator::InMemoryComparisonSort { - ordering_key_count: 1, - partitioned: true, - }, - edge(500, 6_000), - edge(500, 6_000), - )), - ), - ( - "query-1".into(), - evidence(unary_statistics( - PhysicalOperator::Limit { - limit: 20, - offset: 0, - }, - edge(500, 6_000), - edge(20, 240), - )), - ), - ( - "query-0".into(), - evidence(unary_statistics( - PhysicalOperator::PassThrough, - edge(20, 240), - edge(20, 240), - )), - ), - ]); - let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); - - assert_eq!( - dag.nodes - .iter() - .map(|node| node.operator) - .collect::>(), - vec![ - PhysicalOperator::Scan, - PhysicalOperator::Filter { - predicate_operations_per_row: 1, - }, - PhysicalOperator::Project { - expression_operations_per_row: 1, - }, - PhysicalOperator::HashDeduplicate { key_count: 1 }, - PhysicalOperator::InMemoryAnalyticWindow { - partition_key_count: 0, - ordering_key_count: 1, - function_operations_per_row: 1, - }, - PhysicalOperator::InMemoryComparisonSort { - ordering_key_count: 1, - partitioned: true, - }, - PhysicalOperator::Limit { - limit: 20, - offset: 0, - }, - PhysicalOperator::PassThrough, - ] - ); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).is_ok()); - } - - #[test] - fn query_lowering_maps_concat_and_union_all_but_rejects_distinct_set_ops() { - use asap_types::ir::operator::{RelationalSetOpKind, Source}; - use asap_types::ir::schema::{DataType, Field, Schema}; - - let scan = |name: &str| { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: name.into(), - }, - predicates: vec![], - schema: Schema::new(vec![Field::plain("id", DataType::Int64, false)]), - })) - .unwrap() - }; - let union = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::SetOp { - kind: RelationalSetOpKind::Union, - all: true, - left: scan("a"), - right: scan("b"), - })) - .unwrap(); - let scope = scope(vec![ - coverage( - Source::Table { - table_ref: "a".into(), - }, - vec![], - ), - coverage( - Source::Table { - table_ref: "b".into(), - }, - vec![], - ), - ]); - let left_statistics = scan_stats(edge(10, 80), 80); - let right_statistics = scan_stats(edge(20, 160), 160); - let provided = HashMap::from([ - ("query-1".into(), evidence(left_statistics)), - ("query-2".into(), evidence(right_statistics)), - ( - "query-0".into(), - evidence(OperatorStatistics::Concat { - inputs: vec![edge(10, 80), edge(20, 160)], - output: edge(30, 240), - promql: None, - }), - ), - ]); - let dag = lower_query_physical_dag(&union, &scope, &scripted(&provided)).unwrap(); - assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); - - let mut reversed_scope = scope.clone(); - reversed_scope.sources.reverse(); - assert_eq!(validate_comparison_scopes(&scope, &reversed_scope), Ok(1)); - let mut duplicate_scope = scope.clone(); - duplicate_scope.sources.push(scope.sources[0].clone()); - assert_eq!( - duplicate_scope.validate(), - Err(AnalyticalCostError::MissingComparisonScope( - "duplicate scan selection" - )) - ); - - let concat = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![scan("a"), scan("b")], - discriminator_unique_key: None, - })) - .unwrap(); - let dag = lower_query_physical_dag(&concat, &scope, &scripted(&provided)).unwrap(); - assert_eq!(dag.nodes.last().unwrap().operator, PhysicalOperator::Concat); - - let distinct_union = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::SetOp { - kind: RelationalSetOpKind::Union, - all: false, - left: scan("a"), - right: scan("b"), - })) - .unwrap(); - assert_eq!( - lower_query_physical_dag(&distinct_union, &scope, &scripted(&provided)), - Err(AnalyticalCostError::UnsupportedQueryOperator) - ); - } - - #[test] - fn query_lowering_fails_closed_for_missing_or_inconsistent_statistics() { - use asap_types::ir::operator::Source; - use asap_types::ir::schema::{DataType, Field, Schema}; - - let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "events".into(), - }, - predicates: vec![], - schema: Schema::new(vec![Field::plain("id", DataType::Int64, false)]), - })) - .unwrap(); - let root = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - cols: vec![], - qualifier: None, - child: scan, - })) - .unwrap(); - - let comparison_scope = scope(vec![coverage( - Source::Table { - table_ref: "events".into(), - }, - vec![], - )]); - let missing = HashMap::::new(); - assert_eq!( - lower_query_physical_dag(&root, &comparison_scope, &scripted(&missing)), - Err(AnalyticalCostError::MissingOperatorStatistics( - "query-1".into() - )) - ); - struct MissingBuffer; - impl PhysicalNodeEvidenceProvider for MissingBuffer { - fn evidence( - &self, - _request: PhysicalNodeRequest<'_>, - ) -> Result { - Err(AnalyticalCostError::MissingOrStale("output_buffer_bytes")) - } - } - assert_eq!( - lower_query_physical_dag(&root, &comparison_scope, &MissingBuffer), - Err(AnalyticalCostError::MissingOrStale("output_buffer_bytes")) - ); - let scan_statistics = scan_stats(edge(100, 800), 800); - let conflicting = HashMap::from([ - ("query-1".into(), evidence(scan_statistics)), - ( - "query-0".into(), - evidence(unary_statistics( - PhysicalOperator::Project { - expression_operations_per_row: 1, - }, - edge(99, 792), - edge(99, 396), - )), - ), - ]); - assert_eq!( - lower_query_physical_dag(&root, &comparison_scope, &scripted(&conflicting)), - Err(AnalyticalCostError::ConflictingEdgeStatistics { - parent: "query-0".into(), - child: "query-1".into(), - input_index: 0, - }) - ); - - let outside_scope = scope(vec![coverage( - Source::Table { - table_ref: "other".into(), - }, - vec![], - )]); - assert_eq!( - lower_query_physical_dag(&root, &outside_scope, &scripted(&conflicting)), - Err(AnalyticalCostError::ScanOutsideComparisonScope( - "occurrence-1".into() - )) - ); - - let mut second_snapshot = comparison_scope.sources[0].clone(); - second_snapshot.source_snapshot_id = "snapshot-2".into(); - let ambiguous_scope = scope(vec![comparison_scope.sources[0].clone(), second_snapshot]); - assert_eq!( - lower_query_physical_dag(&root, &ambiguous_scope, &scripted(&conflicting)), - Err(AnalyticalCostError::InvalidPhysicalDAG( - "scan selection is ambiguous" - )) - ); - - let mut complete = conflicting.clone(); - complete.get_mut("query-0").unwrap().statistics = unary_statistics( - PhysicalOperator::Project { - expression_operations_per_row: 1, - }, - edge(100, 800), - edge(100, 400), - ); - let extra_scope = scope(vec![ - comparison_scope.sources[0].clone(), - coverage( - Source::Table { - table_ref: "unused".into(), - }, - vec![], - ), - ]); - assert_eq!( - lower_query_physical_dag(&root, &extra_scope, &scripted(&complete)), - Err(AnalyticalCostError::InvalidPhysicalDAG( - "physical scans omit a comparison-scope source" - )) - ); - } - - #[test] - fn query_lowering_accepts_a_consistently_empty_edge() { - use asap_types::ir::operator::{GroupKeys, Source}; - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::schema::{DataType, Field, Schema}; - - let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "events".into(), - }, - predicates: vec![], - schema: Schema::new(vec![Field::plain("id", DataType::Int64, false)]), - })) - .unwrap(); - let filter = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))), - child: scan, - })) - .unwrap(); - let root = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::none(), - child: filter, - })) - .unwrap(); - - let scope = scope(vec![coverage( - Source::Table { - table_ref: "events".into(), - }, - vec![], - )]); - let scan_statistics = scan_stats(edge(100, 800), 800); - let provided = HashMap::from([ - ("query-2".into(), evidence(scan_statistics)), - ( - "query-1".into(), - evidence(unary_statistics( - PhysicalOperator::Filter { - predicate_operations_per_row: 1, - }, - edge(100, 800), - edge(0, 0), - )), - ), - ( - "query-0".into(), - evidence(unary_statistics( - PhysicalOperator::Limit { - limit: 10, - offset: 0, - }, - edge(0, 0), - edge(0, 0), - )), - ), - ]); - let dag = lower_query_physical_dag(&root, &scope, &scripted(&provided)).unwrap(); - let estimate = estimate_physical_dag(&dag.nodes, &dag.root, &scope, &dag.evidence).unwrap(); - assert_eq!(estimate.cpu_ops(), 200.0); - assert_eq!(estimate.scan_bytes(), 800); - } - - #[test] - fn query_lowering_rejects_aggregates_without_a_hash_implementation() { - use asap_types::ir::operator::{AggIntent, GroupKeys, Reduction, Source, WindowFuncKind}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::schema::{DataType, Field, Schema}; - - let scan = || { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "events".into(), - }, - predicates: vec![], - schema: Schema::new(vec![Field::plain("value", DataType::Float64, false)]), - })) - .unwrap() - }; - let exact_quantile = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Quantile { - col: Some(0), - q: 0.99, - accuracy: AccuracyTarget::exact(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: scan(), - })) - .unwrap(); - let empty_sort_limit = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::none(), - child: OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::Sort { - keys: vec![], - partition_by: GroupKeys::none(), - child: scan(), - }, - )) - .unwrap(), - })) - .unwrap(); - let unsupported_window = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::SQLWindowFunc { - func: WindowFuncKind::Lag, - args: vec![ScalarExpr::Column(0)], - partition_by: GroupKeys::none(), - order_by: vec![], - frame: None, - output_name: "lag".into(), - child: scan(), - }, - )) - .unwrap(); - let shifted = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::TimeShift { - shift: asap_types::ir::operator::TimeShift { - offset_ms: 60_000, - at: None, - }, - child: scan(), - })) - .unwrap(); - let scope = scope(vec![coverage( - Source::Table { - table_ref: "events".into(), - }, - vec![], - )]); - let unavailable = HashMap::::new(); - for query in [ - &exact_quantile, - &empty_sort_limit, - &unsupported_window, - &shifted, - ] { - assert_eq!( - lower_query_physical_dag(query, &scope, &scripted(&unavailable)), - Err(AnalyticalCostError::UnsupportedQueryOperator) - ); - } - } - - #[test] - fn scalar_work_counts_every_local_predicate_operation() { - use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; - - let comparison = || ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(0)), - op: CompareOpKind::Eq, - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(1))), - semantics: ExprSemantics::Sql, - }; - let predicate = ScalarExpr::BoolAnd(vec![comparison(), comparison()]); - - assert_eq!(scalar_operation_count(&predicate), Ok(3)); - } - - #[test] - fn promql_presence_is_lowered_with_a_per_step_output_bound() { - use asap_types::ir::operator::{AggIntent, Reduction, Source}; - use asap_types::ir::schema::{DataType, Field, Schema}; - - let source = Source::TimeSeries { - metric: "missing".into(), - }; - let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: source.clone(), - predicates: vec![], - schema: Schema::new(vec![Field::plain("value", DataType::Float64, false)]), - })) - .unwrap(); - let root = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Absent], - output_names: vec![], - filters: vec![], - having: None, - child: scan, - })) - .unwrap(); - let vector = promql_edge(0, 2, PromqlValueKind::Vector); - let scan_statistics = OperatorStatistics::Scan { - edges: promql_unary_edges(edge(0, 0), edge(0, 0), vector, vector), - source_read_bytes: 0, - }; - let output = promql_edge(1, 2, PromqlValueKind::Vector); - let presence_statistics = OperatorStatistics::PromqlPresence { - edges: promql_unary_edges(edge(0, 0), edge(2, 16), vector, output), - }; - let provided = HashMap::from([ - ("query-1".into(), evidence(scan_statistics)), - ("query-0".into(), evidence(presence_statistics.clone())), - ]); - let query_scope = scope(vec![coverage(source, vec![])]); - let dag = lower_query_physical_dag(&root, &query_scope, &scripted(&provided)).unwrap(); - assert!(matches!( - dag.nodes.last().map(|node| node.operator), - Some(PhysicalOperator::PromqlPresence { - kind: PromqlPresenceKind::Absent, - operations_per_row: 1 - }) - )); - - let OperatorStatistics::PromqlPresence { mut edges } = presence_statistics else { - unreachable!() - }; - edges.output = edge(3, 24); - let invalid = HashMap::from([ - ( - "query-1".into(), - evidence(OperatorStatistics::Scan { - edges: promql_unary_edges(edge(0, 0), edge(0, 0), vector, vector), - source_read_bytes: 0, - }), - ), - ( - "query-0".into(), - evidence(OperatorStatistics::PromqlPresence { edges }), - ), - ]); - assert!(matches!( - lower_query_physical_dag(&root, &query_scope, &scripted(&invalid)), - Err(AnalyticalCostError::InconsistentOperatorStatistics(_)) - )); - } - - #[test] - fn promql_range_and_subquery_preserve_internal_steps() { - use asap_types::ir::operator::Source; - use asap_types::ir::schema::{DataType, Field, Schema}; - use std::time::Duration; - - let source = Source::TimeSeries { metric: "m".into() }; - let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: source.clone(), - predicates: vec![], - schema: Schema::new(vec![Field::plain("value", DataType::Float64, false)]), - })) - .unwrap(); - let range = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: scan, - })) - .unwrap(); - let root = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::PromqlSubquery { - range: Duration::from_secs(300), - resolution: Some(Duration::from_secs(60)), - child: range, - }, - )) - .unwrap(); - let vector = promql_edge(10, 6, PromqlValueKind::Vector); - let range_vector = promql_edge(10, 6, PromqlValueKind::RangeVector); - let outer_range = promql_edge(10, 1, PromqlValueKind::RangeVector); - let provided = HashMap::from([ - ( - "query-2".into(), - evidence(OperatorStatistics::Scan { - edges: promql_unary_edges(edge(600, 9_600), edge(600, 9_600), vector, vector), - source_read_bytes: 4_800, - }), - ), - ( - "query-1".into(), - evidence(OperatorStatistics::PromqlRange { - edges: promql_unary_edges( - edge(600, 9_600), - edge(60, 960), - vector, - range_vector, - ), - max_window_samples_per_series: 10, - }), - ), - ( - "query-0".into(), - evidence(OperatorStatistics::PromqlSubquery { - edges: promql_unary_edges( - edge(60, 960), - edge(10, 160), - range_vector, - outer_range, - ), - subquery_steps: 6, - }), - ), - ]); - let query_scope = scope(vec![coverage(source, vec![])]); - let dag = lower_query_physical_dag(&root, &query_scope, &scripted(&provided)).unwrap(); - assert!(matches!( - dag.nodes[1].operator, - PhysicalOperator::PromqlRange { - range_millis: 300_000 - } - )); - assert!(matches!( - dag.nodes[2].operator, - PhysicalOperator::PromqlSubquery { - range_millis: 300_000, - resolution_millis: Some(60_000) - } - )); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &query_scope, &dag.evidence).is_ok()); - } - - #[test] - fn promql_binary_lowering_keeps_operation_and_matching_cardinality() { - use asap_types::ir::operator::{ - BinaryOpKind, GroupSide, Source, VectorGrouping, VectorMatch, VectorMatchKind, - }; - use asap_types::ir::scalar::ArithmeticOpKind; - use asap_types::ir::schema::{DataType, Field, Schema}; - - let left_source = Source::TimeSeries { metric: "a".into() }; - let right_source = Source::TimeSeries { metric: "b".into() }; - let scan = |source| { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source, - predicates: vec![], - schema: Schema::new(vec![Field::plain("value", DataType::Float64, false)]), - })) - .unwrap() - }; - let root = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), - vector_match: Some(VectorMatch { - kind: VectorMatchKind::On, - labels: vec!["service".into()], - grouping: Some(VectorGrouping { - side: GroupSide::Left, - labels: vec!["region".into()], - }), - }), - checked_relative_division: false, - checked_finite_division: false, - }, - return_bool: false, - lhs: scan(left_source.clone()), - rhs: scan(right_source.clone()), - })) - .unwrap(); - let left_promql = promql_edge(10, 10, PromqlValueKind::Vector); - let right_promql = promql_edge(5, 10, PromqlValueKind::Vector); - let output_promql = promql_edge(8, 10, PromqlValueKind::Vector); - let left_edge = edge(100, 1_600); - let right_edge = edge(50, 800); - let output_edge = edge(80, 1_280); - let provided = HashMap::from([ - ( - "query-1".into(), - evidence(OperatorStatistics::Scan { - edges: promql_unary_edges(left_edge, left_edge, left_promql, left_promql), - source_read_bytes: 800, - }), - ), - ( - "query-2".into(), - evidence(OperatorStatistics::Scan { - edges: promql_unary_edges(right_edge, right_edge, right_promql, right_promql), - source_read_bytes: 400, - }), - ), - ( - "query-0".into(), - evidence(OperatorStatistics::PromqlBinary { - edges: BinaryEdgeStatistics { - inputs: [left_edge, right_edge], - output: output_edge, - promql: Some( - crate::cost::physical_operator_statistics::PromqlBinaryEdgeStatistics { - inputs: [left_promql, right_promql], - output: output_promql, - }, - ), - }, - matching_key_bytes: 16, - }), - ), - ]); - let query_scope = scope(vec![ - coverage(left_source, vec![]), - coverage(right_source, vec![]), - ]); - let dag = lower_query_physical_dag(&root, &query_scope, &scripted(&provided)).unwrap(); - assert!(matches!( - dag.nodes.last().map(|node| node.operator), - Some(PhysicalOperator::PromqlBinary { - operation: PromqlBinaryOperation::ArithmeticOrComparison, - operand_mode: PromqlBinaryOperandMode::VectorVector, - cardinality: PromqlVectorCardinality::ManyToOne, - build_side: Some(HashJoinBuildSide::Right), - }) - )); - } - - #[test] - fn promql_relabel_sample_and_per_series_lower_as_a_complete_chain() { - use asap_types::ir::operator::{AggIntent, GroupKeys, Reduction, SampleKind, Source}; - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::schema::{DataType, Field, Schema}; - - let source = Source::TimeSeries { - metric: "requests".into(), - }; - let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: source.clone(), - predicates: vec![], - schema: Schema::new(vec![Field::plain("value", DataType::Float64, false)]), - })) - .unwrap(); - let relabel = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::PromqlRelabel { - dst: "service".into(), - value: ScalarExpr::Literal(ScalarValue::Utf8("api".into())), - child: scan, - }, - )) - .unwrap(); - let sample = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::PromqlSeriesSample { - by: GroupKeys::none(), - kind: SampleKind::LimitK(5), - child: relabel, - }, - )) - .unwrap(); - let root = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: sample, - })) - .unwrap(); - - let input = edge(100, 1_600); - let sampled = edge(50, 800); - let input_promql = promql_edge(10, 10, PromqlValueKind::Vector); - let sampled_promql = promql_edge(5, 10, PromqlValueKind::Vector); - let provided = HashMap::from([ - ( - "query-3".into(), - evidence(OperatorStatistics::Scan { - edges: promql_unary_edges(input, input, input_promql, input_promql), - source_read_bytes: 800, - }), - ), - ( - "query-2".into(), - evidence(OperatorStatistics::PromqlRelabel { - edges: promql_unary_edges(input, input, input_promql, input_promql), - }), - ), - ( - "query-1".into(), - evidence(OperatorStatistics::PromqlSeriesSample { - edges: promql_unary_edges(input, sampled, input_promql, sampled_promql), - group_count: 1, - key_bytes: 8, - }), - ), - ( - "query-0".into(), - evidence(OperatorStatistics::PromqlPerSeries { - edges: promql_unary_edges(sampled, sampled, sampled_promql, sampled_promql), - accumulator_bytes_per_series: 8, - }), - ), - ]); - let query_scope = scope(vec![coverage(source, vec![])]); - - let dag = lower_query_physical_dag(&root, &query_scope, &scripted(&provided)).unwrap(); - assert!(matches!( - dag.nodes.as_slice(), - [ - PhysicalDAGNode { - operator: PhysicalOperator::Scan, - .. - }, - PhysicalDAGNode { - operator: PhysicalOperator::PromqlRelabel { .. }, - .. - }, - PhysicalDAGNode { - operator: PhysicalOperator::PromqlSeriesSample { .. }, - .. - }, - PhysicalDAGNode { - operator: PhysicalOperator::PromqlPerSeries { .. }, - .. - } - ] - )); - assert!(estimate_physical_dag(&dag.nodes, &dag.root, &query_scope, &dag.evidence).is_ok()); - } - - /// `a / scalar(b)` and `a > scalar(b)` read `b` through a - /// `PromqlVectorToScalar` input of the `Project` or `Filter`, after its - /// row input, so the queries lower and cost. - #[test] - fn scalar_reference_in_an_expression_is_a_scalar_input() { - for query in [ - "rate(a[5m]) / scalar(rate(b[5m]))", - "rate(a[5m]) > scalar(rate(b[5m]))", - ] { - assert_scalar_reference_is_a_scalar_input(query, false, 1); - } - } - - /// Each distinct reference is one scalar input of the `Project` or - /// `Filter`; one node read twice is one input. - #[test] - fn scalar_references_are_one_input_per_distinct_reference() { - for query in [ - "rate(a[5m]) / (scalar(rate(b[5m])) + scalar(rate(c[5m])))", - "rate(a[5m]) > (scalar(rate(b[5m])) + scalar(rate(c[5m])))", - ] { - assert_scalar_reference_is_a_scalar_input(query, false, 2); - assert_scalar_reference_is_a_scalar_input(query, true, 1); - } - } - - fn assert_scalar_reference_is_a_scalar_input( - query: &str, - same_reference: bool, - scalar_inputs: usize, - ) { - use crate::cost::physical_operator_statistics::ScalarInputStatistics; - use asap_types::workload::{ - BatchEntry, DataWorkload, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, - }; - use std::cell::RefCell; - - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy: AccuracyTarget::exact(), - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1_000), - )), - }; - let root = asap_frontend_promql::lower_promql_workload(&workload) - .unwrap() - .remove(0); - // Read one node through every reference (as a planner that shares - // the identical references would), so the input dedupe is tested. - let root = if same_reference { - let children = root.children(); - let (row, first) = (Rc::clone(children[0]), Rc::clone(children[1])); - OperatorNode::new_shared(root.operator.map_children(|child| { - if Rc::ptr_eq(child, &row) { - Rc::clone(&row) - } else { - Rc::clone(&first) - } - })) - .unwrap() - } else { - root - }; - - let vector = |series| promql_edge(series, 10, PromqlValueKind::Vector); - let produced: RefCell> = RefCell::default(); - let provider = |request: PhysicalNodeRequest<'_>| { - let produced_now = produced.borrow(); - let inputs: Vec<&OperatorStatistics> = request - .children - .iter() - .map(|id| &produced_now[id]) - .collect(); - let input = || inputs[0].output(); - let promql_input = || inputs[0].promql_output().unwrap(); - let scalar_inputs = || { - inputs[1..] - .iter() - .map(|scalar| ScalarInputStatistics { - edge: scalar.output(), - promql: scalar.promql_output(), - }) - .collect() - }; - let statistics = match request.operator { - PhysicalOperator::Scan => OperatorStatistics::Scan { - edges: promql_unary_edges( - edge(100, 1_600), - edge(100, 1_600), - vector(10), - vector(10), - ), - source_read_bytes: 1_600, - }, - PhysicalOperator::PromqlRange { .. } => OperatorStatistics::PromqlRange { - edges: promql_unary_edges( - input(), - input(), - promql_input(), - promql_edge(10, 10, PromqlValueKind::RangeVector), - ), - max_window_samples_per_series: 10, - }, - PhysicalOperator::PromqlPerSeries { .. } => OperatorStatistics::PromqlPerSeries { - edges: promql_unary_edges(input(), edge(10, 160), promql_input(), vector(10)), - accumulator_bytes_per_series: 8, - }, - PhysicalOperator::PromqlVectorToScalar => { - OperatorStatistics::PromqlVectorToScalar { - edges: promql_unary_edges( - input(), - edge(10, 160), - promql_input(), - promql_edge(0, 10, PromqlValueKind::Scalar), - ), - } - } - PhysicalOperator::Project { .. } => OperatorStatistics::Project { - edges: promql_unary_edges(input(), input(), promql_input(), promql_input()), - scalar_inputs: scalar_inputs(), - }, - PhysicalOperator::Filter { .. } => OperatorStatistics::Filter { - edges: promql_unary_edges(input(), input(), promql_input(), promql_input()), - scalar_inputs: scalar_inputs(), - }, - other => panic!("unexpected physical operator {other:?}"), - }; - drop(produced_now); - let physical_id = format!("query-{}", request.occurrence); - produced - .borrow_mut() - .insert(physical_id.clone(), statistics.clone()); - Ok(PhysicalNodeEvidence { - physical_id, - ..evidence(statistics) - }) - }; - let query_scope = scope( - OperatorNode::reachable(&root) - .iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => Some(coverage(source.clone(), vec![])), - _ => None, - }) - .collect(), - ); - let dag = lower_query_physical_dag(&root, &query_scope, &provider).unwrap(); - let root_node = dag.nodes.iter().find(|node| node.id == dag.root).unwrap(); - assert!( - matches!( - root_node.operator, - PhysicalOperator::Project { .. } | PhysicalOperator::Filter { .. } - ), - "{query}: {:?}", - root_node.operator - ); - assert_eq!(root_node.children.len(), 1 + scalar_inputs, "{query}"); - for child in &root_node.children[1..] { - let scalar = dag.nodes.iter().find(|node| node.id == *child).unwrap(); - assert_eq!(scalar.operator, PhysicalOperator::PromqlVectorToScalar); - } - estimate_physical_dag(&dag.nodes, &dag.root, &query_scope, &dag.evidence).unwrap(); - } -} diff --git a/crates/plan-selection/src/cost/storage_io.rs b/crates/plan-selection/src/cost/storage_io.rs deleted file mode 100644 index ba2363ffe..000000000 --- a/crates/plan-selection/src/cost/storage_io.rs +++ /dev/null @@ -1,261 +0,0 @@ -//! Request counts for explicitly bound physical storage accesses. - -use std::collections::{HashMap, HashSet}; - -use serde::{Deserialize, Serialize}; - -// Keep the original public import path while sharing the sole type definition. -pub use asap_types::workload::resources::StorageResources; - -use crate::cost::analytical_cost::{ - estimate_physical_dag, AnalyticalCostError, EvidenceBackedPhysicalDAG, ExecutionMultiplicity, - PhysicalDAGNode, PhysicalOperator, -}; -use crate::cost::physical_operator_statistics::{ComparisonScope, OperatorStatistics}; - -pub const STORAGE_IO_MODEL_VERSION: &str = "storage-requests-v1"; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum StorageOperation { - DiskRead, - DiskWrite, - ObjectGet, - ObjectPut, -} - -/// Each extent is one independent contiguous disk range, object, or multipart -/// payload. Requests cannot coalesce across extent boundaries. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct StorageAccess { - pub operation: StorageOperation, - pub extent_bytes: Vec, - pub bytes_per_request: u64, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct StorageCalibration { - pub version: String, - pub cost_per_disk_read: f64, - pub cost_per_disk_write: f64, - pub cost_per_object_get: f64, - pub cost_per_object_put: f64, -} - -impl StorageCalibration { - pub fn cost(&self, value: StorageResources) -> Result { - if self.version.trim().is_empty() { - return Err(invalid("blank storage calibration version")); - } - let mut cost = 0.0; - for (coefficient, count) in [ - (self.cost_per_disk_read, value.disk_reads), - (self.cost_per_disk_write, value.disk_writes), - (self.cost_per_object_get, value.object_gets), - (self.cost_per_object_put, value.object_puts), - ] { - if !coefficient.is_finite() || coefficient < 0.0 { - return Err(invalid("invalid storage calibration coefficient")); - } - cost += coefficient * count as f64; - } - if !cost.is_finite() { - return Err(AnalyticalCostError::Overflow); - } - Ok(cost) - } -} - -/// Atomic deployment snapshot. Every reachable physical node needs an entry, -/// including an explicit empty access list for memory-only operators. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct StorageIoProfile { - pub evidence_version: String, - pub observed_at_ms: u64, - pub valid_until_ms: u64, - pub calibration: StorageCalibration, - pub nodes: HashMap, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct StorageNodeEvidence { - pub node: PhysicalDAGNode, - pub statistics: OperatorStatistics, - pub accesses: Vec, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct StorageEstimate { - pub total: StorageResources, - pub per_node: HashMap, - pub model_version: String, - pub evidence_version: String, - pub calibration_version: String, - pub cost: f64, -} - -fn invalid(reason: &'static str) -> AnalyticalCostError { - AnalyticalCostError::InvalidPhysicalDAG(reason) -} - -/// ceil(bytes/request_size), rounded independently per extent and execution. -/// No metadata, retry, or speculative prefetch operations are inferred. -pub fn request_count(access: &StorageAccess) -> Result { - if access.bytes_per_request == 0 { - return Err(invalid("storage request size must be positive")); - } - access.extent_bytes.iter().try_fold(0_u64, |sum, bytes| { - let count = - bytes / access.bytes_per_request + u64::from(bytes % access.bytes_per_request != 0); - sum.checked_add(count).ok_or(AnalyticalCostError::Overflow) - }) -} - -pub fn estimate_storage_io( - dag: &EvidenceBackedPhysicalDAG, - scope: &ComparisonScope, - profile: &StorageIoProfile, - evidence_version: &str, -) -> Result { - // Also prove scan selection, edge consistency, execution legality and DAG - // identity before using supplementary deployment evidence. - estimate_physical_dag(&dag.nodes, &dag.root, scope, dag)?; - let evaluations = scope.validate()?; - if profile.evidence_version.trim().is_empty() - || profile.evidence_version != evidence_version - || profile.observed_at_ms > scope.planning_time.0 - || profile.valid_until_ms <= scope.planning_time.0 - { - return Err(AnalyticalCostError::MissingOrStale("storage I/O profile")); - } - let by_id: HashMap<_, _> = dag - .nodes - .iter() - .map(|node| (node.id.as_str(), node)) - .collect(); - let mut seen = HashSet::new(); - let mut pending = vec![dag.root.as_str()]; - let mut total = StorageResources::default(); - let mut per_node = HashMap::new(); - while let Some(id) = pending.pop() { - if !seen.insert(id) { - continue; - } - let node = by_id[id]; - pending.extend(node.children.iter().map(String::as_str)); - let evidence = profile.nodes.get(id).ok_or_else(|| { - AnalyticalCostError::MissingOperatorStatistics(format!("storage:{id}")) - })?; - if evidence.node != *node || evidence.statistics != dag.evidence[id].statistics { - return Err(invalid( - "storage evidence differs from physical node snapshot", - )); - } - let executions = match node.execution { - ExecutionMultiplicity::Once => 1, - ExecutionMultiplicity::PerEvaluation => evaluations, - }; - let mut local = StorageResources::default(); - let mut read_bytes = 0_u64; - for access in &evidence.accesses { - let count = request_count(access)? - .checked_mul(executions) - .ok_or(AnalyticalCostError::Overflow)?; - let mut term = StorageResources::default(); - match access.operation { - StorageOperation::DiskRead => term.disk_reads = count, - StorageOperation::DiskWrite => term.disk_writes = count, - StorageOperation::ObjectGet => term.object_gets = count, - StorageOperation::ObjectPut => term.object_puts = count, - } - if matches!( - access.operation, - StorageOperation::DiskRead | StorageOperation::ObjectGet - ) { - for bytes in &access.extent_bytes { - read_bytes = read_bytes - .checked_add(*bytes) - .ok_or(AnalyticalCostError::Overflow)?; - } - } - local = local - .checked_add(term) - .ok_or(AnalyticalCostError::Overflow)?; - } - if node.operator == PhysicalOperator::Scan { - let OperatorStatistics::Scan { - source_read_bytes, .. - } = &evidence.statistics - else { - unreachable!() - }; - if read_bytes != *source_read_bytes { - return Err(invalid( - "storage scan access bytes differ from source read bytes", - )); - } - } - total = total - .checked_add(local) - .ok_or(AnalyticalCostError::Overflow)?; - per_node.insert(id.into(), local); - } - Ok(StorageEstimate { - total, - per_node, - model_version: STORAGE_IO_MODEL_VERSION.into(), - evidence_version: profile.evidence_version.clone(), - calibration_version: profile.calibration.version.clone(), - cost: profile.calibration.cost(total)?, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - // Separate objects need separate requests even when their total fits. - #[test] - fn counts_extents_independently_and_is_monotone() { - let mut access = StorageAccess { - operation: StorageOperation::ObjectGet, - extent_bytes: vec![5, 5], - bytes_per_request: 8, - }; - assert_eq!(request_count(&access).unwrap(), 2); - access.bytes_per_request = 4; - assert_eq!(request_count(&access).unwrap(), 4); - access.extent_bytes.push(9); - assert_eq!(request_count(&access).unwrap(), 7); - access.bytes_per_request = 0; - assert!(request_count(&access).is_err()); - } - // Full-width integer evidence must neither wrap nor lose precision. - #[test] - fn checked_rounding_handles_maximum_bytes_and_overflow() { - let mut access = StorageAccess { - operation: StorageOperation::DiskRead, - extent_bytes: vec![u64::MAX], - bytes_per_request: 2, - }; - assert_eq!(request_count(&access).unwrap(), 1_u64 << 63); - access.bytes_per_request = 1; - access.extent_bytes.push(1); - assert_eq!(request_count(&access), Err(AnalyticalCostError::Overflow)); - } - // Invalid deployment coefficients cannot produce a usable cost. - #[test] - fn rejects_non_finite_calibration() { - let calibration = StorageCalibration { - version: "v1".into(), - cost_per_disk_read: f64::NAN, - cost_per_disk_write: 0.0, - cost_per_object_get: 0.0, - cost_per_object_put: 0.0, - }; - assert!(calibration.cost(StorageResources::default()).is_err()); - } -} diff --git a/crates/plan-selection/tests/physical_handoff_cost.rs b/crates/plan-selection/tests/physical_handoff_cost.rs deleted file mode 100644 index 1b9de583b..000000000 --- a/crates/plan-selection/tests/physical_handoff_cost.rs +++ /dev/null @@ -1,447 +0,0 @@ -use asap_plan_selection::cost::analytical_cost::{ - EvidenceBackedPhysicalDAG, ExecutionMultiplicity, PhysicalDAGNode, PhysicalNodeEvidence, - PhysicalOperator, -}; -use asap_plan_selection::cost::physical_operator_statistics::{ - ComparisonScope, EdgeStatistics, OperatorStatistics, ScanSelection, UnaryEdgeStatistics, -}; -use asap_types::ir::operator::operator_properties::Source; -use asap_types::workload::{DataArrival, DurationMs, QueryRecurrence, TimestampMs}; -use std::collections::HashMap; - -fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { - let coverage = ScanSelection { - source: Source::Table { - table_ref: "events".into(), - }, - source_snapshot_id: "snapshot".into(), - predicates: vec![], - info_matchers: vec![], - }; - let scope = ComparisonScope { - data_arrival: DataArrival::AtRest, - planning_time: TimestampMs(100), - horizon: DurationMs(1000), - recurrence: QueryRecurrence::OneTime { - invocations: 3, - execute_at: None, - }, - sources: vec![coverage.clone()], - }; - let edge = EdgeStatistics { - rows: 10, - bytes: 80, - }; - let unary = UnaryEdgeStatistics { - input: edge, - output: edge, - promql: None, - }; - let nodes = vec![ - PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(coverage), - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "left".into(), - operator: PhysicalOperator::PassThrough, - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "right".into(), - operator: PhysicalOperator::PassThrough, - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "root".into(), - operator: PhysicalOperator::Concat, - children: vec!["left".into(), "right".into()], - scan_selection: None, - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - ]; - let evidence = HashMap::from([ - ( - "scan".into(), - PhysicalNodeEvidence { - physical_id: "scan".into(), - statistics: OperatorStatistics::Scan { - edges: unary, - source_read_bytes: 80, - }, - output_buffer_bytes: 0, - }, - ), - ( - "left".into(), - PhysicalNodeEvidence { - physical_id: "left".into(), - statistics: OperatorStatistics::PassThrough { edges: unary }, - output_buffer_bytes: 0, - }, - ), - ( - "right".into(), - PhysicalNodeEvidence { - physical_id: "right".into(), - statistics: OperatorStatistics::PassThrough { edges: unary }, - output_buffer_bytes: 0, - }, - ), - ( - "root".into(), - PhysicalNodeEvidence { - physical_id: "root".into(), - statistics: OperatorStatistics::Concat { - inputs: vec![edge, edge], - output: EdgeStatistics { - rows: 20, - bytes: 160, - }, - promql: None, - }, - output_buffer_bytes: 0, - }, - ), - ]); - ( - EvidenceBackedPhysicalDAG { - nodes, - root: "root".into(), - evidence, - }, - scope, - ) -} - -use asap_plan_selection::cost::physical_handoff_cost::*; - -// Legacy mapping imports and the shared resource API are the very same Rust types. -#[test] -fn mapping_resource_reexports_are_wire_compatible_shared_types() { - let shared = asap_types::workload::resources::PhysicalHandoffBytes { - network_bytes: 480, - materialization_bytes: 40, - }; - let legacy: PhysicalHandoffBytes = shared; - assert_eq!( - serde_json::to_value(legacy).unwrap(), - serde_json::json!({"network_bytes": 480, "materialization_bytes": 40}) - ); - let shared_kind = asap_types::workload::resources::PhysicalHandoffKind::Materialization; - let mut handoff = transfer("persist", None); - handoff.kind = shared_kind; - let json = serde_json::to_value(&handoff).unwrap(); - assert_eq!(json["kind"], serde_json::json!({"kind": "materialization"})); - assert_eq!( - serde_json::from_value::(json).unwrap(), - handoff - ); -} - -fn profile(dag: &EvidenceBackedPhysicalDAG) -> PhysicalHandoffProfile { - PhysicalHandoffProfile { - evidence_version: "evidence-v1".into(), - observed_at_ms: 90, - valid_until_ms: 200, - calibration: PhysicalHandoffCalibration { - version: "bytes-v1".into(), - cost_per_network_byte: 2.0, - cost_per_materialization_byte: 3.0, - }, - plans: vec![PhysicalHandoffPlanEvidence { - root: dag.root.clone(), - nodes: dag - .nodes - .iter() - .map(|node| { - ( - node.id.clone(), - PhysicalHandoffNodeEvidence { - node: node.clone(), - statistics: dag.evidence[&node.id].statistics.clone(), - handoffs: vec![], - }, - ) - }) - .collect(), - }], - } -} - -fn transfer(id: &str, consumer: Option<&str>) -> PhysicalHandoff { - PhysicalHandoff { - id: id.into(), - consumer: consumer.map(str::to_owned), - kind: PhysicalHandoffKind::Network { - source_location: "edge".into(), - destination_location: "backend".into(), - }, - logical_bytes: 80, - encoded_bytes: 40, - copies: 2, - } -} - -fn profiles_for_alternatives(dags: &[&EvidenceBackedPhysicalDAG]) -> PhysicalHandoffProfile { - let mut combined = profile(dags[0]); - combined.plans.clear(); - for dag in dags { - let mut alternative = profile(dag); - alternative.plans[0].nodes.get_mut("scan").unwrap().handoffs = - vec![transfer(&format!("wire-{}", dag.root), Some(&dag.root))]; - combined.plans.extend(alternative.plans); - } - combined -} - -// Alternative consumers may share a producer identity without sharing transfers. -#[test] -fn shared_producer_handoffs_are_scoped_to_each_alternative() { - let (dag, scope) = fixture(); - let alternative = |root: &str| { - let mut value = dag.clone(); - value.root = root.into(); - value - .nodes - .retain(|node| node.id == "scan" || node.id == root); - value.evidence.retain(|id, _| id == "scan" || id == root); - value - }; - let raw = alternative("left"); - let candidate = alternative("right"); - let profile = profiles_for_alternatives(&[&raw, &candidate]); - for plan in [&raw, &candidate] { - let estimate = estimate_physical_handoffs(plan, &scope, &profile, "evidence-v1").unwrap(); - assert_eq!(estimate.total.network_bytes, 240); - assert_eq!(estimate.per_handoff.len(), 1); - assert!(estimate - .per_handoff - .contains_key(&format!("wire-{}", plan.root))); - } -} - -// A handoff binding must identify exactly one complete physical alternative. -#[test] -fn missing_ambiguous_and_mismatched_plan_bindings_are_rejected() { - let (dag, scope) = fixture(); - for case in 0..6 { - let mut profile = profile(&dag); - match case { - 0 => profile.plans.clear(), - 1 => profile.plans.push(profile.plans[0].clone()), - 2 => profile.plans[0].root = "left".into(), - 3 => { - profile.plans[0].nodes.remove("left"); - } - 4 => { - profile.plans[0] - .nodes - .get_mut("scan") - .unwrap() - .node - .execution = ExecutionMultiplicity::Once - } - 5 => { - let OperatorStatistics::Scan { - source_read_bytes, .. - } = &mut profile.plans[0].nodes.get_mut("scan").unwrap().statistics - else { - unreachable!() - }; - *source_read_bytes += 1; - } - _ => unreachable!(), - } - assert!( - estimate_physical_handoffs(&dag, &scope, &profile, "evidence-v1").is_err(), - "case {case}" - ); - } -} - -// Ordinary in-memory edges contribute no traffic; shared transfers count once. -#[test] -fn memory_edges_are_free_and_shared_transfer_is_counted_once() { - let (dag, scope) = fixture(); - let mut profile = profile(&dag); - assert_eq!( - estimate_physical_handoffs(&dag, &scope, &profile, "evidence-v1") - .unwrap() - .total, - PhysicalHandoffBytes::default() - ); - profile.plans[0] - .nodes - .get_mut("scan") - .unwrap() - .handoffs - .push(transfer("shared", None)); - let estimate = estimate_physical_handoffs(&dag, &scope, &profile, "evidence-v1").unwrap(); - assert_eq!(estimate.total.network_bytes, 240); // 40 encoded bytes × 2 replicas × 3 evaluations - assert_eq!(estimate.total.materialization_bytes, 0); - assert_eq!(estimate.cost, 480.0); - assert_eq!(estimate.per_node["scan"].network_bytes, 240); - assert_eq!(estimate.per_handoff["shared"].network_bytes, 240); -} - -// A retained producer materializes once and transfers separately to each reader. -#[test] -fn materialization_once_and_transfers_per_consumer_have_distinct_multiplicity() { - let (mut dag, scope) = fixture(); - dag.nodes[0].execution = ExecutionMultiplicity::Once; - dag.nodes[0].retained_bytes = 80; - let mut profile = profile(&dag); - let mut materialize = transfer("persist", None); - materialize.kind = PhysicalHandoffKind::Materialization; - materialize.copies = 1; - profile.plans[0].nodes.get_mut("scan").unwrap().handoffs = vec![ - materialize, - transfer("left-wire", Some("left")), - transfer("right-wire", Some("right")), - ]; - let estimate = estimate_physical_handoffs(&dag, &scope, &profile, "evidence-v1").unwrap(); - assert_eq!(estimate.total.materialization_bytes, 40); - assert_eq!(estimate.total.network_bytes, 480); - assert_eq!(estimate.cost, 1080.0); -} - -// Invalid endpoint, byte evidence, duplicate identity, and overflow fail closed. -#[test] -fn incompatible_handoff_evidence_is_rejected() { - let (dag, scope) = fixture(); - for case in 0..7 { - let mut profile = profile(&dag); - let mut handoff = transfer("wire", Some("left")); - match case { - 0 => handoff.consumer = Some("root".into()), - 1 => handoff.logical_bytes = 79, - 2 => handoff.copies = 0, - 3 => handoff.encoded_bytes = 0, - 4 => { - handoff.kind = PhysicalHandoffKind::Network { - source_location: "same".into(), - destination_location: "same".into(), - } - } - 5 => handoff.copies = u64::MAX, - 6 => profile.plans[0] - .nodes - .get_mut("scan") - .unwrap() - .handoffs - .push(handoff.clone()), - _ => unreachable!(), - } - profile.plans[0] - .nodes - .get_mut("scan") - .unwrap() - .handoffs - .push(handoff); - assert!( - estimate_physical_handoffs(&dag, &scope, &profile, "evidence-v1").is_err(), - "case {case}" - ); - } -} - -// Missing evidence and non-finite coefficients cannot be treated as zero. -#[test] -fn missing_stale_and_non_finite_evidence_is_rejected() { - let (dag, scope) = fixture(); - let mut evidence = profile(&dag); - evidence.plans[0].nodes.remove("left"); - assert!(estimate_physical_handoffs(&dag, &scope, &evidence, "evidence-v1").is_err()); - evidence = profile(&dag); - evidence.valid_until_ms = 100; - assert!(estimate_physical_handoffs(&dag, &scope, &evidence, "evidence-v1").is_err()); - evidence = profile(&dag); - evidence.calibration.cost_per_network_byte = f64::INFINITY; - assert!(estimate_physical_handoffs(&dag, &scope, &evidence, "evidence-v1").is_err()); - assert!(estimate_physical_handoffs(&dag, &scope, &profile(&dag), "different").is_err()); -} - -// Individual actions may fit while accumulation across actions or nodes overflows. -#[test] -fn handoff_accumulation_and_calibrated_cost_overflow_are_rejected() { - use asap_plan_selection::cost::analytical_cost::AnalyticalCostError; - let (mut dag, mut scope) = fixture(); - scope.recurrence = QueryRecurrence::OneTime { - invocations: 1, - execute_at: None, - }; - for node in &mut dag.nodes { - node.execution = ExecutionMultiplicity::Once; - } - for different_nodes in [false, true] { - let mut evidence = profile(&dag); - for (id, node) in [ - ("first", "scan"), - ("second", if different_nodes { "left" } else { "scan" }), - ] { - let mut handoff = transfer(id, None); - handoff.copies = 1; - handoff.encoded_bytes = u64::MAX / 2 + 1; - evidence.plans[0] - .nodes - .get_mut(node) - .unwrap() - .handoffs - .push(handoff); - } - assert!(matches!( - estimate_physical_handoffs(&dag, &scope, &evidence, "evidence-v1"), - Err(AnalyticalCostError::Overflow) - )); - } - let mut evidence = profile(&dag); - evidence.plans[0] - .nodes - .get_mut("scan") - .unwrap() - .handoffs - .push(transfer("wire", None)); - evidence.calibration.cost_per_network_byte = f64::MAX; - assert!(matches!( - estimate_physical_handoffs(&dag, &scope, &evidence, "evidence-v1"), - Err(AnalyticalCostError::Overflow) - )); -} - -// Renamed Rust fields accept existing evidence and preserve per-action estimate JSON. -#[test] -fn handoff_wire_fields_remain_compatible() { - let (dag, scope) = fixture(); - let mut evidence = profile(&dag); - evidence.plans[0].nodes.get_mut("scan").unwrap().handoffs = vec![transfer("wire", None)]; - let wire = serde_json::to_value(&evidence).unwrap(); - let node = &wire["plans"][0]["nodes"]["scan"]; - assert!(node.get("handoffs").is_none()); - assert_eq!(node["boundaries"][0]["id"], "wire"); - let restored: PhysicalHandoffProfile = serde_json::from_value(wire).unwrap(); - assert_eq!(restored, evidence); - let estimate = estimate_physical_handoffs(&dag, &scope, &restored, "evidence-v1").unwrap(); - let wire = serde_json::to_value(&estimate).unwrap(); - assert!(wire.get("per_handoff").is_none()); - assert_eq!(wire["per_boundary"]["wire"]["network_bytes"], 240); - assert_eq!( - serde_json::from_value::(wire).unwrap(), - estimate - ); -} diff --git a/crates/plan-selection/tests/storage_io.rs b/crates/plan-selection/tests/storage_io.rs deleted file mode 100644 index 3925303f7..000000000 --- a/crates/plan-selection/tests/storage_io.rs +++ /dev/null @@ -1,368 +0,0 @@ -use asap_plan_selection::cost::analytical_cost::{ - EvidenceBackedPhysicalDAG, ExecutionMultiplicity, PhysicalDAGNode, PhysicalNodeEvidence, - PhysicalOperator, -}; -use asap_plan_selection::cost::physical_operator_statistics::{ - ComparisonScope, EdgeStatistics, OperatorStatistics, ScanSelection, UnaryEdgeStatistics, -}; -use asap_types::ir::operator::operator_properties::Source; -use asap_types::workload::{DataArrival, DurationMs, QueryRecurrence, TimestampMs}; -use std::collections::HashMap; - -fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { - let coverage = ScanSelection { - source: Source::Table { - table_ref: "events".into(), - }, - source_snapshot_id: "snapshot".into(), - predicates: vec![], - info_matchers: vec![], - }; - let scope = ComparisonScope { - data_arrival: DataArrival::AtRest, - planning_time: TimestampMs(100), - horizon: DurationMs(1000), - recurrence: QueryRecurrence::OneTime { - invocations: 3, - execute_at: None, - }, - sources: vec![coverage.clone()], - }; - let edge = EdgeStatistics { - rows: 10, - bytes: 80, - }; - let unary = UnaryEdgeStatistics { - input: edge, - output: edge, - promql: None, - }; - let nodes = vec![ - PhysicalDAGNode { - id: "scan".into(), - operator: PhysicalOperator::Scan, - children: vec![], - scan_selection: Some(coverage), - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "left".into(), - operator: PhysicalOperator::PassThrough, - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "right".into(), - operator: PhysicalOperator::PassThrough, - children: vec!["scan".into()], - scan_selection: None, - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - PhysicalDAGNode { - id: "root".into(), - operator: PhysicalOperator::Concat, - children: vec!["left".into(), "right".into()], - scan_selection: None, - output_buffer_bytes: 0, - retained_bytes: 0, - execution: ExecutionMultiplicity::PerEvaluation, - }, - ]; - let evidence = HashMap::from([ - ( - "scan".into(), - PhysicalNodeEvidence { - physical_id: "scan".into(), - statistics: OperatorStatistics::Scan { - edges: unary, - source_read_bytes: 80, - }, - output_buffer_bytes: 0, - }, - ), - ( - "left".into(), - PhysicalNodeEvidence { - physical_id: "left".into(), - statistics: OperatorStatistics::PassThrough { edges: unary }, - output_buffer_bytes: 0, - }, - ), - ( - "right".into(), - PhysicalNodeEvidence { - physical_id: "right".into(), - statistics: OperatorStatistics::PassThrough { edges: unary }, - output_buffer_bytes: 0, - }, - ), - ( - "root".into(), - PhysicalNodeEvidence { - physical_id: "root".into(), - statistics: OperatorStatistics::Concat { - inputs: vec![edge, edge], - output: EdgeStatistics { - rows: 20, - bytes: 160, - }, - promql: None, - }, - output_buffer_bytes: 0, - }, - ), - ]); - ( - EvidenceBackedPhysicalDAG { - nodes, - root: "root".into(), - evidence, - }, - scope, - ) -} - -use asap_plan_selection::cost::storage_io::*; - -// The compatibility import and shared resource namespace expose one Rust type. -#[test] -fn storage_estimates_use_the_shared_resource_type_without_wire_changes() { - let (dag, scope) = fixture(); - let estimate = estimate_storage_io(&dag, &scope, &profile(&dag), "evidence-v1").unwrap(); - let shared: asap_types::workload::resources::StorageResources = estimate.total; - let legacy: asap_plan_selection::cost::storage_io::StorageResources = shared; - assert_eq!(shared, legacy); - let wire = serde_json::to_value(&estimate).unwrap(); - assert_eq!( - wire["total"], - serde_json::json!({ - "disk_reads": 0, "disk_writes": 0, "object_gets": 12, "object_puts": 0, - }) - ); - assert_eq!( - serde_json::from_value::(wire).unwrap(), - estimate - ); -} - -fn profile(dag: &EvidenceBackedPhysicalDAG) -> StorageIoProfile { - StorageIoProfile { - evidence_version: "evidence-v1".into(), - observed_at_ms: 90, - valid_until_ms: 200, - calibration: StorageCalibration { - version: "requests-v1".into(), - cost_per_disk_read: 1.0, - cost_per_disk_write: 2.0, - cost_per_object_get: 3.0, - cost_per_object_put: 4.0, - }, - nodes: dag - .nodes - .iter() - .map(|node| { - ( - node.id.clone(), - StorageNodeEvidence { - node: node.clone(), - statistics: dag.evidence[&node.id].statistics.clone(), - accesses: if node.id == "scan" { - vec![StorageAccess { - operation: StorageOperation::ObjectGet, - extent_bytes: vec![40, 40], - bytes_per_request: 32, - }] - } else { - vec![] - }, - }, - ) - }) - .collect(), - } -} - -// The shared scan reads two objects once per evaluation, despite two parents. -#[test] -fn shared_io_counts_rounding_then_evaluations_and_explicit_writes() { - let (dag, scope) = fixture(); - let mut profile = profile(&dag); - profile - .nodes - .get_mut("root") - .unwrap() - .accesses - .push(StorageAccess { - operation: StorageOperation::DiskWrite, - extent_bytes: vec![160], - bytes_per_request: 64, - }); - let estimate = estimate_storage_io(&dag, &scope, &profile, "evidence-v1").unwrap(); - assert_eq!(estimate.total.object_gets, 12); - assert_eq!(estimate.total.disk_writes, 9); - assert_eq!(estimate.cost, 54.0); - assert_eq!(estimate.per_node["scan"].object_gets, 12); - assert_eq!(estimate.per_node["left"], StorageResources::default()); - assert_eq!(estimate.model_version, STORAGE_IO_MODEL_VERSION); -} - -// A retained scan is read once; downstream in-memory consumers add no reads. -#[test] -fn retained_state_does_not_reread_storage() { - let (mut dag, scope) = fixture(); - dag.nodes[0].execution = ExecutionMultiplicity::Once; - dag.nodes[0].retained_bytes = 80; - let estimate = estimate_storage_io(&dag, &scope, &profile(&dag), "evidence-v1").unwrap(); - assert_eq!(estimate.total.object_gets, 4); -} - -// Incomplete, expired, rebound, or inconsistent scan evidence cannot be ranked. -#[test] -fn incomplete_stale_and_incompatible_evidence_is_rejected() { - let (dag, scope) = fixture(); - let original = profile(&dag); - let mut missing = original.clone(); - missing.nodes.remove("left"); - assert!(estimate_storage_io(&dag, &scope, &missing, "evidence-v1").is_err()); - let mut stale = original.clone(); - stale.valid_until_ms = 100; - assert!(estimate_storage_io(&dag, &scope, &stale, "evidence-v1").is_err()); - stale.valid_until_ms = 200; - stale.observed_at_ms = 101; - assert!(estimate_storage_io(&dag, &scope, &stale, "evidence-v1").is_err()); - let mut mismatch = original.clone(); - mismatch.nodes.get_mut("scan").unwrap().accesses[0].extent_bytes = vec![79]; - assert!(estimate_storage_io(&dag, &scope, &mismatch, "evidence-v1").is_err()); - mismatch = original.clone(); - mismatch.nodes.get_mut("scan").unwrap().node.execution = ExecutionMultiplicity::Once; - assert!(estimate_storage_io(&dag, &scope, &mismatch, "evidence-v1").is_err()); - assert!(estimate_storage_io(&dag, &scope, &original, "another-version").is_err()); -} - -// Zero and non-finite values in wire evidence are rejected without defaults. -#[test] -fn wire_evidence_is_strict() { - let (dag, _) = fixture(); - let mut json = serde_json::to_value(profile(&dag)).unwrap(); - json["nodes"]["scan"]["accesses"][0]["bytes_per_request"] = serde_json::json!(1.5); - assert!(serde_json::from_value::(json).is_err()); - let mut json = serde_json::to_value(profile(&dag)).unwrap(); - json["calibration"]["cost_per_disk_read"] = serde_json::Value::Null; - assert!(serde_json::from_value::(json).is_err()); -} - -// Zero-length payloads add no requests, without turning absent reads into zero. -#[test] -fn zero_extents_and_all_request_kinds_preserve_dimensions() { - let (dag, scope) = fixture(); - let mut profile = profile(&dag); - profile.nodes.get_mut("scan").unwrap().accesses[0] - .extent_bytes - .push(0); - profile.nodes.get_mut("root").unwrap().accesses = [ - StorageOperation::DiskRead, - StorageOperation::DiskWrite, - StorageOperation::ObjectGet, - StorageOperation::ObjectPut, - ] - .into_iter() - .map(|operation| StorageAccess { - operation, - extent_bytes: vec![0, 1, 9], - bytes_per_request: 8, - }) - .collect(); - let estimate = estimate_storage_io(&dag, &scope, &profile, "evidence-v1").unwrap(); - assert_eq!( - estimate.total, - StorageResources { - disk_reads: 9, - disk_writes: 9, - object_gets: 21, - object_puts: 9, - } - ); - assert_eq!(estimate.cost, 126.0); - profile.nodes.get_mut("scan").unwrap().accesses.clear(); - assert!(estimate_storage_io(&dag, &scope, &profile, "evidence-v1").is_err()); -} - -// Valid local integer counts must not wrap when scaled or composed across nodes. -#[test] -fn multiplicity_and_cross_node_overflow_are_unavailable() { - let (dag, mut scope) = fixture(); - let mut profile = profile(&dag); - let large = StorageAccess { - operation: StorageOperation::DiskWrite, - extent_bytes: vec![u64::MAX], - bytes_per_request: 1, - }; - profile - .nodes - .get_mut("root") - .unwrap() - .accesses - .push(large.clone()); - assert_eq!( - estimate_storage_io(&dag, &scope, &profile, "evidence-v1"), - Err(asap_plan_selection::cost::analytical_cost::AnalyticalCostError::Overflow) - ); - scope.recurrence = QueryRecurrence::OneTime { - invocations: 1, - execute_at: None, - }; - profile - .nodes - .get_mut("left") - .unwrap() - .accesses - .push(StorageAccess { - extent_bytes: vec![1], - ..large - }); - assert_eq!( - estimate_storage_io(&dag, &scope, &profile, "evidence-v1"), - Err(asap_plan_selection::cost::analytical_cost::AnalyticalCostError::Overflow) - ); -} - -// Reusing map keys must not bind another node, source snapshot, or statistics. -#[test] -fn storage_node_identity_statistics_and_calibration_provenance_are_bound() { - let (dag, scope) = fixture(); - let original = profile(&dag); - let mut bad = original.clone(); - bad.nodes.get_mut("scan").unwrap().node.id = "another-scan".into(); - assert!(estimate_storage_io(&dag, &scope, &bad, "evidence-v1").is_err()); - bad = original.clone(); - bad.nodes - .get_mut("scan") - .unwrap() - .node - .scan_selection - .as_mut() - .unwrap() - .source_snapshot_id = "another-source".into(); - assert!(estimate_storage_io(&dag, &scope, &bad, "evidence-v1").is_err()); - bad = original.clone(); - if let OperatorStatistics::Scan { - source_read_bytes, .. - } = &mut bad.nodes.get_mut("scan").unwrap().statistics - { - *source_read_bytes = 79; - } - assert!(estimate_storage_io(&dag, &scope, &bad, "evidence-v1").is_err()); - bad = original.clone(); - bad.calibration.version = " \t".into(); - assert!(estimate_storage_io(&dag, &scope, &bad, "evidence-v1").is_err()); - let mut missing = dag.clone(); - missing.evidence.remove("scan"); - assert!(estimate_storage_io(&missing, &scope, &original, "evidence-v1").is_err()); -} diff --git a/crates/types/src/ir/operator/agg_intent.rs b/crates/types/src/ir/operator/agg_intent.rs index 05b056603..086ac00c3 100644 --- a/crates/types/src/ir/operator/agg_intent.rs +++ b/crates/types/src/ir/operator/agg_intent.rs @@ -49,31 +49,25 @@ pub enum AggIntent { accuracy: AccuracyTarget, }, Sum { - #[serde(default)] col: Option, }, Min { - #[serde(default)] col: Option, }, Max { - #[serde(default)] col: Option, }, Avg { - #[serde(default)] col: Option, }, /// Sample standard deviation when `population == false`; population stddev /// otherwise. PromQL `stddev` / `stddev_over_time`; SQL `STDDEV(col)`. StdDev { - #[serde(default)] col: Option, population: bool, }, /// Variance — PromQL `stdvar` / `stdvar_over_time`; SQL `VARIANCE(col)`. Variance { - #[serde(default)] col: Option, population: bool, }, @@ -88,7 +82,6 @@ pub enum AggIntent { /// `median(col)` (φ=0.5); PromQL `quantile(φ, …)` leaves `col` as `None` /// (the sample value). Quantile { - #[serde(default)] col: Option, #[serde(with = "crate::serde_f64")] q: f64, @@ -109,19 +102,16 @@ pub enum AggIntent { /// them. Empty is the PromQL convention "the sample value" — `count_values` /// and `distinct_over_time` leave it so. Cardinality { - #[serde(default = "Vec::new")] cols: Vec, accuracy: AccuracyTarget, }, /// L2 norm of the frequency vector of distinct input values. FrequencyL2 { - #[serde(default)] col: Option, accuracy: AccuracyTarget, }, /// Shannon entropy in bits of the frequency distribution of input values. FrequencyEntropy { - #[serde(default)] col: Option, accuracy: AccuracyTarget, }, @@ -944,39 +934,6 @@ mod tests { ); } } - - /// `col` is `#[serde(default)]`, so a DAG serialized before issue #115 — - /// with no `col` key — still deserializes, as the sample-value convention `None`. - #[test] - fn agg_intent_serde_reads_pre_115_payloads() { - let legacy = r#"{"kind":"quantile","q":0.99,"accuracy":{"max_error":0.0,"failure_probability":0.0,"accuracy_scope":"whole_result"}}"#; - assert_eq!( - serde_json::from_str::(legacy).unwrap(), - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::exact() - } - ); - - let legacy = r#"{"kind":"top_k","k":5,"accuracy":{"max_error":0.0,"failure_probability":0.0,"accuracy_scope":"whole_result"}}"#; - assert_eq!( - serde_json::from_str::(legacy).unwrap(), - AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::exact(), - } - ); - - let legacy = r#"{"kind":"cardinality","accuracy":{"max_error":0.0,"failure_probability":0.0,"accuracy_scope":"whole_result"}}"#; - assert_eq!( - serde_json::from_str::(legacy).unwrap(), - AggIntent::Cardinality { - cols: vec![], - accuracy: AccuracyTarget::exact() - } - ); - } } #[cfg(test)] diff --git a/crates/types/src/ir/schema/mod.rs b/crates/types/src/ir/schema/mod.rs index 61de457dd..c22cbeb92 100644 --- a/crates/types/src/ir/schema/mod.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -234,7 +234,6 @@ pub enum DataType { /// `Aggregate { by, .. }` emits `unique_keys = [by]`; `Dedup { cols }` /// adds `cols`; most other nodes pass through. #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] -#[serde(try_from = "SchemaWire")] pub struct Schema { /// Fields flowing on this edge, in positional order. pub fields: Vec, @@ -267,73 +266,6 @@ pub struct Schema { pub closed: bool, } -/// Deserialization form of [`Schema`]. Also reads the two layouts that -/// predate the unified schema, so plans saved by older builds still load: -/// -/// - pre-ASAP `Schema`: `columns` (not `fields`), each `dtype` a bare -/// [`DataType`] (`"float64"`), with `closed` / `unique_keys` present; -/// - post-ASAP `SummarySchema`: `fields` with tagged dtypes -/// (`{"Plain":"float64"}`), but no `closed` / `unique_keys`. Those were the -/// [`Schema::lifted`] shape, so a missing `closed` next to `fields` is closed. -#[derive(Deserialize)] -struct SchemaWire { - fields: Option>, - columns: Option>, - #[serde(default)] - time_index: Option, - #[serde(default)] - unique_keys: Vec>, - closed: Option, -} - -#[derive(Deserialize)] -struct FieldWire { - name: String, - dtype: FieldDataTypeWire, - nullable: bool, - #[serde(default)] - table: Option, -} - -/// A tagged [`FieldDataType`], or a bare [`DataType`] from a pre-ASAP column. -#[derive(Deserialize)] -#[serde(untagged)] -enum FieldDataTypeWire { - Current(FieldDataType), - Legacy(DataType), -} - -impl TryFrom for Schema { - type Error = String; - - fn try_from(wire: SchemaWire) -> Result { - let legacy_summary = wire.fields.is_some(); - let fields = match (wire.fields, wire.columns) { - (Some(fields), None) | (None, Some(fields)) => fields, - (Some(_), Some(_)) => return Err("schema has both `fields` and `columns`".into()), - (None, None) => return Err("missing field `fields`".into()), - }; - let fields = fields - .into_iter() - .map(|field| Field { - name: field.name, - dtype: match field.dtype { - FieldDataTypeWire::Current(dtype) => dtype, - FieldDataTypeWire::Legacy(dtype) => FieldDataType::Plain(dtype), - }, - nullable: field.nullable, - table: field.table, - }) - .collect(); - Ok(Self { - fields, - time_index: wire.time_index, - unique_keys: wire.unique_keys, - closed: wire.closed.unwrap_or(legacy_summary), - }) - } -} - /// Reserved physical row column carrying canonical JSON of a complete PromQL /// label map. `$` cannot occur in a user PromQL label name. pub const PROMQL_SERIES_IDENTITY: &str = "$promql_series_identity"; @@ -484,52 +416,6 @@ mod tests { assert_eq!(s, back); } - #[test] - fn legacy_pre_asap_schema_deserializes() { - // Pre-unification `Schema`: `columns`, bare dtypes, explicit `closed`. - let json = r#"{"columns":[ - {"name":"ts","dtype":"timestamp","nullable":false,"table":null}, - {"name":"value","dtype":"float64","nullable":true,"table":"t"} - ],"time_index":0,"unique_keys":[[0]],"closed":true}"#; - let back: Schema = serde_json::from_str(json).unwrap(); - let mut expected = Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - Field::plain("value", DataType::Float64, true).with_table("t"), - ], - 0, - vec![vec![0]], - ); - expected.closed = true; - assert_eq!(back, expected); - } - - #[test] - fn legacy_pre_asap_schema_without_closed_or_table_is_open() { - // Older still: written before `closed` and `Field.table` existed. - let json = r#"{"columns":[{"name":"a","dtype":"utf8","nullable":false}]}"#; - let back: Schema = serde_json::from_str(json).unwrap(); - assert_eq!(back, Schema::new(vec![col("a", DataType::Utf8)])); - assert!(!back.closed, "absent `closed` ⇒ open"); - } - - #[test] - fn legacy_summary_schema_deserializes_as_lifted() { - // Pre-unification post-ASAP `SummarySchema`: no `closed`/`unique_keys`. - let json = r#"{"fields":[ - {"name":"ts","dtype":{"Plain":"timestamp"},"nullable":false}, - {"name":"v","dtype":{"Plain":"float64"},"nullable":false} - ],"time_index":0}"#; - let back: Schema = serde_json::from_str(json).unwrap(); - assert_eq!( - back, - Schema::lifted( - vec![col("ts", DataType::Timestamp), col("v", DataType::Float64)], - Some(0) - ) - ); - } - #[test] fn schema_without_fields_is_rejected() { assert!(serde_json::from_str::(r#"{"closed":true}"#).is_err()); diff --git a/crates/types/src/workload/resources.rs b/crates/types/src/workload/resources.rs index 93ba150a3..d54595375 100644 --- a/crates/types/src/workload/resources.rs +++ b/crates/types/src/workload/resources.rs @@ -3,18 +3,10 @@ //! The CPU payload establishes its unit and operation scope; CPU operations //! must never be interpreted as nanoseconds. Byte values can be exact integers //! or measurements carrying uncertainty. `None` means unavailable, not zero. -//! Cache assumptions share this schema namespace but are not additive resource -//! consumption; their numerical interpretation belongs to an estimator. -pub mod cache; pub mod cpu; pub mod measurement; pub mod physical; -pub mod physical_handoff; -pub mod storage; -pub use cache::{CacheCapacityEvidence, CacheEvidence, CacheProfile}; -pub use physical_handoff::{PhysicalHandoffBytes, PhysicalHandoffKind}; -pub use storage::StorageResources; pub use cpu::{MeasuredCpu, ModeledCpu}; pub use measurement::Measurement; diff --git a/crates/types/src/workload/resources/cache.rs b/crates/types/src/workload/resources/cache.rs deleted file mode 100644 index a5dd8e466..000000000 --- a/crates/types/src/workload/resources/cache.rs +++ /dev/null @@ -1,115 +0,0 @@ -//! Shared cache assumptions. Numerical estimation belongs to the cost model. - -use serde::{Deserialize, Serialize}; - -/// Versioned deployment evidence describing query-result and buffer caching. -/// These are deployment assumptions, not additive CPU or byte consumption. -/// Estimators validate and interpret them against a comparison workload. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "profile", rename_all = "snake_case", deny_unknown_fields)] -pub enum CacheProfile { - NoCache { version: String }, - Evidence(CacheEvidence), -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct CacheEvidence { - pub version: String, - /// Evaluations whose parameterization is not identical to a preceding - /// evaluation and therefore cannot use the result cache. - pub distinct_evaluations: u64, - /// Evaluations identical to a preceding evaluation and eligible for a - /// result-cache hit. - pub repeated_identical_evaluations: u64, - pub result_cache: CacheCapacityEvidence, - pub buffer_cache: CacheCapacityEvidence, - /// Fraction of otherwise-resident result entries invalidated by arriving - /// data. Required for ingesting data; `AtRest` permits only - /// zero or omitted invalidation evidence. - pub result_invalidation_ratio: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct CacheCapacityEvidence { - pub working_set_bytes: u64, - pub capacity_bytes: u64, -} - -impl CacheProfile { - pub fn no_cache() -> Self { - Self::NoCache { - version: "no-cache-v1".into(), - } - } - - pub fn version(&self) -> &str { - match self { - Self::NoCache { version } => version, - Self::Evidence(evidence) => &evidence.version, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - // Moving the definitions preserves the established tagged JSON contract. - #[test] - fn existing_cache_json_round_trips_through_shared_types() { - let no_cache = json!({"profile": "no_cache", "version": "no-cache-v1"}); - assert_eq!( - serde_json::to_value(CacheProfile::no_cache()).unwrap(), - no_cache - ); - assert_eq!( - serde_json::from_value::(no_cache) - .unwrap() - .version(), - "no-cache-v1" - ); - let evidence = json!({ - "profile": "evidence", "version": "cache-evidence-7", - "distinct_evaluations": 2, "repeated_identical_evaluations": 4, - "result_cache": {"working_set_bytes": 100, "capacity_bytes": 100}, - "buffer_cache": {"working_set_bytes": 9007199254740993_u64, "capacity_bytes": 50}, - "result_invalidation_ratio": null - }); - let decoded: CacheProfile = serde_json::from_value(evidence.clone()).unwrap(); - assert_eq!(decoded.version(), "cache-evidence-7"); - assert_eq!(serde_json::to_value(decoded).unwrap(), evidence); - } - - // Unknown invalidation stays unknown; required capacity cannot default to zero. - #[test] - fn shared_cache_json_preserves_unknowns_and_rejects_missing_or_extra_dimensions() { - let mut evidence = json!({ - "profile": "evidence", "version": "v1", - "distinct_evaluations": 1, "repeated_identical_evaluations": 0, - "result_cache": {"working_set_bytes": 100, "capacity_bytes": 0}, - "buffer_cache": {"working_set_bytes": 100, "capacity_bytes": 0} - }); - let CacheProfile::Evidence(decoded) = - serde_json::from_value::(evidence.clone()).unwrap() - else { - panic!("wrong variant") - }; - assert_eq!(decoded.result_invalidation_ratio, None); - evidence["result_cache"] - .as_object_mut() - .unwrap() - .remove("capacity_bytes"); - assert!(serde_json::from_value::(evidence).is_err()); - assert!(serde_json::from_value::(json!({ - "working_set_bytes": 100, "capacity_bytes": 10, "cpu_ops": 1 - })) - .is_err()); - assert!(serde_json::from_value::(json!({ - "profile": "no_cache", "version": "v1", "scan_bytes": 0 - })) - .is_err()); - } -} diff --git a/crates/types/src/workload/resources/physical_handoff.rs b/crates/types/src/workload/resources/physical_handoff.rs deleted file mode 100644 index d33461304..000000000 --- a/crates/types/src/workload/resources/physical_handoff.rs +++ /dev/null @@ -1,125 +0,0 @@ -//! Physical transfer and materialization dimensions, independent of planner policy. -//! -//! A handoff is either a network transfer between execution locations or a -//! materialization of an intermediate result. This module defines their kinds -//! and separate byte-work counters; it is not limited to network resources. -//! -//! Counters describe bytes transferred or materialized, not retained memory, -//! allocated disk space, or storage request counts. Estimation, evidence -//! validation, and calibration belong to the cost model, not this schema. - -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum PhysicalHandoffKind { - /// Transfer encoded data between execution locations. - Network { - source_location: String, - destination_location: String, - }, - /// Write an intermediate result for later use. - Materialization, -} - -/// Byte work at explicitly declared physical actions, not storage occupancy. -/// Network traffic and materialization writes remain separate dimensions. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct PhysicalHandoffBytes { - /// Bytes transferred across explicitly declared network handoffs. - pub network_bytes: u64, - /// Bytes written across materialization handoffs, regardless of medium. - pub materialization_bytes: u64, -} - -impl PhysicalHandoffBytes { - pub fn terms(self) -> [(&'static str, u64); 2] { - [ - ("network_bytes", self.network_bytes), - ("materialization_bytes", self.materialization_bytes), - ] - } - - /// Add independent dimensions without wrapping or partially mutating either input. - pub fn checked_add(self, other: Self) -> Option { - Some(Self { - network_bytes: self.network_bytes.checked_add(other.network_bytes)?, - materialization_bytes: self - .materialization_bytes - .checked_add(other.materialization_bytes)?, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Handoff JSON keeps separate counters and only the evidence used for each kind. - #[test] - fn handoff_json_round_trip() { - let bytes = PhysicalHandoffBytes { - network_bytes: 480, - materialization_bytes: 40, - }; - let json = serde_json::json!({"network_bytes": 480, "materialization_bytes": 40}); - assert_eq!(serde_json::to_value(bytes).unwrap(), json); - assert_eq!( - serde_json::from_value::(json).unwrap(), - bytes - ); - let network = PhysicalHandoffKind::Network { - source_location: "edge".into(), - destination_location: "backend".into(), - }; - assert_eq!( - serde_json::to_value(network).unwrap(), - serde_json::json!({ - "kind": "network", "source_location": "edge", "destination_location": "backend" - }) - ); - let kind = PhysicalHandoffKind::Materialization; - let json = serde_json::json!({"kind": "materialization"}); - assert_eq!(serde_json::to_value(&kind).unwrap(), json); - assert_eq!( - serde_json::from_value::(json).unwrap(), - kind - ); - } - - /// Either dimension overflowing returns None without changing the original counters. - #[test] - fn checked_add_preserves_dimensions_and_rejects_overflow() { - let first = PhysicalHandoffBytes { - network_bytes: 2, - materialization_bytes: 3, - }; - let second = PhysicalHandoffBytes { - network_bytes: 5, - materialization_bytes: 7, - }; - assert_eq!( - first.checked_add(second), - Some(PhysicalHandoffBytes { - network_bytes: 7, - materialization_bytes: 10, - }) - ); - for overflowing in [ - PhysicalHandoffBytes { - network_bytes: u64::MAX, - materialization_bytes: 0, - }, - PhysicalHandoffBytes { - network_bytes: 0, - materialization_bytes: u64::MAX, - }, - ] { - assert!(first.checked_add(overflowing).is_none()); - } - assert_eq!( - first.terms(), - [("network_bytes", 2), ("materialization_bytes", 3)] - ); - } -} diff --git a/crates/types/src/workload/resources/storage.rs b/crates/types/src/workload/resources/storage.rs deleted file mode 100644 index e33c47c67..000000000 --- a/crates/types/src/workload/resources/storage.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! Storage request-count dimensions, independent of estimation and calibration. - -use serde::{Deserialize, Serialize}; - -/// Explicit counts of physical data requests, not transferred bytes or CPU work. -/// Unavailable storage evidence is represented by the enclosing optional profile, -/// not by constructing an all-zero count vector. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct StorageResources { - pub disk_reads: u64, - pub disk_writes: u64, - pub object_gets: u64, - pub object_puts: u64, -} - -impl StorageResources { - pub fn terms(self) -> [(&'static str, u64); 4] { - [ - ("disk_read_operations", self.disk_reads), - ("disk_write_operations", self.disk_writes), - ("object_get_operations", self.object_gets), - ("object_put_operations", self.object_puts), - ] - } - - /// Compose exact counts without wrapping or partially updating an input. - pub fn checked_add(self, other: Self) -> Option { - Some(Self { - disk_reads: self.disk_reads.checked_add(other.disk_reads)?, - disk_writes: self.disk_writes.checked_add(other.disk_writes)?, - object_gets: self.object_gets.checked_add(other.object_gets)?, - object_puts: self.object_puts.checked_add(other.object_puts)?, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Moving the type preserves its four required integer wire fields exactly. - #[test] - fn storage_counts_keep_the_existing_wire_shape() { - let wire = serde_json::json!({ - "disk_reads": 1, "disk_writes": 2, "object_gets": 3, "object_puts": 4, - }); - let counts: StorageResources = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(serde_json::to_value(counts).unwrap(), wire); - assert_eq!(counts.terms()[2], ("object_get_operations", 3)); - for invalid in [ - serde_json::json!(null), - serde_json::json!({}), - serde_json::json!({ - "disk_reads": 1.5, "disk_writes": 2, "object_gets": 3, "object_puts": 4, - }), - ] { - assert!(serde_json::from_value::(invalid).is_err()); - } - } - - /// Every independent dimension fails closed on integer addition overflow. - #[test] - fn storage_count_addition_is_checked_in_each_dimension() { - let one = StorageResources { - disk_reads: 1, - disk_writes: 1, - object_gets: 1, - object_puts: 1, - }; - let zero = StorageResources::default(); - assert_eq!(zero.checked_add(one), Some(one)); - for counts in [ - StorageResources { - disk_reads: u64::MAX, - ..zero - }, - StorageResources { - disk_writes: u64::MAX, - ..zero - }, - StorageResources { - object_gets: u64::MAX, - ..zero - }, - StorageResources { - object_puts: u64::MAX, - ..zero - }, - ] { - assert_eq!(counts.checked_add(one), None); - assert_eq!(counts.checked_add(zero), Some(counts)); - } - } -} diff --git a/crates/types/tests/planner_vocabulary.rs b/crates/types/tests/planner_vocabulary.rs index bb04225d2..d1c462149 100644 --- a/crates/types/tests/planner_vocabulary.rs +++ b/crates/types/tests/planner_vocabulary.rs @@ -1,6 +1,5 @@ use asap_types::ir::physical_export::WindowEdgeCompatibility; use asap_types::physical::{validate_pane_coverage, PaneLayout, WindowEdgeCoverage}; -use asap_types::workload::resources::{PhysicalHandoffBytes, PhysicalHandoffKind}; // Renamed pane APIs still read and emit the deployed wire contract. #[test] @@ -27,25 +26,3 @@ fn window_edge_names_preserve_wire_values() { compatibility ); } - -// External consumers can use the new resource names without changing behavior. -// (The schema-resolver half moved with the resolver to `asap-frontend-common`; -// `schema_resolver::tests::bare_source_yields_ts_value_floor` covers it.) -#[test] -fn renamed_handoff_apis_are_public() { - let bytes = PhysicalHandoffBytes { - network_bytes: 12, - materialization_bytes: 4, - }; - assert_eq!( - serde_json::to_value(bytes).unwrap(), - serde_json::json!({ - "network_bytes": 12, "materialization_bytes": 4 - }) - ); - let kind: PhysicalHandoffKind = serde_json::from_value(serde_json::json!({ - "kind": "network", "source_location": "edge", "destination_location": "backend" - })) - .unwrap(); - assert!(matches!(kind, PhysicalHandoffKind::Network { .. })); -} diff --git a/docs/design_docs/architecture/physical-plan-integration.md b/docs/design_docs/architecture/physical-plan-integration.md index 6ce4e12fb..b65d01c44 100644 --- a/docs/design_docs/architecture/physical-plan-integration.md +++ b/docs/design_docs/architecture/physical-plan-integration.md @@ -23,6 +23,13 @@ The arrows are boundaries, not casts. A logical node does not necessarily become one physical node, and a physical operator is not merely a renamed logical variant. +Current state: no code lowers a whole candidate to a `PhysicalOperator` DAG. +The recursive lowering and the physical-DAG estimator that priced its output +were deleted with no remaining reader. Stage 3 prices each node of a physical +candidate with `estimate_operator` (`cost/analytical_cost.rs`) over the +`OperatorStatistics` it derives for that node. The lowering obligations and +comparison rules below are the contract a whole-DAG lowering must meet. + ## Sources of truth Each representation is authoritative for a different concern: @@ -383,7 +390,7 @@ one output row or batch. ## Comparison and failure behavior -Raw and post-ASAP alternatives must cover the same `ComparisonScope`: arrival +Raw and post-ASAP alternatives must cover the same comparison scope: arrival mode, planning time, workload horizon, recurrence, event-time selection, logical sources, ordinary predicates or info-metric matchers, and physical source snapshots. diff --git a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md index 0970b306e..bd3d8534a 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md @@ -5,6 +5,13 @@ > is what Stage 3 measures resource usage with. The physical-plan adapter and > the legacy `CostModel` described below have been removed; Stage 3's cost > models 1–3 replace them. +> The whole-DAG path this document also describes has been deleted, since no +> stage read it: physical query lowering (`query_physical_lowering`), +> `EvidenceBackedPhysicalDAG`, `estimate_physical_dag` and its comparison and +> cache variants, `ComparisonScope`, `CacheProfile`, and the storage and +> handoff estimators. Stage 3 prices each node with `estimate_operator` over +> the `OperatorStatistics` it derives for that node. The sections on those +> deleted parts record the design, not current code. > Unsupported operators, arrival modes and missing evidence remain unavailable; > proposed extensions are not implied by the implemented formulas. @@ -992,15 +999,9 @@ available, that candidate is unavailable rather than costed as in-memory. ## Unsupported and future dimensions -The physical-plan adapter can also estimate explicitly evidenced disk/object -request counts using the optional storage profile. See -[storage operation estimates](../../../develop_docs/storage-operation-costs.md) -for units, formulas, evidence requirements, and calibration. - -The physical-plan adapter supports explicit network and materialization -boundaries through optional deployment evidence. See -[physical boundary estimates](../../../develop_docs/physical-handoff-costs.md) -for kinds, execution multiplicity, evidence, and calibration. +Disk/object request counts and network and materialization handoff bytes are +not estimated. Their estimators (a storage profile and a physical handoff +profile) were deleted because no planning stage supplied or read them. An estimate is unavailable when required evidence, a physical formula, or a finite horizon is missing. Structural node counts are never substituted. diff --git a/docs/develop_docs/README.md b/docs/develop_docs/README.md index 65c63f563..ec6907a21 100644 --- a/docs/develop_docs/README.md +++ b/docs/develop_docs/README.md @@ -12,7 +12,6 @@ formats, evidence, and verification workflows. - [Accuracy implementation companion](end-to-end-accuracy-guarantees.md) - [Offline sketch evidence](offline-sketch-evidence.md) - [Metrics-observability corpora](metrics-observability-corpora.md) -- [Physical handoff cost references](physical-handoff-costs.md), [storage operations](storage-operation-costs.md) - [Physical compile coverage for deployment computation](physical-compile-coverage.md) Historical records of removed APIs: diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index 9acc848e7..7d586e9bf 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -72,8 +72,7 @@ same pipeline behind the `OptimizationPass` trait that `asap-planner` calls. Not yet wired into the stages: `maintained_population.rs` (maintained populations for instant top-k, quantile and sum) in Stage 1, and -`pane_sharing.rs`, `cost/storage_io.rs`, `cost/physical_handoff_cost.rs` and -`cost/query_physical_lowering.rs` in plan selection. +`pane_sharing.rs` in plan selection. ## Algorithm in one pass diff --git a/docs/develop_docs/physical-handoff-costs.md b/docs/develop_docs/physical-handoff-costs.md deleted file mode 100644 index 4d56ad6c1..000000000 --- a/docs/develop_docs/physical-handoff-costs.md +++ /dev/null @@ -1,99 +0,0 @@ -# Physical handoff byte estimates - -`asap_types::workload::resources` owns the canonical `PhysicalHandoffBytes` and -`PhysicalHandoffKind` definitions in `resources/physical_handoff.rs`. The mapping -crate re-exports those same types from `physical_handoff_cost` for import compatibility; -all estimator and export consumers therefore use shared definitions, not copies. -Materialization is encoded as `{"kind":"materialization"}` without a medium; -no current estimator distinguishes storage media. The shared byte counters support checked -addition without depending on planner errors. Snapshot binding, validation, -calibration, and ranking remain in the mapping crate. handoff traffic/write work -is distinct from CPU work, scanned bytes, and stored byte occupancy; it is not -collapsed into the generic CPU/byte resource container. - -The `PhysicalHandoffProfile` estimator is kept in `physical_handoff_cost`, but -its consumer, the legacy physical-plan adapter and its -`PhysicalEvidenceSnapshot`, has been removed. No planning stage or devtool -reads the profile yet. With no profile, these -dimensions remain unestimated and the existing resource objective is preserved. - -The profile's `plans` list binds handoffs to complete physical alternatives. -Each plan supplies its `root` and a `nodes` map containing every physical node, -its authoritative statistics, and an explicit list of handoff actions on its -output. An empty handoff list declares ordinary in-memory dataflow with no -handoff traffic. Exactly one plan must match the root, complete node set, and -node/statistics snapshot; missing or ambiguous matches fail closed. This lets -alternatives reuse a producer identity while declaring different transfers to -their respective consumers. All alternatives share the profile's immutable -evidence generation and calibration. Evidence -must match the immutable snapshot version and be current at planning time: -`observed_at_ms <= planning_time < valid_until_ms`. - -Supported handoffs are: - -| Handoff | Required evidence | Dimension | -|---|---|---| -| Network/exchange/deployment transfer | Distinct nonempty source and destination locations | Network bytes | -| Materialization/persistence | No additional kind-specific evidence | Materialization bytes | - -Every handoff also declares its unique physical ID, output logical bytes, -encoded bytes per execution, and positive copy count. Logical bytes must equal -the producer's output statistic. Encoded bytes capture an explicit compression -or serialization estimate; empty and nonempty payloads must agree with the -logical edge. A remote persistence operation may declare both a network action -and a materialization action with distinct IDs; these contribute to different -dimensions and require separate calibration coefficients. - -```text -handoff_bytes = encoded_bytes * copies * executions -``` - -For a shared handoff (`consumer: null`), execution multiplicity comes from -the producer. For a per-consumer handoff, `consumer` must identify an actual, -reachable parent of the producer; multiplicity comes from that consumer. -`Once` means one execution; `PerEvaluation` uses the comparison scope's demand. -Shared producers are traversed once regardless of fan-out. Duplicate handoff -IDs fail closed instead of being ambiguously counted or silently dropped. -handoff profiles currently require `CacheProfile::NoCache`. Cache evidence does -not identify which physical transfers or materializations are skipped on a hit, -and handoff execution counts derive from the comparison scope rather than a -post-cache schedule. Combining a handoff profile with `CacheProfile::Evidence` -therefore fails closed until cache-aware handoff execution evidence exists. -Supported exports retain the no-cache profile provenance alongside handoff -model/calibration versions. - -Example: a retained producer materializes 40 encoded bytes once. Two consumers -each receive two copies of those 40 bytes over three evaluations. The totals -are 40 materialization bytes and 480 network bytes. Logical in-memory edges -without a handoff action add no traffic. A retained producer can therefore -have a once-only persistence action and repeated transfers to its readers. - -`PhysicalHandoffEstimate` keeps network and materialization totals separate and returns -per-node and per-handoff terms with model, evidence, and calibration provenance. -A `PhysicalHandoffCalibration` supplies finite, nonnegative cost coefficients in the -same cost units as the base resource model, and a nonempty version. Both -alternatives use that profile when ranking. This models byte work, not transfer -latency, bandwidth contention, memory lifetime, or storage requests. -Base CPU, scan, and retained-memory coefficients may all be zero when at least -one handoff coefficient is positive. An absent handoff profile or an entirely -zero objective remains unavailable for ranking. -Both the base and handoff calibration versions must be nonempty, including -when the base coefficients are all zero; combined annotations identify both. - -Annotations expose totals in `bytes`, plus terms named -`physical_node::` and `boundary::`. The existing -viewer cost sidebar displays all terms and the combined formula/calibration -version (`physical-boundary-bytes-v1`) with the evidence version. No network -traffic is inferred from logical edges, operator buffers, or scan bytes. - -Unknown endpoints, mismatched payloads, absent node evidence, duplicate IDs, -stale evidence, invalid coefficients, and integer overflow return typed errors; -ranking reports the comparison as unavailable. This extends the physical -plan adapter; summary-maintenance costing for Stage 2 materialization (#509) is -a separate follow-up integration point. - -Verification: - -```sh -cargo test -p asap-plan-selection --test physical_handoff_cost -``` diff --git a/docs/develop_docs/planner-vocabulary-migration.md b/docs/develop_docs/planner-vocabulary-migration.md index 7a6604474..c0e78485a 100644 --- a/docs/develop_docs/planner-vocabulary-migration.md +++ b/docs/develop_docs/planner-vocabulary-migration.md @@ -56,7 +56,10 @@ input still uses `boundaries`. Resource explanation labels (`boundary:::`; -annotation provenance identifies `storage-requests-v1` and its calibration. -The viewer displays these terms in the existing cost sidebar. The base byte -estimate and storage request estimate remain independently inspectable. - -Missing entries, expired/future evidence, incompatible node snapshots, zero -request sizes, invalid calibration, and overflow return typed analytical -errors. When used by plan ranking/export they make that comparison unavailable. -This profile extends the physical-plan adapter. Combined physical-plan -ranking currently supports storage profiles only with an explicit `NoCache` -profile. `CacheProfile::Evidence` together with storage evidence makes the -comparison unavailable: aggregate cache hit ratios cannot identify which -independently rounded extents issue requests. Supporting that combination -requires cache-aware extent/request evidence; the adapter does not mix -cache-adjusted bytes and CPU with uncached operation counts. - -Verification: - -```sh -cargo test -p asap-plan-selection --test storage_io -cargo test -p asap-plan-selection --lib storage_io -```