From bc988c7376fe715aad9aae183932ebf854aea3ea Mon Sep 17 00:00:00 2001 From: zzylol Date: Tue, 20 Oct 2026 19:30:16 +0000 Subject: [PATCH 1/4] feat!: install the Planner's InstallablePlan; store its stored outputs as summaries The control plane calls asap_planner::plan_installable once and writes the returned InstallablePlan as plan.json; the backend's own Stage 0-3 plumbing, pane-chain expansion and series-identity rewrite are gone. The SDS stores immutable summaries for every stored output the plan lists, built in an ingestion run or in a query run and kept: Sds::from_plan groups outputs by the SDS's own identity (sub-DAG hash, offset-only TimeShifts transparent), records which run builds each definition, keys summaries by (definition, coverage end), and keeps each definition for the largest retention_ms among its outputs (the Planner's retention, which includes range-evaluation spans) instead of "largest offset plus two panes". Each definition lists the stored outputs its builder reads, and Sds::ingestion_build_order orders ingestion-run definitions after their stored inputs. The precompute engine builds ingestion-run definitions in that order, from a RawSampleSelector over the plan's coverage window, and refuses one built from other stored outputs until it can build it once its inputs are sealed. Query-run definitions are left to the query engine; the backend still declares query_time_results_retained: false until that lands. Among Planner crates the workspace depends on asap-planner and asap-executor only, at ASAPPlanner main ecea897f; IR types come through asap_executor::planner. The sync script, its test and the CI README update both. asap-planner pulls in the SQL and MetricsQL frontends until they become optional features upstream. --- Cargo.lock | 1909 +++++++++++++++++++++++++- Cargo.toml | 10 +- README.md | 10 +- control_plane/Cargo.toml | 6 +- control_plane/src/lib.rs | 232 +--- control_plane/src/main.rs | 6 +- control_plane/tests/plan_workload.rs | 67 +- data_plane/Cargo.toml | 6 +- data_plane/src/engine.rs | 13 +- data_plane/src/lib.rs | 2 +- data_plane/src/precompute.rs | 88 +- data_plane/src/sds.rs | 333 ++--- data_plane/tests/common/mod.rs | 19 +- data_plane/tests/precompute.rs | 48 +- data_plane/tests/sds.rs | 163 ++- scripts/ci/README.md | 2 +- scripts/ci/test_sync_planner.py | 9 +- scripts/sync-asapplanner-main.sh | 9 +- 18 files changed, 2315 insertions(+), 617 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18448423..303579a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -12,6 +18,7 @@ dependencies = [ "const-random", "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -25,6 +32,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.6" @@ -90,6 +118,21 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "arrow" version = "58.4.0" @@ -100,7 +143,10 @@ dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", + "arrow-csv", "arrow-data", + "arrow-ipc", + "arrow-json", "arrow-ord", "arrow-row", "arrow-schema", @@ -133,8 +179,9 @@ dependencies = [ "arrow-data", "arrow-schema", "chrono", + "chrono-tz", "half", - "hashbrown", + "hashbrown 0.17.1", "num-complex", "num-integer", "num-traits", @@ -167,12 +214,28 @@ dependencies = [ "atoi", "base64", "chrono", + "comfy-table", "half", "lexical-core", "num-traits", "ryu", ] +[[package]] +name = "arrow-csv" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + [[package]] name = "arrow-data" version = "58.4.0" @@ -186,6 +249,47 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-ipc" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", + "zstd 0.13.3", +] + +[[package]] +name = "arrow-json" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + [[package]] name = "arrow-ord" version = "58.4.0" @@ -217,6 +321,10 @@ name = "arrow-schema" version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +dependencies = [ + "serde_core", + "serde_json", +] [[package]] name = "arrow-select" @@ -277,6 +385,17 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "asap-frontend-metricsql" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=ecea897f78640c377ce1d80846156f330f62dee2#ecea897f78640c377ce1d80846156f330f62dee2" +dependencies = [ + "asap-frontend-common", + "asap-types", + "metricsql_parser", + "thiserror 2.0.20", +] + [[package]] name = "asap-frontend-promql" version = "0.1.0" @@ -287,6 +406,18 @@ dependencies = [ "promql-parser", ] +[[package]] +name = "asap-frontend-sql" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=ecea897f78640c377ce1d80846156f330f62dee2#ecea897f78640c377ce1d80846156f330f62dee2" +dependencies = [ + "asap-frontend-common", + "asap-sql-function-catalog", + "asap-types", + "datafusion", + "serde_json", +] + [[package]] name = "asap-logical-asap-optimization" version = "0.1.0" @@ -323,6 +454,28 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "asap-planner" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=ecea897f78640c377ce1d80846156f330f62dee2#ecea897f78640c377ce1d80846156f330f62dee2" +dependencies = [ + "asap-executor", + "asap-frontend-metricsql", + "asap-frontend-promql", + "asap-frontend-sql", + "asap-logical-asap-optimization", + "asap-plan-selection", + "asap-types", + "serde", + "serde_json", + "thiserror 2.0.20", +] + +[[package]] +name = "asap-sql-function-catalog" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=ecea897f78640c377ce1d80846156f330f62dee2#ecea897f78640c377ce1d80846156f330f62dee2" + [[package]] name = "asap-types" version = "0.1.0" @@ -351,6 +504,18 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "async-compression" +version = "0.4.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -444,6 +609,25 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bincode" version = "1.3.3" @@ -459,6 +643,28 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -468,18 +674,63 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + [[package]] name = "cactus" version = "1.0.7" @@ -493,6 +744,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -542,10 +795,21 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-link", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + [[package]] name = "clap" version = "4.6.6" @@ -592,6 +856,43 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "compression-codecs" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe" +dependencies = [ + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd 0.14.3", + "zstd-safe 8.2.0", +] + +[[package]] +name = "compression-core" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -612,16 +913,20 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "control_plane" version = "0.1.0" dependencies = [ "anyhow", - "asap-frontend-promql", - "asap-logical-asap-optimization", - "asap-plan-selection", - "asap-types", + "asap-planner", "clap", + "futures", "serde", "serde_json", "thiserror 1.0.69", @@ -653,6 +958,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + [[package]] name = "crunchy" version = "0.2.4" @@ -669,13 +989,57 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data_plane" version = "0.1.0" dependencies = [ "anyhow", "asap-executor", - "asap-types", + "asap-planner", "axum", "clap", "control_plane", @@ -685,7 +1049,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "snap", "thiserror 1.0.69", "tokio", @@ -695,20 +1059,689 @@ dependencies = [ ] [[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" - -[[package]] -name = "digest" -version = "0.10.7" +name = "datafusion" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ - "block-buffer", - "crypto-common", -] + "arrow", + "arrow-schema", + "async-trait", + "bzip2", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "flate2", + "futures", + "indexmap", + "itertools 0.14.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "parquet", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", + "zstd 0.13.3", +] + +[[package]] +name = "datafusion-catalog" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools 0.14.0", + "log", + "object_store", +] + +[[package]] +name = "datafusion-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "libc", + "log", + "object_store", + "parquet", + "recursive", + "sqlparser", + "tokio", + "uuid", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" +dependencies = [ + "arrow", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "flate2", + "futures", + "glob", + "itertools 0.14.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "rand 0.9.5", + "tokio", + "tokio-util", + "url", + "zstd 0.13.3", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "parquet", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" + +[[package]] +name = "datafusion-execution" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.5", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap", + "itertools 0.14.0", + "recursive", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-functions" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hex", + "itertools 0.14.0", + "log", + "md-5", + "memchr", + "num-traits", + "rand 0.9.5", + "regex", + "sha2 0.11.1", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "foldhash 0.2.0", + "half", + "log", + "num-traits", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hashbrown 0.17.1", + "itertools 0.14.0", + "itoa", + "log", + "memchr", +] + +[[package]] +name = "datafusion-functions-table" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-functions-window" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" +dependencies = [ + "datafusion-doc", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "datafusion-optimizer" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap", + "itertools 0.14.0", + "log", + "recursive", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "parking_lot", + "petgraph", + "recursive", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "parking_lot", + "pin-project", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "itertools 0.14.0", + "recursive", +] + +[[package]] +name = "datafusion-physical-plan" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" +dependencies = [ + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-pruning" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "log", +] + +[[package]] +name = "datafusion-session" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" +dependencies = [ + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-sql" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-nested", + "indexmap", + "log", + "recursive", + "regex", + "sqlparser", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", +] [[package]] name = "displaydoc" @@ -727,6 +1760,15 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +[[package]] +name = "enquote" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -743,6 +1785,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "filetime" version = "0.2.29" @@ -759,12 +1807,51 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -920,16 +2007,37 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "half" version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "cfg-if", - "crunchy", - "num-traits", - "zerocopy", + "foldhash 0.1.5", ] [[package]] @@ -937,6 +2045,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -944,6 +2057,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "1.5.0" @@ -989,6 +2108,21 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.11.1" @@ -1184,9 +2318,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "ipnet" version = "2.12.2" @@ -1208,12 +2348,31 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.105" @@ -1288,18 +2447,50 @@ dependencies = [ "lexical-util", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "liblzma" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa980ea6744074a5b5e0ac6b164aca94fc970121869b3e78ae12a2b0097dbea7" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c7e3581f367a78d7b7e7ae948d023310556f0cfc13156c2e4e00e25616492b9" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.3" @@ -1321,6 +2512,39 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "logos" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7251356ef8cb7aec833ddf598c6cb24d17b689d20b993f9d11a3d764e34e6458" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f80069600c0d66734f5ff52cc42f2dabd6b29d205f333d61fd7832e9e9963f" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2", + "quote", + "regex-syntax", + "syn 2.0.119", +] + +[[package]] +name = "logos-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24fb722b06a9dc12adb0963ed585f19fc61dc5413e6a9be9422ef92c091e731d" +dependencies = [ + "logos-codegen", +] + [[package]] name = "lrlex" version = "0.13.10" @@ -1381,6 +2605,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + [[package]] name = "matchers" version = "0.2.0" @@ -1396,18 +2629,69 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "metricsql_common" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=ecea897f78640c377ce1d80846156f330f62dee2#ecea897f78640c377ce1d80846156f330f62dee2" +dependencies = [ + "chrono", +] + +[[package]] +name = "metricsql_parser" +version = "0.1.0" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=ecea897f78640c377ce1d80846156f330f62dee2#ecea897f78640c377ce1d80846156f330f62dee2" +dependencies = [ + "ahash", + "chrono", + "enquote", + "logos", + "logos-derive", + "metricsql_common", + "num-traits", + "phf 0.11.3", + "regex", + "scopeguard", + "serde", + "strum", + "strum_macros", + "thiserror 1.0.69", + "tinyvec", + "xxhash-rust", +] + [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.3" @@ -1457,80 +2741,258 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" name = "num-integer" version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http", + "humantime", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "thiserror 2.0.20", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "packedvec" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70" +dependencies = [ + "num-traits", + "serde", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "object_store", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd 0.13.3", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ - "num-traits", + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", + "serde", ] [[package]] -name = "num-traits" -version = "0.2.19" +name = "phf" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "autocfg", - "libm", + "phf_macros", + "phf_shared 0.11.3", ] [[package]] -name = "num_threads" -version = "0.1.7" +name = "phf" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "libc", + "phf_shared 0.12.1", ] [[package]] -name = "once_cell" -version = "1.21.4" +name = "phf_generator" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.8", +] [[package]] -name = "once_cell_polyfill" -version = "1.70.2" +name = "phf_macros" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] -name = "packedvec" -version = "1.2.5" +name = "phf_shared" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ - "num-traits", - "serde", + "siphasher", ] [[package]] -name = "parking_lot" -version = "0.12.5" +name = "phf_shared" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" dependencies = [ - "lock_api", - "parking_lot_core", + "siphasher", ] [[package]] -name = "parking_lot_core" -version = "0.9.12" +name = "pin-project" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", + "pin-project-internal", ] [[package]] -name = "percent-encoding" -version = "2.3.2" +name = "pin-project-internal" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "pin-project-lite" @@ -1538,6 +3000,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "potential_utf" version = "0.1.6" @@ -1601,12 +3069,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "quinn" version = "0.11.11" @@ -1684,6 +3162,15 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" @@ -1715,6 +3202,12 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + [[package]] name = "rand_core" version = "0.9.5" @@ -1739,6 +3232,26 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1854,6 +3367,28 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "rustls" version = "0.23.44" @@ -1901,12 +3436,33 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.229" @@ -2000,7 +3556,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -2028,6 +3595,24 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + [[package]] name = "slab" version = "0.4.12" @@ -2068,12 +3653,47 @@ dependencies = [ "vob 3.0.6", ] +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -2086,6 +3706,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + [[package]] name = "subtle" version = "2.6.1" @@ -2134,6 +3776,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2183,6 +3838,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + [[package]] name = "time" version = "0.3.55" @@ -2287,6 +3953,31 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "tower" version = "0.5.3" @@ -2422,6 +4113,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-width" version = "0.2.2" @@ -2458,6 +4155,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -2501,6 +4209,16 @@ dependencies = [ "serde", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2609,6 +4327,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -2871,8 +4598,60 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe 7.3.0", +] + +[[package]] +name = "zstd" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +dependencies = [ + "zstd-safe 8.2.0", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index bf6734c4..8d1670a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,12 +7,10 @@ edition = "2021" version = "0.1.0" [workspace.dependencies] -# Keep every Planner crate on the same immutable revision. Alias upstream -# asap-types as planner-types to name its origin at use sites. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "ecea897f78640c377ce1d80846156f330f62dee2" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "ecea897f78640c377ce1d80846156f330f62dee2" } -asap-logical-asap-optimization = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "ecea897f78640c377ce1d80846156f330f62dee2" } -asap-plan-selection = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "ecea897f78640c377ce1d80846156f330f62dee2" } +# The backend depends on two Planner crates, at one immutable revision: +# `asap-planner` to plan and `asap-executor` to execute. Planner IR types +# are reached through `asap_executor::planner`. +asap-planner = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "ecea897f78640c377ce1d80846156f330f62dee2" } asap-executor = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "ecea897f78640c377ce1d80846156f330f62dee2" } serde = { version = "1.0", features = ["derive"] } diff --git a/README.md b/README.md index 4747bf47..f1342364 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,10 @@ given. ```text PlanningWorkload (JSON) DeploymentCapabilities + PlanSelectionSettings \ / - control_plane: ASAPPlanner stages 0-3 + control_plane: asap_planner::plan_installable | - plan.json: selected PhysicalASAPDAG + query per root + plan.json: InstallablePlan (selected PhysicalASAPDAG, query per + root, stored outputs with retention, raw windows) | data_plane: Remote Write in, Prometheus query API out ``` @@ -36,6 +37,11 @@ The backend is being rebuilt on the Planner's physical DAG: | Precompute engine: build ingestion-time outputs once per tumbling pane | done | | Query engine: run query-time nodes over stored panes and raw data | planned | +The data plane stores the plan's `stored_outputs`. Outputs that differ only +in which pane they read share one stored summary, kept for the largest +`retention_ms` the plan states among them (which covers range-evaluation +spans); panes ending before the oldest pane still read are evicted. + Panes close `--allowed-lateness-ms` (default 15 s) after the newest sample of a stored summary's metric, capped at the wall clock, passes their end; later samples for them are dropped. The first pane after startup is skipped because it may be partial. diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index c9de2d4f..b20fb2c1 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -4,10 +4,8 @@ version.workspace = true edition.workspace = true [dependencies] -planner-types.workspace = true -asap-frontend-promql.workspace = true -asap-logical-asap-optimization.workspace = true -asap-plan-selection.workspace = true +asap-planner.workspace = true +futures = "0.3" serde.workspace = true serde_json.workspace = true thiserror.workspace = true diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 99fb0eaa..7767e805 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -1,35 +1,19 @@ //! Backend control plane: asks ASAPPlanner for the workload's plan and //! writes it for the data plane to install. //! -//! The Planner runs every planning stage, including plan selection. This -//! crate only supplies the deployment inputs (what the data plane can execute -//! and keep, and how Stage 3 prices candidates) and records which query each -//! root of the selected `PhysicalASAPDAG` answers. +//! The Planner runs every planning stage, including plan selection, and +//! returns the selected DAG ready to install ([`InstallablePlan`]): the DAG, +//! the query each root answers, the outputs to store with how long each +//! must stay available, and the raw-sample windows read at query time. This +//! crate only supplies the deployment inputs: what the data plane can +//! execute and keep, and how Stage 3 prices candidates. use std::path::Path; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_plan_selection::{ - run_stage1_to_stage3, PlanSelectionSettings, PlanningModels, Stage1ToStage3Run, +use asap_planner::{ + plan_installable, DeploymentCapabilities, FrontendInput, InstallablePlan, InstallablePlanError, + PlanSelectionSettings, PlanningModels, PlanningWorkload, UserInput, }; -use planner_types::ir::operator::pane_chain::expand_pane_chains; -use planner_types::ir::physical_export::{compile_physical_asap_workload, PhysicalASAPDAGDocument}; -use planner_types::ir::schema::schema_support::with_promql_series_identity; -use planner_types::ir::QueryRoot; -use planner_types::workload::{PlanningWorkload, QueryLanguage, QueryWorkloadEntry}; -use serde::{Deserialize, Serialize}; - -/// The plan the data plane installs: the Planner's selected physical DAG and -/// the query each of its roots answers. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct InstalledPlan { - /// PromQL text of each root of `dag`, in root order. - pub queries: Vec, - /// The selected candidate's DAG, with every `PaneChain` expanded into - /// the panes the executor runs. - pub dag: PhysicalASAPDAGDocument, -} /// What the backend gives the Planner about its deployment. The Planner /// selects; these only say what can run and what it costs. @@ -44,153 +28,63 @@ pub struct DeploymentInputs { #[derive(Debug, thiserror::Error)] pub enum PlanError { - #[error("only PromQL workloads are supported, got {0:?}")] - UnsupportedLanguage(QueryLanguage), - #[error("workload: {0}")] - Workload(String), - #[error("query {0} lowers to a scalar, which the data plane cannot answer yet")] - ScalarRoot(usize), - #[error("PromQL lowering: {0}")] - Lowering(String), - #[error("query {query}: series identity: {message}")] - SeriesIdentity { query: usize, message: String }, #[error("planning: {0}")] - Planning(String), - #[error("invalid physical DAG: {0}")] - InvalidDag(String), - #[error("the DAG has {roots} roots for {queries} queries")] - RootCount { roots: usize, queries: usize }, + Planning(#[from] asap_planner::PlanError), + #[error("invalid plan: {0}")] + InvalidPlan(#[from] InstallablePlanError), #[error("{path}: {message}")] File { path: String, message: String }, } -impl InstalledPlan { - pub fn validate(&self) -> Result<(), PlanError> { - self.dag - .validate() - .map_err(|e| PlanError::InvalidDag(e.to_string()))?; - let roots = self.dag.dag.roots.len(); - if roots != self.queries.len() { - return Err(PlanError::RootCount { - roots, - queries: self.queries.len(), - }); - } - Ok(()) - } - - /// Reads and validates a plan written by [`InstalledPlan::save`]. - pub fn load(path: &Path) -> Result { - let file_error = |message: String| PlanError::File { - path: path.display().to_string(), - message, - }; - let text = std::fs::read_to_string(path).map_err(|e| file_error(e.to_string()))?; - let plan: Self = serde_json::from_str(&text).map_err(|e| file_error(e.to_string()))?; - plan.validate()?; - Ok(plan) - } - - pub fn save(&self, path: &Path) -> Result<(), PlanError> { - let text = serde_json::to_string_pretty(self).expect("plans serialize"); - std::fs::write(path, text).map_err(|e| PlanError::File { - path: path.display().to_string(), - message: e.to_string(), - }) - } - - /// The installed plan for `roots`, the timed roots of one candidate in - /// query order. - fn from_roots(queries: Vec, roots: &[QueryRoot]) -> Result { - let roots = roots - .iter() - .enumerate() - .map(|(i, root)| match root { - // The executor runs a pane chain as its panes. - QueryRoot::Operator(node) => expand_pane_chains(node) - .map(QueryRoot::Operator) - .map_err(|e| PlanError::InvalidDag(e.to_string())), - QueryRoot::Scalar(_) => Err(PlanError::ScalarRoot(i)), - }) - .collect::, _>>()?; - let dag = compile_physical_asap_workload(&roots) - .map_err(|e| PlanError::InvalidDag(e.to_string()))?; - let plan = Self { - queries, - dag: PhysicalASAPDAGDocument::new(dag), - }; - plan.validate()?; - Ok(plan) - } -} - -/// One planned workload: its queries and what Stages 1–3 returned. -struct Planned { - queries: Vec, - run: Stage1ToStage3Run, -} - -/// Numbers the lowered PromQL roots and makes their rows carry each -/// series' full identity, the representation per-series state needs at -/// runtime. -fn with_series_identity(roots: Vec) -> Result, PlanError> { - roots - .into_iter() - .enumerate() - .map(|(i, root)| match root { - QueryRoot::Operator(node) => with_promql_series_identity(&node) - .map(|node| (i, QueryRoot::Operator(node))) - .map_err(|message| PlanError::SeriesIdentity { query: i, message }), - QueryRoot::Scalar(_) => Err(PlanError::ScalarRoot(i)), - }) - .collect() -} - -/// Lowers `workload` (Stage 0) and runs Stages 1–3 under `inputs`. -fn plan(workload: &PlanningWorkload, inputs: &DeploymentInputs) -> Result { - let language = &workload.query_workload.language; - if *language != QueryLanguage::PromQL { - return Err(PlanError::UnsupportedLanguage(language.clone())); - } - workload - .validate() - .map_err(|e| PlanError::Workload(e.to_string()))?; - let roots = asap_frontend_promql::lower_promql_query_workload(workload) - .map_err(|e| PlanError::Lowering(e.to_string()))?; - let roots = with_series_identity(roots)?; - let entries: Vec = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); +/// Plans `workload` with the Planner (frontend and Stages 1–3) and returns +/// the plan Stage 3 selected, ready to install. Only PromQL workloads are +/// planned: the Planner refuses another language for the PromQL frontend. +/// It rejects every candidate that needs more than +/// `inputs.deployment_capabilities`, and refuses a plan a runtime could not +/// run through the executor. +pub fn plan_workload( + workload: &PlanningWorkload, + inputs: &DeploymentInputs, +) -> Result { let models = PlanningModels::builtin() .with_deployment_capabilities(&inputs.deployment_capabilities) .with_plan_selection(&inputs.plan_selection); - let run = run_stage1_to_stage3(roots, &entries, &data, models) - .map_err(|e| PlanError::Planning(e.to_string()))?; - Ok(Planned { - queries: entries.into_iter().map(|e| e.query.0).collect(), - run, + let input = UserInput::new(workload, FrontendInput::Promql { histograms: None }, models); + // Only the SQL frontend awaits anything; PromQL planning completes on + // its first poll. + let plan = futures::executor::block_on(plan_installable(input))?; + if let Some(selection) = &plan.selection { + tracing::info!(candidate = %selection.selected_candidate_id, "selected physical plan"); + } + Ok(plan) +} + +/// Writes `plan` as the plan file the data plane installs. +pub fn save_plan(plan: &InstallablePlan, path: &Path) -> Result<(), PlanError> { + let text = serde_json::to_string_pretty(plan).expect("plans serialize"); + std::fs::write(path, text).map_err(|e| PlanError::File { + path: path.display().to_string(), + message: e.to_string(), }) } -/// Plans `workload` with the Planner's stage pipeline (Stages 0–3) and -/// returns the plan Stage 3 selected. The Planner rejects every candidate -/// that needs more than `inputs.deployment_capabilities`. -pub fn plan_workload( - workload: &PlanningWorkload, - inputs: &DeploymentInputs, -) -> Result { - let planned = plan(workload, inputs)?; - let selection = planned - .run - .stage3_plan_selection - .map_err(|e| PlanError::Planning(e.to_string()))?; - tracing::info!(candidate = %selection.selected, "selected physical plan"); - InstalledPlan::from_roots(planned.queries, &selection.plan) +/// Reads and validates a plan written by [`save_plan`]. +pub fn load_plan(path: &Path) -> Result { + let file_error = |message: String| PlanError::File { + path: path.display().to_string(), + message, + }; + let text = std::fs::read_to_string(path).map_err(|e| file_error(e.to_string()))?; + let plan: InstallablePlan = + serde_json::from_str(&text).map_err(|e| file_error(e.to_string()))?; + plan.validate()?; + Ok(plan) } /// What this backend's data plane can execute and keep: every summary the /// Planner executor builds, raw samples kept in Prometheus, and no -/// query-time result kept across evaluations (the data plane does not -/// implement `MatAtQueryKeptAcrossEvaluations`). +/// query-time result kept across evaluations: the SDS describes such +/// outputs, but nothing builds them until the query engine does. pub fn backend_capabilities() -> DeploymentCapabilities { DeploymentCapabilities { supported_summaries: None, @@ -209,27 +103,3 @@ impl Default for DeploymentInputs { } } } - -#[cfg(test)] -mod tests { - use super::*; - use planner_types::ir::schema::Schema; - use planner_types::ir::{NonASAPOp, Operator, OperatorNode}; - - /// A root whose rows cannot carry the full series identity is rejected - /// rather than planned without it. - #[test] - fn rejects_a_root_without_a_series_identity_realization() { - let values = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![], - schema: Schema::default(), - })) - .unwrap(); - let error = with_series_identity(vec![QueryRoot::Operator(values)]) - .expect_err("the identity rewrite must fail"); - assert_eq!( - error.to_string(), - "query 0: series identity: operator has no dynamic series-identity realization" - ); - } -} diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index 8773f318..6eefba92 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -1,9 +1,9 @@ use std::path::{Path, PathBuf}; use anyhow::Context; +use asap_planner::PlanningWorkload; use clap::Parser; -use control_plane::{plan_workload, DeploymentInputs}; -use planner_types::workload::PlanningWorkload; +use control_plane::{plan_workload, save_plan, DeploymentInputs}; /// Plan a workload with ASAPPlanner and write the selected plan. #[derive(Parser)] @@ -45,7 +45,7 @@ fn main() -> anyhow::Result<()> { inputs.plan_selection.measured_operation_costs = read_json(path)?; } let plan = plan_workload(&workload, &inputs)?; - plan.save(&args.out)?; + save_plan(&plan, &args.out)?; println!( "wrote {} ({} queries, {} nodes)", args.out.display(), diff --git a/control_plane/tests/plan_workload.rs b/control_plane/tests/plan_workload.rs index 8d936400..f4855511 100644 --- a/control_plane/tests/plan_workload.rs +++ b/control_plane/tests/plan_workload.rs @@ -1,11 +1,11 @@ use std::path::{Path, PathBuf}; -use control_plane::{plan_workload, DeploymentInputs, InstalledPlan, PlanError}; -use planner_types::workload::{PlanningWorkload, QueryLanguage, SqlDialect}; +use asap_planner::{InstallablePlanError, PlanningWorkload}; +use control_plane::{load_plan, plan_workload, save_plan, DeploymentInputs, PlanError}; -fn example1() -> PlanningWorkload { - let path = - Path::new(env!("CARGO_MANIFEST_DIR")).join("../examples/planner-layering-example1.json"); +fn example(name: &str) -> PlanningWorkload { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join(format!("../examples/planner-layering-{name}.json")); serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() } @@ -17,10 +17,13 @@ fn temp_path(name: &str) -> PathBuf { /// query, and the plan survives the file the data plane loads. #[test] fn plans_example1_and_round_trips_the_plan() { - let workload = example1(); + let workload = example("example1"); let plan = plan_workload(&workload, &DeploymentInputs::default()).unwrap(); assert_eq!( - plan.queries, + plan.queries + .iter() + .map(|q| q.query.clone()) + .collect::>(), workload .query_workload .entries() @@ -29,35 +32,59 @@ fn plans_example1_and_round_trips_the_plan() { ); assert_eq!(plan.dag.dag.roots.len(), 2); let path = temp_path("example1.json"); - plan.save(&path).unwrap(); - assert_eq!(InstalledPlan::load(&path).unwrap(), plan); + save_plan(&plan, &path).unwrap(); + assert_eq!(load_plan(&path).unwrap(), plan); std::fs::remove_file(path).unwrap(); } -/// Only PromQL workloads are planned; SQL support comes later. +/// Example 4b without raw data stores its six 10-minute panes, built at +/// ingestion, each kept as long as the Planner states: 60 minutes back for +/// the oldest pane the 60-minute range reads. +#[test] +fn example4b_stores_panes_with_the_planners_retention() { + let mut inputs = DeploymentInputs::default(); + inputs.deployment_capabilities.raw_data_retained = false; + let plan = plan_workload(&example("example4b"), &inputs).unwrap(); + assert_eq!(plan.stored_outputs.len(), 6); + let mut offsets: Vec = plan + .stored_outputs + .iter() + .map(|output| { + assert_eq!(output.pane.width_ms, 600_000); + output.pane.end_offset_ms + }) + .collect(); + offsets.sort(); + assert_eq!(offsets, (0..6).map(|i| i * 600_000).collect::>()); + let retention = plan.stored_outputs.iter().map(|o| o.retention_ms).max(); + assert_eq!(retention, Some(3_600_000)); +} + +/// Only PromQL workloads are planned; the Planner's PromQL frontend refuses +/// SQL. #[test] fn rejects_sql_workloads() { - let mut workload = example1(); - workload.query_workload.language = QueryLanguage::SQL(SqlDialect::ClickhouseSQL); + let mut workload = example("example1"); + workload.query_workload.language = + serde_json::from_str(r#"{"sql": "clickhouse_sql"}"#).unwrap(); assert!(matches!( plan_workload(&workload, &DeploymentInputs::default()), - Err(PlanError::UnsupportedLanguage(_)) + Err(PlanError::Planning(_)) )); } /// A plan whose queries do not match the DAG's roots is not installed. #[test] fn load_rejects_a_plan_without_a_query_per_root() { - let mut plan = plan_workload(&example1(), &DeploymentInputs::default()).unwrap(); + let mut plan = plan_workload(&example("example1"), &DeploymentInputs::default()).unwrap(); plan.queries.pop(); let path = temp_path("mismatched.json"); - plan.save(&path).unwrap(); + save_plan(&plan, &path).unwrap(); assert!(matches!( - InstalledPlan::load(&path), - Err(PlanError::RootCount { - roots: 2, - queries: 1 - }) + load_plan(&path), + Err(PlanError::InvalidPlan( + InstallablePlanError::QueriesDoNotMatchRoots { roots: 2 } + )) )); std::fs::remove_file(path).unwrap(); } diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index cc64affc..5914ecd0 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -4,8 +4,7 @@ version.workspace = true edition.workspace = true [dependencies] -control_plane.workspace = true -planner-types.workspace = true +asap-planner.workspace = true asap-executor.workspace = true sha2 = "0.10" serde.workspace = true @@ -23,7 +22,6 @@ snap = "1.1" futures = "0.3" [dev-dependencies] +control_plane.workspace = true tower = { version = "0.5", features = ["util"] } http-body-util = "0.1" -# The tests check the SDS's frontier against the executor's own. -asap-executor = { workspace = true, features = ["test-support"] } diff --git a/data_plane/src/engine.rs b/data_plane/src/engine.rs index b10d6b05..21f6d082 100644 --- a/data_plane/src/engine.rs +++ b/data_plane/src/engine.rs @@ -6,7 +6,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc::{channel, Receiver, Sender}; use std::sync::{Arc, Mutex}; -use control_plane::InstalledPlan; +use asap_planner::InstallablePlan; use crate::precompute::{Precompute, PrecomputeOptions, PrecomputeStats, RawSample}; use crate::sds::Sds; @@ -185,20 +185,19 @@ struct Engine { impl Engine { fn new(plan_json: &str, options: PrecomputeOptions) -> anyhow::Result<(Self, Vec)> { - let plan: InstalledPlan = serde_json::from_str(plan_json)?; + let plan: InstallablePlan = serde_json::from_str(plan_json)?; plan.validate()?; - let dag = plan.dag.dag; - let frontier = Sds::ingestion_frontier(&dag)?; - let sds = Sds::from_dag(&dag, &frontier)?; + let sds = Sds::from_plan(&plan)?; tracing::info!( definitions = sds.definitions.len(), outputs = sds.outputs.len(), "stored summaries" ); + let queries = plan.queries.iter().map(|q| q.query.clone()).collect(); let engine = Self { - precompute: Precompute::new(dag, &sds, options)?, + precompute: Precompute::new(plan.dag.dag, &sds, options)?, }; - Ok((engine, plan.queries)) + Ok((engine, queries)) } fn serve(mut self, commands: Receiver, shared: Shared) { diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 34b49bf2..fe037eb7 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -45,7 +45,7 @@ pub struct DataPlane { impl DataPlane { /// Installs the plan serialized in `plan_json` (a - /// `control_plane::InstalledPlan`) with [`EngineOptions::default`], except + /// `asap_planner::InstallablePlan`) with [`EngineOptions::default`], except /// that panes close `allowed_lateness_ms` after the newest sample passes /// them. pub fn new( diff --git a/data_plane/src/precompute.rs b/data_plane/src/precompute.rs index 307947f9..7dda6f8f 100644 --- a/data_plane/src/precompute.rs +++ b/data_plane/src/precompute.rs @@ -17,18 +17,17 @@ //! one bad timestamp cannot close every pane up to it. When the watermark //! jumps, panes that would already be evicted are skipped, not built. -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use std::sync::Arc; use asap_executor::operators::Operator; use asap_executor::physical_planner::{compile, CompiledPhysicalDAG, InputContract, Source}; +use asap_executor::planner::ir::physical_export::PhysicalASAPDAG; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, SchemaRef}; use asap_executor::{RawSampleSelector, Sample}; use futures::executor::block_on; use futures::StreamExt; -use planner_types::ir::physical_export::{PhysicalASAPDAG, RawSampleWindow}; -use planner_types::ir::{NonASAPOp, Operator as IrOperator}; use crate::sds::{DefinitionId, Sds, SummaryDefinition, SummaryStore}; @@ -43,6 +42,8 @@ pub struct RawSample { #[derive(Debug, thiserror::Error)] pub enum PrecomputeError { + #[error("stored summaries: {0}")] + Plan(String), #[error("stored summary {0:?}: {1}")] Definition(DefinitionId, String), #[error(transparent)] @@ -99,9 +100,6 @@ struct Builder { selector: RawSampleSelector, raw_schema: SchemaRef, build: CompiledPhysicalDAG, - /// Largest offset any output reads, plus two panes: panes ending at or - /// before `watermark - retention_ms` are no longer read. - retention_ms: i64, /// Samples of open panes, by pane end. open: BTreeMap>, /// End of the first pane seen, which is skipped. `None` until then. @@ -133,57 +131,49 @@ pub struct Precompute { } impl Precompute { - /// Compiles one build per definition of `sds`, a description of `dag`. + /// Compiles one build per definition of `sds` built in an ingestion run, + /// in dependency order; `sds` describes `dag`. pub fn new( dag: PhysicalASAPDAG, sds: &Sds, options: PrecomputeOptions, ) -> Result { let mut builders = Vec::new(); - for definition in sds.definitions.values() { + // Ingestion-run definitions in dependency order; query-run ones are + // the query engine's. + let order = sds + .ingestion_build_order() + .map_err(|e| PrecomputeError::Plan(e.to_string()))?; + for id in &order { + let definition = &sds.definitions[id]; let error = |message: String| PrecomputeError::Definition(definition.id.clone(), message); + if !definition.stored_inputs.is_empty() { + return Err(error( + "built from other stored outputs, which the precompute engine cannot build yet" + .into(), + )); + } if definition.pane_ms <= 0 { return Err(error(format!("pane width {} ms", definition.pane_ms))); } - let scan = scan_below(&dag, definition.raw_input) - .ok_or_else(|| error("no time-series Scan below its TimeRange".into()))?; - let raw_schema = Arc::new(output_schema(&dag, definition.raw_input).clone()); + let raw_schema = Arc::new(output_schema(&dag, definition.raw_input()).clone()); // `window_batch` is called with each pane's end, so the window's // offset from evaluation time is not used. - let selector = RawSampleSelector::new( - &dag, - &RawSampleWindow { - time_range_node_id: definition.raw_input as usize, - width_ms: definition.pane_ms, - end_offset_ms: 0, - scan_node_id: scan as usize, - }, - )?; + let selector = RawSampleSelector::new(&dag, &definition.window)?; let build = compile( &dag, BTreeMap::from([( - definition.raw_input, + definition.raw_input(), InputContract::bounded(raw_schema.clone()), )]), &[definition.builder], )?; - let max_offset = sds - .outputs - .values() - .filter(|(id, _)| id == &definition.id) - .map(|(_, binding)| binding.offset_ms) - .max() - .unwrap_or(0); - let retention_ms = max_offset - .checked_add(definition.pane_ms.saturating_mul(2)) - .ok_or_else(|| error("retention overflows".into()))?; builders.push(Builder { definition: definition.clone(), selector, raw_schema, build, - retention_ms, open: BTreeMap::new(), first_pane_end: None, built_through: None, @@ -287,16 +277,13 @@ impl Precompute { }; let pane_ms = builder.definition.pane_ms; let id = &builder.definition.id; - // Panes ending at or before this are evicted once built: skip them. - let expired_through = watermark.saturating_sub(builder.retention_ms); + // Panes ending before this are no longer read: skip them. + let oldest_kept_end = builder.definition.oldest_kept_end_ms(watermark); let Some(mut end) = built.checked_add(pane_ms) else { continue; }; - if end <= expired_through { - let Some(first_kept) = expired_through - .checked_add(1) - .and_then(|t| pane_end(t, pane_ms)) - else { + if end < oldest_kept_end { + let Some(first_kept) = pane_end(oldest_kept_end, pane_ms) else { continue; }; self.stats.panes_skipped_expired += @@ -324,7 +311,7 @@ impl Precompute { None => break, } } - self.store.evict_through(id, expired_through); + self.store.evict_before(id, oldest_kept_end); } } } @@ -370,7 +357,7 @@ fn build_pane( )?); let build = builder .build - .instantiate(BTreeMap::from([(builder.definition.raw_input, input)]))?; + .instantiate(BTreeMap::from([(builder.definition.raw_input(), input)]))?; let rows = run_root(&build, builder.definition.builder, scope)?; let schema = Arc::new(builder.definition.output_schema.clone()); Ok(Batch::try_new(schema, rows)?) @@ -393,27 +380,10 @@ pub fn run_root( }) } -fn output_schema(dag: &PhysicalASAPDAG, id: u64) -> &planner_types::ir::schema::Schema { +fn output_schema(dag: &PhysicalASAPDAG, id: u64) -> &asap_executor::planner::ir::schema::Schema { &dag.nodes .iter() .find(|n| n.id as u64 == id) .expect("SDS names DAG nodes") .output_schema } - -/// The time-series `Scan` that `id` reads. -fn scan_below(dag: &PhysicalASAPDAG, id: u64) -> Option { - let inputs: HashMap = dag - .edges - .iter() - .map(|e| (e.consumer as u64, e.producer as u64)) - .collect(); - let mut current = id; - loop { - let node = dag.nodes.iter().find(|n| n.id as u64 == current)?; - if let IrOperator::NonASAP(NonASAPOp::Scan { .. }) = &node.payload { - return Some(current); - } - current = *inputs.get(¤t)?; - } -} diff --git a/data_plane/src/sds.rs b/data_plane/src/sds.rs index 771488af..69f5d6e4 100644 --- a/data_plane/src/sds.rs +++ b/data_plane/src/sds.rs @@ -1,29 +1,34 @@ -//! Stored summaries (SDS): what the data plane stores for a plan's -//! ingestion-time outputs, and how a query finds them again. +//! Stored summaries (SDS): the immutable summaries the data plane stores +//! for a plan, and how a query finds them again. //! -//! The Planner's selected DAG marks each output computed at ingestion time -//! (`MatAtIngestion`) and read at query time: the executor's materialization -//! frontier. The control plane installs the DAG with each `PaneChain` -//! expanded into its panes, so each such output reads one tumbling pane of -//! raw samples: +//! The installed plan lists every output it stores across evaluations +//! (`InstallablePlan::stored_outputs`): which run builds it (an ingestion +//! run, or the query run of the first evaluation that reads it), what it +//! covers, and how long it must stay available. Today the Planner's contract +//! states each stored output's coverage as one pane of raw samples: at +//! evaluation time `T` it covers `(T - end_offset - width, T - end_offset]`, +//! the PromQL range convention. //! -//! ```text -//! frontier node <- ... <- TimeRange(width) <- TimeShift(offset)* <- Scan -//! ``` +//! Outputs that differ only in which coverage they read compute the same +//! thing, so they share one [`SummaryDefinition`]: the data plane builds each +//! summary of a definition once, keyed by its coverage end, and every output +//! reads the one it needs. Identity is the SDS's own: the Planner states what +//! each node computes, not which nodes are equal. //! -//! At evaluation time `T` it covers `(T - offset - width, T - offset]`, the -//! PromQL range convention. -//! Outputs that differ only in `offset` compute the same thing over different -//! panes, so they share one [`SummaryDefinition`]: the data plane builds each -//! pane of a definition once, and every output reads the pane it needs. +//! A stored output may in future be built from other stored outputs +//! (rollups over time or across series). [`Sds::ingestion_build_order`] +//! orders definitions so that each one's stored inputs come first; the +//! precompute engine refuses such a definition until it can build it once +//! all its stored inputs are sealed. -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeMap, HashMap}; -use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGNode}; -use planner_types::ir::properties::ExecutionTiming::IngestionTime; -use planner_types::ir::schema::Schema; -use planner_types::ir::{NonASAPOp, Operator}; -use planner_types::physical::Materialization; +use asap_executor::planner::ir::physical_export::{ + PhysicalASAPDAG, PhysicalASAPDAGNode, RawSampleWindow, +}; +use asap_executor::planner::ir::schema::Schema; +use asap_executor::planner::ir::{NonASAPOp, Operator}; +use asap_planner::{InstallablePlan, StoredOutputBuild}; use sha2::{Digest, Sha256}; /// Identifies what a stored summary computes, independent of node ids and of @@ -31,32 +36,63 @@ use sha2::{Digest, Sha256}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct DefinitionId(pub String); -/// One computation the data plane maintains pane by pane. +/// One computation the data plane stores summary by summary. #[derive(Debug, Clone, PartialEq)] pub struct SummaryDefinition { pub id: DefinitionId, - /// Pane width: each stored summary covers `(end - pane_ms, end]`. + /// Which run builds its summaries: an ingestion run, or the query run of + /// the first evaluation that needs one (kept once its coverage is + /// closed). Ingestion when any output of the definition says so. + pub built_at: StoredOutputBuild, + /// Coverage width: each stored summary covers `(end - pane_ms, end]`. pub pane_ms: i64, - /// The frontier output that builds a pane; any output of the definition - /// computes the same, and this is the first in node order. + /// The output that builds a pane; any output of the definition computes + /// the same, and this is the first in node order. pub builder: u64, - /// The raw input of the build: the `TimeRange` node fed one pane's samples. - pub raw_input: u64, + /// The builder's pane: its `TimeRange`, fed one pane's samples, and the + /// `Scan` that selects them. + pub window: RawSampleWindow, pub output_schema: Schema, + /// The definitions of the stored outputs the builder reads, which must + /// be built first. Empty while every stored output reads raw samples. + pub stored_inputs: Vec, + /// How far back from the newest evaluation time a summary stays available: + /// the largest `retention_ms` the plan states for the definition's + /// outputs. A pane that starts earlier is no longer read. + pub retention_ms: i64, +} + +impl SummaryDefinition { + /// The builder's `TimeRange`, which one pane's samples feed. + pub fn raw_input(&self) -> u64 { + self.window.time_range_node_id as u64 + } + + /// The end of the oldest pane still read when the newest evaluation is + /// at `newest_ms`: panes ending before it are evicted. + pub fn oldest_kept_end_ms(&self, newest_ms: i64) -> i64 { + newest_ms + .saturating_sub(self.retention_ms) + .saturating_add(self.pane_ms) + } } -/// How one frontier output reads its definition: at evaluation time `T`, the -/// pane ending at `T - offset_ms`. +/// How one output reads its definition: at evaluation time `T`, the pane +/// ending at `T - offset_ms`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct PaneBinding { pub offset_ms: i64, + /// The output's own `TimeRange` node: where that pane's raw samples + /// feed when the output is computed at query time. + pub raw_input: u64, } /// The stored summaries of one plan. #[derive(Debug, Clone, PartialEq, Default)] pub struct Sds { + /// Every definition, built at ingestion or at query time, by identity. pub definitions: BTreeMap, - /// Every frontier output, with its definition and pane. + /// Every stored output, with its definition and coverage. pub outputs: BTreeMap, } @@ -64,159 +100,134 @@ pub struct Sds { pub enum SdsError { #[error("node {0} is not in the DAG")] UnknownNode(u64), - #[error("ingestion-time output {node} does not read one pane of one Scan: {reason}")] - UnsupportedShape { node: u64, reason: String }, - /// The data plane keeps no query-time result across evaluations; it - /// declares `query_time_results_retained: false`, so the Planner should - /// not select such a plan. - #[error("node {0} is kept across evaluations (MatAtQueryKeptAcrossEvaluations), which the data plane does not support")] - KeptAcrossEvaluations(u64), - #[error("ingestion-time node {0} reads a query-time node")] - QueryTimeFeedsIngestion(u64), + #[error("stored output {0}'s retention does not fit in i64 milliseconds")] + RetentionOverflow(u64), + /// Stored outputs read each other in a cycle. + #[error("stored outputs read each other in a cycle through {0:?}")] + Cycle(DefinitionId), } impl Sds { - /// The plan's materialization frontier: every node computed at ingestion - /// time whose output is read at query time, by a query-time node or as a - /// query's answer (a root, or a node a scalar root reads). In node order. - pub fn ingestion_frontier(dag: &PhysicalASAPDAG) -> Result, SdsError> { - let at_ingestion: HashMap = dag - .nodes - .iter() - .map(|n| (n.id as u64, n.output_state.timing == IngestionTime)) - .collect(); - let timing = |id: u64| { - at_ingestion - .get(&id) - .copied() - .ok_or(SdsError::UnknownNode(id)) - }; - let mut frontier = BTreeSet::new(); - let roots = dag.roots.iter().copied().chain( - dag.scalar_roots - .iter() - .flat_map(|(_, expr)| expr.operator_refs().into_iter().copied()), - ); - for root in roots { - if timing(root as u64)? { - frontier.insert(root as u64); - } - } - for edge in &dag.edges { - let (producer, consumer) = (edge.producer as u64, edge.consumer as u64); - match (timing(producer)?, timing(consumer)?) { - (true, false) => { - frontier.insert(producer); - } - (false, true) => return Err(SdsError::QueryTimeFeedsIngestion(consumer)), - _ => {} - } - } - Ok(frontier.into_iter().collect()) - } - - /// The definitions behind `frontier`, the DAG outputs computed at - /// ingestion time and read at query time. - pub fn from_dag(dag: &PhysicalASAPDAG, frontier: &[u64]) -> Result { + /// The definitions behind `plan`'s stored outputs, built in an ingestion + /// run or in a query run and kept. + pub fn from_plan(plan: &InstallablePlan) -> Result { + let dag = &plan.dag.dag; let nodes: HashMap = dag.nodes.iter().map(|n| (n.id as u64, n)).collect(); - let mut inputs: HashMap> = HashMap::new(); - for edge in &dag.edges { - inputs - .entry(edge.consumer as u64) - .or_default() - .push(edge.producer as u64); - } - if let Some(node) = dag.nodes.iter().find(|n| { - matches!( - n.materialization, - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ) - }) { - return Err(SdsError::KeptAcrossEvaluations(node.id as u64)); - } + let inputs = inputs_of(dag); let mut sds = Sds::default(); let mut hashes = HashMap::new(); - for &node in frontier { - let (raw_input, pane_ms, offset_ms) = pane_of(node, &nodes, &inputs)?; + for output in &plan.stored_outputs { + let node = output.node_id as u64; + let (definitions, outputs) = (&mut sds.definitions, &mut sds.outputs); let id = DefinitionId(hex(&identity(node, &nodes, &inputs, &mut hashes)?)); + let retention_ms = i64::try_from(output.retention_ms) + .map_err(|_| SdsError::RetentionOverflow(node))?; let output_schema = nodes[&node].output_schema.clone(); - let definition = - sds.definitions - .entry(id.clone()) - .or_insert_with(|| SummaryDefinition { - id: id.clone(), - pane_ms, - builder: node, - raw_input, - output_schema, - }); + let definition = definitions + .entry(id.clone()) + .or_insert_with(|| SummaryDefinition { + id: id.clone(), + built_at: output.built_at, + pane_ms: output.pane.width_ms, + builder: node, + window: output.pane.clone(), + output_schema, + stored_inputs: Vec::new(), + retention_ms, + }); + if output.built_at == StoredOutputBuild::IngestionRun { + definition.built_at = StoredOutputBuild::IngestionRun; + } if node < definition.builder { definition.builder = node; - definition.raw_input = raw_input; + definition.window = output.pane.clone(); } - sds.outputs.insert(node, (id, PaneBinding { offset_ms })); + definition.retention_ms = definition.retention_ms.max(retention_ms); + let binding = PaneBinding { + offset_ms: output.pane.end_offset_ms, + raw_input: output.pane.time_range_node_id as u64, + }; + outputs.insert(node, (id, binding)); } + sds.link_stored_inputs(&inputs); + sds.ingestion_build_order()?; Ok(sds) } -} -/// `(raw input, pane width, offset)` of a frontier output: follow its single -/// inputs down to the `TimeRange`, then sum the `TimeShift`s down to the `Scan`. -fn pane_of( - node: u64, - nodes: &HashMap, - inputs: &HashMap>, -) -> Result<(u64, i64, i64), SdsError> { - let unsupported = |reason: &str| SdsError::UnsupportedShape { - node, - reason: reason.into(), - }; - let only_input = |id: u64| match inputs.get(&id).map(Vec::as_slice) { - Some([input]) => Ok(*input), - _ => Err(unsupported( - "every operator above the Scan must have one input", - )), - }; - let mut current = node; - let (raw_input, pane_ms) = loop { - match relational(nodes, current)? { - Some(NonASAPOp::TimeRange { range, .. }) => { - break (current, range.as_millis() as i64); - } - Some(NonASAPOp::TimeShift { .. }) => { - return Err(unsupported("a TimeShift above the TimeRange")); + /// Records, for each definition, the definitions of the stored outputs + /// its builder reads: those found below the builder before any raw + /// input. + fn link_stored_inputs(&mut self, inputs: &HashMap>) { + let stored: HashMap = self + .outputs + .iter() + .map(|(node, (id, _))| (*node, id.clone())) + .collect(); + for definition in self.definitions.values_mut() { + let mut found = Vec::new(); + let mut pending: Vec = + inputs.get(&definition.builder).cloned().unwrap_or_default(); + let mut seen = std::collections::HashSet::new(); + while let Some(node) = pending.pop() { + if !seen.insert(node) { + continue; + } + if let Some(id) = stored.get(&node) { + if !found.contains(id) { + found.push(id.clone()); + } + continue; + } + pending.extend(inputs.get(&node).into_iter().flatten().copied()); } - Some(NonASAPOp::Scan { .. }) => return Err(unsupported("no TimeRange")), - _ => current = only_input(current)?, + found.sort(); + definition.stored_inputs = found; } - }; - let mut offset_ms = 0; - current = only_input(raw_input)?; - loop { - match relational(nodes, current)? { - Some(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_none() => { - offset_ms += shift.offset_ms; - current = only_input(current)?; + } + + /// The definitions built in ingestion runs, each after every definition + /// it reads (`stored_inputs`). Fails on a cycle. + pub fn ingestion_build_order(&self) -> Result, SdsError> { + let mut order = Vec::new(); + let mut state: HashMap<&DefinitionId, bool> = HashMap::new(); // false: visiting + fn visit<'a>( + sds: &'a Sds, + id: &'a DefinitionId, + state: &mut HashMap<&'a DefinitionId, bool>, + order: &mut Vec, + ) -> Result<(), SdsError> { + match state.get(id) { + Some(true) => return Ok(()), + Some(false) => return Err(SdsError::Cycle(id.clone())), + None => {} } - Some(NonASAPOp::Scan { .. }) => return Ok((raw_input, pane_ms, offset_ms)), - _ => { - return Err(unsupported( - "only offset TimeShifts between TimeRange and Scan", - )) + state.insert(id, false); + for input in &sds.definitions[id].stored_inputs { + visit(sds, input, state, order)?; } + state.insert(id, true); + order.push(id.clone()); + Ok(()) + } + for id in self.definitions.keys() { + visit(self, id, &mut state, &mut order)?; } + order.retain(|id| self.definitions[id].built_at == StoredOutputBuild::IngestionRun); + Ok(order) } } -fn relational<'a>( - nodes: &HashMap, - id: u64, -) -> Result>, SdsError> { - match &nodes.get(&id).ok_or(SdsError::UnknownNode(id))?.payload { - Operator::NonASAP(operator) => Ok(Some(operator)), - Operator::ASAP(_) => Ok(None), +/// Each node's inputs, in edge order. +fn inputs_of(dag: &PhysicalASAPDAG) -> HashMap> { + let mut inputs: HashMap> = HashMap::new(); + for edge in &dag.edges { + inputs + .entry(edge.consumer as u64) + .or_default() + .push(edge.producer as u64); } + inputs } /// SHA-256 over a node's payload and output schema, then its inputs' @@ -293,10 +304,10 @@ impl SummaryStore { self.panes.get(definition)?.get(&end_ms) } - /// Drops the panes of `definition` that end at or before `end_ms`. - pub fn evict_through(&mut self, definition: &DefinitionId, end_ms: i64) { + /// Drops the panes of `definition` that end before `end_ms`. + pub fn evict_before(&mut self, definition: &DefinitionId, end_ms: i64) { if let Some(panes) = self.panes.get_mut(definition) { - *panes = panes.split_off(&end_ms.saturating_add(1)); + *panes = panes.split_off(&end_ms); } } diff --git a/data_plane/tests/common/mod.rs b/data_plane/tests/common/mod.rs index 6029a045..a7b7b434 100644 --- a/data_plane/tests/common/mod.rs +++ b/data_plane/tests/common/mod.rs @@ -1,6 +1,7 @@ use std::path::Path; -use control_plane::{plan_workload, DeploymentInputs, InstalledPlan}; +use asap_planner::{InstallablePlan, PlanningWorkload}; +use control_plane::{plan_workload, DeploymentInputs}; /// The deployment inputs of a backend that keeps raw data or not. pub fn inputs(raw_data_retained: bool) -> DeploymentInputs { @@ -9,13 +10,19 @@ pub fn inputs(raw_data_retained: bool) -> DeploymentInputs { inputs } -/// The selected plan of a #509 example workload in `examples/`, for a -/// deployment that keeps raw data or not. -pub fn plan(example: &str, raw_data_retained: bool) -> InstalledPlan { +/// A #509 example workload in `examples/`. +#[allow(dead_code)] +pub fn workload(example: &str) -> PlanningWorkload { let path = Path::new(env!("CARGO_MANIFEST_DIR")) .join(format!("../examples/planner-layering-{example}.json")); - let workload = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); - plan_workload(&workload, &inputs(raw_data_retained)).unwrap() + serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() +} + +/// The selected plan of a #509 example workload, for a deployment that +/// keeps raw data or not. +#[allow(dead_code)] +pub fn plan(example: &str, raw_data_retained: bool) -> InstallablePlan { + plan_workload(&workload(example), &inputs(raw_data_retained)).unwrap() } #[allow(dead_code)] diff --git a/data_plane/tests/precompute.rs b/data_plane/tests/precompute.rs index 553c8911..8dbde308 100644 --- a/data_plane/tests/precompute.rs +++ b/data_plane/tests/precompute.rs @@ -3,10 +3,11 @@ mod common; use std::collections::BTreeMap; use std::sync::Arc; -use control_plane::{plan_workload, InstalledPlan}; +use asap_executor::planner::workload::{PlanningWorkload, Query}; +use asap_planner::InstallablePlan; +use control_plane::plan_workload; use data_plane::precompute::{pane_end, Precompute, PrecomputeOptions, RawSample}; use data_plane::sds::Sds; -use planner_types::workload::{PlanningWorkload, Query}; const MINUTE: i64 = 60_000; const PANE: i64 = 10 * MINUTE; @@ -15,13 +16,13 @@ const DAY: i64 = 24 * 60 * MINUTE; /// The Prometheus stale marker's bits. const STALE_MARKER: u64 = 0x7ff0_0000_0000_0002; -fn precompute(plan: InstalledPlan) -> (Precompute, Sds) { +fn precompute(plan: InstallablePlan) -> (Precompute, Sds) { precompute_with(plan, 0) } -fn precompute_with(plan: InstalledPlan, allowed_lateness_ms: u64) -> (Precompute, Sds) { +fn precompute_with(plan: InstallablePlan, allowed_lateness_ms: u64) -> (Precompute, Sds) { + let sds = Sds::from_plan(&plan).unwrap(); let dag = plan.dag.dag; - let sds = Sds::from_dag(&dag, &Sds::ingestion_frontier(&dag).unwrap()).unwrap(); let options = PrecomputeOptions { allowed_lateness_ms, ..PrecomputeOptions::default() @@ -197,15 +198,17 @@ fn watermark_jump_builds_only_retained_panes() { engine.ingest_at(&samples("latency_ms", &["a"], 0, 25 * MINUTE), 30 * DAY); let built = engine.stats.panes_built; engine.ingest_at(&[sample("latency_ms", "a", 30 * DAY, 1.0)], 30 * DAY); - // Example 4b reads panes up to 50 min back, plus two panes: 70 min. - assert_eq!(engine.stats.panes_built - built, 7); + // The plan keeps Example 4b's panes 60 min back: the six panes ending + // at 30 days - 50 min through 30 days. + assert_eq!(engine.stats.panes_built - built, 6); assert_eq!( engine.stats.panes_skipped_expired, - (30 * DAY - 70 * MINUTE) as u64 / PANE as u64 - 2 + (30 * DAY - 80 * MINUTE) as u64 / PANE as u64 ); assert!(engine.store.get(&id, 30 * DAY).is_some()); + assert!(engine.store.get(&id, 30 * DAY - 6 * PANE).is_none()); assert!(engine.store.get(&id, 2 * PANE).is_none()); - assert_eq!(engine.store.len(), 7); + assert_eq!(engine.store.len(), 6); } /// Timestamps at the ends of `i64` neither overflow nor loop. @@ -231,28 +234,30 @@ fn extreme_timestamps_do_not_overflow() { #[test] fn rejects_zero_width_panes() { let plan = common::plan("example4b", false); + let mut sds = Sds::from_plan(&plan).unwrap(); let dag = plan.dag.dag; - let mut sds = Sds::from_dag(&dag, &Sds::ingestion_frontier(&dag).unwrap()).unwrap(); for definition in sds.definitions.values_mut() { definition.pane_ms = 0; } assert!(Precompute::new(dag, &sds, PrecomputeOptions::default()).is_err()); } -/// Panes older than the retention (largest offset plus two panes) are -/// evicted as the watermark advances. +/// Panes older than the plan's retention are evicted as the watermark +/// advances. #[test] fn evicts_panes_past_retention() { let (mut engine, sds) = precompute(common::plan("example4b", false)); let id = sds.definitions.keys().next().unwrap().clone(); engine.ingest(&samples("latency_ms", &["a"], 0, 25 * MINUTE)); assert!(engine.store.get(&id, 2 * PANE).is_some()); - // Watermark 100 min: panes ending at or before 30 min are evicted. + // Watermark 100 min, retention 60 min: the oldest pane read is + // (40, 50 min], so panes ending before 50 min are evicted. engine.ingest(&[sample("latency_ms", "a", 100 * MINUTE, 1.0)]); assert!(engine.store.get(&id, 2 * PANE).is_none()); - assert!(engine.store.get(&id, 3 * PANE).is_none()); - assert!(engine.store.get(&id, 4 * PANE).is_some()); + assert!(engine.store.get(&id, 4 * PANE).is_none()); + assert!(engine.store.get(&id, 5 * PANE).is_some()); assert!(engine.store.get(&id, 10 * PANE).is_some()); + assert_eq!(engine.store.len(), 6); } /// Out-of-order samples within the allowed lateness reach their pane. @@ -319,3 +324,16 @@ fn far_future_unrouted_sample_counts_as_unrouted() { assert_eq!(engine.stats.samples_unrouted, 1); assert_eq!(engine.stats.samples_dropped_future, 0); } + +/// Summaries built in a query run are not built at ingestion. +#[test] +fn does_not_build_query_run_summaries() { + let mut plan = common::plan("example4b", false); + for output in &mut plan.stored_outputs { + output.built_at = asap_planner::StoredOutputBuild::QueryRun; + } + let (mut engine, _) = precompute(plan); + engine.ingest(&samples("latency_ms", &["a"], 0, 45 * MINUTE)); + assert_eq!(engine.stats.panes_built, 0); + assert_eq!(engine.stats.samples_unrouted, 180); +} diff --git a/data_plane/tests/sds.rs b/data_plane/tests/sds.rs index 0f5361cd..ed5cbbdb 100644 --- a/data_plane/tests/sds.rs +++ b/data_plane/tests/sds.rs @@ -1,38 +1,17 @@ mod common; +use asap_executor::planner::ir::{NonASAPOp, Operator}; +use asap_planner::{InstallablePlan, StoredOutputBuild}; use common::plan; use data_plane::sds::{DefinitionId, Sds, SdsError, SummaryStore}; -use planner_types::ir::physical_export::PhysicalASAPDAG; -use planner_types::ir::{NonASAPOp, Operator}; -use planner_types::physical::{Materialization, Medium}; - -fn sds_of(dag: &PhysicalASAPDAG) -> Result { - Sds::from_dag(dag, &Sds::ingestion_frontier(dag).unwrap()) -} - -/// The SDS's frontier is the executor's: every ingestion-time output read at -/// query time, on every example with and without raw data. -#[test] -fn ingestion_frontier_matches_the_executor() { - for example in ["example1", "example3b", "example4b"] { - for raw_data_retained in [true, false] { - let dag = plan(example, raw_data_retained).dag.dag; - let executor = asap_executor::physical_planner::frontier_from_timing(&dag).unwrap(); - assert_eq!( - Sds::ingestion_frontier(&dag).unwrap(), - executor, - "{example} raw_data_retained={raw_data_retained}" - ); - } - } -} /// Example 4b's six ingestion-time KLL panes are one definition with a -/// 10-minute pane; each output reads the pane `i * 10min` before evaluation. +/// 10-minute pane; each output reads the pane `i * 10min` before evaluation, +/// and the definition is kept for the largest retention among them. #[test] fn example4b_panes_share_one_definition() { let plan = plan("example4b", false); - let sds = sds_of(&plan.dag.dag).unwrap(); + let sds = Sds::from_plan(&plan).unwrap(); assert_eq!(sds.definitions.len(), 1); let definition = sds.definitions.values().next().unwrap(); assert_eq!(definition.pane_ms, 600_000); @@ -46,12 +25,14 @@ fn example4b_panes_share_one_definition() { .collect(); offsets.sort(); assert_eq!(offsets, (0..6).map(|i| i * 600_000).collect::>()); + let largest = plan.stored_outputs.iter().map(|o| o.retention_ms).max(); + assert_eq!(Some(definition.retention_ms as u64), largest); let raw = plan .dag .dag .nodes .iter() - .find(|n| n.id as u64 == definition.raw_input) + .find(|n| n.id as u64 == definition.raw_input()) .unwrap(); assert!(matches!( raw.payload, @@ -59,85 +40,127 @@ fn example4b_panes_share_one_definition() { )); } +/// A definition is kept for the largest retention among its outputs, not +/// its first output's. +#[test] +fn retention_is_the_largest_among_outputs() { + let mut plan = plan("example4b", false); + let last = plan.stored_outputs.len() - 1; + plan.stored_outputs[last].retention_ms = 7 * 3_600_000; + let sds = Sds::from_plan(&plan).unwrap(); + let definition = sds.definitions.values().next().unwrap(); + assert_eq!(definition.retention_ms, 7 * 3_600_000); +} + /// A plan computed entirely at query time stores nothing. #[test] fn query_time_plans_have_no_definitions() { let plan = plan("example1", true); - assert_eq!(sds_of(&plan.dag.dag).unwrap(), Sds::default()); + assert_eq!(Sds::from_plan(&plan).unwrap(), Sds::default()); } -/// Identity depends on what is computed, not on how nodes are numbered. -#[test] -fn identity_ignores_node_numbering() { - let dag = plan("example4b", false).dag.dag; - let mut renumbered = dag.clone(); +/// `plan` with every node id shifted by 100. +fn renumbered(plan: &InstallablePlan) -> InstallablePlan { + let mut plan = plan.clone(); let renumber = |id: &usize| id + 100; - for node in &mut renumbered.nodes { + let dag = &mut plan.dag.dag; + for node in &mut dag.nodes { node.id = renumber(&node.id); node.payload = node.payload.map_children(renumber); } - for edge in &mut renumbered.edges { + for edge in &mut dag.edges { edge.producer = renumber(&edge.producer); edge.consumer = renumber(&edge.consumer); } - renumbered.roots = renumbered.roots.iter().map(renumber).collect(); + dag.roots = dag.roots.iter().map(renumber).collect(); + for output in &mut plan.stored_outputs { + output.node_id = renumber(&output.node_id); + output.pane.time_range_node_id = renumber(&output.pane.time_range_node_id); + output.pane.scan_node_id = renumber(&output.pane.scan_node_id); + } + plan +} + +/// Identity depends on what is computed, not on how nodes are numbered. +#[test] +fn identity_ignores_node_numbering() { + let plan = plan("example4b", false); let ids = |sds: Sds| sds.definitions.into_keys().collect::>(); assert_eq!( - ids(sds_of(&renumbered).unwrap()), - ids(sds_of(&dag).unwrap()) + ids(Sds::from_plan(&renumbered(&plan)).unwrap()), + ids(Sds::from_plan(&plan).unwrap()) ); } /// The same computation over another metric is another definition. #[test] fn identity_depends_on_the_source() { - let dag = plan("example4b", false).dag.dag; - let other: PhysicalASAPDAG = serde_json::from_str( - &serde_json::to_string(&dag) + let plan = plan("example4b", false); + let other: InstallablePlan = serde_json::from_str( + &serde_json::to_string(&plan) .unwrap() .replace("latency_ms", "other"), ) .unwrap(); assert_ne!( - sds_of(&other).unwrap().definitions.into_keys().next(), - sds_of(&dag).unwrap().definitions.into_keys().next() + Sds::from_plan(&other) + .unwrap() + .definitions + .into_keys() + .next(), + Sds::from_plan(&plan) + .unwrap() + .definitions + .into_keys() + .next() ); } -/// An ingestion-time output must read a pane of raw samples. +/// Outputs built in a query run and kept are stored summaries too: their +/// definition says so, and no ingestion run builds them. #[test] -fn rejects_outputs_without_a_pane() { - let dag = plan("example4b", false).dag.dag; - let scan = dag - .nodes - .iter() - .find(|n| matches!(n.payload, Operator::NonASAP(NonASAPOp::Scan { .. }))) - .unwrap(); - let scan = scan.id as u64; - assert_eq!( - Sds::from_dag(&dag, &[scan]), - Err(SdsError::UnsupportedShape { - node: scan, - reason: "no TimeRange".into() - }) - ); +fn query_run_outputs_are_query_run_definitions() { + let mut plan = plan("example4b", false); + for output in &mut plan.stored_outputs { + output.built_at = StoredOutputBuild::QueryRun; + } + let sds = Sds::from_plan(&plan).unwrap(); + assert_eq!(sds.definitions.len(), 1); + let definition = sds.definitions.values().next().unwrap(); + assert_eq!(definition.built_at, StoredOutputBuild::QueryRun); + assert_eq!(sds.outputs.len(), 6); + assert!(sds.ingestion_build_order().unwrap().is_empty()); } -/// A plan that keeps a query-time result across evaluations is not -/// installed: the data plane declares it cannot keep one. +/// A definition shared by an ingestion-run and a query-run output is built +/// at ingestion. #[test] -fn rejects_results_kept_across_evaluations() { - let mut dag = plan("example4b", true).dag.dag; - let node = &mut dag.nodes[0]; - node.materialization = Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory); - let id = node.id as u64; +fn ingestion_wins_when_outputs_share_a_definition() { + let mut plan = plan("example4b", false); + plan.stored_outputs[0].built_at = StoredOutputBuild::QueryRun; + let sds = Sds::from_plan(&plan).unwrap(); + let definition = sds.definitions.values().next().unwrap(); + assert_eq!(definition.built_at, StoredOutputBuild::IngestionRun); assert_eq!( - Sds::from_dag(&dag, &[]), - Err(SdsError::KeptAcrossEvaluations(id)) + sds.ingestion_build_order().unwrap(), + vec![definition.id.clone()] ); } -/// Panes are found by definition and end, and evicted once no query needs them. +/// Every stored output reads raw samples today, so no definition has stored +/// inputs; a definition that read itself would be a cycle. +#[test] +fn build_order_follows_stored_inputs() { + let mut sds = Sds::from_plan(&plan("example4b", false)).unwrap(); + let definition = sds.definitions.values_mut().next().unwrap(); + assert!(definition.stored_inputs.is_empty()); + let id = definition.id.clone(); + definition.stored_inputs = vec![id.clone()]; + assert_eq!(sds.ingestion_build_order(), Err(SdsError::Cycle(id))); +} + +/// Panes are found by definition and end, and evicted once no query needs +/// them. #[test] fn store_finds_and_evicts_panes() { let (a, b) = (DefinitionId("a".into()), DefinitionId("b".into())); @@ -148,7 +171,7 @@ fn store_finds_and_evicts_panes() { store.insert(&b, 60, "b60".to_string()); assert_eq!(store.get(&a, 120).map(String::as_str), Some("a120")); assert_eq!(store.get(&a, 90), None); - store.evict_through(&a, 120); + store.evict_before(&a, 180); assert_eq!(store.get(&a, 120), None); assert_eq!(store.get(&a, 180).map(String::as_str), Some("a180")); assert_eq!(store.get(&b, 60).map(String::as_str), Some("b60")); diff --git a/scripts/ci/README.md b/scripts/ci/README.md index fc215ecc..ffc00818 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -2,7 +2,7 @@ The backend keeps reproducible Planner revisions in `Cargo.toml` and `Cargo.lock`. `Sync ASAPPlanner main` checks upstream main daily (or on manual dispatch), updates -all five Planner dependencies together, and opens or refreshes +both Planner dependencies (`asap-planner`, `asap-executor`) together, and opens or refreshes `automation/sync-asapplanner-main` against backend main. GitHub scheduled runs can be delayed. The existing `ASAP_CI_REPO_TOKEN` secret must have read access to private dependency diff --git a/scripts/ci/test_sync_planner.py b/scripts/ci/test_sync_planner.py index 7c30eff5..c9af27d5 100644 --- a/scripts/ci/test_sync_planner.py +++ b/scripts/ci/test_sync_planner.py @@ -33,8 +33,7 @@ class PlannerSyncTests(unittest.TestCase): revision = "a" * 40 manifest = ''.join( f'{name} = {{ git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "{"b" * 40}" }}\n' - for name in ['planner-types', 'asap-frontend-promql', 'asap-logical-asap-optimization', - 'asap-plan-selection', 'asap-executor'] + for name in ['asap-planner', 'asap-executor'] ) def run_sync(self, manifest, revision=None, cargo_status=0): @@ -59,12 +58,10 @@ def test_updates_all_dependencies_and_lock_resolution(self): root, result = self.run_sync(self.manifest + other) self.assertEqual(result.returncode, 0, result.stderr) updated = (root / 'Cargo.toml').read_text() - self.assertEqual(updated.count(f'rev = "{self.revision}"'), 5) + self.assertEqual(updated.count(f'rev = "{self.revision}"'), 2) self.assertTrue(updated.endswith(other)) self.assertEqual((root / 'cargo-args').read_text().splitlines(), [ - 'update', '-p', 'asap-types@0.1.0', '-p', 'asap-frontend-promql', - '-p', 'asap-logical-asap-optimization', '-p', 'asap-plan-selection', - '-p', 'asap-executor', '--precise', self.revision]) + 'update', '-p', 'asap-planner', '-p', 'asap-executor', '--precise', self.revision]) def test_repository_manifest_updates_only_locked_packages(self): """Every pin in the real Cargo.toml is rewritten and every `-p` names a locked package.""" diff --git a/scripts/sync-asapplanner-main.sh b/scripts/sync-asapplanner-main.sh index 094ec6cf..c4895e62 100755 --- a/scripts/sync-asapplanner-main.sh +++ b/scripts/sync-asapplanner-main.sh @@ -20,16 +20,13 @@ pattern = re.compile( r'(git = "https://github.com/ProjectASAP/ASAPPlanner", rev = ")[0-9a-f]{40}("\s*})' ) updated, replacements = pattern.subn(rf"\g<1>{revision}\2", text) -if replacements != 5: - raise SystemExit(f"expected five ASAPPlanner dependencies, found {replacements}") +if replacements != 2: + raise SystemExit(f"expected two ASAPPlanner dependencies, found {replacements}") path.write_text(updated) PY cargo update \ - -p asap-types@0.1.0 \ - -p asap-frontend-promql \ - -p asap-logical-asap-optimization \ - -p asap-plan-selection \ + -p asap-planner \ -p asap-executor \ --precise "$planner_rev" From 929a81cefdc3526a601a0ec41c5f2a43bd83dad6 Mon Sep 17 00:00:00 2001 From: zzylol Date: Tue, 20 Oct 2026 19:52:23 +0000 Subject: [PATCH 2/4] fix(precompute): measure retention from the newest closed pane An evaluation stored panes can serve is at most at the end of the newest closed pane, not at the watermark: with the watermark at 74:45 and Example 4b's 60-minute retention, the pane ending at 20 min is still read by the evaluation at 70 min, but was skipped as expired. --- data_plane/src/precompute.rs | 7 +++++-- data_plane/tests/precompute.rs | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/data_plane/src/precompute.rs b/data_plane/src/precompute.rs index 7dda6f8f..54c94c88 100644 --- a/data_plane/src/precompute.rs +++ b/data_plane/src/precompute.rs @@ -277,8 +277,11 @@ impl Precompute { }; let pane_ms = builder.definition.pane_ms; let id = &builder.definition.id; - // Panes ending before this are no longer read: skip them. - let oldest_kept_end = builder.definition.oldest_kept_end_ms(watermark); + // The newest evaluation stored panes can serve is at the end of + // the newest closed pane; panes ending before the oldest one it + // reads are no longer read: skip them. + let newest_closed_end = watermark.div_euclid(pane_ms).saturating_mul(pane_ms); + let oldest_kept_end = builder.definition.oldest_kept_end_ms(newest_closed_end); let Some(mut end) = built.checked_add(pane_ms) else { continue; }; diff --git a/data_plane/tests/precompute.rs b/data_plane/tests/precompute.rs index 8dbde308..3eae0253 100644 --- a/data_plane/tests/precompute.rs +++ b/data_plane/tests/precompute.rs @@ -337,3 +337,18 @@ fn does_not_build_query_run_summaries() { assert_eq!(engine.stats.panes_built, 0); assert_eq!(engine.stats.samples_unrouted, 180); } + +/// Retention is measured from the newest evaluation the stored panes can +/// serve, the end of the newest closed pane, not from the watermark: with +/// the watermark at 74:45 the newest closed pane ends at 70 min, and an +/// evaluation at 70 min still reads the pane ending at 20 min. +#[test] +fn keeps_every_pane_the_newest_served_evaluation_reads() { + let (mut engine, sds) = precompute(common::plan("example4b", false)); + let id = sds.definitions.keys().next().unwrap().clone(); + engine.ingest(&samples("latency_ms", &["a"], MINUTE, 75 * MINUTE)); + for end in (2..=7).map(|i| i * PANE) { + assert!(engine.store.get(&id, end).is_some(), "pane ending at {end}"); + } + assert_eq!(engine.stats.panes_skipped_expired, 0); +} From fae43fef8089212d468a3f27433e6e2c4a768fbe Mon Sep 17 00:00:00 2001 From: zzylol Date: Tue, 20 Oct 2026 20:03:20 +0000 Subject: [PATCH 3/4] refactor(sds): key summary instances by definition and selection SummaryStore keys an instance by SummaryInstanceKey { definition, selection }, with a time-only Selection ((end - width, end]) for now. This is ASAPPlanner's summary coverage (definition plus selection) restricted to time selections; when StoredOutput carries the coverage (ASAPPlanner #781), the SDS takes the plan's definition id and selection instead of its own hash and the coverage end. --- data_plane/src/precompute.rs | 12 ++- data_plane/src/sds.rs | 79 ++++++++++++------ data_plane/tests/precompute.rs | 143 ++++++++++++++++++++++++++++----- data_plane/tests/sds.rs | 33 +++++--- 4 files changed, 210 insertions(+), 57 deletions(-) diff --git a/data_plane/src/precompute.rs b/data_plane/src/precompute.rs index 54c94c88..e40a8c23 100644 --- a/data_plane/src/precompute.rs +++ b/data_plane/src/precompute.rs @@ -29,7 +29,9 @@ use asap_executor::{RawSampleSelector, Sample}; use futures::executor::block_on; use futures::StreamExt; -use crate::sds::{DefinitionId, Sds, SummaryDefinition, SummaryStore}; +use crate::sds::{ + DefinitionId, Sds, Selection, SummaryDefinition, SummaryInstanceKey, SummaryStore, +}; /// One raw sample: its series' full label set (including `__name__`), time /// and value. @@ -300,7 +302,10 @@ impl Precompute { let samples = builder.open.remove(&end).unwrap_or_default(); match build_pane(builder, end, &samples) { Ok(state) => { - self.store.insert(id, end, state); + self.store.insert( + SummaryInstanceKey::new(id.clone(), Selection::ending_at(end)), + state, + ); self.stats.panes_built += 1; } Err(error) => { @@ -314,7 +319,8 @@ impl Precompute { None => break, } } - self.store.evict_before(id, oldest_kept_end); + self.store + .evict_before(id, Selection::ending_at(oldest_kept_end)); } } } diff --git a/data_plane/src/sds.rs b/data_plane/src/sds.rs index 69f5d6e4..c71f2e91 100644 --- a/data_plane/src/sds.rs +++ b/data_plane/src/sds.rs @@ -32,7 +32,10 @@ use asap_planner::{InstallablePlan, StoredOutputBuild}; use sha2::{Digest, Sha256}; /// Identifies what a stored summary computes, independent of node ids and of -/// which pane an output reads. Equal for equal computations across plans. +/// which data an output reads: the hash of its coverage definition. Equal for +/// equal computations across plans. When the Planner states each stored +/// output's coverage (ASAPPlanner #781), this becomes the plan's definition +/// id instead of the SDS's own hash. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct DefinitionId(pub String); @@ -268,11 +271,43 @@ fn hex(bytes: &[u8]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() } -/// Built panes, by definition and pane end. A pane is stored once, when it -/// is complete, and never changes. +/// Which data one summary instance covers. For now only time: the interval +/// `(end_ms - width, end_ms]`, the width being its definition's coverage +/// width. When the Planner states each stored output's coverage (definition +/// and selection, ASAPPlanner #781), this becomes the plan's selection. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Selection { + pub end_ms: i64, +} + +impl Selection { + /// The coverage ending at `end_ms`. + pub fn ending_at(end_ms: i64) -> Self { + Self { end_ms } + } +} + +/// One stored summary: what it computes and which data it covers. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SummaryInstanceKey { + pub definition: DefinitionId, + pub selection: Selection, +} + +impl SummaryInstanceKey { + pub fn new(definition: DefinitionId, selection: Selection) -> Self { + Self { + definition, + selection, + } + } +} + +/// Stored summary instances, by definition and selection. An instance is +/// stored once, when its coverage is complete, and never changes. #[derive(Debug)] pub struct SummaryStore { - panes: BTreeMap>, + instances: BTreeMap>, } impl Default for SummaryStore { @@ -284,38 +319,36 @@ impl Default for SummaryStore { impl SummaryStore { pub fn new() -> Self { Self { - panes: BTreeMap::new(), + instances: BTreeMap::new(), } } - pub fn insert(&mut self, definition: &DefinitionId, end_ms: i64, state: S) { - match self.panes.get_mut(definition) { - Some(panes) => { - panes.insert(end_ms, state); - } - None => { - self.panes - .insert(definition.clone(), BTreeMap::from([(end_ms, state)])); - } - } + pub fn insert(&mut self, key: SummaryInstanceKey, state: S) { + self.instances + .entry(key.definition) + .or_default() + .insert(key.selection, state); } - pub fn get(&self, definition: &DefinitionId, end_ms: i64) -> Option<&S> { - self.panes.get(definition)?.get(&end_ms) + /// The instance of `definition` covering `selection`, if stored. Borrows + /// the definition id, so a lookup allocates nothing. + pub fn get(&self, definition: &DefinitionId, selection: Selection) -> Option<&S> { + self.instances.get(definition)?.get(&selection) } - /// Drops the panes of `definition` that end before `end_ms`. - pub fn evict_before(&mut self, definition: &DefinitionId, end_ms: i64) { - if let Some(panes) = self.panes.get_mut(definition) { - *panes = panes.split_off(&end_ms); + /// Drops the instances of `definition` whose coverage ends before + /// `oldest`'s. + pub fn evict_before(&mut self, definition: &DefinitionId, oldest: Selection) { + if let Some(instances) = self.instances.get_mut(definition) { + *instances = instances.split_off(&oldest); } } pub fn len(&self) -> usize { - self.panes.values().map(BTreeMap::len).sum() + self.instances.values().map(BTreeMap::len).sum() } pub fn is_empty(&self) -> bool { - self.panes.values().all(BTreeMap::is_empty) + self.instances.values().all(BTreeMap::is_empty) } } diff --git a/data_plane/tests/precompute.rs b/data_plane/tests/precompute.rs index 3eae0253..93dfec0e 100644 --- a/data_plane/tests/precompute.rs +++ b/data_plane/tests/precompute.rs @@ -7,7 +7,7 @@ use asap_executor::planner::workload::{PlanningWorkload, Query}; use asap_planner::InstallablePlan; use control_plane::plan_workload; use data_plane::precompute::{pane_end, Precompute, PrecomputeOptions, RawSample}; -use data_plane::sds::Sds; +use data_plane::sds::{Sds, Selection}; const MINUTE: i64 = 60_000; const PANE: i64 = 10 * MINUTE; @@ -57,13 +57,26 @@ fn builds_panes_after_the_first_as_the_watermark_passes() { let id = sds.definitions.keys().next().unwrap().clone(); // The first sample, at 1 min, opens pane (0, 10 min], which is skipped. engine.ingest(&samples("latency_ms", &["a", "b"], MINUTE, 35 * MINUTE)); - assert_eq!(engine.store.get(&id, PANE).map(|_| ()), None, "first pane"); + assert_eq!( + engine + .store + .get(&id, Selection::ending_at(PANE)) + .map(|_| ()), + None, + "first pane" + ); for end in [2 * PANE, 3 * PANE] { - let pane = engine.store.get(&id, end).expect("built pane"); + let pane = engine + .store + .get(&id, Selection::ending_at(end)) + .expect("built pane"); assert_eq!(pane.num_rows(), 2, "one state per series"); } assert_eq!( - engine.store.get(&id, 4 * PANE).map(|_| ()), + engine + .store + .get(&id, Selection::ending_at(4 * PANE)) + .map(|_| ()), None, "open pane" ); @@ -104,9 +117,30 @@ fn stores_empty_panes() { let id = sds.definitions.keys().next().unwrap().clone(); engine.ingest(&samples("latency_ms", &["a"], 0, 20 * MINUTE)); engine.ingest(&[sample("latency_ms", "a", 45 * MINUTE, 1.0)]); - assert_eq!(engine.store.get(&id, 2 * PANE).unwrap().num_rows(), 1); - assert_eq!(engine.store.get(&id, 3 * PANE).unwrap().num_rows(), 0); - assert_eq!(engine.store.get(&id, 4 * PANE).unwrap().num_rows(), 0); + assert_eq!( + engine + .store + .get(&id, Selection::ending_at(2 * PANE)) + .unwrap() + .num_rows(), + 1 + ); + assert_eq!( + engine + .store + .get(&id, Selection::ending_at(3 * PANE)) + .unwrap() + .num_rows(), + 0 + ); + assert_eq!( + engine + .store + .get(&id, Selection::ending_at(4 * PANE)) + .unwrap() + .num_rows(), + 0 + ); } /// The plan's label matchers select which series a pane summarizes. @@ -121,7 +155,14 @@ fn applies_label_matchers() { let (mut engine, sds) = precompute(plan_workload(&workload, &common::inputs(false)).unwrap()); let id = sds.definitions.keys().next().unwrap().clone(); engine.ingest(&samples("latency_ms", &["a", "b"], 0, 25 * MINUTE)); - assert_eq!(engine.store.get(&id, 2 * PANE).unwrap().num_rows(), 1); + assert_eq!( + engine + .store + .get(&id, Selection::ending_at(2 * PANE)) + .unwrap() + .num_rows(), + 1 + ); } /// Panes are `(end - width, end]`, like PromQL ranges: a sample at a @@ -152,9 +193,23 @@ fn engine_assigns_boundary_samples_to_the_pane_they_end() { sample("latency_ms", "a", 45 * MINUTE, 1.0), ]); // (10 min, 20 min]: series a and b, the one at its end included. - assert_eq!(engine.store.get(&id, 2 * PANE).unwrap().num_rows(), 2); + assert_eq!( + engine + .store + .get(&id, Selection::ending_at(2 * PANE)) + .unwrap() + .num_rows(), + 2 + ); // (20 min, 30 min]: only c, one millisecond after the boundary. - assert_eq!(engine.store.get(&id, 3 * PANE).unwrap().num_rows(), 1); + assert_eq!( + engine + .store + .get(&id, Selection::ending_at(3 * PANE)) + .unwrap() + .num_rows(), + 1 + ); } /// Samples of a metric no definition reads do not move the watermark, even @@ -205,9 +260,18 @@ fn watermark_jump_builds_only_retained_panes() { engine.stats.panes_skipped_expired, (30 * DAY - 80 * MINUTE) as u64 / PANE as u64 ); - assert!(engine.store.get(&id, 30 * DAY).is_some()); - assert!(engine.store.get(&id, 30 * DAY - 6 * PANE).is_none()); - assert!(engine.store.get(&id, 2 * PANE).is_none()); + assert!(engine + .store + .get(&id, Selection::ending_at(30 * DAY)) + .is_some()); + assert!(engine + .store + .get(&id, Selection::ending_at(30 * DAY - 6 * PANE)) + .is_none()); + assert!(engine + .store + .get(&id, Selection::ending_at(2 * PANE)) + .is_none()); assert_eq!(engine.store.len(), 6); } @@ -249,14 +313,29 @@ fn evicts_panes_past_retention() { let (mut engine, sds) = precompute(common::plan("example4b", false)); let id = sds.definitions.keys().next().unwrap().clone(); engine.ingest(&samples("latency_ms", &["a"], 0, 25 * MINUTE)); - assert!(engine.store.get(&id, 2 * PANE).is_some()); + assert!(engine + .store + .get(&id, Selection::ending_at(2 * PANE)) + .is_some()); // Watermark 100 min, retention 60 min: the oldest pane read is // (40, 50 min], so panes ending before 50 min are evicted. engine.ingest(&[sample("latency_ms", "a", 100 * MINUTE, 1.0)]); - assert!(engine.store.get(&id, 2 * PANE).is_none()); - assert!(engine.store.get(&id, 4 * PANE).is_none()); - assert!(engine.store.get(&id, 5 * PANE).is_some()); - assert!(engine.store.get(&id, 10 * PANE).is_some()); + assert!(engine + .store + .get(&id, Selection::ending_at(2 * PANE)) + .is_none()); + assert!(engine + .store + .get(&id, Selection::ending_at(4 * PANE)) + .is_none()); + assert!(engine + .store + .get(&id, Selection::ending_at(5 * PANE)) + .is_some()); + assert!(engine + .store + .get(&id, Selection::ending_at(10 * PANE)) + .is_some()); assert_eq!(engine.store.len(), 6); } @@ -271,7 +350,14 @@ fn accepts_out_of_order_samples_within_lateness() { engine.ingest(&[sample("latency_ms", "b", 15 * MINUTE, 1.0)]); engine.ingest(&[sample("latency_ms", "a", 26 * MINUTE, 1.0)]); // Watermark 21 min: (10, 20 min] holds b's late sample. - assert_eq!(engine.store.get(&id, 2 * PANE).unwrap().num_rows(), 1); + assert_eq!( + engine + .store + .get(&id, Selection::ending_at(2 * PANE)) + .unwrap() + .num_rows(), + 1 + ); assert_eq!(engine.stats.samples_dropped_late, 0); engine.ingest(&[sample("latency_ms", "c", 18 * MINUTE, 1.0)]); assert_eq!(engine.stats.samples_dropped_late, 1); @@ -291,7 +377,14 @@ fn stale_markers_are_not_samples() { sample("latency_ms", "b", 14 * MINUTE, stale), sample("latency_ms", "a", 25 * MINUTE, 1.0), ]); - assert_eq!(engine.store.get(&id, 2 * PANE).unwrap().num_rows(), 1); + assert_eq!( + engine + .store + .get(&id, Selection::ending_at(2 * PANE)) + .unwrap() + .num_rows(), + 1 + ); } /// One series stamped ahead of the wall clock, though within @@ -307,7 +400,10 @@ fn series_ahead_of_the_clock_does_not_close_panes_early() { let now = base + 9 * MINUTE; engine.ingest_at(&[sample("latency_ms", "b", base + 13 * MINUTE, 1.0)], now); assert!( - engine.store.get(&id, base + PANE).is_none(), + engine + .store + .get(&id, Selection::ending_at(base + PANE)) + .is_none(), "(10, 20 min] still open" ); engine.ingest_at(&[sample("latency_ms", "a", base + 9 * MINUTE, 1.0)], now); @@ -348,7 +444,10 @@ fn keeps_every_pane_the_newest_served_evaluation_reads() { let id = sds.definitions.keys().next().unwrap().clone(); engine.ingest(&samples("latency_ms", &["a"], MINUTE, 75 * MINUTE)); for end in (2..=7).map(|i| i * PANE) { - assert!(engine.store.get(&id, end).is_some(), "pane ending at {end}"); + assert!( + engine.store.get(&id, Selection::ending_at(end)).is_some(), + "pane ending at {end}" + ); } assert_eq!(engine.stats.panes_skipped_expired, 0); } diff --git a/data_plane/tests/sds.rs b/data_plane/tests/sds.rs index ed5cbbdb..90982c08 100644 --- a/data_plane/tests/sds.rs +++ b/data_plane/tests/sds.rs @@ -3,7 +3,7 @@ mod common; use asap_executor::planner::ir::{NonASAPOp, Operator}; use asap_planner::{InstallablePlan, StoredOutputBuild}; use common::plan; -use data_plane::sds::{DefinitionId, Sds, SdsError, SummaryStore}; +use data_plane::sds::{DefinitionId, Sds, SdsError, Selection, SummaryInstanceKey, SummaryStore}; /// Example 4b's six ingestion-time KLL panes are one definition with a /// 10-minute pane; each output reads the pane `i * 10min` before evaluation, @@ -166,14 +166,29 @@ fn store_finds_and_evicts_panes() { let (a, b) = (DefinitionId("a".into()), DefinitionId("b".into())); let mut store = SummaryStore::new(); for end in [60, 120, 180] { - store.insert(&a, end, format!("a{end}")); + store.insert( + SummaryInstanceKey::new(a.clone(), Selection::ending_at(end)), + format!("a{end}"), + ); } - store.insert(&b, 60, "b60".to_string()); - assert_eq!(store.get(&a, 120).map(String::as_str), Some("a120")); - assert_eq!(store.get(&a, 90), None); - store.evict_before(&a, 180); - assert_eq!(store.get(&a, 120), None); - assert_eq!(store.get(&a, 180).map(String::as_str), Some("a180")); - assert_eq!(store.get(&b, 60).map(String::as_str), Some("b60")); + store.insert( + SummaryInstanceKey::new(b.clone(), Selection::ending_at(60)), + "b60".to_string(), + ); + assert_eq!( + store.get(&a, Selection::ending_at(120)).map(String::as_str), + Some("a120") + ); + assert_eq!(store.get(&a, Selection::ending_at(90)), None); + store.evict_before(&a, Selection::ending_at(180)); + assert_eq!(store.get(&a, Selection::ending_at(120)), None); + assert_eq!( + store.get(&a, Selection::ending_at(180)).map(String::as_str), + Some("a180") + ); + assert_eq!( + store.get(&b, Selection::ending_at(60)).map(String::as_str), + Some("b60") + ); assert_eq!(store.len(), 2); } From b401e6aed0a49e96c11b9bd9c0124e17dd9e1614 Mon Sep 17 00:00:00 2001 From: zzylol Date: Tue, 20 Oct 2026 23:23:48 +0000 Subject: [PATCH 4/4] fix(sds): keep one pane beyond the retention; order and check stored inputs - Panes are kept for one extra pane width beyond the plan's retention, as the design doc describes: with the watermark at 84:45 (newest closed pane 80 min), the evaluation at 70 min lost its pane ending at 20 min. The expired-pane skip uses the same bound. - Sds::ingestion_build_order refuses an ingestion-run definition that reads a query-run one, which is built only when an evaluation needs it. - PaneBinding.raw_input is removed: nothing here reads it. --- README.md | 3 +- data_plane/src/precompute.rs | 10 ++++-- data_plane/src/sds.rs | 25 ++++++++++---- data_plane/tests/precompute.rs | 44 ++++++++++++++++++------- data_plane/tests/sds.rs | 59 ++++++++++++++++++++++++++++++++++ 5 files changed, 120 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index f1342364..4794bf52 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ The backend is being rebuilt on the Planner's physical DAG: The data plane stores the plan's `stored_outputs`. Outputs that differ only in which pane they read share one stored summary, kept for the largest `retention_ms` the plan states among them (which covers range-evaluation -spans); panes ending before the oldest pane still read are evicted. +spans), plus one pane width so an evaluation one pane behind the newest +closed pane is still served; older panes are evicted. Panes close `--allowed-lateness-ms` (default 15 s) after the newest sample of a stored summary's metric, capped at the wall clock, passes their end; later samples for them are diff --git a/data_plane/src/precompute.rs b/data_plane/src/precompute.rs index e40a8c23..9658a4fe 100644 --- a/data_plane/src/precompute.rs +++ b/data_plane/src/precompute.rs @@ -280,10 +280,14 @@ impl Precompute { let pane_ms = builder.definition.pane_ms; let id = &builder.definition.id; // The newest evaluation stored panes can serve is at the end of - // the newest closed pane; panes ending before the oldest one it - // reads are no longer read: skip them. + // the newest closed pane. Panes are kept for one extra pane + // width beyond the plan's retention, so an evaluation one pane + // behind it is still served; older panes are no longer read: + // skip them. let newest_closed_end = watermark.div_euclid(pane_ms).saturating_mul(pane_ms); - let oldest_kept_end = builder.definition.oldest_kept_end_ms(newest_closed_end); + let oldest_kept_end = builder + .definition + .oldest_kept_end_ms(newest_closed_end.saturating_sub(pane_ms)); let Some(mut end) = built.checked_add(pane_ms) else { continue; }; diff --git a/data_plane/src/sds.rs b/data_plane/src/sds.rs index c71f2e91..b5354310 100644 --- a/data_plane/src/sds.rs +++ b/data_plane/src/sds.rs @@ -85,9 +85,6 @@ impl SummaryDefinition { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct PaneBinding { pub offset_ms: i64, - /// The output's own `TimeRange` node: where that pane's raw samples - /// feed when the output is computed at query time. - pub raw_input: u64, } /// The stored summaries of one plan. @@ -108,6 +105,13 @@ pub enum SdsError { /// Stored outputs read each other in a cycle. #[error("stored outputs read each other in a cycle through {0:?}")] Cycle(DefinitionId), + /// An ingestion-run definition reads a query-run one, which is built + /// only when an evaluation needs it, so never in time for ingestion. + #[error("ingestion-run summary {definition:?} reads query-run summary {input:?}")] + IngestionReadsQueryRun { + definition: DefinitionId, + input: DefinitionId, + }, } impl Sds { @@ -149,7 +153,6 @@ impl Sds { definition.retention_ms = definition.retention_ms.max(retention_ms); let binding = PaneBinding { offset_ms: output.pane.end_offset_ms, - raw_input: output.pane.time_range_node_id as u64, }; outputs.insert(node, (id, binding)); } @@ -190,7 +193,8 @@ impl Sds { } /// The definitions built in ingestion runs, each after every definition - /// it reads (`stored_inputs`). Fails on a cycle. + /// it reads (`stored_inputs`). Fails on a cycle, and when an + /// ingestion-run definition reads a query-run one. pub fn ingestion_build_order(&self) -> Result, SdsError> { let mut order = Vec::new(); let mut state: HashMap<&DefinitionId, bool> = HashMap::new(); // false: visiting @@ -206,7 +210,16 @@ impl Sds { None => {} } state.insert(id, false); - for input in &sds.definitions[id].stored_inputs { + let definition = &sds.definitions[id]; + for input in &definition.stored_inputs { + if definition.built_at == StoredOutputBuild::IngestionRun + && sds.definitions[input].built_at == StoredOutputBuild::QueryRun + { + return Err(SdsError::IngestionReadsQueryRun { + definition: id.clone(), + input: input.clone(), + }); + } visit(sds, input, state, order)?; } state.insert(id, true); diff --git a/data_plane/tests/precompute.rs b/data_plane/tests/precompute.rs index 93dfec0e..34c1622d 100644 --- a/data_plane/tests/precompute.rs +++ b/data_plane/tests/precompute.rs @@ -253,12 +253,12 @@ fn watermark_jump_builds_only_retained_panes() { engine.ingest_at(&samples("latency_ms", &["a"], 0, 25 * MINUTE), 30 * DAY); let built = engine.stats.panes_built; engine.ingest_at(&[sample("latency_ms", "a", 30 * DAY, 1.0)], 30 * DAY); - // The plan keeps Example 4b's panes 60 min back: the six panes ending - // at 30 days - 50 min through 30 days. - assert_eq!(engine.stats.panes_built - built, 6); + // The plan keeps Example 4b's panes 60 min back, plus one pane: the + // seven panes ending at 30 days - 60 min through 30 days. + assert_eq!(engine.stats.panes_built - built, 7); assert_eq!( engine.stats.panes_skipped_expired, - (30 * DAY - 80 * MINUTE) as u64 / PANE as u64 + (30 * DAY - 90 * MINUTE) as u64 / PANE as u64 ); assert!(engine .store @@ -266,13 +266,13 @@ fn watermark_jump_builds_only_retained_panes() { .is_some()); assert!(engine .store - .get(&id, Selection::ending_at(30 * DAY - 6 * PANE)) + .get(&id, Selection::ending_at(30 * DAY - 7 * PANE)) .is_none()); assert!(engine .store .get(&id, Selection::ending_at(2 * PANE)) .is_none()); - assert_eq!(engine.store.len(), 6); + assert_eq!(engine.store.len(), 7); } /// Timestamps at the ends of `i64` neither overflow nor loop. @@ -317,8 +317,9 @@ fn evicts_panes_past_retention() { .store .get(&id, Selection::ending_at(2 * PANE)) .is_some()); - // Watermark 100 min, retention 60 min: the oldest pane read is - // (40, 50 min], so panes ending before 50 min are evicted. + // Watermark 100 min, retention 60 min plus one pane: the oldest pane + // read by the evaluation at 90 min is (30, 40 min], so panes ending + // before 40 min are evicted. engine.ingest(&[sample("latency_ms", "a", 100 * MINUTE, 1.0)]); assert!(engine .store @@ -326,17 +327,17 @@ fn evicts_panes_past_retention() { .is_none()); assert!(engine .store - .get(&id, Selection::ending_at(4 * PANE)) + .get(&id, Selection::ending_at(3 * PANE)) .is_none()); assert!(engine .store - .get(&id, Selection::ending_at(5 * PANE)) + .get(&id, Selection::ending_at(4 * PANE)) .is_some()); assert!(engine .store .get(&id, Selection::ending_at(10 * PANE)) .is_some()); - assert_eq!(engine.store.len(), 6); + assert_eq!(engine.store.len(), 7); } /// Out-of-order samples within the allowed lateness reach their pane. @@ -451,3 +452,24 @@ fn keeps_every_pane_the_newest_served_evaluation_reads() { } assert_eq!(engine.stats.panes_skipped_expired, 0); } + +/// An evaluation one pane behind the newest closed pane still has every +/// pane it reads: with the watermark at 84:45 the newest closed pane ends +/// at 80 min, and the evaluation at 70 min reads the panes ending 20..70. +/// The extra pane is the only margin: the store stays bounded. +#[test] +fn keeps_one_pane_of_margin_beyond_the_retention() { + let (mut engine, sds) = precompute(common::plan("example4b", false)); + let id = sds.definitions.keys().next().unwrap().clone(); + engine.ingest(&samples("latency_ms", &["a"], MINUTE, 85 * MINUTE)); + for end in (2..=8).map(|i| i * PANE) { + assert!( + engine.store.get(&id, Selection::ending_at(end)).is_some(), + "pane ending at {end}" + ); + } + engine.ingest(&samples("latency_ms", &["a"], 85 * MINUTE, 300 * MINUTE)); + // Panes ending 230..290 min: the six the evaluation at 290 reads, plus + // the one more the evaluation at 280 reads. + assert_eq!(engine.store.len(), 7); +} diff --git a/data_plane/tests/sds.rs b/data_plane/tests/sds.rs index 90982c08..1e27b416 100644 --- a/data_plane/tests/sds.rs +++ b/data_plane/tests/sds.rs @@ -192,3 +192,62 @@ fn store_finds_and_evicts_panes() { ); assert_eq!(store.len(), 2); } + +/// `definition` under another identity, reading `stored_inputs`. +fn renamed( + definition: &data_plane::sds::SummaryDefinition, + name: &str, + stored_inputs: &[&str], +) -> data_plane::sds::SummaryDefinition { + let mut definition = definition.clone(); + definition.id = DefinitionId(name.into()); + definition.stored_inputs = stored_inputs + .iter() + .map(|input| DefinitionId(input.to_string())) + .collect(); + definition +} + +/// Several definitions are ordered after the ones they read, whatever +/// their identities' order: a rollup `a` over `c` and `b`, `b` over `c`. +#[test] +fn build_order_puts_inputs_first() { + let base = Sds::from_plan(&plan("example4b", false)).unwrap(); + let template = base.definitions.values().next().unwrap(); + let mut sds = Sds::default(); + for definition in [ + renamed(template, "a", &["c", "b"]), + renamed(template, "b", &["c"]), + renamed(template, "c", &[]), + ] { + sds.definitions.insert(definition.id.clone(), definition); + } + let order: Vec = sds + .ingestion_build_order() + .unwrap() + .into_iter() + .map(|id| id.0) + .collect(); + assert_eq!(order, ["c", "b", "a"]); +} + +/// An ingestion-run definition cannot read a query-run one: that summary +/// is only built when an evaluation needs it. +#[test] +fn build_order_refuses_ingestion_reading_query_run() { + let base = Sds::from_plan(&plan("example4b", false)).unwrap(); + let template = base.definitions.values().next().unwrap(); + let mut query_run = renamed(template, "q", &[]); + query_run.built_at = StoredOutputBuild::QueryRun; + let mut sds = Sds::default(); + for definition in [renamed(template, "i", &["q"]), query_run] { + sds.definitions.insert(definition.id.clone(), definition); + } + assert_eq!( + sds.ingestion_build_order(), + Err(SdsError::IngestionReadsQueryRun { + definition: DefinitionId("i".into()), + input: DefinitionId("q".into()), + }) + ); +}