From b2501121717c9547b70d9a399db50f3f907fcb78 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 8 Oct 2026 12:19:56 +0200 Subject: [PATCH 1/8] chore(codspeed): drop unused libc dependency The codspeed crate declared libc since its first commit but never used it. --- Cargo.lock | 1 - crates/codspeed/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fb14ef4a..66b6dbc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -590,7 +590,6 @@ dependencies = [ "colored", "getrandom 0.4.3", "glob", - "libc", "nix", "serde", "serde_json", diff --git a/crates/codspeed/Cargo.toml b/crates/codspeed/Cargo.toml index 6a54ce9d..1a3a1f5c 100644 --- a/crates/codspeed/Cargo.toml +++ b/crates/codspeed/Cargo.toml @@ -21,7 +21,6 @@ keywords = ["codspeed", "benchmark"] anyhow = { workspace = true } colored = "3.0.0" glob = "0.3.2" -libc = "^0.2" nix = { version = "0.31.1", features = ["time"] } getrandom = "0.4" serde = { workspace = true } From 06d80d9be885dc2a22037cdf040c0f15043a06d3 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 8 Oct 2026 12:20:12 +0200 Subject: [PATCH 2/8] feat(codspeed): add per-benchmark process isolation Benchmarks of one bench binary share a process, so the allocator state one benchmark leaves behind (raised mmap threshold, fragmented bins, retained caches) changes the measurement of every benchmark after it. Adding or changing one benchmark can then shift unrelated results. Add `isolation::Isolation` for the harnesses. It is on by default and turned off with `CODSPEED_PROCESS_ISOLATION=false`. `Isolation::run` re-executes the bench binary with the same arguments and `CODSPEED_ISOLATED_BENCHMARK` set to one benchmark URI, so each benchmark gets a fresh process with its own heap and threads. The child only runs that benchmark (`only_benchmark`, which overrides the CLI filters), and a failing child's exit code is propagated (128 + N for signal N). Re-exec instead of fork(): fork only copies the calling thread, so threads the bench binary started before the harness (e.g. a Tokio runtime in `main`) would be missing in the child and work sent to them would hang. --- crates/codspeed/src/isolation.rs | 101 +++++++++++++++++++++++++++++++ crates/codspeed/src/lib.rs | 1 + 2 files changed, 102 insertions(+) create mode 100644 crates/codspeed/src/isolation.rs diff --git a/crates/codspeed/src/isolation.rs b/crates/codspeed/src/isolation.rs new file mode 100644 index 00000000..d20f76d3 --- /dev/null +++ b/crates/codspeed/src/isolation.rs @@ -0,0 +1,101 @@ +//! Process isolation: every benchmark runs in a fresh copy of the benchmark +//! executable, so the allocator state and threads that other benchmarks or +//! setup code leave behind can't change its measurement. + +use std::{ + io::Write, + process::{Command, ExitStatus}, + sync::LazyLock, +}; + +static CURRENT: LazyLock = LazyLock::new(Isolation::from_env); + +/// Isolation is on unless this is `0` or `false`. +pub const PROCESS_ISOLATION_ENV: &str = "CODSPEED_PROCESS_ISOLATION"; +/// Set on each child process: the URI of the only benchmark it runs. +const ISOLATED_BENCHMARK_ENV: &str = "CODSPEED_ISOLATED_BENCHMARK"; + +/// Exit code for a child that ended with neither an exit code nor a signal. +const GENERIC_FAILURE_EXIT_CODE: i32 = 1; +/// Shells report a child killed by signal N as exit code 128 + N. +#[cfg(unix)] +const SIGNAL_EXIT_CODE_OFFSET: i32 = 128; + +/// How a harness runs each benchmark. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Isolation { + /// All benchmarks share this process and its allocator state. + SharedProcess, + /// Every benchmark runs in its own child process. + ProcessPerBenchmark, + /// A child spawned by [`Isolation::ProcessPerBenchmark`], restricted to this URI. + IsolatedBenchmark(String), +} + +impl Isolation { + /// The isolation of this process, read once from the environment. + pub fn current() -> &'static Self { + &CURRENT + } + + fn from_env() -> Self { + if let Ok(uri) = std::env::var(ISOLATED_BENCHMARK_ENV) { + return Self::IsolatedBenchmark(uri); + } + + match std::env::var(PROCESS_ISOLATION_ENV).as_deref() { + Err(_) | Ok("1" | "true") => Self::ProcessPerBenchmark, + Ok("0" | "false") => Self::SharedProcess, + Ok(other) => { + panic!("{PROCESS_ISOLATION_ENV} must be `true` or `false`, got `{other}`") + } + } + } + + /// The only benchmark this process may run. It overrides CLI filters. + pub fn only_benchmark(&self) -> Option<&str> { + match self { + Self::IsolatedBenchmark(uri) => Some(uri), + Self::SharedProcess | Self::ProcessPerBenchmark => None, + } + } + + /// Runs the benchmark `uri`. `bench` must contain the whole measurement, + /// from `start_benchmark` to `end_benchmark`. + pub fn run(&self, uri: String, bench: impl FnOnce(String)) { + match self { + Self::SharedProcess | Self::IsolatedBenchmark(_) => bench(uri), + Self::ProcessPerBenchmark => run_in_child_process(&uri), + } + } +} + +/// Runs this executable again with the same arguments, limited to `uri`, and +/// exits with the child's code if it fails. +fn run_in_child_process(uri: &str) { + // Keep the parent's output before the child's. + if let Err(error) = std::io::stdout().flush() { + eprintln!("failed to flush stdout before running {uri}: {error}"); + } + + let exe = std::env::current_exe().expect("failed to locate the benchmark executable"); + let status = Command::new(exe) + .args(std::env::args_os().skip(1)) + .env(ISOLATED_BENCHMARK_ENV, uri) + .status() + .unwrap_or_else(|error| panic!("failed to start the process for {uri}: {error}")); + + if !status.success() { + std::process::exit(exit_code(status)); + } +} + +/// Exit code of a failed child, in the shell convention. +fn exit_code(status: ExitStatus) -> i32 { + #[cfg(unix)] + if let Some(signal) = std::os::unix::process::ExitStatusExt::signal(&status) { + return SIGNAL_EXIT_CODE_OFFSET + signal; + } + + status.code().unwrap_or(GENERIC_FAILURE_EXIT_CODE) +} diff --git a/crates/codspeed/src/lib.rs b/crates/codspeed/src/lib.rs index 929c3662..a4bc1c12 100644 --- a/crates/codspeed/src/lib.rs +++ b/crates/codspeed/src/lib.rs @@ -1,6 +1,7 @@ pub mod codspeed; pub mod instrument_hooks; +pub mod isolation; mod macros; mod measurement; From 43916f8d45a6a4f8409fad2e6841f053547a5416 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 8 Oct 2026 12:20:13 +0200 Subject: [PATCH 3/8] feat(divan): run each benchmark in its own process Route every benchmark of the analysis harness through `Isolation::run`. A child process only matches its own benchmark URI and doesn't repeat the "Skipped" lines the parent already printed. --- crates/divan_compat/src/compat/mod.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/crates/divan_compat/src/compat/mod.rs b/crates/divan_compat/src/compat/mod.rs index 5fa6d558..085136a1 100644 --- a/crates/divan_compat/src/compat/mod.rs +++ b/crates/divan_compat/src/compat/mod.rs @@ -155,13 +155,15 @@ pub mod counter { } } } -use codspeed::codspeed::CodSpeed; +use codspeed::{codspeed::CodSpeed, isolation::Isolation}; use config::Filter; use entry::AnyBenchEntry; use regex::Regex; use std::{cell::RefCell, rc::Rc}; pub fn main() { + let isolation = Isolation::current(); + // Outlined steps of original divan::main and their equivalent in codspeed instrumented mode // 1. Get registered entries let group_entries = &entry::GROUP_ENTRIES; @@ -208,6 +210,9 @@ pub fn main() { .map(|arg_filters| arg_filters.map(parse_filter).collect()); move |uri: &str| { + if let Some(only) = isolation.only_benchmark() { + return uri == only; + } if let Some(filters) = filters.as_ref() { filters.iter().any(|filter| filter.is_match(uri)) } else { @@ -224,8 +229,11 @@ pub fn main() { if let Some(options) = &meta.bench_options { if let Some(true) = options.ignore { - let uri = uri::generate(&entry, entry.display_name()); - println!("Skipped: {uri}"); + // The parent already reported it; child processes stay quiet. + if isolation.only_benchmark().is_none() { + let uri = uri::generate(&entry, entry.display_name()); + println!("Skipped: {uri}"); + } continue; } } @@ -237,7 +245,7 @@ pub fn main() { continue; } - bench_fn(bench::Bencher::new(&codspeed, uri)); + isolation.run(uri, |uri| bench_fn(bench::Bencher::new(&codspeed, uri))); } entry::BenchEntryRunner::Args(bench_runner) => { let bench_runner = bench_runner(); @@ -249,9 +257,9 @@ pub fn main() { continue; } - let bencher = bench::Bencher::new(&codspeed, uri); - - bench_runner.bench(bencher, arg_index); + isolation.run(uri, |uri| { + bench_runner.bench(bench::Bencher::new(&codspeed, uri), arg_index) + }); } } } From 5622bb38d115d2089dcd4f8ede2e0e3c07026dc8 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 8 Oct 2026 12:20:14 +0200 Subject: [PATCH 4/8] feat(criterion): run each benchmark in its own process All criterion compat benchmarks go through `BenchmarkGroup::run_bench`; wrap the measurement in `Isolation::run` like divan. A child process replaces the CLI filter with an exact match on its benchmark URI. --- crates/criterion_compat/src/compat/criterion.rs | 6 +++++- crates/criterion_compat/src/compat/group.rs | 9 ++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/criterion_compat/src/compat/criterion.rs b/crates/criterion_compat/src/compat/criterion.rs index 8031930f..d241b58a 100644 --- a/crates/criterion_compat/src/compat/criterion.rs +++ b/crates/criterion_compat/src/compat/criterion.rs @@ -1,6 +1,6 @@ use std::{cell::RefCell, marker::PhantomData, rc::Rc, time::Duration}; -use codspeed::codspeed::CodSpeed; +use codspeed::{codspeed::CodSpeed, isolation::Isolation}; use criterion::{ measurement::{Measurement, WallTime}, profiler::Profiler, @@ -41,6 +41,10 @@ impl Criterion { fn parse_filter() -> BenchmarkFilter { use clap::{Arg, Command}; + if let Some(uri) = Isolation::current().only_benchmark() { + return BenchmarkFilter::Exact(uri.to_owned()); + } + let matches = Command::new("Criterion Benchmark") .arg( Arg::new("FILTER") diff --git a/crates/criterion_compat/src/compat/group.rs b/crates/criterion_compat/src/compat/group.rs index 0ab50fb4..265720b9 100644 --- a/crates/criterion_compat/src/compat/group.rs +++ b/crates/criterion_compat/src/compat/group.rs @@ -3,6 +3,7 @@ use std::{cell::RefCell, rc::Rc, time::Duration}; use codspeed::{ codspeed::CodSpeed, + isolation::Isolation, utils::{build_uri, get_git_relative_path}, }; use criterion::measurement::WallTime; @@ -83,9 +84,11 @@ impl<'a, M: Measurement> BenchmarkGroup<'a, M> { return; } - let mut codspeed = self.codspeed.borrow_mut(); - let mut b = Bencher::new(&mut codspeed, uri); - f(&mut b, input); + Isolation::current().run(uri, |uri| { + let mut codspeed = self.codspeed.borrow_mut(); + let mut b = Bencher::new(&mut codspeed, uri); + f(&mut b, input); + }); } } From 9ee5ae089be21ab9e50e9ceb54decf70c0473b9f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 8 Oct 2026 12:20:15 +0200 Subject: [PATCH 5/8] test(cargo-codspeed): cover process isolation end to end Build and run a divan fixture in simulation mode and check that: - every benchmark runs in a fresh process by default, including one that uses a multi-threaded Tokio runtime started in `main`; - `CODSPEED_PROCESS_ISOLATION=false` runs them all in one process; - a panicking benchmark fails the run with the panic exit code. --- .github/workflows/ci.yml | 4 +- Cargo.lock | 59 +++++++++++++++++-- crates/cargo-codspeed/Cargo.toml | 1 + .../tests/process-isolation.in/Cargo.toml | 19 ++++++ .../process-isolation.in/benches/isolated.rs | 28 +++++++++ .../process-isolation.in/benches/panicking.rs | 8 +++ .../cargo-codspeed/tests/process-isolation.rs | 51 ++++++++++++++++ 7 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 crates/cargo-codspeed/tests/process-isolation.in/Cargo.toml create mode 100644 crates/cargo-codspeed/tests/process-isolation.in/benches/isolated.rs create mode 100644 crates/cargo-codspeed/tests/process-isolation.in/benches/panicking.rs create mode 100644 crates/cargo-codspeed/tests/process-isolation.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bbbffc2..0467990b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,7 +132,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - partition: [1, 2, 3, 4, 5] + partition: [1, 2, 3, 4, 5, 6, 7, 8] steps: - uses: actions/checkout@v4 with: @@ -142,7 +142,7 @@ jobs: bins: cargo-nextest env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: cargo nextest run -p cargo-codspeed --partition hash:${{ matrix.partition }}/5 + - run: cargo nextest run -p cargo-codspeed --partition hash:${{ matrix.partition }}/8 analysis-integration-test: strategy: diff --git a/Cargo.lock b/Cargo.lock index 66b6dbc7..57086b5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -456,6 +456,7 @@ dependencies = [ "fs_extra", "itertools 0.14.0", "predicates", + "rstest", "serde", "serde_json", "termcolor", @@ -1000,6 +1001,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.31" @@ -1470,9 +1477,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -1577,12 +1584,56 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "rstest" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "948203e6d13b83e51a90d7cf236dd58a0065f23f4b902f00f306e7eb2bd09b9c" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d92eaf7b6e51e12471d71ddc72608e7151573de44099aacfbb1128177c0da4" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] + [[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.37.27" @@ -1942,9 +1993,9 @@ checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "utf8parse" diff --git a/crates/cargo-codspeed/Cargo.toml b/crates/cargo-codspeed/Cargo.toml index 5e7a8e0f..dfe8bd23 100644 --- a/crates/cargo-codspeed/Cargo.toml +++ b/crates/cargo-codspeed/Cargo.toml @@ -34,6 +34,7 @@ codspeed = { path = "../codspeed", version = "=5.0.3" } assert_cmd = "2.0.15" fs_extra = "1.3.0" predicates = "3.1.2" +rstest = "0.27.0" [features] vendored-openssl = [] diff --git a/crates/cargo-codspeed/tests/process-isolation.in/Cargo.toml b/crates/cargo-codspeed/tests/process-isolation.in/Cargo.toml new file mode 100644 index 00000000..43548572 --- /dev/null +++ b/crates/cargo-codspeed/tests/process-isolation.in/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "process-isolation" +version = "0.0.0" +edition = "2021" +publish = false + +[dependencies] +divan = { package = "codspeed-divan-compat", path = "../../../divan_compat" } +tokio = { version = "1.39", features = ["rt-multi-thread"] } + +[workspace] + +[[bench]] +name = "isolated" +harness = false + +[[bench]] +name = "panicking" +harness = false diff --git a/crates/cargo-codspeed/tests/process-isolation.in/benches/isolated.rs b/crates/cargo-codspeed/tests/process-isolation.in/benches/isolated.rs new file mode 100644 index 00000000..fd3c6782 --- /dev/null +++ b/crates/cargo-codspeed/tests/process-isolation.in/benches/isolated.rs @@ -0,0 +1,28 @@ +use divan::Bencher; +use std::sync::LazyLock; +use tokio::runtime::Runtime; + +/// Started in `main`, before any benchmark, so its worker threads belong to +/// the process that runs the harness. +static RUNTIME: LazyLock = + LazyLock::new(|| Runtime::new().expect("failed to start the Tokio runtime")); + +#[divan::bench] +fn first(bencher: Bencher) { + bencher.bench(|| divan::black_box(1)); +} + +#[divan::bench] +fn second(bencher: Bencher) { + bencher.bench(|| divan::black_box(2)); +} + +#[divan::bench] +fn spawn_on_main_runtime(bencher: Bencher) { + bencher.bench(|| RUNTIME.block_on(async { tokio::spawn(async { 3 }).await.unwrap() })); +} + +fn main() { + LazyLock::force(&RUNTIME); + divan::main(); +} diff --git a/crates/cargo-codspeed/tests/process-isolation.in/benches/panicking.rs b/crates/cargo-codspeed/tests/process-isolation.in/benches/panicking.rs new file mode 100644 index 00000000..8303098d --- /dev/null +++ b/crates/cargo-codspeed/tests/process-isolation.in/benches/panicking.rs @@ -0,0 +1,8 @@ +#[divan::bench] +fn panics() { + panic!("benchmark failed"); +} + +fn main() { + divan::main(); +} diff --git a/crates/cargo-codspeed/tests/process-isolation.rs b/crates/cargo-codspeed/tests/process-isolation.rs new file mode 100644 index 00000000..60508b40 --- /dev/null +++ b/crates/cargo-codspeed/tests/process-isolation.rs @@ -0,0 +1,51 @@ +use predicates::str::contains; +use rstest::rstest; +use std::time::Duration; + +mod helpers; +use helpers::*; + +const DIR: &str = "tests/process-isolation.in"; +/// Process isolation only applies to analysis builds. +const MODE: [&str; 2] = ["-m", "simulation"]; +const PROCESS_ISOLATION_ENV: &str = "CODSPEED_PROCESS_ISOLATION"; +/// Exit code of a Rust process whose main thread panicked. +const PANIC_EXIT_CODE: i32 = 101; +/// Fails a run whose benchmark process got stuck instead of blocking the suite. +const RUN_TIMEOUT: Duration = Duration::from_secs(120); + +fn build_and_run(dir: &String, bench: &str, isolation: &str) -> assert_cmd::assert::Assert { + cargo_codspeed(dir) + .args(["build", "--bench", bench]) + .args(MODE) + .assert() + .success(); + cargo_codspeed(dir) + .args(["run", "--bench", bench]) + .args(MODE) + .env(PROCESS_ISOLATION_ENV, isolation) + .timeout(RUN_TIMEOUT) + .assert() +} + +#[rstest] +fn test_isolation_runs_all_benchmarks(#[values("true", "false")] isolation: &str) { + let dir = setup(DIR, Project::Simple); + build_and_run(&dir, "isolated", isolation) + .success() + .stdout(contains("isolated.rs::first")) + .stdout(contains("isolated.rs::second")) + .stdout(contains("isolated.rs::spawn_on_main_runtime")); + teardown(dir); +} + +#[rstest] +fn test_isolation_fails_the_run_when_a_benchmark_panics( + #[values("true", "false")] isolation: &str, +) { + let dir = setup(DIR, Project::Simple); + build_and_run(&dir, "panicking", isolation) + .code(PANIC_EXIT_CODE) + .stderr(contains("benchmark failed")); + teardown(dir); +} From 3218867d7103433528958ddea8b5b13592151fb9 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 8 Oct 2026 12:20:16 +0200 Subject: [PATCH 6/8] test(divan): add allocator ripple benchmark Three identical benchmarks that allocate a zeroed 4 MiB buffer. In one process the first one gets fresh mmap pages from glibc; freeing them raises the mmap threshold, so the next ones get heap memory that calloc must clear. Under callgrind they report 642, 127,593 and 229,862 instructions without isolation, and 642 each with it. `spawn_on_parent_runtime` awaits a task on a Tokio runtime started in `main`, which only works when each isolated process starts its own runtime. --- Cargo.lock | 1 + crates/divan_compat/Cargo.toml | 7 +++ .../divan_compat/benches/allocator_ripple.rs | 43 +++++++++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 crates/divan_compat/benches/allocator_ripple.rs diff --git a/Cargo.lock b/Cargo.lock index 57086b5b..01bb2f1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -664,6 +664,7 @@ dependencies = [ "codspeed-divan-compat-macros", "codspeed-divan-compat-walltime", "regex", + "tokio", ] [[package]] diff --git a/crates/divan_compat/Cargo.toml b/crates/divan_compat/Cargo.toml index d6d7a97c..8c230774 100644 --- a/crates/divan_compat/Cargo.toml +++ b/crates/divan_compat/Cargo.toml @@ -24,6 +24,9 @@ codspeed-divan-compat-macros = { version = "=5.0.3", path = './macros' } regex = "1.11.3" clap = { version = "4", default-features = false, features = ["std", "env"] } +[dev-dependencies] +tokio = { version = "1.39", features = ["rt-multi-thread"] } + [[bench]] name = "basic_example" harness = false @@ -43,3 +46,7 @@ harness = false [[bench]] name = "fib_example" harness = false + +[[bench]] +name = "allocator_ripple" +harness = false diff --git a/crates/divan_compat/benches/allocator_ripple.rs b/crates/divan_compat/benches/allocator_ripple.rs new file mode 100644 index 00000000..ee9371d6 --- /dev/null +++ b/crates/divan_compat/benches/allocator_ripple.rs @@ -0,0 +1,43 @@ +//! Identical benchmarks whose cost only depends on the allocator state left +//! behind by the benchmarks that ran before them in the same process. +//! +//! glibc serves `calloc` requests above `M_MMAP_THRESHOLD` (128 KiB by default) +//! with a fresh `mmap`, which the kernel already zeroed. Freeing such a chunk +//! raises the threshold to the chunk size, so the next allocation of the same +//! size comes from the heap and has to be cleared with `memset`. +//! +//! Simulation mode measures a single run per benchmark. With the default +//! process isolation, every benchmark runs in a fresh process and all of them +//! report the same cost. With `CODSPEED_PROCESS_ISOLATION=false`, they share +//! a process and the benchmarks after the first one pay for clearing the +//! buffer. Walltime mode repeats the routine, so its first iteration already +//! raises the threshold and later samples are similar either way. +//! +//! `spawn_on_parent_runtime` uses a multi-threaded Tokio runtime started in +//! `main`, before the harness runs any benchmark. Each isolated benchmark +//! process starts its own runtime, so awaiting a spawned task completes. + +use std::sync::LazyLock; +use tokio::runtime::Runtime; + +const BUFFER_SIZE: usize = 4 << 20; + +#[codspeed_divan_compat::bench(args = [1, 2, 3])] +fn zeroed_buffer(_copy: usize) -> u8 { + let buffer = vec![0u8; BUFFER_SIZE]; + codspeed_divan_compat::black_box(&buffer)[BUFFER_SIZE / 2] +} + +/// Started in `main`, so its workers only exist in the parent process. +static RUNTIME: LazyLock = + LazyLock::new(|| Runtime::new().expect("failed to start the Tokio runtime")); + +#[codspeed_divan_compat::bench] +fn spawn_on_parent_runtime() -> u32 { + RUNTIME.block_on(async { tokio::spawn(async { 1 }).await.unwrap() }) +} + +fn main() { + LazyLock::force(&RUNTIME); + codspeed_divan_compat::main(); +} From fef0566c677fcb484bbb4b3ecc4ad0b9a57610c5 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 8 Oct 2026 12:20:22 +0200 Subject: [PATCH 7/8] ci: time out the analysis integration tests Every benchmark now runs in its own process. A harness regression that leaves a child stuck must fail the job instead of blocking it until the GitHub Actions default of 6 hours. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0467990b..e8708a8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,6 +160,7 @@ jobs: - package: codspeed-criterion-compat features: async_futures runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 with: From fce04a64e3f13351b2912bd15ac1ac106274268a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 8 Oct 2026 13:01:40 +0200 Subject: [PATCH 8/8] fix(criterion): send benchmark markers around iter_manual rounds --- crates/divan_compat/divan_fork/src/bench/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/divan_compat/divan_fork/src/bench/mod.rs b/crates/divan_compat/divan_fork/src/bench/mod.rs index 5303ae25..8ad21962 100644 --- a/crates/divan_compat/divan_fork/src/bench/mod.rs +++ b/crates/divan_compat/divan_fork/src/bench/mod.rs @@ -29,7 +29,7 @@ mod args; mod defer; mod options; -use ::codspeed::instrument_hooks::InstrumentHooks; +use ::codspeed::instrument_hooks::{BenchmarkMarkers, InstrumentHooks}; use defer::{DeferSlot, DeferStore}; pub use self::{ @@ -658,7 +658,7 @@ impl<'a> BenchContext<'a> { let bench_overheads = timer.bench_overheads(); - let _ = InstrumentHooks::instance().start_benchmark(); + let markers = BenchmarkMarkers::start(); while { // Conditions for when sampling is over: if elapsed_picos >= max_picos { @@ -825,7 +825,7 @@ impl<'a> BenchContext<'a> { elapsed_picos = elapsed_picos.saturating_add(progress_picos); } } - let _ = InstrumentHooks::instance().stop_benchmark(); + drop(markers); // Reset flag for ignoring allocations. crate::alloc::IGNORE_ALLOC.set(false);