feat(divan): export benchmark tracing spans over OTLP behind an otel feature

crates/cargo-codspeed/src/app.rs

@@ -36,7 +36,26 @@ impl Cli {
                 offline,
                 frozen,
                 measurement_mode,
+                keep_tracing,
             } => {
+                if measurement_mode.contains(&MeasurementMode::Telemetry) {
+                    if measurement_mode
+                        .iter()
+                        .any(|m| *m != MeasurementMode::Telemetry)
+                    {
+                        bail!(
+                            "The telemetry mode can't be combined with other measurement modes: \
+                            build it separately with `cargo codspeed build -m telemetry`"
+                        );
+                    }
+                    if keep_tracing {
+                        bail!(
+                            "`--keep-tracing` has no effect in telemetry mode, which always keeps \
+                            `tracing`"
+                        );
+                    }
+                }
+
                 let passthrough_flags = {
                     let mut passthrough_flags = Vec::new();
                     if all_features {
@@ -89,6 +108,7 @@ impl Cli {
                             profile: profile.clone(),
                             quiet: self.quiet,
                             build_mode,
+                            keep_tracing,
                             passthrough_flags: passthrough_flags.clone(),
                         },
                     )?;
@@ -197,6 +217,11 @@ enum Commands {
             env = "CODSPEED_RUNNER_MODE"
         )]
         measurement_mode: Vec<MeasurementMode>,
+
+        /// Keep `tracing` spans and events compiled in. By default, the `max_level_off` feature of
+        /// `tracing` compiles them out of every mode but `telemetry`.
+        #[arg(long, help_heading = COMPILATION_HELP)]
+        keep_tracing: bool,
     },
     /// Run the previously built benchmarks
     Run {

crates/cargo-codspeed/src/build.rs

@@ -9,11 +9,16 @@ use cargo_metadata::{camino::Utf8PathBuf, Message, Metadata, TargetKind};
 use std::collections::HashMap;
 use std::process::{exit, Command, Stdio};
 
+/// Set by `cargo-codspeed` in the builds `tracing` is compiled out of, see the `codspeed` crate
+/// manifest
+const TRACING_OFF_CFG: &str = "codspeed_tracing_off";
+
 struct BuildOptions<'a> {
     bench_target_filters: BenchTargetFilters,
     package_filters: PackageFilters,
     features: &'a Option<Vec<String>>,
     profile: &'a str,
+    keep_tracing: bool,
     passthrough_flags: &'a Vec<String>,
 }
 
@@ -31,6 +36,8 @@ pub struct BuildConfig {
     pub profile: String,
     pub quiet: bool,
     pub build_mode: BuildMode,
+    /// Keep `tracing` compiled in when building for a measurement mode
+    pub keep_tracing: bool,
     pub passthrough_flags: Vec<String>,
 }
 
@@ -226,10 +233,17 @@ See `cargo codspeed build --help` for more information.");
         ];
 
         // Add the codspeed cfg flag if the benchmark should only run once
-        if build_mode == BuildMode::Analysis {
+        if build_mode.runs_once() {
             flags.push("--cfg=codspeed".to_owned());
         }
 
+        // Compile `tracing` out of measured benchmarks, so that it doesn't weigh on the
+        // measurements: the `codspeed` crate enables the `max_level_off` feature of `tracing`
+        // behind this cfg. Only the telemetry build keeps it, to export what it records.
+        if build_mode != BuildMode::Telemetry && !self.keep_tracing {
+            flags.push(format!("--cfg={TRACING_OFF_CFG}"));
+        }
+
         match std::env::var("RUSTFLAGS") {
             Result::Ok(existing_rustflags) => {
                 // Expand already existing RUSTFLAGS env var
@@ -307,6 +321,7 @@ pub fn build_benches(metadata: &Metadata, config: BuildConfig) -> Result<()> {
         package_filters: config.package_filters,
         features: &config.features,
         profile: &config.profile,
+        keep_tracing: config.keep_tracing,
         passthrough_flags: &config.passthrough_flags,
     }
     .build(metadata, config.quiet, build_mode)?;

crates/cargo-codspeed/src/measurement_mode.rs

@@ -7,6 +7,16 @@ pub enum BuildMode {
     #[default]
     Analysis,
     Walltime,
+    /// Runs each benchmark like [`BuildMode::Analysis`], to collect the telemetry it emits rather
+    /// than measure it
+    Telemetry,
+}
+
+impl BuildMode {
+    /// Whether the benchmarks run once, behind `cfg(codspeed)`, rather than in a walltime loop
+    pub fn runs_once(self) -> bool {
+        matches!(self, BuildMode::Analysis | BuildMode::Telemetry)
+    }
 }
 
 impl fmt::Display for BuildMode {
@@ -17,6 +27,7 @@ impl fmt::Display for BuildMode {
             match self {
                 BuildMode::Analysis => "analysis",
                 BuildMode::Walltime => "walltime",
+                BuildMode::Telemetry => "telemetry",
             }
         )
     }
@@ -30,13 +41,16 @@ pub enum MeasurementMode {
     Simulation,
     Walltime,
     Memory,
+    /// Collects the telemetry the benchmarks emit instead of measuring them
+    Telemetry,
 }
 
 impl From<MeasurementMode> for BuildMode {
     fn from(measurement_mode: MeasurementMode) -> Self {
         match measurement_mode {
             MeasurementMode::Simulation | MeasurementMode::Memory => BuildMode::Analysis,
             MeasurementMode::Walltime => BuildMode::Walltime,
+            MeasurementMode::Telemetry => BuildMode::Telemetry,
         }
     }
 }
@@ -50,6 +64,7 @@ impl fmt::Display for MeasurementMode {
                 MeasurementMode::Simulation => "simulation",
                 MeasurementMode::Walltime => "walltime",
                 MeasurementMode::Memory => "memory",
+                MeasurementMode::Telemetry => "telemetry",
             }
         )
     }

crates/cargo-codspeed/tests/tracing.in/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "tracing-example"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+divan = { package = "codspeed-divan-compat", path = "../../../divan_compat" }
+tracing = "0.1"
+
+[workspace]
+
+[[bench]]
+name = "tracing_example"
+harness = false

crates/cargo-codspeed/tests/tracing.in/benches/tracing_example.rs

@@ -0,0 +1,13 @@
+#[divan::bench]
+fn traced() -> u32 {
+    let _span = tracing::info_span!("traced").entered();
+    divan::black_box(1) + 1
+}
+
+fn main() {
+    eprintln!(
+        "tracing static max level: {}",
+        tracing::level_filters::STATIC_MAX_LEVEL
+    );
+    divan::main();
+}

crates/cargo-codspeed/tests/tracing.rs

@@ -0,0 +1,83 @@
+use predicates::str::contains;
+
+mod helpers;
+use helpers::*;
+
+const DIR: &str = "tests/tracing.in";
+
+fn build_and_run(build_args: &[&str], run_args: &[&str], expected_level: &str) {
+    let dir = setup(DIR, Project::Simple);
+    cargo_codspeed(&dir)
+        .arg("build")
+        .args(build_args)
+        .assert()
+        .success();
+    cargo_codspeed(&dir)
+        .arg("run")
+        .args(run_args)
+        .assert()
+        .success()
+        .stderr(contains(format!(
+            "tracing static max level: {expected_level}"
+        )));
+    teardown(dir);
+}
+
+#[test]
+fn test_tracing_compiled_out_for_simulation() {
+    build_and_run(&[], &[], "off");
+}
+
+#[test]
+fn test_tracing_compiled_out_for_walltime() {
+    build_and_run(&["-m", "walltime"], &["-m", "walltime"], "off");
+}
+
+#[test]
+fn test_tracing_kept_with_keep_tracing() {
+    build_and_run(&["--keep-tracing"], &[], "trace");
+}
+
+#[test]
+fn test_tracing_kept_for_telemetry() {
+    build_and_run(&["-m", "telemetry"], &["-m", "telemetry"], "trace");
+}
+
+#[test]
+fn test_telemetry_run_without_build() {
+    let dir = setup(DIR, Project::Simple);
+    cargo_codspeed(&dir)
+        .args(["run", "-m", "telemetry"])
+        .assert()
+        .failure()
+        .stderr(contains(
+            "Error: No benchmarks found for the telemetry mode. Run `cargo codspeed build -m telemetry` first.",
+        ));
+    teardown(dir);
+}
+
+#[test]
+fn test_telemetry_not_combined_with_other_modes() {
+    let dir = setup(DIR, Project::Simple);
+    cargo_codspeed(&dir)
+        .args(["build", "-m", "simulation,telemetry"])
+        .assert()
+        .failure()
+        .stderr(contains(
+            "Error: The telemetry mode can't be combined with other measurement modes",
+        ));
+    teardown(dir);
+}
+
+#[test]
+fn test_telemetry_rejects_keep_tracing() {
+    let dir = setup(DIR, Project::Simple);
+    cargo_codspeed(&dir)
+        .args(["build", "-m", "telemetry", "--keep-tracing"])
+        .assert()
+        .failure()
+        .stderr(contains(
+            "Error: `--keep-tracing` has no effect in telemetry mode",
+        ));
+    teardown(dir);
+}

crates/codspeed/Cargo.toml

@@ -28,6 +28,54 @@ serde = { workspace = true }
 serde_json = { workspace = true }
 statrs = { version = "0.18.0", default-features = false }
 
+opentelemetry = { version = "0.33", default-features = false, features = [
+    "trace",
+    "logs",
+], optional = true }
+opentelemetry_sdk = { version = "0.33", default-features = false, features = [
+    "trace",
+    "logs",
+], optional = true }
+opentelemetry-appender-tracing = { version = "0.33", default-features = false, features = [
+    "experimental_metadata_attributes",
+], optional = true }
+opentelemetry-otlp = { version = "0.33", default-features = false, features = [
+    "trace",
+    "logs",
+    "http-proto",
+    "reqwest-blocking-client",
+], optional = true }
+tracing = { version = "0.1", default-features = false, features = [
+    "std",
+], optional = true }
+tracing-opentelemetry = { version = "0.34", default-features = false, optional = true }
+tracing-subscriber = { version = "0.3", default-features = false, features = [
+    "std",
+    "registry",
+    "env-filter",
+], optional = true }
+# Unused directly: raises the floor of `sharded-slab`'s `lazy_static = "1"`, which fails to build below 1.1
+lazy_static = { version = "1.1", optional = true }
+
+# Unused directly: `cargo-codspeed` sets this cfg when building benchmarks for a measurement mode,
+# so that features unification compiles the `tracing` spans and events of the whole build out.
+[target.'cfg(codspeed_tracing_off)'.dependencies]
+tracing = { version = "0.1", default-features = false, features = [
+    "max_level_off",
+] }
+
+[features]
+otel = [
+    "dep:opentelemetry",
+    "dep:opentelemetry_sdk",
+    "dep:opentelemetry-appender-tracing",
+    "dep:opentelemetry-otlp",
+    "dep:tracing",
+    "dep:tracing-opentelemetry",
+    "dep:tracing-subscriber",
+    "dep:lazy_static",
+]
+
 [[bench]]
 name = "native"
 harness = false

crates/codspeed/src/benchmark_scope.rs

@@ -0,0 +1,19 @@
+//! The benchmark being run, which the exported telemetry is attributed to.
+//!
+//! Without the `otel` feature, entering and exiting a scope does nothing.
+
+/// Marks the start of the benchmark `uri`. Spans started until [`exit`] carry its URI.
+///
+/// Call it outside the measured section.
+pub fn enter(uri: &str) {
+    #[cfg(feature = "otel")]
+    crate::otel::enter_benchmark(uri);
+    #[cfg(not(feature = "otel"))]
+    let _ = uri;
+}
+
+/// Marks the end of the current benchmark.
+pub fn exit() {
+    #[cfg(feature = "otel")]
+    crate::otel::exit_benchmark();
+}

crates/codspeed/src/codspeed.rs

@@ -81,6 +81,7 @@ impl CodSpeed {
     #[inline(always)]
     pub fn start_benchmark(&mut self, name: &str) {
         self.current_benchmark = CString::new(name).expect("CString::new failed");
+        crate::benchmark_scope::enter(name);
 
         if let InstrumentationStatus::InstrumentHooks(hooks_instance) = &self.instrumentation_status
         {
@@ -99,9 +100,15 @@ impl CodSpeed {
         // The overhead in case we are not running valgrind is extremely low, and we do not want to
         // add a conditionnal branch to valgrind measurements.
         measurement::stop(&self.current_benchmark);
-        if let InstrumentationStatus::InstrumentHooks(hooks_instance) = &self.instrumentation_status
-        {
+        let hooks_instance = match &self.instrumentation_status {
+            InstrumentationStatus::InstrumentHooks(hooks_instance) => Some(*hooks_instance),
+            _ => None,
+        };
+        if let Some(hooks_instance) = hooks_instance {
             let _ = hooks_instance.stop_benchmark();
+        }
+        crate::benchmark_scope::exit();
+        if let Some(hooks_instance) = hooks_instance {
             let _ =
                 hooks_instance.set_executed_benchmark(&self.current_benchmark.to_string_lossy());
         }

crates/codspeed/src/instrument_hooks/bindings.rs

@@ -56,6 +56,9 @@ extern "C" {
 extern "C" {
     pub fn instrument_hooks_current_timestamp() -> u64;
 }
+extern "C" {
+    pub fn instrument_hooks_monotonic_offset() -> i64;
+}
 extern "C" {
     pub fn instrument_hooks_callgrind_add_obj_skip(path: *const ::std::os::raw::c_char) -> u8;
 }