From e44f96385a3d2b4e09958a78ca94c7c6d35cdf7d Mon Sep 17 00:00:00 2001 From: Daniel Boros Date: Thu, 12 Nov 2026 23:17:33 +0100 Subject: [PATCH 01/12] chore: update tako to 2.7.0 --- frameworks/tako-gateway/README.md | 2 +- frameworks/tako-grpc/Cargo.lock | 108 ++++++--------------- frameworks/tako-grpc/Cargo.toml | 2 +- frameworks/tako-grpc/README.md | 4 +- frameworks/tako-grpc/meta.json | 2 +- frameworks/tako/Cargo.lock | 154 ++++++------------------------ frameworks/tako/Cargo.toml | 3 +- frameworks/tako/README.md | 6 +- frameworks/tako/meta.json | 2 +- frameworks/tako/src/main.rs | 7 +- 10 files changed, 67 insertions(+), 223 deletions(-) diff --git a/frameworks/tako-gateway/README.md b/frameworks/tako-gateway/README.md index 6a27177435..2edb494e46 100644 --- a/frameworks/tako-gateway/README.md +++ b/frameworks/tako-gateway/README.md @@ -1,6 +1,6 @@ # tako-gateway -The gateway profiles for [Tako](https://github.com/rust-dd/tako) 2.4.2, reported as `tako`. +The gateway profiles for [Tako](https://github.com/rust-dd/tako) 2.7.0, reported as `tako`. The server is built from [`frameworks/tako`](../tako); this directory only holds the compose files and the edge configuration. diff --git a/frameworks/tako-grpc/Cargo.lock b/frameworks/tako-grpc/Cargo.lock index c66705fd26..0e87040ccd 100644 --- a/frameworks/tako-grpc/Cargo.lock +++ b/frameworks/tako-grpc/Cargo.lock @@ -329,7 +329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -986,6 +986,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.47" @@ -1143,16 +1152,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.229" @@ -1211,17 +1210,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha1" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.10.9" @@ -1329,9 +1317,9 @@ dependencies = [ [[package]] name = "tako-rs" -version = "2.4.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa5e097cfb81d8adc24743b24e7f649bc9a717aaf810128da0968532948d7fdf" +checksum = "1407ad0125ae2215830acdeac13196f610a0df4fd1324874189313d643617eae" dependencies = [ "linkme", "tako-rs-core", @@ -1344,23 +1332,21 @@ dependencies = [ [[package]] name = "tako-rs-core" -version = "2.4.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfebf99c7554fa9d7b7612a393c72fa6c161eab5c391cd76de32b0a2648e82e2" +checksum = "dfd800672ed0924b331d21979676d332cc2b62ba42f7b998a6df648fb79f88fb" dependencies = [ "anyhow", "arc-swap", "async-trait", "base64", "bytes", - "cookie", "envy", "futures-util", "http", "http-body", "http-body-util", "hyper", - "hyper-util", "linkme", "matchit", "mime", @@ -1368,65 +1354,48 @@ dependencies = [ "parking_lot", "pin-project-lite", "prost", + "prost-types", "rustls", "scc", "serde", - "serde_bytes", "serde_json", - "serde_urlencoded", - "sha1", - "sha2 0.11.0", "smallvec", "thread_local", "tokio", - "tokio-rustls", - "tokio-stream", - "tokio-util", "tracing", - "url", - "urlencoding", ] [[package]] name = "tako-rs-extractors" -version = "2.4.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20271324f81439e9e86b50f301d85493c11c29640549bb34a0b2d7f1aad091cd" +checksum = "b21362b5148c3d099ce276d64d8c1a8c2f3347b5625555c5dd8b5337e4febdf9" dependencies = [ "anyhow", - "async-trait", "base64", "bytes", "cookie", "futures-util", "hkdf", "http", - "http-body", "http-body-util", "ipnet", "mime", - "mime_guess", - "parking_lot", "serde", - "serde_bytes", "serde_html_form", "serde_json", "serde_urlencoded", - "sha1", "sha2 0.11.0", - "smallvec", "tako-rs-core", "tokio", - "tracing", - "url", "urlencoding", ] [[package]] name = "tako-rs-macros" -version = "2.4.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a60a349ad85a7dd7fa9a10578770029d249850b7dd21f86bad2b5d1cbf17e8cc" +checksum = "64cadbbefdcfb3dfcb15ceda3e7ca994a7f1a278759d8f0023301f6ff2b725ea" dependencies = [ "proc-macro2", "quote", @@ -1435,45 +1404,39 @@ dependencies = [ [[package]] name = "tako-rs-plugins" -version = "2.4.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df77e9e5dbce9805764bba99c806c74d131abb9d3b078a0eccf179212d849a16" +checksum = "23988e23bc380e50feccba1f2d0e89599a691a7a77b47a77928ba4ad7d40bf27" dependencies = [ "anyhow", "async-trait", "base64", "bytes", - "cookie", "futures-util", "http", "http-body", "http-body-util", "httpdate", - "mime", "parking_lot", "pin-project-lite", "scc", "serde", "serde_json", - "sha1", "sha2 0.11.0", - "smallvec", "subtle", "tako-rs-core", "tako-rs-extractors", "tokio", - "tokio-stream", "tracing", "url", - "urlencoding", "uuid", ] [[package]] name = "tako-rs-server" -version = "2.4.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea19bb546daccd8fa4508ae161c56e5d385b8cf8a4c088635656f910a619b8f3" +checksum = "023d57cb4a1d189a64cbd61f940a0f338682b4212208b06a81c6d69093a6530f" dependencies = [ "anyhow", "arc-swap", @@ -1481,7 +1444,6 @@ dependencies = [ "futures-util", "http", "http-body", - "http-body-util", "hyper", "hyper-util", "pin-project-lite", @@ -1489,16 +1451,15 @@ dependencies = [ "tako-rs-core", "tokio", "tokio-rustls", - "tokio-stream", "tokio-util", "tracing", ] [[package]] name = "tako-rs-streams" -version = "2.4.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f435c8191629d3c6a77f270c999047c5ee8153adf021169c783c371d03a0be9" +checksum = "709cec96f9f0524bd8c7da8fdff2db17b3e8a48663585f7678a1162a077515a7" dependencies = [ "anyhow", "bytes", @@ -1507,9 +1468,6 @@ dependencies = [ "http-body", "http-body-util", "httpdate", - "hyper", - "hyper-util", - "mime", "mime_guess", "percent-encoding", "pin-project-lite", @@ -1517,7 +1475,6 @@ dependencies = [ "tokio", "tokio-util", "tracing", - "url", ] [[package]] @@ -1571,9 +1528,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.2" +version = "1.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", "libc", @@ -1606,17 +1563,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-stream" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.20" diff --git a/frameworks/tako-grpc/Cargo.toml b/frameworks/tako-grpc/Cargo.toml index 1ea1b13a35..78235c98e3 100644 --- a/frameworks/tako-grpc/Cargo.toml +++ b/frameworks/tako-grpc/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] prost = "0.14" -tako-rs = { version = "2.4.1", features = ["grpc", "http2", "tls"] } +tako-rs = { version = "2.7.0", features = ["grpc", "http2", "tls"] } tokio = { version = "1", features = ["macros", "net", "rt-multi-thread"] } [profile.release] diff --git a/frameworks/tako-grpc/README.md b/frameworks/tako-grpc/README.md index ae3e3ec2c2..686b9fcbd1 100644 --- a/frameworks/tako-grpc/README.md +++ b/frameworks/tako-grpc/README.md @@ -1,6 +1,6 @@ # tako-grpc -The gRPC profiles for [Tako](https://github.com/rust-dd/tako) 2.4.1, reported as `tako`. +The gRPC profiles for [Tako](https://github.com/rust-dd/tako) 2.7.0, reported as `tako`. ## Why a separate entry @@ -11,7 +11,7 @@ so the gRPC service gets its own container. ## Stack - **Language:** Rust 1.99 -- **Framework:** tako-rs 2.4.1 (`grpc`, `http2`, `tls`) +- **Framework:** tako-rs 2.7.0 (`grpc`, `http2`, `tls`) - **Protobuf:** prost 0.14 - **Build:** Multi-stage, `debian:trixie-slim` runtime diff --git a/frameworks/tako-grpc/meta.json b/frameworks/tako-grpc/meta.json index 582666a8da..0dcae14f9f 100644 --- a/frameworks/tako-grpc/meta.json +++ b/frameworks/tako-grpc/meta.json @@ -4,7 +4,7 @@ "engine": "hyper", "type": "emerging", "mode": "standard", - "description": "Tako 2.4.1 gRPC: benchmark.BenchmarkService/GetSum as a plain tako route with the GrpcRequest extractor and GrpcResponse responder, prost messages. h2c on :8080 and h2 over rustls on :8443, both on tako's multi-thread Server with default configuration.", + "description": "Tako 2.7.0 gRPC: benchmark.BenchmarkService/GetSum as a plain tako route with the GrpcRequest extractor and GrpcResponse responder, prost messages. h2c on :8080 and h2 over rustls on :8443, both on tako's multi-thread Server with default configuration.", "repo": "https://github.com/rust-dd/tako", "website": "https://tako.rust-dd.com", "enabled": true, diff --git a/frameworks/tako/Cargo.lock b/frameworks/tako/Cargo.lock index 802c2fdc53..4726530477 100644 --- a/frameworks/tako/Cargo.lock +++ b/frameworks/tako/Cargo.lock @@ -112,7 +112,7 @@ dependencies = [ "futures-util", "log", "pin-project-lite", - "tungstenite 0.29.0", + "tungstenite", ] [[package]] @@ -522,7 +522,7 @@ dependencies = [ "compio-tls", "futures-util", "pin-project-lite", - "tungstenite 0.29.0", + "tungstenite", ] [[package]] @@ -828,7 +828,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1227,7 +1227,6 @@ name = "httparena-tako" version = "0.1.0" dependencies = [ "deadpool-postgres", - "futures-util", "moka", "serde", "serde_json", @@ -2001,7 +2000,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2175,7 +2174,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2234,7 +2233,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2361,16 +2360,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.229" @@ -2440,17 +2429,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha1" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.1", - "digest 0.11.3", -] - [[package]] name = "sha2" version = "0.10.9" @@ -2648,9 +2626,9 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tako-rs" -version = "2.4.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b14a9fe140df4f1bba8c1dc55c2d3033b59d96e28b8b82f890a2fa77ead1d34" +checksum = "1407ad0125ae2215830acdeac13196f610a0df4fd1324874189313d643617eae" dependencies = [ "linkme", "tako-rs-core", @@ -2664,94 +2642,70 @@ dependencies = [ [[package]] name = "tako-rs-core" -version = "2.4.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a913ace8ee7960be78124eb776c097ba83af6fbbb8c0a3aaaa2b8bb3ed78139f" +checksum = "dfd800672ed0924b331d21979676d332cc2b62ba42f7b998a6df648fb79f88fb" dependencies = [ "anyhow", "arc-swap", "async-trait", "base64", - "brotli", "bytes", - "cookie", "envy", - "flate2", "futures-util", - "h3", - "h3-quinn", "http", "http-body", "http-body-util", "hyper", - "hyper-util", "linkme", "matchit", "mime", "mime_guess", "parking_lot", "pin-project-lite", - "quinn", "rustls", "scc", "serde", - "serde_bytes", "serde_json", - "serde_urlencoded", - "sha1 0.10.7", - "sha2 0.11.0", + "sha1", "smallvec", "thread_local", "tokio", - "tokio-rustls", - "tokio-stream", - "tokio-tungstenite", - "tokio-util", "tracing", "url", - "urlencoding", ] [[package]] name = "tako-rs-extractors" -version = "2.4.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c41a681af6b40fcfd3a3228dd799e8dc5acc6e16ef85120c9be7780eaf81a18" +checksum = "b21362b5148c3d099ce276d64d8c1a8c2f3347b5625555c5dd8b5337e4febdf9" dependencies = [ "anyhow", - "async-trait", "base64", "bytes", "cookie", "futures-util", "hkdf", "http", - "http-body", "http-body-util", "ipnet", "mime", - "mime_guess", - "parking_lot", "serde", - "serde_bytes", "serde_html_form", "serde_json", "serde_urlencoded", - "sha1 0.10.7", "sha2 0.11.0", - "smallvec", "tako-rs-core", "tokio", - "tracing", - "url", "urlencoding", ] [[package]] name = "tako-rs-macros" -version = "2.4.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66453ad4b25d4beef59531f9614d9ec2e6f85ba1321d20bf4b673ed2bb0058c9" +checksum = "64cadbbefdcfb3dfcb15ceda3e7ca994a7f1a278759d8f0023301f6ff2b725ea" dependencies = [ "proc-macro2", "quote", @@ -2760,47 +2714,41 @@ dependencies = [ [[package]] name = "tako-rs-plugins" -version = "2.4.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6eec747af944875d67f719ce15a4a35d43fd5301fe84dbe8ee27ad505df7a13" +checksum = "23988e23bc380e50feccba1f2d0e89599a691a7a77b47a77928ba4ad7d40bf27" dependencies = [ "anyhow", "async-trait", "base64", "brotli", "bytes", - "cookie", "flate2", "futures-util", "http", "http-body", "http-body-util", "httpdate", - "mime", "parking_lot", "pin-project-lite", "scc", "serde", "serde_json", - "sha1 0.10.7", "sha2 0.11.0", - "smallvec", "subtle", "tako-rs-core", "tako-rs-extractors", "tokio", - "tokio-stream", "tracing", "url", - "urlencoding", "uuid", ] [[package]] name = "tako-rs-server" -version = "2.4.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dfd96a982ee7a3b732c0cb930f6b59dfb26863a9bb4af719c24e23331fd291f" +checksum = "023d57cb4a1d189a64cbd61f940a0f338682b4212208b06a81c6d69093a6530f" dependencies = [ "anyhow", "arc-swap", @@ -2811,7 +2759,6 @@ dependencies = [ "h3-quinn", "http", "http-body", - "http-body-util", "hyper", "hyper-util", "pin-project-lite", @@ -2820,16 +2767,15 @@ dependencies = [ "tako-rs-core", "tokio", "tokio-rustls", - "tokio-stream", "tokio-util", "tracing", ] [[package]] name = "tako-rs-server-pt" -version = "2.4.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cdc21ac0cb39e22adfd6fb931fd4aa8df92e224f322ef5f887771de62f1166" +checksum = "54ed007a2d03d51d3043fee313a6b94657310aab26ec0a39a00764f486ba4099" dependencies = [ "core_affinity", "futures-util", @@ -2844,33 +2790,26 @@ dependencies = [ [[package]] name = "tako-rs-streams" -version = "2.4.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374a13704cb651b62c86af2ed9aed5d71ba1c5a6c7bdc5d7461d71a2edf928ee" +checksum = "709cec96f9f0524bd8c7da8fdff2db17b3e8a48663585f7678a1162a077515a7" dependencies = [ "anyhow", - "base64", "bytes", "futures-util", "http", "http-body", "http-body-util", "httpdate", - "hyper", - "hyper-util", - "mime", "mime_guess", "percent-encoding", "pin-project-lite", "quinn", "rustls", - "sha1 0.10.7", "tako-rs-core", "tokio", - "tokio-tungstenite", "tokio-util", "tracing", - "url", ] [[package]] @@ -2959,9 +2898,9 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" -version = "1.53.2" +version = "1.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", "libc", @@ -3020,29 +2959,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-stream" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17a073bfed563fa236697a068031408a93cd9522e08abf9933ead3e73411bd71" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.30.0", -] - [[package]] name = "tokio-util" version = "0.7.20" @@ -3165,23 +3081,7 @@ dependencies = [ "rand 0.9.5", "rustls", "rustls-pki-types", - "sha1 0.10.7", - "thiserror", -] - -[[package]] -name = "tungstenite" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48ac77174b19c110a50ab2128b24215ac9cb40e0e12e093fb602d175c569d22" -dependencies = [ - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.10.3", - "sha1 0.11.0", + "sha1", "thiserror", ] @@ -3445,7 +3345,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/frameworks/tako/Cargo.toml b/frameworks/tako/Cargo.toml index d0286c0675..336e640b50 100644 --- a/frameworks/tako/Cargo.toml +++ b/frameworks/tako/Cargo.toml @@ -5,11 +5,10 @@ edition = "2024" [dependencies] deadpool-postgres = "0.14" -futures-util = "0.3" moka = { version = "0.12", features = ["sync"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -tako-rs = { version = "2.4.2", features = ["http2", "http3", "per-thread", "plugins", "tls", "ws"] } +tako-rs = { version = "2.7.0", features = ["http2", "http3", "per-thread", "plugins", "tls", "ws"] } tokio = { version = "1", features = ["macros", "net", "rt-multi-thread", "time"] } tokio-postgres = { version = "0.7", features = ["with-serde_json-1"] } diff --git a/frameworks/tako/README.md b/frameworks/tako/README.md index d02b48bfa0..670fd418c7 100644 --- a/frameworks/tako/README.md +++ b/frameworks/tako/README.md @@ -1,13 +1,13 @@ # tako -[Tako](https://github.com/rust-dd/tako) 2.4.2 (`tako-rs` on crates.io) on hyper and Tokio, default configuration. +[Tako](https://github.com/rust-dd/tako) 2.7.0 (`tako-rs` on crates.io) on hyper and Tokio, default configuration. gRPC lives in [`tako-grpc`](../tako-grpc) and the gateway stacks in [`tako-gateway`](../tako-gateway); all three report as `tako`. ## Stack - **Language:** Rust 1.99 -- **Framework:** tako-rs 2.4.2 +- **Framework:** tako-rs 2.7.0 - **Build:** Multi-stage, `debian:trixie-slim` runtime ## Listeners @@ -51,7 +51,7 @@ behind the gateway edge listens in cleartext only. - Static files through tako's `ServeDir` with `PrecompressedPolicy::both()`, which picks the `.br`/`.gz` sibling from `Accept-Encoding` and reads from disk on every request -- WebSocket through `TakoWs`, echoing text and binary frames +- WebSocket through `WebSocketUpgrade`, echoing text and binary frames - Postgres through deadpool-postgres and tokio-postgres, pool sized from `DATABASE_MAX_CONN`, statements prepared once per connection - Production-stack reads are cache-aside in process (moka, 1 s TTL for items, diff --git a/frameworks/tako/meta.json b/frameworks/tako/meta.json index c3df4674e3..befde11ac2 100644 --- a/frameworks/tako/meta.json +++ b/frameworks/tako/meta.json @@ -10,7 +10,7 @@ "request": true, "response": true }, - "description": "Tako 2.4.2 on hyper and Tokio, default configuration. Plain HTTP/1.1 and WebSocket on :8080 run tako's per-thread server (one current-thread runtime per CPU, SO_REUSEPORT listeners, PerThreadConfig::default()); h2c (:8082), HTTP/1.1 and h2 over rustls (:8081, :8443) and HTTP/3 over quinn (:8443/udp) run tako's multi-thread Server. Routing, Path/Query/State/body extractors and the Json responder through the tako API, the compression plugin with default settings on the JSON routes, ServeDir with precompressed .br/.gz sidecars read from disk per request, TakoWs for /ws, and deadpool-postgres for async-db.", + "description": "Tako 2.7.0 on hyper and Tokio, default configuration. Plain HTTP/1.1 and WebSocket on :8080 run tako's per-thread server (one current-thread runtime per CPU, SO_REUSEPORT listeners, PerThreadConfig::default()); h2c (:8082), HTTP/1.1 and h2 over rustls (:8081, :8443) and HTTP/3 over quinn (:8443/udp) run tako's multi-thread Server. Routing, Path/Query/State/body extractors and the Json responder through the tako API, the compression plugin with default settings on the JSON routes, ServeDir with precompressed .br/.gz sidecars read from disk per request, WebSocketUpgrade for /ws, and deadpool-postgres for async-db.", "repo": "https://github.com/rust-dd/tako", "website": "https://tako.rust-dd.com", "enabled": true, diff --git a/frameworks/tako/src/main.rs b/frameworks/tako/src/main.rs index 7e5e9cd04e..1c65a05b82 100644 --- a/frameworks/tako/src/main.rs +++ b/frameworks/tako/src/main.rs @@ -2,7 +2,6 @@ mod db; use std::time::Duration; -use futures_util::{SinkExt, StreamExt}; use serde::{Deserialize, Serialize}; use tako::extractors::json::Json; use tako::extractors::path::Path; @@ -13,7 +12,7 @@ use tako::responder::Responder; use tako::router::Router; use tako::r#static::{PrecompressedPolicy, ServeDir}; use tako::types::{BoxError, Request}; -use tako::ws::TakoWs; +use tako::ws::WebSocketUpgrade; use tako::{Bytes, PerThreadConfig, Server, TlsCert}; use tokio::net::TcpListener; @@ -139,8 +138,8 @@ async fn echo(body: Bytes) -> Bytes { } async fn ws_echo(req: Request) -> impl Responder { - TakoWs::new(req, |mut ws| async move { - while let Some(Ok(message)) = ws.next().await { + WebSocketUpgrade::new(req, |mut ws| async move { + while let Some(Ok(message)) = ws.recv().await { if (message.is_text() || message.is_binary()) && ws.send(message).await.is_err() { break; } From bd314f1711e085d56d2e65c587ff424190e66390 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 13 Nov 2026 11:34:03 +0000 Subject: [PATCH 02/12] Benchmark results: 3 frameworks (all tests) [skip ci] --- site/data/frameworks.json | 20 +- site/data/results/tako.json | 540 ++++++++++++++++++------------------ 2 files changed, 280 insertions(+), 280 deletions(-) diff --git a/site/data/frameworks.json b/site/data/frameworks.json index 263dcc81ba..ee1cea2afd 100644 --- a/site/data/frameworks.json +++ b/site/data/frameworks.json @@ -2502,7 +2502,7 @@ }, "tako": { "dir": "tako", - "description": "Tako 2.4.2 on hyper and Tokio, default configuration. Plain HTTP/1.1 and WebSocket on :8080 run tako's per-thread server (one current-thread runtime per CPU, SO_REUSEPORT listeners, PerThreadConfig::default()); h2c (:8082), HTTP/1.1 and h2 over rustls (:8081, :8443) and HTTP/3 over quinn (:8443/udp) run tako's multi-thread Server. Routing, Path/Query/State/body extractors and the Json responder through the tako API, the compression plugin with default settings on the JSON routes, ServeDir with precompressed .br/.gz sidecars read from disk per request, TakoWs for /ws, and deadpool-postgres for async-db.", + "description": "Tako 2.7.0 on hyper and Tokio, default configuration. Plain HTTP/1.1 and WebSocket on :8080 run tako's per-thread server (one current-thread runtime per CPU, SO_REUSEPORT listeners, PerThreadConfig::default()); h2c (:8082), HTTP/1.1 and h2 over rustls (:8081, :8443) and HTTP/3 over quinn (:8443/udp) run tako's multi-thread Server. Routing, Path/Query/State/body extractors and the Json responder through the tako API, the compression plugin with default settings on the JSON routes, ServeDir with precompressed .br/.gz sidecars read from disk per request, WebSocketUpgrade for /ws, and deadpool-postgres for async-db.", "repo": "https://github.com/rust-dd/tako", "type": "emerging", "engine": "hyper", @@ -2532,7 +2532,7 @@ }, { "dir": "tako-grpc", - "description": "Tako 2.4.1 gRPC: benchmark.BenchmarkService/GetSum as a plain tako route with the GrpcRequest extractor and GrpcResponse responder, prost messages. h2c on :8080 and h2 over rustls on :8443, both on tako's multi-thread Server with default configuration.", + "description": "Tako 2.7.0 gRPC: benchmark.BenchmarkService/GetSum as a plain tako route with the GrpcRequest extractor and GrpcResponse responder, prost messages. h2c on :8080 and h2 over rustls on :8443, both on tako's multi-thread Server with default configuration.", "repo": "https://github.com/rust-dd/tako", "type": "emerging", "engine": "hyper", @@ -2748,14 +2748,6 @@ "type": "engine", "engine": "veb" }, - "veil": { - "dir": "veil", - "description": "veil 0.10 reverse proxy from its published container image (full-container build: epoll readiness reactor, no tokio), one process with four listeners. A Proxy-Wasm module loaded into veil answers /baseline11 (including the POST body), /baseline2, /pipeline and /json/{count}, serializing the JSON body per request from the dataset veil loads at startup; /static/* is veil's File backend (sendfile, on-the-fly compression on every protocol, caches that follow the disk). TLS via rustls: HTTP/1.1 on :8081, h2 and h3 (quiche) on :8443, h2c on :8082.", - "repo": "https://github.com/aofusa/veil-proxy", - "type": "infrastructure", - "engine": "veil", - "website": "https://github.com/aofusa/veil-proxy" - }, "veil-api": { "dir": "veil-api", "description": "veil 0.10 API library (crates.io veil-proxy, full-container: epoll readiness reactor, no tokio or hyper). #[veil::route] handlers with path/query extraction, serde_json responses, compression through [api] compression, WebSocket through WebSocketUpgrade, gRPC services generated by veil-build (prost), /static/* through App::serve_dir. One process with four listeners from the configuration file; HTTP/3 via quiche.", @@ -2783,6 +2775,14 @@ "middleware": false } }, + "veil": { + "dir": "veil", + "description": "veil 0.10 reverse proxy from its published container image (full-container build: epoll readiness reactor, no tokio), one process with four listeners. A Proxy-Wasm module loaded into veil answers /baseline11 (including the POST body), /baseline2, /pipeline and /json/{count}, serializing the JSON body per request from the dataset veil loads at startup; /static/* is veil's File backend (sendfile, on-the-fly compression on every protocol, caches that follow the disk). TLS via rustls: HTTP/1.1 on :8081, h2 and h3 (quiche) on :8443, h2c on :8082.", + "repo": "https://github.com/aofusa/veil-proxy", + "type": "infrastructure", + "engine": "veil", + "website": "https://github.com/aofusa/veil-proxy" + }, "velocity": { "dir": "velocity", "description": "Velocity, a lightweight Express.js alternative on node:http with the cluster module for multi-core scaling.", diff --git a/site/data/results/tako.json b/site/data/results/tako.json index ccaf6fe1dc..984896faed 100644 --- a/site/data/results/tako.json +++ b/site/data/results/tako.json @@ -4,44 +4,44 @@ "8gbit-512": { "framework": "tako", "language": "Rust", - "rps": 49100, - "avg_latency": "102.1us", - "p99_latency": "136.0us", - "cpu": "236.7%", - "memory": "101MiB", + "rps": 48980, + "avg_latency": "95.6us", + "p99_latency": "138.0us", + "cpu": "253.5%", + "memory": "93MiB", "connections": 512, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "508.63MB/s", - "input_bw": "479.49MB/s", + "bandwidth": "507.39MB/s", + "input_bw": "478.32MB/s", "reconnects": 0, - "status_2xx": 245583, + "status_2xx": 244971, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "cpu_usec": 12989880, - "cpu_per_req_us": 52.8941, + "cpu_usec": 13042174, + "cpu_per_req_us": 53.2397, "target_rate": 50000, - "rate_ratio": 0.982, - "p99_9_latency": "5810.0us" + "rate_ratio": 0.9796, + "p99_9_latency": "2937.0us" }, "async-32000": { "framework": "tako", "language": "Rust", - "rps": 2244799, - "avg_latency": "14.17ms", - "p99_latency": "17.60ms", - "cpu": "5157.8%", - "memory": "969MiB", + "rps": 2235424, + "avg_latency": "14.21ms", + "p99_latency": "17.70ms", + "cpu": "5199.2%", + "memory": "966MiB", "connections": 32000, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "252.57MB/s", - "input_bw": "102.76MB/s", + "bandwidth": "251.52MB/s", + "input_bw": "102.33MB/s", "reconnects": 0, - "status_2xx": 22447998, + "status_2xx": 22354247, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -49,19 +49,19 @@ "async-db-1024": { "framework": "tako", "language": "Rust", - "rps": 242405, - "avg_latency": "3.69ms", - "p99_latency": "5.86ms", - "cpu": "3689.0%", - "memory": "97MiB", + "rps": 242075, + "avg_latency": "3.64ms", + "p99_latency": "5.96ms", + "cpu": "3681.6%", + "memory": "96MiB", "connections": 1024, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "925.20MB/s", - "input_bw": "16.18MB/s", - "reconnects": 96518, - "status_2xx": 2424052, + "bandwidth": "923.89MB/s", + "input_bw": "16.16MB/s", + "reconnects": 96468, + "status_2xx": 2420750, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -69,19 +69,19 @@ "baseline-4096": { "framework": "tako", "language": "Rust", - "rps": 3282244, - "avg_latency": "1.25ms", - "p99_latency": "2.44ms", - "cpu": "6384.8%", - "memory": "124MiB", + "rps": 3139709, + "avg_latency": "1.30ms", + "p99_latency": "2.60ms", + "cpu": "6652.2%", + "memory": "118MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "369.26MB/s", - "input_bw": "253.55MB/s", + "bandwidth": "353.23MB/s", + "input_bw": "242.54MB/s", "reconnects": 0, - "status_2xx": 16411220, + "status_2xx": 15698546, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -89,18 +89,18 @@ "baseline-h2-1024": { "framework": "tako", "language": "Rust", - "rps": 683442, - "avg_latency": "114.88ms", - "p99_latency": "468.97ms", - "cpu": "4674.6%", - "memory": "1.0GiB", + "rps": 724529, + "avg_latency": "102.12ms", + "p99_latency": "428.66ms", + "cpu": "4806.1%", + "memory": "566MiB", "connections": 1024, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "17.15MB/s", + "bandwidth": "18.18MB/s", "reconnects": 0, - "status_2xx": 3451387, + "status_2xx": 3658872, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -108,18 +108,18 @@ "baseline-h2-256": { "framework": "tako", "language": "Rust", - "rps": 740669, - "avg_latency": "34.19ms", - "p99_latency": "124.60ms", - "cpu": "4605.5%", - "memory": "354MiB", + "rps": 829933, + "avg_latency": "30.01ms", + "p99_latency": "127.79ms", + "cpu": "4755.5%", + "memory": "204MiB", "connections": 256, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "18.49MB/s", + "bandwidth": "20.71MB/s", "reconnects": 0, - "status_2xx": 3725567, + "status_2xx": 4174566, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -127,18 +127,18 @@ "baseline-h2c-1024": { "framework": "tako", "language": "Rust", - "rps": 666838, - "avg_latency": "104.74ms", - "p99_latency": "504.21ms", - "cpu": "4616.7%", - "memory": "1.2GiB", + "rps": 737895, + "avg_latency": "107.49ms", + "p99_latency": "671.76ms", + "cpu": "4758.3%", + "memory": "624MiB", "connections": 1024, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "16.73MB/s", + "bandwidth": "18.55MB/s", "reconnects": 0, - "status_2xx": 3367534, + "status_2xx": 3733752, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -146,18 +146,18 @@ "baseline-h2c-256": { "framework": "tako", "language": "Rust", - "rps": 744038, - "avg_latency": "34.20ms", - "p99_latency": "135.70ms", - "cpu": "4631.5%", - "memory": "393MiB", + "rps": 848574, + "avg_latency": "30.19ms", + "p99_latency": "142.93ms", + "cpu": "4676.3%", + "memory": "176MiB", "connections": 256, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "18.57MB/s", + "bandwidth": "21.18MB/s", "reconnects": 0, - "status_2xx": 3742513, + "status_2xx": 4268331, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -165,18 +165,18 @@ "baseline-h2c-4096": { "framework": "tako", "language": "Rust", - "rps": 656424, - "avg_latency": "264.33ms", - "p99_latency": "1.44s", - "cpu": "4842.2%", - "memory": "2.2GiB", + "rps": 0, + "avg_latency": "", + "p99_latency": "", + "cpu": "3869.5%", + "memory": "977MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "16.63MB/s", + "bandwidth": "0", "reconnects": 0, - "status_2xx": 3334638, + "status_2xx": 0, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -184,18 +184,18 @@ "baseline-h3-64": { "framework": "tako", "language": "Rust", - "rps": 540317, + "rps": 536729, "avg_latency": "0us", "p99_latency": "0us", - "cpu": "2847.7%", - "memory": "98MiB", + "cpu": "2928.1%", + "memory": "107MiB", "connections": 64, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "14.97MB/s", + "bandwidth": "14.87MB/s", "reconnects": 0, - "status_2xx": 2706992, + "status_2xx": 2689016, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -203,18 +203,18 @@ "echo-ws-16384": { "framework": "tako", "language": "Rust", - "rps": 3508819, - "avg_latency": "4.63ms", - "p99_latency": "6.24ms", - "cpu": "6126.4%", - "memory": "390MiB", + "rps": 3698229, + "avg_latency": "4.40ms", + "p99_latency": "7.60ms", + "cpu": "5989.3%", + "memory": "388MiB", "connections": 16384, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "23.93MB/s", + "bandwidth": "25.19MB/s", "reconnects": 0, - "status_2xx": 17544099, + "status_2xx": 18491145, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -222,18 +222,18 @@ "echo-ws-4096": { "framework": "tako", "language": "Rust", - "rps": 4062111, + "rps": 4066143, "avg_latency": "1.01ms", - "p99_latency": "1.08ms", - "cpu": "6445.6%", - "memory": "122MiB", + "p99_latency": "1.17ms", + "cpu": "6444.6%", + "memory": "121MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "27.11MB/s", + "bandwidth": "27.14MB/s", "reconnects": 0, - "status_2xx": 20310557, + "status_2xx": 20330715, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -241,18 +241,18 @@ "echo-ws-512": { "framework": "tako", "language": "Rust", - "rps": 4091198, - "avg_latency": "124us", - "p99_latency": "159us", - "cpu": "6229.5%", - "memory": "38MiB", + "rps": 4191358, + "avg_latency": "121us", + "p99_latency": "156us", + "cpu": "6465.1%", + "memory": "39MiB", "connections": 512, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "27.31MB/s", + "bandwidth": "27.97MB/s", "reconnects": 0, - "status_2xx": 20455991, + "status_2xx": 20956790, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -260,18 +260,18 @@ "echo-ws-limited-4096": { "framework": "tako", "language": "Rust", - "rps": 2543861, - "avg_latency": "1.09ms", - "p99_latency": "1.76ms", - "cpu": "6084.7%", - "memory": "139MiB", + "rps": 2570415, + "avg_latency": "1.12ms", + "p99_latency": "1.75ms", + "cpu": "6357.1%", + "memory": "136MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "57.28MB/s", - "reconnects": 1271617, - "status_2xx": 12719306, + "bandwidth": "57.82MB/s", + "reconnects": 1284692, + "status_2xx": 12852078, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -279,18 +279,18 @@ "echo-ws-limited-512": { "framework": "tako", "language": "Rust", - "rps": 2150090, - "avg_latency": "132us", - "p99_latency": "376us", - "cpu": "6021.7%", + "rps": 2156338, + "avg_latency": "136us", + "p99_latency": "390us", + "cpu": "5894.3%", "memory": "51MiB", "connections": 512, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "48.38MB/s", - "reconnects": 1075097, - "status_2xx": 10750454, + "bandwidth": "48.52MB/s", + "reconnects": 1078197, + "status_2xx": 10781690, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -298,18 +298,18 @@ "echo-ws-pipeline-16384": { "framework": "tako", "language": "Rust", - "rps": 5835140, - "avg_latency": "44.31ms", - "p99_latency": "45.90ms", - "cpu": "6242.5%", - "memory": "386MiB", + "rps": 50973465, + "avg_latency": "5.08ms", + "p99_latency": "11.20ms", + "cpu": "6328.4%", + "memory": "354MiB", "connections": 16384, "threads": 64, "duration": "5s", "pipeline": 16, - "bandwidth": "39.46MB/s", + "bandwidth": "340.71MB/s", "reconnects": 0, - "status_2xx": 29175700, + "status_2xx": 254867328, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -317,18 +317,18 @@ "echo-ws-pipeline-4096": { "framework": "tako", "language": "Rust", - "rps": 6007904, - "avg_latency": "10.91ms", - "p99_latency": "22.40ms", - "cpu": "6172.3%", - "memory": "120MiB", + "rps": 56360143, + "avg_latency": "1.16ms", + "p99_latency": "1.65ms", + "cpu": "6592.1%", + "memory": "122MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 16, - "bandwidth": "40.11MB/s", + "bandwidth": "376.14MB/s", "reconnects": 0, - "status_2xx": 30039522, + "status_2xx": 281800719, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -336,18 +336,18 @@ "echo-ws-pipeline-512": { "framework": "tako", "language": "Rust", - "rps": 5752782, - "avg_latency": "1.42ms", - "p99_latency": "1.83ms", - "cpu": "6461.9%", + "rps": 57721923, + "avg_latency": "141us", + "p99_latency": "240us", + "cpu": "6603.9%", "memory": "40MiB", "connections": 512, "threads": 64, "duration": "5s", "pipeline": 16, - "bandwidth": "38.39MB/s", + "bandwidth": "385.12MB/s", "reconnects": 0, - "status_2xx": 28763911, + "status_2xx": 288609616, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -451,19 +451,19 @@ "json-comp-16384": { "framework": "tako", "language": "Rust", - "rps": 208614, - "avg_latency": "77.60ms", - "p99_latency": "150.00ms", - "cpu": "6430.2%", - "memory": "593MiB", + "rps": 201312, + "avg_latency": "80.23ms", + "p99_latency": "142.20ms", + "cpu": "6485.6%", + "memory": "594MiB", "connections": 16384, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "255.58MB/s", - "input_bw": "15.52MB/s", - "reconnects": 32772, - "status_2xx": 1043071, + "bandwidth": "246.61MB/s", + "input_bw": "14.97MB/s", + "reconnects": 32898, + "status_2xx": 1006560, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -471,19 +471,19 @@ "json-comp-4096": { "framework": "tako", "language": "Rust", - "rps": 211108, - "avg_latency": "19.37ms", - "p99_latency": "28.30ms", - "cpu": "6481.9%", + "rps": 208402, + "avg_latency": "19.65ms", + "p99_latency": "31.50ms", + "cpu": "6229.1%", "memory": "245MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "258.61MB/s", - "input_bw": "15.70MB/s", - "reconnects": 41067, - "status_2xx": 1055540, + "bandwidth": "255.29MB/s", + "input_bw": "15.50MB/s", + "reconnects": 40817, + "status_2xx": 1042014, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -491,18 +491,18 @@ "json-h2c-1024": { "framework": "tako", "language": "Rust", - "rps": 723155, - "avg_latency": "33.12ms", - "p99_latency": "545.11ms", - "cpu": "4403.9%", - "memory": "403MiB", + "rps": 602946, + "avg_latency": "42.56ms", + "p99_latency": "373.07ms", + "cpu": "4047.6%", + "memory": "551MiB", "connections": 1024, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "2.41GB/s", + "bandwidth": "2.01GB/s", "reconnects": 0, - "status_2xx": 3651937, + "status_2xx": 3044882, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -510,18 +510,18 @@ "json-h2c-4096": { "framework": "tako", "language": "Rust", - "rps": 621319, - "avg_latency": "146.64ms", - "p99_latency": "3.97s", - "cpu": "4323.8%", - "memory": "1.2GiB", + "rps": 531594, + "avg_latency": "131.47ms", + "p99_latency": "2.37s", + "cpu": "3839.8%", + "memory": "1.4GiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "2.09GB/s", + "bandwidth": "1.78GB/s", "reconnects": 0, - "status_2xx": 3168730, + "status_2xx": 2705816, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -529,18 +529,18 @@ "json-tls-4096": { "framework": "tako", "language": "Rust", - "rps": 923194, - "avg_latency": "4.57ms", - "p99_latency": "40.63ms", - "cpu": "6425.3%", - "memory": "293MiB", + "rps": 1009044, + "avg_latency": "3.60ms", + "p99_latency": "40.05ms", + "cpu": "6068.4%", + "memory": "222MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "3.14GB", + "bandwidth": "3.43GB", "reconnects": 0, - "status_2xx": 4709022, + "status_2xx": 5145214, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -548,91 +548,91 @@ "latency-10k-1024": { "framework": "tako", "language": "Rust", - "rps": 9981, - "avg_latency": "73.1us", - "p99_latency": "106.0us", - "cpu": "26.0%", - "memory": "50MiB", + "rps": 9983, + "avg_latency": "69.0us", + "p99_latency": "108.0us", + "cpu": "25.8%", + "memory": "49MiB", "connections": 1024, "threads": 64, "duration": "20s", "pipeline": 1, "bandwidth": "1.17MB/s", "reconnects": 0, - "status_2xx": 199650, + "status_2xx": 199673, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "cpu_usec": 5394934, - "cpu_per_req_us": 27.022, + "cpu_usec": 5388405, + "cpu_per_req_us": 26.9861, "target_rate": 10000, - "rate_ratio": 0.9981, - "p99_9_latency": "164.0us" + "rate_ratio": 0.9983, + "p99_9_latency": "146.0us" }, "latency-1m-1024": { "framework": "tako", "language": "Rust", - "rps": 997555, - "avg_latency": "101.5us", - "p99_latency": "177.0us", - "cpu": "2678.2%", - "memory": "51MiB", + "rps": 997908, + "avg_latency": "103.4us", + "p99_latency": "184.0us", + "cpu": "2672.1%", + "memory": "52MiB", "connections": 1024, "threads": 64, "duration": "20s", "pipeline": 1, - "bandwidth": "116.71MB/s", + "bandwidth": "116.76MB/s", "reconnects": 0, - "status_2xx": 19954298, + "status_2xx": 19961584, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "cpu_usec": 485421811, - "cpu_per_req_us": 24.3267, + "cpu_usec": 485410622, + "cpu_per_req_us": 24.3172, "target_rate": 1000000, - "rate_ratio": 0.9976, - "p99_9_latency": "1192.0us" + "rate_ratio": 0.9979, + "p99_9_latency": "2637.0us" }, "latency-500k-8cpu-1024": { "framework": "tako", "language": "Rust", - "rps": 460673, - "avg_latency": "768077.7us", - "p99_latency": "1575424.0us", - "cpu": "799.0%", - "memory": "37MiB", + "rps": 455866, + "avg_latency": "854249.2us", + "p99_latency": "1794560.0us", + "cpu": "800.2%", + "memory": "33MiB", "connections": 1024, "threads": 64, "duration": "20s", "pipeline": 1, - "bandwidth": "53.90MB/s", + "bandwidth": "53.34MB/s", "reconnects": 0, - "status_2xx": 9216445, + "status_2xx": 9120268, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "cpu_usec": 159672169, - "cpu_per_req_us": 17.3247, + "cpu_usec": 159615063, + "cpu_per_req_us": 17.5011, "target_rate": 500000, - "rate_ratio": 0.9213, - "p99_9_latency": "1635840.0us" + "rate_ratio": 0.9117, + "p99_9_latency": "1925632.0us" }, "limited-conn-4096": { "framework": "tako", "language": "Rust", - "rps": 2461724, - "avg_latency": "1.63ms", - "p99_latency": "3.16ms", - "cpu": "6169.4%", + "rps": 2407752, + "avg_latency": "1.67ms", + "p99_latency": "3.29ms", + "cpu": "6144.4%", "memory": "154MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "276.85MB/s", - "input_bw": "190.16MB/s", - "reconnects": 1230602, - "status_2xx": 12308624, + "bandwidth": "270.83MB/s", + "input_bw": "185.99MB/s", + "reconnects": 1202934, + "status_2xx": 12038763, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -640,18 +640,18 @@ "pipelined-4096": { "framework": "tako", "language": "Rust", - "rps": 22398380, - "avg_latency": "2.92ms", - "p99_latency": "3.00ms", - "cpu": "6478.8%", - "memory": "125MiB", + "rps": 20509219, + "avg_latency": "3.20ms", + "p99_latency": "6.06ms", + "cpu": "6354.0%", + "memory": "126MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 16, - "bandwidth": "2.46GB/s", + "bandwidth": "2.25GB/s", "reconnects": 0, - "status_2xx": 111991904, + "status_2xx": 102546096, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -707,18 +707,18 @@ "static-h2-1024": { "framework": "tako", "language": "Rust", - "rps": 126274, - "avg_latency": "219.32ms", - "p99_latency": "629.64ms", - "cpu": "2703.2%", + "rps": 130233, + "avg_latency": "223.43ms", + "p99_latency": "581.57ms", + "cpu": "2740.0%", "memory": "1.0GiB", "connections": 1024, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "1.94GB/s", + "bandwidth": "2.00GB/s", "reconnects": 0, - "status_2xx": 638951, + "status_2xx": 657677, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -726,18 +726,18 @@ "static-h2-256": { "framework": "tako", "language": "Rust", - "rps": 128975, - "avg_latency": "62.18ms", - "p99_latency": "325.80ms", - "cpu": "3132.5%", - "memory": "335MiB", + "rps": 131413, + "avg_latency": "61.09ms", + "p99_latency": "306.30ms", + "cpu": "3003.9%", + "memory": "318MiB", "connections": 256, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "1.97GB/s", + "bandwidth": "2.01GB/s", "reconnects": 0, - "status_2xx": 648747, + "status_2xx": 661009, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -745,18 +745,18 @@ "static-h3-64": { "framework": "tako", "language": "Rust", - "rps": 116563, + "rps": 107569, "avg_latency": "0us", "p99_latency": "0us", - "cpu": "2909.9%", - "memory": "148MiB", + "cpu": "2817.0%", + "memory": "175MiB", "connections": 64, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "1.78GB/s", + "bandwidth": "1.64GB/s", "reconnects": 0, - "status_2xx": 583985, + "status_2xx": 538925, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -764,18 +764,18 @@ "static-tls-1024": { "framework": "tako", "language": "Rust", - "rps": 134333, - "avg_latency": "6.99ms", - "p99_latency": "47.10ms", - "cpu": "2944.7%", - "memory": "162MiB", + "rps": 140842, + "avg_latency": "6.71ms", + "p99_latency": "39.48ms", + "cpu": "3229.7%", + "memory": "157MiB", "connections": 1024, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "2.07GB", + "bandwidth": "2.17GB", "reconnects": 0, - "status_2xx": 685054, + "status_2xx": 718306, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -783,18 +783,18 @@ "unary-grpc-1024": { "framework": "tako", "language": "Rust", - "rps": 513618, - "avg_latency": "163.90ms", - "p99_latency": "656.27ms", - "cpu": "5034.0%", - "memory": "1.1GiB", + "rps": 510326, + "avg_latency": "155.19ms", + "p99_latency": "673.87ms", + "cpu": "5211.5%", + "memory": "720MiB", "connections": 1024, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "18.84MB/s", + "bandwidth": "18.71MB/s", "reconnects": 0, - "status_2xx": 2593775, + "status_2xx": 2577147, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -802,18 +802,18 @@ "unary-grpc-256": { "framework": "tako", "language": "Rust", - "rps": 578019, - "avg_latency": "44.16ms", - "p99_latency": "192.48ms", - "cpu": "5026.6%", - "memory": "359MiB", + "rps": 554095, + "avg_latency": "44.86ms", + "p99_latency": "190.10ms", + "cpu": "5116.2%", + "memory": "196MiB", "connections": 256, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "21.08MB/s", + "bandwidth": "20.17MB/s", "reconnects": 0, - "status_2xx": 2907440, + "status_2xx": 2781557, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -821,18 +821,18 @@ "unary-grpc-tls-1024": { "framework": "tako", "language": "Rust", - "rps": 508813, - "avg_latency": "146.65ms", - "p99_latency": "589.95ms", - "cpu": "4979.4%", - "memory": "1.0GiB", + "rps": 485284, + "avg_latency": "147.51ms", + "p99_latency": "655.26ms", + "cpu": "5040.5%", + "memory": "582MiB", "connections": 1024, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "18.70MB/s", + "bandwidth": "17.84MB/s", "reconnects": 0, - "status_2xx": 2574598, + "status_2xx": 2455540, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -840,18 +840,18 @@ "unary-grpc-tls-256": { "framework": "tako", "language": "Rust", - "rps": 622537, - "avg_latency": "34.90ms", - "p99_latency": "170.64ms", - "cpu": "4906.5%", - "memory": "318MiB", + "rps": 564324, + "avg_latency": "44.62ms", + "p99_latency": "176.46ms", + "cpu": "5267.4%", + "memory": "220MiB", "connections": 256, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "22.66MB/s", + "bandwidth": "20.58MB/s", "reconnects": 0, - "status_2xx": 3125138, + "status_2xx": 2838553, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 From 6b2d03b72c51c175b0b8ab489a6a27c3836bd009 Mon Sep 17 00:00:00 2001 From: Daniel Boros Date: Fri, 13 Nov 2026 16:09:52 +0100 Subject: [PATCH 03/12] fix: reject invalid HTTP/2 and HTTP/3 benchmark results Preserve load-generator failures, validate completed requests, and suppress stale composite scores after failed runs. Update h2load for duration-mode shutdown, fix auth sidecar build caching, and cover both benchmark runners with regression tests. --- .github/workflows/benchmark-pr.yml | 4 + .github/workflows/benchmark-tools.yml | 31 +++++++ docker/h2load.Dockerfile | 34 +++---- frameworks/_shared/authsvc/Dockerfile | 6 +- scripts/benchmark-lite.sh | 43 ++++++--- scripts/benchmark.sh | 55 +++++++++--- scripts/composite_delta.py | 15 +++- scripts/lib/common.sh | 2 +- scripts/lib/tools/h2load-h3.sh | 30 +------ scripts/lib/tools/h2load.sh | 118 ++++++------------------ scripts/lib/tools/h2load_parse.py | 91 +++++++++++++++++++ scripts/tests/test_authsvc_image.py | 27 ++++++ scripts/tests/test_benchmark_run.py | 89 ++++++++++++++++++ scripts/tests/test_composite_status.py | 52 +++++++++++ scripts/tests/test_h2load.py | 119 +++++++++++++++++++++++++ scripts/tests/test_h2load_image.py | 52 +++++++++++ 16 files changed, 607 insertions(+), 161 deletions(-) create mode 100644 .github/workflows/benchmark-tools.yml create mode 100644 scripts/lib/tools/h2load_parse.py create mode 100644 scripts/tests/test_authsvc_image.py create mode 100644 scripts/tests/test_benchmark_run.py create mode 100644 scripts/tests/test_composite_status.py create mode 100644 scripts/tests/test_h2load.py create mode 100644 scripts/tests/test_h2load_image.py diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index b0ada77245..c72817b007 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -376,6 +376,8 @@ jobs: return "".join(f.readlines()[-n:]) def composite(fw): + if fw in failed: + return "\n\nComposite score unavailable: some measurements failed; retained results are not a complete new run." text = tail(logs_dir + "/" + fw + ".composite.md", 50).strip() return "\n\n" + text if text else "" @@ -387,6 +389,8 @@ jobs: rows = parse_rows(logs_dir + "/" + fw + ".log", fw) deltas = parse_deltas(logs_dir + "/" + fw + ".compare.md") head = table(rows, deltas) + composite(fw) if rows else "No results captured" + if fw in failed: + head = "**Benchmark failed; valid partial results follow.**\n\n" + head body = head + "\n\n" + details("Full log", tail(logs_dir + "/" + fw + ".log", 200)) else: heads = [] diff --git a/.github/workflows/benchmark-tools.yml b/.github/workflows/benchmark-tools.yml new file mode 100644 index 0000000000..e34bb84455 --- /dev/null +++ b/.github/workflows/benchmark-tools.yml @@ -0,0 +1,31 @@ +name: Benchmark tool checks + +on: + pull_request: + paths: + - .github/workflows/benchmark-tools.yml + - frameworks/_shared/authsvc/** + - docker/h2load.Dockerfile + - scripts/benchmark.sh + - scripts/benchmark-lite.sh + - scripts/composite_delta.py + - scripts/lib/common.sh + - scripts/lib/tools/h2load* + - scripts/tests/** + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check shell syntax + run: bash -n scripts/benchmark.sh scripts/benchmark-lite.sh scripts/lib/common.sh scripts/lib/tools/h2load.sh scripts/lib/tools/h2load-h3.sh + - name: Build auth sidecar + run: docker build -t httparena-authsvc:test frameworks/_shared/authsvc + - name: Build HTTP/2 load generator + run: docker build -t httparena-h2load:test -f docker/h2load.Dockerfile docker + - name: Test parsers, exit codes and images + env: + HTTPARENA_TEST_AUTHSVC_IMAGE: httparena-authsvc:test + HTTPARENA_TEST_H2LOAD_IMAGE: httparena-h2load:test + run: python3 -B -m unittest discover -s scripts/tests -p 'test_*.py' -v diff --git a/docker/h2load.Dockerfile b/docker/h2load.Dockerfile index 1b53fee978..67dab9d5bc 100644 --- a/docker/h2load.Dockerfile +++ b/docker/h2load.Dockerfile @@ -1,20 +1,22 @@ -# h2load — HTTP/2 load generator from the nghttp2 project. -# -# Uses Ubuntu + apt-installed `nghttp2-client` (glibc build) instead of -# alpine's musl build. The alpine h2load suffers ~20-40% throughput loss -# on high-QPS workloads because of musl's per-call malloc overhead and -# thread primitives — unacceptable for a benchmark load generator. -# -# Self-contained: installs nghttp2 directly from the Ubuntu archive during -# the build; no source checkout, no local file dependencies. - -FROM ubuntu:24.04 +# nghttp2 1.69 includes the duration-mode connection shutdown fix in PR #2613. +# Keep glibc and jemalloc for the high-throughput load generator. +FROM ubuntu:24.04 AS builder ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - nghttp2-client ca-certificates \ - && apt-get clean \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates git cmake ninja-build g++ pkg-config \ + libssl-dev libev-dev libjemalloc-dev zlib1g-dev \ && rm -rf /var/lib/apt/lists/* - +WORKDIR /src +RUN git clone --depth 1 --branch v1.69.0 --recurse-submodules --shallow-submodules https://github.com/nghttp2/nghttp2.git \ + && cmake -S nghttp2 -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=20 \ + -DENABLE_APP=ON -DENABLE_EXAMPLES=OFF -DENABLE_HPACK_TOOLS=OFF \ + -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON \ + && cmake --build build --target h2load -j 4 +FROM ubuntu:24.04 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates libev4t64 libjemalloc2 libssl3t64 zlib1g \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /src/build/src/h2load /usr/local/bin/h2load ENTRYPOINT ["h2load"] diff --git a/frameworks/_shared/authsvc/Dockerfile b/frameworks/_shared/authsvc/Dockerfile index bd5ecae961..e2676d43eb 100644 --- a/frameworks/_shared/authsvc/Dockerfile +++ b/frameworks/_shared/authsvc/Dockerfile @@ -9,11 +9,13 @@ RUN apk add --no-cache musl-dev pkgconfig # Prime the dep cache: build against an empty main.rs so cargo downloads # and compiles the dependency graph once. Actual src/ changes below don't -# invalidate this layer. +# invalidate this layer. Let Cargo remove the placeholder's artifacts so this +# also works when its target-directory layout changes. COPY Cargo.toml . RUN mkdir src && echo 'fn main() {}' > src/main.rs && \ cargo build --release && \ - rm -rf src target/release/deps/authsvc* + cargo clean --release --package authsvc && \ + rm -rf src # Now add the real source and rebuild. COPY src ./src diff --git a/scripts/benchmark-lite.sh b/scripts/benchmark-lite.sh index 796e2d21aa..9b05684bed 100755 --- a/scripts/benchmark-lite.sh +++ b/scripts/benchmark-lite.sh @@ -220,11 +220,11 @@ import json; print(str(json.load(open('$meta')).get('enabled', True)).lower())" return 0 fi - framework_build + framework_build || return 1 local need_pg=false if framework_subscribes_to async-db; then need_pg=true; fi - if $need_pg; then postgres_start; fi + if $need_pg; then postgres_start || return 1; fi local profiles_to_run if [ "${#PROFILE_LIST[@]}" -gt 0 ]; then @@ -233,7 +233,7 @@ import json; print(str(json.load(open('$meta')).get('enabled', True)).lower())" profiles_to_run=("${PROFILE_ORDER[@]}") fi - local profile + local profile failed_profiles=0 for profile in "${profiles_to_run[@]}"; do [ -n "${PROFILES[$profile]+x}" ] || { warn "unknown profile: $profile"; continue; } framework_subscribes_to "$profile" || { info "skip: $fw does not subscribe to $profile"; continue; } @@ -242,7 +242,7 @@ import json; print(str(json.load(open('$meta')).get('enabled', True)).lower())" IFS=',' read -ra CONN_COUNTS <<< "$PROF_CONNS" local CONNS for CONNS in "${CONN_COUNTS[@]}"; do - run_one "$profile" "$CONNS" || continue + run_one "$profile" "$CONNS" || failed_profiles=$((failed_profiles + 1)) done done @@ -250,6 +250,7 @@ import json; print(str(json.load(open('$meta')).get('enabled', True)).lower())" # statement, the &&-list would make run_framework return 1 for non-DB # frameworks and kill the script under `set -e` after a successful run. if $need_pg; then postgres_stop; fi + [ "$failed_profiles" -eq 0 ] } # ── Single (profile, conns) iteration ────────────────────────────────────── @@ -264,8 +265,9 @@ run_one() { tool=$(endpoint_tool "$endpoint") banner "$FRAMEWORK / $profile / ${CONNS}c (tool=$tool)" + if [ "$tool" = h2load ]; then h2load_check_version || return 1; fi - framework_start "$endpoint" "$PROF_CPU" + framework_start "$endpoint" "$PROF_CPU" || return 1 if ! framework_wait_ready "$endpoint"; then warn "$FRAMEWORK did not come up for $profile; skipping" dump_container_logs "$CONTAINER_NAME" "$FRAMEWORK" @@ -287,19 +289,28 @@ run_one() { echo ""; echo "[run $run/$RUNS]" stats_start "$CONTAINER_NAME" - local output - output=$("${tool//-/_}_run" "${gc_args[@]}") + local output run_exit=0 metrics + output=$("${tool//-/_}_run" "${gc_args[@]}") || run_exit=$? stats_stop echo "$output" | grep -Ev '^(Warm-up|Main benchmark duration|Stopped all clients|progress: [0-9]+% of clients started|spawning thread #[0-9]+|[0-9]*Warm-up phase is over for thread #[0-9]+)' || true info "CPU $STATS_AVG_CPU | Mem $STATS_PEAK_MEM" [ -n "$STATS_BREAKDOWN" ] && info " $STATS_BREAKDOWN" + if [ "$run_exit" -ne 0 ]; then + warn "run $run failed: load generator exited with status $run_exit" + continue + fi + if ! metrics=$("${tool//-/_}_parse" "$endpoint" "$output"); then + warn "run $run failed: invalid load-generator output" + continue + fi + declare -A m=() local line while IFS= read -r line; do [[ "$line" == *=* ]] && m["${line%%=*}"]="${line#*=}" - done < <("${tool//-/_}_parse" "$endpoint" "$output") + done <<< "$metrics" local rps_int=${m[rps]:-0} if [ "$rps_int" -gt "$best_rps" ] 2>/dev/null; then @@ -313,6 +324,12 @@ run_one() { sleep 2 done + if [ "$best_rps" -lt 0 ]; then + warn "$profile/$CONNS: no valid measurements; no result saved" + framework_stop + return 1 + fi + echo ""; echo "=== Best: ${best_rps} req/s (CPU: $best_cpu, Mem: $best_mem) ===" if [ "$SAVE_RESULTS" = "true" ]; then @@ -353,6 +370,7 @@ save_result() { "rps": $best_rps, "avg_latency": "${BEST_M[avg_lat]:-}", "p99_latency": "${BEST_M[p99_lat]:-}", + "max_latency": "${BEST_M[max_lat]:-}", "cpu": "$best_cpu", "memory": "$best_mem", "connections": $CONNS, @@ -377,15 +395,16 @@ EOF # ── Driver: either one framework, or loop through every enabled one ──────── FRAMEWORK="" +failed_frameworks=0 if [ -z "$FRAMEWORK_ARG" ]; then info "no framework argument — running every enabled framework" for fw_dir in "$ROOT_DIR"/frameworks/*/; do [ -d "$fw_dir" ] || continue fw=$(basename "$fw_dir") - run_framework "$fw" || warn "$fw run failed, continuing" + run_framework "$fw" || { warn "$fw run failed, continuing"; failed_frameworks=$((failed_frameworks + 1)); } done else - run_framework "$FRAMEWORK_ARG" + run_framework "$FRAMEWORK_ARG" || failed_frameworks=$((failed_frameworks + 1)) fi if [ "$SAVE_RESULTS" = "true" ]; then @@ -393,4 +412,8 @@ if [ "$SAVE_RESULTS" = "true" ]; then python3 "$SCRIPT_DIR/rebuild_site_data.py" --root "$ROOT_DIR" fi +if [ "$failed_frameworks" -gt 0 ]; then + warn "$failed_frameworks framework runs failed" + exit 1 +fi info "done" diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 642846464b..84037cb6c6 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -48,6 +48,11 @@ PROFILE_FILTER="${POSITIONAL[1]:-}" [ -n "$FRAMEWORK_ARG" ] || fail "usage: benchmark.sh [profile[,profile...]] [--save]" +run_status_dir="${HTTPARENA_RUN_STATUS_DIR:-$RESULTS_DIR/run-status}" +mkdir -p "$run_status_dir" +run_status_file="$run_status_dir/$FRAMEWORK_ARG.exit" +printf 'running\n' > "$run_status_file" + # The profile filter is one profile or a comma-separated list (`-t a,b` from a # PR comment lands here as "a,b"), run in the order given, duplicates dropped. # A name that is not a profile fails now, before any image is built or the @@ -90,11 +95,14 @@ cleanup_all() { docker volume prune -f >/dev/null 2>&1 || true docker image prune -f >/dev/null 2>&1 || true } -if [ "${SKIP_TUNE:-}" != "true" ]; then - trap 'cleanup_all; system_restore' EXIT -else - trap 'cleanup_all' EXIT -fi +finish_run() { + local status=$? + cleanup_all || status=1 + if [ "${SKIP_TUNE:-}" != "true" ]; then system_restore || status=1; fi + printf '%s\n' "$status" > "$run_status_file" + exit "$status" +} +trap finish_run EXIT # Clean slate: stop any leftover benchmark containers from a previous # crashed run, AND prune any leftover dangling volumes/images from the @@ -217,7 +225,7 @@ if $need_pg; then postgres_start; fi # the builds and tuning above. # run_one — single (profile, conns) iteration. Returns non-zero if the -# server failed to start; main loop skips to the next profile in that case. +# server failed to start or no valid measurement completed. run_one() { local profile="$1" CONNS="$2" parse_profile "${PROFILES[$profile]}" @@ -226,6 +234,9 @@ run_one() { tool=$(endpoint_tool "$endpoint") banner "$FRAMEWORK / $profile / ${CONNS}c (tool=$tool)" + if [ "$tool" = h2load ]; then + h2load_check_version || return 1 + fi # Reset Postgres before each DB profile so it sees the same clean, # freshly-seeded server a standalone `benchmark.sh ` run gets. @@ -302,12 +313,12 @@ run_one() { stats_start "$CONTAINER_NAME" fi - local output + local output run_exit=0 metrics # Exact cgroup CPU across exactly the window the load is applied in. # Cheap enough to take on every profile — two file reads — and it is # the measurement on the fixed-rate profiles rather than context. cpu_acct_start "$CONTAINER_NAME" - output=$("${tool//-/_}_run" "${gc_args[@]}") + output=$("${tool//-/_}_run" "${gc_args[@]}") || run_exit=$? cpu_acct_stop stats_stop @@ -316,12 +327,21 @@ run_one() { info "CPU $STATS_AVG_CPU | Mem $STATS_PEAK_MEM" [ -n "$STATS_BREAKDOWN" ] && info " $STATS_BREAKDOWN" + if [ "$run_exit" -ne 0 ]; then + warn "run $run failed: load generator exited with status $run_exit" + continue + fi + if ! metrics=$("${tool//-/_}_parse" "$endpoint" "$output"); then + warn "run $run failed: invalid load-generator output" + continue + fi + # Parse into an associative array. declare -A m=() local line while IFS= read -r line; do [[ "$line" == *=* ]] && m["${line%%=*}"]="${line#*=}" - done < <("${tool//-/_}_parse" "$endpoint" "$output") + done <<< "$metrics" local rps_int=${m[rps]:-0} @@ -340,7 +360,7 @@ run_one() { fi if [ -n "$_mdir" ]; then - "${tool//-/_}_parse" "$endpoint" "$output" > "$_mdir/$run.kv" + printf '%s\n' "$metrics" > "$_mdir/$run.kv" printf '%s' "$output" > "$_mdir/$run.out" { printf 'cpu_usec=%s\n' "${CPU_ACCT_USEC:-}" @@ -372,6 +392,13 @@ run_one() { if [ "$CONNS" -ge 32768 ]; then sleep 15; else sleep 2; fi done + if [ "$best_rps" -lt 0 ]; then + warn "$profile/$CONNS: no valid measurements; no result saved" + [ -z "$_mdir" ] || rm -rf "$_mdir" + if $is_gateway; then gateway_down; else framework_stop; fi + return 1 + fi + # ── Latency-1M: pick the run by score, not by any single metric ───── # # The published score weights rate, CPU, mean and p99 latency together, so @@ -584,6 +611,7 @@ save_result() { "rps": $best_rps, "avg_latency": "${BEST_M[avg_lat]:-}", "p99_latency": "${BEST_M[p99_lat]:-}", + "max_latency": "${BEST_M[max_lat]:-}", "cpu": "$best_cpu", "memory": "$best_mem", "connections": $CONNS, @@ -630,6 +658,7 @@ EOF # Iterate profiles × conns. declare -A BEST_M +failed_profiles=0 for profile in "${profiles_to_run[@]}"; do if [ -z "${PROFILES[$profile]+x}" ]; then warn "unknown profile: $profile" @@ -640,7 +669,7 @@ for profile in "${profiles_to_run[@]}"; do parse_profile "${PROFILES[$profile]}" IFS=',' read -ra CONN_COUNTS <<< "$PROF_CONNS" for CONNS in "${CONN_COUNTS[@]}"; do - run_one "$profile" "$CONNS" || continue + run_one "$profile" "$CONNS" || failed_profiles=$((failed_profiles + 1)) done done @@ -651,4 +680,8 @@ if [ "$SAVE_RESULTS" = "true" ]; then python3 "$SCRIPT_DIR/rebuild_site_data.py" --root "$ROOT_DIR" fi +if [ "$failed_profiles" -gt 0 ]; then + warn "$failed_profiles profile/connection measurements failed" + exit 1 +fi info "done" diff --git a/scripts/composite_delta.py b/scripts/composite_delta.py index 9fd1f7badd..7d9983deb7 100644 --- a/scripts/composite_delta.py +++ b/scripts/composite_delta.py @@ -16,6 +16,7 @@ """ from __future__ import annotations +import os import sys from pathlib import Path @@ -54,7 +55,7 @@ def replaced(main, run, names): def display_name(frameworks, fw_dir): for name, m in frameworks.items(): - if m.get("dir") == fw_dir: + if m.get("dir") == fw_dir or any(v.get("dir") == fw_dir for v in m.get("variants", [])): return name return fw_dir @@ -85,6 +86,12 @@ def main(): main_fws = g.load("frameworks.json", main_data) or {} run_fws = g.load("frameworks.json", run_data) or {} names = {display_name(run_fws, d) for d in sys.argv[4:]} + status_dir = Path(os.environ.get("HTTPARENA_RUN_STATUS_DIR", Path(__file__).resolve().parents[1] / "results/run-status")) + incomplete = set() + for fw_dir in sys.argv[4:]: + status = status_dir / f"{fw_dir}.exit" + if not status.is_file() or status.read_text().strip() != "0": + incomplete.add(display_name(run_fws, fw_dir)) main_index = g.load_results(main_data) before = standings(main_fws, main_index) after_fws = {n: m for n, m in main_fws.items() if n not in names} @@ -93,7 +100,11 @@ def main(): out_dir.mkdir(parents=True, exist_ok=True) for fw_dir in sys.argv[4:]: name = display_name(run_fws, fw_dir) - if name in before or name in after: + if name in incomplete: + (out_dir / f"{fw_dir}.composite.md").write_text( + "Composite score unavailable: related measurements failed or did not finish; retained results are not a complete new run.\n", + encoding="utf-8") + elif name in before or name in after: (out_dir / f"{fw_dir}.composite.md").write_text( table(before.get(name, {}), after.get(name, {})) + "\n", encoding="utf-8") diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index dae3580c23..eaebfbd42d 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -55,7 +55,7 @@ fi export GCANNON_CPUS H2LOAD="${H2LOAD:-h2load}" -H2LOAD_IMAGE="${H2LOAD_IMAGE:-h2load:latest}" +H2LOAD_IMAGE="${H2LOAD_IMAGE:-h2load:1.69.0}" H2LOAD_H3="${H2LOAD_H3:-h2load-h3}" H2LOAD_H3_IMAGE="${H2LOAD_H3_IMAGE:-h2load-h3:local}" diff --git a/scripts/lib/tools/h2load-h3.sh b/scripts/lib/tools/h2load-h3.sh index 75ac4eb535..05a402eb35 100644 --- a/scripts/lib/tools/h2load-h3.sh +++ b/scripts/lib/tools/h2load-h3.sh @@ -1,8 +1,8 @@ # scripts/lib/tools/h2load-h3.sh — h2load built with ngtcp2 for HTTP/3. # # Shares the same binary family as h2load but runs with --alpn-list=h3 to -# negotiate HTTP/3 over QUIC. Output format differs from plain h2load — the -# latency table uses "request :" instead of "time for request:". +# negotiate HTTP/3 over QUIC. The shared parser accepts both old summary +# tables and the newer tables with latency percentiles. : "${H2LOAD_H3_CMD:=}" @@ -49,33 +49,11 @@ h2load_h3_build_args() { } h2load_h3_run() { - timeout 45 taskset -c "$GCANNON_CPUS" "$@" 2>&1 || true + timeout 45 taskset -c "$GCANNON_CPUS" "$@" 2>&1 } # ── Parse output ──────────────────────────────────────────────────────────── h2load_h3_parse() { - local output="$2" # $1 = endpoint unused - - # rps from 2xx/duration — see h2load.sh for rationale (avoid inflating - # rps when the server is serving 4xx/5xx responses). - local duration_secs ok - duration_secs=$(echo "$output" | grep -oP 'finished in \K[\d.]+' | head -1) - duration_secs=${duration_secs:-1} - ok=$(echo "$output" | grep -oP '\d+(?= 2xx)' | head -1) - ok=${ok:-0} - echo "rps=$(awk -v ok="$ok" -v dur="$duration_secs" \ - 'BEGIN { if (dur+0 > 0) printf "%d", ok/dur; else print 0 }' 2>/dev/null || echo 0)" - - # h3 emits a tabular "request :" line with mean at column 8 and p99 at col 7. - echo "avg_lat=$(echo "$output" | awk '/^\s*request\s*:/ { print $8; exit }')" - echo "p99_lat=$(echo "$output" | awk '/^\s*request\s*:/ { print $7; exit }')" - - echo "reconnects=0" - echo "bandwidth=$(echo "$output" | grep -oP 'finished in [\d.]+s, [\d.]+ req/s, \K[\d.]+[KMGT]?B/s' | head -1 || echo 0)" - - echo "status_2xx=$(echo "$output" | grep -oP '\d+(?= 2xx)' | head -1 || echo 0)" - echo "status_3xx=$(echo "$output" | grep -oP '\d+(?= 3xx)' | head -1 || echo 0)" - echo "status_4xx=$(echo "$output" | grep -oP '\d+(?= 4xx)' | head -1 || echo 0)" - echo "status_5xx=$(echo "$output" | grep -oP '\d+(?= 5xx)' | head -1 || echo 0)" + python3 "${BASH_SOURCE[0]%/*}/h2load_parse.py" "$1" <<< "$2" } diff --git a/scripts/lib/tools/h2load.sh b/scripts/lib/tools/h2load.sh index f862b46bd2..62d33f630c 100644 --- a/scripts/lib/tools/h2load.sh +++ b/scripts/lib/tools/h2load.sh @@ -16,6 +16,21 @@ _h2load_cmd() { fi } +h2load_check_version() { + if [ "${H2LOAD_VERSION_CHECKED:-false}" = true ]; then return 0; fi + local -a cmd + local version + mapfile -t cmd < <(_h2load_cmd) + version=$("${cmd[@]}" --version) || return 1 + if [[ ! "$version" =~ nghttp2/([0-9]+)\.([0-9]+)\.([0-9]+) ]] \ + || (( BASH_REMATCH[1] < 1 || (BASH_REMATCH[1] == 1 && BASH_REMATCH[2] < 69) )); then + echo "h2load >= 1.69.0 is required: older clients can run forever after -D expires during connection setup. Update H2LOAD or build docker/h2load.Dockerfile." >&2 + return 1 + fi + echo "[info] $version" + H2LOAD_VERSION_CHECKED=true +} + # ── Build arguments ───────────────────────────────────────────────────────── h2load_build_args() { @@ -136,8 +151,9 @@ h2load_run() { if [ "$split_idx" -lt 0 ]; then # Single-invocation path — every profile except production-stack. - timeout 45 taskset -c "$GCANNON_CPUS" "${args[@]}" 2>&1 || true - return + local exit_code=0 + timeout 45 taskset -c "$GCANNON_CPUS" "${args[@]}" 2>&1 || exit_code=$? + return "$exit_code" fi # Split-invocation path — two h2load processes pinned to the same cpuset, @@ -156,108 +172,24 @@ h2load_run() { timeout 45 taskset -c "$GCANNON_CPUS" "${writes_argv[@]}" >"$writes_log" 2>&1 & local writes_pid=$! - wait "$reads_pid" 2>/dev/null || true - wait "$writes_pid" 2>/dev/null || true + local reads_exit=0 writes_exit=0 + wait "$reads_pid" 2>/dev/null || reads_exit=$? + wait "$writes_pid" 2>/dev/null || writes_exit=$? # Emit combined output with sentinels so h2load_parse can sum the two - # sections' 2xx counts and combine bandwidth/duration. + # sections' completed requests and validate both runs. echo "=== h2load-reads ===" cat "$reads_log" echo "=== h2load-writes ===" cat "$writes_log" rm -f "$reads_log" "$writes_log" + if [ "$reads_exit" -ne 0 ]; then return "$reads_exit"; fi + return "$writes_exit" } # ── Parse output ──────────────────────────────────────────────────────────── h2load_parse() { - local endpoint="$1" - local output="$2" - - # rps = successful 2xx responses divided by wall duration. h2load's - # own "finished in Xs, Y req/s" number counts all completed requests - # including 4xx/5xx, which silently inflates rps when the server is - # broken — a stale image serving 404s would look like a throughput win. - # Computing from 2xx + the reported duration makes that impossible. - - if [ "$endpoint" = "production-stack" ]; then - # Split-invocation output: two h2load runs concatenated with - # "=== h2load-reads ===" / "=== h2load-writes ===" sentinels. - # Sum 2xx across both sections, average durations, combine rps. - # Write-side 2xx may appear as "0 2xx" because POST /api/items - # returns 204 (in the 2xx range but h2load's status buckets - # bucket 2xx/3xx/4xx/5xx by first digit, so 204 counts as 2xx). - local reads_ok reads_dur writes_ok writes_dur total_ok total_4xx total_5xx - reads_ok=$(echo "$output" | awk '/=== h2load-reads ===/,/=== h2load-writes ===/' \ - | grep -oP '\d+(?= 2xx)' | head -1) - reads_dur=$(echo "$output" | awk '/=== h2load-reads ===/,/=== h2load-writes ===/' \ - | grep -oP 'finished in \K[\d.]+' | head -1) - writes_ok=$(echo "$output" | awk '/=== h2load-writes ===/,0' \ - | grep -oP '\d+(?= 2xx)' | head -1) - writes_dur=$(echo "$output" | awk '/=== h2load-writes ===/,0' \ - | grep -oP 'finished in \K[\d.]+' | head -1) - - reads_ok=${reads_ok:-0} - writes_ok=${writes_ok:-0} - reads_dur=${reads_dur:-1} - writes_dur=${writes_dur:-1} - total_ok=$(( reads_ok + writes_ok )) - - # Use the reads duration as the primary (they're ~identical — both - # runs use -D $duration and start at the same time). - local dur="$reads_dur" - [ "$dur" = "0" ] && dur="$writes_dur" - [ "$dur" = "0" ] && dur=1 - - echo "rps=$(awk -v ok="$total_ok" -v dur="$dur" \ - 'BEGIN { if (dur+0 > 0) printf "%d", ok/dur; else print 0 }' 2>/dev/null || echo 0)" - - # Report read-path latency (writes have their own latency profile - # that would need a separate column; keeping one number for now). - echo "avg_lat=$(echo "$output" | awk '/=== h2load-reads ===/,/=== h2load-writes ===/' \ - | awk '/time for request:/{print $6}' | head -1)" - echo "p99_lat=$(echo "$output" | awk '/=== h2load-reads ===/,/=== h2load-writes ===/' \ - | awk '/time for request:/{print $5}' | head -1)" - - echo "reconnects=0" - echo "bandwidth=$(echo "$output" | awk '/=== h2load-reads ===/,/=== h2load-writes ===/' \ - | grep -oP 'finished in [\d.]+s, [\d.]+ req/s, \K[\d.]+[KMGT]?B/s' | head -1 || echo 0)" - - # Sum all status counters across both sections. - local r2xx r4xx r5xx w2xx w4xx w5xx - r2xx=$(echo "$output" | awk '/=== h2load-reads ===/,/=== h2load-writes ===/' | grep -oP '\d+(?= 2xx)' | head -1) - r4xx=$(echo "$output" | awk '/=== h2load-reads ===/,/=== h2load-writes ===/' | grep -oP '\d+(?= 4xx)' | head -1) - r5xx=$(echo "$output" | awk '/=== h2load-reads ===/,/=== h2load-writes ===/' | grep -oP '\d+(?= 5xx)' | head -1) - w2xx=$(echo "$output" | awk '/=== h2load-writes ===/,0' | grep -oP '\d+(?= 2xx)' | head -1) - w4xx=$(echo "$output" | awk '/=== h2load-writes ===/,0' | grep -oP '\d+(?= 4xx)' | head -1) - w5xx=$(echo "$output" | awk '/=== h2load-writes ===/,0' | grep -oP '\d+(?= 5xx)' | head -1) - echo "status_2xx=$(( ${r2xx:-0} + ${w2xx:-0} ))" - echo "status_3xx=0" - echo "status_4xx=$(( ${r4xx:-0} + ${w4xx:-0} ))" - echo "status_5xx=$(( ${r5xx:-0} + ${w5xx:-0} ))" - return - fi - - # Single-invocation path — every profile except production-stack. - local duration_secs ok - duration_secs=$(echo "$output" | grep -oP 'finished in \K[\d.]+' | head -1) - duration_secs=${duration_secs:-1} - ok=$(echo "$output" | grep -oP '\d+(?= 2xx)' | head -1) - ok=${ok:-0} - echo "rps=$(awk -v ok="$ok" -v dur="$duration_secs" \ - 'BEGIN { if (dur+0 > 0) printf "%d", ok/dur; else print 0 }' 2>/dev/null || echo 0)" - - # Latency — h2 mode uses "time for request:" one-liner: min/max/mean/sd, no - # percentiles. avg = mean ($6). h2load has no p99, so use max ($5) as the tail. - echo "avg_lat=$(echo "$output" | awk '/time for request:/{print $6}' | head -1)" - echo "p99_lat=$(echo "$output" | awk '/time for request:/{print $5}' | head -1)" - - echo "reconnects=0" - echo "bandwidth=$(echo "$output" | grep -oP 'finished in [\d.]+s, [\d.]+ req/s, \K[\d.]+[KMGT]?B/s' | head -1 || echo 0)" - - echo "status_2xx=$(echo "$output" | grep -oP '\d+(?= 2xx)' | head -1 || echo 0)" - echo "status_3xx=$(echo "$output" | grep -oP '\d+(?= 3xx)' | head -1 || echo 0)" - echo "status_4xx=$(echo "$output" | grep -oP '\d+(?= 4xx)' | head -1 || echo 0)" - echo "status_5xx=$(echo "$output" | grep -oP '\d+(?= 5xx)' | head -1 || echo 0)" + python3 "${BASH_SOURCE[0]%/*}/h2load_parse.py" "$1" <<< "$2" } diff --git a/scripts/lib/tools/h2load_parse.py b/scripts/lib/tools/h2load_parse.py new file mode 100644 index 0000000000..9ad5b220b3 --- /dev/null +++ b/scripts/lib/tools/h2load_parse.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Validate completed h2load runs before emitting benchmark metrics.""" + +import re +import sys +from decimal import Decimal + + +def one(pattern, output, label): + matches = re.findall(pattern, output, re.MULTILINE) + if len(matches) != 1: + raise ValueError(f"expected one {label}, found {len(matches)}") + return matches[0] + + +def parse_run(output): + elapsed, unit, bandwidth = one( + r"^finished in ([\d.]+)(us|ms|s), [\d.]+ req/s, ([\d.]+[KMGT]?B/s)\s*$", + output, "completion summary", + ) + duration = Decimal(elapsed) * {"us": Decimal("0.000001"), "ms": Decimal("0.001"), "s": Decimal(1)}[unit] + if duration <= 0: + raise ValueError("measurement duration must be positive") + counts = one( + r"^requests:\s*(\d+) total,\s*(\d+) started,\s*(\d+) done,\s*(\d+) succeeded,\s*(\d+) failed,\s*(\d+) errored,\s*(\d+) timeout\s*$", + output, "request summary", + ) + _, _, done, succeeded, failed, errored, timeout = map(int, counts) + if failed or errored or timeout: + raise ValueError(f"unsuccessful requests: {failed} failed, {errored} errored, {timeout} timeout") + if succeeded <= 0 or succeeded != done: + raise ValueError("no complete, successful measurement") + statuses = tuple(map(int, one( + r"^status codes:\s*(\d+) 2xx,\s*(\d+) 3xx,\s*(\d+) 4xx,\s*(\d+) 5xx\s*$", + output, "HTTP status summary", + ))) + if statuses[0] < succeeded or any(statuses[1:]): + raise ValueError("successful requests must have HTTP 2xx responses") + + metrics = { + "rps": succeeded / duration, + "duration": f"{duration}s", + "avg_lat": "", + "p99_lat": "", + "max_lat": "", + "reconnects": 0, + "bandwidth": bandwidth, + **{f"status_{code}xx": count for code, count in zip(range(2, 6), statuses)}, + } + latency = re.search(r"^\s*(time for request|request)\s*:\s*(.+)$", output, re.MULTILINE) + if latency: + percentiles = latency[1] == "request" + columns = latency[2].split() + if len(columns) < (8 if percentiles else 5): + raise ValueError("incomplete latency summary") + metrics["max_lat"] = columns[1] + metrics["avg_lat"] = columns[5] if percentiles else columns[2] + if percentiles: + metrics["p99_lat"] = columns[4] + return metrics + + +def parse(output, endpoint): + if endpoint != "production-stack": + return parse_run(output) + if output.count("=== h2load-reads ===") != 1 or output.count("=== h2load-writes ===") != 1: + raise ValueError("missing production read/write output") + _, reads = output.split("=== h2load-reads ===", 1) + reads, writes = reads.split("=== h2load-writes ===", 1) + result = parse_run(reads) + write_metrics = parse_run(writes) + result["rps"] += write_metrics["rps"] + for code in range(2, 6): + result[f"status_{code}xx"] += write_metrics[f"status_{code}xx"] + return result + + +def main(): + try: + metrics = parse(sys.stdin.read(), sys.argv[1]) + except (ValueError, ArithmeticError) as error: + print(f"invalid h2load measurement: {error}", file=sys.stderr) + return 1 + metrics["rps"] = int(metrics["rps"]) + for key, value in metrics.items(): + print(f"{key}={value}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_authsvc_image.py b/scripts/tests/test_authsvc_image.py new file mode 100644 index 0000000000..a806c287fa --- /dev/null +++ b/scripts/tests/test_authsvc_image.py @@ -0,0 +1,27 @@ +import os +import subprocess +import unittest + + +@unittest.skipUnless(os.environ.get("HTTPARENA_TEST_AUTHSVC_IMAGE"), "set HTTPARENA_TEST_AUTHSVC_IMAGE to test a built image") +class AuthImageTests(unittest.TestCase): + def test_image_runs_the_real_health_endpoint(self): + result = subprocess.run( + ["docker", "run", "--rm", "--entrypoint", "sh", os.environ["HTTPARENA_TEST_AUTHSVC_IMAGE"], "-c", """ +set -eu +/usr/local/bin/authsvc & +pid=$! +trap 'kill "$pid" 2>/dev/null || true; wait "$pid" || true' EXIT +for attempt in 1 2 3 4 5; do + if wget -q -O /dev/null http://127.0.0.1:9090/_health; then exit 0; fi + kill -0 "$pid" 2>/dev/null || exit 1 + sleep 1 +done +exit 1 +"""], capture_output=True, text=True, timeout=20, check=False, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_benchmark_run.py b/scripts/tests/test_benchmark_run.py new file mode 100644 index 0000000000..85a3aa9ed5 --- /dev/null +++ b/scripts/tests/test_benchmark_run.py @@ -0,0 +1,89 @@ +import os +import pathlib +import re +import subprocess +import unittest + +from test_h2load import H2 + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +BENCHMARK = ROOT / "scripts/benchmark.sh" + + +class BenchmarkRunTests(unittest.TestCase): + driver = BENCHMARK + + def run_measurement(self, output, exit_code=0, version_supported=True): + function = re.search(r"^run_one\(\) \{\n.*?^\}", self.driver.read_text(), re.MULTILINE | re.DOTALL)[0] + harness = ''' +set -euo pipefail +source "$1" +parse_profile() { PROF_ENDPOINT=h2; PROF_PIPELINE=1; PROF_CPU=1; PROF_REQ=1; } +endpoint_tool() { echo h2load; } +banner() { :; } +info() { echo "$*"; } +warn() { echo "$*" >&2; } +framework_start() { echo STARTED; } +framework_wait_ready() { return 0; } +framework_stop() { echo STOPPED; } +stats_start() { :; } +stats_stop() { :; } +cpu_acct_start() { :; } +cpu_acct_stop() { :; } +sleep() { :; } +h2load_build_args() { echo ignored; } +h2load_check_version() { test "$VERSION_SUPPORTED" = true; } +h2load_run() { printf '%s\\n' "$LOAD_OUTPUT"; return "$LOAD_EXIT"; } +save_result() { echo SAVED; } +FRAMEWORK=sample +CONTAINER_NAME=sample +RUNS=1 +DURATION=5s +SAVE_RESULTS=true +STATS_AVG_CPU=1% +STATS_PEAK_MEM=1MiB +STATS_BREAKDOWN="" +declare -A PROFILES=([baseline-h2]=ignored) +declare -A BEST_M +''' + return subprocess.run( + ["bash", "-c", harness + function + "\nrun_one baseline-h2 256 && exit 0 || exit $?\n", "test", + str(ROOT / "scripts/lib/tools/h2load.sh")], + env={**os.environ, "LOAD_OUTPUT": output, "LOAD_EXIT": str(exit_code), + "VERSION_SUPPORTED": str(version_supported).lower()}, + capture_output=True, text=True, check=False, + ) + + def test_incomplete_run_is_not_saved_as_zero_rps(self): + result = self.run_measurement("starting benchmark...") + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertNotIn("SAVED", result.stdout) + self.assertIn("STOPPED", result.stdout) + + def test_timeout_cannot_publish_a_partial_summary(self): + result = self.run_measurement(H2, exit_code=124) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertNotIn("SAVED", result.stdout) + self.assertIn("STOPPED", result.stdout) + + def test_valid_result_is_saved_and_server_is_stopped(self): + result = self.run_measurement(H2) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("=== Best: 20 req/s", result.stdout) + self.assertIn("SAVED", result.stdout) + self.assertIn("STOPPED", result.stdout) + + def test_unsupported_load_generator_is_rejected_before_server_start(self): + result = self.run_measurement(H2, version_supported=False) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertNotIn("STARTED", result.stdout) + self.assertNotIn("SAVED", result.stdout) + + +class LiteBenchmarkRunTests(BenchmarkRunTests): + driver = ROOT / "scripts/benchmark-lite.sh" + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_composite_status.py b/scripts/tests/test_composite_status.py new file mode 100644 index 0000000000..242fc0c4ed --- /dev/null +++ b/scripts/tests/test_composite_status.py @@ -0,0 +1,52 @@ +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class CompositeStatusTests(unittest.TestCase): + def test_failed_or_unfinished_variants_cannot_reuse_a_retained_composite(self): + for status in (None, "running", "1", "0"): + with self.subTest(status=status), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + statuses = root / "statuses" + statuses.mkdir() + (statuses / "sample.exit").write_text("0\n") + if status is not None: + (statuses / "sample-grpc.exit").write_text(status + "\n") + for data in (root / "main", root / "run"): + (data / "results").mkdir(parents=True) + (data / "frameworks.json").write_text(json.dumps({"sample": { + "dir": "sample", "type": "emerging", "language": "Rust", + "variants": [{"dir": "sample-grpc"}], + }, "rival": {"dir": "rival", "type": "emerging", "language": "Rust"}})) + for name in ("sample", "rival"): + (data / "results" / f"{name}.json").write_text(json.dumps({ + "framework": name, "results": {"baseline-4096": { + "framework": name, "language": "Rust", "rps": 100, + "cpu": "100%", "memory": "10MiB", "connections": 4096, + }}, + })) + result = subprocess.run([ + "python3", "-B", str(ROOT / "scripts/composite_delta.py"), + str(root / "main"), str(root / "run"), str(root / "out"), + "sample", "sample-grpc", + ], env={**os.environ, "HTTPARENA_RUN_STATUS_DIR": str(statuses)}, + capture_output=True, text=True, check=False) + self.assertEqual(result.returncode, 0, result.stderr) + for name in ("sample", "sample-grpc"): + report = (root / "out" / f"{name}.composite.md").read_text() + if status == "0": + self.assertIn("**Composite score**", report) + else: + self.assertIn("Composite score unavailable", report) + self.assertNotIn("| Family |", report) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_h2load.py b/scripts/tests/test_h2load.py new file mode 100644 index 0000000000..2abb76bfeb --- /dev/null +++ b/scripts/tests/test_h2load.py @@ -0,0 +1,119 @@ +import pathlib +import subprocess +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +H2 = """finished in 5.00s, 20.00 req/s, 2.00KB/s +requests: 100 total, 110 started, 100 done, 100 succeeded, 0 failed, 0 errored, 0 timeout +status codes: 105 2xx, 0 3xx, 0 4xx, 0 5xx +time for request: 1ms 9ms 3ms 2ms 68.00% +""" +H3 = H2.replace( + "time for request: 1ms 9ms 3ms 2ms 68.00%", + "request : 1ms 9ms 2ms 4ms 7ms 3ms 2ms 68.00%", +) + + +class H2loadTests(unittest.TestCase): + def parse(self, output, tool="h2load", endpoint="h2"): + return subprocess.run( + ["bash", "-c", 'source "$1"; output=$(cat); "${2}_parse" "$3" "$output"', + "test", str(ROOT / "scripts/lib/tools" / f"{tool}.sh"), + tool.replace("-", "_"), endpoint], + input=output, text=True, capture_output=True, check=False, + ) + + def test_only_completed_successful_responses_count(self): + for tool, output in [("h2load", H2), ("h2load-h3", H3)]: + with self.subTest(tool=tool): + result = self.parse(output, tool) + self.assertEqual(result.returncode, 0, result.stderr) + metrics = dict(line.split("=", 1) for line in result.stdout.splitlines()) + self.assertEqual(metrics["rps"], "20") + self.assertEqual(metrics["duration"], "5.00s") + + def test_failed_streams_with_200_headers_are_rejected(self): + broken = H3.replace("100 succeeded, 0 failed, 0 errored", "0 succeeded, 100 failed, 100 errored") + result = self.parse(broken, "h2load-h3") + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("rps=", result.stdout) + + def test_missing_or_invalid_summaries_are_rejected(self): + for output in ["starting benchmark...", H2.split("requests:")[0], + H2.replace("5.00s", "0.00s"), + H2.replace("0 timeout", "1 timeout"), + H2.replace("0 4xx", "1 4xx")]: + with self.subTest(output=output): + result = self.parse(output) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("rps=", result.stdout) + + def test_maximum_is_not_published_as_p99(self): + result = self.parse(H2) + self.assertEqual(result.returncode, 0, result.stderr) + metrics = dict(line.split("=", 1) for line in result.stdout.splitlines()) + self.assertEqual(metrics["avg_lat"], "3ms") + self.assertEqual(metrics.get("p99_lat", ""), "") + self.assertEqual(metrics["max_lat"], "9ms") + + def test_h3_keeps_its_actual_percentile(self): + result = self.parse(H3, "h2load-h3") + self.assertEqual(result.returncode, 0, result.stderr) + metrics = dict(line.split("=", 1) for line in result.stdout.splitlines()) + self.assertEqual(metrics["p99_lat"], "7ms") + self.assertEqual(metrics["avg_lat"], "3ms") + + def test_new_h2load_keeps_its_actual_percentile(self): + result = self.parse(H3) + self.assertEqual(result.returncode, 0, result.stderr) + metrics = dict(line.split("=", 1) for line in result.stdout.splitlines()) + self.assertEqual(metrics["p99_lat"], "7ms") + self.assertEqual(metrics["avg_lat"], "3ms") + + def test_versions_without_duration_shutdown_fix_are_rejected(self): + for version, expected in [("1.59.0", False), ("1.68.0", False), ("1.69.0", True), ("2.0.0", True)]: + with self.subTest(version=version): + result = subprocess.run( + ["bash", "-c", 'source "$1"; H2LOAD=fake_h2load; TEST_VERSION="$2"; fake_h2load() { echo "h2load nghttp2/$TEST_VERSION"; }; h2load_check_version', + "test", str(ROOT / "scripts/lib/tools/h2load.sh"), version], + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode == 0, expected, result.stderr) + + def test_each_production_invocation_must_complete(self): + for writes in ["starting benchmark...", H2.replace("0 errored", "1 errored")]: + result = self.parse(f"=== h2load-reads ===\n{H2}=== h2load-writes ===\n{writes}", endpoint="production-stack") + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("rps=", result.stdout) + + def test_production_rates_use_each_invocations_duration(self): + writes = H2.replace("5.00s", "10.00s") + result = self.parse(f"=== h2load-reads ===\n{H2}=== h2load-writes ===\n{writes}", endpoint="production-stack") + self.assertEqual(result.returncode, 0, result.stderr) + metrics = dict(line.split("=", 1) for line in result.stdout.splitlines()) + self.assertEqual(metrics["rps"], "30") + + def test_timeout_exit_status_is_preserved(self): + for tool in ["h2load", "h2load-h3"]: + result = subprocess.run( + ["bash", "-c", 'source "$1"; GCANNON_CPUS=0; timeout() { echo partial; return 124; }; "${2}_run" ignored', + "test", str(ROOT / "scripts/lib/tools" / f"{tool}.sh"), tool.replace("-", "_")], + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 124, tool) + self.assertIn("partial", result.stdout) + + def test_split_invocation_failure_is_preserved(self): + result = subprocess.run( + ["bash", "-c", 'source "$1"; GCANNON_CPUS=0; timeout() { echo partial; return 124; }; h2load_run reads --split-- writes', + "test", str(ROOT / "scripts/lib/tools/h2load.sh")], + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 124) + self.assertIn("=== h2load-reads ===", result.stdout) + self.assertIn("=== h2load-writes ===", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_h2load_image.py b/scripts/tests/test_h2load_image.py new file mode 100644 index 0000000000..26a33a8fad --- /dev/null +++ b/scripts/tests/test_h2load_image.py @@ -0,0 +1,52 @@ +import os +import socket +import subprocess +import threading +import unittest +import uuid + + +@unittest.skipUnless(os.environ.get("HTTPARENA_TEST_H2LOAD_IMAGE"), "set HTTPARENA_TEST_H2LOAD_IMAGE to test a built image") +class H2loadImageTests(unittest.TestCase): + def test_duration_ends_while_tls_handshake_is_pending(self): + listener = socket.socket() + listener.bind(("0.0.0.0", 0)) + listener.listen(1) + listener.settimeout(6) + stop = threading.Event() + accepted = threading.Event() + + def hold_handshake(): + try: + conn, _ = listener.accept() + with conn: + accepted.set() + stop.wait(6) + except OSError: + pass + + thread = threading.Thread(target=hold_handshake) + thread.start() + name = "httparena-h2load-test-" + uuid.uuid4().hex + host = os.environ.get("HTTPARENA_TEST_H2LOAD_HOST", "127.0.0.1") + network = ["--network", "host"] if host == "127.0.0.1" else [] + try: + result = subprocess.run( + ["docker", "run", "--rm", "--init", "--name", name, *network, + "--entrypoint", "timeout", os.environ["HTTPARENA_TEST_H2LOAD_IMAGE"], + "-k", "1", "4", "h2load", "-c1", "-t1", "-D1", + f"https://{host}:{listener.getsockname()[1]}/"], + capture_output=True, text=True, timeout=8, check=False, + ) + self.assertTrue(accepted.is_set(), result.stdout + result.stderr) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("finished in", result.stdout) + finally: + stop.set() + listener.close() + thread.join(timeout=7) + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + + +if __name__ == "__main__": + unittest.main() From ffa6e79b3b3a6255482d74f1bd549b78477343d2 Mon Sep 17 00:00:00 2001 From: Daniel Boros Date: Fri, 13 Nov 2026 16:33:49 +0100 Subject: [PATCH 04/12] chore: update tako to 2.7.1 --- frameworks/tako-gateway/README.md | 2 +- frameworks/tako-grpc/Cargo.lock | 30 ++++++++++---------- frameworks/tako-grpc/Cargo.toml | 2 +- frameworks/tako-grpc/README.md | 4 +-- frameworks/tako-grpc/meta.json | 2 +- frameworks/tako/Cargo.lock | 46 +++++++++++++++++-------------- frameworks/tako/Cargo.toml | 2 +- frameworks/tako/README.md | 4 +-- frameworks/tako/meta.json | 2 +- 9 files changed, 49 insertions(+), 45 deletions(-) diff --git a/frameworks/tako-gateway/README.md b/frameworks/tako-gateway/README.md index 2edb494e46..b936c654d5 100644 --- a/frameworks/tako-gateway/README.md +++ b/frameworks/tako-gateway/README.md @@ -1,6 +1,6 @@ # tako-gateway -The gateway profiles for [Tako](https://github.com/rust-dd/tako) 2.7.0, reported as `tako`. +The gateway profiles for [Tako](https://github.com/rust-dd/tako) 2.7.1, reported as `tako`. The server is built from [`frameworks/tako`](../tako); this directory only holds the compose files and the edge configuration. diff --git a/frameworks/tako-grpc/Cargo.lock b/frameworks/tako-grpc/Cargo.lock index 0e87040ccd..d5b9df46bd 100644 --- a/frameworks/tako-grpc/Cargo.lock +++ b/frameworks/tako-grpc/Cargo.lock @@ -329,7 +329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1317,9 +1317,9 @@ dependencies = [ [[package]] name = "tako-rs" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1407ad0125ae2215830acdeac13196f610a0df4fd1324874189313d643617eae" +checksum = "c32a6df727a418c93e5dafc69aa0bc12e1cd65f34421c1dcfeb1326996774ee8" dependencies = [ "linkme", "tako-rs-core", @@ -1332,9 +1332,9 @@ dependencies = [ [[package]] name = "tako-rs-core" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfd800672ed0924b331d21979676d332cc2b62ba42f7b998a6df648fb79f88fb" +checksum = "6b345f3c928e5f2a51f81e219342eeba41ed238c63d8358d05834277a47ce542" dependencies = [ "anyhow", "arc-swap", @@ -1367,9 +1367,9 @@ dependencies = [ [[package]] name = "tako-rs-extractors" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b21362b5148c3d099ce276d64d8c1a8c2f3347b5625555c5dd8b5337e4febdf9" +checksum = "527dbbf0b30579ea4dc5ad3cfe251926b3be638059b08fa41b4ba7328aaec289" dependencies = [ "anyhow", "base64", @@ -1393,9 +1393,9 @@ dependencies = [ [[package]] name = "tako-rs-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cadbbefdcfb3dfcb15ceda3e7ca994a7f1a278759d8f0023301f6ff2b725ea" +checksum = "5f955ebf90049d6b237c637a015db68e940ef407c842400ded6ff248a6fc899d" dependencies = [ "proc-macro2", "quote", @@ -1404,9 +1404,9 @@ dependencies = [ [[package]] name = "tako-rs-plugins" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23988e23bc380e50feccba1f2d0e89599a691a7a77b47a77928ba4ad7d40bf27" +checksum = "987ea0303f3776b185250686fd35fabca0e1d65bd1cfe057c3e4a4460ccd8384" dependencies = [ "anyhow", "async-trait", @@ -1434,9 +1434,9 @@ dependencies = [ [[package]] name = "tako-rs-server" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023d57cb4a1d189a64cbd61f940a0f338682b4212208b06a81c6d69093a6530f" +checksum = "8853a7bab9ac69c1322dc26ee7c6e478551da60e7373c10c667c7916a9ce9019" dependencies = [ "anyhow", "arc-swap", @@ -1457,9 +1457,9 @@ dependencies = [ [[package]] name = "tako-rs-streams" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709cec96f9f0524bd8c7da8fdff2db17b3e8a48663585f7678a1162a077515a7" +checksum = "e6033a3122d6a9c6606cb09a3d928dd42cbfcdf5d05754bfc157ddbf70d0ddf0" dependencies = [ "anyhow", "bytes", diff --git a/frameworks/tako-grpc/Cargo.toml b/frameworks/tako-grpc/Cargo.toml index 78235c98e3..2179ded450 100644 --- a/frameworks/tako-grpc/Cargo.toml +++ b/frameworks/tako-grpc/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] prost = "0.14" -tako-rs = { version = "2.7.0", features = ["grpc", "http2", "tls"] } +tako-rs = { version = "2.7.1", features = ["grpc", "http2", "tls"] } tokio = { version = "1", features = ["macros", "net", "rt-multi-thread"] } [profile.release] diff --git a/frameworks/tako-grpc/README.md b/frameworks/tako-grpc/README.md index 686b9fcbd1..67ffc921ec 100644 --- a/frameworks/tako-grpc/README.md +++ b/frameworks/tako-grpc/README.md @@ -1,6 +1,6 @@ # tako-grpc -The gRPC profiles for [Tako](https://github.com/rust-dd/tako) 2.7.0, reported as `tako`. +The gRPC profiles for [Tako](https://github.com/rust-dd/tako) 2.7.1, reported as `tako`. ## Why a separate entry @@ -11,7 +11,7 @@ so the gRPC service gets its own container. ## Stack - **Language:** Rust 1.99 -- **Framework:** tako-rs 2.7.0 (`grpc`, `http2`, `tls`) +- **Framework:** tako-rs 2.7.1 (`grpc`, `http2`, `tls`) - **Protobuf:** prost 0.14 - **Build:** Multi-stage, `debian:trixie-slim` runtime diff --git a/frameworks/tako-grpc/meta.json b/frameworks/tako-grpc/meta.json index 0dcae14f9f..cb79fdcc23 100644 --- a/frameworks/tako-grpc/meta.json +++ b/frameworks/tako-grpc/meta.json @@ -4,7 +4,7 @@ "engine": "hyper", "type": "emerging", "mode": "standard", - "description": "Tako 2.7.0 gRPC: benchmark.BenchmarkService/GetSum as a plain tako route with the GrpcRequest extractor and GrpcResponse responder, prost messages. h2c on :8080 and h2 over rustls on :8443, both on tako's multi-thread Server with default configuration.", + "description": "Tako 2.7.1 gRPC: benchmark.BenchmarkService/GetSum as a plain tako route with the GrpcRequest extractor and GrpcResponse responder, prost messages. h2c on :8080 and h2 over rustls on :8443, both on tako's multi-thread Server with default configuration.", "repo": "https://github.com/rust-dd/tako", "website": "https://tako.rust-dd.com", "enabled": true, diff --git a/frameworks/tako/Cargo.lock b/frameworks/tako/Cargo.lock index 4726530477..bfbfa6c838 100644 --- a/frameworks/tako/Cargo.lock +++ b/frameworks/tako/Cargo.lock @@ -828,7 +828,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2000,7 +2000,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2174,7 +2174,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2233,7 +2233,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2626,9 +2626,9 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tako-rs" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1407ad0125ae2215830acdeac13196f610a0df4fd1324874189313d643617eae" +checksum = "c32a6df727a418c93e5dafc69aa0bc12e1cd65f34421c1dcfeb1326996774ee8" dependencies = [ "linkme", "tako-rs-core", @@ -2642,9 +2642,9 @@ dependencies = [ [[package]] name = "tako-rs-core" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfd800672ed0924b331d21979676d332cc2b62ba42f7b998a6df648fb79f88fb" +checksum = "6b345f3c928e5f2a51f81e219342eeba41ed238c63d8358d05834277a47ce542" dependencies = [ "anyhow", "arc-swap", @@ -2677,9 +2677,9 @@ dependencies = [ [[package]] name = "tako-rs-extractors" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b21362b5148c3d099ce276d64d8c1a8c2f3347b5625555c5dd8b5337e4febdf9" +checksum = "527dbbf0b30579ea4dc5ad3cfe251926b3be638059b08fa41b4ba7328aaec289" dependencies = [ "anyhow", "base64", @@ -2703,9 +2703,9 @@ dependencies = [ [[package]] name = "tako-rs-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cadbbefdcfb3dfcb15ceda3e7ca994a7f1a278759d8f0023301f6ff2b725ea" +checksum = "5f955ebf90049d6b237c637a015db68e940ef407c842400ded6ff248a6fc899d" dependencies = [ "proc-macro2", "quote", @@ -2714,9 +2714,9 @@ dependencies = [ [[package]] name = "tako-rs-plugins" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23988e23bc380e50feccba1f2d0e89599a691a7a77b47a77928ba4ad7d40bf27" +checksum = "987ea0303f3776b185250686fd35fabca0e1d65bd1cfe057c3e4a4460ccd8384" dependencies = [ "anyhow", "async-trait", @@ -2746,9 +2746,9 @@ dependencies = [ [[package]] name = "tako-rs-server" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023d57cb4a1d189a64cbd61f940a0f338682b4212208b06a81c6d69093a6530f" +checksum = "8853a7bab9ac69c1322dc26ee7c6e478551da60e7373c10c667c7916a9ce9019" dependencies = [ "anyhow", "arc-swap", @@ -2763,6 +2763,8 @@ dependencies = [ "hyper-util", "pin-project-lite", "quinn", + "quinn-proto", + "quinn-udp", "rustls", "tako-rs-core", "tokio", @@ -2773,9 +2775,9 @@ dependencies = [ [[package]] name = "tako-rs-server-pt" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54ed007a2d03d51d3043fee313a6b94657310aab26ec0a39a00764f486ba4099" +checksum = "e90e8b66a454cd17a61dbb7c8f8f34c60c63f578d1360fec2d49385ca0287993" dependencies = [ "core_affinity", "futures-util", @@ -2790,9 +2792,9 @@ dependencies = [ [[package]] name = "tako-rs-streams" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709cec96f9f0524bd8c7da8fdff2db17b3e8a48663585f7678a1162a077515a7" +checksum = "e6033a3122d6a9c6606cb09a3d928dd42cbfcdf5d05754bfc157ddbf70d0ddf0" dependencies = [ "anyhow", "bytes", @@ -2805,6 +2807,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", + "quinn-proto", + "quinn-udp", "rustls", "tako-rs-core", "tokio", @@ -3345,7 +3349,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/frameworks/tako/Cargo.toml b/frameworks/tako/Cargo.toml index 336e640b50..8c6e0e391b 100644 --- a/frameworks/tako/Cargo.toml +++ b/frameworks/tako/Cargo.toml @@ -8,7 +8,7 @@ deadpool-postgres = "0.14" moka = { version = "0.12", features = ["sync"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -tako-rs = { version = "2.7.0", features = ["http2", "http3", "per-thread", "plugins", "tls", "ws"] } +tako-rs = { version = "2.7.1", features = ["http2", "http3", "per-thread", "plugins", "tls", "ws"] } tokio = { version = "1", features = ["macros", "net", "rt-multi-thread", "time"] } tokio-postgres = { version = "0.7", features = ["with-serde_json-1"] } diff --git a/frameworks/tako/README.md b/frameworks/tako/README.md index 670fd418c7..6db3eb3a2e 100644 --- a/frameworks/tako/README.md +++ b/frameworks/tako/README.md @@ -1,13 +1,13 @@ # tako -[Tako](https://github.com/rust-dd/tako) 2.7.0 (`tako-rs` on crates.io) on hyper and Tokio, default configuration. +[Tako](https://github.com/rust-dd/tako) 2.7.1 (`tako-rs` on crates.io) on hyper and Tokio, default configuration. gRPC lives in [`tako-grpc`](../tako-grpc) and the gateway stacks in [`tako-gateway`](../tako-gateway); all three report as `tako`. ## Stack - **Language:** Rust 1.99 -- **Framework:** tako-rs 2.7.0 +- **Framework:** tako-rs 2.7.1 - **Build:** Multi-stage, `debian:trixie-slim` runtime ## Listeners diff --git a/frameworks/tako/meta.json b/frameworks/tako/meta.json index befde11ac2..5e2222d904 100644 --- a/frameworks/tako/meta.json +++ b/frameworks/tako/meta.json @@ -10,7 +10,7 @@ "request": true, "response": true }, - "description": "Tako 2.7.0 on hyper and Tokio, default configuration. Plain HTTP/1.1 and WebSocket on :8080 run tako's per-thread server (one current-thread runtime per CPU, SO_REUSEPORT listeners, PerThreadConfig::default()); h2c (:8082), HTTP/1.1 and h2 over rustls (:8081, :8443) and HTTP/3 over quinn (:8443/udp) run tako's multi-thread Server. Routing, Path/Query/State/body extractors and the Json responder through the tako API, the compression plugin with default settings on the JSON routes, ServeDir with precompressed .br/.gz sidecars read from disk per request, WebSocketUpgrade for /ws, and deadpool-postgres for async-db.", + "description": "Tako 2.7.1 on hyper and Tokio, default configuration. Plain HTTP/1.1 and WebSocket on :8080 run tako's per-thread server (one current-thread runtime per CPU, SO_REUSEPORT listeners, PerThreadConfig::default()); h2c (:8082), HTTP/1.1 and h2 over rustls (:8081, :8443) and HTTP/3 over quinn (:8443/udp) run tako's multi-thread Server. Routing, Path/Query/State/body extractors and the Json responder through the tako API, the compression plugin with default settings on the JSON routes, ServeDir with precompressed .br/.gz sidecars read from disk per request, WebSocketUpgrade for /ws, and deadpool-postgres for async-db.", "repo": "https://github.com/rust-dd/tako", "website": "https://tako.rust-dd.com", "enabled": true, From 6b13bb5613db228236a04b2fefad554c1220e253 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 13 Nov 2026 18:26:36 +0000 Subject: [PATCH 05/12] Benchmark results: 3 frameworks (all tests) [skip ci] --- site/data/frameworks.json | 4 +- site/data/results/tako.json | 447 +++++++++++++++++++----------------- 2 files changed, 238 insertions(+), 213 deletions(-) diff --git a/site/data/frameworks.json b/site/data/frameworks.json index ee1cea2afd..ba268866be 100644 --- a/site/data/frameworks.json +++ b/site/data/frameworks.json @@ -2502,7 +2502,7 @@ }, "tako": { "dir": "tako", - "description": "Tako 2.7.0 on hyper and Tokio, default configuration. Plain HTTP/1.1 and WebSocket on :8080 run tako's per-thread server (one current-thread runtime per CPU, SO_REUSEPORT listeners, PerThreadConfig::default()); h2c (:8082), HTTP/1.1 and h2 over rustls (:8081, :8443) and HTTP/3 over quinn (:8443/udp) run tako's multi-thread Server. Routing, Path/Query/State/body extractors and the Json responder through the tako API, the compression plugin with default settings on the JSON routes, ServeDir with precompressed .br/.gz sidecars read from disk per request, WebSocketUpgrade for /ws, and deadpool-postgres for async-db.", + "description": "Tako 2.7.1 on hyper and Tokio, default configuration. Plain HTTP/1.1 and WebSocket on :8080 run tako's per-thread server (one current-thread runtime per CPU, SO_REUSEPORT listeners, PerThreadConfig::default()); h2c (:8082), HTTP/1.1 and h2 over rustls (:8081, :8443) and HTTP/3 over quinn (:8443/udp) run tako's multi-thread Server. Routing, Path/Query/State/body extractors and the Json responder through the tako API, the compression plugin with default settings on the JSON routes, ServeDir with precompressed .br/.gz sidecars read from disk per request, WebSocketUpgrade for /ws, and deadpool-postgres for async-db.", "repo": "https://github.com/rust-dd/tako", "type": "emerging", "engine": "hyper", @@ -2532,7 +2532,7 @@ }, { "dir": "tako-grpc", - "description": "Tako 2.7.0 gRPC: benchmark.BenchmarkService/GetSum as a plain tako route with the GrpcRequest extractor and GrpcResponse responder, prost messages. h2c on :8080 and h2 over rustls on :8443, both on tako's multi-thread Server with default configuration.", + "description": "Tako 2.7.1 gRPC: benchmark.BenchmarkService/GetSum as a plain tako route with the GrpcRequest extractor and GrpcResponse responder, prost messages. h2c on :8080 and h2 over rustls on :8443, both on tako's multi-thread Server with default configuration.", "repo": "https://github.com/rust-dd/tako", "type": "emerging", "engine": "hyper", diff --git a/site/data/results/tako.json b/site/data/results/tako.json index 984896faed..315fd7423f 100644 --- a/site/data/results/tako.json +++ b/site/data/results/tako.json @@ -4,44 +4,46 @@ "8gbit-512": { "framework": "tako", "language": "Rust", - "rps": 48980, - "avg_latency": "95.6us", - "p99_latency": "138.0us", - "cpu": "253.5%", + "rps": 48904, + "avg_latency": "104.6us", + "p99_latency": "135.0us", + "max_latency": "", + "cpu": "252.0%", "memory": "93MiB", "connections": 512, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "507.39MB/s", - "input_bw": "478.32MB/s", + "bandwidth": "506.60MB/s", + "input_bw": "477.58MB/s", "reconnects": 0, - "status_2xx": 244971, + "status_2xx": 244601, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "cpu_usec": 13042174, - "cpu_per_req_us": 53.2397, + "cpu_usec": 12727740, + "cpu_per_req_us": 52.0347, "target_rate": 50000, - "rate_ratio": 0.9796, - "p99_9_latency": "2937.0us" + "rate_ratio": 0.9781, + "p99_9_latency": "7498.0us" }, "async-32000": { "framework": "tako", "language": "Rust", - "rps": 2235424, - "avg_latency": "14.21ms", + "rps": 2236354, + "avg_latency": "14.19ms", "p99_latency": "17.70ms", - "cpu": "5199.2%", - "memory": "966MiB", + "max_latency": "", + "cpu": "5165.3%", + "memory": "921MiB", "connections": 32000, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "251.52MB/s", - "input_bw": "102.33MB/s", + "bandwidth": "251.62MB/s", + "input_bw": "102.37MB/s", "reconnects": 0, - "status_2xx": 22354247, + "status_2xx": 22363542, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -49,19 +51,20 @@ "async-db-1024": { "framework": "tako", "language": "Rust", - "rps": 242075, - "avg_latency": "3.64ms", - "p99_latency": "5.96ms", - "cpu": "3681.6%", - "memory": "96MiB", + "rps": 235674, + "avg_latency": "3.83ms", + "p99_latency": "6.10ms", + "max_latency": "", + "cpu": "3547.8%", + "memory": "97MiB", "connections": 1024, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "923.89MB/s", - "input_bw": "16.16MB/s", - "reconnects": 96468, - "status_2xx": 2420750, + "bandwidth": "899.14MB/s", + "input_bw": "15.73MB/s", + "reconnects": 93945, + "status_2xx": 2356749, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -69,19 +72,20 @@ "baseline-4096": { "framework": "tako", "language": "Rust", - "rps": 3139709, - "avg_latency": "1.30ms", - "p99_latency": "2.60ms", - "cpu": "6652.2%", - "memory": "118MiB", + "rps": 3201484, + "avg_latency": "1.28ms", + "p99_latency": "2.52ms", + "max_latency": "", + "cpu": "6335.0%", + "memory": "124MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "353.23MB/s", - "input_bw": "242.54MB/s", + "bandwidth": "360.18MB/s", + "input_bw": "247.31MB/s", "reconnects": 0, - "status_2xx": 15698546, + "status_2xx": 16007420, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -184,18 +188,19 @@ "baseline-h3-64": { "framework": "tako", "language": "Rust", - "rps": 536729, - "avg_latency": "0us", - "p99_latency": "0us", - "cpu": "2928.1%", - "memory": "107MiB", + "rps": 534875, + "avg_latency": "7.34ms", + "p99_latency": "89.23ms", + "max_latency": "3.61s", + "cpu": "3061.3%", + "memory": "102MiB", "connections": 64, "threads": 64, - "duration": "5s", + "duration": "5.01s", "pipeline": 1, - "bandwidth": "14.87MB/s", + "bandwidth": "14.82MB/s", "reconnects": 0, - "status_2xx": 2689016, + "status_2xx": 2679745, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -203,18 +208,19 @@ "echo-ws-16384": { "framework": "tako", "language": "Rust", - "rps": 3698229, - "avg_latency": "4.40ms", - "p99_latency": "7.60ms", - "cpu": "5989.3%", - "memory": "388MiB", + "rps": 3711259, + "avg_latency": "4.37ms", + "p99_latency": "5.26ms", + "max_latency": "", + "cpu": "6101.8%", + "memory": "394MiB", "connections": 16384, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "25.19MB/s", + "bandwidth": "25.29MB/s", "reconnects": 0, - "status_2xx": 18491145, + "status_2xx": 18556298, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -222,18 +228,19 @@ "echo-ws-4096": { "framework": "tako", "language": "Rust", - "rps": 4066143, - "avg_latency": "1.01ms", - "p99_latency": "1.17ms", - "cpu": "6444.6%", - "memory": "121MiB", + "rps": 4094900, + "avg_latency": "992us", + "p99_latency": "1.06ms", + "max_latency": "", + "cpu": "6056.1%", + "memory": "113MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "27.14MB/s", + "bandwidth": "27.45MB/s", "reconnects": 0, - "status_2xx": 20330715, + "status_2xx": 20474501, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -241,18 +248,19 @@ "echo-ws-512": { "framework": "tako", "language": "Rust", - "rps": 4191358, - "avg_latency": "121us", - "p99_latency": "156us", - "cpu": "6465.1%", - "memory": "39MiB", + "rps": 4120095, + "avg_latency": "123us", + "p99_latency": "158us", + "max_latency": "", + "cpu": "6328.1%", + "memory": "40MiB", "connections": 512, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "27.97MB/s", + "bandwidth": "27.50MB/s", "reconnects": 0, - "status_2xx": 20956790, + "status_2xx": 20600477, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -260,18 +268,19 @@ "echo-ws-limited-4096": { "framework": "tako", "language": "Rust", - "rps": 2570415, - "avg_latency": "1.12ms", - "p99_latency": "1.75ms", - "cpu": "6357.1%", - "memory": "136MiB", + "rps": 2573728, + "avg_latency": "1.11ms", + "p99_latency": "1.76ms", + "max_latency": "", + "cpu": "6101.9%", + "memory": "139MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "57.82MB/s", - "reconnects": 1284692, - "status_2xx": 12852078, + "bandwidth": "57.98MB/s", + "reconnects": 1285728, + "status_2xx": 12868641, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -279,18 +288,19 @@ "echo-ws-limited-512": { "framework": "tako", "language": "Rust", - "rps": 2156338, + "rps": 2143254, "avg_latency": "136us", - "p99_latency": "390us", - "cpu": "5894.3%", - "memory": "51MiB", + "p99_latency": "370us", + "max_latency": "", + "cpu": "5859.9%", + "memory": "50MiB", "connections": 512, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "48.52MB/s", - "reconnects": 1078197, - "status_2xx": 10781690, + "bandwidth": "48.22MB/s", + "reconnects": 1071643, + "status_2xx": 10716271, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -298,18 +308,19 @@ "echo-ws-pipeline-16384": { "framework": "tako", "language": "Rust", - "rps": 50973465, - "avg_latency": "5.08ms", - "p99_latency": "11.20ms", - "cpu": "6328.4%", - "memory": "354MiB", + "rps": 51593811, + "avg_latency": "5.03ms", + "p99_latency": "6.56ms", + "max_latency": "", + "cpu": "6078.2%", + "memory": "364MiB", "connections": 16384, "threads": 64, "duration": "5s", "pipeline": 16, - "bandwidth": "340.71MB/s", + "bandwidth": "344.83MB/s", "reconnects": 0, - "status_2xx": 254867328, + "status_2xx": 257969056, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -317,18 +328,19 @@ "echo-ws-pipeline-4096": { "framework": "tako", "language": "Rust", - "rps": 56360143, + "rps": 55968937, "avg_latency": "1.16ms", - "p99_latency": "1.65ms", - "cpu": "6592.1%", - "memory": "122MiB", + "p99_latency": "1.64ms", + "max_latency": "", + "cpu": "6260.2%", + "memory": "114MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 16, - "bandwidth": "376.14MB/s", + "bandwidth": "373.64MB/s", "reconnects": 0, - "status_2xx": 281800719, + "status_2xx": 279844688, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -336,18 +348,19 @@ "echo-ws-pipeline-512": { "framework": "tako", "language": "Rust", - "rps": 57721923, - "avg_latency": "141us", - "p99_latency": "240us", - "cpu": "6603.9%", - "memory": "40MiB", + "rps": 57394140, + "avg_latency": "142us", + "p99_latency": "250us", + "max_latency": "", + "cpu": "6602.8%", + "memory": "41MiB", "connections": 512, "threads": 64, "duration": "5s", "pipeline": 16, - "bandwidth": "385.12MB/s", + "bandwidth": "383.05MB/s", "reconnects": 0, - "status_2xx": 288609616, + "status_2xx": 286970704, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -403,67 +416,70 @@ "gateway-h3-256": { "framework": "tako", "language": "Rust", - "rps": 76483, - "avg_latency": "39.77ms", - "p99_latency": "165.95ms", - "cpu": "3389.5%", - "memory": "560MiB", + "rps": 58187, + "avg_latency": "134.06ms", + "p99_latency": "799.05ms", + "max_latency": "1.55s", + "cpu": "2636.9%", + "memory": "1.5GiB", "connections": 256, "threads": 64, - "duration": "5s", + "duration": "5.04s", "pipeline": 1, - "bandwidth": "833.82MB/s", + "bandwidth": "626.06MB/s", "reconnects": 0, - "status_2xx": 385479, + "status_2xx": 298418, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "tpl_static": 115643, - "tpl_baseline": 77095, - "tpl_json": 134917, - "tpl_async_db": 57821, - "cpu_breakdown": "proxy: 2675% 430MiB | server: 715% 130MiB" + "tpl_static": 89525, + "tpl_baseline": 59683, + "tpl_json": 104446, + "tpl_async_db": 44762, + "cpu_breakdown": "proxy: 2062% 1.3GiB | server: 575% 130MiB" }, "gateway-h3-64": { "framework": "tako", "language": "Rust", - "rps": 83042, - "avg_latency": "24.52ms", - "p99_latency": "87.01ms", - "cpu": "3448.6%", - "memory": "427MiB", + "rps": 81857, + "avg_latency": "24.82ms", + "p99_latency": "87.62ms", + "max_latency": "270.78ms", + "cpu": "3477.9%", + "memory": "488MiB", "connections": 64, "threads": 64, - "duration": "5s", + "duration": "5.01s", "pipeline": 1, - "bandwidth": "898.47MB/s", + "bandwidth": "886.71MB/s", "reconnects": 0, - "status_2xx": 416041, + "status_2xx": 410735, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "tpl_static": 124812, - "tpl_baseline": 83208, - "tpl_json": 145614, - "tpl_async_db": 62406, - "cpu_breakdown": "proxy: 2668% 332MiB | server: 781% 95MiB" + "tpl_static": 123220, + "tpl_baseline": 82147, + "tpl_json": 143757, + "tpl_async_db": 61610, + "cpu_breakdown": "proxy: 2691% 381MiB | server: 787% 107MiB" }, "json-comp-16384": { "framework": "tako", "language": "Rust", - "rps": 201312, - "avg_latency": "80.23ms", - "p99_latency": "142.20ms", - "cpu": "6485.6%", - "memory": "594MiB", + "rps": 204554, + "avg_latency": "79.09ms", + "p99_latency": "141.70ms", + "max_latency": "", + "cpu": "6501.0%", + "memory": "592MiB", "connections": 16384, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "246.61MB/s", - "input_bw": "14.97MB/s", - "reconnects": 32898, - "status_2xx": 1006560, + "bandwidth": "250.59MB/s", + "input_bw": "15.22MB/s", + "reconnects": 32773, + "status_2xx": 1022770, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -471,19 +487,20 @@ "json-comp-4096": { "framework": "tako", "language": "Rust", - "rps": 208402, - "avg_latency": "19.65ms", - "p99_latency": "31.50ms", - "cpu": "6229.1%", - "memory": "245MiB", + "rps": 204795, + "avg_latency": "19.94ms", + "p99_latency": "28.90ms", + "max_latency": "", + "cpu": "6478.6%", + "memory": "243MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "255.29MB/s", - "input_bw": "15.50MB/s", - "reconnects": 40817, - "status_2xx": 1042014, + "bandwidth": "250.89MB/s", + "input_bw": "15.23MB/s", + "reconnects": 39301, + "status_2xx": 1023976, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -529,18 +546,19 @@ "json-tls-4096": { "framework": "tako", "language": "Rust", - "rps": 1009044, - "avg_latency": "3.60ms", - "p99_latency": "40.05ms", - "cpu": "6068.4%", - "memory": "222MiB", + "rps": 1001696, + "avg_latency": "3.63ms", + "p99_latency": "39.49ms", + "max_latency": "", + "cpu": "5949.9%", + "memory": "224MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "3.43GB", + "bandwidth": "3.41GB", "reconnects": 0, - "status_2xx": 5145214, + "status_2xx": 5109362, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -548,91 +566,95 @@ "latency-10k-1024": { "framework": "tako", "language": "Rust", - "rps": 9983, - "avg_latency": "69.0us", - "p99_latency": "108.0us", - "cpu": "25.8%", - "memory": "49MiB", + "rps": 9982, + "avg_latency": "72.3us", + "p99_latency": "105.0us", + "max_latency": "", + "cpu": "26.6%", + "memory": "50MiB", "connections": 1024, "threads": 64, "duration": "20s", "pipeline": 1, "bandwidth": "1.17MB/s", "reconnects": 0, - "status_2xx": 199673, + "status_2xx": 199662, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "cpu_usec": 5388405, - "cpu_per_req_us": 26.9861, + "cpu_usec": 5571058, + "cpu_per_req_us": 27.9024, "target_rate": 10000, - "rate_ratio": 0.9983, - "p99_9_latency": "146.0us" + "rate_ratio": 0.9982, + "p99_9_latency": "141.0us" }, "latency-1m-1024": { "framework": "tako", "language": "Rust", - "rps": 997908, - "avg_latency": "103.4us", - "p99_latency": "184.0us", - "cpu": "2672.1%", - "memory": "52MiB", + "rps": 997802, + "avg_latency": "103.7us", + "p99_latency": "180.0us", + "max_latency": "", + "cpu": "2636.7%", + "memory": "49MiB", "connections": 1024, "threads": 64, "duration": "20s", "pipeline": 1, - "bandwidth": "116.76MB/s", + "bandwidth": "116.74MB/s", "reconnects": 0, - "status_2xx": 19961584, + "status_2xx": 19959501, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "cpu_usec": 485410622, - "cpu_per_req_us": 24.3172, + "cpu_usec": 480885387, + "cpu_per_req_us": 24.0931, "target_rate": 1000000, - "rate_ratio": 0.9979, - "p99_9_latency": "2637.0us" + "rate_ratio": 0.9978, + "p99_9_latency": "3041.0us" }, "latency-500k-8cpu-1024": { "framework": "tako", "language": "Rust", - "rps": 455866, - "avg_latency": "854249.2us", - "p99_latency": "1794560.0us", - "cpu": "800.2%", + "rps": 464802, + "avg_latency": "689083.8us", + "p99_latency": "1362432.0us", + "max_latency": "", + "cpu": "795.7%", "memory": "33MiB", "connections": 1024, "threads": 64, "duration": "20s", "pipeline": 1, - "bandwidth": "53.34MB/s", + "bandwidth": "54.38MB/s", "reconnects": 0, - "status_2xx": 9120268, + "status_2xx": 9298008, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "cpu_usec": 159615063, - "cpu_per_req_us": 17.5011, + "cpu_usec": 159476830, + "cpu_per_req_us": 17.1517, "target_rate": 500000, - "rate_ratio": 0.9117, - "p99_9_latency": "1925632.0us" + "rate_ratio": 0.9296, + "p99_9_latency": "1405440.0us" }, "limited-conn-4096": { "framework": "tako", "language": "Rust", - "rps": 2407752, - "avg_latency": "1.67ms", - "p99_latency": "3.29ms", - "cpu": "6144.4%", + "rps": 2421703, + "avg_latency": "1.66ms", + "p99_latency": "3.20ms", + "max_latency": "", + "cpu": "6447.1%", "memory": "154MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "270.83MB/s", - "input_bw": "185.99MB/s", - "reconnects": 1202934, - "status_2xx": 12038763, + "bandwidth": "272.41MB/s", + "input_bw": "187.07MB/s", + "reconnects": 1210538, + "status_2xx": 12108516, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -640,18 +662,19 @@ "pipelined-4096": { "framework": "tako", "language": "Rust", - "rps": 20509219, - "avg_latency": "3.20ms", - "p99_latency": "6.06ms", - "cpu": "6354.0%", - "memory": "126MiB", + "rps": 20799968, + "avg_latency": "3.15ms", + "p99_latency": "3.29ms", + "max_latency": "", + "cpu": "6767.5%", + "memory": "121MiB", "connections": 4096, "threads": 64, "duration": "5s", "pipeline": 16, - "bandwidth": "2.25GB/s", + "bandwidth": "2.29GB/s", "reconnects": 0, - "status_2xx": 102546096, + "status_2xx": 103999840, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -745,18 +768,19 @@ "static-h3-64": { "framework": "tako", "language": "Rust", - "rps": 107569, - "avg_latency": "0us", - "p99_latency": "0us", - "cpu": "2817.0%", - "memory": "175MiB", + "rps": 113798, + "avg_latency": "30.65ms", + "p99_latency": "445.38ms", + "max_latency": "3.89s", + "cpu": "3298.6%", + "memory": "172MiB", "connections": 64, "threads": 64, - "duration": "5s", + "duration": "5.01s", "pipeline": 1, - "bandwidth": "1.64GB/s", + "bandwidth": "1.74GB/s", "reconnects": 0, - "status_2xx": 538925, + "status_2xx": 570488, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -764,18 +788,19 @@ "static-tls-1024": { "framework": "tako", "language": "Rust", - "rps": 140842, - "avg_latency": "6.71ms", - "p99_latency": "39.48ms", - "cpu": "3229.7%", - "memory": "157MiB", + "rps": 136403, + "avg_latency": "6.69ms", + "p99_latency": "43.59ms", + "max_latency": "", + "cpu": "3608.4%", + "memory": "160MiB", "connections": 1024, "threads": 64, "duration": "5s", "pipeline": 1, - "bandwidth": "2.17GB", + "bandwidth": "2.10GB", "reconnects": 0, - "status_2xx": 718306, + "status_2xx": 695639, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 From 598c95a064ae2a2532f7fe7c367ce371dc5838a0 Mon Sep 17 00:00:00 2001 From: Daniel Boros Date: Sun, 15 Nov 2026 09:42:24 +0100 Subject: [PATCH 06/12] fix: prepare a supported HTTP/2 client before benchmarks --- .github/workflows/benchmark-pr.yml | 2 - scripts/benchmark.sh | 10 ++ scripts/lib/common.sh | 1 + scripts/lib/tools/h2load.sh | 27 ++++- scripts/tests/test_h2load_setup.py | 112 ++++++++++++++++++ .../content/docs/load-generators/h2/h2load.md | 10 ++ 6 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 scripts/tests/test_h2load_setup.py diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 8b84730860..0b08a099a8 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -418,8 +418,6 @@ jobs: return "".join(f.readlines()[-n:]) def composite(fw): - if fw in failed: - return "\n\nComposite score unavailable: some measurements failed; retained results are not a complete new run." text = tail(logs_dir + "/" + fw + ".composite.md", 50).strip() return "\n\n" + text if text else "" diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 84037cb6c6..8b0f832b76 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -167,6 +167,16 @@ fi framework_load_meta "$FRAMEWORK_ARG" FRAMEWORK="$FRAMEWORK_ARG" +# Client setup may build an image, so it must precede Docker's tuning restart. +for profile in "${profiles_to_run[@]}"; do + framework_subscribes_to "$profile" || continue + parse_profile "${PROFILES[$profile]}" + if [ "$(endpoint_tool "$PROF_ENDPOINT")" = h2load ]; then + h2load_prepare || fail "HTTP/2 load generator setup failed" + break + fi +done + # Framework-level image build — skipped for compose-only entries because # their compose files build the server image from the repo root context, # not from frameworks//. Covers gateway-64, gateway-h3, production-stack, diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index eaebfbd42d..d91299071e 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -54,6 +54,7 @@ if [ -z "${GCANNON_CPUS:-}" ]; then fi export GCANNON_CPUS +_H2LOAD_EXPLICIT="${H2LOAD:+true}" H2LOAD="${H2LOAD:-h2load}" H2LOAD_IMAGE="${H2LOAD_IMAGE:-h2load:1.69.0}" diff --git a/scripts/lib/tools/h2load.sh b/scripts/lib/tools/h2load.sh index 62d33f630c..54817c763c 100644 --- a/scripts/lib/tools/h2load.sh +++ b/scripts/lib/tools/h2load.sh @@ -24,13 +24,38 @@ h2load_check_version() { version=$("${cmd[@]}" --version) || return 1 if [[ ! "$version" =~ nghttp2/([0-9]+)\.([0-9]+)\.([0-9]+) ]] \ || (( BASH_REMATCH[1] < 1 || (BASH_REMATCH[1] == 1 && BASH_REMATCH[2] < 69) )); then - echo "h2load >= 1.69.0 is required: older clients can run forever after -D expires during connection setup. Update H2LOAD or build docker/h2load.Dockerfile." >&2 + echo "h2load >= 1.69.0 is required (found: $version): older clients can run forever after -D expires during connection setup. Update H2LOAD or build docker/h2load.Dockerfile." >&2 return 1 fi echo "[info] $version" H2LOAD_VERSION_CHECKED=true } +h2load_prepare() { + if h2load_check_version 2>/dev/null; then return 0; fi + # Explicit commands must not silently change to a different client. + if [ -n "$H2LOAD_CMD" ] || [ "${_H2LOAD_EXPLICIT:-}" = true ] || [ "$H2LOAD" != h2load ]; then + h2load_check_version + return $? + fi + + # The HTTP/3 build also speaks HTTP/2 when no h3 ALPN is requested. + H2LOAD="$H2LOAD_H3" + if h2load_check_version 2>/dev/null; then + echo "[info] HTTP/2 load generator: $H2LOAD (native)" + return 0 + fi + H2LOAD=h2load + + if ! docker image inspect "$H2LOAD_IMAGE" >/dev/null 2>&1; then + echo "[info] building $H2LOAD_IMAGE from docker/h2load.Dockerfile (no supported native h2load)" + docker build -t "$H2LOAD_IMAGE" -f "$ROOT_DIR/docker/h2load.Dockerfile" "$ROOT_DIR/docker" || return $? + fi + H2LOAD_CMD="docker run --rm --network host --cpuset-cpus=$GCANNON_CPUS --security-opt seccomp=unconfined --ulimit memlock=-1:-1 --ulimit nofile=1048576:1048576 -v $REQUESTS_DIR:$REQUESTS_DIR:ro $H2LOAD_IMAGE" + h2load_check_version || return $? + echo "[info] HTTP/2 load generator: $H2LOAD_IMAGE (docker)" +} + # ── Build arguments ───────────────────────────────────────────────────────── h2load_build_args() { diff --git a/scripts/tests/test_h2load_setup.py b/scripts/tests/test_h2load_setup.py new file mode 100644 index 0000000000..7acf41b5d3 --- /dev/null +++ b/scripts/tests/test_h2load_setup.py @@ -0,0 +1,112 @@ +import os +import pathlib +import subprocess +import tempfile +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[2] + + +class H2loadSetupTests(unittest.TestCase): + def prepare(self, **overrides): + harness = r''' +set -euo pipefail +source "$1/scripts/lib/common.sh" +source "$1/scripts/lib/tools/h2load.sh" +ROOT_DIR="$1" +REQUESTS_DIR=/requests +GCANNON_CPUS=2-3 +H2C_PORT=8082 +H2THREADS=2 +H2LOAD_IMAGE=h2load:test +H2LOAD_H3=h2load-h3 +native() { + if [ "$2" = --version ]; then + [ "$1" != missing ] || return 127 + echo "h2load nghttp2/$1" + else + echo "MEASURED $1 $*" + fi +} +h2load() { native "$NATIVE_VERSION" "$@"; } +h2load-h3() { native "$H3_VERSION" "$@"; } +custom() { native 1.59.0 "$@"; } +docker() { + printf '%s\n' "$*" >> "$DOCKER_LOG" + case "$1" in + image) return 1 ;; + build) return "$BUILD_EXIT" ;; + run) + if [ "${!#}" = --version ]; then + echo "h2load nghttp2/$IMAGE_VERSION" + else + echo "MEASURED docker $*" + fi + ;; + *) return 99 ;; + esac +} +h2load_prepare +mapfile -t command < <(h2load_build_args h2c 4 1 1s) +"${command[@]}" +''' + with tempfile.TemporaryDirectory() as tmp: + log = pathlib.Path(tmp) / "docker.log" + env = {key: value for key, value in os.environ.items() if key != "H2LOAD"} + result = subprocess.run( + ["bash", "-c", harness, "test", str(ROOT)], + env={**env, "H2LOAD_CMD": "", + "H2LOAD_VERSION_CHECKED": "false", "NATIVE_VERSION": "1.59.0", + "H3_VERSION": "missing", "IMAGE_VERSION": "1.69.0", + "BUILD_EXIT": "0", "DOCKER_LOG": str(log), **overrides}, + capture_output=True, text=True, check=False, + ) + calls = log.read_text() if log.exists() else "" + return result, calls + + def test_supported_native_client_runs_http2_without_docker(self): + for native, h3, selected in [ + ("1.70.0", "1.69.0", "1.70.0"), + ("1.59.0", "1.69.0", "1.69.0"), + ("missing", "1.69.0", "1.69.0"), + ]: + with self.subTest(native=native, h3=h3): + result, calls = self.prepare(NATIVE_VERSION=native, H3_VERSION=h3) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"MEASURED {selected}", result.stdout) + self.assertIn("http://localhost:8082/baseline2?a=1&b=1 -p h2c", result.stdout) + self.assertNotIn("--alpn-list=h3", result.stdout) + self.assertEqual(calls, "") + + def test_missing_or_old_native_clients_use_pinned_http2_image(self): + for native, h3 in [("1.59.0", "1.68.0"), ("missing", "missing")]: + with self.subTest(native=native, h3=h3): + result, calls = self.prepare(NATIVE_VERSION=native, H3_VERSION=h3) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("MEASURED docker", result.stdout) + self.assertIn("build -t h2load:test", calls) + self.assertIn("docker/h2load.Dockerfile", calls) + self.assertIn("--network host --cpuset-cpus=2-3", calls) + self.assertIn("-v /requests:/requests:ro", calls) + self.assertLess(calls.index("build "), calls.index("--version")) + self.assertNotIn("--alpn-list=h3", result.stdout) + + def test_explicit_commands_are_not_silently_replaced(self): + for override in [{"H2LOAD": "h2load"}, {"H2LOAD": "custom"}, {"H2LOAD_CMD": "custom"}]: + with self.subTest(override=override): + result, calls = self.prepare(H3_VERSION="1.69.0", **override) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("MEASURED", result.stdout) + self.assertEqual(calls, "") + + def test_unusable_fallback_stops_before_measurement(self): + for override in [{"BUILD_EXIT": "42"}, {"IMAGE_VERSION": "1.59.0"}]: + with self.subTest(override=override): + result, _ = self.prepare(**override) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("MEASURED", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/site/content/docs/load-generators/h2/h2load.md b/site/content/docs/load-generators/h2/h2load.md index 68bcb3c3fa..e9ef0c626d 100644 --- a/site/content/docs/load-generators/h2/h2load.md +++ b/site/content/docs/load-generators/h2/h2load.md @@ -12,6 +12,16 @@ description: "h2load from the nghttp2 project drives every HTTP/2 profile in Htt sudo apt install nghttp2-client ``` +The benchmark requires nghttp2 1.69.0 or newer to avoid a duration-mode hang +during connection setup. When the default `h2load` is missing or too old, +the full runner first tries a compatible native `h2load-h3` in HTTP/2 mode, +then builds the pinned `docker/h2load.Dockerfile` image for HTTP/2 profiles. +Other load generators keep their configured mode. Setup runs before host +tuning and the selected client is recorded in the log. + +Explicit `H2LOAD` or `H2LOAD_CMD` overrides are validated and fail if unsupported; +they are never replaced automatically. + ## How it's used For `baseline-h2`: From 5bae2d699b5e536dd4d9137ed8935948fcef332b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 15 Nov 2026 11:27:57 +0000 Subject: [PATCH 07/12] Benchmark results: 3 frameworks (baseline-h2,static-h2,baseline-h2c,json-h2c,unary-grpc,unary-grpc-tls,gateway-64,production-stack) [skip ci] --- site/data/results/tako.json | 304 +++++++++++++++++++----------------- 1 file changed, 160 insertions(+), 144 deletions(-) diff --git a/site/data/results/tako.json b/site/data/results/tako.json index 315fd7423f..8cac762c84 100644 --- a/site/data/results/tako.json +++ b/site/data/results/tako.json @@ -93,18 +93,19 @@ "baseline-h2-1024": { "framework": "tako", "language": "Rust", - "rps": 724529, - "avg_latency": "102.12ms", - "p99_latency": "428.66ms", - "cpu": "4806.1%", - "memory": "566MiB", + "rps": 719930, + "avg_latency": "112.60ms", + "p99_latency": "297.44ms", + "max_latency": "451.64ms", + "cpu": "4770.7%", + "memory": "544MiB", "connections": 1024, "threads": 64, - "duration": "5s", + "duration": "5.06s", "pipeline": 1, - "bandwidth": "18.18MB/s", + "bandwidth": "18.11MB/s", "reconnects": 0, - "status_2xx": 3658872, + "status_2xx": 3643184, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -112,18 +113,19 @@ "baseline-h2-256": { "framework": "tako", "language": "Rust", - "rps": 829933, - "avg_latency": "30.01ms", - "p99_latency": "127.79ms", - "cpu": "4755.5%", - "memory": "204MiB", + "rps": 872980, + "avg_latency": "28.83ms", + "p99_latency": "75.56ms", + "max_latency": "120.77ms", + "cpu": "4754.1%", + "memory": "208MiB", "connections": 256, "threads": 64, - "duration": "5s", + "duration": "5.03s", "pipeline": 1, - "bandwidth": "20.71MB/s", + "bandwidth": "21.79MB/s", "reconnects": 0, - "status_2xx": 4174566, + "status_2xx": 4391314, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -131,18 +133,19 @@ "baseline-h2c-1024": { "framework": "tako", "language": "Rust", - "rps": 737895, - "avg_latency": "107.49ms", - "p99_latency": "671.76ms", - "cpu": "4758.3%", - "memory": "624MiB", + "rps": 734010, + "avg_latency": "118.06ms", + "p99_latency": "320.96ms", + "max_latency": "651.85ms", + "cpu": "4776.7%", + "memory": "551MiB", "connections": 1024, "threads": 64, - "duration": "5s", + "duration": "5.06s", "pipeline": 1, - "bandwidth": "18.55MB/s", + "bandwidth": "18.46MB/s", "reconnects": 0, - "status_2xx": 3733752, + "status_2xx": 3714403, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -150,18 +153,19 @@ "baseline-h2c-256": { "framework": "tako", "language": "Rust", - "rps": 848574, - "avg_latency": "30.19ms", - "p99_latency": "142.93ms", - "cpu": "4676.3%", - "memory": "176MiB", + "rps": 865081, + "avg_latency": "29.23ms", + "p99_latency": "75.88ms", + "max_latency": "119.66ms", + "cpu": "4817.3%", + "memory": "209MiB", "connections": 256, "threads": 64, - "duration": "5s", + "duration": "5.03s", "pipeline": 1, - "bandwidth": "21.18MB/s", + "bandwidth": "21.59MB/s", "reconnects": 0, - "status_2xx": 4268331, + "status_2xx": 4351474, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -169,18 +173,19 @@ "baseline-h2c-4096": { "framework": "tako", "language": "Rust", - "rps": 0, - "avg_latency": "", - "p99_latency": "", - "cpu": "3869.5%", - "memory": "977MiB", + "rps": 690356, + "avg_latency": "251.40ms", + "p99_latency": "861.38ms", + "max_latency": "1.34s", + "cpu": "4818.1%", + "memory": "1.3GiB", "connections": 4096, "threads": 64, - "duration": "5s", + "duration": "5.08s", "pipeline": 1, - "bandwidth": "0", + "bandwidth": "17.49MB/s", "reconnects": 0, - "status_2xx": 0, + "status_2xx": 3507520, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -368,50 +373,52 @@ "gateway-64-1024": { "framework": "tako", "language": "Rust", - "rps": 68883, - "avg_latency": "411.11ms", - "p99_latency": "4.02s", - "cpu": "3822.3%", - "memory": "5.3GiB", + "rps": 70030, + "avg_latency": "409.46ms", + "p99_latency": "1.99s", + "max_latency": "3.99s", + "cpu": "3831.0%", + "memory": "5.4GiB", "connections": 1024, "threads": 64, - "duration": "5s", + "duration": "5.06s", "pipeline": 1, - "bandwidth": "818.16MB/s", + "bandwidth": "826.65MB/s", "reconnects": 0, - "status_2xx": 348551, + "status_2xx": 355298, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "tpl_static": 104565, - "tpl_baseline": 69710, - "tpl_json": 121992, - "tpl_async_db": 52282, - "cpu_breakdown": "server: 867% 731MiB | proxy: 2955% 4.6GiB" + "tpl_static": 106589, + "tpl_baseline": 71059, + "tpl_json": 124354, + "tpl_async_db": 53294, + "cpu_breakdown": "server: 869% 763MiB | proxy: 2962% 4.7GiB" }, "gateway-64-512": { "framework": "tako", "language": "Rust", - "rps": 77411, - "avg_latency": "199.07ms", - "p99_latency": "2.16s", - "cpu": "4008.5%", - "memory": "2.8GiB", + "rps": 76150, + "avg_latency": "200.80ms", + "p99_latency": "938.92ms", + "max_latency": "2.32s", + "cpu": "3900.4%", + "memory": "2.6GiB", "connections": 512, "threads": 64, - "duration": "5s", + "duration": "5.05s", "pipeline": 1, - "bandwidth": "871.72MB/s", + "bandwidth": "860.92MB/s", "reconnects": 0, - "status_2xx": 390930, + "status_2xx": 385120, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0, - "tpl_static": 117279, - "tpl_baseline": 78186, - "tpl_json": 136825, - "tpl_async_db": 58639, - "cpu_breakdown": "server: 853% 451MiB | proxy: 3155% 2.3GiB" + "tpl_static": 115536, + "tpl_baseline": 77024, + "tpl_json": 134792, + "tpl_async_db": 57768, + "cpu_breakdown": "server: 822% 346MiB | proxy: 3078% 2.2GiB" }, "gateway-h3-256": { "framework": "tako", @@ -508,18 +515,19 @@ "json-h2c-1024": { "framework": "tako", "language": "Rust", - "rps": 602946, - "avg_latency": "42.56ms", - "p99_latency": "373.07ms", - "cpu": "4047.6%", - "memory": "551MiB", + "rps": 606563, + "avg_latency": "40.05ms", + "p99_latency": "121.95ms", + "max_latency": "564.17ms", + "cpu": "4065.6%", + "memory": "483MiB", "connections": 1024, "threads": 64, - "duration": "5s", + "duration": "5.05s", "pipeline": 1, - "bandwidth": "2.01GB/s", + "bandwidth": "2.02GB/s", "reconnects": 0, - "status_2xx": 3044882, + "status_2xx": 3063253, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -527,18 +535,19 @@ "json-h2c-4096": { "framework": "tako", "language": "Rust", - "rps": 531594, - "avg_latency": "131.47ms", - "p99_latency": "2.37s", - "cpu": "3839.8%", - "memory": "1.4GiB", + "rps": 522847, + "avg_latency": "156.84ms", + "p99_latency": "670.06ms", + "max_latency": "3.98s", + "cpu": "3813.1%", + "memory": "1.3GiB", "connections": 4096, "threads": 64, - "duration": "5s", + "duration": "5.10s", "pipeline": 1, - "bandwidth": "1.78GB/s", + "bandwidth": "1.76GB/s", "reconnects": 0, - "status_2xx": 2705816, + "status_2xx": 2666598, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -682,26 +691,27 @@ "production-stack-1024": { "framework": "tako", "language": "Rust", - "rps": 47171, - "avg_latency": "331.09ms", - "p99_latency": "1.97s", - "cpu": "3451.3%", - "memory": "2.9GiB", + "rps": 47003, + "avg_latency": "345.31ms", + "p99_latency": "837.61ms", + "max_latency": "1.60s", + "cpu": "3375.5%", + "memory": "3.2GiB", "connections": 1024, "threads": 64, - "duration": "5s", + "duration": "5.05s", "pipeline": 1, - "bandwidth": "486.54MB/s", + "bandwidth": "478.07MB/s", "reconnects": 0, - "status_2xx": 238690, + "status_2xx": 237818, "status_3xx": 0, - "status_4xx": 30, + "status_4xx": 0, "status_5xx": 0, - "tpl_static": 71607, - "tpl_baseline": 23869, - "tpl_items": 119345, - "tpl_me": 23869, - "cpu_breakdown": "server: 224% 342MiB | cache: 0% 4MiB | authsvc: 266% 280MiB | edge: 2960% 2.3GiB" + "tpl_static": 71345, + "tpl_baseline": 23781, + "tpl_items": 118909, + "tpl_me": 23781, + "cpu_breakdown": "server: 239% 381MiB | cache: 0% 4MiB | authsvc: 303% 289MiB | edge: 2834% 2.6GiB" }, "production-stack-256": { "framework": "tako", @@ -730,18 +740,19 @@ "static-h2-1024": { "framework": "tako", "language": "Rust", - "rps": 130233, - "avg_latency": "223.43ms", - "p99_latency": "581.57ms", - "cpu": "2740.0%", - "memory": "1.0GiB", + "rps": 129095, + "avg_latency": "220.07ms", + "p99_latency": "416.25ms", + "max_latency": "623.61ms", + "cpu": "3172.9%", + "memory": "977MiB", "connections": 1024, "threads": 64, - "duration": "5s", + "duration": "5.04s", "pipeline": 1, - "bandwidth": "2.00GB/s", + "bandwidth": "1.98GB/s", "reconnects": 0, - "status_2xx": 657677, + "status_2xx": 650766, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -749,18 +760,19 @@ "static-h2-256": { "framework": "tako", "language": "Rust", - "rps": 131413, - "avg_latency": "61.09ms", - "p99_latency": "306.30ms", - "cpu": "3003.9%", - "memory": "318MiB", + "rps": 129744, + "avg_latency": "61.92ms", + "p99_latency": "195.15ms", + "max_latency": "312.62ms", + "cpu": "2846.6%", + "memory": "337MiB", "connections": 256, "threads": 64, - "duration": "5s", + "duration": "5.03s", "pipeline": 1, - "bandwidth": "2.01GB/s", + "bandwidth": "1.99GB/s", "reconnects": 0, - "status_2xx": 661009, + "status_2xx": 652685, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -808,18 +820,19 @@ "unary-grpc-1024": { "framework": "tako", "language": "Rust", - "rps": 510326, - "avg_latency": "155.19ms", - "p99_latency": "673.87ms", - "cpu": "5211.5%", - "memory": "720MiB", + "rps": 493020, + "avg_latency": "139.18ms", + "p99_latency": "469.42ms", + "max_latency": "737.50ms", + "cpu": "5171.3%", + "memory": "532MiB", "connections": 1024, "threads": 64, - "duration": "5s", + "duration": "5.05s", "pipeline": 1, - "bandwidth": "18.71MB/s", + "bandwidth": "18.08MB/s", "reconnects": 0, - "status_2xx": 2577147, + "status_2xx": 2490027, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -827,18 +840,19 @@ "unary-grpc-256": { "framework": "tako", "language": "Rust", - "rps": 554095, - "avg_latency": "44.86ms", - "p99_latency": "190.10ms", - "cpu": "5116.2%", - "memory": "196MiB", + "rps": 578661, + "avg_latency": "42.92ms", + "p99_latency": "110.61ms", + "max_latency": "195.51ms", + "cpu": "5128.5%", + "memory": "201MiB", "connections": 256, "threads": 64, - "duration": "5s", + "duration": "5.03s", "pipeline": 1, - "bandwidth": "20.17MB/s", + "bandwidth": "21.11MB/s", "reconnects": 0, - "status_2xx": 2781557, + "status_2xx": 2910737, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -846,18 +860,19 @@ "unary-grpc-tls-1024": { "framework": "tako", "language": "Rust", - "rps": 485284, - "avg_latency": "147.51ms", - "p99_latency": "655.26ms", - "cpu": "5040.5%", - "memory": "582MiB", + "rps": 498325, + "avg_latency": "139.36ms", + "p99_latency": "443.27ms", + "max_latency": "641.54ms", + "cpu": "5139.5%", + "memory": "584MiB", "connections": 1024, "threads": 64, - "duration": "5s", + "duration": "5.06s", "pipeline": 1, - "bandwidth": "17.84MB/s", + "bandwidth": "18.31MB/s", "reconnects": 0, - "status_2xx": 2455540, + "status_2xx": 2521757, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 @@ -865,18 +880,19 @@ "unary-grpc-tls-256": { "framework": "tako", "language": "Rust", - "rps": 564324, - "avg_latency": "44.62ms", - "p99_latency": "176.46ms", - "cpu": "5267.4%", - "memory": "220MiB", + "rps": 575891, + "avg_latency": "43.85ms", + "p99_latency": "110.69ms", + "max_latency": "179.96ms", + "cpu": "5188.0%", + "memory": "226MiB", "connections": 256, "threads": 64, - "duration": "5s", + "duration": "5.03s", "pipeline": 1, - "bandwidth": "20.58MB/s", + "bandwidth": "21.01MB/s", "reconnects": 0, - "status_2xx": 2838553, + "status_2xx": 2896888, "status_3xx": 0, "status_4xx": 0, "status_5xx": 0 From 1c6faad0dbe4496c6451152bfb60b008e1dbd0f7 Mon Sep 17 00:00:00 2001 From: Daniel Boros Date: Sun, 15 Nov 2026 19:04:22 +0100 Subject: [PATCH 08/12] fix: retain production benchmark failure diagnostics --- scripts/benchmark-lite.sh | 10 +- scripts/benchmark.sh | 41 ++------ scripts/lib/common.sh | 43 ++++++++- scripts/lib/tools/h2load.sh | 26 ++++- scripts/lib/tools/h2load_parse.py | 32 +++++++ scripts/tests/test_benchmark_logs.py | 137 +++++++++++++++++++++++++++ scripts/tests/test_benchmark_run.py | 2 + scripts/tests/test_h2load.py | 65 +++++++++++++ 8 files changed, 315 insertions(+), 41 deletions(-) create mode 100644 scripts/tests/test_benchmark_logs.py diff --git a/scripts/benchmark-lite.sh b/scripts/benchmark-lite.sh index 9b05684bed..876745dcbd 100755 --- a/scripts/benchmark-lite.sh +++ b/scripts/benchmark-lite.sh @@ -270,7 +270,8 @@ run_one() { framework_start "$endpoint" "$PROF_CPU" || return 1 if ! framework_wait_ready "$endpoint"; then warn "$FRAMEWORK did not come up for $profile; skipping" - dump_container_logs "$CONTAINER_NAME" "$FRAMEWORK" + dump_benchmark_logs + if [ "$SAVE_RESULTS" = true ]; then save_benchmark_logs "$profile" "$CONNS"; fi framework_stop return 1 fi @@ -299,10 +300,12 @@ run_one() { if [ "$run_exit" -ne 0 ]; then warn "run $run failed: load generator exited with status $run_exit" + dump_benchmark_logs continue fi if ! metrics=$("${tool//-/_}_parse" "$endpoint" "$output"); then warn "run $run failed: invalid load-generator output" + dump_benchmark_logs continue fi @@ -326,6 +329,7 @@ run_one() { if [ "$best_rps" -lt 0 ]; then warn "$profile/$CONNS: no valid measurements; no result saved" + if [ "$SAVE_RESULTS" = true ]; then save_benchmark_logs "$profile" "$CONNS"; fi framework_stop return 1 fi @@ -387,9 +391,7 @@ save_result() { EOF info "saved results/$profile/$CONNS/${FRAMEWORK}.json" - local log_dir="$ROOT_DIR/site/static/logs/$profile/$CONNS" - mkdir -p "$log_dir" - docker logs "$CONTAINER_NAME" >"$log_dir/${FRAMEWORK}.log" 2>&1 || true + save_benchmark_logs "$profile" "$CONNS" } # ── Driver: either one framework, or loop through every enabled one ──────── diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 8b0f832b76..56096b0dd3 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -279,11 +279,8 @@ run_one() { if ! framework_wait_ready "$endpoint"; then warn "$FRAMEWORK did not come up for $profile; skipping" - if $is_gateway; then - dump_compose_logs "$GATEWAY_PROJECT" - else - dump_container_logs "$CONTAINER_NAME" "$FRAMEWORK" - fi + dump_benchmark_logs + if [ "$SAVE_RESULTS" = true ]; then save_benchmark_logs "$profile" "$CONNS"; fi framework_stop $is_gateway && gateway_down return 1 @@ -339,10 +336,12 @@ run_one() { if [ "$run_exit" -ne 0 ]; then warn "run $run failed: load generator exited with status $run_exit" + dump_benchmark_logs continue fi if ! metrics=$("${tool//-/_}_parse" "$endpoint" "$output"); then warn "run $run failed: invalid load-generator output" + dump_benchmark_logs continue fi @@ -404,6 +403,11 @@ run_one() { if [ "$best_rps" -lt 0 ]; then warn "$profile/$CONNS: no valid measurements; no result saved" + if [ "$endpoint" = production-stack ]; then + h2load_diagnose "${gc_args[@]}" || true + dump_benchmark_logs + fi + if [ "$SAVE_RESULTS" = true ]; then save_benchmark_logs "$profile" "$CONNS"; fi [ -z "$_mdir" ] || rm -rf "$_mdir" if $is_gateway; then gateway_down; else framework_stop; fi return 1 @@ -638,32 +642,7 @@ save_result() { EOF info "saved results/$profile/$CONNS/${FRAMEWORK}.json" - # Persist container logs alongside results for post-mortem. - local log_dir="$ROOT_DIR/site/static/logs/$profile/$CONNS" - mkdir -p "$log_dir" - # Cap what a noisy container can write. A framework that logs per request - # produces a log proportional to throughput: sanic's static run emitted a - # traceback on every request and reached 240MB, which GitHub then refused to - # accept - the push is rejected over the 100MB file limit, after the whole - # benchmark has already run. Keep the tail, which is where a post-mortem - # looks anyway, and say so in the file when it was cut. - local log_max=${HTTPARENA_MAX_LOG_BYTES:-8000000} - local log_file="$log_dir/${FRAMEWORK}.log" - docker logs "$CONTAINER_NAME" >"$log_file" 2>&1 || true - local log_size - log_size=$(stat -c %s "$log_file" 2>/dev/null || echo 0) - if [ "$log_size" -gt "$log_max" ] 2>/dev/null; then - tail -c "$log_max" "$log_file" >"$log_file.tail" 2>/dev/null || true - { - echo "[httparena] container log was ${log_size} bytes and has been" - echo "[httparena] truncated to the last ${log_max}; a framework that" - echo "[httparena] logs per request is usually the cause." - echo - cat "$log_file.tail" - } >"$log_file" - rm -f "$log_file.tail" - warn "$FRAMEWORK $profile/$CONNS: container log was ${log_size} bytes, truncated to ${log_max}" - fi + save_benchmark_logs "$profile" "$CONNS" } # Iterate profiles × conns. diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index d91299071e..2bda2a7d4a 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -95,10 +95,8 @@ banner() { # ── Failure diagnostics ───────────────────────────────────────────────────── # -# Container logs are written to site/static/logs/ only by save_result(), which -# a server that never became ready never reaches — and framework_stop() runs -# `docker rm -f` moments later, so the evidence is gone. These print it while -# it still exists. Bounded by FAIL_LOG_TAIL: a crash-looping server can emit +# Print diagnostics before teardown removes the containers. Bounded by +# FAIL_LOG_TAIL: a crash-looping server can emit # megabytes, and this output is also what lands in the PR comment, which # quotes the last 200 lines of the run. @@ -113,6 +111,10 @@ dump_container_logs() { return 0 fi echo "─── $label — $state" + if [ "$n" = all ]; then + docker logs "$ref" 2>&1 || true + return 0 + fi logs=$(docker logs --tail "$n" "$ref" 2>&1) || true if [ -n "$logs" ]; then echo "─── $label — last $n log lines ───" @@ -133,3 +135,36 @@ dump_compose_logs() { dump_container_logs "$id" "${name:-$id}" done } + +dump_benchmark_logs() { + if [ -n "${GATEWAY_PROJECT:-}" ]; then + dump_compose_logs "$GATEWAY_PROJECT" + else + dump_container_logs "$CONTAINER_NAME" "$FRAMEWORK" + fi +} + +save_benchmark_logs() { + local profile="$1" conns="$2" + local log_dir="$ROOT_DIR/site/static/logs/$profile/$conns" + mkdir -p "$log_dir" + local log_file="$log_dir/${FRAMEWORK}.log" + if [ -n "${GATEWAY_PROJECT:-}" ]; then + FAIL_LOG_TAIL=all dump_compose_logs "$GATEWAY_PROJECT" >"$log_file" 2>&1 + else + docker logs "$CONTAINER_NAME" >"$log_file" 2>&1 || true + fi + + # Keep noisy containers below GitHub's file-size limit. + local log_max=${HTTPARENA_MAX_LOG_BYTES:-8000000} log_size + log_size=$(wc -c < "$log_file") + if [ "$log_size" -gt "$log_max" ] 2>/dev/null; then + tail -c "$log_max" "$log_file" >"$log_file.tail" + { + echo "[httparena] container log was ${log_size} bytes; keeping the last ${log_max}." + cat "$log_file.tail" + } >"$log_file" + rm -f "$log_file.tail" + warn "$FRAMEWORK $profile/$conns: container log was ${log_size} bytes, truncated to ${log_max}" + fi +} diff --git a/scripts/lib/tools/h2load.sh b/scripts/lib/tools/h2load.sh index 54817c763c..6bab708672 100644 --- a/scripts/lib/tools/h2load.sh +++ b/scripts/lib/tools/h2load.sh @@ -192,9 +192,11 @@ h2load_run() { reads_log=$(mktemp) writes_log=$(mktemp) - timeout 45 taskset -c "$GCANNON_CPUS" "${reads_argv[@]}" >"$reads_log" 2>&1 & + # Diagnostic clients reopen stdout with O_APPEND for per-request records. + # Normal stdout must append too, or its buffered summary overwrites them. + timeout 45 taskset -c "$GCANNON_CPUS" "${reads_argv[@]}" >>"$reads_log" 2>&1 & local reads_pid=$! - timeout 45 taskset -c "$GCANNON_CPUS" "${writes_argv[@]}" >"$writes_log" 2>&1 & + timeout 45 taskset -c "$GCANNON_CPUS" "${writes_argv[@]}" >>"$writes_log" 2>&1 & local writes_pid=$! local reads_exit=0 writes_exit=0 @@ -218,3 +220,23 @@ h2load_run() { h2load_parse() { python3 "${BASH_SOURCE[0]%/*}/h2load_parse.py" "$1" <<< "$2" } + +# A separate replay after a failed production profile; never a scored run. +h2load_diagnose() { + local -a args=() + local arg log exit_code=0 + for arg in "$@"; do + if [ "$arg" = --split-- ]; then args+=(--log-file=/dev/stdout); fi + args+=("$arg") + done + args+=(--log-file=/dev/stdout) + log=$(mktemp) + echo "[diagnostic] replaying production traffic with per-request status logging; excluded from results" + h2load_run "${args[@]}" >>"$log" 2>&1 || exit_code=$? + python3 "${BASH_SOURCE[0]%/*}/h2load_parse.py" --diagnostic <"$log" + rm -f "$log" + if [ "$exit_code" -ne 0 ]; then + echo "[diagnostic] load generator exited with status $exit_code" >&2 + fi + return "$exit_code" +} diff --git a/scripts/lib/tools/h2load_parse.py b/scripts/lib/tools/h2load_parse.py index 9ad5b220b3..1f15b8a94d 100644 --- a/scripts/lib/tools/h2load_parse.py +++ b/scripts/lib/tools/h2load_parse.py @@ -3,6 +3,7 @@ import re import sys +from collections import Counter from decimal import Decimal @@ -75,7 +76,38 @@ def parse(output, endpoint): return result +def diagnose(lines): + counts = {} + failures = {} + section = "requests" + for line in lines: + if line.startswith("=== h2load-"): + section = line.strip().removeprefix("=== h2load-").removesuffix(" ===") + record = re.fullmatch(r"(\d+)\t(-?\d+)\t(\d+)\s*", line) + if record: + status = int(record[2]) + counts.setdefault(section, Counter())[status] += 1 + samples = failures.setdefault(section, []) + if not 200 <= status < 300 and len(samples) < 5: + samples.append((record[1], status, record[3])) + elif line.startswith(("finished in ", "requests:", "status codes:")): + print(f"[diagnostic] {section}: {line.strip()}") + for section, statuses in counts.items(): + summary = ", ".join( + f"{'stream error' if code == -1 else f'HTTP {code}'}={count}" + for code, count in sorted(statuses.items()) + ) + print(f"[diagnostic] {section}: {summary}") + for started, status, duration in failures[section]: + print(f"[diagnostic] {section}: start_us={started} status={status} duration_us={duration}") + if not counts: + print("[diagnostic] no per-request records; replay did not complete requests") + + def main(): + if sys.argv[1] == "--diagnostic": + diagnose(sys.stdin) + return 0 try: metrics = parse(sys.stdin.read(), sys.argv[1]) except (ValueError, ArithmeticError) as error: diff --git a/scripts/tests/test_benchmark_logs.py b/scripts/tests/test_benchmark_logs.py new file mode 100644 index 0000000000..f55c4fc3d0 --- /dev/null +++ b/scripts/tests/test_benchmark_logs.py @@ -0,0 +1,137 @@ +import os +import pathlib +import re +import subprocess +import tempfile +import unittest + +from test_h2load import H2 + + +ROOT = pathlib.Path(__file__).resolve().parents[2] + + +class BenchmarkLogTests(unittest.TestCase): + def measure(self, valid, gateway=True, save=True, lite=False, noisy=False): + driver = (ROOT / "scripts" / ("benchmark-lite.sh" if lite else "benchmark.sh")).read_text() + functions = "\n".join( + re.search(rf"^{name}\(\) \{{\n.*?^\}}(?=\n\n)", driver, re.MULTILINE | re.DOTALL)[0] + for name in ("run_one", "save_result") + ) + harness = r''' +set -euo pipefail +source "$1/scripts/lib/common.sh" +source "$1/scripts/lib/tools/h2load.sh" +ROOT_DIR="$2" +RESULTS_DIR="$2/results" +parse_profile() { PROF_ENDPOINT="$ENDPOINT"; PROF_PIPELINE=1; PROF_CPU=1; PROF_REQ=1; } +endpoint_tool() { echo h2load; } +framework_start() { :; } +framework_wait_ready() { return 0; } +framework_stop() { touch "$ROOT_DIR/stopped"; } +gateway_up() { GATEWAY_PROJECT=httparena-sample-production-stack; } +gateway_down() { touch "$ROOT_DIR/stopped"; GATEWAY_PROJECT=""; } +stats_start() { :; } +stats_stop() { :; } +cpu_acct_start() { :; } +cpu_acct_stop() { :; } +sleep() { :; } +h2load_build_args() { echo ignored; } +h2load_check_version() { return 0; } +h2load_run() { printf '%s\n' "$LOAD_OUTPUT"; } +h2load_diagnose() { echo 'DIAGNOSTIC REPLAY'; } +docker() { + [ ! -f "$ROOT_DIR/stopped" ] || { echo 'logs requested after teardown'; return 1; } + case "$1" in + ps) + printf 'edge-id\nserver-id\n' + [ "$2" != -aq ] || echo auth-id ;; + inspect) + if [[ "$*" == *'{{.Name}}'* ]]; then echo "/${!#}"; + else echo 'status=exited exit=1 oom=false error='; fi ;; + logs) + case "${!#}" in + edge-id) echo 'GET /api/me: 401' ;; + auth-id) echo 'auth diagnostic' ;; + server-id|single) + if [ "$NOISY" = true ]; then + python3 -c 'print("x" * 12000)' + fi + echo 'database unavailable' ;; + *) echo "No such container: ${!#}"; return 1 ;; + esac ;; + *) echo "unexpected docker call: $*"; return 1 ;; + esac +} +FRAMEWORK=sample +CONTAINER_NAME=single +[ "$ENDPOINT" != production-stack ] || CONTAINER_NAME=nonexistent-compose-container +GATEWAY_CONTAINERS='edge-id server-id' +DISPLAY_NAME=sample +LANGUAGE=Rust +RUNS=1 +DURATION=5s +STATS_AVG_CPU=1% +STATS_PEAK_MEM=1MiB +STATS_BREAKDOWN="" +declare -A PROFILES=([production-stack]=ignored) +declare -A BEST_M +''' + output = H2 if valid else H2.replace("100 succeeded, 0 failed", "99 succeeded, 1 failed") + if gateway: + output = f"=== h2load-reads ===\n{output}\n=== h2load-writes ===\n{H2}" + with tempfile.TemporaryDirectory() as tmp: + result = subprocess.run( + ["bash", "-c", harness + functions + "\nrun_one production-stack 256 && exit 0 || exit $?\n", + "test", str(ROOT), tmp], + env={**os.environ, "LOAD_OUTPUT": output, "ENDPOINT": "production-stack" if gateway else "h2", + "SAVE_RESULTS": str(save).lower(), "NOISY": str(noisy).lower(), + "HTTPARENA_MAX_LOG_BYTES": "128" if noisy else "8000000"}, + capture_output=True, text=True, check=False, + ) + log_path = pathlib.Path(tmp) / "site/static/logs/production-stack/256/sample.log" + log = log_path.read_text() if log_path.exists() else "" + saved = (pathlib.Path(tmp) / "results/production-stack/256/sample.json").exists() + stopped = (pathlib.Path(tmp) / "stopped").exists() + return result, log, saved, stopped + + def test_compose_logs_survive_success_and_failed_measurements(self): + for valid in (False, True): + with self.subTest(valid=valid): + result, log, saved, stopped = self.measure(valid) + self.assertEqual(result.returncode, 0 if valid else 1, result.stderr) + self.assertEqual(saved, valid) + self.assertTrue(stopped) + for diagnostic in ("GET /api/me: 401", "auth diagnostic", "database unavailable"): + self.assertIn(diagnostic, log) + if not valid: + self.assertIn(diagnostic, result.stdout) + self.assertNotIn("No such container", log) + self.assertNotIn("after teardown", log) + self.assertEqual("DIAGNOSTIC REPLAY" in result.stdout, not valid) + + def test_failed_single_container_logs_respect_save_flag(self): + for lite in (False, True): + for save in (False, True): + with self.subTest(lite=lite, save=save): + result, log, saved, stopped = self.measure(False, gateway=False, save=save, lite=lite) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertFalse(saved) + self.assertTrue(stopped) + self.assertIn("database unavailable", result.stdout) + self.assertEqual("database unavailable" in log, save) + self.assertNotIn("DIAGNOSTIC REPLAY", result.stdout) + + def test_noisy_service_log_is_capped_and_keeps_the_last_error(self): + for gateway in (False, True): + with self.subTest(gateway=gateway): + result, log, saved, _ = self.measure(True, gateway=gateway, noisy=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(saved) + self.assertLess(len(log.encode()), 256) + self.assertIn("keeping the last 128", log) + self.assertIn("database unavailable", log) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_benchmark_run.py b/scripts/tests/test_benchmark_run.py index 85a3aa9ed5..86e9edce85 100644 --- a/scripts/tests/test_benchmark_run.py +++ b/scripts/tests/test_benchmark_run.py @@ -36,6 +36,8 @@ def run_measurement(self, output, exit_code=0, version_supported=True): h2load_check_version() { test "$VERSION_SUPPORTED" = true; } h2load_run() { printf '%s\\n' "$LOAD_OUTPUT"; return "$LOAD_EXIT"; } save_result() { echo SAVED; } +dump_benchmark_logs() { :; } +save_benchmark_logs() { :; } FRAMEWORK=sample CONTAINER_NAME=sample RUNS=1 diff --git a/scripts/tests/test_h2load.py b/scripts/tests/test_h2load.py index 2abb76bfeb..34eb8d1c0c 100644 --- a/scripts/tests/test_h2load.py +++ b/scripts/tests/test_h2load.py @@ -1,5 +1,7 @@ import pathlib import subprocess +import sys +import tempfile import unittest @@ -114,6 +116,69 @@ def test_split_invocation_failure_is_preserved(self): self.assertIn("=== h2load-reads ===", result.stdout) self.assertIn("=== h2load-writes ===", result.stdout) + def test_diagnostic_replay_reports_exact_statuses_without_metrics(self): + harness = r''' +source "$1" +GCANNON_CPUS=0 +timeout() { + [[ "$*" == *--log-file=/dev/stdout* ]] || return 99 + if [[ "$*" == *reads* ]]; then + printf '1000000\t200\t300\n1000001\t401\t400\n1000002\t404\t500\n' + else + printf '1000003\t204\t600\n1000004\t-1\t700\n' + fi + echo 'requests: diagnostic summary' +} +h2load_diagnose reads --split-- writes +''' + result = subprocess.run( + ["bash", "-c", harness, "test", str(ROOT / "scripts/lib/tools/h2load.sh")], + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("reads: HTTP 200=1, HTTP 401=1, HTTP 404=1", result.stdout) + self.assertIn("writes: stream error=1, HTTP 204=1", result.stdout) + self.assertIn("1000001", result.stdout) + self.assertNotIn("rps=", result.stdout) + + def test_diagnostic_replay_keeps_process_failure_visible(self): + result = subprocess.run( + ["bash", "-c", 'source "$1"; GCANNON_CPUS=0; timeout() { echo partial; return 124; }; h2load_diagnose reads --split-- writes', + "test", str(ROOT / "scripts/lib/tools/h2load.sh")], + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 124, result.stderr) + self.assertIn("status 124", result.stderr) + self.assertIn("no per-request records", result.stdout) + + def test_native_diagnostic_records_survive_normal_stdout_writes(self): + client = ''' +import os +print("starting benchmark", flush=True) +fd = os.open("/dev/stdout", os.O_WRONLY | os.O_APPEND) +os.write(fd, b"1000001\\t404\\t500\\n") +print("requests: normal buffered summary written after request log", flush=True) +os.close(fd) +''' + harness = r''' +source "$1" +GCANNON_CPUS=0 +timeout() { shift; "$@"; } +taskset() { shift 2; "$@"; } +h2load_diagnose "$2" "$3" --split-- "$2" "$3" +''' + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) / "client.py" + path.write_text(client) + result = subprocess.run( + ["bash", "-c", harness, "test", str(ROOT / "scripts/lib/tools/h2load.sh"), + sys.executable, str(path)], + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("reads: HTTP 404=1", result.stdout) + self.assertIn("writes: HTTP 404=1", result.stdout) + if __name__ == "__main__": unittest.main() From 4e02e1e2a0e1557f3df707c67eaa1d3794a62482 Mon Sep 17 00:00:00 2001 From: Daniel Boros Date: Sun, 15 Nov 2026 21:55:37 +0100 Subject: [PATCH 09/12] fix: preserve git ancestry in benchmark workflows --- .github/workflows/benchmark-pr.yml | 4 +- .github/workflows/benchmark-tools.yml | 2 + .github/workflows/pr-commands.yml | 6 +-- scripts/tests/test_workflow_merge.py | 74 +++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 scripts/tests/test_workflow_merge.py diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 212925b7c5..1efa84bc37 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -137,7 +137,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git fetch origin main --depth=1 + git fetch origin main if ! git merge --no-edit origin/main; then git merge --abort 2>/dev/null || true gh pr comment "${{ inputs.pr }}" \ @@ -158,7 +158,7 @@ jobs: # replaces staged main's copy of every file under site/data, and copying the # PR's files back over does not delete, so a file the PR removed stayed in the # index and rode along in the results commit. - git fetch origin main --depth=1 + git fetch origin main rm -rf /tmp/main_site_data mkdir -p /tmp/main_site_data git archive origin/main:site/data | tar -x -C /tmp/main_site_data diff --git a/.github/workflows/benchmark-tools.yml b/.github/workflows/benchmark-tools.yml index e34bb84455..53c4202bc3 100644 --- a/.github/workflows/benchmark-tools.yml +++ b/.github/workflows/benchmark-tools.yml @@ -4,6 +4,8 @@ on: pull_request: paths: - .github/workflows/benchmark-tools.yml + - .github/workflows/benchmark-pr.yml + - .github/workflows/pr-commands.yml - frameworks/_shared/authsvc/** - docker/h2load.Dockerfile - scripts/benchmark.sh diff --git a/.github/workflows/pr-commands.yml b/.github/workflows/pr-commands.yml index eb859eb5aa..457412b95e 100644 --- a/.github/workflows/pr-commands.yml +++ b/.github/workflows/pr-commands.yml @@ -172,7 +172,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git fetch origin main --depth=1 + git fetch origin main if ! git merge --no-commit --no-ff origin/main; then git merge --abort 2>/dev/null || true gh pr comment "${{ steps.parse.outputs.pr }}" \ @@ -296,7 +296,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git fetch origin main --depth=1 + git fetch origin main if ! git merge --no-commit --no-ff origin/main; then git merge --abort 2>/dev/null || true gh pr comment "${{ steps.parse.outputs.pr }}" \ @@ -415,7 +415,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git fetch origin main --depth=1 + git fetch origin main if ! git merge --no-commit --no-ff origin/main; then git merge --abort 2>/dev/null || true gh pr comment "${{ steps.parse.outputs.pr }}" \ diff --git a/scripts/tests/test_workflow_merge.py b/scripts/tests/test_workflow_merge.py new file mode 100644 index 0000000000..5a9c65b543 --- /dev/null +++ b/scripts/tests/test_workflow_merge.py @@ -0,0 +1,74 @@ +import os +import pathlib +import re +import shlex +import subprocess +import tempfile +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +WORKFLOWS = ("benchmark-pr.yml", "pr-commands.yml") + + +class WorkflowMergeTests(unittest.TestCase): + def merge_after_fetch(self, command, conflicting): + env = {**os.environ, "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_AUTHOR_NAME": "Test", "GIT_COMMITTER_NAME": "Test", + "GIT_AUTHOR_EMAIL": "test@example.com", "GIT_COMMITTER_EMAIL": "test@example.com"} + + def git(repo, *args, check=True): + return subprocess.run(["git", "-C", str(repo), *args], env=env, + capture_output=True, text=True, check=check) + + with tempfile.TemporaryDirectory() as tmp: + origin = pathlib.Path(tmp) / "origin" + origin.mkdir() + git(origin, "init", "-b", "main") + (origin / "shared").write_text("base\n") + git(origin, "add", ".") + git(origin, "commit", "-m", "test: base") + git(origin, "checkout", "-b", "feature") + (origin / ("shared" if conflicting else "feature")).write_text("feature\n") + git(origin, "add", ".") + git(origin, "commit", "-m", "test: feature") + git(origin, "checkout", "main") + (origin / ("shared" if conflicting else "upstream")).write_text("upstream\n") + git(origin, "add", ".") + git(origin, "commit", "-m", "test: upstream") + + checkout = pathlib.Path(tmp) / "checkout" + git(origin, "clone", "--no-local", str(origin), str(checkout)) + git(checkout, "checkout", "feature") + git(checkout, *shlex.split(command)[1:]) + result = git(checkout, "merge", "--no-commit", "--no-ff", "origin/main", check=False) + files = {p.name: p.read_text() for p in checkout.iterdir() if p.is_file()} + return result, files + + def fetch_commands(self): + for name in WORKFLOWS: + text = (ROOT / ".github/workflows" / name).read_text() + commands = re.findall(r"^\s*(git fetch origin main[^\n]*)$", text, re.MULTILINE) + self.assertTrue(commands, f"no main-fetch commands found in {name}") + for line in commands: + yield name, line.strip() + + def test_diverged_branches_with_compatible_changes_can_merge(self): + for workflow, command in self.fetch_commands(): + with self.subTest(workflow=workflow, command=command): + result, files = self.merge_after_fetch(command, conflicting=False) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(files["feature"], "feature\n") + self.assertEqual(files["upstream"], "upstream\n") + + def test_real_content_conflicts_are_still_rejected(self): + for workflow, command in self.fetch_commands(): + with self.subTest(workflow=workflow, command=command): + result, _ = self.merge_after_fetch(command, conflicting=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("CONFLICT", result.stdout) + self.assertNotIn("unrelated histories", result.stderr) + + +if __name__ == "__main__": + unittest.main() From 397762cad2e3988b67ca965638f2118164514e66 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 15 Nov 2026 21:58:14 +0000 Subject: [PATCH 10/12] Benchmark results: tako-gateway production-stack [skip ci] --- .../production-stack/1024/tako-gateway.log | 31 ++++++++++++++++++- .../production-stack/256/tako-gateway.log | 31 ++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/site/static/logs/production-stack/1024/tako-gateway.log b/site/static/logs/production-stack/1024/tako-gateway.log index fbb3c6b75d..1577380709 100644 --- a/site/static/logs/production-stack/1024/tako-gateway.log +++ b/site/static/logs/production-stack/1024/tako-gateway.log @@ -1 +1,30 @@ -Error response from daemon: No such container: httparena-bench-tako-gateway + +─── httparena-tako-gateway-production-stack-edge-1 — status=running exit=0 oom=false error= +{"level":"info","ts":1794779855.6808357,"msg":"maxprocs: Leaving GOMAXPROCS=30: CPU quota undefined"} +{"level":"info","ts":1794779855.6808484,"msg":"GOMEMLIMIT is updated","GOMEMLIMIT":243039891456,"previous":9223372036854775807} +{"level":"info","ts":1794779855.680851,"msg":"using config from file","file":"/etc/caddy/Caddyfile"} +{"level":"info","ts":1794779855.6808524,"msg":"adapted config to JSON","adapter":"caddyfile"} +{"level":"warn","ts":1794779855.680855,"logger":"admin","msg":"admin endpoint disabled"} +{"level":"info","ts":1794779855.6810577,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0x6f32cf1f780"} +{"level":"info","ts":1794779855.681719,"logger":"http.auto_https","msg":"automatic HTTPS is completely disabled for server","server_name":"srv0"} +{"level":"info","ts":1794779855.6825063,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2"]} +{"level":"info","ts":1794779855.6903713,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"} +{"level":"info","ts":1794779855.6903763,"msg":"serving initial configuration"} +{"level":"info","ts":1794779855.6973524,"logger":"tls","msg":"cleaning storage unit","storage":"FileStorage:/data/caddy"} +{"level":"info","ts":1794779855.7004461,"logger":"tls","msg":"finished cleaning storage units"} + +─── httparena-tako-gateway-production-stack-server-1 — status=running exit=0 oom=false error= + +─── httparena-tako-gateway-production-stack-cache-1 — status=running exit=0 oom=false error= +7:C 15 Nov 2026 21:57:33.387 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. +7:C 15 Nov 2026 21:57:33.387 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo +7:C 15 Nov 2026 21:57:33.387 * Redis version=7.4.8, bits=64, commit=00000000, modified=0, pid=7, just started +7:C 15 Nov 2026 21:57:33.387 * Configuration loaded +7:M 15 Nov 2026 21:57:33.387 * monotonic clock: POSIX clock_gettime +7:M 15 Nov 2026 21:57:33.388 * Running mode=standalone, port=6379. +7:M 15 Nov 2026 21:57:33.389 * Server initialized +7:M 15 Nov 2026 21:57:33.389 * Ready to accept connections tcp +[redis] seeded from /seed.txt + +─── httparena-tako-gateway-production-stack-authsvc-1 — status=running exit=0 oom=false error= +authsvc listening on 0.0.0.0:9090 (JWT HMAC-SHA256) diff --git a/site/static/logs/production-stack/256/tako-gateway.log b/site/static/logs/production-stack/256/tako-gateway.log index fbb3c6b75d..89b9659d34 100644 --- a/site/static/logs/production-stack/256/tako-gateway.log +++ b/site/static/logs/production-stack/256/tako-gateway.log @@ -1 +1,30 @@ -Error response from daemon: No such container: httparena-bench-tako-gateway + +─── httparena-tako-gateway-production-stack-edge-1 — status=running exit=0 oom=false error= +{"level":"info","ts":1794779814.9299517,"msg":"maxprocs: Leaving GOMAXPROCS=30: CPU quota undefined"} +{"level":"info","ts":1794779814.9301875,"msg":"GOMEMLIMIT is updated","GOMEMLIMIT":243039891456,"previous":9223372036854775807} +{"level":"info","ts":1794779814.9301906,"msg":"using config from file","file":"/etc/caddy/Caddyfile"} +{"level":"info","ts":1794779814.9301925,"msg":"adapted config to JSON","adapter":"caddyfile"} +{"level":"warn","ts":1794779814.9301949,"logger":"admin","msg":"admin endpoint disabled"} +{"level":"info","ts":1794779814.9335017,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0x1001bfc6b480"} +{"level":"info","ts":1794779814.9349117,"logger":"http.auto_https","msg":"automatic HTTPS is completely disabled for server","server_name":"srv0"} +{"level":"info","ts":1794779814.9386065,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2"]} +{"level":"info","ts":1794779814.9551582,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"} +{"level":"info","ts":1794779814.955167,"msg":"serving initial configuration"} +{"level":"info","ts":1794779814.958393,"logger":"tls","msg":"cleaning storage unit","storage":"FileStorage:/data/caddy"} +{"level":"info","ts":1794779814.962378,"logger":"tls","msg":"finished cleaning storage units"} + +─── httparena-tako-gateway-production-stack-server-1 — status=running exit=0 oom=false error= + +─── httparena-tako-gateway-production-stack-cache-1 — status=running exit=0 oom=false error= +7:C 15 Nov 2026 21:56:52.498 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. +7:C 15 Nov 2026 21:56:52.498 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo +7:C 15 Nov 2026 21:56:52.498 * Redis version=7.4.8, bits=64, commit=00000000, modified=0, pid=7, just started +7:C 15 Nov 2026 21:56:52.498 * Configuration loaded +7:M 15 Nov 2026 21:56:52.498 * monotonic clock: POSIX clock_gettime +7:M 15 Nov 2026 21:56:52.502 * Running mode=standalone, port=6379. +7:M 15 Nov 2026 21:56:52.502 * Server initialized +7:M 15 Nov 2026 21:56:52.503 * Ready to accept connections tcp +[redis] seeded from /seed.txt + +─── httparena-tako-gateway-production-stack-authsvc-1 — status=running exit=0 oom=false error= +authsvc listening on 0.0.0.0:9090 (JWT HMAC-SHA256) From b8e984726d8c1bb2f65d985329cae1e43191c925 Mon Sep 17 00:00:00 2001 From: Daniel Boros Date: Wed, 18 Nov 2026 00:33:40 +0100 Subject: [PATCH 11/12] fix: compare production failures with a drained control run --- scripts/benchmark.sh | 1 + scripts/lib/tools/h2load.sh | 34 ++++++++++++++++----- scripts/tests/test_h2load.py | 58 +++++++++++++++++++++++++++++++----- 3 files changed, 78 insertions(+), 15 deletions(-) diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 56096b0dd3..d3ff3ba232 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -405,6 +405,7 @@ run_one() { warn "$profile/$CONNS: no valid measurements; no result saved" if [ "$endpoint" = production-stack ]; then h2load_diagnose "${gc_args[@]}" || true + h2load_diagnose --fixed-requests "${gc_args[@]}" || true dump_benchmark_logs fi if [ "$SAVE_RESULTS" = true ]; then save_benchmark_logs "$profile" "$CONNS"; fi diff --git a/scripts/lib/tools/h2load.sh b/scripts/lib/tools/h2load.sh index 6bab708672..d86bbd4f42 100644 --- a/scripts/lib/tools/h2load.sh +++ b/scripts/lib/tools/h2load.sh @@ -223,15 +223,35 @@ h2load_parse() { # A separate replay after a failed production profile; never a scored run. h2load_diagnose() { - local -a args=() - local arg log exit_code=0 - for arg in "$@"; do - if [ "$arg" = --split-- ]; then args+=(--log-file=/dev/stdout); fi - args+=("$arg") + local -a args=() command=() + local arg log duration_index exit_code=0 request_count="" + local description="replaying production traffic with per-request status logging" + if [ "${1:-}" = --fixed-requests ]; then + shift + request_count=100000 + description="fixed-request control: 100000 reads and 10000 writes, draining in-flight requests" + fi + for arg in "$@" --split--; do + if [ "$arg" != --split-- ]; then + command+=("$arg") + continue + fi + if [ -n "$request_count" ]; then + # Replace only the generated suffix; wrappers may have their own -D. + duration_index=$((${#command[@]} - 2)) + if [ "$duration_index" -lt 0 ] || [ "${command[$duration_index]}" != -D ]; then + echo "[diagnostic] fixed-request control requires a trailing -D duration" >&2 + return 2 + fi + command=("${command[@]:0:$duration_index}" -n "$request_count") + request_count=10000 + fi + if [ "${#args[@]}" -gt 0 ]; then args+=(--split--); fi + args+=("${command[@]}" --log-file=/dev/stdout) + command=() done - args+=(--log-file=/dev/stdout) log=$(mktemp) - echo "[diagnostic] replaying production traffic with per-request status logging; excluded from results" + echo "[diagnostic] $description; excluded from results" h2load_run "${args[@]}" >>"$log" 2>&1 || exit_code=$? python3 "${BASH_SOURCE[0]%/*}/h2load_parse.py" --diagnostic <"$log" rm -f "$log" diff --git a/scripts/tests/test_h2load.py b/scripts/tests/test_h2load.py index 34eb8d1c0c..c2a63c1602 100644 --- a/scripts/tests/test_h2load.py +++ b/scripts/tests/test_h2load.py @@ -1,3 +1,4 @@ +import json import pathlib import subprocess import sys @@ -142,14 +143,55 @@ def test_diagnostic_replay_reports_exact_statuses_without_metrics(self): self.assertNotIn("rps=", result.stdout) def test_diagnostic_replay_keeps_process_failure_visible(self): - result = subprocess.run( - ["bash", "-c", 'source "$1"; GCANNON_CPUS=0; timeout() { echo partial; return 124; }; h2load_diagnose reads --split-- writes', - "test", str(ROOT / "scripts/lib/tools/h2load.sh")], - capture_output=True, text=True, check=False, - ) - self.assertEqual(result.returncode, 124, result.stderr) - self.assertIn("status 124", result.stderr) - self.assertIn("no per-request records", result.stdout) + for mode in [[], ["--fixed-requests"]]: + with self.subTest(mode=mode): + result = subprocess.run( + ["bash", "-c", 'source "$1"; shift; GCANNON_CPUS=0; timeout() { echo partial; return 124; }; h2load_diagnose "$@" reads -D 5 --split-- writes -D 5', + "test", str(ROOT / "scripts/lib/tools/h2load.sh"), *mode], + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 124, result.stderr) + self.assertIn("status 124", result.stderr) + self.assertIn("no per-request records", result.stdout) + + def test_fixed_request_diagnostic_drains_both_workloads_without_publishing_scores(self): + client = ''' +import json, pathlib, sys +if sys.argv[1:3] != ["-D", "wrapper-directory"]: + raise SystemExit("wrapper-owned -D option was removed") +role, directory = sys.argv[3:5] +pathlib.Path(directory, role + ".json").write_text(json.dumps(sys.argv[5:])) +status = 499 if role == "reads" else 204 +print(f"1000001\\t{status}\\t500") +''' + harness = r''' +source "$1" +GCANNON_CPUS=0 +timeout() { shift; "$@"; } +taskset() { shift 2; "$@"; } +h2load_diagnose --fixed-requests \ + "$2" "$3" -D wrapper-directory reads "$4" -c 256 -H "Authorization: Bearer fixture" -D 5 \ + --split-- "$2" "$3" -D wrapper-directory writes "$4" -c 32 -d "body fixture.json" -D 5 +''' + with tempfile.TemporaryDirectory() as tmp: + fake = pathlib.Path(tmp) / "client.py" + fake.write_text(client) + result = subprocess.run( + ["bash", "-c", harness, "test", str(ROOT / "scripts/lib/tools/h2load.sh"), + sys.executable, str(fake), tmp], + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + for role, count, connections in [("reads", "100000", "256"), ("writes", "10000", "32")]: + args = json.loads((pathlib.Path(tmp) / (role + ".json")).read_text()) + self.assertNotIn("-D", args) + self.assertEqual(args[args.index("-n") + 1], count) + self.assertEqual(args[args.index("-c") + 1], connections) + self.assertIn("--log-file=/dev/stdout", args) + self.assertIn("Authorization: Bearer fixture" if role == "reads" else "body fixture.json", args) + self.assertIn("reads: HTTP 499=1", result.stdout) + self.assertIn("writes: HTTP 204=1", result.stdout) + self.assertNotIn("rps=", result.stdout) def test_native_diagnostic_records_survive_normal_stdout_writes(self): client = ''' From 2a75a91bd4b5a3045df4242a72535d6112243793 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 18 Nov 2026 09:35:35 +0000 Subject: [PATCH 12/12] Benchmark results: tako-gateway production-stack [skip ci] --- site/data/frameworks.json | 2 +- site/data/results/tako.json | 29 +++++++------- .../production-stack/1024/tako-gateway.log | 40 +++++++++---------- .../production-stack/256/tako-gateway.log | 40 +++++++++---------- 4 files changed, 56 insertions(+), 55 deletions(-) diff --git a/site/data/frameworks.json b/site/data/frameworks.json index 36e97c43e9..e1470c30ea 100644 --- a/site/data/frameworks.json +++ b/site/data/frameworks.json @@ -53,7 +53,7 @@ }, "aegon": { "dir": "aegon", - "description": "Linux-native C++26 asynchronous web framework powered by io_uring multishot primitives, coroutines, stepped SIMD vectorization, Glaze JSON serialization, and native zero-overhead gRPC over HTTP/2 with kTLS TX offload.", + "description": "Linux-native C++26 asynchronous web framework on io_uring: one event loop per core with multishot accept and recv over provided buffer rings, requests parsed in place from the receive buffer, coroutine handlers with userspace timers, Glaze JSON serialization, and gzip/deflate (libdeflate) and brotli compression middleware. Static files are served from a per-worker cache that follows the disk through inotify, with precompressed .br/.gz variants. TLS 1.3 through OpenSSL with a zero-copy connection BIO and ALPN; HTTP/2 and HTTP/3 (QUIC) with pooled streams; WebSocket; native gRPC over HTTP/2.", "repo": "https://github.com/UdayKhare09/Aegon", "type": "experimental", "engine": "io_uring", diff --git a/site/data/results/tako.json b/site/data/results/tako.json index 8cac762c84..4df3a86c66 100644 --- a/site/data/results/tako.json +++ b/site/data/results/tako.json @@ -716,26 +716,27 @@ "production-stack-256": { "framework": "tako", "language": "Rust", - "rps": 54377, - "avg_latency": "76.92ms", - "p99_latency": "528.48ms", - "cpu": "3236.0%", - "memory": "974MiB", + "rps": 54205, + "avg_latency": "77.48ms", + "p99_latency": "223.21ms", + "max_latency": "575.38ms", + "cpu": "3206.2%", + "memory": "1003MiB", "connections": 256, "threads": 64, - "duration": "5s", + "duration": "5.02s", "pipeline": 1, - "bandwidth": "522.86MB/s", + "bandwidth": "520.74MB/s", "reconnects": 0, - "status_2xx": 272973, + "status_2xx": 272425, "status_3xx": 0, - "status_4xx": 1, + "status_4xx": 0, "status_5xx": 0, - "tpl_static": 81891, - "tpl_baseline": 27297, - "tpl_items": 136486, - "tpl_me": 27297, - "cpu_breakdown": "server: 141% 94MiB | cache: 0% 14MiB | authsvc: 266% 69MiB | edge: 2828% 798MiB" + "tpl_static": 81727, + "tpl_baseline": 27242, + "tpl_items": 136212, + "tpl_me": 27242, + "cpu_breakdown": "server: 141% 94MiB | cache: 0% 13MiB | authsvc: 248% 58MiB | edge: 2817% 839MiB" }, "static-h2-1024": { "framework": "tako", diff --git a/site/static/logs/production-stack/1024/tako-gateway.log b/site/static/logs/production-stack/1024/tako-gateway.log index 1577380709..9bb68ed5a2 100644 --- a/site/static/logs/production-stack/1024/tako-gateway.log +++ b/site/static/logs/production-stack/1024/tako-gateway.log @@ -1,29 +1,29 @@ ─── httparena-tako-gateway-production-stack-edge-1 — status=running exit=0 oom=false error= -{"level":"info","ts":1794779855.6808357,"msg":"maxprocs: Leaving GOMAXPROCS=30: CPU quota undefined"} -{"level":"info","ts":1794779855.6808484,"msg":"GOMEMLIMIT is updated","GOMEMLIMIT":243039891456,"previous":9223372036854775807} -{"level":"info","ts":1794779855.680851,"msg":"using config from file","file":"/etc/caddy/Caddyfile"} -{"level":"info","ts":1794779855.6808524,"msg":"adapted config to JSON","adapter":"caddyfile"} -{"level":"warn","ts":1794779855.680855,"logger":"admin","msg":"admin endpoint disabled"} -{"level":"info","ts":1794779855.6810577,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0x6f32cf1f780"} -{"level":"info","ts":1794779855.681719,"logger":"http.auto_https","msg":"automatic HTTPS is completely disabled for server","server_name":"srv0"} -{"level":"info","ts":1794779855.6825063,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2"]} -{"level":"info","ts":1794779855.6903713,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"} -{"level":"info","ts":1794779855.6903763,"msg":"serving initial configuration"} -{"level":"info","ts":1794779855.6973524,"logger":"tls","msg":"cleaning storage unit","storage":"FileStorage:/data/caddy"} -{"level":"info","ts":1794779855.7004461,"logger":"tls","msg":"finished cleaning storage units"} +{"level":"info","ts":1794994493.6816607,"msg":"maxprocs: Leaving GOMAXPROCS=30: CPU quota undefined"} +{"level":"info","ts":1794994493.681677,"msg":"GOMEMLIMIT is updated","GOMEMLIMIT":243039533875,"previous":9223372036854775807} +{"level":"info","ts":1794994493.6816792,"msg":"using config from file","file":"/etc/caddy/Caddyfile"} +{"level":"info","ts":1794994493.681681,"msg":"adapted config to JSON","adapter":"caddyfile"} +{"level":"warn","ts":1794994493.6816835,"logger":"admin","msg":"admin endpoint disabled"} +{"level":"info","ts":1794994493.681928,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0x1c9d6033bb00"} +{"level":"info","ts":1794994493.6820471,"logger":"http.auto_https","msg":"automatic HTTPS is completely disabled for server","server_name":"srv0"} +{"level":"info","ts":1794994493.682789,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2"]} +{"level":"info","ts":1794994493.6940367,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"} +{"level":"info","ts":1794994493.694043,"msg":"serving initial configuration"} +{"level":"info","ts":1794994493.6967988,"logger":"tls","msg":"cleaning storage unit","storage":"FileStorage:/data/caddy"} +{"level":"info","ts":1794994493.6999562,"logger":"tls","msg":"finished cleaning storage units"} ─── httparena-tako-gateway-production-stack-server-1 — status=running exit=0 oom=false error= ─── httparena-tako-gateway-production-stack-cache-1 — status=running exit=0 oom=false error= -7:C 15 Nov 2026 21:57:33.387 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. -7:C 15 Nov 2026 21:57:33.387 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo -7:C 15 Nov 2026 21:57:33.387 * Redis version=7.4.8, bits=64, commit=00000000, modified=0, pid=7, just started -7:C 15 Nov 2026 21:57:33.387 * Configuration loaded -7:M 15 Nov 2026 21:57:33.387 * monotonic clock: POSIX clock_gettime -7:M 15 Nov 2026 21:57:33.388 * Running mode=standalone, port=6379. -7:M 15 Nov 2026 21:57:33.389 * Server initialized -7:M 15 Nov 2026 21:57:33.389 * Ready to accept connections tcp +7:C 18 Nov 2026 09:34:51.401 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. +7:C 18 Nov 2026 09:34:51.401 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo +7:C 18 Nov 2026 09:34:51.401 * Redis version=7.4.8, bits=64, commit=00000000, modified=0, pid=7, just started +7:C 18 Nov 2026 09:34:51.401 * Configuration loaded +7:M 18 Nov 2026 09:34:51.402 * monotonic clock: POSIX clock_gettime +7:M 18 Nov 2026 09:34:51.402 * Running mode=standalone, port=6379. +7:M 18 Nov 2026 09:34:51.403 * Server initialized +7:M 18 Nov 2026 09:34:51.403 * Ready to accept connections tcp [redis] seeded from /seed.txt ─── httparena-tako-gateway-production-stack-authsvc-1 — status=running exit=0 oom=false error= diff --git a/site/static/logs/production-stack/256/tako-gateway.log b/site/static/logs/production-stack/256/tako-gateway.log index 89b9659d34..7ea25ae369 100644 --- a/site/static/logs/production-stack/256/tako-gateway.log +++ b/site/static/logs/production-stack/256/tako-gateway.log @@ -1,29 +1,29 @@ ─── httparena-tako-gateway-production-stack-edge-1 — status=running exit=0 oom=false error= -{"level":"info","ts":1794779814.9299517,"msg":"maxprocs: Leaving GOMAXPROCS=30: CPU quota undefined"} -{"level":"info","ts":1794779814.9301875,"msg":"GOMEMLIMIT is updated","GOMEMLIMIT":243039891456,"previous":9223372036854775807} -{"level":"info","ts":1794779814.9301906,"msg":"using config from file","file":"/etc/caddy/Caddyfile"} -{"level":"info","ts":1794779814.9301925,"msg":"adapted config to JSON","adapter":"caddyfile"} -{"level":"warn","ts":1794779814.9301949,"logger":"admin","msg":"admin endpoint disabled"} -{"level":"info","ts":1794779814.9335017,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0x1001bfc6b480"} -{"level":"info","ts":1794779814.9349117,"logger":"http.auto_https","msg":"automatic HTTPS is completely disabled for server","server_name":"srv0"} -{"level":"info","ts":1794779814.9386065,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2"]} -{"level":"info","ts":1794779814.9551582,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"} -{"level":"info","ts":1794779814.955167,"msg":"serving initial configuration"} -{"level":"info","ts":1794779814.958393,"logger":"tls","msg":"cleaning storage unit","storage":"FileStorage:/data/caddy"} -{"level":"info","ts":1794779814.962378,"logger":"tls","msg":"finished cleaning storage units"} +{"level":"info","ts":1794994457.1503863,"msg":"maxprocs: Leaving GOMAXPROCS=30: CPU quota undefined"} +{"level":"info","ts":1794994457.1506033,"msg":"GOMEMLIMIT is updated","GOMEMLIMIT":243039533875,"previous":9223372036854775807} +{"level":"info","ts":1794994457.150606,"msg":"using config from file","file":"/etc/caddy/Caddyfile"} +{"level":"info","ts":1794994457.1506073,"msg":"adapted config to JSON","adapter":"caddyfile"} +{"level":"warn","ts":1794994457.15061,"logger":"admin","msg":"admin endpoint disabled"} +{"level":"info","ts":1794994457.153106,"logger":"tls.cache.maintenance","msg":"started background certificate maintenance","cache":"0x17d122d6a800"} +{"level":"info","ts":1794994457.1536825,"logger":"http.auto_https","msg":"automatic HTTPS is completely disabled for server","server_name":"srv0"} +{"level":"info","ts":1794994457.1576276,"logger":"http.log","msg":"server running","name":"srv0","protocols":["h1","h2"]} +{"level":"info","ts":1794994457.168737,"msg":"autosaved config (load with --resume flag)","file":"/config/caddy/autosave.json"} +{"level":"info","ts":1794994457.168742,"msg":"serving initial configuration"} +{"level":"info","ts":1794994457.171725,"logger":"tls","msg":"cleaning storage unit","storage":"FileStorage:/data/caddy"} +{"level":"info","ts":1794994457.1747549,"logger":"tls","msg":"finished cleaning storage units"} ─── httparena-tako-gateway-production-stack-server-1 — status=running exit=0 oom=false error= ─── httparena-tako-gateway-production-stack-cache-1 — status=running exit=0 oom=false error= -7:C 15 Nov 2026 21:56:52.498 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. -7:C 15 Nov 2026 21:56:52.498 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo -7:C 15 Nov 2026 21:56:52.498 * Redis version=7.4.8, bits=64, commit=00000000, modified=0, pid=7, just started -7:C 15 Nov 2026 21:56:52.498 * Configuration loaded -7:M 15 Nov 2026 21:56:52.498 * monotonic clock: POSIX clock_gettime -7:M 15 Nov 2026 21:56:52.502 * Running mode=standalone, port=6379. -7:M 15 Nov 2026 21:56:52.502 * Server initialized -7:M 15 Nov 2026 21:56:52.503 * Ready to accept connections tcp +7:C 18 Nov 2026 09:34:14.693 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. +7:C 18 Nov 2026 09:34:14.693 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo +7:C 18 Nov 2026 09:34:14.693 * Redis version=7.4.8, bits=64, commit=00000000, modified=0, pid=7, just started +7:C 18 Nov 2026 09:34:14.693 * Configuration loaded +7:M 18 Nov 2026 09:34:14.693 * monotonic clock: POSIX clock_gettime +7:M 18 Nov 2026 09:34:14.696 * Running mode=standalone, port=6379. +7:M 18 Nov 2026 09:34:14.696 * Server initialized +7:M 18 Nov 2026 09:34:14.697 * Ready to accept connections tcp [redis] seeded from /seed.txt ─── httparena-tako-gateway-production-stack-authsvc-1 — status=running exit=0 oom=false error=