diff --git a/README.md b/README.md index c55ce9ab47..d2aa9329b7 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ The reply states which baseline it used, and profiles the other framework does n | Category | Profiles | Description | |----------|----------|-------------| | Connection | `baseline`, `pipelined` *, `limited-conn` | Mixed GET/POST with query parsing (4K conns), 16× batched pipelining (reference-only, shown faded, excluded from the composite score), short-lived connections that close after 10 requests | -| Concurrency | `async` | `GET /delay/{ms}` with the wait named in the route, 32K held connections. Isolates what a framework does while a request is pending, with no database or I/O in the way. Scored for frameworks and engines, not for infrastructure, since the spec settled at a flat 10 ms over 32K connections | +| Concurrency | `async`, `connections-1m` *, `connections-1m-zrk` * | `GET /delay/{ms}` with the wait named in the route, 32K held connections. Isolates what a framework does while a request is pending, with no database or I/O in the way. Scored for frameworks and engines, not for infrastructure, since the spec settled at a flat 10 ms over 32K connections. `connections-1m` holds one million keep-alive connections at once with 250K req/s paced across them (one request per connection every 4 s), spread over 32 loopback addresses: opt-in and reference-only, it measures memory per held connection, CPU and tail latency, and needs a listener on 0.0.0.0 or [::]. `connections-1m-zrk` is the same test with zrk 3.0.2 as the load generator instead of gcannon | | Efficiency | `latency-1m`, `latency-10k`, `latency-500k-8cpu` * | One million req/s offered at a fixed rate with zrk; the metric is CPU spent per request, read exactly from the container's cgroup rather than sampled. Everyone who finishes serves the same million, so what separates them is the bill. Scored on its own 0-100 formula rather than on rps, since the rate is pinned. `latency-10k` is the identical setup - same cores, connections and endpoint - at a near-idle 10K req/s, so what it reports is the standing cost of being a running server rather than of serving load, and it is scored the same way. Both score CPU at 0.50, p99 at 0.25 and mean latency at 0.25, the mean being the queueing signal: ten times the best mean is worth nothing. `latency-500k-8cpu` pins 500K req/s with the server on four cores plus their SMT siblings, so the load sits near saturation and the queue a server builds shows in its mean; reference-only while the first entries are measured | | Workload | `json-comp`, `json-tls`, `8gbit`, `static-tls` * | JSON serialization, gzip/brotli compression, HTTP/1.1 over TLS, a 10 KB body echoed over TLS at a pinned 50K req/s so ingest and egress load together - scored on the same 0-100 formula as the latency profiles, since its rate is pinned too, 20-file static asset serving over TLS (reference-only for frameworks and engines, scored for infrastructure) | | Database | `async-db` * | Async Postgres sequential scan. Reference-only: the database and its driver dominate it far more than the framework does, so it is measured and shown but no longer decides the ranking | diff --git a/docker/gcannon.Dockerfile b/docker/gcannon.Dockerfile index 389f75ff71..e8779ae4b7 100644 --- a/docker/gcannon.Dockerfile +++ b/docker/gcannon.Dockerfile @@ -5,8 +5,10 @@ ARG GCANNON_REF=main # stale git-clone layer. Without this, edits pushed to the gcannon repo # are invisible until the user manually runs --no-cache. ARG CACHE_BUST=0 +# libssl-dev: gcannon 0.7.0 and later do the TLS handshake with OpenSSL (the +# runtime image's libssl3 serves the binary), and fail to build without it. RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc libc6-dev make git ca-certificates && rm -rf /var/lib/apt/lists/* + gcc libc6-dev make git ca-certificates libssl-dev && rm -rf /var/lib/apt/lists/* WORKDIR /deps RUN git clone --branch liburing-2.9 --depth 1 https://github.com/axboe/liburing.git && \ cd liburing && ./configure --prefix=/usr && make -j"$(nproc)" -C src && make install -C src diff --git a/docker/zrk.Dockerfile b/docker/zrk.Dockerfile index 567452ac8b..c88415bbd0 100644 --- a/docker/zrk.Dockerfile +++ b/docker/zrk.Dockerfile @@ -10,6 +10,9 @@ # across entries and across rounds, so a generator that drifts under it # invalidates every comparison it was measured against, including old ones. # Bumping the version is a deliberate act that re-baselines the profile. +# +# connections-1m-zrk builds this file a second time as zrk:3.0.2, passing +# ZRK_VERSION and ZRK_SHA256 (tools/zrk.sh), and leaves zrk:local as it is. FROM ubuntu:24.04 AS build ARG ZRK_VERSION=2.3.0 ARG ZRK_SHA256=ab6f1c1ce34ce73f52afd106419d15e5c494bd1cedd475afa56babb85b09a816 diff --git a/frameworks/actix/meta.json b/frameworks/actix/meta.json index 4aa8f82ac5..a2848f2dc6 100644 --- a/frameworks/actix/meta.json +++ b/frameworks/actix/meta.json @@ -19,6 +19,8 @@ "baseline", "async", "latency-1m", + "connections-1m", + "connections-1m-zrk", "latency-10k", "latency-500k-8cpu", "pipelined", diff --git a/frameworks/aspnet-minimal/meta.json b/frameworks/aspnet-minimal/meta.json index 8a4f91cfcb..ce54dbc96c 100644 --- a/frameworks/aspnet-minimal/meta.json +++ b/frameworks/aspnet-minimal/meta.json @@ -20,6 +20,8 @@ "baseline", "pipelined", "limited-conn", "async", "latency-1m", "latency-10k", "latency-500k-8cpu", + "connections-1m", + "connections-1m-zrk", "json-comp", "json-tls", "8gbit", diff --git a/frameworks/bun/meta.json b/frameworks/bun/meta.json index baeb3af6a7..249a709f8e 100644 --- a/frameworks/bun/meta.json +++ b/frameworks/bun/meta.json @@ -18,6 +18,8 @@ "baseline", "async", "latency-1m", "latency-10k", "latency-500k-8cpu", + "connections-1m", + "connections-1m-zrk", "pipelined", "limited-conn", "json-comp", diff --git a/frameworks/fib/meta.json b/frameworks/fib/meta.json index 83ddc229c1..1b71f5c631 100644 --- a/frameworks/fib/meta.json +++ b/frameworks/fib/meta.json @@ -20,6 +20,8 @@ "limited-conn", "async", "latency-1m", + "connections-1m", + "connections-1m-zrk", "latency-10k", "latency-500k-8cpu", "json-comp", diff --git a/frameworks/genhttp-ioxide/Dockerfile b/frameworks/genhttp-ioxide/Dockerfile index 4f15e0e31a..1c9d74362c 100644 --- a/frameworks/genhttp-ioxide/Dockerfile +++ b/frameworks/genhttp-ioxide/Dockerfile @@ -3,7 +3,7 @@ FROM mcr.microsoft.com/dotnet/sdk:11.0 AS build # GenHTTP built from source: the branch of Kaliumhexacyanoferrat/GenHTTP#953 # (update-ioxide-0.20.396) rather than the 11.3.0 release. Pinned to one commit, # because a build on cached layers would otherwise keep whichever it cloned first. -ARG GENHTTP_COMMIT=28b0cd107bb36321408a3a1f892f3bb45225e9d7 +ARG GENHTTP_COMMIT=5967f5c6bf64d3d67a0eed7a669fa2f8c846dd02 RUN git init -q /genhttp \ && git -C /genhttp fetch -q --depth 1 https://github.com/Kaliumhexacyanoferrat/GenHTTP.git "$GENHTTP_COMMIT" \ && git -C /genhttp checkout -q FETCH_HEAD diff --git a/frameworks/genhttp-ioxide/meta.json b/frameworks/genhttp-ioxide/meta.json index 1d3c123f75..909c2d5af6 100644 --- a/frameworks/genhttp-ioxide/meta.json +++ b/frameworks/genhttp-ioxide/meta.json @@ -19,6 +19,8 @@ "baseline", "async", "latency-1m", + "connections-1m", + "connections-1m-zrk", "latency-10k", "latency-500k-8cpu", "pipelined", diff --git a/frameworks/node/meta.json b/frameworks/node/meta.json index bd36d466ce..9bf2b98874 100644 --- a/frameworks/node/meta.json +++ b/frameworks/node/meta.json @@ -8,6 +8,8 @@ "enabled": true, "tests": [ "baseline", "latency-1m", "latency-10k", + "connections-1m", + "connections-1m-zrk", "pipelined", "limited-conn", "json-comp", diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 642846464b..4fe3ca6101 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -165,7 +165,7 @@ FRAMEWORK="$FRAMEWORK_ARG" # and any combination thereof. _has_isolated_test=false for t in baseline pipelined limited-conn json-comp json-tls upload \ - static-tls async-db async latency-1m latency-10k latency-500k-8cpu \ + static-tls async-db async latency-1m latency-10k latency-500k-8cpu connections-1m connections-1m-zrk \ baseline-h2 static-h2 baseline-h2c json-h2c \ baseline-h3 static-h3 \ unary-grpc unary-grpc-tls \ @@ -196,10 +196,40 @@ if { framework_subscribes_to "latency-1m" || framework_subscribes_to "latency-10 info "zrk: docker mode ($ZRK_IMAGE)" fi +# connections-1m-zrk runs its own zrk (ZRK_CONN1M_VERSION, tools/zrk.sh): the +# same Dockerfile at another version, so the pinned image above is untouched. +# A failed build is not fatal here: the fan-out asks the image for --version +# and reports the profile as not run, and the other profiles go ahead. +if framework_subscribes_to "connections-1m-zrk" \ + && [[ " ${profiles_to_run[*]} " == *" connections-1m-zrk "* ]] \ + && [ -z "${ZRK_CONN1M_CMD:-}" ]; then + if _zrk_at_least "$ZRK" "$ZRK_CONN1M_VERSION"; then + ZRK_CONN1M_CMD="$ZRK" + else + if ! docker image inspect "$ZRK_CONN1M_IMAGE" >/dev/null 2>&1; then + info "building $ZRK_CONN1M_IMAGE (zrk $ZRK_CONN1M_VERSION) for connections-1m-zrk" + docker build -t "$ZRK_CONN1M_IMAGE" \ + --build-arg ZRK_VERSION="$ZRK_CONN1M_VERSION" \ + --build-arg ZRK_SHA256="$ZRK_CONN1M_SHA256" \ + -f "$ROOT_DIR/docker/zrk.Dockerfile" "$ROOT_DIR/docker" \ + || warn "$ZRK_CONN1M_IMAGE build failed - connections-1m-zrk will not run" + fi + ZRK_CONN1M_CMD="docker run --rm --network host --cpuset-cpus=$GCANNON_CPUS --security-opt seccomp=unconfined --ulimit memlock=-1:-1 --ulimit nofile=1048576:1048576 $ZRK_CONN1M_IMAGE" + fi + info "zrk for connections-1m-zrk: $ZRK_CONN1M_CMD" +fi + # ── System tuning — NOW, after all image builds are complete ─────────────── if [ "${SKIP_TUNE:-}" != "true" ]; then system_tune + # Only when this run will hold a million connections; see system.sh. + if { framework_subscribes_to "connections-1m" \ + && [[ " ${profiles_to_run[*]} " == *" connections-1m "* ]]; } \ + || { framework_subscribes_to "connections-1m-zrk" \ + && [[ " ${profiles_to_run[*]} " == *" connections-1m-zrk "* ]]; }; then + system_tune_connections_1m + fi else info "skipping system tuning as requested (SKIP_TUNE=true)" fi @@ -287,7 +317,7 @@ run_one() { # three runs, which is not known until all three have happened. Every run is # recorded here and the choice is made below. local _mdir="" - case "$endpoint" in latency-1m|latency-10k|latency-500k-8cpu|8gbit) _mdir=$(mktemp -d) ;; esac + case "$endpoint" in latency-1m|latency-10k|latency-500k-8cpu|8gbit|connections-1m|connections-1m-zrk) _mdir=$(mktemp -d) ;; esac BEST_M=() local run @@ -323,6 +353,14 @@ run_one() { [[ "$line" == *=* ]] && m["${line%%=*}"]="${line#*=}" done < <("${tool//-/_}_parse" "$endpoint" "$output") + # connections-1m reports the server's CPU over gcannon's measured + # window only (scripts/conn_hold.py), connections-1m-zrk over the fan- + # out's (scripts/zrk_fanout.py). The reading above spans the whole run, + # a million connects and closes included, so the window replaces it. + if [ -n "${m[cpu_usec_window]:-}" ]; then + CPU_ACCT_USEC="${m[cpu_usec_window]}" + fi + local rps_int=${m[rps]:-0} # Best-of-N keeps the fastest run — except on a fixed-rate profile, @@ -369,7 +407,11 @@ run_one() { # for. tcp_tw_reuse lets those be recycled, but at 48K of a 64.5K-port # range the connect path starts scanning and the ramp lands inside the # measured window. Two seconds is plenty below that. - if [ "$CONNS" -ge 32768 ]; then sleep 15; else sleep 2; fi + # A million at once is longer again: the server has a million closes to + # work through before the next run starts connecting. + if [ "$CONNS" -ge 500000 ]; then sleep 30 + elif [ "$CONNS" -ge 32768 ]; then sleep 15 + else sleep 2; fi done # ── Latency-1M: pick the run by score, not by any single metric ───── @@ -405,7 +447,7 @@ run_one() { echo ""; echo "=== Best: ${best_rps} req/s (CPU: $best_cpu, Mem: $best_mem) ===" # The fixed-rate profiles' headline number, and the check that it counts. - if [ "$endpoint" = "latency-1m" ] || [ "$endpoint" = "latency-10k" ] || [ "$endpoint" = "latency-500k-8cpu" ] || [ "$endpoint" = "8gbit" ]; then + if [ "$endpoint" = "latency-1m" ] || [ "$endpoint" = "latency-10k" ] || [ "$endpoint" = "latency-500k-8cpu" ] || [ "$endpoint" = "8gbit" ] || conn1m_profile "$endpoint"; then # Two different failures that used to print the same line. "No CPU # reading" is a cgroup problem; "no requests" is a generator or server # problem, and saying the former when it is the latter sent the first @@ -440,6 +482,22 @@ run_one() { fi fi + # connections-1m: whether the million was actually held. The rate alone + # cannot say it - a connection that never opened sends nothing, which reads + # as a rate shortfall - so the fan-out counts the server's established + # sockets after the window as well. + if conn1m_profile "$endpoint"; then + if [ -n "${BEST_M[hold_error]:-}" ]; then + warn "$endpoint did not run: ${BEST_M[hold_error]}" + else + echo "=== Connections: ${BEST_M[connections_open]:-n/a} open of $CONNS | connect errors in the window ${BEST_M[connect_errors]:-n/a} ===" + if [ -n "${BEST_M[connections_open]:-}" ] \ + && [ "${BEST_M[connections_open]}" -lt $(( CONNS * 99 / 100 )) ] 2>/dev/null; then + warn "held ${BEST_M[connections_open]} of $CONNS connections through the window" + fi + fi + fi + # 8gbit's generator reports only the bytes it read back, so that figure # is the download half of the echo; the request body is a fixed size # (ZRK_ECHO_BODY_BYTES in scripts/lib/tools/zrk.sh - keep the two in step), @@ -532,7 +590,7 @@ save_result() { # not the load, so the reason ships with the row rather than being inferred # from an rps that looks merely slow. local eff_extra="" - if [ "$profile" = "latency-1m" ] || [ "$profile" = "latency-10k" ] || [ "$profile" = "latency-500k-8cpu" ] || [ "$profile" = "8gbit" ]; then + if [ "$profile" = "latency-1m" ] || [ "$profile" = "latency-10k" ] || [ "$profile" = "latency-500k-8cpu" ] || [ "$profile" = "8gbit" ] || conn1m_profile "$profile"; then local _eff_reqs=${BEST_M[status_2xx]:-0} local _eff_per_req="null" if [ -n "$best_cpu_usec" ] && [ "$_eff_reqs" -gt 0 ] 2>/dev/null; then @@ -544,6 +602,12 @@ save_result() { \"target_rate\": ${BEST_M[target_rate]:-0}, \"rate_ratio\": ${best_rate_ratio:-0}, \"p99_9_latency\": \"${BEST_M[p999_lat]:-}\"" + # How much of the million was held, next to what it cost to hold it. + if conn1m_profile "$profile"; then + eff_extra="$eff_extra, + \"connections_open\": ${BEST_M[connections_open]:-null}, + \"window_connect_errors\": ${BEST_M[connect_errors]:-null}" + fi fi local cpu_extra="" diff --git a/scripts/conn_hold.py b/scripts/conn_hold.py new file mode 100644 index 0000000000..4eaa4c8c2f --- /dev/null +++ b/scripts/conn_hold.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Run gcannon --rate for the connections-1m profile; report a fixed-rate summary. + +connections-1m holds a million keep-alive connections and paces requests +across them. gcannon does the load itself (gcannon >= 0.8.0: --rate, +--addr-span, --connect-limit, --warmup-max) in one process: ~3.3 KiB a +connection, against the ~303 KiB that sank the first attempt with zrk 2.3.0. +(connections-1m-zrk runs the same test on zrk 3.0.2, ~41 KiB a connection, +through scripts/zrk_fanout.py.) +What this wrapper adds is what gcannon cannot know about the server: + + - a listener on every address. The fleet is spread over 127.0.0.1-.32 + (one source address holds ~64.5K connections to one destination), so a + server bound to 127.0.0.1 or localhost refuses most of it. Probed first, + and reported as an error rather than a million connect failures. + - memory. The run is refused when the client, the kernel's share and what + the server may still grow into do not fit in MemAvailable: the first + million-connection run took the bench host down. + - the server's CPU over exactly the measured window. gcannon prints + "gcannon: warmup done" when its window opens and "gcannon: measurement + done" when it closes, while every connection is still up; the server + cgroup's usage_usec is read at each, so neither the connects nor the + million closes afterwards are in the figure. + +stdout is one JSON object in the shape scripts/lib/tools/gcannon.sh's +connections-1m parser reads (rps, latency, status classes, rate_ratio, the +window CPU, connections still open). Usage: + + conn_hold.py --conns N --rate R --port P --probe 127.0.0.2,127.0.0.32 + [--cpu-stat FILE] [--server-mem 96g] [--kib-per-conn 8] + -- --args-- +""" + +from __future__ import annotations + +import argparse +import json +import os +import resource +import socket +import subprocess +import sys +import threading + +KERNEL_KIB_PER_CONN = 8 # two sockets with their file/epoll state, and a conntrack entry +MIN_VERSION = (0, 8, 0) + + +def log(msg: str) -> None: + print(f"[conn-hold] {msg}", file=sys.stderr, flush=True) + + +def _mem_available_kib() -> int | None: + try: + with open("/proc/meminfo") as f: + for line in f: + if line.startswith("MemAvailable:"): + return int(line.split()[1]) + except (OSError, ValueError, IndexError): + pass + return None + + +def _size_kib(s: str | None) -> int: + if not s: + return 0 + s = s.strip().lower() + mult = {"k": 1, "m": 1024, "g": 1024 ** 2, "t": 1024 ** 3} + try: + return int(float(s[:-1]) * mult[s[-1]]) if s[-1] in mult else int(s) // 1024 + except ValueError: + return 0 + + +def _cpu_usec(path: str | None) -> int | None: + if not path: + return None + try: + with open(path) as f: + for line in f: + if line.startswith("usage_usec"): + return int(line.split()[1]) + except (OSError, ValueError, IndexError): + pass + return None + + +def _probe(addr: str, port: int) -> str | None: + try: + with socket.create_connection((addr, port), timeout=3): + return None + except OSError as e: + return f"{type(e).__name__}: {e}" + + +def _version(prefix: list[str]) -> tuple[int, ...] | None: + try: + out = subprocess.run(prefix + ["--version"], capture_output=True, text=True, + timeout=60).stdout + except (OSError, subprocess.TimeoutExpired): + return None + for tok in out.split(): + parts = tok.lstrip("v").split(".") + if len(parts) == 3 and all(p.isdigit() for p in parts): + return tuple(int(p) for p in parts) + return None # an old gcannon has no --version and prints usage instead + + +def _error(args, msg: str) -> dict: + log(msg) + return {"hold_error": msg, "connections": args.conns, "target_rate": args.rate, + "rps": 0, "rate_ratio": 0, "duration_s": 0, "latency_us": {}, + "status_codes": {}, "errors": {}, "connections_open": None, + "server_cpu_usec": None} + + +def run(args, prefix: list[str], gargs: list[str]) -> dict: + ver = _version(prefix) + if ver is None or ver < MIN_VERSION: + # prefix[-1] is the image or the binary: which gcannon was asked + return _error(args, "connections-1m needs gcannon >= %s for --rate/--addr-span; " + "%s %s" % (".".join(map(str, MIN_VERSION)), prefix[-1], + "is " + ".".join(map(str, ver)) if ver else + "did not answer --version (an older gcannon, or an image that did not build)")) + + for addr in [a for a in args.probe.split(",") if a]: + err = _probe(addr, args.port) + if err: + return _error(args, f"no listener reachable on {addr}:{args.port} ({err}): " + "connections-1m spreads its connections over 127.0.0.1-127.0.0.32, " + "so the server must listen on 0.0.0.0 or [::], not 127.0.0.1 or " + "localhost alone") + + avail = _mem_available_kib() + server_kib = _size_kib(args.server_mem) + need = args.conns * (args.kib_per_conn + KERNEL_KIB_PER_CONN) + server_kib + if avail is not None and need > avail * 0.9: + gib = 1024 ** 2 + return _error(args, f"not enough memory to hold {args.conns} connections: the client " + f"needs ~{args.conns * args.kib_per_conn / gib:.1f} GiB, the kernel " + f"~{args.conns * KERNEL_KIB_PER_CONN / gib:.1f} GiB and the server may grow " + f"to {server_kib / gib:.0f} GiB, against {avail / gib:.0f} GiB available") + + native = not any(os.path.basename(p) == "docker" for p in prefix) + if native: + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + want = args.conns + 4096 + if soft < want: + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (min(want, hard), hard)) + except (ValueError, OSError): + pass + soft = resource.getrlimit(resource.RLIMIT_NOFILE)[0] + if soft < want: + return _error(args, f"the open-file limit is {soft}, below the {want} that " + f"{args.conns} connections need (raise the hard limit for this shell)") + + cpu = {} + proc = subprocess.Popen(prefix + gargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + stdin=subprocess.DEVNULL, text=True) + + def watch_stderr(): + assert proc.stderr is not None + for line in proc.stderr: + sys.stderr.write(line) + if line.startswith("gcannon: warmup done"): + cpu["start"] = _cpu_usec(args.cpu_stat) + elif line.startswith("gcannon: measurement done"): + cpu["end"] = _cpu_usec(args.cpu_stat) + sys.stderr.flush() + + t = threading.Thread(target=watch_stderr, daemon=True) + t.start() + # Not communicate(): it reads stderr too, and would race the thread for + # the markers (the "measurement done" line went missing that way). + assert proc.stdout is not None + out = proc.stdout.read() + proc.wait() + t.join(timeout=10) + + start, end = out.find("{"), out.rfind("}") + try: + g = json.loads(out[start:end + 1]) + except ValueError: + return _error(args, f"gcannon exited {proc.returncode} without a JSON summary: " + f"{out.strip()[:300] or ''}") + + el = g.get("elapsed_sec") or 0 + st = g.get("status") or {} + ok = st.get("2xx") or 0 + rps = ok / el if el else 0 + lat = g.get("latency_us") or {} + rate = g.get("rate") or {} + cpu_win = (cpu["end"] - cpu["start"]) if cpu.get("start") is not None \ + and cpu.get("end") is not None and cpu["end"] >= cpu["start"] else None + if g.get("connections_active") is not None: + log(f"{g['connections_active']} of {args.conns} connections open at the end of the window") + return { + "connections": args.conns, + "connections_open": g.get("connections_active"), + "target_rate": args.rate, + "rps": round(rps, 2), + "rate_ratio": round(rps / args.rate, 4) if args.rate else 0, + "duration_s": el, + "bytes_per_sec": g.get("bandwidth_bps") or 0, + "max_schedule_lag_us": rate.get("max_lag_us") or 0, + "latency_us": {"mean": lat.get("avg"), "p50": lat.get("p50"), "p90": lat.get("p90"), + "p99": lat.get("p99"), "p99_9": lat.get("p999")}, + "status_codes": {k: st.get(k) or 0 for k in ("2xx", "3xx", "4xx", "5xx")}, + "errors": g.get("errors") or {}, + "reconnects": g.get("reconnects") or 0, + "server_cpu_usec": cpu_win, + "gcannon": g, + } + + +def main(argv: list[str]) -> int: + if "--" not in argv or "--args--" not in argv: + print(__doc__, file=sys.stderr) + return 2 + i, j = argv.index("--"), argv.index("--args--") + ap = argparse.ArgumentParser() + ap.add_argument("--conns", type=int, required=True) + ap.add_argument("--rate", type=int, required=True) + ap.add_argument("--port", type=int, required=True) + ap.add_argument("--probe", default="") + ap.add_argument("--cpu-stat", dest="cpu_stat") + ap.add_argument("--server-mem", dest="server_mem") + ap.add_argument("--kib-per-conn", dest="kib_per_conn", type=int, default=8) + args = ap.parse_args(argv[:i]) + print(json.dumps(run(args, argv[i + 1:j], argv[j + 1:]), indent=1)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/gen_leaderboard_data.py b/scripts/gen_leaderboard_data.py index ca1d7d3389..a51b8f2c6f 100644 --- a/scripts/gen_leaderboard_data.py +++ b/scripts/gen_leaderboard_data.py @@ -59,6 +59,18 @@ # application handler to await in. ("async", "Async Delay", "A 10ms wait named in the route, at 32K held connections.", [32000], [32000], True,True,False), + # Reference-only while the first field is measured. A million keep-alive + # connections held at once with the baseline GET paced at 250K req/s + # across them, one request per connection every four seconds. Paced, so + # it carries the fixed-rate score (latency_score.FULL_RATE) like the + # Efficiency profiles; it sits here because what it varies is how many + # connections are open, which is this group's question. + ("connections-1m", "1M Connections", "Score out of 100 (reference): CPU, mean and p99 latency at 250K req/s spread over one million held connections.", + [1000000], [1000000], False,False,False), + # The same test with zrk 3.0.2 as the load generator instead of gcannon, + # to compare what the two report on one load. Reference-only too. + ("connections-1m-zrk", "1M Connections (zrk)", "Score out of 100 (reference): 1M Connections with zrk as the load generator instead of gcannon - 250K req/s over one million held connections.", + [1000000], [1000000], False,False,False), ]), ("Efficiency", [ # Scored, on the same terms as latency-1m below: the rate is pinned, so @@ -151,7 +163,7 @@ # Efficiency-only. Emitted like TPL_FIELDS - only where present - so the # other ~2,300 rows in data.js do not each grow four nulls. EFF_FIELDS = ("cpu_usec", "cpu_per_req_us", "rate_ratio", "target_rate", - "p99_9_latency") + "p99_9_latency", "connections_open") # Map each benchmark profile to its Knowledge Base "Implementation Guidelines" # page (docs ids differ from profile ids; TLS gRPC variants share one page). @@ -167,6 +179,8 @@ "latency-1m": "test-profiles/h1/latency-1m/implementation", "latency-10k": "test-profiles/h1/latency-10k/implementation", "latency-500k-8cpu": "test-profiles/h1/latency-500k-8cpu/implementation", + "connections-1m": "test-profiles/h1/connections-1m/implementation", + "connections-1m-zrk": "test-profiles/h1/connections-1m/implementation", "async-db": "test-profiles/h1/async-database/implementation", "fortunes": "test-profiles/h1/fortunes/implementation", "baseline-h2": "test-profiles/h2/baseline-h2/implementation", diff --git a/scripts/latency_score.py b/scripts/latency_score.py index 8dc01df649..8708c60b3c 100755 --- a/scripts/latency_score.py +++ b/scripts/latency_score.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """Scoring for the fixed-rate profiles: `latency-1m`, `latency-10k`, -`latency-500k-8cpu` and `8gbit`. +`latency-500k-8cpu`, `8gbit`, `connections-1m` and `connections-1m-zrk`. Every other profile here ranks on one number, requests per second. This one cannot: the rate is pinned, so every entry that finishes serves the same load @@ -68,6 +68,13 @@ # Keep in step with LAT_FULL in site/leaderboard/index.html - check_badge_parity.js # compares the two and fails the deploy when they disagree. "8gbit": 47_500.0, + # A million held connections at 250K req/s, measured on the window after + # the fleet is connected: 95% of the target, like the others. A connection + # that never opened sends nothing, so a fleet only partly held lands here + # as a rate shortfall. + "connections-1m": 237_500.0, + # The same test on zrk instead of gcannon, scored the same way. + "connections-1m-zrk": 237_500.0, } DEFAULT_PROFILE = "latency-1m" W_CPU, W_P99, W_MEAN = 0.50, 0.25, 0.25 diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index dae3580c23..02769044e0 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -75,6 +75,15 @@ HARD_NOFILE=$(ulimit -Hn 2>/dev/null || echo 1048576) [[ "$HARD_NOFILE" =~ ^[0-9]+$ ]] || HARD_NOFILE=1048576 ulimit -n "$HARD_NOFILE" 2>/dev/null || true +# connections-1m's server container. HARD_NOFILE is whatever the shell running +# the harness was given, which can be below a million; this profile needs a +# million sockets plus the server's own, so its container gets this instead +# (capped at fs.nr_open, which system_tune_connections_1m raises). The memory +# cap keeps an entry that spends ~100 KB or more per connection from taking the +# host down with it: it is OOM-killed and the run reports what it held. +CONN1M_NOFILE="${CONN1M_NOFILE:-2097152}" +CONN1M_MEM_LIMIT="${CONN1M_MEM_LIMIT:-96g}" + # Postgres sidecar. PG_CONTAINER="httparena-postgres" DATABASE_URL="postgres://bench:bench@localhost:5432/benchmark" diff --git a/scripts/lib/framework.sh b/scripts/lib/framework.sh index 839ad24ea6..5a7c7f6f95 100644 --- a/scripts/lib/framework.sh +++ b/scripts/lib/framework.sh @@ -82,11 +82,23 @@ framework_start() { docker stop -t 5 "$CONTAINER_NAME" 2>/dev/null || true docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + # connections-1m (and its zrk twin) holds a million sockets, more than + # HARD_NOFILE may allow; its own limit is capped at fs.nr_open, the + # ceiling the kernel enforces. + local nofile="$HARD_NOFILE" mem_limit="${FRAMEWORK_MEM_LIMIT:-}" + if conn1m_profile "$endpoint"; then + local nr_open + nr_open=$(cat /proc/sys/fs/nr_open 2>/dev/null || echo "$HARD_NOFILE") + nofile="$CONN1M_NOFILE" + [ "$nofile" -gt "$nr_open" ] 2>/dev/null && nofile="$nr_open" + [ -n "$mem_limit" ] || mem_limit="$CONN1M_MEM_LIMIT" + fi + local args=( -d --name "$CONTAINER_NAME" --network host --security-opt seccomp=unconfined --ulimit memlock=-1:-1 - --ulimit nofile="$HARD_NOFILE:$HARD_NOFILE" + --ulimit nofile="$nofile:$nofile" -v "$DATA_DIR/dataset.json:/data/dataset.json:ro" -v "$DATA_DIR/static:/data/static:ro" -v "$CERTS_DIR:/certs:ro" @@ -97,9 +109,10 @@ framework_start() { # Empty is the default and leaves every other caller (benchmark, validate, # spec) exactly as before. --memory-swap equal to --memory disables swap, so # an over-limit server is OOM-killed deterministically instead of silently - # swapping and merely getting slow. - if [ -n "${FRAMEWORK_MEM_LIMIT:-}" ]; then - args+=(--memory="$FRAMEWORK_MEM_LIMIT" --memory-swap="$FRAMEWORK_MEM_LIMIT") + # swapping and merely getting slow. connections-1m sets one of its own + # (CONN1M_MEM_LIMIT) when the caller did not. + if [ -n "$mem_limit" ]; then + args+=(--memory="$mem_limit" --memory-swap="$mem_limit") fi # Profiles that exercise the database get DATABASE_URL + per-profile conn cap. diff --git a/scripts/lib/profiles.sh b/scripts/lib/profiles.sh index 2707f2bcf0..ecf388da18 100644 --- a/scripts/lib/profiles.sh +++ b/scripts/lib/profiles.sh @@ -74,6 +74,31 @@ declare -A PROFILES=( # marginal cost per request and the queue it builds are what get measured. # The rate lives in tools/zrk.sh (ZRK_RATE_LATENCY_500K) like the others. [latency-500k-8cpu]="1|0|0-3,64-67|1024|latency-500k-8cpu" + # Connections-1M: a million keep-alive connections held at once, with the + # baseline GET paced at 250K req/s across them (GCANNON_RATE_CONNECTIONS_1M) + # - one request per connection every four seconds. At that rate the work + # per request is small, so what separates entries is what holding the rest + # of the million costs: memory per connection, the CPU a request carries + # while the others wait, and the tail once a heap, a timer wheel or a poll + # set is that large. Measured on gcannon's window after the whole fleet is + # connected. + # + # One source address can open about 64.5K connections to one destination + # (the async comment above), so gcannon spreads the fleet over the + # loopback addresses 127.0.0.1 .. 127.0.0.32 (--addr-span). An entry must + # listen on 0.0.0.0 or [::] to subscribe; validate.sh checks it, and + # scripts/conn_hold.py probes it before the run. Needs gcannon >= 0.8.0. + # Opt-in and reference-only while the first field is measured. + # + # Not zrk: zrk 2.3.0 keeps ~303 KiB resident per connection, so a million + # needed ~290 GiB and the first run took the 251 GiB bench host down. + # gcannon holds one in ~3.3 KiB. + [connections-1m]="1|0|0-31,64-95|1000000|connections-1m" + # The same test with zrk 3.0.2 as the generator, to compare the two on one + # load: 3.0.2 shares its latency histograms per thread and keeps ~41 KiB a + # connection. zrk takes one URL, so tools/zrk.sh runs one process per + # loopback address (scripts/zrk_fanout.py) and merges their histograms. + [connections-1m-zrk]="1|0|0-31,64-95|1000000|connections-1m-zrk" [json-comp]="1|0|0-31,64-95|4096,16384|json-compressed" [json-tls]="1|0|0-31,64-95|4096|json-tls" # 8Gbit: 100 KB up and the same 100 KB back, over TLS. The only profile @@ -128,8 +153,19 @@ PROFILE_ORDER=( # TIME_WAIT for the kernel's fixed ~60s, so anything scheduled after it # starts against a nearly full port table. async + # After even that, for the same reason twenty times over: a million + # sockets closing at once, and a million conntrack entries ageing out. + connections-1m + connections-1m-zrk ) +# The two million-connection profiles: the same test, one per load generator. +# Everything the harness does around the load (host limits, the server's fd +# and memory caps, the window CPU, the held count) is the same for both. +conn1m_profile() { + [ "$1" = "connections-1m" ] || [ "$1" = "connections-1m-zrk" ] +} + # ── Parsing + validation ──────────────────────────────────────────────────── # Parse a profile spec string into global fields. Exits on malformed input. @@ -151,7 +187,7 @@ endpoint_tool() { # wrk (lua script rotation) static-tls|json-tls) echo "wrk" ;; # zrk — the only paced generator; holds a fixed offered rate - latency-1m|latency-10k|latency-500k-8cpu|8gbit) echo "zrk" ;; + latency-1m|latency-10k|latency-500k-8cpu|8gbit|connections-1m-zrk) echo "zrk" ;; # h2load for all HTTP/2 variants (TLS via ALPN + h2c prior-knowledge) h2|static-h2|h2c|json-h2c|gateway-64|grpc|grpc-tls|production-stack) echo "h2load" ;; # h2load built with ngtcp2 for HTTP/3 diff --git a/scripts/lib/spec.sh b/scripts/lib/spec.sh index f8556c9c4f..ba06db1acf 100644 --- a/scripts/lib/spec.sh +++ b/scripts/lib/spec.sh @@ -14,7 +14,7 @@ SPEC_SUITES="" # listener, ws the /ws echo. spec_profiles_for() { case "$1" in - h1) echo "baseline pipelined limited-conn json-comp async async-db fortunes latency-1m latency-10k latency-500k-8cpu" ;; + h1) echo "baseline pipelined limited-conn json-comp async async-db fortunes latency-1m latency-10k latency-500k-8cpu connections-1m connections-1m-zrk" ;; h2) echo "baseline-h2 static-h2 baseline-h2c json-h2c" ;; h3) echo "baseline-h3 static-h3" ;; ws) echo "echo-ws echo-ws-pipeline echo-ws-limited" ;; diff --git a/scripts/lib/system.sh b/scripts/lib/system.sh index fb97b9a5da..d3fd414207 100644 --- a/scripts/lib/system.sh +++ b/scripts/lib/system.sh @@ -90,6 +90,46 @@ system_tune() { sync } +# Raise a sysctl to at least `want`; never lowers one that is already higher. +_raise_sysctl() { + local key="$1" want="$2" have + have=$(sysctl -n "$key" 2>/dev/null) || { warn "sysctl $key is not available"; return 0; } + [ "$have" -ge "$want" ] 2>/dev/null && return 0 + sudo sysctl -w "$key=$want" >/dev/null 2>&1 || warn "could not raise $key ($have < $want)" +} + +# The limits a million held connections run into, raised only when the run +# includes connections-1m or connections-1m-zrk. Each one is a ceiling rather +# than a setting, so a host left with them raised measures every other profile +# exactly as before; like the governor, they are not put back. +# +# fs.nr_open the most fds a process may be given; the +# default 1048576 leaves a server holding a +# million connections ~48K for everything else +# vm.max_map_count zrk (connections-1m-zrk) maps a stack plus a +# guard per coroutine: ~4 maps a connection, +# ~125K for one process's 31,250, against a +# 65,530 default +# net.netfilter.nf_conntrack_max +# docker's rules switch conntrack on for the +# host, loopback included, and a full table drops +# SYNs. A million live entries plus the last run's +# million in TIME_WAIT (120s) need room for both +# nf_conntrack hashsize so those entries are not chained eight deep in +# a table sized for a quarter of a million +system_tune_connections_1m() { + info "raising host limits for connections-1m" + _raise_sysctl fs.nr_open 4194304 + _raise_sysctl vm.max_map_count 4194304 + if [ -e /proc/sys/net/netfilter/nf_conntrack_max ]; then + _raise_sysctl net.netfilter.nf_conntrack_max 4194304 + local hs=/sys/module/nf_conntrack/parameters/hashsize + if [ -r "$hs" ] && [ "$(cat "$hs" 2>/dev/null || echo 0)" -lt 1048576 ] 2>/dev/null; then + sudo sh -c "echo 1048576 > $hs" 2>/dev/null || warn "could not raise the conntrack hash size" + fi + fi +} + # Revert everything we touched. Call from EXIT trap. system_restore() { info "restoring loopback MTU to 65536" diff --git a/scripts/lib/tools/gcannon.sh b/scripts/lib/tools/gcannon.sh index a133c7dd90..98419a3242 100644 --- a/scripts/lib/tools/gcannon.sh +++ b/scripts/lib/tools/gcannon.sh @@ -4,6 +4,23 @@ # and gRPC plaintext (indirectly via -H and -d, though we use h2load for # that in practice). This file owns the gcannon ↔ driver interface. +# connections-1m: a million keep-alive connections with 250K req/s paced over +# them, one request per connection every four seconds - inside the 5s +# keep-alive default of Node, uvicorn and actix, and a few percent of what a +# fast server carries on this cpuset, so what is left to measure is what +# holding the rest of the million costs. Needs gcannon >= 0.8.0 (--rate, +# --addr-span, --connect-limit, --warmup-max); scripts/conn_hold.py checks. +# +# One source address holds ~64.5K connections to one destination, so the +# fleet spans 127.0.0.1-127.0.0.CONN1M_ADDRS (31,250 on each of 32). First +# connects are capped per thread so a million SYNs never land at once, and +# the warmup waits up to CONN1M_WARMUP_MAX for all of them to open before the +# 20s window starts. +GCANNON_RATE_CONNECTIONS_1M="${GCANNON_RATE_CONNECTIONS_1M:-250000}" +CONN1M_ADDRS="${CONN1M_ADDRS:-32}" +CONN1M_CONNECT_LIMIT="${CONN1M_CONNECT_LIMIT:-64}" +CONN1M_WARMUP_MAX="${CONN1M_WARMUP_MAX:-180}" + # ── Build arguments ───────────────────────────────────────────────────────── # Usage: gcannon_build_args [req_per_conn] @@ -80,6 +97,18 @@ gcannon_build_args() { # redoes the HTTP/1.1 upgrade, which is what that profile measures. [ "$req_per_conn" -gt 0 ] 2>/dev/null && args+=(-r "$req_per_conn") ;; + connections-1m) + # The baseline GET over plaintext: what varies here is how many + # connections are held, so the request stays the one subscribers + # are already validated on. gcannon_run hands the --args-- part to + # scripts/conn_hold.py, which owns everything around the load. + args=(--hold-- "$conns" "$GCANNON_RATE_CONNECTIONS_1M" --args-- + "http://127.0.0.1:$PORT/baseline11?a=1&b=2" + -c "$conns" -t "$THREADS" -d 20s + --rate "$GCANNON_RATE_CONNECTIONS_1M" --addr-span "$CONN1M_ADDRS" + --connect-limit "$CONN1M_CONNECT_LIMIT" + --warmup-max "${CONN1M_WARMUP_MAX}s" --json) + ;; *) fail "gcannon_build_args: unknown endpoint '$endpoint'" ;; @@ -93,6 +122,10 @@ gcannon_build_args() { # Run gcannon with the given args, return output on stdout. # Honors GCANNON_MODE=native|docker. gcannon_run() { + if [ "${1:-}" = "--hold--" ]; then + _gcannon_hold_run "${@:2}" + return 0 + fi local -a args=("$@") if [ "$GCANNON_MODE" = "native" ]; then timeout 45 taskset -c "$GCANNON_CPUS" \ @@ -107,13 +140,83 @@ gcannon_run() { fi } +# connections-1m: gcannon under scripts/conn_hold.py, which probes the +# listener, refuses a run the host cannot hold, and reads the server cgroup's +# CPU at gcannon's window markers. stdout is the wrapper's JSON summary; +# stderr (gcannon's own lines) goes straight to the log. +_gcannon_hold_run() { + local conns="$1" rate="$2" + shift 2 + [ "${1:-}" = "--args--" ] && shift + local -a prefix + if [ "$GCANNON_MODE" = "native" ]; then + prefix=(taskset -c "$GCANNON_CPUS" env LD_LIBRARY_PATH=/usr/lib "$GCANNON") + else + prefix=(docker run --rm --network host --cpuset-cpus="$GCANNON_CPUS" + --security-opt seccomp=unconfined + --ulimit memlock=-1:-1 --ulimit nofile=1048576:1048576 + -v "$REQUESTS_DIR:$REQUESTS_DIR:ro" "$GCANNON_IMAGE") + fi + local cpu_stat="" + if [ -n "${CONTAINER_NAME:-}" ]; then + cpu_stat=$(_cgroup_cpu_path "$CONTAINER_NAME" 2>/dev/null) || cpu_stat="" + fi + timeout $(( CONN1M_WARMUP_MAX + 20 + 180 )) python3 "$SCRIPT_DIR/conn_hold.py" \ + --conns "$conns" --rate "$rate" --port "$PORT" \ + --probe "127.0.0.2,127.0.0.$CONN1M_ADDRS" \ + ${cpu_stat:+--cpu-stat "$cpu_stat"} --server-mem "${CONN1M_MEM_LIMIT:-}" \ + -- "${prefix[@]}" --args-- "$@" || true +} + # ── Parse output ──────────────────────────────────────────────────────────── +# connections-1m: the fixed-rate keys benchmark.sh reads (the same ones +# zrk_parse emits for the latency profiles), from conn_hold.py's summary. +_gcannon_hold_parse() { + python3 - "$1" <<'PY' +import json, sys +raw = sys.argv[1] +s, e = raw.find("{"), raw.rfind("}") +try: + d = json.loads(raw[s:e + 1]) if s >= 0 and e > s else {} +except ValueError: + d = {} +if not d: + sys.stderr.write("[gcannon] no connections-1m summary in the output:\n%s\n" % (raw[:2000] or "")) +lat = d.get("latency_us") or {} +codes = d.get("status_codes") or {} +print("rps=%d" % round(d.get("rps") or 0)) +print("avg_lat=%.1fus" % (lat.get("mean") or 0)) +print("p99_lat=%.1fus" % (lat.get("p99") or 0)) +print("p999_lat=%.1fus" % (lat.get("p99_9") or 0)) +print("reconnects=%d" % (d.get("reconnects") or 0)) +print("bandwidth=%.2fMB/s" % ((d.get("bytes_per_sec") or 0) / 1e6)) +for k in ("2xx", "3xx", "4xx", "5xx"): + print("status_%s=%d" % (k, codes.get(k) or 0)) +print("rate_ratio=%.4f" % (d.get("rate_ratio") or 0)) +print("duration=%.0fs" % (d.get("duration_s") or 0)) +print("target_rate=%d" % (d.get("target_rate") or 0)) +print("max_schedule_lag_us=%d" % (d.get("max_schedule_lag_us") or 0)) +if d.get("connections_open") is not None: + print("connections_open=%d" % d["connections_open"]) +print("connect_errors=%d" % ((d.get("errors") or {}).get("connect") or 0)) +if d.get("server_cpu_usec") is not None: + print("cpu_usec_window=%d" % d["server_cpu_usec"]) +if d.get("hold_error"): + print("hold_error=%s" % " ".join(str(d["hold_error"]).split())) +PY +} + # Usage: gcannon_parse # Echoes KEY=VALUE lines. Caller reads into an assoc array. gcannon_parse() { local endpoint="$1" output="$2" + if [ "$endpoint" = "connections-1m" ]; then + _gcannon_hold_parse "$output" + return 0 + fi + # rps — compute from 2xx (or 3xx for caching, or WS frames for ws-echo) # divided by the actual measured duration. # diff --git a/scripts/lib/tools/zrk.sh b/scripts/lib/tools/zrk.sh index 500c62ccd9..1b62e81d87 100644 --- a/scripts/lib/tools/zrk.sh +++ b/scripts/lib/tools/zrk.sh @@ -49,6 +49,61 @@ ZRK_RATE_LATENCY_10K="${ZRK_RATE_LATENCY_10K:-10000}" ZRK_RATE_LATENCY_500K="${ZRK_RATE_LATENCY_500K:-500000}" ZRK_RATE_8GBIT="${ZRK_RATE_8GBIT:-50000}" +# connections-1m-zrk: connections-1m (tools/gcannon.sh) with zrk as the load +# generator, to compare what the two report on the same load - a million held +# connections, 250K req/s, one request per connection every four seconds. +# +# Its own zrk, 3.0.2, while the profiles above stay on the pinned 2.3.0 (a bump +# re-baselines them). 2.3.0 kept ~303 KiB per connection and took the bench +# host down at a million; 3.0.2 keeps ~41 KiB (measured at 4K and 16K). The +# image is docker/zrk.Dockerfile built with this version and checksum. +ZRK_RATE_CONNECTIONS_1M="${ZRK_RATE_CONNECTIONS_1M:-250000}" +ZRK_CONN1M_VERSION="${ZRK_CONN1M_VERSION:-3.0.2}" +ZRK_CONN1M_SHA256="${ZRK_CONN1M_SHA256:-339ade3ba435f14ea3c15ab8e97a880d931e14604857cde01aea1797e7430cbf}" +ZRK_CONN1M_IMAGE="${ZRK_CONN1M_IMAGE:-zrk:$ZRK_CONN1M_VERSION}" +: "${ZRK_CONN1M_CMD:=}" + +# How the fleet is spread (scripts/zrk_fanout.py has the why). zrk takes one +# URL, so one process per loopback address 127.0.0.1..127.0.0.CONN1M_ADDRS, +# 31,250 connections and 2 threads each (64 in all, like gcannon's -t). Each +# opens all of its connections the moment it starts, so processes start +# STAGGER seconds apart, SETTLE seconds after the last start are discarded, +# WINDOW seconds are measured, and TAIL seconds keep the fleet up while the +# open connections are counted (ss walking a million sockets; a count that +# outlasts the tail would see zrk's own closes). +ZRK_FANOUT_STAGGER="${ZRK_FANOUT_STAGGER:-1}" +ZRK_FANOUT_SETTLE="${ZRK_FANOUT_SETTLE:-10}" +ZRK_FANOUT_WINDOW="${ZRK_FANOUT_WINDOW:-20}" +ZRK_FANOUT_TAIL="${ZRK_FANOUT_TAIL:-20}" +# The fan-out refuses a run the host cannot hold at this much per connection +# and caps each zrk container at half again over it, so an overrun is +# OOM-killed in its own cgroup instead of taking the bench host down. +ZRK_KIB_PER_CONN="${ZRK_KIB_PER_CONN:-48}" + +# The zrk connections-1m-zrk runs: ZRK_CONN1M_CMD when set (benchmark.sh sets +# it at startup), else a zrk on PATH if it is new enough. Printed one word per +# line; the fan-out asks it for --version and refuses anything older. +_zrk_conn1m_cmd() { + if [ -n "$ZRK_CONN1M_CMD" ]; then + printf '%s\n' $ZRK_CONN1M_CMD + elif _zrk_at_least "$ZRK" "$ZRK_CONN1M_VERSION"; then + printf '%s\n' "$ZRK" + else + printf '%s\n' docker run --rm --network host \ + --cpuset-cpus="$GCANNON_CPUS" --security-opt seccomp=unconfined \ + --ulimit memlock=-1:-1 --ulimit nofile=1048576:1048576 "$ZRK_CONN1M_IMAGE" + fi +} + +# True when binary $1 is on PATH and reports a version >= $2. +_zrk_at_least() { + command -v "$1" >/dev/null 2>&1 || return 1 + local have + have=$("$1" --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) + [ -n "$have" ] || return 1 + [ "$(printf '%s\n%s\n' "$2" "$have" | sort -V | head -1)" = "$2" ] +} + # The body zrk posts to /echo. zrk takes a single body (-b @FILE), so unlike the # wrk script this profile used to run there is no eight-way rotation making a # canned response wrong seven times out of eight. The anti-cheat therefore rests @@ -79,6 +134,18 @@ sys.stdout.buffer.write((unit * (n // len(unit) + 1))[:n]) zrk_build_args() { local endpoint="$1" conns="$2" pipeline="$3" duration="$4" local -a cmd + + # The baseline GET over plaintext, as on connections-1m. One zrk cannot + # hold a million connections to one address, so zrk_run fans this out; + # the sentinel carries what it needs ahead of the zrk command itself. + # Handled before _zrk_cmd: this profile runs 3.0.2, not the pinned zrk. + if [ "$endpoint" = "connections-1m-zrk" ]; then + mapfile -t cmd < <(_zrk_conn1m_cmd) + printf '%s\n' --fanout-- "$conns" "$ZRK_RATE_CONNECTIONS_1M" "/baseline11?a=1&b=2" \ + -- "${cmd[@]}" + return 0 + fi + mapfile -t cmd < <(_zrk_cmd) case "$endpoint" in @@ -127,6 +194,10 @@ zrk_build_args() { # dashboard out of the log -- but --format json already suppresses the # dashboard, so all that redirect ever hid was the reason a run failed. zrk_run() { + if [ "${1:-}" = "--fanout--" ]; then + _zrk_fanout_run "${@:2}" + return 0 + fi if [ -n "$ZRK_CMD" ]; then timeout 60 "$@" || true else @@ -134,6 +205,38 @@ zrk_run() { fi } +# connections-1m-zrk: one zrk per loopback address, merged into one summary +# shaped like zrk's own (scripts/zrk_fanout.py). Its stdout is that summary, so +# the caller captures and parses it exactly as it would a single zrk's. +# +# The server cgroup's CPU is read by the fan-out at the edges of the measured +# window and reported as server_cpu_usec; benchmark.sh's own reading spans the +# whole run, a million connects included, so the window figure replaces it. +_zrk_fanout_run() { + local conns="$1" rate="$2" path="$3" + shift 3 + [ "${1:-}" = "--" ] && shift + local -a prefix=("$@") + # docker pins through --cpuset-cpus; a native zrk needs taskset. + [ "$(basename "${prefix[0]}")" = "docker" ] || prefix=(taskset -c "$GCANNON_CPUS" "${prefix[@]}") + local cpu_stat="" + if [ -n "${CONTAINER_NAME:-}" ]; then + cpu_stat=$(_cgroup_cpu_path "$CONTAINER_NAME" 2>/dev/null) || cpu_stat="" + fi + local addrs="${CONN1M_ADDRS:-32}" + local total=$(( (addrs - 1) * ZRK_FANOUT_STAGGER + ZRK_FANOUT_SETTLE \ + + ZRK_FANOUT_WINDOW + ZRK_FANOUT_TAIL )) + timeout $(( total + 300 )) python3 "$SCRIPT_DIR/zrk_fanout.py" \ + --conns "$conns" --rate "$rate" --addrs "$addrs" \ + --port "$PORT" --path "$path" --threads "$THREADS" \ + --stagger "$ZRK_FANOUT_STAGGER" --settle "$ZRK_FANOUT_SETTLE" \ + --window "$ZRK_FANOUT_WINDOW" --tail "$ZRK_FANOUT_TAIL" \ + ${cpu_stat:+--cpu-stat "$cpu_stat"} --ss-cpus "$GCANNON_CPUS" \ + --kib-per-conn "$ZRK_KIB_PER_CONN" --server-mem "${CONN1M_MEM_LIMIT:-}" \ + --min-version "$ZRK_CONN1M_VERSION" \ + -- "${prefix[@]}" || true +} + # ── Parse output ──────────────────────────────────────────────────────────── # zrk writes the JSON summary to stdout under --format json. stderr carries the @@ -192,5 +295,18 @@ print("rate_ratio=%.4f" % (d.get("rate_ratio") or 0)) print("duration=%.0fs" % (d.get("duration_s") or 0)) print("target_rate=%d" % (d.get("target_rate") or 0)) print("max_schedule_lag_us=%d" % (d.get("max_schedule_lag_us") or 0)) + +# connections-1m-zrk's fan-out adds these (scripts/zrk_fanout.py), under the +# keys gcannon's connections-1m parser emits, so benchmark.sh reads both the +# same way. A single zrk summary has none of them. +if "fanout" in d or d.get("hold_error"): + for k in ("launched", "connections_open", "ramp_connect_errors"): + if d.get(k) is not None: + print("%s=%d" % (k, d[k])) + print("connect_errors=%d" % ((d.get("errors") or {}).get("connect") or 0)) +if d.get("server_cpu_usec") is not None: + print("cpu_usec_window=%d" % d["server_cpu_usec"]) +if d.get("hold_error"): + print("hold_error=%s" % " ".join(str(d["hold_error"]).split())) PY } diff --git a/scripts/validate.sh b/scripts/validate.sh index bcb9fb307c..58bcfe3fea 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -1251,7 +1251,7 @@ wait_h2() { # The fixed-rate profiles drive GET /baseline11, so they need the same # handler to be correct and gets its coverage from this section rather than # one of its own -- there is nothing about it a request-shaped check can see. -if has_test "baseline" || has_test "limited-conn" || has_test "latency-1m" || has_test "latency-10k" || has_test "latency-500k-8cpu"; then +if has_test "baseline" || has_test "limited-conn" || has_test "latency-1m" || has_test "latency-10k" || has_test "latency-500k-8cpu" || has_test "connections-1m" || has_test "connections-1m-zrk"; then BASELINE_DOCS="$DOCS_BASE/h1/baseline/validation" echo "[test] baseline endpoints" check "GET /baseline11?a=13&b=42" "55" "$BASELINE_DOCS" \ @@ -1352,6 +1352,23 @@ if has_test "baseline" || has_test "limited-conn" || has_test "latency-1m" || ha fi fi +# ───── Connections-1M: a listener on every loopback address ───── + +# connections-1m (and connections-1m-zrk) spreads its million connections +# over 127.0.0.1-127.0.0.32, because one source address holds at most ~64K +# connections to one destination. +# A server bound to 127.0.0.1 or "localhost" passes every check above and +# refuses all of these but the first, so the profile asks for a listener on +# 0.0.0.0 or [::] and this is where an entry finds out. Both ends of the range. +if has_test "connections-1m" || has_test "connections-1m-zrk"; then + CONN1M_DOCS="$DOCS_BASE/h1/connections-1m/validation" + echo "[test] connections-1m listener on every address" + check "GET /baseline11 on 127.0.0.2" "55" "$CONN1M_DOCS" \ + "http://127.0.0.2:$PORT/baseline11?a=13&b=42" + check "GET /baseline11 on 127.0.0.32" "55" "$CONN1M_DOCS" \ + "http://127.0.0.32:$PORT/baseline11?a=13&b=42" +fi + # ───── Pipelined (GET /pipeline) ───── if has_test "pipelined"; then diff --git a/scripts/zrk_fanout.py b/scripts/zrk_fanout.py new file mode 100755 index 0000000000..2c7a3b0836 --- /dev/null +++ b/scripts/zrk_fanout.py @@ -0,0 +1,644 @@ +#!/usr/bin/env python3 +"""Fan one zrk run out over many loopback addresses and merge the results. + +connections-1m-zrk is connections-1m (a million keep-alive connections, 250K +req/s paced across them) with zrk as the load generator instead of gcannon, +so the two generators can be compared on the same load. It needs zrk >= 3.0.2: +2.3.0 kept a 272 KiB latency histogram per connection, ~303 KiB resident in +all, and the first million-connection run with it took the bench host down. +3.0.2 shares one histogram per thread; measured at 4K and 16K connections it +keeps ~41 KiB a connection (buffers and coroutine stacks), ~40 GiB for the +million. + +The kernel tells TCP connections apart by source address and port plus +destination address and port, so one source address can hold at most one +connection per local port to a given destination: about 64K on the bench host +(ip_local_port_range 1024-65535, less the reserved ports). That is the ceiling +the async profile's comment in scripts/lib/profiles.sh runs into. + +Every 127.x.y.z address is local on Linux, so the fleet is spread over +127.0.0.1 .. 127.0.0.K instead: one zrk process per address, each holding +conns/K connections at rate/K, and each destination address brings its own +64K ports. The server has to accept on all of them, which a listener on +0.0.0.0 or [::] does. One bound to 127.0.0.1 alone refuses the rest, and the +probe below says so before anything is launched. + +Measuring the steady state only. zrk opens all of its connections the moment +it starts, so the processes are launched `stagger` seconds apart, and the +`settle` seconds after the last launch are discarded too. What is reported is +the `window` that follows, with the whole fleet connected. Each process streams +one NDJSON row per second (--timeseries) carrying that second's full latency +histogram (--timeseries-histogram). The rows inside the window are merged +exactly, bucket by bucket, and the percentiles are read off the merged +histogram: averaging per-process percentiles would not be a percentile of +anything. + +Output: one JSON object on stdout with the keys of zrk's own --format json +summary that zrk_parse reads (achieved_rate, latency_us, status_codes, errors, +rate_ratio, duration_s, target_rate, max_schedule_lag_us, bytes_per_sec), and: + + connections offered connection count + launched connections the zrk processes report launching + connections_open server-side ESTABLISHED sockets on the port, counted + with ss(8) right after the window + server_cpu_usec the server cgroup's CPU over the window, or null + ramp_connect_errors connect failures outside the window (the ramp) + fanout how the run was split and timed, per process + hold_error set, with every count zero, when the run could not + start (the key scripts/conn_hold.py uses too) + +status_codes 2xx counts the window, like every other figure here. 4xx and 5xx +count the whole run, ramp included: a non-2xx anywhere is worth seeing. + +Usage (scripts/lib/tools/zrk.sh calls it): + + zrk_fanout.py --conns N --rate R --addrs K --port P --path /x --threads T + [--stagger S] [--settle Z] [--window W] [--tail E] + [--cpu-stat FILE] [--ss-cpus LIST] [--min-version 3.0.2] + -- +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import math +import os +import shutil +import socket +import struct +import subprocess +import sys +import tempfile +import time +import zlib + +# ── HdrHistogram V2, as zrk writes it (src/hdr.zig) ───────────────────────── +# +# zrk records microseconds at 3 significant figures between 1us and 60s. The +# functions below mirror its index arithmetic and its reporting conventions +# exactly -- a percentile is the median-equivalent value of the bucket holding +# the rank round(p/100 * total), the mean weights each bucket's median- +# equivalent value, max is the highest-equivalent value of the top bucket -- so +# the merged figures are the ones zrk would print had one process seen +# everything. tests: decoding a summary's own latency_histogram reproduces the +# latency_us block printed next to it. + +_COMPRESSED_BASE = 0x1C849304 +_ENCODING_BASE = 0x1C849303 +_COOKIE_MASK = 0xFFFFFF0F # zrk masks the word-size nibble: ~0xf0 as a u32 + + +class Layout: + def __init__(self, lowest: int, highest: int, sig_figs: int): + self.key = (lowest, highest, sig_figs) + largest_single_unit = 2 * 10 ** sig_figs + sub_bucket_count_magnitude = (largest_single_unit - 1).bit_length() # ceil(log2) + self.half_mag = max(sub_bucket_count_magnitude - 1, 0) + self.unit_mag = lowest.bit_length() - 1 # floor(log2) + sub_bucket_count = 1 << (self.half_mag + 1) + self.half_count = sub_bucket_count // 2 + self.mask = (sub_bucket_count - 1) << self.unit_mag + + def value_from_index(self, i: int) -> int: + bucket = (i >> self.half_mag) - 1 + sub = (i & (self.half_count - 1)) + self.half_count + if bucket < 0: + sub -= self.half_count + bucket = 0 + return sub << (bucket + self.unit_mag) + + def _bucket(self, v: int) -> int: + return (v | self.mask).bit_length() - self.unit_mag - (self.half_mag + 1) + + def _lowest_equiv(self, v: int) -> int: + b = self._bucket(v) + return (v >> (b + self.unit_mag)) << (b + self.unit_mag) + + def _size(self, v: int) -> int: + return 1 << (self._bucket(v) + self.unit_mag) + + def median_equiv(self, v: int) -> int: + return self._lowest_equiv(v) + (self._size(v) >> 1) + + def highest_equiv(self, v: int) -> int: + return self._lowest_equiv(v) + self._size(v) - 1 + + +def _read_zigzag(buf: bytes, pos: int) -> tuple[int, int]: + value = shift = 0 + for i in range(9): + b = buf[pos] + pos += 1 + if i == 8: # the 9th byte carries the top 8 bits whole + value |= b << 56 + break + value |= (b & 0x7F) << shift + if not b & 0x80: + break + shift += 7 + return (value >> 1) ^ -(value & 1), pos + + +def decode_histogram(b64: str) -> tuple[Layout, list[int]]: + raw = base64.b64decode(b64) + cookie, zlen = struct.unpack(">II", raw[:8]) + if cookie & _COOKIE_MASK != _COMPRESSED_BASE: + raise ValueError("not an HdrHistogram V2 compressed histogram") + enc = zlib.decompress(raw[8:8 + zlen]) + cookie, plen, _offset, sig, lowest, highest, _ratio = struct.unpack(">IIIIQQd", enc[:40]) + if cookie & _COOKIE_MASK != _ENCODING_BASE: + raise ValueError("bad HdrHistogram V2 encoding cookie") + payload = enc[40:40 + plen] + counts: list[int] = [] + pos = 0 + while pos < len(payload): + v, pos = _read_zigzag(payload, pos) + if v < 0: + counts.extend([0] * -v) # a run of zeros + else: + counts.append(v) + return Layout(lowest, highest, sig), counts + + +class Merged: + """Counts summed bucket by bucket across any number of decoded histograms.""" + + def __init__(self): + self.layout: Layout | None = None + self.counts: list[int] = [] + + def add(self, b64: str | None) -> None: + if not b64: + return + layout, counts = decode_histogram(b64) + if self.layout is None: + self.layout = layout + elif layout.key != self.layout.key: + raise ValueError(f"histogram layouts differ: {layout.key} vs {self.layout.key}") + if len(counts) > len(self.counts): + self.counts.extend([0] * (len(counts) - len(self.counts))) + for i, c in enumerate(counts): + if c: + self.counts[i] += c + + @property + def total(self) -> int: + return sum(self.counts) + + def percentile(self, p: float) -> int: + total = self.total + if not total or self.layout is None: + return 0 + wanted = int(math.floor((p / 100.0) * total + 0.5)) # Zig @round: half away from zero + wanted = max(wanted, 1) + running = 0 + for i, c in enumerate(self.counts): + running += c + if running >= wanted: + return self.layout.median_equiv(self.layout.value_from_index(i)) + return self.max() + + def mean(self) -> float: + total = self.total + if not total or self.layout is None: + return 0.0 + acc = 0.0 + for i, c in enumerate(self.counts): + if c: + acc += float(self.layout.median_equiv(self.layout.value_from_index(i))) * c + return acc / total + + def max(self) -> int: + if self.layout is None: + return 0 + for i in range(len(self.counts) - 1, -1, -1): + if self.counts[i]: + return self.layout.highest_equiv(self.layout.value_from_index(i)) + return 0 + + def min(self) -> int: + if self.layout is None: + return 0 + for i, c in enumerate(self.counts): + if c: + return self.layout.value_from_index(i) + return 0 + + def latency_block(self) -> dict: + return { + "min": self.min(), "mean": round(self.mean(), 1), "max": self.max(), + "p50": self.percentile(50), "p75": self.percentile(75), + "p90": self.percentile(90), "p99": self.percentile(99), + "p99_9": self.percentile(99.9), "p99_99": self.percentile(99.99), + } + + +# ── Helpers ───────────────────────────────────────────────────────────────── + +def _summary(text: str) -> dict | None: + """zrk's final report, from a stream that may carry other output around it.""" + start, end = text.find("{"), text.rfind("}") + if start < 0 or end <= start: + return None + try: + return json.loads(text[start:end + 1]) + except json.JSONDecodeError: + return None + + +def _rows(path: str) -> list[dict]: + out = [] + try: + with open(path, errors="replace") as f: + for line in f: + line = line.strip() + if line.startswith("{"): + try: + out.append(json.loads(line)) + except json.JSONDecodeError: + pass + except OSError: + pass + return out + + +def _cpu_usec(path: str | None) -> int | None: + if not path: + return None + try: + with open(path) as f: + for line in f: + if line.startswith("usage_usec"): + return int(line.split()[1]) + except (OSError, ValueError, IndexError): + pass + return None + + +def _count_established(port: int, cpus: str | None) -> int | None: + """Server-side ESTABLISHED sockets on `port`, counted without buffering 1M lines.""" + if not shutil.which("ss"): + return None + cmd = ["ss", "-H", "-t", "-n", "state", "established", f"( sport = :{port} )"] + if cpus and shutil.which("taskset"): + cmd = ["taskset", "-c", cpus] + cmd + try: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + except OSError: + return None + n = 0 + deadline = time.monotonic() + 30 + assert p.stdout is not None + while True: + chunk = p.stdout.read(1 << 20) + if not chunk: + break + n += chunk.count(b"\n") + if time.monotonic() > deadline: + p.kill() + return None + p.wait() + return n if p.returncode == 0 else None + + +def _probe(addr: str, port: int) -> str | None: + try: + with socket.create_connection((addr, port), timeout=3): + return None + except OSError as e: + return f"{type(e).__name__}: {e}" + + +def _split(total: int, parts: int) -> list[int]: + base, extra = divmod(total, parts) + return [base + (1 if i < extra else 0) for i in range(parts)] + + +def _name_containers(prefix: list[str], k: int, mem_mib: int = 0) -> list[str]: + """Name each zrk container so the harness's cleanup sweeps it (httparena-*), + and cap its memory so a client that outgrows its estimate is OOM-killed in + its own cgroup instead of taking the host down with it.""" + for i in range(len(prefix) - 1): + if os.path.basename(prefix[i]) == "docker" and prefix[i + 1] == "run": + extra = ["--name", f"httparena-zrk-{k}"] + if mem_mib > 0: + extra += [f"--memory={mem_mib}m", f"--memory-swap={mem_mib}m"] + return prefix[:i + 2] + extra + prefix[i + 2:] + return prefix + + +def _version(prefix: list[str]) -> tuple[int, ...] | None: + """What `zrk --version` says ("zrk 3.0.2"), through the same prefix the run + uses, so a docker image is asked rather than whatever is on PATH.""" + try: + out = subprocess.run(prefix + ["--version"], capture_output=True, text=True, + timeout=120).stdout + except (OSError, subprocess.TimeoutExpired): + return None + for tok in out.split(): + parts = tok.lstrip("v").split(".") + if len(parts) == 3 and all(p.isdigit() for p in parts): + return tuple(int(p) for p in parts) + return None + + +def _empty(args, error: str) -> dict: + return { + "hold_error": error, "connections": args.conns, "launched": 0, + "connections_open": None, "requests": 0, "achieved_rate": 0, + "target_rate": args.rate, "rate_ratio": 0, "duration_s": 0, + "bytes_per_sec": 0, "max_schedule_lag_us": 0, "server_cpu_usec": None, + "latency_us": {}, "status_codes": {"1xx": 0, "2xx": 0, "3xx": 0, "4xx": 0, "5xx": 0}, + "errors": {"connect": 0}, "ramp_connect_errors": 0, + } + + +def _mem_available_kib() -> int | None: + try: + with open("/proc/meminfo") as f: + for line in f: + if line.startswith("MemAvailable:"): + return int(line.split()[1]) + except (OSError, ValueError, IndexError): + pass + return None + + +def _size_kib(s: str | None) -> int: + """'96g' / '512m' / '1048576k' / bytes -> KiB; 0 when unset or unparseable.""" + if not s: + return 0 + s = s.strip().lower() + mult = {"k": 1, "m": 1024, "g": 1024 ** 2, "t": 1024 ** 3} + try: + if s[-1] in mult: + return int(float(s[:-1]) * mult[s[-1]]) + return int(s) // 1024 + except ValueError: + return 0 + + +# Kernel memory per connection, both ends of it: two sockets with their file, +# dentry and epoll entries (~3.5 KiB each), plus a conntrack entry. +_KERNEL_KIB_PER_CONN = 8 + + +def _sleep_until(t: float) -> None: + d = t - time.monotonic() + if d > 0: + time.sleep(d) + + +def log(msg: str) -> None: + print(f"[zrk-fanout] {msg}", file=sys.stderr, flush=True) + + +# ── The run ───────────────────────────────────────────────────────────────── + +def run(args, prefix: list[str]) -> dict: + K = args.addrs + if not 1 <= K <= 254: + return _empty(args, f"--addrs must be 1..254, got {K}") + addrs = [f"127.0.0.{k + 1}" for k in range(K)] + + # 2.3.0, the version the other zrk profiles pin, would need ~290 GiB here. + want = tuple(int(p) for p in args.min_version.split(".")) + ver = _version(prefix) + if ver is None or ver < want: + msg = ("connections-1m-zrk needs zrk >= %s (per-thread latency histograms); " + "%s %s" % (args.min_version, prefix[-1], + "is " + ".".join(map(str, ver)) if ver else + "did not answer --version (not installed, or an image that did not build)")) + log(msg) + return _empty(args, msg) + + # A listener bound to 127.0.0.1 or "localhost" passes everything else in the + # harness and refuses every address here but the first. Say that, rather + # than launching a million connect errors that read like a broken server. + for probe_addr in (addrs[1], addrs[-1]) if K > 1 else (addrs[0],): + err = _probe(probe_addr, args.port) + if err: + msg = (f"no listener reachable on {probe_addr}:{args.port} ({err}). " + "connections-1m-zrk spreads its connections over 127.0.0.1-" + f"{addrs[-1]}, so the server must listen on 0.0.0.0 or [::], " + "not on 127.0.0.1 or localhost alone") + log(msg) + return _empty(args, msg) + + # Refuse what the host cannot hold. The first million-connection run on the + # bench host took it down: zrk 2.3.0 keeps ~303 KiB resident per connection + # (measured flat from 1K to 8K connections), so a million needed ~290 GiB + # of a 251 GiB machine, with nothing capping it. 3.0.2 keeps ~41 KiB. + # Count the client, the kernel's share and everything the server is still + # allowed to grow into. + avail = _mem_available_kib() + server_kib = _size_kib(args.server_mem) + need = args.conns * (args.kib_per_conn + _KERNEL_KIB_PER_CONN) + server_kib + if avail is not None and need > avail * 0.9: + gib = 1024 ** 2 + msg = (f"not enough memory to hold {args.conns} connections: the load generator " + f"needs ~{args.conns * args.kib_per_conn / gib:.0f} GiB at {args.kib_per_conn} KiB " + f"per connection, the kernel ~{args.conns * _KERNEL_KIB_PER_CONN / gib:.0f} GiB, " + f"and the server may grow to {server_kib / gib:.0f} GiB, against " + f"{avail / gib:.0f} GiB available. Refusing rather than running the host out of memory") + log(msg) + return _empty(args, msg) + + conns = _split(args.conns, K) + rates = _split(args.rate, K) + threads = max(1, args.threads // K) + # Per-process cap: half again over the estimate, plus a fixed allowance. + mem_cap_mib = int(conns[0] * args.kib_per_conn * 1.5 / 1024) + 512 if args.kib_per_conn else 0 + stagger, settle, window, tail = args.stagger, args.settle, args.window, args.tail + w0 = (K - 1) * stagger + settle # window start, seconds after the first launch + total = w0 + window + tail + log(f"{args.conns} connections at {args.rate} req/s over {K} addresses " + f"({addrs[0]}-{addrs[-1]}): {conns[0]} conns, {rates[0]} req/s, {threads} threads each; " + f"launches {stagger}s apart, window {w0}-{w0 + window}s, run {total}s") + + work = tempfile.mkdtemp(prefix="zrk-fanout-") + procs: list[subprocess.Popen | None] = [None] * K + launch = [0.0] * K + ended = [None] * K + files = [] + if any(os.path.basename(x) == "docker" for x in prefix): + for k in range(K): # a name left over from a crashed run would refuse the start + subprocess.run(["docker", "rm", "-f", f"httparena-zrk-{k}"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + t0 = time.monotonic() + try: + for k in range(K): + _sleep_until(t0 + k * stagger) + cmd = _name_containers(prefix, k, mem_cap_mib) + [ + "-t", str(threads), "-c", str(conns[k]), "-d", f"{total - k * stagger}s", + "-R", str(rates[k]), "--interval", "1s", + "--timeseries", "-", "--timeseries-histogram", "--format", "json", + f"http://{addrs[k]}:{args.port}{args.path}", + ] + ts = open(os.path.join(work, f"ts{k}.ndjson"), "wb") + rep = open(os.path.join(work, f"rep{k}.txt"), "wb") + files += [ts, rep] + launch[k] = time.monotonic() - t0 + try: + procs[k] = subprocess.Popen(cmd, stdout=ts, stderr=rep, stdin=subprocess.DEVNULL) + except OSError as e: + log(f"process {k} ({addrs[k]}) did not start: {e}") + + _sleep_until(t0 + w0) + cpu_a = _cpu_usec(args.cpu_stat) + _sleep_until(t0 + w0 + window) + cpu_b = _cpu_usec(args.cpu_stat) + # Counted after the window rather than inside it: walking a million + # sockets is kernel work the measurement should not carry. + conns_open = _count_established(args.port, args.ss_cpus) + + deadline = t0 + total + 90 + while any(p is not None and ended[k] is None for k, p in enumerate(procs)): + for k, p in enumerate(procs): + if p is not None and ended[k] is None and p.poll() is not None: + ended[k] = time.monotonic() - t0 + if time.monotonic() > deadline: + for k, p in enumerate(procs): + if p is not None and ended[k] is None: + log(f"process {k} ({addrs[k]}) overran; terminating") + p.terminate() + try: + p.wait(10) + except subprocess.TimeoutExpired: + p.kill() + ended[k] = time.monotonic() - t0 + break + time.sleep(0.05) + finally: + for f in files: + f.close() + + # ── Merge ─────────────────────────────────────────────────────────────── + hist = Merged() + window_reqs = window_bytes = 0 + rate_sum = bps_sum = 0.0 + covered = [] + errs = {"connect": 0, "read": 0, "write": 0, "timeout": 0, "deadline": 0, "non_2xx_3xx": 0} + run_connect = run_4xx = run_5xx = launched = 0 + lag = 0 + per = [] + for k in range(K): + rep_text = open(os.path.join(work, f"rep{k}.txt"), errors="replace").read() + summ = _summary(rep_text) + rows = _rows(os.path.join(work, f"ts{k}.ndjson")) + info = {"addr": addrs[k], "connections": conns[k], "rate": rates[k], + "launched": None, "window_requests": 0, "window_s": 0.0} + if summ is None: + log(f"process {k} ({addrs[k]}) left no summary: {rep_text.strip()[:300] or ''}") + else: + info["launched"] = (summ.get("config") or {}).get("launched") + launched += info["launched"] or 0 + run_connect += (summ.get("errors") or {}).get("connect") or 0 + codes = summ.get("status_codes") or {} + run_4xx += codes.get("4xx") or 0 + run_5xx += codes.get("5xx") or 0 + # This process's clock started when zrk did, which is its launch plus + # however long `docker run` took to get there. Its own elapsed time, + # taken back from when it ended, places that start better than the + # launch time alone; the slack either way is well inside settle/tail. + start = launch[k] + if summ and summ.get("duration_s") and ended[k] is not None: + start = max(launch[k], ended[k] - float(summ["duration_s"])) + lo, hi = w0 - start, w0 + window - start + prev = 0.0 + cov = 0.0 + reqs = byts = 0 + for r in rows: + t = float(r.get("t") or 0) + interval = t - prev + prev = t + if t - interval < lo - 0.5 or t > hi + 0.5: + continue + cov += interval + reqs += r.get("requests") or 0 + byts += r.get("bytes") or 0 + for kind, n in (r.get("errors_by_kind") or {}).items(): + if kind in errs: + errs[kind] += n or 0 + lag = max(lag, r.get("max_schedule_lag_us") or 0) + hist.add(r.get("latency_histogram")) + info["window_requests"], info["window_s"] = reqs, round(cov, 3) + window_reqs += reqs + window_bytes += byts + if cov > 0: + rate_sum += reqs / cov + bps_sum += byts / cov + covered.append(cov) + per.append(info) + + shutil.rmtree(work, ignore_errors=True) + + if launched and launched < args.conns: + log(f"only {launched} of {args.conns} connections were launched - " + "zrk could not spawn the rest (vm.max_map_count, memory, or fds)") + if conns_open is not None: + log(f"{conns_open} established on :{args.port} after the window " + f"({100.0 * conns_open / args.conns:.1f}% of {args.conns})") + + return { + "fanout": {"processes": K, "addresses": f"{addrs[0]}-{addrs[-1]}", + "stagger_s": stagger, "settle_s": settle, "window_s": window, + "tail_s": tail, "threads_per_process": threads, "per_process": per}, + "connections": args.conns, + "launched": launched, + "connections_open": conns_open, + "duration_s": round(sum(covered) / len(covered), 3) if covered else 0, + "requests": window_reqs, + "bytes": window_bytes, + "achieved_rate": round(rate_sum, 2), + "target_rate": args.rate, + "rate_ratio": round(rate_sum / args.rate, 4) if args.rate else 0, + "bytes_per_sec": round(bps_sum, 2), + "max_schedule_lag_us": lag, + "server_cpu_usec": (cpu_b - cpu_a) if cpu_a is not None and cpu_b is not None and cpu_b >= cpu_a else None, + "latency_us": hist.latency_block(), + "status_codes": {"1xx": 0, "2xx": max(window_reqs - errs["non_2xx_3xx"], 0), "3xx": 0, + "4xx": run_4xx, "5xx": run_5xx}, + "errors": errs, + "ramp_connect_errors": max(run_connect - errs["connect"], 0), + } + + +def main(argv: list[str]) -> int: + if "--" not in argv: + print("usage: zrk_fanout.py [options] -- ", file=sys.stderr) + return 2 + i = argv.index("--") + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--conns", type=int, required=True) + ap.add_argument("--rate", type=int, required=True) + ap.add_argument("--addrs", type=int, required=True) + ap.add_argument("--port", type=int, required=True) + ap.add_argument("--path", required=True) + ap.add_argument("--threads", type=int, default=64) + ap.add_argument("--stagger", type=int, default=1) + ap.add_argument("--settle", type=int, default=10) + ap.add_argument("--window", type=int, default=20) + ap.add_argument("--tail", type=int, default=20) + ap.add_argument("--cpu-stat", dest="cpu_stat") + ap.add_argument("--ss-cpus", dest="ss_cpus") + ap.add_argument("--kib-per-conn", dest="kib_per_conn", type=int, default=48, + help="client memory per connection in KiB (zrk 3.0.2: ~41 measured, " + "2.3.0: ~303)") + ap.add_argument("--min-version", dest="min_version", default="3.0.2", + help="the oldest zrk the run accepts") + ap.add_argument("--server-mem", dest="server_mem", + help="the server container's memory cap (e.g. 96g), counted as headroom") + args = ap.parse_args(argv[:i]) + prefix = argv[i + 1:] + if not prefix: + print("zrk_fanout.py: no zrk command after --", file=sys.stderr) + return 2 + print(json.dumps(run(args, prefix), indent=1)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/site/content/docs/add-framework/meta-json.md b/site/content/docs/add-framework/meta-json.md index 3ba3f42e17..65e61e1fb2 100644 --- a/site/content/docs/add-framework/meta-json.md +++ b/site/content/docs/add-framework/meta-json.md @@ -52,6 +52,8 @@ Create a `meta.json` file in your framework directory: | `latency-1m` | HTTP/1.1 | `/baseline11` (fixed 1M req/s) | | `latency-10k` | HTTP/1.1 | `/baseline11` (fixed 10K req/s) | | `latency-500k-8cpu` | HTTP/1.1 | `/baseline11` (fixed 500K req/s on four cores plus SMT) | +| `connections-1m` | HTTP/1.1 | `/baseline11` (1M held connections at a fixed 250K req/s; opt-in, needs a listener on `0.0.0.0` or `[::]`) | +| `connections-1m-zrk` | HTTP/1.1 | `/baseline11` (`connections-1m` with zrk as the load generator instead of gcannon; same requirements) | | `json-comp` | HTTP/1.1 | `/json/{count}?m=N` (must honor `Accept-Encoding: gzip, br`) | | `json-tls` | HTTP/1.1 + TLS | `/json/{count}?m=N` (port 8081, ALPN `http/1.1`) | | `8gbit` | HTTP/1.1 + TLS | `POST /echo` on port 8081 (10 KB echoed verbatim) | diff --git a/site/content/docs/load-generators/h1/gcannon.md b/site/content/docs/load-generators/h1/gcannon.md index faa7de2034..b89bcedd6d 100644 --- a/site/content/docs/load-generators/h1/gcannon.md +++ b/site/content/docs/load-generators/h1/gcannon.md @@ -83,6 +83,17 @@ Latency is measured per-request from the moment the send SQE is prepared (not wh For pipelined requests, each request in the batch gets its own send timestamp, and latencies are matched FIFO to response completions. +## Fixed-rate mode (`--rate`, gcannon ≥ 0.8.0) + +Everywhere else gcannon sends the next request the moment the last response arrives. With `--rate N` it holds a fixed total rate instead, which is how [1M Connections](/docs/test-profiles/h1/connections-1m/implementation/) runs: + +- every connection has its own schedule, one request per (connections / rate) seconds, the first offset by its share of the interval so the fleet sends evenly rather than in waves; +- latency counts from each request's scheduled time (coordinated-omission correction), so a stall is charged to every request scheduled during it, not only to those already on the wire; +- the event loop sleeps exactly until the next send is due, with 1 ns timer slack, so sends leave within tens of microseconds of their slot; at the same paced load it reads the same p50-p99 as [zrk](../zrk/); +- one process holds a million connections in ~3.3 GiB (~3.3 KiB each), where zrk 2.3.0 needs ~303 KiB per connection and 3.0.2 ~41 KiB ([connections-1m-zrk](/docs/test-profiles/h1/connections-1m/implementation/#connections-1m-zrk-the-same-test-on-zrk) runs the same test on it). + +Three options make a fleet that large possible: `--addr-span 32` spreads it over `127.0.0.1`-`127.0.0.32` (one source address holds ~64.5K connections to one destination), `--connect-limit` caps first connects in flight per thread so a million SYNs never land at once, and `--warmup-max` lets the warmup wait long enough for all of them to open before the window starts. On stderr it marks the window's start and end (`gcannon: warmup done ...`, `gcannon: measurement done`), which `scripts/conn_hold.py` uses to read the server's CPU over exactly that window. + ## Command-line reference ``` @@ -99,3 +110,7 @@ Usage: gcannon -c -t -d | `-p` | Pipeline depth | 1 | | `-r` | Requests per connection (0 = unlimited) | 0 | | `--raw` | Comma-separated raw request template files | -- | +| `--rate` | Fixed total request rate (req/s), paced per connection; needs `-p 1`, no `-r` | closed loop | +| `--addr-span` | Spread connections over N consecutive IPv4 addresses from the target's | 1 | +| `--connect-limit` | First connects in flight per thread | all at once | +| `--warmup-max` | Longest the warmup waits for connections to open | 10s | diff --git a/site/content/docs/load-generators/h1/zrk.md b/site/content/docs/load-generators/h1/zrk.md index b90f8712d7..97bb49ac85 100644 --- a/site/content/docs/load-generators/h1/zrk.md +++ b/site/content/docs/load-generators/h1/zrk.md @@ -36,6 +36,8 @@ The image pins an exact version and verifies its checksum, unlike gcannon which zrk has no distro package. `benchmark.sh` builds the `zrk:local` image from `docker/zrk.Dockerfile` automatically when a framework subscribes to one of these profiles and no `zrk` binary is on the `PATH`. +`connections-1m-zrk` is the exception: it runs zrk **3.0.2** from its own `zrk:3.0.2` image, the same Dockerfile built with that version and checksum, while the profiles above stay on 2.3.0. A million held connections need 3.0.2's per-thread latency histograms: 2.3.0 keeps ~303 KiB resident per connection, 3.0.2 ~41 KiB. It is the [1M Connections](/docs/test-profiles/h1/connections-1m/implementation/#connections-1m-zrk-the-same-test-on-zrk) test with zrk instead of gcannon, to compare the two. + ## Parameters | Profile | Request | Offered rate | Connections | Duration | @@ -44,5 +46,6 @@ zrk has no distro package. `benchmark.sh` builds the `zrk:local` image from `doc | `latency-10k` | `GET /baseline11?a=1&b=2` | 10,000 req/s | 1,024 | 20s | | `latency-500k-8cpu` | `GET /baseline11?a=1&b=2` | 500,000 req/s | 1,024 | 20s | | `8gbit` | `POST /echo` with a 10 KB body, over TLS on port 8081 | 50,000 req/s | 512 | 5s | +| `connections-1m-zrk` | `GET /baseline11?a=1&b=2`, over 127.0.0.1-.32 | 250,000 req/s | 1,000,000 (32 processes) | 20s window | All of them run with 64 threads and 3 runs, keeping the run with the best score. diff --git a/site/content/docs/running-locally/prerequisites.md b/site/content/docs/running-locally/prerequisites.md index caf2b4ab57..70be173669 100644 --- a/site/content/docs/running-locally/prerequisites.md +++ b/site/content/docs/running-locally/prerequisites.md @@ -51,7 +51,7 @@ The one exception is `zrk`, which drives `8gbit`: `benchmark-lite.sh` does not b For the default (native) mode, install each tool the host binary. If anything is missing, set `LOADGEN_DOCKER=true` and the script falls back to the same docker images `benchmark-lite.sh` uses. -- **gcannon** - io_uring HTTP/1.1, WebSocket, and `--raw` multi-template generator. Requires Linux kernel ≥ 6.1 with `io_uring` enabled. +- **gcannon** - io_uring HTTP/1.1, WebSocket, and `--raw` multi-template generator. Requires Linux kernel ≥ 6.1 with `io_uring` enabled. `connections-1m` needs gcannon ≥ 0.8.0 (`--rate`, `--addr-span`); `gcannon --version` prints it, and an older build has no `--version` at all. ```bash git clone --branch liburing-2.9 https://github.com/axboe/liburing.git diff --git a/site/content/docs/running-locally/scripts/benchmark.md b/site/content/docs/running-locally/scripts/benchmark.md index 18f028c2ef..e86496741b 100644 --- a/site/content/docs/running-locally/scripts/benchmark.md +++ b/site/content/docs/running-locally/scripts/benchmark.md @@ -133,6 +133,8 @@ pipeline | req_per_conn | cpu_limit | connections | endpoint | `latency-1m` | 1 | ∞ | `0-31,64-95` | 1024 | zrk | `/baseline11` paced at 1M req/s, 20s runs | | `latency-10k` | 1 | ∞ | `0-31,64-95` | 1024 | zrk | `/baseline11` paced at 10K req/s, 20s runs | | `latency-500k-8cpu` | 1 | ∞ | `0-3,64-67` | 1024 | zrk | `/baseline11` paced at 500K req/s, 20s runs | +| `connections-1m` | 1 | ∞ | `0-31,64-95` | 1000000 | gcannon `--rate` (≥ 0.8.0) | `/baseline11` paced at 250K req/s over 1M held connections on 127.0.0.1-.32, 20s window | +| `connections-1m-zrk` | 1 | ∞ | `0-31,64-95` | 1000000 | zrk 3.0.2, one per address | the same as `connections-1m`, on zrk | | `json-comp` | 1 | ∞ | `0-31,64-95` | 4096, 16384 | gcannon | `/json/{count}` + `Accept-Encoding: gzip, br` | | `json-tls` | 1 | ∞ | `0-31,64-95` | 4096 | wrk | `/json/{count}` over TLS on `H1TLS_PORT` | | `8gbit` | 1 | ∞ | `0-31,64-95` | 512 | zrk | `POST /echo` on TLS `:8081` - 10 KB echoed, paced at 50k req/s | diff --git a/site/content/docs/scoring/composite-score.md b/site/content/docs/scoring/composite-score.md index de118b4654..14a7ffcd56 100644 --- a/site/content/docs/scoring/composite-score.md +++ b/site/content/docs/scoring/composite-score.md @@ -79,6 +79,8 @@ The tables below show whether a profile counts for framework entries. Engines an | Latency-10K | Yes | Fixed 10K req/s; scored on CPU cost and latency (0-100 score, see above) | | Latency-1M | Yes | Fixed 1M req/s; scored on CPU cost and latency (0-100 score, see above) | | Latency-500K/8 | No (*) | Fixed 500K req/s on four cores plus SMT. Reference-only while the first entries are measured | +| 1M Connections | No (*) | One million held connections at a fixed 250K req/s. Opt-in and reference-only while the first entries are measured | +| 1M Connections (zrk) | No (*) | The same test with zrk as the load generator instead of gcannon. Opt-in and reference-only | | JSON Compressed | Yes | JSON with `Accept-Encoding: gzip, br` and multiplier `?m=N` | | JSON TLS | Yes | JSON workload over HTTP/1.1 + TLS on port 8081 | | 8Gbit | Yes | 10 KB echoed over TLS at a fixed 50K req/s; scored on CPU cost and latency (0-100 score, see above) | diff --git a/site/content/docs/test-profiles/h1/_index.md b/site/content/docs/test-profiles/h1/_index.md index 1914d83f49..3d620e7248 100644 --- a/site/content/docs/test-profiles/h1/_index.md +++ b/site/content/docs/test-profiles/h1/_index.md @@ -17,6 +17,7 @@ Every HTTP/1.1 profile measures one thing at a time, so a number can be attribut {{< card link="latency-10k" title="Latency-10K" subtitle="Latency-1M's setup at a near-idle 10K req/s; the metric is the CPU and latency of a server with almost nothing to do." icon="chip" >}} {{< card link="latency-1m" title="Latency-1M" subtitle="One million req/s offered at a fixed rate; the metric is the CPU each framework spends to serve it, read exactly from its cgroup." icon="chip" >}} {{< card link="latency-500k-8cpu" title="Latency-500K/8 *" subtitle="500K req/s pinned with the server on four cores plus their SMT siblings - near saturation, so the queue a server builds shows in its mean latency. Reference-only while the first entries are measured." icon="chip" >}} + {{< card link="connections-1m" title="1M Connections *" subtitle="One million keep-alive connections held at once, 250K req/s paced across them - measures what each held connection costs in memory, CPU and tail latency. Opt-in and reference-only while the first entries are measured." icon="globe" >}} {{< card link="async-database" title="Async Database (Postgres)" subtitle="Async Postgres range query over 100K rows, connection pooling, and JSON serialization. Framework-only benchmark." icon="database" >}} {{< card link="static-tls" title="Static Files over TLS" subtitle="20 static files served over HTTP/1.1 + TLS on port 8081. Reference-only for frameworks and engines, scored for infrastructure." icon="lock-closed" >}} {{< card link="pipelined" title="Pipelined (16x) *" subtitle="16 requests sent back-to-back per connection, testing raw I/O and pipeline batching. Reference-only - not part of the composite score." icon="fast-forward" >}} diff --git a/site/content/docs/test-profiles/h1/connections-1m/_index.md b/site/content/docs/test-profiles/h1/connections-1m/_index.md new file mode 100644 index 0000000000..a241bddc77 --- /dev/null +++ b/site/content/docs/test-profiles/h1/connections-1m/_index.md @@ -0,0 +1,12 @@ +--- +title: 1M Connections +seo_title: "1M Connections: Holding a Million Keep-Alive Connections at Once" +description: "Holds one million HTTP/1.1 keep-alive connections open at once, with 250,000 requests per second paced across them, and measures the memory, CPU and latency it costs the server to carry them." +--- + +One million keep-alive connections, all open at the same time, each sending one request every four seconds. Every other profile keeps a few thousand connections busy; this one keeps a million mostly idle, the way a chat, push or IoT gateway does, so what separates entries is what each held connection costs: memory, the CPU a request carries while the other 999,999 wait, and the tail once the heap, the timer wheel or the poll set is that large. Opt-in, and reference-only while the first entries are measured. + +{{< cards >}} + {{< card link="implementation" title="Implementation Guidelines" subtitle="Endpoint, listener requirements, what is measured, and how it is scored." icon="code" >}} + {{< card link="validation" title="Validation" subtitle="All checks executed by the validation script for this test profile." icon="check-circle" >}} +{{< /cards >}} diff --git a/site/content/docs/test-profiles/h1/connections-1m/implementation.md b/site/content/docs/test-profiles/h1/connections-1m/implementation.md new file mode 100644 index 0000000000..4f28491a8e --- /dev/null +++ b/site/content/docs/test-profiles/h1/connections-1m/implementation.md @@ -0,0 +1,81 @@ +--- +title: Implementation Guidelines +seo_title: "1M Connections Benchmark: Implementation Guide" +description: "What to implement, what the listener must allow, what is measured, and how the one-million-connection profile is scored." +--- +{{< type-rules standard="Nothing new to implement - the profile drives `GET /baseline11`, which the baseline profile already specifies and validates. What it asks of you is a server that can hold a million connections: it must listen on every loopback address (0.0.0.0 or [::], not 127.0.0.1 or localhost), must not cap connections below a million, and must not close an idle keep-alive connection in under five seconds. Memory per connection is the point of the profile, so buffers sized and kept per connection show up directly in the Mem column." tuned="May size buffers, pools and accept loops for a million connections, raise connection caps and idle timeouts, and set the runtime's heap and GC options. All of it is charged: memory is read from the container, which is capped at 96 GiB." engine="Same as above. An engine with a fixed-size connection table, or one that preallocates a full read buffer per connection, finds out here." >}} + + +A million keep-alive connections are opened against the server and held for the whole run, and `GET /baseline11?a=1&b=2` is paced at 250,000 requests per second across them: **one request per connection every four seconds**. At that rate the work per request is a few percent of what a fast server carries on this cpuset, so what is left to measure is what holding the rest of the million costs. + +**Endpoint:** `GET /baseline11?a=1&b=2` · **Connections:** 1,000,000 · **Rate:** 250,000 req/s · **CPU:** cpuset `0-31,64-95` · **Window:** 20s, after the fleet is connected + +## What your server must allow + +The request is the baseline one, so there is no code to write. Four things about the listener decide whether an entry can run here at all: + +- **Listen on every address.** A connection is identified by its source and destination address and port, so one client address can hold at most about 64.5K connections to one server address and port. The million is therefore spread over 32 loopback addresses, `127.0.0.1` to `127.0.0.32`, 31,250 connections each. A listener on `0.0.0.0` or `[::]` accepts on all of them; one bound to `127.0.0.1` or `localhost` refuses 31 of the 32. [Validation](../validation) checks this, and the run refuses to start without it. +- **No connection cap below a million.** Some servers ship with a per-worker or global limit (actix's is 25,000 per worker, which is enough at 64 workers). Raise anything lower. +- **An idle timeout of five seconds or more.** Each connection is idle for four seconds between requests. Node's, uvicorn's and actix's default keep-alive timeouts are five seconds, which is enough; anything shorter closes the fleet between requests and shows up as errors. +- **File descriptors.** The container is started with `--ulimit nofile=2097152` (capped at the host's `fs.nr_open`), so a million sockets fit. A server that sets a lower limit of its own, or spends a descriptor per connection on something besides the socket, has to leave room for them. + +## How the load is run + +One [gcannon](/docs/load-generators/h1/gcannon/) process holds the whole fleet: `--rate 250000` gives each connection its own schedule, one request every four seconds, staggered so the million do not fire in waves, with latency counted from each request's scheduled time. `--addr-span 32` spreads the connections over `127.0.0.1`-`127.0.0.32`, 31,250 on each, and `--connect-limit 64` keeps at most 64 first connects in flight per thread, so they never arrive as one burst of SYNs. + +gcannon's warmup waits until every connection has opened (up to 180s), and **what is reported is the 20 seconds after that**. [`scripts/conn_hold.py`](https://github.com/MDA2AV/HttpArena/blob/main/scripts/conn_hold.py) reads the server's cgroup CPU when gcannon marks the window's start and end, so neither the million connects nor the million closes are in the CPU figure. It also probes the listener and refuses a run the host cannot hold in memory. + +The load generator is gcannon rather than [zrk](/docs/load-generators/h1/zrk/), the other paced generator here, because of memory: zrk 2.3.0 keeps ~303 KiB resident per connection, so a million needed ~290 GiB, and the first attempt ran the 251 GiB bench host out of memory. gcannon holds a connection in ~3.3 KiB. At the same paced load the two report the same p50 to p99. + +## connections-1m-zrk: the same test on zrk + +`connections-1m-zrk` is this profile with zrk as the load generator, so the two generators can be compared on one load: the same million connections, the same 250,000 req/s, the same cpusets, 64 client threads, a 20s window after the fleet is connected, and the same score. Only the generator and how it opens the fleet differ. + +- **zrk 3.0.2, not the pinned 2.3.0.** 3.0.2 shares one latency histogram per thread instead of keeping one per connection, which takes a connection from ~303 KiB to ~41 KiB (measured flat at 4K and 16K connections), ~40 GiB for the million. The other zrk profiles stay on 2.3.0; this one builds its own `zrk:3.0.2` image from the same Dockerfile, and refuses an older zrk. +- **One process per address.** zrk takes one URL, so [`scripts/zrk_fanout.py`](https://github.com/MDA2AV/HttpArena/blob/main/scripts/zrk_fanout.py) runs 32 of them, one per address from `127.0.0.1` to `127.0.0.32`, each holding 31,250 connections at 7,812 req/s on 2 threads. +- **Staggered starts instead of a connect limit.** Each zrk opens all of its connections the moment it starts, so the 32 start one second apart, the 10s after the last start are discarded, and the 20s that follow are measured. +- **Merged histograms.** Each process writes its full latency histogram every second; the rows inside the window are summed bucket by bucket and the percentiles read off the sum, never averaged across processes. +- **Held** is counted from the server's side: its established sockets on :8080, read with `ss` after the window. + +Subscribe by adding `connections-1m-zrk` to `tests`, next to `connections-1m`. Validation is the same for both. + +## What is measured + +| field | meaning | +|---|---| +| `connections_open` | connections still open when the window closed, counted by gcannon ("Held" on the board); on `connections-1m-zrk`, the server's established sockets after the window | +| `memory` | the container's peak memory across the run, a million connections included. Divided by the million it is the cost of a connection, to within the server's baseline | +| `cpu_usec` | server CPU microseconds over the window only, read from cgroup v2 `cpu.stat`, so the million connects are not in it | +| `cpu_per_req_us` | the same divided by requests served in the window | +| `avg_latency`, `p99_latency`, `p99_9_latency` | coordinated-omission-corrected, over the window | +| `rate_ratio` | achieved over offered in the window. A connection that never opened sends nothing, so a fleet only partly held lands here too | +| `window_connect_errors` | connect failures inside the window: connections the server dropped and refused again | + +## Parameters + +| Parameter | Value | +|-----------|-------| +| Endpoint | `GET /baseline11?a=1&b=2` (GET only, no body) | +| Connections | 1,000,000 over 127.0.0.1-127.0.0.32, 31,250 each | +| Offered rate | 250,000 req/s, paced: one request per connection every 4s | +| CPU | cpuset `0-31,64-95` (32 cores, both threads) | +| Memory cap | 96 GiB (`CONN1M_MEM_LIMIT`); a server above it is OOM-killed | +| Ramp | at most 64 first connects in flight per thread; the warmup waits up to 180s for all of them | +| Window | 20s | +| Runs | 3, and the highest-scoring one is kept | +| Load generator | [gcannon](/docs/load-generators/h1/gcannon/) ≥ 0.8.0 with `--rate`, under `scripts/conn_hold.py`; on `connections-1m-zrk`, 32 [zrk](/docs/load-generators/h1/zrk/) 3.0.2 processes under `scripts/zrk_fanout.py` | +| Host limits | `fs.nr_open`, `vm.max_map_count` (zrk maps ~4 regions per connection) and the conntrack table are raised for these two profiles only (`system_tune_connections_1m`) | + +## Scoring + +The fixed-rate score of [Latency-1M](../latency-1m/implementation/#scoring), with the rate threshold moved: + +``` +rateFactor = min(1, achieved_rps / 237,500) +quality = 0.50 x cpuScore + 0.25 x p99Score + 0.25 x meanScore +score = 100 x rateFactor x quality +``` + +Full credit at 237,500 req/s, 95% of the rate offered, computed by [`scripts/latency_score.py`](https://github.com/MDA2AV/HttpArena/blob/main/scripts/latency_score.py) over the window. Memory and the held count are shown next to the score, not inside it. + +**Reference-only and opt-in for now.** The profile is measured and shown but enters no composite score while the first entries are measured, and how memory should weigh against CPU and the tail is still open ([#1759](https://github.com/MDA2AV/HttpArena/issues/1759)). Subscribe by adding `connections-1m` to `tests` in your `meta.json`. diff --git a/site/content/docs/test-profiles/h1/connections-1m/validation.md b/site/content/docs/test-profiles/h1/connections-1m/validation.md new file mode 100644 index 0000000000..b82fe8cbc6 --- /dev/null +++ b/site/content/docs/test-profiles/h1/connections-1m/validation.md @@ -0,0 +1,24 @@ +--- +title: Validation +seo_title: "1M Connections Benchmark: Validation Checks" +description: "How correctness is established for the one-million-connection profile, and the listener check that is specific to it." +--- + +The profile drives `GET /baseline11?a=1&b=2`, the baseline endpoint unchanged, so an entry subscribed to `connections-1m` or `connections-1m-zrk` is run through the full [baseline validation suite](../baseline/validation): the arithmetic, the `Content-Type`, the randomized-input anti-cheat, the TCP fragmentation checks, and the exhaustive per-byte fragmentation sweep. + +## A listener on every address + +One check is specific to this profile. The million connections are spread over `127.0.0.1` to `127.0.0.32`, because one client address can hold only about 64.5K connections to one server address and port. A server bound to `127.0.0.1` or `localhost` passes every baseline check and refuses 31 of the 32 addresses, so validation sends the baseline request to both ends of the range: + +| check | request | expected | +|---|---|---| +| GET /baseline11 on 127.0.0.2 | `GET http://127.0.0.2:8080/baseline11?a=13&b=42` | `55` | +| GET /baseline11 on 127.0.0.32 | `GET http://127.0.0.32:8080/baseline11?a=13&b=42` | `55` | + +To pass, listen on `0.0.0.0` or `[::]` (an IPv6 wildcard listener accepts IPv4 connections too, unless it sets `IPV6_V6ONLY`). The benchmark repeats the probe before it launches anything, and a server that fails it gets an empty row with the reason rather than a million connect errors. + +## What the run checks + +- **Held, not just offered.** gcannon counts the connections still open at the end of the window; on `connections-1m-zrk` the harness counts the server's established sockets on :8080 from the kernel's socket table after it. Either way a server that dropped connections, or never accepted them, shows its real count in the Held column. +- **`rate_ratio`.** Achieved over offered in the window, published with the row. A connection that never opened sends nothing, so a fleet only partly held falls short here as well as in Held, and the score falls proportionally. +- **CPU from outside.** As on the other fixed-rate profiles, CPU is read by the harness from the container's cgroup rather than reported from inside it, over the window only. diff --git a/site/leaderboard/index.html b/site/leaderboard/index.html index d5b4a33fc6..9dbdce5289 100644 --- a/site/leaderboard/index.html +++ b/site/leaderboard/index.html @@ -2272,7 +2272,7 @@

setupTopScroll(); } - function val(r,k){ if(k==='rps')return r.rps||0; if(k==='bw')return bw(r.bandwidth); if(k==='bwreq'){var b=bw(r.bandwidth),q=r.rps||0;return q>0?b/q:0;} if(k==='inbw')return bw(r.input_bw); if(k==='avg')return lat(r.avg_latency); if(k==='p99')return lat(r.p99_latency); if(k==='l1mscore')return latScore(state.profile,r.fw); if(k==='p999')return lat(r.p99_9_latency); if(k==='cpureq')return (r.cpu_per_req_us!=null)?+r.cpu_per_req_us:Infinity; if(k==='cputot'){var t=cpuTotPct(r); return t==null?Infinity:t;} if(k==='cpu')return cpu(r.cpu); if(k==='mem')return mem(r.memory); if(k==='fw')return (r.fw||'').toLowerCase(); return 0; } + function val(r,k){ if(k==='rps')return r.rps||0; if(k==='held')return r.connections_open!=null?+r.connections_open:0; if(k==='bw')return bw(r.bandwidth); if(k==='bwreq'){var b=bw(r.bandwidth),q=r.rps||0;return q>0?b/q:0;} if(k==='inbw')return bw(r.input_bw); if(k==='avg')return lat(r.avg_latency); if(k==='p99')return lat(r.p99_latency); if(k==='l1mscore')return latScore(state.profile,r.fw); if(k==='p999')return lat(r.p99_9_latency); if(k==='cpureq')return (r.cpu_per_req_us!=null)?+r.cpu_per_req_us:Infinity; if(k==='cputot'){var t=cpuTotPct(r); return t==null?Infinity:t;} if(k==='cpu')return cpu(r.cpu); if(k==='mem')return mem(r.memory); if(k==='fw')return (r.fw||'').toLowerCase(); return 0; } function fmtBytes(n){ if(n>=1048576)return (n/1048576).toFixed(2)+' MB'; if(n>=1024)return (n/1024).toFixed(1)+' KB'; return Math.round(n)+' B'; } // Explorer columns - profile-specific (json-comp adds BW/req, upload adds In BW); // low-priority columns drop out below width breakpoints so the grid never overflows. @@ -2284,12 +2284,12 @@

// Offered rate that earns the whole rate factor, per profile. 95% of target: // the generator never quite reaches its own number, so the threshold has to // sit below it. Mirrors FULL_RATE in scripts/latency_score.py. - var LAT_FULL={'latency-1m':950000, 'latency-10k':9500, 'latency-500k-8cpu':475000, '8gbit':47500}; + var LAT_FULL={'latency-1m':950000, 'latency-10k':9500, 'latency-500k-8cpu':475000, '8gbit':47500, 'connections-1m':237500, 'connections-1m-zrk':237500}; // Logical CPUs each fixed-rate profile is pinned to, per scripts/lib/profiles.sh: // cpuset 0-31,64-95 is 32 cores with both threads, so 64; latency-500k-8cpu // gets 0-3,64-67, four cores with both threads. Used to turn CPU-per-request // into "how much of the machine it was allowed did this take". - var LAT_CPUS={'latency-1m':64, 'latency-10k':64, '8gbit':64, 'latency-500k-8cpu':8}; + var LAT_CPUS={'latency-1m':64, 'latency-10k':64, '8gbit':64, 'latency-500k-8cpu':8, 'connections-1m':64, 'connections-1m-zrk':64}; // CPU microseconds per second / 1e6 = cores in use; over the allowance = 0-100%. function cpuTotPct(r,pid){ if(r==null||r.cpu_per_req_us==null) return null; @@ -2361,8 +2361,11 @@

if(isLatProfile(state.profile)){ c.push({k:'l1mscore', label:'Score', num:true, w:'92px', min:0, tip:'0-100. rateFactor \u00d7 (0.50\u00d7CPU + 0.25\u00d7p99 + 0.25\u00d7mean), each measured against the best in the field. rateFactor is 1 at 95% of the offered rate and falls proportionally below it. Higher is better.'}); c.push({k:'rps', label:'Requests / sec', num:true, w:'minmax(150px,1fr)', min:0, tip:TIP.rps}); + // 1M Connections: how much of the million the server actually held. The + // rate cannot say it alone - a connection that never opened sends nothing. + if(state.profile==='connections-1m'||state.profile==='connections-1m-zrk') c.push({k:'held', label:'Held', num:true, w:'104px', min:0, tip:'Connections the server still had established after the measured window, counted from its side of the socket table. 1,000,000 is the whole fleet.'}); c.push({k:'cpureq', label:'CPU / req', num:true, w:'110px', min:0, tip:'CPU microseconds the server spent per request, read exactly out of its cgroup rather than sampled. This is the profile\u2019s metric - lower is better.'}); - c.push({k:'cputot', label:'CPU used', num:true, w:'100px', min:520, tip:'The same CPU cost read as a share of the machine this profile allows: CPU-per-request \u00d7 the rate delivered, over the logical CPUs the profile pins (64 for Latency-1M, Latency-10K and 8Gbit, 8 for Latency-500K/8). 100% would mean the entry saturated everything it was given.'}); + c.push({k:'cputot', label:'CPU used', num:true, w:'100px', min:520, tip:'The same CPU cost read as a share of the machine this profile allows: CPU-per-request \u00d7 the rate delivered, over the logical CPUs the profile pins (64 for Latency-1M, Latency-10K, 8Gbit and 1M Connections, 8 for Latency-500K/8). 100% would mean the entry saturated everything it was given.'}); c.push({k:'p99', label:'p99', num:true, w:'92px', min:0, tip:TIP.p99}); c.push({k:'avg', label:'Avg', num:true, w:'92px', min:640, tip:'Mean latency, coordinated-omission corrected - the queueing signal, in-flight requests over the rate. Weighted at 0.25 in this profile\u2019s score; ten times the best mean scores zero on that term.'}); c.push({k:'mem', label:'Mem', num:true, w:'86px', min:820, tip:TIP.mem}); @@ -2389,6 +2392,7 @@

if(k==='p999') return '
'+esc(r.p99_9_latency||'-')+'
'; if(k==='cpureq') return '
'+(r.cpu_per_req_us!=null?esc((+r.cpu_per_req_us).toFixed(2))+' \u00b5s':'-')+'
'; if(k==='cputot'){ var t=cpuTotPct(r,state.profile); return '
'+(t==null?'-':esc(t.toFixed(1))+'%')+'
'; } + if(k==='held') return '
'+(r.connections_open!=null?fmt(+r.connections_open):'-')+'
'; if(k==='avg') return '
'+esc(r.avg_latency||'-')+'
'; if(k==='p99') return '
'+esc(r.p99_latency||'-')+'
'; if(k==='cpu') return '
'+esc(r.cpu||'-')+'
';