diff --git a/.gitignore b/.gitignore index 5233f61e..3cfb1201 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,5 @@ packages/app/.env # turbo .turbo/ .rollup.cache/ + +.codspeed diff --git a/CLAUDE.md b/CLAUDE.md index b82671a6..00a76a53 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,4 +94,25 @@ Based on the codebase analysis, to add stats access features: - Build: `pnpm turbo run build --filter=` - Typecheck: `pnpm turbo run typecheck --filter=` - Lint: `pnpm turbo run lint --filter=` - - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). \ No newline at end of file + - Run a task across all packages by omitting `--filter` (e.g. `pnpm turbo run build`). + +## Testing a plugin during development + +A plugin behaves differently depending on whether CodSpeed is driving the run, +so exercise all three of these when developing or reviewing a plugin change +(build the plugin first — the benches import from `dist`): + +1. **Fallback (not under CodSpeed).** No env vars. The plugin must stay out of + the way and let the framework run its benchmarks normally (no instrumentation, + no hijacked output). e.g. `pnpm turbo run bench --filter=`. +2. **Instrumentation / simulation.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=simulation` + (or `instrumentation`). The plugin hijacks the run to do a single instrumented + pass per benchmark and prints `Measured/Checked ` instead of the normal + harness output. +3. **Walltime.** `CODSPEED_ENV=true CODSPEED_RUNNER_MODE=walltime`. The plugin + instruments the framework's real benchmark loop and collects walltime results. + +Running these locally outside the CodSpeed runner is expected to log +`instrument-hooks: failed to write environment.json` and skip actual measurement +writes — the point is to verify the plugin's control flow and output per mode, +not to produce real measurements. \ No newline at end of file diff --git a/examples/with-vitest-v4/package.json b/examples/with-vitest-v4/package.json new file mode 100644 index 00000000..8a67c417 --- /dev/null +++ b/examples/with-vitest-v4/package.json @@ -0,0 +1,13 @@ +{ + "name": "with-vitest-v4", + "private": true, + "type": "module", + "scripts": { + "bench-vitest": "vitest bench --run" + }, + "devDependencies": { + "@codspeed/vitest-plugin": "workspace:*", + "typescript": "^5.1.3", + "vitest": "^4.1.9" + } +} diff --git a/examples/with-vitest-v4/src/fibonacci.bench.ts b/examples/with-vitest-v4/src/fibonacci.bench.ts new file mode 100644 index 00000000..227a6e67 --- /dev/null +++ b/examples/with-vitest-v4/src/fibonacci.bench.ts @@ -0,0 +1,20 @@ +import { bench, describe } from "vitest"; +import { iterativeFibonacci, recursiveFibonacci } from "./fibonacci"; + +describe("fibonacci", () => { + bench("recursive fibo 15", () => { + recursiveFibonacci(15); + }); + + bench("recursive fibo 20", () => { + recursiveFibonacci(20); + }); + + bench("iterative fibo 15", () => { + iterativeFibonacci(15); + }); + + bench("iterative fibo 20", () => { + iterativeFibonacci(20); + }); +}); diff --git a/examples/with-vitest-v4/src/fibonacci.ts b/examples/with-vitest-v4/src/fibonacci.ts new file mode 100644 index 00000000..94796660 --- /dev/null +++ b/examples/with-vitest-v4/src/fibonacci.ts @@ -0,0 +1,17 @@ +export function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +export function iterativeFibonacci(n: number): number { + let a = 0; + let b = 1; + for (let i = 0; i < n; i++) { + const temp = a + b; + a = b; + b = temp; + } + return a; +} diff --git a/examples/with-vitest-v4/tsconfig.json b/examples/with-vitest-v4/tsconfig.json new file mode 100644 index 00000000..ace1de03 --- /dev/null +++ b/examples/with-vitest-v4/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "verbatimModuleSyntax": true, + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-vitest-v4/vitest.config.ts b/examples/with-vitest-v4/vitest.config.ts new file mode 100644 index 00000000..4b1290c1 --- /dev/null +++ b/examples/with-vitest-v4/vitest.config.ts @@ -0,0 +1,6 @@ +import codspeedPlugin from "@codspeed/vitest-plugin"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [codspeedPlugin()], +}); diff --git a/examples/with-vitest-v5/package.json b/examples/with-vitest-v5/package.json new file mode 100644 index 00000000..440fd445 --- /dev/null +++ b/examples/with-vitest-v5/package.json @@ -0,0 +1,13 @@ +{ + "name": "with-vitest-v5", + "private": true, + "type": "module", + "scripts": { + "bench-vitest": "vitest bench --run" + }, + "devDependencies": { + "@codspeed/vitest-plugin": "workspace:*", + "typescript": "^5.1.3", + "vitest": "^5.0.0" + } +} diff --git a/examples/with-vitest-v5/src/fibonacci.bench.ts b/examples/with-vitest-v5/src/fibonacci.bench.ts new file mode 100644 index 00000000..3d303e3b --- /dev/null +++ b/examples/with-vitest-v5/src/fibonacci.bench.ts @@ -0,0 +1,38 @@ +import { describe, test } from "vitest"; +import { + iterativeFibonacci as iterativeFibonacciExport, + recursiveFibonacci as recursiveFibonacciExport, +} from "./fibonacci"; + +// Read the imported bindings once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const recursiveFibonacci = recursiveFibonacciExport; +const iterativeFibonacci = iterativeFibonacciExport; + +// Vitest 5 declares benchmarks through the `bench` test-context fixture: each +// `bench()` returns a registration that runs on `.run()`, or as part of a +// `bench.compare()` group. +describe("fibonacci", () => { + test("fibo 15", async ({ bench }) => { + await bench.compare( + bench("recursive", () => { + recursiveFibonacci(15); + }), + bench("iterative", () => { + iterativeFibonacci(15); + }), + ); + }); + + test("fibo 20", async ({ bench }) => { + await bench.compare( + bench("recursive", () => { + recursiveFibonacci(20); + }), + bench("iterative", () => { + iterativeFibonacci(20); + }), + ); + }); +}); diff --git a/examples/with-vitest-v5/src/fibonacci.ts b/examples/with-vitest-v5/src/fibonacci.ts new file mode 100644 index 00000000..94796660 --- /dev/null +++ b/examples/with-vitest-v5/src/fibonacci.ts @@ -0,0 +1,17 @@ +export function recursiveFibonacci(n: number): number { + if (n < 2) { + return n; + } + return recursiveFibonacci(n - 1) + recursiveFibonacci(n - 2); +} + +export function iterativeFibonacci(n: number): number { + let a = 0; + let b = 1; + for (let i = 0; i < n; i++) { + const temp = a + b; + a = b; + b = temp; + } + return a; +} diff --git a/examples/with-vitest-v5/tsconfig.json b/examples/with-vitest-v5/tsconfig.json new file mode 100644 index 00000000..ace1de03 --- /dev/null +++ b/examples/with-vitest-v5/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "verbatimModuleSyntax": true, + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-vitest-v5/vitest.config.ts b/examples/with-vitest-v5/vitest.config.ts new file mode 100644 index 00000000..4b1290c1 --- /dev/null +++ b/examples/with-vitest-v5/vitest.config.ts @@ -0,0 +1,6 @@ +import codspeedPlugin from "@codspeed/vitest-plugin"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [codspeedPlugin()], +}); diff --git a/packages/vitest-plugin/README.md b/packages/vitest-plugin/README.md index 6cba8821..47021a0f 100644 --- a/packages/vitest-plugin/README.md +++ b/packages/vitest-plugin/README.md @@ -41,10 +41,10 @@ pnpm add --save-dev @codspeed/vitest-plugin vitest ## Usage -Let's create a fibonacci function and benchmark it with `vitest.bench`: +Let's create a fibonacci function and benchmark it with Vitest's `bench` fixture: ```ts title="benches/fibo.bench.ts" -import { describe, bench } from "vitest"; +import { describe, test } from "vitest"; function fibonacci(n: number): number { if (n < 2) { @@ -54,16 +54,33 @@ function fibonacci(n: number): number { } describe("fibonacci", () => { - bench("fibonacci10", () => { - fibonacci(10); - }); - - bench("fibonacci15", () => { - fibonacci(15); + test("depth", async ({ bench }) => { + await bench.compare( + bench("fibonacci10", () => { + fibonacci(10); + }), + bench("fibonacci15", () => { + fibonacci(15); + }), + ); }); }); ``` +> [!NOTE] +> The `bench` fixture is a Vitest 5 API. On Vitest 3 and 4, benchmarks are +> declared with the top-level `bench()` export instead: +> +> ```ts +> import { bench, describe } from "vitest"; +> +> describe("fibonacci", () => { +> bench("fibonacci10", () => { +> fibonacci(10); +> }); +> }); +> ``` + Create or update your `vitest.config.ts` file to use the CodSpeed runner: ```ts title="vitest.config.ts" @@ -80,7 +97,6 @@ Finally, run your benchmarks (here with `pnpm`): ```bash $ pnpm vitest bench --run -[CodSpeed] bench detected but no instrumentation found, falling back to default vitest runner ... Regular `vitest bench` output ``` diff --git a/packages/vitest-plugin/benches/flat.bench.ts b/packages/vitest-plugin/benches/flat.bench.ts index 74e67b9d..b9b494a4 100644 --- a/packages/vitest-plugin/benches/flat.bench.ts +++ b/packages/vitest-plugin/benches/flat.bench.ts @@ -1,5 +1,10 @@ -import { bench, describe } from "vitest"; -import parsePr from "./parsePr"; +import { describe, test } from "vitest"; +import parsePrExport from "./parsePr"; + +// Read the imported binding once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const parsePr = parsePrExport; const LONG_BODY = new Array(1_000) @@ -9,12 +14,15 @@ const LONG_BODY = .join("\n") + "fixes #123"; describe("parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test-1", number: 1 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test-2", number: 2 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", () => { + parsePr({ body: "fixes #123", title: "test-1", number: 1 }); + }), + bench("long body", () => { + parsePr({ body: LONG_BODY, title: "test-2", number: 2 }); + }), + ); }); }); @@ -24,10 +32,13 @@ function fibo(n: number): number { } describe("fibo", () => { - bench("fibo 10", () => { - fibo(10); - }); - bench("fibo 15", () => { - fibo(15); + test("depth", async ({ bench }) => { + await bench("fibo 10", () => { + fibo(10); + }).run(); + + await bench("fibo 15", () => { + fibo(15); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/hooks.bench.ts b/packages/vitest-plugin/benches/hooks.bench.ts index d790ca3c..c706dda8 100644 --- a/packages/vitest-plugin/benches/hooks.bench.ts +++ b/packages/vitest-plugin/benches/hooks.bench.ts @@ -1,37 +1,39 @@ -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - bench, - describe, - expect, -} from "vitest"; +import { describe, expect, test } from "vitest"; +// Exercises tinybench's per-benchmark hooks, which Vitest 5 exposes through the +// `bench(name, options, fn)` options object (`beforeAll`/`beforeEach`/...). describe("hooks", () => { let count = 0; - describe("run", () => { - beforeAll(() => { + + const hooks = { + beforeAll: () => { count += 10; - }); - beforeEach(() => { + }, + beforeEach: () => { count += 1; - }); - afterEach(() => { + }, + afterEach: () => { count -= 1; - }); - afterAll(() => { + }, + afterAll: () => { count -= 10; - }); + }, + }; - bench("one", () => { - expect(count).toBe(11); - }); - bench("two", () => { - expect(count).toBe(11); - }); + test("hooked benches", async ({ bench }) => { + await bench.compare( + bench("one", hooks, () => { + expect(count).toBe(11); + }), + bench("two", hooks, () => { + expect(count).toBe(11); + }), + ); }); - bench("end", () => { - expect(count).toBe(0); + + test("after the hooked benches", async ({ bench }) => { + await bench("count is back to zero", () => { + expect(count).toBe(0); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/macos.bench.ts b/packages/vitest-plugin/benches/macos.bench.ts index e05a3082..792e9c3f 100644 --- a/packages/vitest-plugin/benches/macos.bench.ts +++ b/packages/vitest-plugin/benches/macos.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe } from "vitest"; +import { describe, test } from "vitest"; const isMacOS = process.platform === "darwin"; @@ -10,7 +10,9 @@ function fibo(n: number): number { // macOS-only benchmark: skipped on every other platform, so it only runs on // the `codspeed-walltime-macos` CI job (see .github/workflows/codspeed.yml). describe.skipIf(!isMacOS)("macos only", () => { - bench("fibo darwin", () => { - fibo(30); + test("fibo", async ({ bench }) => { + await bench("fibo darwin", () => { + fibo(30); + }).run(); }); }); diff --git a/packages/vitest-plugin/benches/parsePr.bench.ts b/packages/vitest-plugin/benches/parsePr.bench.ts index 91c0adee..60829edf 100644 --- a/packages/vitest-plugin/benches/parsePr.bench.ts +++ b/packages/vitest-plugin/benches/parsePr.bench.ts @@ -1,5 +1,10 @@ -import { bench, describe } from "vitest"; -import parsePr from "./parsePr"; +import { describe, test } from "vitest"; +import parsePrExport from "./parsePr"; + +// Read the imported binding once: Vite's module runner exposes exports through +// getters, and reading one inside the measured loop adds overhead to every +// iteration (Vitest warns about it). +const parsePr = parsePrExport; const LONG_BODY = new Array(1_000) @@ -8,44 +13,49 @@ const LONG_BODY = ) .join("\n") + "fixes #123"; -describe("parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); +function benchShortBody() { + parsePr({ body: "fixes #123", title: "test", number: 124 }); +} - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); +function benchLongBody() { + parsePr({ body: LONG_BODY, title: "test", number: 124 }); +} + +describe("parsePr", () => { + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("deeply nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); }); }); }); describe("another parsePr", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); - }); - - bench("long body", () => { - parsePr({ body: LONG_BODY, title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench.compare( + bench("short body", benchShortBody), + bench("long body", benchLongBody), + ); }); describe("nested suite", () => { - bench("short body", () => { - parsePr({ body: "fixes #123", title: "test", number: 124 }); + test("body size", async ({ bench }) => { + await bench("short body", benchShortBody).run(); }); }); }); diff --git a/packages/vitest-plugin/benches/timing.bench.ts b/packages/vitest-plugin/benches/timing.bench.ts index 83331ad6..fd6aa5f8 100644 --- a/packages/vitest-plugin/benches/timing.bench.ts +++ b/packages/vitest-plugin/benches/timing.bench.ts @@ -1,4 +1,4 @@ -import { bench, describe, type BenchOptions } from "vitest"; +import { describe, test, type BenchCompareOptions } from "vitest"; const busySleep = (ms: number): void => { const end = performance.now() + ms; @@ -7,33 +7,24 @@ const busySleep = (ms: number): void => { } }; -const timingBenchOptions: BenchOptions = { +const timingBenchOptions: BenchCompareOptions = { iterations: 5, warmupIterations: 0, }; describe("timing tests", () => { - bench( - "wait 1ms", - async () => { - busySleep(1); - }, - timingBenchOptions, - ); - - bench( - "wait 500ms", - async () => { - busySleep(500); - }, - timingBenchOptions, - ); - - bench( - "wait 1sec", - async () => { - busySleep(1_000); - }, - timingBenchOptions, - ); + test("busy sleep", async ({ bench }) => { + await bench.compare( + bench("wait 1ms", async () => { + busySleep(1); + }), + bench("wait 500ms", async () => { + busySleep(500); + }), + bench("wait 1sec", async () => { + busySleep(1_000); + }), + timingBenchOptions, + ); + }); }); diff --git a/packages/vitest-plugin/package.json b/packages/vitest-plugin/package.json index 3c935d88..ab23e739 100644 --- a/packages/vitest-plugin/package.json +++ b/packages/vitest-plugin/package.json @@ -41,13 +41,13 @@ "peerDependencies": { "tinybench": ">=2.9.0", "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0", - "vitest": "^3.2 || ^4" + "vitest": "^3.2 || ^4 || ^5" }, "devDependencies": { "@total-typescript/shoehorn": "^0.1.1", "execa": "^8.0.1", "tinybench": "^2.9.0", "vite": "^8.0.0", - "vitest": "^4.1.11" + "vitest": "^5.0.0" } } diff --git a/packages/vitest-plugin/rollup.config.mjs b/packages/vitest-plugin/rollup.config.mjs index 2fda2b6a..b469682d 100644 --- a/packages/vitest-plugin/rollup.config.mjs +++ b/packages/vitest-plugin/rollup.config.mjs @@ -23,18 +23,26 @@ export default defineConfig([ plugins: jsPlugins(pkg.version), external: ["@codspeed/core", /^vitest/], }, + // Vitest imports these by file path (as `test.runner` or `benchmark.provider`), so + // each needs its own output; `resolveFile` expects it at the same path as in `src/`. { - input: "src/analysis.ts", - output: { file: "dist/analysis.mjs", format: "es" }, + input: "src/legacy/analysis.ts", + output: { file: "dist/legacy/analysis.mjs", format: "es" }, // top-level await plugins: jsPlugins(pkg.version, "es2022"), external: ["@codspeed/core", /^vitest/], }, { - input: "src/walltime/index.ts", - output: { file: "dist/walltime.mjs", format: "es" }, + input: "src/legacy/walltime.ts", + output: { file: "dist/legacy/walltime.mjs", format: "es" }, // top-level await plugins: jsPlugins(pkg.version, "es2022"), external: ["@codspeed/core", /^vitest/], }, + { + input: "src/v5/provider.ts", + output: { file: "dist/v5/provider.mjs", format: "es" }, + plugins: jsPlugins(pkg.version), + external: ["@codspeed/core", /^vitest/], + }, ]); diff --git a/packages/vitest-plugin/src/__tests__/globalSetup.test.ts b/packages/vitest-plugin/src/__tests__/globalSetup.test.ts index 178b3ecb..f475f048 100644 --- a/packages/vitest-plugin/src/__tests__/globalSetup.test.ts +++ b/packages/vitest-plugin/src/__tests__/globalSetup.test.ts @@ -4,19 +4,27 @@ import globalSetup from "../globalSetup"; console.log = vi.fn(); describe("globalSetup", () => { - it("should log the correct message on setup and teardown, and fail when teardown is called twice", async () => { + it("should log setup and teardown once, even when Vitest runs them per project", async () => { const teardown = globalSetup(); expect(console.log).toHaveBeenCalledWith( "[CodSpeed] @codspeed/vitest-plugin v1.0.0 - setup", ); + // Vitest 5 runs the same globalSetup for the base project and for the + // benchmark project it clones from it. + globalSetup(); + + expect(console.log).toHaveBeenCalledTimes(1); + teardown(); expect(console.log).toHaveBeenCalledWith( "[CodSpeed] @codspeed/vitest-plugin v1.0.0 - teardown", ); - expect(() => teardown()).toThrowError("teardown called twice"); + teardown(); + + expect(console.log).toHaveBeenCalledTimes(2); }); }); diff --git a/packages/vitest-plugin/src/__tests__/index.test.ts b/packages/vitest-plugin/src/__tests__/index.test.ts index 3aae6e2b..54713449 100644 --- a/packages/vitest-plugin/src/__tests__/index.test.ts +++ b/packages/vitest-plugin/src/__tests__/index.test.ts @@ -1,6 +1,6 @@ import { getV8Flags } from "@codspeed/core"; import { fromPartial } from "@total-typescript/shoehorn"; -import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import codspeedPlugin from "../index"; const coreMocks = vi.hoisted(() => { @@ -12,7 +12,7 @@ const coreMocks = vi.hoisted(() => { }); const fsMocks = vi.hoisted(() => { - let mockVersion = "4.0.18"; // default to v4 + let mockVersion = ""; return { readFileSync: vi.fn((path: string) => { if (path.includes("vitest/package.json")) { @@ -26,11 +26,6 @@ const fsMocks = vi.hoisted(() => { }; }); -const resolvedCodSpeedPlugin = codspeedPlugin(); -const applyPluginFunction = resolvedCodSpeedPlugin.apply; -if (typeof applyPluginFunction !== "function") - throw new Error("applyPluginFunction is not a function"); - vi.mock("@codspeed/core", async (importOriginal) => { const mod = await importOriginal(); return { ...mod, ...coreMocks }; @@ -44,114 +39,131 @@ vi.mock("fs", () => { console.warn = vi.fn(); -describe("codSpeedPlugin", () => { - beforeAll(() => { - // Set environment variables to trigger instrumented mode - process.env.CODSPEED_ENV = "1"; - process.env.CODSPEED_RUNNER_MODE = "instrumentation"; - }); +const plugin = codspeedPlugin(); - afterAll(() => { - // Clean up environment variables - delete process.env.CODSPEED_ENV; - delete process.env.CODSPEED_RUNNER_MODE; +function apply(mode: string) { + if (typeof plugin.apply !== "function") + throw new Error("apply is not a function"); + return plugin.apply({}, fromPartial({ mode })); +} + +function config() { + if (typeof plugin.config !== "function") + throw new Error("config is not a function"); + return plugin.config.call({} as never, {}, fromPartial({})); +} + +describe("codSpeedPlugin", () => { + beforeEach(() => { + vi.stubEnv("CODSPEED_ENV", "1"); + vi.stubEnv("CODSPEED_RUNNER_MODE", "instrumentation"); }); - it("should have a name", async () => { - expect(resolvedCodSpeedPlugin.name).toBe("codspeed:vitest"); + afterEach(() => { + vi.unstubAllEnvs(); }); - it("should enforce to run after the other plugins", async () => { - expect(resolvedCodSpeedPlugin.enforce).toBe("post"); + it("should enforce to run after the other plugins", () => { + expect(plugin.enforce).toBe("post"); }); - describe("apply", () => { - it("should not apply the plugin when the mode is not benchmark", async () => { - const applyPlugin = applyPluginFunction( - {}, - fromPartial({ mode: "test" }), - ); + describe("Vitest 3/4", () => { + beforeEach(() => { + fsMocks.setMockVersion("4.0.18"); + }); - expect(applyPlugin).toBe(false); + it("should not apply the plugin when the mode is not benchmark", () => { + expect(apply("test")).toBe(false); }); - it("should apply the plugin when there is no instrumentation", async () => { + it("should apply the plugin when there is no instrumentation", () => { coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(false); - const applyPlugin = applyPluginFunction( - {}, - fromPartial({ mode: "benchmark" }), - ); - + expect(apply("benchmark")).toBe(true); expect(console.warn).toHaveBeenCalledWith( "[CodSpeed] bench detected but no instrumentation found", ); - expect(applyPlugin).toBe(true); }); - it("should apply the plugin when there is instrumentation", async () => { + it("should apply the plugin when there is instrumentation", () => { coreMocks.InstrumentHooks.isInstrumented.mockReturnValue(true); - const applyPlugin = applyPluginFunction( - {}, - fromPartial({ mode: "benchmark" }), - ); + expect(apply("benchmark")).toBe(true); + }); - expect(applyPlugin).toBe(true); + it("should apply the codspeed config for v4", () => { + expect(config()).toStrictEqual({ + test: { + globalSetup: [ + expect.stringContaining( + "packages/vitest-plugin/src/globalSetup.ts", + ), + ], + pool: "forks", + execArgv: getV8Flags(), + runner: expect.stringContaining( + "packages/vitest-plugin/src/legacy/analysis.ts", + ), + }, + }); }); - }); - it("should apply the codspeed config for v4", () => { - const config = resolvedCodSpeedPlugin.config; - if (typeof config !== "function") - throw new Error("config is not a function"); - - const result = config.call({} as never, {}, fromPartial({})); - - expect(result).toStrictEqual({ - test: { - globalSetup: [ - expect.stringContaining("packages/vitest-plugin/src/globalSetup.ts"), - ], - pool: "forks", - execArgv: getV8Flags(), - runner: expect.stringContaining( - "packages/vitest-plugin/src/analysis.ts", - ), - }, + it("should apply the codspeed config for v3 with poolOptions", () => { + fsMocks.setMockVersion("3.2.0"); + + expect(config()).toStrictEqual({ + test: { + globalSetup: [ + expect.stringContaining( + "packages/vitest-plugin/src/globalSetup.ts", + ), + ], + pool: "forks", + poolOptions: { + forks: { + execArgv: getV8Flags(), + }, + }, + runner: expect.stringContaining( + "packages/vitest-plugin/src/legacy/analysis.ts", + ), + }, + }); }); }); - it("should apply the codspeed config for v3 with poolOptions", () => { - // Set mock version to v3 - fsMocks.setMockVersion("3.2.0"); - - // Create a new plugin instance to pick up the mocked version - const v3Plugin = codspeedPlugin(); - const config = v3Plugin.config; - if (typeof config !== "function") - throw new Error("config is not a function"); - - const result = config.call({} as never, {}, fromPartial({})); - - expect(result).toStrictEqual({ - test: { - globalSetup: [ - expect.stringContaining("packages/vitest-plugin/src/globalSetup.ts"), - ], - pool: "forks", - poolOptions: { - forks: { - execArgv: getV8Flags(), + describe("Vitest 5", () => { + beforeEach(() => { + fsMocks.setMockVersion("5.0.0"); + }); + + it("should apply the plugin in any mode when CodSpeed drives the run", () => { + expect(apply("test")).toBe(true); + }); + + it("should not apply the plugin when CodSpeed is not driving the run", () => { + vi.stubEnv("CODSPEED_ENV", undefined); + + expect(apply("test")).toBe(false); + }); + + it("should wire the benchmark provider", () => { + expect(config()).toStrictEqual({ + test: { + globalSetup: [ + expect.stringContaining( + "packages/vitest-plugin/src/globalSetup.ts", + ), + ], + pool: "forks", + execArgv: getV8Flags(), + benchmark: { + provider: expect.stringContaining( + "packages/vitest-plugin/src/v5/provider.ts", + ), }, }, - runner: expect.stringContaining( - "packages/vitest-plugin/src/analysis.ts", - ), - }, + }); }); - - // Reset mock version back to v4 - fsMocks.setMockVersion("4.0.18"); }); }); diff --git a/packages/vitest-plugin/src/__tests__/instrumented.test.ts b/packages/vitest-plugin/src/__tests__/instrumented.test.ts index 443d62f2..c3e9178a 100644 --- a/packages/vitest-plugin/src/__tests__/instrumented.test.ts +++ b/packages/vitest-plugin/src/__tests__/instrumented.test.ts @@ -1,7 +1,11 @@ import { fromPartial } from "@total-typescript/shoehorn"; import { describe, expect, it, vi, type RunnerTestSuite } from "vitest"; -import { AnalysisRunner as CodSpeedRunner } from "../analysis"; -import { getBenchFn, getBenchOptions } from "../compat"; +import { AnalysisRunner as CodSpeedRunner } from "../legacy/analysis"; +import { getBenchFn, getBenchOptions } from "../legacy/compat"; + +// `legacy/compat` resolves the Vitest 3/4 benchmark backend from the installed +// Vitest, so it is mocked entirely: that backend is gone on Vitest 5, which this +// package is developed against. const coreMocks = vi.hoisted(() => { return { @@ -28,10 +32,16 @@ vi.mock("@codspeed/core", async (importOriginal) => { console.log = vi.fn(); -vi.mock("../compat", async (importOriginal) => { - const actual = await importOriginal(); +vi.mock("../legacy/compat", () => { + class NodeBenchmarkRunner { + async importTinybench() { + return import("tinybench"); + } + } + return { - ...actual, + NodeBenchmarkRunner, + getHooks: vi.fn(), getBenchFn: vi.fn(), getBenchOptions: vi.fn(), }; diff --git a/packages/vitest-plugin/src/compat.ts b/packages/vitest-plugin/src/compat.ts deleted file mode 100644 index d27baa71..00000000 --- a/packages/vitest-plugin/src/compat.ts +++ /dev/null @@ -1,35 +0,0 @@ -type VitestExports = typeof import("vitest"); - -/** - * Vitest 4.1 moved the benchmark runner and the suite helpers to the main - * `vitest` entry point and deprecated the `vitest/runners` and `vitest/suite` - * subpaths, which warn on import. - */ -async function resolveVitestApi() { - const { BenchmarkRunner, TestRunner }: Partial = - await import("vitest"); - - if (BenchmarkRunner && TestRunner) { - return { - NodeBenchmarkRunner: BenchmarkRunner, - getHooks: TestRunner.getSuiteHooks, - getBenchFn: TestRunner.getBenchFn, - getBenchOptions: TestRunner.getBenchOptions, - }; - } - - const [runners, suite] = await Promise.all([ - import("vitest/runners"), - import("vitest/suite"), - ]); - - return { - NodeBenchmarkRunner: runners.NodeBenchmarkRunner, - getHooks: suite.getHooks, - getBenchFn: suite.getBenchFn, - getBenchOptions: suite.getBenchOptions, - }; -} - -export const { NodeBenchmarkRunner, getHooks, getBenchFn, getBenchOptions } = - await resolveVitestApi(); diff --git a/packages/vitest-plugin/src/globalSetup.ts b/packages/vitest-plugin/src/globalSetup.ts index 8c5e1a34..143dafea 100644 --- a/packages/vitest-plugin/src/globalSetup.ts +++ b/packages/vitest-plugin/src/globalSetup.ts @@ -9,13 +9,21 @@ function logCodSpeed(message: string) { console.log(`[CodSpeed] ${message}`); } +let setupHappened = false; let teardownHappened = false; +// Vitest 5 clones a dedicated `(bench)` project from the base one and runs the +// same globalSetup module for both, so setup and teardown each fire twice. +// Report only the first pass and make the repeats no-ops; throwing on the second +// teardown fails the whole run during close. export default function () { - logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`); + if (!setupHappened) { + setupHappened = true; + logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - setup`); + } return () => { - if (teardownHappened) throw new Error("teardown called twice"); + if (teardownHappened) return; teardownHappened = true; logCodSpeed(`@codspeed/vitest-plugin v${__VERSION__} - teardown`); diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index 2bdf59ed..e5e189ba 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -1,5 +1,4 @@ import { - getCodspeedRunnerMode, getInstrumentMode, getV8Flags, InstrumentHooks, @@ -7,49 +6,29 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, } from "@codspeed/core"; -import { readFileSync } from "fs"; -import { createRequire } from "module"; import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; +import { resolveVitestBackend } from "./vitestBackend"; // get this file's directory path from import.meta.url const __dirname = new URL(".", import.meta.url).pathname; const isFileInTs = import.meta.url.endsWith(".ts"); -function getCodSpeedFileFromName(name: string) { +/** + * Resolve a plugin-owned file shipped alongside this module. Source (`.ts`) and + * built (`.mjs`) layouts are identical (see rollup.config.mjs). + */ +function resolveFile(name: string): string { const fileExtension = isFileInTs ? "ts" : "mjs"; - return join(__dirname, `${name}.${fileExtension}`); } -function getVitestMajorVersion(): number | null { - try { - // Resolve vitest from the project's perspective (cwd), not from the plugin's location - // This ensures we detect the vitest version the user has installed - const require = createRequire(join(process.cwd(), "package.json")); - const vitestPkgPath = require.resolve("vitest/package.json"); - const vitestPkg = JSON.parse(readFileSync(vitestPkgPath, "utf-8")); - return parseInt(vitestPkg.version.split(".")[0], 10); - } catch { - return null; - } -} - -function getRunnerFile(): string | undefined { - const instrumentMode = getInstrumentMode(); - if (instrumentMode === "disabled") { - return undefined; - } - - return getCodSpeedFileFromName(instrumentMode); -} - export default function codspeedPlugin(): Plugin { return { name: "codspeed:vitest", apply(_, { mode }) { - if (mode !== "benchmark") { + if (!resolveVitestBackend().isActive(mode)) { return false; } if ( @@ -62,41 +41,16 @@ export default function codspeedPlugin(): Plugin { }, enforce: "post", config(): ViteUserConfig { - const runnerFile = getRunnerFile(); - const runnerMode = getCodspeedRunnerMode(); - const v8Flags = getV8Flags(); - const vitestMajorVersion = getVitestMajorVersion(); - // by default, assume Vitest v4 or higher - const isVitestV4OrHigher = (vitestMajorVersion ?? 4) >= 4; - - const config: ViteUserConfig = { + return { test: { pool: "forks", - ...(isVitestV4OrHigher - ? { execArgv: v8Flags } - : { - // Compat with Vitest v3 - // See: https://vitest.dev/guide/migration.html#pool-rework - // poolOptions only exists in Vitest v3 - poolOptions: { - forks: { - execArgv: v8Flags, - }, - }, - }), - globalSetup: [getCodSpeedFileFromName("globalSetup")], - ...(runnerFile && { - runner: runnerFile, - }), - ...(runnerMode === "walltime" && { - benchmark: { - includeSamples: true, - }, - }), + globalSetup: [resolveFile("globalSetup")], + ...resolveVitestBackend().getBenchmarkTestConfig( + getV8Flags(), + resolveFile, + ), }, }; - - return config; }, }; } diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts new file mode 100644 index 00000000..f1c97d01 --- /dev/null +++ b/packages/vitest-plugin/src/instrument.ts @@ -0,0 +1,192 @@ +import { + calculateQuantiles, + InstrumentHooks, + MARKER_TYPE_BENCHMARK_END, + MARKER_TYPE_BENCHMARK_START, + msToNs, + msToS, + writeWalltimeResults, + type Benchmark, + type BenchmarkStats, +} from "@codspeed/core"; +import type * as tinybench from "tinybench"; + +export type Tinybench = typeof tinybench; + +/** A tinybench task, as read after its run. */ +export interface TinybenchTask { + name: string; + result?: TinybenchTaskResult; +} + +/** tinybench's per-task setup/teardown hook signature. */ +export type TinybenchHook = ( + task: TinybenchTask, + mode: "run" | "warmup", +) => Promise | void; + +/** The mutable subset of a tinybench Bench the runner reaches into. */ +export interface TinybenchBench { + setup: TinybenchHook; + teardown: TinybenchHook; +} + +/** + * The tinybench statistics shape (latency/throughput) shared across the v2 and + * v6 lines. Only the fields the conversion needs are modeled. + */ +interface TinybenchStatistics { + min: number; + max: number; + mean: number; + sd: number; + samples: number[] | undefined; +} + +interface TinybenchTaskResult { + state?: string; + /** Set by tinybench when `state` is `"errored"`. */ + error?: Error; + totalTime: number; + latency: TinybenchStatistics; +} + +/** The subset of tinybench bench options that maps onto a CodSpeed benchmark config. */ +export interface TinybenchOptions { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; +} + +/** + * Drive the instrumentation window from each bench's run-mode setup/teardown + * hooks so it brackets only tinybench's measured loop, excluding the warmup + * that runs beforehand and the statistics computation tinybench performs after + * the loop. Wrapping the whole `Task.run()` would otherwise fold all of that + * framework overhead into the recorded sample. + * + * User-provided hooks are preserved and keep their order relative to the work + * under test. + */ +export function installInstrumentHooks( + bench: TinybenchBench, + getUri: (taskName: string) => string, +): void { + const userSetup = bench.setup; + const userTeardown = bench.teardown; + + // Tasks of a bench run sequentially, so one window is open at a time. + let runStart: bigint | null = null; + + bench.setup = async (task, mode) => { + await userSetup(task, mode); + if (mode === "run") { + InstrumentHooks.startBenchmark(); + runStart = InstrumentHooks.currentTimestamp(); + } + }; + + bench.teardown = async (task, mode) => { + if (mode === "run") { + const pid = process.pid; + // The markers must land inside the sample window, so they go out before + // stopBenchmark(): one sent after it breaks the expected + // SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. + InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, runStart!); + InstrumentHooks.addMarker( + pid, + MARKER_TYPE_BENCHMARK_END, + InstrumentHooks.currentTimestamp(), + ); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(pid, getUri(task.name)); + runStart = null; + } + await userTeardown(task, mode); + }; +} + +/** Persist collected walltime benchmarks, if any, and log a summary. */ +export function writeAndLogWalltimeResults(benchmarks: Benchmark[]): void { + if (benchmarks.length === 0) { + return; + } + writeWalltimeResults(benchmarks); + console.log( + `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`, + ); +} + +/** + * Convert a completed tinybench task into a CodSpeed walltime benchmark. Returns + * null when the task produced no samples (e.g. fully optimized out), in which + * case there is nothing to record. + */ +export function tinybenchTaskToBenchmark( + task: TinybenchTask, + uri: string, + options: TinybenchOptions, +): Benchmark | null { + const stats = tinybenchResultToStats(task.result, options); + if (stats === null) { + return null; + } + + return { + name: task.name, + uri, + config: { + max_rounds: options.iterations ?? null, + max_time_ns: options.time ? msToNs(options.time) : null, + min_round_time_ns: null, // tinybench does not have an option for this + warmup_time_ns: + options.warmupIterations !== 0 && options.warmupTime + ? msToNs(options.warmupTime) + : null, + }, + stats, + }; +} + +function tinybenchResultToStats( + result: TinybenchTaskResult | undefined, + options: TinybenchOptions, +): BenchmarkStats | null { + if (!result) { + throw new Error("No benchmark data available in result"); + } + + const { totalTime, latency } = result; + const { min, max, mean, sd, samples } = latency; + + const sortedTimesNs = (samples ?? []).map(msToNs).sort((a, b) => a - b); + const meanNs = msToNs(mean); + const stdevNs = msToNs(sd); + + if (sortedTimesNs.length == 0) { + // Sometimes the benchmarks can be completely optimized out and not even + // run, but their beforeEach and afterEach hooks are still executed, and the + // task is still considered a success. + return null; + } + + const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = + calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); + + return { + min_ns: msToNs(min), + max_ns: msToNs(max), + mean_ns: meanNs, + stdev_ns: stdevNs, + q1_ns, + median_ns, + q3_ns, + total_time: msToS(totalTime), + iter_per_round: 1, // tinybench runs one iteration per round + rounds: sortedTimesNs.length, + iqr_outlier_rounds, + stdev_outlier_rounds, + warmup_iters: options.warmupIterations ?? 0, + }; +} diff --git a/packages/vitest-plugin/src/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts similarity index 85% rename from packages/vitest-plugin/src/analysis.ts rename to packages/vitest-plugin/src/legacy/analysis.ts index dc50a5e2..f71f0c0a 100644 --- a/packages/vitest-plugin/src/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -7,16 +7,20 @@ import { teardownCore, wrapWithRootFrame, } from "@codspeed/core"; -import type * as tinybench from "tinybench"; -import { Benchmark, type RunnerTestSuite } from "vitest"; +import { type RunnerTestSuite } from "vitest"; +import { type Tinybench } from "../instrument"; + import { callSuiteHook, isVitestTaskBenchmark, patchRootSuiteWithFullFilePath, } from "./common"; -import { getBenchFn, getBenchOptions, NodeBenchmarkRunner } from "./compat"; - -type Tinybench = typeof tinybench; +import { + getBenchFn, + getBenchOptions, + NodeBenchmarkRunner, + type BenchmarkTask, +} from "./compat"; const currentFileName = typeof __filename === "string" @@ -33,7 +37,7 @@ function logCodSpeed(message: string) { } async function runAnalysisBench( - benchmark: Benchmark, + benchmark: BenchmarkTask, suite: RunnerTestSuite, currentSuiteName: string, tinybenchModule: Tinybench, @@ -50,7 +54,6 @@ async function runAnalysisBench( await bench.setup(task, "warmup"); await optimizeFunction(async () => { await callSuiteHook(suite, benchmark, "beforeEach"); - // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench await fn(); await callSuiteHook(suite, benchmark, "afterEach"); }); @@ -62,7 +65,6 @@ async function runAnalysisBench( global.gc?.(); await wrapWithRootFrame(async () => { InstrumentHooks.startBenchmark(); - // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench await fn(); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(process.pid, uri); @@ -88,10 +90,10 @@ async function runAnalysisBenchmarkSuite( for (const task of suite.tasks) { if (task.mode !== "run") continue; - if (isVitestTaskBenchmark(task)) { - await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); - } else if (task.type === "suite") { + if (task.type === "suite") { await runAnalysisBenchmarkSuite(task, tinybenchModule, currentSuiteName); + } else if (isVitestTaskBenchmark(task)) { + await runAnalysisBench(task, suite, currentSuiteName, tinybenchModule); } } diff --git a/packages/vitest-plugin/src/common.ts b/packages/vitest-plugin/src/legacy/common.ts similarity index 74% rename from packages/vitest-plugin/src/common.ts rename to packages/vitest-plugin/src/legacy/common.ts index e853c307..0791b963 100644 --- a/packages/vitest-plugin/src/common.ts +++ b/packages/vitest-plugin/src/legacy/common.ts @@ -1,7 +1,7 @@ import { getGitDir } from "@codspeed/core"; import path from "path"; -import { Benchmark, type RunnerTask, type RunnerTestSuite } from "vitest"; -import { getHooks } from "./compat"; +import { type RunnerTask, type RunnerTestSuite } from "vitest"; +import { getHooks, type BenchmarkTask } from "./compat"; type SuiteHooks = ReturnType; function getSuiteHooks(suite: RunnerTestSuite, name: keyof SuiteHooks) { @@ -19,8 +19,9 @@ export async function callSuiteHook( const hooks = getSuiteHooks(suite, name); - // @ts-expect-error TODO: add support for hooks parameters - await Promise.all(hooks.map((fn) => fn())); + // TODO: add support for hook parameters. The hook signature differs across + // supported Vitest versions, so we call them through a parameterless cast. + await Promise.all((hooks as Array<() => unknown>).map((fn) => fn())); if (name === "afterEach" && suite?.suite) { await callSuiteHook(suite.suite, currentTask, name); @@ -35,6 +36,6 @@ export function patchRootSuiteWithFullFilePath(suite: RunnerTestSuite) { suite.name = path.relative(gitDir, suite.file.filepath); } -export function isVitestTaskBenchmark(task: RunnerTask): task is Benchmark { +export function isVitestTaskBenchmark(task: RunnerTask): task is BenchmarkTask { return task.type === "test" && task.meta.benchmark === true; } diff --git a/packages/vitest-plugin/src/legacy/compat.ts b/packages/vitest-plugin/src/legacy/compat.ts new file mode 100644 index 00000000..a5802de7 --- /dev/null +++ b/packages/vitest-plugin/src/legacy/compat.ts @@ -0,0 +1,93 @@ +import type { RunnerTestCase, RunnerTestSuite } from "vitest"; +import type { Tinybench } from "../instrument"; + +/** + * A Vitest 3/4 benchmark task: a test case flagged as a benchmark, carrying the + * raw tinybench output on its result. + */ +export interface BenchmarkTask extends RunnerTestCase { + meta: RunnerTestCase["meta"] & { benchmark?: boolean }; +} + +/** The tinybench options Vitest 3/4 stores alongside a registered benchmark. */ +export interface LegacyBenchOptions { + time?: number; + warmupTime?: number; + warmupIterations?: number; + iterations?: number; + setup?: ( + task: { name: string }, + mode: "run" | "warmup", + ) => void | Promise; + teardown?: ( + task: { name: string }, + mode: "run" | "warmup", + ) => void | Promise; +} + +/** + * The Vitest 3/4 benchmark backend, which Vitest 5 removed: benchmarks now run + * through a `benchmark.provider` (see `v5/provider.ts`) and the runner is gone. + * The shapes are declared here because neither the removed `vitest/runners` and + * `vitest/suite` subpaths nor the Vitest 5 typings this package compiles against + * describe them. + */ +export interface LegacyBenchmarkApi { + NodeBenchmarkRunner: new (config?: unknown) => { + config: unknown; + runSuite(suite: RunnerTestSuite): Promise; + importTinybench(): Promise; + }; + getHooks: (suite: unknown) => Record unknown>>; + getBenchFn: (benchmark: BenchmarkTask) => () => unknown; + getBenchOptions: (benchmark: BenchmarkTask) => LegacyBenchOptions; +} + +/** + * Vitest 4.1 moved the benchmark runner and the suite helpers to the main + * `vitest` entry point and deprecated the `vitest/runners` and `vitest/suite` + * subpaths, which warn on import. Both lookups have to be dynamic: which of the + * two shapes exists depends on the Vitest the user installed, and the subpaths + * don't resolve at all on Vitest 5. + */ +async function resolveVitestApi(): Promise { + // Vitest 5 exports an unrelated `TestRunner`, so the namespace has to be + // re-typed from scratch rather than narrowed. + const { BenchmarkRunner, TestRunner } = + (await import("vitest")) as unknown as { + BenchmarkRunner?: LegacyBenchmarkApi["NodeBenchmarkRunner"]; + TestRunner?: { + getSuiteHooks: LegacyBenchmarkApi["getHooks"]; + getBenchFn: LegacyBenchmarkApi["getBenchFn"]; + getBenchOptions: LegacyBenchmarkApi["getBenchOptions"]; + }; + }; + + if (BenchmarkRunner && TestRunner) { + return { + NodeBenchmarkRunner: BenchmarkRunner, + getHooks: TestRunner.getSuiteHooks, + getBenchFn: TestRunner.getBenchFn, + getBenchOptions: TestRunner.getBenchOptions, + }; + } + + // These subpaths only exist on Vitest 3/4, so they don't resolve when the + // plugin is installed alongside Vitest 5. + const [runners, suite] = await Promise.all([ + // eslint-disable-next-line import/no-unresolved + import("vitest/runners"), + // eslint-disable-next-line import/no-unresolved + import("vitest/suite"), + ]); + + return { + NodeBenchmarkRunner: runners.NodeBenchmarkRunner, + getHooks: suite.getHooks, + getBenchFn: suite.getBenchFn, + getBenchOptions: suite.getBenchOptions, + }; +} + +export const { NodeBenchmarkRunner, getHooks, getBenchFn, getBenchOptions } = + await resolveVitestApi(); diff --git a/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts new file mode 100644 index 00000000..be55606f --- /dev/null +++ b/packages/vitest-plugin/src/legacy/vitest-legacy.d.ts @@ -0,0 +1,19 @@ +// Vitest 3/4 exposed the benchmark internals through the `vitest/runners` and +// `vitest/suite` subpaths, which Vitest 5 removed. `legacy/compat.ts` imports +// them only when the installed Vitest doesn't expose them from its main entry +// point, i.e. only on Vitest 3/4 — but the package is type-checked against +// whichever Vitest is installed, including 5, where the subpaths don't resolve. +// These declarations keep that code compiling; nothing imports them under v5. +// +// This file must stay a script (no top-level imports): a `declare module` inside +// a module is an augmentation, which requires the module to resolve. + +declare module "vitest/runners" { + export const NodeBenchmarkRunner: import("./compat").LegacyBenchmarkApi["NodeBenchmarkRunner"]; +} + +declare module "vitest/suite" { + export const getHooks: import("./compat").LegacyBenchmarkApi["getHooks"]; + export const getBenchFn: import("./compat").LegacyBenchmarkApi["getBenchFn"]; + export const getBenchOptions: import("./compat").LegacyBenchmarkApi["getBenchOptions"]; +} diff --git a/packages/vitest-plugin/src/legacy/walltime-utils.ts b/packages/vitest-plugin/src/legacy/walltime-utils.ts new file mode 100644 index 00000000..fe8e042a --- /dev/null +++ b/packages/vitest-plugin/src/legacy/walltime-utils.ts @@ -0,0 +1,99 @@ +import { type Benchmark } from "@codspeed/core"; +import { type RunnerTaskResult, type RunnerTestSuite } from "vitest"; +import { tinybenchTaskToBenchmark, type TinybenchTask } from "../instrument"; +import { isVitestTaskBenchmark } from "./common"; +import { getBenchOptions, type BenchmarkTask } from "./compat"; + +export async function extractBenchmarkResults( + suite: RunnerTestSuite, + parentPath = "", +): Promise { + const benchmarks: Benchmark[] = []; + const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; + + for (const task of suite.tasks) { + if (task.type === "suite") { + const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); + benchmarks.push(...nestedBenchmarks); + } else if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { + const benchmark = processBenchmarkTask(task, currentPath); + if (benchmark) { + benchmarks.push(benchmark); + } + } + } + + return benchmarks; +} + +function processBenchmarkTask( + task: BenchmarkTask, + suitePath: string, +): Benchmark | null { + const uri = `${suitePath}::${task.name}`; + + const result = task.result; + if (!result) { + console.warn(` ⚠ No result data available for ${uri}`); + return null; + } + + try { + const benchmark = tinybenchTaskToBenchmark( + adaptLegacyResult(task.name, result), + uri, + getBenchOptions(task), + ); + + if (benchmark === null) { + console.log(` ✔ No walltime data to collect for ${uri}`); + return null; + } + + console.log(` ✔ Collected walltime data for ${uri}`); + return benchmark; + } catch (error) { + console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); + return null; + } +} + +/** tinybench v2's flat statistics, which Vitest 3/4 stores under `result.benchmark`. */ +interface LegacyBenchmarkStats { + totalTime: number; + min: number; + max: number; + mean: number; + sd: number; + samples: number[]; +} + +/** + * Reshape a Vitest 3/4 result into the tinybench v6 form the shared converter + * expects, which nests the statistics under `latency`. + */ +function adaptLegacyResult( + name: string, + result: RunnerTaskResult, +): TinybenchTask { + // `result.benchmark` only exists on the Vitest 3/4 task result; the v5 typings + // (compiled against here) dropped it. + const benchmark = (result as { benchmark?: LegacyBenchmarkStats }).benchmark; + if (!benchmark) { + throw new Error("No benchmark data available in result"); + } + + return { + name, + result: { + totalTime: benchmark.totalTime, + latency: { + min: benchmark.min, + max: benchmark.max, + mean: benchmark.mean, + sd: benchmark.sd, + samples: benchmark.samples, + }, + }, + }; +} diff --git a/packages/vitest-plugin/src/walltime/index.ts b/packages/vitest-plugin/src/legacy/walltime.ts similarity index 50% rename from packages/vitest-plugin/src/walltime/index.ts rename to packages/vitest-plugin/src/legacy/walltime.ts index 880d9279..09271df8 100644 --- a/packages/vitest-plugin/src/walltime/index.ts +++ b/packages/vitest-plugin/src/legacy/walltime.ts @@ -1,54 +1,34 @@ -import { - InstrumentHooks, - MARKER_TYPE_BENCHMARK_END, - MARKER_TYPE_BENCHMARK_START, - setupCore, - wrapWithRootFrame, - writeWalltimeResults, -} from "@codspeed/core"; -import type * as tinybench from "tinybench"; +import { setupCore, wrapWithRootFrame } from "@codspeed/core"; import { RunnerTaskEventPack, RunnerTaskResultPack, type RunnerTestSuite, } from "vitest"; -import { patchRootSuiteWithFullFilePath } from "../common"; -import { NodeBenchmarkRunner } from "../compat"; -import { extractBenchmarkResults } from "./utils"; - -type Tinybench = typeof tinybench; - -/** A tinybench task, exposing the `fn` the runner wraps with the root frame. */ +import { + installInstrumentHooks, + writeAndLogWalltimeResults, + type Tinybench, + type TinybenchBench, +} from "../instrument"; +import { patchRootSuiteWithFullFilePath } from "./common"; +import { NodeBenchmarkRunner } from "./compat"; +import { extractBenchmarkResults } from "./walltime-utils"; + +/** A tinybench v2 task, whose `fn` is a plain, reassignable property. */ interface TinybenchTask { - name: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - fn: (...args: any[]) => any; -} - -/** tinybench's per-task setup/teardown hook signature. */ -type TinybenchHook = ( - task: TinybenchTask, - mode: "run" | "warmup", -) => Promise | void; - -/** The mutable subset of a tinybench Bench the runner reaches into. */ -interface TinybenchBench { - setup: TinybenchHook; - teardown: TinybenchHook; + fn: () => unknown; } /** - * WalltimeRunner uses Vitest's default benchmark execution - * and extracts results from the suite after completion + * Lets tinybench run the benches through Vitest's default benchmark execution, + * instrumenting each measured loop, then extracts the results from the suite + * tree afterwards. */ export class WalltimeRunner extends NodeBenchmarkRunner { private isTinybenchHookedWithCodspeed = false; private suiteUris = new Map(); /// Suite ID of the currently running suite, to allow constructing the URI in the context of tinybench tasks private currentSuiteId: string | null = null; - // Carries the window start timestamp from the setup hook to the teardown - // hook. Tasks run strictly sequentially, so a single field is enough. - private runStart: bigint | null = null; async runSuite(suite: RunnerTestSuite): Promise { patchRootSuiteWithFullFilePath(suite); @@ -59,17 +39,13 @@ export class WalltimeRunner extends NodeBenchmarkRunner { await super.runSuite(suite); const benchmarks = await extractBenchmarkResults(suite); - - if (benchmarks.length > 0) { - writeWalltimeResults(benchmarks); - console.log( - `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`, - ); - } else { + if (benchmarks.length === 0) { console.warn( `[CodSpeed] No benchmark results found after suite execution`, ); + return; } + writeAndLogWalltimeResults(benchmarks); } private populateBenchmarkUris(suite: RunnerTestSuite, parentPath = ""): void { @@ -112,8 +88,7 @@ export class WalltimeRunner extends NodeBenchmarkRunner { /** * Wrap each task's function with the root frame so collected stacks can be - * attributed to a benchmark. The window itself is driven by the bench's - * setup/teardown hooks (see createInstrumentedBench). + * attributed to a benchmark. */ private patchTaskWithRootFrame(tinybench: Tinybench): void { const originalRun = tinybench.Task.prototype.run; @@ -133,16 +108,6 @@ export class WalltimeRunner extends NodeBenchmarkRunner { }; } - /** - * Drive the instrumentation window from each bench's run-mode setup/teardown - * hooks so it brackets only tinybench's measured loop, excluding the warmup - * that Vitest runs beforehand and the statistics computation tinybench - * performs after the loop. Wrapping the whole `Task.run()` would otherwise - * fold all of that framework overhead into the recorded sample. - * - * User-provided hooks are preserved and keep their order relative to the work - * under test. - */ private createInstrumentedBench( tinybench: Tinybench, ): typeof tinybench.Bench { @@ -154,50 +119,15 @@ export class WalltimeRunner extends NodeBenchmarkRunner { // eslint-disable-next-line @typescript-eslint/no-explicit-any constructor(...benchArgs: any[]) { super(...benchArgs); - runner.installInstrumentHooks(this as unknown as TinybenchBench); + installInstrumentHooks(this as unknown as TinybenchBench, (taskName) => + runner.getBenchmarkUri(taskName), + ); } } return InstrumentedBench; } - private installInstrumentHooks(bench: TinybenchBench): void { - const userSetup = bench.setup; - const userTeardown = bench.teardown; - - bench.setup = async (task, mode) => { - await userSetup(task, mode); - if (mode === "run") { - InstrumentHooks.startBenchmark(); - this.runStart = InstrumentHooks.currentTimestamp(); - } - }; - - bench.teardown = async (task, mode) => { - if (mode === "run") { - this.closeInstrumentWindow(this.getBenchmarkUri(task.name)); - } - await userTeardown(task, mode); - }; - } - - private closeInstrumentWindow(uri: string): void { - const runEnd = InstrumentHooks.currentTimestamp(); - const pid = process.pid; - - // Benchmark markers must land inside the sample window opened by - // startBenchmark(), so they have to be emitted before stopBenchmark() - // closes it. The runner consumes the FIFO stream in order, so a marker - // sent after StopBenchmark falls outside the sample and breaks the - // expected SampleStart > BenchmarkStart > BenchmarkEnd > SampleEnd nesting. - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_START, this.runStart!); - InstrumentHooks.addMarker(pid, MARKER_TYPE_BENCHMARK_END, runEnd); - - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(pid, uri); - this.runStart = null; - } - // Allow tinybench to retrieve the path to the currently running suite async onTaskUpdate( _: RunnerTaskResultPack[], diff --git a/packages/vitest-plugin/src/runner.ts b/packages/vitest-plugin/src/runner.ts deleted file mode 100644 index 60a627a6..00000000 --- a/packages/vitest-plugin/src/runner.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { AnalysisRunner } from "./analysis"; - -export default AnalysisRunner; diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts new file mode 100644 index 00000000..d46da02a --- /dev/null +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -0,0 +1,334 @@ +import { + getGitDir, + getInstrumentMode, + InstrumentHooks, + optimizeFunction, + optimizeFunctionSync, + setupCore, + teardownCore, + wrapWithRootFrame, + wrapWithRootFrameSync, + type Benchmark, +} from "@codspeed/core"; +import { createRequire } from "module"; +import path from "path"; +import type { + BenchmarkGroup, + BenchmarkProvider, + BenchRegistrationInput, + BenchResult, + BenchRunOptions, +} from "vitest"; +import { + installInstrumentHooks, + tinybenchTaskToBenchmark, + writeAndLogWalltimeResults, + type TinybenchBench, + type TinybenchOptions, + type TinybenchTask, +} from "../instrument"; + +/** tinybench's per-task lifecycle hooks (a subset of `FnOptions`). */ +interface TinybenchFnOptions { + beforeAll?: (mode?: "run" | "warmup") => unknown; + beforeEach?: (mode?: "run" | "warmup") => unknown; + afterEach?: (mode?: "run" | "warmup") => unknown; + afterAll?: (mode?: "run" | "warmup") => unknown; +} + +type BenchFn = BenchRegistrationInput["fn"]; + +/** tinybench's statistics for one dimension (latency or throughput). */ +type BenchStatistics = BenchResult["latency"]; + +/** + * The subset of the host tinybench Bench the walltime path drives. The timing + * fields are the *resolved* options (tinybench exposes them as readonly + * instance properties), which is what the recorded benchmark config reports. + */ +interface TinybenchWithTasks extends TinybenchBench { + add: ( + name: string, + fn: BenchRegistrationInput["fn"], + fnOpts?: TinybenchFnOptions, + ) => unknown; + run: () => Promise; + tasks: TinybenchTask[]; + readonly iterations: number; + readonly time: number; + readonly warmup: boolean; + readonly warmupIterations: number; + readonly warmupTime: number; +} + +/** + * The host tinybench module, modeled structurally: the plugin's own tinybench + * dependency may be a different major, so its types can't describe the module + * resolved from the installed Vitest. `BenchRunOptions` is Vitest's re-export of + * the tinybench options it accepts, hence of the version it depends on. + */ +interface HostTinybench { + Bench: new (options?: BenchRunOptions) => TinybenchWithTasks; +} + +const isWalltime = getInstrumentMode() === "walltime"; + +// Vitest imports a provider module at most once per worker, and the plugin only +// wires this one up when CodSpeed drives the run (see `V5Backend`), so this is +// the per-worker setup. It must not be redone per benchmark: `setupCore()` +// truncates the process' perf map, which has to cover the whole worker. +setupCore(); +process.once("beforeExit", () => teardownCore()); + +/** + * Build the URI prefix shared by every benchmark of a group: the git-relative + * file path followed by the suite/test path, `::`-separated (e.g. + * `src/a.bench.ts::my suite::my test`). Each registration name is appended to + * it, so a group registering several benchmarks (`bench.compare()`) reports one + * URI per benchmark. + */ +function buildGroupUri(test: BenchmarkGroup["test"]): string { + const filepath = test.file?.filepath; + if (!filepath) { + throw new Error("[CodSpeed] could not resolve the running benchmark file"); + } + const gitDir = getGitDir(filepath); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + const relativeFile = path.relative(gitDir, filepath); + // `fullTestName` uses " > " between suite levels; normalize to "::". + const testPath = test.fullTestName.split(" > ").join("::"); + return [relativeFile, testPath].filter(Boolean).join("::"); +} + +/** + * Resolve the tinybench the *host* Vitest uses so walltime mode returns results + * in the exact shape Vitest serializes. The plugin's own tinybench may be a + * different major, so it is resolved relative to the installed Vitest. + */ +async function importHostTinybench(): Promise { + const require = createRequire(import.meta.url); + const vitestRequire = createRequire(require.resolve("vitest/package.json")); + return import(vitestRequire.resolve("tinybench")); +} + +function analysisResult(name: string): BenchResult { + // Zeroed statistics: enough to satisfy Vitest's reporter and its serializer. + const statistics: BenchStatistics = { + aad: 0, + critical: 0, + df: 0, + mad: 0, + max: 0, + mean: 0, + min: 0, + moe: 0, + p50: 0, + p75: 0, + p99: 0, + p995: 0, + p999: 0, + rme: 0, + samples: undefined, + samplesCount: 0, + sd: 0, + sem: 0, + variance: 0, + }; + + return { + name, + state: "completed", + latency: { ...statistics }, + throughput: { ...statistics }, + period: 0, + totalTime: 0, + runtime: "node", + runtimeVersion: process.versions.node, + timestampProviderName: "codspeed", + }; +} + +/** + * The CodSpeed benchmark provider. Two modes: + * - analysis (instrumentation/simulation): CodSpeed runs each fn itself under a + * tight instrument window and returns zeroed-but-valid results — the real + * measurement is captured by the instrument, not returned here. + * - walltime: tinybench drives the measured loop; CodSpeed brackets it with the + * instrument window and converts each task's stats into a result. + */ +const provider: BenchmarkProvider = { + async run({ test, registrations, options }): Promise { + // Resolve the URI up front, outside any measured window — it walks the + // filesystem (git root lookup), which must not land inside a sample. + const groupUri = buildGroupUri(test); + const getUri = (name: string) => `${groupUri}::${name}`; + + if (isWalltime) { + return runWalltime(registrations, options, getUri, test); + } + return runAnalysis(registrations, getUri); + }, +}; + +async function runAnalysis( + registrations: BenchRegistrationInput[], + getUri: (name: string) => string, +): Promise { + const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + const results: BenchResult[] = []; + for (const { name, fn, fnOpts } of registrations) { + const uri = getUri(name); + await runAnalysisTask(fn, fnOpts, uri); + console.log(`[CodSpeed] ${label} ${uri}`); + results.push(analysisResult(name)); + } + return results; +} + +/** + * Run one benchmark under instrumentation: warm the JIT outside the window, then + * bracket a single `fn()` call with `startBenchmark`/`stopBenchmark` under the + * root frame, with the user hooks around it. + * + * Synchronous benchmarks get a fully synchronous window: awaiting a sync fn + * would splice Node's promise-hook machinery above the root frame. + */ +async function runAnalysisTask( + fn: BenchFn, + fnOpts: TinybenchFnOptions | undefined, + uri: string, +): Promise { + if (fn.constructor?.name === "AsyncFunction") { + await fnOpts?.beforeAll?.("run"); + await optimizeFunction(async () => { + await fnOpts?.beforeEach?.("run"); + await fn(); + await fnOpts?.afterEach?.("run"); + }); + + await fnOpts?.beforeEach?.("run"); + global.gc?.(); + await wrapWithRootFrame(async () => { + InstrumentHooks.startBenchmark(); + await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + await fnOpts?.afterEach?.("run"); + await fnOpts?.afterAll?.("run"); + return; + } + + fnOpts?.beforeAll?.("run"); + optimizeFunctionSync(() => { + fnOpts?.beforeEach?.("run"); + fn(); + fnOpts?.afterEach?.("run"); + }); + + fnOpts?.beforeEach?.("run"); + global.gc?.(); + wrapWithRootFrameSync(() => { + InstrumentHooks.startBenchmark(); + fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + fnOpts?.afterEach?.("run"); + fnOpts?.afterAll?.("run"); +} + +async function runWalltime( + registrations: BenchRegistrationInput[], + options: BenchRunOptions | undefined, + getUri: (name: string) => string, + test: BenchmarkGroup["test"], +): Promise { + const { Bench } = await importHostTinybench(); + const bench = new Bench({ + signal: test.context.signal, + ...options, + // walltime needs per-iteration samples to compute quantiles + retainSamples: true, + }); + + for (const { name, fn, fnOpts } of registrations) { + // tinybench v6 keeps `fn` private, so the root frame is baked in here + bench.add( + name, + wrapWithRootFrame(() => fn()), + fnOpts, + ); + } + + installInstrumentHooks(bench, getUri); + const tasks = await bench.run(); + + throwOnErroredTasks(tasks); + collectWalltimeResults(bench, getUri); + + return tasks.map(tinybenchTaskToBenchResult); +} + +/** + * Surface benchmark failures the way Vitest's own provider does: a single error + * is rethrown as is, several are aggregated, and the test fails instead of + * reporting empty results. + */ +function throwOnErroredTasks(tasks: TinybenchTask[]): void { + const errors = tasks + .filter((task) => task.result?.state === "errored") + .map((task) => task.result?.error); + if (errors.length === 1) { + throw errors[0]; + } + if (errors.length > 1) { + throw new AggregateError(errors, "Some benchmarks failed"); + } +} + +function collectWalltimeResults( + bench: TinybenchWithTasks, + getUri: (name: string) => string, +): void { + const options: TinybenchOptions = { + time: bench.time, + iterations: bench.iterations, + // A bench with warmup turned off keeps its warmup timings, so they'd + // otherwise be reported as if a warmup had run. + warmupTime: bench.warmup ? bench.warmupTime : 0, + warmupIterations: bench.warmup ? bench.warmupIterations : 0, + }; + const benchmarks: Benchmark[] = []; + for (const task of bench.tasks) { + if (task.result?.state !== "completed") continue; + const benchmark = tinybenchTaskToBenchmark( + task, + getUri(task.name), + options, + ); + if (benchmark) { + benchmarks.push(benchmark); + } + } + writeAndLogWalltimeResults(benchmarks); +} + +function tinybenchTaskToBenchResult(task: TinybenchTask): BenchResult { + const result = task.result; + if (result?.state !== "completed") { + throw new Error( + `[CodSpeed] benchmark "${task.name}" did not complete: received "${result?.state ?? "no result"}"`, + ); + } + // tinybench's completed result already carries the full statistics surface + // Vitest serializes; pass it through, keyed by the task name. + return { + ...(result as unknown as Omit), + name: task.name, + }; +} + +export default provider; diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts new file mode 100644 index 00000000..441912dc --- /dev/null +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -0,0 +1,128 @@ +import { getInstrumentMode } from "@codspeed/core"; +import { readFileSync } from "fs"; +import { createRequire } from "module"; +import { join } from "path"; +import { type ViteUserConfig } from "vitest/config"; + +/** + * Everything about integrating with Vitest that depends on which Vitest + * generation the user installed, resolved once so the rest of the plugin reads a + * `VitestBackend` and never inspects the version itself. + */ +export interface VitestBackend { + /** + * Whether the plugin applies for this Vite `mode`; when false it is dropped + * entirely and injects nothing. + */ + isActive(mode: string): boolean; + + /** + * The `test` config fragment that wires the benchmark instrumentation into + * Vitest: the V8 exec args (whose placement moved across versions) plus the + * integration seam. Legacy wires a custom runner subclass (and, in walltime + * mode, asks tinybench to retain samples); v5 wires a `benchmark.provider` + * that owns execution and sample retention entirely. + */ + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"]; +} + +/** + * The integration seam differs across Vitest generations: on 3/4 benchmarks run + * through a `NodeBenchmarkRunner` subclass (via `vitest/runners` / + * `vitest/suite`), while 5+ exposes a `benchmark.provider` API and dropped those + * entrypoints. + * + * When the version cannot be detected we assume the latest supported major. + */ +export function resolveVitestBackend(): VitestBackend { + const major = getVitestMajorVersion() ?? 5; + return major >= 5 ? new V5Backend() : new LegacyBackend(major); +} + +/** + * Resolve the major version of the Vitest the *user's project* depends on, not + * the one bundled alongside this plugin. Returns null when it cannot be found, + * letting `resolveVitestBackend` fall back to the latest supported major. + */ +function getVitestMajorVersion(): number | null { + try { + const require = createRequire(join(process.cwd(), "package.json")); + const vitestPkgPath = require.resolve("vitest/package.json"); + const vitestPkg = JSON.parse(readFileSync(vitestPkgPath, "utf-8")); + return parseInt(vitestPkg.version.split(".")[0], 10); + } catch { + return null; + } +} + +/** + * Vitest 5+. Instrumentation is installed through a `benchmark.provider` that + * owns benchmark execution (see `v5/provider.ts`). + */ +class V5Backend implements VitestBackend { + /** + * `vitest bench` no longer has a dedicated Vite mode, and Vitest clones its + * benchmark project *after* the config hooks ran, so a benchmark run can't be + * read off the config. CodSpeed only drives benchmark runs, so gate on the + * instrument mode: without CodSpeed Vitest runs its own tinybench provider. + */ + isActive(): boolean { + return getInstrumentMode() !== "disabled"; + } + + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"] { + return { + execArgv: v8Flags, + // The provider owns benchmark execution: it runs the registered functions + // under instrumentation (analysis) or drives tinybench itself (walltime). + benchmark: { provider: resolveFile("v5/provider") }, + }; + } +} + +/** + * Vitest 3/4. `vitest bench` runs under a dedicated `"benchmark"` Vite mode, and + * instrumentation is installed through a custom `test.runner` subclass of + * `NodeBenchmarkRunner`, one per instrument mode (`analysis` / `walltime`). + */ +class LegacyBackend implements VitestBackend { + constructor(private readonly major: number) {} + + isActive(mode: string): boolean { + return mode === "benchmark"; + } + + getBenchmarkTestConfig( + v8Flags: string[], + resolveFile: (name: string) => string, + ): ViteUserConfig["test"] { + const instrumentMode = getInstrumentMode(); + const runner = + instrumentMode === "disabled" + ? undefined + : resolveFile(join("legacy", instrumentMode)); + + // Walltime asks tinybench to retain per-iteration samples so the runner can + // compute quantiles. On tinybench v2 (Vitest 3/4) the option is + // `includeSamples` (renamed `retainSamples` in v6). + const benchmark = + instrumentMode === "walltime" ? { includeSamples: true } : undefined; + + return { + // Vitest 3 nests exec args under `poolOptions.forks`; v4 moved them to a + // top-level `test.execArgv`. + // See: https://vitest.dev/guide/migration.html#pool-rework + ...(this.major >= 4 + ? { execArgv: v8Flags } + : { poolOptions: { forks: { execArgv: v8Flags } } }), + ...(runner && { runner }), + ...(benchmark && { benchmark }), + } as ViteUserConfig["test"]; + } +} diff --git a/packages/vitest-plugin/src/walltime/utils.ts b/packages/vitest-plugin/src/walltime/utils.ts deleted file mode 100644 index e0e3313e..00000000 --- a/packages/vitest-plugin/src/walltime/utils.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { - calculateQuantiles, - msToNs, - msToS, - type Benchmark, - type BenchmarkStats, -} from "@codspeed/core"; -import { - type RunnerTaskResult, - type RunnerTestSuite, - type Benchmark as VitestBenchmark, -} from "vitest"; -import { isVitestTaskBenchmark } from "../common"; -import { getBenchOptions } from "../compat"; - -export async function extractBenchmarkResults( - suite: RunnerTestSuite, - parentPath = "", -): Promise { - const benchmarks: Benchmark[] = []; - const currentPath = parentPath ? `${parentPath}::${suite.name}` : suite.name; - - for (const task of suite.tasks) { - if (isVitestTaskBenchmark(task) && task.result?.state === "pass") { - const benchmark = await processBenchmarkTask(task, currentPath); - if (benchmark) { - benchmarks.push(benchmark); - } - } else if (task.type === "suite") { - const nestedBenchmarks = await extractBenchmarkResults(task, currentPath); - benchmarks.push(...nestedBenchmarks); - } - } - - return benchmarks; -} - -async function processBenchmarkTask( - task: VitestBenchmark, - suitePath: string, -): Promise { - const uri = `${suitePath}::${task.name}`; - - const result = task.result; - if (!result) { - console.warn(` ⚠ No result data available for ${uri}`); - return null; - } - - try { - // Get tinybench configuration options from vitest - const benchOptions = getBenchOptions(task); - - const stats = convertVitestResultToBenchmarkStats(result, benchOptions); - - if (stats === null) { - console.log(` ✔ No walltime data to collect for ${uri}`); - return null; - } - - const coreBenchmark: Benchmark = { - name: task.name, - uri, - config: { - max_rounds: benchOptions.iterations ?? null, - max_time_ns: benchOptions.time ? msToNs(benchOptions.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this - warmup_time_ns: - benchOptions.warmupIterations !== 0 && benchOptions.warmupTime - ? msToNs(benchOptions.warmupTime) - : null, - }, - stats, - }; - - console.log(` ✔ Collected walltime data for ${uri}`); - return coreBenchmark; - } catch (error) { - console.warn(` ⚠ Failed to process benchmark result for ${uri}:`, error); - return null; - } -} - -function convertVitestResultToBenchmarkStats( - result: RunnerTaskResult, - benchOptions: { - time?: number; - warmupTime?: number; - warmupIterations?: number; - iterations?: number; - }, -): BenchmarkStats | null { - const benchmark = result.benchmark; - - if (!benchmark) { - throw new Error("No benchmark data available in result"); - } - - const { totalTime, min, max, mean, sd, samples } = benchmark; - - // Get individual sample times in nanoseconds and sort them - const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); - const meanNs = msToNs(mean); - const stdevNs = msToNs(sd); - - if (sortedTimesNs.length == 0) { - // Sometimes the benchmarks can be completely optimized out and not even run, but its beforeEach and afterEach hooks are still executed, and the task is still considered a success. - // This is the case for the hooks.bench.ts example in this package - return null; - } - - const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = - calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); - - return { - min_ns: msToNs(min), - max_ns: msToNs(max), - mean_ns: meanNs, - stdev_ns: stdevNs, - q1_ns, - median_ns, - q3_ns, - total_time: msToS(totalTime), - iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration - rounds: sortedTimesNs.length, - iqr_outlier_rounds, - stdev_outlier_rounds, - warmup_iters: benchOptions.warmupIterations ?? 0, - }; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3e43f6b7..8eb3ec47 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -255,6 +255,30 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/node@26.4.1)(lightningcss@1.33.0)(tsx@4.23.12)(yaml@2.9.0) + examples/with-vitest-v4: + devDependencies: + '@codspeed/vitest-plugin': + specifier: workspace:* + version: link:../../packages/vitest-plugin + typescript: + specifier: ^5.1.3 + version: 5.8.3 + vitest: + specifier: ^4.1.9 + version: 4.1.11(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + + examples/with-vitest-v5: + devDependencies: + '@codspeed/vitest-plugin': + specifier: workspace:* + version: link:../../packages/vitest-plugin + typescript: + specifier: ^5.1.3 + version: 5.8.3 + vitest: + specifier: ^5.0.0 + version: 5.0.2(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + packages/benchmark.js-plugin: dependencies: '@codspeed/core': @@ -368,8 +392,8 @@ importers: specifier: ^8.0.0 version: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) vitest: - specifier: ^4.1.11 - version: 4.1.11(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + specifier: ^5.0.0 + version: 5.0.2(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) packages: @@ -1828,11 +1852,14 @@ packages: '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + '@jridgewell/trace-mapping@0.3.18': resolution: {integrity: sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==} - '@jridgewell/trace-mapping@0.3.29': - resolution: {integrity: sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==} + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} @@ -2762,6 +2789,17 @@ packages: vite: optional: true + '@vitest/mocker@5.0.2': + resolution: {integrity: sha512-Z5FS00Q1SJHkB35xATsmWGdQ5WA1/0MV3CDjqyv7GavHv1OfOj145MNfHOlHk7QLes21dKFDHr8EO2zvL+9WGA==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + '@vitest/pretty-format@3.2.4': resolution: {integrity: sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==} @@ -2798,6 +2836,9 @@ packages: '@vitest/spy@4.1.11': resolution: {integrity: sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==} + '@vitest/spy@5.0.2': + resolution: {integrity: sha512-Ijc7T1nT9efNb5LxvjaBrEqw3f/QwUv5EE0nKqZxgqsaV/FxAAZ8baGylA8X/Z2oS4Lp+K74Jr6dTJsDKxJDeg==} + '@vitest/utils@3.2.4': resolution: {integrity: sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==} @@ -5119,6 +5160,9 @@ packages: resolution: {integrity: sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==} engines: {node: '>=12'} + magic-string@1.4.2: + resolution: {integrity: sha512-vG+rjFRj1PqdIBozIxAGMjPlOhaVe+GXpbttY/iSK7rGcJRMlwNJO7dcUwmUqkymsFLJiNGI06t4D7Fr7yRC9g==} + make-dir@2.1.0: resolution: {integrity: sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==} engines: {node: '>=6'} @@ -5498,6 +5542,10 @@ packages: obug@2.1.1: resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + obug@2.2.1: + resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} + engines: {node: '>=12.20.0'} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -6387,6 +6435,10 @@ packages: resolution: {integrity: sha512-FlHoQpcFvCzeXK5kVPvV7IVgW/hs/B36QWTz876iSdeJguBDfdTSRQmYmaHX+fQNt4hp+gEFB2XXw+8hT4/y8A==} engines: {node: '>=20.0.0'} + tinybench@6.2.0: + resolution: {integrity: sha512-78U2TlB2CnVenajOFzf3BKSm0J6oz5L0NV7g32LCPccvYc0lbWvys4d3uUUCS2B1N8PAf2+aekR8i1KbC3HO7Q==} + engines: {node: '>=20.0.0'} + tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} @@ -6394,6 +6446,10 @@ packages: resolution: {integrity: sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==} engines: {node: '>=18'} + tinyexec@1.3.1: + resolution: {integrity: sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==} + engines: {node: '>=18'} + tinyglobby@0.2.12: resolution: {integrity: sha512-qkf4trmKSIiMTs/E63cxH+ojC2unam7rJ0WrauAzpT3ECNTxGRMlaXxVbfxMUC/w0LaYk6jQ4y/nGR9uBO3tww==} engines: {node: '>=12.0.0'} @@ -6872,6 +6928,47 @@ packages: jsdom: optional: true + vitest@5.0.2: + resolution: {integrity: sha512-7MQrx9pDv5aHiUcovIb/70Ys3tgtkUVgCtledvKdCmEO+/1Dicq5ZqoSxOW034m03oqC+oHOKui2dM6qtMLoJg==} + engines: {node: ^22.12.0 || ^24.0.0 || >=26.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 5.0.2 + '@vitest/browser-preview': 5.0.2 + '@vitest/browser-webdriverio': ^5.0.0-beta.5 || >=5.0.0 + '@vitest/coverage-istanbul': 5.0.2 + '@vitest/coverage-v8': 5.0.2 + '@vitest/ui': 5.0.2 + happy-dom: '*' + jsdom: '*' + vite: ^6.4.0 || ^7.0.0 || ^8.0.0 + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + walk-up-path@3.0.1: resolution: {integrity: sha512-9YlCL/ynK3CTlrSRrDxZvUauLzAswPCrsaCgilqFevUYpeEW0/3ScEjaa3kbW/T0ghhkEr7mv+fpjqn1Y1YuTA==} @@ -6923,6 +7020,11 @@ packages: engines: {node: '>=8'} hasBin: true + why-is-node-running@3.2.2: + resolution: {integrity: sha512-NKUzAelcoCXhXL4dJzKIwXeR8iEVqsA0Lq6Vnd0UXvgaKbzVo4ZTHROF2Jidrv+SgxOQ03fMinnNhzZATxOD3A==} + engines: {node: '>=20.11'} + hasBin: true + wide-align@1.1.5: resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==} @@ -7028,7 +7130,7 @@ snapshots: '@ampproject/remapping@2.3.0': dependencies: '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@apidevtools/json-schema-ref-parser@9.0.9': dependencies: @@ -7098,7 +7200,7 @@ snapshots: dependencies: '@babel/types': 7.22.5 '@jridgewell/gen-mapping': 0.3.3 - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 2.5.2 '@babel/generator@7.28.0': @@ -7106,7 +7208,7 @@ snapshots: '@babel/parser': 7.28.0 '@babel/types': 7.28.1 '@jridgewell/gen-mapping': 0.3.12 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 3.1.0 '@babel/helper-annotate-as-pure@7.22.5': @@ -8655,7 +8757,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@types/node': 26.4.1 chalk: 4.1.2 collect-v8-coverage: 1.0.2 @@ -8687,13 +8789,13 @@ snapshots: '@jest/source-map@29.4.3': dependencies: - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.31 callsites: 3.1.0 graceful-fs: 4.2.11 '@jest/source-map@29.6.3': dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 callsites: 3.1.0 graceful-fs: 4.2.11 @@ -8749,7 +8851,7 @@ snapshots: dependencies: '@babel/core': 7.28.0 '@jest/types': 29.6.3 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 babel-plugin-istanbul: 6.1.1 chalk: 4.1.2 convert-source-map: 2.0.0 @@ -8785,14 +8887,14 @@ snapshots: '@jridgewell/gen-mapping@0.3.12': dependencies: - '@jridgewell/sourcemap-codec': 1.5.5 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/sourcemap-codec': 1.6.0 + '@jridgewell/trace-mapping': 0.3.31 '@jridgewell/gen-mapping@0.3.3': dependencies: '@jridgewell/set-array': 1.1.2 - '@jridgewell/sourcemap-codec': 1.5.4 - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/sourcemap-codec': 1.6.0 + '@jridgewell/trace-mapping': 0.3.31 '@jridgewell/resolve-uri@3.1.0': {} @@ -8810,20 +8912,22 @@ snapshots: '@jridgewell/sourcemap-codec@1.5.5': {} + '@jridgewell/sourcemap-codec@1.6.0': {} + '@jridgewell/trace-mapping@0.3.18': dependencies: '@jridgewell/resolve-uri': 3.1.0 '@jridgewell/sourcemap-codec': 1.4.14 - '@jridgewell/trace-mapping@0.3.29': + '@jridgewell/trace-mapping@0.3.31': dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/sourcemap-codec': 1.6.0 '@jridgewell/trace-mapping@0.3.9': dependencies: '@jridgewell/resolve-uri': 3.1.1 - '@jridgewell/sourcemap-codec': 1.5.4 + '@jridgewell/sourcemap-codec': 1.6.0 '@jsdevtools/ono@7.1.3': {} @@ -9800,6 +9904,15 @@ snapshots: optionalDependencies: vite: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) + '@vitest/mocker@5.0.2(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0))': + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + '@vitest/spy': 5.0.2 + estree-walker: 3.0.3 + magic-string: 1.4.2 + optionalDependencies: + vite: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) + '@vitest/pretty-format@3.2.4': dependencies: tinyrainbow: 2.0.0 @@ -9855,6 +9968,8 @@ snapshots: '@vitest/spy@4.1.11': {} + '@vitest/spy@5.0.2': {} + '@vitest/utils@3.2.4': dependencies: '@vitest/pretty-format': 3.2.4 @@ -12887,6 +13002,10 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.4.15 + magic-string@1.4.2: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + make-dir@2.1.0: dependencies: pify: 4.0.1 @@ -13354,6 +13473,8 @@ snapshots: obug@2.1.1: {} + obug@2.2.1: {} + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -14304,10 +14425,14 @@ snapshots: tinybench@6.0.2: {} + tinybench@6.2.0: {} + tinyexec@0.3.2: {} tinyexec@1.0.2: {} + tinyexec@1.3.1: {} + tinyglobby@0.2.12: dependencies: fdir: 6.5.0(picomatch@4.0.7) @@ -14587,13 +14712,13 @@ snapshots: v8-to-istanbul@9.1.0: dependencies: - '@jridgewell/trace-mapping': 0.3.18 + '@jridgewell/trace-mapping': 0.3.31 '@types/istanbul-lib-coverage': 2.0.4 convert-source-map: 1.9.0 v8-to-istanbul@9.3.0: dependencies: - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.31 '@types/istanbul-lib-coverage': 2.0.6 convert-source-map: 2.0.0 @@ -14744,12 +14869,12 @@ snapshots: es-module-lexer: 2.3.2 expect-type: 1.4.0 magic-string: 0.30.21 - obug: 2.1.1 + obug: 2.2.1 pathe: 2.0.3 picomatch: 4.0.7 std-env: 4.2.0 tinybench: 2.9.0 - tinyexec: 1.0.2 + tinyexec: 1.3.1 tinyglobby: 0.2.17 tinyrainbow: 3.1.1 vite: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) @@ -14759,6 +14884,27 @@ snapshots: transitivePeerDependencies: - msw + vitest@5.0.2(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)): + dependencies: + '@types/chai': 5.2.2 + '@vitest/mocker': 5.0.2(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + chai: 6.2.2 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 1.4.2 + obug: 2.2.1 + picomatch: 4.0.7 + std-env: 4.2.0 + tinybench: 6.2.0 + tinyexec: 1.3.1 + tinyglobby: 0.2.17 + vite: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) + why-is-node-running: 3.2.2 + optionalDependencies: + '@types/node': 26.4.1 + transitivePeerDependencies: + - msw + walk-up-path@3.0.1: {} walker@1.0.8: @@ -14834,6 +14980,8 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 + why-is-node-running@3.2.2: {} + wide-align@1.1.5: dependencies: string-width: 4.2.3