feat: add a new command to compare multiple benchmark results

.github/workflows/benchmark-compare.yml

@@ -0,0 +1,102 @@
+name: Benchmark
+
+# only one can tun at a time.
+# Actions access a common cache entry and may corrupt it.
+concurrency: cd-benchmark-compare-${{ github.ref }}
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  benchmark-nodejs:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Install
+        run: yarn install --frozen-lockfile
+
+      - name: Run performance tests on node
+        run: yarn benchmark --local ./nodejs --noThrow ${{ github.event_name == 'push' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Archive nodejs benchmark artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-benchmark
+          path: nodejs
+
+  benchmark-bun:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Install Dependencies
+        run: bun install
+
+      - name: Run performance tests on bun
+        run: bun run --bun benchmark --local ./bun --noThrow ${{ github.event_name == 'push' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Archive bun benchmark artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: bun-benchmark
+          path: bun
+
+  benchmark-deno:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+
+      - name: Install Dependencies
+        run: deno install
+
+      - name: Run performance tests on bun
+        run: deno run -A benchmark --local ./deno --noThrow ${{ github.event_name == 'push' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Archive deno benchmark artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: deno-benchmark
+          path: deno
+
+  benchmark-compare:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "18"
+
+      - name: Install
+        run: yarn install --frozen-lockfile
+
+      - name: Download all benchmark artifacts
+        uses: actions/download-artifact@v4
+
+      - name: Run comparison
+        run: node --loader ts-node/esm ./src/cli/cli.ts cmp nodejs-benchmark deno-benchmark bun-benchmark

.github/workflows/test.yml

@@ -3,8 +3,8 @@ name: Tests
 on: [pull_request, push]
 
 jobs:
-  test:
-    name: Test
+  test-node:
+    name: Test Node
     strategy:
       fail-fast: false
       matrix:
@@ -25,3 +25,41 @@ jobs:
         run: yarn lint
       - name: Unit tests
         run: yarn test:unit
+
+  test-deno:
+    name: Test Deno
+    runs-on: ubuntu-latest
+    steps:
+      # <common-build> - Uses YAML anchors in the future
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: "22"
+      - uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+      - name: Install
+        run: yarn install --frozen-lockfile
+      - name: Build
+        run: deno run build
+      - name: Unit tests
+        run: deno run -A test:unit
+
+  test-bun:
+    name: Test Bun
+    runs-on: ubuntu-latest
+    steps:
+      # <common-build> - Uses YAML anchors in the future
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: "22"
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+      - name: Install
+        run: yarn install --frozen-lockfile
+      - name: Build
+        run: bun run --bun build
+      - name: Unit tests
+        run: bun run --bun test:unit

README.md

@@ -9,10 +9,10 @@ This tooling provides both a easy to use runner for benchmarking and easy integr
 
 ## Quick start
 
-Create a test mocha test file but use `itBench` instead of `it`
+Create a benchmark test file but use `itBench` instead of `it`
 
 ```ts
-import {itBench, setBenchOpts} from "../../src";
+import {itBench, setBenchOpts, describe} from "../../src";
 
 describe("Sum array benchmark", () => {
   itBench("sum array with reduce", () => {
@@ -21,7 +21,7 @@ describe("Sum array benchmark", () => {
 });
 ```
 
-Then run the CLI, compatible with all mocha options.
+Then run the CLI.
 
 ```
 benchmark 'test/perf/**/*.perf.ts' --local
@@ -36,7 +36,7 @@ Inspect benchmark results in the terminal
 
 ## How does it work?
 
-This tool is a CLI wrapper around mocha, example usage:
+This tool is a CLI tool, example usage:
 
 ```
 benchmark 'test/perf/**/*.perf.ts' --s3
@@ -47,9 +47,7 @@ The above command will:
 - Read benchmark history from the specified provider (AWS S3)
 - Figure out the prev benchmark based on your option (defaults to latest commit in main branch)
 - Run benchmark comparing with previous
-  - Runs mocha programatically against the file globs
-  - Collect benchmark data in-memory while streaming results with a familiar mocha reporter
-  - Note: also runs any test that would regularly be run with mocha
+  - Collect benchmark data in-memory while streaming results
 - Add result to benchmark history and persist them to the specified provider (AWS S3)
 - If in CI, post a PR or commit comment with an expandable summary of the benchmark results comparision
 - If a performance regression was detected, exit 1

bin/index.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+
+require("../lib/esm/cli/cli.js");

package.json

@@ -5,7 +5,7 @@
   "author": "dapplion <[email protected]>",
   "license": "MIT",
   "bin": {
-    "benchmark": "./bin/index.cjs"
+    "benchmark": "./bin/index.js"
   },
   "files": [
     "lib/**/*.d.ts",
@@ -30,43 +30,45 @@
     "build": "yarn build:esm && yarn build:cjs",
     "build:esm": "tsc -p tsconfig.build.esm.json && echo '{\"type\": \"module\"}' > ./lib/esm/package.json",
     "build:cjs": "tsc -p tsconfig.build.cjs.json && echo '{\"type\": \"commonjs\"}' > ./lib/cjs/package.json",
-    "test:unit": "mocha test/unit/**/*.test.ts",
+    "test:unit": "vitest run test/unit/**/*.test.ts",
     "lint": "eslint --color src/ test/",
     "prepublishOnly": "yarn build",
-    "benchmark": "node --loader ts-node/esm ./src/cli.ts 'test/perf/**/*.test.ts'",
+    "benchmark": "node --loader ts-node/esm ./src/cli/cli.ts 'test/perf/**/*.test.ts'",
     "writeDocs": "node --loader ts-node/esm scripts/writeOptionsMd.ts"
   },
   "devDependencies": {
-    "@types/chai": "^4.2.19",
-    "@types/mocha": "^10.0.9",
-    "@types/node": "^18.15.3",
-    "@types/rimraf": "^3.0.0",
+    "@eslint/js": "^9.15.0",
+    "@types/node": "^22.10.2",
     "@types/yargs": "^17.0.33",
     "chai": "^4.5.0",
     "dotenv": "^10.0.0",
     "eslint": "^9.15.0",
-    "@eslint/js": "^9.15.0",
-    "eslint-plugin-import": "^2.31.0",
+    "eslint-config-prettier": "^9.1.0",
     "eslint-import-resolver-typescript": "^3.6.3",
+    "eslint-plugin-import": "^2.31.0",
     "eslint-plugin-prettier": "^5.2.1",
-    "eslint-config-prettier": "^9.1.0",
-    "mocha": "^10.8.2",
     "prettier": "^3.4.0",
-    "rimraf": "^3.0.2",
+    "rimraf": "^5.0.10",
     "ts-node": "^10.9.2",
     "typescript": "^5.6.3",
-    "typescript-eslint": "^8.16.0"
+    "typescript-eslint": "^8.16.0",
+    "vitest": "^2.1.8",
+    "vitest-in-process-pool": "^1.0.0"
   },
   "dependencies": {
     "@actions/cache": "^1.0.7",
     "@actions/github": "^5.0.0",
+    "@vitest/runner": "^2.1.6",
     "ajv": "^8.6.0",
     "aws-sdk": "^2.932.0",
+    "cli-table3": "^0.6.5",
     "csv-parse": "^4.16.0",
     "csv-stringify": "^5.6.2",
+    "glob": "^10.4.5",
+    "log-symbols": "^7.0.0",
     "yargs": "^17.7.2"
   },
-  "peerDependencies": {
-    "mocha": ">10.0.0"
+  "resolutions": {
+    "lru-cache": "10.4.3"
   }
 }

scripts/writeOptionsMd.ts

@@ -1,4 +1,4 @@
-import {options} from "../src/options.js";
+import {options} from "../src/cli/options.js";
 
 const sections: string[] = [];