Merge pull request #2 from ChainSafe/mkeil/perf-unit-test-with-mocha

feat: implement unit and perf tests with Mocha using CommonJS

.benchrc.yaml

@@ -0,0 +1,10 @@
+# Mocha opts
+extension: ["ts"]
+colors: true
+node-option:
+  - "loader=ts-node/register"
+
+# benchmark opts
+threshold: 3
+maxMs: 60_000
+minRuns: 10

.eslintrc.js

@@ -15,7 +15,7 @@ module.exports = {
     project: "./tsconfig.json",
     sourceType: "script",
   },
-  plugins: ["@typescript-eslint", "eslint-plugin-import", "eslint-plugin-node", "prettier", "jest"],
+  plugins: ["@typescript-eslint", "eslint-plugin-import", "eslint-plugin-node", "prettier"],
   extends: [
     "eslint:recommended",
     "plugin:import/errors",

.github/workflows/CI.yml

@@ -150,7 +150,8 @@ jobs:
         run: ls -R .
         shell: bash
       - name: Test bindings
-        run: yarn test
+        run: yarn test:unit
+
   test-linux-x64-gnu-binding:
     name: Test - Linux-x64-gnu - node@${{ matrix.node }}
     needs:
@@ -160,7 +161,7 @@ jobs:
       matrix:
         node:
           - '18'
-          - '22.4.x'
+          - '22.4.1'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -186,7 +187,13 @@ jobs:
       - name: Ensure there are no unstaged changes
         run: git diff --exit-code
       - name: Test bindings
-        run: yarn test
+        run: yarn test:unit
+      # TODO: (@matthewkeil) need to figure out how to get these to post to the PR body
+      # - name: Perf test bindings
+      #   run: yarn test:perf
+      #   if: ${{ matrix.node == '22.4.1' }} 
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   test-linux-x64-musl-binding:
     name: Test - x86_64-unknown-linux-musl - node@${{ matrix.node }}
@@ -225,7 +232,7 @@ jobs:
           options: '--platform linux/amd64 -v ${{ github.workspace }}:/build -w /build'
           run: |
             set -e
-            yarn test
+            yarn test:unit
             ls -la
 
   test-linux-aarch64-gnu-binding:
@@ -266,7 +273,7 @@ jobs:
           options: '--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build'
           run: |
             set -e
-            yarn test
+            yarn test:unit
             ls -la
 
   test-linux-aarch64-musl-binding:
@@ -307,7 +314,7 @@ jobs:
           options: '--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build'
           run: |
             set -e
-            yarn test
+            yarn test:unit
             ls -la
 
   publish:

.gitignore

@@ -1,6 +1,9 @@
 # Created by https://www.toptal.com/developers/gitignore/api/node
 # Edit at https://www.toptal.com/developers/gitignore?templates=node
 
+## Benchmark data
+benchmark_data/
+
 ### Node ###
 # Logs
 logs
@@ -121,7 +124,7 @@ dist
 .AppleDouble
 .LSOverride
 
-# Icon must end with two 
+# Icon must end with two 
 Icon
 
 

.mocharc.yaml

@@ -0,0 +1,3 @@
+colors: true
+require: ts-node/register
+exit: true

index.d.ts

@@ -3,7 +3,8 @@
 
 /* auto-generated by NAPI-RS */
 
-export const SHUFFLE_ROUNDS: number
+export const SHUFFLE_ROUNDS_MINIMAL: number
+export const SHUFFLE_ROUNDS_MAINNET: number
 export declare function shuffleList(activeIndices: Uint32Array, seed: Uint8Array, rounds: number): Uint32Array
 export declare function asyncShuffleList(activeIndices: Uint32Array, seed: Uint8Array, rounds: number): Promise<Uint32Array>
 export declare function unshuffleList(activeIndices: Uint32Array, seed: Uint8Array, rounds: number): Uint32Array

index.js

@@ -310,9 +310,10 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { SHUFFLE_ROUNDS, shuffleList, asyncShuffleList, unshuffleList, asyncUnshuffleList } = nativeBinding
+const { SHUFFLE_ROUNDS_MINIMAL, SHUFFLE_ROUNDS_MAINNET, shuffleList, asyncShuffleList, unshuffleList, asyncUnshuffleList } = nativeBinding
 
-module.exports.SHUFFLE_ROUNDS = SHUFFLE_ROUNDS
+module.exports.SHUFFLE_ROUNDS_MINIMAL = SHUFFLE_ROUNDS_MINIMAL
+module.exports.SHUFFLE_ROUNDS_MAINNET = SHUFFLE_ROUNDS_MAINNET
 module.exports.shuffleList = shuffleList
 module.exports.asyncShuffleList = asyncShuffleList
 module.exports.unshuffleList = unshuffleList

package.json

@@ -3,14 +3,17 @@
   "version": "0.0.1",
   "scripts": {
     "artifacts": "napi artifacts",
-    "build": "napi build --platform --release --js false",
+    "build": "napi build --platform --release",
     "build:debug": "napi build --platform",
     "prepublishOnly": "napi prepublish -t npm",
-    "test": "jest",
+    "test": "yarn test:unit && yarn test:perf",
+    "test:unit": "mocha test/unit/*.test.ts",
+    "test:perf": "node -r ts-node/register node_modules/.bin/benchmark --config .benchrc.yaml test/perf/*.test.ts",
     "lint": "eslint --color --ext .ts test/",
     "universal": "napi universal",
     "version": "napi version"
   },
+  "type": "commonjs",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -27,21 +30,22 @@
   },
   "license": "MIT",
   "devDependencies": {
+    "@dapplion/benchmark": "^0.2.4",
     "@chainsafe/as-sha256": "^0.5.0",
-    "@jest/globals": "^29.7.0",
     "@napi-rs/cli": "^2.18.4",
+    "@types/chai": "^4.3.16",
+    "@types/mocha": "^10.0.7",
     "@types/node": "^22.4.1",
     "@typescript-eslint/eslint-plugin": "^7.15.0",
     "@typescript-eslint/parser": "^7.15.0",
     "bigint-buffer": "^1.1.5",
+    "chai": "^4.3.4",
     "eslint": "^8.57.0",
     "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-jest": "^28.8.3",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-prettier": "^5.1.3",
-    "jest": "^29.7.0",
+    "mocha": "^8.3.2",
     "prettier": "^3.3.2",
-    "ts-jest": "^29.2.5",
     "ts-node": "^9.1.1",
     "typescript": "^5.5.3"
   },

src/lib.rs

@@ -9,7 +9,10 @@ use ethereum_hashing::hash_fixed;
 use ethereum_types::H256;
 
 #[napi]
-pub const SHUFFLE_ROUNDS: u32 = 90;
+pub const SHUFFLE_ROUNDS_MINIMAL: u32 = 10;
+
+#[napi]
+pub const SHUFFLE_ROUNDS_MAINNET: u32 = 90;
 
 const SEED_SIZE: usize = 32;
 const ROUND_SIZE: usize = 1;

test/perf/shuffle.test.ts

@@ -0,0 +1,42 @@
+import {itBench} from "@dapplion/benchmark";
+import {unshuffleList, SHUFFLE_ROUNDS_MINIMAL, asyncUnshuffleList} from "../../index";
+import * as referenceImplementation from "../referenceImplementation";
+
+//          Lightouse  Lodestar
+// 512      254.04 us  1.6034 ms (x6)
+// 16384    6.2046 ms  18.272 ms (x3)
+// 4000000  1.5617 s   4.9690 s  (x3)
+
+for (const listSize of [
+  16384, 250_000, 1_000_000,
+  // Don't run 4_000_000 since it's very slow and not testnet has gotten there yet
+  // 4e6,
+]) {
+  describe(`shuffle list - ${listSize} indices`, () => {
+    const seed = Buffer.alloc(32, 0xac);
+    const input: number[] = [];
+    for (let i = 0; i < listSize; i++) input[i] = i;
+    const indices = new Uint32Array(input);
+
+    itBench<Uint32Array, Uint32Array>({
+      id: `JS   - unshuffleList - ${listSize} indices`,
+      fn: () => {
+        referenceImplementation.unshuffleList(indices, seed, SHUFFLE_ROUNDS_MINIMAL);
+      },
+    });
+
+    itBench<Uint32Array, Uint32Array>({
+      id: `Rust - unshuffleList - ${listSize} indices`,
+      fn: () => {
+        unshuffleList(indices, seed, SHUFFLE_ROUNDS_MINIMAL);
+      },
+    });
+
+    itBench<Uint32Array, Uint32Array>({
+      id: `Rust - asyncUnshuffleList - ${listSize} indices`,
+      fn: async () => {
+        await asyncUnshuffleList(indices, seed, SHUFFLE_ROUNDS_MINIMAL);
+      },
+    });
+  });
+}

test/shuffle.test.ts → test/unit/shuffle.test.ts

@@ -1,7 +1,7 @@
+import {expect} from "chai";
 import {randomBytes} from "node:crypto";
-import {expect, describe, it} from "@jest/globals";
-import * as referenceImplementation from "./referenceImplementation";
-import {shuffleList, asyncShuffleList, unshuffleList, asyncUnshuffleList} from "../index";
+import * as referenceImplementation from "../referenceImplementation";
+import {shuffleList, asyncShuffleList, unshuffleList, asyncUnshuffleList} from "../../index";
 
 interface ShuffleTestCase {
   id: string;
@@ -54,15 +54,15 @@ describe("shuffle", () => {
   it("should throw for invalid seed", () => {
     const test = buildReferenceTestCase(10, 10);
     let invalidSeed = Buffer.alloc(31, 0xac);
-    expect(() => unshuffleList(test.input, invalidSeed, test.rounds)).toThrow("Shuffling seed must be 32 bytes long");
+    expect(() => unshuffleList(test.input, invalidSeed, test.rounds)).to.throw("Shuffling seed must be 32 bytes long");
     invalidSeed = Buffer.alloc(33, 0xac);
-    expect(() => unshuffleList(test.input, invalidSeed, test.rounds)).toThrow("Shuffling seed must be 32 bytes long");
+    expect(() => unshuffleList(test.input, invalidSeed, test.rounds)).to.throw("Shuffling seed must be 32 bytes long");
   });
 
   it("should throw for invalid number of rounds", () => {
     const test = buildReferenceTestCase(10, 10);
-    expect(() => unshuffleList(test.input, test.seed, -1)).toThrow("Rounds must be between 0 and 255");
-    expect(() => unshuffleList(test.input, test.seed, 256)).toThrow("Rounds must be between 0 and 255");
+    expect(() => unshuffleList(test.input, test.seed, -1)).to.throw("Rounds must be between 0 and 255");
+    expect(() => unshuffleList(test.input, test.seed, 256)).to.throw("Rounds must be between 0 and 255");
   });
 
   /**
@@ -72,7 +72,7 @@ describe("shuffle", () => {
   // it("should throw for invalid input array length", () => {
   //   const test = buildReferenceTestCase(10, 10);
   //   const input = Uint32Array.from(Buffer.alloc(2 ** 32, 0xac));
-  //   expect(() => unshuffleList(input, test.seed, 100)).toThrow("ActiveIndices must fit in a u32");
+  //   expect(() => unshuffleList(input, test.seed, 100)).to.throw("ActiveIndices must fit in a u32");
   // });
 
   it("should match spec test results", () => {
@@ -87,7 +87,7 @@ describe("shuffle", () => {
 
     const result = unshuffleList(getInputArray(100), fromHex(seed), rounds);
 
-    expect(Buffer.from(result).toString("hex")).toEqual(expected);
+    expect(Buffer.from(result).toString("hex")).to.equal(expected);
   });
 
   const testCases: ShuffleTestCase[] = [
@@ -98,16 +98,18 @@ describe("shuffle", () => {
     buildReferenceTestCase(256, 192),
   ];
 
-  it.each(testCases)("sync - $id", ({seed, rounds, input, shuffled, unshuffled}) => {
-    const unshuffledResult = unshuffleList(input, seed, rounds);
-    const shuffledResult = shuffleList(input, seed, rounds);
-    expect(Buffer.from(shuffledResult).toString("hex")).toEqual(shuffled);
-    expect(Buffer.from(unshuffledResult).toString("hex")).toEqual(unshuffled);
-  });
-  it.each(testCases)("async - $id", async ({seed, rounds, input, shuffled, unshuffled}) => {
-    const unshuffledResult = await asyncUnshuffleList(input, seed, rounds);
-    const shuffledResult = await asyncShuffleList(input, seed, rounds);
-    expect(Buffer.from(shuffledResult).toString("hex")).toEqual(shuffled);
-    expect(Buffer.from(unshuffledResult).toString("hex")).toEqual(unshuffled);
-  });
+  for (const {id, seed, rounds, input, shuffled, unshuffled} of testCases) {
+    it(`sync - ${id}`, () => {
+      const unshuffledResult = unshuffleList(input, seed, rounds);
+      const shuffledResult = shuffleList(input, seed, rounds);
+      expect(Buffer.from(shuffledResult).toString("hex")).to.equal(shuffled);
+      expect(Buffer.from(unshuffledResult).toString("hex")).to.equal(unshuffled);
+    });
+    it(`async - ${id}`, async () => {
+      const unshuffledResult = await asyncUnshuffleList(input, seed, rounds);
+      const shuffledResult = await asyncShuffleList(input, seed, rounds);
+      expect(Buffer.from(shuffledResult).toString("hex")).to.equal(shuffled);
+      expect(Buffer.from(unshuffledResult).toString("hex")).to.equal(unshuffled);
+    });
+  }
 });

tsconfig.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     "target": "ESNext",
-    "module": "ESNext",
+    "module": "CommonJS",
     "moduleResolution": "Node",
     "esModuleInterop": true,