Update auth api key & export to web/node

.gitignore

@@ -6,6 +6,8 @@
 !.vscode
 !.node-version
 !.env.tpl
+*.csv
+*.cursor
 
 *-linux-x64
 *-macos-arm64

README.md

@@ -5,7 +5,7 @@
 [![License](https://img.shields.io/github/license/substreams-js/substreams-node)](LICENSE)
 [![Try substreams on RunKit](https://badge.runkitcdn.com/@substreams/node.svg)](https://npm.runkit.com/@substreams/node)
 
-> Substreams for `Node.js`
+> Substreams for `Node.js` & Web
 
 ## Install
 
@@ -20,21 +20,24 @@ npm install @substreams/node
 ```typescript
 import { createModuleHashHex, createRegistry, createRequest } from "@substreams/core";
 import { readPackage } from "@substreams/manifest";
-import { BlockEmitter, createNodeTransport } from "@substreams/node";
+import { BlockEmitter } from "@substreams/node";
+import { createNodeTransport } from "@substreams/node/createNodeTransport";
+// or
+import { createWebTransport } from "@substreams/node/createWebTransport";
 
 // auth API token
 // https://app.streamingfast.io/
 // https://app.pinax.network/
-if (!process.env.SUBSTREAMS_API_TOKEN) {
-  throw new Error("SUBSTREAMS_API_TOKEN is require");
+if (!process.env.SUBSTREAMS_API_KEY) {
+  throw new Error("SUBSTREAMS_API_KEY is require");
 }
 
-const token = process.env.SUBSTREAMS_API_TOKEN;
+const token = process.env.SUBSTREAMS_API_KEY;
 const baseUrl = "https://eth.substreams.pinax.network:443";
 
 // User parameters
-const manifest = "https://github.com/pinax-network/subtivity-substreams/releases/download/v0.2.3/subtivity-ethereum-v0.2.3.spkg";
-const outputModule = "map_block_stats";
+const manifest = "https://github.com/pinax-network/substreams-erc20-balance-changes/releases/download/v1.2.0/erc20-balance-changes-mainnet-v1.2.0.spkg";
+const outputModule = "map_balance_changes";
 const startBlockNum = 17381140;
 const stopBlockNum = "+3";
 
@@ -47,7 +50,7 @@ const moduleHash = await createModuleHashHex(substreamPackage.modules, outputMod
 console.log({ moduleHash });
 
 // Connect Transport
-const headers = new Headers({ "User-Agent": "@substreams/node" });
+const headers = new Headers({ "X-User-Agent": "@substreams/node", "X-Api-Key": SUBSTREAMS_API_KEY });
 const registry = createRegistry(substreamPackage);
 const transport = createNodeTransport(baseUrl, token, registry, headers);
 const request = createRequest({

examples/eos-eosio.token-to-CSV.mjs

@@ -1,16 +1,17 @@
 import fs from "fs";
 import { createModuleHashHex, createRegistry, createRequest } from "@substreams/core";
 import { readPackage } from "@substreams/manifest";
-import { BlockEmitter, createNodeTransport } from "@substreams/node";
+import { BlockEmitter } from "@substreams/node";
+import { createNodeTransport } from "@substreams/node/createNodeTransport";
 import LogUpdate from "log-update";
 
 // auth API token
 // https://app.streamingfast.io/
 // https://app.pinax.network/
-if (!process.env.SUBSTREAMS_API_TOKEN) {
-  throw new Error("SUBSTREAMS_API_TOKEN is require");
+if (!process.env.SUBSTREAMS_API_KEY) {
+  throw new Error("SUBSTREAMS_API_KEY is require");
 }
-const token = process.env.SUBSTREAMS_API_TOKEN;
+const token = process.env.SUBSTREAMS_API_KEY;
 
 // User parameters
 const baseUrl = "https://eos.substreams.pinax.network:443";
@@ -30,9 +31,8 @@ const filename = `${moduleHash}-${startBlockNum}`
 const startCursor = fs.existsSync(`${filename}.cursor`) ? fs.readFileSync(`${filename}.cursor`, "utf8") : undefined;
 
 // Connect Transport
-const headers = new Headers({ "User-Agent": "@substreams/node" });
 const registry = createRegistry(substreamPackage);
-const transport = createNodeTransport(baseUrl, token, registry, headers);
+const transport = createNodeTransport(baseUrl, token, registry);
 const request = createRequest({
   substreamPackage,
   outputModule,

examples/eosevm-erc20-transfers.mjs

@@ -1,14 +1,15 @@
 import { createRegistry, createRequest } from "@substreams/core";
 import { readPackage } from "@substreams/manifest";
-import { BlockEmitter, createNodeTransport } from "@substreams/node";
+import { BlockEmitter } from "@substreams/node";
+import { createNodeTransport } from "@substreams/node/createNodeTransport";
 
 // auth API token
 // https://app.streamingfast.io/
 // https://app.pinax.network/
-if (!process.env.SUBSTREAMS_API_TOKEN) {
-  throw new Error("SUBSTREAMS_API_TOKEN is require");
+if (!process.env.SUBSTREAMS_API_KEY) {
+  throw new Error("SUBSTREAMS_API_KEY is require");
 }
-const token = process.env.SUBSTREAMS_API_TOKEN;
+const token = process.env.SUBSTREAMS_API_KEY;
 
 // User parameters
 const baseUrl = "https://eosevm.substreams.pinax.network:443";
@@ -23,9 +24,8 @@ if (!substreamPackage.modules) {
 }
 
 // Connect Transport
-const headers = new Headers({ "User-Agent": "@substreams/node" });
 const registry = createRegistry(substreamPackage);
-const transport = createNodeTransport(baseUrl, token, registry, headers);
+const transport = createNodeTransport(baseUrl, token, registry);
 const request = createRequest({
   substreamPackage,
   outputModule,

examples/ethereum-block_stats.mjs

@@ -1,15 +1,16 @@
-import { createModuleHashHex, createRegistry, createRequest } from "@substreams/core";
+import { createRegistry, createRequest } from "@substreams/core";
 import { readPackage } from "@substreams/manifest";
-import { BlockEmitter, createNodeTransport } from "@substreams/node";
+import { BlockEmitter } from "@substreams/node";
+import { createNodeTransport } from "@substreams/node/createNodeTransport";
 
 // auth API token
 // https://app.streamingfast.io/
 // https://app.pinax.network/
-if (!process.env.SUBSTREAMS_API_TOKEN) {
-  throw new Error("SUBSTREAMS_API_TOKEN is require");
+if (!process.env.SUBSTREAMS_API_KEY) {
+  throw new Error("SUBSTREAMS_API_KEY is require");
 }
 
-const token = process.env.SUBSTREAMS_API_TOKEN;
+const token = process.env.SUBSTREAMS_API_KEY;
 const baseUrl = "https://eth.substreams.pinax.network:443";
 
 // User parameters
@@ -23,13 +24,10 @@ const substreamPackage = await readPackage(manifest);
 if (!substreamPackage.modules) {
   throw new Error("No modules found in substream package");
 }
-const moduleHash = await createModuleHashHex(substreamPackage.modules, outputModule);
-console.log({ moduleHash });
 
 // Connect Transport
-const headers = new Headers({ "User-Agent": "@substreams/node" });
 const registry = createRegistry(substreamPackage);
-const transport = createNodeTransport(baseUrl, token, registry, headers);
+const transport = createNodeTransport(baseUrl, token, registry);
 const request = createRequest({
   substreamPackage,
   outputModule,

examples/ethereum-erc20-balance-changes.mjs

@@ -1,15 +1,16 @@
-import { createModuleHashHex, createRegistry, createRequest } from "@substreams/core";
+import { createRegistry, createRequest } from "@substreams/core";
 import { readPackage } from "@substreams/manifest";
-import { BlockEmitter, createNodeTransport } from "@substreams/node";
+import { BlockEmitter } from "@substreams/node";
+import { createNodeTransport } from "@substreams/node/createNodeTransport";
 
 // auth API token
 // https://app.streamingfast.io/
 // https://app.pinax.network/
-if (!process.env.SUBSTREAMS_API_TOKEN) {
-  throw new Error("SUBSTREAMS_API_TOKEN is require");
+if (!process.env.SUBSTREAMS_API_KEY) {
+  throw new Error("SUBSTREAMS_API_KEY is require");
 }
 
-const token = process.env.SUBSTREAMS_API_TOKEN;
+const token = process.env.SUBSTREAMS_API_KEY;
 const baseUrl = "https://eth.substreams.pinax.network:443";
 
 // User parameters
@@ -22,11 +23,9 @@ const substreamPackage = await readPackage(manifest);
 if (!substreamPackage.modules) {
   throw new Error("No modules found in substream package");
 }
-const moduleHash = await createModuleHashHex(substreamPackage.modules, outputModule);
-console.log({ moduleHash });
 
 // Connect Transport
-const headers = new Headers({ "User-Agent": "@substreams/node" });
+const headers = new Headers({ "X-User-Agent": "@substreams/node", "X-Api-Key": SUBSTREAMS_API_KEY });
 const registry = createRegistry(substreamPackage);
 const transport = createNodeTransport(baseUrl, token, registry, headers);
 const request = createRequest({

examples/package.json

@@ -3,7 +3,7 @@
     "@substreams/core": "latest",
     "@substreams/manifest": "latest",
     "@substreams/node": "latest",
-    "@wharfkit/antelope": "^1.0.6",
+    "@wharfkit/antelope": "latest",
     "log-update": "latest"
   }
 }

examples/wax-delband.mjs

@@ -1,15 +1,16 @@
 import { createRegistry, createRequest, applyParams } from "@substreams/core";
 import { readPackage } from "@substreams/manifest";
-import { BlockEmitter, createNodeTransport } from "@substreams/node";
-import { APIClient, Bytes, Serializer } from "@wharfkit/antelope";
+import { BlockEmitter } from "@substreams/node";
+import { createNodeTransport } from "@substreams/node/createNodeTransport";
+import { APIClient, Serializer } from "@wharfkit/antelope";
 
 // auth API token
 // https://app.streamingfast.io/
 // https://app.pinax.network/
-if (!process.env.SUBSTREAMS_API_TOKEN) {
-  throw new Error("SUBSTREAMS_API_TOKEN is require");
+if (!process.env.SUBSTREAMS_API_KEY) {
+  throw new Error("SUBSTREAMS_API_KEY is require");
 }
-const token = process.env.SUBSTREAMS_API_TOKEN;
+const token = process.env.SUBSTREAMS_API_KEY;
 
 // RPC API Client
 const rpc = new APIClient({ url: "https://wax.api.eosnation.io" });

index.ts

@@ -1,5 +1,2 @@
 export { BlockEmitter } from "./src/BlockEmitter.js";
-export { createDefaultTransport } from "./src/createDefaultTransport.js";
-export { createNodeTransport } from "./src/createNodeTransport.js";
-export { createWebTransport } from "./src/createWebTransport.js";
-export { createHeadersInterceptor } from "./src/createHeadersInterceptor.js";
+export { createHeadersInterceptor, createAuthInterceptor, createInterceptors, defaultHeadersInterceptor } from "./src/interceptors.js";

package.json

@@ -1,13 +1,28 @@
 {
   "name": "@substreams/node",
-  "version": "0.5.3",
-  "description": "Substreams for Node.js",
+  "version": "0.6.0",
+  "description": "Substreams for Node.js & Web",
   "license": "MIT",
   "repository": "substreams-js/substreams-node",
   "type": "module",
-  "exports": "./dist/index.js",
   "types": "./dist/index.d.ts",
-  "runkitExampleFilename": "example.js",
+  "exports": {
+    ".": {
+      "default": "./dist/index.js",
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./createNodeTransport": {
+      "default": "./dist/src/createNodeTransport.js",
+      "import": "./dist/src/createNodeTransport.js",
+      "types": "./dist/src/createNodeTransport.d.ts"
+    },
+    "./createWebTransport": {
+      "default": "./dist/src/createWebTransport.js",
+      "import": "./dist/src/createWebTransport.js",
+      "types": "./dist/src/createWebTransport.d.ts"
+    }
+  },
   "sideEffects": false,
   "engines": {
     "node": ">=18"
@@ -16,7 +31,7 @@
     "build": "tsc",
     "prepublishOnly": "npm run build"
   },
-  "files": ["dist", "src", "example.js"],
+  "files": ["dist", "src"],
   "keywords": ["substreams", "streamingfast", "thegraph", "graph"],
   "dependencies": {
     "@bufbuild/protobuf": "latest",

src/BlockEmitter.ts

@@ -1,4 +1,3 @@
-// import { type Request, type Response, Stream } from "@substreams/core/proto";
 import { AnyMessage, IMessageTypeRegistry, JsonObject, Message } from "@bufbuild/protobuf";
 import { type CallOptions, type ConnectError, type Transport, createCallbackClient } from "@connectrpc/connect";
 import { isEmptyMessage, unpackMapOutput } from "@substreams/core";

src/createNodeTransport.ts

@@ -1,18 +1,13 @@
-import { createHeadersInterceptor } from "./createHeadersInterceptor.js";
 import type { IMessageTypeRegistry } from "@bufbuild/protobuf";
 import { Transport } from "@connectrpc/connect";
 import { createGrpcWebTransport } from "@connectrpc/connect-node";
-import { createAuthInterceptor } from "@substreams/core";
+import { createInterceptors } from "./interceptors.js";
 
 export function createNodeTransport(baseUrl: string, token: string, registry: IMessageTypeRegistry, headers?: Headers): Transport {
-  const interceptors = [createHeadersInterceptor(headers)];
-  if (token) {
-    interceptors.push(createAuthInterceptor(token));
-  }
   return createGrpcWebTransport({
     baseUrl,
     httpVersion: "2",
-    interceptors,
+    interceptors: createInterceptors(token, headers),
     jsonOptions: {
       typeRegistry: registry,
     },

src/createWebTransport.ts

@@ -1,17 +1,12 @@
-import { createHeadersInterceptor } from "./createHeadersInterceptor.js";
 import type { IMessageTypeRegistry } from "@bufbuild/protobuf";
 import { Transport } from "@connectrpc/connect";
 import { createGrpcWebTransport } from "@connectrpc/connect-web";
-import { createAuthInterceptor } from "@substreams/core";
+import { createInterceptors } from "./interceptors.js";
 
 export function createWebTransport(baseUrl: string, token: string, registry: IMessageTypeRegistry, headers?: Headers): Transport {
-  const interceptors = [createHeadersInterceptor(headers)];
-  if (token) {
-    interceptors.push(createAuthInterceptor(token));
-  }
   return createGrpcWebTransport({
     baseUrl,
-    interceptors,
+    interceptors: createInterceptors(token, headers),
     jsonOptions: {
       typeRegistry: registry,
     },

src/interceptors.ts

@@ -0,0 +1,31 @@
+import type { Interceptor } from "@connectrpc/connect";
+
+export function createAuthInterceptor(token: string) {
+    const headers = new Headers({"X-Api-Key": token});
+    return createHeadersInterceptor(headers)
+}
+
+export function defaultHeadersInterceptor() {
+    const headers = new Headers({"X-User-Agent": "@substreams/node"});
+    return createHeadersInterceptor(headers)
+}
+
+export function createHeadersInterceptor(headers: Headers): Interceptor {
+  return (next) => async (req) => {
+    for (const [key, value] of Array.from(headers)) {
+      req.header.set(key, value);
+    }
+    return next(req);
+  };
+}
+
+export function createInterceptors(token?: string, headers?: Headers) {
+    const interceptors = [defaultHeadersInterceptor()];
+    if (token) {
+      interceptors.push(createAuthInterceptor(token));
+    }
+    if (headers) {
+      interceptors.push(createHeadersInterceptor(headers));
+    }
+    return interceptors;
+}

tsconfig.json

@@ -10,6 +10,7 @@
 		"sourceMap": true
 	},
 	"include": [
-		"index.ts"
+		"index.ts",
+		"src/**.ts"
 	]
 }