From f1f44969bffe9c75b8c65239b1ab58d7b3f69b36 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Wed, 7 Feb 2024 19:20:42 -0500 Subject: [PATCH] Update auth api key --- .gitignore | 2 ++ README.md | 19 +++++++------ examples/eos-eosio.token-to-CSV.mjs | 12 ++++---- examples/eosevm-erc20-transfers.mjs | 12 ++++---- examples/ethereum-block_stats.mjs | 16 +++++------ examples/ethereum-erc20-balance-changes.mjs | 15 +++++----- examples/package.json | 2 +- examples/wax-delband.mjs | 11 ++++---- index.ts | 5 +--- package.json | 25 +++++++++++++---- src/BlockEmitter.ts | 1 - src/createDefaultTransport.ts | 4 --- src/createHeadersInterceptor.ts | 11 -------- src/createNodeTransport.ts | 9 ++---- src/createWebTransport.ts | 9 ++---- src/interceptors.ts | 31 +++++++++++++++++++++ tsconfig.json | 3 +- 17 files changed, 104 insertions(+), 83 deletions(-) delete mode 100644 src/createDefaultTransport.ts delete mode 100644 src/createHeadersInterceptor.ts create mode 100644 src/interceptors.ts diff --git a/.gitignore b/.gitignore index c2ea34e..191f917 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ !.vscode !.node-version !.env.tpl +*.csv +*.cursor *-linux-x64 *-macos-arm64 diff --git a/README.md b/README.md index af04d93..f408f67 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![License](https://img.shields.io/github/license/substreams-js/substreams-node)](LICENSE) [![Try substreams on RunKit](https://badge.runkitcdn.com/@substreams/node.svg)](https://npm.runkit.com/@substreams/node) -> Substreams for `Node.js` +> Substreams for `Node.js` & Web ## Install @@ -20,21 +20,24 @@ npm install @substreams/node ```typescript import { createModuleHashHex, createRegistry, createRequest } from "@substreams/core"; import { readPackage } from "@substreams/manifest"; -import { BlockEmitter, createNodeTransport } from "@substreams/node"; +import { BlockEmitter } from "@substreams/node"; +import { createNodeTransport } from "@substreams/node/createNodeTransport"; +// or +import { createWebTransport } from "@substreams/node/createWebTransport"; // auth API token // https://app.streamingfast.io/ // https://app.pinax.network/ -if (!process.env.SUBSTREAMS_API_TOKEN) { - throw new Error("SUBSTREAMS_API_TOKEN is require"); +if (!process.env.SUBSTREAMS_API_KEY) { + throw new Error("SUBSTREAMS_API_KEY is require"); } -const token = process.env.SUBSTREAMS_API_TOKEN; +const token = process.env.SUBSTREAMS_API_KEY; const baseUrl = "https://eth.substreams.pinax.network:443"; // User parameters -const manifest = "https://github.com/pinax-network/subtivity-substreams/releases/download/v0.2.3/subtivity-ethereum-v0.2.3.spkg"; -const outputModule = "map_block_stats"; +const manifest = "https://github.com/pinax-network/substreams-erc20-balance-changes/releases/download/v1.2.0/erc20-balance-changes-mainnet-v1.2.0.spkg"; +const outputModule = "map_balance_changes"; const startBlockNum = 17381140; const stopBlockNum = "+3"; @@ -47,7 +50,7 @@ const moduleHash = await createModuleHashHex(substreamPackage.modules, outputMod console.log({ moduleHash }); // Connect Transport -const headers = new Headers({ "User-Agent": "@substreams/node" }); +const headers = new Headers({ "X-User-Agent": "@substreams/node", "X-Api-Key": SUBSTREAMS_API_KEY }); const registry = createRegistry(substreamPackage); const transport = createNodeTransport(baseUrl, token, registry, headers); const request = createRequest({ diff --git a/examples/eos-eosio.token-to-CSV.mjs b/examples/eos-eosio.token-to-CSV.mjs index 7fe1c74..af8dc94 100644 --- a/examples/eos-eosio.token-to-CSV.mjs +++ b/examples/eos-eosio.token-to-CSV.mjs @@ -1,16 +1,17 @@ import fs from "fs"; import { createModuleHashHex, createRegistry, createRequest } from "@substreams/core"; import { readPackage } from "@substreams/manifest"; -import { BlockEmitter, createNodeTransport } from "@substreams/node"; +import { BlockEmitter } from "@substreams/node"; +import { createNodeTransport } from "@substreams/node/createNodeTransport"; import LogUpdate from "log-update"; // auth API token // https://app.streamingfast.io/ // https://app.pinax.network/ -if (!process.env.SUBSTREAMS_API_TOKEN) { - throw new Error("SUBSTREAMS_API_TOKEN is require"); +if (!process.env.SUBSTREAMS_API_KEY) { + throw new Error("SUBSTREAMS_API_KEY is require"); } -const token = process.env.SUBSTREAMS_API_TOKEN; +const token = process.env.SUBSTREAMS_API_KEY; // User parameters const baseUrl = "https://eos.substreams.pinax.network:443"; @@ -30,9 +31,8 @@ const filename = `${moduleHash}-${startBlockNum}` const startCursor = fs.existsSync(`${filename}.cursor`) ? fs.readFileSync(`${filename}.cursor`, "utf8") : undefined; // Connect Transport -const headers = new Headers({ "User-Agent": "@substreams/node" }); const registry = createRegistry(substreamPackage); -const transport = createNodeTransport(baseUrl, token, registry, headers); +const transport = createNodeTransport(baseUrl, token, registry); const request = createRequest({ substreamPackage, outputModule, diff --git a/examples/eosevm-erc20-transfers.mjs b/examples/eosevm-erc20-transfers.mjs index 2ee7377..1a282b4 100644 --- a/examples/eosevm-erc20-transfers.mjs +++ b/examples/eosevm-erc20-transfers.mjs @@ -1,14 +1,15 @@ import { createRegistry, createRequest } from "@substreams/core"; import { readPackage } from "@substreams/manifest"; -import { BlockEmitter, createNodeTransport } from "@substreams/node"; +import { BlockEmitter } from "@substreams/node"; +import { createNodeTransport } from "@substreams/node/createNodeTransport"; // auth API token // https://app.streamingfast.io/ // https://app.pinax.network/ -if (!process.env.SUBSTREAMS_API_TOKEN) { - throw new Error("SUBSTREAMS_API_TOKEN is require"); +if (!process.env.SUBSTREAMS_API_KEY) { + throw new Error("SUBSTREAMS_API_KEY is require"); } -const token = process.env.SUBSTREAMS_API_TOKEN; +const token = process.env.SUBSTREAMS_API_KEY; // User parameters const baseUrl = "https://eosevm.substreams.pinax.network:443"; @@ -23,9 +24,8 @@ if (!substreamPackage.modules) { } // Connect Transport -const headers = new Headers({ "User-Agent": "@substreams/node" }); const registry = createRegistry(substreamPackage); -const transport = createNodeTransport(baseUrl, token, registry, headers); +const transport = createNodeTransport(baseUrl, token, registry); const request = createRequest({ substreamPackage, outputModule, diff --git a/examples/ethereum-block_stats.mjs b/examples/ethereum-block_stats.mjs index e29fed3..cd5bc37 100644 --- a/examples/ethereum-block_stats.mjs +++ b/examples/ethereum-block_stats.mjs @@ -1,15 +1,16 @@ -import { createModuleHashHex, createRegistry, createRequest } from "@substreams/core"; +import { createRegistry, createRequest } from "@substreams/core"; import { readPackage } from "@substreams/manifest"; -import { BlockEmitter, createNodeTransport } from "@substreams/node"; +import { BlockEmitter } from "@substreams/node"; +import { createNodeTransport } from "@substreams/node/createNodeTransport"; // auth API token // https://app.streamingfast.io/ // https://app.pinax.network/ -if (!process.env.SUBSTREAMS_API_TOKEN) { - throw new Error("SUBSTREAMS_API_TOKEN is require"); +if (!process.env.SUBSTREAMS_API_KEY) { + throw new Error("SUBSTREAMS_API_KEY is require"); } -const token = process.env.SUBSTREAMS_API_TOKEN; +const token = process.env.SUBSTREAMS_API_KEY; const baseUrl = "https://eth.substreams.pinax.network:443"; // User parameters @@ -23,13 +24,10 @@ const substreamPackage = await readPackage(manifest); if (!substreamPackage.modules) { throw new Error("No modules found in substream package"); } -const moduleHash = await createModuleHashHex(substreamPackage.modules, outputModule); -console.log({ moduleHash }); // Connect Transport -const headers = new Headers({ "User-Agent": "@substreams/node" }); const registry = createRegistry(substreamPackage); -const transport = createNodeTransport(baseUrl, token, registry, headers); +const transport = createNodeTransport(baseUrl, token, registry); const request = createRequest({ substreamPackage, outputModule, diff --git a/examples/ethereum-erc20-balance-changes.mjs b/examples/ethereum-erc20-balance-changes.mjs index 968af8c..22fd57c 100644 --- a/examples/ethereum-erc20-balance-changes.mjs +++ b/examples/ethereum-erc20-balance-changes.mjs @@ -1,15 +1,16 @@ -import { createModuleHashHex, createRegistry, createRequest } from "@substreams/core"; +import { createRegistry, createRequest } from "@substreams/core"; import { readPackage } from "@substreams/manifest"; -import { BlockEmitter, createNodeTransport } from "@substreams/node"; +import { BlockEmitter } from "@substreams/node"; +import { createNodeTransport } from "@substreams/node/createNodeTransport"; // auth API token // https://app.streamingfast.io/ // https://app.pinax.network/ -if (!process.env.SUBSTREAMS_API_TOKEN) { - throw new Error("SUBSTREAMS_API_TOKEN is require"); +if (!process.env.SUBSTREAMS_API_KEY) { + throw new Error("SUBSTREAMS_API_KEY is require"); } -const token = process.env.SUBSTREAMS_API_TOKEN; +const token = process.env.SUBSTREAMS_API_KEY; const baseUrl = "https://eth.substreams.pinax.network:443"; // User parameters @@ -22,11 +23,9 @@ const substreamPackage = await readPackage(manifest); if (!substreamPackage.modules) { throw new Error("No modules found in substream package"); } -const moduleHash = await createModuleHashHex(substreamPackage.modules, outputModule); -console.log({ moduleHash }); // Connect Transport -const headers = new Headers({ "User-Agent": "@substreams/node" }); +const headers = new Headers({ "X-User-Agent": "@substreams/node", "X-Api-Key": SUBSTREAMS_API_KEY }); const registry = createRegistry(substreamPackage); const transport = createNodeTransport(baseUrl, token, registry, headers); const request = createRequest({ diff --git a/examples/package.json b/examples/package.json index 4bce9e0..1fabc46 100644 --- a/examples/package.json +++ b/examples/package.json @@ -3,7 +3,7 @@ "@substreams/core": "latest", "@substreams/manifest": "latest", "@substreams/node": "latest", - "@wharfkit/antelope": "^1.0.6", + "@wharfkit/antelope": "latest", "log-update": "latest" } } diff --git a/examples/wax-delband.mjs b/examples/wax-delband.mjs index 8795349..798c3b5 100644 --- a/examples/wax-delband.mjs +++ b/examples/wax-delband.mjs @@ -1,15 +1,16 @@ import { createRegistry, createRequest, applyParams } from "@substreams/core"; import { readPackage } from "@substreams/manifest"; -import { BlockEmitter, createNodeTransport } from "@substreams/node"; -import { APIClient, Bytes, Serializer } from "@wharfkit/antelope"; +import { BlockEmitter } from "@substreams/node"; +import { createNodeTransport } from "@substreams/node/createNodeTransport"; +import { APIClient, Serializer } from "@wharfkit/antelope"; // auth API token // https://app.streamingfast.io/ // https://app.pinax.network/ -if (!process.env.SUBSTREAMS_API_TOKEN) { - throw new Error("SUBSTREAMS_API_TOKEN is require"); +if (!process.env.SUBSTREAMS_API_KEY) { + throw new Error("SUBSTREAMS_API_KEY is require"); } -const token = process.env.SUBSTREAMS_API_TOKEN; +const token = process.env.SUBSTREAMS_API_KEY; // RPC API Client const rpc = new APIClient({ url: "https://wax.api.eosnation.io" }); diff --git a/index.ts b/index.ts index ce6905f..a3f46e6 100644 --- a/index.ts +++ b/index.ts @@ -1,5 +1,2 @@ export { BlockEmitter } from "./src/BlockEmitter.js"; -export { createDefaultTransport } from "./src/createDefaultTransport.js"; -export { createNodeTransport } from "./src/createNodeTransport.js"; -export { createWebTransport } from "./src/createWebTransport.js"; -export { createHeadersInterceptor } from "./src/createHeadersInterceptor.js"; +export { createHeadersInterceptor, createAuthInterceptor, createInterceptors, defaultHeadersInterceptor } from "./src/interceptors.js"; diff --git a/package.json b/package.json index 4bcdee0..62696ee 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,28 @@ { "name": "@substreams/node", - "version": "0.5.3", - "description": "Substreams for Node.js", + "version": "0.6.0", + "description": "Substreams for Node.js & Web", "license": "MIT", "repository": "substreams-js/substreams-node", "type": "module", - "exports": "./dist/index.js", "types": "./dist/index.d.ts", - "runkitExampleFilename": "example.js", + "exports": { + ".": { + "default": "./dist/index.js", + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + }, + "./createNodeTransport": { + "default": "./dist/src/createNodeTransport.js", + "import": "./dist/src/createNodeTransport.js", + "types": "./dist/src/createNodeTransport.d.ts" + }, + "./createWebTransport": { + "default": "./dist/src/createWebTransport.js", + "import": "./dist/src/createWebTransport.js", + "types": "./dist/src/createWebTransport.d.ts" + } + }, "sideEffects": false, "engines": { "node": ">=18" @@ -16,7 +31,7 @@ "build": "tsc", "prepublishOnly": "npm run build" }, - "files": ["dist", "src", "example.js"], + "files": ["dist", "src"], "keywords": ["substreams", "streamingfast", "thegraph", "graph"], "dependencies": { "@bufbuild/protobuf": "latest", diff --git a/src/BlockEmitter.ts b/src/BlockEmitter.ts index 2bd227b..5ea6e2f 100644 --- a/src/BlockEmitter.ts +++ b/src/BlockEmitter.ts @@ -1,4 +1,3 @@ -// import { type Request, type Response, Stream } from "@substreams/core/proto"; import { AnyMessage, IMessageTypeRegistry, JsonObject, Message } from "@bufbuild/protobuf"; import { type CallOptions, type ConnectError, type Transport, createCallbackClient } from "@connectrpc/connect"; import { isEmptyMessage, unpackMapOutput } from "@substreams/core"; diff --git a/src/createDefaultTransport.ts b/src/createDefaultTransport.ts deleted file mode 100644 index 744dc22..0000000 --- a/src/createDefaultTransport.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { createNodeTransport } from "./createNodeTransport.js"; - -// make it backwards compatible for those still using `createDefaultTransport` -export const createDefaultTransport = createNodeTransport; diff --git a/src/createHeadersInterceptor.ts b/src/createHeadersInterceptor.ts deleted file mode 100644 index 81376e0..0000000 --- a/src/createHeadersInterceptor.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { Interceptor } from "@connectrpc/connect"; - -export function createHeadersInterceptor(headers?: Headers): Interceptor { - return (next) => async (req) => { - for (const header in headers?.entries()) { - const [key, value] = header; - req.header.set(key, value); - } - return next(req); - }; -} diff --git a/src/createNodeTransport.ts b/src/createNodeTransport.ts index 0a3ac3a..0a8e5a2 100644 --- a/src/createNodeTransport.ts +++ b/src/createNodeTransport.ts @@ -1,18 +1,13 @@ -import { createHeadersInterceptor } from "./createHeadersInterceptor.js"; import type { IMessageTypeRegistry } from "@bufbuild/protobuf"; import { Transport } from "@connectrpc/connect"; import { createGrpcWebTransport } from "@connectrpc/connect-node"; -import { createAuthInterceptor } from "@substreams/core"; +import { createInterceptors } from "./interceptors.js"; export function createNodeTransport(baseUrl: string, token: string, registry: IMessageTypeRegistry, headers?: Headers): Transport { - const interceptors = [createHeadersInterceptor(headers)]; - if (token) { - interceptors.push(createAuthInterceptor(token)); - } return createGrpcWebTransport({ baseUrl, httpVersion: "2", - interceptors, + interceptors: createInterceptors(token, headers), jsonOptions: { typeRegistry: registry, }, diff --git a/src/createWebTransport.ts b/src/createWebTransport.ts index 4c4e0b0..f02a8ea 100644 --- a/src/createWebTransport.ts +++ b/src/createWebTransport.ts @@ -1,17 +1,12 @@ -import { createHeadersInterceptor } from "./createHeadersInterceptor.js"; import type { IMessageTypeRegistry } from "@bufbuild/protobuf"; import { Transport } from "@connectrpc/connect"; import { createGrpcWebTransport } from "@connectrpc/connect-web"; -import { createAuthInterceptor } from "@substreams/core"; +import { createInterceptors } from "./interceptors.js"; export function createWebTransport(baseUrl: string, token: string, registry: IMessageTypeRegistry, headers?: Headers): Transport { - const interceptors = [createHeadersInterceptor(headers)]; - if (token) { - interceptors.push(createAuthInterceptor(token)); - } return createGrpcWebTransport({ baseUrl, - interceptors, + interceptors: createInterceptors(token, headers), jsonOptions: { typeRegistry: registry, }, diff --git a/src/interceptors.ts b/src/interceptors.ts new file mode 100644 index 0000000..a284044 --- /dev/null +++ b/src/interceptors.ts @@ -0,0 +1,31 @@ +import type { Interceptor } from "@connectrpc/connect"; + +export function createAuthInterceptor(token: string) { + const headers = new Headers({"X-Api-Key": token}); + return createHeadersInterceptor(headers) +} + +export function defaultHeadersInterceptor() { + const headers = new Headers({"X-User-Agent": "@substreams/node"}); + return createHeadersInterceptor(headers) +} + +export function createHeadersInterceptor(headers: Headers): Interceptor { + return (next) => async (req) => { + for (const [key, value] of Array.from(headers)) { + req.header.set(key, value); + } + return next(req); + }; +} + +export function createInterceptors(token?: string, headers?: Headers) { + const interceptors = [defaultHeadersInterceptor()]; + if (token) { + interceptors.push(createAuthInterceptor(token)); + } + if (headers) { + interceptors.push(createHeadersInterceptor(headers)); + } + return interceptors; +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index ab13394..b13d00d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,6 +10,7 @@ "sourceMap": true }, "include": [ - "index.ts" + "index.ts", + "src/**.ts" ] } \ No newline at end of file