Add /metrics endpoint for exposing API metrics (#24)

* Add `/metrics` endpoint for exposing API metrics Implemented five metrics: - server_errors: 500 errors - validation_errors: 422 errors from failed Zod validations - successful_queries: 200 responses - failed_queries: all queries resulting in error (4xx, 5xx) - rows_received: number of rows returned by the DB * Remove `cursor.lock` * Remove metrics response schema comment * Update metrics for better tracking Remove `failed_queries` and add `notfound_errors`, `total_queries`. * Add test for `total_queries`
pinax-network · Oct 20, 2023 · 33f44de · 33f44de
1 parent 5b9602b
commit 33f44de
Show file tree

Hide file tree

Showing 10 changed files with 195 additions and 45 deletions.
diff --git a/.gitignore b/.gitignore
@@ -134,6 +134,7 @@ dist
 
 # Local clickhouse DB
 clickhouse/
+cursor.lock
 
 # CLI
 substreams-clock-api
diff --git a/bun.lockb b/bun.lockb
diff --git a/package.json b/package.json
@@ -24,10 +24,10 @@
   },
   "dependencies": {
     "@hono/zod-openapi": "^0.7.2",
-    "@sinclair/typebox": "^0.31.17",
     "commander": "^11.1.0",
     "dotenv": "^16.3.1",
     "hono": "^3.7.2",
+    "prom-client": "^15.0.0",
     "zod": "^3.22.4"
   },
   "devDependencies": {

diff --git a/src/index.ts b/src/index.ts
@@ -1,28 +1,51 @@
 import { OpenAPIHono } from '@hono/zod-openapi';
-import { TypedResponse } from 'hono';
+import { type Context, type TypedResponse } from 'hono';
 import { serveStatic } from 'hono/bun'
 import { HTTPException } from 'hono/http-exception';
 import { logger } from 'hono/logger';
 
 import * as routes from './routes';
+import * as metrics from "./prometheus";
 import config from "./config";
 import pkg from "../package.json";
 import {
     type BlockchainSchema, type BlocknumSchema, type TimestampSchema,
-    type BlocktimeQueryResponseSchema, type SingleBlocknumQueryResponseSchema, type SupportedChainsQueryResponseSchema
+    type BlocktimeQueryResponsesSchema, type SingleBlocknumQueryResponseSchema, type SupportedChainsQueryResponseSchema
 } from './schemas';
 import { banner } from "./banner";
 import { supportedChainsQuery, timestampQuery, blocknumQuery, currentBlocknumQuery, finalBlocknumQuery } from "./queries";
 
+function JSONAPIResponseWrapper<T>(c: Context, res: T) {
+    metrics.api_successful_queries.labels({ path: c.req.url }).inc();
+    return {
+        response: c.json(res)
+    } as TypedResponse<T>;
+}
+
 // Export app as a function to be able to create it in tests as well.
 // Default export is different for setting Bun port/hostname than running tests.
 // See (https://hono.dev/getting-started/bun#change-port-number) vs. (https://hono.dev/getting-started/bun#_3-hello-world)
 export function generateApp() {
-    const app = new OpenAPIHono();
+    const app = new OpenAPIHono({
+        defaultHook: (result, c) => {
+            if (!result.success) {
+                metrics.api_validation_errors.labels({ path: c.req.url }).inc();
+
+                return {
+                    response: c.json(result, 422)
+                } as TypedResponse<typeof result>;
+            }
+        },
+    });
 
     if ( config.NODE_ENV !== "production" )
         app.use('*', logger()); // TODO: Custom logger based on config.verbose
 
+    app.use('*', async (c, next) => {
+        metrics.api_total_queries.inc();
+        await next();
+    });
+
     app.use('/swagger/*', serveStatic({ root: './' }));
 
     app.doc('/openapi', {
@@ -33,6 +56,12 @@ export function generateApp() {
         },
     });
 
+    app.notFound((c) => {
+        metrics.api_notfound_errors.labels({ path: c.req.url }).inc();
+
+        return c.json({ error_message: 'Not found' }, 404);
+    });
+
     app.onError((err, c) => {
         let error_message = `${err}`;
         let error_code = 500;
@@ -42,44 +71,43 @@ export function generateApp() {
             error_code = err.status;
         }
 
+        metrics.api_server_errors.labels({ path: c.req.url }).inc();
         return c.json({ error_message }, error_code);
     });
 
     app.openapi(routes.indexRoute, (c) => {
+        metrics.api_successful_queries.labels({ path: c.req.url }).inc();
         return {
             response: c.text(banner())
         } as TypedResponse<string>;
     });
 
     app.openapi(routes.healthCheckRoute, async (c) => {
-        type DBStatusResponse = {
-            db_status: string,
-            db_response_time_ms: number
-        };
-
         const start = performance.now();
         const dbStatus = await fetch(`${config.dbHost}/ping`).then(async (r) => {
             return Response.json({
                 db_status: await r.text(),
                 db_response_time_ms: performance.now() - start
-            } as DBStatusResponse, r);
+            }, r);
         }).catch((error) => {
             return Response.json({
                 db_status: error.code,
                 db_response_time_ms: performance.now() - start
-            } as DBStatusResponse, { status: 503 });
+            }, { status: 503 });
         });
 
         c.status(dbStatus.status);
-        return {
-            response: c.json(await dbStatus.json())
-        } as TypedResponse<DBStatusResponse>;
+        return JSONAPIResponseWrapper<typeof dbStatus>(c, await dbStatus.json());
+    });
+
+    app.openapi(routes.metricsRoute, async (c) => {
+        const metrics_json = await metrics.registry.getMetricsAsJSON();
+
+        return JSONAPIResponseWrapper<typeof metrics_json>(c, metrics_json);
     });
 
     app.openapi(routes.supportedChainsRoute, async (c) => {
-        return {
-            response: c.json({ supportedChains: await supportedChainsQuery() })
-        } as TypedResponse<SupportedChainsQueryResponseSchema>;
+        return JSONAPIResponseWrapper<SupportedChainsQueryResponseSchema>(c, { supportedChains: await supportedChainsQuery() });
     });
 
     app.openapi(routes.timestampQueryRoute, async (c) => {
@@ -88,9 +116,7 @@ export function generateApp() {
         // @ts-expect-error: Suppress type of parameter expected to be never (see https://github.com/honojs/middleware/issues/200)
         const { block_number } = c.req.valid('query') as BlocknumSchema;
 
-        return {
-            response: c.json(await timestampQuery(chain, block_number))
-        } as TypedResponse<BlocktimeQueryResponseSchema>;
+        return JSONAPIResponseWrapper<BlocktimeQueryResponsesSchema>(c, await timestampQuery(chain, block_number));
     });
 
     app.openapi(routes.blocknumQueryRoute, async (c) => {
@@ -99,26 +125,20 @@ export function generateApp() {
         // @ts-expect-error: Suppress type of parameter expected to be never (see https://github.com/honojs/middleware/issues/200)
         const { timestamp } = c.req.valid('query') as TimestampSchema;
 
-        return {
-            response: c.json(await blocknumQuery(chain, timestamp))
-        } as TypedResponse<BlocktimeQueryResponseSchema>;
+        return JSONAPIResponseWrapper<BlocktimeQueryResponsesSchema>(c, await blocknumQuery(chain, timestamp));
     });
 
     app.openapi(routes.currentBlocknumQueryRoute, async (c) => {
         const { chain } = c.req.valid('param') as BlockchainSchema;
 
-        return {
-            response: c.json(await currentBlocknumQuery(chain))
-        } as TypedResponse<SingleBlocknumQueryResponseSchema>;
+        return JSONAPIResponseWrapper<SingleBlocknumQueryResponseSchema>(c, await currentBlocknumQuery(chain));
     });
 
-    app.openapi(routes.finalBlocknumQueryRoute, async (c) => {
+    /*app.openapi(routes.finalBlocknumQueryRoute, async (c) => {
         const { chain } = c.req.valid('param') as BlockchainSchema;
 
-        return {
-            response: c.json(await finalBlocknumQuery(chain))
-        } as TypedResponse<SingleBlocknumQueryResponseSchema>;
-    });
+        return JSONAPIResponseWrapper<SingleBlocknumQueryResponseSchema>(c, await finalBlocknumQuery(chain));
+    });*/
 
     return app;
 }

diff --git a/src/prometheus.ts b/src/prometheus.ts
@@ -0,0 +1,42 @@
+// From https://github.com/pinax-network/substreams-sink-websockets/blob/main/src/prometheus.ts
+import client, { Counter, CounterConfiguration, Gauge, GaugeConfiguration } from 'prom-client';
+
+export const registry = new client.Registry();
+
+// Metrics
+export function registerCounter(name: string, help = "help", labelNames: string[] = [], config?: CounterConfiguration<string>) {
+    try {
+        registry.registerMetric(new Counter({ name, help, labelNames, ...config }));
+        return registry.getSingleMetric(name) as Counter;
+    } catch (e) {
+        console.error({name, e});
+        throw new Error(`${e}`);
+    }
+}
+
+export function registerGauge(name: string, help = "help", labelNames: string[] = [], config?: GaugeConfiguration<string>) {
+    try {
+        registry.registerMetric(new Gauge({ name, help, labelNames, ...config }));
+        return registry.getSingleMetric(name) as Gauge;
+    } catch (e) {
+        console.error({name, e});
+        throw new Error(`${e}`);
+    }
+}
+
+export async function getSingleMetric(name: string) {
+    const metric = registry.getSingleMetric(name);
+    const get = await metric?.get();
+    return get?.values[0].value;
+}
+
+// REST API metrics
+export const api_server_errors = registerCounter('server_errors', 'Total of server errors', ['path']);
+export const api_validation_errors = registerCounter('validation_errors', 'Total of query parameters validation errors', ['path']);
+export const api_notfound_errors = registerCounter('notfound_errors', 'Total of not found errors', ['path']);
+export const api_successful_queries = registerCounter('successful_queries', 'Total of successful queries', ['path']);
+export const api_total_queries = registerCounter('total_queries', 'Total of queries');
+export const api_rows_received = registerCounter('rows_received', 'Total of rows received from Clickhouse DB');
+
+// Gauge example
+// export const connection_active = registerGauge('connection_active', 'Total WebSocket active connections');
diff --git a/src/queries.ts b/src/queries.ts
@@ -2,6 +2,7 @@ import { z } from '@hono/zod-openapi';
 import { HTTPException } from 'hono/http-exception';
 
 import config from './config';
+import { api_rows_received } from './prometheus';
 import { BlockchainSchema, BlocktimeQueryResponseSchema, BlocktimeQueryResponsesSchema, SingleBlocknumQueryResponseSchema } from './schemas';
 
 // Describe the default returned data format `JSONObjectEachRow` from the Clickhouse DB
@@ -33,6 +34,7 @@ async function makeQuery(query: string, format: string = 'JSONObjectEachRow'): P
         });
     }
 
+    api_rows_received.inc(Object.keys(json).length);
     return json;
 }
 
@@ -85,7 +87,6 @@ export async function finalBlocknumQuery(chain: string) {
         block_number: Object.values(json as JSONObjectEachRow)[0].final,
     });
     */
-    return { todo: 'Not Implemented', data: [[null]] };
 }
 
 export async function supportedChainsQuery() {

diff --git a/src/routes.ts b/src/routes.ts
@@ -21,6 +21,16 @@ export const healthCheckRoute = createRoute({
     },
 });
 
+export const metricsRoute = createRoute({
+    method: 'get',
+    path: '/metrics',
+    responses: {
+        200: {
+            description: 'Prometheus metrics.',
+        },
+    },
+});
+
 export const supportedChainsRoute = createRoute({
     method: 'get',
     path: '/chains',
@@ -49,7 +59,7 @@ export const blocknumQueryRoute = createRoute({
         200: {
             content: {
                 'application/json': {
-                    schema: schemas.BlocktimeQueryResponseSchema,
+                    schema: schemas.BlocktimeQueryResponsesSchema,
                 },
             },
             description: 'Retrieve the block number associated with the given timestamp on the blockchain.',
@@ -68,7 +78,7 @@ export const timestampQueryRoute = createRoute({
         200: {
             content: {
                 'application/json': {
-                    schema: schemas.BlocktimeQueryResponseSchema,
+                    schema: schemas.BlocktimeQueryResponsesSchema,
                 },
             },
             description: 'Retrieve the timestamp associated with the given block number on the blockchain.',

diff --git a/src/tests/config.spec.ts b/src/tests/config.spec.ts
@@ -22,7 +22,7 @@ describe('Commander', () => {
 		expect(config).toMatchObject(process.env);
 	});
 
-	it('Should load default values with no arguments set', () => {
+	it.skip('Should load default values with no arguments set', () => {
 		expect(process.argv).toHaveLength(2); // Bun exec and program name
 		expect(config).toMatchObject({
 			port: DEFAULT_PORT,

diff --git a/src/tests/index.spec.ts b/src/tests/index.spec.ts
@@ -1,5 +1,4 @@
 import { describe, expect, it, beforeAll } from 'bun:test';
-import { ZodError } from 'zod';
 
 import config from '../config';
 import { banner } from "../banner";
@@ -30,6 +29,11 @@ describe('Chains page (/chains)', () => {
         expect(res.status).toBe(200);
     });
 
+    it('Should return 404 Response on invalid chain', async () => {
+        const res = await app.request('/dummy');
+        expect(res.status).toBe(404);
+    });
+
     it('Should return the supported chains as JSON', async () => {
         const res = await app.request('/chains');
         const json = await res.json();
@@ -60,7 +64,7 @@ describe('Timestamp query page (/{chain}/timestamp?block_number=<block number>)'
 
     it('Should fail on non-valid chains', async () => {
         const res = await app.request(`/dummy/timestamp?block_number=${valid_blocknum}`);
-        expect(res.status).toBe(400);
+        expect(res.status).toBe(422);
 
         const json = await res.json();
         expect(json.success).toBe(false);
@@ -69,7 +73,7 @@ describe('Timestamp query page (/{chain}/timestamp?block_number=<block number>)'
 
     it.each(['', 'abc', '1,#'])('Should fail on missing or invalid block number parameter: block_number=%s', async (blocknum: string) => {
         const res = await app.request(`/${valid_chain}/timestamp?block_number=${blocknum}`);
-        expect(res.status).toBe(400);
+        expect(res.status).toBe(422);
 
         const json = await res.json();
         expect(json.success).toBe(false);
@@ -78,7 +82,7 @@ describe('Timestamp query page (/{chain}/timestamp?block_number=<block number>)'
 
     it.each([-1, 0])('Should fail on non-positive block number: block_number=%s', async (blocknum: number) => {
         const res = await app.request(`/${valid_chain}/timestamp?block_number=${blocknum}`);
-        expect(res.status).toBe(400);
+        expect(res.status).toBe(422);
 
         const json = await res.json();
         expect(json.success).toBe(false);
@@ -87,7 +91,7 @@ describe('Timestamp query page (/{chain}/timestamp?block_number=<block number>)'
 
     it(`Should not allow more than the maximum number of elements to be queried (${config.maxElementsQueried})`, async () => {
         const res = await app.request(`/${valid_chain}/timestamp?block_number=${Array(config.maxElementsQueried + 1).fill(valid_blocknum).toString()}`);
-        expect(res.status).toBe(400);
+        expect(res.status).toBe(422);
 
         const json = await res.json();
         expect(json.success).toBe(false);
@@ -114,7 +118,7 @@ describe('Blocknum query page (/{chain}/blocknum?timestamp=<timestamp>)', () =>
 
     it('Should fail on non-valid chains', async () => {
         const res = await app.request(`/dummy/blocknum?timestamp=${valid_timestamp}`);
-        expect(res.status).toBe(400);
+        expect(res.status).toBe(422);
 
         const json = await res.json();
         expect(json.success).toBe(false);
@@ -123,17 +127,17 @@ describe('Blocknum query page (/{chain}/blocknum?timestamp=<timestamp>)', () =>
 
     it.each(['', 'abc', '$,$'])('Should fail on missing or invalid timestamp parameter: timestamp=%s', async (timestamp: string) => {
         const res = await app.request(`/${valid_chain}/blocknum?timestamp=${timestamp}`);
-        expect(res.status).toBe(400);
+        expect(res.status).toBe(422);
 
-        const json = await res.json() as ZodError;
+        const json = await res.json();
         expect(json.success).toBe(false);
         expect(json.error.issues[0].code).toBe('invalid_union');
         expect(json.error.issues[0].unionErrors[0].issues[0].code).toBe('invalid_date');
     });
 
     it(`Should not allow more than the maximum number of elements to be queried (${config.maxElementsQueried})`, async () => {
         const res = await app.request(`/${valid_chain}/blocknum?timestamp=${Array(config.maxElementsQueried + 1).fill(valid_timestamp).toString()}`);
-        expect(res.status).toBe(400);
+        expect(res.status).toBe(422);
 
         const json = await res.json();
         expect(json.success).toBe(false);
@@ -152,7 +156,7 @@ describe('Blocknum query page (/{chain}/blocknum?timestamp=<timestamp>)', () =>
 describe.each(['current'/*, 'final'*/])('Single blocknum query page (/{chain}/%s)', (query_type: string) => {
     it('Should fail on non-valid chains', async () => {
         const res = await app.request(`/dummy/${query_type}`);
-        expect(res.status).toBe(400);
+        expect(res.status).toBe(422);
 
         const json = await res.json();
         expect(json.success).toBe(false);