task: Expose prometheus metrics (#2586)

## About the changes
This connects our backend with Prometheus (or compatible) metrics
service, and exposes raw data (i.e. acting as a proxy)

Co-authored-by: Christopher Kolstad <chriswk@getunleash.ai>

src/lib/__snapshots__/create-config.test.ts.snap

@@ -121,6 +121,7 @@ exports[`should create default config 1`] = `
   },
   "preHook": undefined,
   "preRouterHook": undefined,
+  "prometheusApi": undefined,
   "secureHeaders": false,
   "segmentValuesLimit": 100,
   "server": {

src/lib/create-config.ts

@@ -460,6 +460,7 @@ export function createConfig(options: IUnleashOptions): IUnleashConfig {
 
     const clientFeatureCaching = loadClientCachingOptions(options);
 
+    const prometheusApi = options.prometheusApi || process.env.PROMETHEUS_API;
     return {
         db,
         session,
@@ -490,6 +491,7 @@ export function createConfig(options: IUnleashOptions): IUnleashConfig {
         strategySegmentsLimit,
         clientFeatureCaching,
         accessControlMaxAge,
+        prometheusApi,
     };
 }
 

src/lib/openapi/index.ts

@@ -85,6 +85,8 @@ import {
     publicSignupTokensSchema,
     publicSignupTokenUpdateSchema,
     resetPasswordSchema,
+    requestsPerSecondSchema,
+    requestsPerSecondSegmentedSchema,
     roleSchema,
     sdkContextSchema,
     searchEventsSchema,
@@ -214,6 +216,8 @@ export const schemas = {
     publicSignupTokensSchema,
     publicSignupTokenUpdateSchema,
     resetPasswordSchema,
+    requestsPerSecondSchema,
+    requestsPerSecondSegmentedSchema,
     roleSchema,
     sdkContextSchema,
     searchEventsSchema,

src/lib/openapi/spec/index.ts

@@ -119,3 +119,5 @@ export * from './feature-strategy-segment-schema';
 export * from './public-signup-token-create-schema';
 export * from './public-signup-token-update-schema';
 export * from './feature-environment-metrics-schema';
+export * from './requests-per-second-schema';
+export * from './requests-per-second-segmented-schema';

src/lib/openapi/spec/requests-per-second-schema.ts

@@ -0,0 +1,58 @@
+import { FromSchema } from 'json-schema-to-ts';
+
+export const requestsPerSecondSchema = {
+    $id: '#/components/schemas/requestsPerSecondSchema',
+    type: 'object',
+    properties: {
+        status: {
+            type: 'string',
+        },
+        data: {
+            type: 'object',
+            properties: {
+                resultType: {
+                    type: 'string',
+                },
+                result: {
+                    description:
+                        'An array of values per metric. Each one represents a line in the graph labeled by its metric name',
+                    type: 'array',
+                    items: {
+                        type: 'object',
+                        properties: {
+                            metric: {
+                                description:
+                                    'A key value set representing the metric',
+                                type: 'object',
+                                properties: {
+                                    appName: {
+                                        type: 'string',
+                                    },
+                                },
+                            },
+                            values: {
+                                description:
+                                    'An array of arrays. Each element of the array is an array of size 2 consisting of the 2 axis for the graph: in position zero the x axis represented as a number and position one the y axis represented as string',
+                                type: 'array',
+                                items: {
+                                    type: 'array',
+                                    items: {
+                                        anyOf: [
+                                            { type: 'string' },
+                                            { type: 'number' },
+                                        ],
+                                    },
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    },
+    components: {},
+} as const;
+
+export type RequestsPerSecondSchema = FromSchema<
+    typeof requestsPerSecondSchema
+>;

src/lib/openapi/spec/requests-per-second-segmented-schema.ts

@@ -0,0 +1,23 @@
+import { FromSchema } from 'json-schema-to-ts';
+import { requestsPerSecondSchema } from './requests-per-second-schema';
+export const requestsPerSecondSegmentedSchema = {
+    $id: '#/components/schemas/requestsPerSecondSegmentedSchema',
+    type: 'object',
+    properties: {
+        clientMetrics: {
+            $ref: '#/components/schemas/requestsPerSecondSchema',
+        },
+        adminMetrics: {
+            $ref: '#/components/schemas/requestsPerSecondSchema',
+        },
+    },
+    components: {
+        schemas: {
+            requestsPerSecondSchema,
+        },
+    },
+} as const;
+
+export type RequestsPerSecondSegmentedSchema = FromSchema<
+    typeof requestsPerSecondSegmentedSchema
+>;

src/lib/routes/admin-api/metrics.test.ts

@@ -12,12 +12,17 @@ async function getSetup() {
         preRouterHook: perms.hook,
     });
     const services = createServices(stores, config);
+    jest.spyOn(
+        services.clientInstanceService,
+        'getRPSForPath',
+    ).mockImplementation(async () => jest.fn());
     const app = await getApp(config, stores, services);
 
     return {
         request: supertest(app),
         stores,
         perms,
+        config,
         destroy: () => {
             services.versionService.destroy();
             services.clientInstanceService.destroy();
@@ -29,12 +34,14 @@ async function getSetup() {
 let stores;
 let request;
 let destroy;
+let config;
 
 beforeEach(async () => {
     const setup = await getSetup();
     stores = setup.stores;
     request = setup.request;
     destroy = setup.destroy;
+    config = setup.config;
 });
 
 afterEach(() => {
@@ -113,3 +120,20 @@ test('should delete application', () => {
         .delete(`/api/admin/metrics/applications/${appName}`)
         .expect(200);
 });
+
+test('/api/admin/metrics/rps with flag disabled', () => {
+    return request.get('/api/admin/metrics/rps').expect(404);
+});
+
+test('/api/admin/metrics/rps should return data with flag enabled', () => {
+    const mockedResponse = {};
+    config.experimental.flags.networkView = true;
+    expect(config.flagResolver.isEnabled('networkView')).toBeTruthy();
+    return request
+        .get('/api/admin/metrics/rps')
+        .expect(200)
+        .expect((res) => {
+            const metrics = res.body;
+            expect(metrics).toStrictEqual(mockedResponse);
+        });
+});

src/lib/routes/admin-api/metrics.ts

@@ -10,10 +10,15 @@ import { createResponseSchema } from '../../openapi/util/create-response-schema'
 import { ApplicationSchema } from '../../openapi/spec/application-schema';
 import { ApplicationsSchema } from '../../openapi/spec/applications-schema';
 import { emptyResponse } from '../../openapi/util/standard-responses';
+import { RequestsPerSecondSegmentedSchema } from 'lib/openapi/spec/requests-per-second-segmented-schema';
+import { IFlagResolver } from 'lib/types';
 
+type RpsError = string;
 class MetricsController extends Controller {
     private logger: Logger;
 
+    private flagResolver: IFlagResolver;
+
     private clientInstanceService: ClientInstanceService;
 
     constructor(
@@ -25,6 +30,7 @@ class MetricsController extends Controller {
     ) {
         super(config);
         this.logger = config.getLogger('/admin-api/metrics.ts');
+        this.flagResolver = config.flagResolver;
 
         this.clientInstanceService = clientInstanceService;
 
@@ -96,6 +102,24 @@ class MetricsController extends Controller {
                 }),
             ],
         });
+
+        this.route({
+            method: 'get',
+            path: '/rps',
+            handler: this.getRps,
+            permission: NONE,
+            middleware: [
+                openApiService.validPath({
+                    tags: ['Metrics'],
+                    operationId: 'getRequestsPerSecond',
+                    responses: {
+                        200: createResponseSchema(
+                            'requestsPerSecondSegmentedSchema',
+                        ),
+                    },
+                }),
+            ],
+        });
     }
 
     async deprecated(req: Request, res: Response): Promise<void> {
@@ -152,5 +176,34 @@ class MetricsController extends Controller {
         );
         res.json(appDetails);
     }
+
+    async getRps(
+        req: Request,
+        res: Response<RequestsPerSecondSegmentedSchema | RpsError>,
+    ): Promise<void> {
+        if (!this.flagResolver.isEnabled('networkView')) {
+            res.status(404).send('Not enabled');
+            return;
+        }
+        try {
+            const hoursToQuery = 6;
+            const [clientMetrics, adminMetrics] = await Promise.all([
+                this.clientInstanceService.getRPSForPath(
+                    '/api/client/.*',
+                    hoursToQuery,
+                ),
+                this.clientInstanceService.getRPSForPath(
+                    '/api/admin/.*',
+                    hoursToQuery,
+                ),
+            ]);
+            res.json({
+                clientMetrics,
+                adminMetrics,
+            });
+        } catch (e) {
+            res.status(500).send('Error fetching RPS metrics');
+        }
+    }
 }
 export default MetricsController;

src/lib/services/client-metrics/instance-service.test.ts

@@ -1,8 +1,8 @@
 import ClientInstanceService from './instance-service';
-import getLogger from '../../../test/fixtures/no-logger';
 import { IClientApp } from '../../types/model';
 import { secondsToMilliseconds } from 'date-fns';
 import FakeEventStore from '../../../test/fixtures/fake-event-store';
+import { createTestConfig } from '../../../test/config/test-config';
 
 /**
  * A utility to wait for any pending promises in the test subject code.
@@ -37,7 +37,10 @@ import FakeEventStore from '../../../test/fixtures/fake-event-store';
 function flushPromises() {
     return Promise.resolve(setImmediate);
 }
-
+let config;
+beforeAll(() => {
+    config = createTestConfig({});
+});
 test('Multiple registrations of same appname and instanceid within same time period should only cause one registration', async () => {
     jest.useFakeTimers();
     const appStoreSpy = jest.fn();
@@ -57,7 +60,7 @@ test('Multiple registrations of same appname and instanceid within same time per
             clientInstanceStore,
             eventStore: new FakeEventStore(),
         },
-        { getLogger },
+        config,
     );
     const client1: IClientApp = {
         appName: 'test_app',
@@ -107,7 +110,7 @@ test('Multiple unique clients causes multiple registrations', async () => {
             clientInstanceStore,
             eventStore: new FakeEventStore(),
         },
-        { getLogger },
+        config,
     );
     const client1 = {
         appName: 'test_app',
@@ -160,7 +163,7 @@ test('Same client registered outside of dedup interval will be registered twice'
             clientInstanceStore,
             eventStore: new FakeEventStore(),
         },
-        { getLogger },
+        config,
         bulkInterval,
     );
     const client1 = {
@@ -213,7 +216,7 @@ test('No registrations during a time period will not call stores', async () => {
             clientInstanceStore,
             eventStore: new FakeEventStore(),
         },
-        { getLogger },
+        config,
     );
     jest.advanceTimersByTime(6000);
     expect(appStoreSpy).toHaveBeenCalledTimes(0);

src/lib/services/client-metrics/instance-service.ts

@@ -2,7 +2,7 @@ import { applicationSchema } from './schema';
 import { APPLICATION_CREATED, CLIENT_REGISTER } from '../../types/events';
 import { IApplication } from './models';
 import { IUnleashStores } from '../../types/stores';
-import { IUnleashConfig } from '../../types/option';
+import { IServerOption, IUnleashConfig } from '../../types/option';
 import { IEventStore } from '../../types/stores/event-store';
 import {
     IClientApplication,
@@ -19,6 +19,7 @@ import { minutesToMilliseconds, secondsToMilliseconds } from 'date-fns';
 import { IClientMetricsStoreV2 } from '../../types/stores/client-metrics-store-v2';
 import { clientMetricsSchema } from './schema';
 import { PartialSome } from '../../types/partial';
+import fetch from 'make-fetch-happen';
 
 export default class ClientInstanceService {
     apps = {};
@@ -45,6 +46,10 @@ export default class ClientInstanceService {
 
     private announcementInterval: number;
 
+    private serverOption: IServerOption;
+
+    readonly prometheusApi;
+
     constructor(
         {
             clientMetricsStoreV2,
@@ -62,7 +67,11 @@ export default class ClientInstanceService {
             | 'clientInstanceStore'
             | 'eventStore'
         >,
-        { getLogger }: Pick<IUnleashConfig, 'getLogger'>,
+        {
+            getLogger,
+            prometheusApi,
+            server,
+        }: Pick<IUnleashConfig, 'getLogger' | 'prometheusApi' | 'server'>,
         bulkInterval = secondsToMilliseconds(5),
         announcementInterval = minutesToMilliseconds(5),
     ) {
@@ -72,7 +81,8 @@ export default class ClientInstanceService {
         this.clientApplicationsStore = clientApplicationsStore;
         this.clientInstanceStore = clientInstanceStore;
         this.eventStore = eventStore;
-
+        this.prometheusApi = prometheusApi;
+        this.serverOption = server;
         this.logger = getLogger(
             '/services/client-metrics/client-instance-service.ts',
         );
@@ -210,6 +220,34 @@ export default class ClientInstanceService {
         await this.clientApplicationsStore.upsert(applicationData);
     }
 
+    private toEpoch(d: Date) {
+        return (d.getTime() - d.getMilliseconds()) / 1000;
+    }
+
+    async getRPSForPath(path: string, hoursToQuery: number): Promise<any> {
+        const timeoutSeconds = 5;
+        const basePath = this.serverOption.baseUriPath;
+        const compositePath = `${basePath}/${path}`.replaceAll('//', '/');
+        const step = '5m'; // validate: I'm using the step both for step in query_range and for irate
+        const query = `sum by(appName) (irate (http_request_duration_milliseconds_count{path=~"${compositePath}"} [${step}]))`;
+        const end = new Date();
+        const start = new Date();
+        start.setHours(end.getHours() - hoursToQuery);
+
+        const params = `timeout=${timeoutSeconds}s&start=${this.toEpoch(
+            start,
+        )}&end=${this.toEpoch(end)}&step=${step}&query=${encodeURI(query)}`;
+        const url = `${this.prometheusApi}/api/v1/query_range?${params}`;
+        let metrics;
+        const response = await fetch(url);
+        if (response.ok) {
+            metrics = await response.json();
+        } else {
+            throw new Error(response.statusText);
+        }
+        return metrics;
+    }
+
     destroy(): void {
         this.timers.forEach(clearInterval);
     }

src/lib/types/option.ts

@@ -119,6 +119,7 @@ export interface IUnleashOptions {
     clientFeatureCaching?: Partial<IClientCachingOption>;
     flagResolver?: IFlagResolver;
     accessControlMaxAge?: number;
+    prometheusApi?: string;
 }
 
 export interface IEmailOption {
@@ -205,4 +206,5 @@ export interface IUnleashConfig {
     strategySegmentsLimit: number;
     clientFeatureCaching: IClientCachingOption;
     accessControlMaxAge: number;
+    prometheusApi?: string;
 }

src/server-dev.ts

@@ -43,6 +43,7 @@ process.nextTick(async () => {
                         toggleTagFiltering: true,
                         favorites: true,
                         variantsPerEnvironment: true,
+                        networkView: true,
                     },
                 },
                 authentication: {

src/test/e2e/api/openapi/__snapshots__/openapi.e2e.test.ts.snap

@@ -2703,6 +2703,68 @@ exports[`should serve the OpenAPI spec 1`] = `
         ],
         "type": "object",
       },
+      "requestsPerSecondSchema": {
+        "properties": {
+          "data": {
+            "properties": {
+              "result": {
+                "description": "An array of values per metric. Each one represents a line in the graph labeled by its metric name",
+                "items": {
+                  "properties": {
+                    "metric": {
+                      "description": "A key value set representing the metric",
+                      "properties": {
+                        "appName": {
+                          "type": "string",
+                        },
+                      },
+                      "type": "object",
+                    },
+                    "values": {
+                      "description": "An array of arrays. Each element of the array is an array of size 2 consisting of the 2 axis for the graph: in position zero the x axis represented as a number and position one the y axis represented as string",
+                      "items": {
+                        "items": {
+                          "anyOf": [
+                            {
+                              "type": "string",
+                            },
+                            {
+                              "type": "number",
+                            },
+                          ],
+                        },
+                        "type": "array",
+                      },
+                      "type": "array",
+                    },
+                  },
+                  "type": "object",
+                },
+                "type": "array",
+              },
+              "resultType": {
+                "type": "string",
+              },
+            },
+            "type": "object",
+          },
+          "status": {
+            "type": "string",
+          },
+        },
+        "type": "object",
+      },
+      "requestsPerSecondSegmentedSchema": {
+        "properties": {
+          "adminMetrics": {
+            "$ref": "#/components/schemas/requestsPerSecondSchema",
+          },
+          "clientMetrics": {
+            "$ref": "#/components/schemas/requestsPerSecondSchema",
+          },
+        },
+        "type": "object",
+      },
       "resetPasswordSchema": {
         "additionalProperties": false,
         "properties": {
@@ -5015,6 +5077,26 @@ If the provided project does not exist, the list of events will be empty.",
         ],
       },
     },
+    "/api/admin/metrics/rps": {
+      "get": {
+        "operationId": "getRequestsPerSecond",
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/requestsPerSecondSegmentedSchema",
+                },
+              },
+            },
+            "description": "requestsPerSecondSegmentedSchema",
+          },
+        },
+        "tags": [
+          "Metrics",
+        ],
+      },
+    },
     "/api/admin/playground": {
       "post": {
         "description": "Use the provided \`context\`, \`environment\`, and \`projects\` to evaluate toggles on this Unleash instance. Returns a list of all toggles that match the parameters and what they evaluate to. The response also contains the input parameters that were provided.",

src/test/e2e/services/client-metrics-service.e2e.test.ts

@@ -1,6 +1,8 @@
 import ClientInstanceService from '../../../lib/services/client-metrics/instance-service';
 import { IClientApp } from '../../../lib/types/model';
 import { secondsToMilliseconds } from 'date-fns';
+import { createTestConfig } from '../../config/test-config';
+import { IUnleashConfig } from '../../../lib/types';
 
 const faker = require('faker');
 const dbInit = require('../helpers/database-init');
@@ -10,17 +12,17 @@ const { APPLICATION_CREATED } = require('../../../lib/types/events');
 let stores;
 let db;
 let clientInstanceService;
-
+let config: IUnleashConfig;
 beforeAll(async () => {
     db = await dbInit('client_metrics_service_serial', getLogger);
     stores = db.stores;
-
+    config = createTestConfig({});
     const bulkInterval = secondsToMilliseconds(0.5);
     const announcementInterval = secondsToMilliseconds(2);
 
     clientInstanceService = new ClientInstanceService(
         stores,
-        { getLogger },
+        config,
         bulkInterval,
         announcementInterval,
     );