From 5fe238c8968091cc9342b1a827a08650c7319c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gast=C3=B3n=20Fournier?= Date: Mon, 12 Dec 2022 14:05:56 +0100 Subject: [PATCH] task: Expose prometheus metrics (#2586) ## About the changes This connects our backend with Prometheus (or compatible) metrics service, and exposes raw data (i.e. acting as a proxy) Co-authored-by: Christopher Kolstad --- .../__snapshots__/create-config.test.ts.snap | 1 + src/lib/create-config.ts | 2 + src/lib/openapi/index.ts | 4 + src/lib/openapi/spec/index.ts | 2 + .../spec/requests-per-second-schema.ts | 58 +++++++++++++ .../requests-per-second-segmented-schema.ts | 23 ++++++ src/lib/routes/admin-api/metrics.test.ts | 24 ++++++ src/lib/routes/admin-api/metrics.ts | 53 ++++++++++++ .../client-metrics/instance-service.test.ts | 15 ++-- .../client-metrics/instance-service.ts | 44 +++++++++- src/lib/types/option.ts | 2 + src/server-dev.ts | 1 + .../__snapshots__/openapi.e2e.test.ts.snap | 82 +++++++++++++++++++ .../client-metrics-service.e2e.test.ts | 8 +- 14 files changed, 307 insertions(+), 12 deletions(-) create mode 100644 src/lib/openapi/spec/requests-per-second-schema.ts create mode 100644 src/lib/openapi/spec/requests-per-second-segmented-schema.ts diff --git a/src/lib/__snapshots__/create-config.test.ts.snap b/src/lib/__snapshots__/create-config.test.ts.snap index 3b70a4c0fb..11f6788257 100644 --- a/src/lib/__snapshots__/create-config.test.ts.snap +++ b/src/lib/__snapshots__/create-config.test.ts.snap @@ -121,6 +121,7 @@ exports[`should create default config 1`] = ` }, "preHook": undefined, "preRouterHook": undefined, + "prometheusApi": undefined, "secureHeaders": false, "segmentValuesLimit": 100, "server": { diff --git a/src/lib/create-config.ts b/src/lib/create-config.ts index f891b6a00e..921a75b011 100644 --- a/src/lib/create-config.ts +++ b/src/lib/create-config.ts @@ -460,6 +460,7 @@ export function createConfig(options: IUnleashOptions): IUnleashConfig { const clientFeatureCaching = loadClientCachingOptions(options); + const prometheusApi = options.prometheusApi || process.env.PROMETHEUS_API; return { db, session, @@ -490,6 +491,7 @@ export function createConfig(options: IUnleashOptions): IUnleashConfig { strategySegmentsLimit, clientFeatureCaching, accessControlMaxAge, + prometheusApi, }; } diff --git a/src/lib/openapi/index.ts b/src/lib/openapi/index.ts index 3b1bdbc873..1529843cd5 100644 --- a/src/lib/openapi/index.ts +++ b/src/lib/openapi/index.ts @@ -85,6 +85,8 @@ import { publicSignupTokensSchema, publicSignupTokenUpdateSchema, resetPasswordSchema, + requestsPerSecondSchema, + requestsPerSecondSegmentedSchema, roleSchema, sdkContextSchema, searchEventsSchema, @@ -214,6 +216,8 @@ export const schemas = { publicSignupTokensSchema, publicSignupTokenUpdateSchema, resetPasswordSchema, + requestsPerSecondSchema, + requestsPerSecondSegmentedSchema, roleSchema, sdkContextSchema, searchEventsSchema, diff --git a/src/lib/openapi/spec/index.ts b/src/lib/openapi/spec/index.ts index 279bcffbb6..0c8e50e660 100644 --- a/src/lib/openapi/spec/index.ts +++ b/src/lib/openapi/spec/index.ts @@ -119,3 +119,5 @@ export * from './feature-strategy-segment-schema'; export * from './public-signup-token-create-schema'; export * from './public-signup-token-update-schema'; export * from './feature-environment-metrics-schema'; +export * from './requests-per-second-schema'; +export * from './requests-per-second-segmented-schema'; diff --git a/src/lib/openapi/spec/requests-per-second-schema.ts b/src/lib/openapi/spec/requests-per-second-schema.ts new file mode 100644 index 0000000000..0c54f263bb --- /dev/null +++ b/src/lib/openapi/spec/requests-per-second-schema.ts @@ -0,0 +1,58 @@ +import { FromSchema } from 'json-schema-to-ts'; + +export const requestsPerSecondSchema = { + $id: '#/components/schemas/requestsPerSecondSchema', + type: 'object', + properties: { + status: { + type: 'string', + }, + data: { + type: 'object', + properties: { + resultType: { + type: 'string', + }, + result: { + description: + 'An array of values per metric. Each one represents a line in the graph labeled by its metric name', + type: 'array', + items: { + type: 'object', + properties: { + metric: { + description: + 'A key value set representing the metric', + type: 'object', + properties: { + appName: { + type: 'string', + }, + }, + }, + values: { + description: + 'An array of arrays. Each element of the array is an array of size 2 consisting of the 2 axis for the graph: in position zero the x axis represented as a number and position one the y axis represented as string', + type: 'array', + items: { + type: 'array', + items: { + anyOf: [ + { type: 'string' }, + { type: 'number' }, + ], + }, + }, + }, + }, + }, + }, + }, + }, + }, + components: {}, +} as const; + +export type RequestsPerSecondSchema = FromSchema< + typeof requestsPerSecondSchema +>; diff --git a/src/lib/openapi/spec/requests-per-second-segmented-schema.ts b/src/lib/openapi/spec/requests-per-second-segmented-schema.ts new file mode 100644 index 0000000000..9767a2404d --- /dev/null +++ b/src/lib/openapi/spec/requests-per-second-segmented-schema.ts @@ -0,0 +1,23 @@ +import { FromSchema } from 'json-schema-to-ts'; +import { requestsPerSecondSchema } from './requests-per-second-schema'; +export const requestsPerSecondSegmentedSchema = { + $id: '#/components/schemas/requestsPerSecondSegmentedSchema', + type: 'object', + properties: { + clientMetrics: { + $ref: '#/components/schemas/requestsPerSecondSchema', + }, + adminMetrics: { + $ref: '#/components/schemas/requestsPerSecondSchema', + }, + }, + components: { + schemas: { + requestsPerSecondSchema, + }, + }, +} as const; + +export type RequestsPerSecondSegmentedSchema = FromSchema< + typeof requestsPerSecondSegmentedSchema +>; diff --git a/src/lib/routes/admin-api/metrics.test.ts b/src/lib/routes/admin-api/metrics.test.ts index 27063b6283..4a84027df3 100644 --- a/src/lib/routes/admin-api/metrics.test.ts +++ b/src/lib/routes/admin-api/metrics.test.ts @@ -12,12 +12,17 @@ async function getSetup() { preRouterHook: perms.hook, }); const services = createServices(stores, config); + jest.spyOn( + services.clientInstanceService, + 'getRPSForPath', + ).mockImplementation(async () => jest.fn()); const app = await getApp(config, stores, services); return { request: supertest(app), stores, perms, + config, destroy: () => { services.versionService.destroy(); services.clientInstanceService.destroy(); @@ -29,12 +34,14 @@ async function getSetup() { let stores; let request; let destroy; +let config; beforeEach(async () => { const setup = await getSetup(); stores = setup.stores; request = setup.request; destroy = setup.destroy; + config = setup.config; }); afterEach(() => { @@ -113,3 +120,20 @@ test('should delete application', () => { .delete(`/api/admin/metrics/applications/${appName}`) .expect(200); }); + +test('/api/admin/metrics/rps with flag disabled', () => { + return request.get('/api/admin/metrics/rps').expect(404); +}); + +test('/api/admin/metrics/rps should return data with flag enabled', () => { + const mockedResponse = {}; + config.experimental.flags.networkView = true; + expect(config.flagResolver.isEnabled('networkView')).toBeTruthy(); + return request + .get('/api/admin/metrics/rps') + .expect(200) + .expect((res) => { + const metrics = res.body; + expect(metrics).toStrictEqual(mockedResponse); + }); +}); diff --git a/src/lib/routes/admin-api/metrics.ts b/src/lib/routes/admin-api/metrics.ts index e6d44d1868..c3c2bddf8e 100644 --- a/src/lib/routes/admin-api/metrics.ts +++ b/src/lib/routes/admin-api/metrics.ts @@ -10,10 +10,15 @@ import { createResponseSchema } from '../../openapi/util/create-response-schema' import { ApplicationSchema } from '../../openapi/spec/application-schema'; import { ApplicationsSchema } from '../../openapi/spec/applications-schema'; import { emptyResponse } from '../../openapi/util/standard-responses'; +import { RequestsPerSecondSegmentedSchema } from 'lib/openapi/spec/requests-per-second-segmented-schema'; +import { IFlagResolver } from 'lib/types'; +type RpsError = string; class MetricsController extends Controller { private logger: Logger; + private flagResolver: IFlagResolver; + private clientInstanceService: ClientInstanceService; constructor( @@ -25,6 +30,7 @@ class MetricsController extends Controller { ) { super(config); this.logger = config.getLogger('/admin-api/metrics.ts'); + this.flagResolver = config.flagResolver; this.clientInstanceService = clientInstanceService; @@ -96,6 +102,24 @@ class MetricsController extends Controller { }), ], }); + + this.route({ + method: 'get', + path: '/rps', + handler: this.getRps, + permission: NONE, + middleware: [ + openApiService.validPath({ + tags: ['Metrics'], + operationId: 'getRequestsPerSecond', + responses: { + 200: createResponseSchema( + 'requestsPerSecondSegmentedSchema', + ), + }, + }), + ], + }); } async deprecated(req: Request, res: Response): Promise { @@ -152,5 +176,34 @@ class MetricsController extends Controller { ); res.json(appDetails); } + + async getRps( + req: Request, + res: Response, + ): Promise { + if (!this.flagResolver.isEnabled('networkView')) { + res.status(404).send('Not enabled'); + return; + } + try { + const hoursToQuery = 6; + const [clientMetrics, adminMetrics] = await Promise.all([ + this.clientInstanceService.getRPSForPath( + '/api/client/.*', + hoursToQuery, + ), + this.clientInstanceService.getRPSForPath( + '/api/admin/.*', + hoursToQuery, + ), + ]); + res.json({ + clientMetrics, + adminMetrics, + }); + } catch (e) { + res.status(500).send('Error fetching RPS metrics'); + } + } } export default MetricsController; diff --git a/src/lib/services/client-metrics/instance-service.test.ts b/src/lib/services/client-metrics/instance-service.test.ts index 860c8442b9..dc3b490780 100644 --- a/src/lib/services/client-metrics/instance-service.test.ts +++ b/src/lib/services/client-metrics/instance-service.test.ts @@ -1,8 +1,8 @@ import ClientInstanceService from './instance-service'; -import getLogger from '../../../test/fixtures/no-logger'; import { IClientApp } from '../../types/model'; import { secondsToMilliseconds } from 'date-fns'; import FakeEventStore from '../../../test/fixtures/fake-event-store'; +import { createTestConfig } from '../../../test/config/test-config'; /** * A utility to wait for any pending promises in the test subject code. @@ -37,7 +37,10 @@ import FakeEventStore from '../../../test/fixtures/fake-event-store'; function flushPromises() { return Promise.resolve(setImmediate); } - +let config; +beforeAll(() => { + config = createTestConfig({}); +}); test('Multiple registrations of same appname and instanceid within same time period should only cause one registration', async () => { jest.useFakeTimers(); const appStoreSpy = jest.fn(); @@ -57,7 +60,7 @@ test('Multiple registrations of same appname and instanceid within same time per clientInstanceStore, eventStore: new FakeEventStore(), }, - { getLogger }, + config, ); const client1: IClientApp = { appName: 'test_app', @@ -107,7 +110,7 @@ test('Multiple unique clients causes multiple registrations', async () => { clientInstanceStore, eventStore: new FakeEventStore(), }, - { getLogger }, + config, ); const client1 = { appName: 'test_app', @@ -160,7 +163,7 @@ test('Same client registered outside of dedup interval will be registered twice' clientInstanceStore, eventStore: new FakeEventStore(), }, - { getLogger }, + config, bulkInterval, ); const client1 = { @@ -213,7 +216,7 @@ test('No registrations during a time period will not call stores', async () => { clientInstanceStore, eventStore: new FakeEventStore(), }, - { getLogger }, + config, ); jest.advanceTimersByTime(6000); expect(appStoreSpy).toHaveBeenCalledTimes(0); diff --git a/src/lib/services/client-metrics/instance-service.ts b/src/lib/services/client-metrics/instance-service.ts index b973763891..5a324743f9 100644 --- a/src/lib/services/client-metrics/instance-service.ts +++ b/src/lib/services/client-metrics/instance-service.ts @@ -2,7 +2,7 @@ import { applicationSchema } from './schema'; import { APPLICATION_CREATED, CLIENT_REGISTER } from '../../types/events'; import { IApplication } from './models'; import { IUnleashStores } from '../../types/stores'; -import { IUnleashConfig } from '../../types/option'; +import { IServerOption, IUnleashConfig } from '../../types/option'; import { IEventStore } from '../../types/stores/event-store'; import { IClientApplication, @@ -19,6 +19,7 @@ import { minutesToMilliseconds, secondsToMilliseconds } from 'date-fns'; import { IClientMetricsStoreV2 } from '../../types/stores/client-metrics-store-v2'; import { clientMetricsSchema } from './schema'; import { PartialSome } from '../../types/partial'; +import fetch from 'make-fetch-happen'; export default class ClientInstanceService { apps = {}; @@ -45,6 +46,10 @@ export default class ClientInstanceService { private announcementInterval: number; + private serverOption: IServerOption; + + readonly prometheusApi; + constructor( { clientMetricsStoreV2, @@ -62,7 +67,11 @@ export default class ClientInstanceService { | 'clientInstanceStore' | 'eventStore' >, - { getLogger }: Pick, + { + getLogger, + prometheusApi, + server, + }: Pick, bulkInterval = secondsToMilliseconds(5), announcementInterval = minutesToMilliseconds(5), ) { @@ -72,7 +81,8 @@ export default class ClientInstanceService { this.clientApplicationsStore = clientApplicationsStore; this.clientInstanceStore = clientInstanceStore; this.eventStore = eventStore; - + this.prometheusApi = prometheusApi; + this.serverOption = server; this.logger = getLogger( '/services/client-metrics/client-instance-service.ts', ); @@ -210,6 +220,34 @@ export default class ClientInstanceService { await this.clientApplicationsStore.upsert(applicationData); } + private toEpoch(d: Date) { + return (d.getTime() - d.getMilliseconds()) / 1000; + } + + async getRPSForPath(path: string, hoursToQuery: number): Promise { + const timeoutSeconds = 5; + const basePath = this.serverOption.baseUriPath; + const compositePath = `${basePath}/${path}`.replaceAll('//', '/'); + const step = '5m'; // validate: I'm using the step both for step in query_range and for irate + const query = `sum by(appName) (irate (http_request_duration_milliseconds_count{path=~"${compositePath}"} [${step}]))`; + const end = new Date(); + const start = new Date(); + start.setHours(end.getHours() - hoursToQuery); + + const params = `timeout=${timeoutSeconds}s&start=${this.toEpoch( + start, + )}&end=${this.toEpoch(end)}&step=${step}&query=${encodeURI(query)}`; + const url = `${this.prometheusApi}/api/v1/query_range?${params}`; + let metrics; + const response = await fetch(url); + if (response.ok) { + metrics = await response.json(); + } else { + throw new Error(response.statusText); + } + return metrics; + } + destroy(): void { this.timers.forEach(clearInterval); } diff --git a/src/lib/types/option.ts b/src/lib/types/option.ts index 918286f4d3..9773e3541a 100644 --- a/src/lib/types/option.ts +++ b/src/lib/types/option.ts @@ -119,6 +119,7 @@ export interface IUnleashOptions { clientFeatureCaching?: Partial; flagResolver?: IFlagResolver; accessControlMaxAge?: number; + prometheusApi?: string; } export interface IEmailOption { @@ -205,4 +206,5 @@ export interface IUnleashConfig { strategySegmentsLimit: number; clientFeatureCaching: IClientCachingOption; accessControlMaxAge: number; + prometheusApi?: string; } diff --git a/src/server-dev.ts b/src/server-dev.ts index 0309571e2c..d3e394c78d 100644 --- a/src/server-dev.ts +++ b/src/server-dev.ts @@ -43,6 +43,7 @@ process.nextTick(async () => { toggleTagFiltering: true, favorites: true, variantsPerEnvironment: true, + networkView: true, }, }, authentication: { diff --git a/src/test/e2e/api/openapi/__snapshots__/openapi.e2e.test.ts.snap b/src/test/e2e/api/openapi/__snapshots__/openapi.e2e.test.ts.snap index ea382a685d..a5ce6781f8 100644 --- a/src/test/e2e/api/openapi/__snapshots__/openapi.e2e.test.ts.snap +++ b/src/test/e2e/api/openapi/__snapshots__/openapi.e2e.test.ts.snap @@ -2703,6 +2703,68 @@ exports[`should serve the OpenAPI spec 1`] = ` ], "type": "object", }, + "requestsPerSecondSchema": { + "properties": { + "data": { + "properties": { + "result": { + "description": "An array of values per metric. Each one represents a line in the graph labeled by its metric name", + "items": { + "properties": { + "metric": { + "description": "A key value set representing the metric", + "properties": { + "appName": { + "type": "string", + }, + }, + "type": "object", + }, + "values": { + "description": "An array of arrays. Each element of the array is an array of size 2 consisting of the 2 axis for the graph: in position zero the x axis represented as a number and position one the y axis represented as string", + "items": { + "items": { + "anyOf": [ + { + "type": "string", + }, + { + "type": "number", + }, + ], + }, + "type": "array", + }, + "type": "array", + }, + }, + "type": "object", + }, + "type": "array", + }, + "resultType": { + "type": "string", + }, + }, + "type": "object", + }, + "status": { + "type": "string", + }, + }, + "type": "object", + }, + "requestsPerSecondSegmentedSchema": { + "properties": { + "adminMetrics": { + "$ref": "#/components/schemas/requestsPerSecondSchema", + }, + "clientMetrics": { + "$ref": "#/components/schemas/requestsPerSecondSchema", + }, + }, + "type": "object", + }, "resetPasswordSchema": { "additionalProperties": false, "properties": { @@ -5015,6 +5077,26 @@ If the provided project does not exist, the list of events will be empty.", ], }, }, + "/api/admin/metrics/rps": { + "get": { + "operationId": "getRequestsPerSecond", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/requestsPerSecondSegmentedSchema", + }, + }, + }, + "description": "requestsPerSecondSegmentedSchema", + }, + }, + "tags": [ + "Metrics", + ], + }, + }, "/api/admin/playground": { "post": { "description": "Use the provided \`context\`, \`environment\`, and \`projects\` to evaluate toggles on this Unleash instance. Returns a list of all toggles that match the parameters and what they evaluate to. The response also contains the input parameters that were provided.", diff --git a/src/test/e2e/services/client-metrics-service.e2e.test.ts b/src/test/e2e/services/client-metrics-service.e2e.test.ts index 8be77210cb..732e99b14c 100644 --- a/src/test/e2e/services/client-metrics-service.e2e.test.ts +++ b/src/test/e2e/services/client-metrics-service.e2e.test.ts @@ -1,6 +1,8 @@ import ClientInstanceService from '../../../lib/services/client-metrics/instance-service'; import { IClientApp } from '../../../lib/types/model'; import { secondsToMilliseconds } from 'date-fns'; +import { createTestConfig } from '../../config/test-config'; +import { IUnleashConfig } from '../../../lib/types'; const faker = require('faker'); const dbInit = require('../helpers/database-init'); @@ -10,17 +12,17 @@ const { APPLICATION_CREATED } = require('../../../lib/types/events'); let stores; let db; let clientInstanceService; - +let config: IUnleashConfig; beforeAll(async () => { db = await dbInit('client_metrics_service_serial', getLogger); stores = db.stores; - + config = createTestConfig({}); const bulkInterval = secondsToMilliseconds(0.5); const announcementInterval = secondsToMilliseconds(2); clientInstanceService = new ClientInstanceService( stores, - { getLogger }, + config, bulkInterval, announcementInterval, );