feat: custom metrics poc (#10018)

Now we can receive custom metrics, return those for UI and have extra
prometheus endpoint for it.

---------

Co-authored-by: Christopher Kolstad <chriswk@getunleash.io>

src/lib/features/metrics/custom/custom-metrics-controller.ts

@@ -0,0 +1,108 @@
+import type { Response } from 'express';
+import Controller from '../../../routes/controller.js';
+import type { IUnleashConfig } from '../../../types/index.js';
+import type { Logger } from '../../../logger.js';
+import type { IAuthRequest } from '../../../routes/unleash-types.js';
+import { NONE } from '../../../types/permissions.js';
+import type {
+    IUnleashServices,
+    OpenApiService,
+} from '../../../services/index.js';
+import { emptyResponse } from '../../../openapi/util/standard-responses.js';
+import type { CustomMetricsService } from './custom-metrics-service.js';
+
+export default class CustomMetricsController extends Controller {
+    logger: Logger;
+    openApiService: OpenApiService;
+    customMetricsService: CustomMetricsService;
+
+    constructor(
+        {
+            customMetricsService,
+            openApiService,
+        }: Pick<IUnleashServices, 'customMetricsService' | 'openApiService'>,
+        config: IUnleashConfig,
+    ) {
+        super(config);
+        const { getLogger } = config;
+
+        this.logger = getLogger('/admin-api/custom-metrics');
+        this.openApiService = openApiService;
+        this.customMetricsService = customMetricsService;
+
+        this.route({
+            method: 'get',
+            path: '',
+            handler: this.getCustomMetrics,
+            permission: NONE,
+            middleware: [
+                this.openApiService.validPath({
+                    tags: ['Metrics'],
+                    summary: 'Get stored custom metrics',
+                    description: `Retrieves the stored custom metrics data.`,
+                    operationId: 'getCustomMetrics',
+                    responses: {
+                        200: emptyResponse,
+                    },
+                }),
+            ],
+        });
+
+        this.route({
+            method: 'get',
+            path: '/prometheus',
+            handler: this.getPrometheusMetrics,
+            permission: NONE,
+            middleware: [
+                this.openApiService.validPath({
+                    tags: ['Metrics'],
+                    summary: 'Get metrics in Prometheus format',
+                    description: `Exposes all custom metrics in Prometheus text format for scraping.`,
+                    operationId: 'getPrometheusMetrics',
+                    responses: {
+                        200: {
+                            description: 'Prometheus formatted metrics',
+                            content: {
+                                'text/plain': {
+                                    schema: {
+                                        type: 'string',
+                                    },
+                                },
+                            },
+                        },
+                    },
+                }),
+            ],
+        });
+    }
+
+    async getCustomMetrics(req: IAuthRequest, res: Response): Promise<void> {
+        try {
+            const allMetrics = this.customMetricsService.getMetrics();
+
+            res.json({
+                metrics: allMetrics,
+                count: allMetrics.length,
+                metricNames: this.customMetricsService.getMetricNames(),
+            });
+        } catch (e) {
+            this.logger.error('Error retrieving custom metrics', e);
+            res.status(500).end();
+        }
+    }
+
+    async getPrometheusMetrics(
+        req: IAuthRequest,
+        res: Response,
+    ): Promise<void> {
+        try {
+            const output = this.customMetricsService.getPrometheusMetrics();
+
+            res.set('Content-Type', 'text/plain');
+            res.send(output);
+        } catch (e) {
+            this.logger.error('Error generating Prometheus metrics', e);
+            res.status(500).end();
+        }
+    }
+}

src/lib/features/metrics/custom/custom-metrics-service.ts

@@ -0,0 +1,47 @@
+import type { Logger } from '../../../logger.js';
+import type { IUnleashConfig } from '../../../types/index.js';
+import {
+    CustomMetricsStore,
+    type ICustomMetricsStore,
+    type StoredCustomMetric,
+} from './custom-metrics-store.js';
+
+export class CustomMetricsService {
+    private logger: Logger;
+    private store: ICustomMetricsStore;
+
+    constructor(config: IUnleashConfig) {
+        this.logger = config.getLogger('custom-metrics-service');
+        this.store = new CustomMetricsStore(config);
+    }
+
+    addMetric(metric: Omit<StoredCustomMetric, 'timestamp'>): void {
+        this.store.addMetric(metric);
+    }
+
+    addMetrics(metrics: Omit<StoredCustomMetric, 'timestamp'>[]): void {
+        this.store.addMetrics(metrics);
+    }
+
+    getMetrics(): StoredCustomMetric[] {
+        return this.store.getMetrics();
+    }
+
+    getMetricNames(): string[] {
+        return this.store.getMetricNames();
+    }
+
+    getPrometheusMetrics(): string {
+        return this.store.getPrometheusMetrics();
+    }
+
+    clearMetricsForTesting(): void {
+        if (this.store instanceof CustomMetricsStore) {
+            (this.store as any).customMetricsStore = new Map();
+        } else {
+            this.logger.warn(
+                'Cannot clear metrics - store is not an instance of CustomMetricsStore',
+            );
+        }
+    }
+}

src/lib/features/metrics/custom/custom-metrics-store.ts

@@ -0,0 +1,168 @@
+import type { Logger } from '../../../logger.js';
+import type { IUnleashConfig } from '../../../types/index.js';
+
+export interface StoredCustomMetric {
+    name: string;
+    value: number;
+    labels?: Record<string, string>;
+    timestamp: Date;
+}
+
+export interface ICustomMetricsStore {
+    addMetric(metric: Omit<StoredCustomMetric, 'timestamp'>): void;
+    addMetrics(metrics: Omit<StoredCustomMetric, 'timestamp'>[]): void;
+    getMetrics(): StoredCustomMetric[];
+    getMetricsByName(name: string): StoredCustomMetric[];
+    getMetricNames(): string[];
+    getPrometheusMetrics(): string;
+}
+
+export class CustomMetricsStore implements ICustomMetricsStore {
+    private logger: Logger;
+    private customMetricsStore: Map<string, StoredCustomMetric> = new Map();
+
+    constructor(config: IUnleashConfig) {
+        this.logger = config.getLogger('custom-metrics-store');
+    }
+
+    private roundToMinute(date: Date): Date {
+        const rounded = new Date(date);
+        rounded.setSeconds(0);
+        rounded.setMilliseconds(0);
+        return rounded;
+    }
+
+    private getMetricKey(
+        metric: Omit<StoredCustomMetric, 'timestamp'>,
+        timestamp: Date,
+    ): string {
+        const roundedTimestamp = this.roundToMinute(timestamp);
+        const timeKey = roundedTimestamp.toISOString();
+
+        let key = `${metric.name}:${timeKey}`;
+
+        if (metric.labels && Object.keys(metric.labels).length > 0) {
+            const labelEntries = Object.entries(metric.labels).sort(
+                ([keyA], [keyB]) => keyA.localeCompare(keyB),
+            );
+
+            const labelString = labelEntries
+                .map(([key, value]) => `${key}=${value}`)
+                .join(',');
+
+            key += `:${labelString}`;
+        }
+
+        return key;
+    }
+
+    addMetric(metric: Omit<StoredCustomMetric, 'timestamp'>): void {
+        const now = new Date();
+        const roundedTimestamp = this.roundToMinute(now);
+        const metricKey = this.getMetricKey(metric, now);
+
+        const storedMetric: StoredCustomMetric = {
+            ...metric,
+            timestamp: roundedTimestamp,
+        };
+
+        this.customMetricsStore.set(metricKey, storedMetric);
+    }
+
+    addMetrics(metrics: Omit<StoredCustomMetric, 'timestamp'>[]): void {
+        let storedCount = 0;
+        metrics.forEach((metric) => {
+            this.addMetric(metric);
+            storedCount++;
+        });
+        this.logger.debug(`Stored ${storedCount} custom metrics`);
+    }
+
+    getMetrics(): StoredCustomMetric[] {
+        return Array.from(this.customMetricsStore.values());
+    }
+
+    getMetricsByName(name: string): StoredCustomMetric[] {
+        return Array.from(this.customMetricsStore.values()).filter(
+            (metric) => metric.name === name,
+        );
+    }
+
+    getMetricNames(): string[] {
+        const names = new Set<string>();
+        for (const metric of this.customMetricsStore.values()) {
+            names.add(metric.name);
+        }
+        return Array.from(names);
+    }
+
+    getPrometheusMetrics(): string {
+        let output = '';
+        const metricsByName = new Map<
+            string,
+            Map<string, StoredCustomMetric>
+        >();
+
+        for (const metric of this.customMetricsStore.values()) {
+            if (!metricsByName.has(metric.name)) {
+                metricsByName.set(
+                    metric.name,
+                    new Map<string, StoredCustomMetric>(),
+                );
+            }
+
+            let labelKey = '';
+            if (metric.labels && Object.keys(metric.labels).length > 0) {
+                const labelEntries = Object.entries(metric.labels).sort(
+                    ([keyA], [keyB]) => keyA.localeCompare(keyB),
+                );
+
+                labelKey = labelEntries
+                    .map(([key, value]) => `${key}=${value}`)
+                    .join(',');
+            }
+
+            const metricsForName = metricsByName.get(metric.name)!;
+
+            if (
+                !metricsForName.has(labelKey) ||
+                metricsForName.get(labelKey)!.timestamp < metric.timestamp
+            ) {
+                metricsForName.set(labelKey, metric);
+            }
+        }
+
+        for (const [metricName, metricsMap] of metricsByName.entries()) {
+            if (metricsMap.size === 0) continue;
+
+            output += `# HELP ${metricName} Custom metric reported to Unleash\n`;
+            output += `# TYPE ${metricName} counter\n`;
+
+            for (const metric of metricsMap.values()) {
+                let labelStr = '';
+                if (metric.labels && Object.keys(metric.labels).length > 0) {
+                    const labelParts = Object.entries(metric.labels)
+                        .map(
+                            ([key, value]) =>
+                                `${key}="${this.escapePrometheusString(value)}"`,
+                        )
+                        .join(',');
+                    labelStr = `{${labelParts}}`;
+                }
+
+                output += `${metricName}${labelStr} ${metric.value}\n`;
+            }
+
+            output += '\n';
+        }
+
+        return output;
+    }
+
+    private escapePrometheusString(str: string): string {
+        return str
+            .replace(/\\/g, '\\\\')
+            .replace(/"/g, '\\"')
+            .replace(/\n/g, '\\n');
+    }
+}

src/lib/features/metrics/custom/custom-metrics.e2e.test.ts

@@ -0,0 +1,243 @@
+import {
+    type IUnleashTest,
+    setupAppWithCustomConfig,
+} from '../../../../test/e2e/helpers/test-helper.js';
+import dbInit, {
+    type ITestDb,
+} from '../../../../test/e2e/helpers/database-init.js';
+import getLogger from '../../../../test/fixtures/no-logger.js';
+import type { CustomMetricsService } from './custom-metrics-service.js';
+import type { StoredCustomMetric } from './custom-metrics-store.js';
+
+let app: IUnleashTest;
+let db: ITestDb;
+
+beforeAll(async () => {
+    db = await dbInit('metrics_api_admin_custom', getLogger);
+    app = await setupAppWithCustomConfig(db.stores, {
+        experimental: {
+            flags: {
+                responseTimeMetricsFix: true,
+            },
+        },
+    });
+});
+
+afterEach(async () => {
+    const service = app.services.customMetricsService as CustomMetricsService;
+    service.clearMetricsForTesting();
+});
+
+afterAll(async () => {
+    await app.destroy();
+    await db.destroy();
+});
+
+test('should store custom metrics in memory and be able to retrieve them', async () => {
+    const customMetricsExample = {
+        metrics: [
+            {
+                name: 'test_metric',
+                value: 42,
+                labels: {
+                    env: 'test',
+                    component: 'api',
+                },
+            },
+        ],
+    };
+
+    await app.request
+        .post('/api/client/metrics/custom')
+        .send(customMetricsExample)
+        .expect(202);
+
+    const response = await app.request
+        .get('/api/admin/custom-metrics')
+        .expect(200);
+
+    expect(response.body).toHaveProperty('metrics');
+    expect(response.body).toHaveProperty('count');
+    expect(response.body).toHaveProperty('metricNames');
+    expect(response.body.count).toBeGreaterThan(0);
+    expect(response.body.metricNames).toContain('test_metric');
+
+    const metrics = response.body.metrics;
+    const found = metrics.some(
+        (metric) =>
+            metric.name === 'test_metric' &&
+            metric.value === 42 &&
+            metric.labels &&
+            metric.labels.env === 'test' &&
+            metric.labels.component === 'api',
+    );
+
+    expect(found).toBe(true);
+});
+
+test('should expose metrics in Prometheus format', async () => {
+    await app.request
+        .post('/api/client/metrics/custom')
+        .send({
+            metrics: [
+                {
+                    name: 'api_requests_total',
+                    value: 10,
+                    labels: {
+                        status: '200',
+                        endpoint: '/api/test',
+                    },
+                },
+                {
+                    name: 'api_requests_total',
+                    value: 5,
+                    labels: {
+                        status: '404',
+                        endpoint: '/api/missing',
+                    },
+                },
+                {
+                    name: 'memory_usage',
+                    value: 1024,
+                    labels: {
+                        application: 'unleash',
+                    },
+                },
+            ],
+        })
+        .expect(202);
+
+    const response = await app.request
+        .get('/api/admin/custom-metrics/prometheus')
+        .expect(200);
+
+    expect(response.headers['content-type']).toContain('text/plain');
+
+    const metricsText = response.text;
+
+    expect(metricsText).toContain('# HELP api_requests_total');
+    expect(metricsText).toContain('# TYPE api_requests_total counter');
+    expect(metricsText).toContain('# HELP memory_usage');
+    expect(metricsText).toContain('# TYPE memory_usage counter');
+
+    expect(metricsText).toMatch(
+        /api_requests_total{status="200",endpoint="\/api\/test"} 10/,
+    );
+    expect(metricsText).toMatch(
+        /api_requests_total{status="404",endpoint="\/api\/missing"} 5/,
+    );
+    expect(metricsText).toMatch(/memory_usage{application="unleash"} 1024/);
+});
+
+test('should deduplicate metrics, round timestamps, and preserve different labels', async () => {
+    await app.request
+        .post('/api/client/metrics/custom')
+        .send({
+            metrics: [
+                {
+                    name: 'test_counter',
+                    value: 1,
+                    labels: {
+                        instance: 'server1',
+                    },
+                },
+                {
+                    name: 'test_counter',
+                    value: 5,
+                    labels: {
+                        instance: 'server2',
+                    },
+                },
+                {
+                    name: 'memory_usage',
+                    value: 100,
+                    labels: {
+                        server: 'main',
+                    },
+                },
+            ],
+        })
+        .expect(202);
+
+    await app.request
+        .post('/api/client/metrics/custom')
+        .send({
+            metrics: [
+                {
+                    name: 'test_counter',
+                    value: 2,
+                    labels: {
+                        instance: 'server1',
+                    },
+                },
+                {
+                    name: 'memory_usage',
+                    value: 200,
+                    labels: {
+                        server: 'main',
+                    },
+                },
+                {
+                    name: 'memory_usage',
+                    value: 150,
+                    labels: {
+                        server: 'backup',
+                    },
+                },
+            ],
+        })
+        .expect(202);
+
+    const response = await app.request
+        .get('/api/admin/custom-metrics')
+        .expect(200);
+
+    expect(response.body).toHaveProperty('metrics');
+    expect(response.body).toHaveProperty('count');
+    expect(response.body).toHaveProperty('metricNames');
+
+    const metrics = response.body.metrics as StoredCustomMetric[];
+
+    expect(response.body.count).toBe(4);
+
+    const testCounterServer1 = metrics.find(
+        (m) => m.name === 'test_counter' && m.labels?.instance === 'server1',
+    );
+    expect(testCounterServer1).toBeDefined();
+    expect(testCounterServer1?.value).toBe(2);
+
+    const testCounterServer2 = metrics.find(
+        (m) => m.name === 'test_counter' && m.labels?.instance === 'server2',
+    );
+    expect(testCounterServer2).toBeDefined();
+    expect(testCounterServer2?.value).toBe(5);
+
+    const memoryUsageMain = metrics.find(
+        (m) => m.name === 'memory_usage' && m.labels?.server === 'main',
+    );
+    expect(memoryUsageMain).toBeDefined();
+    expect(memoryUsageMain?.value).toBe(200);
+
+    const memoryUsageBackup = metrics.find(
+        (m) => m.name === 'memory_usage' && m.labels?.server === 'backup',
+    );
+    expect(memoryUsageBackup).toBeDefined();
+    expect(memoryUsageBackup?.value).toBe(150);
+
+    metrics.forEach((metric) => {
+        const date = new Date(metric.timestamp);
+        expect(date.getSeconds()).toBe(0);
+        expect(date.getMilliseconds()).toBe(0);
+    });
+
+    const prometheusResponse = await app.request
+        .get('/api/admin/custom-metrics/prometheus')
+        .expect(200);
+
+    const prometheusOutput = prometheusResponse.text;
+
+    expect(prometheusOutput).toMatch(/test_counter{instance="server1"} 2/);
+    expect(prometheusOutput).toMatch(/test_counter{instance="server2"} 5/);
+    expect(prometheusOutput).toMatch(/memory_usage{server="main"} 200/);
+    expect(prometheusOutput).toMatch(/memory_usage{server="backup"} 150/);
+});

src/lib/features/metrics/instance/metrics.ts

@@ -18,9 +18,15 @@ import {
 import rateLimit from 'express-rate-limit';
 import { minutesToMilliseconds } from 'date-fns';
 import type { BulkMetricsSchema } from '../../../openapi/spec/bulk-metrics-schema.js';
-import { clientMetricsEnvBulkSchema } from '../shared/schema.js';
+import {
+    clientMetricsEnvBulkSchema,
+    customMetricsSchema,
+} from '../shared/schema.js';
 import type { IClientMetricsEnv } from '../client-metrics/client-metrics-store-v2-type.js';
 import { CLIENT_METRICS } from '../../../events/index.js';
+import type { CustomMetricsSchema } from '../../../openapi/spec/custom-metrics-schema.js';
+import type { StoredCustomMetric } from '../custom/custom-metrics-store.js';
+import type { CustomMetricsService } from '../custom/custom-metrics-service.js';
 
 export default class ClientMetricsController extends Controller {
     logger: Logger;
@@ -31,6 +37,8 @@ export default class ClientMetricsController extends Controller {
 
     metricsV2: ClientMetricsServiceV2;
 
+    customMetricsService: CustomMetricsService;
+
     flagResolver: IFlagResolver;
 
     constructor(
@@ -38,11 +46,13 @@ export default class ClientMetricsController extends Controller {
             clientInstanceService,
             clientMetricsServiceV2,
             openApiService,
+            customMetricsService,
         }: Pick<
             IUnleashServices,
             | 'clientInstanceService'
             | 'clientMetricsServiceV2'
             | 'openApiService'
+            | 'customMetricsService'
         >,
         config: IUnleashConfig,
     ) {
@@ -53,6 +63,7 @@ export default class ClientMetricsController extends Controller {
         this.clientInstanceService = clientInstanceService;
         this.openApiService = openApiService;
         this.metricsV2 = clientMetricsServiceV2;
+        this.customMetricsService = customMetricsService;
         this.flagResolver = config.flagResolver;
 
         this.route({
@@ -102,6 +113,35 @@ export default class ClientMetricsController extends Controller {
                 }),
             ],
         });
+
+        this.route({
+            method: 'post',
+            path: '/custom',
+            handler: this.customMetrics,
+            permission: NONE,
+            middleware: [
+                this.openApiService.validPath({
+                    tags: ['Client'],
+                    summary: 'Send custom metrics',
+                    description: `This operation accepts custom metrics from clients. These metrics will be exposed via Prometheus in Unleash.`,
+                    operationId: 'clientCustomMetrics',
+                    requestBody: createRequestSchema('customMetricsSchema'),
+                    responses: {
+                        202: emptyResponse,
+                        ...getStandardResponses(400),
+                    },
+                }),
+                rateLimit({
+                    windowMs: minutesToMilliseconds(1),
+                    max: config.metricsRateLimiting.clientMetricsMaxPerMinute,
+                    validate: false,
+                    standardHeaders: true,
+                    legacyHeaders: false,
+                }),
+            ],
+        });
+
+        // Note: Custom metrics GET endpoints are now handled by the admin API
     }
 
     async registerMetrics(req: IAuthRequest, res: Response): Promise<void> {
@@ -130,6 +170,48 @@ export default class ClientMetricsController extends Controller {
         }
     }
 
+    async customMetrics(
+        req: IAuthRequest<void, void, CustomMetricsSchema>,
+        res: Response<void>,
+    ): Promise<void> {
+        if (this.config.flagResolver.isEnabled('disableMetrics')) {
+            res.status(204).end();
+        } else {
+            try {
+                const { body } = req;
+
+                console.log(body);
+
+                // Use Joi validation for custom metrics
+                await customMetricsSchema.validateAsync(body);
+
+                // Process and store custom metrics
+                if (body.metrics && Array.isArray(body.metrics)) {
+                    const validMetrics = body.metrics.filter(
+                        (metric) =>
+                            typeof metric.name === 'string' &&
+                            typeof metric.value === 'number',
+                    );
+
+                    if (validMetrics.length < body.metrics.length) {
+                        this.logger.warn(
+                            'Some invalid metric types found, skipping',
+                        );
+                    }
+
+                    this.customMetricsService.addMetrics(
+                        validMetrics as Omit<StoredCustomMetric, 'timestamp'>[],
+                    );
+                }
+
+                res.status(202).end();
+            } catch (e) {
+                this.logger.error('Failed to process custom metrics', e);
+                res.status(400).end();
+            }
+        }
+    }
+
     async bulkMetrics(
         req: IAuthRequest<void, void, BulkMetricsSchema>,
         res: Response<void>,

src/lib/features/metrics/shared/schema.ts

@@ -69,6 +69,22 @@ export const applicationSchema = joi
         announced: joi.boolean().optional().default(false),
     });
 
+export const customMetricSchema = joi
+    .object()
+    .options({ stripUnknown: true })
+    .keys({
+        name: joi.string().required(),
+        value: joi.number().required(),
+        labels: joi.object().pattern(joi.string(), joi.string()).optional(),
+    });
+
+export const customMetricsSchema = joi
+    .object()
+    .options({ stripUnknown: true })
+    .keys({
+        metrics: joi.array().items(customMetricSchema).required(),
+    });
+
 export const batchMetricsSchema = joi
     .object()
     .options({ stripUnknown: true })

src/lib/openapi/spec/custom-metric-schema.ts

@@ -0,0 +1,34 @@
+import type { FromSchema } from 'json-schema-to-ts';
+
+export const customMetricSchema = {
+    $id: '#/components/schemas/customMetricSchema',
+    type: 'object' as const,
+    required: ['name', 'value'],
+    description: 'A custom metric with name, value and optional labels',
+    properties: {
+        name: {
+            type: 'string' as const,
+            description: 'Name of the custom metric',
+            example: 'http_responses_total',
+        },
+        value: {
+            type: 'number' as const,
+            description: 'Value of the custom metric',
+            example: 1,
+        },
+        labels: {
+            type: 'object' as const,
+            description: 'Labels to categorize the metric',
+            additionalProperties: {
+                type: 'string' as const,
+            },
+            example: {
+                status: '200',
+                method: 'GET',
+            },
+        },
+    },
+    components: {},
+};
+
+export type CustomMetricSchema = FromSchema<typeof customMetricSchema>;

src/lib/openapi/spec/custom-metrics-schema.ts

@@ -0,0 +1,27 @@
+import type { FromSchema } from 'json-schema-to-ts';
+import { dateSchema } from './date-schema.js';
+import { customMetricSchema } from './custom-metric-schema.js';
+
+export const customMetricsSchema = {
+    $id: '#/components/schemas/customMetricsSchema',
+    type: 'object' as const,
+    required: ['metrics'],
+    description: 'A collection of custom metrics',
+    properties: {
+        metrics: {
+            type: 'array' as const,
+            description: 'Array of custom metrics',
+            items: {
+                $ref: '#/components/schemas/customMetricSchema',
+            },
+        },
+    },
+    components: {
+        schemas: {
+            customMetricSchema,
+            dateSchema,
+        },
+    },
+} as const;
+
+export type CustomMetricsSchema = FromSchema<typeof customMetricsSchema>;

src/lib/openapi/spec/index.ts

@@ -58,6 +58,8 @@ export * from './create-strategy-variant-schema.js';
 export * from './create-tag-schema.js';
 export * from './create-user-response-schema.js';
 export * from './create-user-schema.js';
+export * from './custom-metric-schema.js';
+export * from './custom-metrics-schema.js';
 export * from './date-schema.js';
 export * from './dependencies-exist-schema.js';
 export * from './dependent-feature-schema.js';

src/lib/routes/admin-api/index.ts

@@ -37,6 +37,7 @@ import { SearchApi } from './search/index.js';
 import PersonalDashboardController from '../../features/personal-dashboard/personal-dashboard-controller.js';
 import FeatureLifecycleCountController from '../../features/feature-lifecycle/feature-lifecycle-count-controller.js';
 import type { IUnleashServices } from '../../services/index.js';
+import CustomMetricsController from '../../features/metrics/custom/custom-metrics-controller.js';
 
 export class AdminApi extends Controller {
     constructor(
@@ -77,6 +78,10 @@ export class AdminApi extends Controller {
             '/client-metrics',
             new ClientMetricsController(config, services).router,
         );
+        this.app.use(
+            '/custom-metrics',
+            new CustomMetricsController(services, config).router,
+        );
         this.app.use('/user', new UserController(config, services).router);
         this.app.use(
             '/user/tokens',

src/lib/routes/backstage.ts

@@ -4,19 +4,42 @@ import { join } from 'path';
 import { register as prometheusRegister } from 'prom-client';
 import Controller from './controller.js';
 import type { IUnleashConfig } from '../types/option.js';
+import type { IFlagResolver } from '../types/index.js';
+import type { CustomMetricsService } from '../features/metrics/custom/custom-metrics-service.js';
+import type { IUnleashServices } from '../services/index.js';
 
 class BackstageController extends Controller {
     logger: any;
+    private flagResolver: IFlagResolver;
+    private customMetricsService: CustomMetricsService;
 
-    constructor(config: IUnleashConfig) {
+    constructor(
+        config: IUnleashConfig,
+        {
+            customMetricsService,
+        }: Pick<IUnleashServices, 'customMetricsService'>,
+    ) {
         super(config);
 
         this.logger = config.getLogger('backstage.js');
+        this.flagResolver = config.flagResolver;
+        this.customMetricsService = customMetricsService;
 
         if (config.server.serverMetrics) {
             this.get('/prometheus', async (req, res) => {
                 res.set('Content-Type', prometheusRegister.contentType);
-                res.end(await prometheusRegister.metrics());
+
+                let metricsOutput = await prometheusRegister.metrics();
+
+                if (this.flagResolver.isEnabled('customMetrics')) {
+                    const customMetrics =
+                        this.customMetricsService.getPrometheusMetrics();
+                    if (customMetrics) {
+                        metricsOutput = `${metricsOutput}\n${customMetrics}`;
+                    }
+                }
+
+                res.end(metricsOutput);
             });
         }
 

src/lib/routes/index.ts

@@ -30,7 +30,10 @@ class IndexRouter extends Controller {
             '/invite',
             new PublicInviteController(config, services).router,
         );
-        this.use('/internal-backstage', new BackstageController(config).router);
+        this.use(
+            '/internal-backstage',
+            new BackstageController(config, services).router,
+        );
         this.use('/logout', new LogoutController(config, services).router);
         this.useWithMiddleware(
             '/auth/simple',

src/lib/services/index.ts

@@ -6,6 +6,7 @@ import HealthService from './health-service.js';
 import ProjectService from '../features/project/project-service.js';
 import ClientInstanceService from '../features/metrics/instance/instance-service.js';
 import ClientMetricsServiceV2 from '../features/metrics/client-metrics/metrics-service-v2.js';
+import { CustomMetricsService } from '../features/metrics/custom/custom-metrics-service.js';
 import TagTypeService from '../features/tag-type/tag-type-service.js';
 import TagService from './tag-service.js';
 import StrategyService from './strategy-service.js';
@@ -204,12 +205,16 @@ export const createServices = (
 
     const unknownFlagsService = new UnknownFlagsService(stores, config);
 
+    // Initialize custom metrics service
+    const customMetricsService = new CustomMetricsService(config);
+
     const clientMetricsServiceV2 = new ClientMetricsServiceV2(
         stores,
         config,
         lastSeenService,
         unknownFlagsService,
     );
+
     const dependentFeaturesReadModel = db
         ? new DependentFeaturesReadModel(db)
         : new FakeDependentFeaturesReadModel();
@@ -453,6 +458,7 @@ export const createServices = (
         tagService,
         clientInstanceService,
         clientMetricsServiceV2,
+        customMetricsService,
         contextService,
         transactionalContextService,
         versionService,
@@ -574,6 +580,7 @@ export interface IUnleashServices {
     apiTokenService: ApiTokenService;
     clientInstanceService: ClientInstanceService;
     clientMetricsServiceV2: ClientMetricsServiceV2;
+    customMetricsService: CustomMetricsService;
     contextService: ContextService;
     transactionalContextService: WithTransactional<ContextService>;
     emailService: EmailService;

src/lib/types/experimental.ts

@@ -64,7 +64,8 @@ export type IFlagKey =
     | 'projectLinkTemplates'
     | 'reportUnknownFlags'
     | 'lastSeenBulkQuery'
-    | 'newGettingStartedEmail';
+    | 'newGettingStartedEmail'
+    | 'customMetrics';
 
 export type IFlags = Partial<{ [key in IFlagKey]: boolean | Variant }>;
 

src/server-dev.ts

@@ -57,6 +57,7 @@ process.nextTick(async () => {
                         featureLinks: true,
                         projectLinkTemplates: true,
                         reportUnknownFlags: true,
+                        customMetrics: true,
                     },
                 },
                 authentication: {

src/test/e2e/api/client/metrics.e2e.test.ts

@@ -79,6 +79,50 @@ test('should create instance if does not exist', async () => {
     expect(finalInstances.length).toBe(1);
 });
 
+test('should accept custom metrics', async () => {
+    const customMetricsExample = {
+        metrics: [
+            {
+                name: 'http_responses_total',
+                value: 1,
+                labels: {
+                    status: '200',
+                    method: 'GET',
+                },
+            },
+            {
+                name: 'http_responses_total',
+                value: 1,
+                labels: {
+                    status: '304',
+                    method: 'GET',
+                },
+            },
+        ],
+    };
+
+    return app.request
+        .post('/api/client/metrics/custom')
+        .send(customMetricsExample)
+        .expect(202);
+});
+
+test('should reject invalid custom metrics', async () => {
+    const invalidCustomMetrics = {
+        data: [
+            {
+                name: 'http_responses_total',
+                value: 1,
+            },
+        ],
+    };
+
+    return app.request
+        .post('/api/client/metrics/custom')
+        .send(invalidCustomMetrics)
+        .expect(400);
+});
+
 test('should emit response time metrics data in the correct path', async () => {
     const badMetrics = {
         ...metricsExample,