feat: Metrics api returning daily data (#5830)

src/lib/routes/admin-api/client-metrics.ts

@@ -1,7 +1,7 @@
 import { Request, Response } from 'express';
 import Controller from '../controller';
 import { IUnleashConfig } from '../../types/option';
-import { IUnleashServices } from '../../types';
+import { IFlagResolver, IUnleashServices } from '../../types';
 import { Logger } from '../../logger';
 import ClientMetricsServiceV2 from '../../services/client-metrics/metrics-service-v2';
 import { NONE } from '../../types/permissions';
@@ -33,10 +33,14 @@ class ClientMetricsController extends Controller {
 
     private openApiService: OpenApiService;
 
+    private flagResolver: Pick<IFlagResolver, 'isEnabled'>;
+
     private static HOURS_BACK_MIN = 1;
 
     private static HOURS_BACK_MAX = 48;
 
+    private static HOURS_BACK_MAX_V2 = 24 * 31 * 3; // 3 months
+
     constructor(
         config: IUnleashConfig,
         {
@@ -49,6 +53,7 @@ class ClientMetricsController extends Controller {
 
         this.metrics = clientMetricsServiceV2;
         this.openApiService = openApiService;
+        this.flagResolver = config.flagResolver;
 
         this.route({
             method: 'get',
@@ -130,11 +135,11 @@ class ClientMetricsController extends Controller {
         }
 
         const parsed = Number(param);
+        const max = this.flagResolver.isEnabled('extendedUsageMetrics')
+            ? ClientMetricsController.HOURS_BACK_MAX_V2
+            : ClientMetricsController.HOURS_BACK_MAX;
 
-        if (
-            parsed >= ClientMetricsController.HOURS_BACK_MIN &&
-            parsed <= ClientMetricsController.HOURS_BACK_MAX
-        ) {
+        if (parsed >= ClientMetricsController.HOURS_BACK_MIN && parsed <= max) {
             return parsed;
         }
     }

src/lib/services/client-metrics/metrics-service-v2.test.ts

@@ -5,7 +5,9 @@ import getLogger from '../../../test/fixtures/no-logger';
 import createStores from '../../../test/fixtures/store';
 import EventEmitter from 'events';
 import { LastSeenService } from './last-seen/last-seen-service';
-import { IUnleashConfig } from 'lib/types';
+import { IClientMetricsStoreV2, IUnleashConfig } from 'lib/types';
+import { endOfDay, startOfHour, subDays, subHours } from 'date-fns';
+import { IClientMetricsEnv } from '../../types/stores/client-metrics-store-v2';
 
 function initClientMetrics(flagEnabled = true) {
     const stores = createStores();
@@ -120,3 +122,115 @@ test('process metrics properly even when some names are not url friendly, with d
     expect(eventBus.emit).toHaveBeenCalledTimes(1);
     expect(lastSeenService.updateLastSeen).toHaveBeenCalledTimes(1);
 });
+
+test('get daily client metrics for a toggle', async () => {
+    const yesterday = subDays(new Date(), 1);
+    const twoDaysAgo = subDays(new Date(), 2);
+    const threeDaysAgo = subDays(new Date(), 3);
+    const baseData = {
+        featureName: 'feature',
+        appName: 'test',
+        environment: 'development',
+        yes: 0,
+        no: 0,
+    };
+    const clientMetricsStoreV2 = {
+        getMetricsForFeatureToggleV2(
+            featureName: string,
+            hoursBack?: number,
+        ): Promise<IClientMetricsEnv[]> {
+            return Promise.resolve([
+                {
+                    ...baseData,
+                    timestamp: endOfDay(yesterday),
+                    yes: 2,
+                    no: 1,
+                    variants: { a: 1, b: 1 },
+                },
+            ]);
+        },
+    } as IClientMetricsStoreV2;
+    const config = {
+        flagResolver: {
+            isEnabled() {
+                return true;
+            },
+        },
+        getLogger() {},
+    } as unknown as IUnleashConfig;
+    const lastSeenService = {} as LastSeenService;
+    const service = new ClientMetricsServiceV2(
+        { clientMetricsStoreV2 },
+        config,
+        lastSeenService,
+    );
+
+    const metrics = await service.getClientMetricsForToggle('feature', 3 * 24);
+
+    expect(metrics).toMatchObject([
+        { ...baseData, timestamp: endOfDay(threeDaysAgo) },
+        { ...baseData, timestamp: endOfDay(twoDaysAgo) },
+        {
+            ...baseData,
+            timestamp: endOfDay(yesterday),
+            yes: 2,
+            no: 1,
+            variants: { a: 1, b: 1 },
+        },
+    ]);
+});
+
+test('get hourly client metrics for a toggle', async () => {
+    const hourAgo = startOfHour(subHours(new Date(), 1));
+    const thisHour = startOfHour(new Date());
+    const baseData = {
+        featureName: 'feature',
+        appName: 'test',
+        environment: 'development',
+        yes: 0,
+        no: 0,
+    };
+    const clientMetricsStoreV2 = {
+        getMetricsForFeatureToggleV2(
+            featureName: string,
+            hoursBack?: number,
+        ): Promise<IClientMetricsEnv[]> {
+            return Promise.resolve([
+                {
+                    ...baseData,
+                    timestamp: thisHour,
+                    yes: 2,
+                    no: 1,
+                    variants: { a: 1, b: 1 },
+                },
+            ]);
+        },
+    } as IClientMetricsStoreV2;
+    const config = {
+        flagResolver: {
+            isEnabled() {
+                return true;
+            },
+        },
+        getLogger() {},
+    } as unknown as IUnleashConfig;
+    const lastSeenService = {} as LastSeenService;
+    const service = new ClientMetricsServiceV2(
+        { clientMetricsStoreV2 },
+        config,
+        lastSeenService,
+    );
+
+    const metrics = await service.getClientMetricsForToggle('feature', 2);
+
+    expect(metrics).toMatchObject([
+        { ...baseData, timestamp: hourAgo },
+        {
+            ...baseData,
+            timestamp: thisHour,
+            yes: 2,
+            no: 1,
+            variants: { a: 1, b: 1 },
+        },
+    ]);
+});

src/lib/services/client-metrics/metrics-service-v2.ts

@@ -14,7 +14,11 @@ import { ALL } from '../../types/models/api-token';
 import { IUser } from '../../types/user';
 import { collapseHourlyMetrics } from '../../util/collapseHourlyMetrics';
 import { LastSeenService } from './last-seen/last-seen-service';
-import { generateHourBuckets } from '../../util/time-utils';
+import {
+    generateDayBuckets,
+    generateHourBuckets,
+    HourBucket,
+} from '../../util/time-utils';
 import { ClientMetricsSchema } from 'lib/openapi';
 import { nameSchema } from '../../schema/feature-schema';
 
@@ -183,17 +187,26 @@ export default class ClientMetricsServiceV2 {
         featureName: string,
         hoursBack: number = 24,
     ): Promise<IClientMetricsEnv[]> {
-        const metrics = this.flagResolver.isEnabled('extendedUsageMetrics')
-            ? await this.clientMetricsStoreV2.getMetricsForFeatureToggleV2(
-                  featureName,
-                  hoursBack,
-              )
-            : await this.clientMetricsStoreV2.getMetricsForFeatureToggle(
-                  featureName,
-                  hoursBack,
-              );
-
-        const hours = generateHourBuckets(hoursBack);
+        let hours: HourBucket[];
+        let metrics: IClientMetricsEnv[];
+        if (this.flagResolver.isEnabled('extendedUsageMetrics')) {
+            metrics =
+                await this.clientMetricsStoreV2.getMetricsForFeatureToggleV2(
+                    featureName,
+                    hoursBack,
+                );
+            hours =
+                hoursBack > 48
+                    ? generateDayBuckets(Math.floor(hoursBack / 24))
+                    : generateHourBuckets(hoursBack);
+        } else {
+            metrics =
+                await this.clientMetricsStoreV2.getMetricsForFeatureToggle(
+                    featureName,
+                    hoursBack,
+                );
+            hours = generateHourBuckets(hoursBack);
+        }
 
         const environments = [...new Set(metrics.map((x) => x.environment))];
 

src/lib/util/time-utils.test.ts

@@ -1,7 +1,16 @@
-import { generateHourBuckets } from './time-utils';
+import { generateDayBuckets, generateHourBuckets } from './time-utils';
+import { endOfDay, subDays } from 'date-fns';
 
 test('generateHourBuckets', () => {
     const result = generateHourBuckets(24);
 
     expect(result).toHaveLength(24);
 });
+
+test('generateDayBuckets', () => {
+    const result = generateDayBuckets(7);
+    const endOfDayYesterday = endOfDay(subDays(new Date(), 1));
+
+    expect(result).toHaveLength(7);
+    expect(result[0]).toMatchObject({ timestamp: endOfDayYesterday });
+});

src/lib/util/time-utils.ts

@@ -1,4 +1,4 @@
-import { startOfHour, subHours } from 'date-fns';
+import { endOfDay, startOfHour, subDays, subHours } from 'date-fns';
 
 export interface HourBucket {
     timestamp: Date;
@@ -14,3 +14,15 @@ export function generateHourBuckets(hours: number): HourBucket[] {
     }
     return result;
 }
+
+// Generate last x days starting from end of yesterday
+export function generateDayBuckets(days: number): HourBucket[] {
+    const start = endOfDay(subDays(new Date(), 1));
+
+    const result = [];
+
+    for (let i = 0; i < days; i++) {
+        result.push({ timestamp: subDays(start, i) });
+    }
+    return result;
+}

src/test/e2e/api/admin/client-metrics.e2e.test.ts

@@ -25,6 +25,7 @@ beforeAll(async () => {
             experimental: {
                 flags: {
                     strictSchemaValidation: true,
+                    extendedUsageMetrics: true,
                 },
             },
         },
@@ -143,18 +144,21 @@ test('should support the hoursBack query param for raw metrics', async () => {
     ];
 
     await db.stores.clientMetricsStoreV2.batchInsertMetrics(metrics);
+    await db.stores.clientMetricsStoreV2.aggregateDailyMetrics();
 
     const hours1 = await fetchHoursBack(1);
     const hours24 = await fetchHoursBack(24);
     const hours48 = await fetchHoursBack(48);
     const hoursTooFew = await fetchHoursBack(-999);
-    const hoursTooMany = await fetchHoursBack(999);
+    const hoursTooMany = await fetchHoursBack(24 * 31 * 3 + 1); // 3 months + 1 hour
+    const days = await fetchHoursBack(48 + 1); // switch to days after 48 hours
 
     expect(hours1.data).toHaveLength(1);
     expect(hours24.data).toHaveLength(24);
     expect(hours48.data).toHaveLength(48);
     expect(hoursTooFew.data).toHaveLength(24);
     expect(hoursTooMany.data).toHaveLength(24);
+    expect(days.data).toHaveLength(2); // two days of data
 });
 
 test('should return toggle summary', async () => {