feat: read extended metrics from more than 48 hours (#5822)

src/lib/db/client-metrics-store-v2.test.ts

@@ -1,7 +1,7 @@
 import dbInit from '../../test/e2e/helpers/database-init';
 import getLogger from '../../test/fixtures/no-logger';
 import { IClientMetricsStoreV2 } from '../types';
-import { setHours, startOfDay, subDays } from 'date-fns';
+import { endOfDay, setHours, startOfDay, startOfHour, subDays } from 'date-fns';
 
 let stores;
 let db;
@@ -53,39 +53,93 @@ test('aggregate daily metrics from previous day', async () => {
 
     await clientMetricsStore.aggregateDailyMetrics();
 
-    // TODO: change to store methods once we build them
+    const hourlyMetrics = await clientMetricsStore.getMetricsForFeatureToggleV2(
-    const results = await db.rawDatabase
+        'feature',
-        .table('client_metrics_env_daily')
+        48,
-        .select('*');
+    );
-    expect(results).toMatchObject([
+    expect(hourlyMetrics).toMatchObject([
         {
-            feature_name: 'feature',
+            featureName: 'feature',
-            app_name: 'test',
+            appName: 'test',
             environment: 'development',
-            yes: 2,
+            timestamp: startOfHour(setHours(yesterday, 10)),
+            yes: 1,
+            no: 0,
+            variants: { a: 1, b: 0 },
+        },
+        {
+            featureName: 'feature',
+            appName: 'test',
+            environment: 'development',
+            timestamp: startOfHour(setHours(yesterday, 11)),
+            yes: 1,
             no: 1,
-            date: startOfDay(yesterday),
+            variants: { a: 0, b: 1 },
         },
     ]);
-    const variantResults = await db.rawDatabase
+    const dailyMetrics = await clientMetricsStore.getMetricsForFeatureToggleV2(
-        .table('client_metrics_env_variants_daily')
+        'feature',
-        .select('*');
+        49,
-    expect(variantResults).toMatchObject([
+    );
+    expect(dailyMetrics).toMatchObject([
         {
-            feature_name: 'feature',
+            featureName: 'feature',
-            app_name: 'test',
+            appName: 'test',
             environment: 'development',
-            date: startOfDay(yesterday),
+            timestamp: endOfDay(yesterday),
-            variant: 'a',
+            yes: 2,
-            count: 1,
+            no: 1,
+            variants: { a: 1, b: 1 },
+        },
+    ]);
+});
+
+test('clear daily metrics', async () => {
+    const yesterday = subDays(new Date(), 1);
+    const twoDaysAgo = subDays(new Date(), 2);
+    await clientMetricsStore.batchInsertMetrics([
+        {
+            appName: 'test',
+            featureName: 'feature',
+            environment: 'development',
+            timestamp: yesterday,
+            no: 0,
+            yes: 1,
+            variants: {
+                a: 0,
+                b: 1,
+            },
         },
         {
-            feature_name: 'feature',
+            appName: 'test',
-            app_name: 'test',
+            featureName: 'feature',
             environment: 'development',
-            date: startOfDay(yesterday),
+            timestamp: twoDaysAgo,
-            variant: 'b',
+            no: 0,
-            count: 1,
+            yes: 2,
+            variants: {
+                a: 1,
+                b: 1,
+            },
+        },
+    ]);
+    await clientMetricsStore.aggregateDailyMetrics();
+
+    await clientMetricsStore.clearDailyMetrics(2);
+
+    const dailyMetrics = await clientMetricsStore.getMetricsForFeatureToggleV2(
+        'feature',
+        49,
+    );
+    expect(dailyMetrics).toMatchObject([
+        {
+            featureName: 'feature',
+            appName: 'test',
+            environment: 'development',
+            timestamp: endOfDay(yesterday),
+            yes: 1,
+            no: 0,
+            variants: { a: 0, b: 1 },
         },
     ]);
 });

src/lib/db/client-metrics-store-v2.ts

@@ -6,7 +6,7 @@ import {
     IClientMetricsStoreV2,
 } from '../types/stores/client-metrics-store-v2';
 import NotFoundError from '../error/notfound-error';
-import { startOfHour } from 'date-fns';
+import { endOfDay, startOfHour } from 'date-fns';
 import {
     collapseHourlyMetrics,
     spreadVariants,
@@ -95,6 +95,39 @@ const variantRowReducer = (acc, tokenRow) => {
     return acc;
 };
 
+const variantRowReducerV2 = (acc, tokenRow) => {
+    const {
+        feature_name: featureName,
+        app_name: appName,
+        environment,
+        timestamp,
+        date,
+        yes,
+        no,
+        variant,
+        count,
+    } = tokenRow;
+    const key = `${featureName}_${appName}_${environment}_${
+        timestamp || date
+    }_${yes}_${no}`;
+    if (!acc[key]) {
+        acc[key] = {
+            featureName,
+            appName,
+            environment,
+            timestamp: timestamp || endOfDay(date),
+            yes: Number(yes),
+            no: Number(no),
+            variants: {},
+        };
+    }
+    if (variant) {
+        acc[key].variants[variant] = count;
+    }
+
+    return acc;
+};
+
 export class ClientMetricsStoreV2 implements IClientMetricsStoreV2 {
     private db: Db;
 
@@ -226,6 +259,41 @@ export class ClientMetricsStoreV2 implements IClientMetricsStoreV2 {
         return Object.values(tokens);
     }
 
+    async getMetricsForFeatureToggleV2(
+        featureName: string,
+        hoursBack: number = 24,
+    ): Promise<IClientMetricsEnv[]> {
+        const mainTable = hoursBack <= 48 ? TABLE : DAILY_TABLE;
+        const variantsTable =
+            hoursBack <= 48 ? TABLE_VARIANTS : DAILY_TABLE_VARIANTS;
+        const dateTime = hoursBack <= 48 ? 'timestamp' : 'date';
+
+        const rows = await this.db<ClientMetricsEnvTable>(mainTable)
+            .select([`${mainTable}.*`, 'variant', 'count'])
+            .leftJoin(variantsTable, function () {
+                this.on(
+                    `${variantsTable}.feature_name`,
+                    `${mainTable}.feature_name`,
+                )
+                    .on(`${variantsTable}.app_name`, `${mainTable}.app_name`)
+                    .on(
+                        `${variantsTable}.environment`,
+                        `${mainTable}.environment`,
+                    )
+                    .on(
+                        `${variantsTable}.${dateTime}`,
+                        `${mainTable}.${dateTime}`,
+                    );
+            })
+            .where(`${mainTable}.feature_name`, featureName)
+            .andWhereRaw(
+                `${mainTable}.${dateTime} >= NOW() - INTERVAL '${hoursBack} hours'`,
+            );
+
+        const tokens = rows.reduce(variantRowReducerV2, {});
+        return Object.values(tokens);
+    }
+
     async getSeenAppsForFeatureToggle(
         featureName: string,
         hoursBack: number = 24,

src/lib/services/client-metrics/metrics-service-v2.ts

@@ -183,11 +183,15 @@ export default class ClientMetricsServiceV2 {
         featureName: string,
         hoursBack: number = 24,
     ): Promise<IClientMetricsEnv[]> {
-        const metrics =
+        const metrics = this.flagResolver.isEnabled('extendedUsageMetrics')
-            await this.clientMetricsStoreV2.getMetricsForFeatureToggle(
+            ? await this.clientMetricsStoreV2.getMetricsForFeatureToggleV2(
-                featureName,
+                  featureName,
-                hoursBack,
+                  hoursBack,
-            );
+              )
+            : await this.clientMetricsStoreV2.getMetricsForFeatureToggle(
+                  featureName,
+                  hoursBack,
+              );
 
         const hours = generateHourBuckets(hoursBack);
 

src/lib/types/stores/client-metrics-store-v2.ts

@@ -25,6 +25,10 @@ export interface IClientMetricsStoreV2
         featureName: string,
         hoursBack?: number,
     ): Promise<IClientMetricsEnv[]>;
+    getMetricsForFeatureToggleV2(
+        featureName: string,
+        hoursBack?: number,
+    ): Promise<IClientMetricsEnv[]>;
     getSeenAppsForFeatureToggle(
         featureName: string,
         hoursBack?: number,

src/test/fixtures/fake-client-metrics-store-v2.ts

@@ -44,6 +44,12 @@ export default class FakeClientMetricsStoreV2
     ): Promise<IClientMetricsEnv[]> {
         throw new Error('Method not implemented.');
     }
+    getMetricsForFeatureToggleV2(
+        featureName: string,
+        hoursBack?: number,
+    ): Promise<IClientMetricsEnv[]> {
+        throw new Error('Method not implemented.');
+    }
     batchInsertMetrics(metrics: IClientMetricsEnv[]): Promise<void> {
         metrics.forEach((m) => this.metrics.push(m));
         return Promise.resolve();