feat: Aggregate daily metrics (#5804)

src/lib/db/client-metrics-store-v2.test.ts

@@ -0,0 +1,86 @@
+import dbInit from '../../test/e2e/helpers/database-init';
+import getLogger from '../../test/fixtures/no-logger';
+import { IClientMetricsStoreV2 } from '../types';
+import { setHours, startOfDay, subDays } from 'date-fns';
+
+let stores;
+let db;
+let clientMetricsStore: IClientMetricsStoreV2;
+
+beforeAll(async () => {
+    db = await dbInit('client_metrics_aggregation', getLogger);
+    stores = db.stores;
+    clientMetricsStore = stores.clientMetricsStoreV2;
+});
+
+afterAll(async () => {
+    await db.destroy();
+});
+
+test('aggregate daily metrics from previous day', async () => {
+    const yesterday = subDays(new Date(), 1);
+    await clientMetricsStore.batchInsertMetrics([
+        {
+            appName: 'test',
+            featureName: 'feature',
+            environment: 'development',
+            timestamp: setHours(yesterday, 10),
+            no: 0,
+            yes: 1,
+            variants: {
+                a: 1,
+                b: 0,
+            },
+        },
+        {
+            appName: 'test',
+            featureName: 'feature',
+            environment: 'development',
+            timestamp: setHours(yesterday, 11),
+            no: 1,
+            yes: 1,
+            variants: {
+                a: 0,
+                b: 1,
+            },
+        },
+    ]);
+
+    await clientMetricsStore.aggregateDailyMetrics();
+
+    // TODO: change to store methods once we build them
+    const results = await db.rawDatabase
+        .table('client_metrics_env_daily')
+        .select('*');
+    expect(results).toMatchObject([
+        {
+            feature_name: 'feature',
+            app_name: 'test',
+            environment: 'development',
+            yes: 2,
+            no: 1,
+            date: startOfDay(yesterday),
+        },
+    ]);
+    const variantResults = await db.rawDatabase
+        .table('client_metrics_env_variants_daily')
+        .select('*');
+    expect(variantResults).toMatchObject([
+        {
+            feature_name: 'feature',
+            app_name: 'test',
+            environment: 'development',
+            date: startOfDay(yesterday),
+            variant: 'a',
+            count: 1,
+        },
+        {
+            feature_name: 'feature',
+            app_name: 'test',
+            environment: 'development',
+            date: startOfDay(yesterday),
+            variant: 'b',
+            count: 1,
+        },
+    ]);
+});

src/lib/db/client-metrics-store-v2.ts

@@ -32,7 +32,9 @@ interface ClientMetricsEnvVariantTable extends ClientMetricsBaseTable {
 }
 
 const TABLE = 'client_metrics_env';
+const DAILY_TABLE = 'client_metrics_env_daily';
 const TABLE_VARIANTS = 'client_metrics_env_variants';
+const DAILY_TABLE_VARIANTS = 'client_metrics_env_variants_daily';
 
 const fromRow = (row: ClientMetricsEnvTable) => ({
     featureName: row.feature_name,
@@ -253,4 +255,50 @@ export class ClientMetricsStoreV2 implements IClientMetricsStoreV2 {
             .whereRaw(`timestamp <= NOW() - INTERVAL '${hoursAgo} hours'`)
             .del();
     }
+
+    // aggregates all hourly metrics from a previous day into daily metrics
+    async aggregateDailyMetrics(): Promise<void> {
+        const rawQuery: string = `
+          INSERT INTO ${DAILY_TABLE} (feature_name, app_name, environment, date, yes, no)
+          SELECT
+              feature_name,
+              app_name,
+              environment,
+              CURRENT_DATE - INTERVAL '1 day' as date,
+              SUM(yes) as yes,
+              SUM(no) as no
+          FROM
+              ${TABLE}
+          WHERE
+              timestamp >= CURRENT_DATE - INTERVAL '1 day'
+              AND timestamp < CURRENT_DATE
+          GROUP BY
+              feature_name, app_name, environment
+          ON CONFLICT (feature_name, app_name, environment, date)
+          DO UPDATE SET yes = EXCLUDED.yes, no = EXCLUDED.no;
+        `;
+        const rawVariantsQuery: string = `
+          INSERT INTO ${DAILY_TABLE_VARIANTS} (feature_name, app_name, environment, date, variant, count)
+          SELECT
+              feature_name,
+              app_name,
+              environment,
+              CURRENT_DATE - INTERVAL '1 day' as date,
+              variant,
+              SUM(count) as count
+          FROM
+              ${TABLE_VARIANTS}
+          WHERE
+              timestamp >= CURRENT_DATE - INTERVAL '1 day'
+              AND timestamp < CURRENT_DATE
+          GROUP BY
+              feature_name, app_name, environment, variant
+          ON CONFLICT (feature_name, app_name, environment, date, variant)
+          DO UPDATE SET count = EXCLUDED.count;
+        `;
+
+        // have to be run serially since variants table has FK on yes/no metrics
+        await this.db.raw(rawQuery);
+        await this.db.raw(rawVariantsQuery);
+    }
 }

src/lib/features/scheduler/schedule-services.ts

@@ -145,6 +145,14 @@ export const scheduleServices = async (
         'clearMetrics',
     );
 
+    schedulerService.schedule(
+        () => {
+            clientMetricsServiceV2.aggregateDailyMetrics().catch(console.error);
+        },
+        hoursToMilliseconds(24),
+        'aggregateDailyMetrics',
+    );
+
     schedulerService.schedule(
         accountService.updateLastSeen.bind(accountService),
         minutesToMilliseconds(3),

src/lib/services/client-metrics/metrics-service-v2.ts

@@ -49,6 +49,12 @@ export default class ClientMetricsServiceV2 {
         return this.clientMetricsStoreV2.clearMetrics(hoursAgo);
     }
 
+    async aggregateDailyMetrics() {
+        if (this.flagResolver.isEnabled('extendedUsageMetrics')) {
+            await this.clientMetricsStoreV2.aggregateDailyMetrics();
+        }
+    }
+
     async filterValidToggleNames(toggleNames: string[]): Promise<string[]> {
         const nameValidations: Promise<
             PromiseFulfilledResult<{ name: string }> | PromiseRejectedResult

src/lib/types/stores/client-metrics-store-v2.ts

@@ -34,4 +34,5 @@ export interface IClientMetricsStoreV2
         hoursBack?: number,
     ): Promise<string[]>;
     clearMetrics(hoursAgo: number): Promise<void>;
+    aggregateDailyMetrics(): Promise<void>;
 }

src/test/fixtures/fake-client-metrics-store-v2.ts

@@ -26,6 +26,9 @@ export default class FakeClientMetricsStoreV2
     clearMetrics(hoursBack: number): Promise<void> {
         return Promise.resolve();
     }
+    aggregateDailyMetrics(): Promise<void> {
+        return Promise.resolve();
+    }
     getSeenAppsForFeatureToggle(
         featureName: string,
         hoursBack?: number,