diff --git a/src/lib/db/client-metrics-store-v2.test.ts b/src/lib/db/client-metrics-store-v2.test.ts new file mode 100644 index 0000000000..120f64ad17 --- /dev/null +++ b/src/lib/db/client-metrics-store-v2.test.ts @@ -0,0 +1,86 @@ +import dbInit from '../../test/e2e/helpers/database-init'; +import getLogger from '../../test/fixtures/no-logger'; +import { IClientMetricsStoreV2 } from '../types'; +import { setHours, startOfDay, subDays } from 'date-fns'; + +let stores; +let db; +let clientMetricsStore: IClientMetricsStoreV2; + +beforeAll(async () => { + db = await dbInit('client_metrics_aggregation', getLogger); + stores = db.stores; + clientMetricsStore = stores.clientMetricsStoreV2; +}); + +afterAll(async () => { + await db.destroy(); +}); + +test('aggregate daily metrics from previous day', async () => { + const yesterday = subDays(new Date(), 1); + await clientMetricsStore.batchInsertMetrics([ + { + appName: 'test', + featureName: 'feature', + environment: 'development', + timestamp: setHours(yesterday, 10), + no: 0, + yes: 1, + variants: { + a: 1, + b: 0, + }, + }, + { + appName: 'test', + featureName: 'feature', + environment: 'development', + timestamp: setHours(yesterday, 11), + no: 1, + yes: 1, + variants: { + a: 0, + b: 1, + }, + }, + ]); + + await clientMetricsStore.aggregateDailyMetrics(); + + // TODO: change to store methods once we build them + const results = await db.rawDatabase + .table('client_metrics_env_daily') + .select('*'); + expect(results).toMatchObject([ + { + feature_name: 'feature', + app_name: 'test', + environment: 'development', + yes: 2, + no: 1, + date: startOfDay(yesterday), + }, + ]); + const variantResults = await db.rawDatabase + .table('client_metrics_env_variants_daily') + .select('*'); + expect(variantResults).toMatchObject([ + { + feature_name: 'feature', + app_name: 'test', + environment: 'development', + date: startOfDay(yesterday), + variant: 'a', + count: 1, + }, + { + feature_name: 'feature', + app_name: 'test', + environment: 'development', + date: startOfDay(yesterday), + variant: 'b', + count: 1, + }, + ]); +}); diff --git a/src/lib/db/client-metrics-store-v2.ts b/src/lib/db/client-metrics-store-v2.ts index 616dbb9281..9607a10129 100644 --- a/src/lib/db/client-metrics-store-v2.ts +++ b/src/lib/db/client-metrics-store-v2.ts @@ -32,7 +32,9 @@ interface ClientMetricsEnvVariantTable extends ClientMetricsBaseTable { } const TABLE = 'client_metrics_env'; +const DAILY_TABLE = 'client_metrics_env_daily'; const TABLE_VARIANTS = 'client_metrics_env_variants'; +const DAILY_TABLE_VARIANTS = 'client_metrics_env_variants_daily'; const fromRow = (row: ClientMetricsEnvTable) => ({ featureName: row.feature_name, @@ -253,4 +255,50 @@ export class ClientMetricsStoreV2 implements IClientMetricsStoreV2 { .whereRaw(`timestamp <= NOW() - INTERVAL '${hoursAgo} hours'`) .del(); } + + // aggregates all hourly metrics from a previous day into daily metrics + async aggregateDailyMetrics(): Promise { + const rawQuery: string = ` + INSERT INTO ${DAILY_TABLE} (feature_name, app_name, environment, date, yes, no) + SELECT + feature_name, + app_name, + environment, + CURRENT_DATE - INTERVAL '1 day' as date, + SUM(yes) as yes, + SUM(no) as no + FROM + ${TABLE} + WHERE + timestamp >= CURRENT_DATE - INTERVAL '1 day' + AND timestamp < CURRENT_DATE + GROUP BY + feature_name, app_name, environment + ON CONFLICT (feature_name, app_name, environment, date) + DO UPDATE SET yes = EXCLUDED.yes, no = EXCLUDED.no; + `; + const rawVariantsQuery: string = ` + INSERT INTO ${DAILY_TABLE_VARIANTS} (feature_name, app_name, environment, date, variant, count) + SELECT + feature_name, + app_name, + environment, + CURRENT_DATE - INTERVAL '1 day' as date, + variant, + SUM(count) as count + FROM + ${TABLE_VARIANTS} + WHERE + timestamp >= CURRENT_DATE - INTERVAL '1 day' + AND timestamp < CURRENT_DATE + GROUP BY + feature_name, app_name, environment, variant + ON CONFLICT (feature_name, app_name, environment, date, variant) + DO UPDATE SET count = EXCLUDED.count; + `; + + // have to be run serially since variants table has FK on yes/no metrics + await this.db.raw(rawQuery); + await this.db.raw(rawVariantsQuery); + } } diff --git a/src/lib/features/scheduler/schedule-services.ts b/src/lib/features/scheduler/schedule-services.ts index 0e276270c9..4546c01940 100644 --- a/src/lib/features/scheduler/schedule-services.ts +++ b/src/lib/features/scheduler/schedule-services.ts @@ -145,6 +145,14 @@ export const scheduleServices = async ( 'clearMetrics', ); + schedulerService.schedule( + () => { + clientMetricsServiceV2.aggregateDailyMetrics().catch(console.error); + }, + hoursToMilliseconds(24), + 'aggregateDailyMetrics', + ); + schedulerService.schedule( accountService.updateLastSeen.bind(accountService), minutesToMilliseconds(3), diff --git a/src/lib/services/client-metrics/metrics-service-v2.ts b/src/lib/services/client-metrics/metrics-service-v2.ts index 90a0ff3806..a1ac2b0e63 100644 --- a/src/lib/services/client-metrics/metrics-service-v2.ts +++ b/src/lib/services/client-metrics/metrics-service-v2.ts @@ -49,6 +49,12 @@ export default class ClientMetricsServiceV2 { return this.clientMetricsStoreV2.clearMetrics(hoursAgo); } + async aggregateDailyMetrics() { + if (this.flagResolver.isEnabled('extendedUsageMetrics')) { + await this.clientMetricsStoreV2.aggregateDailyMetrics(); + } + } + async filterValidToggleNames(toggleNames: string[]): Promise { const nameValidations: Promise< PromiseFulfilledResult<{ name: string }> | PromiseRejectedResult diff --git a/src/lib/types/stores/client-metrics-store-v2.ts b/src/lib/types/stores/client-metrics-store-v2.ts index 8d99b28265..6bda292bb5 100644 --- a/src/lib/types/stores/client-metrics-store-v2.ts +++ b/src/lib/types/stores/client-metrics-store-v2.ts @@ -34,4 +34,5 @@ export interface IClientMetricsStoreV2 hoursBack?: number, ): Promise; clearMetrics(hoursAgo: number): Promise; + aggregateDailyMetrics(): Promise; } diff --git a/src/test/fixtures/fake-client-metrics-store-v2.ts b/src/test/fixtures/fake-client-metrics-store-v2.ts index 3c5f51b62d..fd2c080a77 100644 --- a/src/test/fixtures/fake-client-metrics-store-v2.ts +++ b/src/test/fixtures/fake-client-metrics-store-v2.ts @@ -26,6 +26,9 @@ export default class FakeClientMetricsStoreV2 clearMetrics(hoursBack: number): Promise { return Promise.resolve(); } + aggregateDailyMetrics(): Promise { + return Promise.resolve(); + } getSeenAppsForFeatureToggle( featureName: string, hoursBack?: number,