From d71108526eed7216f1ff82bc2038b698edd96563 Mon Sep 17 00:00:00 2001 From: Mateusz Kwasniewski Date: Wed, 10 Jan 2024 11:48:06 +0100 Subject: [PATCH] feat: read extended metrics from more than 48 hours (#5822) --- src/lib/db/client-metrics-store-v2.test.ts | 102 +++++++++++++----- src/lib/db/client-metrics-store-v2.ts | 70 +++++++++++- .../client-metrics/metrics-service-v2.ts | 14 ++- .../types/stores/client-metrics-store-v2.ts | 4 + .../fixtures/fake-client-metrics-store-v2.ts | 6 ++ 5 files changed, 166 insertions(+), 30 deletions(-) diff --git a/src/lib/db/client-metrics-store-v2.test.ts b/src/lib/db/client-metrics-store-v2.test.ts index 234055c489..d59abb5fe7 100644 --- a/src/lib/db/client-metrics-store-v2.test.ts +++ b/src/lib/db/client-metrics-store-v2.test.ts @@ -1,7 +1,7 @@ import dbInit from '../../test/e2e/helpers/database-init'; import getLogger from '../../test/fixtures/no-logger'; import { IClientMetricsStoreV2 } from '../types'; -import { setHours, startOfDay, subDays } from 'date-fns'; +import { endOfDay, setHours, startOfDay, startOfHour, subDays } from 'date-fns'; let stores; let db; @@ -53,39 +53,93 @@ test('aggregate daily metrics from previous day', async () => { await clientMetricsStore.aggregateDailyMetrics(); - // TODO: change to store methods once we build them - const results = await db.rawDatabase - .table('client_metrics_env_daily') - .select('*'); - expect(results).toMatchObject([ + const hourlyMetrics = await clientMetricsStore.getMetricsForFeatureToggleV2( + 'feature', + 48, + ); + expect(hourlyMetrics).toMatchObject([ { - feature_name: 'feature', - app_name: 'test', + featureName: 'feature', + appName: 'test', environment: 'development', - yes: 2, + timestamp: startOfHour(setHours(yesterday, 10)), + yes: 1, + no: 0, + variants: { a: 1, b: 0 }, + }, + { + featureName: 'feature', + appName: 'test', + environment: 'development', + timestamp: startOfHour(setHours(yesterday, 11)), + yes: 1, no: 1, - date: startOfDay(yesterday), + variants: { a: 0, b: 1 }, }, ]); - const variantResults = await db.rawDatabase - .table('client_metrics_env_variants_daily') - .select('*'); - expect(variantResults).toMatchObject([ + const dailyMetrics = await clientMetricsStore.getMetricsForFeatureToggleV2( + 'feature', + 49, + ); + expect(dailyMetrics).toMatchObject([ { - feature_name: 'feature', - app_name: 'test', + featureName: 'feature', + appName: 'test', environment: 'development', - date: startOfDay(yesterday), - variant: 'a', - count: 1, + timestamp: endOfDay(yesterday), + yes: 2, + no: 1, + variants: { a: 1, b: 1 }, + }, + ]); +}); + +test('clear daily metrics', async () => { + const yesterday = subDays(new Date(), 1); + const twoDaysAgo = subDays(new Date(), 2); + await clientMetricsStore.batchInsertMetrics([ + { + appName: 'test', + featureName: 'feature', + environment: 'development', + timestamp: yesterday, + no: 0, + yes: 1, + variants: { + a: 0, + b: 1, + }, }, { - feature_name: 'feature', - app_name: 'test', + appName: 'test', + featureName: 'feature', environment: 'development', - date: startOfDay(yesterday), - variant: 'b', - count: 1, + timestamp: twoDaysAgo, + no: 0, + yes: 2, + variants: { + a: 1, + b: 1, + }, + }, + ]); + await clientMetricsStore.aggregateDailyMetrics(); + + await clientMetricsStore.clearDailyMetrics(2); + + const dailyMetrics = await clientMetricsStore.getMetricsForFeatureToggleV2( + 'feature', + 49, + ); + expect(dailyMetrics).toMatchObject([ + { + featureName: 'feature', + appName: 'test', + environment: 'development', + timestamp: endOfDay(yesterday), + yes: 1, + no: 0, + variants: { a: 0, b: 1 }, }, ]); }); diff --git a/src/lib/db/client-metrics-store-v2.ts b/src/lib/db/client-metrics-store-v2.ts index 81035de947..ee4218b8d0 100644 --- a/src/lib/db/client-metrics-store-v2.ts +++ b/src/lib/db/client-metrics-store-v2.ts @@ -6,7 +6,7 @@ import { IClientMetricsStoreV2, } from '../types/stores/client-metrics-store-v2'; import NotFoundError from '../error/notfound-error'; -import { startOfHour } from 'date-fns'; +import { endOfDay, startOfHour } from 'date-fns'; import { collapseHourlyMetrics, spreadVariants, @@ -95,6 +95,39 @@ const variantRowReducer = (acc, tokenRow) => { return acc; }; +const variantRowReducerV2 = (acc, tokenRow) => { + const { + feature_name: featureName, + app_name: appName, + environment, + timestamp, + date, + yes, + no, + variant, + count, + } = tokenRow; + const key = `${featureName}_${appName}_${environment}_${ + timestamp || date + }_${yes}_${no}`; + if (!acc[key]) { + acc[key] = { + featureName, + appName, + environment, + timestamp: timestamp || endOfDay(date), + yes: Number(yes), + no: Number(no), + variants: {}, + }; + } + if (variant) { + acc[key].variants[variant] = count; + } + + return acc; +}; + export class ClientMetricsStoreV2 implements IClientMetricsStoreV2 { private db: Db; @@ -226,6 +259,41 @@ export class ClientMetricsStoreV2 implements IClientMetricsStoreV2 { return Object.values(tokens); } + async getMetricsForFeatureToggleV2( + featureName: string, + hoursBack: number = 24, + ): Promise { + const mainTable = hoursBack <= 48 ? TABLE : DAILY_TABLE; + const variantsTable = + hoursBack <= 48 ? TABLE_VARIANTS : DAILY_TABLE_VARIANTS; + const dateTime = hoursBack <= 48 ? 'timestamp' : 'date'; + + const rows = await this.db(mainTable) + .select([`${mainTable}.*`, 'variant', 'count']) + .leftJoin(variantsTable, function () { + this.on( + `${variantsTable}.feature_name`, + `${mainTable}.feature_name`, + ) + .on(`${variantsTable}.app_name`, `${mainTable}.app_name`) + .on( + `${variantsTable}.environment`, + `${mainTable}.environment`, + ) + .on( + `${variantsTable}.${dateTime}`, + `${mainTable}.${dateTime}`, + ); + }) + .where(`${mainTable}.feature_name`, featureName) + .andWhereRaw( + `${mainTable}.${dateTime} >= NOW() - INTERVAL '${hoursBack} hours'`, + ); + + const tokens = rows.reduce(variantRowReducerV2, {}); + return Object.values(tokens); + } + async getSeenAppsForFeatureToggle( featureName: string, hoursBack: number = 24, diff --git a/src/lib/services/client-metrics/metrics-service-v2.ts b/src/lib/services/client-metrics/metrics-service-v2.ts index f4a7e72034..5288ba0ebb 100644 --- a/src/lib/services/client-metrics/metrics-service-v2.ts +++ b/src/lib/services/client-metrics/metrics-service-v2.ts @@ -183,11 +183,15 @@ export default class ClientMetricsServiceV2 { featureName: string, hoursBack: number = 24, ): Promise { - const metrics = - await this.clientMetricsStoreV2.getMetricsForFeatureToggle( - featureName, - hoursBack, - ); + const metrics = this.flagResolver.isEnabled('extendedUsageMetrics') + ? await this.clientMetricsStoreV2.getMetricsForFeatureToggleV2( + featureName, + hoursBack, + ) + : await this.clientMetricsStoreV2.getMetricsForFeatureToggle( + featureName, + hoursBack, + ); const hours = generateHourBuckets(hoursBack); diff --git a/src/lib/types/stores/client-metrics-store-v2.ts b/src/lib/types/stores/client-metrics-store-v2.ts index ebe75d54b6..9f148c8879 100644 --- a/src/lib/types/stores/client-metrics-store-v2.ts +++ b/src/lib/types/stores/client-metrics-store-v2.ts @@ -25,6 +25,10 @@ export interface IClientMetricsStoreV2 featureName: string, hoursBack?: number, ): Promise; + getMetricsForFeatureToggleV2( + featureName: string, + hoursBack?: number, + ): Promise; getSeenAppsForFeatureToggle( featureName: string, hoursBack?: number, diff --git a/src/test/fixtures/fake-client-metrics-store-v2.ts b/src/test/fixtures/fake-client-metrics-store-v2.ts index 3484dd10d6..19a35293a3 100644 --- a/src/test/fixtures/fake-client-metrics-store-v2.ts +++ b/src/test/fixtures/fake-client-metrics-store-v2.ts @@ -44,6 +44,12 @@ export default class FakeClientMetricsStoreV2 ): Promise { throw new Error('Method not implemented.'); } + getMetricsForFeatureToggleV2( + featureName: string, + hoursBack?: number, + ): Promise { + throw new Error('Method not implemented.'); + } batchInsertMetrics(metrics: IClientMetricsEnv[]): Promise { metrics.forEach((m) => this.metrics.push(m)); return Promise.resolve();