From 992062561929ee6f36bf0763012dcbfe5778f88f Mon Sep 17 00:00:00 2001 From: Jaanus Sellin Date: Thu, 18 Apr 2024 09:02:33 +0300 Subject: [PATCH] Merge --- .../__snapshots__/create-config.test.ts.snap | 1 + src/lib/db/client-applications-store.ts | 164 +++++++++++++++++- src/lib/types/experimental.ts | 7 +- src/server-dev.ts | 1 + .../e2e/api/admin/applications.e2e.test.ts | 1 + 5 files changed, 172 insertions(+), 2 deletions(-) diff --git a/src/lib/__snapshots__/create-config.test.ts.snap b/src/lib/__snapshots__/create-config.test.ts.snap index 7f6ee2a33d..9211b8a9a1 100644 --- a/src/lib/__snapshots__/create-config.test.ts.snap +++ b/src/lib/__snapshots__/create-config.test.ts.snap @@ -75,6 +75,7 @@ exports[`should create default config 1`] = ` "experiments": { "adminTokenKillSwitch": false, "anonymiseEventLog": false, + "applicationOverviewNewQuery": false, "automatedActions": false, "bearerTokenMiddleware": false, "caseInsensitiveInOperators": false, diff --git a/src/lib/db/client-applications-store.ts b/src/lib/db/client-applications-store.ts index b3502c8ed5..6521fa36d6 100644 --- a/src/lib/db/client-applications-store.ts +++ b/src/lib/db/client-applications-store.ts @@ -10,6 +10,9 @@ import type { Logger, LogProvider } from '../logger'; import type { Db } from './db'; import type { IApplicationOverview } from '../features/metrics/instance/models'; import { applySearchFilters } from '../features/feature-search/search-utils'; +import type { IFlagResolver } from '../types'; +import metricsHelper from '../util/metrics-helper'; +import { DB_TIME } from '../metric-events'; const COLUMNS = [ 'app_name', @@ -125,9 +128,23 @@ export default class ClientApplicationsStore private logger: Logger; - constructor(db: Db, eventBus: EventEmitter, getLogger: LogProvider) { + private timer: Function; + + private flagResolver: IFlagResolver; + + constructor( + db: Db, + eventBus: EventEmitter, + getLogger: LogProvider, + flagResolver: IFlagResolver, + ) { this.db = db; this.logger = getLogger('client-applications-store.ts'); + this.timer = (action: string) => + metricsHelper.wrapTimer(eventBus, DB_TIME, { + store: 'client-applications', + action, + }); } async upsert(details: Partial): Promise { @@ -291,6 +308,60 @@ export default class ClientApplicationsStore async getApplicationOverview( appName: string, ): Promise { + if (!this.flagResolver.isEnabled('applicationOverviewNewQuery')) { + return this.getOldApplicationOverview(appName); + } + const stopTimer = this.timer('getApplicationOverview'); + const query = this.db + .with('metrics', (qb) => { + qb.select([ + 'cme.app_name', + 'cme.environment', + 'f.project', + this.db.raw( + 'array_agg(DISTINCT cme.feature_name) as features', + ), + ]) + .from('client_metrics_env as cme') + .leftJoin('features as f', 'f.name', 'cme.feature_name') + .groupBy('cme.app_name', 'cme.environment', 'f.project'); + }) + .select([ + 'm.project', + 'm.environment', + 'm.features', + 'ci.instance_id', + 'ci.sdk_version', + 'ci.last_seen', + 'a.strategies', + ]) + .from({ a: 'client_applications' }) + .leftJoin('metrics as m', 'm.app_name', 'a.app_name') + .leftJoin('client_instances as ci', function () { + this.on('ci.app_name', '=', 'm.app_name').andOn( + 'ci.environment', + '=', + 'm.environment', + ); + }) + .where('a.app_name', appName) + .orderBy('m.environment', 'asc'); + const rows = await query; + stopTimer(); + if (!rows.length) { + throw new NotFoundError(`Could not find appName=${appName}`); + } + const existingStrategies: string[] = await this.db + .select('name') + .from('strategies') + .pluck('name'); + return this.mapApplicationOverviewData(rows, existingStrategies); + } + + async getOldApplicationOverview( + appName: string, + ): Promise { + const stopTimer = this.timer('getApplicationOverviewOld'); const query = this.db .with('metrics', (qb) => { qb.distinct( @@ -321,6 +392,7 @@ export default class ClientApplicationsStore .where('a.app_name', appName) .orderBy('cme.environment', 'asc'); const rows = await query; + stopTimer(); if (!rows.length) { throw new NotFoundError(`Could not find appName=${appName}`); } @@ -334,6 +406,96 @@ export default class ClientApplicationsStore mapApplicationOverviewData( rows: any[], existingStrategies: string[], + ): IApplicationOverview { + if (!this.flagResolver.isEnabled('applicationOverviewNewQuery')) { + return this.mapOldApplicationOverviewData(rows, existingStrategies); + } + const featureCount = new Set(rows.flatMap((row) => row.features)).size; + const missingStrategies: Set = new Set(); + + const environments = rows.reduce((acc, row) => { + const { + environment, + instance_id, + sdk_version, + last_seen, + project, + features, + strategies, + } = row; + + if (!environment) return acc; + + strategies.forEach((strategy) => { + if ( + !DEPRECATED_STRATEGIES.includes(strategy) && + !existingStrategies.includes(strategy) + ) { + missingStrategies.add(strategy); + } + }); + + const featuresNotMappedToProject = !project; + + let env = acc.find((e) => e.name === environment); + if (!env) { + env = { + name: environment, + instanceCount: instance_id ? 1 : 0, + sdks: sdk_version ? [sdk_version] : [], + lastSeen: last_seen, + uniqueInstanceIds: new Set( + instance_id ? [instance_id] : [], + ), + issues: { + missingFeatures: featuresNotMappedToProject + ? features + : [], + }, + }; + acc.push(env); + } else { + if (instance_id) { + env.uniqueInstanceIds.add(instance_id); + env.instanceCount = env.uniqueInstanceIds.size; + } + if (featuresNotMappedToProject) { + env.issues.missingFeatures = features; + } + if (sdk_version && !env.sdks.includes(sdk_version)) { + env.sdks.push(sdk_version); + } + if (new Date(last_seen) > new Date(env.lastSeen)) { + env.lastSeen = last_seen; + } + } + + return acc; + }, []); + environments.forEach((env) => { + delete env.uniqueInstanceIds; + env.sdks.sort(); + }); + + return { + projects: [ + ...new Set( + rows + .filter((row) => row.project != null) + .map((row) => row.project), + ), + ], + featureCount, + environments, + issues: { + missingStrategies: [...missingStrategies], + }, + }; + } + + private mapOldApplicationOverviewData( + rows: any[], + existingStrategies: string[], ): IApplicationOverview { const featureCount = new Set(rows.map((row) => row.feature_name)).size; const missingStrategies: Set = new Set(); diff --git a/src/lib/types/experimental.ts b/src/lib/types/experimental.ts index 78b86dd143..28a6ee0610 100644 --- a/src/lib/types/experimental.ts +++ b/src/lib/types/experimental.ts @@ -58,7 +58,8 @@ export type IFlagKey = | 'variantDependencies' | 'disableShowContextFieldSelectionValues' | 'bearerTokenMiddleware' - | 'projectOverviewRefactorFeedback'; + | 'projectOverviewRefactorFeedback' + | 'applicationOverviewNewQuery'; export type IFlags = Partial<{ [key in IFlagKey]: boolean | Variant }>; @@ -288,6 +289,10 @@ const flags: IFlags = { process.env.UNLEASH_EXPERIMENTAL_PROJECT_OVERVIEW_REFACTOR_FEEDBACK, false, ), + applicationOverviewNewQuery: parseEnvVarBoolean( + process.env.UNLEASH_EXPERIMENTAL_APPLICATION_OVERVIEW_NEW_QUERY, + false, + ), }; export const defaultExperimentalOptions: IExperimentalOptions = { diff --git a/src/server-dev.ts b/src/server-dev.ts index c66a8b3748..a3cc6952ee 100644 --- a/src/server-dev.ts +++ b/src/server-dev.ts @@ -54,6 +54,7 @@ process.nextTick(async () => { disableShowContextFieldSelectionValues: false, variantDependencies: true, projectOverviewRefactorFeedback: true, + applicationOverviewNewQuery: true, }, }, authentication: { diff --git a/src/test/e2e/api/admin/applications.e2e.test.ts b/src/test/e2e/api/admin/applications.e2e.test.ts index 41bd760193..659fd779f3 100644 --- a/src/test/e2e/api/admin/applications.e2e.test.ts +++ b/src/test/e2e/api/admin/applications.e2e.test.ts @@ -56,6 +56,7 @@ beforeAll(async () => { experimental: { flags: { strictSchemaValidation: true, + applicationOverviewNewQuery: true, }, }, },