From 3c4457af004c7ff8294dc984dafab75ed3e68d68 Mon Sep 17 00:00:00 2001 From: Jaanus Sellin Date: Thu, 22 Feb 2024 08:20:57 +0200 Subject: [PATCH] feat: application overview backend (#6303) --- src/lib/db/client-applications-store.ts | 80 +++++++++++ .../metrics/instance/instance-service.ts | 8 +- src/lib/features/metrics/instance/models.ts | 16 +++ .../spec/application-overview-schema.ts | 4 +- src/lib/routes/admin-api/metrics.ts | 29 +++- .../types/stores/client-applications-store.ts | 2 + .../e2e/api/admin/applications.e2e.test.ts | 136 ++++++++++++++++++ .../fake-client-applications-store.ts | 5 + 8 files changed, 276 insertions(+), 4 deletions(-) create mode 100644 src/test/e2e/api/admin/applications.e2e.test.ts diff --git a/src/lib/db/client-applications-store.ts b/src/lib/db/client-applications-store.ts index 1ca45fdd23..8cbe91ed74 100644 --- a/src/lib/db/client-applications-store.ts +++ b/src/lib/db/client-applications-store.ts @@ -7,6 +7,7 @@ import { import { Logger, LogProvider } from '../logger'; import { IApplicationQuery } from '../types/query'; import { Db } from './db'; +import { IApplicationOverview } from '../features/metrics/instance/models'; const COLUMNS = [ 'app_name', @@ -249,4 +250,83 @@ export default class ClientApplicationsStore return mapRow(row); } + + async getApplicationOverview( + appName: string, + ): Promise { + const query = this.db + .select([ + 'f.project', + 'cme.environment', + 'cme.feature_name', + 'ci.instance_id', + 'ci.sdk_version', + 'ci.last_seen', + ]) + .from({ a: 'client_applications' }) + .leftJoin('client_metrics_env as cme', 'cme.app_name', 'a.app_name') + .leftJoin('features as f', 'cme.feature_name', 'f.name') + .leftJoin('client_instances as ci', function () { + this.on('ci.app_name', '=', 'cme.app_name').andOn( + 'ci.environment', + '=', + 'cme.environment', + ); + }) + .where('a.app_name', appName); + + const rows = await query; + if (!rows.length) { + throw new NotFoundError(`Could not find appName=${appName}`); + } + + return this.mapApplicationOverviewData(rows); + } + + mapApplicationOverviewData(rows: any[]): IApplicationOverview { + const featureCount = new Set(rows.map((row) => row.feature_name)).size; + + const environments = rows.reduce((acc, row) => { + const { environment, instance_id, sdk_version, last_seen } = row; + let env = acc.find((e) => e.name === environment); + if (!env) { + env = { + name: environment, + instanceCount: 1, + sdks: sdk_version ? [sdk_version] : [], + lastSeen: last_seen, + uniqueInstanceIds: new Set([instance_id]), + }; + acc.push(env); + } else { + env.uniqueInstanceIds.add(instance_id); + env.instanceCount = env.uniqueInstanceIds.size; + if (sdk_version && !env.sdks.includes(sdk_version)) { + env.sdks.push(sdk_version); + } + if (new Date(last_seen) > new Date(env.lastSeen)) { + env.lastSeen = last_seen; + } + } + + return acc; + }, []); + + environments.forEach((env) => { + delete env.uniqueInstanceIds; + env.sdks.sort(); + }); + + return { + projects: [ + ...new Set( + rows + .filter((row) => row.project != null) + .map((row) => row.project), + ), + ], + featureCount, + environments, + }; + } } diff --git a/src/lib/features/metrics/instance/instance-service.ts b/src/lib/features/metrics/instance/instance-service.ts index 56339bc325..f50c65f23a 100644 --- a/src/lib/features/metrics/instance/instance-service.ts +++ b/src/lib/features/metrics/instance/instance-service.ts @@ -1,5 +1,5 @@ import { APPLICATION_CREATED, CLIENT_REGISTER } from '../../../types/events'; -import { IApplication } from './models'; +import { IApplication, IApplicationOverview } from './models'; import { IUnleashStores } from '../../../types/stores'; import { IUnleashConfig } from '../../../types/option'; import { IEventStore } from '../../../types/stores/event-store'; @@ -212,6 +212,12 @@ export default class ClientInstanceService { }; } + async getApplicationOverview( + appName: string, + ): Promise { + return this.clientApplicationsStore.getApplicationOverview(appName); + } + async deleteApplication(appName: string): Promise { await this.clientInstanceStore.deleteForApplication(appName); await this.clientApplicationsStore.delete(appName); diff --git a/src/lib/features/metrics/instance/models.ts b/src/lib/features/metrics/instance/models.ts index 05bb83c0c6..03bafde72d 100644 --- a/src/lib/features/metrics/instance/models.ts +++ b/src/lib/features/metrics/instance/models.ts @@ -1,4 +1,6 @@ import { IClientInstance } from '../../../types/stores/client-instance-store'; +import { ApplicationOverviewSchema } from '../../../openapi/spec/application-overview-schema'; +import { ApplicationOverviewEnvironmentSchema } from '../../../openapi/spec/application-overview-environment-schema'; export interface IYesNoCount { yes: number; @@ -29,3 +31,17 @@ export interface IApplication { environment?: string; links?: Record; } + +type IApplicationOverviewEnvironment = Omit< + ApplicationOverviewEnvironmentSchema, + 'lastSeen' +> & { + lastSeen: Date; +}; + +export type IApplicationOverview = Omit< + ApplicationOverviewSchema, + 'environments' +> & { + environments: IApplicationOverviewEnvironment[]; +}; diff --git a/src/lib/openapi/spec/application-overview-schema.ts b/src/lib/openapi/spec/application-overview-schema.ts index e0c4310e36..cbdf038494 100644 --- a/src/lib/openapi/spec/application-overview-schema.ts +++ b/src/lib/openapi/spec/application-overview-schema.ts @@ -33,7 +33,9 @@ export const applicationOverviewSchema = { }, }, components: { - applicationOverviewEnvironmentSchema, + schemas: { + applicationOverviewEnvironmentSchema, + }, }, } as const; diff --git a/src/lib/routes/admin-api/metrics.ts b/src/lib/routes/admin-api/metrics.ts index 4abc8b8e8f..79033445cc 100644 --- a/src/lib/routes/admin-api/metrics.ts +++ b/src/lib/routes/admin-api/metrics.ts @@ -16,12 +16,23 @@ import { import { CreateApplicationSchema } from '../../openapi/spec/create-application-schema'; import { IAuthRequest } from '../unleash-types'; import { extractUserIdFromUser } from '../../util'; +import { IFlagResolver, serializeDates } from '../../types'; +import { NotFoundError } from '../../error'; +import { + ApplicationOverviewSchema, + applicationOverviewSchema, +} from '../../openapi/spec/application-overview-schema'; +import { OpenApiService } from '../../services'; class MetricsController extends Controller { private logger: Logger; private clientInstanceService: ClientInstanceService; + private flagResolver: IFlagResolver; + + private openApiService: OpenApiService; + constructor( config: IUnleashConfig, { @@ -33,6 +44,8 @@ class MetricsController extends Controller { this.logger = config.getLogger('/admin-api/metrics.ts'); this.clientInstanceService = clientInstanceService; + this.openApiService = openApiService; + this.flagResolver = config.flagResolver; // deprecated routes this.get('/seen-toggles', this.deprecated); @@ -195,9 +208,21 @@ class MetricsController extends Controller { } async getApplicationOverview( req: Request, - res: Response, + res: Response, ): Promise { - throw new Error('Not implemented'); + if (!this.flagResolver.isEnabled('sdkReporting')) { + throw new NotFoundError(); + } + const { appName } = req.params; + const overview = + await this.clientInstanceService.getApplicationOverview(appName); + + this.openApiService.respondWithValidation( + 200, + res, + applicationOverviewSchema.$id, + serializeDates(overview), + ); } } export default MetricsController; diff --git a/src/lib/types/stores/client-applications-store.ts b/src/lib/types/stores/client-applications-store.ts index 45433858e0..dc0ec008ad 100644 --- a/src/lib/types/stores/client-applications-store.ts +++ b/src/lib/types/stores/client-applications-store.ts @@ -1,5 +1,6 @@ import { Store } from './store'; import { IApplicationQuery } from '../query'; +import { IApplicationOverview } from '../../features/metrics/instance/models'; export interface IClientApplicationUsage { project: string; @@ -28,4 +29,5 @@ export interface IClientApplicationsStore getAppsForStrategy(query: IApplicationQuery): Promise; getUnannounced(): Promise; setUnannouncedToAnnounced(): Promise; + getApplicationOverview(appName: string): Promise; } diff --git a/src/test/e2e/api/admin/applications.e2e.test.ts b/src/test/e2e/api/admin/applications.e2e.test.ts new file mode 100644 index 0000000000..2304faa043 --- /dev/null +++ b/src/test/e2e/api/admin/applications.e2e.test.ts @@ -0,0 +1,136 @@ +import dbInit, { ITestDb } from '../../helpers/database-init'; +import { + IUnleashTest, + setupAppWithCustomConfig, +} from '../../helpers/test-helper'; +import getLogger from '../../../fixtures/no-logger'; +import { + ApiTokenType, + IApiToken, +} from '../../../../lib/types/models/api-token'; + +let app: IUnleashTest; +let db: ITestDb; +let defaultToken: IApiToken; + +const metrics = { + appName: 'appName', + instanceId: 'instanceId', + bucket: { + start: '2016-11-03T07:16:43.572Z', + stop: '2016-11-03T07:16:53.572Z', + toggles: { + 'toggle-name-1': { + yes: 123, + no: 321, + variants: { + 'variant-1': 123, + 'variant-2': 321, + }, + }, + 'toggle-name-2': { + yes: 123, + no: 321, + variants: { + 'variant-1': 123, + 'variant-2': 321, + }, + }, + 'toggle-name-3': { + yes: 123, + no: 321, + variants: { + 'variant-1': 123, + 'variant-2': 321, + }, + }, + }, + }, +}; + +beforeAll(async () => { + db = await dbInit('applications_serial', getLogger, {}); + app = await setupAppWithCustomConfig( + db.stores, + { + experimental: { + flags: { + strictSchemaValidation: true, + sdkReporting: true, + }, + }, + }, + db.rawDatabase, + ); + + defaultToken = + await app.services.apiTokenService.createApiTokenWithProjects({ + type: ApiTokenType.CLIENT, + projects: ['default'], + environment: 'default', + tokenName: 'tester', + }); +}); + +afterEach(async () => { + await Promise.all([ + db.stores.clientMetricsStoreV2.deleteAll(), + db.stores.clientInstanceStore.deleteAll(), + db.stores.featureToggleStore.deleteAll(), + ]); +}); + +afterAll(async () => { + await app.destroy(); + await db.destroy(); +}); + +test('should show correct number of total', async () => { + await Promise.all([ + app.createFeature('toggle-name-1'), + app.createFeature('toggle-name-2'), + app.createFeature('toggle-name-3'), + app.request.post('/api/client/register').send({ + appName: metrics.appName, + instanceId: metrics.instanceId, + strategies: ['default'], + sdkVersion: 'unleash-client-test', + started: Date.now(), + interval: 10, + }), + app.request.post('/api/client/register').send({ + appName: metrics.appName, + instanceId: 'another-instance', + strategies: ['default'], + sdkVersion: 'unleash-client-test2', + started: Date.now(), + interval: 10, + }), + ]); + await app.services.clientInstanceService.bulkAdd(); + await app.request + .post('/api/client/metrics') + .set('Authorization', defaultToken.secret) + .send(metrics) + .expect(202); + + await app.services.clientMetricsServiceV2.bulkAdd(); + + const { body } = await app.request + .get(`/api/admin/metrics/applications/${metrics.appName}/overview`) + .expect(200); + + const expected = { + projects: ['default'], + environments: [ + { + instanceCount: 2, + name: 'default', + sdks: ['unleash-client-test', 'unleash-client-test2'], + }, + ], + featureCount: 3, + }; + + expect(body).toMatchObject(expected); +}); diff --git a/src/test/fixtures/fake-client-applications-store.ts b/src/test/fixtures/fake-client-applications-store.ts index f3bfd43142..b1f76aef27 100644 --- a/src/test/fixtures/fake-client-applications-store.ts +++ b/src/test/fixtures/fake-client-applications-store.ts @@ -4,6 +4,7 @@ import { } from '../../lib/types/stores/client-applications-store'; import NotFoundError from '../../lib/error/notfound-error'; import { IApplicationQuery } from '../../lib/types/query'; +import { IApplicationOverview } from '../../lib/features/metrics/instance/models'; export default class FakeClientApplicationsStore implements IClientApplicationsStore @@ -78,4 +79,8 @@ export default class FakeClientApplicationsStore await this.delete(details.appName); return this.bulkUpsert([details]); } + + getApplicationOverview(appName: string): Promise { + throw new Error('Method not implemented.'); + } }