From cd49ae2a26231c6c301b0ec3f3cc6e2b0900a762 Mon Sep 17 00:00:00 2001 From: Jaanus Sellin Date: Wed, 8 May 2024 15:19:23 +0300 Subject: [PATCH] feat: add project id to prometheus and feature flag (#7008) Now we are also sending project id to prometheus, also querying from database. This sets us up for grafana dashboard. Also put the metrics behind flag, just incase it causes cpu/memory issues. --- .../fake-feature-lifecycle-store.ts | 7 +++-- .../feature-lifecycle-service.test.ts | 5 +++ .../feature-lifecycle-service.ts | 7 +++-- .../feature-lifecycle-store-type.ts | 5 +-- .../feature-lifecycle-store.ts | 31 ++++++++++++------- .../instance-stats/instance-stats-service.ts | 18 +++++++++-- src/lib/metrics.test.ts | 6 ++++ src/lib/metrics.ts | 3 +- src/lib/routes/admin-api/instance-admin.ts | 1 + src/lib/types/experimental.ts | 1 + src/lib/types/model.ts | 1 + 11 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/lib/features/feature-lifecycle/fake-feature-lifecycle-store.ts b/src/lib/features/feature-lifecycle/fake-feature-lifecycle-store.ts index cea1c52a11..74ff64cf6a 100644 --- a/src/lib/features/feature-lifecycle/fake-feature-lifecycle-store.ts +++ b/src/lib/features/feature-lifecycle/fake-feature-lifecycle-store.ts @@ -2,7 +2,7 @@ import type { FeatureLifecycleStage, IFeatureLifecycleStore, FeatureLifecycleView, - FeatureLifecycleFullItem, + FeatureLifecycleProjectItem, } from './feature-lifecycle-store-type'; export class FakeFeatureLifecycleStore implements IFeatureLifecycleStore { @@ -36,12 +36,13 @@ export class FakeFeatureLifecycleStore implements IFeatureLifecycleStore { return this.lifecycles[feature] || []; } - async getAll(): Promise { + async getAll(): Promise { const result = Object.entries(this.lifecycles).flatMap( - ([key, items]): FeatureLifecycleFullItem[] => + ([key, items]): FeatureLifecycleProjectItem[] => items.map((item) => ({ ...item, feature: key, + project: 'fake-project', })), ); return result; diff --git a/src/lib/features/feature-lifecycle/feature-lifecycle-service.test.ts b/src/lib/features/feature-lifecycle/feature-lifecycle-service.test.ts index 3ce51ac04a..e3dccd902f 100644 --- a/src/lib/features/feature-lifecycle/feature-lifecycle-service.test.ts +++ b/src/lib/features/feature-lifecycle/feature-lifecycle-service.test.ts @@ -144,26 +144,31 @@ test('can find feature lifecycle stage timings', async () => { { feature: 'a', stage: 'initial', + project: 'default', enteredStageAt: minusTenMinutes, }, { feature: 'b', stage: 'initial', + project: 'default', enteredStageAt: minusTenMinutes, }, { feature: 'a', stage: 'pre-live', + project: 'default', enteredStageAt: minusOneMinute, }, { feature: 'b', stage: 'live', + project: 'default', enteredStageAt: minusOneMinute, }, { feature: 'c', stage: 'initial', + project: 'default', enteredStageAt: minusTenMinutes, }, ]); diff --git a/src/lib/features/feature-lifecycle/feature-lifecycle-service.ts b/src/lib/features/feature-lifecycle/feature-lifecycle-service.ts index ef3be33029..38b56ffa47 100644 --- a/src/lib/features/feature-lifecycle/feature-lifecycle-service.ts +++ b/src/lib/features/feature-lifecycle/feature-lifecycle-service.ts @@ -14,7 +14,7 @@ import { type IUnleashConfig, } from '../../types'; import type { - FeatureLifecycleFullItem, + FeatureLifecycleProjectItem, FeatureLifecycleView, IFeatureLifecycleStore, } from './feature-lifecycle-store-type'; @@ -215,10 +215,10 @@ export class FeatureLifecycleService extends EventEmitter { } public calculateStageDurations( - featureLifeCycles: FeatureLifecycleFullItem[], + featureLifeCycles: FeatureLifecycleProjectItem[], ) { const groupedByFeature = featureLifeCycles.reduce<{ - [feature: string]: FeatureLifecycleFullItem[]; + [feature: string]: FeatureLifecycleProjectItem[]; }>((acc, curr) => { if (!acc[curr.feature]) { acc[curr.feature] = []; @@ -247,6 +247,7 @@ export class FeatureLifecycleService extends EventEmitter { times.push({ feature: stage.feature, stage: stage.stage, + project: stage.project, duration, }); }); diff --git a/src/lib/features/feature-lifecycle/feature-lifecycle-store-type.ts b/src/lib/features/feature-lifecycle/feature-lifecycle-store-type.ts index 9deaef74ef..668891c7a6 100644 --- a/src/lib/features/feature-lifecycle/feature-lifecycle-store-type.ts +++ b/src/lib/features/feature-lifecycle/feature-lifecycle-store-type.ts @@ -7,14 +7,15 @@ export type FeatureLifecycleStage = { export type FeatureLifecycleView = IFeatureLifecycleStage[]; -export type FeatureLifecycleFullItem = FeatureLifecycleStage & { +export type FeatureLifecycleProjectItem = FeatureLifecycleStage & { enteredStageAt: Date; + project: string; }; export interface IFeatureLifecycleStore { insert(featureLifecycleStages: FeatureLifecycleStage[]): Promise; get(feature: string): Promise; - getAll(): Promise; + getAll(): Promise; stageExists(stage: FeatureLifecycleStage): Promise; delete(feature: string): Promise; deleteStage(stage: FeatureLifecycleStage): Promise; diff --git a/src/lib/features/feature-lifecycle/feature-lifecycle-store.ts b/src/lib/features/feature-lifecycle/feature-lifecycle-store.ts index 1c8a843edf..2eedea9844 100644 --- a/src/lib/features/feature-lifecycle/feature-lifecycle-store.ts +++ b/src/lib/features/feature-lifecycle/feature-lifecycle-store.ts @@ -2,17 +2,21 @@ import type { FeatureLifecycleStage, IFeatureLifecycleStore, FeatureLifecycleView, - FeatureLifecycleFullItem, + FeatureLifecycleProjectItem, } from './feature-lifecycle-store-type'; import type { Db } from '../../db/db'; import type { StageName } from '../../types'; type DBType = { - feature: string; stage: StageName; created_at: string; }; +type DBProjectType = DBType & { + feature: string; + project: string; +}; + export class FeatureLifecycleStore implements IFeatureLifecycleStore { private db: Db; @@ -58,17 +62,20 @@ export class FeatureLifecycleStore implements IFeatureLifecycleStore { })); } - async getAll(): Promise { - const results = await this.db('feature_lifecycles').orderBy( - 'created_at', - 'asc', - ); + async getAll(): Promise { + const results = await this.db('feature_lifecycles as flc') + .select('flc.feature', 'flc.stage', 'flc.created_at', 'f.project') + .leftJoin('features f', 'f.name', 'flc.feature') + .orderBy('created_at', 'asc'); - return results.map(({ feature, stage, created_at }: DBType) => ({ - feature, - stage, - enteredStageAt: new Date(created_at), - })); + return results.map( + ({ feature, stage, created_at, project }: DBProjectType) => ({ + feature, + stage, + project, + enteredStageAt: new Date(created_at), + }), + ); } async delete(feature: string): Promise { diff --git a/src/lib/features/instance-stats/instance-stats-service.ts b/src/lib/features/instance-stats/instance-stats-service.ts index 9f48bc5f9b..72ee21afe4 100644 --- a/src/lib/features/instance-stats/instance-stats-service.ts +++ b/src/lib/features/instance-stats/instance-stats-service.ts @@ -23,6 +23,7 @@ import { FEATURES_IMPORTED, type IApiTokenStore, type IFeatureLifecycleStageDuration, + type IFlagResolver, } from '../../types'; import { CUSTOM_ROOT_ROLE_TYPE } from '../../util'; import type { GetActiveUsers } from './getActiveUsers'; @@ -105,6 +106,8 @@ export class InstanceStatsService { private clientMetricsStore: IClientMetricsStoreV2; + private flagResolver: IFlagResolver; + private appCount?: Partial<{ [key in TimeRange]: number }>; private getActiveUsers: GetActiveUsers; @@ -144,7 +147,10 @@ export class InstanceStatsService { | 'apiTokenStore' | 'clientMetricsStoreV2' >, - { getLogger }: Pick, + { + getLogger, + flagResolver, + }: Pick, versionService: VersionService, getActiveUsers: GetActiveUsers, getProductionChanges: GetProductionChanges, @@ -169,6 +175,7 @@ export class InstanceStatsService { this.getProductionChanges = getProductionChanges; this.apiTokenStore = apiTokenStore; this.clientMetricsStore = clientMetricsStoreV2; + this.flagResolver = flagResolver; } async refreshAppCountSnapshot(): Promise< @@ -274,7 +281,7 @@ export class InstanceStatsService { this.eventStore.filteredCount({ type: FEATURES_IMPORTED }), this.getProductionChanges(), this.clientMetricsStore.countPreviousDayHourlyMetricsBuckets(), - this.featureLifecycleService.getAllWithStageDuration(), + this.getAllWithStageDuration(), ]); return { @@ -341,4 +348,11 @@ export class InstanceStatsService { ); return { ...instanceStats, sum, projects: totalProjects }; } + + async getAllWithStageDuration(): Promise { + if (this.flagResolver.isEnabled('featureLifecycleMetrics')) { + return this.featureLifecycleService.getAllWithStageDuration(); + } + return []; + } } diff --git a/src/lib/metrics.test.ts b/src/lib/metrics.test.ts index 42d6826864..74e73e0299 100644 --- a/src/lib/metrics.test.ts +++ b/src/lib/metrics.test.ts @@ -30,11 +30,17 @@ let environmentStore: IEnvironmentStore; let statsService: InstanceStatsService; let stores: IUnleashStores; let schedulerService: SchedulerService; + beforeAll(async () => { const config = createTestConfig({ server: { serverMetrics: true, }, + experimental: { + flags: { + featureLifecycleMetrics: true, + }, + }, }); stores = createStores(); eventStore = stores.eventStore; diff --git a/src/lib/metrics.ts b/src/lib/metrics.ts index ff646b5f3f..1f778f2d86 100644 --- a/src/lib/metrics.ts +++ b/src/lib/metrics.ts @@ -261,7 +261,7 @@ export default class MetricsMonitor { const featureLifecycleStageDuration = createHistogram({ name: 'feature_lifecycle_stage_duration', - labelNames: ['feature_id', 'stage'], + labelNames: ['feature_id', 'stage', 'project_id'], help: 'Duration of feature lifecycle stages', }); @@ -294,6 +294,7 @@ export default class MetricsMonitor { .labels({ feature_id: stage.feature, stage: stage.stage, + project_id: stage.project, }) .observe(stage.duration); }); diff --git a/src/lib/routes/admin-api/instance-admin.ts b/src/lib/routes/admin-api/instance-admin.ts index 48716c140a..bbe7f96858 100644 --- a/src/lib/routes/admin-api/instance-admin.ts +++ b/src/lib/routes/admin-api/instance-admin.ts @@ -128,6 +128,7 @@ class InstanceAdminController extends Controller { featureLifeCycles: [ { feature: 'feature1', + project: 'default', stage: 'archived', duration: 2000, }, diff --git a/src/lib/types/experimental.ts b/src/lib/types/experimental.ts index f7753d257e..c203845baa 100644 --- a/src/lib/types/experimental.ts +++ b/src/lib/types/experimental.ts @@ -54,6 +54,7 @@ export type IFlagKey = | 'disableShowContextFieldSelectionValues' | 'projectOverviewRefactorFeedback' | 'featureLifecycle' + | 'featureLifecycleMetrics' | 'projectListFilterMyProjects' | 'projectsListNewCards' | 'parseProjectFromSession' diff --git a/src/lib/types/model.ts b/src/lib/types/model.ts index 06ceefda52..a3304edbd9 100644 --- a/src/lib/types/model.ts +++ b/src/lib/types/model.ts @@ -168,6 +168,7 @@ export interface IFeatureLifecycleStage { export type IFeatureLifecycleStageDuration = FeatureLifecycleStage & { duration: number; + project: string; }; export interface IFeatureDependency {