From 02440dfed2e11f2cd5f834478ef5f77f2c4e431e Mon Sep 17 00:00:00 2001 From: Jaanus Sellin Date: Wed, 8 May 2024 11:33:51 +0300 Subject: [PATCH] feat: duration in stage, add feature lifecycle prometheus metrics (#6973) Introduce a new concept. Duration in stage. Also add it into prometheus metric. --- .../fake-feature-lifecycle-store.ts | 12 ++++ .../feature-lifecycle-service.test.ts | 68 +++++++++++++++++++ .../feature-lifecycle-service.ts | 50 ++++++++++++++ .../feature-lifecycle-store-type.ts | 5 ++ .../feature-lifecycle-store.ts | 18 ++++- .../createInstanceStatsService.ts | 13 ++++ .../instance-stats-service.test.ts | 2 + .../instance-stats/instance-stats-service.ts | 10 +++ src/lib/metrics.test.ts | 9 +++ src/lib/metrics.ts | 46 +++++++++++-- src/lib/routes/admin-api/instance-admin.ts | 7 ++ src/lib/types/model.ts | 6 ++ 12 files changed, 238 insertions(+), 8 deletions(-) diff --git a/src/lib/features/feature-lifecycle/fake-feature-lifecycle-store.ts b/src/lib/features/feature-lifecycle/fake-feature-lifecycle-store.ts index ff686bffe2..cea1c52a11 100644 --- a/src/lib/features/feature-lifecycle/fake-feature-lifecycle-store.ts +++ b/src/lib/features/feature-lifecycle/fake-feature-lifecycle-store.ts @@ -2,6 +2,7 @@ import type { FeatureLifecycleStage, IFeatureLifecycleStore, FeatureLifecycleView, + FeatureLifecycleFullItem, } from './feature-lifecycle-store-type'; export class FakeFeatureLifecycleStore implements IFeatureLifecycleStore { @@ -35,6 +36,17 @@ export class FakeFeatureLifecycleStore implements IFeatureLifecycleStore { return this.lifecycles[feature] || []; } + async getAll(): Promise { + const result = Object.entries(this.lifecycles).flatMap( + ([key, items]): FeatureLifecycleFullItem[] => + items.map((item) => ({ + ...item, + feature: key, + })), + ); + return result; + } + async delete(feature: string): Promise { this.lifecycles[feature] = []; } diff --git a/src/lib/features/feature-lifecycle/feature-lifecycle-service.test.ts b/src/lib/features/feature-lifecycle/feature-lifecycle-service.test.ts index 82f5f962bc..3ce51ac04a 100644 --- a/src/lib/features/feature-lifecycle/feature-lifecycle-service.test.ts +++ b/src/lib/features/feature-lifecycle/feature-lifecycle-service.test.ts @@ -128,3 +128,71 @@ test('ignores lifecycle state updates when flag disabled', async () => { expect(lifecycle).toEqual([]); }); + +test('can find feature lifecycle stage timings', async () => { + const eventBus = new EventEmitter(); + const { featureLifecycleService, eventStore, environmentStore } = + createFakeFeatureLifecycleService({ + flagResolver: { isEnabled: () => false }, + eventBus, + getLogger: noLoggerProvider, + } as unknown as IUnleashConfig); + const now = new Date(); + const minusOneMinute = new Date(now.getTime() - 1 * 60 * 1000); + const minusTenMinutes = new Date(now.getTime() - 10 * 60 * 1000); + const durations = featureLifecycleService.calculateStageDurations([ + { + feature: 'a', + stage: 'initial', + enteredStageAt: minusTenMinutes, + }, + { + feature: 'b', + stage: 'initial', + enteredStageAt: minusTenMinutes, + }, + { + feature: 'a', + stage: 'pre-live', + enteredStageAt: minusOneMinute, + }, + { + feature: 'b', + stage: 'live', + enteredStageAt: minusOneMinute, + }, + { + feature: 'c', + stage: 'initial', + enteredStageAt: minusTenMinutes, + }, + ]); + + expect(durations).toMatchObject([ + { + feature: 'a', + stage: 'initial', + duration: 9, + }, + { + feature: 'a', + stage: 'pre-live', + duration: 1, + }, + { + feature: 'b', + stage: 'initial', + duration: 9, + }, + { + feature: 'b', + stage: 'live', + duration: 1, + }, + { + feature: 'c', + stage: 'initial', + duration: 10, + }, + ]); +}); diff --git a/src/lib/features/feature-lifecycle/feature-lifecycle-service.ts b/src/lib/features/feature-lifecycle/feature-lifecycle-service.ts index 351a1e7a71..ef3be33029 100644 --- a/src/lib/features/feature-lifecycle/feature-lifecycle-service.ts +++ b/src/lib/features/feature-lifecycle/feature-lifecycle-service.ts @@ -9,10 +9,12 @@ import { type IEnvironmentStore, type IEventStore, type IFeatureEnvironmentStore, + type IFeatureLifecycleStageDuration, type IFlagResolver, type IUnleashConfig, } from '../../types'; import type { + FeatureLifecycleFullItem, FeatureLifecycleView, IFeatureLifecycleStore, } from './feature-lifecycle-store-type'; @@ -20,6 +22,7 @@ import EventEmitter from 'events'; import type { Logger } from '../../logger'; import type EventService from '../events/event-service'; import type { ValidatedClientMetrics } from '../metrics/shared/schema'; +import { differenceInMinutes } from 'date-fns'; export const STAGE_ENTERED = 'STAGE_ENTERED'; @@ -203,4 +206,51 @@ export class FeatureLifecycleService extends EventEmitter { await this.featureLifecycleStore.delete(feature); await this.featureInitialized(feature); } + + public async getAllWithStageDuration(): Promise< + IFeatureLifecycleStageDuration[] + > { + const featureLifeCycles = await this.featureLifecycleStore.getAll(); + return this.calculateStageDurations(featureLifeCycles); + } + + public calculateStageDurations( + featureLifeCycles: FeatureLifecycleFullItem[], + ) { + const groupedByFeature = featureLifeCycles.reduce<{ + [feature: string]: FeatureLifecycleFullItem[]; + }>((acc, curr) => { + if (!acc[curr.feature]) { + acc[curr.feature] = []; + } + acc[curr.feature].push(curr); + return acc; + }, {}); + + const times: IFeatureLifecycleStageDuration[] = []; + Object.values(groupedByFeature).forEach((stages) => { + stages.sort( + (a, b) => + a.enteredStageAt.getTime() - b.enteredStageAt.getTime(), + ); + + stages.forEach((stage, index) => { + const nextStage = stages[index + 1]; + const endTime = nextStage + ? nextStage.enteredStageAt + : new Date(); + const duration = differenceInMinutes( + endTime, + stage.enteredStageAt, + ); + + times.push({ + feature: stage.feature, + stage: stage.stage, + duration, + }); + }); + }); + return times; + } } diff --git a/src/lib/features/feature-lifecycle/feature-lifecycle-store-type.ts b/src/lib/features/feature-lifecycle/feature-lifecycle-store-type.ts index 52e44fc107..9deaef74ef 100644 --- a/src/lib/features/feature-lifecycle/feature-lifecycle-store-type.ts +++ b/src/lib/features/feature-lifecycle/feature-lifecycle-store-type.ts @@ -7,9 +7,14 @@ export type FeatureLifecycleStage = { export type FeatureLifecycleView = IFeatureLifecycleStage[]; +export type FeatureLifecycleFullItem = FeatureLifecycleStage & { + enteredStageAt: Date; +}; + export interface IFeatureLifecycleStore { insert(featureLifecycleStages: FeatureLifecycleStage[]): Promise; get(feature: string): Promise; + getAll(): Promise; stageExists(stage: FeatureLifecycleStage): Promise; delete(feature: string): Promise; deleteStage(stage: FeatureLifecycleStage): Promise; diff --git a/src/lib/features/feature-lifecycle/feature-lifecycle-store.ts b/src/lib/features/feature-lifecycle/feature-lifecycle-store.ts index d34f22a617..1c8a843edf 100644 --- a/src/lib/features/feature-lifecycle/feature-lifecycle-store.ts +++ b/src/lib/features/feature-lifecycle/feature-lifecycle-store.ts @@ -2,6 +2,7 @@ import type { FeatureLifecycleStage, IFeatureLifecycleStore, FeatureLifecycleView, + FeatureLifecycleFullItem, } from './feature-lifecycle-store-type'; import type { Db } from '../../db/db'; import type { StageName } from '../../types'; @@ -9,7 +10,7 @@ import type { StageName } from '../../types'; type DBType = { feature: string; stage: StageName; - created_at: Date; + created_at: string; }; export class FeatureLifecycleStore implements IFeatureLifecycleStore { @@ -53,7 +54,20 @@ export class FeatureLifecycleStore implements IFeatureLifecycleStore { return results.map(({ stage, created_at }: DBType) => ({ stage, - enteredStageAt: created_at, + enteredStageAt: new Date(created_at), + })); + } + + async getAll(): Promise { + const results = await this.db('feature_lifecycles').orderBy( + 'created_at', + 'asc', + ); + + return results.map(({ feature, stage, created_at }: DBType) => ({ + feature, + stage, + enteredStageAt: new Date(created_at), })); } diff --git a/src/lib/features/instance-stats/createInstanceStatsService.ts b/src/lib/features/instance-stats/createInstanceStatsService.ts index 8784d4b78e..ddbf0302ba 100644 --- a/src/lib/features/instance-stats/createInstanceStatsService.ts +++ b/src/lib/features/instance-stats/createInstanceStatsService.ts @@ -40,6 +40,10 @@ import FakeSettingStore from '../../../test/fixtures/fake-setting-store'; import FakeSegmentStore from '../../../test/fixtures/fake-segment-store'; import FakeStrategiesStore from '../../../test/fixtures/fake-strategies-store'; import FakeFeatureStrategiesStore from '../feature-toggle/fakes/fake-feature-strategies-store'; +import { + createFakeFeatureLifecycleService, + createFeatureLifecycleService, +} from '../feature-lifecycle/createFeatureLifecycle'; export const createInstanceStatsService = (db: Db, config: IUnleashConfig) => { const { eventBus, getLogger, flagResolver } = config; @@ -84,6 +88,10 @@ export const createInstanceStatsService = (db: Db, config: IUnleashConfig) => { getLogger, flagResolver, ); + const { featureLifecycleService } = createFeatureLifecycleService( + db, + config, + ); const instanceStatsServiceStores = { featureToggleStore, userStore, @@ -125,6 +133,7 @@ export const createInstanceStatsService = (db: Db, config: IUnleashConfig) => { versionService, getActiveUsers, getProductionChanges, + featureLifecycleService, ); return instanceStatsService; @@ -146,6 +155,9 @@ export const createFakeInstanceStatsService = (config: IUnleashConfig) => { const eventStore = new FakeEventStore(); const apiTokenStore = new FakeApiTokenStore(); const clientMetricsStoreV2 = new FakeClientMetricsStoreV2(); + + const { featureLifecycleService } = + createFakeFeatureLifecycleService(config); const instanceStatsServiceStores = { featureToggleStore, userStore, @@ -182,6 +194,7 @@ export const createFakeInstanceStatsService = (config: IUnleashConfig) => { versionService, getActiveUsers, getProductionChanges, + featureLifecycleService, ); return instanceStatsService; diff --git a/src/lib/features/instance-stats/instance-stats-service.test.ts b/src/lib/features/instance-stats/instance-stats-service.test.ts index d3f423bec1..6b11afc68b 100644 --- a/src/lib/features/instance-stats/instance-stats-service.test.ts +++ b/src/lib/features/instance-stats/instance-stats-service.test.ts @@ -4,6 +4,7 @@ import createStores from '../../../test/fixtures/store'; import VersionService from '../../services/version-service'; import { createFakeGetActiveUsers } from './getActiveUsers'; import { createFakeGetProductionChanges } from './getProductionChanges'; +import { createFakeFeatureLifecycleService } from '../feature-lifecycle/createFeatureLifecycle'; let instanceStatsService: InstanceStatsService; let versionService: VersionService; @@ -23,6 +24,7 @@ beforeEach(() => { versionService, createFakeGetActiveUsers(), createFakeGetProductionChanges(), + createFakeFeatureLifecycleService(config).featureLifecycleService, ); jest.spyOn(instanceStatsService, 'refreshAppCountSnapshot'); diff --git a/src/lib/features/instance-stats/instance-stats-service.ts b/src/lib/features/instance-stats/instance-stats-service.ts index 04e6ec30d1..9f48bc5f9b 100644 --- a/src/lib/features/instance-stats/instance-stats-service.ts +++ b/src/lib/features/instance-stats/instance-stats-service.ts @@ -22,11 +22,13 @@ import { FEATURES_EXPORTED, FEATURES_IMPORTED, type IApiTokenStore, + type IFeatureLifecycleStageDuration, } from '../../types'; import { CUSTOM_ROOT_ROLE_TYPE } from '../../util'; import type { GetActiveUsers } from './getActiveUsers'; import type { ProjectModeCount } from '../project/project-store'; import type { GetProductionChanges } from './getProductionChanges'; +import type { FeatureLifecycleService } from '../feature-lifecycle/feature-lifecycle-service'; export type TimeRange = 'allTime' | '30d' | '7d'; @@ -60,6 +62,7 @@ export interface InstanceStats { enabledCount: number; variantCount: number; }; + featureLifeCycles: IFeatureLifecycleStageDuration[]; } export type InstanceStatsSigned = Omit & { @@ -90,6 +93,8 @@ export class InstanceStatsService { private eventStore: IEventStore; + private featureLifecycleService: FeatureLifecycleService; + private apiTokenStore: IApiTokenStore; private versionService: VersionService; @@ -143,6 +148,7 @@ export class InstanceStatsService { versionService: VersionService, getActiveUsers: GetActiveUsers, getProductionChanges: GetProductionChanges, + featureLifecycleService: FeatureLifecycleService, ) { this.strategyStore = strategyStore; this.userStore = userStore; @@ -157,6 +163,7 @@ export class InstanceStatsService { this.settingStore = settingStore; this.eventStore = eventStore; this.clientInstanceStore = clientInstanceStore; + this.featureLifecycleService = featureLifecycleService; this.logger = getLogger('services/stats-service.js'); this.getActiveUsers = getActiveUsers; this.getProductionChanges = getProductionChanges; @@ -243,6 +250,7 @@ export class InstanceStatsService { featureImports, productionChanges, previousDayMetricsBucketsCount, + featureLifeCycles, ] = await Promise.all([ this.getToggleCount(), this.getArchivedToggleCount(), @@ -266,6 +274,7 @@ export class InstanceStatsService { this.eventStore.filteredCount({ type: FEATURES_IMPORTED }), this.getProductionChanges(), this.clientMetricsStore.countPreviousDayHourlyMetricsBuckets(), + this.featureLifecycleService.getAllWithStageDuration(), ]); return { @@ -298,6 +307,7 @@ export class InstanceStatsService { featureImports, productionChanges, previousDayMetricsBucketsCount, + featureLifeCycles, }; } diff --git a/src/lib/metrics.test.ts b/src/lib/metrics.test.ts index f1be259d33..42d6826864 100644 --- a/src/lib/metrics.test.ts +++ b/src/lib/metrics.test.ts @@ -20,6 +20,7 @@ import type { IEnvironmentStore, IUnleashStores } from './types'; import FakeEnvironmentStore from './features/project-environments/fake-environment-store'; import { SchedulerService } from './services'; import noLogger from '../test/fixtures/no-logger'; +import { createFakeFeatureLifecycleService } from './features'; const monitor = createMetricsMonitor(); const eventBus = new EventEmitter(); @@ -45,12 +46,15 @@ beforeAll(async () => { createFakeGetActiveUsers(), createFakeGetProductionChanges(), ); + const { featureLifecycleService } = + createFakeFeatureLifecycleService(config); statsService = new InstanceStatsService( stores, config, versionService, createFakeGetActiveUsers(), createFakeGetProductionChanges(), + featureLifecycleService, ); schedulerService = new SchedulerService( @@ -275,3 +279,8 @@ test('should collect metrics for project disabled numbers', async () => { /project_environments_disabled{project_id=\"default\"} 1/, ); }); + +test('should collect metrics for lifecycle', async () => { + const metrics = await prometheusRegister.metrics(); + expect(metrics).toMatch(/feature_lifecycle_stage_duration/); +}); diff --git a/src/lib/metrics.ts b/src/lib/metrics.ts index 927c1811d2..ff646b5f3f 100644 --- a/src/lib/metrics.ts +++ b/src/lib/metrics.ts @@ -259,6 +259,12 @@ export default class MetricsMonitor { help: 'Duration of mapFeaturesForClient function', }); + const featureLifecycleStageDuration = createHistogram({ + name: 'feature_lifecycle_stage_duration', + labelNames: ['feature_id', 'stage'], + help: 'Duration of feature lifecycle stages', + }); + const projectEnvironmentsDisabled = createCounter({ name: 'project_environments_disabled', help: 'How many "environment disabled" events we have received for each project', @@ -283,6 +289,15 @@ export default class MetricsMonitor { serviceAccounts.reset(); serviceAccounts.set(stats.serviceAccounts); + stats.featureLifeCycles.forEach((stage) => { + featureLifecycleStageDuration + .labels({ + feature_id: stage.feature, + stage: stage.stage, + }) + .observe(stage.duration); + }); + apiTokens.reset(); for (const [type, value] of stats.apiTokens) { @@ -358,7 +373,10 @@ export default class MetricsMonitor { rateLimits.reset(); rateLimits - .labels({ endpoint: '/api/client/metrics', method: 'POST' }) + .labels({ + endpoint: '/api/client/metrics', + method: 'POST', + }) .set(config.metricsRateLimiting.clientMetricsMaxPerMinute); rateLimits .labels({ @@ -389,7 +407,10 @@ export default class MetricsMonitor { }) .set(config.rateLimiting.createUserMaxPerMinute); rateLimits - .labels({ endpoint: '/auth/simple', method: 'POST' }) + .labels({ + endpoint: '/auth/simple', + method: 'POST', + }) .set(config.rateLimiting.simpleLoginMaxPerMinute); rateLimits .labels({ @@ -407,7 +428,6 @@ export default class MetricsMonitor { ); } catch (e) {} } - await schedulerService.schedule( collectStaticCounters.bind(this), hoursToMilliseconds(2), @@ -419,7 +439,12 @@ export default class MetricsMonitor { events.REQUEST_TIME, ({ path, method, time, statusCode, appName }) => { requestDuration - .labels({ path, method, status: statusCode, appName }) + .labels({ + path, + method, + status: statusCode, + appName, + }) .observe(time); }, ); @@ -432,7 +457,10 @@ export default class MetricsMonitor { events.FUNCTION_TIME, ({ functionName, className, time }) => { functionDuration - .labels({ functionName, className }) + .labels({ + functionName, + className, + }) .observe(time); }, ); @@ -446,7 +474,12 @@ export default class MetricsMonitor { }); eventBus.on(events.DB_TIME, ({ store, action, time }) => { - dbDuration.labels({ store, action }).observe(time); + dbDuration + .labels({ + store, + action, + }) + .observe(time); }); eventBus.on(events.PROXY_REPOSITORY_CREATED, () => { @@ -704,6 +737,7 @@ export default class MetricsMonitor { } } } + export function createMetricsMonitor(): MetricsMonitor { return new MetricsMonitor(); } diff --git a/src/lib/routes/admin-api/instance-admin.ts b/src/lib/routes/admin-api/instance-admin.ts index 65e1b457b7..48716c140a 100644 --- a/src/lib/routes/admin-api/instance-admin.ts +++ b/src/lib/routes/admin-api/instance-admin.ts @@ -125,6 +125,13 @@ class InstanceAdminController extends Controller { variantCount: 100, enabledCount: 200, }, + featureLifeCycles: [ + { + feature: 'feature1', + stage: 'archived', + duration: 2000, + }, + ], }; } diff --git a/src/lib/types/model.ts b/src/lib/types/model.ts index 63460ab199..e3a2eb4ead 100644 --- a/src/lib/types/model.ts +++ b/src/lib/types/model.ts @@ -7,6 +7,7 @@ import type { IProjectStats } from '../features/project/project-service'; import type { CreateFeatureStrategySchema } from '../openapi'; import type { ProjectEnvironment } from '../features/project/project-store-type'; import type { FeatureSearchEnvironmentSchema } from '../openapi/spec/feature-search-environment-schema'; +import type { FeatureLifecycleStage } from '../features/feature-lifecycle/feature-lifecycle-store-type'; export type Operator = (typeof ALL_OPERATORS)[number]; @@ -159,11 +160,16 @@ export type StageName = | 'live' | 'completed' | 'archived'; + export interface IFeatureLifecycleStage { stage: StageName; enteredStageAt: Date; } +export type IFeatureLifecycleStageDuration = FeatureLifecycleStage & { + duration: number; +}; + export interface IFeatureDependency { feature: string; dependency: IDependency;