feat: duration in stage, add feature lifecycle prometheus metrics (#6973)

Introduce a new concept. Duration in stage.
Also add it into prometheus metric.

src/lib/features/feature-lifecycle/fake-feature-lifecycle-store.ts

@@ -2,6 +2,7 @@ import type {
     FeatureLifecycleStage,
     IFeatureLifecycleStore,
     FeatureLifecycleView,
+    FeatureLifecycleFullItem,
 } from './feature-lifecycle-store-type';
 
 export class FakeFeatureLifecycleStore implements IFeatureLifecycleStore {
@@ -35,6 +36,17 @@ export class FakeFeatureLifecycleStore implements IFeatureLifecycleStore {
         return this.lifecycles[feature] || [];
     }
 
+    async getAll(): Promise<FeatureLifecycleFullItem[]> {
+        const result = Object.entries(this.lifecycles).flatMap(
+            ([key, items]): FeatureLifecycleFullItem[] =>
+                items.map((item) => ({
+                    ...item,
+                    feature: key,
+                })),
+        );
+        return result;
+    }
+
     async delete(feature: string): Promise<void> {
         this.lifecycles[feature] = [];
     }

src/lib/features/feature-lifecycle/feature-lifecycle-service.test.ts

@@ -128,3 +128,71 @@ test('ignores lifecycle state updates when flag disabled', async () => {
 
     expect(lifecycle).toEqual([]);
 });
+
+test('can find feature lifecycle stage timings', async () => {
+    const eventBus = new EventEmitter();
+    const { featureLifecycleService, eventStore, environmentStore } =
+        createFakeFeatureLifecycleService({
+            flagResolver: { isEnabled: () => false },
+            eventBus,
+            getLogger: noLoggerProvider,
+        } as unknown as IUnleashConfig);
+    const now = new Date();
+    const minusOneMinute = new Date(now.getTime() - 1 * 60 * 1000);
+    const minusTenMinutes = new Date(now.getTime() - 10 * 60 * 1000);
+    const durations = featureLifecycleService.calculateStageDurations([
+        {
+            feature: 'a',
+            stage: 'initial',
+            enteredStageAt: minusTenMinutes,
+        },
+        {
+            feature: 'b',
+            stage: 'initial',
+            enteredStageAt: minusTenMinutes,
+        },
+        {
+            feature: 'a',
+            stage: 'pre-live',
+            enteredStageAt: minusOneMinute,
+        },
+        {
+            feature: 'b',
+            stage: 'live',
+            enteredStageAt: minusOneMinute,
+        },
+        {
+            feature: 'c',
+            stage: 'initial',
+            enteredStageAt: minusTenMinutes,
+        },
+    ]);
+
+    expect(durations).toMatchObject([
+        {
+            feature: 'a',
+            stage: 'initial',
+            duration: 9,
+        },
+        {
+            feature: 'a',
+            stage: 'pre-live',
+            duration: 1,
+        },
+        {
+            feature: 'b',
+            stage: 'initial',
+            duration: 9,
+        },
+        {
+            feature: 'b',
+            stage: 'live',
+            duration: 1,
+        },
+        {
+            feature: 'c',
+            stage: 'initial',
+            duration: 10,
+        },
+    ]);
+});

src/lib/features/feature-lifecycle/feature-lifecycle-service.ts

@@ -9,10 +9,12 @@ import {
     type IEnvironmentStore,
     type IEventStore,
     type IFeatureEnvironmentStore,
+    type IFeatureLifecycleStageDuration,
     type IFlagResolver,
     type IUnleashConfig,
 } from '../../types';
 import type {
+    FeatureLifecycleFullItem,
     FeatureLifecycleView,
     IFeatureLifecycleStore,
 } from './feature-lifecycle-store-type';
@@ -20,6 +22,7 @@ import EventEmitter from 'events';
 import type { Logger } from '../../logger';
 import type EventService from '../events/event-service';
 import type { ValidatedClientMetrics } from '../metrics/shared/schema';
+import { differenceInMinutes } from 'date-fns';
 
 export const STAGE_ENTERED = 'STAGE_ENTERED';
 
@@ -203,4 +206,51 @@ export class FeatureLifecycleService extends EventEmitter {
         await this.featureLifecycleStore.delete(feature);
         await this.featureInitialized(feature);
     }
+
+    public async getAllWithStageDuration(): Promise<
+        IFeatureLifecycleStageDuration[]
+    > {
+        const featureLifeCycles = await this.featureLifecycleStore.getAll();
+        return this.calculateStageDurations(featureLifeCycles);
+    }
+
+    public calculateStageDurations(
+        featureLifeCycles: FeatureLifecycleFullItem[],
+    ) {
+        const groupedByFeature = featureLifeCycles.reduce<{
+            [feature: string]: FeatureLifecycleFullItem[];
+        }>((acc, curr) => {
+            if (!acc[curr.feature]) {
+                acc[curr.feature] = [];
+            }
+            acc[curr.feature].push(curr);
+            return acc;
+        }, {});
+
+        const times: IFeatureLifecycleStageDuration[] = [];
+        Object.values(groupedByFeature).forEach((stages) => {
+            stages.sort(
+                (a, b) =>
+                    a.enteredStageAt.getTime() - b.enteredStageAt.getTime(),
+            );
+
+            stages.forEach((stage, index) => {
+                const nextStage = stages[index + 1];
+                const endTime = nextStage
+                    ? nextStage.enteredStageAt
+                    : new Date();
+                const duration = differenceInMinutes(
+                    endTime,
+                    stage.enteredStageAt,
+                );
+
+                times.push({
+                    feature: stage.feature,
+                    stage: stage.stage,
+                    duration,
+                });
+            });
+        });
+        return times;
+    }
 }

src/lib/features/feature-lifecycle/feature-lifecycle-store-type.ts

@@ -7,9 +7,14 @@ export type FeatureLifecycleStage = {
 
 export type FeatureLifecycleView = IFeatureLifecycleStage[];
 
+export type FeatureLifecycleFullItem = FeatureLifecycleStage & {
+    enteredStageAt: Date;
+};
+
 export interface IFeatureLifecycleStore {
     insert(featureLifecycleStages: FeatureLifecycleStage[]): Promise<void>;
     get(feature: string): Promise<FeatureLifecycleView>;
+    getAll(): Promise<FeatureLifecycleFullItem[]>;
     stageExists(stage: FeatureLifecycleStage): Promise<boolean>;
     delete(feature: string): Promise<void>;
     deleteStage(stage: FeatureLifecycleStage): Promise<void>;

src/lib/features/feature-lifecycle/feature-lifecycle-store.ts

@@ -2,6 +2,7 @@ import type {
     FeatureLifecycleStage,
     IFeatureLifecycleStore,
     FeatureLifecycleView,
+    FeatureLifecycleFullItem,
 } from './feature-lifecycle-store-type';
 import type { Db } from '../../db/db';
 import type { StageName } from '../../types';
@@ -9,7 +10,7 @@ import type { StageName } from '../../types';
 type DBType = {
     feature: string;
     stage: StageName;
-    created_at: Date;
+    created_at: string;
 };
 
 export class FeatureLifecycleStore implements IFeatureLifecycleStore {
@@ -53,7 +54,20 @@ export class FeatureLifecycleStore implements IFeatureLifecycleStore {
 
         return results.map(({ stage, created_at }: DBType) => ({
             stage,
-            enteredStageAt: created_at,
+            enteredStageAt: new Date(created_at),
+        }));
+    }
+
+    async getAll(): Promise<FeatureLifecycleFullItem[]> {
+        const results = await this.db('feature_lifecycles').orderBy(
+            'created_at',
+            'asc',
+        );
+
+        return results.map(({ feature, stage, created_at }: DBType) => ({
+            feature,
+            stage,
+            enteredStageAt: new Date(created_at),
         }));
     }
 

src/lib/features/instance-stats/createInstanceStatsService.ts

@@ -40,6 +40,10 @@ import FakeSettingStore from '../../../test/fixtures/fake-setting-store';
 import FakeSegmentStore from '../../../test/fixtures/fake-segment-store';
 import FakeStrategiesStore from '../../../test/fixtures/fake-strategies-store';
 import FakeFeatureStrategiesStore from '../feature-toggle/fakes/fake-feature-strategies-store';
+import {
+    createFakeFeatureLifecycleService,
+    createFeatureLifecycleService,
+} from '../feature-lifecycle/createFeatureLifecycle';
 
 export const createInstanceStatsService = (db: Db, config: IUnleashConfig) => {
     const { eventBus, getLogger, flagResolver } = config;
@@ -84,6 +88,10 @@ export const createInstanceStatsService = (db: Db, config: IUnleashConfig) => {
         getLogger,
         flagResolver,
     );
+    const { featureLifecycleService } = createFeatureLifecycleService(
+        db,
+        config,
+    );
     const instanceStatsServiceStores = {
         featureToggleStore,
         userStore,
@@ -125,6 +133,7 @@ export const createInstanceStatsService = (db: Db, config: IUnleashConfig) => {
         versionService,
         getActiveUsers,
         getProductionChanges,
+        featureLifecycleService,
     );
 
     return instanceStatsService;
@@ -146,6 +155,9 @@ export const createFakeInstanceStatsService = (config: IUnleashConfig) => {
     const eventStore = new FakeEventStore();
     const apiTokenStore = new FakeApiTokenStore();
     const clientMetricsStoreV2 = new FakeClientMetricsStoreV2();
+
+    const { featureLifecycleService } =
+        createFakeFeatureLifecycleService(config);
     const instanceStatsServiceStores = {
         featureToggleStore,
         userStore,
@@ -182,6 +194,7 @@ export const createFakeInstanceStatsService = (config: IUnleashConfig) => {
         versionService,
         getActiveUsers,
         getProductionChanges,
+        featureLifecycleService,
     );
 
     return instanceStatsService;

src/lib/features/instance-stats/instance-stats-service.test.ts

@@ -4,6 +4,7 @@ import createStores from '../../../test/fixtures/store';
 import VersionService from '../../services/version-service';
 import { createFakeGetActiveUsers } from './getActiveUsers';
 import { createFakeGetProductionChanges } from './getProductionChanges';
+import { createFakeFeatureLifecycleService } from '../feature-lifecycle/createFeatureLifecycle';
 
 let instanceStatsService: InstanceStatsService;
 let versionService: VersionService;
@@ -23,6 +24,7 @@ beforeEach(() => {
         versionService,
         createFakeGetActiveUsers(),
         createFakeGetProductionChanges(),
+        createFakeFeatureLifecycleService(config).featureLifecycleService,
     );
 
     jest.spyOn(instanceStatsService, 'refreshAppCountSnapshot');

src/lib/features/instance-stats/instance-stats-service.ts

@@ -22,11 +22,13 @@ import {
     FEATURES_EXPORTED,
     FEATURES_IMPORTED,
     type IApiTokenStore,
+    type IFeatureLifecycleStageDuration,
 } from '../../types';
 import { CUSTOM_ROOT_ROLE_TYPE } from '../../util';
 import type { GetActiveUsers } from './getActiveUsers';
 import type { ProjectModeCount } from '../project/project-store';
 import type { GetProductionChanges } from './getProductionChanges';
+import type { FeatureLifecycleService } from '../feature-lifecycle/feature-lifecycle-service';
 
 export type TimeRange = 'allTime' | '30d' | '7d';
 
@@ -60,6 +62,7 @@ export interface InstanceStats {
         enabledCount: number;
         variantCount: number;
     };
+    featureLifeCycles: IFeatureLifecycleStageDuration[];
 }
 
 export type InstanceStatsSigned = Omit<InstanceStats, 'projects'> & {
@@ -90,6 +93,8 @@ export class InstanceStatsService {
 
     private eventStore: IEventStore;
 
+    private featureLifecycleService: FeatureLifecycleService;
+
     private apiTokenStore: IApiTokenStore;
 
     private versionService: VersionService;
@@ -143,6 +148,7 @@ export class InstanceStatsService {
         versionService: VersionService,
         getActiveUsers: GetActiveUsers,
         getProductionChanges: GetProductionChanges,
+        featureLifecycleService: FeatureLifecycleService,
     ) {
         this.strategyStore = strategyStore;
         this.userStore = userStore;
@@ -157,6 +163,7 @@ export class InstanceStatsService {
         this.settingStore = settingStore;
         this.eventStore = eventStore;
         this.clientInstanceStore = clientInstanceStore;
+        this.featureLifecycleService = featureLifecycleService;
         this.logger = getLogger('services/stats-service.js');
         this.getActiveUsers = getActiveUsers;
         this.getProductionChanges = getProductionChanges;
@@ -243,6 +250,7 @@ export class InstanceStatsService {
             featureImports,
             productionChanges,
             previousDayMetricsBucketsCount,
+            featureLifeCycles,
         ] = await Promise.all([
             this.getToggleCount(),
             this.getArchivedToggleCount(),
@@ -266,6 +274,7 @@ export class InstanceStatsService {
             this.eventStore.filteredCount({ type: FEATURES_IMPORTED }),
             this.getProductionChanges(),
             this.clientMetricsStore.countPreviousDayHourlyMetricsBuckets(),
+            this.featureLifecycleService.getAllWithStageDuration(),
         ]);
 
         return {
@@ -298,6 +307,7 @@ export class InstanceStatsService {
             featureImports,
             productionChanges,
             previousDayMetricsBucketsCount,
+            featureLifeCycles,
         };
     }
 

src/lib/metrics.test.ts

@@ -20,6 +20,7 @@ import type { IEnvironmentStore, IUnleashStores } from './types';
 import FakeEnvironmentStore from './features/project-environments/fake-environment-store';
 import { SchedulerService } from './services';
 import noLogger from '../test/fixtures/no-logger';
+import { createFakeFeatureLifecycleService } from './features';
 
 const monitor = createMetricsMonitor();
 const eventBus = new EventEmitter();
@@ -45,12 +46,15 @@ beforeAll(async () => {
         createFakeGetActiveUsers(),
         createFakeGetProductionChanges(),
     );
+    const { featureLifecycleService } =
+        createFakeFeatureLifecycleService(config);
     statsService = new InstanceStatsService(
         stores,
         config,
         versionService,
         createFakeGetActiveUsers(),
         createFakeGetProductionChanges(),
+        featureLifecycleService,
     );
 
     schedulerService = new SchedulerService(
@@ -275,3 +279,8 @@ test('should collect metrics for project disabled numbers', async () => {
         /project_environments_disabled{project_id=\"default\"} 1/,
     );
 });
+
+test('should collect metrics for lifecycle', async () => {
+    const metrics = await prometheusRegister.metrics();
+    expect(metrics).toMatch(/feature_lifecycle_stage_duration/);
+});

src/lib/metrics.ts

@@ -259,6 +259,12 @@ export default class MetricsMonitor {
             help: 'Duration of mapFeaturesForClient function',
         });
 
+        const featureLifecycleStageDuration = createHistogram({
+            name: 'feature_lifecycle_stage_duration',
+            labelNames: ['feature_id', 'stage'],
+            help: 'Duration of feature lifecycle stages',
+        });
+
         const projectEnvironmentsDisabled = createCounter({
             name: 'project_environments_disabled',
             help: 'How many "environment disabled" events we have received for each project',
@@ -283,6 +289,15 @@ export default class MetricsMonitor {
                 serviceAccounts.reset();
                 serviceAccounts.set(stats.serviceAccounts);
 
+                stats.featureLifeCycles.forEach((stage) => {
+                    featureLifecycleStageDuration
+                        .labels({
+                            feature_id: stage.feature,
+                            stage: stage.stage,
+                        })
+                        .observe(stage.duration);
+                });
+
                 apiTokens.reset();
 
                 for (const [type, value] of stats.apiTokens) {
@@ -358,7 +373,10 @@ export default class MetricsMonitor {
 
                 rateLimits.reset();
                 rateLimits
-                    .labels({ endpoint: '/api/client/metrics', method: 'POST' })
+                    .labels({
+                        endpoint: '/api/client/metrics',
+                        method: 'POST',
+                    })
                     .set(config.metricsRateLimiting.clientMetricsMaxPerMinute);
                 rateLimits
                     .labels({
@@ -389,7 +407,10 @@ export default class MetricsMonitor {
                     })
                     .set(config.rateLimiting.createUserMaxPerMinute);
                 rateLimits
-                    .labels({ endpoint: '/auth/simple', method: 'POST' })
+                    .labels({
+                        endpoint: '/auth/simple',
+                        method: 'POST',
+                    })
                     .set(config.rateLimiting.simpleLoginMaxPerMinute);
                 rateLimits
                     .labels({
@@ -407,7 +428,6 @@ export default class MetricsMonitor {
                     );
             } catch (e) {}
         }
-
         await schedulerService.schedule(
             collectStaticCounters.bind(this),
             hoursToMilliseconds(2),
@@ -419,7 +439,12 @@ export default class MetricsMonitor {
             events.REQUEST_TIME,
             ({ path, method, time, statusCode, appName }) => {
                 requestDuration
-                    .labels({ path, method, status: statusCode, appName })
+                    .labels({
+                        path,
+                        method,
+                        status: statusCode,
+                        appName,
+                    })
                     .observe(time);
             },
         );
@@ -432,7 +457,10 @@ export default class MetricsMonitor {
             events.FUNCTION_TIME,
             ({ functionName, className, time }) => {
                 functionDuration
-                    .labels({ functionName, className })
+                    .labels({
+                        functionName,
+                        className,
+                    })
                     .observe(time);
             },
         );
@@ -446,7 +474,12 @@ export default class MetricsMonitor {
         });
 
         eventBus.on(events.DB_TIME, ({ store, action, time }) => {
-            dbDuration.labels({ store, action }).observe(time);
+            dbDuration
+                .labels({
+                    store,
+                    action,
+                })
+                .observe(time);
         });
 
         eventBus.on(events.PROXY_REPOSITORY_CREATED, () => {
@@ -704,6 +737,7 @@ export default class MetricsMonitor {
         }
     }
 }
+
 export function createMetricsMonitor(): MetricsMonitor {
     return new MetricsMonitor();
 }

src/lib/routes/admin-api/instance-admin.ts

@@ -125,6 +125,13 @@ class InstanceAdminController extends Controller {
                 variantCount: 100,
                 enabledCount: 200,
             },
+            featureLifeCycles: [
+                {
+                    feature: 'feature1',
+                    stage: 'archived',
+                    duration: 2000,
+                },
+            ],
         };
     }
 

src/lib/types/model.ts

@@ -7,6 +7,7 @@ import type { IProjectStats } from '../features/project/project-service';
 import type { CreateFeatureStrategySchema } from '../openapi';
 import type { ProjectEnvironment } from '../features/project/project-store-type';
 import type { FeatureSearchEnvironmentSchema } from '../openapi/spec/feature-search-environment-schema';
+import type { FeatureLifecycleStage } from '../features/feature-lifecycle/feature-lifecycle-store-type';
 
 export type Operator = (typeof ALL_OPERATORS)[number];
 
@@ -159,11 +160,16 @@ export type StageName =
     | 'live'
     | 'completed'
     | 'archived';
+
 export interface IFeatureLifecycleStage {
     stage: StageName;
     enteredStageAt: Date;
 }
 
+export type IFeatureLifecycleStageDuration = FeatureLifecycleStage & {
+    duration: number;
+};
+
 export interface IFeatureDependency {
     feature: string;
     dependency: IDependency;