feat: store memory footprints to grafana (#9001)

When there is new revision, we will start storing memory footprint for
old client-api and the new delta-api.
We will be sending it as prometheus metrics.

The memory size will only be recalculated if revision changes, which
does not happen very often.

src/lib/features/client-feature-toggles/client-feature-toggle.controller.ts

@@ -35,6 +35,7 @@ import {
 import type ConfigurationRevisionService from '../feature-toggle/configuration-revision-service';
 import type { ClientFeatureToggleService } from './client-feature-toggle-service';
 import {
+    CLIENT_FEATURES_MEMORY,
     CLIENT_METRICS_NAMEPREFIX,
     CLIENT_METRICS_TAGS,
 } from '../../internals';
@@ -69,6 +70,8 @@ export default class FeatureController extends Controller {
 
     private eventBus: EventEmitter;
 
+    private clientFeaturesCacheMap = new Map<string, number>();
+
     private featuresAndSegments: (
         query: IFeatureToggleQuery,
         etag: string,
@@ -162,6 +165,32 @@ export default class FeatureController extends Controller {
     private async resolveFeaturesAndSegments(
         query?: IFeatureToggleQuery,
     ): Promise<[FeatureConfigurationClient[], IClientSegment[]]> {
+        if (this.flagResolver.isEnabled('deltaApi')) {
+            const features =
+                await this.clientFeatureToggleService.getClientFeatures(query);
+
+            const segments =
+                await this.clientFeatureToggleService.getActiveSegmentsForClient();
+
+            try {
+                const featuresSize = this.getCacheSizeInBytes(features);
+                const segmentsSize = this.getCacheSizeInBytes(segments);
+                this.clientFeaturesCacheMap.set(
+                    JSON.stringify(query),
+                    featuresSize + segmentsSize,
+                );
+
+                await this.clientFeatureToggleService.getClientDelta(
+                    undefined,
+                    query!,
+                );
+                this.storeFootprint();
+            } catch (e) {
+                this.logger.error('Delta diff failed', e);
+            }
+
+            return [features, segments];
+        }
         return Promise.all([
             this.clientFeatureToggleService.getClientFeatures(query),
             this.clientFeatureToggleService.getActiveSegmentsForClient(),
@@ -270,7 +299,6 @@ export default class FeatureController extends Controller {
             query,
             etag,
         );
-
         if (this.clientSpecService.requestSupportsSpec(req, 'segments')) {
             this.openApiService.respondWithValidation(
                 200,
@@ -335,4 +363,17 @@ export default class FeatureController extends Controller {
             },
         );
     }
+
+    storeFootprint() {
+        let memory = 0;
+        for (const value of this.clientFeaturesCacheMap.values()) {
+            memory += value;
+        }
+        this.eventBus.emit(CLIENT_FEATURES_MEMORY, { memory });
+    }
+
+    getCacheSizeInBytes(value: any): number {
+        const jsonString = JSON.stringify(value);
+        return Buffer.byteLength(jsonString, 'utf8');
+    }
 }

src/lib/features/client-feature-toggles/delta/client-feature-toggle-delta.ts

@@ -5,6 +5,7 @@ import type {
     IFeatureToggleQuery,
     IFlagResolver,
     ISegmentReadModel,
+    IUnleashConfig,
 } from '../../../types';
 import type ConfigurationRevisionService from '../../feature-toggle/configuration-revision-service';
 import { UPDATE_REVISION } from '../../feature-toggle/configuration-revision-service';
@@ -13,6 +14,9 @@ import type {
     FeatureConfigurationDeltaClient,
     IClientFeatureToggleDeltaReadModel,
 } from './client-feature-toggle-delta-read-model-type';
+import { CLIENT_DELTA_MEMORY } from '../../../metric-events';
+import type EventEmitter from 'events';
+import type { Logger } from '../../../logger';
 
 type DeletedFeature = {
     name: string;
@@ -86,7 +90,6 @@ export const calculateRequiredClientRevision = (
     const targetedRevisions = revisions.filter(
         (revision) => revision.revisionId > requiredRevisionId,
     );
-    console.log('targeted revisions', targetedRevisions);
     const projectFeatureRevisions = targetedRevisions.map((revision) =>
         filterRevisionByProject(revision, projects),
     );
@@ -105,20 +108,23 @@ export class ClientFeatureToggleDelta {
 
     private currentRevisionId: number = 0;
 
-    private interval: NodeJS.Timer;
-
     private flagResolver: IFlagResolver;
 
     private configurationRevisionService: ConfigurationRevisionService;
 
     private readonly segmentReadModel: ISegmentReadModel;
 
+    private eventBus: EventEmitter;
+
+    private readonly logger: Logger;
+
     constructor(
         clientFeatureToggleDeltaReadModel: IClientFeatureToggleDeltaReadModel,
         segmentReadModel: ISegmentReadModel,
         eventStore: IEventStore,
         configurationRevisionService: ConfigurationRevisionService,
         flagResolver: IFlagResolver,
+        config: IUnleashConfig,
     ) {
         this.eventStore = eventStore;
         this.configurationRevisionService = configurationRevisionService;
@@ -126,6 +132,8 @@ export class ClientFeatureToggleDelta {
             clientFeatureToggleDeltaReadModel;
         this.flagResolver = flagResolver;
         this.segmentReadModel = segmentReadModel;
+        this.eventBus = config.eventBus;
+        this.logger = config.getLogger('delta/client-feature-toggle-delta.js');
         this.onUpdateRevisionEvent = this.onUpdateRevisionEvent.bind(this);
         this.delta = {};
 
@@ -161,6 +169,8 @@ export class ClientFeatureToggleDelta {
             await this.updateSegments();
         }
 
+        // TODO: 19.12 this logic seems to be not logical, when no revisionId is coming, it should not go to db, but take latest from cache
+
         // Should get the latest state if revision does not exist or if sdkRevision is not present
         // We should be able to do this without going to the database by merging revisions from the delta with
         // the base case
@@ -203,12 +213,13 @@ export class ClientFeatureToggleDelta {
 
     private async onUpdateRevisionEvent() {
         if (this.flagResolver.isEnabled('deltaApi')) {
-            await this.listenToRevisionChange();
+            await this.updateFeaturesDelta();
             await this.updateSegments();
+            this.storeFootprint();
         }
     }
 
-    public async listenToRevisionChange() {
+    public async updateFeaturesDelta() {
         const keys = Object.keys(this.delta);
 
         if (keys.length === 0) return;
@@ -248,7 +259,6 @@ export class ClientFeatureToggleDelta {
                 removed,
             });
         }
-
         this.currentRevisionId = latestRevision;
     }
 
@@ -279,8 +289,9 @@ export class ClientFeatureToggleDelta {
                 removed: [],
             },
         ]);
-
         this.delta[environment] = delta;
+
+        this.storeFootprint();
     }
 
     async getClientFeatures(
@@ -294,4 +305,20 @@ export class ClientFeatureToggleDelta {
     private async updateSegments(): Promise<void> {
         this.segments = await this.segmentReadModel.getActiveForClient();
     }
+
+    storeFootprint() {
+        try {
+            const featuresMemory = this.getCacheSizeInBytes(this.delta);
+            const segmentsMemory = this.getCacheSizeInBytes(this.segments);
+            const memory = featuresMemory + segmentsMemory;
+            this.eventBus.emit(CLIENT_DELTA_MEMORY, { memory });
+        } catch (e) {
+            this.logger.error('Client delta footprint error', e);
+        }
+    }
+
+    getCacheSizeInBytes(value: any): number {
+        const jsonString = JSON.stringify(value);
+        return Buffer.byteLength(jsonString, 'utf8');
+    }
 }

src/lib/features/client-feature-toggles/delta/createClientFeatureToggleDelta.ts

@@ -28,6 +28,7 @@ export const createClientFeatureToggleDelta = (
         eventStore,
         configurationRevisionService,
         flagResolver,
+        config,
     );
 
     return clientFeatureToggleDelta;

src/lib/features/client-feature-toggles/delta/revision-delta.ts

@@ -12,7 +12,7 @@ export class RevisionDelta {
     private delta: Revision[];
     private maxLength: number;
 
-    constructor(data: Revision[] = [], maxLength: number = 100) {
+    constructor(data: Revision[] = [], maxLength: number = 20) {
         this.delta = data;
         this.maxLength = maxLength;
     }

src/lib/metric-events.ts

@@ -16,6 +16,8 @@ const REQUEST_ORIGIN = 'request_origin' as const;
 const ADDON_EVENTS_HANDLED = 'addon-event-handled' as const;
 const CLIENT_METRICS_NAMEPREFIX = 'client-api-nameprefix';
 const CLIENT_METRICS_TAGS = 'client-api-tags';
+const CLIENT_FEATURES_MEMORY = 'client_features_memory';
+const CLIENT_DELTA_MEMORY = 'client_delta_memory';
 
 type MetricEvent =
     | typeof REQUEST_TIME
@@ -32,7 +34,9 @@ type MetricEvent =
     | typeof EXCEEDS_LIMIT
     | typeof REQUEST_ORIGIN
     | typeof CLIENT_METRICS_NAMEPREFIX
-    | typeof CLIENT_METRICS_TAGS;
+    | typeof CLIENT_METRICS_TAGS
+    | typeof CLIENT_FEATURES_MEMORY
+    | typeof CLIENT_DELTA_MEMORY;
 
 type RequestOriginEventPayload = {
     type: 'UI' | 'API';
@@ -82,6 +86,8 @@ export {
     ADDON_EVENTS_HANDLED,
     CLIENT_METRICS_NAMEPREFIX,
     CLIENT_METRICS_TAGS,
+    CLIENT_FEATURES_MEMORY,
+    CLIENT_DELTA_MEMORY,
     type MetricEvent,
     type MetricEventPayload,
     emitMetricEvent,

src/lib/metrics.ts

@@ -624,6 +624,16 @@ export function registerPrometheusMetrics(
         help: 'Number of API tokens without a project',
     });
 
+    const clientFeaturesMemory = createGauge({
+        name: 'client_features_memory',
+        help: 'The amount of memory client features endpoint is using for caching',
+    });
+
+    const clientDeltaMemory = createGauge({
+        name: 'client_delta_memory',
+        help: 'The amount of memory client features delta endpoint is using for caching',
+    });
+
     const orphanedTokensActive = createGauge({
         name: 'orphaned_api_tokens_active',
         help: 'Number of API tokens without a project, last seen within 3 months',
@@ -752,6 +762,16 @@ export function registerPrometheusMetrics(
         tagsUsed.inc();
     });
 
+    eventBus.on(events.CLIENT_FEATURES_MEMORY, (event: { memory: number }) => {
+        clientFeaturesMemory.reset();
+        clientFeaturesMemory.set(event.memory);
+    });
+
+    eventBus.on(events.CLIENT_DELTA_MEMORY, (event: { memory: number }) => {
+        clientDeltaMemory.reset();
+        clientDeltaMemory.set(event.memory);
+    });
+
     events.onMetricEvent(
         eventBus,
         events.REQUEST_ORIGIN,