chore: unknown flags with environment (#10325)

https://linear.app/unleash/issue/2-3681/add-environment-to-unknown-flag-reports

This does a few things:
 - Adds environment to our unknown flag reports
 - Increases our max unknown flag reports from 10 to 100
 - Reduces our clean up interval from 24 to 2 hours
- Flattens our response (easier to get started with and display in a
table, we can later decide if we want to group this data)

<img width="496" alt="image"
src="https://github.com/user-attachments/assets/11350639-9f7f-4011-8b39-b135c820ca21"
/>

src/lib/features/metrics/client-metrics/metrics-service-v2.ts

@@ -26,10 +26,7 @@ import {
 import type { ClientMetricsSchema } from '../../../../lib/openapi/index.js';
 import { nameSchema } from '../../../schema/feature-schema.js';
 import memoizee from 'memoizee';
-import {
+import type { UnknownFlagsService } from '../unknown-flags/unknown-flags-service.js';
-    MAX_UNKNOWN_FLAGS,
-    type UnknownFlagsService,
-} from '../unknown-flags/unknown-flags-service.js';
 import {
     type Metric,
     MetricsTranslator,
@@ -133,23 +130,17 @@ export default class ClientMetricsServiceV2 {
         validatedToggleNames: string[];
         unknownToggleNames: string[];
     }> {
-        let unknownToggleNames: string[] = [];
-
         const existingFlags = await this.cachedFeatureNames();
         const existingNames = toggleNames.filter((name) =>
             existingFlags.includes(name),
         );
 
+        let unknownToggleNames: string[] = [];
         if (this.flagResolver.isEnabled('reportUnknownFlags')) {
             try {
-                const nonExistingNames = toggleNames.filter(
+                unknownToggleNames = toggleNames.filter(
                     (name) => !existingFlags.includes(name),
                 );
-
-                unknownToggleNames = nonExistingNames.slice(
-                    0,
-                    MAX_UNKNOWN_FLAGS,
-                );
             } catch (e) {
                 this.logger.error(e);
             }
@@ -244,16 +235,19 @@ export default class ClientMetricsServiceV2 {
             this.config.eventBus.emit(CLIENT_REGISTER, heartbeatEvent);
         }
 
+        const environment = value.environment ?? 'default';
+
         if (unknownToggleNames.length > 0) {
             const unknownFlags = unknownToggleNames.map((name) => ({
                 name,
                 appName: value.appName,
                 seenAt: value.bucket.stop,
+                environment,
             }));
             this.logger.info(
-                `Registering ${unknownFlags.length} unknown flags from ${value.appName}, i.e.: ${unknownFlags
+                `Registering ${unknownFlags.length} unknown flags from ${value.appName} in the ${environment} environment. Some of the unknown flag names include: ${unknownFlags
                     .slice(0, 10)
-                    .map((f) => f.name)
+                    .map(({ name }) => `"${name}"`)
                     .join(', ')}`,
             );
             this.unknownFlagsService.register(unknownFlags);
@@ -264,7 +258,7 @@ export default class ClientMetricsServiceV2 {
                 (name) => ({
                     featureName: name,
                     appName: value.appName,
-                    environment: value.environment ?? 'default',
+                    environment,
                     timestamp: value.bucket.stop, //we might need to approximate between start/stop...
                     yes: value.bucket.toggles[name].yes ?? 0,
                     no: value.bucket.toggles[name].no ?? 0,

src/lib/features/metrics/unknown-flags/fake-unknown-flags-store.ts

@@ -1,21 +1,27 @@
-import type { IUnknownFlagsStore, UnknownFlag } from './unknown-flags-store.js';
+import type {
+    IUnknownFlagsStore,
+    UnknownFlag,
+    QueryParams,
+} from './unknown-flags-store.js';
 
 export class FakeUnknownFlagsStore implements IUnknownFlagsStore {
     private unknownFlagMap = new Map<string, UnknownFlag>();
 
     private getKey(flag: UnknownFlag): string {
-        return `${flag.name}:${flag.appName}`;
+        return `${flag.name}:${flag.appName}:${flag.environment}`;
     }
 
-    async replaceAll(flags: UnknownFlag[]): Promise<void> {
+    async insert(flags: UnknownFlag[]): Promise<void> {
         this.unknownFlagMap.clear();
         for (const flag of flags) {
             this.unknownFlagMap.set(this.getKey(flag), flag);
         }
     }
 
-    async getAll(): Promise<UnknownFlag[]> {
+    async getAll({ limit }: QueryParams = {}): Promise<UnknownFlag[]> {
-        return Array.from(this.unknownFlagMap.values());
+        const flags = Array.from(this.unknownFlagMap.values());
+        if (!limit) return flags;
+        return flags.slice(0, limit);
     }
 
     async clear(hoursAgo: number): Promise<void> {

src/lib/features/metrics/unknown-flags/unknown-flags-controller.ts

@@ -45,7 +45,7 @@ export default class UnknownFlagsController extends Controller {
                     tags: ['Unstable'],
                     summary: 'Get latest reported unknown flag names',
                     description:
-                        'Returns a list of unknown flag names reported in the last 24 hours, if any. Maximum of 10.',
+                        'Returns a list of unknown flag reports from the last 7 days, if any. Maximum of 1000.',
                     responses: {
                         200: createResponseSchema('unknownFlagsResponseSchema'),
                     },
@@ -61,8 +61,9 @@ export default class UnknownFlagsController extends Controller {
         if (!this.flagResolver.isEnabled('reportUnknownFlags')) {
             throw new NotFoundError();
         }
-        const unknownFlags =
+        const unknownFlags = await this.unknownFlagsService.getAll({
-            await this.unknownFlagsService.getGroupedUnknownFlags();
+            limit: 1000,
+        });
 
         this.openApiService.respondWithValidation(
             200,

src/lib/features/metrics/unknown-flags/unknown-flags-service.ts

@@ -7,8 +7,6 @@ import type {
 import type { IUnleashStores } from '../../../types/index.js';
 import type { UnknownFlag } from './unknown-flags-store.js';
 
-export const MAX_UNKNOWN_FLAGS = 10;
-
 export class UnknownFlagsService {
     private logger: Logger;
 
@@ -31,25 +29,13 @@ export class UnknownFlagsService {
     }
 
     private getKey(flag: UnknownFlag) {
-        return `${flag.name}:${flag.appName}`;
+        return `${flag.name}:${flag.appName}:${flag.environment}`;
     }
 
     register(unknownFlags: UnknownFlag[]) {
+        if (!this.flagResolver.isEnabled('reportUnknownFlags')) return;
         for (const flag of unknownFlags) {
             const key = this.getKey(flag);
-
-            if (this.unknownFlagsCache.has(key)) {
-                this.unknownFlagsCache.set(key, flag);
-                continue;
-            }
-
-            if (this.unknownFlagsCache.size >= MAX_UNKNOWN_FLAGS) {
-                const oldestKey = [...this.unknownFlagsCache.entries()].sort(
-                    (a, b) => a[1].seenAt.getTime() - b[1].seenAt.getTime(),
-                )[0][0];
-                this.unknownFlagsCache.delete(oldestKey);
-            }
-
             this.unknownFlagsCache.set(key, flag);
         }
     }
@@ -58,46 +44,15 @@ export class UnknownFlagsService {
         if (!this.flagResolver.isEnabled('reportUnknownFlags')) return;
         if (this.unknownFlagsCache.size === 0) return;
 
-        const existing = await this.unknownFlagsStore.getAll();
         const cached = Array.from(this.unknownFlagsCache.values());
 
-        const merged = [...existing, ...cached];
+        await this.unknownFlagsStore.insert(cached);
-        const mergedMap = new Map<string, UnknownFlag>();
-
-        for (const flag of merged) {
-            const key = this.getKey(flag);
-            const existing = mergedMap.get(key);
-            if (!existing || flag.seenAt > existing.seenAt) {
-                mergedMap.set(key, flag);
-            }
-        }
-
-        const latest = Array.from(mergedMap.values())
-            .sort((a, b) => b.seenAt.getTime() - a.seenAt.getTime())
-            .slice(0, MAX_UNKNOWN_FLAGS);
-
-        await this.unknownFlagsStore.replaceAll(latest);
         this.unknownFlagsCache.clear();
     }
 
-    async getGroupedUnknownFlags(): Promise<
+    async getAll({ limit }: { limit?: number }): Promise<UnknownFlag[]> {
-        { name: string; reportedBy: { appName: string; seenAt: Date }[] }[]
+        if (!this.flagResolver.isEnabled('reportUnknownFlags')) return [];
-    > {
+        return this.unknownFlagsStore.getAll({ limit });
-        const unknownFlags = await this.unknownFlagsStore.getAll();
-
-        const grouped = new Map<string, { appName: string; seenAt: Date }[]>();
-
-        for (const { name, appName, seenAt } of unknownFlags) {
-            if (!grouped.has(name)) {
-                grouped.set(name, []);
-            }
-            grouped.get(name)!.push({ appName, seenAt });
-        }
-
-        return Array.from(grouped.entries()).map(([name, reportedBy]) => ({
-            name,
-            reportedBy,
-        }));
     }
 
     async clear(hoursAgo: number) {

src/lib/features/metrics/unknown-flags/unknown-flags-store.ts

@@ -1,5 +1,4 @@
 import type { Db } from '../../../db/db.js';
-import { MAX_UNKNOWN_FLAGS } from './unknown-flags-service.js';
 
 const TABLE = 'unknown_flags';
 
@@ -7,11 +6,16 @@ export type UnknownFlag = {
     name: string;
     appName: string;
     seenAt: Date;
+    environment: string;
+};
+
+export type QueryParams = {
+    limit?: number;
 };
 
 export interface IUnknownFlagsStore {
-    replaceAll(flags: UnknownFlag[]): Promise<void>;
+    insert(flags: UnknownFlag[]): Promise<void>;
-    getAll(): Promise<UnknownFlag[]>;
+    getAll(params?: QueryParams): Promise<UnknownFlag[]>;
     clear(hoursAgo: number): Promise<void>;
     deleteAll(): Promise<void>;
     count(): Promise<number>;
@@ -24,32 +28,40 @@ export class UnknownFlagsStore implements IUnknownFlagsStore {
         this.db = db;
     }
 
-    async replaceAll(flags: UnknownFlag[]): Promise<void> {
+    async insert(flags: UnknownFlag[]): Promise<void> {
-        await this.db.transaction(async (tx) => {
-            await tx(TABLE).delete();
         if (flags.length > 0) {
             const rows = flags.map((flag) => ({
                 name: flag.name,
                 app_name: flag.appName,
                 seen_at: flag.seenAt,
+                environment: flag.environment,
             }));
-                await tx(TABLE)
+            await this.db(TABLE)
                 .insert(rows)
-                    .onConflict(['name', 'app_name'])
+                .onConflict(['name', 'app_name', 'environment'])
                 .merge(['seen_at']);
         }
-        });
     }
 
-    async getAll(): Promise<UnknownFlag[]> {
+    async getAll({ limit }: QueryParams = {}): Promise<UnknownFlag[]> {
-        const rows = await this.db(TABLE)
+        let query = this.db(TABLE).select(
-            .select('name', 'app_name', 'seen_at')
+            'name',
-            .orderBy('seen_at', 'desc')
+            'app_name',
-            .limit(MAX_UNKNOWN_FLAGS);
+            'seen_at',
+            'environment',
+        );
+
+        if (limit) {
+            query = query.limit(limit);
+        }
+
+        const rows = await query;
+
         return rows.map((row) => ({
             name: row.name,
             appName: row.app_name,
             seenAt: new Date(row.seen_at),
+            environment: row.environment,
         }));
     }
 

src/lib/features/scheduler/schedule-services.ts

@@ -203,7 +203,7 @@ export const scheduleServices = (
     );
 
     schedulerService.schedule(
-        unknownFlagsService.clear.bind(unknownFlagsService, 24),
+        unknownFlagsService.clear.bind(unknownFlagsService, 24 * 7),
         hoursToMilliseconds(24),
         'clearUnknownFlags',
     );

src/lib/openapi/spec/unknown-flag-schema.ts

@@ -4,7 +4,7 @@ export const unknownFlagSchema = {
     $id: '#/components/schemas/unknownFlagSchema',
     type: 'object',
     additionalProperties: false,
-    required: ['name', 'reportedBy'],
+    required: ['name', 'appName', 'seenAt', 'environment'],
     description: 'An unknown flag that has been reported by the system',
     properties: {
         name: {
@@ -12,15 +12,6 @@ export const unknownFlagSchema = {
             description: 'The name of the unknown flag.',
             example: 'my-unknown-flag',
         },
-        reportedBy: {
-            description:
-                'Details about the application that reported the unknown flag.',
-            type: 'array',
-            items: {
-                type: 'object',
-                additionalProperties: false,
-                required: ['appName', 'seenAt'],
-                properties: {
         appName: {
             type: 'string',
             description:
@@ -34,8 +25,11 @@ export const unknownFlagSchema = {
                 'The date and time when the unknown flag was reported.',
             example: '2023-10-01T12:00:00Z',
         },
-                },
+        environment: {
-            },
+            type: 'string',
+            description:
+                'The environment in which the unknown flag was reported.',
+            example: 'production',
         },
     },
     components: {},

src/migrations/20250707153020-unknown-flags-environment.js

@@ -0,0 +1,17 @@
+exports.up = function(db, cb) {
+  db.runSql(`
+    ALTER TABLE unknown_flags
+      ADD COLUMN IF NOT EXISTS environment TEXT NOT NULL DEFAULT 'default',
+      DROP CONSTRAINT unknown_flags_pkey,
+      ADD PRIMARY KEY (name, app_name, environment);
+  `, cb);
+};
+
+exports.down = function(db, cb) {
+  db.runSql(`
+    ALTER TABLE unknown_flags
+      DROP CONSTRAINT unknown_flags_pkey,
+      ADD PRIMARY KEY (name, app_name),
+      DROP COLUMN IF EXISTS environment;
+  `, cb);
+};