diff --git a/frontend/src/component/unknownFlags/UnknownFlagsSeenInUnleashCell.tsx b/frontend/src/component/unknownFlags/UnknownFlagsLastEventCell.tsx similarity index 95% rename from frontend/src/component/unknownFlags/UnknownFlagsSeenInUnleashCell.tsx rename to frontend/src/component/unknownFlags/UnknownFlagsLastEventCell.tsx index 69324a6b18..9bc6b71936 100644 --- a/frontend/src/component/unknownFlags/UnknownFlagsSeenInUnleashCell.tsx +++ b/frontend/src/component/unknownFlags/UnknownFlagsLastEventCell.tsx @@ -11,7 +11,7 @@ interface IUnknownFlagsSeenInUnleashCellProps extends ITimeAgoCellProps { unknownFlag: UnknownFlag; } -export const UnknownFlagsSeenInUnleashCell = ({ +export const UnknownFlagsLastEventCell = ({ unknownFlag, ...props }: IUnknownFlagsSeenInUnleashCellProps) => { diff --git a/frontend/src/component/unknownFlags/UnknownFlagsLastReportedCell.tsx b/frontend/src/component/unknownFlags/UnknownFlagsLastReportedCell.tsx new file mode 100644 index 0000000000..90b69037a2 --- /dev/null +++ b/frontend/src/component/unknownFlags/UnknownFlagsLastReportedCell.tsx @@ -0,0 +1,128 @@ +import { TextCell } from 'component/common/Table/cells/TextCell/TextCell'; +import { TooltipLink } from 'component/common/TooltipLink/TooltipLink'; +import type { UnknownFlag } from './hooks/useUnknownFlags.js'; +import { TimeAgo } from 'component/common/TimeAgo/TimeAgo'; +import { formatDateYMDHMS } from 'utils/formatDate.js'; +import { useLocationSettings } from 'hooks/useLocationSettings.js'; +import { styled } from '@mui/material'; +import { Highlighter } from 'component/common/Highlighter/Highlighter.js'; +import { useSearchHighlightContext } from 'component/common/Table/SearchHighlightContext/SearchHighlightContext.js'; + +const REPORT_APP_LIMIT = 20; +const REPORT_ENV_LIMIT = 10; + +const StyledTooltip = styled('div')(({ theme }) => ({ + display: 'flex', + flexDirection: 'column', + gap: theme.spacing(1), +})); + +const StyledReport = styled('div')(({ theme }) => ({ + display: 'flex', + flexDirection: 'column', + '& > ul': { + padding: theme.spacing(0, 3), + margin: 0, + }, +})); + +interface IUnknownFlagsLastReportedCellProps { + row: { original: UnknownFlag }; +} + +const UnknownFlagsLastReportedCellTooltip = ({ + unknownFlag, + searchQuery, +}: { unknownFlag: UnknownFlag; searchQuery: string }) => { + const { locationSettings } = useLocationSettings(); + const lastReported = formatDateYMDHMS( + unknownFlag.lastSeenAt, + locationSettings.locale, + ); + + return ( + + Last reported: {lastReported} + {unknownFlag.reports + .slice(0, REPORT_APP_LIMIT) + .map(({ appName, environments }) => ( + + + + {appName} + + + + + ))} + {unknownFlag.reports.length > REPORT_APP_LIMIT && ( + + and {unknownFlag.reports.length - REPORT_APP_LIMIT} more + + )} + + ); +}; + +export const UnknownFlagsLastReportedCell = ({ + row, +}: IUnknownFlagsLastReportedCellProps) => { + const { original: unknownFlag } = row; + const { searchQuery } = useSearchHighlightContext(); + + const searchableAppNames = Array.from( + new Set( + unknownFlag.reports.map((report) => report.appName.toLowerCase()), + ), + ).join('\n'); + const searchableEnvironments = Array.from( + new Set( + unknownFlag.reports.flatMap((report) => + report.environments.map((env) => env.environment.toLowerCase()), + ), + ), + ).join('\n'); + + return ( + + + } + highlighted={ + searchQuery.length > 0 && + (searchableAppNames.includes(searchQuery.toLowerCase()) || + searchableEnvironments.includes( + searchQuery.toLowerCase(), + )) + } + > + + + + ); +}; diff --git a/frontend/src/component/unknownFlags/UnknownFlagsTable.tsx b/frontend/src/component/unknownFlags/UnknownFlagsTable.tsx index 714ec76bd5..7f575fc3ca 100644 --- a/frontend/src/component/unknownFlags/UnknownFlagsTable.tsx +++ b/frontend/src/component/unknownFlags/UnknownFlagsTable.tsx @@ -11,14 +11,14 @@ import { Search } from 'component/common/Search/Search'; import { useSearch } from 'hooks/useSearch'; import { type UnknownFlag, useUnknownFlags } from './hooks/useUnknownFlags.js'; import theme from 'themes/theme.js'; -import { TimeAgoCell } from 'component/common/Table/cells/TimeAgoCell/TimeAgoCell.js'; import { formatDateYMDHMS } from 'utils/formatDate.js'; import { HighlightCell } from 'component/common/Table/cells/HighlightCell/HighlightCell.js'; import { useUiFlag } from 'hooks/useUiFlag.js'; import NotFound from 'component/common/NotFound/NotFound.js'; -import { UnknownFlagsSeenInUnleashCell } from './UnknownFlagsSeenInUnleashCell.js'; +import { UnknownFlagsLastEventCell } from './UnknownFlagsLastEventCell.js'; import { HelpIcon } from 'component/common/HelpIcon/HelpIcon.js'; import { UnknownFlagsActionsCell } from './UnknownFlagsActionsCell.js'; +import { UnknownFlagsLastReportedCell } from './UnknownFlagsLastReportedCell.js'; const StyledAlert = styled(Alert)(({ theme }) => ({ marginBottom: theme.spacing(3), @@ -51,36 +51,19 @@ export const UnknownFlagsTable = () => { minWidth: 100, searchable: true, }, - { - Header: 'Application', - accessor: 'appName', - searchable: true, - minWidth: 100, - }, - { - Header: 'Environment', - accessor: 'environment', - searchable: true, - }, { Header: ( - Reported + Last reported ), - accessor: 'seenAt', - Cell: ({ value }) => ( - `Reported: ${date}`} - dateFormat={formatDateYMDHMS} - /> - ), - width: 150, + accessor: 'lastSeenAt', + Cell: UnknownFlagsLastReportedCell, + width: 170, }, { Header: ( @@ -98,7 +81,7 @@ export const UnknownFlagsTable = () => { }: { row: { original: UnknownFlag }; }) => ( - @@ -116,12 +99,34 @@ export const UnknownFlagsTable = () => { width: 100, disableSortBy: true, }, + // Always hidden -- for search + { + accessor: (row: UnknownFlag) => + row.reports.map(({ appName }) => appName).join('\n'), + id: 'appNames', + searchable: true, + }, + { + accessor: (row: UnknownFlag) => + Array.from( + new Set( + row.reports.flatMap(({ environments }) => + environments.map( + ({ environment }) => environment, + ), + ), + ), + ).join('\n'), + id: 'environments', + searchable: true, + }, ], [], ); const [initialState] = useState({ sortBy: [{ id: 'name', desc: false }], + hiddenColumns: ['appNames', 'environments'], }); const { data, getSearchText } = useSearch( @@ -155,7 +160,7 @@ export const UnknownFlagsTable = () => { isLoading={loading} header={ { > -

+

Clean up unknown flags to keep your code and configuration in sync @@ -195,9 +200,9 @@ export const UnknownFlagsTable = () => {
Unknown flags are feature flags that your SDKs tried to evaluate but which Unleash doesn't recognize. -

+
-

+

Unknown flags can include:
  • @@ -209,14 +214,7 @@ export const UnknownFlagsTable = () => { unexpected names, unsupported by Unleash.
-

- -

- Why do I see the same flag name multiple times? -
- The same flag name will appear multiple times if it's - been seen in different applications or environments. -

+
diff --git a/frontend/src/component/unknownFlags/hooks/useUnknownFlags.ts b/frontend/src/component/unknownFlags/hooks/useUnknownFlags.ts index 1923a4276c..3747ce4035 100644 --- a/frontend/src/component/unknownFlags/hooks/useUnknownFlags.ts +++ b/frontend/src/component/unknownFlags/hooks/useUnknownFlags.ts @@ -5,12 +5,21 @@ import { useConditionalSWR } from 'hooks/api/getters/useConditionalSWR/useCondit import handleErrorResponses from 'hooks/api/getters/httpErrorResponseHandler'; import type { SWRConfiguration } from 'swr'; +type UnknownFlagEnvReport = { + environment: string; + seenAt: Date; +}; + +type UnknownFlagAppReport = { + appName: string; + environments: UnknownFlagEnvReport[]; +}; + export type UnknownFlag = { name: string; - appName: string; - seenAt: Date; - environment: string; - lastEventAt: Date; + lastSeenAt: Date; + lastEventAt?: Date; + reports: UnknownFlagAppReport[]; }; type UnknownFlagsResponse = { diff --git a/src/lib/features/metrics/client-metrics/metrics-service-v2.ts b/src/lib/features/metrics/client-metrics/metrics-service-v2.ts index 4b430f526f..659a9d251b 100644 --- a/src/lib/features/metrics/client-metrics/metrics-service-v2.ts +++ b/src/lib/features/metrics/client-metrics/metrics-service-v2.ts @@ -32,7 +32,7 @@ import { MetricsTranslator, } from '../impact/metrics-translator.js'; import { impactRegister } from '../impact/impact-register.js'; -import type { UnknownFlag } from '../unknown-flags/unknown-flags-store.js'; +import type { UnknownFlagReport } from '../unknown-flags/unknown-flags-store.js'; export default class ClientMetricsServiceV2 { private config: IUnleashConfig; @@ -209,14 +209,14 @@ export default class ClientMetricsServiceV2 { `Got ${toggleNames.length} metrics (${invalidCount > 0 ? `${invalidCount} invalid` : 'all valid'}).`, ); - const unknownFlags: UnknownFlag[] = []; + const unknownFlags: UnknownFlagReport[] = []; for (const [featureName, group] of metricsByToggle) { if (unknownSet.has(featureName)) { for (const m of group) { unknownFlags.push({ name: featureName, appName: m.appName, - seenAt: m.timestamp, + lastSeenAt: m.timestamp, environment: m.environment, }); } diff --git a/src/lib/features/metrics/unknown-flags/fake-unknown-flags-store.ts b/src/lib/features/metrics/unknown-flags/fake-unknown-flags-store.ts index 5284212477..33d0697821 100644 --- a/src/lib/features/metrics/unknown-flags/fake-unknown-flags-store.ts +++ b/src/lib/features/metrics/unknown-flags/fake-unknown-flags-store.ts @@ -1,25 +1,67 @@ import type { IUnknownFlagsStore, UnknownFlag, + UnknownFlagReport, QueryParams, } from './unknown-flags-store.js'; export class FakeUnknownFlagsStore implements IUnknownFlagsStore { - private unknownFlagMap = new Map(); + private unknownFlagMap = new Map(); - private getKey(flag: UnknownFlag): string { + private getKey(flag: UnknownFlagReport): string { return `${flag.name}:${flag.appName}:${flag.environment}`; } - async insert(flags: UnknownFlag[]): Promise { + async insert(flags: UnknownFlagReport[]): Promise { this.unknownFlagMap.clear(); for (const flag of flags) { this.unknownFlagMap.set(this.getKey(flag), flag); } } + private groupFlags(flags: UnknownFlagReport[]): UnknownFlag[] { + const byName = new Map>>(); + + for (const f of flags) { + const apps = + byName.get(f.name) ?? new Map>(); + const envs = apps.get(f.appName) ?? new Map(); + const prev = envs.get(f.environment); + if (!prev || f.lastSeenAt > prev) + envs.set(f.environment, f.lastSeenAt); + apps.set(f.appName, envs); + byName.set(f.name, apps); + } + + const out: UnknownFlag[] = []; + for (const [name, appsMap] of byName) { + let lastSeenAt: Date | null = null; + + const reports = Array.from(appsMap.entries()).map( + ([appName, envMap]) => { + const environments = Array.from(envMap.entries()).map( + ([environment, seenAt]) => { + if (!lastSeenAt || seenAt > lastSeenAt) + lastSeenAt = seenAt; + return { environment, seenAt }; + }, + ); + return { appName, environments }; + }, + ); + + out.push({ + name, + lastSeenAt: lastSeenAt ?? new Date(0), + reports, + }); + } + return out; + } + async getAll({ limit, orderBy }: QueryParams = {}): Promise { - const flags = Array.from(this.unknownFlagMap.values()); + const flat = Array.from(this.unknownFlagMap.values()); + const flags = this.groupFlags(flat); if (orderBy) { flags.sort((a, b) => { for (const { column, order } of orderBy) { @@ -36,7 +78,7 @@ export class FakeUnknownFlagsStore implements IUnknownFlagsStore { async clear(hoursAgo: number): Promise { const cutoff = Date.now() - hoursAgo * 60 * 60 * 1000; for (const [key, flag] of this.unknownFlagMap.entries()) { - if (flag.seenAt.getTime() < cutoff) { + if (flag.lastSeenAt.getTime() < cutoff) { this.unknownFlagMap.delete(key); } } diff --git a/src/lib/features/metrics/unknown-flags/unknown-flags-service.ts b/src/lib/features/metrics/unknown-flags/unknown-flags-service.ts index 1e316b94b9..0ca96e0de5 100644 --- a/src/lib/features/metrics/unknown-flags/unknown-flags-service.ts +++ b/src/lib/features/metrics/unknown-flags/unknown-flags-service.ts @@ -5,7 +5,11 @@ import type { IUnleashConfig, } from '../../../types/index.js'; import type { IUnleashStores } from '../../../types/index.js'; -import type { QueryParams, UnknownFlag } from './unknown-flags-store.js'; +import type { + QueryParams, + UnknownFlag, + UnknownFlagReport, +} from './unknown-flags-store.js'; export class UnknownFlagsService { private logger: Logger; @@ -14,7 +18,7 @@ export class UnknownFlagsService { private unknownFlagsStore: IUnknownFlagsStore; - private unknownFlagsCache: Map; + private unknownFlagsCache: Map; constructor( { unknownFlagsStore }: Pick, @@ -25,14 +29,14 @@ export class UnknownFlagsService { this.logger = config.getLogger( '/features/metrics/unknown-flags/unknown-flags-service.ts', ); - this.unknownFlagsCache = new Map(); + this.unknownFlagsCache = new Map(); } - private getKey(flag: UnknownFlag) { + private getKey(flag: UnknownFlagReport) { return `${flag.name}:${flag.appName}:${flag.environment}`; } - register(unknownFlags: UnknownFlag[]) { + register(unknownFlags: UnknownFlagReport[]) { if (!this.flagResolver.isEnabled('reportUnknownFlags')) return; for (const flag of unknownFlags) { const key = this.getKey(flag); diff --git a/src/lib/features/metrics/unknown-flags/unknown-flags-store.ts b/src/lib/features/metrics/unknown-flags/unknown-flags-store.ts index 9ac4d8098f..480f282a38 100644 --- a/src/lib/features/metrics/unknown-flags/unknown-flags-store.ts +++ b/src/lib/features/metrics/unknown-flags/unknown-flags-store.ts @@ -5,12 +5,28 @@ const TABLE = 'unknown_flags'; const TABLE_EVENTS = 'events'; const MAX_INSERT_BATCH_SIZE = 100; +type UnknownFlagEnvReport = { + environment: string; + seenAt: Date; +}; + +type UnknownFlagAppReport = { + appName: string; + environments: UnknownFlagEnvReport[]; +}; + export type UnknownFlag = { name: string; - appName: string; - seenAt: Date; - environment: string; + lastSeenAt: Date; lastEventAt?: Date; + reports: UnknownFlagAppReport[]; +}; + +export type UnknownFlagReport = { + name: string; + appName: string; + lastSeenAt: Date; + environment: string; }; export type QueryParams = { @@ -22,7 +38,7 @@ export type QueryParams = { }; export interface IUnknownFlagsStore { - insert(flags: UnknownFlag[]): Promise; + insert(flags: UnknownFlagReport[]): Promise; getAll(params?: QueryParams): Promise; clear(hoursAgo: number): Promise; deleteAll(): Promise; @@ -39,15 +55,17 @@ export class UnknownFlagsStore implements IUnknownFlagsStore { this.logger = getLogger('unknown-flags-store.ts'); } - async insert(flags: UnknownFlag[]): Promise { + async insert(flags: UnknownFlagReport[]): Promise { if (!flags.length) return; - const rows = flags.map(({ name, appName, seenAt, environment }) => ({ - name, - app_name: appName, - seen_at: seenAt, - environment, - })); + const rows = flags.map( + ({ name, appName, lastSeenAt, environment }) => ({ + name, + app_name: appName, + seen_at: lastSeenAt, + environment, + }), + ); for (let i = 0; i < rows.length; i += MAX_INSERT_BATCH_SIZE) { const chunk = rows.slice(i, i + MAX_INSERT_BATCH_SIZE); @@ -66,42 +84,67 @@ export class UnknownFlagsStore implements IUnknownFlagsStore { } async getAll({ limit, orderBy }: QueryParams = {}): Promise { - let query = this.db(`${TABLE} AS uf`) - .select( - 'uf.name', - 'uf.app_name', - 'uf.seen_at', - 'uf.environment', - this.db.raw( - `(SELECT MAX(e.created_at) - FROM ${TABLE_EVENTS} AS e - WHERE e.feature_name = uf.name) - AS last_event_at`, - ), + const base = this.db + .with('base', (qb) => + qb + .from(`${TABLE} as uf`) + .leftJoin('features as f', 'f.name', 'uf.name') + .whereNull('f.name') + .select('uf.name', 'uf.app_name', 'uf.environment') + .max({ seen_at: 'uf.seen_at' }) + .groupBy('uf.name', 'uf.app_name', 'uf.environment'), ) - .whereNotExists( - this.db('features as f') - .select(this.db.raw('1')) - .whereRaw('f.name = uf.name'), + .select( + 'b.name', + this.db.raw('MAX(b.seen_at) as last_seen_at'), + this.db.raw( + `(SELECT MAX(e.created_at) FROM ${TABLE_EVENTS} e WHERE e.feature_name = b.name) as last_event_at`, + ), + this.db.raw(` + jsonb_object_agg( + b.app_name, + ( + SELECT jsonb_object_agg(env_row.environment, env_row.seen_at) + FROM ( + SELECT environment, MAX(seen_at) AS seen_at + FROM base + WHERE name = b.name AND app_name = b.app_name + GROUP BY environment + ) env_row + ) + ) as reports + `), + ) + .from('base as b') + .groupBy('b.name'); + + let q = base; + if (orderBy) q = q.orderBy(orderBy); + if (limit) q = q.limit(limit); + + const rows = await q; + + return rows.map((r) => { + const reportsObj = r.reports ?? {}; + const reports = Object.entries(reportsObj).map( + ([appName, envs]) => ({ + appName, + environments: Object.entries( + envs as Record, + ).map(([environment, seenAt]) => ({ + environment, + seenAt: new Date(seenAt), + })), + }), ); - if (orderBy) { - query = query.orderBy(orderBy); - } - - if (limit) { - query = query.limit(limit); - } - - const rows = await query; - - return rows.map((row) => ({ - name: row.name, - appName: row.app_name, - seenAt: row.seen_at, - environment: row.environment, - lastEventAt: row.last_event_at, - })); + return { + name: r.name, + lastSeenAt: r.last_seen_at, + lastEventAt: r.last_event_at, + reports, + }; + }); } async clear(hoursAgo: number): Promise { diff --git a/src/lib/features/metrics/unknown-flags/unknown-flags.e2e.test.ts b/src/lib/features/metrics/unknown-flags/unknown-flags.e2e.test.ts index be89d2811d..3b7c8f7a2d 100644 --- a/src/lib/features/metrics/unknown-flags/unknown-flags.e2e.test.ts +++ b/src/lib/features/metrics/unknown-flags/unknown-flags.e2e.test.ts @@ -105,9 +105,18 @@ describe('should register unknown flags', () => { expect(unknownFlags).toHaveLength(1); expect(unknownFlags[0]).toMatchObject({ name: 'unknown_flag', - environment: 'development', - appName: 'demo', - seenAt: expect.any(Date), + lastSeenAt: expect.any(Date), + reports: [ + { + appName: 'demo', + environments: [ + { + environment: 'development', + seenAt: expect.any(Date), + }, + ], + }, + ], }); expect(eventBus.emit).toHaveBeenCalledWith( CLIENT_METRICS, @@ -167,9 +176,18 @@ describe('should register unknown flags', () => { expect(unknownFlags).toHaveLength(1); expect(unknownFlags[0]).toMatchObject({ name: 'unknown_flag_bulk', - environment: 'development', - appName: 'demo', - seenAt: expect.any(Date), + lastSeenAt: expect.any(Date), + reports: [ + { + appName: 'demo', + environments: [ + { + environment: 'development', + seenAt: expect.any(Date), + }, + ], + }, + ], }); expect(eventBus.emit).toHaveBeenCalledWith( CLIENT_METRICS, @@ -242,15 +260,35 @@ describe('should fetch unknown flags', () => { expect(res.body.unknownFlags).toEqual([ expect.objectContaining({ name: 'unknown_flag_1', - environment: 'development', - appName: 'demo', + lastSeenAt: expect.any(String), lastEventAt: null, + reports: [ + { + appName: 'demo', + environments: [ + { + environment: 'development', + seenAt: expect.any(String), + }, + ], + }, + ], }), expect.objectContaining({ name: 'unknown_flag_2', - environment: 'development', - appName: 'demo', + lastSeenAt: expect.any(String), lastEventAt: null, + reports: [ + { + appName: 'demo', + environments: [ + { + environment: 'development', + seenAt: expect.any(String), + }, + ], + }, + ], }), ]); }); @@ -312,9 +350,18 @@ describe('should fetch unknown flags', () => { expect(res.body.unknownFlags).toEqual([ expect.objectContaining({ name: 'unknown_flag_2', - environment: 'development', - appName: 'demo', - lastEventAt: null, + lastSeenAt: expect.any(String), + reports: [ + { + appName: 'demo', + environments: [ + { + environment: 'development', + seenAt: expect.any(String), + }, + ], + }, + ], }), ]); }); diff --git a/src/lib/openapi/spec/unknown-flag-schema.ts b/src/lib/openapi/spec/unknown-flag-schema.ts index cb6394cdb6..34c8343eca 100644 --- a/src/lib/openapi/spec/unknown-flag-schema.ts +++ b/src/lib/openapi/spec/unknown-flag-schema.ts @@ -4,7 +4,7 @@ export const unknownFlagSchema = { $id: '#/components/schemas/unknownFlagSchema', type: 'object', additionalProperties: false, - required: ['name', 'appName', 'seenAt', 'environment'], + required: ['name', 'lastSeenAt'], description: 'An unknown flag report', properties: { name: { @@ -12,25 +12,13 @@ export const unknownFlagSchema = { description: 'The name of the unknown flag.', example: 'my-unknown-flag', }, - appName: { - type: 'string', - description: - 'The name of the application that reported the unknown flag.', - example: 'my-app', - }, - seenAt: { + lastSeenAt: { type: 'string', format: 'date-time', description: - 'The date and time when the unknown flag was reported.', + 'The date and time when the unknown flag was last reported.', example: '2023-10-01T12:00:00Z', }, - environment: { - type: 'string', - description: - 'The environment in which the unknown flag was reported.', - example: 'production', - }, lastEventAt: { type: 'string', format: 'date-time', @@ -39,6 +27,48 @@ export const unknownFlagSchema = { example: '2023-10-01T12:00:00Z', nullable: true, }, + reports: { + type: 'array', + description: 'The list of reports for this unknown flag.', + items: { + type: 'object', + additionalProperties: false, + required: ['appName', 'environments'], + properties: { + appName: { + type: 'string', + description: + 'The name of the application that reported the unknown flag.', + example: 'my-app', + }, + environments: { + type: 'array', + description: + 'The list of environments where this application reported the unknown flag.', + items: { + type: 'object', + additionalProperties: false, + required: ['environment', 'seenAt'], + properties: { + environment: { + type: 'string', + description: + 'The environment in which the unknown flag was reported.', + example: 'production', + }, + seenAt: { + type: 'string', + format: 'date-time', + description: + 'The date and time when the unknown flag was last seen in this environment.', + example: '2023-10-01T12:00:00Z', + }, + }, + }, + }, + }, + }, + }, }, components: {}, } as const;