diff --git a/src/lib/db/feature-environment-store.ts b/src/lib/db/feature-environment-store.ts index 5e868b8bc5..ec114157c8 100644 --- a/src/lib/db/feature-environment-store.ts +++ b/src/lib/db/feature-environment-store.ts @@ -88,6 +88,7 @@ export class FeatureEnvironmentStore implements IFeatureEnvironmentStore { featureName, environment, variants: md.variants, + lastSeenAt: md.last_seen_at, }; } throw new NotFoundError( @@ -123,6 +124,7 @@ export class FeatureEnvironmentStore implements IFeatureEnvironmentStore { featureName: r.feature_name, environment: r.environment, variants: r.variants, + lastSeenAt: r.last_seen_at, })); } @@ -196,6 +198,7 @@ export class FeatureEnvironmentStore implements IFeatureEnvironmentStore { environment: r.environment, variants: r.variants || [], enabled: r.enabled, + lastSeenAt: r.last_seen_at, })); } return []; diff --git a/src/lib/db/feature-strategy-store.ts b/src/lib/db/feature-strategy-store.ts index 5e2d709822..297d4bd5d8 100644 --- a/src/lib/db/feature-strategy-store.ts +++ b/src/lib/db/feature-strategy-store.ts @@ -348,13 +348,14 @@ class FeatureStrategiesStore implements IFeatureStrategiesStore { acc.description = r.description; acc.project = r.project; acc.stale = r.stale; + acc.lastSeenAt = r.last_seen_at; acc.createdAt = r.created_at; - acc.lastSeenAt = r.last_seen_at; acc.type = r.type; if (!acc.environments[r.environment]) { acc.environments[r.environment] = { name: r.environment, + lastSeenAt: r.env_last_seen_at, }; } const env = acc.environments[r.environment]; @@ -443,6 +444,7 @@ class FeatureStrategiesStore implements IFeatureStrategiesStore { type: r.environment_type, sortOrder: r.environment_sort_order, variantCount: r.variants?.length || 0, + lastSeenAt: r.env_last_seen_at, }; } @@ -523,6 +525,7 @@ class FeatureStrategiesStore implements IFeatureStrategiesStore { 'feature_environments.enabled as enabled', 'feature_environments.environment as environment', 'feature_environments.variants as variants', + 'feature_environments.last_seen_at as env_last_seen_at', 'environments.type as environment_type', 'environments.sort_order as environment_sort_order', 'ft.tag_value as tag_value', diff --git a/src/lib/db/feature-toggle-client-store.ts b/src/lib/db/feature-toggle-client-store.ts index 285d0d7163..af7e88afc5 100644 --- a/src/lib/db/feature-toggle-client-store.ts +++ b/src/lib/db/feature-toggle-client-store.ts @@ -75,9 +75,10 @@ export default class FeatureToggleClientStore 'features.project as project', 'features.stale as stale', 'features.impression_data as impression_data', - 'fe.variants as variants', - 'features.created_at as created_at', 'features.last_seen_at as last_seen_at', + 'features.created_at as created_at', + 'fe.variants as variants', + 'fe.last_seen_at as env_last_seen_at', 'fe.enabled as enabled', 'fe.environment as environment', 'fs.id as strategy_id', @@ -109,6 +110,7 @@ export default class FeatureToggleClientStore 'enabled', 'environment', 'variants', + 'last_seen_at', ) .where({ environment }) .as('fe'), @@ -200,6 +202,7 @@ export default class FeatureToggleClientStore feature.project = r.project; feature.stale = r.stale; feature.type = r.type; + feature.lastSeenAt = r.last_seen_at; feature.variants = r.variants || []; feature.project = r.project; if (isAdmin) { diff --git a/src/lib/db/feature-toggle-store.ts b/src/lib/db/feature-toggle-store.ts index 54c73a77d9..64ab1278ff 100644 --- a/src/lib/db/feature-toggle-store.ts +++ b/src/lib/db/feature-toggle-store.ts @@ -7,6 +7,9 @@ import { Logger, LogProvider } from '../logger'; import { FeatureToggle, FeatureToggleDTO, IVariant } from '../types/model'; import { IFeatureToggleStore } from '../types/stores/feature-toggle-store'; import { Db } from './db'; +import { LastSeenInput } from '../services/client-metrics/last-seen-service'; + +export type EnvironmentFeatureNames = { [key: string]: string[] }; const FEATURE_COLUMNS = [ 'name', @@ -165,24 +168,60 @@ export default class FeatureToggleStore implements IFeatureToggleStore { return present; } - async setLastSeen(toggleNames: string[]): Promise { + async setLastSeen(data: LastSeenInput[]): Promise { const now = new Date(); + const environmentArrays = this.mapMetricDataToEnvBuckets(data); try { - await this.db(TABLE) - .update({ last_seen_at: now }) - .whereIn( - 'name', - this.db(TABLE) - .select('name') - .whereIn('name', toggleNames) - .forUpdate() - .skipLocked(), - ); + for (const env of Object.keys(environmentArrays)) { + const toggleNames = environmentArrays[env]; + await this.db(FEATURE_ENVIRONMENTS_TABLE) + .update({ last_seen_at: now }) + .where('environment', env) + .whereIn( + 'feature_name', + this.db(FEATURE_ENVIRONMENTS_TABLE) + .select('feature_name') + .whereIn('feature_name', toggleNames) + .forUpdate() + .skipLocked(), + ); + + // Updating the toggle's last_seen_at also for backwards compatibility + await this.db(TABLE) + .update({ last_seen_at: now }) + .whereIn( + 'name', + this.db(TABLE) + .select('name') + .whereIn('name', toggleNames) + .forUpdate() + .skipLocked(), + ); + } } catch (err) { this.logger.error('Could not update lastSeen, error: ', err); } } + private mapMetricDataToEnvBuckets( + data: LastSeenInput[], + ): EnvironmentFeatureNames { + return data.reduce( + (acc: EnvironmentFeatureNames, feature: LastSeenInput) => { + const { environment, featureName } = feature; + + if (!acc[environment]) { + acc[environment] = []; + } + + acc[environment].push(featureName); + + return acc; + }, + {}, + ); + } + static filterByArchived: Knex.QueryCallbackWithArgs = ( queryBuilder: Knex.QueryBuilder, archived: boolean, diff --git a/src/lib/features/export-import-toggles/export-import-service.ts b/src/lib/features/export-import-toggles/export-import-service.ts index ba3cb7d56b..e86ed13e88 100644 --- a/src/lib/features/export-import-toggles/export-import-service.ts +++ b/src/lib/features/export-import-toggles/export-import-service.ts @@ -643,6 +643,7 @@ export default class ExportImportService { featureEnvironments: featureEnvironments.map((item) => ({ ...item, name: item.featureName, + lastSeenAt: item.lastSeenAt?.toISOString(), })), contextFields: filteredContextFields.map((item) => { const { createdAt, ...rest } = item; diff --git a/src/lib/openapi/spec/feature-environment-schema.ts b/src/lib/openapi/spec/feature-environment-schema.ts index edc1ea6acf..110140db5e 100644 --- a/src/lib/openapi/spec/feature-environment-schema.ts +++ b/src/lib/openapi/spec/feature-environment-schema.ts @@ -63,6 +63,14 @@ export const featureEnvironmentSchema = { }, description: 'A list of variants for the feature environment', }, + lastSeenAt: { + type: 'string', + format: 'date-time', + nullable: true, + example: '2023-01-28T16:21:39.975Z', + description: + 'The date when metrics where last collected for the feature environment', + }, }, components: { schemas: { diff --git a/src/lib/openapi/spec/feature-schema.ts b/src/lib/openapi/spec/feature-schema.ts index 2e51676988..cc1d11ad61 100644 --- a/src/lib/openapi/spec/feature-schema.ts +++ b/src/lib/openapi/spec/feature-schema.ts @@ -84,9 +84,10 @@ export const featureSchema = { type: 'string', format: 'date-time', nullable: true, + deprecated: true, example: '2023-01-28T16:21:39.975Z', description: - 'The date when metrics where last collected for the feature', + 'The date when metrics where last collected for the feature. This field is deprecated, use the one in featureEnvironmentSchema', }, environments: { type: 'array', diff --git a/src/lib/services/client-metrics/last-seen-service.ts b/src/lib/services/client-metrics/last-seen-service.ts index a6c95b82f0..640a87f57a 100644 --- a/src/lib/services/client-metrics/last-seen-service.ts +++ b/src/lib/services/client-metrics/last-seen-service.ts @@ -5,10 +5,15 @@ import { IUnleashStores } from '../../types'; import { IClientMetricsEnv } from '../../types/stores/client-metrics-store-v2'; import { IFeatureToggleStore } from '../../types/stores/feature-toggle-store'; +export type LastSeenInput = { + featureName: string; + environment: string; +}; + export class LastSeenService { private timers: NodeJS.Timeout[] = []; - private lastSeenToggles: Set = new Set(); + private lastSeenToggles: Set = new Set(); private logger: Logger; @@ -48,7 +53,10 @@ export class LastSeenService { (clientMetric) => clientMetric.yes > 0 || clientMetric.no > 0, ) .forEach((clientMetric) => - this.lastSeenToggles.add(clientMetric.featureName), + this.lastSeenToggles.add({ + featureName: clientMetric.featureName, + environment: clientMetric.environment, + }), ); } diff --git a/src/lib/types/model.ts b/src/lib/types/model.ts index c1b02ec0e9..769c602b09 100644 --- a/src/lib/types/model.ts +++ b/src/lib/types/model.ts @@ -114,6 +114,7 @@ export interface IFeatureEnvironment { environment: string; featureName: string; enabled: boolean; + lastSeenAt?: Date; variants?: IVariant[]; } @@ -170,6 +171,7 @@ export interface IEnvironmentBase { enabled: boolean; type: string; sortOrder: number; + lastSeenAt: Date; } export interface IEnvironmentOverview extends IEnvironmentBase { diff --git a/src/lib/types/stores/feature-toggle-store.ts b/src/lib/types/stores/feature-toggle-store.ts index 60694f73c6..9108519fb1 100644 --- a/src/lib/types/stores/feature-toggle-store.ts +++ b/src/lib/types/stores/feature-toggle-store.ts @@ -1,5 +1,6 @@ import { FeatureToggle, FeatureToggleDTO, IVariant } from '../model'; import { Store } from './store'; +import { LastSeenInput } from '../../services/client-metrics/last-seen-service'; export interface IFeatureToggleQuery { archived: boolean; @@ -10,7 +11,7 @@ export interface IFeatureToggleQuery { export interface IFeatureToggleStore extends Store { count(query?: Partial): Promise; - setLastSeen(toggleNames: string[]): Promise; + setLastSeen(data: LastSeenInput[]): Promise; getProjectId(name: string): Promise; create(project: string, data: FeatureToggleDTO): Promise; update(project: string, data: FeatureToggleDTO): Promise; diff --git a/src/migrations/20230802141830-add-feature-and-environment-last-seen-at-to-features-view.js b/src/migrations/20230802141830-add-feature-and-environment-last-seen-at-to-features-view.js new file mode 100644 index 0000000000..d47ba94962 --- /dev/null +++ b/src/migrations/20230802141830-add-feature-and-environment-last-seen-at-to-features-view.js @@ -0,0 +1,90 @@ +'use strict'; + +exports.up = function (db, callback) { + db.runSql( + ` + ALTER TABLE feature_environments ADD COLUMN IF NOT EXISTS last_seen_at timestamp with time zone; + + DROP VIEW features_view; + + CREATE VIEW features_view AS + SELECT + features.name as name, + features.description as description, + features.type as type, + features.project as project, + features.stale as stale, + features.impression_data as impression_data, + features.created_at as created_at, + features.archived_at as archived_at, + features.last_seen_at as last_seen_at, + feature_environments.last_seen_at as env_last_seen_at, + feature_environments.enabled as enabled, + feature_environments.environment as environment, + feature_environments.variants as variants, + environments.name as environment_name, + environments.type as environment_type, + environments.sort_order as environment_sort_order, + feature_strategies.id as strategy_id, + feature_strategies.strategy_name as strategy_name, + feature_strategies.parameters as parameters, + feature_strategies.constraints as constraints, + feature_strategies.sort_order as sort_order, + fss.segment_id as segments, + feature_strategies.title as strategy_title, + feature_strategies.disabled as strategy_disabled, + feature_strategies.variants as strategy_variants + FROM + features + LEFT JOIN feature_environments ON feature_environments.feature_name = features.name + LEFT JOIN feature_strategies ON feature_strategies.feature_name = feature_environments.feature_name + and feature_strategies.environment = feature_environments.environment + LEFT JOIN environments ON feature_environments.environment = environments.name + LEFT JOIN feature_strategy_segment as fss ON fss.feature_strategy_id = feature_strategies.id; + `, + callback, + ); +}; + +exports.down = function (db, callback) { + db.runSql( + ` + DROP VIEW features_view; + + CREATE VIEW features_view AS + SELECT + features.name as name, + features.description as description, + features.type as type, + features.project as project, + features.stale as stale, + features.impression_data as impression_data, + features.created_at as created_at, + features.archived_at as archived_at, + feature_environments.last_seen_at as last_seen_at, + feature_environments.enabled as enabled, + feature_environments.environment as environment, + feature_environments.variants as variants, + environments.name as environment_name, + environments.type as environment_type, + environments.sort_order as environment_sort_order, + feature_strategies.id as strategy_id, + feature_strategies.strategy_name as strategy_name, + feature_strategies.parameters as parameters, + feature_strategies.constraints as constraints, + feature_strategies.sort_order as sort_order, + fss.segment_id as segments, + feature_strategies.title as strategy_title, + feature_strategies.disabled as strategy_disabled, + feature_strategies.variants as strategy_variants + FROM + features + LEFT JOIN feature_environments ON feature_environments.feature_name = features.name + LEFT JOIN feature_strategies ON feature_strategies.feature_name = feature_environments.feature_name + and feature_strategies.environment = feature_environments.environment + LEFT JOIN environments ON feature_environments.environment = environments.name + LEFT JOIN feature_strategy_segment as fss ON fss.feature_strategy_id = feature_strategies.id; + `, + callback, + ); +}; diff --git a/src/test/e2e/api/client/metricsV2.e2e.test.ts b/src/test/e2e/api/client/metricsV2.e2e.test.ts index f7ad1d00b9..f7cd95c171 100644 --- a/src/test/e2e/api/client/metricsV2.e2e.test.ts +++ b/src/test/e2e/api/client/metricsV2.e2e.test.ts @@ -98,7 +98,7 @@ test('should pick up environment from token', async () => { expect(metrics[0].appName).toBe('some-fancy-app'); }); -test('should set lastSeen for toggles with metrics', async () => { +test('should set lastSeen for toggles with metrics both for toggle and toggle env', async () => { const start = Date.now(); await app.services.featureToggleServiceV2.createFeatureToggle( 'default', @@ -110,6 +110,7 @@ test('should set lastSeen for toggles with metrics', async () => { { name: 't2' }, 'tester', ); + const token = await app.services.apiTokenService.createApiToken({ type: ApiTokenType.CLIENT, project: 'default', @@ -142,8 +143,24 @@ test('should set lastSeen for toggles with metrics', async () => { await app.services.clientMetricsServiceV2.bulkAdd(); await app.services.lastSeenService.store(); - const t1 = await db.stores.featureToggleStore.get('t1'); - const t2 = await db.stores.featureToggleStore.get('t2'); - expect(t1.lastSeenAt.getTime()).toBeGreaterThanOrEqual(start); - expect(t2.lastSeenAt).toBeDefined(); + const t1 = await app.services.featureToggleServiceV2.getFeature({ + featureName: 't1', + archived: false, + environmentVariants: true, + projectId: 'default', + }); + const t2 = await app.services.featureToggleServiceV2.getFeature({ + featureName: 't2', + archived: false, + environmentVariants: true, + projectId: 'default', + }); + + const t1Env = t1.environments.find((e) => e.name === 'default'); + const t2Env = t2.environments.find((e) => e.name === 'default'); + + expect(t1.lastSeenAt?.getTime()).toBeGreaterThanOrEqual(start); + expect(t1Env?.lastSeenAt.getTime()).toBeGreaterThanOrEqual(start); + expect(t2?.lastSeenAt).toBeDefined(); + expect(t2Env?.lastSeenAt).toBeDefined(); }); diff --git a/src/test/e2e/services/environment-service.test.ts b/src/test/e2e/services/environment-service.test.ts index 65d05539b3..39981838e5 100644 --- a/src/test/e2e/services/environment-service.test.ts +++ b/src/test/e2e/services/environment-service.test.ts @@ -62,6 +62,7 @@ test('Can connect environment to project', async () => { sortOrder: 9999, type: 'production', variantCount: 0, + lastSeenAt: null, }, ]); }); @@ -89,6 +90,7 @@ test('Can remove environment from project', async () => { sortOrder: 9999, type: 'production', variantCount: 0, + lastSeenAt: null, }, ]); }); diff --git a/src/test/fixtures/fake-feature-toggle-store.ts b/src/test/fixtures/fake-feature-toggle-store.ts index 5b50350554..6acd59c937 100644 --- a/src/test/fixtures/fake-feature-toggle-store.ts +++ b/src/test/fixtures/fake-feature-toggle-store.ts @@ -9,6 +9,8 @@ import { IFeatureEnvironment, IVariant, } from 'lib/types/model'; +import { LastSeenInput } from '../../lib/services/client-metrics/last-seen-service'; +import { EnvironmentFeatureNames } from '../../lib/db/feature-toggle-store'; export default class FakeFeatureToggleStore implements IFeatureToggleStore { features: FeatureToggle[] = []; @@ -161,13 +163,33 @@ export default class FakeFeatureToggleStore implements IFeatureToggleStore { throw new NotFoundError('Could not find feature to update'); } - async setLastSeen(toggleNames: string[]): Promise { - toggleNames.forEach((t) => { - const toUpdate = this.features.find((f) => f.name === t); - if (toUpdate) { - toUpdate.lastSeenAt = new Date(); + async setLastSeen(data: LastSeenInput[]): Promise { + const envArrays = data.reduce( + (acc: EnvironmentFeatureNames, feature: LastSeenInput) => { + const { environment, featureName } = feature; + + if (!acc[environment]) { + acc[environment] = []; + } + + acc[environment].push(featureName); + + return acc; + }, + {}, + ); + + for (const env of Object.keys(envArrays)) { + const toggleNames = envArrays[env]; + if (toggleNames && Array.isArray(toggleNames)) { + toggleNames.forEach((t) => { + const toUpdate = this.features.find((f) => f.name === t); + if (toUpdate) { + toUpdate.lastSeenAt = new Date(); + } + }); } - }); + } } async getVariants(featureName: string): Promise {