feat: create global repository for frontend repositories (#6460)

Co-authored-by: kwasniew <kwasniewski.mateusz@gmail.com>

src/lib/proxy/client-feature-toggle-read-model.ts

@@ -0,0 +1,234 @@
+import { Knex } from 'knex';
+import {
+    IFeatureToggleClient,
+    IFeatureToggleQuery,
+    IStrategyConfig,
+    ITag,
+    PartialDeep,
+} from '../types';
+import { ensureStringValue, mapValues } from '../util';
+import { Db } from '../db/db';
+import FeatureToggleStore from '../features/feature-toggle/feature-toggle-store';
+import Raw = Knex.Raw;
+import metricsHelper from '../util/metrics-helper';
+import { DB_TIME } from '../metric-events';
+import EventEmitter from 'events';
+
+export interface IGetAllFeatures {
+    featureQuery?: IFeatureToggleQuery;
+    archived: boolean;
+    userId?: number;
+}
+
+export default class ClientFeatureToggleReadModel {
+    private db: Db;
+
+    private timer: Function;
+
+    constructor(db: Db, eventBus: EventEmitter) {
+        this.db = db;
+        this.timer = (action) =>
+            metricsHelper.wrapTimer(eventBus, DB_TIME, {
+                store: 'client-feature-toggle-read-model',
+                action,
+            });
+    }
+
+    private async getAll(): Promise<Record<string, IFeatureToggleClient[]>> {
+        const stopTimer = this.timer(`getAll`);
+        const selectColumns = [
+            'features.name as name',
+            'features.description as description',
+            'features.type as type',
+            'features.project as project',
+            'features.stale as stale',
+            'features.impression_data as impression_data',
+            'features.created_at as created_at',
+            'fe.variants as variants',
+            'fe.enabled as enabled',
+            'fe.environment as environment',
+            'fs.id as strategy_id',
+            'fs.strategy_name as strategy_name',
+            'fs.title as strategy_title',
+            'fs.disabled as strategy_disabled',
+            'fs.parameters as parameters',
+            'fs.constraints as constraints',
+            'fs.sort_order as sort_order',
+            'fs.variants as strategy_variants',
+            'segments.id as segment_id',
+            'segments.constraints as segment_constraints',
+            'df.parent as parent',
+            'df.variants as parent_variants',
+            'df.enabled as parent_enabled',
+        ] as (string | Raw<any>)[];
+
+        let query = this.db('features')
+            .modify(FeatureToggleStore.filterByArchived, false)
+            .leftJoin(
+                this.db('feature_environments')
+                    .select(
+                        'feature_name',
+                        'enabled',
+                        'environment',
+                        'variants',
+                    )
+                    .as('fe'),
+                'fe.feature_name',
+                'features.name',
+            )
+            .leftJoin('feature_strategies as fs', function () {
+                this.on('fs.feature_name', '=', 'features.name').andOn(
+                    'fs.environment',
+                    '=',
+                    'fe.environment',
+                );
+            })
+            .leftJoin(
+                'feature_strategy_segment as fss',
+                `fss.feature_strategy_id`,
+                `fs.id`,
+            )
+            .leftJoin('segments', `segments.id`, `fss.segment_id`)
+            .leftJoin('dependent_features as df', 'df.child', 'features.name');
+
+        query = query.select(selectColumns);
+        const rows = await query;
+        stopTimer();
+
+        const data = this.getAggregatedData(rows);
+        return data;
+    }
+
+    getAggregatedData(rows): Record<string, IFeatureToggleClient[]> {
+        const featureTogglesByEnv: Record<string, IFeatureToggleClient[]> = {};
+
+        rows.forEach((row) => {
+            const environment = row.environment;
+
+            if (!featureTogglesByEnv[environment]) {
+                featureTogglesByEnv[environment] = [];
+            }
+
+            let feature = featureTogglesByEnv[environment].find(
+                (f) => f.name === row.name,
+            );
+
+            if (!feature) {
+                feature = {
+                    name: row.name,
+                    strategies: [],
+                    variants: row.variants || [],
+                    impressionData: row.impression_data,
+                    enabled: !!row.enabled,
+                    description: row.description,
+                    project: row.project,
+                    stale: row.stale,
+                    type: row.type,
+                };
+                featureTogglesByEnv[environment].push(feature);
+            } else {
+                if (this.isNewTag(feature, row)) {
+                    this.addTag(feature, row);
+                }
+            }
+            if (row.parent) {
+                feature.dependencies = feature.dependencies || [];
+                feature.dependencies.push({
+                    feature: row.parent,
+                    enabled: row.parent_enabled,
+                    ...(row.parent_enabled
+                        ? { variants: row.parent_variants }
+                        : {}),
+                });
+            }
+
+            if (
+                this.isUnseenStrategyRow(feature, row) &&
+                !row.strategy_disabled
+            ) {
+                feature.strategies?.push(this.rowToStrategy(row));
+            }
+        });
+        Object.keys(featureTogglesByEnv).forEach((envKey) => {
+            featureTogglesByEnv[envKey] = featureTogglesByEnv[envKey].map(
+                (featureToggle) => ({
+                    ...featureToggle,
+                    strategies: featureToggle.strategies
+                        ?.sort((strategy1, strategy2) => {
+                            if (
+                                typeof strategy1.sortOrder === 'number' &&
+                                typeof strategy2.sortOrder === 'number'
+                            ) {
+                                return (
+                                    strategy1.sortOrder - strategy2.sortOrder
+                                );
+                            }
+                            return 0;
+                        })
+                        .map(({ id, title, sortOrder, ...strategy }) => ({
+                            ...strategy,
+                        })),
+                }),
+            );
+        });
+
+        return featureTogglesByEnv;
+    }
+
+    private rowToStrategy(row: Record<string, any>): IStrategyConfig {
+        const strategy: IStrategyConfig = {
+            id: row.strategy_id,
+            name: row.strategy_name,
+            title: row.strategy_title,
+            constraints: row.constraints || [],
+            parameters: mapValues(row.parameters || {}, ensureStringValue),
+            sortOrder: row.sort_order,
+        };
+        strategy.variants = row.strategy_variants || [];
+        return strategy;
+    }
+
+    private static rowToTag(row: Record<string, any>): ITag {
+        return {
+            value: row.tag_value,
+            type: row.tag_type,
+        };
+    }
+
+    private isUnseenStrategyRow(
+        feature: PartialDeep<IFeatureToggleClient>,
+        row: Record<string, any>,
+    ): boolean {
+        return (
+            row.strategy_id &&
+            !feature.strategies?.find((s) => s?.id === row.strategy_id)
+        );
+    }
+
+    private addTag(
+        feature: Record<string, any>,
+        row: Record<string, any>,
+    ): void {
+        const tags = feature.tags || [];
+        const newTag = ClientFeatureToggleReadModel.rowToTag(row);
+        feature.tags = [...tags, newTag];
+    }
+
+    private isNewTag(
+        feature: PartialDeep<IFeatureToggleClient>,
+        row: Record<string, any>,
+    ): boolean {
+        return (
+            row.tag_type &&
+            row.tag_value &&
+            !feature.tags?.some(
+                (tag) =>
+                    tag?.type === row.tag_type && tag?.value === row.tag_value,
+            )
+        );
+    }
+
+    async getClient(): Promise<Record<string, IFeatureToggleClient[]>> {
+        return this.getAll();
+    }
+}

src/lib/proxy/frontend-api-repository.ts

@@ -0,0 +1,72 @@
+import EventEmitter from 'events';
+import { RepositoryInterface } from 'unleash-client/lib/repository';
+import { Segment } from 'unleash-client/lib/strategy/strategy';
+import {
+    EnhancedFeatureInterface,
+    FeatureInterface,
+} from 'unleash-client/lib/feature';
+import { IApiUser } from '../types/api-user';
+import { IUnleashConfig } from '../types';
+import { UnleashEvents } from 'unleash-client';
+import { Logger } from '../logger';
+import { GlobalFrontendApiRepository } from './global-frontend-api-repository';
+
+type Config = Pick<IUnleashConfig, 'getLogger'>;
+
+export class FrontendApiRepository
+    extends EventEmitter
+    implements RepositoryInterface
+{
+    private readonly config: Config;
+
+    private readonly logger: Logger;
+
+    private readonly token: IApiUser;
+
+    private globalFrontendApiRepository: GlobalFrontendApiRepository;
+
+    private running: boolean;
+
+    constructor(
+        config: Config,
+        globalFrontendApiRepository: GlobalFrontendApiRepository,
+        token: IApiUser,
+    ) {
+        super();
+        this.config = config;
+        this.logger = config.getLogger('frontend-api-repository.ts');
+        this.token = token;
+        this.globalFrontendApiRepository = globalFrontendApiRepository;
+    }
+
+    getTogglesWithSegmentData(): EnhancedFeatureInterface[] {
+        // TODO: add real implementation
+        return [];
+    }
+
+    getSegment(id: number): Segment | undefined {
+        return this.globalFrontendApiRepository.getSegment(id);
+    }
+
+    getToggle(name: string): FeatureInterface {
+        //@ts-ignore (we must update the node SDK to allow undefined)
+        return this.globalFrontendApiRepository
+            .getToggles(this.token)
+            .find((feature) => feature.name);
+    }
+
+    getToggles(): FeatureInterface[] {
+        return this.globalFrontendApiRepository.getToggles(this.token);
+    }
+
+    async start(): Promise<void> {
+        this.running = true;
+
+        this.emit(UnleashEvents.Ready);
+        this.emit(UnleashEvents.Changed);
+    }
+
+    stop(): void {
+        this.running = false;
+    }
+}

src/lib/proxy/global-frontend-api-repository.ts

@@ -0,0 +1,106 @@
+import EventEmitter from 'events';
+import { Segment } from 'unleash-client/lib/strategy/strategy';
+import { FeatureInterface } from 'unleash-client/lib/feature';
+import { IApiUser } from '../types/api-user';
+import { ISegmentReadModel, IUnleashConfig } from '../types';
+import {
+    mapFeaturesForClient,
+    mapSegmentsForClient,
+} from '../features/playground/offline-unleash-client';
+import { ALL_ENVS } from '../util/constants';
+import { Logger } from '../logger';
+import ConfigurationRevisionService, {
+    UPDATE_REVISION,
+} from '../features/feature-toggle/configuration-revision-service';
+import ClientFeatureToggleReadModel from './client-feature-toggle-read-model';
+import { mapValues } from '../util';
+
+type Config = Pick<IUnleashConfig, 'getLogger' | 'frontendApi' | 'eventBus'>;
+
+export class GlobalFrontendApiRepository extends EventEmitter {
+    private readonly config: Config;
+
+    private readonly logger: Logger;
+
+    private readonly clientFeatureToggleReadModel: ClientFeatureToggleReadModel;
+
+    private readonly segmentReadModel: ISegmentReadModel;
+
+    private readonly configurationRevisionService: ConfigurationRevisionService;
+
+    private featuresByEnvironment: Record<string, FeatureInterface[]>;
+
+    private segments: Segment[];
+
+    private interval: number;
+
+    private running: boolean;
+
+    constructor(
+        config: Config,
+        segmentReadModel: ISegmentReadModel,
+        clientFeatureToggleReadModel: ClientFeatureToggleReadModel,
+        configurationRevisionService: ConfigurationRevisionService,
+    ) {
+        super();
+        this.config = config;
+        this.logger = config.getLogger('proxy-repository.ts');
+        this.clientFeatureToggleReadModel = clientFeatureToggleReadModel;
+        this.configurationRevisionService = configurationRevisionService;
+        this.segmentReadModel = segmentReadModel;
+        this.onUpdateRevisionEvent = this.onUpdateRevisionEvent.bind(this);
+        this.interval = config.frontendApi.refreshIntervalInMs;
+        this.refreshData();
+        this.configurationRevisionService.on(
+            UPDATE_REVISION,
+            this.onUpdateRevisionEvent,
+        );
+    }
+
+    getSegment(id: number): Segment | undefined {
+        return this.segments.find((segment) => segment.id === id);
+    }
+
+    getToggles(token: IApiUser): FeatureInterface[] {
+        return this.featuresByEnvironment[
+            this.environmentNameForToken(token)
+        ].filter(
+            (feature) =>
+                token.projects.includes('*') ||
+                (feature.project && token.projects.includes(feature.project)),
+        );
+    }
+
+    private async getAllFeatures(): Promise<
+        Record<string, FeatureInterface[]>
+    > {
+        const features = await this.clientFeatureToggleReadModel.getClient();
+        return mapValues(features, mapFeaturesForClient);
+    }
+
+    private async getAllSegments(): Promise<Segment[]> {
+        return mapSegmentsForClient(await this.segmentReadModel.getAll());
+    }
+
+    // TODO: fetch only relevant projects/environments based on tokens
+    // TODO: also consider not fetching disabled features, because those are not returned by frontend API
+    private async refreshData() {
+        try {
+            this.featuresByEnvironment = await this.getAllFeatures();
+            this.segments = await this.getAllSegments();
+        } catch (e) {
+            this.logger.error('Cannot load data for token', e);
+        }
+    }
+
+    private async onUpdateRevisionEvent() {
+        await this.refreshData();
+    }
+
+    private environmentNameForToken(token: IApiUser): string {
+        if (token.environment === ALL_ENVS) {
+            return 'default';
+        }
+        return token.environment;
+    }
+}

src/lib/services/index.ts

@@ -1,4 +1,4 @@
-import { IUnleashConfig, IUnleashStores, IUnleashServices } from '../types';
+import { IUnleashConfig, IUnleashServices, IUnleashStores } from '../types';
 import FeatureTypeService from './feature-type-service';
 import EventService from '../features/events/event-service';
 import HealthService from './health-service';
@@ -96,8 +96,8 @@ import {
 } from '../features/client-feature-toggles/createClientFeatureToggleService';
 import { ClientFeatureToggleService } from '../features/client-feature-toggles/client-feature-toggle-service';
 import {
-    createFeatureSearchService,
     createFakeFeatureSearchService,
+    createFeatureSearchService,
 } from '../features/feature-search/createFeatureSearchService';
 import { FeatureSearchService } from '../features/feature-search/feature-search-service';
 import {

src/lib/services/proxy-service.ts

@@ -8,7 +8,6 @@ import {
     Unleash,
     UnleashEvents,
 } from 'unleash-client';
-import { ProxyRepository } from '../proxy';
 import { ApiTokenType } from '../types/models/api-token';
 import {
     FrontendSettings,
@@ -17,6 +16,7 @@ import {
 import { validateOrigins } from '../util';
 import { BadDataError, InvalidTokenError } from '../error';
 import { PROXY_REPOSITORY_CREATED } from '../metric-events';
+import { ProxyRepository } from '../proxy';
 
 type Config = Pick<
     IUnleashConfig,
@@ -68,7 +68,6 @@ export class ProxyService {
     ): Promise<ProxyFeatureSchema[]> {
         const client = await this.clientForProxyToken(token);
         const definitions = client.getFeatureToggleDefinitions() || [];
-
         const sessionId = context.sessionId || String(Math.random());
 
         return definitions
@@ -125,7 +124,6 @@ export class ProxyService {
             this.services,
             token,
         );
-
         const client = new Unleash({
             appName: 'proxy',
             url: 'unused',