Feat/configure scheduled created by migration (#6821)

## About the changes

- Removes the feature flag for the created_by migrations.
- Adds a configuration option in IServerOption for
`ENABLE_SCHEDULED_CREATED_BY_MIGRATION` that defaults to `false`
- the new configuration option when set on startup enables scheduling of
the two created_by migration services (features+events)
- Removes the dependency on flag provider in EventStore as it's no
longer needed
- Adds a brief description of the new configuration option in
`configuring-unleash.md`
- Sets the events created_by migration interval to 15 minutes, up from
2.

---------

Co-authored-by: Gastón Fournier <gaston@getunleash.io>

src/lib/__snapshots__/create-config.test.ts.snap

@@ -80,7 +80,6 @@ exports[`should create default config 1`] = `
       "caseInsensitiveInOperators": false,
       "celebrateUnleash": false,
       "collectTrafficDataUsage": false,
-      "createdByUserIdDataMigration": false,
       "demo": false,
       "disableBulkToggle": false,
       "disableMetrics": false,
@@ -205,6 +204,7 @@ exports[`should create default config 1`] = `
     "disableCompression": false,
     "enableHeapSnapshotEnpoint": false,
     "enableRequestLogger": false,
+    "enableScheduledCreatedByMigration": false,
     "gracefulShutdownEnable": true,
     "gracefulShutdownTimeout": 1000,
     "headersTimeout": 61000,

src/lib/create-config.ts

@@ -259,6 +259,10 @@ const defaultServerOption: IServerOption = {
         secondsToMilliseconds(1),
     ),
     secret: process.env.UNLEASH_SECRET || 'super-secret',
+    enableScheduledCreatedByMigration: parseEnvVarBoolean(
+        process.env.ENABLE_SCHEDULED_CREATED_BY_MIGRATION,
+        false,
+    ),
 };
 
 const defaultVersionOption: IVersionOption = {

src/lib/db/index.ts

@@ -48,8 +48,8 @@ export const createStores = (
     config: IUnleashConfig,
     db: Db,
 ): IUnleashStores => {
-    const { getLogger, eventBus, flagResolver } = config;
-    const eventStore = new EventStore(db, getLogger, flagResolver);
+    const { getLogger, eventBus } = config;
+    const eventStore = new EventStore(db, getLogger);
 
     return {
         eventStore,

src/lib/features/events/createEventsService.ts

@@ -10,11 +10,7 @@ export const createEventsService: (
     db: Db,
     config: IUnleashConfig,
 ) => EventService = (db, config) => {
-    const eventStore = new EventStore(
-        db,
-        config.getLogger,
-        config.flagResolver,
-    );
+    const eventStore = new EventStore(db, config.getLogger);
     const featureTagStore = new FeatureTagStore(
         db,
         config.eventBus,

src/lib/features/events/event-created-by-migration.test.ts

@@ -1,20 +1,13 @@
 import EventStore from './event-store';
 import getLogger from '../../../test/fixtures/no-logger';
 import dbInit, { type ITestDb } from '../../../test/e2e/helpers/database-init';
-import { defaultExperimentalOptions } from '../../types/experimental';
-import FlagResolver from '../../util/flag-resolver';
 import { EventEmitter } from 'stream';
 import EventService from './event-service';
 import { EVENTS_CREATED_BY_PROCESSED } from '../../metric-events';
 
 let db: ITestDb;
-let resolver: FlagResolver;
 
 beforeAll(async () => {
-    resolver = new FlagResolver({
-        ...defaultExperimentalOptions,
-        flags: { createdByUserIdDataMigration: true },
-    });
     db = await dbInit('events_test', getLogger);
 });
 
@@ -25,7 +18,7 @@ afterAll(async () => {
 });
 
 test('sets created_by_user_id on events with user username/email set as created_by', async () => {
-    const store = new EventStore(db.rawDatabase, getLogger, resolver);
+    const store = new EventStore(db.rawDatabase, getLogger);
 
     await db.rawDatabase('users').insert({ username: 'test1' });
     await db.rawDatabase('events').insert({
@@ -55,7 +48,7 @@ test('sets created_by_user_id on events with user username/email set as created_
 });
 
 test('sets created_by_user_id on a mix of events and created_bys', async () => {
-    const store = new EventStore(db.rawDatabase, getLogger, resolver);
+    const store = new EventStore(db.rawDatabase, getLogger);
 
     await db.rawDatabase('users').insert({ username: 'test2' });
 
@@ -126,7 +119,7 @@ test('sets created_by_user_id on a mix of events and created_bys', async () => {
 });
 
 test('emits events with details on amount of updated rows', async () => {
-    const store = new EventStore(db.rawDatabase, getLogger, resolver);
+    const store = new EventStore(db.rawDatabase, getLogger);
 
     const eventBus = new EventEmitter();
     const service = new EventService(

src/lib/features/events/event-store.test.ts

@@ -3,13 +3,8 @@ import EventStore from './event-store';
 import getLogger from '../../../test/fixtures/no-logger';
 import { subHours, formatRFC3339 } from 'date-fns';
 import dbInit from '../../../test/e2e/helpers/database-init';
-import { defaultExperimentalOptions } from '../../types/experimental';
-import FlagResolver from '../../util/flag-resolver';
-
-let resolver: FlagResolver;
 
 beforeAll(() => {
-    resolver = new FlagResolver(defaultExperimentalOptions);
     getLogger.setMuteError(true);
 });
 
@@ -21,7 +16,7 @@ test('Trying to get events if db fails should yield empty list', async () => {
     const db = knex({
         client: 'pg',
     });
-    const store = new EventStore(db, getLogger, resolver);
+    const store = new EventStore(db, getLogger);
     const events = await store.getEvents();
     expect(events.length).toBe(0);
     await db.destroy();
@@ -31,7 +26,7 @@ test('Trying to get events by name if db fails should yield empty list', async (
     const db = knex({
         client: 'pg',
     });
-    const store = new EventStore(db, getLogger, resolver);
+    const store = new EventStore(db, getLogger);
     const events = await store.searchEvents({ type: 'application-created' });
     expect(events).toBeTruthy();
     expect(events.length).toBe(0);
@@ -51,7 +46,7 @@ test('Find unannounced events returns all events', async () => {
     }));
     await db.rawDatabase('events').insert(allEvents).returning(['id']);
 
-    const store = new EventStore(db.rawDatabase, getLogger, resolver);
+    const store = new EventStore(db.rawDatabase, getLogger);
     const events = await store.setUnannouncedToAnnounced();
     expect(events).toBeTruthy();
     expect(events.length).toBe(505);

src/lib/features/events/event-store.ts

@@ -14,11 +14,7 @@ import { sharedEventEmitter } from '../../util/anyEventEmitter';
 import type { Db } from '../../db/db';
 import type { Knex } from 'knex';
 import type EventEmitter from 'events';
-import {
-    ADMIN_TOKEN_USER,
-    type IFlagResolver,
-    SYSTEM_USER_ID,
-} from '../../types';
+import { ADMIN_TOKEN_USER, SYSTEM_USER_ID } from '../../types';
 
 const EVENT_COLUMNS = [
     'id',
@@ -97,15 +93,12 @@ class EventStore implements IEventStore {
     // only one shared event emitter should exist across all event store instances
     private eventEmitter: EventEmitter = sharedEventEmitter;
 
-    private flagResolver: IFlagResolver;
-
     private logger: Logger;
 
     // a new DB has to be injected per transaction
-    constructor(db: Db, getLogger: LogProvider, flagResolver: IFlagResolver) {
+    constructor(db: Db, getLogger: LogProvider) {
         this.db = db;
         this.logger = getLogger('event-store');
-        this.flagResolver = flagResolver;
     }
 
     async store(event: IBaseEvent): Promise<void> {
@@ -443,10 +436,6 @@ class EventStore implements IEventStore {
     async setCreatedByUserId(batchSize: number): Promise<number | undefined> {
         const API_TOKEN_TABLE = 'api_tokens';
 
-        if (!this.flagResolver.isEnabled('createdByUserIdDataMigration')) {
-            return undefined;
-        }
-
         const toUpdate = await this.db(`${TABLE} as e`)
             .joinRaw(
                 'LEFT OUTER JOIN users AS u ON e.created_by = u.username OR e.created_by = u.email',

src/lib/features/export-import-toggles/createExportImportService.ts

@@ -180,7 +180,7 @@ export const deferredExportImportTogglesService = (
             eventBus,
             getLogger,
         );
-        const eventStore = new EventStore(db, getLogger, flagResolver);
+        const eventStore = new EventStore(db, getLogger);
         const accessService = createAccessService(db, config);
         const featureToggleService = createFeatureToggleService(db, config);
         const privateProjectChecker = createPrivateProjectChecker(db, config);

src/lib/features/feature-lifecycle/createFeatureLifecycle.ts

@@ -13,7 +13,7 @@ export const createFeatureLifecycleService = (
     config: IUnleashConfig,
 ) => {
     const { eventBus, getLogger, flagResolver } = config;
-    const eventStore = new EventStore(db, getLogger, flagResolver);
+    const eventStore = new EventStore(db, getLogger);
     const featureLifecycleStore = new FeatureLifecycleStore(db);
     const environmentStore = new EnvironmentStore(db, eventBus, getLogger);
     const featureLifecycleService = new FeatureLifecycleService(

src/lib/features/feature-toggle/feature-toggle-store.ts

@@ -730,9 +730,6 @@ export default class FeatureToggleStore implements IFeatureToggleStore {
         const USERS_TABLE = 'users';
         const API_TOKEN_TABLE = 'api_tokens';
 
-        if (!this.flagResolver.isEnabled('createdByUserIdDataMigration')) {
-            return undefined;
-        }
         const toUpdate = await this.db(`${TABLE} as f`)
             .joinRaw(`JOIN ${EVENTS_TABLE} AS ev ON ev.feature_name = f.name`)
             .joinRaw(

src/lib/features/instance-stats/createInstanceStatsService.ts

@@ -77,7 +77,7 @@ export const createInstanceStatsService = (db: Db, config: IUnleashConfig) => {
         eventBus,
         getLogger,
     );
-    const eventStore = new EventStore(db, getLogger, flagResolver);
+    const eventStore = new EventStore(db, getLogger);
     const apiTokenStore = new ApiTokenStore(db, eventBus, getLogger);
     const clientMetricsStoreV2 = new ClientMetricsStoreV2(
         db,

src/lib/features/project/createProjectService.ts

@@ -47,7 +47,7 @@ export const createProjectService = (
     config: IUnleashConfig,
 ): ProjectService => {
     const { eventBus, getLogger, flagResolver } = config;
-    const eventStore = new EventStore(db, getLogger, flagResolver);
+    const eventStore = new EventStore(db, getLogger);
     const projectStore = new ProjectStore(
         db,
         eventBus,

src/lib/features/scheduler/schedule-services.ts

@@ -3,7 +3,7 @@ import {
     minutesToMilliseconds,
     secondsToMilliseconds,
 } from 'date-fns';
-import type { IUnleashServices } from '../../server-impl';
+import type { IUnleashConfig, IUnleashServices } from '../../server-impl';
 
 /**
  * Schedules service methods.
@@ -13,6 +13,7 @@ import type { IUnleashServices } from '../../server-impl';
  */
 export const scheduleServices = async (
     services: IUnleashServices,
+    config: IUnleashConfig,
 ): Promise<void> => {
     const {
         accountService,
@@ -150,16 +151,18 @@ export const scheduleServices = async (
         'updateAccountLastSeen',
     );
 
-    schedulerService.schedule(
-        eventService.setEventCreatedByUserId.bind(eventService),
-        minutesToMilliseconds(2),
-        'setEventCreatedByUserId',
-    );
-    schedulerService.schedule(
-        featureToggleService.setFeatureCreatedByUserIdFromEvents.bind(
-            featureToggleService,
-        ),
-        minutesToMilliseconds(15),
-        'setFeatureCreatedByUserIdFromEvents',
-    );
+    if (config.server.enableScheduledCreatedByMigration) {
+        schedulerService.schedule(
+            eventService.setEventCreatedByUserId.bind(eventService),
+            minutesToMilliseconds(15),
+            'setEventCreatedByUserId',
+        );
+        schedulerService.schedule(
+            featureToggleService.setFeatureCreatedByUserIdFromEvents.bind(
+                featureToggleService,
+            ),
+            minutesToMilliseconds(15),
+            'setFeatureCreatedByUserIdFromEvents',
+        );
+    }
 };

src/lib/server-impl.ts

@@ -47,7 +47,7 @@ async function createApp(
     const stores = createStores(config, db);
     const services = createServices(stores, config, db);
     if (!config.disableScheduler) {
-        await scheduleServices(services);
+        await scheduleServices(services, config);
     }
 
     const metricsMonitor = createMetricsMonitor();

src/lib/types/experimental.ts

@@ -38,7 +38,6 @@ export type IFlagKey =
     | 'executiveDashboard'
     | 'executiveDashboardUI'
     | 'feedbackComments'
-    | 'createdByUserIdDataMigration'
     | 'showInactiveUsers'
     | 'inMemoryScheduledChangeRequests'
     | 'collectTrafficDataUsage'
@@ -207,10 +206,6 @@ const flags: IFlags = {
                 '',
         },
     },
-    createdByUserIdDataMigration: parseEnvVarBoolean(
-        process.env.CREATED_BY_USERID_DATA_MIGRATION,
-        false,
-    ),
     showInactiveUsers: parseEnvVarBoolean(
         process.env.UNLEASH_EXPERIMENTAL_SHOW_INACTIVE_USERS,
         false,

src/lib/types/option.ts

@@ -100,6 +100,7 @@ export interface IServerOption {
     gracefulShutdownEnable: boolean;
     gracefulShutdownTimeout: number;
     secret: string;
+    enableScheduledCreatedByMigration: boolean;
 }
 
 export interface IClientCachingOption {

src/test/config/test-config.ts

@@ -27,7 +27,6 @@ export function createTestConfig(config?: IUnleashOptions): IUnleashConfig {
             flags: {
                 embedProxy: true,
                 embedProxyFrontend: true,
-                createdByUserIdDataMigration: true,
             },
         },
         publicFolder: path.join(__dirname, '../examples'),

website/docs/using-unleash/deploy/configuring-unleash.md

@@ -121,6 +121,7 @@ unleash.start(unleashOptions);
   - _unleashUrl_ (string) - Used to specify the official URL this instance of Unleash can be accessed at for an end user. Can also be configured through the environment variable `UNLEASH_URL`.
   - _gracefulShutdownEnable_: (boolean) - Used to control if Unleash should shutdown gracefully (close connections, stop tasks,). Defaults to true. `GRACEFUL_SHUTDOWN_ENABLE`
   - _gracefulShutdownTimeout_: (number) - Used to control the timeout, in milliseconds, for shutdown Unleash gracefully. Will kill all connections regardless if this timeout is exceeded. Defaults to 1000ms `GRACEFUL_SHUTDOWN_TIMEOUT`
+  - _enableScheduledCreatedByMigration_: (boolean) - Schedules migrations that fills the field created_by_user_id for past events and features. It does it gradually in chunks and runs every 15 minutes so it doesn't affect the application's performance. Defaults to false. `ENABLE_SCHEDULED_CREATED_BY_MIGRATION`
 - **session** - The session config object takes the following options:
   - _ttlHours_ (number) - The number of hours a user session is allowed to live before a new sign-in is required. Defaults to 48 hours. `SESSION_TTL_HOURS`
   - _clearSiteDataOnLogout_ (boolean) - When `true`, a logout action will return a Clear Site Data response header instructing the browser to clear all cookies on the same domain Unleash is running on. If disabled unleash will only destroy and clear the session cookie. Defaults to _true_. `SESSION_CLEAR_SITE_DATA_ON_LOGOUT`