feat: maintenance mode disables scheduler (#3854)

src/lib/server-impl.ts

@@ -43,7 +43,7 @@ async function createApp(
     const db = createDb(config);
     const stores = createStores(config, db);
     const services = createServices(stores, config, db);
-    scheduleServices(services);
+    await scheduleServices(services);
 
     const metricsMonitor = createMetricsMonitor();
     const unleashSession = sessionDb(config, db);

src/lib/services/index.ts

@@ -60,7 +60,9 @@ import ConfigurationRevisionService from '../features/feature-toggle/configurati
 import { createFeatureToggleService } from '../features';
 
 // TODO: will be moved to scheduler feature directory
-export const scheduleServices = (services: IUnleashServices): void => {
+export const scheduleServices = async (
+    services: IUnleashServices,
+): Promise<void> => {
     const {
         schedulerService,
         apiTokenService,
@@ -69,8 +71,13 @@ export const scheduleServices = (services: IUnleashServices): void => {
         projectService,
         projectHealthService,
         configurationRevisionService,
+        maintenanceService,
     } = services;
 
+    if (await maintenanceService.isMaintenanceMode()) {
+        schedulerService.pause();
+    }
+
     schedulerService.schedule(
         apiTokenService.fetchActiveTokens.bind(apiTokenService),
         minutesToMilliseconds(1),
@@ -224,14 +231,15 @@ export const createServices = (
         versionService,
     );
 
+    const schedulerService = new SchedulerService(config.getLogger);
+
     const maintenanceService = new MaintenanceService(
         stores,
         config,
         settingService,
+        schedulerService,
     );
 
-    const schedulerService = new SchedulerService(config.getLogger);
-
     return {
         accessService,
         accountService,

src/lib/services/maintenance-service.ts

@@ -5,6 +5,7 @@ import { IEventStore } from '../types/stores/event-store';
 import SettingService from './setting-service';
 import { maintenanceSettingsKey } from '../types/settings/maintenance-settings';
 import { MaintenanceSchema } from '../openapi/spec/maintenance-schema';
+import { SchedulerService } from './scheduler-service';
 
 export default class MaintenanceService {
     private config: IUnleashConfig;
@@ -17,6 +18,8 @@ export default class MaintenanceService {
 
     private settingService: SettingService;
 
+    private schedulerService: SchedulerService;
+
     constructor(
         {
             patStore,
@@ -24,12 +27,14 @@ export default class MaintenanceService {
         }: Pick<IUnleashStores, 'patStore' | 'eventStore'>,
         config: IUnleashConfig,
         settingService: SettingService,
+        schedulerService: SchedulerService,
     ) {
         this.config = config;
         this.logger = config.getLogger('services/pat-service.ts');
         this.patStore = patStore;
         this.eventStore = eventStore;
         this.settingService = settingService;
+        this.schedulerService = schedulerService;
     }
 
     async isMaintenanceMode(): Promise<boolean> {
@@ -51,6 +56,11 @@ export default class MaintenanceService {
         setting: MaintenanceSchema,
         user: string,
     ): Promise<void> {
+        if (setting.enabled) {
+            this.schedulerService.pause();
+        } else {
+            this.schedulerService.resume();
+        }
         return this.settingService.insert(
             maintenanceSettingsKey,
             setting,

src/lib/services/scheduler-service.test.ts

@@ -1,4 +1,5 @@
 import { SchedulerService } from './scheduler-service';
+import { LogProvider } from '../logger';
 
 function ms(timeMs) {
     return new Promise((resolve) => setTimeout(resolve, timeMs));
@@ -6,7 +7,7 @@ function ms(timeMs) {
 
 const getLogger = () => {
     const records: any[] = [];
-    const logger = () => ({
+    const logger: LogProvider = () => ({
         error(...args: any[]) {
             records.push(args);
         },
@@ -14,17 +15,15 @@ const getLogger = () => {
         info() {},
         warn() {},
         fatal() {},
-        getRecords() {
-            return records;
-        },
     });
-    logger.getRecords = () => records;
+    const getRecords = () => records;
 
-    return logger;
+    return { logger, getRecords };
 };
 
 test('Schedules job immediately', async () => {
-    const schedulerService = new SchedulerService(getLogger());
+    const { logger } = getLogger();
+    const schedulerService = new SchedulerService(logger);
     const job = jest.fn();
 
     schedulerService.schedule(job, 10);
@@ -33,8 +32,21 @@ test('Schedules job immediately', async () => {
     schedulerService.stop();
 });
 
+test('Does not schedule job immediately when paused', async () => {
+    const { logger } = getLogger();
+    const schedulerService = new SchedulerService(logger);
+    const job = jest.fn();
+
+    schedulerService.pause();
+    schedulerService.schedule(job, 10);
+
+    expect(job).toBeCalledTimes(0);
+    schedulerService.stop();
+});
+
 test('Can schedule a single regular job', async () => {
-    const schedulerService = new SchedulerService(getLogger());
+    const { logger } = getLogger();
+    const schedulerService = new SchedulerService(logger);
     const job = jest.fn();
 
     schedulerService.schedule(job, 50);
@@ -44,8 +56,36 @@ test('Can schedule a single regular job', async () => {
     schedulerService.stop();
 });
 
+test('Scheduled job ignored in a paused mode', async () => {
+    const { logger } = getLogger();
+    const schedulerService = new SchedulerService(logger);
+    const job = jest.fn();
+
+    schedulerService.pause();
+    schedulerService.schedule(job, 50);
+    await ms(75);
+
+    expect(job).toBeCalledTimes(0);
+    schedulerService.stop();
+});
+
+test('Can resume paused job', async () => {
+    const { logger } = getLogger();
+    const schedulerService = new SchedulerService(logger);
+    const job = jest.fn();
+
+    schedulerService.pause();
+    schedulerService.schedule(job, 50);
+    schedulerService.resume();
+    await ms(75);
+
+    expect(job).toBeCalledTimes(1);
+    schedulerService.stop();
+});
+
 test('Can schedule multiple jobs at the same interval', async () => {
-    const schedulerService = new SchedulerService(getLogger());
+    const { logger } = getLogger();
+    const schedulerService = new SchedulerService(logger);
     const job = jest.fn();
     const anotherJob = jest.fn();
 
@@ -59,7 +99,8 @@ test('Can schedule multiple jobs at the same interval', async () => {
 });
 
 test('Can schedule multiple jobs at the different intervals', async () => {
-    const schedulerService = new SchedulerService(getLogger());
+    const { logger } = getLogger();
+    const schedulerService = new SchedulerService(logger);
     const job = jest.fn();
     const anotherJob = jest.fn();
 
@@ -73,7 +114,7 @@ test('Can schedule multiple jobs at the different intervals', async () => {
 });
 
 test('Can handle crash of a async job', async () => {
-    const logger = getLogger();
+    const { logger, getRecords } = getLogger();
     const schedulerService = new SchedulerService(logger);
     const job = async () => {
         await Promise.reject('async reason');
@@ -83,14 +124,14 @@ test('Can handle crash of a async job', async () => {
     await ms(75);
 
     schedulerService.stop();
-    expect(logger.getRecords()).toEqual([
+    expect(getRecords()).toEqual([
         ['scheduled job failed', 'async reason'],
         ['scheduled job failed', 'async reason'],
     ]);
 });
 
 test('Can handle crash of a sync job', async () => {
-    const logger = getLogger();
+    const { logger, getRecords } = getLogger();
     const schedulerService = new SchedulerService(logger);
     const job = () => {
         throw new Error('sync reason');
@@ -100,7 +141,7 @@ test('Can handle crash of a sync job', async () => {
     await ms(75);
 
     schedulerService.stop();
-    expect(logger.getRecords()).toEqual([
+    expect(getRecords()).toEqual([
         ['scheduled job failed', new Error('sync reason')],
         ['scheduled job failed', new Error('sync reason')],
     ]);

src/lib/services/scheduler-service.ts

@@ -1,12 +1,17 @@
 import { Logger, LogProvider } from '../logger';
 
+export type SchedulerMode = 'active' | 'paused';
+
 export class SchedulerService {
     private intervalIds: NodeJS.Timer[] = [];
 
+    private mode: SchedulerMode;
+
     private logger: Logger;
 
     constructor(getLogger: LogProvider) {
         this.logger = getLogger('/services/scheduler-service.ts');
+        this.mode = 'active';
     }
 
     async schedule(
@@ -16,14 +21,18 @@ export class SchedulerService {
         this.intervalIds.push(
             setInterval(async () => {
                 try {
-                    await scheduledFunction();
+                    if (this.mode === 'active') {
+                        await scheduledFunction();
+                    }
                 } catch (e) {
                     this.logger.error('scheduled job failed', e);
                 }
             }, timeMs).unref(),
         );
         try {
-            await scheduledFunction();
+            if (this.mode === 'active') {
+                await scheduledFunction();
+            }
         } catch (e) {
             this.logger.error('scheduled job failed', e);
         }
@@ -32,4 +41,16 @@ export class SchedulerService {
     stop(): void {
         this.intervalIds.forEach(clearInterval);
     }
+
+    pause(): void {
+        this.mode = 'paused';
+    }
+
+    resume(): void {
+        this.mode = 'active';
+    }
+
+    getMode(): SchedulerMode {
+        return this.mode;
+    }
 }