fix: add option for graceful shutdown (#872)

* fix: add option for graceful shutdown

* fix: gracefulShutdown should close idle keep-alive connections

* fix: eslint import order

* docs: add config options to docs as well

package.json

@@ -99,6 +99,7 @@
     "prom-client": "^13.1.0",
     "response-time": "^2.3.2",
     "serve-favicon": "^2.5.0",
+    "stoppable": "^1.1.0",
     "unleash-frontend": "4.0.4",
     "uuid": "^8.3.2"
   },
@@ -110,6 +111,7 @@
     "@types/node-fetch": "^2.5.10",
     "@types/nodemailer": "^6.4.1",
     "@types/owasp-password-strength-test": "^1.3.0",
+    "@types/stoppable": "^1.1.1",
     "@typescript-eslint/eslint-plugin": "^4.22.0",
     "@typescript-eslint/parser": "^4.22.0",
     "copyfiles": "^2.4.1",
@@ -132,8 +134,8 @@
     "superagent": "^6.1.0",
     "supertest": "^6.1.3",
     "ts-jest": "^27.0.0",
-    "ts-node": "^9.1.1",
+    "ts-node": "^10.0.0",
-    "tsc-watch": "^4.2.9",
+    "tsc-watch": "^4.4.0",
     "typescript": "^4.2.4"
   },
   "resolutions": {

src/lib/create-config.ts

@@ -90,6 +90,14 @@ const defaultServerOption: IServerOption = {
     keepAliveTimeout: 60 * 1000,
     headersTimeout: 61 * 1000,
     enableRequestLogger: false,
+    gracefulShutdownEnable: safeBoolean(
+        process.env.GRACEFUL_SHUTDOWN_ENABLE,
+        true,
+    ),
+    gracefulShutdownTimeout: safeNumber(
+        process.env.GRACEFUL_SHUTDOWN_TIMEOUT,
+        1000,
+    ),
     secret: process.env.UNLEASH_SECRET || 'super-secret',
 };
 

src/lib/db/db-pool.ts

@@ -1,10 +1,6 @@
-'use strict';
+import { knex, Knex } from 'knex';
-
-import { Knex } from 'knex';
 import { IUnleashConfig } from '../types/option';
 
-const knex = require('knex');
-
 export function createDb({
     db,
     getLogger,
@@ -19,12 +15,13 @@ export function createDb({
         asyncStackTraces: true,
         log: {
             debug: msg => logger.debug(msg),
-            info: msg => logger.info(msg),
             warn: msg => logger.warn(msg),
             error: msg => logger.error(msg),
         },
     });
 }
+
+// for backward compatibility
 module.exports = {
     createDb,
 };

src/lib/server-impl.ts

@@ -1,13 +1,14 @@
 'use strict';
 
 import EventEmitter from 'events';
-import { Server } from 'http';
+import stoppable, { StoppableServer } from 'stoppable';
+import { promisify } from 'util';
 import { IUnleash } from './types/core';
 import { IUnleashConfig, IUnleashOptions } from './types/option';
 import version from './util/version';
 import migrator from '../migrator';
 import getApp from './app';
-import MetricsMonitor, { createMetricsMonitor } from './metrics';
+import { createMetricsMonitor } from './metrics';
 import { createStores } from './db';
 import { createServices } from './services';
 import { createConfig } from './create-config';
@@ -20,22 +21,11 @@ import { addEventHook } from './event-hook';
 import registerGracefulShutdown from './util/graceful-shutdown';
 import { IUnleashStores } from './types/stores';
 
-async function closeServer(
-    server: Server,
-    metricsMonitor: MetricsMonitor,
-): Promise<void> {
-    metricsMonitor.stopMonitoring();
-
-    return new Promise((resolve, reject) => {
-        server.close(err => (err ? reject(err) : resolve()));
-    });
-}
-
 async function destroyDatabase(stores: IUnleashStores): Promise<void> {
     const { db, clientInstanceStore, clientMetricsStore } = stores;
     clientInstanceStore.destroy();
     clientMetricsStore.destroy();
-    return db.destroy();
+    await db.destroy();
 }
 
 async function createApp(
@@ -48,6 +38,17 @@ async function createApp(
     const eventBus = new EventEmitter();
     const stores = createStores(config, eventBus);
     const services = createServices(stores, config);
+    const metricsMonitor = createMetricsMonitor();
+
+    const stopUnleash = async (server?: StoppableServer) => {
+        logger.info('Shutting down Unleash...');
+        if (server) {
+            const stopServer = promisify(server.stop);
+            await stopServer();
+        }
+        metricsMonitor.stopMonitoring();
+        await destroyDatabase(stores);
+    };
 
     if (!config.server.secret) {
         const secret = await stores.settingStore.get('unleash.secret');
@@ -55,7 +56,7 @@ async function createApp(
         config.server.secret = secret;
     }
     const app = getApp(config, stores, services, eventBus);
-    const metricsMonitor = createMetricsMonitor();
+
     if (typeof config.eventHook === 'function') {
         addEventHook(config.eventHook, stores.eventStore);
     }
@@ -80,30 +81,25 @@ async function createApp(
 
     return new Promise((resolve, reject) => {
         if (startApp) {
-            const server = app.listen(config.listen, () =>
+            const server = stoppable(
+                app.listen(config.listen, () =>
                     logger.info('Unleash has started.', server.address()),
+                ),
+                config.server.gracefulShutdownTimeout,
             );
-            const stop = async () => {
-                logger.info('Shutting down Unleash...');
-
-                await closeServer(server, metricsMonitor);
-                return destroyDatabase(stores);
-            };
 
             server.keepAliveTimeout = config.server.keepAliveTimeout;
             server.headersTimeout = config.server.headersTimeout;
             server.on('listening', () => {
-                resolve({ ...unleash, server, stop });
+                resolve({
+                    ...unleash,
+                    server,
+                    stop: () => stopUnleash(server),
+                });
             });
             server.on('error', reject);
         } else {
-            const stop = async () => {
+            resolve({ ...unleash, stop: stopUnleash });
-                logger.info('Shutting down Unleash...');
-                metricsMonitor.stopMonitoring();
-                return destroyDatabase(stores);
-            };
-
-            resolve({ ...unleash, stop });
         }
     });
 }
@@ -125,8 +121,9 @@ async function start(opts: IUnleashOptions = {}): Promise<IUnleash> {
     }
 
     const unleash = await createApp(config, true);
-    logger.info('register graceful shutdown');
+    if (config.server.gracefulShutdownEnable) {
         registerGracefulShutdown(unleash, logger);
+    }
     return unleash;
 }
 

src/lib/types/option.ts

@@ -68,6 +68,8 @@ export interface IServerOption {
     unleashUrl: string;
     serverMetrics: boolean;
     enableRequestLogger: boolean;
+    gracefulShutdownEnable: boolean;
+    gracefulShutdownTimeout: number;
     secret: string;
 }
 

src/lib/util/graceful-shutdown.ts

@@ -2,27 +2,23 @@ import { Logger } from '../logger';
 import { IUnleash } from '../types/core';
 
 function registerGracefulShutdown(unleash: IUnleash, logger: Logger): void {
-    process.on('SIGINT', async () => {
+    const unleashCloser = (signal: string) => async () => {
         try {
-            logger.info('Graceful shutdown signal (SIGINT) received.');
+            logger.info(`Graceful shutdown signal (${signal}) received.`);
             await unleash.stop();
+            logger.info('Unleash has been successfully stopped.');
             process.exit(0);
         } catch (e) {
-            logger.error('Unable to shutdown Unleash. Hard exit!', e);
+            logger.error('Unable to shutdown Unleash. Hard exit!');
             process.exit(1);
         }
-    });
+    };
 
-    process.on('SIGTERM', async () => {
+    logger.info('Registering graceful shutdown');
-        try {
+
-            logger.info('Graceful shutdown signal (SIGTERM) received.');
+    process.on('SIGINT', unleashCloser('SIGINT'));
-            await unleash.stop();
+    process.on('SIGHUP', unleashCloser('SIGHUP'));
-            process.exit(0);
+    process.on('SIGTERM', unleashCloser('SIGTERM'));
-        } catch (e) {
-            logger.error('Unable to shutdown Unleash. Hard exit!', e);
-            process.exit(1);
-        }
-    });
 }
 
 export default registerGracefulShutdown;

src/server-dev.ts

@@ -4,7 +4,9 @@ import unleash from './lib/server-impl';
 import { createConfig } from './lib/create-config';
 import { LogLevel } from './lib/logger';
 
-unleash.start(
+process.nextTick(async () => {
+    try {
+        await unleash.start(
             createConfig({
                 db: {
                     user: 'unleash_user',
@@ -17,6 +19,8 @@ unleash.start(
                 server: {
                     enableRequestLogger: true,
                     baseUriPath: '',
+                    // keepAliveTimeout: 1,
+                    gracefulShutdownEnable: true,
                 },
                 logLevel: LogLevel.debug,
                 enableOAS: true,
@@ -25,3 +29,14 @@ unleash.start(
                 },
             }),
         );
+    } catch (error) {
+        if (error.code === 'EADDRINUSE') {
+            // eslint-disable-next-line no-console
+            console.warn('Port in use. You might want to reload once more.');
+        } else {
+            // eslint-disable-next-line no-console
+            console.error(error);
+            process.exit();
+        }
+    }
+}, 0);

websitev2/docs/deploy/configuring-unleash.md

@@ -88,6 +88,8 @@ unleash.start(unleashOptions);
   - _serverMetrics_ (boolean) - use this option to turn on/off prometheus metrics.
   - _baseUriPath_ (string) - use to register a base path for all routes on the application. For example `/my/unleash/base` (note the starting /). Defaults to `/`. Can also be configured through the environment variable `BASE_URI_PATH`.
   - _unleashUrl_ (string) - Used to specify the official URL this instance of Unleash can be accessed at for an end user. Can also be configured through the environment variable `UNLEASH_URL`.
+  - \_gracefulShutdownEnable: (boolean) - Used to control if Unleash should shutdown gracefully (close connections, stop tasks,). Defaults to true. `GRACEFUL_SHUTDOWN_ENABLE`
+  - \_gracefulShutdownTimeout: (number) - Used to control the timeout, in milliseconds, for shutdown Unleash gracefully. Will kill all connections regardless if this timeout is exceeded. Defaults to 1000ms `GRACEFUL_SHUTDOWN_TIMEOUT`
 - **preHook** (function) - this is a hook if you need to provide any middlewares to express before `unleash` adds any. Express app instance is injected as first argument.
 - **preRouterHook** (function) - use this to register custom express middlewares before the `unleash` specific routers are added.
 - **authentication** - (object) - An object for configuring/implementing custom admin authentication

yarn.lock

@@ -582,6 +582,26 @@
   resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
   integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==
 
+"@tsconfig/node10@^1.0.7":
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.8.tgz#c1e4e80d6f964fbecb3359c43bd48b40f7cadad9"
+  integrity sha512-6XFfSQmMgq0CFLY1MslA/CPUfhIL919M1rMsa5lP2P097N2Wd1sSX0tx1u4olM16fLNhtHZpRhedZJphNJqmZg==
+
+"@tsconfig/node12@^1.0.7":
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.8.tgz#a883d62f049a64fea1e56a6bbe66828d11c6241b"
+  integrity sha512-LM6XwBhjZRls1qJGpiM/It09SntEwe9M0riXRfQ9s6XlJQG0JPGl92ET18LtGeYh/GuOtafIXqwZeqLOd0FNFQ==
+
+"@tsconfig/node14@^1.0.0":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.1.tgz#95f2d167ffb9b8d2068b0b235302fafd4df711f2"
+  integrity sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg==
+
+"@tsconfig/node16@^1.0.1":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.1.tgz#a6ca6a9a0ff366af433f42f5f0e124794ff6b8f1"
+  integrity sha512-FTgBI767POY/lKNDNbIzgAX6miIDBs6NTCbdlDb8TrWovHsSvaVIZDlTqym29C6UqhzwcJx4CYr+AlrMywA0cA==
+
 "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14":
   version "7.1.14"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.14.tgz#faaeefc4185ec71c389f4501ee5ec84b170cc402"
@@ -763,6 +783,13 @@
   resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.0.tgz#7036640b4e21cc2f259ae826ce843d277dad8cff"
   integrity sha512-RJJrrySY7A8havqpGObOB4W92QXKJo63/jFLLgpvOtsGUqbQZ9Sbgl35KMm1DjC6j7AvmmU2bIno+3IyEaemaw==
 
+"@types/stoppable@^1.1.1":
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/@types/stoppable/-/stoppable-1.1.1.tgz#a6f1f280e29f8f3c743277534425e0a75041d2f9"
+  integrity sha512-b8N+fCADRIYYrGZOcmOR8ZNBOqhktWTB/bMUl5LvGtT201QKJZOOH5UsFyI3qtteM6ZAJbJqZoBcLqqxKIwjhw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/yargs-parser@*":
   version "20.2.0"
   resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.0.tgz#dd3e6699ba3237f0348cd085e4698780204842f9"
@@ -6222,6 +6249,11 @@ static-extend@^0.1.1:
   resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c"
   integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=
 
+stoppable@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/stoppable/-/stoppable-1.1.0.tgz#32da568e83ea488b08e4d7ea2c3bcc9d75015d5b"
+  integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==
+
 stream-combiner@~0.0.4:
   version "0.0.4"
   resolved "https://registry.yarnpkg.com/stream-combiner/-/stream-combiner-0.0.4.tgz#4d5e433c185261dde623ca3f44c586bcf5c4ad14"
@@ -6640,11 +6672,15 @@ ts-jest@^27.0.0:
     semver "7.x"
     yargs-parser "20.x"
 
-ts-node@^9.1.1:
+ts-node@^10.0.0:
-  version "9.1.1"
+  version "10.0.0"
-  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-9.1.1.tgz#51a9a450a3e959401bda5f004a72d54b936d376d"
+  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.0.0.tgz#05f10b9a716b0b624129ad44f0ea05dac84ba3be"
-  integrity sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==
+  integrity sha512-ROWeOIUvfFbPZkoDis0L/55Fk+6gFQNZwwKPLinacRl6tsxstTF1DbAcLKkovwnpKMVvOMHP1TIbnwXwtLg1gg==
   dependencies:
+    "@tsconfig/node10" "^1.0.7"
+    "@tsconfig/node12" "^1.0.7"
+    "@tsconfig/node14" "^1.0.0"
+    "@tsconfig/node16" "^1.0.1"
     arg "^4.1.0"
     create-require "^1.1.0"
     diff "^4.0.1"
@@ -6652,10 +6688,10 @@ ts-node@^9.1.1:
     source-map-support "^0.5.17"
     yn "3.1.1"
 
-tsc-watch@^4.2.9:
+tsc-watch@^4.4.0:
-  version "4.2.9"
+  version "4.4.0"
-  resolved "https://registry.yarnpkg.com/tsc-watch/-/tsc-watch-4.2.9.tgz#d93fc74233ca4ef7ee6b12d08c0fe6aca3e19044"
+  resolved "https://registry.yarnpkg.com/tsc-watch/-/tsc-watch-4.4.0.tgz#3ebbf1db54bcef6bfe534b330fa87284a4139320"
-  integrity sha512-DlTaoDs74+KUpyWr7dCGhuscAUKCz6CiFduBN7R9RbLJSSN1moWdwoCLASE7+zLgGvV5AwXfYDiEMAsPGaO+Vw==
+  integrity sha512-+0Yey6ptOOXAnt44OKTk2/EnQnmA0auL7VWXm9d9abMS4tabt0Xdr9B4AK6OJbWAre9ZdLA81+Nk8sz9unptyA==
   dependencies:
     cross-spawn "^7.0.3"
     node-cleanup "^2.1.2"