From 4f9deee2ed767d7f1f65030fc271b8b7488247a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivar=20Conradi=20=C3=98sthus?= Date: Thu, 17 Jun 2021 20:33:34 +0200 Subject: [PATCH] fix: add option for graceful shutdown (#872) * fix: add option for graceful shutdown * fix: gracefulShutdown should close idle keep-alive connections * fix: eslint import order * docs: add config options to docs as well --- package.json | 6 +- src/lib/create-config.ts | 8 +++ src/lib/db/db-pool.ts | 9 +-- src/lib/server-impl.ts | 63 ++++++++++---------- src/lib/types/option.ts | 2 + src/lib/util/graceful-shutdown.ts | 24 ++++---- src/server-dev.ts | 57 +++++++++++------- websitev2/docs/deploy/configuring-unleash.md | 2 + yarn.lock | 52 +++++++++++++--- 9 files changed, 139 insertions(+), 84 deletions(-) diff --git a/package.json b/package.json index 783efd6bf0..9f6e7077ee 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "prom-client": "^13.1.0", "response-time": "^2.3.2", "serve-favicon": "^2.5.0", + "stoppable": "^1.1.0", "unleash-frontend": "4.0.4", "uuid": "^8.3.2" }, @@ -110,6 +111,7 @@ "@types/node-fetch": "^2.5.10", "@types/nodemailer": "^6.4.1", "@types/owasp-password-strength-test": "^1.3.0", + "@types/stoppable": "^1.1.1", "@typescript-eslint/eslint-plugin": "^4.22.0", "@typescript-eslint/parser": "^4.22.0", "copyfiles": "^2.4.1", @@ -132,8 +134,8 @@ "superagent": "^6.1.0", "supertest": "^6.1.3", "ts-jest": "^27.0.0", - "ts-node": "^9.1.1", - "tsc-watch": "^4.2.9", + "ts-node": "^10.0.0", + "tsc-watch": "^4.4.0", "typescript": "^4.2.4" }, "resolutions": { diff --git a/src/lib/create-config.ts b/src/lib/create-config.ts index 6a136979df..138893bbd7 100644 --- a/src/lib/create-config.ts +++ b/src/lib/create-config.ts @@ -90,6 +90,14 @@ const defaultServerOption: IServerOption = { keepAliveTimeout: 60 * 1000, headersTimeout: 61 * 1000, enableRequestLogger: false, + gracefulShutdownEnable: safeBoolean( + process.env.GRACEFUL_SHUTDOWN_ENABLE, + true, + ), + gracefulShutdownTimeout: safeNumber( + process.env.GRACEFUL_SHUTDOWN_TIMEOUT, + 1000, + ), secret: process.env.UNLEASH_SECRET || 'super-secret', }; diff --git a/src/lib/db/db-pool.ts b/src/lib/db/db-pool.ts index 1ff628e189..8cbd96a4f8 100644 --- a/src/lib/db/db-pool.ts +++ b/src/lib/db/db-pool.ts @@ -1,10 +1,6 @@ -'use strict'; - -import { Knex } from 'knex'; +import { knex, Knex } from 'knex'; import { IUnleashConfig } from '../types/option'; -const knex = require('knex'); - export function createDb({ db, getLogger, @@ -19,12 +15,13 @@ export function createDb({ asyncStackTraces: true, log: { debug: msg => logger.debug(msg), - info: msg => logger.info(msg), warn: msg => logger.warn(msg), error: msg => logger.error(msg), }, }); } + +// for backward compatibility module.exports = { createDb, }; diff --git a/src/lib/server-impl.ts b/src/lib/server-impl.ts index f9e03c0e53..a9826bee7d 100644 --- a/src/lib/server-impl.ts +++ b/src/lib/server-impl.ts @@ -1,13 +1,14 @@ 'use strict'; import EventEmitter from 'events'; -import { Server } from 'http'; +import stoppable, { StoppableServer } from 'stoppable'; +import { promisify } from 'util'; import { IUnleash } from './types/core'; import { IUnleashConfig, IUnleashOptions } from './types/option'; import version from './util/version'; import migrator from '../migrator'; import getApp from './app'; -import MetricsMonitor, { createMetricsMonitor } from './metrics'; +import { createMetricsMonitor } from './metrics'; import { createStores } from './db'; import { createServices } from './services'; import { createConfig } from './create-config'; @@ -20,22 +21,11 @@ import { addEventHook } from './event-hook'; import registerGracefulShutdown from './util/graceful-shutdown'; import { IUnleashStores } from './types/stores'; -async function closeServer( - server: Server, - metricsMonitor: MetricsMonitor, -): Promise { - metricsMonitor.stopMonitoring(); - - return new Promise((resolve, reject) => { - server.close(err => (err ? reject(err) : resolve())); - }); -} - async function destroyDatabase(stores: IUnleashStores): Promise { const { db, clientInstanceStore, clientMetricsStore } = stores; clientInstanceStore.destroy(); clientMetricsStore.destroy(); - return db.destroy(); + await db.destroy(); } async function createApp( @@ -48,6 +38,17 @@ async function createApp( const eventBus = new EventEmitter(); const stores = createStores(config, eventBus); const services = createServices(stores, config); + const metricsMonitor = createMetricsMonitor(); + + const stopUnleash = async (server?: StoppableServer) => { + logger.info('Shutting down Unleash...'); + if (server) { + const stopServer = promisify(server.stop); + await stopServer(); + } + metricsMonitor.stopMonitoring(); + await destroyDatabase(stores); + }; if (!config.server.secret) { const secret = await stores.settingStore.get('unleash.secret'); @@ -55,7 +56,7 @@ async function createApp( config.server.secret = secret; } const app = getApp(config, stores, services, eventBus); - const metricsMonitor = createMetricsMonitor(); + if (typeof config.eventHook === 'function') { addEventHook(config.eventHook, stores.eventStore); } @@ -80,30 +81,25 @@ async function createApp( return new Promise((resolve, reject) => { if (startApp) { - const server = app.listen(config.listen, () => - logger.info('Unleash has started.', server.address()), + const server = stoppable( + app.listen(config.listen, () => + logger.info('Unleash has started.', server.address()), + ), + config.server.gracefulShutdownTimeout, ); - const stop = async () => { - logger.info('Shutting down Unleash...'); - - await closeServer(server, metricsMonitor); - return destroyDatabase(stores); - }; server.keepAliveTimeout = config.server.keepAliveTimeout; server.headersTimeout = config.server.headersTimeout; server.on('listening', () => { - resolve({ ...unleash, server, stop }); + resolve({ + ...unleash, + server, + stop: () => stopUnleash(server), + }); }); server.on('error', reject); } else { - const stop = async () => { - logger.info('Shutting down Unleash...'); - metricsMonitor.stopMonitoring(); - return destroyDatabase(stores); - }; - - resolve({ ...unleash, stop }); + resolve({ ...unleash, stop: stopUnleash }); } }); } @@ -125,8 +121,9 @@ async function start(opts: IUnleashOptions = {}): Promise { } const unleash = await createApp(config, true); - logger.info('register graceful shutdown'); - registerGracefulShutdown(unleash, logger); + if (config.server.gracefulShutdownEnable) { + registerGracefulShutdown(unleash, logger); + } return unleash; } diff --git a/src/lib/types/option.ts b/src/lib/types/option.ts index 4f21f17edf..e941d38112 100644 --- a/src/lib/types/option.ts +++ b/src/lib/types/option.ts @@ -68,6 +68,8 @@ export interface IServerOption { unleashUrl: string; serverMetrics: boolean; enableRequestLogger: boolean; + gracefulShutdownEnable: boolean; + gracefulShutdownTimeout: number; secret: string; } diff --git a/src/lib/util/graceful-shutdown.ts b/src/lib/util/graceful-shutdown.ts index eaa5f13841..f5cb519ff6 100644 --- a/src/lib/util/graceful-shutdown.ts +++ b/src/lib/util/graceful-shutdown.ts @@ -2,27 +2,23 @@ import { Logger } from '../logger'; import { IUnleash } from '../types/core'; function registerGracefulShutdown(unleash: IUnleash, logger: Logger): void { - process.on('SIGINT', async () => { + const unleashCloser = (signal: string) => async () => { try { - logger.info('Graceful shutdown signal (SIGINT) received.'); + logger.info(`Graceful shutdown signal (${signal}) received.`); await unleash.stop(); + logger.info('Unleash has been successfully stopped.'); process.exit(0); } catch (e) { - logger.error('Unable to shutdown Unleash. Hard exit!', e); + logger.error('Unable to shutdown Unleash. Hard exit!'); process.exit(1); } - }); + }; - process.on('SIGTERM', async () => { - try { - logger.info('Graceful shutdown signal (SIGTERM) received.'); - await unleash.stop(); - process.exit(0); - } catch (e) { - logger.error('Unable to shutdown Unleash. Hard exit!', e); - process.exit(1); - } - }); + logger.info('Registering graceful shutdown'); + + process.on('SIGINT', unleashCloser('SIGINT')); + process.on('SIGHUP', unleashCloser('SIGHUP')); + process.on('SIGTERM', unleashCloser('SIGTERM')); } export default registerGracefulShutdown; diff --git a/src/server-dev.ts b/src/server-dev.ts index 29d1ed870c..6c9c056a39 100644 --- a/src/server-dev.ts +++ b/src/server-dev.ts @@ -4,24 +4,39 @@ import unleash from './lib/server-impl'; import { createConfig } from './lib/create-config'; import { LogLevel } from './lib/logger'; -unleash.start( - createConfig({ - db: { - user: 'unleash_user', - password: 'passord', - host: 'localhost', - port: 5432, - database: 'unleash', - ssl: false, - }, - server: { - enableRequestLogger: true, - baseUriPath: '', - }, - logLevel: LogLevel.debug, - enableOAS: true, - versionCheck: { - enable: false, - }, - }), -); +process.nextTick(async () => { + try { + await unleash.start( + createConfig({ + db: { + user: 'unleash_user', + password: 'passord', + host: 'localhost', + port: 5432, + database: 'unleash', + ssl: false, + }, + server: { + enableRequestLogger: true, + baseUriPath: '', + // keepAliveTimeout: 1, + gracefulShutdownEnable: true, + }, + logLevel: LogLevel.debug, + enableOAS: true, + versionCheck: { + enable: false, + }, + }), + ); + } catch (error) { + if (error.code === 'EADDRINUSE') { + // eslint-disable-next-line no-console + console.warn('Port in use. You might want to reload once more.'); + } else { + // eslint-disable-next-line no-console + console.error(error); + process.exit(); + } + } +}, 0); diff --git a/websitev2/docs/deploy/configuring-unleash.md b/websitev2/docs/deploy/configuring-unleash.md index cda235f4bf..2d50a46b82 100644 --- a/websitev2/docs/deploy/configuring-unleash.md +++ b/websitev2/docs/deploy/configuring-unleash.md @@ -88,6 +88,8 @@ unleash.start(unleashOptions); - _serverMetrics_ (boolean) - use this option to turn on/off prometheus metrics. - _baseUriPath_ (string) - use to register a base path for all routes on the application. For example `/my/unleash/base` (note the starting /). Defaults to `/`. Can also be configured through the environment variable `BASE_URI_PATH`. - _unleashUrl_ (string) - Used to specify the official URL this instance of Unleash can be accessed at for an end user. Can also be configured through the environment variable `UNLEASH_URL`. + - \_gracefulShutdownEnable: (boolean) - Used to control if Unleash should shutdown gracefully (close connections, stop tasks,). Defaults to true. `GRACEFUL_SHUTDOWN_ENABLE` + - \_gracefulShutdownTimeout: (number) - Used to control the timeout, in milliseconds, for shutdown Unleash gracefully. Will kill all connections regardless if this timeout is exceeded. Defaults to 1000ms `GRACEFUL_SHUTDOWN_TIMEOUT` - **preHook** (function) - this is a hook if you need to provide any middlewares to express before `unleash` adds any. Express app instance is injected as first argument. - **preRouterHook** (function) - use this to register custom express middlewares before the `unleash` specific routers are added. - **authentication** - (object) - An object for configuring/implementing custom admin authentication diff --git a/yarn.lock b/yarn.lock index e80e60ede5..2064981c8a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -582,6 +582,26 @@ resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== +"@tsconfig/node10@^1.0.7": + version "1.0.8" + resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.8.tgz#c1e4e80d6f964fbecb3359c43bd48b40f7cadad9" + integrity sha512-6XFfSQmMgq0CFLY1MslA/CPUfhIL919M1rMsa5lP2P097N2Wd1sSX0tx1u4olM16fLNhtHZpRhedZJphNJqmZg== + +"@tsconfig/node12@^1.0.7": + version "1.0.8" + resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.8.tgz#a883d62f049a64fea1e56a6bbe66828d11c6241b" + integrity sha512-LM6XwBhjZRls1qJGpiM/It09SntEwe9M0riXRfQ9s6XlJQG0JPGl92ET18LtGeYh/GuOtafIXqwZeqLOd0FNFQ== + +"@tsconfig/node14@^1.0.0": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.1.tgz#95f2d167ffb9b8d2068b0b235302fafd4df711f2" + integrity sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg== + +"@tsconfig/node16@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.1.tgz#a6ca6a9a0ff366af433f42f5f0e124794ff6b8f1" + integrity sha512-FTgBI767POY/lKNDNbIzgAX6miIDBs6NTCbdlDb8TrWovHsSvaVIZDlTqym29C6UqhzwcJx4CYr+AlrMywA0cA== + "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14": version "7.1.14" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.14.tgz#faaeefc4185ec71c389f4501ee5ec84b170cc402" @@ -763,6 +783,13 @@ resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.0.tgz#7036640b4e21cc2f259ae826ce843d277dad8cff" integrity sha512-RJJrrySY7A8havqpGObOB4W92QXKJo63/jFLLgpvOtsGUqbQZ9Sbgl35KMm1DjC6j7AvmmU2bIno+3IyEaemaw== +"@types/stoppable@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@types/stoppable/-/stoppable-1.1.1.tgz#a6f1f280e29f8f3c743277534425e0a75041d2f9" + integrity sha512-b8N+fCADRIYYrGZOcmOR8ZNBOqhktWTB/bMUl5LvGtT201QKJZOOH5UsFyI3qtteM6ZAJbJqZoBcLqqxKIwjhw== + dependencies: + "@types/node" "*" + "@types/yargs-parser@*": version "20.2.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.0.tgz#dd3e6699ba3237f0348cd085e4698780204842f9" @@ -6222,6 +6249,11 @@ static-extend@^0.1.1: resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= +stoppable@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/stoppable/-/stoppable-1.1.0.tgz#32da568e83ea488b08e4d7ea2c3bcc9d75015d5b" + integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw== + stream-combiner@~0.0.4: version "0.0.4" resolved "https://registry.yarnpkg.com/stream-combiner/-/stream-combiner-0.0.4.tgz#4d5e433c185261dde623ca3f44c586bcf5c4ad14" @@ -6640,11 +6672,15 @@ ts-jest@^27.0.0: semver "7.x" yargs-parser "20.x" -ts-node@^9.1.1: - version "9.1.1" - resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-9.1.1.tgz#51a9a450a3e959401bda5f004a72d54b936d376d" - integrity sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg== +ts-node@^10.0.0: + version "10.0.0" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.0.0.tgz#05f10b9a716b0b624129ad44f0ea05dac84ba3be" + integrity sha512-ROWeOIUvfFbPZkoDis0L/55Fk+6gFQNZwwKPLinacRl6tsxstTF1DbAcLKkovwnpKMVvOMHP1TIbnwXwtLg1gg== dependencies: + "@tsconfig/node10" "^1.0.7" + "@tsconfig/node12" "^1.0.7" + "@tsconfig/node14" "^1.0.0" + "@tsconfig/node16" "^1.0.1" arg "^4.1.0" create-require "^1.1.0" diff "^4.0.1" @@ -6652,10 +6688,10 @@ ts-node@^9.1.1: source-map-support "^0.5.17" yn "3.1.1" -tsc-watch@^4.2.9: - version "4.2.9" - resolved "https://registry.yarnpkg.com/tsc-watch/-/tsc-watch-4.2.9.tgz#d93fc74233ca4ef7ee6b12d08c0fe6aca3e19044" - integrity sha512-DlTaoDs74+KUpyWr7dCGhuscAUKCz6CiFduBN7R9RbLJSSN1moWdwoCLASE7+zLgGvV5AwXfYDiEMAsPGaO+Vw== +tsc-watch@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/tsc-watch/-/tsc-watch-4.4.0.tgz#3ebbf1db54bcef6bfe534b330fa87284a4139320" + integrity sha512-+0Yey6ptOOXAnt44OKTk2/EnQnmA0auL7VWXm9d9abMS4tabt0Xdr9B4AK6OJbWAre9ZdLA81+Nk8sz9unptyA== dependencies: cross-spawn "^7.0.3" node-cleanup "^2.1.2"