From e952ae20a85043074742c3a44bb8559cb2cc87c3 Mon Sep 17 00:00:00 2001 From: Christopher Kolstad Date: Thu, 4 Feb 2021 14:14:46 +0100 Subject: [PATCH] Added explicit pool settings in options.db object - Also adds metrics for min and max pool size - Metrics for free/used connections. - Metrics for pending and current acquires fixes: #705 --- CHANGELOG.md | 3 ++ docs/configuring-unleash.md | 14 ++++++++ lib/db/db-pool.js | 10 ++---- lib/event-type.js | 1 + lib/metrics.js | 60 +++++++++++++++++++++++++++++++ lib/metrics.test.js | 22 ++++++++++++ lib/options.js | 29 +++++++++++++++ lib/options.test.js | 39 ++++++++++++++++++++ test/e2e/helpers/database-init.js | 4 +-- 9 files changed, 171 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96af40967f..bec52c038f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,9 @@ - docs: update getting started guide with docker options (#697) - fix typo in /api/client/features docs (#694) - fix: website: require immer 8.0.1 or higher +- fix: Add support for configuring database pool size (#705) +- feat: Set default min dbpool size to 0 +- feat: Set default max dbpool size to 4 ## 3.10.1 diff --git a/docs/configuring-unleash.md b/docs/configuring-unleash.md index ba4b4d284a..4b27527629 100644 --- a/docs/configuring-unleash.md +++ b/docs/configuring-unleash.md @@ -16,6 +16,11 @@ const unleashOptions = { port: 5432, database: 'unleash', ssl: false, + pool: { + min: 0, + max: 4, + idleTimeoutMillis: 30000, + }, }, enableRequestLogger: true, }; @@ -33,6 +38,10 @@ unleash.start(unleashOptions); - _database_ - the database name to be used (`DATABASE_NAME`) - _ssl_ - an object describing ssl options, see https://node-postgres.com/features/ssl (`DATABASE_SSL`, as a stringified json object) - _version_ - the postgres database version. Used to connect a non-standard database. Defaults to `undefined`, which let the underlying adapter to detect the version automatically. (`DATABASE_VERSION`) + - _pool_ - an object describing pool options, see https://knexjs.org/#Installation-pooling. We support the following three fields: + - _min_ - minimum connections in connections pool (defaults to 0) (`DATABASE_POOL_MIN`) + - _max_ - maximum connections in connections pool (defaults to 4) (`DATABASE_POOL_MAX`) + - _idleTimeoutMillis_ - time in milliseconds a connection must be idle before being marked as a candidate for eviction (defaults to 30000) (`DATABASE_POOL_IDLE_TIMEOUT_MS`) - **databaseUrl** - (_deprecated_) the postgres database url to connect to. Only used if _db_ object is not specified. Should include username/password. This value may also be set via the `DATABASE_URL` environment variable. Alternatively, if you would like to read the database url from a file, you may set the `DATABASE_URL_FILE` environment variable with the full file path. The contents of the file must be the database url exactly. - **databaseSchema** - the postgres database schema to use. Defaults to 'public'. (`DATABASE_SCHEMA`) - **port** - which port the unleash-server should bind to. If port is omitted or is 0, the operating system will assign an arbitrary unused port. Will be ignored if pipe is specified. This value may also be set via the `HTTP_PORT` environment variable @@ -93,3 +102,8 @@ function getLogger(name) { ``` The logger interface with its `debug`, `info`, `warn` and `error` methods expects format string support as seen in `debug` or the JavaScript `console` object. Many commonly used logging implementations cover this API, e.g., bunyan, pino or winston. + +## Database pooling connection timeouts + +- Please be aware of the default values of connection pool about idle session handling. +- If you have a network component which closes idle sessions on tcp layer, please ensure, that the connection pool idleTimeoutMillis setting is lower than the timespan as the network component will close the idle connection. diff --git a/lib/db/db-pool.js b/lib/db/db-pool.js index 8fd7b39031..21e6316ae8 100644 --- a/lib/db/db-pool.js +++ b/lib/db/db-pool.js @@ -2,19 +2,13 @@ const knex = require('knex'); -module.exports.createDb = function({ - db, - poolMin = 2, - poolMax = 20, - databaseSchema, - getLogger, -}) { +module.exports.createDb = function({ db, databaseSchema, getLogger }) { const logger = getLogger('db-pool.js'); return knex({ client: 'pg', version: db.version, connection: db, - pool: { min: poolMin, max: poolMax }, + pool: db.pool, searchPath: databaseSchema, log: { debug: msg => logger.debug(msg), diff --git a/lib/event-type.js b/lib/event-type.js index b4b6066fcd..49a431455a 100644 --- a/lib/event-type.js +++ b/lib/event-type.js @@ -31,4 +31,5 @@ module.exports = { ADDON_CONFIG_CREATED: 'addon-config-created', ADDON_CONFIG_UPDATED: 'addon-config-updated', ADDON_CONFIG_DELETED: 'addon-config-deleted', + DB_POOL_UPDATE: 'db-pool-update', }; diff --git a/lib/metrics.js b/lib/metrics.js index ada1fed5a8..6ded97973f 100644 --- a/lib/metrics.js +++ b/lib/metrics.js @@ -7,9 +7,11 @@ const { FEATURE_UPDATED, FEATURE_ARCHIVED, FEATURE_REVIVED, + DB_POOL_UPDATE, } = require('./event-type'); const THREE_HOURS = 3 * 60 * 60 * 1000; +const ONE_MINUTE = 60 * 1000; class MetricsMonitor { constructor() { @@ -102,11 +104,69 @@ class MetricsMonitor { .inc(no); } }); + + this.configureDbMetrics(stores, eventStore); } stopMonitoring() { clearInterval(this.timer); } + + configureDbMetrics(stores, eventStore) { + if (stores.db && stores.db.client) { + const dbPoolMin = new client.Gauge({ + name: 'db_pool_min', + help: 'Minimum DB pool size', + }); + dbPoolMin.set(stores.db.client.pool.min); + const dbPoolMax = new client.Gauge({ + name: 'db_pool_max', + help: 'Maximum DB pool size', + }); + dbPoolMax.set(stores.db.client.pool.max); + const dbPoolFree = new client.Gauge({ + name: 'db_pool_free', + help: 'Current free connections in DB pool', + }); + const dbPoolUsed = new client.Gauge({ + name: 'db_pool_used', + help: 'Current connections in use in DB pool', + }); + const dbPoolPendingCreates = new client.Gauge({ + name: 'db_pool_pending_creates', + help: + 'how many asynchronous create calls are running in DB pool', + }); + const dbPoolPendingAcquires = new client.Gauge({ + name: 'db_pool_pending_acquires', + help: + 'how many acquires are waiting for a resource to be released in DB pool', + }); + + eventStore.on(DB_POOL_UPDATE, data => { + dbPoolFree.set(data.free); + dbPoolUsed.set(data.used); + dbPoolPendingCreates.set(data.pendingCreates); + dbPoolPendingAcquires.set(data.pendingAcquires); + }); + + this.registerPoolMetrics(stores.db.client.pool, eventStore); + setInterval( + () => + this.registerPoolMetrics(stores.db.client.pool, eventStore), + ONE_MINUTE, + ); + } + } + + registerPoolMetrics(pool, eventStore) { + eventStore.emit(DB_POOL_UPDATE, { + used: pool.numUsed(), + free: pool.numFree(), + pendingCreates: pool.numPendingCreates(), + pendingAcquires: pool.numPendingAcquires(), + }); + } } module.exports = { diff --git a/lib/metrics.test.js b/lib/metrics.test.js index 74dad47d29..f4ace72da7 100644 --- a/lib/metrics.test.js +++ b/lib/metrics.test.js @@ -24,6 +24,18 @@ test.before(() => { eventStore, clientMetricsStore, featureToggleStore, + db: { + client: { + pool: { + min: 0, + max: 4, + numUsed: () => 2, + numFree: () => 2, + numPendingAcquires: () => 0, + numPendingCreates: () => 1, + }, + }, + }, }, version: '3.4.1', }; @@ -95,3 +107,13 @@ test('should collect metrics for feature toggle size', async t => { const metrics = await prometheusRegister.metrics(); t.regex(metrics, /feature_toggles_total{version="(.*)"} 123/); }); + +test('Should collect metrics for database', async t => { + const metrics = await prometheusRegister.metrics(); + t.regex(metrics, /db_pool_max/); + t.regex(metrics, /db_pool_min/); + t.regex(metrics, /db_pool_used/); + t.regex(metrics, /db_pool_free/); + t.regex(metrics, /db_pool_pending_creates/); + t.regex(metrics, /db_pool_pending_acquires/); +}); diff --git a/lib/options.js b/lib/options.js index de4b83ef34..d6ecf711ce 100644 --- a/lib/options.js +++ b/lib/options.js @@ -19,6 +19,18 @@ function defaultDatabaseUrl() { return undefined; } +function safeNumber(envVar, defaultVal) { + if (envVar) { + try { + return Number.parseInt(envVar, 10); + } catch (err) { + return defaultVal; + } + } else { + return defaultVal; + } +} + function defaultOptions() { return { databaseUrl: defaultDatabaseUrl(), @@ -35,6 +47,14 @@ function defaultOptions() { : false, driver: 'postgres', version: process.env.DATABASE_VERSION, + pool: { + min: safeNumber(process.env.DATABASE_POOL_MIN, 0), + max: safeNumber(process.env.DATABASE_POOL_MAX, 4), + idleTimeoutMillis: safeNumber( + process.env.DATABASE_POOL_IDLE_TIMEOUT_MS, + 30000, + ), + }, }, port: process.env.HTTP_PORT || process.env.PORT || 4242, host: process.env.HTTP_HOST, @@ -71,6 +91,15 @@ module.exports = { // Use DATABASE_URL when 'db' not defined. if (!opts.db && options.databaseUrl) { options.db = parseDbUrl(options.databaseUrl); + options.db.pool = defaultOptions().db.pool; + } + + // If poolMin and poolMax is set, override pool settings + if (opts.poolMin) { + options.db.pool.min = opts.poolMin; + } + if (opts.poolMax) { + options.db.pool.max = opts.poolMax; } if (!options.db.host) { diff --git a/lib/options.test.js b/lib/options.test.js index d724f8a8c6..ddaa29657c 100644 --- a/lib/options.test.js +++ b/lib/options.test.js @@ -100,6 +100,11 @@ test('should expand databaseUrl from options', t => { password: 'p', port: '5432', user: 'u', + pool: { + idleTimeoutMillis: 30000, + max: 4, + min: 0, + }, }); }); @@ -136,6 +141,11 @@ test('should prefer custom db connection options', t => { ssl: false, driver: 'postgres', version: '10', + pool: { + max: 4, + min: 0, + idleTimeoutMillis: 30000, + }, }; const options = createOptions({ databaseUrl, db }); @@ -149,3 +159,32 @@ test('should baseUriPath', t => { t.deepEqual(options.baseUriPath, baseUriPath); }); + +test('should allow setting pool size', t => { + const min = 4; + const max = 20; + const db = { + user: 'db_user', + password: 'db_password', + host: 'db-host', + port: 5432, + database: 'unleash', + pool: { + min, + max, + }, + }; + const options = createOptions({ db }); + t.is(options.db.pool.min, min); + t.is(options.db.pool.max, max); + t.is(options.db.driver, 'postgres'); +}); + +test('Should allow using outer poolMin and poolMax to set poolsize', t => { + const databaseUrl = 'postgres://u:p@localhost:5432/options'; + const poolMin = 10; + const poolMax = 20; + const options = createOptions({ databaseUrl, poolMax, poolMin }); + t.is(options.db.pool.min, 10); + t.is(options.db.pool.max, 20); +}); diff --git a/test/e2e/helpers/database-init.js b/test/e2e/helpers/database-init.js index 9a27e17f8e..a4e9d7e5d0 100644 --- a/test/e2e/helpers/database-init.js +++ b/test/e2e/helpers/database-init.js @@ -82,10 +82,8 @@ async function setupDatabase(stores) { module.exports = async function init(databaseSchema = 'test', getLogger) { const options = { - db: dbConfig.getDb(), + db: { ...dbConfig.getDb(), pool: { min: 2, max: 8 } }, databaseSchema, - minPool: 1, - maxPool: 1, getLogger, };