Merge pull request #710 from Unleash/fix-705-db-pool-tuning

Added explicit pool settings in options.db object

CHANGELOG.md

@@ -21,6 +21,9 @@
 - docs: update getting started guide with docker options (#697)
 - fix typo in /api/client/features docs (#694)
 - fix: website: require immer 8.0.1 or higher
+- fix: Add support for configuring database pool size (#705)
+- feat: Set default min dbpool size to 0
+- feat: Set default max dbpool size to 4
 
 ## 3.10.1
 

docs/configuring-unleash.md

@@ -16,6 +16,11 @@ const unleashOptions = {
     port: 5432,
     database: 'unleash',
     ssl: false,
+    pool: {
+      min: 0,
+      max: 4,
+      idleTimeoutMillis: 30000,
+    },
   },
   enableRequestLogger: true,
 };
@@ -33,6 +38,10 @@ unleash.start(unleashOptions);
   - _database_ - the database name to be used (`DATABASE_NAME`)
   - _ssl_ - an object describing ssl options, see https://node-postgres.com/features/ssl (`DATABASE_SSL`, as a stringified json object)
   - _version_ - the postgres database version. Used to connect a non-standard database. Defaults to `undefined`, which let the underlying adapter to detect the version automatically. (`DATABASE_VERSION`)
+  - _pool_ - an object describing pool options, see https://knexjs.org/#Installation-pooling. We support the following three fields:
+    - _min_ - minimum connections in connections pool (defaults to 0) (`DATABASE_POOL_MIN`)
+    - _max_ - maximum connections in connections pool (defaults to 4) (`DATABASE_POOL_MAX`)
+    - _idleTimeoutMillis_ - time in milliseconds a connection must be idle before being marked as a candidate for eviction (defaults to 30000) (`DATABASE_POOL_IDLE_TIMEOUT_MS`)
 - **databaseUrl** - (_deprecated_) the postgres database url to connect to. Only used if _db_ object is not specified. Should include username/password. This value may also be set via the `DATABASE_URL` environment variable. Alternatively, if you would like to read the database url from a file, you may set the `DATABASE_URL_FILE` environment variable with the full file path. The contents of the file must be the database url exactly.
 - **databaseSchema** - the postgres database schema to use. Defaults to 'public'. (`DATABASE_SCHEMA`)
 - **port** - which port the unleash-server should bind to. If port is omitted or is 0, the operating system will assign an arbitrary unused port. Will be ignored if pipe is specified. This value may also be set via the `HTTP_PORT` environment variable
@@ -93,3 +102,8 @@ function getLogger(name) {
 ```
 
 The logger interface with its `debug`, `info`, `warn` and `error` methods expects format string support as seen in `debug` or the JavaScript `console` object. Many commonly used logging implementations cover this API, e.g., bunyan, pino or winston.
+
+## Database pooling connection timeouts
+
+- Please be aware of the default values of connection pool about idle session handling.
+- If you have a network component which closes idle sessions on tcp layer, please ensure, that the connection pool idleTimeoutMillis setting is lower than the timespan as the network component will close the idle connection.

lib/db/db-pool.js

@@ -2,19 +2,13 @@
 
 const knex = require('knex');
 
-module.exports.createDb = function({
-    db,
-    poolMin = 2,
-    poolMax = 20,
-    databaseSchema,
-    getLogger,
-}) {
+module.exports.createDb = function({ db, databaseSchema, getLogger }) {
     const logger = getLogger('db-pool.js');
     return knex({
         client: 'pg',
         version: db.version,
         connection: db,
-        pool: { min: poolMin, max: poolMax },
+        pool: db.pool,
         searchPath: databaseSchema,
         log: {
             debug: msg => logger.debug(msg),

lib/event-type.js

@@ -31,4 +31,5 @@ module.exports = {
     ADDON_CONFIG_CREATED: 'addon-config-created',
     ADDON_CONFIG_UPDATED: 'addon-config-updated',
     ADDON_CONFIG_DELETED: 'addon-config-deleted',
+    DB_POOL_UPDATE: 'db-pool-update',
 };

lib/metrics.js

@@ -7,9 +7,11 @@ const {
     FEATURE_UPDATED,
     FEATURE_ARCHIVED,
     FEATURE_REVIVED,
+    DB_POOL_UPDATE,
 } = require('./event-type');
 
 const THREE_HOURS = 3 * 60 * 60 * 1000;
+const ONE_MINUTE = 60 * 1000;
 
 class MetricsMonitor {
     constructor() {
@@ -102,11 +104,69 @@ class MetricsMonitor {
                     .inc(no);
             }
         });
+
+        this.configureDbMetrics(stores, eventStore);
     }
 
     stopMonitoring() {
         clearInterval(this.timer);
     }
+
+    configureDbMetrics(stores, eventStore) {
+        if (stores.db && stores.db.client) {
+            const dbPoolMin = new client.Gauge({
+                name: 'db_pool_min',
+                help: 'Minimum DB pool size',
+            });
+            dbPoolMin.set(stores.db.client.pool.min);
+            const dbPoolMax = new client.Gauge({
+                name: 'db_pool_max',
+                help: 'Maximum DB pool size',
+            });
+            dbPoolMax.set(stores.db.client.pool.max);
+            const dbPoolFree = new client.Gauge({
+                name: 'db_pool_free',
+                help: 'Current free connections in DB pool',
+            });
+            const dbPoolUsed = new client.Gauge({
+                name: 'db_pool_used',
+                help: 'Current connections in use in DB pool',
+            });
+            const dbPoolPendingCreates = new client.Gauge({
+                name: 'db_pool_pending_creates',
+                help:
+                    'how many asynchronous create calls are running in DB pool',
+            });
+            const dbPoolPendingAcquires = new client.Gauge({
+                name: 'db_pool_pending_acquires',
+                help:
+                    'how many acquires are waiting for a resource to be released in DB pool',
+            });
+
+            eventStore.on(DB_POOL_UPDATE, data => {
+                dbPoolFree.set(data.free);
+                dbPoolUsed.set(data.used);
+                dbPoolPendingCreates.set(data.pendingCreates);
+                dbPoolPendingAcquires.set(data.pendingAcquires);
+            });
+
+            this.registerPoolMetrics(stores.db.client.pool, eventStore);
+            setInterval(
+                () =>
+                    this.registerPoolMetrics(stores.db.client.pool, eventStore),
+                ONE_MINUTE,
+            );
+        }
+    }
+
+    registerPoolMetrics(pool, eventStore) {
+        eventStore.emit(DB_POOL_UPDATE, {
+            used: pool.numUsed(),
+            free: pool.numFree(),
+            pendingCreates: pool.numPendingCreates(),
+            pendingAcquires: pool.numPendingAcquires(),
+        });
+    }
 }
 
 module.exports = {

lib/metrics.test.js

@@ -24,6 +24,18 @@ test.before(() => {
             eventStore,
             clientMetricsStore,
             featureToggleStore,
+            db: {
+                client: {
+                    pool: {
+                        min: 0,
+                        max: 4,
+                        numUsed: () => 2,
+                        numFree: () => 2,
+                        numPendingAcquires: () => 0,
+                        numPendingCreates: () => 1,
+                    },
+                },
+            },
         },
         version: '3.4.1',
     };
@@ -95,3 +107,13 @@ test('should collect metrics for feature toggle size', async t => {
     const metrics = await prometheusRegister.metrics();
     t.regex(metrics, /feature_toggles_total{version="(.*)"} 123/);
 });
+
+test('Should collect metrics for database', async t => {
+    const metrics = await prometheusRegister.metrics();
+    t.regex(metrics, /db_pool_max/);
+    t.regex(metrics, /db_pool_min/);
+    t.regex(metrics, /db_pool_used/);
+    t.regex(metrics, /db_pool_free/);
+    t.regex(metrics, /db_pool_pending_creates/);
+    t.regex(metrics, /db_pool_pending_acquires/);
+});

lib/options.js

@@ -19,6 +19,18 @@ function defaultDatabaseUrl() {
     return undefined;
 }
 
+function safeNumber(envVar, defaultVal) {
+    if (envVar) {
+        try {
+            return Number.parseInt(envVar, 10);
+        } catch (err) {
+            return defaultVal;
+        }
+    } else {
+        return defaultVal;
+    }
+}
+
 function defaultOptions() {
     return {
         databaseUrl: defaultDatabaseUrl(),
@@ -35,6 +47,14 @@ function defaultOptions() {
                     : false,
             driver: 'postgres',
             version: process.env.DATABASE_VERSION,
+            pool: {
+                min: safeNumber(process.env.DATABASE_POOL_MIN, 0),
+                max: safeNumber(process.env.DATABASE_POOL_MAX, 4),
+                idleTimeoutMillis: safeNumber(
+                    process.env.DATABASE_POOL_IDLE_TIMEOUT_MS,
+                    30000,
+                ),
+            },
         },
         port: process.env.HTTP_PORT || process.env.PORT || 4242,
         host: process.env.HTTP_HOST,
@@ -71,6 +91,15 @@ module.exports = {
         // Use DATABASE_URL when 'db' not defined.
         if (!opts.db && options.databaseUrl) {
             options.db = parseDbUrl(options.databaseUrl);
+            options.db.pool = defaultOptions().db.pool;
+        }
+
+        // If poolMin and poolMax is set, override pool settings
+        if (opts.poolMin) {
+            options.db.pool.min = opts.poolMin;
+        }
+        if (opts.poolMax) {
+            options.db.pool.max = opts.poolMax;
         }
 
         if (!options.db.host) {

lib/options.test.js

@@ -100,6 +100,11 @@ test('should expand databaseUrl from options', t => {
         password: 'p',
         port: '5432',
         user: 'u',
+        pool: {
+            idleTimeoutMillis: 30000,
+            max: 4,
+            min: 0,
+        },
     });
 });
 
@@ -136,6 +141,11 @@ test('should prefer custom db connection options', t => {
         ssl: false,
         driver: 'postgres',
         version: '10',
+        pool: {
+            max: 4,
+            min: 0,
+            idleTimeoutMillis: 30000,
+        },
     };
     const options = createOptions({ databaseUrl, db });
 
@@ -149,3 +159,32 @@ test('should baseUriPath', t => {
 
     t.deepEqual(options.baseUriPath, baseUriPath);
 });
+
+test('should allow setting pool size', t => {
+    const min = 4;
+    const max = 20;
+    const db = {
+        user: 'db_user',
+        password: 'db_password',
+        host: 'db-host',
+        port: 5432,
+        database: 'unleash',
+        pool: {
+            min,
+            max,
+        },
+    };
+    const options = createOptions({ db });
+    t.is(options.db.pool.min, min);
+    t.is(options.db.pool.max, max);
+    t.is(options.db.driver, 'postgres');
+});
+
+test('Should allow using outer poolMin and poolMax to set poolsize', t => {
+    const databaseUrl = 'postgres://u:p@localhost:5432/options';
+    const poolMin = 10;
+    const poolMax = 20;
+    const options = createOptions({ databaseUrl, poolMax, poolMin });
+    t.is(options.db.pool.min, 10);
+    t.is(options.db.pool.max, 20);
+});

test/e2e/helpers/database-init.js

@@ -82,10 +82,8 @@ async function setupDatabase(stores) {
 
 module.exports = async function init(databaseSchema = 'test', getLogger) {
     const options = {
-        db: dbConfig.getDb(),
+        db: { ...dbConfig.getDb(), pool: { min: 2, max: 8 } },
         databaseSchema,
-        minPool: 1,
-        maxPool: 1,
         getLogger,
     };