feat: Handle database connection errors with 500 (#725)

* feat: Handle database connection errors with 500

- If database goes away while unleash is running, unleash now stays
  running, but all api endpoints will return 500.
- This includes our health endpoint, which allows k8s or similar
  orchestrators to decide what should be done, rather than Unleash
  terminating unexpectedly

src/lib/db/client-metrics-db.js

@@ -23,12 +23,15 @@ class ClientMetricsDb {
     }
 
     async removeMetricsOlderThanOneHour() {
-        const rows = await this.db(TABLE)
-            .whereRaw("created_at < now() - interval '1 hour'")
-            .del();
-
-        if (rows > 0) {
-            this.logger.debug(`Deleted ${rows} metrics`);
+        try {
+            const rows = await this.db(TABLE)
+                .whereRaw("created_at < now() - interval '1 hour'")
+                .del();
+            if (rows > 0) {
+                this.logger.debug(`Deleted ${rows} metrics`);
+            }
+        } catch (e) {
+            this.logger.warn(`Error when deleting metrics ${e}`);
         }
     }
 
@@ -39,26 +42,34 @@ class ClientMetricsDb {
 
     // Used at startup to load all metrics last week into memory!
     async getMetricsLastHour() {
-        const result = await this.db
-            .select(METRICS_COLUMNS)
-            .from(TABLE)
-            .limit(2000)
-            .whereRaw("created_at > now() - interval '1 hour'")
-            .orderBy('created_at', 'asc');
-
-        return result.map(mapRow);
+        try {
+            const result = await this.db
+                .select(METRICS_COLUMNS)
+                .from(TABLE)
+                .limit(2000)
+                .whereRaw("created_at > now() - interval '1 hour'")
+                .orderBy('created_at', 'asc');
+            return result.map(mapRow);
+        } catch (e) {
+            this.logger.warn(`error when getting metrics last hour ${e}`);
+        }
+        return [];
     }
 
     // Used to poll for new metrics
     async getNewMetrics(lastKnownId) {
-        const result = await this.db
-            .select(METRICS_COLUMNS)
-            .from(TABLE)
-            .limit(1000)
-            .where('id', '>', lastKnownId)
-            .orderBy('created_at', 'asc');
-
-        return result.map(mapRow);
+        try {
+            const res = await this.db
+                .select(METRICS_COLUMNS)
+                .from(TABLE)
+                .limit(1000)
+                .where('id', '>', lastKnownId)
+                .orderBy('created_at', 'asc');
+            return res.map(mapRow);
+        } catch (e) {
+            this.logger.warn(`error when getting new metrics ${e}`);
+        }
+        return [];
     }
 
     destroy() {

src/lib/db/db-pool.js

@@ -10,6 +10,7 @@ module.exports.createDb = function({ db, databaseSchema, getLogger }) {
         connection: db,
         pool: db.pool,
         searchPath: databaseSchema,
+        asyncStackTraces: true,
         log: {
             debug: msg => logger.debug(msg),
             info: msg => logger.info(msg),

src/lib/db/event-store.js

@@ -13,48 +13,60 @@ const EVENT_COLUMNS = [
 ];
 
 class EventStore extends EventEmitter {
-    constructor(db) {
+    constructor(db, getLogger) {
         super();
         this.db = db;
+        this.logger = getLogger('lib/db/event-store.js');
     }
 
     async store(event) {
-        await this.db('events').insert({
-            type: event.type,
-            created_by: event.createdBy, // eslint-disable-line
-            data: event.data,
-            tags: event.tags ? JSON.stringify(event.tags) : [],
-        });
-        process.nextTick(() => this.emit(event.type, event));
+        try {
+            await this.db('events').insert({
+                type: event.type,
+                created_by: event.createdBy, // eslint-disable-line
+                data: event.data,
+                tags: event.tags ? JSON.stringify(event.tags) : [],
+            });
+            process.nextTick(() => this.emit(event.type, event));
+        } catch (e) {
+            this.logger.warn(`Failed to store event ${e}`);
+        }
     }
 
     async getEvents() {
-        const rows = await this.db
-            .select(EVENT_COLUMNS)
-            .from('events')
-            .limit(100)
-            .orderBy('created_at', 'desc');
+        try {
+            const rows = await this.db
+                .select(EVENT_COLUMNS)
+                .from('events')
+                .limit(100)
+                .orderBy('created_at', 'desc');
 
-        return rows.map(this.rowToEvent);
+            return rows.map(this.rowToEvent);
+        } catch (err) {
+            return [];
+        }
     }
 
     async getEventsFilterByName(name) {
-        const rows = await this.db
-            .select(EVENT_COLUMNS)
-            .from('events')
-            .limit(100)
-            .whereRaw("data ->> 'name' = ?", [name])
-            .andWhere(
-                'id',
-                '>=',
-                this.db
-                    .select(this.db.raw('coalesce(max(id),0) as id'))
-                    .from('events')
-                    .where({ type: DROP_FEATURES }),
-            )
-            .orderBy('created_at', 'desc');
-
-        return rows.map(this.rowToEvent);
+        try {
+            const rows = await this.db
+                .select(EVENT_COLUMNS)
+                .from('events')
+                .limit(100)
+                .whereRaw("data ->> 'name' = ?", [name])
+                .andWhere(
+                    'id',
+                    '>=',
+                    this.db
+                        .select(this.db.raw('coalesce(max(id),0) as id'))
+                        .from('events')
+                        .where({ type: DROP_FEATURES }),
+                )
+                .orderBy('created_at', 'desc');
+            return rows.map(this.rowToEvent);
+        } catch (err) {
+            return [];
+        }
     }
 
     rowToEvent(row) {

src/lib/metrics.js

@@ -57,7 +57,13 @@ class MetricsMonitor {
 
         async function collectFeatureToggleMetrics() {
             featureTogglesTotal.reset();
-            const togglesCount = await featureToggleStore.count();
+            let togglesCount;
+            try {
+                togglesCount = await featureToggleStore.count();
+                // eslint-disable-next-line no-empty
+            } catch (e) {}
+
+            togglesCount = togglesCount || 0;
             featureTogglesTotal.labels(version).set(togglesCount);
         }
 
@@ -105,14 +111,14 @@ class MetricsMonitor {
             }
         });
 
-        this.configureDbMetrics(stores, eventStore);
+        this.configureDbMetrics(stores, eventBus);
     }
 
     stopMonitoring() {
         clearInterval(this.timer);
     }
 
-    configureDbMetrics(stores, eventStore) {
+    configureDbMetrics(stores, eventBus) {
         if (stores.db && stores.db.client) {
             const dbPoolMin = new client.Gauge({
                 name: 'db_pool_min',
@@ -143,29 +149,31 @@ class MetricsMonitor {
                     'how many acquires are waiting for a resource to be released in DB pool',
             });
 
-            eventStore.on(DB_POOL_UPDATE, data => {
+            eventBus.on(DB_POOL_UPDATE, data => {
                 dbPoolFree.set(data.free);
                 dbPoolUsed.set(data.used);
                 dbPoolPendingCreates.set(data.pendingCreates);
                 dbPoolPendingAcquires.set(data.pendingAcquires);
             });
 
-            this.registerPoolMetrics(stores.db.client.pool, eventStore);
+            this.registerPoolMetrics(stores.db.client.pool, eventBus);
             setInterval(
-                () =>
-                    this.registerPoolMetrics(stores.db.client.pool, eventStore),
+                () => this.registerPoolMetrics(stores.db.client.pool, eventBus),
                 ONE_MINUTE,
             );
         }
     }
 
-    registerPoolMetrics(pool, eventStore) {
-        eventStore.emit(DB_POOL_UPDATE, {
-            used: pool.numUsed(),
-            free: pool.numFree(),
-            pendingCreates: pool.numPendingCreates(),
-            pendingAcquires: pool.numPendingAcquires(),
-        });
+    registerPoolMetrics(pool, eventBus) {
+        try {
+            eventBus.emit(DB_POOL_UPDATE, {
+                used: pool.numUsed(),
+                free: pool.numFree(),
+                pendingCreates: pool.numPendingCreates(),
+                pendingAcquires: pool.numPendingAcquires(),
+            });
+            // eslint-disable-next-line no-empty
+        } catch (e) {}
     }
 }
 

src/lib/metrics.test.js

@@ -15,7 +15,7 @@ const monitor = createMetricsMonitor();
 
 test.before(() => {
     const featureToggleStore = {
-        count: () => 123,
+        count: async () => 123,
     };
     const config = {
         serverMetrics: true,

src/lib/routes/admin-api/archive.js

@@ -1,5 +1,7 @@
 'use strict';
 
+import { handleErrors } from './util';
+
 const Controller = require('../controller');
 
 const extractUser = require('../../extract-user');
@@ -16,8 +18,12 @@ class ArchiveController extends Controller {
     }
 
     async getArchivedFeatures(req, res) {
-        const features = await this.featureService.getArchivedFeatures();
-        res.json({ features });
+        try {
+            const features = await this.featureService.getArchivedFeatures();
+            res.json({ features });
+        } catch (err) {
+            handleErrors(res, this.logger, err);
+        }
     }
 
     async reviveFeatureToggle(req, res) {

src/lib/routes/admin-api/context.js

@@ -34,10 +34,14 @@ class ContextController extends Controller {
     }
 
     async getContextFields(req, res) {
-        const fields = await this.contextService.getAll();
-        res.status(200)
-            .json(fields)
-            .end();
+        try {
+            const fields = await this.contextService.getAll();
+            res.status(200)
+                .json(fields)
+                .end();
+        } catch (e) {
+            handleErrors(res, this.logger, e);
+        }
     }
 
     async getContextField(req, res) {

src/lib/routes/admin-api/event.js

@@ -1,5 +1,7 @@
 'use strict';
 
+import { handleErrors } from './util';
+
 const Controller = require('../controller');
 
 const eventDiffer = require('../../event-differ');
@@ -15,20 +17,30 @@ class EventController extends Controller {
     }
 
     async getEvents(req, res) {
-        const events = await this.eventStore.getEvents();
-        eventDiffer.addDiffs(events);
-        res.json({ version, events });
+        try {
+            const events = await this.eventStore.getEvents();
+            eventDiffer.addDiffs(events);
+            res.json({ version, events });
+        } catch (e) {
+            handleErrors(res, this.logger, e);
+        }
     }
 
     async getEventsForToggle(req, res) {
         const toggleName = req.params.name;
-        const events = await this.eventStore.getEventsFilterByName(toggleName);
+        try {
+            const events = await this.eventStore.getEventsFilterByName(
+                toggleName,
+            );
 
-        if (events) {
-            eventDiffer.addDiffs(events);
-            res.json({ toggleName, events });
-        } else {
-            res.status(404).json({ error: 'Could not find events' });
+            if (events) {
+                eventDiffer.addDiffs(events);
+                res.json({ toggleName, events });
+            } else {
+                res.status(404).json({ error: 'Could not find events' });
+            }
+        } catch (e) {
+            handleErrors(res, this.logger, e);
         }
     }
 }

src/lib/routes/admin-api/feature-type.js

@@ -1,5 +1,7 @@
 'use strict';
 
+import { handleErrors } from './util';
+
 const Controller = require('../controller');
 
 const version = 1;
@@ -14,8 +16,12 @@ class FeatureTypeController extends Controller {
     }
 
     async getAllFeatureTypes(req, res) {
-        const types = await this.featureTypeStore.getAll();
-        res.json({ version, types });
+        try {
+            const types = await this.featureTypeStore.getAll();
+            res.json({ version, types });
+        } catch (e) {
+            handleErrors(res, this.logger, e);
+        }
     }
 }
 

src/lib/routes/admin-api/feature.js

@@ -49,11 +49,15 @@ class FeatureController extends Controller {
     }
 
     async getAllToggles(req, res) {
-        const features = await this.featureService.getFeatures(
-            req.query,
-            fields,
-        );
-        res.json({ version, features });
+        try {
+            const features = await this.featureService.getFeatures(
+                req.query,
+                fields,
+            );
+            res.json({ version, features });
+        } catch (err) {
+            handleErrors(res, this.logger, err);
+        }
     }
 
     async getToggle(req, res) {
@@ -67,8 +71,14 @@ class FeatureController extends Controller {
     }
 
     async listTags(req, res) {
-        const tags = await this.featureService.listTags(req.params.featureName);
-        res.json({ version, tags });
+        try {
+            const tags = await this.featureService.listTags(
+                req.params.featureName,
+            );
+            res.json({ version, tags });
+        } catch (err) {
+            handleErrors(res, this.logger, err);
+        }
     }
 
     async addTag(req, res) {
@@ -89,12 +99,16 @@ class FeatureController extends Controller {
     async removeTag(req, res) {
         const { featureName, type, value } = req.params;
         const userName = extractUser(req);
-        await this.featureService.removeTag(
-            featureName,
-            { type, value },
-            userName,
-        );
-        res.status(200).end();
+        try {
+            await this.featureService.removeTag(
+                featureName,
+                { type, value },
+                userName,
+            );
+            res.status(200).end();
+        } catch (err) {
+            handleErrors(res, this.logger, err);
+        }
     }
 
     async validate(req, res) {

src/lib/routes/admin-api/feature.test.js

@@ -16,9 +16,9 @@ const {
 
 const eventBus = new EventEmitter();
 
-function getSetup() {
+function getSetup(databaseIsUp = true) {
     const base = `/random${Math.round(Math.random() * 1000)}`;
-    const stores = store.createStores();
+    const stores = store.createStores(databaseIsUp);
     const perms = permissions();
     const config = {
         baseUriPath: base,
@@ -614,3 +614,9 @@ test('Tags should be included in updated events', async t => {
     t.is(events[0].tags[0].type, 'simple');
     t.is(events[0].tags[0].value, 'tag');
 });
+
+test('Trying to get features while database is down should yield 500', t => {
+    t.plan(0);
+    const { request, base } = getSetup(false);
+    return request.get(`${base}/api/admin/features`).expect(500);
+});

src/lib/routes/admin-api/metrics.js

@@ -28,29 +28,45 @@ class MetricsController extends Controller {
     }
 
     async getSeenToggles(req, res) {
-        const seenAppToggles = await this.metrics.getAppsWithToggles();
-        res.json(seenAppToggles);
+        try {
+            const seenAppToggles = await this.metrics.getAppsWithToggles();
+            res.json(seenAppToggles);
+        } catch (e) {
+            handleErrors(res, this.logger, e);
+        }
     }
 
     async getSeenApps(req, res) {
-        const seenApps = await this.metrics.getSeenApps();
-        res.json(seenApps);
+        try {
+            const seenApps = await this.metrics.getSeenApps();
+            res.json(seenApps);
+        } catch (e) {
+            handleErrors(res, this.logger, e);
+        }
     }
 
     async getFeatureToggles(req, res) {
-        const toggles = await this.metrics.getTogglesMetrics();
-        res.json(toggles);
+        try {
+            const toggles = await this.metrics.getTogglesMetrics();
+            res.json(toggles);
+        } catch (e) {
+            handleErrors(res, this.logger, e);
+        }
     }
 
     async getFeatureToggle(req, res) {
-        const { name } = req.params;
-        const data = await this.metrics.getTogglesMetrics();
-        const lastHour = data.lastHour[name] || {};
-        const lastMinute = data.lastMinute[name] || {};
-        res.json({
-            lastHour,
-            lastMinute,
-        });
+        try {
+            const { name } = req.params;
+            const data = await this.metrics.getTogglesMetrics();
+            const lastHour = data.lastHour[name] || {};
+            const lastMinute = data.lastMinute[name] || {};
+            res.json({
+                lastHour,
+                lastMinute,
+            });
+        } catch (e) {
+            handleErrors(res, this.logger, e);
+        }
     }
 
     async deleteApplication(req, res) {

src/lib/routes/admin-api/strategy.js

@@ -36,8 +36,12 @@ class StrategyController extends Controller {
     }
 
     async getAllStratgies(req, res) {
-        const strategies = await this.strategyService.getStrategies();
-        res.json({ version, strategies });
+        try {
+            const strategies = await this.strategyService.getStrategies();
+            res.json({ version, strategies });
+        } catch (err) {
+            handleErrors(res, this.logger, err);
+        }
     }
 
     async getStrategy(req, res) {

src/lib/routes/admin-api/strategy.test.js

@@ -16,10 +16,10 @@ const {
 
 const eventBus = new EventEmitter();
 
-function getSetup() {
+function getSetup(databaseIsUp = true) {
     const base = `/random${Math.round(Math.random() * 1000)}`;
     const perms = permissions();
-    const stores = store.createStores();
+    const stores = store.createStores(databaseIsUp);
     const config = {
         baseUriPath: base,
         stores,
@@ -261,3 +261,9 @@ test(`deprecating 'default' strategy will yield 403`, t => {
         .set('Content-Type', 'application/json')
         .expect(403);
 });
+
+test('Getting strategies while database is down should yield 500', t => {
+    t.plan(0);
+    const { request, base } = getSetup(false);
+    return request.get(`${base}/api/admin/strategies`).expect(500);
+});

src/lib/routes/admin-api/tag-type.js

@@ -22,8 +22,12 @@ class TagTypeController extends Controller {
     }
 
     async getTagTypes(req, res) {
-        const tagTypes = await this.tagTypeService.getAll();
-        res.json({ version, tagTypes });
+        try {
+            const tagTypes = await this.tagTypeService.getAll();
+            res.json({ version, tagTypes });
+        } catch (e) {
+            handleErrors(res, this.logger, e);
+        }
     }
 
     async validate(req, res) {

src/lib/routes/admin-api/tag.js

@@ -22,13 +22,21 @@ class TagController extends Controller {
     }
 
     async getTags(req, res) {
-        const tags = await this.tagService.getTags();
-        res.json({ version, tags });
+        try {
+            const tags = await this.tagService.getTags();
+            res.json({ version, tags });
+        } catch (e) {
+            handleErrors(res, this.logger, e);
+        }
     }
 
     async getTagsByType(req, res) {
-        const tags = await this.tagService.getTagsByType(req.params.type);
-        res.json({ version, tags });
+        try {
+            const tags = await this.tagService.getTagsByType(req.params.type);
+            res.json({ version, tags });
+        } catch (e) {
+            handleErrors(res, this.logger, e);
+        }
     }
 
     async getTag(req, res) {

src/lib/routes/admin-api/tag.test.js

@@ -12,9 +12,9 @@ const { UPDATE_FEATURE } = require('../../permissions');
 
 const eventBus = new EventEmitter();
 
-function getSetup() {
+function getSetup(databaseIsUp = true) {
     const base = `/random${Math.round(Math.random() * 1000)}`;
-    const stores = store.createStores();
+    const stores = store.createStores(databaseIsUp);
     const perms = permissions();
     const config = {
         baseUriPath: base,
@@ -118,3 +118,9 @@ test('should be able to filter by type', t => {
             t.is(res.body.tags[0].value, 'TeamRed');
         });
 });
+
+test('Getting tags while database is down should be a 500', t => {
+    t.plan(0);
+    const { request, base } = getSetup(false);
+    return request.get(`${base}/api/admin/tags`).expect(500);
+});

src/lib/routes/client-api/feature.js

@@ -1,5 +1,7 @@
 'use strict';
 
+import { handleErrors } from '../admin-api/util';
+
 const Controller = require('../controller');
 
 const version = 1;
@@ -23,11 +25,15 @@ class FeatureController extends Controller {
     }
 
     async getAll(req, res) {
-        const features = await this.toggleService.getFeatures(
-            req.query,
-            FEATURE_COLUMNS_CLIENT,
-        );
-        res.json({ version, features });
+        try {
+            const features = await this.toggleService.getFeatures(
+                req.query,
+                FEATURE_COLUMNS_CLIENT,
+            );
+            res.json({ version, features });
+        } catch (e) {
+            handleErrors(res, this.logger, e);
+        }
     }
 
     async getFeatureToggle(req, res) {

src/test/fixtures/fake-feature-toggle-store.js

@@ -1,12 +1,15 @@
 'use strict';
 
-module.exports = () => {
+module.exports = (databaseIsUp = true) => {
     const _features = [];
     const _archive = [];
     const _featureTags = {};
 
     return {
         getFeature: name => {
+            if (!databaseIsUp) {
+                return Promise.reject(new Error('No database connection'));
+            }
             const toggle = _features.find(f => f.name === name);
             if (toggle) {
                 return Promise.resolve(toggle);
@@ -63,6 +66,9 @@ module.exports = () => {
         },
         importFeature: feat => Promise.resolve(_features.push(feat)),
         getFeatures: query => {
+            if (!databaseIsUp) {
+                return Promise.reject(new Error('No database connection'));
+            }
             if (query) {
                 const activeQueryKeys = Object.keys(query).filter(
                     t => query[t],

src/test/fixtures/fake-strategies-store.js

@@ -2,13 +2,18 @@
 
 const NotFoundError = require('../../lib/error/notfound-error');
 
-module.exports = () => {
+module.exports = (databaseIsUp = true) => {
     const _strategies = [
         { name: 'default', editable: false, parameters: {}, deprecated: false },
     ];
 
     return {
-        getStrategies: () => Promise.resolve(_strategies),
+        getStrategies: () => {
+            if (databaseIsUp) {
+                return Promise.resolve(_strategies);
+            }
+            return Promise.reject(new Error('No database connection'));
+        },
         getEditableStrategies: () =>
             Promise.resolve(_strategies.filter(s => s.editable)),
         getStrategy: name => {

src/test/fixtures/fake-tag-store.js

@@ -1,9 +1,12 @@
 const NotFoundError = require('../../lib/error/notfound-error');
 
-module.exports = () => {
+module.exports = (databaseIsUp = true) => {
     const _tags = [];
     return {
         getTagsByType: type => {
+            if (!databaseIsUp) {
+                return Promise.reject(new Error('No database connection'));
+            }
             const tags = _tags.filter(t => t.type === type);
             return Promise.resolve(tags);
         },
@@ -18,7 +21,12 @@ module.exports = () => {
                 1,
             );
         },
-        getAll: () => Promise.resolve(_tags),
+        getAll: () => {
+            if (!databaseIsUp) {
+                return Promise.reject(new Error('No database connection'));
+            }
+            return Promise.resolve(_tags);
+        },
         getTag: (type, value) => {
             const tag = _tags.find(t => t.type === type && t.value === value);
             if (tag) {

src/test/fixtures/store.js

@@ -13,7 +13,7 @@ const settingStore = require('./fake-setting-store');
 const addonStore = require('./fake-addon-store');
 
 module.exports = {
-    createStores: () => {
+    createStores: (databaseIsUp = true) => {
         const db = {
             select: () => ({
                 from: () => Promise.resolve(),
@@ -22,17 +22,17 @@ module.exports = {
 
         return {
             db,
-            clientApplicationsStore: clientApplicationsStore(),
-            clientMetricsStore: new ClientMetricsStore(),
-            clientInstanceStore: clientInstanceStore(),
-            featureToggleStore: featureToggleStore(),
-            tagStore: tagStore(),
-            tagTypeStore: tagTypeStore(),
-            eventStore: new EventStore(),
-            strategyStore: strategyStore(),
-            contextFieldStore: contextFieldStore(),
-            settingStore: settingStore(),
-            addonStore: addonStore(),
+            clientApplicationsStore: clientApplicationsStore(databaseIsUp),
+            clientMetricsStore: new ClientMetricsStore(databaseIsUp),
+            clientInstanceStore: clientInstanceStore(databaseIsUp),
+            featureToggleStore: featureToggleStore(databaseIsUp),
+            tagStore: tagStore(databaseIsUp),
+            tagTypeStore: tagTypeStore(databaseIsUp),
+            eventStore: new EventStore(databaseIsUp),
+            strategyStore: strategyStore(databaseIsUp),
+            contextFieldStore: contextFieldStore(databaseIsUp),
+            settingStore: settingStore(databaseIsUp),
+            addonStore: addonStore(databaseIsUp),
         };
     },
 };