feat: Added import & export through stateService #395

lib/db/event-store.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const { DROP_FEATURES } = require('../event-type');
 const { EventEmitter } = require('events');
 
 const EVENT_COLUMNS = ['id', 'type', 'created_by', 'created_at', 'data'];
@@ -35,6 +36,14 @@ class EventStore extends EventEmitter {
             .from('events')
             .limit(100)
             .whereRaw("data ->> 'name' = ?", [name])
+            .andWhere(
+                'id',
+                '>=',
+                this.db
+                    .select(this.db.raw('coalesce(max(id),0) as id'))
+                    .from('events')
+                    .where({ type: DROP_FEATURES })
+            )
             .orderBy('created_at', 'desc')
             .map(this.rowToEvent);
     }

lib/db/feature-toggle-store.js

@@ -5,6 +5,8 @@ const {
     FEATURE_UPDATED,
     FEATURE_ARCHIVED,
     FEATURE_REVIVED,
+    FEATURE_IMPORT,
+    DROP_FEATURES,
 } = require('../event-type');
 const logger = require('../logger')('client-toggle-store.js');
 const NotFoundError = require('../error/notfound-error');
@@ -33,6 +35,8 @@ class FeatureToggleStore {
         eventStore.on(FEATURE_REVIVED, event =>
             this._reviveFeature(event.data)
         );
+        eventStore.on(FEATURE_IMPORT, event => this._importFeature(event.data));
+        eventStore.on(DROP_FEATURES, () => this._dropFeatures());
     }
 
     getFeatures() {
@@ -137,6 +141,29 @@ class FeatureToggleStore {
                 logger.error('Could not archive feature, error was: ', err)
             );
     }
+
+    _importFeature(data) {
+        data = this.eventDataToRow(data);
+        return this.db
+            .raw(`? ON CONFLICT (name) DO ?`, [
+                this.db(TABLE).insert(data),
+                this.db
+                    .queryBuilder()
+                    .update(data)
+                    .update('archived', 0),
+            ])
+            .catch(err =>
+                logger.error('Could not import feature, error was: ', err)
+            );
+    }
+
+    _dropFeatures() {
+        return this.db(TABLE)
+            .delete()
+            .catch(err =>
+                logger.error('Could not drop features, error was: ', err)
+            );
+    }
 }
 
 module.exports = FeatureToggleStore;

lib/db/strategy-store.js

@@ -4,6 +4,8 @@ const {
     STRATEGY_CREATED,
     STRATEGY_DELETED,
     STRATEGY_UPDATED,
+    STRATEGY_IMPORT,
+    DROP_STRATEGIES,
 } = require('../event-type');
 const logger = require('../logger')('strategy-store.js');
 const NotFoundError = require('../error/notfound-error');
@@ -19,14 +21,13 @@ class StrategyStore {
         eventStore.on(STRATEGY_UPDATED, event =>
             this._updateStrategy(event.data)
         );
-        eventStore.on(STRATEGY_DELETED, event => {
+        eventStore.on(STRATEGY_DELETED, event =>
-            db(TABLE)
+            this._deleteStrategy(event.data)
-                .where('name', event.data.name)
+        );
-                .del()
+        eventStore.on(STRATEGY_IMPORT, event =>
-                .catch(err => {
+            this._importStrategy(event.data)
-                    logger.error('Could not delete strategy, error was: ', err);
+        );
-                });
+        eventStore.on(DROP_STRATEGIES, () => this._dropStrategies());
-        });
     }
 
     getStrategies() {
@@ -81,6 +82,36 @@ class StrategyStore {
                 logger.error('Could not update strategy, error was: ', err)
             );
     }
+
+    _deleteStrategy({ name }) {
+        return this.db(TABLE)
+            .where({ name })
+            .del()
+            .catch(err => {
+                logger.error('Could not delete strategy, error was: ', err);
+            });
+    }
+
+    _importStrategy(data) {
+        data = this.eventDataToRow(data);
+        return this.db
+            .raw(`? ON CONFLICT (name) DO ?`, [
+                this.db(TABLE).insert(data),
+                this.db.queryBuilder().update(data).where(`${TABLE}.built_in`, 0), // eslint-disable-line
+            ])
+            .catch(err =>
+                logger.error('Could not import strategy, error was: ', err)
+            );
+    }
+
+    _dropStrategies() {
+        return this.db(TABLE)
+            .where({ built_in: 0 }) // eslint-disable-line
+            .delete()
+            .catch(err =>
+                logger.error('Could not drop strategies, error was: ', err)
+            );
+    }
 }
 
 module.exports = StrategyStore;

lib/event-differ.js

@@ -4,20 +4,32 @@ const {
     STRATEGY_CREATED,
     STRATEGY_DELETED,
     STRATEGY_UPDATED,
+    STRATEGY_IMPORT,
+    DROP_STRATEGIES,
     FEATURE_CREATED,
     FEATURE_UPDATED,
     FEATURE_ARCHIVED,
     FEATURE_REVIVED,
+    FEATURE_IMPORT,
+    DROP_FEATURES,
 } = require('./event-type');
 const diff = require('deep-diff').diff;
 
-const strategyTypes = [STRATEGY_CREATED, STRATEGY_DELETED, STRATEGY_UPDATED];
+const strategyTypes = [
+    STRATEGY_CREATED,
+    STRATEGY_DELETED,
+    STRATEGY_UPDATED,
+    STRATEGY_IMPORT,
+    DROP_STRATEGIES,
+];
 
 const featureTypes = [
     FEATURE_CREATED,
     FEATURE_UPDATED,
     FEATURE_ARCHIVED,
     FEATURE_REVIVED,
+    FEATURE_IMPORT,
+    DROP_FEATURES,
 ];
 
 function baseTypeFor(event) {

lib/event-type.js

@@ -5,7 +5,11 @@ module.exports = {
     FEATURE_UPDATED: 'feature-updated',
     FEATURE_ARCHIVED: 'feature-archived',
     FEATURE_REVIVED: 'feature-revived',
+    FEATURE_IMPORT: 'feature-import',
+    DROP_FEATURES: 'drop-features',
     STRATEGY_CREATED: 'strategy-created',
     STRATEGY_DELETED: 'strategy-deleted',
     STRATEGY_UPDATED: 'strategy-updated',
+    STRATEGY_IMPORT: 'strategy-import',
+    DROP_STRATEGIES: 'drop-strategies',
 };

lib/options.js

@@ -20,6 +20,8 @@ const DEFAULT_OPTIONS = {
     sessionAge: THIRTY_DAYS,
     adminAuthentication: 'unsecure',
     ui: {},
+    importFile: undefined,
+    dropBeforeImport: false,
 };
 
 module.exports = {

lib/routes/admin-api/api-def.json

@@ -15,6 +15,9 @@
     },
     "metrics": {
       "uri": "/api/admin/metrics"
+    },
+    "state": {
+      "uri": "/api/admin/state"
     }
   }
 }

lib/routes/admin-api/index.js

@@ -8,6 +8,7 @@ const StrategyController = require('./strategy');
 const MetricsController = require('./metrics');
 const UserController = require('./user');
 const ConfigController = require('./config');
+const StateController = require('./state');
 const apiDef = require('./api-def.json');
 
 class AdminApi extends Controller {
@@ -22,6 +23,7 @@ class AdminApi extends Controller {
         this.app.use('/metrics', new MetricsController(config).router);
         this.app.use('/user', new UserController(config).router);
         this.app.use('/ui-config', new ConfigController(config).router);
+        this.app.use('/state', new StateController(config).router);
     }
 
     index(req, res) {

lib/routes/admin-api/state.js

@@ -0,0 +1,80 @@
+'use strict';
+
+const Controller = require('../controller');
+const { ADMIN } = require('../../permissions');
+const extractUser = require('../../extract-user');
+const { handleErrors } = require('./util');
+const YAML = require('js-yaml');
+const moment = require('moment');
+const multer = require('multer');
+const upload = multer({ limits: { fileSize: 5242880 } });
+
+class ImportController extends Controller {
+    constructor(config) {
+        super(config);
+        this.fileupload('/import', upload.single('file'), this.import, ADMIN);
+        this.get('/export', this.export, ADMIN);
+    }
+
+    async import(req, res) {
+        const userName = extractUser(req);
+        const { drop } = req.query;
+
+        let data;
+        if (req.file) {
+            if (req.file.mimetype === 'text/yaml') {
+                data = YAML.safeLoad(req.file.buffer);
+            } else {
+                data = JSON.parse(req.file.buffer);
+            }
+        } else {
+            data = req.body;
+        }
+
+        try {
+            await this.config.stateService.import({
+                data,
+                userName,
+                dropBeforeImport: drop,
+            });
+            res.sendStatus(202);
+        } catch (err) {
+            handleErrors(res, err);
+        }
+    }
+
+    async export(req, res) {
+        const { format } = req.query;
+
+        let strategies = 'strategies' in req.query;
+        let featureToggles = 'features' in req.query;
+
+        if (!strategies && !featureToggles) {
+            strategies = true;
+            featureToggles = true;
+        }
+
+        try {
+            const data = await this.config.stateService.export({
+                strategies,
+                featureToggles,
+            });
+            const timestamp = moment().format('YYYY-MM-DD_HH-mm-ss');
+            if (format === 'yaml') {
+                if ('download' in req.query) {
+                    res.attachment(`export-${timestamp}.yml`);
+                }
+                res.type('yaml').send(YAML.safeDump(data));
+            } else {
+                if ('download' in req.query) {
+                    res.attachment(`export-${timestamp}.json`);
+                }
+                res.json(data);
+            }
+        } catch (err) {
+            handleErrors(res, err);
+        }
+    }
+}
+
+module.exports = ImportController;

lib/routes/controller.js

@@ -44,6 +44,15 @@ class Controller {
         );
     }
 
+    fileupload(path, filehandler, handler, permission) {
+        this.app.post(
+            path,
+            checkPermission(this.config, permission),
+            filehandler,
+            handler.bind(this)
+        );
+    }
+
     use(path, router) {
         this.app.use(path, router);
     }

lib/server-impl.js

@@ -10,10 +10,11 @@ const getApp = require('./app');
 const { startMonitoring } = require('./metrics');
 const { createStores } = require('./db');
 const { createOptions } = require('./options');
+const StateService = require('./state-service');
 const User = require('./user');
 const AuthenticationRequired = require('./authentication-required');
 
-function createApp(options) {
+async function createApp(options) {
     // Database dependecies (statefull)
     const stores = createStores(options);
     const eventBus = new EventEmitter();
@@ -35,12 +36,29 @@ function createApp(options) {
         stores.clientMetricsStore
     );
 
+    const stateService = new StateService(config);
+    config.stateService = stateService;
+    if (config.importFile) {
+        await stateService.importFile({
+            importFile: config.importFile,
+            dropBeforeImport: config.dropBeforeImport,
+            userName: 'importer',
+        });
+    }
+
     const server = app.listen({ port: options.port, host: options.host }, () =>
         logger.info(`Unleash started on port ${server.address().port}`)
     );
 
-    return new Promise((resolve, reject) => {
+    return await new Promise((resolve, reject) => {
-        server.on('listening', () => resolve({ app, server, eventBus }));
+        server.on('listening', () =>
+            resolve({
+                app,
+                server,
+                eventBus,
+                stateService,
+            })
+        );
         server.on('error', reject);
     });
 }

lib/state-service.js

@@ -0,0 +1,122 @@
+'use strict';
+
+const joi = require('joi');
+const fs = require('fs');
+const mime = require('mime');
+const { featureShema } = require('./routes/admin-api/feature-schema');
+const strategySchema = require('./routes/admin-api/strategy-schema');
+const getLogger = require('./logger');
+const yaml = require('js-yaml');
+const {
+    FEATURE_IMPORT,
+    DROP_FEATURES,
+    STRATEGY_IMPORT,
+    DROP_STRATEGIES,
+} = require('./event-type');
+
+const logger = getLogger('state-service.js');
+
+const dataSchema = joi.object().keys({
+    features: joi
+        .array()
+        .optional()
+        .items(featureShema),
+    strategies: joi
+        .array()
+        .optional()
+        .items(strategySchema),
+});
+
+class StateService {
+    constructor(config) {
+        this.config = config;
+    }
+
+    async importFile({ importFile, dropBeforeImport, userName }) {
+        let data = await new Promise((resolve, reject) =>
+            fs.readFile(importFile, (err, v) =>
+                err ? reject(err) : resolve(v)
+            )
+        );
+        if (mime.lookup(importFile) === 'text/yaml') {
+            data = yaml.safeLoad(data);
+        }
+
+        await this.import({
+            data,
+            dropBeforeImport,
+            userName,
+        });
+    }
+
+    async import({ data, userName, dropBeforeImport }) {
+        const { eventStore } = this.config.stores;
+
+        if (typeof data === 'string') {
+            data = JSON.parse(data);
+        }
+
+        data = await joi.validate(data, dataSchema);
+
+        if (data.features) {
+            logger.info(`Importing ${data.features.length} features`);
+            if (dropBeforeImport) {
+                logger.info(`Dropping existing features`);
+                await eventStore.store({
+                    type: DROP_FEATURES,
+                    createdBy: userName,
+                    data: { name: 'all-features' },
+                });
+            }
+            for (const feature of data.features) {
+                await eventStore.store({
+                    type: FEATURE_IMPORT,
+                    createdBy: userName,
+                    data: feature,
+                });
+            }
+        }
+
+        if (data.strategies) {
+            logger.info(`Importing ${data.strategies.length} strategies`);
+            if (dropBeforeImport) {
+                logger.info(`Dropping existing strategies`);
+                await eventStore.store({
+                    type: DROP_STRATEGIES,
+                    createdBy: userName,
+                    data: { name: 'all-strategies' },
+                });
+            }
+            for (const strategy of data.strategies) {
+                await eventStore.store({
+                    type: STRATEGY_IMPORT,
+                    createdBy: userName,
+                    data: strategy,
+                });
+            }
+        }
+    }
+
+    async export({ strategies, featureToggles }) {
+        const { featureToggleStore, strategyStore } = this.config.stores;
+        const result = {};
+
+        if (featureToggles) {
+            result.features = await featureToggleStore.getFeatures();
+        }
+
+        if (strategies) {
+            result.strategies = (await strategyStore.getStrategies())
+                .filter(strat => strat.editable)
+                .map(strat => {
+                    strat = Object.assign({}, strat);
+                    delete strat.editable;
+                    return strat;
+                });
+        }
+
+        return result;
+    }
+}
+
+module.exports = StateService;

lib/state-service.test.js

@@ -0,0 +1,148 @@
+'use strict';
+
+const test = require('ava');
+
+const store = require('./../test/fixtures/store');
+const StateService = require('./state-service');
+const {
+    FEATURE_IMPORT,
+    DROP_FEATURES,
+    STRATEGY_IMPORT,
+    DROP_STRATEGIES,
+} = require('./event-type');
+
+function getSetup() {
+    const stores = store.createStores();
+    return { stateService: new StateService({ stores }), stores };
+}
+
+test('should import a feature', async t => {
+    const { stateService, stores } = getSetup();
+
+    const data = {
+        features: [
+            {
+                name: 'new-feature',
+                enabled: true,
+                strategies: [{ name: 'default' }],
+            },
+        ],
+    };
+
+    await stateService.import({ data });
+
+    const events = await stores.eventStore.getEvents();
+    t.is(events.length, 1);
+    t.is(events[0].type, FEATURE_IMPORT);
+    t.is(events[0].data.name, 'new-feature');
+});
+
+test('should drop feature before import if specified', async t => {
+    const { stateService, stores } = getSetup();
+
+    const data = {
+        features: [
+            {
+                name: 'new-feature',
+                enabled: true,
+                strategies: [{ name: 'default' }],
+            },
+        ],
+    };
+
+    await stateService.import({ data, dropBeforeImport: true });
+
+    const events = await stores.eventStore.getEvents();
+    t.is(events.length, 2);
+    t.is(events[0].type, DROP_FEATURES);
+    t.is(events[1].type, FEATURE_IMPORT);
+    t.is(events[1].data.name, 'new-feature');
+});
+
+test('should import a strategy', async t => {
+    const { stateService, stores } = getSetup();
+
+    const data = {
+        strategies: [
+            {
+                name: 'new-strategy',
+                parameters: [],
+            },
+        ],
+    };
+
+    await stateService.import({ data });
+
+    const events = await stores.eventStore.getEvents();
+    t.is(events.length, 1);
+    t.is(events[0].type, STRATEGY_IMPORT);
+    t.is(events[0].data.name, 'new-strategy');
+});
+
+test('should drop strategies before import if specified', async t => {
+    const { stateService, stores } = getSetup();
+
+    const data = {
+        strategies: [
+            {
+                name: 'new-strategy',
+                parameters: [],
+            },
+        ],
+    };
+
+    await stateService.import({ data, dropBeforeImport: true });
+
+    const events = await stores.eventStore.getEvents();
+    t.is(events.length, 2);
+    t.is(events[0].type, DROP_STRATEGIES);
+    t.is(events[1].type, STRATEGY_IMPORT);
+    t.is(events[1].data.name, 'new-strategy');
+});
+
+test('should drop neither features nor strategies when neither is imported', async t => {
+    const { stateService, stores } = getSetup();
+
+    const data = {};
+
+    await stateService.import({ data, dropBeforeImport: true });
+
+    const events = await stores.eventStore.getEvents();
+    t.is(events.length, 0);
+});
+
+test('should not accept gibberish', async t => {
+    const { stateService } = getSetup();
+
+    const data1 = {
+        type: 'gibberish',
+        flags: { evil: true },
+    };
+    const data2 = '{somerandomtext/';
+
+    await t.throwsAsync(stateService.import({ data: data1 }));
+
+    await t.throwsAsync(stateService.import({ data: data2 }));
+});
+
+test('should export featureToggles', async t => {
+    const { stateService, stores } = getSetup();
+
+    stores.featureToggleStore.addFeature({ name: 'a-feature' });
+
+    const data = await stateService.export({ featureToggles: true });
+
+    t.is(data.features.length, 1);
+    t.is(data.features[0].name, 'a-feature');
+});
+
+test('should export strategies', async t => {
+    const { stateService, stores } = getSetup();
+
+    stores.strategyStore.addStrategy({ name: 'a-strategy', editable: true });
+
+    const data = await stateService.export({ strategies: true });
+
+    t.is(data.strategies.length, 1);
+    t.is(data.strategies[0].name, 'a-strategy');
+});

package.json

@@ -69,9 +69,12 @@
     "gravatar": "^1.8.0",
     "install": "^0.12.2",
     "joi": "^14.3.1",
+    "js-yaml": "^3.12.2",
     "knex": "^0.16.3",
     "log4js": "^4.0.0",
+    "mime": "^1.4.1",
     "moment": "^2.24.0",
+    "multer": "^1.4.1",
     "parse-database-url": "^0.3.0",
     "pg": "^7.8.1",
     "pkginfo": "^0.4.1",

test/fixtures/fake-event-store.js

@@ -6,7 +6,7 @@ module.exports = () => {
     return {
         store: event => {
             events.push(event);
-            Promise.resolve();
+            return Promise.resolve();
         },
         getEvents: () => Promise.resolve(events),
     };