feat: Allow migration style import (#645)

docs/api/admin/state-api.md

@@ -11,7 +11,7 @@ The api endpoint `/api/admin/state/export` will export feature-toggles and strat
 You can customize the export with queryparameters:
 
 | Parameter | Default | Description |
-| -------------- | ------- | --------------------------------------------------- |
+| --- | --- | --- |
 | format | `json` | Export format, either `json` or `yaml` |
 | download | `false` | If the exported data should be downloaded as a file |
 | featureToggles | `true` | Include feature-toggles in the exported data |
@@ -63,9 +63,12 @@ strategies:
 You can import feature-toggles and strategies by POSTing to the `/api/admin/state/import` endpoint.\
 You can either send the data as JSON in the POST-body or send a `file` parameter with `multipart/form-data` (YAML files are also accepted here).
 
-Specify the `drop` query parameter, if you want the database to be cleaned before import (all strategies and features will be removed).
+**Query Paramters**
 
-> You should never use this in production environments.
+- **drop** - Use this paramter if you want the database to be cleaned before import (all strategies and features will be removed).
+- **keep** - Use this query parameter if you want keep all exiting feature toggle (and strategy) configurations as is (no override), and only insert missing feature toggles from the data provided.
+
+> You should be careful useing the `drop` parameter in production environments.
 
 Success: `202 Accepted`\
 Error: `400 Bad Request`

docs/import-export.md

@@ -30,7 +30,7 @@ unleash.start({...})
 
 If you want the database to be cleaned before import (all strategies and features will be removed), set the `dropBeforeImport` parameter.
 
-> You should never use this in production environments.
+It also possible to not override exiting feature toggles (and strategies) by using the `keepExisting` parameter.
 
 ### API Export
 

lib/options.js

@@ -48,6 +48,7 @@ function defaultOptions() {
         adminAuthentication: process.env.ADMIN_AUTHENTICATION || 'unsecure',
         ui: {},
         importFile: undefined,
+        importKeepExisting: false,
         dropBeforeImport: false,
         getLogger: defaultLogProvider,
         customContextFields: [],

lib/routes/admin-api/state.js

@@ -21,7 +21,7 @@ class StateController extends Controller {
 
     async import(req, res) {
         const userName = extractUser(req);
-        const { drop } = req.query;
+        const { drop, keep } = req.query;
 
         try {
             let data;
@@ -39,6 +39,7 @@ class StateController extends Controller {
                 data,
                 userName,
                 dropBeforeImport: drop,
+                keepExisting: keep,
             });
             res.sendStatus(202);
         } catch (err) {

lib/server-impl.js

@@ -8,7 +8,7 @@ const getApp = require('./app');
 const { startMonitoring } = require('./metrics');
 const { createStores } = require('./db');
 const { createOptions } = require('./options');
-const StateService = require('./state-service');
+const StateService = require('./services/state-service');
 const User = require('./user');
 const permissions = require('./permissions');
 const AuthenticationRequired = require('./authentication-required');
@@ -44,6 +44,7 @@ async function createApp(options) {
             file: config.importFile,
             dropBeforeImport: config.dropBeforeImport,
             userName: 'import',
+            keepExisting: config.importKeepExisting,
         });
     }
 

lib/services/state-schema.js

@@ -0,0 +1,20 @@
+const joi = require('joi');
+const { featureShema } = require('../routes/admin-api/feature-schema');
+const strategySchema = require('../routes/admin-api/strategy-schema');
+
+// TODO: Extract to seperate file
+const stateSchema = joi.object().keys({
+    version: joi.number(),
+    features: joi
+        .array()
+        .optional()
+        .items(featureShema),
+    strategies: joi
+        .array()
+        .optional()
+        .items(strategySchema),
+});
+
+module.exports = {
+    stateSchema,
+};

lib/services/state-service.js

@@ -0,0 +1,138 @@
+const { stateSchema } = require('./state-schema');
+const {
+    FEATURE_IMPORT,
+    DROP_FEATURES,
+    STRATEGY_IMPORT,
+    DROP_STRATEGIES,
+} = require('../event-type');
+
+const {
+    readFile,
+    parseFile,
+    filterExisitng,
+    filterEqual,
+} = require('./state-util');
+
+class StateService {
+    constructor({ stores, getLogger }) {
+        this.eventStore = stores.eventStore;
+        this.toggleStore = stores.featureToggleStore;
+        this.strategyStore = stores.strategyStore;
+        this.logger = getLogger('services/state-service.js');
+    }
+
+    importFile({ file, dropBeforeImport, userName, keepExisting }) {
+        return readFile(file)
+            .then(data => parseFile(file, data))
+            .then(data =>
+                this.import({ data, userName, dropBeforeImport, keepExisting }),
+            );
+    }
+
+    async import({ data, userName, dropBeforeImport, keepExisting }) {
+        const importData = await stateSchema.validateAsync(data);
+
+        if (importData.features) {
+            await this.importFeatures({
+                features: data.features,
+                userName,
+                dropBeforeImport,
+                keepExisting,
+            });
+        }
+
+        if (importData.strategies) {
+            await this.importStrategies({
+                strategies: data.strategies,
+                userName,
+                dropBeforeImport,
+                keepExisting,
+            });
+        }
+    }
+
+    async importFeatures({
+        features,
+        userName,
+        dropBeforeImport,
+        keepExisting,
+    }) {
+        this.logger.info(`Importing ${features.length} feature toggles`);
+        const oldToggles = dropBeforeImport
+            ? []
+            : await this.toggleStore.getFeatures();
+
+        if (dropBeforeImport) {
+            this.logger.info(`Dropping existing feature toggles`);
+            await this.eventStore.store({
+                type: DROP_FEATURES,
+                createdBy: userName,
+                data: { name: 'all-features' },
+            });
+        }
+
+        await Promise.all(
+            features
+                .filter(filterExisitng(keepExisting, oldToggles))
+                .filter(filterEqual(oldToggles))
+                .map(feature =>
+                    this.eventStore.store({
+                        type: FEATURE_IMPORT,
+                        createdBy: userName,
+                        data: feature,
+                    }),
+                ),
+        );
+    }
+
+    async importStrategies({
+        strategies,
+        userName,
+        dropBeforeImport,
+        keepExisting,
+    }) {
+        this.logger.info(`Importing ${strategies.length} strategies`);
+        const oldStrategies = dropBeforeImport
+            ? []
+            : await this.strategyStore.getStrategies();
+
+        if (dropBeforeImport) {
+            this.logger.info(`Dropping existing strategies`);
+            await this.eventStore.store({
+                type: DROP_STRATEGIES,
+                createdBy: userName,
+                data: { name: 'all-strategies' },
+            });
+        }
+
+        await Promise.all(
+            strategies
+                .filter(filterExisitng(keepExisting, oldStrategies))
+                .filter(filterEqual(oldStrategies))
+                .map(strategy =>
+                    this.eventStore.store({
+                        type: STRATEGY_IMPORT,
+                        createdBy: userName,
+                        data: strategy,
+                    }),
+                ),
+        );
+    }
+
+    async export({ includeFeatureToggles = true, includeStrategies = true }) {
+        return Promise.all([
+            includeFeatureToggles
+                ? this.toggleStore.getFeatures()
+                : Promise.resolve(),
+            includeStrategies
+                ? this.strategyStore.getEditableStrategies()
+                : Promise.resolve(),
+        ]).then(([features, strategies]) => ({
+            version: 1,
+            features,
+            strategies,
+        }));
+    }
+}
+
+module.exports = StateService;

lib/services/state-service.test.js

@@ -2,8 +2,8 @@
 
 const test = require('ava');
 
-const store = require('../test/fixtures/store');
-const getLogger = require('../test/fixtures/no-logger');
+const store = require('../../test/fixtures/store');
+const getLogger = require('../../test/fixtures/no-logger');
 
 const StateService = require('./state-service');
 const {
@@ -11,7 +11,7 @@ const {
     DROP_FEATURES,
     STRATEGY_IMPORT,
     DROP_STRATEGIES,
-} = require('./event-type');
+} = require('../event-type');
 
 function getSetup() {
     const stores = store.createStores();
@@ -39,6 +39,54 @@ test('should import a feature', async t => {
     t.is(events[0].data.name, 'new-feature');
 });
 
+test('should not import an existing feature', async t => {
+    const { stateService, stores } = getSetup();
+
+    const data = {
+        features: [
+            {
+                name: 'new-feature',
+                enabled: true,
+                strategies: [{ name: 'default' }],
+            },
+        ],
+    };
+
+    await stores.featureToggleStore.addFeature(data.features[0]);
+
+    await stateService.import({ data, keepExisting: true });
+
+    const events = await stores.eventStore.getEvents();
+    t.is(events.length, 0);
+});
+
+test('should not keep existing feature if drop-before-import', async t => {
+    const { stateService, stores } = getSetup();
+
+    const data = {
+        features: [
+            {
+                name: 'new-feature',
+                enabled: true,
+                strategies: [{ name: 'default' }],
+            },
+        ],
+    };
+
+    await stores.featureToggleStore.addFeature(data.features[0]);
+
+    await stateService.import({
+        data,
+        keepExisting: true,
+        dropBeforeImport: true,
+    });
+
+    const events = await stores.eventStore.getEvents();
+    t.is(events.length, 2);
+    t.is(events[0].type, DROP_FEATURES);
+    t.is(events[1].type, FEATURE_IMPORT);
+});
+
 test('should drop feature before import if specified', async t => {
     const { stateService, stores } = getSetup();
 
@@ -81,6 +129,26 @@ test('should import a strategy', async t => {
     t.is(events[0].data.name, 'new-strategy');
 });
 
+test('should not import an exiting strategy', async t => {
+    const { stateService, stores } = getSetup();
+
+    const data = {
+        strategies: [
+            {
+                name: 'new-strategy',
+                parameters: [],
+            },
+        ],
+    };
+
+    await stores.strategyStore.addStrategy(data.strategies[0]);
+
+    await stateService.import({ data, keepExisting: true });
+
+    const events = await stores.eventStore.getEvents();
+    t.is(events.length, 0);
+});
+
 test('should drop strategies before import if specified', async t => {
     const { stateService, stores } = getSetup();
 

lib/services/state-util.js

@@ -0,0 +1,42 @@
+const fs = require('fs');
+const mime = require('mime');
+const YAML = require('js-yaml');
+
+const readFile = file => {
+    return new Promise((resolve, reject) =>
+        fs.readFile(file, (err, v) => (err ? reject(err) : resolve(v))),
+    );
+};
+
+const parseFile = (file, data) => {
+    return mime.getType(file) === 'text/yaml'
+        ? YAML.safeLoad(data)
+        : JSON.parse(data);
+};
+
+const filterExisitng = (keepExisting, exitingArray) => {
+    return item => {
+        if (keepExisting) {
+            const found = exitingArray.find(t => t.name === item.name);
+            return !found;
+        }
+        return true;
+    };
+};
+
+const filterEqual = exitingArray => {
+    return item => {
+        const toggle = exitingArray.find(t => t.name === item.name);
+        if (toggle) {
+            return JSON.stringify(toggle) !== JSON.stringify(item);
+        }
+        return true;
+    };
+};
+
+module.exports = {
+    readFile,
+    parseFile,
+    filterExisitng,
+    filterEqual,
+};

lib/state-service.js

@@ -1,122 +0,0 @@
-'use strict';
-
-const joi = require('joi');
-const fs = require('fs');
-const mime = require('mime');
-const YAML = require('js-yaml');
-const { featureShema } = require('./routes/admin-api/feature-schema');
-const strategySchema = require('./routes/admin-api/strategy-schema');
-const {
-    FEATURE_IMPORT,
-    DROP_FEATURES,
-    STRATEGY_IMPORT,
-    DROP_STRATEGIES,
-} = require('./event-type');
-
-const dataSchema = joi.object().keys({
-    version: joi.number(),
-    features: joi
-        .array()
-        .optional()
-        .items(featureShema),
-    strategies: joi
-        .array()
-        .optional()
-        .items(strategySchema),
-});
-
-function readFile(file) {
-    return new Promise((resolve, reject) =>
-        fs.readFile(file, (err, v) => (err ? reject(err) : resolve(v))),
-    );
-}
-
-function parseFile(file, data) {
-    return mime.getType(file) === 'text/yaml'
-        ? YAML.safeLoad(data)
-        : JSON.parse(data);
-}
-
-class StateService {
-    constructor(config) {
-        this.config = config;
-        this.logger = config.getLogger('state-service.js');
-    }
-
-    importFile({ file, dropBeforeImport, userName }) {
-        return readFile(file)
-            .then(data => parseFile(file, data))
-            .then(data => this.import({ data, userName, dropBeforeImport }));
-    }
-
-    async import({ data, userName, dropBeforeImport }) {
-        const { eventStore } = this.config.stores;
-
-        const importData = await dataSchema.validateAsync(data);
-
-        if (importData.features) {
-            this.logger.info(
-                `Importing ${importData.features.length} features`,
-            );
-            if (dropBeforeImport) {
-                this.logger.info(`Dropping existing features`);
-                await eventStore.store({
-                    type: DROP_FEATURES,
-                    createdBy: userName,
-                    data: { name: 'all-features' },
-                });
-            }
-            await Promise.all(
-                importData.features.map(feature =>
-                    eventStore.store({
-                        type: FEATURE_IMPORT,
-                        createdBy: userName,
-                        data: feature,
-                    }),
-                ),
-            );
-        }
-
-        if (importData.strategies) {
-            this.logger.info(
-                `Importing ${importData.strategies.length} strategies`,
-            );
-            if (dropBeforeImport) {
-                this.logger.info(`Dropping existing strategies`);
-                await eventStore.store({
-                    type: DROP_STRATEGIES,
-                    createdBy: userName,
-                    data: { name: 'all-strategies' },
-                });
-            }
-            await Promise.all(
-                importData.strategies.map(strategy =>
-                    eventStore.store({
-                        type: STRATEGY_IMPORT,
-                        createdBy: userName,
-                        data: strategy,
-                    }),
-                ),
-            );
-        }
-    }
-
-    async export({ includeFeatureToggles = true, includeStrategies = true }) {
-        const { featureToggleStore, strategyStore } = this.config.stores;
-
-        return Promise.all([
-            includeFeatureToggles
-                ? featureToggleStore.getFeatures()
-                : Promise.resolve(),
-            includeStrategies
-                ? strategyStore.getEditableStrategies()
-                : Promise.resolve(),
-        ]).then(([features, strategies]) => ({
-            version: 1,
-            features,
-            strategies,
-        }));
-    }
-}
-
-module.exports = StateService;

test/e2e/helpers/test-helper.js

@@ -7,7 +7,7 @@ const supertest = require('supertest');
 const { EventEmitter } = require('events');
 const getApp = require('../../../lib/app');
 const getLogger = require('../../fixtures/no-logger');
-const StateService = require('../../../lib/state-service');
+const StateService = require('../../../lib/services/state-service');
 
 const eventBus = new EventEmitter();