Mirrors/unleash.unleash
1
0
Fork 0
mirror of https://github.com/Unleash/unleash.git synced 2025-01-31 00:16:47 +01:00
Code Issues Projects Releases Wiki Activity

feat: Added import & export through stateService #395

Browse Source
This commit is contained in:
Benjamin Ludewig 2019-03-13 19:10:13 +01:00 committed by Ivar Conradi Østhus
parent db3d88e6cd
commit 5e8059dcf1
15 changed files with 484 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
lib/db/event-store.js
View File

@ -1,5 +1,6 @@
'use strict';
const { DROP_FEATURES } = require('../event-type');
const { EventEmitter } = require('events');
const EVENT_COLUMNS = ['id', 'type', 'created_by', 'created_at', 'data'];
@ -14,7 +15,7 @@ class EventStore extends EventEmitter {
return this.db('events')
.insert({
type: event.type,
created_by: event.createdBy, // eslint-disable-line
created_by: event.createdBy, // eslint-disable-line
data: event.data,
})
.then(() => this.emit(event.type, event));
@ -35,6 +36,14 @@ class EventStore extends EventEmitter {
.from('events')
.limit(100)
.whereRaw("data ->> 'name' = ?", [name])
.andWhere(
'id',
'>=',
this.db
.select(this.db.raw('coalesce(max(id),0) as id'))
.from('events')
.where({ type: DROP_FEATURES })
)
.orderBy('created_at', 'desc')
.map(this.rowToEvent);
}

27
lib/db/feature-toggle-store.js
View File

@ -5,6 +5,8 @@ const {
FEATURE_UPDATED,
FEATURE_ARCHIVED,
FEATURE_REVIVED,
FEATURE_IMPORT,
DROP_FEATURES,
} = require('../event-type');
const logger = require('../logger')('client-toggle-store.js');
const NotFoundError = require('../error/notfound-error');
@ -33,6 +35,8 @@ class FeatureToggleStore {
eventStore.on(FEATURE_REVIVED, event =>
this._reviveFeature(event.data)
);
eventStore.on(FEATURE_IMPORT, event => this._importFeature(event.data));
eventStore.on(DROP_FEATURES, () => this._dropFeatures());
}
getFeatures() {
@ -137,6 +141,29 @@ class FeatureToggleStore {
logger.error('Could not archive feature, error was: ', err)
);
}
_importFeature(data) {
data = this.eventDataToRow(data);
return this.db
.raw(`? ON CONFLICT (name) DO ?`, [
this.db(TABLE).insert(data),
this.db
.queryBuilder()
.update(data)
.update('archived', 0),
])
.catch(err =>
logger.error('Could not import feature, error was: ', err)
);
}
_dropFeatures() {
return this.db(TABLE)
.delete()
.catch(err =>
logger.error('Could not drop features, error was: ', err)
);
}
}
module.exports = FeatureToggleStore;

47
lib/db/strategy-store.js
View File

@ -4,6 +4,8 @@ const {
STRATEGY_CREATED,
STRATEGY_DELETED,
STRATEGY_UPDATED,
STRATEGY_IMPORT,
DROP_STRATEGIES,
} = require('../event-type');
const logger = require('../logger')('strategy-store.js');
const NotFoundError = require('../error/notfound-error');
@ -19,14 +21,13 @@ class StrategyStore {
eventStore.on(STRATEGY_UPDATED, event =>
this._updateStrategy(event.data)
);
eventStore.on(STRATEGY_DELETED, event => {
db(TABLE)
.where('name', event.data.name)
.del()
.catch(err => {
logger.error('Could not delete strategy, error was: ', err);
});
});
eventStore.on(STRATEGY_DELETED, event =>
this._deleteStrategy(event.data)
);
eventStore.on(STRATEGY_IMPORT, event =>
this._importStrategy(event.data)
);
eventStore.on(DROP_STRATEGIES, () => this._dropStrategies());
}
getStrategies() {
@ -81,6 +82,36 @@ class StrategyStore {
logger.error('Could not update strategy, error was: ', err)
);
}
_deleteStrategy({ name }) {
return this.db(TABLE)
.where({ name })
.del()
.catch(err => {
logger.error('Could not delete strategy, error was: ', err);
});
}
_importStrategy(data) {
data = this.eventDataToRow(data);
return this.db
.raw(`? ON CONFLICT (name) DO ?`, [
this.db(TABLE).insert(data),
this.db.queryBuilder().update(data).where(`${TABLE}.built_in`, 0), // eslint-disable-line
])
.catch(err =>
logger.error('Could not import strategy, error was: ', err)
);
}
_dropStrategies() {
return this.db(TABLE)
.where({ built_in: 0 }) // eslint-disable-line
.delete()
.catch(err =>
logger.error('Could not drop strategies, error was: ', err)
);
}
}
module.exports = StrategyStore;

14
lib/event-differ.js
View File

@ -4,20 +4,32 @@ const {
STRATEGY_CREATED,
STRATEGY_DELETED,
STRATEGY_UPDATED,
STRATEGY_IMPORT,
DROP_STRATEGIES,
FEATURE_CREATED,
FEATURE_UPDATED,
FEATURE_ARCHIVED,
FEATURE_REVIVED,
FEATURE_IMPORT,
DROP_FEATURES,
} = require('./event-type');
const diff = require('deep-diff').diff;
const strategyTypes = [STRATEGY_CREATED, STRATEGY_DELETED, STRATEGY_UPDATED];
const strategyTypes = [
STRATEGY_CREATED,
STRATEGY_DELETED,
STRATEGY_UPDATED,
STRATEGY_IMPORT,
DROP_STRATEGIES,
];
const featureTypes = [
FEATURE_CREATED,
FEATURE_UPDATED,
FEATURE_ARCHIVED,
FEATURE_REVIVED,
FEATURE_IMPORT,
DROP_FEATURES,
];
function baseTypeFor(event) {

4
lib/event-type.js
View File

@ -5,7 +5,11 @@ module.exports = {
FEATURE_UPDATED: 'feature-updated',
FEATURE_ARCHIVED: 'feature-archived',
FEATURE_REVIVED: 'feature-revived',
FEATURE_IMPORT: 'feature-import',
DROP_FEATURES: 'drop-features',
STRATEGY_CREATED: 'strategy-created',
STRATEGY_DELETED: 'strategy-deleted',
STRATEGY_UPDATED: 'strategy-updated',
STRATEGY_IMPORT: 'strategy-import',
DROP_STRATEGIES: 'drop-strategies',
};

2
lib/options.js
View File

@ -20,6 +20,8 @@ const DEFAULT_OPTIONS = {
sessionAge: THIRTY_DAYS,
adminAuthentication: 'unsecure',
ui: {},
importFile: undefined,
dropBeforeImport: false,
};
module.exports = {

3
lib/routes/admin-api/api-def.json
View File

@ -15,6 +15,9 @@
},
"metrics": {
"uri": "/api/admin/metrics"
},
"state": {
"uri": "/api/admin/state"
}
}
}

2
lib/routes/admin-api/index.js
View File

@ -8,6 +8,7 @@ const StrategyController = require('./strategy');
const MetricsController = require('./metrics');
const UserController = require('./user');
const ConfigController = require('./config');
const StateController = require('./state');
const apiDef = require('./api-def.json');
class AdminApi extends Controller {
@ -22,6 +23,7 @@ class AdminApi extends Controller {
this.app.use('/metrics', new MetricsController(config).router);
this.app.use('/user', new UserController(config).router);
this.app.use('/ui-config', new ConfigController(config).router);
this.app.use('/state', new StateController(config).router);
}
index(req, res) {

80
lib/routes/admin-api/state.js Normal file
View File

@ -0,0 +1,80 @@
'use strict';
const Controller = require('../controller');
const { ADMIN } = require('../../permissions');
const extractUser = require('../../extract-user');
const { handleErrors } = require('./util');
const YAML = require('js-yaml');
const moment = require('moment');
const multer = require('multer');
const upload = multer({ limits: { fileSize: 5242880 } });
class ImportController extends Controller {
constructor(config) {
super(config);
this.fileupload('/import', upload.single('file'), this.import, ADMIN);
this.get('/export', this.export, ADMIN);
}
async import(req, res) {
const userName = extractUser(req);
const { drop } = req.query;
let data;
if (req.file) {
if (req.file.mimetype === 'text/yaml') {
data = YAML.safeLoad(req.file.buffer);
} else {
data = JSON.parse(req.file.buffer);
}
} else {
data = req.body;
}
try {
await this.config.stateService.import({
data,
userName,
dropBeforeImport: drop,
});
res.sendStatus(202);
} catch (err) {
handleErrors(res, err);
}
}
async export(req, res) {
const { format } = req.query;
let strategies = 'strategies' in req.query;
let featureToggles = 'features' in req.query;
if (!strategies && !featureToggles) {
strategies = true;
featureToggles = true;
}
try {
const data = await this.config.stateService.export({
strategies,
featureToggles,
});
const timestamp = moment().format('YYYY-MM-DD_HH-mm-ss');
if (format === 'yaml') {
if ('download' in req.query) {
res.attachment(`export-${timestamp}.yml`);
}
res.type('yaml').send(YAML.safeDump(data));
} else {
if ('download' in req.query) {
res.attachment(`export-${timestamp}.json`);
}
res.json(data);
}
} catch (err) {
handleErrors(res, err);
}
}
}
module.exports = ImportController;

9
lib/routes/controller.js
View File

@ -44,6 +44,15 @@ class Controller {
);
}
fileupload(path, filehandler, handler, permission) {
this.app.post(
path,
checkPermission(this.config, permission),
filehandler,
handler.bind(this)
);
}
use(path, router) {
this.app.use(path, router);
}

24
lib/server-impl.js
View File

@ -10,10 +10,11 @@ const getApp = require('./app');
const { startMonitoring } = require('./metrics');
const { createStores } = require('./db');
const { createOptions } = require('./options');
const StateService = require('./state-service');
const User = require('./user');
const AuthenticationRequired = require('./authentication-required');
function createApp(options) {
async function createApp(options) {
// Database dependecies (statefull)
const stores = createStores(options);
const eventBus = new EventEmitter();
@ -35,12 +36,29 @@ function createApp(options) {
stores.clientMetricsStore
);
const stateService = new StateService(config);
config.stateService = stateService;
if (config.importFile) {
await stateService.importFile({
importFile: config.importFile,
dropBeforeImport: config.dropBeforeImport,
userName: 'importer',
});
}
const server = app.listen({ port: options.port, host: options.host }, () =>
logger.info(`Unleash started on port ${server.address().port}`)
);
return new Promise((resolve, reject) => {
server.on('listening', () => resolve({ app, server, eventBus }));
return await new Promise((resolve, reject) => {
server.on('listening', () =>
resolve({
app,
server,
eventBus,
stateService,
})
);
server.on('error', reject);
});
}

122
lib/state-service.js Normal file
View File

@ -0,0 +1,122 @@
'use strict';
const joi = require('joi');
const fs = require('fs');
const mime = require('mime');
const { featureShema } = require('./routes/admin-api/feature-schema');
const strategySchema = require('./routes/admin-api/strategy-schema');
const getLogger = require('./logger');
const yaml = require('js-yaml');
const {
FEATURE_IMPORT,
DROP_FEATURES,
STRATEGY_IMPORT,
DROP_STRATEGIES,
} = require('./event-type');
const logger = getLogger('state-service.js');
const dataSchema = joi.object().keys({
features: joi
.array()
.optional()
.items(featureShema),
strategies: joi
.array()
.optional()
.items(strategySchema),
});
class StateService {
constructor(config) {
this.config = config;
}
async importFile({ importFile, dropBeforeImport, userName }) {
let data = await new Promise((resolve, reject) =>
fs.readFile(importFile, (err, v) =>
err ? reject(err) : resolve(v)
)
);
if (mime.lookup(importFile) === 'text/yaml') {
data = yaml.safeLoad(data);
}
await this.import({
data,
dropBeforeImport,
userName,
});
}
async import({ data, userName, dropBeforeImport }) {
const { eventStore } = this.config.stores;
if (typeof data === 'string') {
data = JSON.parse(data);
}
data = await joi.validate(data, dataSchema);
if (data.features) {
logger.info(`Importing ${data.features.length} features`);
if (dropBeforeImport) {
logger.info(`Dropping existing features`);
await eventStore.store({
type: DROP_FEATURES,
createdBy: userName,
data: { name: 'all-features' },
});
}
for (const feature of data.features) {
await eventStore.store({
type: FEATURE_IMPORT,
createdBy: userName,
data: feature,
});
}
}
if (data.strategies) {
logger.info(`Importing ${data.strategies.length} strategies`);
if (dropBeforeImport) {
logger.info(`Dropping existing strategies`);
await eventStore.store({
type: DROP_STRATEGIES,
createdBy: userName,
data: { name: 'all-strategies' },
});
}
for (const strategy of data.strategies) {
await eventStore.store({
type: STRATEGY_IMPORT,
createdBy: userName,
data: strategy,
});
}
}
}
async export({ strategies, featureToggles }) {
const { featureToggleStore, strategyStore } = this.config.stores;
const result = {};
if (featureToggles) {
result.features = await featureToggleStore.getFeatures();
}
if (strategies) {
result.strategies = (await strategyStore.getStrategies())
.filter(strat => strat.editable)
.map(strat => {
strat = Object.assign({}, strat);
delete strat.editable;
return strat;
});
}
return result;
}
}
module.exports = StateService;

148
lib/state-service.test.js Normal file
View File

@ -0,0 +1,148 @@
'use strict';
const test = require('ava');
const store = require('./../test/fixtures/store');
const StateService = require('./state-service');
const {
FEATURE_IMPORT,
DROP_FEATURES,
STRATEGY_IMPORT,
DROP_STRATEGIES,
} = require('./event-type');
function getSetup() {
const stores = store.createStores();
return { stateService: new StateService({ stores }), stores };
}
test('should import a feature', async t => {
const { stateService, stores } = getSetup();
const data = {
features: [
{
name: 'new-feature',
enabled: true,
strategies: [{ name: 'default' }],
},
],
};
await stateService.import({ data });
const events = await stores.eventStore.getEvents();
t.is(events.length, 1);
t.is(events[0].type, FEATURE_IMPORT);
t.is(events[0].data.name, 'new-feature');
});
test('should drop feature before import if specified', async t => {
const { stateService, stores } = getSetup();
const data = {
features: [
{
name: 'new-feature',
enabled: true,
strategies: [{ name: 'default' }],
},
],
};
await stateService.import({ data, dropBeforeImport: true });
const events = await stores.eventStore.getEvents();
t.is(events.length, 2);
t.is(events[0].type, DROP_FEATURES);
t.is(events[1].type, FEATURE_IMPORT);
t.is(events[1].data.name, 'new-feature');
});
test('should import a strategy', async t => {
const { stateService, stores } = getSetup();
const data = {
strategies: [
{
name: 'new-strategy',
parameters: [],
},
],
};
await stateService.import({ data });
const events = await stores.eventStore.getEvents();
t.is(events.length, 1);
t.is(events[0].type, STRATEGY_IMPORT);
t.is(events[0].data.name, 'new-strategy');
});
test('should drop strategies before import if specified', async t => {
const { stateService, stores } = getSetup();
const data = {
strategies: [
{
name: 'new-strategy',
parameters: [],
},
],
};
await stateService.import({ data, dropBeforeImport: true });
const events = await stores.eventStore.getEvents();
t.is(events.length, 2);
t.is(events[0].type, DROP_STRATEGIES);
t.is(events[1].type, STRATEGY_IMPORT);
t.is(events[1].data.name, 'new-strategy');
});
test('should drop neither features nor strategies when neither is imported', async t => {
const { stateService, stores } = getSetup();
const data = {};
await stateService.import({ data, dropBeforeImport: true });
const events = await stores.eventStore.getEvents();
t.is(events.length, 0);
});
test('should not accept gibberish', async t => {
const { stateService } = getSetup();
const data1 = {
type: 'gibberish',
flags: { evil: true },
};
const data2 = '{somerandomtext/';
await t.throwsAsync(stateService.import({ data: data1 }));
await t.throwsAsync(stateService.import({ data: data2 }));
});
test('should export featureToggles', async t => {
const { stateService, stores } = getSetup();
stores.featureToggleStore.addFeature({ name: 'a-feature' });
const data = await stateService.export({ featureToggles: true });
t.is(data.features.length, 1);
t.is(data.features[0].name, 'a-feature');
});
test('should export strategies', async t => {
const { stateService, stores } = getSetup();
stores.strategyStore.addStrategy({ name: 'a-strategy', editable: true });
const data = await stateService.export({ strategies: true });
t.is(data.strategies.length, 1);
t.is(data.strategies[0].name, 'a-strategy');
});

3
package.json
View File

@ -69,9 +69,12 @@
"gravatar": "^1.8.0",
"install": "^0.12.2",
"joi": "^14.3.1",
"js-yaml": "^3.12.2",
"knex": "^0.16.3",
"log4js": "^4.0.0",
"mime": "^1.4.1",
"moment": "^2.24.0",
"multer": "^1.4.1",
"parse-database-url": "^0.3.0",
"pg": "^7.8.1",
"pkginfo": "^0.4.1",

2
test/fixtures/fake-event-store.js vendored
View File

@ -6,7 +6,7 @@ module.exports = () => {
return {
store: event => {
events.push(event);
Promise.resolve();
return Promise.resolve();
},
getEvents: () => Promise.resolve(events),
};