1
0
mirror of https://github.com/Unleash/unleash.git synced 2025-05-31 01:16:01 +02:00

feat: remove old metrics service

This commit is contained in:
Ivar Conradi Østhus 2021-12-09 21:02:58 +01:00
parent 6334486a7c
commit 4a9939ccb1
48 changed files with 145 additions and 1507 deletions

View File

@ -39,6 +39,12 @@ Object {
},
"enableOAS": false,
"enterpriseVersion": undefined,
"eventBus": EventEmitter {
"_events": Object {},
"_eventsCount": 0,
"_maxListeners": undefined,
Symbol(kCapture): false,
},
"eventHook": undefined,
"experimental": Object {},
"getLogger": [Function],

View File

@ -1,6 +1,5 @@
import { publicFolder } from 'unleash-frontend';
import fs from 'fs';
import EventEmitter from 'events';
import express, { Application, RequestHandler } from 'express';
import cors from 'cors';
import compression from 'compression';
@ -29,7 +28,6 @@ export default function getApp(
config: IUnleashConfig,
stores: IUnleashStores,
services: IUnleashServices,
eventBus?: EventEmitter,
unleashSession?: RequestHandler,
): Application {
const app = express();
@ -47,8 +45,8 @@ export default function getApp(
app.set('port', config.server.port);
app.locals.baseUriPath = baseUriPath;
if (config.server.serverMetrics && eventBus) {
app.use(responseTimeMetrics(eventBus));
if (config.server.serverMetrics && config.eventBus) {
app.use(responseTimeMetrics(config.eventBus));
}
app.use(requestLogger(config));

View File

@ -20,6 +20,7 @@ import { getDefaultLogProvider, LogLevel, validateLogProvider } from './logger';
import { defaultCustomAuthDenyAll } from './default-custom-auth-deny-all';
import { formatBaseUri } from './util/format-base-uri';
import { minutesToMilliseconds, secondsToMilliseconds } from 'date-fns';
import EventEmitter from 'events';
const safeToUpper = (s: string) => (s ? s.toUpperCase() : s);
@ -275,6 +276,7 @@ export function createConfig(options: IUnleashOptions): IUnleashConfig {
preRouterHook: options.preRouterHook,
eventHook: options.eventHook,
enterpriseVersion: options.enterpriseVersion,
eventBus: new EventEmitter(),
};
}

View File

@ -1,108 +0,0 @@
import { Knex } from 'knex';
import { Logger, LogProvider } from '../logger';
import { IClientMetric } from '../types/stores/client-metrics-db';
import { minutesToMilliseconds } from 'date-fns';
const METRICS_COLUMNS = ['id', 'created_at', 'metrics'];
const TABLE = 'client_metrics';
const mapRow = (row) => ({
id: row.id,
createdAt: row.created_at,
metrics: row.metrics,
});
export class ClientMetricsDb {
private readonly logger: Logger;
private readonly timer: NodeJS.Timeout;
constructor(private db: Knex, getLogger: LogProvider) {
this.logger = getLogger('client-metrics-db.js');
// Clear old metrics regularly
const clearer = () => this.removeMetricsOlderThanOneHour();
setTimeout(clearer, 10).unref();
this.timer = setInterval(clearer, minutesToMilliseconds(1)).unref();
}
async removeMetricsOlderThanOneHour(): Promise<void> {
try {
const rows = await this.db(TABLE)
.whereRaw("created_at < now() - interval '1 hour'")
.del();
if (rows > 0) {
this.logger.debug(`Deleted ${rows} metrics`);
}
} catch (e) {
this.logger.warn(`Error when deleting metrics ${e}`);
}
}
async delete(id: number): Promise<void> {
await this.db(TABLE).where({ id }).del();
}
async deleteAll(): Promise<void> {
await this.db(TABLE).del();
}
// Insert new client metrics
async insert(metrics: IClientMetric): Promise<void> {
return this.db(TABLE).insert({ metrics });
}
// Used at startup to load all metrics last week into memory!
async getMetricsLastHour(): Promise<IClientMetric[]> {
try {
const result = await this.db
.select(METRICS_COLUMNS)
.from(TABLE)
.limit(2000)
.whereRaw("created_at > now() - interval '1 hour'")
.orderBy('created_at', 'asc');
return result.map(mapRow);
} catch (e) {
this.logger.warn(`error when getting metrics last hour ${e}`);
}
return [];
}
async get(id: number): Promise<IClientMetric> {
const result = await this.db
.select(METRICS_COLUMNS)
.from(TABLE)
.where({ id })
.first();
return mapRow(result);
}
async exists(id: number): Promise<boolean> {
const result = await this.db.raw(
`SELECT EXISTS (SELECT 1 FROM ${TABLE} WHERE id = ?) AS present`,
[id],
);
const { present } = result.rows[0];
return present;
}
// Used to poll for new metrics
async getNewMetrics(lastKnownId: number): Promise<IClientMetric[]> {
try {
const res = await this.db
.select(METRICS_COLUMNS)
.from(TABLE)
.limit(1000)
.where('id', '>', lastKnownId)
.orderBy('created_at', 'asc');
return res.map(mapRow);
} catch (e) {
this.logger.warn(`error when getting new metrics ${e}`);
}
return [];
}
destroy(): void {
clearInterval(this.timer);
}
}

View File

@ -156,6 +156,18 @@ export class ClientMetricsStoreV2 implements IClientMetricsStoreV2 {
.orderBy('app_name');
}
async getSeenTogglesForApp(
appName: string,
hoursBack: number = 24,
): Promise<string[]> {
return this.db<ClientMetricsEnvTable>(TABLE)
.distinct()
.where({ app_name: appName })
.andWhereRaw(`timestamp >= NOW() - INTERVAL '${hoursBack} hours'`)
.pluck('feature_name')
.orderBy('feature_name');
}
async clearMetrics(hoursAgo: number): Promise<void> {
return this.db<ClientMetricsEnvTable>(TABLE)
.whereRaw(`timestamp <= NOW() - INTERVAL '${hoursAgo} hours'`)

View File

@ -1,96 +0,0 @@
import EventEmitter from 'events';
import { ClientMetricsStore } from './client-metrics-store';
import getLogger from '../../test/fixtures/no-logger';
function getMockDb() {
const list = [
{ id: 4, metrics: { appName: 'test' } },
{ id: 3, metrics: { appName: 'test' } },
{ id: 2, metrics: { appName: 'test' } },
];
return {
getMetricsLastHour() {
return Promise.resolve([{ id: 1, metrics: { appName: 'test' } }]);
},
getNewMetrics() {
return Promise.resolve([list.pop() || { id: 0 }]);
},
destroy() {
// noop
},
};
}
test('should call database on startup', (done) => {
jest.useFakeTimers('modern');
const mock = getMockDb();
const ee = new EventEmitter();
const store = new ClientMetricsStore(mock as any, ee, getLogger);
jest.runAllTicks();
expect.assertions(2);
store.on('metrics', (metrics) => {
expect(store.highestIdSeen).toBe(1);
expect(metrics.appName).toBe('test');
store.destroy();
done();
});
});
test('should start poller even if initial database fetch fails', (done) => {
jest.useFakeTimers('modern');
getLogger.setMuteError(true);
const mock = getMockDb();
mock.getMetricsLastHour = () => Promise.reject(new Error('oops'));
const ee = new EventEmitter();
const store = new ClientMetricsStore(mock as any, ee, getLogger, 100);
jest.runAllTicks();
const metrics = [];
store.on('metrics', (m) => metrics.push(m));
store.on('ready', () => {
jest.useFakeTimers('modern');
expect(metrics).toHaveLength(0);
jest.advanceTimersByTime(300);
jest.useRealTimers();
process.nextTick(() => {
expect(metrics).toHaveLength(3);
expect(store.highestIdSeen).toBe(4);
store.destroy();
done();
});
});
getLogger.setMuteError(false);
});
test('should poll for updates', (done) => {
jest.useFakeTimers('modern');
const mock = getMockDb();
const ee = new EventEmitter();
const store = new ClientMetricsStore(mock as any, ee, getLogger, 100);
jest.runAllTicks();
const metrics = [];
store.on('metrics', (m) => metrics.push(m));
expect(metrics).toHaveLength(0);
store.on('ready', () => {
jest.useFakeTimers('modern');
expect(metrics).toHaveLength(1);
jest.advanceTimersByTime(300);
jest.useRealTimers();
process.nextTick(() => {
expect(metrics).toHaveLength(4);
expect(store.highestIdSeen).toBe(4);
store.destroy();
done();
});
});
});

View File

@ -1,111 +0,0 @@
import EventEmitter from 'events';
import { Logger, LogProvider } from '../logger';
import metricsHelper from '../util/metrics-helper';
import { DB_TIME } from '../metric-events';
import { ClientMetricsDb } from './client-metrics-db';
import { IClientMetric } from '../types/stores/client-metrics-db';
import { IClientMetricsStore } from '../types/stores/client-metrics-store';
import { secondsToMilliseconds } from 'date-fns';
export class ClientMetricsStore
extends EventEmitter
implements IClientMetricsStore
{
private logger: Logger;
highestIdSeen = 0;
private startTimer: Function;
private timer: NodeJS.Timeout;
constructor(
private metricsDb: ClientMetricsDb,
eventBus: EventEmitter,
getLogger: LogProvider,
pollInterval = secondsToMilliseconds(10),
) {
super();
this.logger = getLogger('client-metrics-store.ts.js');
this.metricsDb = metricsDb;
this.highestIdSeen = 0;
this.startTimer = (action) =>
metricsHelper.wrapTimer(eventBus, DB_TIME, {
store: 'metrics',
action,
});
process.nextTick(async () => {
await this._init(pollInterval);
});
}
async _init(pollInterval: number): Promise<void> {
try {
const metrics = await this.metricsDb.getMetricsLastHour();
this._emitMetrics(metrics);
} catch (err) {
this.logger.error('Error fetching metrics last hour', err);
}
this._startPoller(pollInterval);
this.emit('ready');
}
_startPoller(pollInterval: number): void {
this.timer = setInterval(() => this._fetchNewAndEmit(), pollInterval);
this.timer.unref();
}
_fetchNewAndEmit(): void {
this.metricsDb
.getNewMetrics(this.highestIdSeen)
.then((metrics) => this._emitMetrics(metrics));
}
_emitMetrics(metrics: IClientMetric[]): void {
if (metrics && metrics.length > 0) {
this.highestIdSeen = metrics[metrics.length - 1].id;
metrics.forEach((m) => this.emit('metrics', m.metrics));
}
}
/**
* Insert client metrics. In the future we will isolate "appName" and "environment"
* in separate columns in the database to make it easier to query the data.
*
* @param metrics sent from the client SDK.
*/
async insert(metrics: IClientMetric): Promise<void> {
const stopTimer = this.startTimer('insert');
await this.metricsDb.insert(metrics);
stopTimer();
}
destroy(): void {
clearInterval(this.timer);
this.metricsDb.destroy();
}
async delete(key: number): Promise<void> {
await this.metricsDb.delete(key);
}
async deleteAll(): Promise<void> {
await this.metricsDb.deleteAll();
}
async exists(key: number): Promise<boolean> {
return this.metricsDb.exists(key);
}
async get(key: number): Promise<IClientMetric> {
return this.metricsDb.get(key);
}
async getAll(): Promise<IClientMetric[]> {
return this.metricsDb.getMetricsLastHour();
}
}

View File

@ -8,8 +8,6 @@ import FeatureToggleStore from './feature-toggle-store';
import FeatureTypeStore from './feature-type-store';
import StrategyStore from './strategy-store';
import ClientInstanceStore from './client-instance-store';
import { ClientMetricsDb } from './client-metrics-db';
import { ClientMetricsStore } from './client-metrics-store';
import ClientApplicationsStore from './client-applications-store';
import ContextFieldStore from './context-field-store';
import SettingStore from './setting-store';
@ -33,12 +31,10 @@ import UserSplashStore from './user-splash-store';
export const createStores = (
config: IUnleashConfig,
eventBus: EventEmitter,
db: Knex,
): IUnleashStores => {
const { getLogger } = config;
const { getLogger, eventBus } = config;
const eventStore = new EventStore(db, getLogger);
const clientMetricsDb = new ClientMetricsDb(db, getLogger);
return {
eventStore,
@ -51,11 +47,6 @@ export const createStores = (
getLogger,
),
clientInstanceStore: new ClientInstanceStore(db, eventBus, getLogger),
clientMetricsStore: new ClientMetricsStore(
clientMetricsDb,
eventBus,
getLogger,
),
clientMetricsStoreV2: new ClientMetricsStoreV2(db, getLogger),
contextFieldStore: new ContextFieldStore(db, getLogger),
settingStore: new SettingStore(db, getLogger),

View File

@ -2,7 +2,7 @@ import { register } from 'prom-client';
import EventEmitter from 'events';
import { createTestConfig } from '../test/config/test-config';
import { REQUEST_TIME, DB_TIME } from './metric-events';
import { FEATURE_UPDATED } from './types/events';
import { CLIENT_METRICS, FEATURE_UPDATED } from './types/events';
import { createMetricsMonitor } from './metrics';
import createStores from '../test/fixtures/store';
@ -64,7 +64,7 @@ test('should collect metrics for updated toggles', async () => {
});
test('should collect metrics for client metric reports', async () => {
stores.clientMetricsStore.emit('metrics', {
eventBus.emit(CLIENT_METRICS, {
bucket: {
toggles: {
TestToggle: {

View File

@ -11,6 +11,7 @@ import {
FEATURE_STRATEGY_REMOVE,
FEATURE_STRATEGY_UPDATE,
FEATURE_UPDATED,
CLIENT_METRICS,
} from './types/events';
import { IUnleashConfig } from './types/option';
import { IUnleashStores } from './types/stores';
@ -38,13 +39,8 @@ export default class MetricsMonitor {
return;
}
const {
eventStore,
clientMetricsStore,
featureToggleStore,
userStore,
projectStore,
} = stores;
const { eventStore, featureToggleStore, userStore, projectStore } =
stores;
client.collectDefaultMetrics();
@ -148,7 +144,7 @@ export default class MetricsMonitor {
featureToggleUpdateTotal.labels(featureName).inc();
});
clientMetricsStore.on('metrics', (m) => {
eventBus.on(CLIENT_METRICS, (m) => {
// eslint-disable-next-line no-restricted-syntax
for (const entry of Object.entries(m.bucket.toggles)) {
featureToggleUsageTotal

View File

@ -1,5 +1,4 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import { createServices } from '../services';
import { createTestConfig } from '../../test/config/test-config';
@ -9,8 +8,6 @@ import getApp from '../app';
import User from '../types/user';
import sessionDb from './session-db';
const eventBus = new EventEmitter();
function getSetup(preRouterHook) {
const base = `/random${Math.round(Math.random() * 1000)}`;
const config = createTestConfig({
@ -26,7 +23,7 @@ function getSetup(preRouterHook) {
const stores = createStores();
const services = createServices(stores, config);
const unleashSession = sessionDb(config, undefined);
const app = getApp(config, stores, services, eventBus, unleashSession);
const app = getApp(config, stores, services, unleashSession);
return {
base,

View File

@ -30,7 +30,7 @@ class ClientMetricsController extends Controller {
const data = await this.metrics.getClientMetricsForToggle(name);
res.json({
version: 1,
maturity: 'experimental',
maturity: 'stable',
data,
});
}
@ -40,7 +40,7 @@ class ClientMetricsController extends Controller {
const data = await this.metrics.getFeatureToggleMetricsSummary(name);
res.json({
version: 1,
maturity: 'experimental',
maturity: 'stable',
...data,
});
}

View File

@ -1,13 +1,10 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import { createTestConfig } from '../../../test/config/test-config';
import createStores from '../../../test/fixtures/store';
import getApp from '../../app';
import { createServices } from '../../services';
const eventBus = new EventEmitter();
const uiConfig = {
headerBackground: 'red',
slogan: 'hello',
@ -22,7 +19,7 @@ function getSetup() {
const stores = createStores();
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return {
base,

View File

@ -1,13 +1,10 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import { createTestConfig } from '../../../test/config/test-config';
import createStores from '../../../test/fixtures/store';
import { createServices } from '../../services';
import permissions from '../../../test/fixtures/permissions';
import getApp from '../../app';
const eventBus = new EventEmitter();
function getSetup() {
const base = `/random${Math.round(Math.random() * 1000)}`;
const perms = permissions();
@ -18,7 +15,7 @@ function getSetup() {
const stores = createStores();
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return {
base,

View File

@ -1,13 +1,10 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import { createTestConfig } from '../../../test/config/test-config';
import createStores from '../../../test/fixtures/store';
import { createServices } from '../../services';
import permissions from '../../../test/fixtures/permissions';
import getApp from '../../app';
const eventBus = new EventEmitter();
function getSetup() {
const base = `/random${Math.round(Math.random() * 1000)}`;
const stores = createStores();
@ -18,7 +15,7 @@ function getSetup() {
});
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return {
base,

View File

@ -1,5 +1,4 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import { createServices } from '../../services';
import { createTestConfig } from '../../../test/config/test-config';
@ -7,8 +6,6 @@ import createStores from '../../../test/fixtures/store';
import getApp from '../../app';
const eventBus = new EventEmitter();
function getSetup() {
const base = `/random${Math.round(Math.random() * 1000)}`;
const stores = createStores();
@ -16,7 +13,7 @@ function getSetup() {
server: { baseUriPath: base },
});
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return { base, eventStore: stores.eventStore, request: supertest(app) };
}

View File

@ -1,13 +1,10 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import createStores from '../../../test/fixtures/store';
import permissions from '../../../test/fixtures/permissions';
import getApp from '../../app';
import { createTestConfig } from '../../../test/config/test-config';
import { createServices } from '../../services';
const eventBus = new EventEmitter();
function getSetup() {
const stores = createStores();
const perms = permissions();
@ -15,7 +12,7 @@ function getSetup() {
preRouterHook: perms.hook,
});
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return {
request: supertest(app),
@ -44,77 +41,15 @@ afterEach(() => {
destroy();
});
test('should return seen toggles even when there is nothing', () => {
expect.assertions(1);
return request
.get('/api/admin/metrics/seen-toggles')
.expect(200)
.expect((res) => {
expect(res.body.length === 0).toBe(true);
});
test('/api/admin/metrics/seen-toggles is deprecated', () => {
return request.get('/api/admin/metrics/seen-toggles').expect(410);
});
test('should return list of seen-toggles per app', () => {
expect.assertions(3);
const appName = 'asd!23';
stores.clientMetricsStore.emit('metrics', {
appName,
instanceId: 'instanceId',
bucket: {
start: new Date(),
stop: new Date(),
toggles: {
toggleX: { yes: 123, no: 0 },
toggleY: { yes: 123, no: 0 },
},
},
});
return request
.get('/api/admin/metrics/seen-toggles')
.expect(200)
.expect((res) => {
const seenAppsWithToggles = res.body;
expect(seenAppsWithToggles.length === 1).toBe(true);
expect(seenAppsWithToggles[0].appName === appName).toBe(true);
expect(seenAppsWithToggles[0].seenToggles.length === 2).toBe(true);
});
});
test('should return feature-toggles metrics even when there is nothing', () => {
expect.assertions(0);
return request.get('/api/admin/metrics/feature-toggles').expect(200);
});
test('should return metrics for all toggles', () => {
expect.assertions(2);
const appName = 'asd!23';
stores.clientMetricsStore.emit('metrics', {
appName,
instanceId: 'instanceId',
bucket: {
start: new Date(),
stop: new Date(),
toggles: {
toggleX: { yes: 123, no: 0 },
toggleY: { yes: 123, no: 0 },
},
},
});
return request
.get('/api/admin/metrics/feature-toggles')
.expect(200)
.expect((res) => {
const metrics = res.body;
expect(metrics.lastHour !== undefined).toBe(true);
expect(metrics.lastMinute !== undefined).toBe(true);
});
test('/api/admin/metrics/feature-toggles is deprecated', () => {
return request.get('/api/admin/metrics/feature-toggles').expect(410);
});
test('should return empty list of client applications', () => {
expect.assertions(1);
return request
.get('/api/admin/metrics/applications')
.expect(200)

View File

@ -22,10 +22,13 @@ class MetricsController extends Controller {
this.metrics = clientMetricsService;
this.get('/seen-toggles', this.getSeenToggles);
this.get('/seen-apps', this.getSeenApps);
this.get('/feature-toggles', this.getFeatureToggles);
this.get('/feature-toggles/:name', this.getFeatureToggle);
// deprecated routes
this.get('/seen-toggles', this.deprecated);
this.get('/seen-apps', this.deprecated);
this.get('/feature-toggles', this.deprecated);
this.get('/feature-toggles/:name', this.deprecated);
// in use
this.post(
'/applications/:appName',
this.createApplication,
@ -40,29 +43,11 @@ class MetricsController extends Controller {
this.get('/applications/:appName', this.getApplication);
}
async getSeenToggles(req: Request, res: Response): Promise<void> {
const seenAppToggles = await this.metrics.getAppsWithToggles();
res.json(seenAppToggles);
}
async getSeenApps(req: Request, res: Response): Promise<void> {
const seenApps = await this.metrics.getSeenApps();
res.json(seenApps);
}
async getFeatureToggles(req: Request, res: Response): Promise<void> {
const toggles = await this.metrics.getTogglesMetrics();
res.json(toggles);
}
async getFeatureToggle(req: Request, res: Response): Promise<void> {
const { name } = req.params;
const data = await this.metrics.getTogglesMetrics();
const lastHour = data.lastHour[name] || {};
const lastMinute = data.lastMinute[name] || {};
res.json({
lastHour,
lastMinute,
async deprecated(req: Request, res: Response): Promise<void> {
res.status(410).json({
lastHour: {},
lastMinute: {},
maturity: 'deprecated',
});
}
@ -95,4 +80,3 @@ class MetricsController extends Controller {
}
}
export default MetricsController;
module.exports = MetricsController;

View File

@ -1,12 +1,10 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import { createTestConfig } from '../../../test/config/test-config';
import createStores from '../../../test/fixtures/store';
import permissions from '../../../test/fixtures/permissions';
import getApp from '../../app';
import { createServices } from '../../services';
const eventBus = new EventEmitter();
let destroy;
function getSetup() {
@ -18,7 +16,7 @@ function getSetup() {
preRouterHook: perms.hook,
});
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
destroy = () => {
services.versionService.destroy();

View File

@ -1,13 +1,10 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import createStores from '../../../test/fixtures/store';
import permissions from '../../../test/fixtures/permissions';
import getApp from '../../app';
import { createTestConfig } from '../../../test/config/test-config';
import { createServices } from '../../services';
const eventBus = new EventEmitter();
function getSetup() {
const base = `/random${Math.round(Math.random() * 1000)}`;
const stores = createStores();
@ -17,7 +14,7 @@ function getSetup() {
preRouterHook: perms.hook,
});
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return {
base,

View File

@ -1,5 +1,4 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import { createServices } from '../../services';
import { createTestConfig } from '../../../test/config/test-config';
@ -7,8 +6,6 @@ import createStores from '../../../test/fixtures/store';
import getApp from '../../app';
import User from '../../types/user';
const eventBus = new EventEmitter();
const currentUser = new User({ id: 1337, email: 'test@mail.com' });
async function getSetup() {
@ -26,7 +23,7 @@ async function getSetup() {
server: { baseUriPath: base },
});
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return {
base,
userStore: stores.userStore,

View File

@ -1,20 +1,17 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import { createServices } from '../services';
import { createTestConfig } from '../../test/config/test-config';
import createStores from '../../test/fixtures/store';
import getApp from '../app';
const eventBus = new EventEmitter();
test('should enable prometheus', async () => {
expect.assertions(0);
const stores = createStores();
const config = createTestConfig();
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
const request = supertest(app);

View File

@ -1,5 +1,4 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import createStores from '../../../test/fixtures/store';
import getLogger from '../../../test/fixtures/no-logger';
import getApp from '../../app';
@ -8,8 +7,6 @@ import FeatureController from './feature';
import { createTestConfig } from '../../../test/config/test-config';
import { secondsToMilliseconds } from 'date-fns';
const eventBus = new EventEmitter();
function getSetup() {
const base = `/random${Math.round(Math.random() * 1000)}`;
const stores = createStores();
@ -18,7 +15,7 @@ function getSetup() {
});
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return {
base,

View File

@ -1,5 +1,4 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import createStores from '../../../test/fixtures/store';
import getApp from '../../app';
import { createTestConfig } from '../../../test/config/test-config';
@ -8,14 +7,12 @@ import { createServices } from '../../services';
import { IUnleashStores } from '../../types';
import { IUnleashOptions } from '../../server-impl';
const eventBus = new EventEmitter();
function getSetup(opts?: IUnleashOptions) {
const stores = createStores();
const config = createTestConfig(opts);
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return {
request: supertest(app),

View File

@ -1,18 +1,15 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import { createTestConfig } from '../../../test/config/test-config';
import createStores from '../../../test/fixtures/store';
import getLogger from '../../../test/fixtures/no-logger';
import getApp from '../../app';
import { createServices } from '../../services';
const eventBus = new EventEmitter();
function getSetup() {
const stores = createStores();
const config = createTestConfig();
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return {
request: supertest(app),

View File

@ -1,5 +1,4 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import { createServices } from '../services';
import { createTestConfig } from '../../test/config/test-config';
@ -8,13 +7,11 @@ import getLogger from '../../test/fixtures/no-logger';
import getApp from '../app';
import { IUnleashStores } from '../types';
const eventBus = new EventEmitter();
function getSetup() {
const stores = createStores();
const config = createTestConfig();
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return {
request: supertest(app),
@ -57,7 +54,7 @@ test('should give 500 when db is failing', () => {
// @ts-ignore
const services = createServices(failingStores, config);
// @ts-ignore
const app = getApp(createTestConfig(), failingStores, services, eventBus);
const app = getApp(createTestConfig(), failingStores, services);
request = supertest(app);
getLogger.setMuteError(true);
expect.assertions(2);

View File

@ -1,12 +1,9 @@
import supertest from 'supertest';
import { EventEmitter } from 'events';
import { createTestConfig } from '../../test/config/test-config';
import createStores from '../../test/fixtures/store';
import getApp from '../app';
import { createServices } from '../services';
const eventBus = new EventEmitter();
function getSetup() {
const base = `/random${Math.round(Math.random() * 1000)}`;
const stores = createStores();
@ -14,7 +11,7 @@ function getSetup() {
server: { baseUriPath: base },
});
const services = createServices(stores, config);
const app = getApp(config, stores, services, eventBus);
const app = getApp(config, stores, services);
return {
base,

View File

@ -1,4 +1,3 @@
import EventEmitter from 'events';
import stoppable, { StoppableServer } from 'stoppable';
import { promisify } from 'util';
import version from './util/version';
@ -33,9 +32,8 @@ async function createApp(
// Database dependencies (stateful)
const logger = config.getLogger('server-impl.js');
const serverVersion = version;
const eventBus = new EventEmitter();
const db = createDb(config);
const stores = createStores(config, eventBus, db);
const stores = createStores(config, db);
const services = createServices(stores, config);
const metricsMonitor = createMetricsMonitor();
@ -49,7 +47,6 @@ async function createApp(
}
metricsMonitor.stopMonitoring();
stores.clientInstanceStore.destroy();
stores.clientMetricsStore.destroy();
services.clientMetricsServiceV2.destroy();
await db.destroy();
};
@ -59,15 +56,21 @@ async function createApp(
// eslint-disable-next-line no-param-reassign
config.server.secret = secret;
}
const app = getApp(config, stores, services, eventBus, unleashSession);
const app = getApp(config, stores, services, unleashSession);
if (typeof config.eventHook === 'function') {
addEventHook(config.eventHook, stores.eventStore);
}
metricsMonitor.startMonitoring(config, stores, serverVersion, eventBus, db);
metricsMonitor.startMonitoring(
config,
stores,
serverVersion,
config.eventBus,
db,
);
const unleash: Omit<IUnleash, 'stop'> = {
stores,
eventBus,
eventBus: config.eventBus,
services,
app,
config,

View File

@ -2,16 +2,7 @@ import EventEmitter from 'events';
import ClientMetricsService from './index';
import getLogger from '../../../test/fixtures/no-logger';
import { IClientApp } from '../../types/model';
import {
addHours,
addMinutes,
hoursToMilliseconds,
minutesToMilliseconds,
secondsToMilliseconds,
subHours,
subMinutes,
subSeconds,
} from 'date-fns';
import { secondsToMilliseconds } from 'date-fns';
/**
* A utility to wait for any pending promises in the test subject code.
@ -47,393 +38,8 @@ function flushPromises() {
return Promise.resolve(setImmediate);
}
const appName = 'appName';
const instanceId = 'instanceId';
const createMetricsService = (cms) =>
new ClientMetricsService(
{
clientMetricsStore: cms,
strategyStore: null,
featureToggleStore: null,
clientApplicationsStore: null,
clientInstanceStore: null,
eventStore: null,
},
{ getLogger },
);
test('should work without state', () => {
const clientMetricsStore = new EventEmitter();
const metrics = createMetricsService(clientMetricsStore);
expect(metrics.getAppsWithToggles()).toBeTruthy();
expect(metrics.getTogglesMetrics()).toBeTruthy();
metrics.destroy();
});
test('data should expire', () => {
jest.useFakeTimers('modern');
const clientMetricsStore = new EventEmitter();
const metrics = createMetricsService(clientMetricsStore);
metrics.addPayload({
appName,
instanceId,
bucket: {
start: subSeconds(Date.now(), 2),
stop: subSeconds(Date.now(), 1),
toggles: {
toggleX: {
yes: 123,
no: 0,
},
},
},
});
let lastHourExpires = 0;
metrics.lastHourList.on('expire', () => {
lastHourExpires++;
});
let lastMinExpires = 0;
metrics.lastMinuteList.on('expire', () => {
lastMinExpires++;
});
jest.advanceTimersByTime(minutesToMilliseconds(1));
expect(lastMinExpires).toBe(1);
expect(lastHourExpires).toBe(0);
jest.advanceTimersByTime(hoursToMilliseconds(1));
expect(lastMinExpires).toBe(1);
expect(lastHourExpires).toBe(1);
jest.useRealTimers();
metrics.destroy();
});
test('should listen to metrics from store', () => {
const clientMetricsStore = new EventEmitter();
const metrics = createMetricsService(clientMetricsStore);
clientMetricsStore.emit('metrics', {
appName,
instanceId,
bucket: {
start: new Date(),
stop: new Date(),
toggles: {
toggleX: {
yes: 123,
no: 0,
},
},
},
});
expect(metrics.apps[appName].count).toBe(123);
expect(metrics.globalCount).toBe(123);
expect(metrics.getTogglesMetrics().lastHour.toggleX).toEqual({
yes: 123,
no: 0,
});
expect(metrics.getTogglesMetrics().lastMinute.toggleX).toEqual({
yes: 123,
no: 0,
});
metrics.addPayload({
appName,
instanceId,
bucket: {
start: new Date(),
stop: new Date(),
toggles: {
toggleX: {
yes: 10,
no: 10,
},
},
},
});
expect(metrics.globalCount).toBe(143);
expect(metrics.getTogglesMetrics().lastHour.toggleX).toEqual({
yes: 133,
no: 10,
});
expect(metrics.getTogglesMetrics().lastMinute.toggleX).toEqual({
yes: 133,
no: 10,
});
metrics.destroy();
});
test('should build up list of seen toggles when new metrics arrives', () => {
const clientMetricsStore = new EventEmitter();
const metrics = createMetricsService(clientMetricsStore);
clientMetricsStore.emit('metrics', {
appName,
instanceId,
bucket: {
start: new Date(),
stop: new Date(),
toggles: {
toggleX: {
yes: 123,
no: 0,
},
toggleY: {
yes: 50,
no: 50,
},
},
},
});
const appToggles = metrics.getAppsWithToggles();
const togglesForApp = metrics.getSeenTogglesByAppName(appName);
expect(appToggles.length).toBe(1);
expect(appToggles[0].seenToggles.length).toBe(2);
expect(appToggles[0].seenToggles).toContain('toggleX');
expect(appToggles[0].seenToggles).toContain('toggleY');
expect(togglesForApp.length === 2);
expect(togglesForApp).toContain('toggleX');
expect(togglesForApp).toContain('toggleY');
metrics.destroy();
});
test('should handle a lot of toggles', () => {
const clientMetricsStore = new EventEmitter();
const metrics = createMetricsService(clientMetricsStore);
const toggleCounts = {};
for (let i = 0; i < 100; i++) {
toggleCounts[`toggle${i}`] = { yes: i, no: i };
}
clientMetricsStore.emit('metrics', {
appName,
instanceId,
bucket: {
start: new Date(),
stop: new Date(),
toggles: toggleCounts,
},
});
const seenToggles = metrics.getSeenTogglesByAppName(appName);
expect(seenToggles.length).toBe(100);
metrics.destroy();
});
test('should have correct values for lastMinute', () => {
jest.useFakeTimers('modern');
const clientMetricsStore = new EventEmitter();
const metrics = createMetricsService(clientMetricsStore);
const now = new Date();
const input = [
{
start: subHours(now, 1),
stop: subMinutes(now, 59),
toggles: {
toggle: { yes: 10, no: 10 },
},
},
{
start: subMinutes(now, 30),
stop: subMinutes(now, 29),
toggles: {
toggle: { yes: 10, no: 10 },
},
},
{
start: subMinutes(now, 2),
stop: subMinutes(now, 1),
toggles: {
toggle: { yes: 10, no: 10 },
},
},
{
start: subMinutes(now, 2),
stop: subSeconds(now, 59),
toggles: {
toggle: { yes: 10, no: 10 },
},
},
{
start: now,
stop: subSeconds(now, 30),
toggles: {
toggle: { yes: 10, no: 10 },
},
},
];
input.forEach((bucket) => {
clientMetricsStore.emit('metrics', {
appName,
instanceId,
bucket,
});
});
const seenToggles = metrics.getSeenTogglesByAppName(appName);
expect(seenToggles.length).toBe(1);
// metrics.se
let c = metrics.getTogglesMetrics();
expect(c.lastMinute.toggle).toEqual({ yes: 20, no: 20 });
jest.advanceTimersByTime(10_000);
c = metrics.getTogglesMetrics();
expect(c.lastMinute.toggle).toEqual({ yes: 10, no: 10 });
jest.advanceTimersByTime(20_000);
c = metrics.getTogglesMetrics();
expect(c.lastMinute.toggle).toEqual({ yes: 0, no: 0 });
metrics.destroy();
jest.useRealTimers();
});
test('should have correct values for lastHour', () => {
jest.useFakeTimers('modern');
const clientMetricsStore = new EventEmitter();
const metrics = createMetricsService(clientMetricsStore);
const now = Date.now();
const input = [
{
start: subHours(now, 1),
stop: subMinutes(now, 59),
toggles: {
toggle: { yes: 10, no: 10 },
},
},
{
start: subMinutes(now, 30),
stop: subMinutes(now, 29),
toggles: {
toggle: { yes: 10, no: 10 },
},
},
{
start: subMinutes(now, 15),
stop: subMinutes(now, 14),
toggles: {
toggle: { yes: 10, no: 10 },
},
},
{
start: addMinutes(now, 59),
stop: addHours(now, 1),
toggles: {
toggle: { yes: 11, no: 11 },
},
},
];
input.forEach((bucket) => {
clientMetricsStore.emit('metrics', {
appName,
instanceId,
bucket,
});
});
const seenToggles = metrics.getSeenTogglesByAppName(appName);
expect(seenToggles.length).toBe(1);
// metrics.se
let c = metrics.getTogglesMetrics();
expect(c.lastHour.toggle).toEqual({ yes: 41, no: 41 });
jest.advanceTimersByTime(10_000);
c = metrics.getTogglesMetrics();
expect(c.lastHour.toggle).toEqual({ yes: 41, no: 41 });
// at 30
jest.advanceTimersByTime(minutesToMilliseconds(30));
c = metrics.getTogglesMetrics();
expect(c.lastHour.toggle).toEqual({ yes: 31, no: 31 });
// at 45
jest.advanceTimersByTime(minutesToMilliseconds(15));
c = metrics.getTogglesMetrics();
expect(c.lastHour.toggle).toEqual({ yes: 21, no: 21 });
// at 1:15
jest.advanceTimersByTime(minutesToMilliseconds(30));
c = metrics.getTogglesMetrics();
expect(c.lastHour.toggle).toEqual({ yes: 11, no: 11 });
// at 2:00
jest.advanceTimersByTime(minutesToMilliseconds(45));
c = metrics.getTogglesMetrics();
expect(c.lastHour.toggle).toEqual({ yes: 0, no: 0 });
metrics.destroy();
jest.useRealTimers();
});
test('should not fail when toggle metrics is missing yes/no field', () => {
const clientMetricsStore = new EventEmitter();
const metrics = createMetricsService(clientMetricsStore);
clientMetricsStore.emit('metrics', {
appName,
instanceId,
bucket: {
start: new Date(),
stop: new Date(),
toggles: {
toggleX: {
yes: 123,
no: 0,
},
},
},
});
metrics.addPayload({
appName,
instanceId,
bucket: {
start: new Date(),
stop: new Date(),
toggles: {
toggleX: {
blue: 10,
green: 10,
},
},
},
});
expect(metrics.globalCount).toBe(123);
expect(metrics.getTogglesMetrics().lastMinute.toggleX).toEqual({
yes: 123,
no: 0,
});
metrics.destroy();
});
test('Multiple registrations of same appname and instanceid within same time period should only cause one registration', async () => {
jest.useFakeTimers('modern');
const clientMetricsStore: any = new EventEmitter();
const appStoreSpy = jest.fn();
const bulkSpy = jest.fn();
const clientApplicationsStore: any = {
@ -444,14 +50,14 @@ test('Multiple registrations of same appname and instanceid within same time per
};
const clientMetrics = new ClientMetricsService(
{
clientMetricsStore,
clientMetricsStoreV2: null,
strategyStore: null,
featureToggleStore: null,
clientApplicationsStore,
clientInstanceStore,
eventStore: null,
},
{ getLogger },
{ getLogger, eventBus: new EventEmitter() },
);
const client1: IClientApp = {
appName: 'test_app',
@ -483,7 +89,6 @@ test('Multiple registrations of same appname and instanceid within same time per
test('Multiple unique clients causes multiple registrations', async () => {
jest.useFakeTimers('modern');
const clientMetricsStore: any = new EventEmitter();
const appStoreSpy = jest.fn();
const bulkSpy = jest.fn();
const clientApplicationsStore: any = {
@ -495,14 +100,14 @@ test('Multiple unique clients causes multiple registrations', async () => {
const clientMetrics = new ClientMetricsService(
{
clientMetricsStore,
clientMetricsStoreV2: null,
strategyStore: null,
featureToggleStore: null,
clientApplicationsStore,
clientInstanceStore,
eventStore: null,
},
{ getLogger },
{ getLogger, eventBus: new EventEmitter() },
);
const client1 = {
appName: 'test_app',
@ -535,7 +140,6 @@ test('Multiple unique clients causes multiple registrations', async () => {
});
test('Same client registered outside of dedup interval will be registered twice', async () => {
jest.useFakeTimers('modern');
const clientMetricsStore: any = new EventEmitter();
const appStoreSpy = jest.fn();
const bulkSpy = jest.fn();
const clientApplicationsStore: any = {
@ -549,14 +153,14 @@ test('Same client registered outside of dedup interval will be registered twice'
const clientMetrics = new ClientMetricsService(
{
clientMetricsStore,
clientMetricsStoreV2: null,
strategyStore: null,
featureToggleStore: null,
clientApplicationsStore,
clientInstanceStore,
eventStore: null,
},
{ getLogger },
{ getLogger, eventBus: new EventEmitter() },
bulkInterval,
);
const client1 = {
@ -592,7 +196,6 @@ test('Same client registered outside of dedup interval will be registered twice'
test('No registrations during a time period will not call stores', async () => {
jest.useFakeTimers('modern');
const clientMetricsStore: any = new EventEmitter();
const appStoreSpy = jest.fn();
const bulkSpy = jest.fn();
const clientApplicationsStore: any = {
@ -603,14 +206,14 @@ test('No registrations during a time period will not call stores', async () => {
};
new ClientMetricsService(
{
clientMetricsStore,
clientMetricsStoreV2: null,
strategyStore: null,
featureToggleStore: null,
clientApplicationsStore,
clientInstanceStore,
eventStore: null,
},
{ getLogger },
{ getLogger, eventBus: new EventEmitter() },
);
jest.advanceTimersByTime(6000);
expect(appStoreSpy).toHaveBeenCalledTimes(0);

View File

@ -1,8 +1,7 @@
import { applicationSchema } from './metrics-schema';
import { Projection } from './projection';
import { clientMetricsSchema } from './client-metrics-schema';
import { APPLICATION_CREATED, IBaseEvent } from '../../types/events';
import { IApplication, IYesNoCount } from './models';
import { APPLICATION_CREATED, CLIENT_METRICS } from '../../types/events';
import { IApplication } from './models';
import { IUnleashStores } from '../../types/stores';
import { IUnleashConfig } from '../../types/option';
import { IEventStore } from '../../types/stores/event-store';
@ -12,45 +11,25 @@ import {
} from '../../types/stores/client-applications-store';
import { IFeatureToggleStore } from '../../types/stores/feature-toggle-store';
import { IStrategyStore } from '../../types/stores/strategy-store';
import { IClientMetricsStore } from '../../types/stores/client-metrics-store';
import { IClientInstanceStore } from '../../types/stores/client-instance-store';
import { IApplicationQuery } from '../../types/query';
import { IClientApp, IMetricCounts, IMetricsBucket } from '../../types/model';
import { IClientApp } from '../../types/model';
import { clientRegisterSchema } from './register-schema';
import {
minutesToMilliseconds,
parseISO,
secondsToMilliseconds,
} from 'date-fns';
import TTLList from './ttl-list';
import { minutesToMilliseconds, secondsToMilliseconds } from 'date-fns';
import EventEmitter from 'events';
import { IClientMetricsStoreV2 } from '../../types/stores/client-metrics-store-v2';
export default class ClientMetricsService {
globalCount = 0;
apps = {};
lastHourProjection = new Projection();
lastMinuteProjection = new Projection();
lastHourList = new TTLList<IMetricCounts>({
interval: secondsToMilliseconds(10),
});
logger = null;
lastMinuteList = new TTLList<IMetricCounts>({
interval: secondsToMilliseconds(10),
expireType: 'minutes',
expireAmount: 1,
});
seenClients: Record<string, IClientApp> = {};
private timers: NodeJS.Timeout[] = [];
private clientMetricsStore: IClientMetricsStore;
private clientMetricsStoreV2: IClientMetricsStoreV2;
private strategyStore: IStrategyStore;
@ -66,9 +45,11 @@ export default class ClientMetricsService {
private announcementInterval: number;
private eventBus: EventEmitter;
constructor(
{
clientMetricsStore,
clientMetricsStoreV2,
strategyStore,
featureToggleStore,
clientInstanceStore,
@ -76,46 +57,29 @@ export default class ClientMetricsService {
eventStore,
}: Pick<
IUnleashStores,
| 'clientMetricsStore'
| 'clientMetricsStoreV2'
| 'strategyStore'
| 'featureToggleStore'
| 'clientApplicationsStore'
| 'clientInstanceStore'
| 'eventStore'
>,
{ getLogger }: Pick<IUnleashConfig, 'getLogger'>,
{ getLogger, eventBus }: Pick<IUnleashConfig, 'getLogger' | 'eventBus'>,
bulkInterval = secondsToMilliseconds(5),
announcementInterval = minutesToMilliseconds(5),
) {
this.clientMetricsStore = clientMetricsStore;
this.clientMetricsStoreV2 = clientMetricsStoreV2;
this.strategyStore = strategyStore;
this.featureToggleStore = featureToggleStore;
this.clientApplicationsStore = clientApplicationsStore;
this.clientInstanceStore = clientInstanceStore;
this.eventStore = eventStore;
this.eventBus = eventBus;
this.logger = getLogger('/services/client-metrics/index.ts');
this.bulkInterval = bulkInterval;
this.announcementInterval = announcementInterval;
this.lastHourList.on('expire', (toggles) => {
Object.keys(toggles).forEach((toggleName) => {
this.lastHourProjection.substract(
toggleName,
this.createCountObject(toggles[toggleName]),
);
});
});
this.lastMinuteList.on('expire', (toggles) => {
Object.keys(toggles).forEach((toggleName) => {
this.lastMinuteProjection.substract(
toggleName,
this.createCountObject(toggles[toggleName]),
);
});
});
this.timers.push(
setInterval(() => this.bulkAdd(), this.bulkInterval).unref(),
);
@ -125,26 +89,37 @@ export default class ClientMetricsService {
this.announcementInterval,
).unref(),
);
clientMetricsStore.on('metrics', (m) => this.addPayload(m));
}
async registerClientMetrics(
public async registerClientMetrics(
data: IClientApp,
clientIp: string,
): Promise<void> {
const value = await clientMetricsSchema.validateAsync(data);
const toggleNames = Object.keys(value.bucket.toggles);
if (toggleNames.length > 0) {
await this.featureToggleStore.setLastSeen(toggleNames);
await this.clientMetricsStore.insert(value);
}
await this.clientInstanceStore.insert({
appName: value.appName,
instanceId: value.instanceId,
clientIp,
});
// TODO: move to new service
const toggleNames = Object.keys(value.bucket.toggles);
if (toggleNames.length > 0) {
await this.featureToggleStore.setLastSeen(toggleNames);
}
this.eventBus.emit(CLIENT_METRICS, value);
}
public async registerClient(
data: IClientApp,
clientIp: string,
): Promise<void> {
const value = await clientRegisterSchema.validateAsync(data);
value.clientIp = clientIp;
value.createdBy = clientIp;
this.seenClients[this.clientKey(value)] = value;
}
async announceUnannounced(): Promise<void> {
@ -162,13 +137,6 @@ export default class ClientMetricsService {
}
}
async registerClient(data: IClientApp, clientIp: string): Promise<void> {
const value = await clientRegisterSchema.validateAsync(data);
value.clientIp = clientIp;
value.createdBy = clientIp;
this.seenClients[this.clientKey(value)] = value;
}
clientKey(client: IClientApp): string {
return `${client.appName}_${client.instanceId}`;
}
@ -202,51 +170,6 @@ export default class ClientMetricsService {
}
}
appToEvent(app: IClientApp): IBaseEvent {
return {
type: APPLICATION_CREATED,
createdBy: app.clientIp,
data: app,
};
}
getAppsWithToggles(): IClientApp[] {
const apps = [];
Object.keys(this.apps).forEach((appName) => {
const seenToggles = Object.keys(this.apps[appName].seenToggles);
const metricsCount = this.apps[appName].count;
apps.push({ appName, seenToggles, metricsCount });
});
return apps;
}
getSeenTogglesByAppName(appName: string): string[] {
return this.apps[appName]
? Object.keys(this.apps[appName].seenToggles)
: [];
}
async getSeenApps(): Promise<Record<string, IApplication[]>> {
const seenApps = this.getSeenAppsPerToggle();
const applications: IClientApplication[] =
await this.clientApplicationsStore.getAll();
const metaData = applications.reduce((result, entry) => {
// eslint-disable-next-line no-param-reassign
result[entry.appName] = entry;
return result;
}, {});
Object.keys(seenApps).forEach((key) => {
seenApps[key] = seenApps[key].map((entry) => {
if (metaData[entry.appName]) {
return { ...entry, ...metaData[entry.appName] };
}
return entry;
});
});
return seenApps;
}
async getApplications(
query: IApplicationQuery,
): Promise<IClientApplication[]> {
@ -254,9 +177,9 @@ export default class ClientMetricsService {
}
async getApplication(appName: string): Promise<IApplication> {
const seenToggles = this.getSeenTogglesByAppName(appName);
const [application, instances, strategies, features] =
const [seenToggles, application, instances, strategies, features] =
await Promise.all([
this.clientMetricsStoreV2.getSeenTogglesForApp(appName),
this.clientApplicationsStore.get(appName),
this.clientInstanceStore.getByAppName(appName),
this.strategyStore.getAll(),
@ -285,90 +208,6 @@ export default class ClientMetricsService {
};
}
getSeenAppsPerToggle(): Record<string, IApplication[]> {
const toggles = {};
Object.keys(this.apps).forEach((appName) => {
Object.keys(this.apps[appName].seenToggles).forEach(
(seenToggleName) => {
if (!toggles[seenToggleName]) {
toggles[seenToggleName] = [];
}
toggles[seenToggleName].push({ appName });
},
);
});
return toggles;
}
getTogglesMetrics(): Record<string, Record<string, IYesNoCount>> {
return {
lastHour: this.lastHourProjection.getProjection(),
lastMinute: this.lastMinuteProjection.getProjection(),
};
}
addPayload(data: IClientApp): void {
const { appName, bucket } = data;
const app = this.getApp(appName);
this.addBucket(app, bucket);
}
getApp(appName: string): IClientApp {
this.apps[appName] = this.apps[appName] || {
seenToggles: {},
count: 0,
};
return this.apps[appName];
}
createCountObject(entry: IMetricCounts): IYesNoCount {
let yes = typeof entry.yes === 'number' ? entry.yes : 0;
let no = typeof entry.no === 'number' ? entry.no : 0;
if (entry.variants) {
Object.entries(entry.variants).forEach(([key, value]) => {
if (key === 'disabled') {
no += value;
} else {
yes += value;
}
});
}
return { yes, no };
}
addBucket(app: IClientApp, bucket: IMetricsBucket): void {
let count = 0;
// TODO stop should be createdAt
const { stop, toggles } = bucket;
const toggleNames = Object.keys(toggles);
toggleNames.forEach((n) => {
const countObj = this.createCountObject(toggles[n]);
this.lastHourProjection.add(n, countObj);
this.lastMinuteProjection.add(n, countObj);
count += countObj.yes + countObj.no;
});
const timestamp = typeof stop === 'string' ? parseISO(stop) : stop;
this.lastHourList.add(toggles, timestamp);
this.lastMinuteList.add(toggles, timestamp);
this.globalCount += count;
// eslint-disable-next-line no-param-reassign
app.count += count;
this.addSeenToggles(app, toggleNames);
}
addSeenToggles(app: IClientApp, toggleNames: string[]): void {
toggleNames.forEach((t) => {
// eslint-disable-next-line no-param-reassign
app.seenToggles[t] = true;
});
}
async deleteApplication(appName: string): Promise<void> {
await this.clientInstanceStore.deleteForApplication(appName);
await this.clientApplicationsStore.delete(appName);
@ -380,8 +219,6 @@ export default class ClientMetricsService {
}
destroy(): void {
this.lastHourList.destroy();
this.lastMinuteList.destroy();
this.timers.forEach(clearInterval);
}
}

View File

@ -1,31 +0,0 @@
import { Projection } from './projection';
test('should return set empty if missing', () => {
const projection = new Projection();
projection.substract('name-1', { yes: 1, no: 2 });
expect(projection.getProjection()['name-1']).toEqual({ yes: 0, no: 0 });
});
test('should add and substract', () => {
const projection = new Projection();
expect(projection.store).toBeTruthy();
projection.add('name-1', { yes: 1, no: 2 });
expect(projection.getProjection()['name-1']).toEqual({ yes: 1, no: 2 });
projection.add('name-1', { yes: 1, no: 2 });
expect(projection.getProjection()['name-1']).toEqual({ yes: 2, no: 4 });
projection.substract('name-1', { yes: 1, no: 2 });
expect(projection.getProjection()['name-1']).toEqual({ yes: 1, no: 2 });
projection.substract('name-1', { yes: 1, no: 2 });
expect(projection.getProjection()['name-1']).toEqual({ yes: 0, no: 0 });
projection.substract('name-2', { yes: 23213, no: 23213 });
projection.add('name-2', { yes: 3, no: 2 });
expect(projection.getProjection()['name-2']).toEqual({ yes: 3, no: 2 });
});

View File

@ -1,33 +0,0 @@
import { IYesNoCount } from './models';
export class Projection {
store: Record<string, IYesNoCount> = {};
getProjection(): Record<string, IYesNoCount> {
return this.store;
}
add(name: string, countObj: IYesNoCount): void {
if (this.store[name]) {
this.store[name].yes += countObj.yes;
this.store[name].no += countObj.no;
} else {
this.store[name] = {
yes: countObj.yes,
no: countObj.no,
};
}
}
substract(name: string, countObj: IYesNoCount): void {
if (this.store[name]) {
this.store[name].yes -= countObj.yes;
this.store[name].no -= countObj.no;
} else {
this.store[name] = {
yes: 0,
no: 0,
};
}
}
}

View File

@ -1,85 +0,0 @@
import { addMilliseconds } from 'date-fns';
import TTLList from './ttl-list';
test('should emit expire', (done) => {
jest.useFakeTimers('modern');
const list = new TTLList<{ n: number }>({
interval: 20,
expireAmount: 10,
expireType: 'milliseconds',
});
list.on('expire', (entry) => {
list.destroy();
expect(entry.n).toBe(1);
done();
});
list.add({ n: 1 });
jest.advanceTimersByTime(21);
jest.useRealTimers();
});
test('should slice off list', () => {
jest.useFakeTimers('modern');
const list = new TTLList<{ n: string }>({
interval: 10,
expireAmount: 10,
expireType: 'milliseconds',
});
list.add({ n: '1' }, addMilliseconds(Date.now(), 1));
list.add({ n: '2' }, addMilliseconds(Date.now(), 50));
list.add({ n: '3' }, addMilliseconds(Date.now(), 200));
list.add({ n: '4' }, addMilliseconds(Date.now(), 300));
const expired = [];
list.on('expire', (entry) => {
// console.timeEnd(entry.n);
expired.push(entry);
});
expect(expired).toHaveLength(0);
expect(list.list.toArray()).toHaveLength(4);
jest.advanceTimersByTime(21);
expect(expired).toHaveLength(1);
expect(list.list.toArray()).toHaveLength(3);
jest.advanceTimersByTime(51);
expect(expired).toHaveLength(2);
expect(list.list.toArray()).toHaveLength(2);
jest.advanceTimersByTime(201);
expect(expired).toHaveLength(3);
expect(list.list.toArray()).toHaveLength(1);
jest.advanceTimersByTime(301);
expect(expired).toHaveLength(4);
expect(list.list.toArray()).toHaveLength(0);
list.destroy();
jest.useRealTimers();
});
test('should add item created in the past but expiring in the future', () => {
jest.useFakeTimers('modern');
const list = new TTLList<{ n: string }>({
interval: 10,
expireAmount: 10,
expireType: 'milliseconds',
});
const expireCallback = jest.fn();
list.on('expire', expireCallback);
list.add({ n: '1' }, new Date());
expect(expireCallback).not.toHaveBeenCalled();
expect(list.list.toArray()).toHaveLength(1);
jest.useRealTimers();
});

View File

@ -1,87 +0,0 @@
import { EventEmitter } from 'events';
import List from './list';
import {
add,
addMilliseconds,
secondsToMilliseconds,
Duration,
isFuture,
} from 'date-fns';
interface ConstructorArgs {
interval: number;
expireAmount: number;
expireType: keyof Duration | 'milliseconds';
}
// this list must have entries with sorted ttl range
export default class TTLList<T> extends EventEmitter {
private readonly interval: number;
private readonly expireAmount: number;
private readonly expireType: keyof Duration | 'milliseconds';
public list: List<{ ttl: Date; value: T }>;
private timer: NodeJS.Timeout;
private readonly getExpiryFrom: (timestamp) => Date;
constructor({
interval = secondsToMilliseconds(1),
expireAmount = 1,
expireType = 'hours',
}: Partial<ConstructorArgs> = {}) {
super();
this.interval = interval;
this.expireAmount = expireAmount;
this.expireType = expireType;
this.getExpiryFrom = (timestamp) => {
if (this.expireType === 'milliseconds') {
return addMilliseconds(timestamp, expireAmount);
} else {
return add(timestamp, { [expireType]: expireAmount });
}
};
this.list = new List();
this.list.on('evicted', ({ value, ttl }) => {
this.emit('expire', value, ttl);
});
this.startTimer();
}
startTimer(): void {
if (this.list) {
this.timer = setTimeout(() => {
if (this.list) {
this.timedCheck();
}
}, this.interval);
this.timer.unref();
}
}
add(value: T, timestamp = new Date()): void {
const ttl = this.getExpiryFrom(timestamp);
if (isFuture(ttl)) {
this.list.add({ ttl, value });
} else {
this.emit('expire', value, ttl);
}
}
timedCheck(): void {
this.list.reverseRemoveUntilTrue(({ value }) => isFuture(value.ttl));
this.startTimer();
}
destroy(): void {
clearTimeout(this.timer);
this.timer = null;
this.list = null;
}
}

View File

@ -58,6 +58,8 @@ export const USER_DELETED = 'user-deleted';
export const DROP_ENVIRONMENTS = 'drop-environments';
export const ENVIRONMENT_IMPORT = 'environment-import';
export const CLIENT_METRICS = 'client-metrics';
export interface IBaseEvent {
type: string;
createdBy: string;

View File

@ -1,3 +1,4 @@
import EventEmitter from 'events';
import { LogLevel, LogProvider } from '../logger';
export type EventHook = (eventName: string, data: object) => void;
@ -151,4 +152,5 @@ export interface IUnleashConfig {
preRouterHook?: Function;
eventHook?: EventHook;
enterpriseVersion?: string;
eventBus: EventEmitter;
}

View File

@ -4,7 +4,6 @@ import { IFeatureTypeStore } from './stores/feature-type-store';
import { IStrategyStore } from './stores/strategy-store';
import { IClientApplicationsStore } from './stores/client-applications-store';
import { IClientInstanceStore } from './stores/client-instance-store';
import { IClientMetricsStore } from './stores/client-metrics-store';
import { IFeatureToggleStore } from './stores/feature-toggle-store';
import { IContextFieldStore } from './stores/context-field-store';
import { ISettingStore } from './stores/settings-store';
@ -31,7 +30,6 @@ export interface IUnleashStores {
apiTokenStore: IApiTokenStore;
clientApplicationsStore: IClientApplicationsStore;
clientInstanceStore: IClientInstanceStore;
clientMetricsStore: IClientMetricsStore;
clientMetricsStoreV2: IClientMetricsStoreV2;
contextFieldStore: IContextFieldStore;
environmentStore: IEnvironmentStore;

View File

@ -1,12 +0,0 @@
export interface IClientMetric {
id: number;
createdAt: Date;
metrics: any;
}
export interface IClientMetricsDb {
removeMetricsOlderThanOneHour(): Promise<void>;
insert(metrics: IClientMetric);
getMetricsLastHour(): Promise<IClientMetric[]>;
getNewMetrics(lastKnownId: number): Promise<IClientMetric[]>;
}

View File

@ -23,5 +23,9 @@ export interface IClientMetricsStoreV2
featureName: string,
hoursBack?: number,
): Promise<string[]>;
getSeenTogglesForApp(
appName: string,
hoursBack?: number,
): Promise<string[]>;
clearMetrics(hoursAgo: number): Promise<void>;
}

View File

@ -1,9 +0,0 @@
import EventEmitter from 'events';
import { IClientMetric } from './client-metrics-db';
import { Store } from './store';
export interface IClientMetricsStore
extends Store<IClientMetric, number>,
EventEmitter {
insert(metrics: IClientMetric): Promise<void>;
}

View File

@ -52,24 +52,6 @@ beforeEach(async () => {
started: clientStartedDate,
interval: 10,
});
await app.services.clientMetricsService.addPayload({
appName: 'demo-app-1',
instanceId: '123',
bucket: {
start: Date.now(),
stop: Date.now(),
toggles: {
someToggle: {
yes: 100,
no: 0,
},
anotherToggle: {
yes: 0,
no: 1,
},
},
},
});
});
afterAll(async () => {
@ -83,7 +65,6 @@ afterEach(async () => {
});
test('should get application details', async () => {
expect.assertions(2);
return app.request
.get('/api/admin/metrics/applications/demo-app-1')
.expect('Content-Type', /json/)
@ -105,51 +86,6 @@ test('should get list of applications', async () => {
});
});
test('should get list of seen seen-apps', async () => {
return app.request
.get('/api/admin/metrics/seen-apps')
.expect('Content-Type', /json/)
.expect(200)
.expect((res) => {
expect(res.body.someToggle).toBeDefined();
});
});
test('should get list of seen seen-toggles', async () => {
return app.request
.get('/api/admin/metrics/seen-toggles')
.expect('Content-Type', /json/)
.expect(200)
.expect((res) => {
expect(res.body).toHaveLength(1);
expect(res.body[0].seenToggles).toContain('someToggle');
});
});
test('should get list of feature-toggle metrics', async () => {
return app.request
.get('/api/admin/metrics/feature-toggles')
.expect('Content-Type', /json/)
.expect(200)
.expect((res) => {
expect(res.body.lastHour).toBeDefined();
expect(res.body.lastHour.anotherToggle).toBeDefined();
expect(res.body.lastMinute).toBeDefined();
expect(res.body.lastMinute.anotherToggle).toBeDefined();
});
});
test('should get feature-toggle metrics', async () => {
return app.request
.get('/api/admin/metrics/feature-toggles/anotherToggle')
.expect('Content-Type', /json/)
.expect(200)
.expect((res) => {
expect(res.body.lastHour).toBeDefined();
expect(res.body.lastMinute).toBeDefined();
});
});
test('should delete application', async () => {
expect.assertions(2);
await app.request

View File

@ -1,11 +1,11 @@
import { setupAppWithAuth } from '../../helpers/test-helper';
import { IUnleashTest, setupAppWithAuth } from '../../helpers/test-helper';
import metricsExample from '../../../examples/client-metrics.json';
import dbInit from '../../helpers/database-init';
import dbInit, { ITestDb } from '../../helpers/database-init';
import getLogger from '../../../fixtures/no-logger';
import { ApiTokenType } from '../../../../lib/types/models/api-token';
let app;
let db;
let app: IUnleashTest;
let db: ITestDb;
beforeAll(async () => {
db = await dbInit('metrics_api_e2e_access_client', getLogger);
@ -19,7 +19,7 @@ afterAll(async () => {
test('should enrich metrics with environment from api-token', async () => {
const { apiTokenService } = app.services;
const { environmentStore, clientMetricsStore } = db.stores;
const { environmentStore, clientMetricsStoreV2 } = db.stores;
await environmentStore.create({
name: 'some',
@ -39,6 +39,6 @@ test('should enrich metrics with environment from api-token', async () => {
.send(metricsExample)
.expect(202);
const all = await clientMetricsStore.getAll();
expect(all[0].metrics.environment).toBe('some');
const all = await clientMetricsStoreV2.getAll();
expect(all[0].environment).toBe('some');
});

View File

@ -1,4 +1,3 @@
import { EventEmitter } from 'events';
import { migrateDb } from '../../../migrator';
import { createStores } from '../../../lib/db';
import { createDb } from '../../../lib/db/db-pool';
@ -92,7 +91,6 @@ export default async function init(
});
const db = createDb(config);
const eventBus = new EventEmitter();
await db.raw(`DROP SCHEMA IF EXISTS ${config.db.schema} CASCADE`);
await db.raw(`CREATE SCHEMA IF NOT EXISTS ${config.db.schema}`);
@ -100,8 +98,7 @@ export default async function init(
await migrateDb({ ...config, databaseSchema: config.db.schema });
await db.destroy();
const testDb = createDb(config);
const stores = await createStores(config, eventBus, testDb);
stores.clientMetricsStore.setMaxListeners(0);
const stores = await createStores(config, testDb);
stores.eventStore.setMaxListeners(0);
await resetDatabase(testDb);
await setupDatabase(stores);
@ -113,10 +110,9 @@ export default async function init(
await setupDatabase(stores);
},
destroy: async () => {
const { clientInstanceStore, clientMetricsStore } = stores;
const { clientInstanceStore } = stores;
return new Promise<void>((resolve, reject) => {
clientInstanceStore.destroy();
clientMetricsStore.destroy();
testDb.destroy((error) => (error ? reject(error) : resolve()));
});
},

View File

@ -38,13 +38,7 @@ function createApp(
const unleashSession = sessionDb(config, undefined);
const emitter = new EventEmitter();
emitter.setMaxListeners(0);
const app = getApp(
config,
stores,
services,
new EventEmitter(),
unleashSession,
);
const app = getApp(config, stores, services, unleashSession);
const request = supertest.agent(app);
const destroy = async () => {

View File

@ -1,6 +1,7 @@
import ClientMetricsService from '../../../lib/services/client-metrics';
import { IClientApp } from '../../../lib/types/model';
import { secondsToMilliseconds } from 'date-fns';
import EventEmitter from 'events';
const faker = require('faker');
const dbInit = require('../helpers/database-init');
@ -14,13 +15,14 @@ let clientMetricsService;
beforeAll(async () => {
db = await dbInit('client_metrics_service_serial', getLogger);
stores = db.stores;
const eventBus = new EventEmitter();
const bulkInterval = secondsToMilliseconds(0.5);
const announcementInterval = secondsToMilliseconds(2);
clientMetricsService = new ClientMetricsService(
stores,
{ getLogger },
{ getLogger, eventBus },
bulkInterval,
announcementInterval,
);

View File

@ -1,7 +1,6 @@
/* eslint-disable @typescript-eslint/lines-between-class-members */
/* eslint-disable @typescript-eslint/no-unused-vars */
import EventEmitter from 'events';
import { IClientMetric } from '../../lib/types/stores/client-metrics-db';
import {
IClientMetricsEnv,
IClientMetricsEnvKey,
@ -18,6 +17,12 @@ export default class FakeClientMetricsStoreV2
super();
this.setMaxListeners(0);
}
getSeenTogglesForApp(
appName: string,
hoursBack?: number,
): Promise<string[]> {
throw new Error('Method not implemented.');
}
clearMetrics(hoursBack: number): Promise<void> {
return Promise.resolve();
}
@ -50,7 +55,7 @@ export default class FakeClientMetricsStoreV2
throw new Error('Method not implemented.');
}
async getMetricsLastHour(): Promise<IClientMetric[]> {
async getMetricsLastHour(): Promise<[]> {
return Promise.resolve([]);
}

View File

@ -1,53 +0,0 @@
import EventEmitter from 'events';
import { IClientMetricsStore } from '../../lib/types/stores/client-metrics-store';
import { IClientMetric } from '../../lib/types/stores/client-metrics-db';
import NotFoundError from '../../lib/error/notfound-error';
export default class FakeClientMetricsStore
extends EventEmitter
implements IClientMetricsStore
{
metrics: IClientMetric[] = [];
constructor() {
super();
this.setMaxListeners(0);
}
async getMetricsLastHour(): Promise<IClientMetric[]> {
return Promise.resolve([]);
}
async insert(): Promise<void> {
return Promise.resolve();
}
async delete(key: number): Promise<void> {
this.metrics.splice(
this.metrics.findIndex((m) => m.id === key),
1,
);
}
async deleteAll(): Promise<void> {
return Promise.resolve(undefined);
}
destroy(): void {}
async exists(key: number): Promise<boolean> {
return this.metrics.some((m) => m.id === key);
}
async get(key: number): Promise<IClientMetric> {
const metric = this.metrics.find((m) => m.id === key);
if (metric) {
return metric;
}
throw new NotFoundError(`Could not find metric with key: ${key}`);
}
async getAll(): Promise<IClientMetric[]> {
return this.metrics;
}
}

View File

@ -1,5 +1,4 @@
import FakeFeatureStrategiesStore from './fake-feature-strategies-store';
import FakeClientMetricsStore from './fake-client-metrics-store';
import FakeClientInstanceStore from './fake-client-instance-store';
import FakeClientApplicationsStore from './fake-client-applications-store';
import FakeFeatureToggleStore from './fake-feature-toggle-store';
@ -36,7 +35,6 @@ const createStores: () => IUnleashStores = () => {
return {
db,
clientApplicationsStore: new FakeClientApplicationsStore(),
clientMetricsStore: new FakeClientMetricsStore(),
clientMetricsStoreV2: new FakeClientMetricsStoreV2(),
clientInstanceStore: new FakeClientInstanceStore(),
featureToggleStore: new FakeFeatureToggleStore(),