diff --git a/packages/unleash-api/lib/client-metrics.js b/packages/unleash-api/lib/client-metrics/index.js similarity index 57% rename from packages/unleash-api/lib/client-metrics.js rename to packages/unleash-api/lib/client-metrics/index.js index 52d1892329..c432c32097 100644 --- a/packages/unleash-api/lib/client-metrics.js +++ b/packages/unleash-api/lib/client-metrics/index.js @@ -1,29 +1,42 @@ 'use strict'; +const Projection = require('./projection.js'); +const TTLList = require('./ttl-list.js'); + module.exports = class UnleashClientMetrics { constructor () { this.globalCount = 0; - this.apps = []; + this.apps = {}; this.clients = {}; this.strategies = {}; this.buckets = {}; + + this.hourProjectionValue = new Projection(); + this.oneHourLruCache = new TTLList(); + this.oneHourLruCache.on('expire', (toggles) => { + Object.keys(toggles).forEach(toggleName => { + this.hourProjectionValue.substract(toggleName, toggles[toggleName]); + }); + }); } toJSON () { - return JSON.stringify(this.getState(), null, 4); + return JSON.stringify(this.getMetricsOverview(), null, 4); } - getState () { - // TODO need to flatten the store / possibly evict/flag stale clients + getMetricsOverview () { return { globalCount: this.globalCount, apps: this.apps, clients: this.clients, strategies: this.strategies, - buckets: this.buckets, }; } + getTogglesMetrics () { + return this.hourProjectionValue.getProjection(); + } + registerClient (data) { this.addClient(data.appName, data.instanceId, data.started); this.addStrategies(data.appName, data.strategies); @@ -35,23 +48,19 @@ module.exports = class UnleashClientMetrics { } addBucket (appName, instanceId, bucket) { - // TODO normalize time client-server-time / NTP? let count = 0; - const { start, stop, toggles } = bucket; - Object.keys(toggles).forEach((n) => { - if (this.buckets[n]) { - this.buckets[n].yes.push({ start, stop, count: toggles[n].yes }); - this.buckets[n].no.push({ start, stop, count: toggles[n].no }); - } else { - this.buckets[n] = { - yes: [{ start, stop, count: toggles[n].yes }], - no: [{ start, stop, count: toggles[n].no }], - }; - } + // TODO stop should be createdAt + const { stop, toggles } = bucket; - count += (toggles[n].yes + toggles[n].no); + Object.keys(toggles).forEach((n) => { + const entry = toggles[n]; + this.hourProjectionValue.add(n, entry); + count += (entry.yes + entry.no); }); - this.addClientCount(instanceId, count); + + this.oneHourLruCache.add(toggles, stop); + + this.addClientCount(appName, instanceId, count); } addStrategies (appName, strategyNames) { @@ -63,7 +72,7 @@ module.exports = class UnleashClientMetrics { }); } - addClientCount (instanceId, count) { + addClientCount (appName, instanceId, count) { if (typeof count === 'number' && count > 0) { this.globalCount += count; if (this.clients[instanceId]) { @@ -73,7 +82,7 @@ module.exports = class UnleashClientMetrics { } addClient (appName, instanceId, started = new Date()) { - this.addApp(appName); + this.addApp(appName, instanceId); if (instanceId) { if (this.clients[instanceId]) { this.clients[instanceId].ping = new Date(); @@ -89,9 +98,16 @@ module.exports = class UnleashClientMetrics { } } - addApp (v) { - if (v && !this.apps.includes(v)) { - this.apps.push(v); + addApp (appName, instanceId) { + if (appName && !this.apps[appName]) { + this.apps[appName] = { + count: 0, + clients: [], + }; + } + + if (instanceId && !this.apps[appName].clients.includes(instanceId)) { + this.apps[appName].clients.push(instanceId); } } }; diff --git a/packages/unleash-api/lib/client-metrics/projection.js b/packages/unleash-api/lib/client-metrics/projection.js new file mode 100644 index 0000000000..f43fa2a968 --- /dev/null +++ b/packages/unleash-api/lib/client-metrics/projection.js @@ -0,0 +1,35 @@ +'use strict'; + +module.exports = class Projection { + constructor () { + this.store = {}; + } + + getProjection () { + return this.store; + } + + add (name, countObj) { + if (this.store[name]) { + this.store[name].yes += countObj.yes; + this.store[name].no += countObj.no; + } else { + this.store[name] = { + yes: countObj.yes, + no: countObj.no, + }; + } + } + + substract (name, countObj) { + if (this.store[name]) { + this.store[name].yes -= countObj.yes; + this.store[name].no -= countObj.no; + } else { + this.store[name] = { + yes: 0, + no: 0, + }; + } + } +} diff --git a/packages/unleash-api/lib/client-metrics/ttl-list.js b/packages/unleash-api/lib/client-metrics/ttl-list.js new file mode 100644 index 0000000000..dc0e6868dd --- /dev/null +++ b/packages/unleash-api/lib/client-metrics/ttl-list.js @@ -0,0 +1,48 @@ +'use strict'; + +const { EventEmitter } = require('events'); +const yallist = require('yallist'); +const moment = require('moment'); + +// this list must have entires with sorted ttl range +module.exports = class TTLList extends EventEmitter { + constructor () { + super(); + this.cache = yallist.create(); + setInterval(() => { + this.timedCheck(); + }, 1000); + } + + expire (entry) { + this.emit('expire', entry.value); + } + + add (value, timestamp) { + const ttl = moment(timestamp).add(1, 'hour'); + this.cache.push({ ttl, value }); + } + + timedCheck () { + const now = moment(new Date()); + // find index to remove + let done = false; + // TODO: might use internal linkedlist + this.cache.forEachReverse((entry, index) => { + console.log(now.format(), entry.ttl.format()); + if (done) { + return; + } else if (now.isBefore(entry.ttl)) { + // When we hit a valid ttl, remove next items in list (iteration is reversed) + this.cache = this.cache.slice(0, index + 1); + done = true; + } else if (index === 0) { + this.expire(entry); + // if rest of list has timed out, let it DIE! + this.cache = yallist.create(); // empty= + } else { + this.expire(entry); + } + }); + } +} diff --git a/packages/unleash-api/lib/routes/metrics.js b/packages/unleash-api/lib/routes/metrics.js index b2194afd6e..f100da6d10 100644 --- a/packages/unleash-api/lib/routes/metrics.js +++ b/packages/unleash-api/lib/routes/metrics.js @@ -14,22 +14,25 @@ module.exports = function (app, config) { service.on('metrics', (entries) => { - entries.forEach((m) => metrics.addPayload(m.metrics)); - }); - - app.get('/service-metrics', (req, res) => { - res.json(service.getMetrics()); + entries.forEach((m) => { + metrics.addPayload(m.metrics); + }); }); app.get('/metrics', (req, res) => { - res.json(metrics.getState()); + res.json(metrics.getMetricsOverview()); + }); + + app.get('/toggle-metrics', (req, res) => { + res.json(metrics.getTogglesMetrics()); }); app.post('/client/metrics', (req, res) => { try { const data = typeof req.body === 'string' ? JSON.parse(req.body) : req.body; - metrics.addPayload(data); - service.insert(data); + service + .insert(data) + .catch(e => logger.error('Error inserting metrics data', e)); } catch (e) { logger.error('Error receiving metrics', e); } diff --git a/packages/unleash-api/package.json b/packages/unleash-api/package.json index cfd7becd07..e74c41e5e8 100644 --- a/packages/unleash-api/package.json +++ b/packages/unleash-api/package.json @@ -59,8 +59,10 @@ "install": "^0.8.1", "knex": "^0.11.10", "log4js": "^0.6.38", + "moment": "^2.15.2", "pg": "^6.1.0", - "serve-favicon": "^2.3.0" + "serve-favicon": "^2.3.0", + "yallist": "^2.0.0" }, "devDependencies": { "coveralls": "^2.11.12",