From ab201db754d3d84789bc383103d27a25794c5990 Mon Sep 17 00:00:00 2001
From: sveisvei <sveinung.rosaker@gmail.com>
Date: Fri, 4 Nov 2016 16:16:55 +0100
Subject: [PATCH] add metrics projections for last hour
---
.../index.js} | 64 ++++++++++++-------
.../lib/client-metrics/projection.js | 35 ++++++++++
.../lib/client-metrics/ttl-list.js | 48 ++++++++++++++
packages/unleash-api/lib/routes/metrics.js | 19 +++---
packages/unleash-api/package.json | 4 +-
5 files changed, 137 insertions(+), 33 deletions(-)
rename packages/unleash-api/lib/{client-metrics.js => client-metrics/index.js} (57%)
create mode 100644 packages/unleash-api/lib/client-metrics/projection.js
create mode 100644 packages/unleash-api/lib/client-metrics/ttl-list.js
diff --git a/packages/unleash-api/lib/client-metrics.js b/packages/unleash-api/lib/client-metrics/index.js
similarity index 57%
rename from packages/unleash-api/lib/client-metrics.js
rename to packages/unleash-api/lib/client-metrics/index.js
index 52d1892329..c432c32097 100644
--- a/packages/unleash-api/lib/client-metrics.js
+++ b/packages/unleash-api/lib/client-metrics/index.js
@@ -1,29 +1,42 @@
'use strict';
+const Projection = require('./projection.js');
+const TTLList = require('./ttl-list.js');
+
module.exports = class UnleashClientMetrics {
constructor () {
this.globalCount = 0;
- this.apps = [];
+ this.apps = {};
this.clients = {};
this.strategies = {};
this.buckets = {};
+
+ this.hourProjectionValue = new Projection();
+ this.oneHourLruCache = new TTLList();
+ this.oneHourLruCache.on('expire', (toggles) => {
+ Object.keys(toggles).forEach(toggleName => {
+ this.hourProjectionValue.substract(toggleName, toggles[toggleName]);
+ });
+ });
}
toJSON () {
- return JSON.stringify(this.getState(), null, 4);
+ return JSON.stringify(this.getMetricsOverview(), null, 4);
}
- getState () {
- // TODO need to flatten the store / possibly evict/flag stale clients
+ getMetricsOverview () {
return {
globalCount: this.globalCount,
apps: this.apps,
clients: this.clients,
strategies: this.strategies,
- buckets: this.buckets,
};
}
+ getTogglesMetrics () {
+ return this.hourProjectionValue.getProjection();
+ }
+
registerClient (data) {
this.addClient(data.appName, data.instanceId, data.started);
this.addStrategies(data.appName, data.strategies);
@@ -35,23 +48,19 @@ module.exports = class UnleashClientMetrics {
}
addBucket (appName, instanceId, bucket) {
- // TODO normalize time client-server-time / NTP?
let count = 0;
- const { start, stop, toggles } = bucket;
- Object.keys(toggles).forEach((n) => {
- if (this.buckets[n]) {
- this.buckets[n].yes.push({ start, stop, count: toggles[n].yes });
- this.buckets[n].no.push({ start, stop, count: toggles[n].no });
- } else {
- this.buckets[n] = {
- yes: [{ start, stop, count: toggles[n].yes }],
- no: [{ start, stop, count: toggles[n].no }],
- };
- }
+ // TODO stop should be createdAt
+ const { stop, toggles } = bucket;
- count += (toggles[n].yes + toggles[n].no);
+ Object.keys(toggles).forEach((n) => {
+ const entry = toggles[n];
+ this.hourProjectionValue.add(n, entry);
+ count += (entry.yes + entry.no);
});
- this.addClientCount(instanceId, count);
+
+ this.oneHourLruCache.add(toggles, stop);
+
+ this.addClientCount(appName, instanceId, count);
}
addStrategies (appName, strategyNames) {
@@ -63,7 +72,7 @@ module.exports = class UnleashClientMetrics {
});
}
- addClientCount (instanceId, count) {
+ addClientCount (appName, instanceId, count) {
if (typeof count === 'number' && count > 0) {
this.globalCount += count;
if (this.clients[instanceId]) {
@@ -73,7 +82,7 @@ module.exports = class UnleashClientMetrics {
}
addClient (appName, instanceId, started = new Date()) {
- this.addApp(appName);
+ this.addApp(appName, instanceId);
if (instanceId) {
if (this.clients[instanceId]) {
this.clients[instanceId].ping = new Date();
@@ -89,9 +98,16 @@ module.exports = class UnleashClientMetrics {
}
}
- addApp (v) {
- if (v && !this.apps.includes(v)) {
- this.apps.push(v);
+ addApp (appName, instanceId) {
+ if (appName && !this.apps[appName]) {
+ this.apps[appName] = {
+ count: 0,
+ clients: [],
+ };
+ }
+
+ if (instanceId && !this.apps[appName].clients.includes(instanceId)) {
+ this.apps[appName].clients.push(instanceId);
}
}
};
diff --git a/packages/unleash-api/lib/client-metrics/projection.js b/packages/unleash-api/lib/client-metrics/projection.js
new file mode 100644
index 0000000000..f43fa2a968
--- /dev/null
+++ b/packages/unleash-api/lib/client-metrics/projection.js
@@ -0,0 +1,35 @@
+'use strict';
+
+module.exports = class Projection {
+ constructor () {
+ this.store = {};
+ }
+
+ getProjection () {
+ return this.store;
+ }
+
+ add (name, countObj) {
+ if (this.store[name]) {
+ this.store[name].yes += countObj.yes;
+ this.store[name].no += countObj.no;
+ } else {
+ this.store[name] = {
+ yes: countObj.yes,
+ no: countObj.no,
+ };
+ }
+ }
+
+ substract (name, countObj) {
+ if (this.store[name]) {
+ this.store[name].yes -= countObj.yes;
+ this.store[name].no -= countObj.no;
+ } else {
+ this.store[name] = {
+ yes: 0,
+ no: 0,
+ };
+ }
+ }
+}
diff --git a/packages/unleash-api/lib/client-metrics/ttl-list.js b/packages/unleash-api/lib/client-metrics/ttl-list.js
new file mode 100644
index 0000000000..dc0e6868dd
--- /dev/null
+++ b/packages/unleash-api/lib/client-metrics/ttl-list.js
@@ -0,0 +1,48 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+const yallist = require('yallist');
+const moment = require('moment');
+
+// this list must have entires with sorted ttl range
+module.exports = class TTLList extends EventEmitter {
+ constructor () {
+ super();
+ this.cache = yallist.create();
+ setInterval(() => {
+ this.timedCheck();
+ }, 1000);
+ }
+
+ expire (entry) {
+ this.emit('expire', entry.value);
+ }
+
+ add (value, timestamp) {
+ const ttl = moment(timestamp).add(1, 'hour');
+ this.cache.push({ ttl, value });
+ }
+
+ timedCheck () {
+ const now = moment(new Date());
+ // find index to remove
+ let done = false;
+ // TODO: might use internal linkedlist
+ this.cache.forEachReverse((entry, index) => {
+ console.log(now.format(), entry.ttl.format());
+ if (done) {
+ return;
+ } else if (now.isBefore(entry.ttl)) {
+ // When we hit a valid ttl, remove next items in list (iteration is reversed)
+ this.cache = this.cache.slice(0, index + 1);
+ done = true;
+ } else if (index === 0) {
+ this.expire(entry);
+ // if rest of list has timed out, let it DIE!
+ this.cache = yallist.create(); // empty=
+ } else {
+ this.expire(entry);
+ }
+ });
+ }
+}
diff --git a/packages/unleash-api/lib/routes/metrics.js b/packages/unleash-api/lib/routes/metrics.js
index b2194afd6e..f100da6d10 100644
--- a/packages/unleash-api/lib/routes/metrics.js
+++ b/packages/unleash-api/lib/routes/metrics.js
@@ -14,22 +14,25 @@ module.exports = function (app, config) {
service.on('metrics', (entries) => {
- entries.forEach((m) => metrics.addPayload(m.metrics));
- });
-
- app.get('/service-metrics', (req, res) => {
- res.json(service.getMetrics());
+ entries.forEach((m) => {
+ metrics.addPayload(m.metrics);
+ });
});
app.get('/metrics', (req, res) => {
- res.json(metrics.getState());
+ res.json(metrics.getMetricsOverview());
+ });
+
+ app.get('/toggle-metrics', (req, res) => {
+ res.json(metrics.getTogglesMetrics());
});
app.post('/client/metrics', (req, res) => {
try {
const data = typeof req.body === 'string' ? JSON.parse(req.body) : req.body;
- metrics.addPayload(data);
- service.insert(data);
+ service
+ .insert(data)
+ .catch(e => logger.error('Error inserting metrics data', e));
} catch (e) {
logger.error('Error receiving metrics', e);
}
diff --git a/packages/unleash-api/package.json b/packages/unleash-api/package.json
index cfd7becd07..e74c41e5e8 100644
--- a/packages/unleash-api/package.json
+++ b/packages/unleash-api/package.json
@@ -59,8 +59,10 @@
"install": "^0.8.1",
"knex": "^0.11.10",
"log4js": "^0.6.38",
+ "moment": "^2.15.2",
"pg": "^6.1.0",
- "serve-favicon": "^2.3.0"
+ "serve-favicon": "^2.3.0",
+ "yallist": "^2.0.0"
},
"devDependencies": {
"coveralls": "^2.11.12",