add metrics projections for last hour

packages/unleash-api/lib/client-metrics/index.js

@@ -1,29 +1,42 @@
 'use strict';
 
+const Projection = require('./projection.js');
+const TTLList = require('./ttl-list.js');
+
 module.exports = class UnleashClientMetrics {
     constructor () {
         this.globalCount = 0;
-        this.apps = [];
+        this.apps = {};
         this.clients = {};
         this.strategies = {};
         this.buckets = {};
+
+        this.hourProjectionValue = new Projection();
+        this.oneHourLruCache = new TTLList();
+        this.oneHourLruCache.on('expire', (toggles) => {
+            Object.keys(toggles).forEach(toggleName => {
+                this.hourProjectionValue.substract(toggleName, toggles[toggleName]);
+            });
+        });
     }
 
     toJSON () {
-        return JSON.stringify(this.getState(), null, 4);
+        return JSON.stringify(this.getMetricsOverview(), null, 4);
     }
 
-    getState () {
-        // TODO need to flatten the store / possibly evict/flag stale clients
+    getMetricsOverview () {
         return {
             globalCount: this.globalCount,
             apps: this.apps,
             clients: this.clients,
             strategies: this.strategies,
-            buckets: this.buckets,
         };
     }
 
+    getTogglesMetrics () {
+        return this.hourProjectionValue.getProjection();
+    }
+
     registerClient (data) {
         this.addClient(data.appName, data.instanceId, data.started);
         this.addStrategies(data.appName, data.strategies);
@@ -35,23 +48,19 @@ module.exports = class UnleashClientMetrics {
     }
 
     addBucket (appName, instanceId, bucket) {
-        // TODO normalize time client-server-time / NTP?
         let count = 0;
-        const { start, stop, toggles } = bucket;
-        Object.keys(toggles).forEach((n) => {
-            if (this.buckets[n]) {
-                this.buckets[n].yes.push({ start, stop, count: toggles[n].yes });
-                this.buckets[n].no.push({ start, stop, count: toggles[n].no });
-            } else {
-                this.buckets[n] = {
-                    yes: [{ start, stop, count: toggles[n].yes }],
-                    no: [{ start, stop, count: toggles[n].no }],
-                };
-            }
+        // TODO stop should be createdAt
+        const { stop, toggles } = bucket;
 
-            count += (toggles[n].yes + toggles[n].no);
+        Object.keys(toggles).forEach((n) => {
+            const entry = toggles[n];
+            this.hourProjectionValue.add(n, entry);
+            count += (entry.yes + entry.no);
         });
-        this.addClientCount(instanceId, count);
+
+        this.oneHourLruCache.add(toggles, stop);
+
+        this.addClientCount(appName, instanceId, count);
     }
 
     addStrategies (appName, strategyNames) {
@@ -63,7 +72,7 @@ module.exports = class UnleashClientMetrics {
         });
     }
 
-    addClientCount (instanceId, count) {
+    addClientCount (appName, instanceId, count) {
         if (typeof count === 'number' && count > 0) {
             this.globalCount += count;
             if (this.clients[instanceId]) {
@@ -73,7 +82,7 @@ module.exports = class UnleashClientMetrics {
     }
 
     addClient (appName, instanceId, started = new Date()) {
-        this.addApp(appName);
+        this.addApp(appName, instanceId);
         if (instanceId) {
             if (this.clients[instanceId]) {
                 this.clients[instanceId].ping = new Date();
@@ -89,9 +98,16 @@ module.exports = class UnleashClientMetrics {
         }
     }
 
-    addApp (v) {
-        if (v && !this.apps.includes(v)) {
-            this.apps.push(v);
+    addApp (appName, instanceId) {
+        if (appName && !this.apps[appName]) {
+            this.apps[appName] = {
+                count: 0,
+                clients: [],
+            };
+        }
+
+        if (instanceId && !this.apps[appName].clients.includes(instanceId)) {
+            this.apps[appName].clients.push(instanceId);
         }
     }
 };

packages/unleash-api/lib/client-metrics/projection.js

@@ -0,0 +1,35 @@
+'use strict';
+
+module.exports = class Projection {
+    constructor () {
+        this.store = {};
+    }
+
+    getProjection () {
+        return this.store;
+    }
+
+    add (name, countObj) {
+        if (this.store[name]) {
+            this.store[name].yes += countObj.yes;
+            this.store[name].no += countObj.no;
+        } else {
+            this.store[name] = {
+                yes: countObj.yes,
+                no: countObj.no,
+            };
+        }
+    }
+
+    substract (name, countObj) {
+        if (this.store[name]) {
+            this.store[name].yes -= countObj.yes;
+            this.store[name].no -= countObj.no;
+        } else {
+            this.store[name] = {
+                yes: 0,
+                no: 0,
+            };
+        }
+    }
+}

packages/unleash-api/lib/client-metrics/ttl-list.js

@@ -0,0 +1,48 @@
+'use strict';
+
+const { EventEmitter } = require('events');
+const yallist = require('yallist');
+const moment = require('moment');
+
+// this list must have entires with sorted ttl range
+module.exports = class TTLList extends EventEmitter {
+    constructor () {
+        super();
+        this.cache = yallist.create();
+        setInterval(() => {
+            this.timedCheck();
+        }, 1000);
+    }
+
+    expire (entry) {
+        this.emit('expire', entry.value);
+    }
+
+    add (value, timestamp) {
+        const ttl = moment(timestamp).add(1, 'hour');
+        this.cache.push({ ttl, value });
+    }
+
+    timedCheck () {
+        const now = moment(new Date());
+        // find index to remove
+        let done = false;
+        // TODO: might use internal linkedlist
+        this.cache.forEachReverse((entry, index) => {
+            console.log(now.format(), entry.ttl.format());
+            if (done) {
+                return;
+            } else if (now.isBefore(entry.ttl)) {
+                // When we hit a valid ttl, remove next items in list (iteration is reversed)
+                this.cache = this.cache.slice(0, index + 1);
+                done = true;
+            } else if (index === 0) {
+                this.expire(entry);
+                // if rest of list has timed out, let it DIE!
+                this.cache = yallist.create(); // empty=
+            } else {
+                this.expire(entry);
+            }
+        });
+    }
+}

packages/unleash-api/lib/routes/metrics.js

@@ -14,22 +14,25 @@ module.exports = function (app, config) {
 
 
     service.on('metrics', (entries) => {
-        entries.forEach((m) => metrics.addPayload(m.metrics));
-    });
-
-    app.get('/service-metrics', (req, res) => {
-        res.json(service.getMetrics());
+        entries.forEach((m) => {
+            metrics.addPayload(m.metrics);
+        });
     });
 
     app.get('/metrics', (req, res) => {
-        res.json(metrics.getState());
+        res.json(metrics.getMetricsOverview());
+    });
+
+    app.get('/toggle-metrics', (req, res) => {
+        res.json(metrics.getTogglesMetrics());
     });
 
     app.post('/client/metrics', (req, res) => {
         try {
             const data = typeof req.body === 'string' ? JSON.parse(req.body) : req.body;
-            metrics.addPayload(data);
-            service.insert(data);
+            service
+                .insert(data)
+                .catch(e => logger.error('Error inserting metrics data', e));
         } catch (e) {
             logger.error('Error receiving metrics', e);
         }

packages/unleash-api/package.json

@@ -59,8 +59,10 @@
     "install": "^0.8.1",
     "knex": "^0.11.10",
     "log4js": "^0.6.38",
+    "moment": "^2.15.2",
     "pg": "^6.1.0",
-    "serve-favicon": "^2.3.0"
+    "serve-favicon": "^2.3.0",
+    "yallist": "^2.0.0"
   },
   "devDependencies": {
     "coveralls": "^2.11.12",