Fixes. Refactory endpoints. Timeout

docker-compose.yml

@@ -177,6 +177,8 @@ services:
       context: ./statsd/endpoint/json
     volumes:
       - ./statsd/endpoint/json:/opt/datasource
+    environment:
+      TIMEOUT: 60000
       
   mysql_jsonendpoint:
     restart: always

statsd/endpoint/json/index.js

@@ -8,6 +8,8 @@ var app = express();
 
 var request = require('request');
 
+var timeout = process.env.TIMEOUT;
+
 app.use(morgan('combined'));
 app.use(bodyParser.json());
 
@@ -25,20 +27,28 @@ app.all('/', function(req, res) {
 
 function search(url, body, res)
 {
-    request.post(
-        url,
-        body,
+    request.post({
+            url: url,
+            body: JSON.stringify(body),
+            timeout: timeout
+        },
         function (error, response, body) {
             if (!error && response.statusCode == 200) {
-                if (result.length == 0) {
-                    result = JSON.parse(body);
+                if (search_result.length == 0) {
+                    search_result = JSON.parse(body);
                 } else {
-                    result.concat(JSON.parse(body));
+                    var values = JSON.parse(body);
+                    for (var i = 0; i < values.length - 1; i++) {
+                        if (search_result.indexOf(values[i]) === -1) {
+                            search_result.push(values[i]);
+                        }
+                    }
+                    search_result.sort();
                 }
                 
-                if (res) {
+                if (res !== undefined) {
                     setCORSHeaders(res);
-                    res.json(result);
+                    res.json(search_result);
                     res.end();
                 }
             } else {
@@ -49,23 +59,24 @@ function search(url, body, res)
 }
 
 function query(url, req, res)
-{
+{       
     request.post({
-        url: url,
-        headers: req.headers,
-        body: JSON.stringify(req.body)
-    },
+            url: url,
+            headers: req.headers,
+            body: JSON.stringify(req.body),
+            timeout: timeout
+        },
         function (error, response, body) {
             if (!error && response.statusCode == 200) {
-                if (queryResult.length == 0) {
-                    queryResult = JSON.parse(body);
+                if (query_result.length == 0) {
+                    query_result = JSON.parse(body);
                 } else {
-                    queryResult.concat(JSON.parse(body));
+                    query_result = query_result.concat(JSON.parse(body));
                 }
                 
-                if (res) {
+                if (res !== undefined) {
                     setCORSHeaders(res);
-                    res.json(queryResult);
+                    res.json(query_result);
                     res.end();
                 }
             } else {
@@ -75,16 +86,16 @@ function query(url, req, res)
     );
 }
 
-var result = [];
+var search_result = [];
 app.all('/search', function (req, res) {
-    result = [];
+    search_result = [];
     search('http://mysql_jsonendpoint:8000/search', req.body);
     search('http://mongodb_jsonendpoint:8000/search', req.body, res);
 });
 
-var queryResult = [];
+var query_result = [];
 app.all('/query', function (req, res) {
-    queryResult = [];
+    query_result = [];
     query('http://mysql_jsonendpoint:8000/query', req);
     query('http://mongodb_jsonendpoint:8000/query', req, res);
 });

statsd/endpoint/mongodb/index.js

@@ -36,9 +36,7 @@ app.all('/', function(req, res) {
   res.end();
 });
 
-var result = [];
 app.all('/search', function (req, res) {
-    setCORSHeaders(res);
     mongo.connect("mongodb://" + options.host + "/" + options.name, function (err, db) {
         if (err) {
             console.log(err);
@@ -47,20 +45,22 @@ app.all('/search', function (req, res) {
             console.log("Connected successfully to server");
         }
         db.listCollections().toArray(function(err, collInfos) {
-            result = [];
-            for (var i = 0; i <= collInfos.length -1; i++) {
-                if (collInfos[i]) {
-                    result.push(collInfos[i].name);
+            var mongo_search_result = [];
+            _.each(collInfos, function(collInfo) {
+                if (mongo_search_result.indexOf(collInfo.name) === -1) {
+                    mongo_search_result.push(collInfo.name);
                 }
-            } 
+            });
+            
+            setCORSHeaders(res);
+            res.json(mongo_search_result);
+            res.end();
         });
-        res.json(result);
-        res.end();
     });
 });
 
-var tsResult = [];
 app.all('/query', function (req, res) {
+    var mongo_query_result = [];
     var from = new Date(req.body.range.from);
     var to = new Date(req.body.range.to);
     var from_str = Math.floor(from.getTime() / 1000);
@@ -68,7 +68,6 @@ app.all('/query', function (req, res) {
     var names = _.map(req.body.targets, function (t) {
         return t.target;
     });
-    var name = names.join('", "');
     var interval = req.body.intervalMs / 1000;
     var maxDataPoints = req.body.maxDataPoints;
 
@@ -77,39 +76,43 @@ app.all('/query', function (req, res) {
             console.log(err);
         }
 //        https://docs.mongodb.com/manual/reference/method/db.collection.find/#db.collection.find
-        db.collection(""+name).find({ time: { $gte: from_str, $lte: to_str } }, { type: 1, time: 1, count:1, durations: 1, gauge: 1, set: 1, $slice: maxDataPoints }).sort({ time: 1 }).toArray(function (err, docs) {
-            if (err) {
-                console.log(err);
-            }
-            var result = {};
-            result[name] = new Array();
-            _.each(docs, function (d) {
-                var value = 0;
-                if (d.type == 'counters') {
-                    value = d.count;
-                } else if (d.type == 'timers') {
-                    value = d.durations;
-                } else if (d.type == 'gauges') {
-                    value = d.gauge;
-                } else if (d.type == 'sets') {
-                    value = d.set;
+        _.each(names, function(name, index) {
+            db.collection(""+name).find({ time: { $gte: from_str, $lte: to_str } }, { type: 1, time: 1, count:1, durations: 1, gauge: 1, set: 1, $slice: maxDataPoints }).sort({ time: 1 }).toArray(function (err, docs) {
+                if (err) {
+                    console.log(err);
                 }
-                (result[name]).push([value, 1000 * d.time]);
-            });
+                var result = {};
+                result[name] = new Array();
+                _.each(docs, function (doc) {
+                    var value = 0;
+                    if (doc.type == 'counters') {
+                        value = doc.count;
+                    } else if (doc.type == 'timers') {
+                        value = doc.durations;
+                    } else if (doc.type == 'gauges') {
+                        value = doc.gauge;
+                    } else if (doc.type == 'sets') {
+                        value = doc.set;
+                    }
+                    (result[name]).push([value, 1000 * doc.time]);
+                });
 
-            tsResult = [];
-            _.each(_.keys(result), function (d) {
-                var data = {
-                    target: d,
-                    datapoints: result[d]
-                };
-                tsResult.push(data);
+                _.each(_.keys(result), function (key) {
+                    var data = {
+                        target: key,
+                        datapoints: result[key]
+                    };
+                    mongo_query_result.push(data);
+                });
+                
+                if (index === names.length -1) {
+                    setCORSHeaders(res);
+                    res.json(mongo_query_result);
+                    res.end();
+                }
             });
-            
-            setCORSHeaders(res);
-            res.json(tsResult);
-            res.end();
         });
+        
     });
 });
 

statsd/endpoint/mysql/index.js

@@ -4,7 +4,6 @@ var bodyParser = require('body-parser');
 var _ = require('lodash');
 var app = express();
 
-
 var mysql      = require('mysql');
 var connection =  mysql.createConnection({
   host     : 'mysql',
@@ -17,57 +16,6 @@ var connection =  mysql.createConnection({
 app.use(morgan('combined'))
 
 app.use(bodyParser.json());
-
-var timeserie = [
-	{'target': 's1', 'datapoints':  [ [0,0], [1,0], [2,0], [3,0], [4,0] , [5,0], [6,0] , [7,0], [8,0], [9,0], [10,0], [11,0], [12,0], [13,0], [14,0], [15,0]]},
-	{'target': 's2', 'datapoints':  [ [15,0], [14,0], [13,0], [12,0], [11,0] , [10,0], [9,0] , [8,0], [7,0], [6,0], [5,0], [4,0], [3,0], [2,0], [1,0], [0,0]]},
-	{'target': 's3', 'datapoints':  [ [0,0], [1,0], [2,0], [3,0], [4,0] , [5,0], [6,0] , [7,0], [8,0], [9,0], [10,0], [11,0], [12,0], [13,0], [14,0], [15,0]]},
-	{'target': 's4', 'datapoints':  [ [15,0], [14,0], [13,0], [12,0], [11,0] , [10,0], [9,0] , [8,0], [7,0], [6,0], [5,0], [4,0], [3,0], [2,0], [1,0], [0,0]]},
-	{'target': 's5', 'datapoints':  [ [0,0], [1,0], [2,0], [3,0], [4,0] , [5,0], [6,0] , [7,0], [8,0], [9,0], [10,0], [11,0], [12,0], [13,0], [14,0], [15,0]]},
-	{'target': 's6', 'datapoints':  [ [15,0], [14,0], [13,0], [12,0], [11,0] , [10,0], [9,0] , [8,0], [7,0], [6,0], [5,0], [4,0], [3,0], [2,0], [1,0], [0,0]]}
-	]; //require('./series');
-
-var now = Date.now();
-
-for (var i = timeserie.length -1; i >= 0; i--) {
-  var series = timeserie[i];
-  var decreaser = 0;
-  for (var y = series.datapoints.length -1; y >= 0; y--) {
-    series.datapoints[y][1] = Math.round((now - decreaser) /1000) * 1000;
-    decreaser += 50000;
-  }
-}
-
-var annotation = {
-  name : "annotation name",
-  enabled: true,
-  datasource: "generic datasource",
-  showLine: true,
-}
-
-var annotations = [
-  { annotation: annotation, "title": "Donlad trump is kinda funny", "time": 1450754160000, text: "teeext", tags: "taaags" },
-  { annotation: annotation, "title": "Wow he really won", "time": 1450754160000, text: "teeext", tags: "taaags" },
-  { annotation: annotation, "title": "When is the next ", "time": 1450754160000, text: "teeext", tags: "taaags" }
-];
-
-var now = Date.now();
-var decreaser = 0;
-for (var i = 0;i < annotations.length; i++) {
-  var anon = annotations[i];
-  anon.time = (now - decreaser);
-  decreaser += 1000000
-}
-
-var table =
-  {
-    columns: [{text: 'Time', type: 'time'}, {text: 'Country', type: 'string'}, {text: 'Number', type: 'number'}],
-    values: [
-      [ 1234567, 'SE', 123 ],
-      [ 1234567, 'DE', 231 ],
-      [ 1234567, 'US', 321 ],
-    ]
-  };
   
 function setCORSHeaders(res) {
   res.setHeader("Access-Control-Allow-Origin", "*");
@@ -75,49 +23,45 @@ function setCORSHeaders(res) {
   res.setHeader("Access-Control-Allow-Headers", "accept, content-type");  
 }
 
-
-var now = Date.now();
-var decreaser = 0;
-for (var i = 0;i < table.values.length; i++) {
-  var anon = table.values[i];
-
-  anon[0] = (now - decreaser);
-  decreaser += 1000000
-}
-
 app.all('/', function(req, res) {
   setCORSHeaders(res);
   res.send('https://grafana.com/plugins/grafana-simple-json-datasource\n');
   res.end();
 });
 
-var result = [];
-function search(table)
+function search(table, res)
 {
-    connection.query('SELECT `name` FROM `' + table + '` GROUP BY `name` ORDER BY `name`', function (err, rows, fields) {
-        if (err)
+    connection.query('SELECT DISTINCT(`name`) FROM `' + table + '` WHERE `name` NOT IN ("' + mysql_search_result.join('", "') + '") ORDER BY `name`', function (err, rows, fields) {
+        if (err) {
+            console.log(err);
             throw err;
-        _.each(rows, function (ts) {
-            if (result.indexOf(ts.name) === -1) { // Se agregan solo los name que no esten en result
-                result.push(ts.name);
+        }
+        for (var i = 0; i <= rows.length - 1; i++) {
+            if (mysql_search_result.indexOf(rows[i].name) === -1) { // Se agregan solo los name que no esten en result
+                mysql_search_result.push(rows[i].name);
             }
-        });
+        }
     });
-
 }
+
+var mysql_search_result = [];
+
+search('gauges_statistics');
+search('counters_statistics');
+search('timers_statistics');
+search('sets_statistics');
+
 app.all('/search', function (req, res) {
     setCORSHeaders(res);
+    res.json(mysql_search_result);
+    res.end();
     
     search('gauges_statistics');
     search('counters_statistics');
     search('timers_statistics');
     search('sets_statistics');
-        
-    res.json(result);
-    res.end();
 });
 
-var tsResult = [];
 function query(table, req, res)
 {
     var from = new Date(req.body.range.from);
@@ -139,37 +83,41 @@ function query(table, req, res)
     var maxDataPoints = req.body.maxDataPoints;
     var limit = ' LIMIT ' + maxDataPoints + ';';
     sql += group_by + order_by + limit;  
-    
+        
     connection.query(sql, function (err, rows, fields) {
-        var result = {};
         if (err) {
             console.log(err);
+            throw err;
         }
-        _.each(rows[1], function (d) {
-            if (d.name) {
-                if (!(d.name in result)) {
-                    result[d.name] = new Array();
+        
+        var result = [];
+        for (var i = 0; i < rows.length - 1; i++) {
+            if (rows[i].name !== undefined) {
+                if (result.indexOf(rows[i].name) === -1) {
+                    result[rows[i].name] = new Array();   
                 }
-                (result[d.name]).push([d.value, 1000 * d.timestamp]);
+                (result[rows[i].name]).push([rows[i].value, 1000 * rows[i].timestamp]);
             }
-        });
-        _.each(_.keys(result), function (d) {
+        }
+        var keys = _.keys(result);
+        for (var i = 0; i < keys.length; i++) {
             var data = {
-                target: d,
-                datapoints: result[d]
+                target: keys[i],
+                datapoints: result[keys[i]]
             };
-            tsResult.push(data);
-        });
-        if (res) {
+            mysql_query_result.push(data);
+        }
+        if (res !== undefined) {
             setCORSHeaders(res);
-            res.json(tsResult);
+            res.json(mysql_query_result);
             res.end();
         }
     });
 }
 
+var mysql_query_result = [];
 app.all('/query', function (req, res) {
-    tsResult = [];
+    mysql_query_result = [];
     query('gauges_statistics', req);
     query('counters_statistics', req);
     query('timers_statistics', req);