v0.2.0 release

new settings: - keepFilesOnDelete - graphMaxPoints Graph legend div reformat + total raw points count Exporting raw data in graph time window to CSV
LowPowerLab · Aug 2, 2016 · 719e9fa · 719e9fa
1 parent c6f498d
commit 719e9fa
Show file tree

Hide file tree

Showing 6 changed files with 100 additions and 23 deletions.
diff --git a/gateway.js b/gateway.js
@@ -182,7 +182,7 @@ io.sockets.on('connection', function (socket) {
       io.sockets.emit('NODELISTREORDER', newOrder);
     });
   });
-          
+
   socket.on('UPDATENODESETTINGS', function (node) {
     db.find({ _id : node._id }, function (err, entries) {
       if (entries.length == 1)
@@ -248,15 +248,31 @@ io.sockets.on('connection', function (socket) {
   });
 
   socket.on('DELETENODE', function (nodeId) {
+    //delete all node-metric log files
+    console.log("DELETENODE settings.general.keepMetricLogsOnDelete.value: " + settings.general.keepMetricLogsOnDelete.value);
+    if (settings.general.keepMetricLogsOnDelete.value != 'true')
+      db.find({ _id : nodeId }, function (err, entries) {
+        if (entries.length == 1)
+        {
+          var dbNode = entries[0];
+          Object.keys(dbNode.metrics).forEach(function(mKey,index) {
+            if (dbNode.metrics[mKey].graph == 1)
+              dbLog.removeMetricLog(path.join(__dirname, dbDir, dbLog.getLogName(dbNode._id, mKey)));
+          });
+        }
+      });
+
+    //delete the node from the DB
     db.remove({ _id : nodeId }, function (err, removedCount) {
       console.log('DELETED entries: ' + removedCount);
-      
+
       //pull all nodes from the database and send them to client
       db.find({ _id : { $exists: true } }, function (err, entries) {
         io.sockets.emit('UPDATENODES', sortNodes(entries));
       });
     });
 
+    //remove scheduled events associated with the deleted node
     for(var s in scheduledEvents)
       if (scheduledEvents[s].nodeId == nodeId)
       {
@@ -273,6 +289,8 @@ io.sockets.on('connection', function (socket) {
         var dbNode = entries[0];
         dbNode.metrics[metricKey] = undefined;
         db.update({ _id: dbNode._id }, { $set : dbNode}, {}, function (err, numReplaced) { console.log('DELETENODEMETRIC DB-Replaced:' + numReplaced); });
+        if (settings.general.keepMetricLogsOnDelete.value != 'true')
+          dbLog.removeMetricLog(path.join(__dirname, dbDir, dbLog.getLogName(dbNode._id, metricKey)));
         io.sockets.emit('UPDATENODE', dbNode); //post it back to all clients to confirm UI changes
       }
     });
@@ -323,11 +341,11 @@ io.sockets.on('connection', function (socket) {
     else socket.emit('LOG', 'CANNOT INJECT NODE, INVALID NEW ID: ' + node.nodeId);
   });
 
-  socket.on('GETGRAPHDATA', function (nodeId, metricKey, start, end) {
+  socket.on('GETGRAPHDATA', function (nodeId, metricKey, start, end, exportMode) {
     var sts = Math.floor(start / 1000); //get timestamp in whole seconds
     var ets = Math.floor(end / 1000); //get timestamp in whole seconds
     var logfile = path.join(__dirname, dbDir, dbLog.getLogName(nodeId,metricKey));
-    var graphData = dbLog.getData(logfile, sts, ets);
+    var graphData = dbLog.getData(logfile, sts, ets, exportMode ? 100000 : settings.general.graphMaxPoints.value); //100k points when exporting, more points is really pointless
     var graphOptions={};
     for(var k in metricsDef.metrics)
     {
@@ -339,7 +357,11 @@ io.sockets.on('connection', function (socket) {
       }
     }
     graphOptions.metricName=metricKey;
-    socket.emit('GRAPHDATAREADY', { graphData:graphData, options : graphOptions });
+
+    if (exportMode)
+      socket.emit('EXPORTDATAREADY', { graphData:graphData, options : graphOptions });
+    else
+      socket.emit('GRAPHDATAREADY', { graphData:graphData, options : graphOptions });
   });
 
   socket.on('UPDATESETTINGSDEF', function (newSettings) {

diff --git a/logUtil.js b/logUtil.js
@@ -14,15 +14,16 @@
 //     NOTE: timestamps are whole integers, in seconds, so javascript timestamps have to be divided by 1000 before being passed in
 // *******************************************************************************
 var fs = require('fs');
-var metrics = require(require('path').resolve(__dirname,'metrics.js'));
+var path = require('path');
+var metrics = require(path.resolve(__dirname,'metrics.js'));
 
 exports.getLogName = function(nodeId, metricId) {
   return ('0000' + nodeId).slice(-4) + '_' + metricId + '.bin'; //left pad log names with zeros
 }
 
 exports.getData = function(filename, start, end, dpcount) {
-  dpcount = dpcount || 600;
-  if (dpcount>1500) dpcount = 1500;
+  dpcount = dpcount || 1500;
+  //if (dpcount>1500) dpcount = 1500;
   if (dpcount<1) dpcount = 1;
   if (dpcount<1 || start > end) return {};
 
@@ -56,7 +57,7 @@ exports.getData = function(filename, start, end, dpcount) {
       value = buff.readInt32BE(5);
       data.push({t:timetmp*1000, v:value/10000});
     }
-    return {data:data, queryTime:(new Date() - ts)};
+    return {data:data, queryTime:(new Date() - ts), totalIntervalDatapoints: (posEnd-posStart)/9+1 };
   }
 
   //too many data points, use binarySearch to aggregate
@@ -76,7 +77,7 @@ exports.getData = function(filename, start, end, dpcount) {
   }
   fs.closeSync(fd);
 
-  return {data:data, queryTime:(new Date() - ts)};
+  return {data:data, queryTime:(new Date() - ts), totalIntervalDatapoints: (posEnd-posStart)/9+1 };
 }
 
 exports.postData = function post(filename, timestamp, value) {
@@ -189,4 +190,14 @@ exports.binarySearchExact = function(fileDescriptor, timestamp, filesize) {
 
 exports.fileSize = function(filename) {
   return fs.existsSync(filename) ? fs.statSync(filename)['size'] : -1;
+}
+
+exports.removeMetricLog = function(logfile) {
+  if (exports.fileSize(logfile) >= 0)
+  {
+    fs.unlinkSync(logfile);
+    console.warn('removeMetricLog(): removed (' + logfile + ')');
+  }
+  else
+    console.log('removeMetricLog(): no log file found (' + logfile + ')');
 }
diff --git a/metrics.js b/metrics.js
@@ -72,7 +72,11 @@ exports.metrics = {
   START         : { name:'START', regexp:/START/i, value:'Started'},
 
   //WeatherShield metrics
+  //uncomment FtoC if you want a F:1234 to be valuated as a Centigrade isntead of F (the first match is picked up and will evaluate, any following defs are ignored)
+  //FtoC : { name:'C', regexp:/F\:(-?\d+\.\d+)/i, value:'', valuation:function(value) {return (value - 32) * 5/9;}, unit:'°', pin:1, graph:1, graphValSuffix:'C', graphOptions:{ legendLbl:'Temperature', lines: { lineWidth:1 }}}
   F : { name:'F', regexp:/F\:(-?\d+\.\d+)/i, value:'', unit:'°', pin:1, graph:1, graphValSuffix:'F', graphOptions:{ legendLbl:'Temperature', lines: { lineWidth:1 } }},
+  //uncomment FHtoC if you want a F:1234 to be valuated as a Centigrade isntead of F (the first match is picked up and will evaluate, any following defs are ignored)
+  //FHtoC : { name:'C', regexp:/F\:(-?\d+)/i, value:'', valuation:function(value) {return (value/100 - 32) * 5/9;}, unit:'°', pin:1, graph:1, graphValSuffix:'C', graphOptions:{ legendLbl:'Temperature', lines: { lineWidth:1 }}}
   FH : { name:'F', regexp:/F\:(-?\d+)/i, value:'', valuation:function(value) {return value/100;}, unit:'°', pin:1, graph:1, graphValSuffix:'F', graphOptions:{ legendLbl:'Temperature', lines: { lineWidth:1 }}},
   C : { name:'C', regexp:/C\:([-\d\.]+)/i, value:'', unit:'°', pin:1, graph:1, graphValSuffix:'C', graphOptions:{ legendLbl:'Temperature' }},
   H : { name:'H', regexp:/H\:([\d\.]+)/i, value:'', unit:'%', pin:1, graph:1, graphOptions:{ legendLbl:'Humidity', lines: { lineWidth:1 }}},
@@ -103,7 +107,7 @@ exports.metrics = {
   FSTATE : { name:'FSTATE', regexp:/FSTATE\:(AUTO|AUTOCIRC|ON)/i, value:''},
 
   //special metrics
-  V : { name:'V', regexp:/(?:V?BAT|VOLTS|V)\:(\d\.\d+)v?/i, value:'', unit:'v'},
+  V : { name:'V', regexp:/(?:V?BAT|VOLTS|V)\:(\d+\.\d+)v?/i, value:'', unit:'v'},
   //catchAll : { name:'CatchAll', regexp:/(\w+)\:(\w+)/i, value:''},
 };
 
@@ -122,7 +126,7 @@ exports.events = {
   motionSMS : { label:'Motion : SMS', icon:'comment', descr:'Send SMS when MOTION is detected', serverExecute:function(node) { if (node.metrics['M'] && node.metrics['M'].value == 'MOTION' && (Date.now() - new Date(node.metrics['M'].updated).getTime() < 2000)) { sendSMS('MOTION DETECTED', 'MOTION WAS DETECTED ON NODE: [' + node._id + ':' + node.label + '] @ ' + (new Date().toLocaleTimeString() + (new Date().getHours() > 12 ? 'PM':'AM'))); }; } },
   mailboxSMS : { label:'Mailbox open : SMS', icon:'comment', descr:'Send SMS when mailbox is opened', serverExecute:function(node) { if (node.metrics['M'] && node.metrics['M'].value == 'MOTION' && (Date.now() - new Date(node.metrics['M'].updated).getTime() < 2000)) { sendSMS('MAILBOX OPENED', 'Mailbox opened [' + node._id + ':' + node.label + '] @ ' + (new Date().toLocaleTimeString() + (new Date().getHours() > 12 ? 'PM':'AM'))); }; } },
   motionLightON23 : { label:'Motion: SM23 ON!', icon:'action', descr:'Turn SwitchMote:23 ON when MOTION is detected', serverExecute:function(node) { if (node.metrics['M'] && node.metrics['M'].value == 'MOTION' && (Date.now() - new Date(node.metrics['M'].updated).getTime() < 2000)) { sendMessageToNode({nodeId:23, action:'MOT:1'}); }; } },
-  
+
   doorbellSound : { label:'Doorbell : Sound', icon:'audio', descr:'Play sound when doorbell rings', serverExecute:function(node) { if (node.metrics['RING'] && node.metrics['RING'].value == 'RING' && (Date.now() - new Date(node.metrics['RING'].updated).getTime() < 2000)) { io.sockets.emit('PLAYSOUND', 'sounds/doorbell.wav'); }; } },
   doorbellSMS : { label:'Doorbell : SMS', icon:'comment', descr:'Send SMS when Doorbell button is pressed', serverExecute:function(node) { if (node.metrics['RING'] && node.metrics['RING'].value == 'RING' && (Date.now() - new Date(node.metrics['RING'].updated).getTime() < 2000)) { sendSMS('DOORBELL', 'DOORBELL WAS RINGED: [' + node._id + '] ' + node.label + ' @ ' + (new Date().toLocaleTimeString() + (new Date().getHours() > 12 ? 'PM':'AM'))); }; } },
   sumpSMS : { label:'SumpPump : SMS (below 20cm)', icon:'comment', descr:'Send SMS if water < 20cm below surface', serverExecute:function(node) { if (node.metrics['CM'] && node.metrics['CM'].value < 20 && (Date.now() - new Date(node.metrics['CM'].updated).getTime() < 2000)) { sendSMS('SUMP PUMP ALERT', 'Water is only 20cm below surface and rising - [' + node._id + '] ' + node.label + ' @ ' + (new Date().toLocaleTimeString() + (new Date().getHours() > 12 ? 'PM':'AM'))); }; } },
@@ -161,7 +165,7 @@ exports.events = {
       else console.log('thermostat_H73_PM IF(FAIL): day=' + (new Date().getDay()));
     }
   },
-    
+
   thermostat_H73_PM : { label:'Thermostat heat 73° @ 4:00PM weekdays', icon:'clock', descr:'Request heat point of 73° weekdays at 4pm',
     nextSchedule:function(node) { return exports.timeoutOffset(16,0); }, //ie 16:00 (4pm)
     scheduledExecute:function(node) {

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "RaspberryPi-Gateway",
-  "version": "0.1.0",
-  "description": "RaspberryPi Home Automation Gateway Sample Implementation",
+  "version": "0.2.0",
+  "description": "RaspberryPi Home Automation Gateway",
   "main": "gateway.js",
   "repository": "https://github.com/LowPowerLab/RaspberryPi-Gateway.git",
   "author": "Felix Rusu (https://github.com/LowPowerLab)",

diff --git a/settings.json5 b/settings.json5
@@ -56,6 +56,12 @@
       },
       genNodeIfNoMatch: {
         value: "false"
+      },
+	  keepMetricLogsOnDelete: {
+        value: "false"
+      },
+      graphMaxPoints: {
+        value: 800
       }
     },
     radiothermostat: {

diff --git a/www/index.html b/www/index.html
@@ -251,7 +251,7 @@ <h3><span id="metricDetailTitle">Metric details</span> <span class="metricUpdate
       <div class="ui-field-contain">
         <label for="metricLabel" class="labelbold">Label:</label>
         <input type="text" name="metricLabel" id="metricLabel" placeholder="metric label..." />
-        <label for="metricValue" class="labelbold">Value:</label>
+        <label for="metricValue" class="labelbold">Last value:</label>
         <input type="text" name="metricValue" id="metricValue" placeholder="value" readonly />
       </div>
       <div id="metricGraphWrapper" style="width:100%; height:280px;">
@@ -265,6 +265,7 @@ <h3><span id="metricDetailTitle">Metric details</span> <span class="metricUpdate
           <a id="graphZoomOut" href="#" data-role="button" data-icon="minus" data-iconpos="notext" data-theme="b" title="zoom out">ZoomOut</a>
           <a id="graphPanLeft" href="#" data-role="button" data-icon="arrow-l" data-iconpos="notext" data-theme="b" title="scroll left">Left</a>
           <a id="graphPanRight" href="#" data-role="button" data-icon="arrow-r" data-iconpos="notext" data-theme="b" title="scroll right">Right</a>
+          <a id="graphExport" href="#" data-role="button" data-icon="fa-download" data-iconpos="notext" data-theme="b" title="export raw data">Right</a>
           <!--<a id="graphReset" href="#" data-role="button" data-icon="refresh" data-iconpos="notext" data-theme="b" title="reset graph">Reset</a>-->
         </div>
       </div>
@@ -579,6 +580,30 @@ <h1>Settings</h1>
       //ask socket for the data
       socket.emit('GETGRAPHDATA', selectedNodeId, selectedMetricKey, graphView.start, graphView.end);
     }
+
+    function exportGraph() {
+      socket.emit('GETGRAPHDATA', selectedNodeId, selectedMetricKey, graphView.start, graphView.end, true);
+    }
+
+    socket.on('EXPORTDATAREADY', function(rawData) {
+      //package and stream the data to the browser
+      graphData = [];
+      var csv = 'data:text/csv;charset=utf-8,unix_timestamp(ms),'+rawData.options.legendLbl+'\n';
+
+      rawData.graphData.data.forEach(function(logItem, index){
+         dataString = logItem.t + ',' + logItem.v;
+         csv += index < rawData.graphData.data.length ? dataString+ '\n' : dataString;
+      });
+
+      var encodedUri = encodeURI(csv);
+      var link = document.createElement('a');
+      link.setAttribute('id', 'exportLink');
+      link.setAttribute('href', encodedUri);
+      link.setAttribute('download', selectedNodeId+'_'+rawData.options.legendLbl+'.csv');
+      document.body.appendChild(link); // Required for FF
+      link.click(); //make it stream to the browser
+      $('#exportLink').remove();
+    });
 
     socket.on('GRAPHDATAREADY', function(rawData){
       graphData = [];
@@ -604,11 +629,17 @@ <h1>Settings</h1>
       graphOptions.yaxis.max = max;
 
       graphOptions = $.extend(true, graphOptions, rawData.options); //http://stackoverflow.com/questions/171251/how-can-i-merge-properties-of-two-javascript-objects-dynamically
-      $('#graphStat').html(rawData.graphData.msg != undefined ? rawData.graphData.msg : (rawData.graphData.data.length + 'pts, ' + rawData.graphData.queryTime + 'ms'));
+      $(graphStat).html(rawData.graphData.msg != undefined ? rawData.graphData.msg : (rawData.graphData.data.length + (rawData.graphData.totalIntervalDatapoints != rawData.graphData.data.length ? ' / '+rawData.graphData.totalIntervalDatapoints : '') +'pts ('+ rawData.graphData.queryTime+'ms)'));
       //need to defer plotting until after pageshow is finished rendering, otherwise the wrapper will return an incorrect width of "100"
       if (metricGraphWrapper.width()==100)
-        $(document).on("pageshow", "#metricdetails", renderPlot);
-      else renderPlot();
+        $(document).on("pageshow", "#metricdetails", function() { 
+          renderPlot();
+          $(graphStat).clone().appendTo('#metricGraph');
+        });
+      else {
+        renderPlot();
+        $(graphStat).clone().appendTo('#metricGraph');
+      }
     });
 
     $(window).resize(function(){
@@ -618,6 +649,7 @@ <h1>Settings</h1>
         graphOptions.xaxis.min = graphView.start;
         graphOptions.xaxis.max = graphView.end;
         $.plot(metricGraph, [{label:graphOptions.legendLbl, data:graphData}], graphOptions);
+        $(graphStat).clone().appendTo('#metricGraph');
       }
     });
 
@@ -628,6 +660,7 @@ <h1>Settings</h1>
     $('#graphPanRight').click(function () {graphView.panright(); refreshGraph(true);});
     $('#graphPanLeft').click(function () {graphView.panleft(); refreshGraph(true);});
     //$('#graphReset').click(function () {graphView.resetDomain(); refreshGraph();});
+    $('#graphExport').click(function (e) { e.preventDefault(); exportGraph(); });
     $('.graphControl').click(function () {graphView.setDomain($(this).attr("hours")); refreshGraph();});
     metricGraph.bind("plotselected", function (event, ranges)
     {
@@ -649,6 +682,7 @@ <h1>Settings</h1>
       //plot.setData([graphData]);
       //plot.setupGrid(); //only necessary if your new data will change the axes or grid
       //plot.draw();
+      $(graphStat).clone().appendTo('#metricGraph');
     }
 
     function metricsValues(metrics) {
@@ -1131,15 +1165,15 @@ <h1>Settings</h1>
 		}).appendTo("body");
 
     //graph query statistics
-    $("<span id='graphStat'></span>").css({
+    var graphStat = $("<span id='graphStat'></span>").css({
 			position: "relative",
-      bottom:"85px",
-      right:"-35px",
+      top:"5px",
+      right:"-45px",
       fontSize: "10px",
       color:'#00ff11',
       'font-weight':'bold',
       'text-shadow':'0 1px 0 black',
-		}).appendTo("#metricGraphWrapper");
+		});
   });
   }
   </script>