All node data CSV export & other fixes/improves

Export all node data (csv)
Add "excel timestamp" columns in CSV exports
Datepicker support for export date range popup
Settings description + tooltips
Settings page listview trigger create bug fix
Moved "Restart App" in main menu
Moved "Uptime" status after main menu
Added _example.js event override sample
fixed README.md titles
Fixed AM/PM bug in SMS messages
Updated exports.timeoutOffset
Setup script mkdir /data/db so pi would chown it

.setup/gatewaysetup.sh

@@ -96,6 +96,11 @@ sudo npm install --unsafe-perm --build-from-source
 sudo npm cache clean    #clear any caches/incomplete installs
 sudo mkdir $APPSRVDIR/logs -p
 
+#create db and empty placeholders so chown pi will override root permissions
+sudo mkdir $APPSRVDIR/data/db -p
+touch $APPSRVDIR/data/db/gateway.db
+touch $APPSRVDIR/data/db/gateway_nonmatches.db
+
 #create self signed certificate
 #WARNING: must do this *AFTER* the gateway app was git-cloned
 echo -e "${CYAN}************* STEP: Create self signed HTTPS certificate (5 year) *************${NC}"
@@ -158,7 +163,8 @@ sudo systemctl enable gateway.service
 sudo systemctl start gateway.service
 
 echo -e "${RED}Make sure: ${YLW}to edit your gateway settings from the UI or from settings.json5 (and restart to apply changes)${NC}"
-echo -e "${RED}By default ${YLW}the gateway app uses the GPIO serial port. If you use MoteinoUSB or another serial port you must edit the serial port setting or else the app will not receive messages from your Moteino nodes.${NC}"
+echo -e "${RED}By default ${YLW}the gateway app uses the GPIO serial port. Run ${GRN}raspi-config${NC} and ensure the GPIO serial is enabled and GPIO console is disabled.${NC}"
+echo -e "${YLW}If you use MoteinoUSB or another serial port you must edit the serial port setting or the app will not receive messages from your Moteino nodes.${NC}"
 echo -e "${RED}App restarts ${YLW}can be requested from the Gateway UI (power symbol button on settings page, or from the terminal via ${RED}sudo systemctl restart gateway.service${NC}"
 echo -e "${RED}Don't forget: ${YLW}install proftpd (choose standalone mode) if you plan to FTP transfer files to your Pi (very useful!) with ${GRN}sudo apt-get install proftpd${NC}"
 echo -e "${RED}Don't forget: ${YLW}install minicom - useful for serial port debugging with ${GRN}sudo apt-get install minicom${NC}"

README.md

@@ -3,7 +3,7 @@ RaspberryPi Gateway Web Interface for the Moteino IoT Framework
 By Felix Rusu (lowpowerlab.com/contact)
 <br/>
 
-###Features:
+### Features:
 - SSL Encrypted with self signed certificate
 - `auth_basic` authenticated
 - realtime websocket driven using node.js and socket.io
@@ -13,15 +13,15 @@ By Felix Rusu (lowpowerlab.com/contact)
 - [nodemailer](https://github.com/andris9/Nodemailer) for sending email (and SMS relayed via email)
 - [Font-awesome](http://htmlpreview.github.io/?https://github.com/dotcastle/jquery-mobile-font-awesome/blob/master/index.html) icons for jQuery-Mobile
 
-###License
+### License
 This source code is released under GPL 3.0 with the following ammendment:<br/>
 You are free to use, copy, distribute and transmit this Software for non-commercial purposes.
 For more details see [LICENSE](https://github.com/LowPowerLab/RaspberryPi-Gateway/blob/master/LICENSE)
 
-###Details & Setup Guide
+### Details & Setup Guide
 The full details of how to install this stack along with supporting webserver are published [here](http://lowpowerlab.com/gateway). There you will also find a pre-compiled Pi image that has this stack installed and ready to go. In addition there are hardware guidelines and requirements.
 
-###Quick reference:
+### Quick reference:
 - Copy the contents of this directory in `/home/pi/gateway`
 - run `npm install` in the `/home/pi/gateway` directory to install all node dependencies
 - Adjust any email/password/SMS settings in `settings.json5`
@@ -32,8 +32,8 @@ The full details of how to install this stack along with supporting webserver ar
 - if you are using a wi-fi dongle, edit your wifi password in `/etc/wpa_supplicant/wpa_supplicant.conf`
 - Ensure your `gateway.js` script runs at boot (see the [Pi Stack Setup guide for how to set that up with upstart](http://lowpowerlab.com/gateway/#pisetup) and the [Gateway app setup](http://lowpowerlab.com/gateway/#sourcecode)). You can always use the pre-compiled Pi image that has all these things ready to go (except the settings which you should revisit anyway); this image also has upstart already configured to run the `gateway.js` app at startup. Otherwise if you want to manually start the `gateway.js` app or see the output it generates to the console start it with `node gateway.js &`. If you want to manually start it and ensure it persists after you logout use `nohup node gateway.js &`
 
-###Video Overview & Demo
+### Video Overview & Demo
 https://www.youtube.com/watch?v=F15dEqZ4pMM
 
-###3rd party custom gateway setup overview
+### 3rd party custom gateway setup overview
 https://www.youtube.com/watch?v=DP83RJeTpUY

gateway.js

@@ -125,7 +125,7 @@ global.sendEmail = function(SUBJECT, BODY) {
 
 global.sendSMS = function(SUBJECT, BODY) {
   var mailOptions = {
-      from: 'Moteino Gateway <[email protected]>',
+      from: 'Gateway <[email protected]>',
       to: settings.credentials.smsAlertsTo.value, //your mobile carrier should have an email address that will generate a SMS to your phone
       subject: SUBJECT,
       text: BODY
@@ -297,7 +297,7 @@ io.sockets.on('connection', function (socket) {
         if (entries.length == 1)
         {
           var dbNode = entries[0];
-          Object.keys(dbNode.metrics).forEach(function(mKey,index) {
+          Object.keys(dbNode.metrics).forEach(function(mKey,index) { //syncronous/blocking call
             if (dbNode.metrics[mKey].graph == 1)
               dbLog.removeMetricLog(path.join(__dirname, dbDir, dbLog.getLogName(dbNode._id, mKey)));
           });
@@ -405,7 +405,29 @@ io.sockets.on('connection', function (socket) {
     else
       socket.emit('GRAPHDATAREADY', { graphData:graphData, options : graphOptions });
   });
-
+
+  socket.on('EXPORTNODELOGSCSV', function (nodeId, start, end, howManyPoints) {
+    var sts = Math.floor(start / 1000); //get timestamp in whole seconds
+    var ets = Math.floor(end / 1000); //get timestamp in whole seconds
+    var sets = [];
+
+    db.find({ _id : nodeId }, function (err, entries) {
+      if (entries.length == 1)
+      {
+        var dbNode = entries[0];
+        Object.keys(dbNode.metrics).forEach(function(mKey,index) { //syncronous/blocking call
+          if (dbNode.metrics[mKey].graph == 1) {
+            var logfile = path.join(__dirname, dbDir, dbLog.getLogName(dbNode._id, mKey));
+            var theData = dbLog.getData(logfile, sts, ets, howManyPoints /*settings.general.graphMaxPoints.value*/);
+            theData.label = dbNode.metrics[mKey].label || mKey;
+            sets.push(theData); //100k points when exporting, more points is really pointless
+          }
+        });
+        socket.emit('EXPORTNODELOGSCSVREADY', { sets:sets });
+      }
+    });
+  });
+
   socket.on('UPDATESETTINGSDEF', function (newSettings) {
     var settings = nconf.get('settings');
 

logUtil.js

@@ -33,6 +33,18 @@ exports.getData = function(filename, start, end, dpcount) {
   filesize = exports.fileSize(filename);
   if (filesize == -1) return {data:data, queryTime:0, msg:'no log data'};
   fd = fs.openSync(filename, 'r');
+
+  //truncate start/end to log time limits if necessary - this ensures good data resolution when time limits are out of bounds
+  var buff = new Buffer(9);
+  fs.readSync(fd, buff, 0, 9, 0);
+  var firstLogTimestamp = buff.readUInt32BE(1);
+  fs.readSync(fd, buff, 0, 9, filesize-9);
+  var lastLogTimestamp = buff.readUInt32BE(1); //read timestamp (bytes 0-3 in buffer)
+  if (start < firstLogTimestamp) start = firstLogTimestamp;
+  if (end > lastLogTimestamp) end = lastLogTimestamp;
+
+  //console.info('getData() [start,end] = ' + start + ', ' + end);
+
   interval = (end - start) / dpcount;
 
   // Ensure that interval request is less than 1, adjust number of datapoints to request if interval = 1
@@ -42,7 +54,6 @@ exports.getData = function(filename, start, end, dpcount) {
   }
 
   timetmp = 0;
-  buff = new Buffer(9);
 
   //first check if sequential reads (much faster) make sense
   posStart = exports.binarySearch(fd,start-interval,filesize);