feat(metadata): handle CSV comments (pelias#405)

lib/tasks/meta.js

@@ -11,16 +11,19 @@ var fs = require('fs'),
     request = require('request'),
     JSONStream = require('JSONStream'),
     through = require('through2'),
+    split = require('split2'),
     metafiles = require('./metafiles');
 
 // tsv parser
-var parse = require('csv-parse');
+var parse = require('csv-parse').parse;
 var parser = function( columns )
 {
   var options = {
     delimiter: '\t',
+    comment: '',
     quote: false,
     trim: true,
+    bom: true,
     columns: columns
   };
 
@@ -54,6 +57,12 @@ var download = function( filename )
 
   // Parse tsv and write to jsonify stream
   download
+    .pipe( split() )
+    .pipe( through(function (row, enc, next) {
+      var line = row.toString('utf8');
+      if (line[0] !== '#'){ this.push(line + '\n'); }
+      next();
+    }))
     .pipe( parser( fileoptions.columns ) )
     .pipe( through.obj( function( row, enc, next ){
       var jsonIndex = row[ fileoptions.index ].replace( /[^\w]/g, '' );

package.json

@@ -6,7 +6,7 @@
   "homepage": "https://pelias.io",
   "license": "MIT",
   "scripts": {
-    "download_metadata": "mkdirp metadata && node bin/updateMetadata.js",
+    "download_metadata": "mkdir -p metadata && node bin/updateMetadata.js",
     "download": "./bin/download",
     "countryCodes": "node bin/viewCountryCodes.js",
     "functional": "./bin/functional",
@@ -33,22 +33,22 @@
     "url": "https://github.com/pelias/geonames/issues"
   },
   "engines": {
-    "node": ">=l2.0.0"
+    "node": ">=12.0.0"
   },
   "dependencies": {
-    "JSONStream": "^1.0.7",
     "cli-table": "^0.3.0",
-    "csv-parse": "^4.8.2",
+    "csv-parse": "^5.0.4",
     "geonames-stream": "^2.0.3",
+    "JSONStream": "^1.0.7",
     "lodash": "^4.17.15",
-    "mkdirp": "^1.0.0",
     "pelias-blacklist-stream": "^1.2.0",
     "pelias-config": "^4.12.0",
     "pelias-dbclient": "^2.13.0",
     "pelias-logger": "^1.4.1",
     "pelias-model": "^9.0.0",
     "pelias-wof-admin-lookup": "^7.3.0",
     "request": "^2.34.0",
+    "split2": "^4.1.0",
     "through2": "^3.0.0",
     "through2-filter": "^3.0.0",
     "through2-sink": "^1.0.0",