diff --git a/lib/tasks/meta.js b/lib/tasks/meta.js index 9402daa3..992deaa2 100644 --- a/lib/tasks/meta.js +++ b/lib/tasks/meta.js @@ -11,16 +11,19 @@ var fs = require('fs'), request = require('request'), JSONStream = require('JSONStream'), through = require('through2'), + split = require('split2'), metafiles = require('./metafiles'); // tsv parser -var parse = require('csv-parse'); +var parse = require('csv-parse').parse; var parser = function( columns ) { var options = { delimiter: '\t', + comment: '', quote: false, trim: true, + bom: true, columns: columns }; @@ -54,6 +57,12 @@ var download = function( filename ) // Parse tsv and write to jsonify stream download + .pipe( split() ) + .pipe( through(function (row, enc, next) { + var line = row.toString('utf8'); + if (line[0] !== '#'){ this.push(line + '\n'); } + next(); + })) .pipe( parser( fileoptions.columns ) ) .pipe( through.obj( function( row, enc, next ){ var jsonIndex = row[ fileoptions.index ].replace( /[^\w]/g, '' ); diff --git a/package.json b/package.json index 41d821c5..2d0ca960 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "homepage": "https://pelias.io", "license": "MIT", "scripts": { - "download_metadata": "mkdirp metadata && node bin/updateMetadata.js", + "download_metadata": "mkdir -p metadata && node bin/updateMetadata.js", "download": "./bin/download", "countryCodes": "node bin/viewCountryCodes.js", "functional": "./bin/functional", @@ -33,15 +33,14 @@ "url": "https://github.com/pelias/geonames/issues" }, "engines": { - "node": ">=l2.0.0" + "node": ">=12.0.0" }, "dependencies": { - "JSONStream": "^1.0.7", "cli-table": "^0.3.0", - "csv-parse": "^4.8.2", + "csv-parse": "^5.0.4", "geonames-stream": "^2.0.3", + "JSONStream": "^1.0.7", "lodash": "^4.17.15", - "mkdirp": "^1.0.0", "pelias-blacklist-stream": "^1.2.0", "pelias-config": "^4.12.0", "pelias-dbclient": "^2.13.0", @@ -49,6 +48,7 @@ "pelias-model": "^9.0.0", "pelias-wof-admin-lookup": "^7.3.0", "request": "^2.34.0", + "split2": "^4.1.0", "through2": "^3.0.0", "through2-filter": "^3.0.0", "through2-sink": "^1.0.0",