Merge pull request #467 from pelias/remove_venue_parameter

feat(venue): Add option to disable venue import

README.md

@@ -59,6 +59,12 @@ See [the config](https://github.com/pelias/config) documentation for details on
 }
 ```
 
+The importer has the possibility to download or not the OSM venues.
+This ability is managed by the parameter "importVenues" as described below:
+
+| key | required | default | description |
+| `imports.openstreetmap.importVenues` | no | true | set to `true` to include venues in the data download and import process |
+
 ### Environment Settings
 
 - `imports.openstreetmap.datapath` - this is the directory which you downloaded the pbf file to

config/features.js

@@ -4,8 +4,13 @@
  imports. @see: https://github.com/pelias/pbf2json for more info.
 **/
 
+// default tags imported
 var tags = [
-  'addr:housenumber+addr:street',
+  'addr:housenumber+addr:street'
+];
+
+// tags corresponding to venues
+var venue_tags = [
   'amenity+name',
   'building+name',
   'shop+name',
@@ -40,4 +45,4 @@ var tags = [
   'aeroway~airport+name'
 ];
 
-module.exports = tags;
+module.exports = {tags,venue_tags};

schema.js

@@ -6,6 +6,7 @@ const Joi = require('joi');
 // datapath: string (required)
 // leveldbpath: string (required)
 // import: array of objects containing filename (optional)
+// importVenues: boolean (optional)
 // download: array of objects containing sourceURL (optional)
 // deduplicate: boolean (optional)
 module.exports = Joi.object().keys({
@@ -14,7 +15,8 @@ module.exports = Joi.object().keys({
       datapath: Joi.string(),
       leveldbpath: Joi.string(),
       import: Joi.array().items(Joi.object().keys({
-        filename: Joi.string()
+        filename: Joi.string(),
+        importVenues: Joi.boolean().default(true).truthy('yes').falsy('no').insensitive(true)
       }).requiredKeys('filename').unknown(true)),
       download: Joi.array().items(Joi.object().keys({
         sourceURL: Joi.string()

stream/multiple_pbfs.js

@@ -8,7 +8,11 @@ function createCombinedStream(){
   var defaultPath= require('pelias-config').generate().imports.openstreetmap;
 
   defaultPath.import.forEach(function( importObject){
-    var conf = {file: path.join(defaultPath.datapath, importObject.filename), leveldb: defaultPath.leveldbpath};
+    var conf = {
+      file: path.join(defaultPath.datapath, importObject.filename),
+      leveldb: defaultPath.leveldbpath,
+      importVenues: importObject.importVenues
+    };
     fullStream.append(function(next){
       logger.info('Creating read stream for: ' + conf.file);
       next(pbf.parser(conf));

stream/pbf.js

@@ -6,7 +6,7 @@
 
 var fs = require('fs'),
     pbf2json = require('pbf2json'),
-    settings = require('pelias-config').generate(),
+    settings = require('pelias-config').generate(require('../schema')),
     features = require('../config/features'),
     path = require('path');
 
@@ -44,7 +44,13 @@ function config(opts){
 
   // Use default parser tags
   if(!opts.tags){
-    opts.tags = features;
+    // check if we import venues
+    opts.importVenues = settings.imports.openstreetmap.import[0].importVenues;
+    if(opts.importVenues){
+      opts.tags = features.tags.concat(features.venue_tags);
+    } else {
+      opts.tags = features.tags;
+    }
   }
   return opts;
 }

test/config/features.js

@@ -6,59 +6,68 @@ module.exports.tests = {};
 // test exports
 module.exports.tests.interface = function(test, common) {
   test('interface: features', function(t) {
-    t.true(Array.isArray(features), 'valid taglist');
+    t.true(Array.isArray(features.tags), 'valid taglist');
     t.end();
   });
 };
 
 // ensure some tags are excluded
 module.exports.tests.blacklist = function(test, common) {
-  test('blacklist', function(t) {
+  test('blacklist default tags', function(t) {
     // see: https://github.com/pelias/openstreetmap/pull/280
-    t.true( features.indexOf('aeroway+name') <0 );
-    t.true( features.indexOf('aeroway~gate+name') <0 );
-    t.true( features.indexOf('railway+name') <0 );
-    t.true( features.indexOf('railway~rail+name') <0 );
+    t.true( features.tags.indexOf('aeroway+name') <0 );
+    t.true( features.tags.indexOf('aeroway~gate+name') <0 );
+    t.true( features.tags.indexOf('railway+name') <0 );
+    t.true( features.tags.indexOf('railway~rail+name') <0 );
     t.end();
   });
 };
 
 // ensure some tags are included
+// we exclude by default tags corresponding to venues
 module.exports.tests.whitelist = function(test, common) {
-  test('whitelist', function(t) {
-    t.false( features.indexOf('addr:housenumber+addr:street') <0 );
-    t.false( features.indexOf('amenity+name') <0 );
-    t.false( features.indexOf('building+name') <0 );
-    t.false( features.indexOf('shop+name') <0 );
-    t.false( features.indexOf('office+name') <0 );
-    t.false( features.indexOf('public_transport+name') <0 );
-    t.false( features.indexOf('cuisine+name') <0 );
-    t.false( features.indexOf('railway~tram_stop+name') <0 );
-    t.false( features.indexOf('railway~station+name') <0 );
-    t.false( features.indexOf('railway~halt+name') <0 );
-    t.false( features.indexOf('railway~subway_entrance+name') <0 );
-    t.false( features.indexOf('railway~train_station_entrance+name') <0 );
-    t.false( features.indexOf('sport+name') <0 );
-    t.false( features.indexOf('natural+name') <0 );
-    t.false( features.indexOf('tourism+name') <0 );
-    t.false( features.indexOf('leisure+name') <0 );
-    t.false( features.indexOf('historic+name') <0 );
-    t.false( features.indexOf('man_made+name') <0 );
-    t.false( features.indexOf('landuse+name') <0 );
-    t.false( features.indexOf('waterway+name') <0 );
-    t.false( features.indexOf('aerialway+name') <0 );
-    t.false( features.indexOf('craft+name') <0 );
-    t.false( features.indexOf('military+name') <0 );
-    t.false( features.indexOf('aeroway~terminal+name') <0 );
-    t.false( features.indexOf('aeroway~aerodrome+name') <0 );
-    t.false( features.indexOf('aeroway~helipad+name') <0 );
-    t.false( features.indexOf('aeroway~airstrip+name') <0 );
-    t.false( features.indexOf('aeroway~heliport+name') <0 );
-    t.false( features.indexOf('aeroway~areodrome+name') <0 );
-    t.false( features.indexOf('aeroway~spaceport+name') <0 );
-    t.false( features.indexOf('aeroway~landing_strip+name') <0 );
-    t.false( features.indexOf('aeroway~airfield+name') <0 );
-    t.false( features.indexOf('aeroway~airport+name') <0 );
+  test('whitelist default tags', function(t) {
+    t.false( features.tags.indexOf('addr:housenumber+addr:street') <0 );
+    t.true( features.venue_tags.indexOf('amenity+name') <0 );
+    t.end();
+  });
+};
+
+// ensure some venue tags are included
+module.exports.tests.whitelist = function(test, common) {
+  test('whitelist venue tags', function(t) {
+    t.false( features.venue_tags.indexOf('amenity+name') <0 );
+    t.false( features.venue_tags.indexOf('building+name') <0 );
+    t.false( features.venue_tags.indexOf('shop+name') <0 );
+    t.false( features.venue_tags.indexOf('office+name') <0 );
+    t.false( features.venue_tags.indexOf('public_transport+name') <0 );
+    t.false( features.venue_tags.indexOf('cuisine+name') <0 );
+    t.false( features.venue_tags.indexOf('railway~tram_stop+name') <0 );
+    t.false( features.venue_tags.indexOf('railway~station+name') <0 );
+    t.false( features.venue_tags.indexOf('railway~halt+name') <0 );
+    t.false( features.venue_tags.indexOf('railway~subway_entrance+name') <0 );
+    t.false( features.venue_tags.indexOf('railway~train_station_entrance+name') <0 );
+    t.false( features.venue_tags.indexOf('sport+name') <0 );
+    t.false( features.venue_tags.indexOf('natural+name') <0 );
+    t.false( features.venue_tags.indexOf('tourism+name') <0 );
+    t.false( features.venue_tags.indexOf('leisure+name') <0 );
+    t.false( features.venue_tags.indexOf('historic+name') <0 );
+    t.false( features.venue_tags.indexOf('man_made+name') <0 );
+    t.false( features.venue_tags.indexOf('landuse+name') <0 );
+    t.false( features.venue_tags.indexOf('waterway+name') <0 );
+    t.false( features.venue_tags.indexOf('aerialway+name') <0 );
+    t.false( features.venue_tags.indexOf('craft+name') <0 );
+    t.false( features.venue_tags.indexOf('military+name') <0 );
+    t.false( features.venue_tags.indexOf('aeroway~terminal+name') <0 );
+    t.false( features.venue_tags.indexOf('aeroway~aerodrome+name') <0 );
+    t.false( features.venue_tags.indexOf('aeroway~helipad+name') <0 );
+    t.false( features.venue_tags.indexOf('aeroway~airstrip+name') <0 );
+    t.false( features.venue_tags.indexOf('aeroway~heliport+name') <0 );
+    t.false( features.venue_tags.indexOf('aeroway~areodrome+name') <0 );
+    t.false( features.venue_tags.indexOf('aeroway~spaceport+name') <0 );
+    t.false( features.venue_tags.indexOf('aeroway~landing_strip+name') <0 );
+    t.false( features.venue_tags.indexOf('aeroway~airfield+name') <0 );
+    t.false( features.venue_tags.indexOf('aeroway~airport+name') <0 );
     t.end();
   });
 };

test/stream/pbf.js

@@ -94,6 +94,26 @@ module.exports.tests.parser = function(test, common) {
   });
 };
 
+// Disable venues import
+module.exports.tests.parser = function(test, common) {
+  test('Disable venues import', function(t) {
+    var localFakeGeneratedConfig = JSON.parse(JSON.stringify(fakeGeneratedConfig));  // "JSON.parse" to get a deep copy
+    var localFakeConfig = {
+      generate: function fakeGenerate() {
+        return localFakeGeneratedConfig;
+      }
+    };
+    var pbf = proxyquire('../../stream/pbf', {'pelias-config': localFakeConfig});
+    localFakeGeneratedConfig.imports.openstreetmap.import[0].importVenues = false;
+    var conf = pbf.config();
+    var expected = {
+      importVenues: false
+    };
+    t.equal(conf.importVenues, expected.importVenues, 'disable importVenues');
+    t.end();
+  });
+};
+
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {