feat(batch): clean up source normalization code

lib/parameters.js

@@ -1,13 +1,12 @@
-var fs = require( 'fs' );
-var util = require( 'util' );
-var glob = require( 'glob' );
-var path = require( 'path' );
-var _ = require('lodash');
+const fs = require('fs');
+const util = require('util');
+const glob = require('glob');
+const path = require('path');
+const _ = require('lodash');
+const minimist = require('minimist');
 
-var minimist = require( 'minimist' );
-
-var peliasConfig = require( 'pelias-config' ).generate();
-const logger = require('pelias-logger').get('openaddresses-import');
+const peliasConfig = require('pelias-config').generate();
+const OpenAddressesAPI = require('../utils/OpenAddressesAPI');
 
 /**
  * Interprets the command-line arguments passed to the script.
@@ -102,15 +101,8 @@ function getFullFileList(peliasConfig, args) {
     // otherwise return the requested files with full path
     return files.map(file => {
 
-      // remove file extension from source
-      const source = file.replace(/\.[^\/.]+$/, '');
-
-      // source definitions previously required a file extension.
-      // please remove file extensions from your ~/pelias.json file
-      // to silence these warning messages.
-      if (source !== file) {
-        logger.warn(`source definitions no longer require a file extension '${file}'`);
-      }
+      // normalize source
+      const source = OpenAddressesAPI.normalize(file);
 
       // search for files matching this source id, ending in either .geojson or .csv
       const found = glob.sync(`${source}.{csv,geojson}`, { cwd: args.dirPath, absolute: true });

utils/OpenAddressesAPI.js

@@ -1,6 +1,7 @@
 const _ = require('lodash');
 const axios = require('axios');
 const config = require('pelias-config');
+const logger = require('pelias-logger').get('openaddresses');
 const HOST = 'https://batch.openaddresses.io';
 
 class OpenAddressesAPI {
@@ -10,13 +11,22 @@ class OpenAddressesAPI {
   }
 
   // remove file extensions from 'source'
-  normalize(source) {
+  static normalize(source) {
     if (!_.isString(source)) { return source; }
-    return source.replace(/\.[^/.]+$/, '');
+    const norm = source.replace(/\.[^/.]+$/, '');
+
+    // source definitions previously required a file extension.
+    // please remove file extensions from your ~/pelias.json file
+    // to silence these warning messages.
+    if (source !== norm) {
+      logger.warn(`source definitions no longer require a file extension '${source}'`);
+    }
+
+    return norm;
   }
 
   // return the http url for a specific job id
-  url(job) {
+  static url(job) {
     return `${HOST}/api/job/${job}/output/source.geojson.gz`;
   }
 
@@ -25,11 +35,10 @@ class OpenAddressesAPI {
     return _.get(this.config, 'validated') === true;
   }
 
-  async lookup(filename) {
-    // normalize 'source' property
+  async lookup(source) {
     // support the 'validated' property for financial supporters
     const params = {
-      source: this.normalize(filename),
+      source,
       layer: 'addresses',
       validated: this.isValidatedModeEnabled() ? 'true' : 'false'
     };

utils/download_filtered.js

@@ -7,6 +7,9 @@ const temp = require('temp');
 const logger = require('pelias-logger').get('openaddresses-download');
 const Bottleneck = require('bottleneck/es5');
 
+const OpenAddressesAPI = require('./OpenAddressesAPI');
+const oa = new OpenAddressesAPI();
+
 function downloadFiltered(config, callback) {
   const targetDir = config.imports.openaddresses.datapath;
   const errorsFatal = config.get('imports.openaddresses.missingFilesAreFatal');
@@ -56,18 +59,10 @@ function downloadFiltered(config, callback) {
 }
 
 async function getSources(files) {
-  const OpenAddressesAPI = require('./OpenAddressesAPI');
-  const oa = new OpenAddressesAPI();
-
   return await Promise.all(files.map(async file => {
 
-    // source definitions previously required a file extension.
-    // please remove file extensions from your ~/pelias.json file
-    // to silence these warning messages.
-    let id = file.replace(/\.[^\/.]+$/, '');
-    if (id !== file) {
-      logger.warn(`source definitions no longer require a file extension '${file}'`);
-    }
+    // normalize source
+    let id = OpenAddressesAPI.normalize(file);
 
     // lookup the source using the OpenAddresses API
     // to find the most current job id and ensure validity
@@ -80,7 +75,7 @@ async function getSources(files) {
     }
 
     // valid source
-    return { id, url: oa.url(version.job) };
+    return { id, url: OpenAddressesAPI.url(version.job) };
   }));
 }