Refactor Staging/Production Alternate Host Implementation

.github/workflows/s3-bucket.yml

@@ -0,0 +1,44 @@
+name: S3 Bucket Test
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  test-on-os-node-matrix:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        node: [18, 20, 22]
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      S3_BUCKET: ${{ secrets.S3_BUCKET }}
+
+    name: Test S3 Bucket - Node ${{ matrix.node }} on ${{ matrix.os }}
+
+    steps:
+      - name: Checkout ${{ github.ref }}
+        uses: actions/checkout@v4
+
+      - name: Setup node ${{ matrix.node }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: NPM Install
+        run: npm install
+
+      - name: Show Environment Info
+        run: |
+          printenv
+          node --version
+          npm --version
+
+      - name: Run S3 Tests (against ${{ env.S3_BUCKET }} bucket)
+        run: |
+          npm run bucket ${{ env.S3_BUCKET }}
+          npm run test:s3
+        if: ${{ env.S3_BUCKET != '' }}
+

lib/install.js

@@ -233,3 +233,10 @@ function install(gyp, argv, callback) {
     });
   }
 }
+
+// setting an environment variable: node_pre_gyp_mock_s3 to any value
+// enables intercepting outgoing http requests to s3 (using nock) and
+// serving them from a mocked S3 file system (using mock-aws-s3)
+if (process.env.node_pre_gyp_mock_s3) {
+  require('./mock/http')();
+}

lib/main.js

@@ -72,12 +72,6 @@ function run() {
     return;
   }
 
-  // set binary.host when appropriate. host determines the s3 target bucket.
-  const target = prog.setBinaryHostProperty(command.name);
-  if (target && ['install', 'publish', 'unpublish', 'info'].indexOf(command.name) >= 0) {
-    log.info('using binary.host: ' + prog.package_json.binary.host);
-  }
-
   prog.commands[command.name](command.args, function(err) {
     if (err) {
       log.error(command.name + ' error');

lib/mock/http.js

@@ -0,0 +1,39 @@
+'use strict';
+
+module.exports = exports = http_mock;
+
+const fs = require('fs');
+const path = require('path');
+const nock = require('nock');
+const os = require('os');
+
+const log = require('npmlog');
+log.disableProgress(); // disable the display of a progress bar
+log.heading = 'node-pre-gyp'; // differentiate node-pre-gyp's logs from npm's
+
+function http_mock() {
+  log.warn('mocking http requests to s3');
+
+  const basePath = `${os.tmpdir()}/mock`;
+
+  nock(new RegExp('([a-z0-9]+[.])*s3[.]us-east-1[.]amazonaws[.]com'))
+    .persist()
+    .get(() => true) //a function that always returns true is a catch all for nock
+    .reply(
+      (uri) => {
+        const bucket = 'npg-mock-bucket';
+        const mockDir = uri.indexOf(bucket) === -1 ? `${basePath}/${bucket}` : basePath;
+        const filepath = path.join(mockDir, uri.replace(new RegExp('%2B', 'g'), '+'));
+
+        try {
+          fs.accessSync(filepath, fs.constants.R_OK);
+        } catch (e) {
+          return [404, 'not found\n'];
+        }
+
+        // mock s3 functions write to disk
+        // return what is read from it.
+        return [200, fs.createReadStream(filepath)];
+      }
+    );
+}

lib/mock/s3.js

@@ -0,0 +1,42 @@
+'use strict';
+
+module.exports = exports = s3_mock;
+
+const AWSMock = require('mock-aws-s3');
+const os = require('os');
+
+const log = require('npmlog');
+log.disableProgress(); // disable the display of a progress bar
+log.heading = 'node-pre-gyp'; // differentiate node-pre-gyp's logs from npm's
+
+function s3_mock() {
+  log.warn('mocking s3 operations');
+
+  AWSMock.config.basePath = `${os.tmpdir()}/mock`;
+
+  const s3 = AWSMock.S3();
+
+  // wrapped callback maker. fs calls return code of ENOENT but AWS.S3 returns
+  // NotFound.
+  const wcb = (fn) => (err, ...args) => {
+    if (err && err.code === 'ENOENT') {
+      err.code = 'NotFound';
+    }
+    return fn(err, ...args);
+  };
+
+  return {
+    listObjects(params, callback) {
+      return s3.listObjects(params, wcb(callback));
+    },
+    headObject(params, callback) {
+      return s3.headObject(params, wcb(callback));
+    },
+    deleteObject(params, callback) {
+      return s3.deleteObject(params, wcb(callback));
+    },
+    putObject(params, callback) {
+      return s3.putObject(params, wcb(callback));
+    }
+  };
+}

lib/node-pre-gyp.js

@@ -10,18 +10,13 @@ module.exports = exports;
  * Module dependencies.
  */
 
-// load mocking control function for accessing s3 via https. the function is a noop always returning
-// false if not mocking.
-exports.mockS3Http = require('./util/s3_setup').get_mockS3Http();
-exports.mockS3Http('on');
-const mocking = exports.mockS3Http('get');
-
-
 const fs = require('fs');
 const path = require('path');
 const nopt = require('nopt');
 const log = require('npmlog');
-log.disableProgress();
+log.disableProgress(); // disable the display of a progress bar
+log.heading = 'node-pre-gyp'; // differentiate node-pre-gyp's logs from npm's
+
 const napi = require('./util/napi.js');
 
 const EE = require('events').EventEmitter;
@@ -43,12 +38,6 @@ const cli_commands = [
 ];
 const aliases = {};
 
-// differentiate node-pre-gyp's logs from npm's
-log.heading = 'node-pre-gyp';
-
-if (mocking) {
-  log.warn(`mocking s3 to ${process.env.node_pre_gyp_mock_s3}`);
-}
 
 // this is a getter to avoid circular reference warnings with node v14.
 Object.defineProperty(exports, 'find', {
@@ -88,11 +77,8 @@ function Run({ package_json_path = './package.json', argv }) {
   });
 
   this.parseArgv(argv);
-
-  // this is set to true after the binary.host property was set to
-  // either staging_host or production_host.
-  this.binaryHostSet = false;
 }
+
 inherits(Run, EE);
 exports.Run = Run;
 const proto = Run.prototype;
@@ -216,67 +202,6 @@ proto.parseArgv = function parseOpts(argv) {
   log.resume();
 };
 
-/**
- * allow the binary.host property to be set at execution time.
- *
- * for this to take effect requires all the following to be true.
- * - binary is a property in package.json
- * - binary.host is falsey
- * - binary.staging_host is not empty
- * - binary.production_host is not empty
- *
- * if any of the previous checks fail then the function returns an empty string
- * and makes no changes to package.json's binary property.
- *
- *
- * if command is "publish" then the default is set to "binary.staging_host"
- * if command is not "publish" the the default is set to "binary.production_host"
- *
- * if the command-line option '--s3_host' is set to "staging" or "production" then
- * "binary.host" is set to the specified "staging_host" or "production_host". if
- * '--s3_host' is any other value an exception is thrown.
- *
- * if '--s3_host' is not present then "binary.host" is set to the default as above.
- *
- * this strategy was chosen so that any command other than "publish" or "unpublish" uses "production"
- * as the default without requiring any command-line options but that "publish" and "unpublish" require
- * '--s3_host production_host' to be specified in order to *really* publish (or unpublish). publishing
- * to staging can be done freely without worrying about disturbing any production releases.
- */
-proto.setBinaryHostProperty = function(command) {
-  if (this.binaryHostSet) {
-    return this.package_json.binary.host;
-  }
-  const p = this.package_json;
-  // don't set anything if host is present. it must be left blank to trigger this.
-  if (!p || !p.binary || p.binary.host) {
-    return '';
-  }
-  // and both staging and production must be present. errors will be reported later.
-  if (!p.binary.staging_host || !p.binary.production_host) {
-    return '';
-  }
-  let target = 'production_host';
-  if (command === 'publish' || command === 'unpublish') {
-    target = 'staging_host';
-  }
-  // the environment variable has priority over the default or the command line. if
-  // either the env var or the command line option are invalid throw an error.
-  const npg_s3_host = process.env.node_pre_gyp_s3_host;
-  if (npg_s3_host === 'staging' || npg_s3_host === 'production') {
-    target = `${npg_s3_host}_host`;
-  } else if (this.opts['s3_host'] === 'staging' || this.opts['s3_host'] === 'production') {
-    target = `${this.opts['s3_host']}_host`;
-  } else if (this.opts['s3_host'] || npg_s3_host) {
-    throw new Error(`invalid s3_host ${this.opts['s3_host'] || npg_s3_host}`);
-  }
-
-  p.binary.host = p.binary[target];
-  this.binaryHostSet = true;
-
-  return p.binary.host;
-};
-
 /**
  * Returns the usage instructions for node-pre-gyp.
  */

lib/pre-binding.js

@@ -19,7 +19,6 @@ exports.find = function(package_json_path, opts) {
     throw new Error(package_json_path + 'does not exist');
   }
   const prog = new npg.Run({ package_json_path, argv: process.argv });
-  prog.setBinaryHostProperty();
   const package_json = prog.package_json;
 
   versioning.validate_config(package_json, opts);