From 484f7ca0db6fe1a429f661d36695c79803fd013a Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Tue, 10 May 2022 12:29:31 -0700 Subject: [PATCH 01/14] Refactored mock setup. - mocking setup can be more easily used by automated tests and/or manually by developer. - mock modules moved to lib/mock. - mock modules are self contained and have a signature similar to that of other modules in the package. - simplified http mock to remove no longer needed interface. - modified http mock to support path style bucket access. - set http mock to work with default npg-mock-bucket bucket. --- lib/install.js | 7 +++ lib/mock/http.js | 40 ++++++++++++++++ lib/mock/s3.js | 42 +++++++++++++++++ lib/node-pre-gyp.js | 17 ++----- lib/util/s3_setup.js | 106 ++----------------------------------------- 5 files changed, 96 insertions(+), 116 deletions(-) create mode 100644 lib/mock/http.js create mode 100644 lib/mock/s3.js diff --git a/lib/install.js b/lib/install.js index 617dd866..bdbb3b9d 100644 --- a/lib/install.js +++ b/lib/install.js @@ -233,3 +233,10 @@ function install(gyp, argv, callback) { }); } } + +// setting an environment variable: node_pre_gyp_mock_s3 to any value +// enables intercepting outgoing http requests to s3 (using nock) and +// serving them from a mocked S3 file system (using mock-aws-s3) +if (process.env.node_pre_gyp_mock_s3) { + require('./mock/http')(); +} diff --git a/lib/mock/http.js b/lib/mock/http.js new file mode 100644 index 00000000..9f9bad10 --- /dev/null +++ b/lib/mock/http.js @@ -0,0 +1,40 @@ +'use strict'; + +module.exports = exports = http_mock; + +const fs = require('fs'); +const path = require('path'); +const nock = require('nock'); +const os = require('os'); + +const log = require('npmlog'); +log.disableProgress(); // disable the display of a progress bar +log.heading = 'node-pre-gyp'; // differentiate node-pre-gyp's logs from npm's + +function http_mock() { + log.warn('mocking http requests to s3'); + + const baseHostname = 's3.us-east-1.amazonaws.com'; + const basePath = `${os.tmpdir()}/mock`; + + nock(new RegExp('^.*' + baseHostname)) + .persist() + .get(() => true) //a function that always returns true is a catch all for nock + .reply( + (uri) => { + const bucket = 'npg-mock-bucket'; + const mockDir = uri.indexOf(bucket) === -1 ? `${basePath}/${bucket}` : basePath; + const filepath = path.join(mockDir, uri.replace('%2B', '+')); + + try { + fs.accessSync(filepath, fs.constants.R_OK); + } catch (e) { + return [404, 'not found\n']; + } + + // mock s3 functions write to disk + // return what is read from it. + return [200, fs.createReadStream(filepath)]; + } + ); +} diff --git a/lib/mock/s3.js b/lib/mock/s3.js new file mode 100644 index 00000000..076b995b --- /dev/null +++ b/lib/mock/s3.js @@ -0,0 +1,42 @@ +'use strict'; + +module.exports = exports = s3_mock; + +const AWSMock = require('mock-aws-s3'); +const os = require('os'); + +const log = require('npmlog'); +log.disableProgress(); // disable the display of a progress bar +log.heading = 'node-pre-gyp'; // differentiate node-pre-gyp's logs from npm's + +function s3_mock() { + log.warn('mocking s3 operations'); + + AWSMock.config.basePath = `${os.tmpdir()}/mock`; + + const s3 = AWSMock.S3(); + + // wrapped callback maker. fs calls return code of ENOENT but AWS.S3 returns + // NotFound. + const wcb = (fn) => (err, ...args) => { + if (err && err.code === 'ENOENT') { + err.code = 'NotFound'; + } + return fn(err, ...args); + }; + + return { + listObjects(params, callback) { + return s3.listObjects(params, wcb(callback)); + }, + headObject(params, callback) { + return s3.headObject(params, wcb(callback)); + }, + deleteObject(params, callback) { + return s3.deleteObject(params, wcb(callback)); + }, + putObject(params, callback) { + return s3.putObject(params, wcb(callback)); + } + }; +} diff --git a/lib/node-pre-gyp.js b/lib/node-pre-gyp.js index dc18e749..f6c262de 100644 --- a/lib/node-pre-gyp.js +++ b/lib/node-pre-gyp.js @@ -10,18 +10,13 @@ module.exports = exports; * Module dependencies. */ -// load mocking control function for accessing s3 via https. the function is a noop always returning -// false if not mocking. -exports.mockS3Http = require('./util/s3_setup').get_mockS3Http(); -exports.mockS3Http('on'); -const mocking = exports.mockS3Http('get'); - - const fs = require('fs'); const path = require('path'); const nopt = require('nopt'); const log = require('npmlog'); -log.disableProgress(); +log.disableProgress(); // disable the display of a progress bar +log.heading = 'node-pre-gyp'; // differentiate node-pre-gyp's logs from npm's + const napi = require('./util/napi.js'); const EE = require('events').EventEmitter; @@ -43,12 +38,6 @@ const cli_commands = [ ]; const aliases = {}; -// differentiate node-pre-gyp's logs from npm's -log.heading = 'node-pre-gyp'; - -if (mocking) { - log.warn(`mocking s3 to ${process.env.node_pre_gyp_mock_s3}`); -} // this is a getter to avoid circular reference warnings with node v14. Object.defineProperty(exports, 'find', { diff --git a/lib/util/s3_setup.js b/lib/util/s3_setup.js index 52839e3b..0095cd34 100644 --- a/lib/util/s3_setup.js +++ b/lib/util/s3_setup.js @@ -3,8 +3,6 @@ module.exports = exports; const url = require('url'); -const fs = require('fs'); -const path = require('path'); module.exports.detect = function(opts) { const config = {}; @@ -60,40 +58,11 @@ module.exports.detect = function(opts) { }; module.exports.get_s3 = function(config) { - + // setting an environment variable: node_pre_gyp_mock_s3 to any value + // enables intercepting outgoing http requests to s3 (using nock) and + // serving them from a mocked S3 file system (using mock-aws-s3) if (process.env.node_pre_gyp_mock_s3) { - // here we're mocking. node_pre_gyp_mock_s3 is the scratch directory - // for the mock code. - const AWSMock = require('mock-aws-s3'); - const os = require('os'); - - AWSMock.config.basePath = `${os.tmpdir()}/mock`; - - const s3 = AWSMock.S3(); - - // wrapped callback maker. fs calls return code of ENOENT but AWS.S3 returns - // NotFound. - const wcb = (fn) => (err, ...args) => { - if (err && err.code === 'ENOENT') { - err.code = 'NotFound'; - } - return fn(err, ...args); - }; - - return { - listObjects(params, callback) { - return s3.listObjects(params, wcb(callback)); - }, - headObject(params, callback) { - return s3.headObject(params, wcb(callback)); - }, - deleteObject(params, callback) { - return s3.deleteObject(params, wcb(callback)); - }, - putObject(params, callback) { - return s3.putObject(params, wcb(callback)); - } - }; + return require('../mock/s3')(); } // if not mocking then setup real s3. @@ -117,71 +86,4 @@ module.exports.get_s3 = function(config) { return s3.putObject(params, callback); } }; - - - }; - -// -// function to get the mocking control function. if not mocking it returns a no-op. -// -// if mocking it sets up the mock http interceptors that use the mocked s3 file system -// to fulfill responses. -module.exports.get_mockS3Http = function() { - let mock_s3 = false; - if (!process.env.node_pre_gyp_mock_s3) { - return () => mock_s3; - } - - const nock = require('nock'); - // the bucket used for testing, as addressed by https. - const host = 'https://mapbox-node-pre-gyp-public-testing-bucket.s3.us-east-1.amazonaws.com'; - const mockDir = process.env.node_pre_gyp_mock_s3 + '/mapbox-node-pre-gyp-public-testing-bucket'; - - // function to setup interceptors. they are "turned off" by setting mock_s3 to false. - const mock_http = () => { - // eslint-disable-next-line no-unused-vars - function get(uri, requestBody) { - const filepath = path.join(mockDir, uri.replace('%2B', '+')); - - try { - fs.accessSync(filepath, fs.constants.R_OK); - } catch (e) { - return [404, 'not found\n']; - } - - // the mock s3 functions just write to disk, so just read from it. - return [200, fs.createReadStream(filepath)]; - } - - // eslint-disable-next-line no-unused-vars - return nock(host) - .persist() - .get(() => mock_s3) // mock any uri for s3 when true - .reply(get); - }; - - // setup interceptors. they check the mock_s3 flag to determine whether to intercept. - mock_http(nock, host, mockDir); - // function to turn matching all requests to s3 on/off. - const mockS3Http = (action) => { - const previous = mock_s3; - if (action === 'off') { - mock_s3 = false; - } else if (action === 'on') { - mock_s3 = true; - } else if (action !== 'get') { - throw new Error(`illegal action for setMockHttp ${action}`); - } - return previous; - }; - - // call mockS3Http with the argument - // - 'on' - turn it on - // - 'off' - turn it off (used by fetch.test.js so it doesn't interfere with redirects) - // - 'get' - return true or false for 'on' or 'off' - return mockS3Http; -}; - - - From 728653c244a69a8a85d4c364c7186ca533563d2e Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Tue, 10 May 2022 12:30:32 -0700 Subject: [PATCH 02/14] Refactored test setup. - separated s3 tests from build tests. - refactored fetch test and proxy-bcrypt test to work with refactored mock setup. --- test/s3.test.js | 230 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 test/s3.test.js diff --git a/test/s3.test.js b/test/s3.test.js new file mode 100644 index 00000000..907b3bf8 --- /dev/null +++ b/test/s3.test.js @@ -0,0 +1,230 @@ +'use strict'; + +const test = require('tape'); +const run = require('./run.util.js'); +const existsSync = require('fs').existsSync || require('path').existsSync; +const fs = require('fs'); +const rm = require('rimraf'); +const path = require('path'); +const napi = require('../lib/util/napi.js'); +const versioning = require('../lib/util/versioning.js'); + +const localVer = [versioning.get_runtime_abi('node'), process.platform, process.arch].join('-'); +const SOEXT = { 'darwin': 'dylib', 'linux': 'so', 'win32': 'dll' }[process.platform]; + +// The list of different sample apps that we use to test +// apps with . in name are variation of app with different binary hosting setting +const apps = [ + { + 'name': 'app1', + 'args': '', + 'files': { + 'base': ['binding/app1.node'] + } + }, + { + 'name': 'app1.1', + 'args': '', + 'files': { + 'base': ['binding/app1.1.node'] + } + }, + { + 'name': 'app1.2', + 'args': '', + 'files': { + 'base': ['binding/app1.2.node'] + } + }, + { + 'name': 'app2', + 'args': '--custom_include_path=../include --debug', + 'files': { + 'base': ['node-pre-gyp-test-app2/app2.node'] + } + }, + { + 'name': 'app2', + 'args': '--custom_include_path=../include --toolset=cpp11', + 'files': { + 'base': ['node-pre-gyp-test-app2/app2.node'] + } + }, + { + 'name': 'app3', + 'args': '', + 'files': { + 'base': [[localVer, 'app3.node'].join('/')] + } + }, + { + 'name': 'app4', + 'args': '', + 'files': { + 'base': [[localVer, 'app4.node'].join('/'), [localVer, 'mylib.' + SOEXT].join('/')] + } + }, + { + 'name': 'app7', + 'args': '' + } +]; + + +// https://stackoverflow.com/questions/38599457/how-to-write-a-custom-assertion-for-testing-node-or-javascript-with-tape-or-che +test.Test.prototype.stringContains = function(actual, contents, message) { + this._assert(actual.indexOf(contents) > -1, { + message: message || 'should contain ' + contents, + operator: 'stringContains', + actual: actual, + expected: contents + }); +}; + +// Tests run for all apps + +apps.forEach((app) => { + + if (app.name === 'app7' && !napi.get_napi_version()) return; + + // clear out entire binding directory + // to ensure no stale builds. This is needed + // because "node-pre-gyp clean" only removes + // the current target and not alternative builds + test('cleanup of app', (t) => { + const binding_directory = path.join(__dirname, app.name, 'lib/binding'); + if (fs.existsSync(binding_directory)) { + rm.sync(binding_directory); + } + t.end(); + }); + + test(app.name + ' build ' + app.args, (t) => { + run('node-pre-gyp', 'rebuild', '--fallback-to-build', app, {}, (err, stdout, stderr) => { + t.ifError(err); + if (err) { + console.log(stdout); + console.log(stderr); + } + t.end(); + }); + }); + + + test(app.name + ' package ' + app.args, (t) => { + run('node-pre-gyp', 'package', '', app, {}, (err) => { + t.ifError(err); + // Make sure a tarball was created + run('node-pre-gyp', 'reveal', 'staged_tarball --silent', app, {}, (err2, stdout) => { + t.ifError(err2); + let staged_tarball = stdout.trim(); + if (staged_tarball.indexOf('\n') !== -1) { // take just the first line + staged_tarball = staged_tarball.substr(0, staged_tarball.indexOf('\n')); + } + const tarball_path = path.join(__dirname, app.name, staged_tarball); + t.ok(existsSync(tarball_path), 'staged tarball is a valid file'); + if (!app.files) { + return t.end(); + } + t.end(); + }); + }); + }); + + test(app.name + ' package is valid ' + app.args, (t) => { + run('node-pre-gyp', 'testpackage', '', app, {}, (err) => { + t.ifError(err); + t.end(); + }); + }); + + if (process.env.AWS_ACCESS_KEY_ID || process.env.node_pre_gyp_accessKeyId || process.env.node_pre_gyp_mock_s3) { + + test(app.name + ' publishes ' + app.args, (t) => { + run('node-pre-gyp', 'unpublish publish', '', app, {}, (err, stdout) => { + t.ifError(err); + t.notEqual(stdout, ''); + t.end(); + }); + }); + + test(app.name + ' info shows it ' + app.args, (t) => { + run('node-pre-gyp', 'reveal', 'package_name', app, {}, (err, stdout) => { + t.ifError(err); + let package_name = stdout.trim(); + if (package_name.indexOf('\n') !== -1) { // take just the first line + package_name = package_name.substr(0, package_name.indexOf('\n')); + } + run('node-pre-gyp', 'info', '', app, {}, (err2, stdout2) => { + t.ifError(err2); + t.stringContains(stdout2, package_name); + t.end(); + }); + }); + }); + + test(app.name + ' can be uninstalled ' + app.args, (t) => { + run('node-pre-gyp', 'clean', '', app, {}, (err, stdout) => { + t.ifError(err); + t.notEqual(stdout, ''); + t.end(); + }); + }); + + test(app.name + ' can be installed via remote ' + app.args, (t) => { + const opts = { + cwd: path.join(__dirname, app.name), + npg_debug: false + }; + run('npm', 'install', '--fallback-to-build=false', app, opts, (err, stdout) => { + t.ifError(err); + t.notEqual(stdout, ''); + t.end(); + }); + }); + + test(app.name + ' can be reinstalled via remote ' + app.args, (t) => { + const opts = { + cwd: path.join(__dirname, app.name), + npg_debug: false + }; + run('npm', 'install', '--update-binary --fallback-to-build=false', app, opts, (err, stdout) => { + t.ifError(err); + t.notEqual(stdout, ''); + t.end(); + }); + }); + + test(app.name + ' via remote passes tests ' + app.args, (t) => { + const opts = { + cwd: path.join(__dirname, app.name), + npg_debug: false + }; + run('npm', 'install', '', app, opts, (err, stdout) => { + t.ifError(err); + t.notEqual(stdout, ''); + t.end(); + }); + }); + + test(app.name + ' unpublishes ' + app.args, (t) => { + run('node-pre-gyp', 'unpublish', '', app, {}, (err, stdout) => { + t.ifError(err); + t.notEqual(stdout, ''); + t.end(); + }); + }); + + } else { + test.skip(app.name + ' publishes ' + app.args, () => {}); + } + + // note: the above test will result in a non-runnable binary, so the below test must succeed otherwise all following tests will fail + + test(app.name + ' builds with custom --target ' + app.args, (t) => { + run('node-pre-gyp', 'rebuild', '--loglevel=error --fallback-to-build --target=' + process.versions.node, app, {}, (err) => { + t.ifError(err); + t.end(); + }); + }); +}); From 937131c560cd881fa2188a19e4aa90ea8ed2fc88 Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Tue, 10 May 2022 12:32:42 -0700 Subject: [PATCH 03/14] Refactored test apps and increased test coverage - set hosts of test apps to point to mock bucket. - added app1.1 - identical to app1 but using production and staging binary host option. - added app1.2 - identical to app1 but using explicit host, region, bucket options. --- test/app1.1/.gitignore | 5 +++++ test/app1.1/README.md | 4 ++++ test/app1.1/app1.1.cc | 14 ++++++++++++++ test/app1.1/binding.gyp | 19 +++++++++++++++++++ test/app1.1/index.js | 6 ++++++ test/app1.1/package.json | 24 ++++++++++++++++++++++++ test/app1.2/.gitignore | 5 +++++ test/app1.2/README.md | 4 ++++ test/app1.2/app1.2.cc | 14 ++++++++++++++ test/app1.2/binding.gyp | 19 +++++++++++++++++++ test/app1.2/index.js | 6 ++++++ test/app1.2/package.json | 26 ++++++++++++++++++++++++++ test/app1/package.json | 2 +- test/app2/package.json | 2 +- test/app3/package.json | 2 +- test/app4/package.json | 2 +- test/app7/package.json | 2 +- 17 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 test/app1.1/.gitignore create mode 100644 test/app1.1/README.md create mode 100644 test/app1.1/app1.1.cc create mode 100644 test/app1.1/binding.gyp create mode 100644 test/app1.1/index.js create mode 100644 test/app1.1/package.json create mode 100644 test/app1.2/.gitignore create mode 100644 test/app1.2/README.md create mode 100644 test/app1.2/app1.2.cc create mode 100644 test/app1.2/binding.gyp create mode 100644 test/app1.2/index.js create mode 100644 test/app1.2/package.json diff --git a/test/app1.1/.gitignore b/test/app1.1/.gitignore new file mode 100644 index 00000000..f6a05031 --- /dev/null +++ b/test/app1.1/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +build/ +lib/binding/ +node_modules +npm-debug.log \ No newline at end of file diff --git a/test/app1.1/README.md b/test/app1.1/README.md new file mode 100644 index 00000000..1210b152 --- /dev/null +++ b/test/app1.1/README.md @@ -0,0 +1,4 @@ +# Test app + +Demonstrates a simple configuration that uses node-pre-gyp. +Identical to app1 but using production and staging binary host option. diff --git a/test/app1.1/app1.1.cc b/test/app1.1/app1.1.cc new file mode 100644 index 00000000..4c8a8d87 --- /dev/null +++ b/test/app1.1/app1.1.cc @@ -0,0 +1,14 @@ +#include + +Napi::Value get_hello(Napi::CallbackInfo const& info) { + Napi::Env env = info.Env(); + Napi::EscapableHandleScope scope(env); + return scope.Escape(Napi::String::New(env, "hello")); +} + +Napi::Object start(Napi::Env env, Napi::Object exports) { + exports.Set("hello", Napi::Function::New(env, get_hello)); + return exports; +} + +NODE_API_MODULE(app1, start) diff --git a/test/app1.1/binding.gyp b/test/app1.1/binding.gyp new file mode 100644 index 00000000..05ef4f1c --- /dev/null +++ b/test/app1.1/binding.gyp @@ -0,0 +1,19 @@ +{ + "targets": [ + { + "target_name": "<(module_name)", + "sources": [ "<(module_name).cc" ], + 'product_dir': '<(module_path)', + 'include_dirs': ["../../node_modules/node-addon-api/"], + 'cflags!': [ '-fno-exceptions' ], + 'cflags_cc!': [ '-fno-exceptions' ], + "xcode_settings": { + 'GCC_ENABLE_CPP_EXCEPTIONS': 'YES', + "CLANG_CXX_LIBRARY": "libc++" + }, + 'msvs_settings': { + 'VCCLCompilerTool': { 'ExceptionHandling': 1 }, + } + } + ] +} diff --git a/test/app1.1/index.js b/test/app1.1/index.js new file mode 100644 index 00000000..084fc583 --- /dev/null +++ b/test/app1.1/index.js @@ -0,0 +1,6 @@ +var binary = require('node-pre-gyp'); +var path = require('path') +var binding_path = binary.find(path.resolve(path.join(__dirname,'./package.json'))); +var binding = require(binding_path); + +require('assert').equal(binding.hello(),"hello"); \ No newline at end of file diff --git a/test/app1.1/package.json b/test/app1.1/package.json new file mode 100644 index 00000000..ddc9233b --- /dev/null +++ b/test/app1.1/package.json @@ -0,0 +1,24 @@ +{ + "name": "node-pre-gyp-test-app1.1", + "author": "Dane Springmeyer ", + "description": "node-pre-gyp test", + "repository": { + "type": "git", + "url": "git://github.com/mapbox/node-pre-gyp.git" + }, + "license": "BSD-3-Clause", + "version": "0.1.0", + "main": "./index.js", + "binary": { + "module_name": "app1.1", + "module_path": "./lib/binding/", + "staging_host": "https://npg-mock-bucket.s3.us-east-1.amazonaws.com", + "production_host": "https://npg-mock-bucket.s3.us-east-1.amazonaws.com", + "remote_path": "./node-pre-gyp/{name}/v{version}/{configuration}/{toolset}/", + "package_name": "{node_abi}-{platform}-{arch}.tar.gz" + }, + "scripts": { + "install": "node-pre-gyp install --fallback-to-build", + "test": "node index.js" + } +} diff --git a/test/app1.2/.gitignore b/test/app1.2/.gitignore new file mode 100644 index 00000000..f6a05031 --- /dev/null +++ b/test/app1.2/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +build/ +lib/binding/ +node_modules +npm-debug.log \ No newline at end of file diff --git a/test/app1.2/README.md b/test/app1.2/README.md new file mode 100644 index 00000000..5cbf7616 --- /dev/null +++ b/test/app1.2/README.md @@ -0,0 +1,4 @@ +# Test app + +Demonstrates a simple configuration that uses node-pre-gyp. +Identical to app1 but using explicit host, region, bucket options. diff --git a/test/app1.2/app1.2.cc b/test/app1.2/app1.2.cc new file mode 100644 index 00000000..4c8a8d87 --- /dev/null +++ b/test/app1.2/app1.2.cc @@ -0,0 +1,14 @@ +#include + +Napi::Value get_hello(Napi::CallbackInfo const& info) { + Napi::Env env = info.Env(); + Napi::EscapableHandleScope scope(env); + return scope.Escape(Napi::String::New(env, "hello")); +} + +Napi::Object start(Napi::Env env, Napi::Object exports) { + exports.Set("hello", Napi::Function::New(env, get_hello)); + return exports; +} + +NODE_API_MODULE(app1, start) diff --git a/test/app1.2/binding.gyp b/test/app1.2/binding.gyp new file mode 100644 index 00000000..05ef4f1c --- /dev/null +++ b/test/app1.2/binding.gyp @@ -0,0 +1,19 @@ +{ + "targets": [ + { + "target_name": "<(module_name)", + "sources": [ "<(module_name).cc" ], + 'product_dir': '<(module_path)', + 'include_dirs': ["../../node_modules/node-addon-api/"], + 'cflags!': [ '-fno-exceptions' ], + 'cflags_cc!': [ '-fno-exceptions' ], + "xcode_settings": { + 'GCC_ENABLE_CPP_EXCEPTIONS': 'YES', + "CLANG_CXX_LIBRARY": "libc++" + }, + 'msvs_settings': { + 'VCCLCompilerTool': { 'ExceptionHandling': 1 }, + } + } + ] +} diff --git a/test/app1.2/index.js b/test/app1.2/index.js new file mode 100644 index 00000000..084fc583 --- /dev/null +++ b/test/app1.2/index.js @@ -0,0 +1,6 @@ +var binary = require('node-pre-gyp'); +var path = require('path') +var binding_path = binary.find(path.resolve(path.join(__dirname,'./package.json'))); +var binding = require(binding_path); + +require('assert').equal(binding.hello(),"hello"); \ No newline at end of file diff --git a/test/app1.2/package.json b/test/app1.2/package.json new file mode 100644 index 00000000..eac6c266 --- /dev/null +++ b/test/app1.2/package.json @@ -0,0 +1,26 @@ +{ + "name": "node-pre-gyp-test-app1.2", + "author": "Dane Springmeyer ", + "description": "node-pre-gyp test", + "repository": { + "type": "git", + "url": "git://github.com/mapbox/node-pre-gyp.git" + }, + "license": "BSD-3-Clause", + "version": "0.1.0", + "main": "./index.js", + "binary": { + "module_name": "app1.2", + "module_path": "./lib/binding/", + "host": "https://s3.us-east-1.amazonaws.com", + "bucket": "npg-mock-bucket", + "region": "us-east-1", + "s3ForcePathStyle": true, + "remote_path": "./node-pre-gyp/{name}/v{version}/{configuration}/{toolset}/", + "package_name": "{node_abi}-{platform}-{arch}.tar.gz" + }, + "scripts": { + "install": "node-pre-gyp install --fallback-to-build", + "test": "node index.js" + } +} diff --git a/test/app1/package.json b/test/app1/package.json index 3cc13212..c1b5bf79 100644 --- a/test/app1/package.json +++ b/test/app1/package.json @@ -12,7 +12,7 @@ "binary": { "module_name": "app1", "module_path": "./lib/binding/", - "host": "https://mapbox-node-pre-gyp-public-testing-bucket.s3.us-east-1.amazonaws.com", + "host": "https://npg-mock-bucket.s3.us-east-1.amazonaws.com", "remote_path": "./node-pre-gyp/{name}/v{version}/{configuration}/{toolset}/", "package_name": "{node_abi}-{platform}-{arch}.tar.gz" }, diff --git a/test/app2/package.json b/test/app2/package.json index d86e49a6..edfc6dbc 100644 --- a/test/app2/package.json +++ b/test/app2/package.json @@ -14,7 +14,7 @@ "module_path": "./lib/binding/{configuration}/{name}", "remote_path": "./node-pre-gyp/{name}/v{version}/{configuration}/{version}/{toolset}/", "package_name": "{module_name}-v{major}.{minor}.{patch}-{prerelease}+{build}-{node_abi}-{platform}-{arch}.tar.gz", - "host": "https://mapbox-node-pre-gyp-public-testing-bucket.s3.us-east-1.amazonaws.com" + "host": "https://npg-mock-bucket.s3.us-east-1.amazonaws.com" }, "scripts": { "install": "node-pre-gyp install --fallback-to-build", diff --git a/test/app3/package.json b/test/app3/package.json index faf15dae..c0d1c4a0 100644 --- a/test/app3/package.json +++ b/test/app3/package.json @@ -14,7 +14,7 @@ "module_path": "./lib/binding/{node_abi}-{platform}-{arch}", "remote_path": "./node-pre-gyp/{module_name}/v{version}", "package_name": "{node_abi}-{platform}-{arch}.tar.gz", - "host": "https://mapbox-node-pre-gyp-public-testing-bucket.s3.us-east-1.amazonaws.com" + "host": "https://npg-mock-bucket.s3.us-east-1.amazonaws.com" }, "scripts": { "install": "node-pre-gyp install --fallback-to-build", diff --git a/test/app4/package.json b/test/app4/package.json index 388fa4e8..998eb423 100644 --- a/test/app4/package.json +++ b/test/app4/package.json @@ -14,7 +14,7 @@ "module_path": "./lib/binding/{node_abi}-{platform}-{arch}", "remote_path": "./node-pre-gyp/{module_name}/v{version}", "package_name": "{node_abi}-{platform}-{arch}.tar.gz", - "host": "https://mapbox-node-pre-gyp-public-testing-bucket.s3.us-east-1.amazonaws.com" + "host": "https://npg-mock-bucket.s3.us-east-1.amazonaws.com" }, "scripts": { "install": "node-pre-gyp install --fallback-to-build", diff --git a/test/app7/package.json b/test/app7/package.json index 27e30b31..ba8e7b00 100644 --- a/test/app7/package.json +++ b/test/app7/package.json @@ -14,7 +14,7 @@ "module_path": "./lib/binding/napi-v{napi_build_version}", "remote_path": "./node-pre-gyp/{module_name}/v{version}/{configuration}/", "package_name": "{module_name}-v{version}-{platform}-{arch}-napi-v{napi_build_version}-node-{node_abi}.tar.gz", - "host": "https://mapbox-node-pre-gyp-public-testing-bucket.s3.us-east-1.amazonaws.com", + "host": "https://npg-mock-bucket.s3.us-east-1.amazonaws.com", "napi_versions": [ 1, 2 From 628d4746820a53a75bcc644026353977af3dc832 Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Tue, 10 May 2022 12:33:54 -0700 Subject: [PATCH 04/14] Added service script to switch test apps bucket from mock to S3 and vis versa. Removed previous one. Fix CodeQL errors. --- lib/mock/http.js | 4 +- package.json | 3 +- scripts/set-bucket.js | 47 +++++++++++++++++++++++ scripts/switch-s3-hosts.js | 76 -------------------------------------- 4 files changed, 51 insertions(+), 79 deletions(-) create mode 100644 scripts/set-bucket.js delete mode 100644 scripts/switch-s3-hosts.js diff --git a/lib/mock/http.js b/lib/mock/http.js index 9f9bad10..7d4dd0b2 100644 --- a/lib/mock/http.js +++ b/lib/mock/http.js @@ -17,14 +17,14 @@ function http_mock() { const baseHostname = 's3.us-east-1.amazonaws.com'; const basePath = `${os.tmpdir()}/mock`; - nock(new RegExp('^.*' + baseHostname)) + nock(new RegExp('^\.*' + baseHostname)) .persist() .get(() => true) //a function that always returns true is a catch all for nock .reply( (uri) => { const bucket = 'npg-mock-bucket'; const mockDir = uri.indexOf(bucket) === -1 ? `${basePath}/${bucket}` : basePath; - const filepath = path.join(mockDir, uri.replace('%2B', '+')); + const filepath = path.join(mockDir, uri.replace(new RegExp('%2B', 'g'), '+')); try { fs.accessSync(filepath, fs.constants.R_OK); diff --git a/package.json b/package.json index af25cf10..3548b298 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "lint": "eslint bin/node-pre-gyp lib/*js lib/util/*js test/*js scripts/*js", "fix": "npm run lint -- --fix", "update-crosswalk": "node scripts/abi_crosswalk.js", - "test": "tape test/*test.js" + "test": "tape test/*test.js", + "bucket": "node scripts/set-bucket.js" } } diff --git a/scripts/set-bucket.js b/scripts/set-bucket.js new file mode 100644 index 00000000..894d77b4 --- /dev/null +++ b/scripts/set-bucket.js @@ -0,0 +1,47 @@ +'use strict'; + +// script changes the bucket name set in package.json of the test apps. + +const fs = require('fs'); +const path = require('path'); + +// http mock (lib/mock/http.js) sets 'npg-mock-bucket' as default bucket name. +// when providing no bucket name as argument, script will set +// all apps back to default mock settings. +const bucket = process.argv[2] || 'npg-mock-bucket'; + +const root = '../test'; +const rootPath = path.resolve(__dirname, root); +const dirs = fs.readdirSync(rootPath).filter((fileorDir) => fs.lstatSync(path.resolve(rootPath, fileorDir)).isDirectory()); + +dirs.forEach((dir) => { + const pkg = require(`${root}/${dir}/package.json`); // relative path + + // bucket specified as part of s3 virtual host format (auto detected by node-pre-gyp) + const keys = ['host', 'staging_host', 'production_host']; + keys.forEach((item) => { + if (pkg.binary[item]) { + + // match the bucket part of the url + const match = pkg.binary[item].match(/^https:\/\/(.+)(?:\.s3[-.].*)$/i); + if (match) { + pkg.binary[item] = pkg.binary[item].replace(match[1], bucket); + console.log(`Success: set ${dir} ${item} to ${pkg.binary[item]}`); + } + } + }); + // bucket is specified explicitly + if (pkg.binary.bucket) { + pkg.binary.bucket = bucket; + console.log(`Set ${dir} bucket to ${pkg.binary.bucket}`); + } + + // make sure bucket name is set in the package (somewhere) else this is an obvious error. + // most likely due to manual editing of the json resulting in unusable format + const str = JSON.stringify(pkg, null, 4); + if (str.indexOf(bucket) !== -1) { + fs.writeFileSync(path.join(path.resolve(rootPath, dir), 'package.json'), str + '\n'); + } else { + throw new Error(`Error: could not set ${dir}. Manually check package.json`); + } +}); diff --git a/scripts/switch-s3-hosts.js b/scripts/switch-s3-hosts.js deleted file mode 100644 index 3dd25bbf..00000000 --- a/scripts/switch-s3-hosts.js +++ /dev/null @@ -1,76 +0,0 @@ -'use strict'; - -// -// utility to switch s3 targets for local testing. if the s3 buckets are left -// pointing to the mapbox-node-pre-gyp-public-testing-bucket and you don't have -// write permissions to those buckets then the tests will fail. switching the -// target allows the tests to be run locally (even though the CI tests will fail -// if you are not a collaborator to the mapbox/node-pre-gyp repository). -// -// this replaces the mapbox-specific s3 URLs with an URL pointing to an S3 -// bucket which can be written to. each person using this will need to supply -// their own `toLocal.target` and `toMapbox.source` values that refer to their -// s3 buckets (and set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY -// appropriately). -// -// reset to the mapbox settings before committing. -// - -const fs = require('fs'); -const walk = require('action-walk'); // eslint-disable-line node/no-missing-require - -const [maj, min] = process.versions.node.split('.'); -if (`${maj}.${min}` < 10.1) { - console.error('requires node >= 10.1 for fs.promises'); - process.exit(1); -} -if (process.argv[2] !== 'toLocal' && process.argv[2] !== 'toMapbox') { - console.error('argument must be toLocal or toMapbox, not', process.argv[2]); - process.exit(1); -} - -const direction = { - toLocal: { - source: /mapbox-node-pre-gyp-public-testing-bucket/g, - target: 'bmac-pre-gyp-test' - }, - toMapbox: { - source: /bmac-pre-gyp-test/g, - target: 'mapbox-node-pre-gyp-public-testing-bucket' - } -}; - -const repl = direction[process.argv[2]]; - -console.log('replacing:'); -console.log(' ', repl.source); -console.log('with:'); -console.log(' ', repl.target); - - -function dirAction(path) { - if (path.startsWith('./node_modules/')) { - return 'skip'; - } -} - -function fileAction(path) { - if (path.endsWith('/package.json') || path.endsWith('/fetch.test.js') || path.endsWith('/lib/util/s3_setup.js')) { - const file = fs.readFileSync(path, 'utf8'); - const changed = file.replace(repl.source, repl.target); - if (file !== changed) { - console.log('replacing in:', path); - // eslint-disable-next-line node/no-unsupported-features/node-builtins - return fs.promises.writeFile(path, changed); - } else { - console.log('target not found in:', path); - } - } -} - -const options = { - dirAction, - fileAction -}; - -walk('.', options); From 7e50d3da49dc92442c1a0017690b9f253324293e Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Fri, 28 Jun 2024 18:19:44 -0700 Subject: [PATCH 05/14] Fix CodeQL and rebase errors. --- lib/mock/http.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/mock/http.js b/lib/mock/http.js index 7d4dd0b2..43f9ac8d 100644 --- a/lib/mock/http.js +++ b/lib/mock/http.js @@ -14,10 +14,9 @@ log.heading = 'node-pre-gyp'; // differentiate node-pre-gyp's logs from npm's function http_mock() { log.warn('mocking http requests to s3'); - const baseHostname = 's3.us-east-1.amazonaws.com'; const basePath = `${os.tmpdir()}/mock`; - nock(new RegExp('^\.*' + baseHostname)) + nock(new RegExp('([a-z0-9]+[.])*s3[.]us-east-1[.]amazonaws[.]com')) .persist() .get(() => true) //a function that always returns true is a catch all for nock .reply( From f081f2568ad8ab3f6e1542f20000c24ec5019bd7 Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Thu, 12 May 2022 12:00:05 -0700 Subject: [PATCH 06/14] Created GitHub Actions Test Setup - Added a GitHub Actions workflow that runs whenever there is a push to the repo. - Workflow includes two jobs: - A matrix job of node versions (10, 12, 14, 16, 18) and operating systems (Linux (ubuntu), Mac and Windows (2019 Enterprise)) that runs all tests against mock and then runs s3 tests against a bucket (located at us-east-1-bucket) specified as a repo secret. - A matrix job of and NW.js versions (0.64.0, 0.50.2) and node versions (10, 12, ,14, 16) that runs the NW.js test script. --- .github/workflows/push.yml | 74 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/push.yml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 00000000..c0d9396b --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,74 @@ +name: Push - Matrix Tests + +on: + push: + workflow_dispatch: + +jobs: + test-on-os-node-matrix: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-2019] # due to node-gyp & node compatibility issues, windows 2022 won't work for all node versions + node: [10, 12, 14, 16, 18] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} + + name: Test Node ${{ matrix.node }} on ${{ matrix.os }} + + steps: + - name: Checkout ${{ github.ref }} + uses: actions/checkout@v2 + + - name: Setup node ${{ matrix.node }} + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node }} + + - name: NPM Install + run: npm install + + - name: Configure Windows 2019 + run: | + echo "/c/Program Files/Microsoft Visual Studio/2019/Enterprise/MSBuild/Current/Bin/" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + npm config set msvs_version 2019 + if: ${{ matrix.os == 'windows-2019' }} + + - name: Show Environment Info + run: | + printenv + node --version + npm --version + + - name: Run All Tests (against mock) + run: npm test + env: + node_pre_gyp_mock_s3 : true + + - name: Run S3 Tests (against ${{ env.S3_BUCKET }} bucket) + run: | + npm run bucket ${{ env.S3_BUCKET }} + npm run test:s3 + if: ${{ env.S3_BUCKET != '' }} + + test-nw: + runs-on: ubuntu-18.04 # at current config the nw test requires python 2 as default. hence use older linux version + strategy: + matrix: + node: [10, 12, 14, 16] # node 18 requires glibc GLIBC_2.28 not available on older version of linux + nw: [0.64.0, 0.50.2] # current version as of may 2022 and the one tested before introduction of this action. + name: NW.js ${{ matrix.nw }} on Node ${{ matrix.node }} + + steps: + - name: Checkout ${{ github.ref }} + uses: actions/checkout@v2 + + - name: Setup node ${{ matrix.node }} + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node }} + + - name: Run Script + run: ./scripts/test-node-webkit.sh ${{ matrix.nw }} From ade877de39cb845995f3415f85722b509b717754 Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Thu, 12 May 2022 12:00:17 -0700 Subject: [PATCH 07/14] Modified existing configurations to work with GitHub Actions Test setup. - Modified `scripts/test-node-webkit.sh` so that it can now accept an NW.js version as input. This allows running the script in a GitHub Actions matrix. - Modified `test/run.util.js` so that it does not set `--msvs_version=2015` when running in GitHub Actions. This is required because current GitHub Actions runner do not support VS Studio 2015. - Added npm script command `test:s3` to `package.json` that runs only the s3 tests. This is required because invoking `npx tape test/s3.test.js` on windows does not work as expected. - Modified `test/proxy-bcrypt.test.js`. Removed uneeded CI conditionals and modified download directory setup/cleanup. Latter was required due to concurrency issues with running tests on Mac, resulting in uncatchable errors during directory removal originating from `rimraf`. --- package.json | 1 + scripts/test-node-webkit.sh | 4 +++- test/s3.test.js | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 3548b298..86c958cc 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "fix": "npm run lint -- --fix", "update-crosswalk": "node scripts/abi_crosswalk.js", "test": "tape test/*test.js", + "test:s3": "tape test/s3.test.js", "bucket": "node scripts/set-bucket.js" } } diff --git a/scripts/test-node-webkit.sh b/scripts/test-node-webkit.sh index ac81ff79..8bf555bd 100755 --- a/scripts/test-node-webkit.sh +++ b/scripts/test-node-webkit.sh @@ -1,5 +1,7 @@ #!/bin/bash +nw_version=${1:-"0.50.2"} + set -eu set -o pipefail @@ -9,7 +11,7 @@ export PATH=`pwd`/bin:$PATH BASE=$(pwd) -export NODE_WEBKIT_VERSION="0.50.2" +export NODE_WEBKIT_VERSION="${nw_version}" export NW_INSTALL_URL="https://dl.nwjs.io" if [[ `uname -s` == 'Darwin' ]]; then diff --git a/test/s3.test.js b/test/s3.test.js index 907b3bf8..52123c50 100644 --- a/test/s3.test.js +++ b/test/s3.test.js @@ -110,7 +110,6 @@ apps.forEach((app) => { }); }); - test(app.name + ' package ' + app.args, (t) => { run('node-pre-gyp', 'package', '', app, {}, (err) => { t.ifError(err); From 6f8a730b26cf216c7bc34cefe62430edbdb503f8 Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Fri, 28 Jun 2024 20:28:28 -0700 Subject: [PATCH 08/14] Updated github actions to only test node 18, 20, 22. --- .github/workflows/push.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index c0d9396b..ad66f7e0 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-2019] # due to node-gyp & node compatibility issues, windows 2022 won't work for all node versions - node: [10, 12, 14, 16, 18] + node: [18, 20, 22] env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -57,7 +57,7 @@ jobs: runs-on: ubuntu-18.04 # at current config the nw test requires python 2 as default. hence use older linux version strategy: matrix: - node: [10, 12, 14, 16] # node 18 requires glibc GLIBC_2.28 not available on older version of linux + node: [18, 20, 22] nw: [0.64.0, 0.50.2] # current version as of may 2022 and the one tested before introduction of this action. name: NW.js ${{ matrix.nw }} on Node ${{ matrix.node }} From de64a25e41b1cf4536e850ac1d5ac017053dc264 Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Fri, 28 Jun 2024 20:31:31 -0700 Subject: [PATCH 09/14] Trying windows-latest --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ad66f7e0..5870f2fc 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -9,7 +9,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-2019] # due to node-gyp & node compatibility issues, windows 2022 won't work for all node versions + os: [ubuntu-latest, macos-latest, windows-latest] # due to node-gyp & node compatibility issues, windows 2022 won't work for all node versions node: [18, 20, 22] env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} From 0afc89935fc831d452387097f6aa8488e6c47cb0 Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Fri, 28 Jun 2024 23:31:57 -0700 Subject: [PATCH 10/14] Removed nw tests from GitHub push actions as they are not working right now. --- .github/workflows/push.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5870f2fc..27e8e433 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -53,22 +53,3 @@ jobs: npm run test:s3 if: ${{ env.S3_BUCKET != '' }} - test-nw: - runs-on: ubuntu-18.04 # at current config the nw test requires python 2 as default. hence use older linux version - strategy: - matrix: - node: [18, 20, 22] - nw: [0.64.0, 0.50.2] # current version as of may 2022 and the one tested before introduction of this action. - name: NW.js ${{ matrix.nw }} on Node ${{ matrix.node }} - - steps: - - name: Checkout ${{ github.ref }} - uses: actions/checkout@v2 - - - name: Setup node ${{ matrix.node }} - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node }} - - - name: Run Script - run: ./scripts/test-node-webkit.sh ${{ matrix.nw }} From 2953b85a1be501b042025a33ad0798e267c3fe4c Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Sat, 29 Jun 2024 00:35:35 -0700 Subject: [PATCH 11/14] Changed push to bucket only tests. --- .github/workflows/{push.yml => s3-bucket.yml} | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) rename .github/workflows/{push.yml => s3-bucket.yml} (57%) diff --git a/.github/workflows/push.yml b/.github/workflows/s3-bucket.yml similarity index 57% rename from .github/workflows/push.yml rename to .github/workflows/s3-bucket.yml index 27e8e433..09f7378c 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/s3-bucket.yml @@ -1,4 +1,4 @@ -name: Push - Matrix Tests +name: S3 Bucket Test on: push: @@ -9,7 +9,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] # due to node-gyp & node compatibility issues, windows 2022 won't work for all node versions + os: [ubuntu-latest, macos-latest, windows-latest] node: [18, 20, 22] env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -20,33 +20,22 @@ jobs: steps: - name: Checkout ${{ github.ref }} - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup node ${{ matrix.node }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node }} - name: NPM Install run: npm install - - name: Configure Windows 2019 - run: | - echo "/c/Program Files/Microsoft Visual Studio/2019/Enterprise/MSBuild/Current/Bin/" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - npm config set msvs_version 2019 - if: ${{ matrix.os == 'windows-2019' }} - - name: Show Environment Info run: | printenv node --version npm --version - - name: Run All Tests (against mock) - run: npm test - env: - node_pre_gyp_mock_s3 : true - - name: Run S3 Tests (against ${{ env.S3_BUCKET }} bucket) run: | npm run bucket ${{ env.S3_BUCKET }} From c139ad304e93ad138788390a91dc8d22a705f96b Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Sat, 29 Jun 2024 00:43:40 -0700 Subject: [PATCH 12/14] Changed test name --- .github/workflows/s3-bucket.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/s3-bucket.yml b/.github/workflows/s3-bucket.yml index 09f7378c..d496c91c 100644 --- a/.github/workflows/s3-bucket.yml +++ b/.github/workflows/s3-bucket.yml @@ -16,8 +16,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_BUCKET: ${{ secrets.S3_BUCKET }} - name: Test Node ${{ matrix.node }} on ${{ matrix.os }} - steps: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 From 09171e5cb809639a7a628840ddb567d595b640af Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Sat, 29 Jun 2024 00:47:56 -0700 Subject: [PATCH 13/14] Changed test name --- .github/workflows/s3-bucket.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/s3-bucket.yml b/.github/workflows/s3-bucket.yml index d496c91c..06df7c20 100644 --- a/.github/workflows/s3-bucket.yml +++ b/.github/workflows/s3-bucket.yml @@ -16,6 +16,8 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_BUCKET: ${{ secrets.S3_BUCKET }} + name: Test S3 Bucket - Node ${{ matrix.node }} on ${{ matrix.os }} + steps: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 From 4e407be1186ab66f15c12a7e8e9cc304fbc4fdde Mon Sep 17 00:00:00 2001 From: Ron Ilan Date: Wed, 18 May 2022 12:56:38 -0700 Subject: [PATCH 14/14] Refactor PR 533 and Fix issue 653 - Moved logic regarding host selection to versioning where all user defined values from package.json are transformed into command options. - Moved testing of feature from `run.test.js` to `versioning.test.js`. - Added `development_host` option. Becomes default option for `publish` `unpublish` when present. - Changed behavior when alternate hosts are defined. Now `production_host` acts as alias to host. Defining `staging_host` or `development_host` is enough to default `publish` and `unpublish` away from production. - When a chain of commands that includes `publish` or `unpublish`, when host not specifically set via command line or environment variable, ALL commands in the chain default away from production. - An invalid `s3_host` option does not result in error and is instead silently ignored. - Change is backwards compatible with previously valid configurations. --- lib/main.js | 6 - lib/node-pre-gyp.js | 66 +--- lib/pre-binding.js | 1 - lib/util/versioning.js | 83 +++-- test/run.test.js | 141 +-------- test/versioning.test.js | 652 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 708 insertions(+), 241 deletions(-) diff --git a/lib/main.js b/lib/main.js index bae32acb..9b54638f 100644 --- a/lib/main.js +++ b/lib/main.js @@ -72,12 +72,6 @@ function run() { return; } - // set binary.host when appropriate. host determines the s3 target bucket. - const target = prog.setBinaryHostProperty(command.name); - if (target && ['install', 'publish', 'unpublish', 'info'].indexOf(command.name) >= 0) { - log.info('using binary.host: ' + prog.package_json.binary.host); - } - prog.commands[command.name](command.args, function(err) { if (err) { log.error(command.name + ' error'); diff --git a/lib/node-pre-gyp.js b/lib/node-pre-gyp.js index f6c262de..6c8a1534 100644 --- a/lib/node-pre-gyp.js +++ b/lib/node-pre-gyp.js @@ -77,11 +77,8 @@ function Run({ package_json_path = './package.json', argv }) { }); this.parseArgv(argv); - - // this is set to true after the binary.host property was set to - // either staging_host or production_host. - this.binaryHostSet = false; } + inherits(Run, EE); exports.Run = Run; const proto = Run.prototype; @@ -205,67 +202,6 @@ proto.parseArgv = function parseOpts(argv) { log.resume(); }; -/** - * allow the binary.host property to be set at execution time. - * - * for this to take effect requires all the following to be true. - * - binary is a property in package.json - * - binary.host is falsey - * - binary.staging_host is not empty - * - binary.production_host is not empty - * - * if any of the previous checks fail then the function returns an empty string - * and makes no changes to package.json's binary property. - * - * - * if command is "publish" then the default is set to "binary.staging_host" - * if command is not "publish" the the default is set to "binary.production_host" - * - * if the command-line option '--s3_host' is set to "staging" or "production" then - * "binary.host" is set to the specified "staging_host" or "production_host". if - * '--s3_host' is any other value an exception is thrown. - * - * if '--s3_host' is not present then "binary.host" is set to the default as above. - * - * this strategy was chosen so that any command other than "publish" or "unpublish" uses "production" - * as the default without requiring any command-line options but that "publish" and "unpublish" require - * '--s3_host production_host' to be specified in order to *really* publish (or unpublish). publishing - * to staging can be done freely without worrying about disturbing any production releases. - */ -proto.setBinaryHostProperty = function(command) { - if (this.binaryHostSet) { - return this.package_json.binary.host; - } - const p = this.package_json; - // don't set anything if host is present. it must be left blank to trigger this. - if (!p || !p.binary || p.binary.host) { - return ''; - } - // and both staging and production must be present. errors will be reported later. - if (!p.binary.staging_host || !p.binary.production_host) { - return ''; - } - let target = 'production_host'; - if (command === 'publish' || command === 'unpublish') { - target = 'staging_host'; - } - // the environment variable has priority over the default or the command line. if - // either the env var or the command line option are invalid throw an error. - const npg_s3_host = process.env.node_pre_gyp_s3_host; - if (npg_s3_host === 'staging' || npg_s3_host === 'production') { - target = `${npg_s3_host}_host`; - } else if (this.opts['s3_host'] === 'staging' || this.opts['s3_host'] === 'production') { - target = `${this.opts['s3_host']}_host`; - } else if (this.opts['s3_host'] || npg_s3_host) { - throw new Error(`invalid s3_host ${this.opts['s3_host'] || npg_s3_host}`); - } - - p.binary.host = p.binary[target]; - this.binaryHostSet = true; - - return p.binary.host; -}; - /** * Returns the usage instructions for node-pre-gyp. */ diff --git a/lib/pre-binding.js b/lib/pre-binding.js index e110fe38..9fd4407f 100644 --- a/lib/pre-binding.js +++ b/lib/pre-binding.js @@ -19,7 +19,6 @@ exports.find = function(package_json_path, opts) { throw new Error(package_json_path + 'does not exist'); } const prog = new npg.Run({ package_json_path, argv: process.argv }); - prog.setBinaryHostProperty(); const package_json = prog.package_json; versioning.validate_config(package_json, opts); diff --git a/lib/util/versioning.js b/lib/util/versioning.js index 70c3f85a..ad3048ce 100644 --- a/lib/util/versioning.js +++ b/lib/util/versioning.js @@ -186,12 +186,6 @@ function get_runtime_abi(runtime, target_version) { } module.exports.get_runtime_abi = get_runtime_abi; -const required_parameters = [ - 'module_name', - 'module_path', - 'host' -]; - function validate_config(package_json, opts) { const msg = package_json.name + ' package.json is not node-pre-gyp ready:\n'; const missing = []; @@ -207,25 +201,33 @@ function validate_config(package_json, opts) { if (!package_json.binary) { missing.push('binary'); } - const o = package_json.binary; - if (o) { - required_parameters.forEach((p) => { - if (!o[p] || typeof o[p] !== 'string') { - missing.push('binary.' + p); - } - }); + + if (package_json.binary) { + if (!package_json.binary.module_name) { + missing.push('binary.module_name'); + } + if (!package_json.binary.module_path) { + missing.push('binary.module_path'); + } + if (!package_json.binary.host && !package_json.binary.production_host) { + missing.push('binary.host'); + } } if (missing.length >= 1) { throw new Error(msg + 'package.json must declare these properties: \n' + missing.join('\n')); } - if (o) { - // enforce https over http - const protocol = url.parse(o.host).protocol; - if (protocol === 'http:') { - throw new Error("'host' protocol (" + protocol + ") is invalid - only 'https:' is accepted"); - } + + if (package_json.binary) { + // for all possible host definitions - verify https usage + ['host', 'production_host', 'staging_host', 'development_host'].filter((item) => package_json.binary[item]).forEach((item) => { + const protocol = url.parse(package_json.binary[item]).protocol; + if (protocol === 'http:') { + throw new Error(msg + "'" + item + "' protocol (" + protocol + ") is invalid - only 'https:' is accepted"); + } + }); } + napi.validate_package_json(package_json, opts); } @@ -309,11 +311,46 @@ module.exports.evaluate = function(package_json, options, napi_build_version) { region: package_json.binary.region, s3ForcePathStyle: package_json.binary.s3ForcePathStyle || false }; - // support host mirror with npm config `--{module_name}_binary_host_mirror` - // e.g.: https://github.com/node-inspector/v8-profiler/blob/master/package.json#L25 - // > npm install v8-profiler --profiler_binary_host_mirror=https://npm.taobao.org/mirrors/node-inspector/ + + // user can define a target host key to use (development_host, staging_host, production_host) + // by setting the name of the host (development, staging, production) + // into an environment variable or via a command line option. + // the environment variable has priority over the the command line. + let targetHost = process.env.node_pre_gyp_s3_host || options.s3_host; + + // if value is not one of the allowed or the matching key is not found in package.json + // silently ignore the option + if (['production', 'staging', 'development'].indexOf(targetHost) === -1 || !package_json.binary[`${targetHost}_host`]) { + targetHost = ''; + } + + // the production host is as specified in 'host' key (default) + // unless there is none and alias production_host is specified (backwards compatibility) + // note: package.json is verified in validate_config to include at least one of the two. + let host = package_json.binary.host || package_json.binary.production_host; + + // when a valid target is specified by user, the host is from that target (or 'host') + if (targetHost === 'staging') { + host = package_json.binary.staging_host; + } else if (targetHost === 'development') { + host = package_json.binary.development_host; + } else if (!targetHost && (package_json.binary.development_host || package_json.binary.staging_host)) { + // when host not specifically set via command line or environment variable + // but staging and/or development host are present in package.json + // for any command (or command chain) that includes publish or unpublish + // default to lower host (development, and if not preset, staging). + if (options.argv && options.argv.remain.some((item) => (item === 'publish' || item === 'unpublish'))) { + host = package_json.binary.development_host || package_json.binary.staging_host; + } + } + + // support host mirror with npm config `--{module_name}_binary_host_mirror` + // e.g.: https://github.com/node-inspector/v8-profiler/blob/master/package.json#L25 + // > npm install v8-profiler --profiler_binary_host_mirror=https://npm.taobao.org/mirrors/node-inspector/ const validModuleName = opts.module_name.replace('-', '_'); - const host = process.env['npm_config_' + validModuleName + '_binary_host_mirror'] || package_json.binary.host; + // explicitly set mirror overrides everything set above + host = process.env['npm_config_' + validModuleName + '_binary_host_mirror'] || host; + opts.host = fix_slashes(eval_template(host, opts)); opts.module_path = eval_template(package_json.binary.module_path, opts); // now we resolve the module_path to ensure it is absolute so that binding.gyp variables work predictably diff --git a/test/run.test.js b/test/run.test.js index c45e900f..e121df47 100644 --- a/test/run.test.js +++ b/test/run.test.js @@ -28,10 +28,6 @@ const package_json_template = { } }; - -const all_commands = ['build', 'clean', 'configure', 'info', 'install', 'package', 'publish', 'rebuild', - 'reinstall', 'reveal', 'testbinary', 'testpackage', 'unpublish']; - /** * before testing create a scratch directory to run tests in. */ @@ -67,82 +63,6 @@ test.onFinish(() => { rimraf(scratch).then(() => undefined, () => undefined); }); -test('should set staging and production hosts', (t) => { - // make sure it's good when specifying host. - const mock_package_json = makePackageJson(); - - let { prog } = setupTest(dir, mock_package_json); - t.deepEqual(prog.package_json, mock_package_json); - t.equal(prog.binaryHostSet, false, 'binary host should not be flagged as set'); - - // test with no s3_host option - all_commands.forEach((cmd) => { - const mpj = clone(mock_package_json); - mpj.binary.host = ''; - const opts = { argv: [cmd] }; - ({ prog } = setupTest(dir, mpj, opts)); - mpj.binary.host = (cmd === 'publish' || cmd === 'unpublish') ? mpj.binary.staging_host : mpj.binary.production_host; - t.deepEqual(prog.package_json, mpj, 'host should be correct for command: ' + cmd); - t.equal(prog.binaryHostSet, true, 'binary host should be flagged as set'); - }); - - // test with s3_host set to staging - all_commands.forEach((cmd) => { - const mpj = clone(mock_package_json); - mpj.binary.host = ''; - const opts = { argv: [cmd, '--s3_host=staging'] }; - ({ prog } = setupTest(dir, mpj, opts)); - mpj.binary.host = mpj.binary.staging_host; - t.deepEqual(prog.package_json, mpj, 'host should be correct for command: ' + cmd); - t.equal(prog.binaryHostSet, true, 'binary host should be flagged as set'); - }); - - // test with s3_host set to production - all_commands.forEach((cmd) => { - const mpj = clone(mock_package_json); - mpj.binary.host = ''; - const opts = { argv: [cmd, '--s3_host=production'] }; - ({ prog } = setupTest(dir, mpj, opts)); - mpj.binary.host = mpj.binary.production_host; - t.deepEqual(prog.package_json, mpj, 'host should be correct for command: ' + cmd); - t.equal(prog.binaryHostSet, true, 'binary host should be flagged as set'); - }); - - t.end(); -}); - -test('should execute setBinaryHostProperty() properly', (t) => { - // it only --s3_host only takes effect if host is falsey. - const mock_package_json = makePackageJson({ binary: { host: '' } }); - - const opts = { argv: ['publish', '--s3_host=staging'] }; - - let { prog, binaryHost } = setupTest(dir, mock_package_json, opts); - t.equal(binaryHost, mock_package_json.binary.staging_host); - - // set it again to verify that it returns the already set value - binaryHost = prog.setBinaryHostProperty('publish'); - t.equal(binaryHost, mock_package_json.binary.staging_host); - - // now do this again but expect an empty binary host value because - // staging_host is missing. - const mpj = clone(mock_package_json); - delete mpj.binary.staging_host; - ({ prog, binaryHost } = setupTest(dir, mpj, opts)); - t.equal(binaryHost, ''); - - // one more time but with an invalid value for s3_host - opts.argv = ['publish', '--s3_host=bad-news']; - try { - ({ prog, binaryHost } = setupTest(dir, mock_package_json, opts)); - t.fail('should throw with --s3_host=bad-news'); - } catch (e) { - t.equal(e.message, 'invalid s3_host bad-news'); - } - - t.end(); -}); - test('verify that the --directory option works', (t) => { const initial = process.cwd(); @@ -223,6 +143,10 @@ test('verify that a non-existent package.json fails', (t) => { // test helpers. // +// helper to clone mock package.json. +// // https://stackoverflow.com/questions/4459928/how-to-deep-clone-in-javascript +const clone = (obj) => JSON.parse(JSON.stringify(obj)); + function makePackageJson(options = {}) { const package_json = clone(package_json_template); // override binary values if supplied @@ -233,60 +157,3 @@ function makePackageJson(options = {}) { } return package_json; } - -// helper to write package.json to disk so Run() can be instantiated with it. -function setupTest(directory, package_json, opts) { - opts = opts || {}; - let argv = ['node', 'program']; - if (opts.argv) { - argv = argv.concat(opts.argv); - } - const prev_dir = process.cwd(); - if (!opts.noChdir) { - try { - fs.mkdirSync(directory); - } catch (e) { - if (e.code !== 'EEXIST') { - throw e; - } - } - process.chdir(directory); - } - - try { - fs.writeFileSync('package.json', JSON.stringify(package_json)); - const prog = new npg.Run({ package_json_path: './package.json', argv }); - const binaryHost = prog.setBinaryHostProperty(prog.todo[0] && prog.todo[0].name); - return { prog, binaryHost }; - } finally { - process.chdir(prev_dir); - } -} - -// helper to clone mock package.json. it's overkill for existing tests -// but is future-proof. -// https://stackoverflow.com/questions/4459928/how-to-deep-clone-in-javascript -function clone(obj, hash = new WeakMap()) { - if (Object(obj) !== obj) return obj; // primitives - if (hash.has(obj)) return hash.get(obj); // cyclic reference - let result; - - if (obj instanceof Set) { - result = new Set(obj); // treat set as a value - } else if (obj instanceof Map) { - result = new Map(Array.from(obj, ([key, val]) => [key, clone(val, hash)])); - } else if (obj instanceof Date) { - result = new Date(obj); - } else if (obj instanceof RegExp) { - result = new RegExp(obj.source, obj.flags); - } else if (obj.constructor) { - result = new obj.constructor(); - } else { - result = Object.create(null); - } - hash.set(obj, result); - return Object.assign(result, ...Object.keys(obj).map((key) => { - return { [key]: clone(obj[key], hash) }; - })); -} - diff --git a/test/versioning.test.js b/test/versioning.test.js index c59344ee..48f491df 100644 --- a/test/versioning.test.js +++ b/test/versioning.test.js @@ -222,6 +222,107 @@ test('should verify that the binary property has required properties', (t) => { t.end(); }); +test('should allow production_host to act as alias to host (when host not preset)', (t) => { + const mock_package_json = { + 'name': 'test', + 'main': 'test.js', + 'version': '0.1.0', + 'binary': { + 'module_name': 'binary-module-name', + 'module_path': 'binary-module-path', + 'production_host': 's3-production-path' + } + }; + + const package_json = Object.assign({}, mock_package_json); + const opts = versioning.evaluate(package_json, { module_root: '/root' }); + t.equal(opts.host, mock_package_json.binary.production_host + '/'); + t.equal(opts.hosted_path, mock_package_json.binary.production_host + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary.production_host + '/' + opts.package_name); + + t.end(); +}); + +test('should use host over production_host (when both are preset)', (t) => { + const mock_package_json = { + 'name': 'test', + 'main': 'test.js', + 'version': '0.1.0', + 'binary': { + 'module_name': 'binary-module-name', + 'module_path': 'binary-module-path', + 'production_host': 's3-production-path', + 'host': 'binary-path' + } + }; + + const package_json = Object.assign({}, mock_package_json); + const opts = versioning.evaluate(package_json, { module_root: '/root' }); + t.equal(opts.host, mock_package_json.binary.host + '/'); + t.equal(opts.hosted_path, mock_package_json.binary.host + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary.host + '/' + opts.package_name); + + t.end(); +}); + +test('should verify that the host url protocol is https', (t) => { + const mock_package_json = { + 'name': 'test', + 'main': 'test.js', + 'version': '0.1.0', + 'binary': { + 'module_name': 'binary-module-name', + 'module_path': 'binary-module-path', + 'host': 'http://your_module.s3-us-west-1.amazonaws.com' + } + }; + + const package_json = Object.assign({}, mock_package_json); + + try { + // eslint-disable-next-line no-unused-vars + const opts = versioning.evaluate(package_json, {}); + } catch (e) { + // name won't be there if it's missing but both messages say 'undefined' + const msg = package_json.name + ' package.json is not node-pre-gyp ready:\n'; + const expectedMessage = msg + '\'host\' protocol (http:) is invalid - only \'https:\' is accepted'; + t.equal(e.message, expectedMessage); + } + + t.end(); +}); + +test('should verify that alternate hosts url protocol is https', (t) => { + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + host: 'https://your_module.s3-us-west-1.amazonaws.com' + } + }; + + const hosts = ['production', 'staging', 'development']; + hosts.forEach((host) => { + const package_json = Object.assign({}, mock_package_json); + package_json[`${host}_host`] = `https://${host}_bucket.s3-us-west-1.amazonaws.com`; + + try { + // eslint-disable-next-line no-unused-vars + const opts = versioning.evaluate(package_json, {}); + } catch (e) { + // name won't be there if it's missing but both messages say 'undefined' + const msg = package_json.name + ' package.json is not node-pre-gyp ready:\n'; + const expectedMessage = msg + `'${host}_host' protocol (http:) is invalid - only 'https:' is accepted`; + t.equal(e.message, expectedMessage); + } + }); + + t.end(); +}); + test('should not add bucket name to hosted_path when s3ForcePathStyle is false', (t) => { const mock_package_json = { 'name': 'test', @@ -266,7 +367,53 @@ test('should add bucket name to hosted_path when s3ForcePathStyle is true', (t) t.end(); }); -test('should verify host overrides staging and production values', (t) => { +test('should use host key by default for install, info, publish and unpublish commands (when no other hosts specified)', (t) => { + const makeOoptions = (cmd) => { + return { + argv: { + remain: [cmd], + cooked: [cmd], + original: [cmd] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + host: 'binary-path' + } + }; + + const cmds = ['install', 'info', 'publish', 'unpublish']; + cmds.forEach((cmd) => { + try { + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); + t.equal(opts.host, mock_package_json.binary.host + '/'); + t.equal(opts.hosted_path, mock_package_json.binary.host + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary.host + '/' + opts.package_name); + } catch (e) { + t.ifError(e, 'staging_host and production_host should be silently ignored'); + } + }); + t.end(); +}); + +test('should use production_host as alias for host for install and info commands (when host not preset)', (t) => { + const makeOoptions = (cmd) => { + return { + argv: { + remain: [cmd], + cooked: [cmd], + original: [cmd] + } + }; + }; + const mock_package_json = { name: 'test', main: 'test.js', @@ -274,25 +421,512 @@ test('should verify host overrides staging and production values', (t) => { binary: { module_name: 'binary-module-name', module_path: 'binary-module-path', - host: 'binary-path', - staging_host: 's3-staging-path', production_host: 's3-production-path' } }; - try { - const opts = versioning.evaluate(mock_package_json, { module_root: '/root' }); + const cmds = ['install', 'info']; + cmds.forEach((cmd) => { + + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); + t.equal(opts.host, mock_package_json.binary.production_host + '/'); + t.equal(opts.hosted_path, mock_package_json.binary.production_host + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary.production_host + '/' + opts.package_name); + + }); + t.end(); +}); + +test('should use host over production_host for install and info commands (when both are preset)', (t) => { + const makeOoptions = (cmd) => { + return { + argv: { + remain: [cmd], + cooked: [cmd], + original: [cmd] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + production_host: 's3-production-path', + host: 'binary-path' + } + }; + + const cmds = ['install', 'info']; + cmds.forEach((cmd) => { + + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); t.equal(opts.host, mock_package_json.binary.host + '/'); t.equal(opts.hosted_path, mock_package_json.binary.host + '/'); t.equal(opts.hosted_tarball, mock_package_json.binary.host + '/' + opts.package_name); - } catch (e) { - t.ifError(e, 'staging_host and production_host should be silently ignored'); - } + }); + t.end(); +}); + +test('should use host by default for install and info commands (overriding alternate hosts, production_host not present)', (t) => { + const options = { + argv: { + remain: ['install', 'info'], + cooked: ['install', 'info'], + original: ['install', 'info'] + } + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + host: 'binary-path', + development_host: 's3-development-path', + staging_host: 's3-staging-path' + } + }; + + + const opts = versioning.evaluate(mock_package_json, options); + t.equal(opts.host, mock_package_json.binary.host + '/'); + t.equal(opts.hosted_path, mock_package_json.binary.host + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary.host + '/' + opts.package_name); + + + t.end(); +}); + +test('should use host by default for install and info commands (overriding alternate hosts, host is present)', (t) => { + const options = { + argv: { + remain: ['install', 'info'], + cooked: ['install', 'info'], + original: ['install', 'info'] + } + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + host: 'binary-path', + development_host: 's3-development-path', + staging_host: 's3-staging-path', + production_host: 's3-production-path' + } + }; + + + const opts = versioning.evaluate(mock_package_json, options); + t.equal(opts.host, mock_package_json.binary.host + '/'); + t.equal(opts.hosted_path, mock_package_json.binary.host + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary.host + '/' + opts.package_name); + + + t.end(); +}); + +test('should use development_host key by default for publish and unpublish commands (when it is specified)', (t) => { + const makeOoptions = (cmd) => { + return { + argv: { + remain: [cmd], + cooked: [cmd], + original: [cmd] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + host: 'binary-path', + development_host: 's3-development-path', + staging_host: 's3-staging-path', + production_host: 's3-production-path' + } + }; + + const cmds = ['publish', 'unpublish']; + cmds.forEach((cmd) => { + + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); + t.equal(opts.host, mock_package_json.binary.development_host + '/'); + t.equal(opts.hosted_path, mock_package_json.binary.development_host + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary.development_host + '/' + opts.package_name); + + }); + t.end(); +}); + +test('should use staging_host key by default for publish and unpublish commands (when it is specified and no development_host)', (t) => { + const makeOoptions = (cmd) => { + return { + argv: { + remain: [cmd], + cooked: [cmd], + original: [cmd] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + host: 'binary-path', + staging_host: 's3-staging-path', + production_host: 's3-production-path' + } + }; + + const cmds = ['publish', 'unpublish']; + cmds.forEach((cmd) => { + + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); + t.equal(opts.host, mock_package_json.binary.staging_host + '/'); + t.equal(opts.hosted_path, mock_package_json.binary.staging_host + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary.staging_host + '/' + opts.package_name); + + }); + t.end(); +}); + +test('should use development_host key by default for publish and unpublish commands in a chain (when it is specified)', (t) => { + const makeOoptions = (cmd) => { + return { + argv: { + remain: ['info', cmd], + cooked: ['info', cmd], + original: ['info', cmd] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + host: 'binary-path', + development_host: 's3-development-path', + staging_host: 's3-staging-path', + production_host: 's3-production-path' + } + }; + + const cmds = ['publish', 'unpublish']; + cmds.forEach((cmd) => { + + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); + t.equal(opts.host, mock_package_json.binary.development_host + '/'); + t.equal(opts.hosted_path, mock_package_json.binary.development_host + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary.development_host + '/' + opts.package_name); + + }); + t.end(); +}); + +test('should use host specified by the --s3_host option', (t) => { + const makeOoptions = (cmd, host) => { + return { + s3_host: host, + argv: { + remain: [cmd], + cooked: [cmd, '--s3_host', host], + original: [cmd, `--s3_host=${host}`] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + // host: 'binary-path', + development_host: 's3-development-path', + staging_host: 's3-staging-path', + production_host: 's3-production-path' + } + }; + + const hosts = ['production', 'staging', 'development']; + const cmds = ['install', 'info', 'publish', 'unpublish']; + + cmds.forEach((cmd) => { + hosts.forEach((host) => { + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd, host)); + t.equal(opts.host, mock_package_json.binary[`${host}_host`] + '/'); + t.equal(opts.hosted_path, mock_package_json.binary[`${host}_host`] + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary[`${host}_host`] + '/' + opts.package_name); + }); + }); + t.end(); +}); + +test('should use defaults when --s3_host option is invalid', (t) => { + const makeOoptions = (cmd) => { + return { + s3_host: 'not-valid', + argv: { + remain: [cmd], + cooked: [cmd, '--s3_host', 'not-valid'], + original: [cmd, '--s3_host=not-valid'] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + host: 'binary-path', + development_host: 's3-development-path', + staging_host: 's3-staging-path', + production_host: 's3-production-path' + } + }; + + const cmds = ['install', 'info', 'publish', 'unpublish']; + + cmds.forEach((cmd) => { + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); + const host = cmd.indexOf('publish') === -1 ? 'host' : 'development_host'; + + t.equal(opts.host, mock_package_json.binary[host] + '/'); + t.equal(opts.hosted_path, mock_package_json.binary[host] + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary[host] + '/' + opts.package_name); + + }); + t.end(); +}); + +test('should use host specified by the s3_host environment variable', (t) => { + const makeOoptions = (cmd) => { + return { + argv: { + remain: [cmd], + cooked: [cmd], + original: [cmd] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + // host: 'binary-path', + development_host: 's3-development-path', + staging_host: 's3-staging-path', + production_host: 's3-production-path' + } + }; + + const hosts = ['production', 'staging', 'development']; + const cmds = ['install', 'info', 'publish', 'unpublish']; + + cmds.forEach((cmd) => { + hosts.forEach((host) => { + process.env.node_pre_gyp_s3_host = host; + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); + t.equal(opts.host, mock_package_json.binary[`${host}_host`] + '/'); + t.equal(opts.hosted_path, mock_package_json.binary[`${host}_host`] + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary[`${host}_host`] + '/' + opts.package_name); + }); + }); + t.end(); +}); + +test('should use defaults when s3_host environment variable is invalid', (t) => { + const makeOoptions = (cmd) => { + return { + argv: { + remain: [cmd], + cooked: [cmd], + original: [cmd] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + host: 'binary-path', + development_host: 's3-development-path', + staging_host: 's3-staging-path', + production_host: 's3-production-path' + } + }; + + const cmds = ['install', 'info', 'publish', 'unpublish']; + + cmds.forEach((cmd) => { + process.env.node_pre_gyp_s3_host = 'not-valid'; + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); + const host = cmd.indexOf('publish') === -1 ? 'host' : 'development_host'; + + t.equal(opts.host, mock_package_json.binary[host] + '/'); + t.equal(opts.hosted_path, mock_package_json.binary[host] + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary[host] + '/' + opts.package_name); + + }); + t.end(); +}); + +test('should use defaults when s3_host environment is valid but package.json does not match (production_host is default)', (t) => { + const makeOoptions = (cmd) => { + return { + argv: { + remain: [cmd], + cooked: [cmd], + original: [cmd] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + // no development_host + staging_host: 's3-staging-path', + // production_host not host + production_host: 's3-production-path' + } + }; + + const cmds = ['install', 'info', 'publish', 'unpublish']; + + cmds.forEach((cmd) => { + process.env.node_pre_gyp_s3_host = 'development'; // specify development_host + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); + const host = cmd.indexOf('publish') === -1 ? 'production_host' : 'staging_host'; // defaults + + t.equal(opts.host, mock_package_json.binary[host] + '/'); + t.equal(opts.hosted_path, mock_package_json.binary[host] + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary[host] + '/' + opts.package_name); + + }); + t.end(); +}); + +test('should use defaults when s3_host environment is valid but package.json does not match (host is default)', (t) => { + const makeOoptions = (cmd) => { + return { + argv: { + remain: [cmd], + cooked: [cmd], + original: [cmd] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + // host not production_host + host: 'binary-path', + // no development_host + staging_host: 's3-staging-path' + } + }; + + const cmds = ['install', 'info', 'publish', 'unpublish']; + + cmds.forEach((cmd) => { + process.env.node_pre_gyp_s3_host = 'development'; // specify development_host + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); + const host = cmd.indexOf('publish') === -1 ? 'host' : 'staging_host'; // defaults + + t.equal(opts.host, mock_package_json.binary[host] + '/'); + t.equal(opts.hosted_path, mock_package_json.binary[host] + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary[host] + '/' + opts.package_name); + + }); + t.end(); +}); + +test('should use host specified by environment variable overriding --s3_host option', (t) => { + const makeOoptions = (cmd) => { + return { + s3_host: 'staging', // from command line + argv: { + remain: [cmd], + cooked: [cmd, '--s3_host', 'staging'], + original: [cmd, '--s3_host=staging'] + } + }; + }; + + const mock_package_json = { + name: 'test', + main: 'test.js', + version: '0.1.0', + binary: { + module_name: 'binary-module-name', + module_path: 'binary-module-path', + // host: 'binary-path', + development_host: 's3-development-path', + staging_host: 's3-staging-path', + production_host: 's3-production-path' + } + }; + + + const cmds = ['install', 'info', 'publish', 'unpublish']; + cmds.forEach((cmd) => { + process.env.node_pre_gyp_s3_host = 'production'; + const opts = versioning.evaluate(mock_package_json, makeOoptions(cmd)); + t.equal(opts.host, mock_package_json.binary.production_host + '/'); + t.equal(opts.hosted_path, mock_package_json.binary.production_host + '/'); + t.equal(opts.hosted_tarball, mock_package_json.binary.production_host + '/' + opts.package_name); + + }); t.end(); }); -test('should replace "-" with "_" in custom binary host', (t) => { +test('should replace "-" with "_" in mirror binary host', (t) => { const mock_package_json = { name: 'test', main: 'test.js',