From 2a0da992ee3972b6990c15bf2a9965b7350f93d6 Mon Sep 17 00:00:00 2001 From: Olup Date: Mon, 1 Mar 2021 09:21:04 +0100 Subject: [PATCH 1/9] fix(package): exclude complete bundle directory --- src/pack-individually.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pack-individually.ts b/src/pack-individually.ts index 9e93b8bb..1877c880 100644 --- a/src/pack-individually.ts +++ b/src/pack-individually.ts @@ -48,7 +48,7 @@ export async function packIndividually() { // get a list of every function bundle const buildResults = this.buildResults; - const bundlePathList = buildResults.map(b => b.bundlePath); + const bundlePathList = buildResults.map(b => path.dirname(b.bundlePath)); // get a list of external dependencies already listed in package.json const externals = without(this.buildOptions.exclude, this.buildOptions.external); @@ -63,9 +63,9 @@ export async function packIndividually() { const startZip = Date.now(); const name = func.name; - const excludedFiles = [ + const excludedFilesOrDirectory = [ ...excludedFilesDefault, - ...bundlePathList.filter(p => p !== bundlePath), + ...bundlePathList.filter(p => !bundlePath.startsWith(p)), ]; // allowed external dependencies in the final zip @@ -89,8 +89,8 @@ export async function packIndividually() { zip.pipe(output); files.forEach((filePath: string) => { - // exclude non individual files - if (excludedFiles.includes(filePath)) return; + // exclude non individual files based on file or dir path + if (excludedFilesOrDirectory.find(p => filePath.startsWith(p))) return; // exclude generated zip TODO:better logic if (filePath.endsWith('.zip')) return; From 62da4519172fb6e14c792804125b15742b8af129 Mon Sep 17 00:00:00 2001 From: Olup Date: Mon, 1 Mar 2021 14:10:52 +0100 Subject: [PATCH 2/9] fix(package): change logic to fix #90 Here we prevent the change of serverless.config.servicepath --- src/index.ts | 37 +++++++------- src/pack-externals.ts | 49 ++++++++++++++----- src/pack-individually.ts | 101 ++++++++++++++++----------------------- src/utils.ts | 32 +++++++++++++ 4 files changed, 127 insertions(+), 92 deletions(-) diff --git a/src/index.ts b/src/index.ts index 57c1166f..f4585699 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,7 @@ import { packIndividually } from './pack-individually'; export const SERVERLESS_FOLDER = '.serverless'; export const BUILD_FOLDER = '.build'; +export const WORK_FOLDER = '.esbuild'; interface OptionsExtended extends Serverless.Options { verbose?: boolean; @@ -43,7 +44,8 @@ const DEFAULT_BUILD_OPTIONS: Partial = { }; export class EsbuildPlugin implements Plugin { - private originalServicePath: string; + workDirPath: string; + buildDirPath: string; serverless: Serverless; options: OptionsExtended; @@ -63,6 +65,9 @@ export class EsbuildPlugin implements Plugin { this.packExternalModules = packExternalModules.bind(this); this.packIndividually = packIndividually.bind(this); + this.workDirPath = path.join(this.serverless.config.servicePath, WORK_FOLDER); + this.buildDirPath = path.join(this.workDirPath, BUILD_FOLDER); + const withDefaultOptions = mergeRight(DEFAULT_BUILD_OPTIONS); this.buildOptions = withDefaultOptions( this.serverless.service.custom?.esbuild ?? {} @@ -129,7 +134,7 @@ export class EsbuildPlugin implements Plugin { get rootFileNames() { return extractFileNames( - this.originalServicePath, + this.serverless.config.servicePath, this.serverless.service.provider.name, this.functions ); @@ -152,6 +157,7 @@ export class EsbuildPlugin implements Plugin { } prepare() { + fs.mkdirpSync(this.buildDirPath); // exclude serverless-esbuild for (const fnName in this.functions) { const fn = this.serverless.service.getFunction(fnName); @@ -171,20 +177,13 @@ export class EsbuildPlugin implements Plugin { this.prepare(); this.serverless.cli.log('Compiling with esbuild...'); - if (!this.originalServicePath) { - // Save original service path and functions - this.originalServicePath = this.serverless.config.servicePath; - // Fake service path so that serverless will know what to zip - this.serverless.config.servicePath = path.join(this.originalServicePath, BUILD_FOLDER); - } - return Promise.all( this.rootFileNames.map(async ({ entry, func }) => { const config: Omit = { ...this.buildOptions, external: [...this.buildOptions.external, ...this.buildOptions.exclude], entryPoints: [entry], - outdir: path.join(this.originalServicePath, BUILD_FOLDER, path.dirname(entry)), + outdir: path.join(this.buildDirPath, path.dirname(entry)), platform: 'node', incremental, }; @@ -216,7 +215,7 @@ export class EsbuildPlugin implements Plugin { const files = await globby(service.package.include); for (const filename of files) { - const destFileName = path.resolve(path.join(BUILD_FOLDER, filename)); + const destFileName = path.resolve(path.join(this.buildDirPath, filename)); const dirname = path.dirname(destFileName); if (!fs.existsSync(dirname)) { @@ -224,7 +223,7 @@ export class EsbuildPlugin implements Plugin { } if (!fs.existsSync(destFileName)) { - fs.copySync(path.resolve(filename), path.resolve(path.join(BUILD_FOLDER, filename))); + fs.copySync(path.resolve(filename), path.resolve(path.join(this.buildDirPath, filename))); } } } @@ -238,14 +237,14 @@ export class EsbuildPlugin implements Plugin { const { service } = this.serverless; await fs.copy( - path.join(this.originalServicePath, BUILD_FOLDER, SERVERLESS_FOLDER), - path.join(this.originalServicePath, SERVERLESS_FOLDER) + path.join(this.workDirPath, SERVERLESS_FOLDER), + path.join(this.serverless.config.servicePath, SERVERLESS_FOLDER) ); if (this.options.function) { const fn = service.getFunction(this.options.function); fn.package.artifact = path.join( - this.originalServicePath, + this.serverless.config.servicePath, SERVERLESS_FOLDER, path.basename(fn.package.artifact) ); @@ -256,7 +255,7 @@ export class EsbuildPlugin implements Plugin { const functionNames = service.getAllFunctions(); functionNames.forEach(name => { service.getFunction(name).package.artifact = path.join( - this.originalServicePath, + this.serverless.config.servicePath, SERVERLESS_FOLDER, path.basename(service.getFunction(name).package.artifact) ); @@ -265,7 +264,7 @@ export class EsbuildPlugin implements Plugin { } service.package.artifact = path.join( - this.originalServicePath, + this.serverless.config.servicePath, SERVERLESS_FOLDER, path.basename(service.package.artifact) ); @@ -273,10 +272,8 @@ export class EsbuildPlugin implements Plugin { async cleanup(): Promise { await this.moveArtifacts(); - // Restore service path - this.serverless.config.servicePath = this.originalServicePath; // Remove temp build folder - fs.removeSync(path.join(this.originalServicePath, BUILD_FOLDER)); + fs.removeSync(path.join(this.workDirPath)); } } diff --git a/src/pack-externals.ts b/src/pack-externals.ts index 1b279843..0be546d0 100644 --- a/src/pack-externals.ts +++ b/src/pack-externals.ts @@ -44,7 +44,11 @@ function rebaseFileReferences(pathToPackageRoot: string, moduleVersion: string) /** * Add the given modules to a package json's dependencies. */ -function addModulesToPackageJson(externalModules: string[], packageJson: JSONObject, pathToPackageRoot: string) { +function addModulesToPackageJson( + externalModules: string[], + packageJson: JSONObject, + pathToPackageRoot: string +) { forEach(externalModule => { const splitModule = split('@', externalModule); // If we have a scoped module we have to re-add the @ @@ -88,17 +92,29 @@ function getProdModules(externalModules: { external: string }[], packageJsonPath externalModule.external, 'package.json' ); - const peerDependencies = require(modulePackagePath).peerDependencies as Record; + const peerDependencies = require(modulePackagePath).peerDependencies as Record< + string, + string + >; if (!isEmpty(peerDependencies)) { - this.options.verbose && this.serverless.cli.log(`Adding explicit peers for dependency ${externalModule.external}`); - const peerModules = getProdModules.call(this, - compose(map(([external]) => ({ external })), toPairs)(peerDependencies), + this.options.verbose && + this.serverless.cli.log( + `Adding explicit peers for dependency ${externalModule.external}` + ); + const peerModules = getProdModules.call( + this, + compose( + map(([external]) => ({ external })), + toPairs + )(peerDependencies), packageJsonPath ); Array.prototype.push.apply(prodModules, peerModules); } } catch (e) { - this.serverless.cli.log(`WARNING: Could not check for peer dependencies of ${externalModule.external}`); + this.serverless.cli.log( + `WARNING: Could not check for peer dependencies of ${externalModule.external}` + ); } } else { if (!packageJson.devDependencies || !packageJson.devDependencies[externalModule.external]) { @@ -113,7 +129,9 @@ function getProdModules(externalModules: { external: string }[], packageJsonPath this.serverless.cli.log( `ERROR: Runtime dependency '${externalModule.external}' found in devDependencies.` ); - throw new this.serverless.classes.Error(`Serverless-webpack dependency error: ${externalModule.external}.`); + throw new this.serverless.classes.Error( + `Serverless-webpack dependency error: ${externalModule.external}.` + ); } this.options.verbose && @@ -148,7 +166,8 @@ export async function packExternalModules(this: EsbuildPlugin) { } // Read plugin configuration - const packageJsonPath = this.buildOptions.packagePath || path.join(findProjectRoot(), './package.json'); + const packageJsonPath = + this.buildOptions.packagePath || path.join(findProjectRoot(), './package.json'); // Determine and create packager const packager = await Packagers.get(this.buildOptions.packager); @@ -158,7 +177,8 @@ export async function packExternalModules(this: EsbuildPlugin) { const packageJson = this.serverless.utils.readFileSync(packageJsonPath); const packageSections = pick(sectionNames, packageJson); if (!isEmpty(packageSections)) { - this.options.verbose && this.serverless.cli.log(`Using package.json sections ${join(', ', keys(packageSections))}`); + this.options.verbose && + this.serverless.cli.log(`Using package.json sections ${join(', ', keys(packageSections))}`); } // Get first level dependency graph @@ -166,7 +186,9 @@ export async function packExternalModules(this: EsbuildPlugin) { // (1) Generate dependency composition const externalModules = map(external => ({ external }), externals); - const compositeModules: JSONObject = uniq(getProdModules.call(this, externalModules, packageJsonPath)); + const compositeModules: JSONObject = uniq( + getProdModules.call(this, externalModules, packageJsonPath) + ); if (isEmpty(compositeModules)) { // The compiled code does not reference any external modules at all @@ -175,7 +197,7 @@ export async function packExternalModules(this: EsbuildPlugin) { } // (1.a) Install all needed modules - const compositeModulePath = this.serverless.config.servicePath; + const compositeModulePath = this.buildDirPath; const compositePackageJson = path.join(compositeModulePath, 'package.json'); // (1.a.1) Create a package.json @@ -190,7 +212,10 @@ export async function packExternalModules(this: EsbuildPlugin) { ); const relativePath = path.relative(compositeModulePath, path.dirname(packageJsonPath)); addModulesToPackageJson(compositeModules, compositePackage, relativePath); - this.serverless.utils.writeFileSync(compositePackageJson, JSON.stringify(compositePackage, null, 2)); + this.serverless.utils.writeFileSync( + compositePackageJson, + JSON.stringify(compositePackage, null, 2) + ); // (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades const packageLockPath = path.join(path.dirname(packageJsonPath), packager.lockfileName); diff --git a/src/pack-individually.ts b/src/pack-individually.ts index 1877c880..15c543d8 100644 --- a/src/pack-individually.ts +++ b/src/pack-individually.ts @@ -1,13 +1,12 @@ -import * as archiver from 'archiver'; import * as fs from 'fs-extra'; import * as glob from 'glob'; import * as path from 'path'; import { intersection, isEmpty, path as get, without } from 'ramda'; import * as semver from 'semver'; -import { SERVERLESS_FOLDER } from '.'; +import { EsbuildPlugin, SERVERLESS_FOLDER } from '.'; import { doSharePath, flatDep, getDepsFromBundle } from './helper'; import * as Packagers from './packagers'; -import { humanSize } from './utils'; +import { humanSize, zip } from './utils'; function setArtifactPath(func, artifactPath) { const version = this.serverless.getVersion(); @@ -27,25 +26,24 @@ function setArtifactPath(func, artifactPath) { const excludedFilesDefault = ['package-lock.json', 'yarn.lock', 'package.json']; -export async function packIndividually() { - // If individually is not set, ignore this part - if (!this.serverless?.service?.package?.individually) return null; - - const packager = await Packagers.get(this.buildOptions.packager); - const buildDir = this.serverless.config.servicePath; - +export async function packIndividually(this: EsbuildPlugin) { // get a list of all path in build const files = glob.sync('**', { - cwd: buildDir, + cwd: this.buildDirPath, dot: true, silent: true, follow: true, }); if (isEmpty(files)) { - throw new this.serverless.classes.Error('Packaging: No files found'); + throw new Error('Packaging: No files found'); } + // If individually is not set, ignore this part + if (!this.serverless?.service?.package?.individually) return null; + + const packager = await Packagers.get(this.buildOptions.packager); + // get a list of every function bundle const buildResults = this.buildResults; const bundlePathList = buildResults.map(b => path.dirname(b.bundlePath)); @@ -55,12 +53,13 @@ export async function packIndividually() { const hasExternals = !!externals?.length; // get a tree of all production dependencies - const packagerDependenciesList = hasExternals ? await packager.getProdDependencies(buildDir) : {}; + const packagerDependenciesList = hasExternals + ? await packager.getProdDependencies(this.buildDirPath) + : {}; // package each function await Promise.all( buildResults.map(async ({ func, bundlePath }) => { - const startZip = Date.now(); const name = func.name; const excludedFilesOrDirectory = [ @@ -72,68 +71,50 @@ export async function packIndividually() { let depWhiteList = []; if (hasExternals) { - const bundleDeps = getDepsFromBundle(path.join(buildDir, bundlePath)); + const bundleDeps = getDepsFromBundle(path.join(this.buildDirPath, bundlePath)); const bundleExternals = intersection(bundleDeps, externals); depWhiteList = flatDep(packagerDependenciesList.dependencies, bundleExternals); } - // Create zip and open it - const zip = archiver.create('zip'); const zipName = `${name}.zip`; - const artifactPath = path.join(buildDir, SERVERLESS_FOLDER, zipName); - this.serverless.utils.writeFileDir(artifactPath); - const output = fs.createWriteStream(artifactPath); - - // write zip - output.on('open', () => { - zip.pipe(output); + const artifactPath = path.join(this.workDirPath, SERVERLESS_FOLDER, zipName); - files.forEach((filePath: string) => { + // filter files + const filesPathList = files + .filter((filePath: string) => { // exclude non individual files based on file or dir path - if (excludedFilesOrDirectory.find(p => filePath.startsWith(p))) return; - - // exclude generated zip TODO:better logic - if (filePath.endsWith('.zip')) return; + if (excludedFilesOrDirectory.find(p => filePath.startsWith(p))) return false; // exclude non whitelisted dependencies if (filePath.startsWith('node_modules')) { - if (!hasExternals) return; + if (!hasExternals) return false; if ( // this is needed for dependencies that maps to a path (like scopped ones) !depWhiteList.find(dep => doSharePath(filePath, 'node_modules/' + dep)) ) - return; + return false; } - // exclude directories - const fullPath = path.resolve(buildDir, filePath); - const stats = fs.statSync(fullPath); - if (stats.isDirectory()) return; - - zip.append(fs.readFileSync(fullPath), { - name: filePath, - mode: stats.mode, - date: new Date(0), // necessary to get the same hash when zipping the same content - }); - }); - - zip.finalize(); - }); - - return new Promise((resolve, reject) => { - output.on('close', () => { - // log zip results - const { size } = fs.statSync(artifactPath); - this.serverless.cli.log( - `Zip function: ${func.name} - ${humanSize(size)} [${Date.now() - startZip} ms]` - ); - - // defined present zip as output artifact - setArtifactPath.call(this, func, path.relative(this.originalServicePath, artifactPath)); - resolve(artifactPath); - }); - zip.on('error', err => reject(err)); - }); + return true; + }) + // get absolute path + .map(name => ({ path: path.join(this.buildDirPath, name), name })); + + const startZip = Date.now(); + await zip(artifactPath, filesPathList); + + const { size } = fs.statSync(artifactPath); + + this.serverless.cli.log( + `Zip function: ${func.name} - ${humanSize(size)} [${Date.now() - startZip} ms]` + ); + + // defined present zip as output artifact + setArtifactPath.call( + this, + func, + path.relative(this.serverless.config.servicePath, artifactPath) + ); }) ); } diff --git a/src/utils.ts b/src/utils.ts index ea0ce166..b91dc8bc 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,3 +1,4 @@ +import * as archiver from 'archiver'; import * as childProcess from 'child_process'; import * as fs from 'fs-extra'; import * as path from 'path'; @@ -89,3 +90,34 @@ export const humanSize = (size: number) => { const sanitized = (size / Math.pow(1024, i)).toFixed(2); return `${sanitized} ${['B', 'KB', 'MB', 'GB', 'TB'][i]}`; }; + +export const zip = (zipPath: string, filesPathList: { path: string; name: string }[]) => { + fs.mkdirpSync(path.dirname(zipPath)); + const zip = archiver.create('zip'); + const output = fs.createWriteStream(zipPath); + + // write zip + output.on('open', () => { + zip.pipe(output); + + filesPathList.forEach(file => { + const stats = fs.statSync(file.path); + if (stats.isDirectory()) return; + + zip.append(fs.readFileSync(file.path), { + name: file.name, + mode: stats.mode, + date: new Date(0), // necessary to get the same hash when zipping the same content + }); + }); + + zip.finalize(); + }); + + return new Promise((resolve, reject) => { + output.on('close', () => { + resolve(zipPath); + }); + zip.on('error', err => reject(err)); + }); +}; From 9954a14baad7d85666ab7dd24928673521d640fc Mon Sep 17 00:00:00 2001 From: Olup Date: Mon, 1 Mar 2021 16:46:08 +0100 Subject: [PATCH 3/9] fix(package): refacto, second part We do the zipping, even when individually is not set. --- src/index.ts | 22 +++++-- src/{pack-individually.ts => pack.ts} | 82 ++++++++++++++++----------- src/pre-local.ts | 6 ++ src/pre-offline.ts | 11 ++++ src/utils.ts | 9 +-- 5 files changed, 86 insertions(+), 44 deletions(-) rename src/{pack-individually.ts => pack.ts} (58%) create mode 100644 src/pre-local.ts create mode 100644 src/pre-offline.ts diff --git a/src/index.ts b/src/index.ts index f4585699..cc61cd1d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,9 @@ import * as chokidar from 'chokidar'; import { extractFileNames } from './helper'; import { packExternalModules } from './pack-externals'; -import { packIndividually } from './pack-individually'; +import { pack } from './pack'; +import { preOffline } from './pre-offline'; +import { preLocal } from './pre-local'; export const SERVERLESS_FOLDER = '.serverless'; export const BUILD_FOLDER = '.build'; @@ -39,7 +41,7 @@ const DEFAULT_BUILD_OPTIONS: Partial = { packager: 'npm', watch: { pattern: './**/*.(js|ts)', - ignore: [BUILD_FOLDER, 'dist', 'node_modules', SERVERLESS_FOLDER], + ignore: [WORK_FOLDER, 'dist', 'node_modules', SERVERLESS_FOLDER], }, }; @@ -57,13 +59,17 @@ export class EsbuildPlugin implements Plugin { func: any; }[]; packExternalModules: () => Promise; - packIndividually: () => Promise; + pack: () => Promise; + preOffline: () => Promise; + preLocal: () => void; constructor(serverless: Serverless, options: OptionsExtended) { this.serverless = serverless; this.options = options; this.packExternalModules = packExternalModules.bind(this); - this.packIndividually = packIndividually.bind(this); + this.pack = pack.bind(this); + this.preOffline = preOffline.bind(this); + this.preLocal = preLocal.bind(this); this.workDirPath = path.join(this.serverless.config.servicePath, WORK_FOLDER); this.buildDirPath = path.join(this.workDirPath, BUILD_FOLDER); @@ -83,19 +89,21 @@ export class EsbuildPlugin implements Plugin { await this.bundle(); await this.packExternalModules(); await this.copyExtras(); + await this.preOffline(); this.watch(); }, 'before:offline:start:init': async () => { await this.bundle(); await this.packExternalModules(); await this.copyExtras(); + await this.preOffline(); this.watch(); }, 'before:package:createDeploymentArtifacts': async () => { await this.bundle(); await this.packExternalModules(); await this.copyExtras(); - await this.packIndividually(); + await this.pack(); }, 'after:package:createDeploymentArtifacts': async () => { await this.cleanup(); @@ -104,7 +112,7 @@ export class EsbuildPlugin implements Plugin { await this.bundle(); await this.packExternalModules(); await this.copyExtras(); - await this.packIndividually(); + await this.pack(); }, 'after:deploy:function:packageFunction': async () => { await this.cleanup(); @@ -113,6 +121,7 @@ export class EsbuildPlugin implements Plugin { await this.bundle(); await this.packExternalModules(); await this.copyExtras(); + await this.preLocal(); }, }; } @@ -158,6 +167,7 @@ export class EsbuildPlugin implements Plugin { prepare() { fs.mkdirpSync(this.buildDirPath); + fs.mkdirpSync(path.join(this.workDirPath, SERVERLESS_FOLDER)); // exclude serverless-esbuild for (const fnName in this.functions) { const fn = this.serverless.service.getFunction(fnName); diff --git a/src/pack-individually.ts b/src/pack.ts similarity index 58% rename from src/pack-individually.ts rename to src/pack.ts index 15c543d8..e5385ef3 100644 --- a/src/pack-individually.ts +++ b/src/pack.ts @@ -8,7 +8,7 @@ import { doSharePath, flatDep, getDepsFromBundle } from './helper'; import * as Packagers from './packagers'; import { humanSize, zip } from './utils'; -function setArtifactPath(func, artifactPath) { +function setFunctionArtifactPath(this: EsbuildPlugin, func, artifactPath) { const version = this.serverless.getVersion(); // Serverless changed the artifact path location in version 1.18 if (semver.lt(version, '1.18.0')) { @@ -26,22 +26,42 @@ function setArtifactPath(func, artifactPath) { const excludedFilesDefault = ['package-lock.json', 'yarn.lock', 'package.json']; -export async function packIndividually(this: EsbuildPlugin) { +export async function pack(this: EsbuildPlugin) { // get a list of all path in build - const files = glob.sync('**', { - cwd: this.buildDirPath, - dot: true, - silent: true, - follow: true, - }); + const files: { localPath: string; rootPath: string }[] = glob + .sync('**', { + cwd: this.buildDirPath, + dot: true, + silent: true, + follow: true, + }) + .filter(p => !excludedFilesDefault.includes(p)) + .map(localPath => ({ localPath, rootPath: path.join(this.buildDirPath, localPath) })); if (isEmpty(files)) { throw new Error('Packaging: No files found'); } - // If individually is not set, ignore this part - if (!this.serverless?.service?.package?.individually) return null; + // 1) If individually is not set, just zip the all build dir and return + if (!this.serverless?.service?.package?.individually) { + const zipName = `${this.serverless.service.service}.zip`; + const artifactPath = path.join(this.workDirPath, SERVERLESS_FOLDER, zipName); + + const startZip = Date.now(); + await zip(artifactPath, files); + const { size } = fs.statSync(artifactPath); + + this.serverless.cli.log( + `Zip service ${this.serverless.service.service} - ${humanSize(size)} [${ + Date.now() - startZip + } ms]` + ); + // defined present zip as output artifact + this.serverless.service.package.artifact = artifactPath; + return; + } + // 2) If individually is set, we'll optimize files and zip per-function const packager = await Packagers.get(this.buildOptions.packager); // get a list of every function bundle @@ -62,10 +82,7 @@ export async function packIndividually(this: EsbuildPlugin) { buildResults.map(async ({ func, bundlePath }) => { const name = func.name; - const excludedFilesOrDirectory = [ - ...excludedFilesDefault, - ...bundlePathList.filter(p => !bundlePath.startsWith(p)), - ]; + const excludedFilesOrDirectory = bundlePathList.filter(p => !bundlePath.startsWith(p)); // allowed external dependencies in the final zip let depWhiteList = []; @@ -80,25 +97,22 @@ export async function packIndividually(this: EsbuildPlugin) { const artifactPath = path.join(this.workDirPath, SERVERLESS_FOLDER, zipName); // filter files - const filesPathList = files - .filter((filePath: string) => { - // exclude non individual files based on file or dir path - if (excludedFilesOrDirectory.find(p => filePath.startsWith(p))) return false; - - // exclude non whitelisted dependencies - if (filePath.startsWith('node_modules')) { - if (!hasExternals) return false; - if ( - // this is needed for dependencies that maps to a path (like scopped ones) - !depWhiteList.find(dep => doSharePath(filePath, 'node_modules/' + dep)) - ) - return false; - } - - return true; - }) - // get absolute path - .map(name => ({ path: path.join(this.buildDirPath, name), name })); + const filesPathList = files.filter(({ rootPath, localPath }) => { + // exclude non individual files based on file or dir path + if (excludedFilesOrDirectory.find(p => localPath.startsWith(p))) return false; + + // exclude non whitelisted dependencies + if (localPath.startsWith('node_modules')) { + if (!hasExternals) return false; + if ( + // this is needed for dependencies that maps to a path (like scopped ones) + !depWhiteList.find(dep => doSharePath(localPath, 'node_modules/' + dep)) + ) + return false; + } + + return true; + }); const startZip = Date.now(); await zip(artifactPath, filesPathList); @@ -110,7 +124,7 @@ export async function packIndividually(this: EsbuildPlugin) { ); // defined present zip as output artifact - setArtifactPath.call( + setFunctionArtifactPath.call( this, func, path.relative(this.serverless.config.servicePath, artifactPath) diff --git a/src/pre-local.ts b/src/pre-local.ts new file mode 100644 index 00000000..55714a11 --- /dev/null +++ b/src/pre-local.ts @@ -0,0 +1,6 @@ +import { EsbuildPlugin } from '.'; +export function preLocal(this: EsbuildPlugin) { + this.serverless.config.servicePath = this.buildDirPath; + // Set service path as CWD to allow accessing bundled files correctly + process.chdir(this.serverless.config.servicePath); +} diff --git a/src/pre-offline.ts b/src/pre-offline.ts new file mode 100644 index 00000000..3d3cff89 --- /dev/null +++ b/src/pre-offline.ts @@ -0,0 +1,11 @@ +import { EsbuildPlugin } from '.'; +import { relative } from 'path'; +export function preOffline(this: EsbuildPlugin) { + // Set offline location automatically if not set manually + if (!this.serverless?.service?.custom?.['serverless-offline']?.location) { + this.serverless.service.custom['serverless-offline'].location = relative( + this.serverless.config.servicePath, + this.buildDirPath + ); + } +} diff --git a/src/utils.ts b/src/utils.ts index b91dc8bc..4f7de55b 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -91,8 +91,9 @@ export const humanSize = (size: number) => { return `${sanitized} ${['B', 'KB', 'MB', 'GB', 'TB'][i]}`; }; -export const zip = (zipPath: string, filesPathList: { path: string; name: string }[]) => { +export const zip = (zipPath: string, filesPathList: { rootPath: string; localPath: string }[]) => { fs.mkdirpSync(path.dirname(zipPath)); + const zip = archiver.create('zip'); const output = fs.createWriteStream(zipPath); @@ -101,11 +102,11 @@ export const zip = (zipPath: string, filesPathList: { path: string; name: string zip.pipe(output); filesPathList.forEach(file => { - const stats = fs.statSync(file.path); + const stats = fs.statSync(file.rootPath); if (stats.isDirectory()) return; - zip.append(fs.readFileSync(file.path), { - name: file.name, + zip.append(fs.readFileSync(file.rootPath), { + name: file.localPath, mode: stats.mode, date: new Date(0), // necessary to get the same hash when zipping the same content }); From 9e4450666304069b5abb0fe5975f399affafb72e Mon Sep 17 00:00:00 2001 From: Olup Date: Mon, 1 Mar 2021 16:52:10 +0100 Subject: [PATCH 4/9] fix(readme): update readme --- README.md | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 6d9ac08b..26f8d840 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -💨 serverless-esbuild -============== +# 💨 serverless-esbuild Serverless plugin for zero-config JavaScript and TypeScript code bundling using promising fast & furious [`esbuild`](https://github.com/evanw/esbuild) bundler and minifier @@ -9,16 +8,15 @@ Serverless plugin for zero-config JavaScript and TypeScript code bundling using [![build status](https://img.shields.io/github/workflow/status/floydspace/serverless-esbuild/release)](https://github.com/floydspace/serverless-esbuild/actions) [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) - ## Features -* Zero-config: Works out of the box without the need to install any other compiler or plugins -* Supports ESNext syntax with transforming limitations (See *Note*) -* Supports `sls package`, `sls deploy` and `sls deploy function` -* Supports `sls invoke local` -* Integrates nicely with [`serverless-offline`](https://github.com/dherault/serverless-offline) +- Zero-config: Works out of the box without the need to install any other compiler or plugins +- Supports ESNext syntax with transforming limitations (See _Note_) +- Supports `sls package`, `sls deploy` and `sls deploy function` +- Supports `sls invoke local` +- Integrates nicely with [`serverless-offline`](https://github.com/dherault/serverless-offline) -*Note*: The default JavaScript syntax target is set to [`ES2017`](https://node.green/#ES2017), so the final bundle will be supported by all [AWS Lambda Node.js runtimes](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html). If you still using an old lambda runtime and have to respect it you can play with esbuild `target` option, see [JavaScript syntax support](https://github.com/evanw/esbuild#javascript-syntax-support) for more details about syntax transform limitations. +_Note_: The default JavaScript syntax target is set to [`ES2017`](https://node.green/#ES2017), so the final bundle will be supported by all [AWS Lambda Node.js runtimes](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html). If you still using an old lambda runtime and have to respect it you can play with esbuild `target` option, see [JavaScript syntax support](https://github.com/evanw/esbuild#javascript-syntax-support) for more details about syntax transform limitations. ## Install @@ -55,7 +53,6 @@ See [example folder](example) for a minimal example. All files from `package/include` will be included in the final build file. See [Exclude/Include](https://serverless.com/framework/docs/providers/aws/guide/packaging#exclude--include) - ## Usage ### Automatic compilation @@ -73,13 +70,13 @@ simulate AWS Lambda and AWS API Gateway locally. Add the plugins to your `serverless.yml` file and make sure that `serverless-esbuild` precedes `serverless-offline` as the order is important: + ```yaml - plugins: - ... - - serverless-esbuild - ... - - serverless-offline - ... +plugins: ... + - serverless-esbuild + ... + - serverless-offline + ... ``` Run `serverless offline` or `serverless offline start` to start the Lambda/API simulation. @@ -109,11 +106,12 @@ Note: When overriding ignore pattern, remember to ignore `.build` directory to a Configure your service the same as mentioned above, but additionally add the `serverless-dynamodb-local` plugin as follows: + ```yaml - plugins: - - serverless-esbuild - - serverless-dynamodb-local - - serverless-offline +plugins: + - serverless-esbuild + - serverless-dynamodb-local + - serverless-offline ``` Run `serverless offline start`. @@ -136,4 +134,8 @@ Options are: [Victor Korzunin](https://floydspace.github.io/) -Inspired by [serverless-plugin-typescript](https://github.com/prisma-labs/serverless-plugin-typescript) +## Contributors + +[Loup Topalian](https://github.com/olup) + +Inspired by [serverless-plugin-typescript](https://github.com/prisma-labs/serverless-plugin-typescript) and [serverless-webpack](https://github.com/serverless-heaven/serverless-webpack) From 26bcff5c72667db723c90117665ddff634351e43 Mon Sep 17 00:00:00 2001 From: Olup Date: Tue, 2 Mar 2021 17:01:11 +0100 Subject: [PATCH 5/9] fix(offline): fix pre-offline function --- src/pre-offline.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/pre-offline.ts b/src/pre-offline.ts index 3d3cff89..c0716043 100644 --- a/src/pre-offline.ts +++ b/src/pre-offline.ts @@ -1,11 +1,14 @@ -import { EsbuildPlugin } from '.'; import { relative } from 'path'; +import { assocPath } from 'ramda'; +import { EsbuildPlugin } from '.'; + export function preOffline(this: EsbuildPlugin) { // Set offline location automatically if not set manually if (!this.serverless?.service?.custom?.['serverless-offline']?.location) { - this.serverless.service.custom['serverless-offline'].location = relative( - this.serverless.config.servicePath, - this.buildDirPath + assocPath( + ['service', 'custom', 'serverless-offline', 'location'], + relative(this.serverless.config.servicePath, this.buildDirPath), + this.serverless ); } } From da11e33f5cd869e6adbf179d913dabb1d980a29c Mon Sep 17 00:00:00 2001 From: Olup Date: Wed, 3 Mar 2021 09:36:47 +0100 Subject: [PATCH 6/9] fix(package): better google function handle --- src/pack-externals.ts | 12 ++++++------ src/pack.ts | 11 ++++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/pack-externals.ts b/src/pack-externals.ts index 0be546d0..ac99f2ba 100644 --- a/src/pack-externals.ts +++ b/src/pack-externals.ts @@ -238,6 +238,12 @@ export async function packExternalModules(this: EsbuildPlugin) { } } + // GOOGLE: Copy modules only if not google-cloud-functions + // GCF Auto installs the package json + if (get(['service', 'provider', 'name'], this.serverless) === 'google') { + return; + } + const start = Date.now(); this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', ')); await packager.install(compositeModulePath); @@ -248,10 +254,4 @@ export async function packExternalModules(this: EsbuildPlugin) { await packager.prune(compositeModulePath); this.options.verbose && this.serverless.cli.log(`Prune: ${compositeModulePath} [${Date.now() - startPrune} ms]`); - - // GOOGLE: Copy modules only if not google-cloud-functions - // GCF Auto installs the package json - if (get(['service', 'provider', 'name'], this.serverless) === 'google') { - await fse.remove(path.join(compositeModulePath, 'node_modules')); - } } diff --git a/src/pack.ts b/src/pack.ts index e5385ef3..0541da46 100644 --- a/src/pack.ts +++ b/src/pack.ts @@ -27,6 +27,10 @@ function setFunctionArtifactPath(this: EsbuildPlugin, func, artifactPath) { const excludedFilesDefault = ['package-lock.json', 'yarn.lock', 'package.json']; export async function pack(this: EsbuildPlugin) { + // GOOGLE Provider requires a package.json and NO node_modules + const isGoogleProvider = this.serverless?.service?.provider?.name === 'google'; + const excludedFiles = isGoogleProvider ? [] : excludedFilesDefault; + // get a list of all path in build const files: { localPath: string; rootPath: string }[] = glob .sync('**', { @@ -35,7 +39,7 @@ export async function pack(this: EsbuildPlugin) { silent: true, follow: true, }) - .filter(p => !excludedFilesDefault.includes(p)) + .filter(p => !excludedFiles.includes(p)) .map(localPath => ({ localPath, rootPath: path.join(this.buildDirPath, localPath) })); if (isEmpty(files)) { @@ -68,7 +72,7 @@ export async function pack(this: EsbuildPlugin) { const buildResults = this.buildResults; const bundlePathList = buildResults.map(b => path.dirname(b.bundlePath)); - // get a list of external dependencies already listed in package.json + // get a list of externals const externals = without(this.buildOptions.exclude, this.buildOptions.external); const hasExternals = !!externals?.length; @@ -103,7 +107,8 @@ export async function pack(this: EsbuildPlugin) { // exclude non whitelisted dependencies if (localPath.startsWith('node_modules')) { - if (!hasExternals) return false; + // if no externals is set or if the provider is google, we do not need any files from node_modules + if (!hasExternals || isGoogleProvider) return false; if ( // this is needed for dependencies that maps to a path (like scopped ones) !depWhiteList.find(dep => doSharePath(localPath, 'node_modules/' + dep)) From 92468491d2010cb7e0a4c3f21aa7fe03771e1968 Mon Sep 17 00:00:00 2001 From: Olup Date: Wed, 3 Mar 2021 11:50:05 +0100 Subject: [PATCH 7/9] fix(package): properly assign offline location --- src/pre-offline.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pre-offline.ts b/src/pre-offline.ts index c0716043..fe1b53f6 100644 --- a/src/pre-offline.ts +++ b/src/pre-offline.ts @@ -5,10 +5,11 @@ import { EsbuildPlugin } from '.'; export function preOffline(this: EsbuildPlugin) { // Set offline location automatically if not set manually if (!this.serverless?.service?.custom?.['serverless-offline']?.location) { - assocPath( + const newServerless = assocPath( ['service', 'custom', 'serverless-offline', 'location'], relative(this.serverless.config.servicePath, this.buildDirPath), this.serverless ); + this.serverless.service.custom = newServerless.service.custom; } } From 8f93ed8aa17808a57de3347f2c7a10fd1506ee0a Mon Sep 17 00:00:00 2001 From: Olup Date: Fri, 5 Mar 2021 01:57:57 +0100 Subject: [PATCH 8/9] fix(package): revert naming --- src/{pack.ts => pack-individually.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{pack.ts => pack-individually.ts} (100%) diff --git a/src/pack.ts b/src/pack-individually.ts similarity index 100% rename from src/pack.ts rename to src/pack-individually.ts From 66d856d7395fe330c7fc9406dc1d99d3e18a9978 Mon Sep 17 00:00:00 2001 From: Olup Date: Fri, 5 Mar 2021 02:03:47 +0100 Subject: [PATCH 9/9] fix(package): post merge change filename --- src/index.ts | 2 +- src/{pack-individually.ts => pack.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{pack-individually.ts => pack.ts} (100%) diff --git a/src/index.ts b/src/index.ts index aace9d04..cc61cd1d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,7 @@ import * as chokidar from 'chokidar'; import { extractFileNames } from './helper'; import { packExternalModules } from './pack-externals'; -import { pack } from './pack-individually'; +import { pack } from './pack'; import { preOffline } from './pre-offline'; import { preLocal } from './pre-local'; diff --git a/src/pack-individually.ts b/src/pack.ts similarity index 100% rename from src/pack-individually.ts rename to src/pack.ts