From 4abced52c2fa0476683ff47ab1a07e559446e45e Mon Sep 17 00:00:00 2001 From: Sebastian Wessel Date: Wed, 10 Jul 2024 22:10:37 +0100 Subject: [PATCH 1/3] feat: Typescript support - Run Typescript in the QuickJS sandbox #20 --- README.md | 8 +- docs/README.md | 6 +- docs/sandboxed-code.md | 51 +++++++++++ example/server/worker.ts | 9 ++ jsr.json | 2 +- package.json | 24 ++++- src/createVirtualFileSystem.ts | 6 +- src/getTypescriptSupport.ts | 157 +++++++++++++++++++++++++++++++++ src/index.ts | 2 + src/quickJS.ts | 7 +- src/types/RuntimeOptions.ts | 10 +++ 11 files changed, 267 insertions(+), 15 deletions(-) create mode 100644 src/getTypescriptSupport.ts diff --git a/README.md b/README.md index 3ca80e6..0d31541 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# QuickJS - Execute JavaScript in a WebAssembly QuickJS Sandbox +# QuickJS - Execute JavaScript and TypeScript in a WebAssembly QuickJS Sandbox -This TypeScript package allows you to safely execute JavaScript code within a WebAssembly sandbox using the QuickJS engine. Perfect for isolating and running untrusted code securely, it leverages the lightweight and fast QuickJS engine compiled to WebAssembly, providing a robust environment for code execution. +This TypeScript package allows you to safely execute **JavaScript AND TypeScript code** within a WebAssembly sandbox using the QuickJS engine. Perfect for isolating and running untrusted code securely, it leverages the lightweight and fast QuickJS engine compiled to WebAssembly, providing a robust environment for code execution. ## Features -- **Security**: Run untrusted JavaScript code in a safe, isolated environment. +- **Security**: Run untrusted JavaScript and TypeScript code in a safe, isolated environment. - **Basic Node.js modules**: Provides basic standard Node.js module support for common use cases. - **File System**: Can mount a virtual file system. - **Custom Node Modules**: Custom node modules are mountable. @@ -12,7 +12,7 @@ This TypeScript package allows you to safely execute JavaScript code within a We - **Test-Runner**: Includes a test runner and chai based `expect`. - **Performance**: Benefit from the lightweight and efficient QuickJS engine. - **Versatility**: Easily integrate with existing TypeScript projects. -- **Simplicity**: User-friendly API for executing and managing JavaScript code in the sandbox. +- **Simplicity**: User-friendly API for executing and managing JavaScript and TypeScript code in the sandbox. **[View the full documentation](https://sebastianwessel.github.io/quickjs/)** diff --git a/docs/README.md b/docs/README.md index cb8475a..081982a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # Package @sebastianwessel/quickjs -This TypeScript package allows you to safely execute JavaScript code within a WebAssembly sandbox using the QuickJS engine. Perfect for isolating and running untrusted code securely, it leverages the lightweight and fast QuickJS engine compiled to WebAssembly, providing a robust environment for code execution. +This TypeScript package allows you to safely execute **JavaScript and TypeScript code** within a WebAssembly sandbox using the QuickJS engine. Perfect for isolating and running untrusted code securely, it leverages the lightweight and fast QuickJS engine compiled to WebAssembly, providing a robust environment for code execution. ## Documentation @@ -21,7 +21,7 @@ This TypeScript package allows you to safely execute JavaScript code within a We ## Features -- **Security**: Run untrusted JavaScript code in a safe, isolated environment. +- **Security**: Run untrusted JavaScript and TypeScript code in a safe, isolated environment. - **Basic Node.js modules**: Provides basic standard Node.js module support for common use cases. - **File System**: Can mount a virtual file system. - **Custom Node Modules**: Custom node modules are mountable. @@ -29,7 +29,7 @@ This TypeScript package allows you to safely execute JavaScript code within a We - **Test-Runner**: Includes a test runner and chai based `expect`. - **Performance**: Benefit from the lightweight and efficient QuickJS engine.. - **Versatility**: Easily integrate with existing TypeScript projects. -- **Simplicity**: User-friendly API for executing and managing JavaScript code in the sandbox. +- **Simplicity**: User-friendly API for executing and managing JavaScript and TypeScript code in the sandbox. ## Basic Usage diff --git a/docs/sandboxed-code.md b/docs/sandboxed-code.md index fa45ad4..5282a37 100644 --- a/docs/sandboxed-code.md +++ b/docs/sandboxed-code.md @@ -9,6 +9,8 @@ The most common use case is, to execute a give JavaScript code in the QuickJS we The `evalCode` function described below, is intend to be used, when a given JavaScript code should be executed and optional a result value is returned. It is recommended, to always return a value, for better validation on the host side, that the code was executed as expected. +In the sandbox, the executed code "lives" in `/src/index.js`. If custom files are added via configuration, source files should be placed below `/src`. Nested directories are supported. + ```typescript import { quickJS } from '@sebastianwessel/quickjs' @@ -84,3 +86,52 @@ console.log(result) The `validateCode` functions returns a unified result object, and does not throw. For further information, it is hight recommended to [read "Basic Understanding"](./basic.md). + +## Execute TypeScript in the Sandbox + +Executing TypeScript in the sandboxed runtime, is similar to executing JavaScript. An additional transpile step will be applied to the given code. Additionally each file below `/src` with file extension`.ts` in the custom file system will be transpiled. + +The TypeScript code is only transpiled, but not type-checked! +If checking types is required, it should be done and handled, before using this library. + +**Requirements:** + +- optional dependency package `typescript` must be installed on the host system +- `createRuntime` option `transformTypescript` must be set to `true` + +Example: + +```typescript +import { quickJS } from '@sebastianwessel/quickjs' + +const { createRuntime } = await quickJS() + +// Create a runtime instance (sandbox) +const { evalCode } = await createRuntime({ + transformTypescript: true, + mountFs: { + src: { + 'test.ts': `export const testFn = (value: string): string => { + console.log(value) + return value + }`, + }, + }, +}) + + +const result = await evalCode(` +import { testFn } from './test.js' + +const t = (value:string):number=>value.length + +console.log(t('abc')) + +export default testFn('hello') +`) + +console.log(result) // { ok: true, data: 'hello' } +// console log on host: +// 3 +// hello +``` diff --git a/example/server/worker.ts b/example/server/worker.ts index dc92fe4..4bbe7cb 100644 --- a/example/server/worker.ts +++ b/example/server/worker.ts @@ -28,6 +28,15 @@ class MyThreadWorker extends ThreadWorker { allowFetch: true, enableTestUtils: true, env: {}, + transformTypescript: true, + mountFs: { + src: { + 'test.ts': `export const testFn = (value: string): string => { + console.log(value) + return value + }`, + }, + }, }) const result = await evalCode(data.content) diff --git a/jsr.json b/jsr.json index 0e0b357..27ebd0c 100644 --- a/jsr.json +++ b/jsr.json @@ -2,7 +2,7 @@ "$schema": "https://jsr.io/schema/config-file.v1.json", "name": "@sebastianwessel/quickjs", "version": "1.2.0", - "description": "A typescript package to execute javascript code in a webassembly quickjs sandbox", + "description": "A typescript package to execute JavaScript and TypeScript code in a webassembly quickjs sandbox", "exports": "./dist/esm/index.js", "publish": { "include": ["dist/**/*.js", "dist/**/*.d.ts", "README.md", "package.json"], diff --git a/package.json b/package.json index adc9fa8..dff878b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@sebastianwessel/quickjs", "version": "1.2.0", - "description": "A typescript package to execute javascript code in a webassembly quickjs sandbox", + "description": "A typescript package to execute JavaScript and TypeScript code in a webassembly quickjs sandbox", "type": "module", "keywords": [ "typescript", @@ -17,10 +17,18 @@ "package", "library" ], - "files": ["dist"], + "files": [ + "dist" + ], "tshy": { - "exclude": ["src/**/*.test.ts", "vendor"], - "dialects": ["esm", "commonjs"], + "exclude": [ + "src/**/*.test.ts", + "vendor" + ], + "dialects": [ + "esm", + "commonjs" + ], "exports": { "./package.json": "./package.json", ".": "./src/index.ts" @@ -75,6 +83,14 @@ "quickjs-emscripten-core": "^0.29.2", "rate-limiter-flexible": "^5.0.3" }, + "peerDependencies": { + "typescript": ">= 5.5.3" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + }, "exports": { "./package.json": "./package.json", ".": { diff --git a/src/createVirtualFileSystem.ts b/src/createVirtualFileSystem.ts index 3710ccc..e1b0a42 100644 --- a/src/createVirtualFileSystem.ts +++ b/src/createVirtualFileSystem.ts @@ -18,13 +18,15 @@ import type { RuntimeOptions } from './types/RuntimeOptions.js' * @returns filesystem fs and volume vol */ export const createVirtualFileSystem = (runtimeOptions: RuntimeOptions = {}) => { + const customFileSystem = runtimeOptions.mountFs + // setup node_modules folder const virtualFS: Record = { '/': { - ...runtimeOptions.mountFs, + ...customFileSystem, src: { // biome-ignore lint/complexity/useLiteralKeys: - ...(runtimeOptions.mountFs?.['src'] ? (runtimeOptions.mountFs?.['src'] as NestedDirectoryJSON) : {}), + ...(customFileSystem?.['src'] ? (customFileSystem?.['src'] as NestedDirectoryJSON) : {}), }, node_modules: { ...runtimeOptions?.nodeModules, diff --git a/src/getTypescriptSupport.ts b/src/getTypescriptSupport.ts new file mode 100644 index 0000000..1c92eef --- /dev/null +++ b/src/getTypescriptSupport.ts @@ -0,0 +1,157 @@ +import { join } from 'node:path' +import type { IFs, NestedDirectoryJSON } from 'memfs' +import type { default as TS } from 'typescript' + +export type TranspileNestedJsonOptions = { + fileExtensions?: string[] +} + +export type TranspileVirtualFsOptions = { + fileExtensions?: string[] + startPath?: string +} + +/** + * Add support for handling typescript files and code. + * Requires the optional dependency 'typescript'. + */ +export const getTypescriptSupport = async (enabled = false, options?: TS.CompilerOptions) => { + if (!enabled) { + return { + transpileFile: ( + value: string, + _compilerOptions?: TS.CompilerOptions, + _fileName?: string, + _diagnostics?: TS.Diagnostic[], + _moduleName?: string, + ) => value, + transpileNestedDirectoryJSON: ( + mountFsJson: NestedDirectoryJSON, + _option?: TranspileNestedJsonOptions, + ): NestedDirectoryJSON => mountFsJson, + transpileVirtualFs: (fs: IFs, _options?: TranspileVirtualFsOptions): IFs => { + return fs + }, + } + } + + let ts: typeof TS + try { + ts = await import('typescript') + } catch (error) { + throw new Error('Package "typescript" is missing') + } + + const compilerOptions: TS.CompilerOptions = { + module: 99, // ESNext + target: 99, // ES2023 + //lib: ['ESNext'], + allowJs: true, + // moduleResolution: 100, + skipLibCheck: true, + esModuleInterop: true, + strict: false, + allowSyntheticDefaultImports: true, + ...options, + } + + /** + * Transpile a single File + * + * @param params source typescript code + * @returns javascript code + */ + const transpileFile: typeof ts.transpile = ( + input: string, + cpOptions = compilerOptions, + fileName?: string, + diagnostics?: TS.Diagnostic[], + moduleName?: string, + ) => ts.transpile(input, cpOptions, fileName, diagnostics, moduleName) + + /** + * Iterates through the given JSON - NestedDirectoryJSON for defining the virtual file system. + * Replace every typescript file with the transpiled javascript version and renames the file to *.js + * + * @param mountFsJson + * @param fileExtensions + * @returns + */ + const transpileNestedDirectoryJSON = ( + mountFsJson: NestedDirectoryJSON, + option?: TranspileNestedJsonOptions, + ): NestedDirectoryJSON => { + const opt = { + fileExtensions: ['ts'], + ...option, + } + + const transformJson = ( + obj: NestedDirectoryJSON, + transformer: (key: string, value: any) => [string, any], + ): NestedDirectoryJSON => { + if (typeof obj === 'object' && obj !== null) { + const newObj: any = Array.isArray(obj) ? [] : {} + for (const key in obj) { + if (obj[key]) { + const [newKey, newValue] = transformer(key, obj[key]) + newObj[newKey] = transformJson(newValue, transformer) + } + } + return newObj + } + return obj + } + + const transpileTypescript = (key: string, value: string | Buffer): [string, any] => { + if (!opt.fileExtensions.includes(key)) { + return [key, value] + } + const newFileName = key.replace('.ts', '.js') + const tsSource = typeof value === 'string' ? value : value.toString() + const jsSource = transpileFile(tsSource, compilerOptions) + return [newFileName, jsSource] + } + + return transformJson(mountFsJson, transpileTypescript) + } + + const transpileVirtualFs = (fs: IFs, options?: TranspileVirtualFsOptions) => { + const opt = { + startPath: '/src', + ...options, + } + + const transformFileSystem = (startPath: string): void => { + if (!fs.existsSync(startPath)) { + throw new Error(`Directory not found: ${startPath}`) + } + + const files = fs.readdirSync(startPath) + for (const file of files) { + const filePath = join(startPath, file as string) + const stat = fs.lstatSync(filePath) + + if (stat.isDirectory()) { + // ignore node_modules + if ((file as string) !== 'node_modules') { + transformFileSystem(filePath) + } + } else if (stat.isFile()) { + const newFilePath = join(startPath, (file as string).replace('.ts', '.js')) + const content = fs.readFileSync(filePath, 'utf8') + const tsSource = typeof content === 'string' ? content : content.toString() + const jsSource = transpileFile(tsSource, compilerOptions, newFilePath) + fs.renameSync(filePath, newFilePath) + fs.writeFileSync(newFilePath, jsSource) + } + } + } + + transformFileSystem(opt.startPath) + + return fs + } + + return { transpileFile, transpileNestedDirectoryJSON, transpileVirtualFs } +} diff --git a/src/index.ts b/src/index.ts index 9cc7ea9..3ca2bc7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -63,3 +63,5 @@ export * from './types/QuickJSEvalReturn.js' export * from './adapter/fetch.js' export * from './createVirtualFileSystem.js' export * from './types/OkResponseCheck.js' +export * from './getTypescriptSupport.js' +export * from './modulePathNormalizer.js' diff --git a/src/quickJS.ts b/src/quickJS.ts index 16805d5..1c79964 100644 --- a/src/quickJS.ts +++ b/src/quickJS.ts @@ -15,6 +15,7 @@ import type { IFs } from 'memfs' import { createTimeInterval } from './createTimeInterval.js' import { createVirtualFileSystem } from './createVirtualFileSystem.js' import { getModuleLoader } from './getModuleLoader.js' +import { getTypescriptSupport } from './getTypescriptSupport.js' import { modulePathNormalizer } from './modulePathNormalizer.js' import type { OkResponseCheck } from './types/OkResponseCheck.js' @@ -31,6 +32,9 @@ export const quickJS = async (wasmVariantName = '@jitl/quickjs-ng-wasmfile-relea const fs = existingFs ?? createVirtualFileSystem(runtimeOptions).fs + const { transpileVirtualFs, transpileFile } = await getTypescriptSupport(runtimeOptions.transformTypescript) + transpileVirtualFs(fs) + vm.runtime.setModuleLoader(getModuleLoader(fs, runtimeOptions), modulePathNormalizer) const arena = new Arena(vm, { isMarshalable: true }) @@ -105,7 +109,8 @@ export const quickJS = async (wasmVariantName = '@jitl/quickjs-ng-wasmfile-relea }, 0) try { - const evalResult = arena.evalCode(code, filename, { + const jsCode = transpileFile(code) + const evalResult = arena.evalCode(jsCode, filename, { strict: true, strip: true, backtraceBarrier: true, diff --git a/src/types/RuntimeOptions.ts b/src/types/RuntimeOptions.ts index a2dfa29..503c698 100644 --- a/src/types/RuntimeOptions.ts +++ b/src/types/RuntimeOptions.ts @@ -1,4 +1,5 @@ import type { NestedDirectoryJSON } from 'memfs' +import type { default as TS } from 'typescript' export type RuntimeOptions = { /** @@ -85,4 +86,13 @@ export type RuntimeOptions = { * This means, the values on the host, can be set by the guest system */ dangerousSync?: Record + /** + * Transpile all typescript files to javascript file in mountFs + * Requires dependency typescript to be installed + */ + transformTypescript?: boolean + /** + * The Typescript compiler options for transpiling files from typescript to JavaScript + */ + transformCompilerOptions?: TS.CompilerOptions } From a10e4988f2ac8119d2e424153d0d0e183a37b3d4 Mon Sep 17 00:00:00 2001 From: Sebastian Wessel Date: Thu, 11 Jul 2024 18:13:27 +0100 Subject: [PATCH 2/3] feat: Improve customization #19 --- src/quickJS.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/quickJS.ts b/src/quickJS.ts index 1c79964..79ad3c4 100644 --- a/src/quickJS.ts +++ b/src/quickJS.ts @@ -51,7 +51,7 @@ export const quickJS = async (wasmVariantName = '@jitl/quickjs-ng-wasmfile-relea `) const dispose = () => { - let err: unknown | undefined + let err: unknown try { arena.dispose() } catch (error) { @@ -204,7 +204,7 @@ export const quickJS = async (wasmVariantName = '@jitl/quickjs-ng-wasmfile-relea } } - return { vm: arena, dispose, evalCode, validateCode } + return { vm: arena, dispose, evalCode, validateCode, mountedFs: fs } } return { createRuntime } From 4adc3ae0e49b1376f4e925c8b4cb3321a98c7b1f Mon Sep 17 00:00:00 2001 From: Sebastian Wessel Date: Thu, 11 Jul 2024 18:14:07 +0100 Subject: [PATCH 3/3] chore: cleanup --- package.json | 14 +++----------- src/types/InitResponseType.ts | 32 +++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index dff878b..0ec6c6b 100644 --- a/package.json +++ b/package.json @@ -17,18 +17,10 @@ "package", "library" ], - "files": [ - "dist" - ], + "files": ["dist"], "tshy": { - "exclude": [ - "src/**/*.test.ts", - "vendor" - ], - "dialects": [ - "esm", - "commonjs" - ], + "exclude": ["src/**/*.test.ts", "vendor"], + "dialects": ["esm", "commonjs"], "exports": { "./package.json": "./package.json", ".": "./src/index.ts" diff --git a/src/types/InitResponseType.ts b/src/types/InitResponseType.ts index f561b37..ab450ad 100644 --- a/src/types/InitResponseType.ts +++ b/src/types/InitResponseType.ts @@ -6,17 +6,47 @@ import type { OkResponse } from './OkResponse.js' import type { OkResponseCheck } from './OkResponseCheck.js' export type InitResponseType = { + /** + * The QuickJS Arena runtime + */ vm: Arena + /** + * Dispose the sandbox. + * It should not be needed to call this, as this is done automatically + */ dispose: () => void + /** + * Execute code once and cleanup after execution. + * + * The result of the code execution must be exported with export default. + * If the code is async, it needs to be awaited on export. + * + * @example + * ```js + * const result = await evalCode('export default await asyncFunction()') + * ``` + */ evalCode: ( code: string, filename?: string, options?: Omit & { executionTimeout?: number }, ) => Promise + /** + * Compile code only, but does not execute the code. + * + * @example + * ```js + * const result = await validateCode('export default await asyncFunction()') + * ``` + */ validateCode: ( code: string, filename?: string, options?: Omit & { executionTimeout?: number }, ) => Promise - mountedFs?: IFs + /** + * the virtual filesystem + * + */ + mountedFs: IFs }