Merge pull request #1666 from DanielXMoore/worker-fixes

Worker threads fixes and SourceMap revamp

build/test.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # Translate CIVET_THREADS into Mocha's --parallel
 # In particular, CIVET_THREADS Workers don't work within Mocha
 args=""
-if [ ! -z "${CIVET_THREADS:-}" ]; then
+if [ "${CIVET_THREADS:-0}" != 0 ]; then
   args="--parallel -j $CIVET_THREADS"
   export CIVET_THREADS=
 fi

civet.dev/public/playground.worker.js

@@ -17,7 +17,7 @@ onmessage = async (e) => {
     if (errors.length) {
       // Rerun with SourceMap to get error location
       errors = []
-      tsCode = Civet.generate(ast, { errors, sourceMap: Civet.SourceMap(code) });
+      tsCode = Civet.generate(ast, { errors, sourceMap: new Civet.SourceMap(code) });
       error = errors[0]
     }
   } catch (e) {
@@ -61,7 +61,7 @@ onmessage = async (e) => {
       if (errors.length) {
         // Rerun with SourceMap to get error location
         errors = []
-        jsCode = Civet.generate(ast, { js: true, errors, sourceMap: Civet.SourceMap(code) });
+        jsCode = Civet.generate(ast, { js: true, errors, sourceMap: new Civet.SourceMap(code) });
         // Don't postError(errors[0]) here so that we still display TypeScript
         // transpilation; only show error when trying to Run code
         throw errors[0]

integration/eslint/source/index.civet

@@ -27,7 +27,7 @@ export function civet(options: Options = {js: true}): ESLint.Plugin
     // Length 1 corresponding to the return value of `preprocess`
     [messages] := _messages
     if sourceMap := sourceMaps.get filename
-      sourceMapLines := sourceMap.data.lines
+      sourceMapLines := sourceMap.lines ?? sourceMap.data.lines // older Civet
       for each message of messages
         if message.line?
           [message.line, message.column] = remap message.line, message.column

lsp/source/lib/typescript-service.mts

@@ -46,9 +46,8 @@ try {
 }
 
 interface SourceMap {
-  data: {
-    lines: CivetSourceMap["data"]["lines"]
-  }
+  lines: CivetSourceMap["lines"]
+  data: CivetSourceMap["data"]
 }
 
 // ts doesn't have this key in the type
@@ -57,7 +56,7 @@ interface ResolvedModuleWithFailedLookupLocations extends ts.ResolvedModuleWithF
 }
 
 export interface FileMeta {
-  sourcemapLines: SourceMap["data"]["lines"] | undefined
+  sourcemapLines: SourceMap["lines"] | undefined
   transpiledDoc: TextDocument | undefined
   parseErrors: (Error | ParseError)[] | undefined
   fatal: boolean // whether errors were fatal during compilation, so no doc
@@ -384,7 +383,7 @@ function TSHost(compilationSettings: CompilerOptions, initialFileNames: string[]
     return snapshot
   }
 
-  function createOrUpdateMeta(path: string, transpiledDoc: TextDocument, sourcemapLines?: SourceMap["data"]["lines"], parseErrors?: (Error | ParseError)[], fatal?: boolean) {
+  function createOrUpdateMeta(path: string, transpiledDoc: TextDocument, sourcemapLines?: SourceMap["lines"], parseErrors?: (Error | ParseError)[], fatal?: boolean) {
     let meta = fileMetaData.get(path)
 
     if (!meta) {
@@ -415,7 +414,8 @@ function TSHost(compilationSettings: CompilerOptions, initialFileNames: string[]
 
     if (result) {
       const { code: transpiledCode, sourceMap, errors } = result
-      createOrUpdateMeta(sourcePath, transpiledDoc, sourceMap?.data.lines, errors, false)
+      const sourceMapLines = sourceMap?.lines ?? sourceMap?.data.lines // older Civet
+      createOrUpdateMeta(sourcePath, transpiledDoc, sourceMapLines, errors, false)
       TextDocument.update(transpiledDoc, [{ text: transpiledCode }], version)
 
       return transpiledCode

lsp/source/lib/util.mts

@@ -24,7 +24,7 @@ import {
   remapRange,
 } from '@danielx/civet/ts-diagnostic';
 
-export type SourcemapLines = SourceMap['data']['lines'];
+export type SourcemapLines = SourceMap['lines'];
 
 export {
   flattenDiagnosticMessageText,

lsp/test/util.civet

@@ -1,5 +1,5 @@
 // TODO: figure out the magic ts-note/TypeScript config to make this work without destructuring from default import
-{ intersectRanges, containsRange, makeRange, remapPosition, forwardMap, convertCoffeeScriptSourceMap } from ../source/lib/util.mjs
+{ intersectRanges, containsRange, makeRange, remapPosition, forwardMap } from ../source/lib/util.mjs
 assert from assert
 Civet from @danielx/civet
 
@@ -25,7 +25,7 @@ describe "util", ->
       sourceMap: true
     })
 
-    linesMap := sourceMap.data.lines
+    linesMap := sourceMap.lines
     // console.log code, linesMap
 
     assert.deepEqual remapPosition({
@@ -52,7 +52,7 @@ describe "util", ->
 
     [0..6].forEach (i) ->
       srcColumn := i + 13
-      pos := forwardMap(sourceMap.data.lines, {line: 0, character: srcColumn})
+      pos := forwardMap(sourceMap.lines, {line: 0, character: srcColumn})
       srcStr := src.slice(srcColumn, srcColumn + 7)
       assert.equal srcStr.replace(" ", "("), generatedLines[pos.line].slice(pos.character, pos.character + 7)
 
@@ -71,7 +71,7 @@ describe "util", ->
     generatedLines := code.split("\n")
 
     [2..6].forEach (srcColumn, i) ->
-      pos := forwardMap(sourceMap.data.lines, {line: 1, character: srcColumn})
+      pos := forwardMap(sourceMap.lines, {line: 1, character: srcColumn})
       srcStr := srcLines[1].slice(srcColumn, srcColumn + 5 - i)
       generatedStr := generatedLines[pos.line].slice(pos.character, pos.character + 5 - i)
       assert.equal srcStr, generatedStr

source/cli.civet

@@ -339,7 +339,7 @@ export function repl(args: string[], options: Options)
         if errors#
           // Rerun with sourceMap
           errors = []
-          generate ast, { ...options, errors, sourceMap: SourceMap(input) }
+          generate ast, { ...options, errors, sourceMap: new SourceMap input }
           showError errors[0]
           return callback null, undefined
 

source/esm.civet

@@ -48,10 +48,15 @@ export async function load(url: string, context: any, next: any)
   if context.format is "civet"
     path := fileURLToPath url
     source := readFileSync path, "utf8"
-    {code: tsSource, sourceMap} := await compile source,
-      filename: path
-      sourceMap: true
-      js: true
+    let tsSource, sourceMap
+    try
+      {code: tsSource, sourceMap} = await compile source,
+        filename: path
+        sourceMap: true
+        js: true
+    catch e
+      console.error `Civet failed to compile ${url}:`, e
+      throw e
 
     // NOTE: Append .tsx to URL so ts-node treats as TypeScript
     transpiledUrl := url + ".tsx"

source/generate.civet

@@ -5,7 +5,9 @@ type { ASTNode } from './parser/types.civet'
 export type Options =
   sourceMap?: undefined |
     updateSourceMap: (token: string, pos?: number) => void
-    data: { srcLine: number, srcColumn: number, srcOffset: number }
+    srcLine: number
+    srcColumn: number
+    srcOffset: number
   js?: boolean
   filename?: string
   errors?: ParseError[]
@@ -17,7 +19,7 @@ function stringify(node: ASTNode): string
     return `${node}`
 
 function gen(root: ASTNode, options: Options): string
-  updateSourceMap := options?.sourceMap?.updateSourceMap
+  updateSourceMap := options?.sourceMap?@updateSourceMap
   return recurse root
 
   function recurse(node: ASTNode): string
@@ -26,7 +28,7 @@ function gen(root: ASTNode, options: Options): string
     if node <? "string"
       // increment output line/column
       updateSourceMap? node
-      //console.log 'advance', JSON.stringify(node), options.sourceMap.data.srcLine, options.sourceMap.data.srcColumn if options?.sourceMap?
+      //console.log 'advance', JSON.stringify(node), options.sourceMap.srcLine, options.sourceMap.srcColumn if options?.sourceMap?
 
       return node
 
@@ -50,10 +52,10 @@ function gen(root: ASTNode, options: Options): string
           if node.$loc?
             sourceMap.updateSourceMap "", node.$loc.pos
           // Convert 0-based to 1-based
-          line = sourceMap.data.srcLine + 1
-          column = sourceMap.data.srcColumn + 1
+          line = sourceMap.srcLine + 1
+          column = sourceMap.srcColumn + 1
           //console.log 'error', node.message, 'at', line, column, offset
-          offset = sourceMap.data.srcOffset
+          offset = sourceMap.srcOffset
         options.errors ?= []
         options.errors.push new ParseError
           node.message
@@ -67,7 +69,7 @@ function gen(root: ASTNode, options: Options): string
       if "$loc" in node
         {token, $loc} := node
         updateSourceMap? token, $loc.pos if $loc?
-        //console.log 'set', node, options.sourceMap.data.srcLine, options.sourceMap.data.srcColumn if options?.sourceMap?
+        //console.log 'set', node, options.sourceMap.srcLine, options.sourceMap.srcColumn if options?.sourceMap?
         return token
 
       unless node.children

source/main.civet

@@ -192,7 +192,7 @@ export function compile<const T extends CompilerOptions>(src: string, options?:
 
     if options.sourceMap or options.inlineMap
       //@ts-ignore sourceMap option for generate
-      options.sourceMap = SourceMap(src)
+      options.sourceMap = new SourceMap src
       code := generate ast, options
       checkErrors()
 
@@ -211,7 +211,7 @@ export function compile<const T extends CompilerOptions>(src: string, options?:
     if options!.errors?.length
       delete options.errors
       //@ts-ignore sourceMap option for generate
-      options.sourceMap = SourceMap(src)
+      options.sourceMap = new SourceMap src
       generate ast, options
       checkErrors()
 

source/node-worker.civet

@@ -15,6 +15,11 @@ async do
           result = await (compile as any) ...args
         else
           throw `Unknown operation: ${op}`
-      parentPort!.postMessage {id, result}
+      // If we pass in an `errors` option, its modification is part of output
+      parentPort!.postMessage {id, result, errors: args[1]?.errors}
     catch error
-      parentPort!.postMessage {id, error}
+      console.log `Civet worker failed to compile:`, error
+      parentPort!.postMessage {id, error: {
+        type: error.constructor.name
+        error.{name,message}
+      }}