LSP unit testing of completions

lsp/.gitignore

@@ -1,3 +1,4 @@
 *.metafile
 *.vsix
 open-vsx-token
+server.log

lsp/package.json

@@ -89,7 +89,8 @@
     "package": "bash ./build/build.sh && vsce package --yarn",
     "vsce-publish": "bash ./build/build.sh && vsce publish",
     "test": "mocha",
-    "watch": "node build.mjs --watch"
+    "watch": "node build.mjs --watch",
+    "postinstall": "patch-package"
   },
   "dependencies": {},
   "devDependencies": {
@@ -98,17 +99,20 @@
     "@types/mocha": "^10.0.8",
     "@types/node": "^14.17.0",
     "@types/vscode": "^1.63.0",
+    "@volar/language-server": "^2.4.8",
+    "@volar/test-utils": "^2.4.8",
     "@vscode/test-electron": "^2.1.2",
     "@vscode/vsce": "^2.19.0",
     "esbuild": "0.24.0",
     "esbuild-visualizer": "^0.3.1",
     "mocha": "^10.7.3",
     "nyc": "^15.1.0",
+    "patch-package": "^8.0.0",
     "source-map-support": "^0.5.21",
     "ts-node": "^10.6.0",
     "typescript": "^5.6.3",
     "vscode-languageclient": "^8.0.2",
     "vscode-languageserver": "^8.0.2",
-    "vscode-languageserver-textdocument": "^1.0.7"
+    "vscode-languageserver-textdocument": "^1.0.12"
   }
 }

lsp/patches/@volar+test-utils+2.4.8.patch

@@ -0,0 +1,28 @@
+diff --git a/node_modules/@volar/test-utils/index.js b/node_modules/@volar/test-utils/index.js
+index 830c066..816c26b 100644
+--- a/node_modules/@volar/test-utils/index.js
++++ b/node_modules/@volar/test-utils/index.js
+@@ -17,6 +17,14 @@ function startLanguageServer(serverModule, cwd) {
+         cwd,
+         stdio: 'pipe',
+     });
++    const logFilename = 'server.log';
++    const logFile = fs.createWriteStream
++        ? fs.createWriteStream(logFilename, { flags: 'w' })
++        : null;
++    if (logFile) {
++        childProcess.stdout.pipe(logFile);
++        childProcess.stderr.pipe(logFile);
++    }
+     const connection = _.createProtocolConnection(childProcess.stdout, childProcess.stdin);
+     const documentVersions = new Map();
+     const openedDocuments = new Map();
+@@ -153,8 +161,6 @@ function startLanguageServer(serverModule, cwd) {
+                 textDocument: { uri },
+                 position,
+             });
+-            // @volar/language-server only returns CompletionList
+-            assert(!Array.isArray(result));
+             return result;
+         },
+         sendCompletionResolveRequest(item) {

lsp/source/lib/typescript-service.mts

@@ -232,7 +232,7 @@ function TSHost(compilationSettings: CompilerOptions, initialFileNames: string[]
       const extension = getExtensionFromPath(path)
       const transpiler = transpilers.get(extension)
 
-      // console.log("addOrUpdateDocument", path, extension, transpiler)
+      // logger.log("addOrUpdateDocument", path, extension, transpiler)
 
       if (transpiler) {
         const { target } = transpiler
@@ -300,7 +300,7 @@ function TSHost(compilationSettings: CompilerOptions, initialFileNames: string[]
       return Array.from(scriptFileNames)
     },
     writeFile(fileName: string, content: string) {
-      console.log("write", fileName, content)
+      // console.log("write", fileName, content)
     }
   });
 
@@ -445,7 +445,11 @@ function TSHost(compilationSettings: CompilerOptions, initialFileNames: string[]
 }
 
 function TSService(projectURL = "./") {
-  const logger = console
+  const logger = {
+    info: (...args: unknown[]) => void 0,
+    error: (...args: unknown[]) => void 0,
+    warn: (...args: unknown[]) => void 0
+  }
 
   logger.info("CIVET VSCODE PLUGIN", version)
   logger.info("TYPESCRIPT", typescriptVersion)
@@ -495,27 +499,27 @@ function TSService(projectURL = "./") {
   const civetPath = "@danielx/civet"
   try {
     projectRequire(`${civetPath}/lsp/package.json`)
-    console.info("USING DEVELOPMENT VERSION OF CIVET -- BE SURE TO yarn build")
+    logger.info("USING DEVELOPMENT VERSION OF CIVET -- BE SURE TO yarn build")
   } catch (e) {}
   try {
     Civet = projectRequire(civetPath)
     CivetConfig = projectRequire(`${civetPath}/config`)
     const CivetVersion = projectRequire(`${civetPath}/package.json`).version
-    console.info(`LOADED PROJECT CIVET ${CivetVersion}: ${path.join(projectURL, civetPath)} \n\n`)
+    logger.info(`LOADED PROJECT CIVET ${CivetVersion}: ${path.join(projectURL, civetPath)} \n\n`)
   } catch (e) {
-    console.info("USING BUNDLED CIVET")
+    logger.info("USING BUNDLED CIVET")
   }
 
   let civetConfig: CompileOptions = {}
   CivetConfig.findConfig(projectPath).then(async (configPath) => {
     if (configPath) {
-      console.info("Loading Civet config @", configPath)
+      logger.info("Loading Civet config @", configPath)
       const config = await CivetConfig.loadConfig(configPath)
-      console.info("Found civet config!")
+      logger.info("Found civet config!")
       civetConfig = config
-    } else console.info("No Civet config found")
+    } else logger.info("No Civet config found")
   }).catch((e: unknown) => {
-    console.error("Error loading Civet config", e)
+    logger.error("Error loading Civet config", e)
   })
 
   return Object.assign({}, service, {
@@ -534,7 +538,7 @@ function TSService(projectURL = "./") {
         .map(file => pathToFileURL(file).toString())
 
       for (const filePath of pluginFiles) {
-        console.info("Loading plugin", filePath)
+        logger.info("Loading plugin", filePath)
         await loadPlugin(filePath)
       }
     }
@@ -543,13 +547,13 @@ function TSService(projectURL = "./") {
   async function loadPlugin(path: string) {
     await import(path)
       .then(({ default: plugin }: { default: Plugin }) => {
-        console.info("Loaded plugin", plugin)
+        logger.info("Loaded plugin", plugin)
         plugin.transpilers?.forEach((transpiler: Transpiler) => {
           transpilers.set(transpiler.extension, transpiler)
         })
       })
       .catch(e => {
-        console.error("Error loading plugin", path, e)
+        logger.error("Error loading plugin", path, e)
       })
   }
 

lsp/source/lib/util.mts

@@ -358,7 +358,7 @@ export function logTiming<R, A extends unknown[]>(name: string, fn: (...args: A)
     const start = performance.now(),
       result = fn(...args),
       end = performance.now();
-    console.log(`${name.padStart(32)}${(end - start).toFixed(2).padStart(8)}ms`)
+    // console.log(`${name.padStart(32)}${(end - start).toFixed(2).padStart(8)}ms`)
 
     return result;
   }

lsp/source/server.mts

@@ -98,7 +98,7 @@ const ensureServiceForSourcePath = async (sourcePath: string) => {
   await projectPathToPendingPromiseMap.get(projPath)
   let service = projectPathToServiceMap.get(projPath)
   if (service) return service
-  console.log("Spawning language server for project path: ", projPath)
+  // console.log("Spawning language server for project path: ", projPath)
   service = TSService(projPath)
   const initP = service.loadPlugins()
   projectPathToPendingPromiseMap.set(projPath, initP)
@@ -113,6 +113,7 @@ const diagnosticsDelay = 16;  // ms delay for primary updated file
 const diagnosticsPropagationDelay = 100;  // ms delay for other files
 
 connection.onInitialize(async (params: InitializeParams) => {
+  connection.console.log(`Initializing civet server at ${new Date().toLocaleTimeString()}`);
   const capabilities = params.capabilities;
 
   // Does the client support the `workspace/configuration` request?
@@ -143,7 +144,6 @@ connection.onInitialize(async (params: InitializeParams) => {
       definitionProvider: true,
       hoverProvider: true,
       referencesProvider: true,
-
     }
   };
 
@@ -155,15 +155,20 @@ connection.onInitialize(async (params: InitializeParams) => {
     };
   }
 
-  // TODO: currently only using the first workspace folder
-  const baseDir = params.workspaceFolders?.[0]?.uri.toString()
-  if (!baseDir)
-    throw new Error("Could not initialize without workspace folders")
+  if (!params.rootUri) {
+    // TODO: currently only using the first workspace folder
+    const baseDir = params.workspaceFolders?.[0]?.uri.toString()
+    if (!baseDir)
+      throw new Error("Could not initialize without workspace folders")
 
-  rootUri = baseDir + "/"
-  rootDir = fileURLToPath(rootUri)
+    rootUri = baseDir + "/"
+    rootDir = fileURLToPath(rootUri)
+  } else {
+    rootUri = params.rootUri
+    rootDir = fileURLToPath(rootUri)
+  }
 
-  console.log("Init", rootDir)
+  // console.log("Init", rootDir)
   return result;
 });
 
@@ -241,6 +246,7 @@ connection.onHover(async ({ textDocument, position }) => {
 
 // This handler provides the initial list of the completion items.
 connection.onCompletion(async ({ textDocument, position, context: _context }) => {
+  connection.console.log('onCompletion');
   const completionConfiguration = {
     useCodeSnippetsOnMethodSuggest: false,
     pathSuggestions: true,
@@ -265,7 +271,7 @@ connection.onCompletion(async ({ textDocument, position, context: _context }) =>
   const service = await ensureServiceForSourcePath(sourcePath)
   if (!service) return
 
-  console.log("completion", sourcePath, position)
+  connection.console.log(`completion ${JSON.stringify({ sourcePath, position }, null, 2)}`);
 
   await updating(textDocument)
   if (sourcePath.match(tsSuffix)) { // non-transpiled
@@ -288,7 +294,7 @@ connection.onCompletion(async ({ textDocument, position, context: _context }) =>
   // Don't map for files that don't have a sourcemap (plain .ts for example)
   if (sourcemapLines) {
     position = forwardMap(sourcemapLines, position)
-    console.log('remapped')
+    // console.log('remapped')
   }
 
   const p = transpiledDoc.offsetAt(position)
@@ -484,11 +490,11 @@ connection.onDocumentSymbol(async ({ textDocument }) => {
 
 // TODO
 documents.onDidClose(({ document }) => {
-  console.log("close", document.uri)
+  // console.log("close", document.uri)
 });
 
 documents.onDidOpen(async ({ document }) => {
-  console.log("open", document.uri)
+  // console.log("open", document.uri)
 })
 
 // Buffer up changes to documents so we don't stack transpilations and become unresponsive
@@ -503,7 +509,7 @@ async function executeQueue() {
   // Reset queue to allow accumulating jobs while this queue runs
   const changed = changeQueue
   changeQueue = new Set
-  console.log("executeQueue", changed.size)
+  // console.log("executeQueue", changed.size)
   // Run all jobs in queue (preventing livelock).
   for (const document of changed) {
     await updateDiagnosticsForDoc(document)
@@ -532,14 +538,14 @@ async function scheduleExecuteQueue() {
 // The content of a text document has changed. This event is emitted
 // when the text document first opened or when its content has changed.
 documents.onDidChangeContent(async ({ document }) => {
-  console.log("onDidChangeContent", document.uri)
+  // console.log("onDidChangeContent", document.uri)
   documentUpdateStatus.set(document.uri, withResolvers())
   changeQueue.add(document)
   scheduleExecuteQueue()
 });
 
 async function updateDiagnosticsForDoc(document: TextDocument) {
-  console.log("Updating diagnostics for doc:", document.uri)
+  // console.log("Updating diagnostics for doc:", document.uri)
   const sourcePath = documentToSourcePath(document)
   assert(sourcePath)
 
@@ -565,7 +571,7 @@ async function updateDiagnosticsForDoc(document: TextDocument) {
   // Transpiled file
   const meta = service.host.getMeta(sourcePath)
   if (!meta) {
-    console.log("no meta for ", sourcePath)
+    // console.log("no meta for ", sourcePath)
     return
   }
   const { sourcemapLines, transpiledDoc, parseErrors, fatal } = meta

lsp/test/completions.civet

@@ -0,0 +1,26 @@
+assert from node:assert
+
+{ getLanguageServer, resolveByProject, prepareDocument } from ./utils.civet
+
+requestCompletions := (fileName: string, languageId: string, content: string) =>
+  offset := content.indexOf "|"
+  assert.notEqual offset, -1
+  content = content[..offset - 1] + content[offset + 1..]
+
+  server := await getLanguageServer()
+  document .= await prepareDocument fileName, languageId, content
+
+  position := document.positionAt(offset)
+  return await server.sendCompletionRequest document.uri, position
+  assert.notEqual return.value, undefined
+
+describe "Completions", =>
+  it "obejct field", async =>
+    completions := await requestCompletions
+      resolveByProject("./index.civet")
+      "civet"
+      ```
+      foo := name: 'foo', type: 'string', description: 'foo'
+      foo.n|
+      ```
+    ['completions', completions] |> console.log

lsp/test/fixtures/empty/package.json

@@ -0,0 +1,4 @@
+{
+    "private": true,
+    "name": "empty"
+}

lsp/test/fixtures/empty/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "compilerOptions": {
+    "esModuleInterop": true,
+    "module": "NodeNext",
+    "target": "es2020",
+    "jsx": "preserve"
+  }
+}

lsp/test/utils.civet

@@ -0,0 +1,39 @@
+{ resolve, dirname } from node:path
+{ createRequire } from node:module
+
+{ PublishDiagnosticsNotification } from @volar/language-server
+{ startLanguageServer, LanguageServerHandle } from @volar/test-utils
+{ TextDocument } from vscode-languageserver-textdocument
+{ URI } from vscode-uri
+
+require := createRequire import.meta.url
+
+__dirname := import.meta.url |> .slice 7 |> dirname
+export resolveByDir := (...paths: string[]) => resolve __dirname, ...paths
+projectRoot := resolveByDir "./fixtures/empty/"
+export resolveByProject = (...paths: string[]) => resolve projectRoot, ...paths
+
+serverHandle: LanguageServerHandle | undefined .= undefined
+
+openedDocuments: TextDocument[] := []
+
+export getLanguageServer := (cwd = projectRoot) =>
+  if !serverHandle
+    serverHandle = startLanguageServer require.resolve("../dist/server.js"), cwd
+    serverHandle.connection.onNotification PublishDiagnosticsNotification.method, =>
+    await serverHandle.initialize
+      URI.file(cwd).toString()
+      {
+        typescript:
+          tsdk: dirname require.resolve('typescript/lib/typescript.js')
+          disableAutoImportCache: true
+      }
+  serverHandle!
+
+export prepareDocument := async (fileName: string, languageId: string, content: string, cwd?: string) =>
+  server := await getLanguageServer cwd
+  uri := URI.file(fileName).toString()
+  document := await server.openInMemoryDocument uri.toString(), languageId, content
+  if openedDocuments.every .uri !== document.uri
+    openedDocuments.push document
+  document