feat(file imports): adds an experimental IFC parser

.gitignore

@@ -72,6 +72,9 @@ postgres-data/
 redis-data/
 
 .tshy-build
+obj/
+bin/
+
 
 # Server
 multiregion.json

packages/fileimport-service/Dockerfile

@@ -1,97 +1,74 @@
 ARG NODE_ENV=production
 
-FROM node:18-bookworm-slim@sha256:408f8cbbb7b33a5bb94bdb8862795a94d2b64c2d516856824fd86c4a5594a443 as build-stage
+FROM mcr.microsoft.com/dotnet/sdk:8.0-noble AS dotnet-build-stage
+WORKDIR /app
+COPY packages/ifc-converter .
+RUN dotnet publish ifc-converter.csproj -c Release -o output/
 
-ARG NODE_ENV
-ENV NODE_ENV=${NODE_ENV}
+
+FROM mcr.microsoft.com/dotnet/runtime:8.0-noble AS runtime
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 WORKDIR /speckle-server
 
 # install tini
 ARG TINI_VERSION=v0.19.0
 ENV TINI_VERSION=${TINI_VERSION}
-ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini ./tini
-RUN chmod +x ./tini
-
-# install wait
-ARG WAIT_VERSION=2.8.0
-ENV WAIT_VERSION=${WAIT_VERSION}
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/${WAIT_VERSION}/wait ./wait
-RUN chmod +x ./wait
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
+RUN chmod +x /usr/bin/tini
+
+RUN apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+     --no-install-recommends \
+     curl=8.5.0-2ubuntu10.5 \
+    && curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+     --no-install-recommends \
+     nodejs=18.20.5-1nodesource1 \
+    && npm install -g [email protected] \
+    && DEBIAN_FRONTEND=noninteractive apt-get remove curl -y \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # download yarn dependencies for building shared libraries
 COPY .yarnrc.yml .
 COPY .yarn ./.yarn
 COPY package.json yarn.lock ./
 
-COPY packages/frontend-2/type-augmentations/stubs ./packages/frontend-2/type-augmentations/stubs/
-COPY packages/shared/package.json ./packages/shared/
-COPY packages/fileimport-service/package.json ./packages/fileimport-service/
+COPY packages/frontend-2/type-augmentations/stubs packages/frontend-2/type-augmentations/stubs/
+COPY packages/shared/package.json packages/shared/
+COPY packages/fileimport-service/package.json packages/fileimport-service/
 
 RUN yarn workspaces focus --all
 
 # build shared libraries
-COPY packages/shared ./packages/shared/
-COPY packages/fileimport-service ./packages/fileimport-service/
+COPY packages/shared packages/shared/
+COPY packages/fileimport-service packages/fileimport-service/
 RUN yarn workspaces foreach -W run build
 
 # Install python virtual env and python dependencies
-RUN apt-get update && \
-  DEBIAN_FRONTEND=noninteractive apt-get install -y \
-  --no-install-recommends \
-  python3-venv=3.11.2-1+b1 \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/* \
-  && python3 -m venv /venv
+RUN apt-get update -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+     --no-install-recommends \
+     python3.12=3.12.3-1ubuntu0.3 \
+     python3-pip=24.0+dfsg-1ubuntu1.1 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY packages/fileimport-service/requirements.txt /speckle-server/
-RUN /venv/bin/pip install --disable-pip-version-check --no-cache-dir --requirement /speckle-server/requirements.txt
-
-FROM node:18-bookworm-slim@sha256:408f8cbbb7b33a5bb94bdb8862795a94d2b64c2d516856824fd86c4a5594a443 as dependency-stage
-# installing just the production dependencies
-# separate stage to avoid including development dependencies
-ARG NODE_ENV
-ENV NODE_ENV=${NODE_ENV}
-
-WORKDIR /speckle-server
-COPY .yarnrc.yml .
-COPY .yarn ./.yarn
-COPY package.json yarn.lock ./
-
-COPY packages/frontend-2/type-augmentations/stubs ./packages/frontend-2/type-augmentations/stubs/
-COPY packages/shared/package.json ./packages/shared/
-COPY packages/fileimport-service/package.json ./packages/fileimport-service/
-
-WORKDIR /speckle-server/packages/fileimport-service
-RUN yarn workspaces focus --production
-
-FROM gcr.io/distroless/python3-debian12:nonroot@sha256:14c62b8925d3bb30319de2f346bde203fe18103a68898284a62db9d4aa54c794 as python-image
-
-FROM gcr.io/distroless/nodejs18-debian12:nonroot@sha256:afdea027580f7afcaf1f316b2b3806690c297cb3ce6ddc5cf6a15804dc1c790f as distributable-stage
+RUN pip install --break-system-packages --disable-pip-version-check --no-cache-dir --requirement /speckle-server/requirements.txt
 
 ARG NODE_ENV
 ENV NODE_ENV=${NODE_ENV}
-ARG NODE_BINARY_PATH=/nodejs/bin/node
+ARG NODE_BINARY_PATH=/usr/bin/node
 ENV NODE_BINARY_PATH=${NODE_BINARY_PATH}
-ARG PYTHON_BINARY_PATH=/venv/bin/python3
+ARG PYTHON_BINARY_PATH=/usr/bin/python3
 ENV PYTHON_BINARY_PATH=${PYTHON_BINARY_PATH}
+ARG DOTNET_BINARY_PATH=/usr/bin/dotnet
+ENV DOTNET_BINARY_PATH=${DOTNET_BINARY_PATH}
 
-WORKDIR /speckle-server
-
-COPY --from=python-image / /
-COPY --from=build-stage /speckle-server/tini /usr/bin/tini
-COPY --from=build-stage /speckle-server/wait /usr/bin/wait
-COPY --from=build-stage /speckle-server/packages/shared ./packages/shared
-COPY --from=build-stage /speckle-server/packages/fileimport-service ./packages/fileimport-service
-COPY --from=build-stage /venv /venv
-COPY --from=dependency-stage /speckle-server/node_modules ./node_modules
+COPY --from=dotnet-build-stage /app/output packages/ifc-converter
 
 WORKDIR /speckle-server/packages/fileimport-service
 
-# Prefixing PATH with our virtual environment should seek required binaries
-# from virtual environment first.
-# Unsetting python home
-ENV PATH=/venv/bin:${PATH} \
-    PYTHONHOME=
-
-ENTRYPOINT [ "tini", "--", "/nodejs/bin/node", "--no-experimental-fetch", "src/daemon.js"]
+ENTRYPOINT [ "tini", "--", "node", "--no-experimental-fetch", "src/daemon.js"]

packages/fileimport-service/src/daemon.js

@@ -129,17 +129,13 @@ async function doTask(mainDb, regionName, taskDb, task) {
     if (info.fileType.toLowerCase() === 'ifc') {
       await runProcessWithTimeout(
         taskLogger,
-        process.env['NODE_BINARY_PATH'] || 'node',
+        process.env['DOTNET_BINARY_PATH'] || 'dotnet',
         [
-          '--no-experimental-fetch',
-          './ifc/import_file.js',
+          '/speckle-server/packages/ifc-converter/ifc-converter.dll',
           TMP_FILE_PATH,
           TMP_RESULTS_PATH,
-          info.userId,
           info.streamId,
-          info.branchName,
           `File upload: ${info.fileName}`,
-          info.id,
           existingBranch?.id,
           regionName
         ],

packages/ifc-converter/.config/dotnet-tools.json

@@ -0,0 +1,12 @@
+{
+  "version": 1,
+  "isRoot": true,
+  "tools": {
+    "csharpier": {
+      "version": "0.30.1",
+      "commands": [
+        "dotnet-csharpier"
+      ]
+    }
+  }
+}

packages/ifc-converter/ConsoleProgress.cs

@@ -0,0 +1,33 @@
+using Speckle.Sdk.Transports;
+
+namespace Speckle.Converter;
+
+public class ConsoleProgress : IProgress<ProgressArgs>
+{
+  private readonly TimeSpan DEBOUNCE = TimeSpan.FromSeconds(1);
+  private DateTime _lastTime = DateTime.UtcNow;
+
+  private long _totalBytes;
+
+  public void Report(ProgressArgs value)
+  {
+    if (value.ProgressEvent == ProgressEvent.DownloadBytes)
+    {
+      Interlocked.Add(ref _totalBytes, value.Count);
+    }
+    var now = DateTime.UtcNow;
+    if (now - _lastTime >= DEBOUNCE)
+    {
+      if (value.ProgressEvent == ProgressEvent.DownloadBytes)
+      {
+        Console.WriteLine(value.ProgressEvent + " t " + _totalBytes);
+      }
+      else
+      {
+        Console.WriteLine(value.ProgressEvent + " c " + value.Count + " t " + value.Total);
+      }
+
+      _lastTime = now;
+    }
+  }
+}

packages/ifc-converter/Program.cs

@@ -0,0 +1,47 @@
+using System.CommandLine;
+using System.Text.Json;
+using Speckle.Sdk.Common;
+using Speckle.WebIfc.Importer;
+
+var filePathArgument = new Argument<string>(name: "filePath");
+var outputPathArgument = new Argument<string>("outputPath");
+var streamIdArgument = new Argument<string>("streamId");
+var commitMessageArgument = new Argument<string>("commitMessage");
+var modelIdArgument = new Argument<string>("modelId");
+var regionNameArgument = new Argument<string>("regionName");
+
+var rootCommand = new RootCommand
+{
+  filePathArgument,
+  outputPathArgument,
+  streamIdArgument,
+  commitMessageArgument,
+  modelIdArgument,
+  regionNameArgument,
+};
+rootCommand.SetHandler(
+  async (filePath, outputPath, streamId, commitMessage, modelId, _) =>
+  {
+    try
+    {
+      var token = Environment.GetEnvironmentVariable("USER_TOKEN").NotNull("USER_TOKEN is missing");
+      var url = Environment.GetEnvironmentVariable("SPECKLE_SERVER_URL") ?? "http://127.0.0.1:3000";
+      var commitId = await Import.Ifc(url, filePath, streamId, modelId, commitMessage, token);
+      File.WriteAllText(outputPath, JsonSerializer.Serialize(new { success = true, commitId }));
+    }
+    catch (Exception e)
+    {
+      File.WriteAllText(
+        outputPath,
+        JsonSerializer.Serialize(new { success = false, error = e.ToString() })
+      );
+    }
+  },
+  filePathArgument,
+  outputPathArgument,
+  streamIdArgument,
+  commitMessageArgument,
+  modelIdArgument,
+  regionNameArgument
+);
+await rootCommand.InvokeAsync(args);

packages/ifc-converter/ifc-converter.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
+        <RootNamespace>Speckle.Converter</RootNamespace>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
+      <PackageReference Include="Speckle.WebIfc.Importer" Version="0.0.3" />
+      <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+    </ItemGroup>
+
+</Project>

packages/ifc-converter/ifc-converter.sln

@@ -0,0 +1,16 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ifc-converter", "ifc-converter.csproj", "{4D63FBD3-8ABF-4F51-A08F-740B17BDCA28}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{4D63FBD3-8ABF-4F51-A08F-740B17BDCA28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4D63FBD3-8ABF-4F51-A08F-740B17BDCA28}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4D63FBD3-8ABF-4F51-A08F-740B17BDCA28}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4D63FBD3-8ABF-4F51-A08F-740B17BDCA28}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal