feature: Allow to inject custom library dependencies

.github/actions/build-project/action.yaml

@@ -13,6 +13,9 @@ inputs:
   disabled-scalac-options:
     description: "List of scalacOptions which should be filted out when building projects."
     default: ""
+  extra-library-dependencies:
+    description: "List of library dependencies which should be injected when building projects, in format org:artifact:version, cross versions are not supported. Multiple entires should be seperated by a single semicolon character `;`"
+    default: ""
   custom-build-id:
     description: "Custom buildId to use instead of autogenerated github job id"
     default: ""
@@ -107,7 +110,7 @@ runs:
       uses: addnab/docker-run-action@v3
       if: steps.check-history.outputs.can-skip-build != 'true'
       with:
-        image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.3.3"
+        image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.3.4"
         options: -v ${{ github.workspace }}:/opencb/ -v ${{ github.workspace }}/github_key:/root/.ssh/github_key:ro
         run: |
           # Setup ssh required for downloading submodules
@@ -139,7 +142,9 @@ runs:
             '1.6.2' \
             "$(config .config // ${DefaultConfig})" \
             "${{ inputs.extra-scalac-options }}" \
-            "${{ inputs.disabled-scalac-options }}" 2>&1 | tee build-logs.txt
+            "${{ inputs.disabled-scalac-options }}" \
+            "${{ inputs.extra-library-dependencies }}" \
+            2>&1 | tee build-logs.txt
 
           # Store results
           mv build-logs.txt /opencb/

.github/actions/setup-build/action.yaml

@@ -75,7 +75,7 @@ runs:
       uses: addnab/docker-run-action@v3
       if: steps.check-published.outputs.is-compiler-published == 'false'
       with:
-        image: "virtuslab/scala-community-build-compiler-builder:v0.3.3"
+        image: "virtuslab/scala-community-build-compiler-builder:v0.3.4"
         options: -v ${{ github.workspace }}/compiler:/compiler/
         run: |
           Version="${{ steps.calc-version.outputs.effective-scala-version }}"

.github/workflows/buildExecuteCustom-A.yaml

@@ -21,11 +21,15 @@ on:
         default: "main"
       extra-scalac-options:
         type: string
-        description: "List of scalacOptions which should be used when building projects. Multiple entires should be seperated by a single comma character `,`"
+        description: "List of scalacOptions which should be used when building projects. Multiple entires should be seperated by a single comma character `,`. (Best effort)"
         default: ""
       disabled-scalac-options:
         type: string
-        description: "List of scalacOptions which should be filtered out when building projects."
+        description: "List of scalacOptions which should be filtered out when building projects. (Best effort)"
+        default: ""
+      extra-library-dependencies:
+        type: string
+        description: "List of library dependencies which should be injected when building projects, in format org:artifact:version, cross versions are not supported. Multiple entires should be seperated by a single semicolon character `;`. (Best effort)"
         default: ""
       push-to-gh-pages:
         type: boolean
@@ -41,6 +45,7 @@ jobs:
       repository-branch: ${{ inputs.repository-branch }}
       extra-scalac-options: ${{ inputs.extra-scalac-options }}
       disabled-scalac-options: ${{ inputs.disabled-scalac-options }}
+      extra-library-dependencies: ${{ inputs.extra-library-dependencies }}
       custom-build-id: ${{ inputs.build-name }}
     secrets: inherit
 

.github/workflows/buildExecuteCustom-B.yaml

@@ -21,11 +21,15 @@ on:
         default: "main"
       extra-scalac-options:
         type: string
-        description: "List of scalacOptions which should be used when building projects. Multiple entires should be seperated by a single comma character `,`"
+        description: "List of scalacOptions which should be used when building projects. Multiple entires should be seperated by a single comma character `,` (Best effort)"
         default: ""
       disabled-scalac-options:
         type: string
-        description: "List of scalacOptions which should be filtered out when building projects."
+        description: "List of scalacOptions which should be filtered out when building projects. (Best effort)"
+        default: ""
+      extra-library-dependencies:
+        type: string
+        description: "List of library dependencies which should be injected when building projects, in format org:artifact:version, cross versions are not supported. Multiple entires should be seperated by a single semicolon character `;`. (Best effort)"
         default: ""
       push-to-gh-pages:
         type: boolean
@@ -41,6 +45,7 @@ jobs:
       repository-branch: ${{ inputs.repository-branch }}
       extra-scalac-options: ${{ inputs.extra-scalac-options }}
       disabled-scalac-options: ${{ inputs.disabled-scalac-options }}
+      extra-library-dependencies: ${{ inputs.extra-library-dependencies }}
       custom-build-id: ${{ inputs.build-name }}
     secrets: inherit
 

cli/scb-cli.scala

@@ -31,7 +31,7 @@ class FailedProjectException(msg: String)
     with NoStackTrace
 
 val communityBuildVersion =
-  sys.props.getOrElse("communitybuild.version", "v0.3.3")
+  sys.props.getOrElse("communitybuild.version", "v0.3.4")
 private val CBRepoName = "VirtusLab/community-build3"
 val projectBuilderUrl =
   s"https://raw.githubusercontent.com/$CBRepoName/master/project-builder"
@@ -767,7 +767,8 @@ object MinikubeReproducer:
                   params.enforcedSbtVersion.getOrElse("1.6.2"),
                   params.config.getOrElse("{}"),
                   /* extra-scalac-options    = */ "",
-                  /* disabled-scalac-options = */ ""
+                  /* disabled-scalac-options = */ "",
+                  /* extra-library-deps = */ ""
                 )
               )
             ),

coordinator/configs/projects-config.conf

@@ -207,7 +207,7 @@ disneystreaming_smithy4s {
 disneystreaming_smithy-translate {
   source-patches = [{
     path = "buildSetup.sc"
-    pattern      = "case ZincWorkerUtil.DottyVersion("0","
+    pattern      = "case ZincWorkerUtil.DottyVersion(\"0\","
     replace-with = "case ZincWorkerUtil.Scala3Version("
   }]
 }

coordinator/src/main/scala/buildPlan.scala

@@ -532,6 +532,10 @@ def createGithubActionJob(
     |        type: string
     |        description: "List of scalacOptions which should be filtered out when building projects."
     |        default: ""
+    |      extra-library-dependencies:
+    |        type: string
+    |        description: "List of library dependencies which should be injected when building projects, in format org:artifact:version, cross versions are not supported. Multiple entires should be seperated by a single semicolon character `;`. (Best effort)"
+    |        default: ""
     |      custom-build-id:
     |        type: string
     |        description: "Custom buildId to use instead of autogenerated github job id"
@@ -600,6 +604,7 @@ def createGithubActionJob(
           println("    custom-build-id: ${{ inputs.custom-build-id }}")
           println("    extra-scalac-options: ${{ inputs.extra-scalac-options }}")
           println("    disabled-scalac-options: ${{ inputs.disabled-scalac-options }}")
+          println("    extra-library-dependencies: ${{ inputs.extra-library-dependencies }}")
           println(s"    scala-version: $${{ $setupOutputs.scala-version }}")
           println(s"    maven-repo-url: $${{ $setupOutputs.maven-repo-url }}")
           println("    elastic-user: ${{ secrets.OPENCB_ELASTIC_USER }}")

k8s/project-builder-mill-test.yaml

@@ -25,6 +25,7 @@ spec:
               '{"projects": {"exclude": ["dummy.org%dummy-.*"], "overrides": {"upickle-core": {"tests": "compile-only"}, "ujson": {"tests": "disabled"}} } }' \
               "" \
               "" \
+              "" \
             && echo \
             && echo "Community project published successfully") || true
       restartPolicy: Never

k8s/project-builder-sbt-test.yaml

@@ -28,6 +28,7 @@ spec:
                 '{"sbt": {"commands": ["show crossScalaVersions", "show scalaVersion", "show scalacOptions"] }, "projects": {"exclude": ["dummy.org%dummy-.*"], "overrides": {"shapeless3-deriving": {"tests": "compile-only"}, "shapeless3-typeable": {"tests": "disabled"}} } }' \
                 "" \
                 "" \
+                "" \
               && echo \
               && echo "Community project published successfully") || true
       restartPolicy: Never

project-builder/build-revision.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 set -e
 
-if [ $# -ne 11 ]; then
-  echo "Wrong number of script arguments, got $# expected 11"
+if [ $# -ne 12 ]; then
+  echo "Wrong number of script arguments, got $# expected 12"
   exit 1
 fi
 
@@ -17,6 +17,7 @@ enforcedSbtVersion="$8" # e.g. '1.5.5' or empty ''
 projectConfig="$9"
 extraScalacOptions="${10}" # e.g '' or "-Wunused:all -Ylightweight-lazy-vals"
 disabledScalacOption="${11}"
+extraLibraryDeps="${12}" # format org:artifact:version, eg. org.scala-lang:scala2-library-tasty_3:3.4.0-RC1
 
 scriptDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 export OPENCB_SCRIPT_DIR=$scriptDir
@@ -54,6 +55,12 @@ if [ -z $disabledScalacOption ];then disabledScalacOption="$commonRemoveScalacOp
 else disabledScalacOption="$disabledScalacOption,$commonRemoveScalacOptions"; 
 fi
 
+if [[ ! -z $extraLibraryDeps ]]; then
+  echo "Would try to append extra library dependencies (best-effort, sbt/scala-cli only): ${extraLibraryDeps}"
+fi
+
+echo ""
+echo "----"
 if [ -f "repo/mill" ] || [ -f "repo/build.sc" ]; then
   echo "Mill project found: ${isMillProject}"
   echo "mill" > $buildToolFile
@@ -64,12 +71,12 @@ elif [ -f "repo/build.sbt" ]; then
   echo "sbt project found: ${isSbtProject}"
   echo "sbt" > $buildToolFile
   $scriptDir/sbt/prepare-project.sh "$project" repo "$enforcedSbtVersion" "$scalaVersion" "$projectConfig"
-  $scriptDir/sbt/build.sh repo "$scalaVersion" "$version" "$targets" "$mvnRepoUrl" "$projectConfig" "$extraScalacOptions" "$disabledScalacOption"
+  $scriptDir/sbt/build.sh repo "$scalaVersion" "$version" "$targets" "$mvnRepoUrl" "$projectConfig" "$extraScalacOptions" "$disabledScalacOption" "$extraLibraryDeps"
 else
   echo "Not found sbt or mill build files, assuming scala-cli project"
   ls -l repo/
   echo "scala-cli" > $buildToolFile
   scala-cli clean $scriptDir/scala-cli/
   scala-cli clean repo
-  scala-cli $scriptDir/scala-cli/build.scala -- repo "$scalaVersion" "$projectConfig" "$mvnRepoUrl"
+  scala-cli $scriptDir/scala-cli/build.scala -- repo "$scalaVersion" "$projectConfig" "$mvnRepoUrl" "$extraLibraryDeps"
 fi

project-builder/mill/MillCommunityBuild.sc

@@ -112,7 +112,7 @@ def mapCrossVersions[T](
     version <- Seq(mappedCross, crossEntry).distinct
   } yield {
     if (version != crossEntry) {
-      println(s"Use cross-version $mappedCross instead of $crossEntry")
+      logOnce(s"Use cross-version $mappedCross instead of $crossEntry")
     }
     version.asInstanceOf[T]
   }

project-builder/sbt/CommunityBuildPlugin.scala

@@ -85,11 +85,18 @@ object CommunityBuildPlugin extends AutoPlugin {
     }
     .getOrElse(Nil)
 
-  override def projectSettings =
-    Seq(
-      // Fix for cyclic dependency when trying to use crossScalaVersion ~= ???
-      crossScalaVersions := (thisProjectRef / crossScalaVersions).value
-    ) ++ mvnRepoPublishSettings
+  override def projectSettings = Seq(
+    // Fix for cyclic dependency when trying to use crossScalaVersion ~= ???
+    crossScalaVersions := (thisProjectRef / crossScalaVersions).value,
+    libraryDependencies ++= extraLibraryDependencies((thisProjectRef / scalaVersion).value)
+  ) ++ mvnRepoPublishSettings
+
+  private def extraLibraryDependencies(scalaVersion: String) =
+    if (!scalaVersion.startsWith("3.")) Nil
+    else
+      Utils.extraLibraryDependencies.map { case Utils.LibraryDependency(org, artifact, version) =>
+        org % artifact % version
+      }
 
   private def stripScala3Suffix(s: String) = s match {
     case WithExtractedScala3Suffix(prefix, _) => prefix; case _ => s

project-builder/sbt/build.sh

@@ -2,8 +2,8 @@
 set -e
 set -o pipefail
 
-if [ $# -ne 8 ]; then
-  echo "Wrong number of script arguments, expected $0 <repo_dir> <scala-version> <version> <targets> <maven_repo> <project_config?>, got $#: $@"
+if [ $# -ne 9 ]; then
+  echo "Wrong number of script arguments, expected $0 <repo_dir> <scala-version> <version> <targets> <maven_repo> <project_config?> <extraScalacOpts?> <removeScalacOpts?> <extraDeps?>, got $#: $@"
   exit 1
 fi
 
@@ -15,6 +15,7 @@ export CB_MVN_REPO_URL="$5" # e.g. https://mvn-repo/maven2/2021-05-23_1
 projectConfig="$6"
 extraScalacOptions="$7"
 disabledScalacOption="$8"
+extraLibraryDeps="$9"
 
 if [[ -z "$projectConfig" ]]; then
   projectConfig="{}"
@@ -48,7 +49,8 @@ fi
 sbtSettings=(
   --batch
   --verbose
-  -Dcommunitybuild.version="$version"
+  "-Dcommunitybuild.version=$version"
+  "-Dcommunitybuild.project.dependencies.add=$extraLibraryDeps"
   ${memorySettings[@]}
   $(echo $projectConfig | jq -r '.sbt.options? // [] | join(" ")' | sed "s/<SCALA_VERSION>/${scalaVersion}/g")
 )

project-builder/scala-cli/build.scala

@@ -13,8 +13,13 @@ import os.CommandResult
     repositoryDir: String,
     scalaVersion: String,
     configJson: String,
-    mavenRepoURL: String
+    mavenRepoURL: String,
+    extraLibraryDependenciesString: String
 ): Unit = {
+  // For compatibility with other build tools s use logic from CBCore
+  // Set system property if missing
+  sys.props.getOrElseUpdate(Utils.ExtraLibraryDependenciesProp, extraLibraryDependenciesString)
+  println(System.getProperty(Utils.ExtraLibraryDependenciesProp))
   println(s"Build config: ${configJson}")
   val config =
     if (configJson.isEmpty()) ProjectBuildConfig()
@@ -102,7 +107,9 @@ import os.CommandResult
 case class CliCommand[T](
     command: Seq[String],
     errHandler: (CommandResult, EvalResult.Failure) => EvalResult[T]
-)
+){
+  override def toString(): String = s"${command.mkString(" ")}"
+}
 def cmd(args: String*) = CliCommand[Unit](args, (_, failure) => failure)
 class CliTaskEvaluator(scalaVersion: String, repositoryDir: String, mavenRepoURL: Option[String])
     extends TaskEvaluator[CliCommand] {
@@ -120,6 +127,9 @@ class CliTaskEvaluator(scalaVersion: String, repositoryDir: String, mavenRepoURL
   }
 
   def eval[T](task: CliCommand[T]): EvalResult[T] = {
+    val extraLibraryDependencies = Utils.extraLibraryDependencies.map{
+      case Utils.LibraryDependency(org, artifact, version) => s"--dependency=$org:$artifact:$version"
+    }
     val evalStart = System.currentTimeMillis()
     val proc = os
       .proc(
@@ -130,20 +140,21 @@ class CliTaskEvaluator(scalaVersion: String, repositoryDir: String, mavenRepoURL
         s"--server=false",
         s"--scala-version=${scalaVersion}",
         "--scalac-option=-J-Xss10M",
-        "--scalac-option=-J-Xmx=7G",
-        "--scalac-option=-J-Xms=4G",
-        mavenRepoURL.map(s"--repository=" + _).getOrElse("")
+        "--scalac-option=-J-Xmx7G",
+        "--scalac-option=-J-Xms4G",
+        mavenRepoURL.map(s"--repository=" + _).toList,
+        extraLibraryDependencies
       )
       .call(check = false, stderr = os.Pipe)
     val result = proc.exitCode
     val tookMillis = (System.currentTimeMillis() - evalStart).toInt
     def nullT = null.asInstanceOf[T]
     result match {
       case 0 =>
-        println(s"Successfully evaluated $task")
+        println(s"Successfully evaluated: $task")
         EvalResult.Value(nullT, evalTime = tookMillis)
       case exitCode =>
-        println(s"Failed to evaluated $task: exitCode ${exitCode}")
+        println(s"Failed to evaluated: $task, exitCode ${exitCode}")
         proc.err.lines().foreach(System.err.println)
         val failure = EvalResult.Failure(
           EvaluationFailure(proc.err.toString()) :: Nil,

project-builder/shared/CommunityBuildCore.scala

@@ -414,6 +414,23 @@ object Scala3CommunityBuild {
             definedSourceVersion.isEmpty && isMatching("is dangling source version",SourceVersionPattern.findFirstIn(s.trim()))     
         } ++ appendSettings.distinct
     }
-
+
+    case class LibraryDependency(organization: String, artifact: String, version: String)
+    final val ExtraLibraryDependenciesProp = "communitybuild.project.dependencies.add"
+    lazy val extraLibraryDependencies: Seq[LibraryDependency] =      sys.props
+        .getOrElse(ExtraLibraryDependenciesProp, "")
+        .split(';')
+        .filter(_.nonEmpty)
+        .map(dep => dep.trim().split(':'))
+        .flatMap {
+          case Array(org, artifact, version) => 
+            val dep = LibraryDependency(org, artifact, version)
+            logOnce(s"Would include extra dependency: org=$org, artifact=$artifact, version=$version")
+            Some(dep)
+          case segments =>
+            logOnce(s"Invalid dependency format, segments=${segments.toList}")
+            None
+        }
   }
+
 }

scripts/bisect.scala

@@ -9,7 +9,7 @@ import java.nio.file.attribute.PosixFilePermissions
 import java.nio.charset.StandardCharsets
 import java.nio.file._
 
-val communityBuildVersion = "v0.3.3"
+val communityBuildVersion = "v0.3.4"
 
 @main def run(args: String*): Unit =
   val config = scopt.OParser
@@ -156,6 +156,7 @@ object ValidationScript:
       then ""
       else """* { "tests": "compile-only"} """
     val revision = projectRevision.getOrElse("$(config .revision)")
+    val extraLibraryDependencies = ""
     raw"""
     |#!/usr/bin/env bash
     |set -e
@@ -188,7 +189,8 @@ object ValidationScript:
     |  '1.6.2' \
     |  "$$(config .config '$configPatch' // $${DefaultConfig} '$configPatch')" \
     |  "$extraScalacOptions" \
-    |  "$disabledScalacOption"
+    |  "$disabledScalacOption" \
+    |  "$extraLibraryDependencies"
     |
     |grep -q "success" build-status.txt;
     |exit $$?

scripts/build-all.sh

@@ -7,7 +7,7 @@ if [ $# -ne 1 ]; then
 fi
 
 VERSION="$1"
-export PREV_CB_VERSION="v0.3.2"
+export PREV_CB_VERSION="v0.3.3"
 
 javaDefault=11
 javaAccessoryVersions=(8 17 21)