WIP: Improve all public APIs by dropping (using Context) (#538)

* removed most of (using Context) from public-facing APIs * split rpc from core * Try to speed up builds by @prolativ * increase IT logging to debug deadlock on CI * drop most sonatype:snapshots and remove all autogenerated project.scala in IT from git --------- Co-authored-by: Michał Pałka <[email protected]>
VirtusLab · Oct 2, 2024 · 96f1f75 · 96f1f75
1 parent 2f59f60
commit 96f1f75
Show file tree

Hide file tree

Showing 228 changed files with 1,996 additions and 1,954 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -57,9 +57,9 @@ jobs:
     - uses: extractions/setup-just@v1
       with:
         just-version: '1.14.0' # just for sanity
-    - uses: pulumi/actions@v4
+    - uses: pulumi/actions@v5
       with:
-        pulumi-version: '3.116.1'
+        pulumi-version: '3.134.0'
     - uses: coursier/[email protected]
     - uses: VirtusLab/[email protected]
       with:

diff --git a/.scalafmt.conf b/.scalafmt.conf
@@ -1,7 +1,7 @@
 version = 3.5.2
 runner.dialect = scala3
 project.excludePaths = [
-    "glob:**/src/main/scala/besom/rpc/**.scala"
+    "glob:**/besom-rpc/src/main/scala/**.scala"
 ]
 project.git = true
 align = most
@@ -11,4 +11,4 @@ align.tokens = [{code = "=>", owner = "Case"}, "<-", "%", "%%", "="]
 indent.defnSite = 2
 maxColumn = 140
 
-rewrite.scala3.insertEndMarkerMinLines = 40
+rewrite.scala3.insertEndMarkerMinLines = 40
diff --git a/Justfile b/Justfile
diff --git a/README.md b/README.md
@@ -130,12 +130,15 @@ import besom.api.aws
 > Besom **does NOT depend on Pulumi Java SDK**, it is a completely separate implementation.
 
 ## Tips
-- Whenever you use Besom outside the `Pulumi.run` block, pass [`Context`](https://virtuslab.github.io/besom/docs/context) with `(using besom.Context)`
 - Resources are initialized lazily. To make them appear in your physical infrastructure make sure 
-their evaluation is triggered directly or transitively from the main for-comprehension block of your Pulumi program.
+their evaluation is triggered directly or transitively (by referencing them or resources that depend on them) by 
+including them in the final `Stack` structure of your program.
 - Use whatever Scala concepts you are familiar with, infrastructure as code in Besom is still a Scala program, 
 so you have the full potential of the language to work with.
 - Pay attention to the types. You will be instantiating case classes to pass parameters, note their package of origin.
+- Some Pulumi functions that allow you to access the metadata of the current stack require access to the `Context`. 
+`Context` is available as a given (implicitly) in the `Pulumi.run` block, but you can also require it explicitly 
+in any of your functions with `(using Context)`.
 
 ## Requirements
 

diff --git a/auto/project.scala b/auto/project.scala
@@ -22,4 +22,3 @@
 //> using publish.repository "central"
 //> using publish.developer "lbialy|Łukasz Biały|https://github.com/lbialy"
 //> using publish.developer "pawelprazak|Paweł Prażak|https://github.com/pawelprazak"
-//> using repository sonatype:snapshots
diff --git a/besom-cats/project.scala b/besom-cats/project.scala
@@ -15,4 +15,4 @@
 //> using publish.developer "lbialy|Łukasz Biały|https://github.com/lbialy"
 //> using publish.developer "prolativ|Michał Pałka|https://github.com/prolativ"
 //> using publish.developer "KacperFKorban|Kacper Korban|https://github.com/KacperFKorban"
-//> using repository sonatype:snapshots
+
diff --git a/besom-cats/src/main/scala/runtime.scala b/besom-cats/src/main/scala/runtime.scala
@@ -47,8 +47,6 @@ trait CatsEffectModule extends BesomModule:
   implicit val toFutureCatsEffectIO: Result.ToFuture[Eff] = new Result.ToFuture[IO]:
     def eval[A](fa: => IO[A]): () => Future[A] = () => fa.uncancelable.unsafeToFuture()(using ioRuntime)
 
-  // override def run(program: Context ?=> Output[Exports]): Future[Unit] = ???
-
 object Pulumi extends CatsEffectModule
 export Pulumi.{component => _, *, given}
 
@@ -70,10 +68,10 @@ import scala.reflect.Typeable
   * @return
   *   The component resource.
   */
-def component[A <: ComponentResource & Product: RegistersOutputs: Typeable](using ctx: Context)(
+def component[A <: ComponentResource & Product: RegistersOutputs: Typeable](
   name: NonEmptyString,
   typ: ResourceType,
   opts: ComponentResourceOptions = ComponentResourceOptions()
 )(
-  f: Context ?=> ComponentBase ?=> A
+  f: ComponentBase ?=> A
 ): Output[A] = Pulumi.component(name, typ, opts)(f)
diff --git a/besom-cfg/k8s/project.scala b/besom-cfg/k8s/project.scala
@@ -21,4 +21,4 @@
 //> using publish.developer "prolativ|Michał Pałka|https://github.com/prolativ"
 //> using publish.developer "KacperFKorban|Kacper Korban|https://github.com/KacperFKorban"
 //> using publish.developer "pawelprazak|Paweł Prażak|https://github.com/pawelprazak"
-//> using repository sonatype:snapshots
+
diff --git a/besom-cfg/k8s/src/main/scala/ConfiguredContainerArgs.scala b/besom-cfg/k8s/src/main/scala/ConfiguredContainerArgs.scala
@@ -1,7 +1,7 @@
 package besom.cfg.k8s
 
 import besom.cfg.internal.*
-import besom.types.{Input, Context, Output}
+import besom.types.{Input, Output}
 import besom.cfg.*
 import besom.json.*
 import besom.cfg.containers.*
@@ -15,7 +15,7 @@ import besom.cfg.k8s.syntax.*
 
 object syntax:
   extension (s: Struct)
-    def foldedToEnvVarArgs(using Context): Output[List[EnvVarArgs]] =
+    def foldedToEnvVarArgs: Output[List[EnvVarArgs]] =
       s.foldToEnv.map(_.map { case (k, v) => EnvVarArgs(name = k, value = v) })
 
 object ConfiguredContainerArgs:
@@ -48,7 +48,7 @@ object ConfiguredContainerArgs:
     volumeDevices: Input.Optional[List[Input[VolumeDeviceArgs]]] = None,
     volumeMounts: Input.Optional[List[Input[VolumeMountArgs]]] = None,
     workingDir: Input.Optional[String] = None
-  )(using ctx: Context) = ${
+  ) = ${
     applyImpl(
       'name,
       'image,
@@ -74,8 +74,7 @@ object ConfiguredContainerArgs:
       'tty,
       'volumeDevices,
       'volumeMounts,
-      'workingDir,
-      'ctx
+      'workingDir
     )
   }
 
@@ -104,8 +103,7 @@ object ConfiguredContainerArgs:
     tty: Expr[Input.Optional[Boolean]],
     volumeDevices: Expr[Input.Optional[List[Input[VolumeDeviceArgs]]]],
     volumeMounts: Expr[Input.Optional[List[Input[VolumeMountArgs]]]],
-    workingDir: Expr[Input.Optional[String]],
-    context: Expr[Context]
+    workingDir: Expr[Input.Optional[String]]
   )(using Quotes): Expr[ContainerArgs] =
     import quotes.reflect.*
 
@@ -130,9 +128,10 @@ object ConfiguredContainerArgs:
 
       case Right(()) =>
         val envExpr = '{
-          val envOutput                 = ${ env }.asOptionOutput()(using ${ context })
+          val envOutput                 = ${ env }.asOptionOutput()
           val conf                      = ${ configuration }
-          val configurationAsEnvVarArgs = conf.foldedToEnvVarArgs(using $context)
+          val configurationAsEnvVarArgs = conf.foldedToEnvVarArgs
+
           envOutput.zip(configurationAsEnvVarArgs).map {
             case (Some(envVarArgsList), envVarArgsListFromConf) => envVarArgsList ++ envVarArgsListFromConf
             case (None, envVarArgsListFromConf)                 => envVarArgsListFromConf
@@ -165,7 +164,7 @@ object ConfiguredContainerArgs:
             volumeDevices = $volumeDevices,
             volumeMounts = $volumeMounts,
             workingDir = $workingDir
-          )(using $context)
+          )
         }
     end match
   end applyImpl

diff --git a/besom-cfg/k8s/src/test/scala/DummyContext.scala b/besom-cfg/k8s/src/test/scala/DummyContext.scala
@@ -57,7 +57,7 @@ object DummyContext:
       config       <- Config(runInfo.project, isProjectName = true, configMap = configMap, configSecretKeys = configSecretKeys)
       resources    <- Resources()
       given Context = Context.create(runInfo, featureSupport, config, logger, monitor, engine, taskTracker, resources, memo, stackPromise)
-      _ <- stackPromise.fulfill(StackResource()(using ComponentBase(Output(besom.types.URN.empty))))
+      _ <- stackPromise.fulfill(StackResource()(using ComponentBase(Output.pure(besom.types.URN.empty))))
     yield summon[Context]
 
 end DummyContext
diff --git a/besom-cfg/k8s/src/test/scala/RunOutput.scala b/besom-cfg/k8s/src/test/scala/RunOutput.scala
@@ -3,11 +3,11 @@ package besom.internal
 import scala.concurrent.Future
 
 trait RunOutput[F[+_]]:
-  def run[A](output: Output[A]): Option[A]
+  def run[A](output: Output[A])(using Context): Option[A]
 
 object RunOutput:
   given runOutputForFuture(using RunResult[Future]): RunOutput[Future] = new RunOutput[Future]:
-    def run[A](output: Output[A]): Option[A] = RunResult.unsafeRunSync(output.getData)().getValue
+    def run[A](output: Output[A])(using Context): Option[A] = RunResult.unsafeRunSync(output.getData)().getValue
 
-  extension [F[+_], A](output: Output[A])(using ro: RunOutput[F]) def unsafeRunSync(): Option[A] = ro.run(output)
-  extension [F[+_], A](result: Result[A])(using rr: RunResult[F]) def unsafeRunSync(): A         = rr.run(result)
+  extension [F[+_], A](output: Output[A])(using ro: RunOutput[F]) def unsafeRunSync()(using Context): Option[A] = ro.run(output)
+  extension [F[+_], A](result: Result[A])(using rr: RunResult[F]) def unsafeRunSync(): A                        = rr.run(result)
diff --git a/besom-cfg/k8s/src/test/scala/RunResult.scala b/besom-cfg/k8s/src/test/scala/RunResult.scala
@@ -8,7 +8,7 @@ trait RunResult[F[+_]]:
 object RunResult:
   given RunResult[Future] = new RunResult[Future]:
     given ExecutionContext = ExecutionContext.fromExecutorService(
-      null, // FJP does seem to swallow fatals
+      null, // global Future EC reports fatals and uncaught exceptions to stderr, we want to quit on fatals
       (t: Throwable) =>
         // TODO this has to contain a link to github issue tracker to allow user to easily create a bug report, this is EXTREMELY IMPORTANT
         scribe.error("Uncaught fatal error in Future Runtime", t)

diff --git a/besom-cfg/lib/project.scala b/besom-cfg/lib/project.scala
@@ -16,4 +16,3 @@
 //> using publish.developer "prolativ|Michał Pałka|https://github.com/prolativ"
 //> using publish.developer "KacperFKorban|Kacper Korban|https://github.com/KacperFKorban"
 //> using publish.developer "pawelprazak|Paweł Prażak|https://github.com/pawelprazak"
-//> using repository sonatype:snapshots
diff --git a/besom-cfg/lib/src/main/scala/Struct.scala b/besom-cfg/lib/src/main/scala/Struct.scala
@@ -4,7 +4,7 @@ import scala.language.dynamics
 import scala.quoted.*
 import scala.collection.immutable.ListMap
 import besom.cfg.internal.MetaUtils
-import besom.types.{Output, Context}
+import besom.types.Output
 import scala.util.chaining.*
 
 // possible types under `Any`:
@@ -18,7 +18,7 @@ class Struct private (val _values: ListMap[String, Any]) extends Selectable:
     onStruct: Map[String, Output[B]] => Output[B],
     onList: List[B] => Output[B],
     onValue: Any => Output[B]
-  )(using Context): Output[B] =
+  ): Output[B] =
     val onOutput: Output[_] => Output[B] = _.flatMap {
       case s: Struct => s.fold(onStruct, onList, onValue)
       case a         => onValue(a)
@@ -91,7 +91,7 @@ object Struct extends Dynamic:
         )
 
   extension (s: Struct)
-    def foldToEnv(using Context): Output[List[(String, String)]] = s.fold[List[(String, String)]](
+    def foldToEnv: Output[List[(String, String)]] = s.fold[List[(String, String)]](
       onStruct = { mapB =>
         mapB.foldLeft(Output(List.empty[(String, String)])) { case (acc, (k, v)) =>
           acc.flatMap { accList =>

diff --git a/besom-rpc/project.scala b/besom-rpc/project.scala
@@ -0,0 +1,18 @@
+//> using scala 3.3.3
+
+//> using options "-java-output-version:11"
+
+// TODO dependency bumps
+//> using dep "com.google.protobuf:protobuf-java-util:3.25.1"
+//> using dep "com.thesamet.scalapb::scalapb-runtime:0.11.15"
+//> using dep "com.thesamet.scalapb::scalapb-runtime-grpc:0.11.15"
+
+//> using publish.name "besom-rpc"
+//> using publish.organization "org.virtuslab"
+//> using publish.url "https://github.com/VirtusLab/besom"
+//> using publish.vcs "github:VirtusLab/besom"
+//> using publish.license "Apache-2.0"
+//> using publish.repository "central"
+//> using publish.developer "lbialy|Łukasz Biały|https://github.com/lbialy"
+//> using publish.developer "prolativ|Michał Pałka|https://github.com/prolativ"
+
diff --git a/...om/rpc/com/google/rpc/status/Status.scala → .../scala/com/google/rpc/status/Status.scala b/...om/rpc/com/google/rpc/status/Status.scala → .../scala/com/google/rpc/status/Status.scala
@@ -76,8 +76,8 @@ final case class Status(
     unknownFields: _root_.scalapb.UnknownFieldSet = _root_.scalapb.UnknownFieldSet.empty
     ) extends scalapb.GeneratedMessage with scalapb.lenses.Updatable[Status] {
     @transient
-    private[this] var __serializedSizeMemoized: _root_.scala.Int = 0
-    private[this] def __computeSerializedSize(): _root_.scala.Int = {
+    private var __serializedSizeMemoized: _root_.scala.Int = 0
+    private def __computeSerializedSize(): _root_.scala.Int = {
       var __size = 0
 
       {
@@ -208,21 +208,21 @@ object Status extends scalapb.GeneratedMessageCompanion[com.google.rpc.status.St
   }
   def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = StatusProto.javaDescriptor.getMessageTypes().get(0)
   def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = StatusProto.scalaDescriptor.messages(0)
-  def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = {
-    var __out: _root_.scalapb.GeneratedMessageCompanion[_] = null
+  def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[?] = {
+    var __out: _root_.scalapb.GeneratedMessageCompanion[?] = null
     (__number: @_root_.scala.unchecked) match {
       case 3 => __out = com.google.protobuf.any.Any
     }
     __out
   }
-  lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty
-  def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber)
+  lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[? <: _root_.scalapb.GeneratedMessage]] = Seq.empty
+  def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[?] = throw new MatchError(__fieldNumber)
   lazy val defaultInstance = com.google.rpc.status.Status(
     code = 0,
     message = "",
     details = _root_.scala.Seq.empty
   )
-  implicit class StatusLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, com.google.rpc.status.Status]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, com.google.rpc.status.Status](_l) {
+  implicit class StatusLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, com.google.rpc.status.Status]) extends _root_.scalapb.lenses.MessageLens[UpperPB, com.google.rpc.status.Status](_l) {
     def code: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Int] = field(_.code)((c_, f_) => c_.copy(code = f_))
     def message: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.message)((c_, f_) => c_.copy(message = f_))
     def details: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Seq[com.google.protobuf.any.Any]] = field(_.details)((c_, f_) => c_.copy(details = f_))

diff --git a/...c/com/google/rpc/status/StatusProto.scala → ...a/com/google/rpc/status/StatusProto.scala b/...c/com/google/rpc/status/StatusProto.scala → ...a/com/google/rpc/status/StatusProto.scala
@@ -9,8 +9,8 @@ object StatusProto extends _root_.scalapb.GeneratedFileObject {
   lazy val dependencies: Seq[_root_.scalapb.GeneratedFileObject] = Seq(
     com.google.protobuf.any.AnyProto
   )
-  lazy val messagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] =
-    Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]](
+  lazy val messagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[? <: _root_.scalapb.GeneratedMessage]] =
+    Seq[_root_.scalapb.GeneratedMessageCompanion[? <: _root_.scalapb.GeneratedMessage]](
       com.google.rpc.status.Status
     )
   private lazy val ProtoBytes: _root_.scala.Array[Byte] =