diff --git a/Justfile b/Justfile index 276a7112..dfb8187b 100644 --- a/Justfile +++ b/Justfile @@ -1,7 +1,6 @@ # Big idea behind using a Justfile is so that we can have modules like in sbt. besom-version := `cat version.txt` -besom-cfg-version := `cat besom-cfg/version.txt` is-snapshot := if "{{besom-version}}" =~ '.*-SNAPSHOT' { "true" } else { "false" } no-bloop := if env_var_or_default('BESOM_BUILD_NO_BLOOP', "") == "true" { "--server=false" } else { "" } @@ -44,20 +43,20 @@ build-packages-for-templates-and-examples: grep -hr "0.3-SNAPSHOT" examples/**/*.scala templates/**/*.scala | sed -n 's/.*besom-\([^:]*:[^"]*\).*-core.0.3-SNAPSHOT.*/\1/p' | sort -u | tr '\n' ' ' | xargs -I {} just cli packages local {} # Cleans everything -clean-all: clean-json clean-rpc clean-sdk clean-auto clean-out clean-compiler-plugin clean-codegen clean-scripts clean-test-integration clean-cfg clean-test-templates clean-test-examples clean-test-markdown +clean-all: clean-json clean-model clean-rpc clean-sdk clean-auto clean-out clean-compiler-plugin clean-codegen clean-scripts clean-test-integration clean-cfg clean-test-templates clean-test-examples clean-test-markdown # Compiles everything -compile-all: compile-json compile-rpc compile-sdk compile-auto compile-codegen compile-scripts compile-compiler-plugin build-language-plugin +compile-all: compile-json compile-model compile-rpc compile-sdk compile-auto compile-codegen compile-scripts compile-compiler-plugin build-language-plugin # Tests everything -test-all: test-json test-sdk test-auto test-codegen test-scripts test-integration build-packages-for-templates-and-examples test-templates test-examples test-markdown +test-all: test-json test-model test-sdk test-auto test-codegen test-scripts test-integration build-packages-for-templates-and-examples test-templates test-examples test-markdown # Publishes everything locally -publish-local-all: publish-local-json publish-local-rpc publish-local-sdk publish-local-auto publish-local-codegen publish-local-scripts install-language-plugin +publish-local-all: publish-local-json publish-local-model publish-local-rpc publish-local-sdk publish-local-auto publish-local-codegen publish-local-scripts install-language-plugin # Publishes everything to Maven # TODO add publish-maven-auto once stable -publish-maven-all: publish-maven-json publish-maven-rpc publish-maven-sdk publish-maven-codegen publish-maven-scripts +publish-maven-all: publish-maven-json publish-maven-model publish-maven-rpc publish-maven-sdk publish-maven-codegen publish-maven-scripts # Runs all necessary checks before committing before-commit: compile-all test-all @@ -215,7 +214,7 @@ publish-maven-json: #################### # Compiles auto module -compile-auto: publish-local-core +compile-auto: publish-local-core publish-local-model scala-cli --power compile auto --suppress-experimental-feature-warning # Runs tests for auto module @@ -314,65 +313,106 @@ publish-language-plugins-all: package-language-plugins-all compile-cfg-lib: publish-local-json publish-local-core scala-cli --power compile besom-cfg/lib --suppress-experimental-feature-warning +compile-cfg-containers: publish-local-cfg-lib publish-local-model + scala-cli --power compile besom-cfg/containers --suppress-experimental-feature-warning + # Compiles besom-cfg k8s module -compile-cfg-k8s: publish-local-cfg-lib +compile-cfg-k8s: publish-local-cfg-lib publish-local-cfg-containers just cli packages local kubernetes:4.17.1 scala-cli --power compile besom-cfg/k8s --suppress-experimental-feature-warning -# Compiles all besom-cfg modules -compile-cfg: compile-cfg-lib compile-cfg-k8s - # Publishes locally besom-cfg lib module publish-local-cfg-lib: - scala-cli --power publish local besom-cfg/lib --project-version {{besom-cfg-version}} --suppress-experimental-feature-warning + scala-cli --power publish local besom-cfg/lib --project-version {{besom-version}} --suppress-experimental-feature-warning + +publish-local-cfg-containers: compile-cfg-containers + scala-cli --power publish local besom-cfg/containers --project-version {{besom-version}} --suppress-experimental-feature-warning # Publishes locally besom-cfg k8s module publish-local-cfg-k8s: compile-cfg-k8s - scala-cli --power publish local besom-cfg/k8s --project-version {{besom-cfg-version}} --suppress-experimental-feature-warning + scala-cli --power publish local besom-cfg/k8s --project-version {{besom-version}} --suppress-experimental-feature-warning # Publishes locally all besom-cfg modules -publish-local-cfg: publish-local-cfg-lib publish-local-cfg-k8s +publish-local-cfg: publish-local-cfg-lib publish-local-cfg-containers publish-local-cfg-k8s # Publishes besom-cfg lib module to Maven publish-maven-cfg-lib: - scala-cli --power publish besom-cfg/lib --project-version {{besom-cfg-version}} {{publish-maven-auth-options}} --suppress-experimental-feature-warning + scala-cli --power publish besom-cfg/lib --project-version {{besom-version}} {{publish-maven-auth-options}} --suppress-experimental-feature-warning + +publish-maven-cfg-containers: + scala-cli --power publish besom-cfg/containers --project-version {{besom-version}} {{publish-maven-auth-options}} --suppress-experimental-feature-warning # Publishes besom-cfg k8s module to Maven publish-maven-cfg-k8s: - scala-cli --power publish besom-cfg/k8s --project-version {{besom-cfg-version}} {{publish-maven-auth-options}} --suppress-experimental-feature-warning + scala-cli --power publish besom-cfg/k8s --project-version {{besom-version}} {{publish-maven-auth-options}} --suppress-experimental-feature-warning # Tests besom-cfg lib module test-cfg-lib: compile-cfg-lib scala-cli --power test besom-cfg/lib --suppress-experimental-feature-warning +# Tests besom-cfg containers module +test-cfg-containers: compile-cfg-containers + scala-cli --power test besom-cfg/containers --suppress-experimental-feature-warning + # Tests besom-cfg k8s module -test-cfg-k8s: publish-local-cfg-lib compile-cfg-k8s +test-cfg-k8s: compile-cfg-k8s scala-cli --power test besom-cfg/k8s --suppress-experimental-feature-warning +# Compiles all besom-cfg modules +compile-cfg: compile-cfg-lib compile-cfg-containers compile-cfg-k8s + # Runs all tests of besom-cfg -test-cfg: test-cfg-lib test-cfg-k8s +test-cfg: test-cfg-lib test-cfg-containers test-cfg-k8s # Cleans besom-cfg-lib build clean-cfg-lib: scala-cli clean besom-cfg/lib +# Cleans besom-cfg-containers build +clean-cfg-containers: + scala-cli clean besom-cfg/containers + # Cleans besom-cfg-k8s build clean-cfg-k8s: scala-cli clean besom-cfg/k8s # Cleans all besom-cfg builds -clean-cfg: clean-cfg-lib clean-cfg-k8s +clean-cfg: clean-cfg-lib clean-cfg-containers clean-cfg-k8s + +#################### +# Model +#################### + +# Compiles model module +compile-model: + scala-cli --power compile {{no-bloop}} model --suppress-experimental-feature-warning + +# Runs tests for model module +test-model: + scala-cli --power test {{no-bloop}} model --suppress-experimental-feature-warning + +# Cleans model module +clean-model: + scala-cli --power clean model + +# Publishes locally model module +publish-local-model: test-model + scala-cli --power publish local {{no-bloop}} model --project-version {{besom-version}} --suppress-experimental-feature-warning + +# Publishes model module +publish-maven-model: test-model + scala-cli --power publish {{no-bloop}} model --project-version {{besom-version}} {{publish-maven-auth-options}} --suppress-experimental-feature-warning #################### # Codegen #################### # Compiles Besom codegen module -compile-codegen: +compile-codegen: publish-local-model scala-cli --power compile {{no-bloop}} codegen --suppress-experimental-feature-warning # Runs tests for Besom codegen -test-codegen: +test-codegen: compile-codegen scala-cli --power test {{no-bloop}} codegen --suppress-experimental-feature-warning # Cleans codegen build diff --git a/auto/project.scala b/auto/project.scala index 41f69fa5..5600b713 100644 --- a/auto/project.scala +++ b/auto/project.scala @@ -4,6 +4,7 @@ //> using dep org.virtuslab::besom-json:0.4.0-SNAPSHOT //> using dep org.virtuslab::besom-core:0.4.0-SNAPSHOT +//> using dep org.virtuslab::besom-model:0.4.0-SNAPSHOT //> using dep org.virtuslab::scala-yaml:0.0.8 //> using dep com.lihaoyi::os-lib:0.10.0 //> using dep com.lihaoyi::os-lib-watch:0.10.0 diff --git a/auto/src/main/scala/besom/model/SemanticVersion.scala b/auto/src/main/scala/besom/model/SemanticVersion.scala deleted file mode 100644 index 38f6d8f1..00000000 --- a/auto/src/main/scala/besom/model/SemanticVersion.scala +++ /dev/null @@ -1,123 +0,0 @@ -package besom.model - -// TODO: move to separate module -// NOTICE: keep in sync with codegen/src/model/SemanticVersion.scala - -/** A semantic version as defined by https://semver.org/ - * - * @param major - * Major version number - * @param minor - * Minor version number - * @param patch - * Patch version number - * @param preRelease - * Pre-release version identifier - * @param buildMetadata - * Build metadata version identifier - */ -case class SemanticVersion( - major: Int, - minor: Int, - patch: Int, - preRelease: Option[String] = None, - buildMetadata: Option[String] = None -) extends Ordered[SemanticVersion]: - require(major >= 0, "major version must be non-negative") - require(minor >= 0, "minor version must be non-negative") - require(patch >= 0, "patch version must be non-negative") - - override def compare(that: SemanticVersion): Int = - import math.Ordered.orderingToOrdered - - val mainCompared = (major, minor, patch).compare((that.major, that.minor, that.patch)) - - // for pre release compare each dot separated identifier from left to right - lazy val thisPreRelease: Vector[Option[String]] = preRelease.map(_.split('.')).getOrElse(Array.empty[String]).toVector.map(Some(_)) - lazy val thatPreRelease: Vector[Option[String]] = that.preRelease.map(_.split('.')).getOrElse(Array.empty[String]).toVector.map(Some(_)) - - def comparePreReleaseIdentifier(thisIdentifeir: Option[String], thatIdentifier: Option[String]): Int = - (thisIdentifeir, thatIdentifier) match - case (Some(thisId), Some(thatId)) => - val thisIsNumeric = thisId.forall(_.isDigit) - val thatIsNumeric = thatId.forall(_.isDigit) - (thisIsNumeric, thatIsNumeric) match - case (true, true) => thisId.toInt.compare(thatId.toInt) - case (false, false) => thisId.compare(thatId) - case (true, false) => -1 // numeric identifiers have always lower precedence than non-numeric identifiers - case (false, true) => 1 // numeric identifiers have always lower precedence than non-numeric identifiers - /* A larger set of pre-release fields has a higher precedence than a smaller set, if all of the preceding identifiers are equal. */ - case (Some(_), None) => 1 // larger set of pre-release fields has higher precedence - case (None, Some(_)) => -1 // larger set of pre-release fields has higher precedence - case (None, None) => 0 - - lazy val preCompared: Int = - thisPreRelease - .zipAll(thatPreRelease, None, None) - .map(comparePreReleaseIdentifier.tupled) - .find(_ != 0) - .getOrElse( - thisPreRelease.length.compare(thatPreRelease.length) - ) // if all identifiers are equal, the version with fewer fields has lower precedence - - // ignore build metadata when comparing versions per semver spec https://semver.org/#spec-item-10 - - if mainCompared != 0 - then mainCompared - else if thisPreRelease.isEmpty && thatPreRelease.nonEmpty - then 1 // normal version has higher precedence than a pre-release version - else if thisPreRelease.nonEmpty && thatPreRelease.isEmpty - then -1 // normal version has higher precedence than a pre-release version - else preCompared // pre-release version has lower precedence than a normal version - end compare - - def isSnapshot: Boolean = preRelease.contains("SNAPSHOT") - - lazy val preReleaseString: String = preRelease.map("-" + _).getOrElse("") - lazy val buildMetadataString: String = buildMetadata.map("+" + _).getOrElse("") - - override def toString: String = s"$major.$minor.$patch$preReleaseString$buildMetadataString" - def toShortString: String = - val xyz = List(major) ++ Option.when(minor > 0)(minor) ++ Option.when(patch > 0)(patch) - xyz.mkString(".") + preReleaseString + buildMetadataString -end SemanticVersion - -//noinspection ScalaFileName -object SemanticVersion { - // https://semver.org/spec/v2.0.0.html#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string - private val versionRegex = - """^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$""".r - - def apply(major: Int, minor: Int, patch: Int): SemanticVersion = - new SemanticVersion(major, minor, patch, None, None) - - def apply(major: Int, minor: Int, patch: Int, preRelease: String): SemanticVersion = - new SemanticVersion(major, minor, patch, Some(preRelease), None) - - def apply(major: Int, minor: Int, patch: Int, preRelease: String, buildMetadata: String): SemanticVersion = - new SemanticVersion(major, minor, patch, Some(preRelease), Some(buildMetadata)) - - def parse(version: String): Either[Exception, SemanticVersion] = { - version match { - case versionRegex(major, minor, patch, preRelease, buildMetadata) => - Right(SemanticVersion(major.toInt, minor.toInt, patch.toInt, Option(preRelease), Option(buildMetadata))) - case _ => Left(Exception(s"Cannot parse as semantic version: '$version'")) - } - } - - /** ParseTolerant allows for certain version specifications that do not strictly adhere to semver specs to be parsed by this library. It - * does so by normalizing versions before passing them to [[parse]]. It currently trims spaces, removes a "v" prefix, and adds a 0 patch - * number to versions with only major and minor components specified. - */ - def parseTolerant(version: String): Either[Exception, SemanticVersion] = { - val str = version.trim.stripPrefix("v") - - // Split into major.minor.(patch+pr+meta) - val parts = str.split("\\.", 3) - if parts.length < 3 then - if parts.last.contains("+") || parts.last.contains("-") then - Left(Exception("Short version cannot contain PreRelease/Build meta data")) - else parse((parts.toList ::: List.fill(3 - parts.length)("0")).mkString(".")) - else parse(str) - } -} diff --git a/besom-cfg/containers/.scalafmt.conf b/besom-cfg/containers/.scalafmt.conf new file mode 100644 index 00000000..f4d4b655 --- /dev/null +++ b/besom-cfg/containers/.scalafmt.conf @@ -0,0 +1,11 @@ +version = 3.5.2 +runner.dialect = scala3 +project.git = true +align = most +align.openParenCallSite = false +align.openParenDefnSite = false +align.tokens = [{code = "=>", owner = "Case"}, "<-", "%", "%%", "="] +indent.defnSite = 2 +maxColumn = 140 + +rewrite.scala3.insertEndMarkerMinLines = 40 \ No newline at end of file diff --git a/besom-cfg/containers/project.scala b/besom-cfg/containers/project.scala new file mode 100644 index 00000000..142130c5 --- /dev/null +++ b/besom-cfg/containers/project.scala @@ -0,0 +1,18 @@ +//> using scala 3.3.3 + +//> using dep com.lihaoyi::os-lib::0.9.3 +//> using dep org.virtuslab::besom-cfg:0.4.0-SNAPSHOT +//> using dep org.virtuslab::besom-model:0.4.0-SNAPSHOT + +//> using test.dep org.scalameta::munit:1.0.0 + +//> using publish.name "besom-cfg-containers" +//> using publish.organization "org.virtuslab" +//> using publish.url "https://github.com/VirtusLab/besom" +//> using publish.vcs "github:VirtusLab/besom" +//> using publish.license "Apache-2.0" +//> using publish.repository "central" +//> using publish.developer "lbialy|Łukasz Biały|https://github.com/lbialy" +//> using publish.developer "prolativ|Michał Pałka|https://github.com/prolativ" +//> using publish.developer "KacperFKorban|Kacper Korban|https://github.com/KacperFKorban" +//> using publish.developer "pawelprazak|Paweł Prażak|https://github.com/pawelprazak" diff --git a/besom-cfg/k8s/src/main/scala/containers.scala b/besom-cfg/containers/src/main/scala/containers.scala similarity index 52% rename from besom-cfg/k8s/src/main/scala/containers.scala rename to besom-cfg/containers/src/main/scala/containers.scala index 57d5da68..82d3fd68 100644 --- a/besom-cfg/k8s/src/main/scala/containers.scala +++ b/besom-cfg/containers/src/main/scala/containers.scala @@ -1,8 +1,8 @@ package besom.cfg.containers -// this should be a separate package, base for all container integrations - import besom.cfg.internal.Schema +import besom.cfg.Configured +import besom.model.SemanticVersion import besom.json.* import scala.util.Try @@ -26,38 +26,45 @@ def saveToCache(image: String, content: String): Unit = os.makeDir.all(os.Path(s"$cacheDir/besom-cfg")) os.write.over(os.Path(s"$cacheDir/besom-cfg/$sanitized"), content) -def resolveMetadataFromImage(image: String): String = +def resolveMetadataFromImage(image: String, overrideClasspathPath: Option[String] = None): String = lazy val sbtNativePackagerFormatCall = + val classpathPath = overrideClasspathPath.getOrElse("/opt/docker/lib/*") os - .proc("docker", "run", "--rm", "--entrypoint", "java", image, "-cp", "/opt/docker/lib/*", "besom.cfg.SummonConfiguration") + .proc("docker", "run", "--rm", "--entrypoint", "java", image, "-cp", classpathPath, "besom.cfg.SummonConfiguration") .call(check = false) lazy val customDockerFormatCall = + val classpathPath = overrideClasspathPath.getOrElse("/app/main") os - .proc("docker", "run", "--rm", "--entrypoint", "java", image, "-cp", "/app/main", "besom.cfg.SummonConfiguration") + .proc("docker", "run", "--rm", "--entrypoint", "java", image, "-cp", classpathPath, "besom.cfg.SummonConfiguration") .call(check = false) if sbtNativePackagerFormatCall.exitCode == 0 then sbtNativePackagerFormatCall.out.text().trim() else if customDockerFormatCall.exitCode == 0 then customDockerFormatCall.out.text().trim() else throw RuntimeException(s"Failed to get configuration from $image") -def getDockerImageMetadata(image: String): Either[Throwable, Schema] = +def getDockerImageMetadata(image: String, dontUseCache: Boolean, overrideClasspathPath: Option[String] = None): Either[Throwable, Schema] = Try { - // 1. cache result per image in /tmp DONE - // 2. verify the version of the library used, fail macro if we are older than it - // 3. parse the json to correct structure DONE - // next: - // - support different image setups, autodetect which one is used somehow? somewhat DONE - // - cp argument should be configurable - val json = fetchFromCache(image) match { + val maybeCachedJson = if dontUseCache then None else fetchFromCache(image) + + val json = maybeCachedJson match { case Some(cachedJson) => cachedJson case None => - val json = resolveMetadataFromImage(image) + val json = resolveMetadataFromImage(image, overrideClasspathPath) saveToCache(image, json) json } - summon[JsonFormat[Schema]].read(json.parseJson) + val schema = summon[JsonFormat[Schema]].read(json.parseJson) + val obtainedSchemaVersion = SemanticVersion.parse(schema.version).toTry.get + val besomCfgVersionFromClasspath = SemanticVersion.parse(besom.cfg.Version).toTry.get + + if obtainedSchemaVersion > besomCfgVersionFromClasspath then + throw Exception( + s"Version of besom-cfg-lib used in image $image is '$obtainedSchemaVersion' and is newer than the present library version '$besomCfgVersionFromClasspath'. Please update the besom-cfg extension in your besom project." + ) + else schema + }.toEither diff --git a/besom-cfg/k8s/project.scala b/besom-cfg/k8s/project.scala index 14f849d9..f3bbf5b2 100644 --- a/besom-cfg/k8s/project.scala +++ b/besom-cfg/k8s/project.scala @@ -1,7 +1,8 @@ //> using scala 3.3.3 //> using dep com.lihaoyi::os-lib::0.9.3 -//> using dep org.virtuslab::besom-cfg:0.2.0-SNAPSHOT +//> using dep org.virtuslab::besom-cfg:0.4.0-SNAPSHOT +//> using dep org.virtuslab::besom-cfg-containers:0.4.0-SNAPSHOT //> using dep org.virtuslab::besom-kubernetes:4.17.1-core.0.4-SNAPSHOT //> using dep com.lihaoyi::fansi::0.5.0 //> using dep com.lihaoyi::fastparse:3.1.0 diff --git a/besom-cfg/k8s/src/main/scala/ConfiguredContainerArgs.scala b/besom-cfg/k8s/src/main/scala/ConfiguredContainerArgs.scala index d95d75b2..01c92cec 100644 --- a/besom-cfg/k8s/src/main/scala/ConfiguredContainerArgs.scala +++ b/besom-cfg/k8s/src/main/scala/ConfiguredContainerArgs.scala @@ -11,9 +11,9 @@ import scala.util.* import scala.quoted.* import besom.cfg.k8s.syntax.* -// this is besom-cfg-kubernetes entrypoint - object syntax: + import besom.cfg.from.env.* + // this should be somehow bound to implementation of medium backend extension (s: Struct) def foldedToEnvVarArgs: Output[List[EnvVarArgs]] = s.foldToEnv.map(_.map { case (k, v) => EnvVarArgs(name = k, value = v) }) @@ -47,7 +47,9 @@ object ConfiguredContainerArgs: tty: Input.Optional[Boolean] = None, volumeDevices: Input.Optional[List[Input[VolumeDeviceArgs]]] = None, volumeMounts: Input.Optional[List[Input[VolumeMountArgs]]] = None, - workingDir: Input.Optional[String] = None + workingDir: Input.Optional[String] = None, + dontUseCache: Boolean = false, + overrideClasspathPath: Option[String] = None ) = ${ applyImpl( 'name, @@ -74,7 +76,9 @@ object ConfiguredContainerArgs: 'tty, 'volumeDevices, 'volumeMounts, - 'workingDir + 'workingDir, + 'dontUseCache, + 'overrideClasspathPath ) } @@ -103,7 +107,9 @@ object ConfiguredContainerArgs: tty: Expr[Input.Optional[Boolean]], volumeDevices: Expr[Input.Optional[List[Input[VolumeDeviceArgs]]]], volumeMounts: Expr[Input.Optional[List[Input[VolumeMountArgs]]]], - workingDir: Expr[Input.Optional[String]] + workingDir: Expr[Input.Optional[String]], + dontUseCache: Expr[Boolean], + overrideClasspathPath: Expr[Option[String]] )(using Quotes): Expr[ContainerArgs] = import quotes.reflect.* @@ -115,10 +121,18 @@ object ConfiguredContainerArgs: case None => report.errorAndAbort("Image name has to be a literal!", image) case Some(value) => value - val schema = getDockerImageMetadata(dockerImage) match + val classpathPath = overrideClasspathPath.value.flatten + + val schema = getDockerImageMetadata(dockerImage, dontUseCache.value.getOrElse(false), classpathPath) match case Left(throwable) => report.errorAndAbort(s"Failed to get metadata for image $dockerImage:$NL${pprint(throwable)}", image) case Right(schema) => schema + if schema.medium != Configured.FromEnv.MediumIdentifier then + report.errorAndAbort( + s"Medium ${schema.medium} requested by besom-cfg schema for image $dockerImage is not supported by k8s container configuration yet.", + image + ) + Diff.performDiff(schema, configuration) match case Left(prettyDiff) => // TODO maybe strip all the ansi codes if in CI? report.errorAndAbort( diff --git a/besom-cfg/k8s/src/test/scala/ErrorsTest.scala b/besom-cfg/k8s/src/test/scala/ErrorsTest.scala index 3e053908..b41ddd77 100644 --- a/besom-cfg/k8s/src/test/scala/ErrorsTest.scala +++ b/besom-cfg/k8s/src/test/scala/ErrorsTest.scala @@ -198,7 +198,8 @@ class ErrorsTest extends munit.FunSuite: "name": "shouldBeAListOfStructsButItsAString" } ], - "version": "0.1.0" + "version": "0.1.0", + "medium": "env" }""", struct ) match diff --git a/besom-cfg/k8s/src/test/scala/StructSerdeTest.scala b/besom-cfg/k8s/src/test/scala/StructSerdeTest.scala index b4ab580a..5c18c218 100644 --- a/besom-cfg/k8s/src/test/scala/StructSerdeTest.scala +++ b/besom-cfg/k8s/src/test/scala/StructSerdeTest.scala @@ -2,6 +2,7 @@ package besom.cfg import besom.cfg.k8s.* import besom.cfg.* +import besom.cfg.from.env.* import besom.internal.DummyContext import besom.internal.RunOutput.* diff --git a/besom-cfg/lib/src/main/scala/Configured.scala b/besom-cfg/lib/src/main/scala/Configured.scala index 196662e2..474af952 100644 --- a/besom-cfg/lib/src/main/scala/Configured.scala +++ b/besom-cfg/lib/src/main/scala/Configured.scala @@ -3,6 +3,10 @@ package besom.cfg import scala.quoted.* import besom.json.* import besom.cfg.internal.* +import besom.util.Validated +import scala.util.control.NoStackTrace + +final val Version = "0.4.0-SNAPSHOT" // trait Constraint[A]: // def validate(a: A): Boolean @@ -26,75 +30,89 @@ import besom.cfg.internal.* // object NonBlank extends Constraint[String]: // def validate(a: String): Boolean = a.trim.nonEmpty -// trait FromEnv[A]: -// def fromEnv(parentKey: String, selected: Map[String, String]): A - -// object StringFromEnv extends FromEnv[String]: -// def fromEnv(parentKey: String, selected: Map[String, String]): String = -// selected.getOrElse(parentKey, throw new Exception(s"Key $parentKey not found in env")) - -// given ListFromEnv[A](using FromEnv[A]): FromEnv[List[A]] with -// def fromEnv(parentKey: String, selected: Map[String, String]): List[A] = -// val prefix = s"$parentKey." -// val subselected = selected.filter(_._1.startsWith(prefix)) -// subselected.keys -// .map { k => -// val index = k.stripPrefix(prefix) -// val value = summon[FromEnv[A]].fromEnv(k, selected) -// index.toInt -> value -// } -// .toList -// .sortBy(_._1) -// .map(_._2) +case class ConfigurationError(errors: Iterable[Throwable]) extends Exception(ConfigurationError.render(errors)) with NoStackTrace: + override def toString(): String = getMessage() + +object ConfigurationError: + def render(errors: Iterable[Throwable]): String = + s"""Start of the application was impossible due to the following configuration errors: + |${errors.map(_.getMessage).mkString(" * ", "\n * ", "")} + |""".stripMargin + +trait Default[A]: + def default: A trait Configured[A]: + type INPUT def schema: Schema - def newInstanceFromEnv(env: Map[String, String] = sys.env): A + def newInstance(input: INPUT): A object Configured: - val Version = "0.1.0" - - inline def derived[A <: Product]: Configured[A] = ${ derivedImpl[A] } - - def derivedImpl[A <: Product: Type](using ctx: Quotes): Expr[Configured[A]] = - import ctx.reflect.* - - val tpe = TypeRepr.of[A] - val fields = tpe.typeSymbol.caseFields.map { case sym => - val name = Expr(sym.name) - val ftpe: Expr[ConfiguredType[_]] = - tpe.memberType(sym).dealias.asType match - case '[t] => - Expr.summon[ConfiguredType[t]].getOrElse { - report.error( - s"Cannot find ConfiguredType for type ${tpe.memberType(sym).dealias.show}" - ) - throw new Exception("Cannot find ConfiguredType") - } - case _ => - report.error("Unsupported type") - throw new Exception("Unsupported type") - - '{ Field(${ name }, ${ ftpe }.toFieldType) } - } - - val fromEnvExpr = Expr.summon[FromEnv[A]].getOrElse { - report.error(s"Cannot find FromEnv for type ${tpe.show}") - throw new Exception("Cannot find FromEnv") - } - - val schemaExpr = '{ Schema(${ Expr.ofList(fields) }.toList, ${ Expr(Version) }) } - - '{ - new Configured[A] { - def schema = $schemaExpr - def newInstanceFromEnv(env: Map[String, String] = sys.env): A = - $fromEnvExpr.decode(env, "").getOrElse { - throw new Exception("Failed to decode") - } + + trait FromEnv[A] extends Configured[A]: + type INPUT = FromEnv.EnvData + + object FromEnv: + val MediumIdentifier = "env" + + opaque type EnvData >: Map[String, String] = Map[String, String] + extension (env: EnvData) def unwrap: Map[String, String] = env + object EnvData: + given Default[EnvData] = new Default[EnvData]: + def default: EnvData = sys.env + + inline def derived[A <: Product]: Configured.FromEnv[A] = ${ derivedImpl[A] } + + def derivedImpl[A <: Product: Type](using ctx: Quotes): Expr[Configured.FromEnv[A]] = + import ctx.reflect.* + + val tpe = TypeRepr.of[A] + val fields = tpe.typeSymbol.caseFields.map { case sym => + val name = Expr(sym.name) + val ftpe: Expr[ConfiguredType[_]] = + tpe.memberType(sym).dealias.asType match + case '[t] => + Expr.summon[ConfiguredType[t]].getOrElse { + report.errorAndAbort( + s"Cannot find ConfiguredType for type ${tpe.memberType(sym).dealias.show}" + ) + } + + case _ => report.errorAndAbort("Unsupported type") + + '{ Field(${ name }, ${ ftpe }.toFieldType) } + } + + val fromEnvExpr = Expr.summon[from.env.ReadFromEnvVars[A]].getOrElse { + report.errorAndAbort(s"Cannot find FromEnv for type ${tpe.show}") + } + + val schemaExpr = '{ Schema(${ Expr.ofList(fields) }.toList, ${ Expr(Version) }, ${ Expr(FromEnv.MediumIdentifier) }) } + + '{ + new Configured.FromEnv[A] { + + def schema = $schemaExpr + def newInstance(input: INPUT): A = + $fromEnvExpr.decode(input.unwrap, from.env.EnvPath.Root) match + case Validated.Valid(a) => a + case Validated.Invalid(errors) => throw ConfigurationError(errors.toVector) + } } - } + end derivedImpl + end FromEnv end Configured -def resolveConfiguration[A](using c: Configured[A]): A = - c.newInstanceFromEnv() +def resolveConfiguration[A](using c: Configured[A], d: Default[c.INPUT]): A = + c.newInstance(d.default) + +def resolveConfiguration[A](using c: Configured[A])(input: c.INPUT): A = + c.newInstance(input) + +def resolveConfigurationEither[A](using c: Configured[A], d: Default[c.INPUT]): Either[ConfigurationError, A] = + try Right(resolveConfiguration[A]) + catch case e: ConfigurationError => Left(e) + +def resolveConfigurationEither[A](using c: Configured[A])(input: c.INPUT): Either[ConfigurationError, A] = + try Right(resolveConfiguration[A](input)) + catch case e: ConfigurationError => Left(e) diff --git a/besom-cfg/lib/src/main/scala/FromEnv.scala b/besom-cfg/lib/src/main/scala/FromEnv.scala deleted file mode 100644 index b43a87d4..00000000 --- a/besom-cfg/lib/src/main/scala/FromEnv.scala +++ /dev/null @@ -1,87 +0,0 @@ -package besom.cfg - -// TODO do not use Option[T], use something with a proper error channel and missing value channel -// TODO rationale: if a value is provided but it's not valid (e.g. empty string for an Int), we want to know -// TODO if a value is missing, but the type is optional in configuration, that's fine -trait FromEnv[A]: - def decode(env: Map[String, String], path: String): Option[A] - -object FromEnv: - - import scala.deriving.* - import scala.compiletime.{erasedValue, summonInline} - - inline def summonAllInstances[T <: Tuple]: List[FromEnv[?]] = - inline erasedValue[T] match - case _: (t *: ts) => summonInline[FromEnv[t]] :: summonAllInstances[ts] - case _: EmptyTuple => Nil - - inline def summonLabels[T <: Tuple]: List[String] = - inline erasedValue[T] match - case _: EmptyTuple => Nil - case _: (t *: ts) => - summonInline[ValueOf[t]].value.asInstanceOf[String] :: summonLabels[ts] - - given [A: FromEnv]: FromEnv[Option[A]] with - def decode(env: Map[String, String], path: String): Option[Option[A]] = - Some(summon[FromEnv[A]].decode(env, path)) - - given FromEnv[Int] with - def decode(env: Map[String, String], path: String): Option[Int] = - env.get(path).flatMap(s => scala.util.Try(s.toInt).toOption) - - given FromEnv[Long] with - def decode(env: Map[String, String], path: String): Option[Long] = - env.get(path).flatMap(s => scala.util.Try(s.toLong).toOption) - - given FromEnv[String] with - def decode(env: Map[String, String], path: String): Option[String] = - env.get(path) - - given FromEnv[Double] with - def decode(env: Map[String, String], path: String): Option[Double] = - env.get(path).flatMap(s => scala.util.Try(s.toDouble).toOption) - - given FromEnv[Float] with - def decode(env: Map[String, String], path: String): Option[Float] = - env.get(path).flatMap(s => scala.util.Try(s.toFloat).toOption) - - given FromEnv[Boolean] with - def decode(env: Map[String, String], path: String): Option[Boolean] = - env.get(path).flatMap(s => scala.util.Try(s.toBoolean).toOption) - - given [A: FromEnv]: FromEnv[List[A]] with - def decode(env: Map[String, String], path: String): Option[List[A]] = - Iterator.from(0).map(i => summon[FromEnv[A]].decode(env, s"$path.$i")).takeWhile(_.isDefined).toList.sequence - - given [A: FromEnv]: FromEnv[Vector[A]] with - def decode(env: Map[String, String], path: String): Option[Vector[A]] = - Iterator.from(0).map(i => summon[FromEnv[A]].decode(env, s"$path.$i")).takeWhile(_.isDefined).toVector.sequence - - inline given derived[A](using m: Mirror.ProductOf[A]): FromEnv[A] = new FromEnv[A]: - def decode(env: Map[String, String], path: String): Option[A] = - val elemDecoders = summonAllInstances[m.MirroredElemTypes] - val labels = summonLabels[m.MirroredElemLabels] - - val elemValues = elemDecoders.zip(labels).map { case (decoder, label) => - // handle top-level gracefully (empty path) - decoder.asInstanceOf[FromEnv[Any]].decode(env, if path.isBlank() then label else s"$path.$label") - } - - if elemValues.forall(_.isDefined) then Some(m.fromProduct(Tuple.fromArray(elemValues.flatten.toArray))) - else None - - // Helper to sequence a List[Option[A]] into Option[List[A]] - extension [A](xs: List[Option[A]]) - def sequence: Option[List[A]] = xs.foldRight(Option(List.empty[A])) { - case (Some(a), Some(acc)) => Some(a :: acc) - case _ => None - } - - // Helper to sequence a Vector[Option[A]] into Option[Vector[A]] - extension [A](xs: Vector[Option[A]]) - def sequence: Option[Vector[A]] = xs.foldLeft(Option(Vector.empty[A])) { - case (Some(acc), Some(a)) => Some(acc :+ a) - case _ => None - } -end FromEnv diff --git a/besom-cfg/lib/src/main/scala/Struct.scala b/besom-cfg/lib/src/main/scala/Struct.scala index 36f79e17..7fee0c96 100644 --- a/besom-cfg/lib/src/main/scala/Struct.scala +++ b/besom-cfg/lib/src/main/scala/Struct.scala @@ -71,11 +71,14 @@ object Struct extends Dynamic: args match case Varargs(argExprs) => - val refinementTypes = argExprs.toList.map { case '{ ($key: String, $value: v) } => - (key.valueOrAbort, TypeRepr.of[v]) + val refinementTypes = argExprs.toList.map { + case '{ ($key: String, $value: v) } => (key.valueOrAbort, TypeRepr.of[v]) + case _ => report.errorAndAbort("Expected explicit named varargs sequence. Notation `args*` is not supported.", args) } - val exprs = argExprs.map { case '{ ($key: String, $value: v) } => - '{ ($key, $value) } + + val exprs = argExprs.map { + case '{ ($key: String, $value: v) } => '{ ($key, $value) } + case _ => report.errorAndAbort("Expected explicit named varargs sequence. Notation `args*` is not supported.", args) } val argsExpr = Expr.ofSeq(exprs) @@ -84,36 +87,5 @@ object Struct extends Dynamic: '{ Struct.make(${ argsExpr }.to(ListMap)).asInstanceOf[t] } case _ => - report.errorAndAbort( - "Expected explicit varargs sequence. " + - "Notation `args*` is not supported.", - args - ) - - extension (s: Struct) - def foldToEnv: Output[List[(String, String)]] = s.fold[List[(String, String)]]( - onStruct = { mapB => - mapB.foldLeft(Output(List.empty[(String, String)])) { case (acc, (k, v)) => - acc.flatMap { accList => - v.map { vList => - accList ++ vList.map { case (k2, v2) => - // println(s"struct, serializing '$k' '$k2' to ${if k2.isBlank() then s"$k -> $v2" else s"$k.$k2 -> $v2"}") - if k2.isBlank then k -> v2 else s"$k.$k2" -> v2 - } - } - } - } - }, - onList = { list => - Output(list.zipWithIndex.flatMap { (lst, idx) => - lst.map { case (k, v) => - // println(s"list: serializing $k, $v to $k$idx -> $v") - if k.isBlank() then s"$k$idx" -> v else s"$idx.$k" -> v - } - }) - }, - onValue = a => - // println(s"serializing $a to List(\"\" -> $a)") - Output(List("" -> a.toString)) - ) + report.errorAndAbort("Expected explicit named varargs sequence. Notation `args*` is not supported.", args) end Struct diff --git a/besom-cfg/lib/src/main/scala/from/env/FromEnv.scala b/besom-cfg/lib/src/main/scala/from/env/FromEnv.scala new file mode 100644 index 00000000..baebdbc2 --- /dev/null +++ b/besom-cfg/lib/src/main/scala/from/env/FromEnv.scala @@ -0,0 +1,126 @@ +package besom.cfg.from.env + +import besom.util.* +import scala.util.Try +import besom.cfg.Struct +import besom.types.{Output, Context} + +extension (s: Struct) + def foldToEnv: Output[List[(String, String)]] = s.fold[List[(String, String)]]( + onStruct = { mapB => + mapB.foldLeft(Output(List.empty[(String, String)])) { case (acc, (k, v)) => + acc.flatMap { accList => + v.map { vList => + accList ++ vList.map { case (k2, v2) => + if k2.isBlank then k -> v2 else s"$k.$k2" -> v2 + } + } + } + } + }, + onList = { list => + Output(list.zipWithIndex.flatMap { (lst, idx) => + lst.map { case (k, v) => + if k.isBlank() then s"$k$idx" -> v else s"$idx.$k" -> v + } + }) + }, + onValue = a => Output(List("" -> a.toString)) + ) + +enum EnvPath: + case Root + case Sub(path: String) + + def subpath(path: String): EnvPath = + this match + case Root => Sub(path) + case Sub(p) => Sub(s"$p.$path") + + def asKey: String = this match + case Root => ReadFromEnvVars.Prefix + case Sub(path) => s"${ReadFromEnvVars.Prefix}_$path" + +trait ReadFromEnvVars[A]: + def decode(env: Map[String, String], path: EnvPath): Validated[Throwable, A] + +object ReadFromEnvVars: + + private[cfg] val Prefix = "BESOM_CFG" + + case class MissingKey(path: EnvPath) + extends Exception(s"Missing value for key: ${path.asKey.stripPrefix(Prefix + "_")} (env var: `${path.asKey}`)") + + extension (env: Map[String, String]) + private[ReadFromEnvVars] def lookup(path: EnvPath): Validated[Throwable, String] = + env.get(path.asKey).filter(_.nonEmpty).toValidatedOrError(MissingKey(path)) + + import scala.deriving.* + import scala.compiletime.{erasedValue, summonInline} + + inline def summonAllInstances[T <: Tuple]: List[ReadFromEnvVars[?]] = + inline erasedValue[T] match + case _: (t *: ts) => summonInline[ReadFromEnvVars[t]] :: summonAllInstances[ts] + case _: EmptyTuple => Nil + + inline def summonLabels[T <: Tuple]: List[String] = + inline erasedValue[T] match + case _: EmptyTuple => Nil + case _: (t *: ts) => + summonInline[ValueOf[t]].value.asInstanceOf[String] :: summonLabels[ts] + + given [A: ReadFromEnvVars]: ReadFromEnvVars[Option[A]] with + def decode(env: Map[String, String], path: EnvPath): Validated[Throwable, Option[A]] = + summon[ReadFromEnvVars[A]].decode(env, path).redeem(_ => None, Some(_)) + + given ReadFromEnvVars[Int] with + def decode(env: Map[String, String], path: EnvPath): Validated[Throwable, Int] = + env.lookup(path).flatMap(s => Try(s.toInt).toValidated) + + given ReadFromEnvVars[Long] with + def decode(env: Map[String, String], path: EnvPath): Validated[Throwable, Long] = + env.lookup(path).flatMap(s => Try(s.toLong).toValidated) + + given ReadFromEnvVars[String] with + def decode(env: Map[String, String], path: EnvPath): Validated[Throwable, String] = + env.lookup(path) + + given ReadFromEnvVars[Double] with + def decode(env: Map[String, String], path: EnvPath): Validated[Throwable, Double] = + env.lookup(path).flatMap(s => Try(s.toDouble).toValidated) + + given ReadFromEnvVars[Float] with + def decode(env: Map[String, String], path: EnvPath): Validated[Throwable, Float] = + env.lookup(path).flatMap(s => Try(s.toFloat).toValidated) + + given ReadFromEnvVars[Boolean] with + def decode(env: Map[String, String], path: EnvPath): Validated[Throwable, Boolean] = + env.lookup(path).flatMap(s => Try(s.toBoolean).toValidated) + + given [A: ReadFromEnvVars]: ReadFromEnvVars[List[A]] with + def decode(env: Map[String, String], path: EnvPath): Validated[Throwable, List[A]] = + Iterator.from(0).map(i => summon[ReadFromEnvVars[A]].decode(env, path.subpath(i.toString()))).takeWhile(_.isValid).toList.sequenceL + + given [A: ReadFromEnvVars]: ReadFromEnvVars[Vector[A]] with + def decode(env: Map[String, String], path: EnvPath): Validated[Throwable, Vector[A]] = + Iterator.from(0).map(i => summon[ReadFromEnvVars[A]].decode(env, path.subpath(i.toString()))).takeWhile(_.isValid).toVector.sequenceV + + inline given derived[A](using m: Mirror.ProductOf[A]): ReadFromEnvVars[A] = new ReadFromEnvVars[A]: + def decode(env: Map[String, String], path: EnvPath): Validated[Throwable, A] = + val elemDecoders = summonAllInstances[m.MirroredElemTypes] + val labels = summonLabels[m.MirroredElemLabels] + + val elemValues = elemDecoders.zip(labels).map { case (decoder, label) => + // handle top-level gracefully (empty path) + val computedPath = path match + case EnvPath.Root => EnvPath.Sub(label) + case s: EnvPath.Sub => s.subpath(label) + decoder.asInstanceOf[ReadFromEnvVars[Any]].decode(env, computedPath) + } + + elemValues.sequenceL.map { values => + val product = m.fromProduct(Tuple.fromArray(values.toArray)) + product.asInstanceOf[A] + } + +end ReadFromEnvVars diff --git a/besom-cfg/lib/src/main/scala/internal/FieldType.scala b/besom-cfg/lib/src/main/scala/internal/FieldType.scala new file mode 100644 index 00000000..49ac375a --- /dev/null +++ b/besom-cfg/lib/src/main/scala/internal/FieldType.scala @@ -0,0 +1,116 @@ +package besom.cfg.internal + +import scala.quoted.* +import besom.json.* + +enum FieldType: + case Int, Long, Float, Double, String, Boolean + case Array(inner: FieldType) + case Struct(fields: (String, FieldType)*) + case Optional(inner: FieldType) + +object FieldType: + given ToExpr[FieldType] with + def apply(fieldType: FieldType)(using Quotes): Expr[FieldType] = + import quotes.reflect.* + fieldType match + case FieldType.Int => '{ FieldType.Int } + case FieldType.Long => '{ FieldType.Long } + case FieldType.Float => '{ FieldType.Float } + case FieldType.Double => '{ FieldType.Double } + case FieldType.String => '{ FieldType.String } + case FieldType.Boolean => '{ FieldType.Boolean } + case FieldType.Array(inner) => '{ FieldType.Array(${ Expr(inner) }) } + case FieldType.Struct(fields: _*) => '{ FieldType.Struct(${ Expr(fields) }: _*) } + case FieldType.Optional(inner) => '{ FieldType.Optional(${ Expr(inner) }) } + + given FromExpr[FieldType] with + def unapply(expr: Expr[FieldType])(using Quotes): Option[FieldType] = + import quotes.reflect.* + expr match + case '{ FieldType.Int } => Some(FieldType.Int) + case '{ FieldType.Long } => Some(FieldType.Long) + case '{ FieldType.Float } => Some(FieldType.Float) + case '{ FieldType.Double } => Some(FieldType.Double) + case '{ FieldType.String } => Some(FieldType.String) + case '{ FieldType.Boolean } => Some(FieldType.Boolean) + case '{ FieldType.Array($inner) } => Some(FieldType.Array(inner.valueOrAbort)) + case '{ FieldType.Struct($fields: _*) } => Some(FieldType.Struct(fields.valueOrAbort: _*)) + case '{ FieldType.Optional($inner) } => Some(FieldType.Optional(inner.valueOrAbort)) + case _ => println("didn't match in FieldType"); None + + given JsonFormat[FieldType] with + def write(fieldType: FieldType): JsValue = fieldType match + case FieldType.Int => JsObject("type" -> JsString("int")) + case FieldType.Long => JsObject("type" -> JsString("long")) + case FieldType.Float => JsObject("type" -> JsString("float")) + case FieldType.Double => JsObject("type" -> JsString("double")) + case FieldType.String => JsObject("type" -> JsString("string")) + case FieldType.Boolean => JsObject("type" -> JsString("boolean")) + case FieldType.Array(inner) => JsObject("type" -> JsString("array"), "inner" -> write(inner)) + case FieldType.Struct(fields: _*) => + JsObject( + "type" -> JsString("struct"), + "fields" -> JsObject(fields.map { case (k, v) => k -> write(v) }.toMap) + ) + case FieldType.Optional(inner) => JsObject("type" -> JsString("optional"), "inner" -> write(inner)) + + def read(json: JsValue): FieldType = json match + case JsObject(fields) => + fields.get("type") match + case Some(JsString("int")) => FieldType.Int + case Some(JsString("long")) => FieldType.Long + case Some(JsString("float")) => FieldType.Float + case Some(JsString("double")) => FieldType.Double + case Some(JsString("string")) => FieldType.String + case Some(JsString("boolean")) => FieldType.Boolean + case Some(JsString("array")) => + fields.get("inner") match + case Some(inner) => FieldType.Array(read(inner)) + case _ => throw new Exception("Invalid JSON: array.inner must be present") + case Some(JsString("struct")) => + fields.get("fields") match + case Some(JsObject(innerFields)) => + val structFields = innerFields.map { case (k, v) => k -> read(v) } + FieldType.Struct(structFields.toVector: _*) + case None => throw new Exception("Invalid JSON: struct.fields must be present") + case _ => throw new Exception("Invalid JSON: struct.fields must be an object") + case Some(JsString("optional")) => + fields.get("inner") match + case Some(inner) => FieldType.Optional(read(inner)) + case _ => throw new Exception("Invalid JSON: optional.inner must be present") + case Some(what) => + throw new Exception(s"Invalid JSON: unknown type $what") + case None => throw new Exception("Invalid JSON: type must present") + case _ => throw new Exception("Invalid JSON: expected object") + end given +end FieldType + +case class Field(name: String, `type`: FieldType) +object Field: + given ToExpr[Field] with + def apply(field: Field)(using Quotes): Expr[Field] = + import quotes.reflect.* + '{ Field(${ Expr(field.name) }, ${ Expr(field.`type`) }) } + + given FromExpr[Field] with + def unapply(expr: Expr[Field])(using Quotes): Option[Field] = + import quotes.reflect.* + expr match + case '{ Field($name, $fieldType) } => Some(Field(name.valueOrAbort, fieldType.valueOrAbort)) + case _ => println("didn't match in Field"); None + + given fieldGiven(using fieldTypeWriter: JsonFormat[FieldType]): JsonFormat[Field] with + def write(field: Field): JsValue = + JsObject("name" -> JsString(field.name), "details" -> fieldTypeWriter.write(field.`type`)) + def read(json: JsValue): Field = + json match + case JsObject(fields) => + val name = fields.get("name") match + case Some(JsString(name)) => name + case _ => throw new Exception("Invalid JSON: field.name must be present") + val details = fields.get("details") match + case Some(details) => fieldTypeWriter.read(details) + case _ => throw new Exception("Invalid JSON: field.details must be present") + Field(name, details) + case _ => throw new Exception("Invalid JSON: expected object") diff --git a/besom-cfg/lib/src/main/scala/internal/Schema.scala b/besom-cfg/lib/src/main/scala/internal/Schema.scala index 1256e34d..bb4e8a09 100644 --- a/besom-cfg/lib/src/main/scala/internal/Schema.scala +++ b/besom-cfg/lib/src/main/scala/internal/Schema.scala @@ -3,131 +3,19 @@ package besom.cfg.internal import scala.quoted.* import besom.json.* -enum FieldType: - case Int, Long, Float, Double, String, Boolean - case Array(inner: FieldType) - case Struct(fields: (String, FieldType)*) - case Optional(inner: FieldType) - -object FieldType: - given ToExpr[FieldType] with - def apply(fieldType: FieldType)(using Quotes): Expr[FieldType] = - import quotes.reflect.* - fieldType match - case FieldType.Int => '{ FieldType.Int } - case FieldType.Long => '{ FieldType.Long } - case FieldType.Float => '{ FieldType.Float } - case FieldType.Double => '{ FieldType.Double } - case FieldType.String => '{ FieldType.String } - case FieldType.Boolean => '{ FieldType.Boolean } - case FieldType.Array(inner) => '{ FieldType.Array(${ Expr(inner) }) } - case FieldType.Struct(fields: _*) => '{ FieldType.Struct(${ Expr(fields) }: _*) } - case FieldType.Optional(inner) => '{ FieldType.Optional(${ Expr(inner) }) } - - given FromExpr[FieldType] with - def unapply(expr: Expr[FieldType])(using Quotes): Option[FieldType] = - import quotes.reflect.* - expr match - case '{ FieldType.Int } => Some(FieldType.Int) - case '{ FieldType.Long } => Some(FieldType.Long) - case '{ FieldType.Float } => Some(FieldType.Float) - case '{ FieldType.Double } => Some(FieldType.Double) - case '{ FieldType.String } => Some(FieldType.String) - case '{ FieldType.Boolean } => Some(FieldType.Boolean) - case '{ FieldType.Array($inner) } => Some(FieldType.Array(inner.valueOrAbort)) - case '{ FieldType.Struct($fields: _*) } => Some(FieldType.Struct(fields.valueOrAbort: _*)) - case '{ FieldType.Optional($inner) } => Some(FieldType.Optional(inner.valueOrAbort)) - case _ => println("didn't match in FieldType"); None - - given JsonFormat[FieldType] with - def write(fieldType: FieldType): JsValue = fieldType match - case FieldType.Int => JsObject("type" -> JsString("int")) - case FieldType.Long => JsObject("type" -> JsString("long")) - case FieldType.Float => JsObject("type" -> JsString("float")) - case FieldType.Double => JsObject("type" -> JsString("double")) - case FieldType.String => JsObject("type" -> JsString("string")) - case FieldType.Boolean => JsObject("type" -> JsString("boolean")) - case FieldType.Array(inner) => JsObject("type" -> JsString("array"), "inner" -> write(inner)) - case FieldType.Struct(fields: _*) => - JsObject( - "type" -> JsString("struct"), - "fields" -> JsObject(fields.map { case (k, v) => k -> write(v) }.toMap) - ) - case FieldType.Optional(inner) => JsObject("type" -> JsString("optional"), "inner" -> write(inner)) - - def read(json: JsValue): FieldType = json match - case JsObject(fields) => - fields.get("type") match - case Some(JsString("int")) => FieldType.Int - case Some(JsString("long")) => FieldType.Long - case Some(JsString("float")) => FieldType.Float - case Some(JsString("double")) => FieldType.Double - case Some(JsString("string")) => FieldType.String - case Some(JsString("boolean")) => FieldType.Boolean - case Some(JsString("array")) => - fields.get("inner") match - case Some(inner) => FieldType.Array(read(inner)) - case _ => throw new Exception("Invalid JSON: array.inner must be present") - case Some(JsString("struct")) => - fields.get("fields") match - case Some(JsObject(innerFields)) => - val structFields = innerFields.map { case (k, v) => k -> read(v) } - FieldType.Struct(structFields.toVector: _*) - case None => throw new Exception("Invalid JSON: struct.fields must be present") - case _ => throw new Exception("Invalid JSON: struct.fields must be an object") - case Some(JsString("optional")) => - fields.get("inner") match - case Some(inner) => FieldType.Optional(read(inner)) - case _ => throw new Exception("Invalid JSON: optional.inner must be present") - case Some(what) => - throw new Exception(s"Invalid JSON: unknown type $what") - case None => throw new Exception("Invalid JSON: type must present") - case _ => throw new Exception("Invalid JSON: expected object") - end given -end FieldType - -case class Field(name: String, `type`: FieldType) -object Field: - given ToExpr[Field] with - def apply(field: Field)(using Quotes): Expr[Field] = - import quotes.reflect.* - '{ Field(${ Expr(field.name) }, ${ Expr(field.`type`) }) } - - given FromExpr[Field] with - def unapply(expr: Expr[Field])(using Quotes): Option[Field] = - import quotes.reflect.* - expr match - case '{ Field($name, $fieldType) } => Some(Field(name.valueOrAbort, fieldType.valueOrAbort)) - case _ => println("didn't match in Field"); None - - given fieldGiven(using fieldTypeWriter: JsonFormat[FieldType]): JsonFormat[Field] with - def write(field: Field): JsValue = - JsObject("name" -> JsString(field.name), "details" -> fieldTypeWriter.write(field.`type`)) - def read(json: JsValue): Field = - json match - case JsObject(fields) => - val name = fields.get("name") match - case Some(JsString(name)) => name - case _ => throw new Exception("Invalid JSON: field.name must be present") - val details = fields.get("details") match - case Some(details) => fieldTypeWriter.read(details) - case _ => throw new Exception("Invalid JSON: field.details must be present") - Field(name, details) - case _ => throw new Exception("Invalid JSON: expected object") - -case class Schema(fields: List[Field], version: String) +case class Schema(fields: List[Field], version: String, medium: String) object Schema: given ToExpr[Schema] with def apply(schema: Schema)(using Quotes): Expr[Schema] = import quotes.reflect.* - '{ Schema(${ Expr(schema.fields) }, ${ Expr(schema.version) }) } + '{ Schema(${ Expr(schema.fields) }, ${ Expr(schema.version) }, ${ Expr(schema.medium) }) } given FromExpr[Schema] with def unapply(expr: Expr[Schema])(using Quotes): Option[Schema] = import quotes.reflect.* expr match - case '{ Schema($fields, $version) } => Some(Schema(fields.valueOrAbort, version.valueOrAbort)) - case _ => println("didn't match in Schema"); None + case '{ Schema($fields, $version, $medium) } => Some(Schema(fields.valueOrAbort, version.valueOrAbort, medium.valueOrAbort)) + case _ => println("didn't match in Schema"); None given schemaGiven(using fieldWriter: JsonFormat[Field]): JsonFormat[Schema] with def write(schema: Schema): JsValue = @@ -145,5 +33,8 @@ object Schema: val schema = fields.get("schema") match case Some(JsArray(fields)) => fields.map(fieldWriter.read).toList case _ => throw new Exception("Invalid JSON: schema.schema must be present") - Schema(schema, version) + val medium = fields.get("medium") match + case Some(JsString(medium)) => medium + case _ => throw new Exception("Invalid JSON: schema.medium must be present") + Schema(schema, version, medium) case _ => throw new Exception("Invalid JSON: expected object") diff --git a/besom-cfg/lib/src/test/scala/ConfiguredTest.scala b/besom-cfg/lib/src/test/scala/ConfiguredTest.scala index d39b084f..65b1fdbe 100644 --- a/besom-cfg/lib/src/test/scala/ConfiguredTest.scala +++ b/besom-cfg/lib/src/test/scala/ConfiguredTest.scala @@ -1,8 +1,10 @@ package besom.cfg -case class Test1(los: List[String]) derives Configured +import besom.cfg.Configured.FromEnv -case class Test2(name: String, int: Int, struct: First, list: List[Double]) derives Configured +case class Test1(los: List[String]) derives Configured.FromEnv + +case class Test2(name: String, int: Int, struct: First, list: List[Double]) derives Configured.FromEnv case class First(d: Int, e: String) case class Test3( @@ -14,17 +16,21 @@ case class Test3( lo: List[String], ol: List[String], os: Third -) derives Configured +) derives Configured.FromEnv case class Second(f1: List[Fourth], f2: String) case class Third(oh: String, it: String) case class Fourth(deep: String) class ConfiguredTest extends munit.FunSuite: + extension (m: Map[String, String]) + def withBesomCfgPrefix: Map[String, String] = + m.map { case (k, v) => s"${from.env.ReadFromEnvVars.Prefix}_$k" -> v } + test("very simple case class") { - val env = Map("los.0" -> "test", "los.1" -> "test2") + val env = Map("los.0" -> "test", "los.1" -> "test2").withBesomCfgPrefix - summon[Configured[Test1]].newInstanceFromEnv(env) match + summon[Configured.FromEnv[Test1]].newInstance(env) match case Test1(los) => assertEquals(los, List("test", "test2")) } @@ -38,9 +44,9 @@ class ConfiguredTest extends munit.FunSuite: "list.0" -> "1.2", "list.1" -> "2.3", "list.2" -> "3.4" - ) + ).withBesomCfgPrefix - summon[Configured[Test2]].newInstanceFromEnv(env) match + summon[Configured.FromEnv[Test2]].newInstance(env) match case Test2(name, int, s, l) => assertEquals(name, "test") assertEquals(int, 23) @@ -67,9 +73,9 @@ class ConfiguredTest extends munit.FunSuite: "ol.2" -> "z", "os.oh" -> "yeah", "os.it" -> "works!" - ) + ).withBesomCfgPrefix - summon[Configured[Test3]].newInstanceFromEnv(env) match + summon[Configured.FromEnv[Test3]].newInstance(env) match case Test3(name, int, s, l, ls, lo, ol, os) => assertEquals(name, "test") assertEquals(int, 23) @@ -80,4 +86,61 @@ class ConfiguredTest extends munit.FunSuite: assertEquals(ol, List("x", "y", "z")) assertEquals(os, Third("yeah", "works!")) } + + test("proper error messages") { + val confErr = intercept[ConfigurationError](summon[Configured.FromEnv[Test2]].newInstance(Map.empty)) + assertEquals(confErr.errors.size, 4) + assertNoDiff( + confErr.getMessage(), + """Start of the application was impossible due to the following configuration errors: + * Missing value for key: name (env var: `BESOM_CFG_name`) + * Missing value for key: int (env var: `BESOM_CFG_int`) + * Missing value for key: struct.d (env var: `BESOM_CFG_struct.d`) + * Missing value for key: struct.e (env var: `BESOM_CFG_struct.e`)""" + ) + } + + val env = Map("los.0" -> "test", "los.1" -> "test2").withBesomCfgPrefix + + test("resolve configuration - use default") { + given Default[FromEnv.EnvData] = new Default[FromEnv.EnvData]: + def default: FromEnv.EnvData = env + + try + resolveConfiguration[Test1] match + case Test1(los) => + assertEquals(los, List("test", "test2")) + catch + case e: ConfigurationError => + fail(e.getMessage()) + } + + test("resolve configuration - use direct argument") { + try + resolveConfiguration[Test1](env) match + case Test1(los) => + assertEquals(los, List("test", "test2")) + catch + case e: ConfigurationError => + fail(e.getMessage()) + } + + test("resolve configuration - use default and either variant") { + given Default[FromEnv.EnvData] = new Default[FromEnv.EnvData]: + def default: FromEnv.EnvData = env + + resolveConfigurationEither[Test1] match + case Right(Test1(los)) => + assertEquals(los, List("test", "test2")) + case Left(cfgError) => + fail(cfgError.getMessage()) + } + + test("resolve configuration - use direct argument and either variant") { + resolveConfigurationEither[Test1](env) match + case Right(Test1(los)) => + assertEquals(los, List("test", "test2")) + case Left(cfgError) => + fail(cfgError.getMessage()) + } end ConfiguredTest diff --git a/besom-cfg/version.txt b/besom-cfg/version.txt deleted file mode 100644 index 9b17816c..00000000 --- a/besom-cfg/version.txt +++ /dev/null @@ -1 +0,0 @@ -0.2.0-SNAPSHOT \ No newline at end of file diff --git a/codegen/project.scala b/codegen/project.scala index 68d0b9de..d4f97dfa 100644 --- a/codegen/project.scala +++ b/codegen/project.scala @@ -1,6 +1,7 @@ //> using scala 3.3.1 //> using options -release:11 -deprecation -Werror -Wunused:all -Wvalue-discard -Wnonunit-statement +//> using dep org.virtuslab::besom-model:0.4.0-SNAPSHOT //> using dep org.virtuslab::scala-yaml:0.1.0 //> using dep org.scalameta:scalameta_2.13:4.8.15 //> using dep com.lihaoyi::upickle:3.1.4 diff --git a/core/src/main/scala/besom/util/Validated.scala b/core/src/main/scala/besom/util/Validated.scala index 9e85c7c3..1e845a63 100644 --- a/core/src/main/scala/besom/util/Validated.scala +++ b/core/src/main/scala/besom/util/Validated.scala @@ -1,6 +1,7 @@ package besom.util import besom.internal.Zippable +import scala.util.Try enum Validated[+E, +A]: case Valid(a: A) extends Validated[E, A] @@ -21,6 +22,16 @@ enum Validated[+E, +A]: case Valid(a) => Valid(f(a)) case i @ Invalid(_) => i.asInstanceOf[Validated[E, B]] + def bimap[EE, B](f: E => EE, g: A => B): Validated[EE, B] = + this match + case Valid(a) => Valid(g(a)) + case Invalid(e) => Invalid(e.map(f)) + + def redeem[B](fe: NonEmptyVector[E] => B, fa: A => B): Validated[Nothing, B] = + this match + case Valid(a) => Valid(fa(a)) + case Invalid(e) => Valid(fe(e)) + def flatMap[EE >: E, B](f: A => Validated[EE, B]): Validated[EE, B] = this match case Valid(a) => f(a) @@ -45,6 +56,13 @@ enum Validated[+E, +A]: this match case Valid(a) => Valid(a) case Invalid(e) => Invalid(e.map(f)) + + def isValid: Boolean = this match + case Valid(_) => true + case _ => false + + def isInvalid: Boolean = !isValid + end Validated extension [A](a: A) @@ -61,6 +79,10 @@ extension [E, A](e: Either[E, A]) def toValidatedResult: Validated.ValidatedResult[E, A] = e.fold(_.invalidResult, _.validResult) +extension [A](t: Try[A]) + def toValidated: Validated[Throwable, A] = + t.fold(_.invalid, _.valid) + extension [A](a: Option[A]) def toValidatedOrError[E](e: => E): Validated[E, A] = a.fold(e.invalid)(_.valid) @@ -72,11 +94,26 @@ extension [A](vec: Vector[A]) vec.foldLeft[Validated[E, Vector[B]]](Vector.empty[B].valid) { (acc, a) => acc.zipWith(f(a))(_ :+ _) } + + def sequenceV[E, R](using ev: A =:= Validated[E, R]): Validated[E, Vector[R]] = + vec.traverseV(v => ev(v)) + def traverseVR[E, B](f: A => Validated.ValidatedResult[E, B]): Validated.ValidatedResult[E, Vector[B]] = vec.foldLeft[Validated.ValidatedResult[E, Vector[B]]](Vector.empty[B].validResult) { (acc, a) => acc.zipWith(f(a))(_ :+ _) } +extension [A](vec: List[A]) + def traverseL[E, B](f: A => Validated[E, B]): Validated[E, List[B]] = + vec + .foldLeft[Validated[E, Vector[B]]](Vector.empty[B].valid) { (acc, a) => + acc.zipWith(f(a))(_ :+ _) + } + .map(_.toList) + + def sequenceL[E, R](using ev: A =:= Validated[E, R]): Validated[E, List[R]] = + vec.traverseL(v => ev(v)) + extension [E, A](vec: Vector[Validated[E, A]]) def sequenceV: Validated[E, Vector[A]] = vec.traverseV(identity) diff --git a/codegen/src/model/SemanticVersion.scala b/model/SemanticVersion.scala similarity index 100% rename from codegen/src/model/SemanticVersion.scala rename to model/SemanticVersion.scala diff --git a/codegen/src/model/SemanticVersion.test.scala b/model/SemanticVersion.test.scala similarity index 100% rename from codegen/src/model/SemanticVersion.test.scala rename to model/SemanticVersion.test.scala diff --git a/model/project.scala b/model/project.scala new file mode 100644 index 00000000..4015a810 --- /dev/null +++ b/model/project.scala @@ -0,0 +1,17 @@ +//> using scala 3.3.1 +//> using options -release:11 -deprecation -Werror -Wunused:all -Wvalue-discard -Wnonunit-statement + +//> using test.dep org.scalameta::munit::1.0.0 + +//> using exclude "resources/*" + +//> using publish.name "besom-model" +//> using publish.organization "org.virtuslab" +//> using publish.url "https://github.com/VirtusLab/besom" +//> using publish.vcs "github:VirtusLab/besom" +//> using publish.license "Apache-2.0" +//> using publish.repository "central" +//> using publish.developer "lbialy|Łukasz Biały|https://github.com/lbialy" +//> using publish.developer "prolativ|Michał Pałka|https://github.com/prolativ" +//> using publish.developer "KacperFKorban|Kacper Korban|https://github.com/KacperFKorban" +//> using publish.developer "pawelprazak|Paweł Prażak|https://github.com/pawelprazak"