From 9728bc66e2516124ed9499a0616cb180995a040c Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 24 Jan 2022 16:20:22 +0000 Subject: [PATCH 1/8] Migrate build to sbt-typelevel-ci-release --- .github/workflows/ci.yml | 210 +++++++++++++++++++++++++++------------ build.sbt | 169 ++++++++++++++----------------- project/plugins.sbt | 8 +- 3 files changed, 230 insertions(+), 157 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8103e44c..06aa40c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,11 +1,25 @@ +# This file was automatically generated by sbt-github-actions using the +# githubWorkflowGenerate task. You should add and commit this file to +# your git repository. It goes without saying that you shouldn't edit +# this file by hand! Instead, if you wish to make changes, you should +# change your sbt build configuration to revise the workflow description +# to meet your needs, then regenerate this file. + name: Continuous Integration on: pull_request: branches: ['**'] push: - branches: ['master'] - tags: ['v*'] + branches: ['**'] + tags: [v*] + +env: + PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + PGP_SECRET: ${{ secrets.PGP_SECRET }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: build: @@ -13,21 +27,34 @@ jobs: strategy: matrix: os: [ubuntu-latest] - scala: [2.12.15, 2.13.8] - java: [8] - distribution: [temurin] + scala: [2.13.8, 2.12.15] + java: [temurin@8] + project: [root-spark30, root-spark31, root-spark32] runs-on: ${{ matrix.os }} - env: - SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@v2 + - name: Checkout current branch (full) + uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: coursier/cache-action@v6 - - uses: actions/setup-java@v2 + + - name: Setup Java (temurin@8) + if: matrix.java == 'temurin@8' + uses: actions/setup-java@v2 with: - distribution: ${{ matrix.distribution }} - java-version: ${{ matrix.java }} + distribution: temurin + java-version: 8 + + - name: Cache sbt + uses: actions/cache@v2 + with: + path: | + ~/.sbt + ~/.ivy2/cache + ~/.coursier/cache/v1 + ~/.cache/coursier/v1 + ~/AppData/Local/Coursier/Cache/v1 + ~/Library/Caches/Coursier/v1 + key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - name: Setup Python uses: actions/setup-python@v2 @@ -37,83 +64,144 @@ jobs: - name: Setup codecov run: pip install codecov - - name: Test & Compute Coverage - run: sbt ++${{ matrix.scala }} coverage frameless-test + - name: Check that workflows are up to date + run: sbt '++${{ matrix.scala }}' 'project ${{ matrix.project }}' 'project /' githubWorkflowCheck + + - name: Test & Compute Coverage Spark + env: + SPARK_LOCAL_IP: localhost + run: sbt '++${{ matrix.scala }}' 'project ${{ matrix.project }}' coverage test test/coverageReport - - name: Test & Compute Coverage Spark 3.1.x - run: sbt ++${{ matrix.scala }} coverage frameless-test-spark31 + - name: Check binary compatibility + run: sbt '++${{ matrix.scala }}' 'project ${{ matrix.project }}' mimaReportBinaryIssues - - name: Test & Compute Coverage Spark 3.0.x - run: sbt ++${{ matrix.scala }} coverage frameless-test-spark30 + - name: Generate API documentation + run: sbt '++${{ matrix.scala }}' 'project ${{ matrix.project }}' doc - name: Upload Codecov Results run: codecov -F ${{ matrix.scala }} - - name: Binary compatibility check - run: sbt ++${{ matrix.scala }} frameless-mimaReportBinaryIssues + - name: Make target directories + run: mkdir -p refined-spark30/target refined/target ml-spark30/target target mdocs/target ml-spark31/target refined-spark31/target .spark31/target ml/target cats-spark31/target dataset-spark31/target dataset/target cats-spark30/target .spark30/target .spark32/target cats/target core/target dataset-spark30/target project/target + + - name: Compress target directories + run: tar cf targets.tar refined-spark30/target refined/target ml-spark30/target target mdocs/target ml-spark31/target refined-spark31/target .spark31/target ml/target cats-spark31/target dataset-spark31/target dataset/target cats-spark30/target .spark30/target .spark32/target cats/target core/target dataset-spark30/target project/target + + - name: Upload target directories + uses: actions/upload-artifact@v2 + with: + name: target-${{ matrix.os }}-${{ matrix.scala }}-${{ matrix.java }} + path: targets.tar publish: name: Publish Artifacts needs: [build] - if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v')) + if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master') strategy: matrix: os: [ubuntu-latest] - java: [8] - distribution: [temurin] + scala: [2.12.15] + java: [temurin@8] runs-on: ${{ matrix.os }} - env: - PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} - PGP_SECRET: ${{ secrets.PGP_SECRET }} - SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} - SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} - CI_SNAPSHOT_RELEASE: "frameless-publish" - CI_RELEASE: "frameless-publishSigned" - steps: - - uses: actions/checkout@v2 + - name: Checkout current branch (full) + uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: coursier/cache-action@v6 - - uses: actions/setup-java@v2 + + - name: Setup Java (temurin@8) + if: matrix.java == 'temurin@8' + uses: actions/setup-java@v2 with: - distribution: ${{ matrix.distribution }} - java-version: ${{ matrix.java }} + distribution: temurin + java-version: 8 - # Sequentially publish different artifacts for different Scala versions. - # Doing that in parallel may cause unclosed staging repositories drop. - - name: Publish Scala 2.12.15 artifacts - run: sbt ++$SCALA_VERSION ci-release - env: - SCALA_VERSION: 2.12.15 - if: ${{ env.SONATYPE_PASSWORD != '' && env.SONATYPE_USERNAME != '' }} + - name: Cache sbt + uses: actions/cache@v2 + with: + path: | + ~/.sbt + ~/.ivy2/cache + ~/.coursier/cache/v1 + ~/.cache/coursier/v1 + ~/AppData/Local/Coursier/Cache/v1 + ~/Library/Caches/Coursier/v1 + key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - name: Publish Scala 2.13.8 artifacts - run: sbt ++$SCALA_VERSION ci-release - env: - SCALA_VERSION: 2.13.8 - if: ${{ env.SONATYPE_PASSWORD != '' && env.SONATYPE_USERNAME != '' }} + - name: Download target directories (2.13.8) + uses: actions/download-artifact@v2 + with: + name: target-${{ matrix.os }}-2.13.8-${{ matrix.java }} + + - name: Inflate target directories (2.13.8) + run: | + tar xf targets.tar + rm targets.tar + + - name: Download target directories (2.12.15) + uses: actions/download-artifact@v2 + with: + name: target-${{ matrix.os }}-2.12.15-${{ matrix.java }} + + - name: Inflate target directories (2.12.15) + run: | + tar xf targets.tar + rm targets.tar + + - name: Import signing key + if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE == '' + run: echo $PGP_SECRET | base64 -d | gpg --import - docs: - name: Documentation + - name: Import signing key and strip passphrase + if: env.PGP_SECRET != '' && env.PGP_PASSPHRASE != '' + run: | + echo "$PGP_SECRET" | base64 -d > /tmp/signing-key.gpg + echo "$PGP_PASSPHRASE" | gpg --pinentry-mode loopback --passphrase-fd 0 --import /tmp/signing-key.gpg + (echo "$PGP_PASSPHRASE"; echo; echo) | gpg --command-fd 0 --pinentry-mode loopback --change-passphrase $(gpg --list-secret-keys --with-colons 2> /dev/null | grep '^sec:' | cut --delimiter ':' --fields 5 | tail -n 1) + + - name: Publish + run: sbt '++${{ matrix.scala }}' tlRelease + + site: + name: Generate Site strategy: matrix: os: [ubuntu-latest] scala: [2.12.15] - java: [8] - distribution: [temurin] + java: [temurin@8] runs-on: ${{ matrix.os }} - env: - SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@v2 + - name: Checkout current branch (full) + uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: coursier/cache-action@v6 - - uses: actions/setup-java@v2 + + - name: Setup Java (temurin@8) + if: matrix.java == 'temurin@8' + uses: actions/setup-java@v2 with: - distribution: ${{ matrix.distribution }} - java-version: ${{ matrix.java }} + distribution: temurin + java-version: 8 - - name: Documentation - run: sbt ++${{ matrix.scala }} doc mdoc + - name: Cache sbt + uses: actions/cache@v2 + with: + path: | + ~/.sbt + ~/.ivy2/cache + ~/.coursier/cache/v1 + ~/.cache/coursier/v1 + ~/AppData/Local/Coursier/Cache/v1 + ~/Library/Caches/Coursier/v1 + key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} + + - name: Generate site + run: sbt '++${{ matrix.scala }}' tlSite + + - name: Publish site + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + uses: peaceiris/actions-gh-pages@v3.8.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: mdocs/target/docs/site + publish_branch: gh-pages diff --git a/build.sbt b/build.sbt index 7614bc95..8882bb74 100644 --- a/build.sbt +++ b/build.sbt @@ -13,66 +13,33 @@ val refinedVersion = "0.9.28" val Scala212 = "2.12.15" val Scala213 = "2.13.8" -val previousVersion = "0.10.1" +ThisBuild / tlBaseVersion := "0.11" -/** A list of projects that can be safely compiled across Scala versions. */ -val projectsCrossVersion = "core" :: "cats" :: "dataset" :: "refined" :: "ml" :: Nil -val projectsSpark31 = projectsCrossVersion.head :: projectsCrossVersion.tail.map(_ + "-spark31") -val projectsSpark30 = projectsCrossVersion.head :: projectsCrossVersion.tail.map(_ + "-spark30") +ThisBuild / crossScalaVersions := Seq(Scala213, Scala212) +ThisBuild / scalaVersion := Scala212 -ThisBuild / versionScheme := Some("semver-spec") +lazy val root = project.in(file(".")) + .enablePlugins(NoPublishPlugin) + .aggregate(`root-spark32`, `root-spark31`, `root-spark30`, docs) -ThisBuild / crossScalaVersions := Seq(Scala212, Scala213) -ThisBuild / scalaVersion := Scala212 +lazy val `root-spark32` = project + .in(file(".spark32")) + .enablePlugins(NoPublishPlugin) + .aggregate(core, cats, dataset, refined, ml) -ThisBuild / mimaFailOnNoPrevious := false - -lazy val root = Project("frameless", file("." + "frameless")).in(file(".")) - .aggregate( - core, - cats, - `cats-spark31`, - `cats-spark30`, - dataset, - `dataset-spark31`, - `dataset-spark30`, - refined, - `refined-spark31`, - `refined-spark30`, - ml, - `ml-spark31`, - `ml-spark30`, - docs - ) - .settings(framelessSettings) - .settings(noPublishSettings) - .settings(mimaPreviousArtifacts := Set.empty) - .settings( - /** Not all Spark versions support Scala 2.13. These commands are launched for the supported subset of projects only. */ - commands ++= Seq( - // run tests separately for different Spark versions to reduce pressure on CI - command("frameless-test")(projectsCrossVersion.map(_ + "/test") ::: projectsCrossVersion.map(_ + "/test/coverageReport")).value, - command212("frameless-test-spark31")(projectsSpark31.map(_ + "/test") ::: projectsSpark31.map(_ + "/test/coverageReport")).value, - command212("frameless-test-spark30")(projectsSpark30.map(_ + "/test") ::: projectsSpark30.map(_ + "/test/coverageReport")).value, - commandCrossVersion("frameless-mimaReportBinaryIssues")(projectsCrossVersion.map(_ + "/mimaReportBinaryIssues"), "mimaReportBinaryIssues" :: Nil).value, - commandCrossVersion("frameless-publish")(projectsCrossVersion.map(_ + "/publish"), "publish" :: Nil).value, - commandCrossVersion("frameless-publishSigned")(projectsCrossVersion.map(_ + "/publishSigned"), "publishSigned" :: Nil).value, - ) - ) +lazy val `root-spark31` = project + .in(file(".spark31")) + .enablePlugins(NoPublishPlugin) + .aggregate(core, `cats-spark31`, `dataset-spark31`, `refined-spark31`, `ml-spark31`) -def command(name: String)(commands: List[String]) = commandCrossVersion(name)(commands, commands) -def command212(name: String)(commands212: List[String]) = commandCrossVersion(name)(Nil, commands212) -def commandCrossVersion(name: String)(commands213: List[String], commands212: List[String]) = Def.setting { Command.command(name) { currentState => - CrossVersion.partialVersion(scalaVersion.value) match { - case Some((2, 13)) => commands213 ::: currentState - case _ => commands212 ::: currentState - } -} } +lazy val `root-spark30` = project + .in(file(".spark30")) + .enablePlugins(NoPublishPlugin) + .aggregate(core, `cats-spark30`, `dataset-spark30`, `refined-spark30`, `ml-spark30`) lazy val core = project .settings(name := "frameless-core") .settings(framelessSettings) - .settings(publishSettings) lazy val cats = project .settings(name := "frameless-cats") @@ -166,16 +133,14 @@ lazy val `ml-spark30` = project lazy val docs = project .in(file("mdocs")) .settings(framelessSettings) - .settings(noPublishSettings) .settings(scalacOptions --= Seq("-Xfatal-warnings", "-Ywarn-unused-import")) - .enablePlugins(MdocPlugin) + .enablePlugins(TypelevelSitePlugin) .settings(sparkDependencies(sparkVersion, Compile)) .settings(sparkMlDependencies(sparkVersion, Compile)) .settings( addCompilerPlugin("org.typelevel" % "kind-projector" % "0.13.2" cross CrossVersion.full), scalacOptions += "-Ydelambdafy:inline" ) - .settings(mimaPreviousArtifacts := Set()) .dependsOn(dataset, cats, ml) def sparkDependencies(sparkVersion: String, scope: Configuration = Provided) = Seq( @@ -188,7 +153,7 @@ def sparkDependencies(sparkVersion: String, scope: Configuration = Provided) = S def sparkMlDependencies(sparkVersion: String, scope: Configuration = Provided) = Seq(libraryDependencies += "org.apache.spark" %% "spark-mllib" % sparkVersion % scope) -lazy val catsSettings = framelessSettings ++ publishSettings ++ Seq( +lazy val catsSettings = framelessSettings ++ Seq( addCompilerPlugin("org.typelevel" % "kind-projector" % "0.13.2" cross CrossVersion.full), libraryDependencies ++= Seq( "org.typelevel" %% "cats-core" % catsCoreVersion, @@ -198,7 +163,7 @@ lazy val catsSettings = framelessSettings ++ publishSettings ++ Seq( ) ) -lazy val datasetSettings = framelessSettings ++ framelessTypedDatasetREPL ++ publishSettings ++ Seq( +lazy val datasetSettings = framelessSettings ++ framelessTypedDatasetREPL ++ Seq( mimaBinaryIssueFilters ++= { import com.typesafe.tools.mima.core._ @@ -219,12 +184,12 @@ lazy val datasetSettings = framelessSettings ++ framelessTypedDatasetREPL ++ pub } ) -lazy val refinedSettings = framelessSettings ++ framelessTypedDatasetREPL ++ publishSettings ++ Seq( +lazy val refinedSettings = framelessSettings ++ framelessTypedDatasetREPL ++ Seq( mimaPreviousArtifacts := Set.empty, libraryDependencies += "eu.timepit" %% "refined" % refinedVersion ) -lazy val mlSettings = framelessSettings ++ framelessTypedDatasetREPL ++ publishSettings +lazy val mlSettings = framelessSettings ++ framelessTypedDatasetREPL lazy val scalac212Options = Seq( "-Xlint:-missing-interpolator,-unused,_", @@ -272,10 +237,7 @@ lazy val scalacOptionSettings = Def.setting { } lazy val framelessSettings = Seq( - organization := "org.typelevel", scalacOptions ++= scalacOptionSettings.value, - licenses += ("Apache-2.0", url("http://opensource.org/licenses/Apache-2.0")), - homepage := Some(url("https://typelevel.org/frameless")), Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-oDF"), libraryDependencies ++= Seq( "com.chuusai" %% "shapeless" % shapeless, @@ -286,12 +248,11 @@ lazy val framelessSettings = Seq( Test / javaOptions ++= Seq("-Xmx1G", "-ea"), Test / fork := true, Test / parallelExecution := false, - mimaPreviousArtifacts := Def.setting { - CrossVersion.partialVersion(scalaVersion.value) match { - // TODO: remove once 2.13 artifacts published - case Some((2, 13)) => Set.empty[ModuleID] - case _ => Set("org.typelevel" %% name.value % previousVersion) - } }.value + mimaPreviousArtifacts := { + if (scalaBinaryVersion.value == "2.13") + Set.empty + else mimaPreviousArtifacts.value + }, ) ++ consoleSettings lazy val consoleSettings = Seq( @@ -326,32 +287,56 @@ lazy val framelessTypedDatasetREPL = Seq( """.stripMargin ) -lazy val publishSettings = Seq( - Test / publishArtifact := false, - ThisBuild / developers := List( - "OlivierBlanvillain" -> "Olivier Blanvillain", - "adelbertc" -> "Adelbert Chang", - "imarios" -> "Marios Iliofotou", - "kanterov" -> "Gleb Kanterov", - "non" -> "Erik Osheim", - "jeremyrsmith" -> "Jeremy Smith", - "cchantep" -> "Cédric Chantepie", - "pomadchin" -> "Grigory Pomadchin" - ).map { case (username, fullName) => - Developer(username, fullName, s"@$username", url(s"https://github.com/$username")) - } -) +ThisBuild / organization := "org.typelevel" +ThisBuild / licenses := List("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0")) +ThisBuild / developers := List( + "OlivierBlanvillain" -> "Olivier Blanvillain", + "adelbertc" -> "Adelbert Chang", + "imarios" -> "Marios Iliofotou", + "kanterov" -> "Gleb Kanterov", + "non" -> "Erik Osheim", + "jeremyrsmith" -> "Jeremy Smith", + "cchantep" -> "Cédric Chantepie", + "pomadchin" -> "Grigory Pomadchin" +).map { case (username, fullName) => + tlGitHubDev(username, fullName) +} + +ThisBuild / tlCiReleaseBranches := Seq("master") -lazy val noPublishSettings = Seq( - publish := (()), - publishLocal := (()), - publishArtifact := false +ThisBuild / githubWorkflowBuildPreamble ++= Seq( + WorkflowStep.Use( + UseRef.Public("actions", "setup-python", "v2"), + name = Some("Setup Python"), + params = Map( + "python-version" -> "3.x" + ) + ), + WorkflowStep.Run( + List("pip install codecov"), + name = Some("Setup codecov") + ) ) -lazy val copyReadme = taskKey[Unit]("copy for website generation") -lazy val copyReadmeImpl = Def.task { - val from = baseDirectory.value / "README.md" - val to = baseDirectory.value / "docs" / "src" / "main" / "tut" / "README.md" - sbt.IO.copy(List((from, to)), overwrite = true, preserveLastModified = true, preserveExecutable = true) +ThisBuild / githubWorkflowBuildMatrixAdditions += + "project" -> List("root-spark30", "root-spark31", "root-spark32") +ThisBuild / githubWorkflowBuildSbtStepPreamble += s"project $${{ matrix.project }}" + +ThisBuild / githubWorkflowBuild ~= { steps => + steps.map { // replace the test step + case _ @ WorkflowStep.Sbt(List("test"), _, _, _, _, _) => + WorkflowStep.Sbt( + List("coverage", "test", "test/coverageReport"), + name = Some("Test & Compute Coverage Spark"), + env = Map("SPARK_LOCAL_IP" -> "localhost") + ) + case step => step + } } -copyReadme := copyReadmeImpl.value + +ThisBuild / githubWorkflowBuildPostamble ++= Seq( + WorkflowStep.Run( + List(s"codecov -F $${{ matrix.scala }}"), + name = Some("Upload Codecov Results") + ) +) diff --git a/project/plugins.sbt b/project/plugins.sbt index 279db366..2b1a36f6 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ -addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.10") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.3") -addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.2.24") -addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "1.0.1") +val sbtTypelevelVersion = "0.4.1" +addSbtPlugin("org.typelevel" % "sbt-typelevel-ci-release" % sbtTypelevelVersion) +addSbtPlugin("org.typelevel" % "sbt-typelevel-site" % sbtTypelevelVersion) +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.3") From 8d2581b4c2bfed21fb45db3fa9a12235ad91c498 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 24 Jan 2022 16:21:43 +0000 Subject: [PATCH 2/8] Adjust mima config --- build.sbt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.sbt b/build.sbt index 8882bb74..332e54a3 100644 --- a/build.sbt +++ b/build.sbt @@ -248,6 +248,9 @@ lazy val framelessSettings = Seq( Test / javaOptions ++= Seq("-Xmx1G", "-ea"), Test / fork := true, Test / parallelExecution := false, + mimaPreviousArtifacts ~= { + _.filterNot(_.revision == "0.11.0") // didn't release properly + }, mimaPreviousArtifacts := { if (scalaBinaryVersion.value == "2.13") Set.empty From 52c4cc53c474404cdf3327b4c1de27c21e859e7e Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 24 Jan 2022 16:27:04 +0000 Subject: [PATCH 3/8] Migrate site to laika --- docs/README.md | 1 + docs/SUMMARY.md | 9 --------- docs/directory.conf | 12 ++++++++++++ 3 files changed, 13 insertions(+), 9 deletions(-) create mode 120000 docs/README.md delete mode 100644 docs/SUMMARY.md create mode 100644 docs/directory.conf diff --git a/docs/README.md b/docs/README.md new file mode 120000 index 00000000..32d46ee8 --- /dev/null +++ b/docs/README.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md deleted file mode 100644 index 4b43c5a4..00000000 --- a/docs/SUMMARY.md +++ /dev/null @@ -1,9 +0,0 @@ -- [TypedDataset: Feature Overview](FeatureOverview.md) -- [Comparing TypedDatasets with Spark's Datasets](TypedDatasetVsSparkDataset.md) -- [Working with CSV and Parquet](WorkingWithCsvParquetJson.md) -- [Typed Encoders in Frameless](TypedEncoder.md) -- [Injection: Creating Custom Encoders](Injection.md) -- [Job\[A\]](Job.md) -- [Using Cats with RDDs](Cats.md) -- [Using Spark ML with TypedDataset](TypedML.md) -- [Proof of Concept: TypedDataFrame](TypedDataFrame.md) diff --git a/docs/directory.conf b/docs/directory.conf new file mode 100644 index 00000000..cee676f1 --- /dev/null +++ b/docs/directory.conf @@ -0,0 +1,12 @@ +laika.title = frameless +laika.navigationOrder = [ + README.md + FeatureOverview.md + TypedDatasetVsSparkDataset.md + WorkingWithCsvParquetJson.md + Injection.md + Job.md + Cats.md + TypedML.md + TypedDataFrame.md +] \ No newline at end of file From 5f0c06edc55a02f459f0010caf0d3eaf27e92bf3 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 24 Jan 2022 16:29:10 +0000 Subject: [PATCH 4/8] Make SPARK_LOCAL_IP global env --- .github/workflows/ci.yml | 3 +-- build.sbt | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06aa40c3..0232ef9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,7 @@ env: SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} PGP_SECRET: ${{ secrets.PGP_SECRET }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SPARK_LOCAL_IP: localhost jobs: build: @@ -68,8 +69,6 @@ jobs: run: sbt '++${{ matrix.scala }}' 'project ${{ matrix.project }}' 'project /' githubWorkflowCheck - name: Test & Compute Coverage Spark - env: - SPARK_LOCAL_IP: localhost run: sbt '++${{ matrix.scala }}' 'project ${{ matrix.project }}' coverage test test/coverageReport - name: Check binary compatibility diff --git a/build.sbt b/build.sbt index 332e54a3..e26e9bfb 100644 --- a/build.sbt +++ b/build.sbt @@ -307,6 +307,7 @@ ThisBuild / developers := List( ThisBuild / tlCiReleaseBranches := Seq("master") +ThisBuild / githubWorkflowEnv += "SPARK_LOCAL_IP" -> "localhost" ThisBuild / githubWorkflowBuildPreamble ++= Seq( WorkflowStep.Use( UseRef.Public("actions", "setup-python", "v2"), @@ -330,8 +331,7 @@ ThisBuild / githubWorkflowBuild ~= { steps => case _ @ WorkflowStep.Sbt(List("test"), _, _, _, _, _) => WorkflowStep.Sbt( List("coverage", "test", "test/coverageReport"), - name = Some("Test & Compute Coverage Spark"), - env = Map("SPARK_LOCAL_IP" -> "localhost") + name = Some("Test & Compute Coverage Spark") ) case step => step } From dc2ccffd74d34d6b6845745bc5e8feae9ba32b45 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 24 Jan 2022 16:34:58 +0000 Subject: [PATCH 5/8] Add matrix exclude --- .github/workflows/ci.yml | 3 +++ build.sbt | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0232ef9c..8f78b884 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,9 @@ jobs: scala: [2.13.8, 2.12.15] java: [temurin@8] project: [root-spark30, root-spark31, root-spark32] + exclude: + - scala: 2.13.8 + project: root-spark30 runs-on: ${{ matrix.os }} steps: - name: Checkout current branch (full) diff --git a/build.sbt b/build.sbt index e26e9bfb..10218db2 100644 --- a/build.sbt +++ b/build.sbt @@ -325,6 +325,8 @@ ThisBuild / githubWorkflowBuildPreamble ++= Seq( ThisBuild / githubWorkflowBuildMatrixAdditions += "project" -> List("root-spark30", "root-spark31", "root-spark32") ThisBuild / githubWorkflowBuildSbtStepPreamble += s"project $${{ matrix.project }}" +ThisBuild / githubWorkflowBuildMatrixExclusions += + MatrixExclude(Map("scala" -> Scala213, "project" -> "root-spark30")) ThisBuild / githubWorkflowBuild ~= { steps => steps.map { // replace the test step From 4403b29210609d72a897ea1acc0ad806e8492d5b Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 24 Jan 2022 16:41:55 +0000 Subject: [PATCH 6/8] Another exclusion --- .github/workflows/ci.yml | 2 ++ build.sbt | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f78b884..6fe479e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,8 @@ jobs: exclude: - scala: 2.13.8 project: root-spark30 + - scala: 2.13.8 + project: root-spark31 runs-on: ${{ matrix.os }} steps: - name: Checkout current branch (full) diff --git a/build.sbt b/build.sbt index 10218db2..096c4e64 100644 --- a/build.sbt +++ b/build.sbt @@ -322,11 +322,13 @@ ThisBuild / githubWorkflowBuildPreamble ++= Seq( ) ) +val roots = List("root-spark30", "root-spark31", "root-spark32") ThisBuild / githubWorkflowBuildMatrixAdditions += - "project" -> List("root-spark30", "root-spark31", "root-spark32") + "project" -> roots ThisBuild / githubWorkflowBuildSbtStepPreamble += s"project $${{ matrix.project }}" -ThisBuild / githubWorkflowBuildMatrixExclusions += - MatrixExclude(Map("scala" -> Scala213, "project" -> "root-spark30")) +ThisBuild / githubWorkflowBuildMatrixExclusions ++= roots.init.map { project => + MatrixExclude(Map("scala" -> Scala213, "project" -> project)) +} ThisBuild / githubWorkflowBuild ~= { steps => steps.map { // replace the test step From b00c76b0e8b43be0497e15cfcc787b88450bc789 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 24 Jan 2022 16:53:40 +0000 Subject: [PATCH 7/8] Tweak settings --- build.sbt | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index 096c4e64..cba67293 100644 --- a/build.sbt +++ b/build.sbt @@ -50,14 +50,14 @@ lazy val `cats-spark31` = project .settings(name := "frameless-cats-spark31") .settings(sourceDirectory := (cats / sourceDirectory).value) .settings(catsSettings) - .settings(mimaPreviousArtifacts := Set.empty) + .settings(spark31Settings) .dependsOn(`dataset-spark31` % "test->test;compile->compile;provided->provided") lazy val `cats-spark30` = project .settings(name := "frameless-cats-spark30") .settings(sourceDirectory := (cats / sourceDirectory).value) .settings(catsSettings) - .settings(mimaPreviousArtifacts := Set.empty) + .settings(spark30Settings) .dependsOn(`dataset-spark30` % "test->test;compile->compile;provided->provided") lazy val dataset = project @@ -71,7 +71,7 @@ lazy val `dataset-spark31` = project .settings(sourceDirectory := (dataset / sourceDirectory).value) .settings(datasetSettings) .settings(sparkDependencies(spark31Version)) - .settings(mimaPreviousArtifacts := Set.empty) + .settings(spark31Settings) .dependsOn(core % "test->test;compile->compile") lazy val `dataset-spark30` = project @@ -79,7 +79,7 @@ lazy val `dataset-spark30` = project .settings(sourceDirectory := (dataset / sourceDirectory).value) .settings(datasetSettings) .settings(sparkDependencies(spark30Version)) - .settings(mimaPreviousArtifacts := Set.empty) + .settings(spark30Settings) .dependsOn(core % "test->test;compile->compile") lazy val refined = project @@ -91,12 +91,14 @@ lazy val `refined-spark31` = project .settings(name := "frameless-refined-spark31") .settings(sourceDirectory := (refined / sourceDirectory).value) .settings(refinedSettings) + .settings(spark31Settings) .dependsOn(`dataset-spark31` % "test->test;compile->compile;provided->provided") lazy val `refined-spark30` = project .settings(name := "frameless-refined-spark30") .settings(sourceDirectory := (refined / sourceDirectory).value) .settings(refinedSettings) + .settings(spark30Settings) .dependsOn(`dataset-spark30` % "test->test;compile->compile;provided->provided") lazy val ml = project @@ -113,7 +115,7 @@ lazy val `ml-spark31` = project .settings(sourceDirectory := (ml / sourceDirectory).value) .settings(mlSettings) .settings(sparkMlDependencies(spark31Version)) - .settings(mimaPreviousArtifacts := Set.empty) + .settings(spark31Settings) .dependsOn( core % "test->test;compile->compile", `dataset-spark31` % "test->test;compile->compile;provided->provided" @@ -124,7 +126,7 @@ lazy val `ml-spark30` = project .settings(sourceDirectory := (ml / sourceDirectory).value) .settings(mlSettings) .settings(sparkMlDependencies(spark30Version)) - .settings(mimaPreviousArtifacts := Set.empty) + .settings(spark30Settings) .dependsOn( core % "test->test;compile->compile", `dataset-spark30` % "test->test;compile->compile;provided->provided" @@ -258,6 +260,14 @@ lazy val framelessSettings = Seq( }, ) ++ consoleSettings +lazy val spark30Settings = Seq( + crossScalaVersions := Seq(Scala212) +) + +lazy val spark31Settings = Seq( + crossScalaVersions := Seq(Scala212) +) + lazy val consoleSettings = Seq( Compile / console / scalacOptions ~= {_.filterNot("-Ywarn-unused-import" == _)}, Test / console / scalacOptions := (Compile / console / scalacOptions).value From 2d875f71100f2d2706a1965fa986fa33a5462455 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 24 Jan 2022 17:43:53 +0000 Subject: [PATCH 8/8] Minor fixes to workflow --- .github/workflows/ci.yml | 4 ++-- build.sbt | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fe479e7..7acaf76e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: - name: Check that workflows are up to date run: sbt '++${{ matrix.scala }}' 'project ${{ matrix.project }}' 'project /' githubWorkflowCheck - - name: Test & Compute Coverage Spark + - name: Test & Compute Coverage run: sbt '++${{ matrix.scala }}' 'project ${{ matrix.project }}' coverage test test/coverageReport - name: Check binary compatibility @@ -203,7 +203,7 @@ jobs: run: sbt '++${{ matrix.scala }}' tlSite - name: Publish site - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/master' uses: peaceiris/actions-gh-pages@v3.8.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/build.sbt b/build.sbt index cba67293..557f8035 100644 --- a/build.sbt +++ b/build.sbt @@ -316,6 +316,7 @@ ThisBuild / developers := List( } ThisBuild / tlCiReleaseBranches := Seq("master") +ThisBuild / tlSitePublishBranch := Some("master") ThisBuild / githubWorkflowEnv += "SPARK_LOCAL_IP" -> "localhost" ThisBuild / githubWorkflowBuildPreamble ++= Seq( @@ -345,7 +346,7 @@ ThisBuild / githubWorkflowBuild ~= { steps => case _ @ WorkflowStep.Sbt(List("test"), _, _, _, _, _) => WorkflowStep.Sbt( List("coverage", "test", "test/coverageReport"), - name = Some("Test & Compute Coverage Spark") + name = Some("Test & Compute Coverage") ) case step => step }