From 40cf9b42ff521f13380f80169e874f086c9c8b80 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 28 May 2024 13:19:49 +0200 Subject: [PATCH 1/4] Update configs --- coordinator/configs/filtered-projects.txt | 5 ++- coordinator/configs/projects-config.conf | 7 +++- .../shared/CommunityBuildCore.scala | 36 +++++++++++++++---- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/coordinator/configs/filtered-projects.txt b/coordinator/configs/filtered-projects.txt index e34bacea..e73c85d4 100644 --- a/coordinator/configs/filtered-projects.txt +++ b/coordinator/configs/filtered-projects.txt @@ -258,4 +258,7 @@ bot4s:zmatrix:.* ## Requires MacOS JDK ### Uses com.apple.eawt.Application which is a part of MacOS JDK distribution -dicklieber:wa9nnn-util:.* \ No newline at end of file +dicklieber:wa9nnn-util:.* + +## Requires JDK 22, no Scala source code +armanbilge:cheshire:.* diff --git a/coordinator/configs/projects-config.conf b/coordinator/configs/projects-config.conf index 46fcce62..2c9025c7 100644 --- a/coordinator/configs/projects-config.conf +++ b/coordinator/configs/projects-config.conf @@ -29,6 +29,7 @@ ajozwik_protoquill-generic { "removeScalacOptionsStartingWith -P:wartremover" ] } +ajozwik_pekko-smtp-server.tests=compile-only akka_akka { // Based on Scala3 managed community build @@ -218,6 +219,7 @@ com-lihaoyi_unroll { replace-with = "override def init(options: List[String]): List[PluginPhase]" }] } +com-lihaoyi_sourcecode.tests = compile-only coursier_class-path-util.tests = compile-only // no tests, scala-cli dacr_jassh.tests = compile-only @@ -362,6 +364,7 @@ geirolz_fly4s.tests = compile-only // flaky geirolz_scope.tests = compile-only geirolz_secret.source-version=3.4 // implicts gemini-hlsw_gsp-graphql.tests = compile-only // uses Docker +gemini-hlsw_lucuma-itc.java.version = 17 getkyo_kyo { source-version = 3.4 tests = compile-only @@ -511,7 +514,7 @@ jodersky_scala-argparse{ jodersky_scalapb-ujson{ tests = disabled // starts failing process (proto gen?) } -jodersky_redic.tests = compile-only +jodersky_redicl.tests = compile-only joernio_joern{ projects.overrides{ x2cpg.tests = compile-only // Requires mvn binary @@ -970,6 +973,8 @@ rssh_cps-async-connect { java.version = 21 } +quafadas_live-server-scala-cli-js.tests = compile-only + t2v_holidays.tests = compile-only // invalid classpath at runtime taig_backmail.source-version=3.4 taig_scala-linguist.projects.exclude=["io.taig%scala-linguist-graalvm-ruby"] // needs GraalVM diff --git a/project-builder/shared/CommunityBuildCore.scala b/project-builder/shared/CommunityBuildCore.scala index 99b4ba8e..da8d4abf 100644 --- a/project-builder/shared/CommunityBuildCore.scala +++ b/project-builder/shared/CommunityBuildCore.scala @@ -84,7 +84,7 @@ object Scala3CommunityBuild { sourceVersion.map(v => s""""sourceVersion": "$v"""") ).flatten val optionalsString = - if (optionals.isEmpty) "" + if (optionals.isEmpty) "" else optionals.mkString(", ", ", ", "") s"""{$commonJsonFields, "warnings": ${warnings}, "errors": ${errors}$optionalsString}""" } @@ -334,20 +334,37 @@ object Scala3CommunityBuild { } private val multiStringSettings = Seq( - "-scalajs-mapSourceURI", "-language", - "-Wconf","-Xmacro-settings","-Yimports","-Yfrom-tasty-ignore-list" + "-scalajs-mapSourceURI", + "-language", + "-Wconf", + "-Xmacro-settings", + "-Yimports", + "-Yfrom-tasty-ignore-list" ) - object logOnce extends Function[String, Unit]{ + object logOnce extends Function[String, Unit] { val logged = collection.mutable.Set.empty[String] - override def apply(v :String): Unit = if(logged.add(v)) println(s"OpenCB::$v") + override def apply(v: String): Unit = if (logged.add(v)) println(s"OpenCB::$v") } + // Invalid scalac options but not checked before 3.5.0-RC + val invalidLanguageOptions = 0.until(5).map(v => s"3.$v") ++ Seq( + "_", + "experimental", + "future", + "scala3", + "Scala3", + "scala2", + "Scala2", + "scala2Compat", + "Scala2Compat" + ) + val ignoredScalacOptions = invalidLanguageOptions.map(v => s"-language:$v") def mapScalacOptions( current: Seq[String], append: Seq[String], remove: Seq[String] ): Seq[String] = { - val (removeMatchSettings, removeSettings) = remove.partition { _.startsWith("MATCH:") } + val (removeMatchSettings, removeSettings) = (remove ++ ignoredScalacOptions).partition { _.startsWith("MATCH:") } val matchPatterns = removeMatchSettings.map(_.stripPrefix("MATCH:")) val (required, standardAppendSettings) = append.partition(_.startsWith("REQUIRE:")) @@ -401,6 +418,11 @@ object Scala3CommunityBuild { .apply(setting) } current + .flatMap { + case opt if opt.startsWith("-language:") => + opt.stripPrefix("-language:").split(',').map(v => s"-language:$v") + case opt => List(opt) + } .filterNot { s => def isMatching(reason: String, found: Option[String]): Boolean = found match { case Some(matched) => @@ -417,7 +439,7 @@ object Scala3CommunityBuild { case class LibraryDependency(organization: String, artifact: String, version: String, crossScalaVersion: Boolean){ override def toString(): String = { - val crossVersion = if(crossScalaVersion) "%%" else "%" + val crossVersion = if (crossScalaVersion) "%%" else "%" s"$organization $crossVersion $artifact % $version" } } From 8ba31fad8e6516dfac3805f075ba1540c098943f Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 28 May 2024 13:24:49 +0200 Subject: [PATCH 2/4] Don't execute runtime tests, make compile-only the default setting --- coordinator/configs/projects-config.conf | 229 ------------------ .../shared/CommunityBuildCore.scala | 2 +- 2 files changed, 1 insertion(+), 230 deletions(-) diff --git a/coordinator/configs/projects-config.conf b/coordinator/configs/projects-config.conf index 2c9025c7..a1210488 100644 --- a/coordinator/configs/projects-config.conf +++ b/coordinator/configs/projects-config.conf @@ -1,7 +1,5 @@ # Content of this file is expected to be moved into actual projects in the future -47degrees_sbt-energymonitor.tests=compile-only 7mind_izumi { - tests=compile-only source-patches = [ { path = "project/Deps.sc" @@ -29,33 +27,23 @@ ajozwik_protoquill-generic { "removeScalacOptionsStartingWith -P:wartremover" ] } -ajozwik_pekko-smtp-server.tests=compile-only akka_akka { // Based on Scala3 managed community build - tests = compile-only sbt.commands = ["disableFatalWarnings"] } akka_akka-httptests = compile-only -akka_akka-persistence-cassandra.tests = compile-only -akka_alpakka.tests = compile-only -akka_akka-diagnostics.tests = compile-only -akka_akka-management.tests = compile-only -akka_akka-persistence-r2dbc.tests = compile-only alejandrohdezma_http4s-munit { - tests = compile-only sbt.commands = [ // Does not compile with any version of Scala 3 """set `http4s-munit`/Test/unmanagedSources/excludeFilter ~= {_ || "Http4sTestHttpRoutesSuiteSuite.scala" || "Http4sTestAuthedRoutesSuiteSuite.scala" }""" ] } -alexarchambault_case-app.tests = compile-only almond-sh_almond{ projects.exclude = ["sh.almond%launcher"] # next RC only, can't resolve } apache_pekko { java.version = 8 - tests = compile-only } apache_pekko-grpc{ java.version = 8 @@ -65,7 +53,6 @@ apache_pekko-grpc{ ] } apache_pekko-http { - tests = compile-only java.version = 8 projects.exclude = [ # Scalafix supports only Scala2 @@ -77,15 +64,8 @@ apache_pekko-http { } apache_pekko-connectors{ java.version = 8 // for scaladoc - tests = compile-only } -apache_pekko-connectors-kafka.tests = compile-only -apache_pekko-persistence-cassandra.tests = compile-only -apache_pekko-persistence-dynamodb.tests = compile-only -apache_pekko-persistence-r2dbc.tests = compile-only -apimorphism_telegramium.tests = compile-only appliedtopology_tda4j.source-version = 3.4 // implicits order -armanbilge_calico.tests = compile-only armanbilge_litter.sbt.commands = ["set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat"] armanbilge_feral.projects.exclude = [ // Not present in the repo @@ -97,7 +77,6 @@ armanbilge_feral.projects.exclude = [ armanbilge_gcp4s { // Needs reordeding of given instances (since 3.4), fails to compile under -source:3.4 sbt.commands = ["""set every excludeFilter ~= { _ || "EndToEndSuite.scala" }"""] - tests = compile-only } armanbilge_van-cats.source-version=3.4 // override -source:future assist-iot-sripas_scala-mqtt-wrapper { @@ -136,7 +115,6 @@ beangle_doc { java.version=17 sbt.commands = ["styleFormat"] // Was failing due to missing license header in v0.1.7 } -bilal-fazlani_akka-http-jwt-auth.tests = compile-only bilal-fazlani_circe-json-schema.tests = compile-only // Missing test files at runtime bitlap_rolls { source-patches = [{ @@ -191,10 +169,6 @@ cheleb_zio-pravega { "removeScalacOptionsStartingWith -P:wartremover" ] } -clevercloud_pulsar4s.tests = compile-only -clockworkconsulting_peregrin.tests = compile-only -clulab_fatdynet.tests = compile-only -clulab_processors.tests = compile-only coralogix_zio-k8s{ java.version = 17 projects.overrides { @@ -203,14 +177,11 @@ coralogix_zio-k8s{ } } cognitedata_cognite-sdk-scala { - tests = compile-only sbt.commands = [ "excludeLibraryDependency org.wartremover:wartremover_{scalaVersion}", "removeScalacOptionsStartingWith -P:wartremover" ] } -com-lihaoyi_requests-scala.tests = compile-only -com-lihaoyi_scalatags.tests = compile-only com-lihaoyi_fastparse.projects.exclude = ["com.lihaoyi%pythonparse"] // Needs python com-lihaoyi_unroll { source-patches = [{ @@ -219,33 +190,24 @@ com-lihaoyi_unroll { replace-with = "override def init(options: List[String]): List[PluginPhase]" }] } -com-lihaoyi_sourcecode.tests = compile-only coursier_class-path-util.tests = compile-only // no tests, scala-cli -dacr_jassh.tests = compile-only dacr_zio-lmdb.java.version = 17 -darrenjw_scala-glm.tests = compile-only danslapman_morphling{ projects.exclude = [morphling-circe] // Needs update due to source compatible change } dapperware_dappermongo.tests = compile-only // test containers -datastrophic-games_dice.tests = compile-only davenverse_fuuid{ sbt.commands = [ // Disable intergration tests using test containers """set doobie/Test/unmanagedSources/excludeFilter ~= {_ || "PostgresInstanceSpec.scala" || "PostgresTraversalSpec.scala"} """ ] } -davenverse_rediculous.tests = compile-only -davenverse_rediculous-concurrent.tests = compile-only -davenverse_whale-tail.tests = compile-only -delimobil_cabbit.tests = compile-only delimobil_fs2-hbase.tests = disabled // Does not compile with any Scala version dfianthdl_dfiant.source-version = 3.4 // implicits ordering disneystreaming_smithy4s { projects.overrides { // Infinite loop in OperationSpec - smithy4s-dynamic.tests = compile-only } } disneystreaming_smithy-translate { @@ -258,8 +220,6 @@ disneystreaming_smithy-translate { disneystreaming_weaver-test { sbt.options=["-Dcommunitybuild.dualVersion=minor:+1"] } -dobrynya_zio-jms.tests = compile-only -dream11_zio-http.tests = compile-only earogov_ordset{ source-patches = [{ @@ -271,8 +231,6 @@ earogov_ordset{ ordset.tests = disabled // https://github.com/scala/scala3/issues/15820 does not compile } } -eed3si9n_verify.tests = compile-only -eed3si9n_expecty.tests = compile-only eed3si9n_ifdef { source-patches = [{ path = "compiler-plugin/src/main/scala-3/IfDefPlugin.scala" @@ -281,14 +239,12 @@ eed3si9n_ifdef { }] } eikek_binny { - tests = compile-only java.version = 11 } epfl-lara_lisa{ projects.exclude=["scallion", "scallion-examples", "scallion-benchmarks"] tests = compile-only // exit on failure } -emergentorder_onnx-scala.tests = compile-only endless4s_endless { sbt.commands = [ "excludeLibraryDependency org.wartremover:wartremover_{scalaVersion}", @@ -307,7 +263,6 @@ endpoints4s_endpoints4s{ } } -enragedginger_akka-quartz-scheduler.tests = compile-only enriquerodbe_borsh4s { sbt.commands = [ "excludeLibraryDependency org.wartremover:wartremover_{scalaVersion}", @@ -322,8 +277,6 @@ etorreborre_specs2 { projects{ exclude = ["specs2-guide", "specs2-cats", "specs2-scalaz", "xml"] overrides { - specs2-form.tests = compile-only - specs2-examples.tests = compile-only } } sbt.commands = [ @@ -332,18 +285,14 @@ etorreborre_specs2 { ] } -fasterxml_jackson-module-scala.tests = compile-only -fenixedu_fenixedu-scala-sdk.tests = compile-only fd4s_fs2-kafka{ sbt.commands = [ // Disable intergration tests using test containers """set core/Test/unmanagedSources/excludeFilter ~= {_ || "KafkaProducerConnectionSpec.scala" || "KafkaConsumerSpec.scala" || "KafkaProducerSpec.scala" || "TransactionalKafkaProducerSpec.scala" || "TransactionalKafkaProducerTimeoutSpec.scala" || "KafkaAdminClientSpec.scala"} """ ] } -findify_flink-scala-api.tests = compile-only fgrutsch_emergence.source-version = 3.4 firstbirdtech_emergence.source-version = 3.4 -foldables-io_skunk-tables.tests = compile-only fomkin_korolev { projects.exclude=[ "org.fomkin%korolev-http4s", # Conflicting cross Scala versions _3 vs _2.13 @@ -351,25 +300,19 @@ fomkin_korolev { ] tests = compile-only //deadlocks in the CB } -foundweekends_pamflet.tests = compile-only frawa_typed-json.projects.overrides { typed-json.tests = disabled // tries to read non-existing file } -frugalmechanic_fm-http.tests = compile-only -fs2-blobstore_fs2-blobstore.tests = compile-only garlandus_augment.java.version=17 gchudnov_swearwolf.sbt.commands = ["disableFatalWarnings"] geirolz_fly4s.tests = compile-only // flaky -geirolz_scope.tests = compile-only geirolz_secret.source-version=3.4 // implicts gemini-hlsw_gsp-graphql.tests = compile-only // uses Docker gemini-hlsw_lucuma-itc.java.version = 17 getkyo_kyo { source-version = 3.4 - tests = compile-only } -gonzih_cats-nats.tests = compile-only guntiso_mojoz.java.version = 8 guymers_ceesvee { sbt.commands = [ @@ -379,25 +322,15 @@ guymers_ceesvee { tests = compile-only // flaky } guymers_foobie { - tests = compile-only sbt.commands = [ "excludeLibraryDependency org.wartremover:wartremover_{scalaVersion}", "removeScalacOptionsStartingWith -P:wartremover" ] } giabao_asm-flow.tests = disabled // Does not compile -giabao_paho-akka.tests = compile-only -giiita_refuel.tests = compile-only -gitbucket_blocking-slick.tests = compile-only ghostdogpr_caliban.java.version = 17 -gmethvin_directory-watcher.tests = compile-only -greenleafoss_green-leaf-mongo.tests = compile-only -greenfossil_commons-json.tests = compile-only -greenfossil_thorium.tests = compile-only -guardian_fastly-api-client.tests = compile-only guardian_etag-caching.java.version = 17 gvolpe_trading { - tests = compile-only sbt.commands = [ "excludeLibraryDependency com.github.ghik:zerowaste_{scalaVersion}" ] @@ -417,18 +350,12 @@ hedgehogqa_scala-hedgehog{ sbt.commands = ["disableFatalWarnings"] } haifengl_smile { - tests = compile-only java.version = 17 } -herminiogg_dmaog.tests = compile-only -herminiogg_label2thesaurus.tests = compile-only -herminiogg_shexml.tests = compile-only higherkindness_droste { sbt.commands = ["disableFatalWarnings"] } -hjfruit_scalikejdbc-helper.tests = compile-only hnaderi_edomata{ - tests = compile-only projects.exclude = ["edomata-docs"] // Requires Scala Native compiler plugin } hnaderi_named-codec{ @@ -438,21 +365,11 @@ hnaderi_yaml4s { projects.exclude = ["yaml4s-docs"] // Requires Scala Native compiler plugin } hsci-r_lucene-perfieldpostingsformatordtermvectorscodec.tests = disabled // does not compile -hshn_slick-codegen-patch.tests = compile-only -http4s_blaze.tests = compile-only http4s_http4s.sbt.commands = [ "set every unidoc/unidocAllSources := Nil" // unidoc sometimes leads to deadlock/timeout when run in container ] -http4s_http4s-tomcat.tests = compile-only -hughsimpson_scalameter.tests = compile-only -ij-plugins_ijp-color.tests = compile-only -ij-plugins_ijp-javacv.tests = compile-only -ij-plugins_ijp-toolkit.tests = compile-only -ij-plugins_ijp-scala-console.tests = compile-only -indoorvivants_sn-bindgen.tests = compile-only indoorvivants_subatomic{ - tests = compile-only sbt.commands = [ """set every Test/unmanagedSources/excludeFilter := HiddenFileFilter || "MdocJSTests.scala"""" "disableFatalWarnings" @@ -461,21 +378,12 @@ indoorvivants_subatomic{ indoorvivants_weaver-playwright.tests = compile-only // uses scripts ingarabr_gcs-lock.tests = compile-only // uses gcloud irevive_union-derivation.tests = compile-only // fails due to changes in report messages -ist-dsi_scala-cinder-client.tests = compile-only -ist-dsi_scala-designate-client.tests = compile-only ist-dsi_scala-keystone-client.tests = compile-only // uses docker -ist-dsi_scala-neutron-client.tests = compile-only -ist-dsi_scala-nova-client.tests = compile-only -itv_quartz4s.tests = compile-only -japgolly_scala-graal.tests = compile-only japgolly_test-state { projects.overrides{ - ext-selenium.tests = compile-only - dom-zipper-selenium.tests = compile-only } } -japgolly_clear-config.tests = compile-only jbwheatley_pact4s.sbt.commands = ["""set shared/scalacOptions --= Seq("-Wconf:cat=deprecation:i", "-Xfatal-warnings") """] jchapuis_fs2-kafka-mock { sbt.commands = [ @@ -491,7 +399,6 @@ jchapuis_fs2-kafka-mock { ] } jcouyang_dhall-generic.sbt.commands = ["set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat"] -jmcardon_tsec.tests=compile-only joan38_kubernetes-client { source-patches = [ { @@ -514,30 +421,20 @@ jodersky_scala-argparse{ jodersky_scalapb-ujson{ tests = disabled // starts failing process (proto gen?) } -jodersky_redicl.tests = compile-only joernio_joern{ projects.overrides{ x2cpg.tests = compile-only // Requires mvn binary jssrc2cpg.tests = compile-only // Requires joernio/astgen installed via npm - javasrc2cpg.tests = compile-only } projects.exclude=[php2cpg] // Malformed setup file } json4s_json4s.source-version=3.3 -julianpeeters_avrohugger.tests = compile-only jwojnowski_googlecloud4s.tests = compile-only // uses docker -juliano_pokeapi-scala.tests = compile-only jvican_dijon.projects.exclude = ["me.vican.jorge%monix-grpc-codegen", "me.vican.jorge%monix-grpc-runtime"] // missing -j5ik2o_akka-persistence-s3.tests = compile-only j5ik2o_akka-persistence-dynamodb.projects.exclude = ["com.github.j5ik2o%event-store-adapter-scala"] // missing -j5ik2o_docker-controller-scala.tests = compile-only -j5ik2o_pekko-kinesis.tests = compile-only -j5ik2o_pekko-persistence-dynamod.tests = compile-only -kaizen-solutions_trace4cats-zio-extras.tests = compile-only kamon-io_kamon{ java.version = 8 - tests = compile-only } karelcemus_play-redis { sbt.commands = [ @@ -548,7 +445,6 @@ karelcemus_play-redis { katrix_minejson.tests = disabled // Does not compile, build issue kierendavies_skunk-migrate.tests = compile-only kitlangton_neotype.java.version = 17 -kirill5k_mongo4cats.tests = compile-only kevin-lee_extras{ sbt.commands = [ "excludeLibraryDependency org.wartremover:wartremover_{scalaVersion}", @@ -604,8 +500,6 @@ kostaskougios_functions-remote.java.version = 21 # io.github.kostaskougios%terminal21-mathjax # ] # } -kovstas_fs2-throttler.tests = compile-only -krrrr38_protoquill-finagle-mysql.tests = compile-only kubukoz_drops { sbt.commands = ["excludeLibraryDependency org.polyvariant:better-tostring_{scalaVersion}"] } @@ -614,11 +508,9 @@ kubukoz_spotify-next { } kubukoz_sup { sbt.commands = ["excludeLibraryDependency org.polyvariant:better-tostring_{scalaVersion}"] - tests = compile-only } laserdisc-io_fs2-aws{ - tests = compile-only sbt.commands = ["disableFatalWarnings"] } lichess-org_lila.java.version = 21 @@ -648,11 +540,8 @@ lorandszakacs_enclosure.sbt.commands = ["set every Test/classLoaderLayeringStrat losizm_little-sql.java.version = 8 losizm_scamper{ java.version = 8 - tests = compile-only } -makiftutuncu_effect.tests = compile-only markehammons_slinc { - tests = compile-only java.version = 21 } mateuszkubuszok_pipez{ @@ -666,9 +555,7 @@ mateuszkubuszok_pipez{ } marchliu_jaskell-dotty.sbt.commands = ["""set Compile/sources ~= (_.filter(_.getName != "NewLine.scala")) """] // Duplicates with Newline.scala martinhh_scalacheck-derived.source-version=3.3 -matsluni_aws-spi-akka-http.tests = compile-only mercurievv_minuscles.projects.exclude=["io.github.mercurievv.minuscles%conversions"] // missing -metarank_cfor.tests = compile-only metarank_ltrlib.tests = disabled // used Scala 2 macro mccarthy_fetch{ # https://github.com/scala/scala3/issues/15843 @@ -680,12 +567,8 @@ medeia_medeia{ "removeScalacOptionsStartingWith -P:wartremover" ] } -minosiants_pencil.tests = compile-only mjakubowski84_parquet4s.java.version = 8 -mkroli_dns4s.tests = compile-only -mpollmeier_scala-repl-pp.tests = compile-only monix_minitest.sbt.commands = ["set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat"] -monix_monix-connect.tests = compile-only monix_implicitbox.sbt.commands = ["set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat"] mvv_sager { source-version = 3.4 @@ -700,41 +583,20 @@ nafg_css-dsl.projects.exclude = [ "io.github.nafg.css-dsl%bulma_scalatags", "io.github.nafg.css-dsl%bulma_scalajsreact" ] -nafg_slick-migration-api.tests = compile-only nau_scalus { source-version = 3.4 - tests = compile-only } neandertech_langoustine.source-version = 3.4 -naoh87_lettucef.tests = compile-only -nrkno_bigquery-scala.tests = compile-only noelwelsh_mads.sbt.commands = ["disableFatalWarnings"] -oheger_cloud-files.tests = compile-only -ohze_couchbase-scala.tests = compile-only -ohze_php-utils.tests = compile-only -ollls_quartz-h2.tests = compile-only -ollls_zio-quartz-h2.tests = compile-only -outr_media4s.tests = compile-only -outr_robobrowser.tests = compile-only -outr_scarango.tests = compile-only -outr_profig.tests = compile-only - -palanga_zio-cassandra.tests = compile-only palanga_parana.tests = compile-only // usses cassandra instance -pathikrit_better-files.tests = compile-only -permutive-engineering_fs2-google-pubsub.tests = compile-only -phenoscape_scowl.tests = compile-only pityka_lamp{ java.version=17 tests = compile-only // uses native libraries } pjfanning_akka-rabbitmq.sbt.commands = ["""set every Test/unmanagedSources/excludeFilter := HiddenFileFilter || "PublishSubscribeSpec.scala""""] -pjfanning_aws-spi-pekko-http.tests = compile-only pjfanning_jackson-scala3-reflection-extensions.tests = compile-only // uses pjfanning_pekko-http-session.java.version = 17 -pjfanning_pekko-rabbitmq.tests = compile-only -pjfanning_scala-faker.tests = compile-only pjfanning_scala3-reflection { source-patches = [{ path = "src/main/scala/co.blocke.scala_reflection/ReflectionWorker.scala" @@ -742,8 +604,6 @@ pjfanning_scala3-reflection { replace-with = "override def init(options: List[String]): List[PluginPhase]" }] } -playframework_playframework.tests = compile-only -playframework_play-json.tests = compile-only plokhotnyuk_jsoniter-scala { sbt.commands = ["""set every Test/unmanagedSources/excludeFilter := HiddenFileFilter || "JsonCodecMakerNewTypeSpec.scala" """] } @@ -771,7 +631,6 @@ polyvariant_smithy4s-caliban { polyvariant_treesitter4s { sbt.commands = ["excludeLibraryDependency org.polyvariant:better-tostring_{scalaVersion}"] } -postgresql-async_postgresql-async.tests = compile-only paulbutcher_scalamock { source-patches = [{ path = "jvm/src/test/scala/com.paulbutcher.test/mock/JavaMocksTest.scala" @@ -788,30 +647,22 @@ pureharm_pureharm-db-core{ "com.busymachines%pureharm-db-testkit" ] } -purplekingdomgames_ultraviolet.tests = compile-only -ranyitz_brush.tests = compile-only -rediscala_rediscala.tests = compile-only -reactivecore_rc-circe-json-schema.tests = compile-only reactivemongo_reactivemongo { source-version = 3.4 projects.exclude = ["org.reactivemongo%reactivemongo-alias"] // missing projects.overrides = { reactivemongo { // Actual tests needs env set up - tests = compile-only } } } reactivemongo_reactivemongo-bson { projects.exclude = ["reactivemongo-jmx", "reactivemongo-datadog", "reactivemongo-kamon"] projects.overrides = { - reactivemongo-bson-api.tests = compile-only - reactivemongo-bson-geo.tests = compile-only } } reactivemongo_reactivemongo-play-json.tests = compile-only // fails due to changes in reported error messages -rescala-lang_rescala.tests = compile-only rmgk_slips.java.version = 17 ruimo_csvparser.tests = disabled // does not compile ruimo_graphics.tests = disabled // does not compile @@ -819,24 +670,19 @@ ruimo_scoins.tests = disabled // does not compile sageserpent-open_americium{ java.version = 17 - tests = compile-only } sageserpent-open_kineticmerge{ java.version=17 - tests = compile-only } scala_toolkit.tests = disabled // no tests scala-cli_libdaemon-jvm { java.version=17 - tests = compile-only } scala-native_scala-native.tests = compile-only -scalafx_scalafx.tests = compile-only scalacenter_simple-rockets-compiler { # multi string settings need to use `:` since 3.2 sbt.commands = ["""set core/Compile/doc/scalacOptions ~= {_.diff(Seq("-comment-syntax", "wiki")) ++ Seq("-comment-syntax:wiki")}"""] } -scalacenter_tasty-mima.tests = compile-only scalacenter_tasty-query.tests = disabled // does not compile scalamolecule_molecule.tests = disabled // does not compile, try restore in the future scalatest_scalatest { @@ -858,8 +704,6 @@ scalatest_scalatestplus-mockito { sbt.commands = ["set Compile/doc/skip := tr scalatest_scalatestplus-scalacheck { sbt.commands = ["set Compile/doc/skip := true"] } scalatest_scalatestplus-selenium { sbt.commands = ["set Compile/doc/skip := true"] } scalatest_scalatestplus-scalacheck { sbt.commands = ["set scalatestPlusScalaCheckJVM/Compile/doc/skip := true"] } -scala-native_scala-native-cli.tests = compile-only -scalanlp_breeze.tests = compile-only scalapatisserie_muffin.source-version = 3.4 // implicits scalapb_scalapb { // Upstream is compiled with Java 11 @@ -874,46 +718,25 @@ scalapb_scalapb { replace-with = "" }] } -scalapb_protobuf-scala-runtime.tests = compile-only -scalapb-json_scalapb-argonaut.tests = compile-only -scalapb-json_scalapb-json-common.tests = compile-only -scalapy_scalapy.tests = compile-only -scalapy_python-native-libs.tests = compile-only scalaz_scalaz { sbt { commands = ["set every unidoc/unidocAllSources := Nil"] // in Scala 3.1.0+ gets into infinite loop when run in containers } } -scalikejdbc_scalikejdbc.tests = compile-only -scalikejdbc_scalikejdbc-async.tests = compile-only -scanamo_scanamo.tests = compile-only scf37_hottie.tests = compile-only // Uses dotc.Main, works locally, fails in container sciss_soundprocesses{ - tests = compile-only projects.exclude=["de.sciss%soundprocesses-compiler"] // Fails to compile after changes to compiler API in 3.1.3 } -sciss_patterns.tests = compile-only -sciss_wolkenpumpe.tests = compile-only scodec_scodec-cats.sbt.commands = ["set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat"] -scodec_scodec-stream.tests = compile-only scottweaver_testcontainers-for-zio { sbt.commands = ["set every semanticdbEnabled := false"] - tests = compile-only } scoverage_scalac-scoverage-plugin{ projects.overrides { # Fails in runtime times due to changed error messages - scalac-scoverage-reporter.tests = compile-only } } -seratch_awscala.tests = compile-only -sky-uk_fs2-kafka-topic-loader.tests = compile-only -shawjef3_tryutils.tests = compile-only -shiftleftsecurity_js2cpg.tests = compile-only simy4_coregex.java.version = 11 // scaldoc/javadoc fails in Java 8 due to incorrect flags -sirthias_scala-ssh.tests = compile-only -sirthias_parboiled2.tests = compile-only -sksamuel_elastic4s.tests = compile-only sky-uk_kafka-topic-loader.tests = compile-only // timeouts softwaremill_akka-http-session { sbt.commands = [ @@ -929,7 +752,6 @@ softwaremill_sttp { sbt { commands=["set every closeClassLoaders := false"] } - tests = compile-only projects.exclude=[ # Does not compile - we publish ZIO 1.x "com.softwaremill.sttp.client3%zio.*" @@ -940,25 +762,15 @@ softwaremill_tapir { sbt.options=["-J-XX:+UseG1GC"] sbt.commands=["set vertxServerZio.jvm(true)/Compile/doc/sources := Nil"] projects.overrides { - tapir-datadog-metrics.tests = compile-only - tapir-http4s-client.tests = compile-only tapir-sttp-client-ws-zio.tests = compile-only - tapir-sttp-client.tests = compile-only // Deadlocks when executing in minikube - tapir-vertx-server.tests = compile-only // Timeout - tapir-play-client.tests = compile-only - tapir-play29-client.tests = compile-only } } softwaremill_tapir-loom { java.version=21 } staeff777_scalaveusz.tests = disabled // Does not compile with any Scala version -svroonland_rezilience.tests = compile-only -svroonland_zio-kinesis.tests = compile-only -svroonland_zio-amqp.tests = compile-only -swagger-akka-http_swagger-enumeratum-module.tests = compile-only swagger-akka-http_swagger-pekko-http.tests = compile-only // depends on experimental compiler systemfw_dynosaur.java.version = 11 @@ -967,30 +779,24 @@ reactivemongo_reactivemongo { "reactivemongo%reactivemongo-alias" # Not a part of the main repo, defined in reactivemongo-shaded ] } -regis-leray_fs2-ftp.tests = compile-only rssh_cps-async-connect { source-version = 3.4 java.version = 21 } -quafadas_live-server-scala-cli-js.tests = compile-only t2v_holidays.tests = compile-only // invalid classpath at runtime taig_backmail.source-version=3.4 taig_scala-linguist.projects.exclude=["io.taig%scala-linguist-graalvm-ruby"] // needs GraalVM taig_scala-pygments.projects.exclude=["io.taig%scala-pygments-graalvm-python"] -takezoe_solr-scala-client.tests = compile-only -tinkoff_phobos.tests = compile-only testcontainers_testcontainers-scala{ java.version = 17 - tests = compile-only } tharwaninitin_etlflow { sbt.commands = [ "excludeLibraryDependency org.wartremover:wartremover_{scalaVersion}", "removeScalacOptionsStartingWith -P:wartremover" ] - tests = compile-only } tharwaninitin_cron4zio { sbt.commands = [ @@ -999,13 +805,11 @@ tharwaninitin_cron4zio { ] } tharwaninitin_gcp4zio { - tests = compile-only sbt.commands = [ "excludeLibraryDependency org.wartremover:wartremover_{scalaVersion}", "removeScalacOptionsStartingWith -P:wartremover" ] } -thatscalaguy_nats4cats.tests = compile-only thoughtworksinc_binding.scala { // Not a part of the repositroy projects.exclude=[ @@ -1017,16 +821,11 @@ thoughtworksinc_binding.scala { "com.thoughtworks.binding%streamt" ] } -tomasmikula_libretto.tests = compile-only torrentdam_bencode.tests = disabled // needs ScalaNative nscplugin -tototoshi_dbcache.tests = compile-only tpolecat_doobie { projects.overrides{ // as per https://github.com/scala/community-builds/pull/835#issuecomment-455729365, // these subprojects require a Postgres instance for the tests to run - doobie-hikari.tests = compile-only - doobie-postgres.tests = compile-only - doobie-postgres-circe.tests = compile-only } } trace4cats_trace4cats { @@ -1036,15 +835,10 @@ trace4cats_trace4cats { "io.janstenpickle%trace4cats-dynamic-sampling-http4s" ] } -trace4cats_trace4cats-datadog.tests = compile-only -trace4cats_trace4cats-jaeger.tests = compile-only -trace4cats_trace4cats-opentelemetry.tests = compile-only -trace4cats_trace4cats-zipkin.tests = compile-only travisbrown_dhallj { sbt.commands=["javacc"] projects.overrides { // Uses data fetched from outside repo - dhall-parser.tests = compile-only } } twitter_util { @@ -1061,7 +855,6 @@ typelevel_cats-effect { replace-with = "(t.asLeft, limit - 1).asRight" }] } -typelevel_cats-effect-testing.tests = compile-only typelevel_cats-mtl { sbt.commands=[ "set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat" @@ -1077,16 +870,13 @@ typelevel_discipline-scalatest { "set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat" ] } -typelevel_grackle.tests = compile-only typelevel_jawn { projects.exclude=[ "com.eed3si9n%shaded-jawn-parser" ] } -typelevel_keypool.tests = compile-only typelevel_laika { java.version = 11 - tests = compile-only projects.exclude = ["org.typelevel%laika-docs"] } @@ -1102,9 +892,6 @@ wvlet_airframe { ] overrides{ //https://github.com/wvlet/airframe/issues/1077 - airframe-codec.tests = compile-only - airframe.tests = compile-only - airframe-ulid.tests = compile-only } } } @@ -1118,9 +905,6 @@ vigoo_prox.tests = compile-only virtuslab_scala-cli { java.version=17 projects.overrides{ - build-module.tests = compile-only - build-macros.tests = compile-only - cli.tests = compile-only directives.tests = compile-only // uses Scala 3.0.2 runner } } @@ -1137,23 +921,18 @@ vladkopanev_cats-saga.source-patches = [ } ] -xuwei-k_httpz.tests = compile-only xuwei-k_wartremover-scalikejdbc { sbt.commands = [ "excludeLibraryDependency org.wartremover:wartremover_{scalaVersion}", "removeScalacOptionsStartingWith -P:wartremover" ] } -y-yu_slick-bulk-insert.tests = compile-only zaneli_scalikejdbc-athena{ sbt.commands=[ "set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat" ] } -zengularity_benji.tests = compile-only -zio_zio-aws.tests = compile-only -zio_izumi-reflect.tests = compile-only zio_interop-cats.source-patches = [ { path = "project/BuildHelper.scala" @@ -1169,16 +948,11 @@ zio_zio-config { zio-config-refined.tests = disabled } } -zio_zio-connect.tests = compile-only -zio_zio_ftp.tests = compile-only zio_zio_http{ - tests = compile-only projects.exclude = ["dev.zio%zio-http-testkit-shaded", "dev.zio%zio-http-shaded"] // missing } zio_zio-json { projects.overrides { - zio-json-yaml.tests = compile-only - zio-json-golden.tests = compile-only zio-json-interop-refined.tests = disabled } } @@ -1196,10 +970,7 @@ zio_zio-protoquill { sbt.commands=[ "set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat" ] - tests = compile-only } -zio_zio-redis.tests = compile-only zio_zio-quill { sbt.options=["-Dquill.scala.version="] } -zio_zio-s3.tests = compile-only diff --git a/project-builder/shared/CommunityBuildCore.scala b/project-builder/shared/CommunityBuildCore.scala index da8d4abf..e5512e47 100644 --- a/project-builder/shared/CommunityBuildCore.scala +++ b/project-builder/shared/CommunityBuildCore.scala @@ -12,7 +12,7 @@ object Scala3CommunityBuild { // Community projects configs case class ProjectBuildConfig( projects: ProjectsConfig = ProjectsConfig(), - tests: TestingMode = TestingMode.Full + tests: TestingMode = TestingMode.CompileOnly ) case class ProjectOverrides(tests: Option[TestingMode] = None) From fd12fdf4b7aa5493fb2496a635916cc6ef242c71 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 28 May 2024 15:22:20 +0200 Subject: [PATCH 3/4] Don't split projects by dependencies - no longer usefull --- coordinator/src/main/scala/Scaladex.scala | 11 +++++ coordinator/src/main/scala/buildPlan.scala | 52 ++-------------------- 2 files changed, 14 insertions(+), 49 deletions(-) diff --git a/coordinator/src/main/scala/Scaladex.scala b/coordinator/src/main/scala/Scaladex.scala index fef497cd..e83588e4 100644 --- a/coordinator/src/main/scala/Scaladex.scala +++ b/coordinator/src/main/scala/Scaladex.scala @@ -1,6 +1,7 @@ import java.time.ZonedDateTime import java.util.concurrent.TimeUnit.SECONDS import scala.concurrent.* +import scala.concurrent.duration.* object Scaladex { case class Pagination(current: Int, pageCount: Int, totalSize: Int) @@ -40,6 +41,12 @@ object Scaladex { ) SECONDS.sleep(backoffSeconds) tryFetch((backoffSeconds * 2).min(60)) + case _: requests.TimeoutException => + Console.err.println( + s"Failed to fetch artifact metadata, Scaladex request timeout, retry with backoff ${backoffSeconds}s for $groupId:$artifactId" + ) + SECONDS.sleep(backoffSeconds) + tryFetch((backoffSeconds * 2).min(60)) } tryFetch(1) } @@ -63,6 +70,10 @@ object Scaladex { Option.unless(response.contentLength.contains(0)) { fromJson[ProjectSummary](response.text()) } + }.recoverWith{ + case _: requests.TimeoutException => + Thread.sleep(scala.util.Random.nextInt(10.seconds.toMillis.toInt)) + projectSummary(organization, repository, scalaBinaryVersion) } } diff --git a/coordinator/src/main/scala/buildPlan.scala b/coordinator/src/main/scala/buildPlan.scala index d4bf28ac..4a9cbe1a 100644 --- a/coordinator/src/main/scala/buildPlan.scala +++ b/coordinator/src/main/scala/buildPlan.scala @@ -398,57 +398,11 @@ def splitIntoStages( ): StagedBuildPlan = { val deps = projects.map(v => (v.project, v)).toMap // GitHub Actions limits to 255 elements in matrix - val maxStageSize = 255 - @scala.annotation.tailrec - def groupByDeps( - remaining: Set[ProjectBuildDef], - done: Set[Project], - acc: List[Set[ProjectBuildDef]] - ): List[Set[ProjectBuildDef]] = { - if remaining.isEmpty then acc.reverse - else - var (currentStage, newRemainings) = remaining.partition { - _.dependencies.filter(deps.contains).forall(done.contains) - } - if currentStage.isEmpty then { - def hasCyclicDependencies(p: ProjectBuildDef) = - p.dependencies.exists( - deps.get(_).fold(false)(_.dependencies.contains(p.project)) - ) - val cyclicDeps = newRemainings.filter(hasCyclicDependencies) - if cyclicDeps.nonEmpty then { - currentStage ++= cyclicDeps - newRemainings --= cyclicDeps - cyclicDeps.foreach(v => - println( - s"Mitigated cyclic dependency in ${v.project} -> ${v.dependencies.toList - .filterNot(done.contains)}" - ) - ) - } else { - val (minDeps, tieBreakers) = newRemainings - .groupBy(_.dependencies.count(!done.contains(_))) - .toSeq - .sortBy { (depsCount, _) => depsCount } - .head - def showCurrent = - tieBreakers.map(_.project.coordinates).mkString(", ") - System.err.println( - s"Not found projects without already resolved dependencies, using [${tieBreakers.size}] projects with minimal dependency size=$minDeps : ${showCurrent}" - ) - currentStage ++= tieBreakers - newRemainings --= tieBreakers - } - } - val names = currentStage.map(_.project) - val currentStages = currentStage.grouped(maxStageSize).toList - groupByDeps(newRemainings, done ++ names, currentStages ::: acc) - } - - val (longRunningDefs, toSplit) = projects.toSet + val MaxStageSize = 255 + val (longRunningDefs, toSplit) = projects.toSet .partition(p => longBuildingProjects.contains(p.project)) val longRunning = longRunningDefs.map(_.project) - val staged = groupByDeps(toSplit, done = longRunning, Nil) + val staged = toSplit.grouped(MaxStageSize).toList val all = longRunningDefs :: staged all.map(_.toList.sortBy(_.project)) From f1034b2135dedbbee2de33ee930f377be8d3f8aa Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 28 May 2024 15:25:02 +0200 Subject: [PATCH 4/4] Set version to 0.3.12 --- .github/actions/build-project/action.yaml | 2 +- .github/actions/setup-build/action.yaml | 2 +- cli/scb-cli.scala | 2 +- scripts/bisect.scala | 2 +- scripts/build-all.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/build-project/action.yaml b/.github/actions/build-project/action.yaml index 4fe83516..899c3f0c 100644 --- a/.github/actions/build-project/action.yaml +++ b/.github/actions/build-project/action.yaml @@ -110,7 +110,7 @@ runs: uses: addnab/docker-run-action@v3 if: steps.check-history.outputs.can-skip-build != 'true' with: - image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.3.11" + image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.3.12" options: -v ${{ github.workspace }}:/opencb/ -v ${{ github.workspace }}/github_key:/root/.ssh/github_key:ro run: | # Setup ssh required for downloading submodules diff --git a/.github/actions/setup-build/action.yaml b/.github/actions/setup-build/action.yaml index a3cd12a5..260b101f 100644 --- a/.github/actions/setup-build/action.yaml +++ b/.github/actions/setup-build/action.yaml @@ -75,7 +75,7 @@ runs: uses: addnab/docker-run-action@v3 if: steps.check-published.outputs.is-compiler-published == 'false' with: - image: "virtuslab/scala-community-build-compiler-builder:v0.3.11" + image: "virtuslab/scala-community-build-compiler-builder:v0.3.12" options: -v ${{ github.workspace }}/compiler:/compiler/ run: | Version="${{ steps.calc-version.outputs.effective-scala-version }}" diff --git a/cli/scb-cli.scala b/cli/scb-cli.scala index 2581b0f6..6c957032 100755 --- a/cli/scb-cli.scala +++ b/cli/scb-cli.scala @@ -31,7 +31,7 @@ class FailedProjectException(msg: String) with NoStackTrace val communityBuildVersion = - sys.props.getOrElse("communitybuild.version", "v0.3.11") + sys.props.getOrElse("communitybuild.version", "v0.3.12") private val CBRepoName = "VirtusLab/community-build3" val projectBuilderUrl = s"https://raw.githubusercontent.com/$CBRepoName/master/project-builder" diff --git a/scripts/bisect.scala b/scripts/bisect.scala index 9ad5e849..e30adcbb 100755 --- a/scripts/bisect.scala +++ b/scripts/bisect.scala @@ -9,7 +9,7 @@ import java.nio.file.attribute.PosixFilePermissions import java.nio.charset.StandardCharsets import java.nio.file._ -val communityBuildVersion = "v0.3.11" +val communityBuildVersion = "v0.3.12" @main def run(args: String*): Unit = val config = scopt.OParser diff --git a/scripts/build-all.sh b/scripts/build-all.sh index 5fa6544f..3a8ab773 100755 --- a/scripts/build-all.sh +++ b/scripts/build-all.sh @@ -7,7 +7,7 @@ if [ $# -ne 1 ]; then fi VERSION="$1" -export PREV_CB_VERSION="v0.3.10" +export PREV_CB_VERSION="v0.3.11" javaDefault=11 javaAccessoryVersions=(8 17 21)