From 4ab322768f7b5813312aeb553fed8a2caf35e4a4 Mon Sep 17 00:00:00 2001 From: Piotr Chabelski Date: Thu, 19 Dec 2024 18:22:26 +0100 Subject: [PATCH] Retry some more flaky tests on the CI (#3382) --- .../scala/cli/integration/GitHubTests.scala | 22 ++-- .../integration/RunGistTestDefinitions.scala | 18 +-- .../cli/integration/RunTestDefinitions.scala | 2 +- .../cli/integration/TestTestDefinitions.scala | 121 +++++++++--------- .../scala/cli/integration/UpdateTests.scala | 2 +- 5 files changed, 89 insertions(+), 76 deletions(-) diff --git a/modules/integration/src/test/scala/scala/cli/integration/GitHubTests.scala b/modules/integration/src/test/scala/scala/cli/integration/GitHubTests.scala index 34f93f64da..1c8e761845 100644 --- a/modules/integration/src/test/scala/scala/cli/integration/GitHubTests.scala +++ b/modules/integration/src/test/scala/scala/cli/integration/GitHubTests.scala @@ -51,20 +51,24 @@ class GitHubTests extends ScalaCliSuite { } } - override def munitFlakyOK: Boolean = TestUtil.isCI && Properties.isMac + override def munitFlakyOK: Boolean = TestUtil.isCI + + def createSecret(): Unit = { + try + createSecretTest() + catch { + case e: UnsatisfiedLinkError if e.getMessage.contains("libsodium") => + fail("libsodium, couldn't be loaded") + } + + } // currently having issues loading libsodium from the static launcher // that launcher is mainly meant to be used on CIs or from docker, missing // that feature shouldn't be a big deal there if (TestUtil.cliKind != "native-static") - test("create secret".flaky) { - try - createSecretTest() - catch { - case e: UnsatisfiedLinkError if e.getMessage.contains("libsodium") => - fail("libsodium, couldn't be loaded") - } - } + if (Properties.isMac) test("create secret".flaky)(createSecret()) + else test("create secret")(TestUtil.retryOnCi()(createSecret())) } diff --git a/modules/integration/src/test/scala/scala/cli/integration/RunGistTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/RunGistTestDefinitions.scala index 8cfdb61198..3e1818e8c7 100644 --- a/modules/integration/src/test/scala/scala/cli/integration/RunGistTestDefinitions.scala +++ b/modules/integration/src/test/scala/scala/cli/integration/RunGistTestDefinitions.scala @@ -46,14 +46,16 @@ trait RunGistTestDefinitions { _: RunTestDefinitions => } test("Github Gists Script URL") { - val url = - "https://gist.github.com/alexarchambault/7b4ec20c4033690dd750ffd601e540ec" - val message = "Hello" - emptyInputs.fromRoot { root => - val output = os.proc(TestUtil.cli, extraOptions, escapedUrls(url)) - .call(cwd = root) - .out.trim() - expect(output == message) + TestUtil.retryOnCi() { + val url = + "https://gist.github.com/alexarchambault/7b4ec20c4033690dd750ffd601e540ec" + val message = "Hello" + emptyInputs.fromRoot { root => + val output = os.proc(TestUtil.cli, extraOptions, escapedUrls(url)) + .call(cwd = root) + .out.trim() + expect(output == message) + } } } } diff --git a/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala index 767ad9767d..0507c9b6e7 100644 --- a/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala +++ b/modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala @@ -1905,7 +1905,7 @@ abstract class RunTestDefinitions test( s"offline mode should fail on missing artifacts (with Scala $actualAnnouncedScalaVersion)" ) { - TestUtil.retryOnCi() { + TestUtil.retryOnCi(maxAttempts = 5) { // Kill bloop deamon to test scalac fallback os.proc(TestUtil.cli, "--power", "bloop", "exit") .call(cwd = os.pwd) diff --git a/modules/integration/src/test/scala/scala/cli/integration/TestTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/TestTestDefinitions.scala index 8d0bc3eaf6..84235258d2 100644 --- a/modules/integration/src/test/scala/scala/cli/integration/TestTestDefinitions.scala +++ b/modules/integration/src/test/scala/scala/cli/integration/TestTestDefinitions.scala @@ -556,70 +556,77 @@ abstract class TestTestDefinitions extends ScalaCliSuite with TestScalaVersionAr for ((platformName, platformArgs) <- platforms) test(s"custom test framework $platformName") { - val inputs = TestInputs( - os.rel / "MyTests.test.scala" -> - s"""//> using dep com.lihaoyi::utest::$utestVersion - | - |package mytests - |import utest._ - | - |object MyTests extends TestSuite { - | val tests = Tests { - | test("foo") { - | assert(2 + 2 == 4) - | println("Hello from " + "tests") - | } - | } - |} - |""".stripMargin, - os.rel / "CustomFramework.test.scala" -> - """package custom - | - |class CustomFramework extends utest.runner.Framework { - | override def setup(): Unit = - | println("Hello from CustomFramework") - |} - |""".stripMargin - ) - inputs.fromRoot { root => - val baseRes = os.proc(TestUtil.cli, "test", extraOptions, platformArgs, ".") - .call(cwd = root) - val baseOutput = baseRes.out.text() - expect(baseOutput.contains("Hello from tests")) - expect(!baseOutput.contains("Hello from CustomFramework")) - - val cmd = Seq[os.Shellable]( - TestUtil.cli, - "test", - extraOptions, - platformArgs, - ".", - "--test-framework", - "custom.CustomFramework" + TestUtil.retryOnCi(maxAttempts = if (platformName.contains("native")) 3 else 1) { + val inputs = TestInputs( + os.rel / "MyTests.test.scala" -> + s"""//> using dep com.lihaoyi::utest::$utestVersion + | + |package mytests + |import utest._ + | + |object MyTests extends TestSuite { + | val tests = Tests { + | test("foo") { + | assert(2 + 2 == 4) + | println("Hello from " + "tests") + | } + | } + |} + |""".stripMargin, + os.rel / "CustomFramework.test.scala" -> + """package custom + | + |class CustomFramework extends utest.runner.Framework { + | override def setup(): Unit = + | println("Hello from CustomFramework") + |} + |""".stripMargin ) - val res = os.proc(cmd).call(cwd = root) - val output = res.out.text() - expect(output.contains("Hello from tests")) - expect(output.contains("Hello from CustomFramework")) + inputs.fromRoot { root => + val baseRes = os.proc(TestUtil.cli, "test", extraOptions, platformArgs, ".") + .call(cwd = root) + val baseOutput = baseRes.out.text() + expect(baseOutput.contains("Hello from tests")) + expect(!baseOutput.contains("Hello from CustomFramework")) + + val cmd = Seq[os.Shellable]( + TestUtil.cli, + "test", + extraOptions, + platformArgs, + ".", + "--test-framework", + "custom.CustomFramework" + ) + val res = os.proc(cmd).call(cwd = root) + val output = res.out.text() + expect(output.contains("Hello from tests")) + expect(output.contains("Hello from CustomFramework")) + } } } for ((platformName, platformArgs) <- platforms) test(s"Fail if no tests were run $platformName") { - val inputs = TestInputs( - os.rel / "MyTests.test.scala" -> - s"""//> using dep org.scalameta::munit::$munitVersion - | - |object MyTests - |""".stripMargin - ) + TestUtil.retryOnCi(maxAttempts = if (platformName.contains("native")) 3 else 1) { + val inputs = TestInputs( + os.rel / "MyTests.test.scala" -> + s"""//> using dep org.scalameta::munit::$munitVersion + | + |object MyTests + |""".stripMargin + ) - inputs.fromRoot { root => - val res = os.proc(TestUtil.cli, "test", extraOptions, "--require-tests", platformArgs, ".") - .call(cwd = root, stderr = os.Pipe, mergeErrIntoOut = true, check = false) - expect(res.exitCode != 0) - val output = res.out.text() - expect(output.contains("Error: no tests were run") || output.contains("No tests were run")) + inputs.fromRoot { root => + val res = + os.proc(TestUtil.cli, "test", extraOptions, "--require-tests", platformArgs, ".") + .call(cwd = root, stderr = os.Pipe, mergeErrIntoOut = true, check = false) + expect(res.exitCode != 0) + val output = res.out.text() + expect( + output.contains("Error: no tests were run") || output.contains("No tests were run") + ) + } } } diff --git a/modules/integration/src/test/scala/scala/cli/integration/UpdateTests.scala b/modules/integration/src/test/scala/scala/cli/integration/UpdateTests.scala index 24fec67afa..c6c52be6de 100644 --- a/modules/integration/src/test/scala/scala/cli/integration/UpdateTests.scala +++ b/modules/integration/src/test/scala/scala/cli/integration/UpdateTests.scala @@ -108,7 +108,7 @@ class UpdateTests extends ScalaCliSuite { if (!Properties.isWin && Constants.ghOrg == "VirtusLab" && Constants.ghName == "scala-cli") test("updating dummy scala-cli using update command") { - runUpdate() + TestUtil.retryOnCi()(runUpdate()) } test("run update before run/test/compile should not return exit code") {