Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use initial Seed in CheckConfig and failure message. #39

Merged
merged 7 commits into from
May 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,18 +1,40 @@
package weaver
package scalacheck

case class CheckConfig(
minimumSuccessful: Int,
maximumDiscardRatio: Int,
maximumGeneratorSize: Int,
perPropertyParallelism: Int,
initialSeed: Option[Long]
) {
import org.scalacheck.rng.Seed
import org.typelevel.scalaccompat.annotation.unused

case class CheckConfig private (
val minimumSuccessful: Int,
val maximumDiscardRatio: Int,
val maximumGeneratorSize: Int,
val perPropertyParallelism: Int,
val initialSeed: Option[Seed]) {
assert(maximumDiscardRatio >= 0)
assert(maximumDiscardRatio <= 100)
assert(minimumSuccessful > 0)

def maximumDiscarded = minimumSuccessful * maximumDiscardRatio / 100

def withMinimumSuccessful(minimumSuccessful: Int) = copy(
minimumSuccessful = minimumSuccessful
)

def withMaximumDiscardRatio(maximumDiscardRatio: Int) = copy(
maximumDiscardRatio = maximumDiscardRatio
)

def withMaximumGeneratorSize(maximumGeneratorSize: Int) = copy(
maximumGeneratorSize = maximumGeneratorSize
)

def withPerPropertyParallelism(perPropertyParallelism: Int) = copy(
perPropertyParallelism = perPropertyParallelism
)

def withInitialSeed(initialSeed: Option[Seed]) = copy(
initialSeed = initialSeed
)
}

object CheckConfig {
Expand All @@ -23,4 +45,19 @@ object CheckConfig {
perPropertyParallelism = 10,
initialSeed = None
)

def apply(
minimumSuccessful: Int,
maximumDiscardRatio: Int,
maximumGeneratorSize: Int,
perPropertyParallelism: Int,
initialSeed: Option[Seed]): CheckConfig =
new CheckConfig(minimumSuccessful,
maximumDiscardRatio,
maximumGeneratorSize,
perPropertyParallelism,
initialSeed)

@unused
private def unapply(c: CheckConfig): Some[CheckConfig] = Some(c)
}
Original file line number Diff line number Diff line change
Expand Up @@ -106,54 +106,33 @@ trait Checkers {

private def forall_[A: Show](gen: Gen[A], f: A => F[Expectations])(
implicit loc: SourceLocation): F[Expectations] = {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The changes here are a bit dense. To summarize:

  • paramStream: fs2.Stream[F, (Gen.Parameters, Seed)] has a fixed Gen.Parameters value, so can be better expressed as (Gen.Parameters, Stream[F, Seed]).
    Since we need the first seed, I've expressed it as:
    val (params, initialSeed) = startParams
    seedStream(initialSeed)
  • mapAccumulate(...).map(_.1) can be reduced to scan.
  • takeRight(1).compile.last can be reduced to compile.last.

paramStream
.parEvalMap(config.perPropertyParallelism) {
testOneTupled(gen, f)
}
.mapAccumulate(Status.start[A]) { case (oldStatus, testResult) =>
val newStatus = testResult match {
val params = Gen.Parameters.default.withNoInitialSeed.withSize(
config.maximumGeneratorSize)
val initialSeed = config.initialSeed.getOrElse(Seed.random())
seedStream(initialSeed)
.parEvalMap(config.perPropertyParallelism)(testOne(gen, f)(params, _))
.scan(Status.start[A]) { case (oldStatus, testResult) =>
testResult match {
case TestResult.Success => oldStatus.addSuccess
case TestResult.Discard => oldStatus.addDiscard
case TestResult.Failure(input, seed, exp) =>
oldStatus.addFailure(input, seed, exp)
case TestResult.Failure(input, exp) =>
oldStatus.addFailure(input, initialSeed, exp)
}
(newStatus, newStatus)
}
.map(_._1)
.takeWhile(_.shouldContinue(config), takeFailure = true)
.takeRight(1) // getting the first error (which finishes the stream)
.compile
.last
.map { (x: Option[Status[A]]) =>
x match {
case Some(status) => status.endResult(config)
case None => Expectations.Helpers.success
}
}
}

private def paramStream: fs2.Stream[F, (Gen.Parameters, Seed)] = {
val initial = startSeed(
Gen.Parameters.default
.withSize(config.maximumGeneratorSize)
.withInitialSeed(config.initialSeed.map(Seed(_))))

fs2.Stream.iterate(initial) {
case (p, s) => (p, s.slide)
}
.lastOrError // This will never fail as there will always be at least one status
.map { status => status.endResult(config) }
}

private def seedStream(initial: Seed): fs2.Stream[F, Seed] =
fs2.Stream.iterate[F, Seed](initial)(_.slide)
}

object forall extends PartiallyAppliedForall(checkConfig) {
def withConfig(config: CheckConfig) = new PartiallyAppliedForall(config)
}

private def testOneTupled[T: Show](
gen: Gen[T],
f: T => F[Expectations])(ps: (Gen.Parameters, Seed)) =
testOne(gen, f)(ps._1, ps._2)

private def testOne[T: Show](
gen: Gen[T],
f: T => F[Expectations])(
Expand All @@ -165,19 +144,13 @@ trait Checkers {
.map { (x: Option[(T, Expectations)]) =>
x match {
case Some((_, ex)) if ex.run.isValid => TestResult.Success
case Some((t, ex)) => TestResult.Failure(t.show, seed, ex)
case Some((t, ex)) => TestResult.Failure(t.show, ex)
case None => TestResult.Discard
}
}
}
}

def startSeed(params: Gen.Parameters): (Gen.Parameters, Seed) =
params.initialSeed match {
case Some(seed) => (params.withNoInitialSeed, seed)
case None => (params, Seed.random())
}

private[scalacheck] case class Status[T](
succeeded: Int,
discarded: Int,
Expand All @@ -192,7 +165,10 @@ trait Checkers {
val ith = succeeded + discarded + 1
val failure = Expectations.Helpers
.failure(
s"Property test failed on try $ith with seed ${seed} and input $input")
s"""Property test failed on try $ith with seed $seed and input $input.
|You can reproduce this by adding the following override to your suite:
|
|override def checkConfig = super.checkConfig.withInitialSeed($seed.toOption)""".stripMargin)
.and(exp)
copy(failure = Some(failure))
} else this
Expand Down Expand Up @@ -242,7 +218,7 @@ object Checkers {
private object TestResult {
case object Success extends TestResult
case object Discard extends TestResult
case class Failure(input: String, seed: Seed, exp: Expectations)
case class Failure(input: String, exp: Expectations)
extends TestResult
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ object CheckersConcurrencyTest extends SimpleIOSuite {
object CheckersConcurrencyTestNested extends SimpleIOSuite with Checkers {

override def checkConfig: CheckConfig =
super.checkConfig.copy(perPropertyParallelism = minTests * 5,
minimumSuccessful = minTests,
maximumDiscardRatio = 10)
super.checkConfig
.withPerPropertyParallelism(minTests * 5)
.withMinimumSuccessful(minTests)
.withMaximumDiscardRatio(10)

loggedTest("nested") { log =>
val atomicInt = new AtomicInteger(0)
Expand Down Expand Up @@ -57,9 +58,9 @@ object CheckersConcurrencyTest extends SimpleIOSuite {
object CheckersConcurrencyTestNested extends SimpleIOSuite with Checkers {

override def checkConfig: CheckConfig =
super.checkConfig.copy(perPropertyParallelism = 50,
minimumSuccessful = 10,
maximumDiscardRatio = 10)
super.checkConfig.withPerPropertyParallelism(50)
.withMinimumSuccessful(10)
.withMaximumDiscardRatio(10)

test("nested") {
val atomicInt = new AtomicInteger(0)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import org.scalacheck.Gen
object CheckersTest extends SimpleIOSuite with Checkers {

override def checkConfig: CheckConfig =
super.checkConfig.copy(perPropertyParallelism = 100)
super.checkConfig.withPerPropertyParallelism(100)

test("universal") {
forall(Gen.posNum[Int]) { a =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import cats.effect.{ IO, Resource }
import weaver.framework._

import org.scalacheck.Gen
import org.scalacheck.rng.Seed

object PropertyDogFoodTest extends IOSuite {

Expand All @@ -24,24 +25,29 @@ object PropertyDogFoodTest extends IOSuite {
case LoggedEvent.Error(msg) => msg
}
exists(errorLogs) { log =>
val seed = Meta.FailedChecks.initialSeed.toBase64
// Go into software engineering they say
// Learn how to make amazing algorithms
// Build robust and deterministic software
val (attempt, value, seed) =
val (attempt, value) =
if (ScalaCompat.isScala3) {
("4",
"-2147483648",
"""Seed.fromBase64("AkTFK0oQzv-BOkf-rqnsdb_Etapzkj9gQD9rHj7UnKM=")""")
("4", "-2147483648")
} else {
("2",
"0",
"""Seed.fromBase64("Nj62qCHF96VYEMGcD2OBlfmuyihbPQQhQLH9acYL5RA=")""")
("2", "0")
}

val expectedMessage =
s"Property test failed on try $attempt with seed $seed and input $value"

expect(log.contains(expectedMessage))
val actualLines = log.split(System.lineSeparator()).toList
val expectedLines = s"""foobar
|Property test failed on try $attempt with seed Seed.fromBase64("$seed") and input $value.
|You can reproduce this by adding the following override to your suite:
|
|override def checkConfig = super.checkConfig.withInitialSeed(Seed.fromBase64("$seed").toOption)"""
.stripMargin
.linesIterator.toList

forEach(actualLines.zip(expectedLines))({ case (actual, expected) =>
expect(actual.contains(expected))
})
}
}
}
Expand Down Expand Up @@ -102,13 +108,17 @@ object Meta {

override def checkConfig: CheckConfig =
super.checkConfig
.copy(perPropertyParallelism = 100, minimumSuccessful = 100)
.withPerPropertyParallelism(100)
.withMinimumSuccessful(100)
}

object FailedChecks extends SimpleIOSuite with Checkers {

val initialSeed = Seed(5L)
override def checkConfig: CheckConfig =
super.checkConfig.copy(perPropertyParallelism = 1, initialSeed = Some(5L))
super.checkConfig
.withPerPropertyParallelism(1)
.withInitialSeed(Some(initialSeed))

test("foobar") {
forall { (x: Int) =>
Expand All @@ -127,8 +137,9 @@ object Meta {
object ConfigOverrideChecks extends DiscardedChecks {

val configOverride =
super.checkConfig.copy(minimumSuccessful = 200,
perPropertyParallelism = 1)
super.checkConfig
.withMinimumSuccessful(200)
.withPerPropertyParallelism(1)

override def partiallyAppliedForall: PartiallyAppliedForall =
forall.withConfig(configOverride)
Expand All @@ -139,8 +150,10 @@ object Meta {
override def partiallyAppliedForall: PartiallyAppliedForall = forall

override def checkConfig =
super.checkConfig.copy(minimumSuccessful = 100,
// to avoid overcounting of discarded checks
perPropertyParallelism = 1)
super.checkConfig
.withMinimumSuccessful(100)
.withPerPropertyParallelism(
1
) // to avoid overcounting of discarded checks
}
}