[WIP] [DO NOT MERGE] improved integration tests a little bit

.github/CODEOWNERS

@@ -0,0 +1 @@
+.github/ @lbialy @tgodzik

.gitignore

@@ -38,10 +38,13 @@ target/
 # test files
 *.tml
 *.yaml
-tests/repositories/
+integration-tests/repositories/
 
 #general
 *.class
 *.log
 
 generated-docs
+
+# java=17.0.9-graalce if you use sdkman
+.sdkmanrc 

core/shared/src/main/scala/org/virtuslab/yaml/Node.scala

@@ -23,11 +23,45 @@ sealed trait Node {
 }
 
 object Node {
-  final case class ScalarNode private[yaml] (value: String, tag: Tag, pos: Option[Range] = None)
-      extends Node
+
+  import org.virtuslab.yaml.internal.load.reader.token.{ScalarStyle => ParserScalarStyle}
+
+  sealed abstract class ScalarStyle(indicator: Char) {
+    def toParserStyle: ParserScalarStyle = this match {
+      case ScalarStyle.Plain        => ParserScalarStyle.Plain
+      case ScalarStyle.DoubleQuoted => ParserScalarStyle.DoubleQuoted
+      case ScalarStyle.SingleQuoted => ParserScalarStyle.SingleQuoted
+      case ScalarStyle.Folded       => ParserScalarStyle.Folded
+      case ScalarStyle.Literal      => ParserScalarStyle.Literal
+    }
+  }
+  object ScalarStyle {
+
+    def fromParserStyle(style: ParserScalarStyle): ScalarStyle = style match {
+      case ParserScalarStyle.Plain        => ScalarStyle.Plain
+      case ParserScalarStyle.DoubleQuoted => ScalarStyle.DoubleQuoted
+      case ParserScalarStyle.SingleQuoted => ScalarStyle.SingleQuoted
+      case ParserScalarStyle.Folded       => ScalarStyle.Folded
+      case ParserScalarStyle.Literal      => ScalarStyle.Literal
+    }
+
+    case object Plain        extends ScalarStyle(' ')
+    case object DoubleQuoted extends ScalarStyle('"')
+    case object SingleQuoted extends ScalarStyle('\'')
+    case object Folded       extends ScalarStyle('>')
+    case object Literal      extends ScalarStyle('|')
+  }
+
+  final case class ScalarNode private[yaml] (
+      value: String,
+      tag: Tag,
+      style: ScalarStyle,
+      pos: Option[Range] = None
+  ) extends Node
 
   object ScalarNode {
-    def apply(value: String): ScalarNode = new ScalarNode(value, Tag.resolveTag(value))
+    def apply(value: String): ScalarNode =
+      new ScalarNode(value, style = ScalarStyle.Plain, tag = Tag.resolveTag(value))
     def unapply(node: ScalarNode): Option[(String, Tag)] = Some((node.value, node.tag))
   }
 

core/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala

@@ -48,7 +48,7 @@ object YamlEncoder extends YamlEncoderCrossCompanionCompat {
 
   implicit def forOption[T](implicit encoder: YamlEncoder[T]): YamlEncoder[Option[T]] = {
     case Some(t) => encoder.asNode(t)
-    case None    => Node.ScalarNode("", Tag.nullTag)
+    case None    => Node.ScalarNode("", Tag.nullTag, Node.ScalarStyle.Plain)
   }
 
   implicit def forSet[T](implicit encoder: YamlEncoder[T]): YamlEncoder[Set[T]] = (nodes) =>

core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/present/PresenterImpl.scala

@@ -8,6 +8,7 @@ import org.virtuslab.yaml.Tag
 import org.virtuslab.yaml.internal.load.parse.EventKind
 import org.virtuslab.yaml.internal.load.parse.EventKind._
 import org.virtuslab.yaml.internal.load.parse.NodeEventMetadata
+import org.virtuslab.yaml.internal.load.reader.token.ScalarStyle
 
 object PresenterImpl extends Presenter {
   override def asString(events: Seq[EventKind]): String = {
@@ -30,11 +31,18 @@ object PresenterImpl extends Presenter {
               insertSequencePadding()
               pushAndIncreaseIndent(SequenceStart())
               parseSequence(tail)
-            case Scalar(value, _, NodeEventMetadata(_, tag)) =>
+            case Scalar(value, style, NodeEventMetadata(_, tag)) =>
               insertSequencePadding()
               // todo escape string using doublequotes
               if (tag.contains(Tag.nullTag)) sb.append("!!null")
-              else sb.append(value)
+              else
+                style match {
+                  case ScalarStyle.Plain        => sb.append(value)
+                  case ScalarStyle.DoubleQuoted => sb.append(s"\"$value\"")
+                  case ScalarStyle.SingleQuoted => sb.append(s"'$value'")
+                  case ScalarStyle.Folded       => sb.append(value)
+                  case ScalarStyle.Literal      => sb.append(value)
+                }
               sb.append(newline)
               tail
             case DocumentStart(_) => parseNode(tail)
@@ -50,8 +58,8 @@ object PresenterImpl extends Presenter {
         case MappingEnd :: tail =>
           popAndDecreaseIndent()
           tail
-        case Scalar(value, _, _) :: tail =>
-          appendKey(value)
+        case Scalar(value, style, _) :: tail =>
+          appendKey(value, style)
           val rest = parseNode(tail)
           parseMapping(rest)
         case _ => events
@@ -69,9 +77,15 @@ object PresenterImpl extends Presenter {
           parseSequence(rest)
       }
 
-    def appendKey(value: String) = {
+    def appendKey(value: String, style: ScalarStyle) = {
       sb.append(" " * indent)
-      sb.append(value)
+      style match {
+        case ScalarStyle.Plain        => sb.append(value)
+        case ScalarStyle.DoubleQuoted => sb.append(s"\"$value\"")
+        case ScalarStyle.SingleQuoted => sb.append(s"'$value'")
+        case ScalarStyle.Folded       => sb.append(value)
+        case ScalarStyle.Literal      => sb.append(value)
+      }
       sb.append(": ")
     }
 

core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/serialize/SerializerImpl.scala

@@ -29,6 +29,12 @@ object SerializerImpl extends Serializer {
   }
 
   private def convertScalarNode(node: Node.ScalarNode): Seq[EventKind] =
-    Seq(Scalar(node.value, metadata = NodeEventMetadata(tag = node.tag)))
+    Seq(
+      Scalar(
+        node.value,
+        style = node.style.toParserStyle,
+        metadata = NodeEventMetadata(tag = node.tag)
+      )
+    )
 
 }

core/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala

@@ -48,7 +48,8 @@ object ComposerImpl extends Composer {
           composeMappingNode(tail, anchor, aliases)
         case s: EventKind.Scalar =>
           val tag: Tag = s.metadata.tag.getOrElse(Tag.resolveTag(s.value, Some(s.style)))
-          val node     = Node.ScalarNode(s.value, tag, head.pos)
+          val node =
+            Node.ScalarNode(s.value, tag, Node.ScalarStyle.fromParserStyle(s.style), head.pos)
           s.metadata.anchor.foreach(anchor => aliases.put(anchor, node))
           Right(Result(node, tail))
         // todo #88

core/shared/src/test/scala/org/virtuslab/yaml/traverse/NodeVisitorSuite.scala

@@ -27,17 +27,17 @@ class NodeVisitorSuite extends munit.FunSuite {
     val modifiedYaml = modifiedNode.asYaml
 
     val exptectedYaml =
-      s"""version: 3.9
+      s"""version: "3.9"
          |services: 
          |  web: 
          |    build: .
          |    volumes: 
          |      - .:/code
          |      - logvolume01:/var/log
          |    ports: 
-         |      - 6000:6000
+         |      - "6000:6000"
          |  redis: 
-         |    image: redis:alpine
+         |    image: "redis:alpine"
          |""".stripMargin
 
     assertEquals(modifiedYaml, exptectedYaml)
@@ -71,17 +71,17 @@ class NodeVisitorSuite extends munit.FunSuite {
     val modifiedYaml = modifiedNode.asYaml
 
     val exptectedYaml =
-      s"""version: 3.9
+      s"""version: "3.9"
          |services: 
          |  web: 
          |    build: .
          |    volumes: 
          |      - .:/code
          |      - logvolume01:/var/log
          |    ports: 
-         |      - 5000:5000:6000
+         |      - "5000:5000:6000"
          |  redis: 
-         |    image: redis:alpine:latest
+         |    image: "redis:alpine:latest"
          |""".stripMargin
 
     assertEquals(modifiedYaml, exptectedYaml)
@@ -115,7 +115,7 @@ class NodeVisitorSuite extends munit.FunSuite {
     val modifiedYaml = modifiedNode.asYaml
 
     val exptectedYaml =
-      s"""version: 3.9
+      s"""version: "3.9"
          |services: 
          |  web: 
          |    build: .

integration-tests/README.md

@@ -1,3 +1,9 @@
 ### Test suite
 
-Yaml test suite are used in integration testing - https://github.com/yaml/yaml-test-suite
+Yaml test suite are used in integration testing - https://github.com/yaml/yaml-test-suite
+
+#### How to run integration tests
+
+1. Figure out if `yamlpp-events` is installed on your system or install it if not, it's perl's yaml pp module
+2. Run `downloadYamlConfigs.sh` script to download yaml test suites
+3. `sbt integration/test` to run tests

integration-tests/downloadYamlConfigs.sh

@@ -26,13 +26,14 @@ do
 
 done
 
-mkdir ./test-suite/jvm/src/it/resources/yaml/configs/
-find ./repositories -name '*.yaml' -exec cp -prv '{}' './test-suite/jvm/src/it/resources/yaml/configs/' ';'
+mkdir -p ./src/test/resources/yaml/configs/
+find ./repositories -name '*.yaml' -exec cp -prv '{}' './src/test/resources/yaml/configs/' ';'
 
-LIB_YAML_PATH="" # Set the path to libyaml
+# lbialy: I think the process used here is 'yamlpp-events' so `which yamlpp-events` should be used
+LIB_YAML_PATH=`which yamlpp-events` # Set the path to libyaml
 
 # In downloaded repositories contains some invalid yaml, below instructions can remove this yaml
-for f in ./test-suite/jvm/src/it/resources/yaml/configs/*.yaml; do
+for f in ./src/test/resources/yaml/configs/*.yaml; do
     cat $f | $LIB_YAML_PATH >/dev/null
 
     # if libyaml return error exit code, these means, that yaml is invalid

integration-tests/src/main/scala/org/virtuslab/yaml/RunnerResult.scala

@@ -20,6 +20,11 @@ object RunnerResult {
     else Error(eventsAsStr, error)
   }
 
+  case class InvalidOutYaml(eventsResult: RunnerResult, outYaml: String, expectedOutYaml: String)
+      extends RunnerResult {
+    override val isValid = false
+  }
+
   case class Success(events: String) extends RunnerResult {
     override val isValid = true
   }

integration-tests/src/main/scala/org/virtuslab/yaml/TestMlEntry.scala

@@ -5,6 +5,7 @@ case class TestMlEntry(
     from: String,
     tags: String,
     inYaml: String,
+    outYaml: Option[String],
     seqEvent: String
 )
 
@@ -24,6 +25,21 @@ case object TestMlEntry {
       .head
   }
 
+  private def extractOutYaml(testMl: String): Option[String] = {
+    val patternOut =
+      raw"--- out-yaml(\(<\)|\(\+\)|\(<\+\)|)(([^\n]*\n+)+?)--- (in-json|error|emit-yaml|test-event)".r
+
+    patternOut
+      .findAllIn(testMl)
+      .matchData
+      .map { m =>
+        m.group(2)
+      }
+      .toList
+      .headOption
+      .map(_.strip())
+  }
+
   private def extractSeqEvent(testMl: String): String = {
     val patternEvent = raw"--- test-event(([^\n]*\n+)+).*".r
 
@@ -42,6 +58,7 @@ case object TestMlEntry {
       from = "",
       tags = "",
       inYaml = extractInYaml(content),
+      outYaml = extractOutYaml(content),
       seqEvent = extractSeqEvent(content)
     )
   }

integration-tests/src/main/scala/org/virtuslab/yaml/TestRunner.scala

@@ -16,7 +16,9 @@ import org.virtuslab.yaml.internal.load.reader.token.ScalarStyle
 import org.virtuslab.yaml.internal.load.reader.Tokenizer
 
 trait TestRunner {
+  def testOutYaml: Boolean
   def inYaml: String
+  def outYaml: Option[String] = None
   def expectedEvents: String
 
   def run(): RunnerResult = {
@@ -35,7 +37,28 @@ trait TestRunner {
           RunnerResult(acc.toList, expectedEvents, error)
       }
     }
-    loop()
+
+    val result = loop()
+
+    if (testOutYaml) {
+      // we don't compare output yaml if there is an error in event parsing
+      if (
+        outYaml.isDefined && (result match {
+          case RunnerResult.Error(_, _) => false; case _ => true
+        })
+      ) {
+        val roundtrip = inYaml.asNode match
+          case Left(error) =>
+            println(s"Error while parsing inYaml:\n$error")
+            s"!!! ERROR: $error"
+          case Right(value) => value.asYaml
+
+        if (roundtrip != outYaml.map(_.appended('\n')).get)
+          RunnerResult.InvalidOutYaml(result, roundtrip, outYaml.get)
+        else result
+      } else result
+
+    } else result
   }
 
 }
@@ -69,10 +92,11 @@ object TestRunnerUtils {
         case MappingEnd              => "-MAP"
         case Alias(alias)            => s"=ALI *$alias"
         case Scalar(value, style, data) =>
+          val escapedValue = value.replace("\\", "\\\\").replace("\n", "\\n")
           style match {
             case ScalarStyle.Plain        => s"=VAL${data.asString} :$value"
             case ScalarStyle.DoubleQuoted => s"""=VAL${data.asString} "$value"""
-            case ScalarStyle.SingleQuoted => s"=VAL${data.asString} '$value"
+            case ScalarStyle.SingleQuoted => s"=VAL${data.asString} '$escapedValue"
             case ScalarStyle.Folded       => s"=VAL${data.asString} >$value"
             case ScalarStyle.Literal      => s"=VAL${data.asString} |$value"
           }
@@ -81,7 +105,8 @@ object TestRunnerUtils {
 
 }
 
-case class K8sYamlTestRunner(yamlPath: os.Path, libYaml: os.Path) extends TestRunner {
+case class K8sYamlTestRunner(yamlPath: os.Path, libYaml: os.Path, testOutYaml: Boolean)
+    extends TestRunner {
   override val inYaml = os.read(yamlPath)
   override val expectedEvents = os
     .proc(libYaml, yamlPath)
@@ -97,9 +122,10 @@ case class K8sYamlTestRunner(yamlPath: os.Path, libYaml: os.Path) extends TestRu
 
 }
 
-case class YamlSuiteTestRunner(testYamlML: os.Path) extends TestRunner {
+case class YamlSuiteTestRunner(testYamlML: os.Path, testOutYaml: Boolean) extends TestRunner {
   private val testMl = TestMlEntry.from(testYamlML)
 
+  override val outYaml        = testMl.outYaml
   override val inYaml         = testMl.inYaml
   override val expectedEvents = testMl.seqEvent
 

integration-tests/src/test/scala/org/virtuslab/yaml/K8sConfigSpec.scala

@@ -4,7 +4,8 @@ class K8sConfigSpec extends YamlRunnerSpec {
 
   val libYamlPath = os.Path(System.getenv("LIB_YAML_PATH"))
 
-  override def resourcePath: String                   = "/yaml/configs"
-  def createTestRunner(yamlPath: os.Path): TestRunner = K8sYamlTestRunner(yamlPath, libYamlPath)
+  override def resourcePath: String = "/yaml/configs"
+  def createTestRunner(yamlPath: os.Path): TestRunner =
+    K8sYamlTestRunner(yamlPath, libYamlPath, testOutYaml)
 
 }

integration-tests/src/test/scala/org/virtuslab/yaml/OfficialTestSuiteSpec.scala

@@ -1,7 +1,17 @@
 package org.virtuslab.yaml
 
+import scala.util.matching.Regex
+
 class OfficialTestSuiteSpec extends YamlRunnerSpec {
 
-  override def resourcePath: String                            = "/yaml/test-suite"
-  override def createTestRunner(yamlPath: os.Path): TestRunner = YamlSuiteTestRunner(yamlPath)
+  override val verbose: Boolean           = true
+  override val testOutYaml: Boolean       = false
+  override val skipErrors: Boolean        = false
+  override val skipInvalidEvents: Boolean = false
+
+  // override def runMatching: Regex = ".*LQZ7.*".r
+
+  override def resourcePath: String = "/yaml/test-suite"
+  override def createTestRunner(yamlPath: os.Path): TestRunner =
+    YamlSuiteTestRunner(yamlPath, testOutYaml)
 }