diff --git a/build.sbt b/build.sbt index 6909479..271d8ed 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ ThisBuild / organization := "io.appthreat" ThisBuild / version := "1.6.0" ThisBuild / scalaVersion := "3.3.1" -val chenVersion = "0.6.3" +val chenVersion = "1.0.0" lazy val atom = Projects.atom @@ -170,6 +170,8 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Compile / doc / sources := Seq.empty Compile / packageDoc / publishArtifact := false +wartremoverWarnings ++= Seq(Wart.NoNeedImport, Wart.ArrayEquals, Wart.Any, Wart.FinalCaseClass, Wart.FinalVal, Wart.ToString, Wart.TryPartial) + githubOwner := "appthreat" githubRepository := "atom" githubSuppressPublicationWarning := true diff --git a/project/plugins.sbt b/project/plugins.sbt index 86cd746..3e24acb 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -6,3 +6,4 @@ addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16") addSbtPlugin("io.shiftleft" % "sbt-overflowdb" % "2.29") addSbtPlugin("com.codecommit" % "sbt-github-packages" % "0.5.2") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.1") +addSbtPlugin("org.wartremover" % "sbt-wartremover" % "3.1.5") diff --git a/src/main/scala/io/appthreat/atom/Atom.scala b/src/main/scala/io/appthreat/atom/Atom.scala index 096ecc4..46e6e0e 100644 --- a/src/main/scala/io/appthreat/atom/Atom.scala +++ b/src/main/scala/io/appthreat/atom/Atom.scala @@ -33,6 +33,7 @@ import io.shiftleft.codepropertygraph.generated.Languages import io.shiftleft.semanticcpg.layers.LayerCreatorContext import scopt.OptionParser +import java.util.Locale import scala.language.postfixOps import scala.util.{Failure, Properties, Success} @@ -363,7 +364,7 @@ object Atom: .withMethodAnnotationFilter(x.methodAnnotationFilter) private def generateAtom(config: BaseConfig, language: String): Either[String, String] = - generateForLanguage(language.toUpperCase, config) + generateForLanguage(language.toUpperCase(Locale.ROOT), config) private def generateForLanguage(language: String, config: BaseConfig): Either[String, String] = val outputAtomFile = config match diff --git a/src/main/scala/io/appthreat/atom/dataflows/DataFlowGraph.scala b/src/main/scala/io/appthreat/atom/dataflows/DataFlowGraph.scala index 20eaae3..9d1f9bb 100644 --- a/src/main/scala/io/appthreat/atom/dataflows/DataFlowGraph.scala +++ b/src/main/scala/io/appthreat/atom/dataflows/DataFlowGraph.scala @@ -12,15 +12,12 @@ private class DataFlowGraph(nodes: Set[Option[DFNode]]): // Maximum number of data-flow paths to compute private val MAX_PATHS = 100 - private val USEFUL_PATH_LABELS = List("METHOD_PARAMETER_IN", "CALL") - def paths: Set[Path] = implicit val finalSet: mutable.Set[Path] = mutable.Set.empty implicit val nMap: Map[Long, DFNode] = nodes.map(x => x.get.id -> x.get).toMap nodes.foreach { n => - val currPath = List(n.get.id) - val currLabelPath = List(n.get.label) - follow(currPath, currLabelPath, n.get.out.flatMap(nMap.get)) + val currPath = List(n.get.id) + follow(currPath, n.get.out.flatMap(nMap.get)) } finalSet.toSet @@ -32,39 +29,30 @@ private class DataFlowGraph(nodes: Set[Option[DFNode]]): private def isSubList[A](lst: List[A])(implicit finalSet: mutable.Set[Path]): Boolean = finalSet.filterNot(_.size < lst.size).exists(xs => isSubList(lst, xs)) - /** A given path is useful if it starts with a METHOD_PARAMETER_IN contains at least 1 CALL and - * a METHOD_PARAMETER_IN nodes - */ - private def isUsefulPath(finalSet: mutable.Set[Path], path: List[String]): Boolean = - path.last == "METHOD_PARAMETER_IN" || (path.contains( - "CALL" - ) && path.size > 5) || path.count(x => - x != "IDENTIFIER" - ) > 2 - /** Is there an existing path that starts and ends with the same node */ private def isDuplicate(finalSet: mutable.Set[Path], path: Path): Boolean = - finalSet.exists(apath => apath.head == path.head && apath.last == path.last) + finalSet.exists(apath => + apath.headOption == path.headOption && apath.lastOption == path.lastOption + ) - private def follow(currPath: List[Long], currLabelPath: List[String], outNodes: Set[DFNode])( + private def follow(currPath: List[Long], outNodes: Set[DFNode])( implicit nMap: Map[Long, DFNode], finalSet: mutable.Set[Path] ): Unit = outNodes.foreach { x => - val path = currPath :+ x.id - val labelPath = currLabelPath :+ x.label - val queue = x.out.filterNot(currPath.contains) + val path = currPath :+ x.id + val queue = x.out.filterNot(currPath.contains) if queue.isEmpty then if !isDuplicate(finalSet, path) && !isSubList(path) then finalSet.add(path) else if finalSet.size < MAX_PATHS then - follow(path, labelPath, queue.flatMap(nMap.get)) + follow(path, queue.flatMap(nMap.get)) } end DataFlowGraph -private case class DFNode( +private final case class DFNode( id: Long, isExternal: Boolean, label: String, diff --git a/src/main/scala/io/appthreat/atom/passes/SafeConcurrentCpgPass.scala b/src/main/scala/io/appthreat/atom/passes/SafeConcurrentCpgPass.scala index 87378ad..689d997 100644 --- a/src/main/scala/io/appthreat/atom/passes/SafeConcurrentCpgPass.scala +++ b/src/main/scala/io/appthreat/atom/passes/SafeConcurrentCpgPass.scala @@ -43,35 +43,31 @@ abstract class SafeConcurrentCpgPass[T <: AnyRef]( writerThread.setName("Writer") writerThread.start() implicit val ec: ExecutionContext = ExecutionContextProvider.getExecutionContext + var done = false try + while !done || completedParts < nParts do + if completionQueue.size < producerQueueCapacity && partIter.hasNext then + val next = partIter.next() + completionQueue.prepend(Future.apply { + val builder = new DiffGraphBuilder + runOnPart(builder, next.asInstanceOf[T]) + builder + }) + else if completionQueue.nonEmpty then + val future = completionQueue.removeLast() + val res = Await.result(future, Duration.Inf).build() + nDiff += res.size + writer.queue.put(Some(res)) + completedParts += 1 + else + writer.queue.put(None) + completedParts += 1 + done = true + finally try - var done = false - while !done || completedParts < nParts do - if completionQueue.size < producerQueueCapacity && partIter.hasNext then - val next = partIter.next() - completionQueue.prepend(Future.apply { - val builder = new DiffGraphBuilder - runOnPart(builder, next.asInstanceOf[T]) - builder - }) - else if completionQueue.nonEmpty then - val future = completionQueue.removeLast() - val res = Await.result(future, Duration.Inf).build() - nDiff += res.size - writer.queue.put(Some(res)) - completedParts += 1 - else { - writer.queue.put(None) - completedParts += 1 - done = true - } + writerThread.join() finally - try - writerThread.join() - finally finish() - finally { - // pass - } + finish() end try end createApplySerializeAndStore @@ -83,11 +79,12 @@ abstract class SafeConcurrentCpgPass[T <: AnyRef]( ) override def run(): Unit = - try - nDiffT = 0 - var terminate = false - var index: Int = 0 - while !terminate do + var terminate = false + var index: Int = 0 + nDiffT = 0 + var hadErrors = false + while !terminate do + try queue.take() match case None => terminate = true @@ -96,8 +93,7 @@ abstract class SafeConcurrentCpgPass[T <: AnyRef]( .applyDiff(cpg.graph, diffGraph, keyPool.orNull, null) .transitiveModifications() index += 1 - finally { - // pass - } + finally + hadErrors = true end Writer end SafeConcurrentCpgPass diff --git a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala index 4832409..8b44a41 100644 --- a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala +++ b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala @@ -72,7 +72,7 @@ object ReachableSlicing: val addedPaths = mutable.Set[String]() val purls = mutable.Set[String]() path.elements.foreach { astNode => - val lineNumber = astNode.lineNumber.getOrElse("").toString + val lineNumber = astNode.lineNumber.map(_.intValue()) val fileName = astNode.file.name.headOption.getOrElse("").replace("", "") var fileLocation = s"${fileName}#${lineNumber}" var tags: String = tagAsString(astNode.tag)