Merge pull request #3 from geotrellis/feature/npz/mamlkind

Add typechecking, interpretation, and demo usage

.gitignore

@@ -47,3 +47,4 @@ index.js
 nohup.out
 
 site/
+.metadata/

build.sbt

@@ -1,6 +1,7 @@
 name := "MAML"
 
-scalaVersion in ThisBuild := "2.11.8"
+scalaVersion in ThisBuild := "2.11.11"
+
 
 lazy val root = project.in(file(".")).
   aggregate(mamlJS, mamlJVM).
@@ -15,19 +16,46 @@ lazy val maml = crossProject.in(file(".")).
     version := "0.1-SNAPSHOT",
     resolvers += Resolver.sonatypeRepo("releases"),
     addCompilerPlugin("org.scalamacros" % "paradise" % "2.1.0" cross CrossVersion.full),
+    scalacOptions := Seq(
+      "-deprecation",
+      "-unchecked",
+      "-feature",
+      "-language:implicitConversions",
+      "-language:reflectiveCalls",
+      "-language:higherKinds",
+      "-language:postfixOps",
+      "-language:existentials",
+      "-language:experimental.macros",
+      "-feature",
+      "-Ypartial-unification",
+      "-Ypatmat-exhaust-depth", "100"
+    ),
     libraryDependencies ++= Seq(
-      "org.scalacheck" %% "scalacheck"           % "1.13.4" % "test",
-      "org.scalatest"  %% "scalatest"            % "3.0.1"  % "test",
-      "io.circe"      %%% "circe-core"           % "0.8.0",
-      "io.circe"      %%% "circe-generic"        % "0.8.0",
-      "io.circe"      %%% "circe-generic-extras" % "0.8.0",
-      "io.circe"      %%% "circe-parser"         % "0.8.0",
-      "io.circe"      %%% "circe-optics"         % "0.8.0"
+      "org.scalacheck" %% "scalacheck"              % "1.13.4" % "test",
+      "org.scalatest"  %% "scalatest"               % "3.0.1"  % "test",
+      "org.typelevel"  %% "cats"                    % "0.9.0",
+      "io.circe"      %%% "circe-core"              % "0.8.0",
+      "io.circe"      %%% "circe-generic"           % "0.8.0",
+      "io.circe"      %%% "circe-generic-extras"    % "0.8.0",
+      "io.circe"      %%% "circe-parser"            % "0.8.0",
+      "io.circe"      %%% "circe-optics"            % "0.8.0"
     )
   )
   .jvmSettings(
+    resolvers += Resolver.bintrayRepo("hseeberger", "maven"),
     libraryDependencies ++= Seq(
-      "org.locationtech.geotrellis" %% "geotrellis-raster" % "1.1.1"
+      "org.locationtech.geotrellis" %% "geotrellis-raster"    % "1.1.1",
+      "org.locationtech.geotrellis" %% "geotrellis-spark"     % "1.1.1",
+      "org.locationtech.geotrellis" %% "geotrellis-s3"        % "1.1.1",
+      "org.apache.spark"            %% "spark-core"           % "2.0.0",
+      "com.typesafe.akka"           %% "akka-actor"           % "2.5.3",
+      "com.typesafe.akka"           %% "akka-stream"          % "2.5.3",
+      "com.typesafe.akka"           %% "akka-testkit"         % "2.5.3",
+      "com.typesafe.akka"           %% "akka-http"            % "10.0.9",
+      "com.typesafe.akka"           %% "akka-http-spray-json" % "10.0.9",
+      "com.typesafe.akka"           %% "akka-http-testkit"    % "10.0.9",
+      "de.heikoseeberger"           %% "akka-http-circe"      % "1.17.0"
+
     )
   )
   .jsSettings()

jvm/src/main/resources/application.conf

@@ -0,0 +1,9 @@
+akka {
+  loglevel = DEBUG
+}
+
+http {
+  interface = "0.0.0.0"
+  port = 9000
+}
+

jvm/src/main/scala/ast/jvm/TileLiteral.scala

@@ -0,0 +1,12 @@
+package maml.ast.jvm
+
+import maml.ast._
+
+import geotrellis.raster.Tile
+
+import java.util.UUID
+
+case class TileLiteral(tile: Tile) extends Source {
+  val kind = MamlKind.Tile
+  def id = UUID.randomUUID.toString
+}

jvm/src/main/scala/ast/jvm/ValueReaderTileSource.scala

@@ -0,0 +1,24 @@
+package maml.ast
+
+import io.circe._
+import io.circe.syntax._
+import io.circe.generic.extras.semiauto._
+import io.circe.generic.extras.Configuration
+
+import java.security.InvalidParameterException
+
+
+case class ValueReaderTileSource(bucket: String, root: String, layerId: String) extends Source {
+  val kind = MamlKind.Tile
+  def id = s"ValueReaderTileSrc-$bucket-$root-$layerId"
+}
+
+
+object ValueReaderTileSource {
+  implicit def conf: Configuration =
+    Configuration.default.withDefaults.withDiscriminator("type")
+
+  implicit lazy val decodeValueReaderTileSource: Decoder[ValueReaderTileSource] = deriveDecoder
+  implicit lazy val encodeValueReaderTileSource: Encoder[ValueReaderTileSource] = deriveEncoder
+}
+

jvm/src/main/scala/dsl/jvm/JvmLiterals.scala

@@ -0,0 +1,10 @@
+package maml.dsl
+
+import maml.ast.jvm._
+
+import geotrellis.raster.Tile
+
+
+trait JvmLiterals {
+  implicit def tileIsTileLiteral(tile: Tile): TileLiteral = TileLiteral(tile)
+}

jvm/src/main/scala/dsl/jvm/package.scala

@@ -0,0 +1,4 @@
+package maml.dsl
+
+
+package object jvm extends Literals with JvmLiterals with Operations

jvm/src/main/scala/eval/BufferingInterpreter.scala

@@ -0,0 +1,43 @@
+package maml.eval
+
+import maml.ast._
+import maml.error._
+
+import cats._
+import cats.implicits._
+import cats.data.Validated._
+import cats.data.{NonEmptyList => NEL, _}
+import geotrellis.raster.GridBounds
+
+import scala.reflect.ClassTag
+
+
+case class BufferingInterpreter(
+  directives: List[ScopedDirective[BufferingInterpreter.Scope]],
+  options: BufferingInterpreter.Options = BufferingInterpreter.Options(256)
+) extends ScopedInterpreter[BufferingInterpreter.Scope] {
+
+  def scopeFor(exp: Expression, previous: Option[BufferingInterpreter.Scope]): BufferingInterpreter.Scope = {
+    val scope = previous.getOrElse(BufferingInterpreter.Scope(0, options.tileSize))
+    exp match {
+      case f: FocalExpression => scope.copy(buffer = scope.buffer + f.neighborhood.extent)
+      case _ => scope
+    }
+  }
+
+  val fallbackDirective: ScopedDirective[BufferingInterpreter.Scope] =
+    { case (exp, res, scope) => Invalid(NEL.of(UnhandledCase(exp, exp.kind))) }
+
+  def instructions(expression: Expression, children: Seq[Result], scope: BufferingInterpreter.Scope): Interpreted[Result] =
+    directives.reduceLeft(_ orElse _).orElse(fallbackDirective)((expression, children, scope))
+}
+
+
+object BufferingInterpreter {
+  case class Options(tileSize: Int)
+  case class Scope(buffer: Int, tileSize: Int)
+
+  def gridbounds(expectedTileSize: Int, buffer: Int, extent: Int): GridBounds =
+    GridBounds(extent, extent, expectedTileSize - 1 + buffer * 2 + extent, expectedTileSize - 1 + buffer * 2 + extent)
+}
+

jvm/src/main/scala/eval/Interpreter.scala

@@ -0,0 +1,28 @@
+package maml.eval
+
+import maml.ast._
+import maml.error._
+
+import cats._
+import cats.implicits._
+import cats.data.Validated._
+import cats.data.{NonEmptyList => NEL, _}
+
+import scala.reflect.ClassTag
+
+
+trait Interpreter {
+  def fallbackDirective: Directive
+  def instructions(expression: Expression, children: List[Result]): Interpreted[Result]
+  def apply(exp: Expression): Interpreted[Result] = {
+    val children: Interpreted[List[Result]] = exp.children.map(apply).sequence
+    children.andThen({ childRes => instructions(exp, childRes) })
+  }
+}
+
+object Interpreter {
+  def naive(directives: Directive*) = NaiveInterpreter(directives.toList)
+
+  def buffering(directives: ScopedDirective[BufferingInterpreter.Scope]*) = BufferingInterpreter(directives.toList)
+}
+

jvm/src/main/scala/eval/NaiveInterpreter.scala

@@ -0,0 +1,21 @@
+package maml.eval
+
+import maml.ast._
+import maml.error._
+
+import cats._
+import cats.implicits._
+import cats.data.Validated._
+import cats.data.{NonEmptyList => NEL, _}
+
+import scala.reflect.ClassTag
+
+
+case class NaiveInterpreter(directives: List[Directive]) extends Interpreter {
+  val fallbackDirective: Directive =
+    { case (exp, res) => Invalid(NEL.of(UnhandledCase(exp, exp.kind))) }
+
+  def instructions(expression: Expression, children: List[Result]): Interpreted[Result] =
+    directives.reduceLeft(_ orElse _).orElse(fallbackDirective)((expression, children))
+}
+

jvm/src/main/scala/eval/Resolver.scala

@@ -0,0 +1,43 @@
+package maml.eval
+
+import maml.ast._
+import maml.ast.jvm.TileLiteral
+import maml.error._
+
+import geotrellis.raster._
+import geotrellis.raster.render._
+import geotrellis.spark._
+import geotrellis.spark.io._
+import geotrellis.spark.io.s3._
+
+import cats.data.{NonEmptyList => NEL}
+import cats.data.Validated._
+import cats.implicits._
+import scala.concurrent._
+import scala.util.{Try, Success, Failure}
+
+
+object Resolver {
+  def tmsLiteral(exp: Expression)(implicit ec: ExecutionContext): (Int, Int, Int) => Future[Interpreted[Expression]] = (z: Int, x: Int, y: Int) => {
+    exp match {
+      case ValueReaderTileSource(bucket, root, layerId) => Future {
+        val reader = S3ValueReader(bucket, root).reader[SpatialKey, Tile](LayerId(layerId, z))
+        Try {
+          reader.read(x, y)
+        } match {
+          case Success(tile) => Valid(TileLiteral(tile))
+          case Failure(e: ValueNotFoundError) => Invalid(NEL.of(S3TileResolutionError(exp, Some((z, x, y)))))
+          case Failure(e) =>  Invalid(NEL.of(UnknownTileResolutionError(exp, Some((z, x, y)))))
+        }
+      }
+      case _ =>
+        exp.children
+          .map({ child => tmsLiteral(child)(ec)(z, x, y) })
+          .toList.sequence
+          .map({ futureValidChildren => futureValidChildren.toList.sequence })
+          .map({ children =>
+            children.map({ exp.withChildren(_) })
+          })
+    }
+  }
+}

jvm/src/main/scala/eval/Result.scala

@@ -0,0 +1,81 @@
+package maml.eval
+
+import maml.error._
+import maml.ast._
+import maml.eval._
+import maml.eval.tile._
+
+import geotrellis.raster.Tile
+import geotrellis.vector.Geometry
+import cats.data.{NonEmptyList => NEL, _}
+import Validated._
+
+import scala.reflect.ClassTag
+
+
+sealed trait Result {
+  def as[T](implicit ct: ClassTag[T]): Interpreted[T]
+  def kind: MamlKind
+}
+
+case class DoubleResult(res: Double) extends Result {
+  def as[T](implicit ct: ClassTag[T]): Interpreted[T] = {
+    val cls = ct.runtimeClass
+    if (classOf[Int] isAssignableFrom cls)
+      Valid(res.toInt.asInstanceOf[T])
+    else if (classOf[Double] isAssignableFrom cls)
+      Valid(res.asInstanceOf[T])
+    else
+      Invalid(NEL.of(EvalTypeError(cls.getName, List("int", "double"))))
+  }
+  def kind: MamlKind = MamlKind.Double
+}
+
+case class IntResult(res: Int) extends Result {
+  def as[T](implicit ct: ClassTag[T]): Interpreted[T] = {
+    val cls = ct.runtimeClass
+    if (classOf[Int] isAssignableFrom cls)
+      Valid(res.toInt.asInstanceOf[T])
+    else if (classOf[Double] isAssignableFrom cls)
+      Valid(res.toDouble.asInstanceOf[T])
+    else
+      Invalid(NEL.of(EvalTypeError(cls.getName, List("int", "double"))))
+  }
+  def kind: MamlKind = MamlKind.Int
+}
+
+case class GeomResult(res: Geometry) extends Result {
+  def as[T](implicit ct: ClassTag[T]): Interpreted[T] = {
+    val cls = ct.runtimeClass
+    if (classOf[Geometry] isAssignableFrom cls)
+      Valid(res.asInstanceOf[T])
+    else
+      Invalid(NEL.of(EvalTypeError(cls.getName, List("geom"))))
+  }
+  def kind: MamlKind = MamlKind.Geom
+}
+
+case class TileResult(res: LazyTile) extends Result {
+  def as[T](implicit ct: ClassTag[T]): Interpreted[T] = {
+    val cls = ct.runtimeClass
+    if (classOf[Tile] isAssignableFrom cls)
+      Valid(res.evaluate.asInstanceOf[T])
+    else if (classOf[LazyTile] isAssignableFrom cls)
+      Valid(res.asInstanceOf[T])
+    else
+      Invalid(NEL.of(EvalTypeError(cls.getName, List("Tile"))))
+  }
+  def kind: MamlKind = MamlKind.Tile
+}
+
+case class BoolResult(res: Boolean) extends Result {
+  def as[T](implicit ct: ClassTag[T]): Interpreted[T] = {
+    val cls = ct.runtimeClass
+    if (classOf[Boolean] isAssignableFrom cls)
+      Valid(res.asInstanceOf[T])
+    else
+      Invalid(NEL.of(EvalTypeError(cls.getName, List("bool"))))
+  }
+  def kind: MamlKind = MamlKind.Bool
+}
+

jvm/src/main/scala/eval/ScopedInterpreter.scala

@@ -0,0 +1,28 @@
+package maml.eval
+
+import maml.ast._
+import maml.error._
+
+import cats._
+import cats.implicits._
+import cats.data.Validated._
+import cats.data.{NonEmptyList => NEL, _}
+import geotrellis.raster.GridBounds
+
+
+trait ScopedInterpreter[Scope] {
+  def scopeFor(exp: Expression, previous: Option[Scope]): Scope
+  def fallbackDirective: ScopedDirective[Scope]
+  def instructions(expression: Expression, children: Seq[Result], scope: Scope): Interpreted[Result]
+
+  def apply(exp: Expression, maybeScope: Option[Scope] = None): Interpreted[Result] = {
+    val currentScope = scopeFor(exp, maybeScope)
+    val children: Interpreted[List[Result]] = exp.children.map({ childTree =>
+      val childScope = scopeFor(childTree, Some(currentScope))
+      apply(childTree, Some(childScope))
+    }).sequence
+
+    children.andThen({ childResult => instructions(exp, childResult, currentScope) })
+  }
+}
+