Add AST dump as outputter and inputter

.cirrus.yml

@@ -36,6 +36,7 @@ task:
     - luarocks51 install luarepl
     - luarocks51 install luautf8
     - luarocks51 install penlight
+    - luarocks51 install serpent
     - luarocks51 install vstruct
   bootstrap_script:
     - git fetch --prune --tags ||:

build-aux/pkg.nix

@@ -45,6 +45,7 @@ let
     luasocket
     luautf8
     penlight
+    serpent
     vstruct
     # lua packages needed for testing
     busted

configure.ac

@@ -215,6 +215,7 @@ AM_COND_IF([SYSTEM_LUAROCKS], [
     AX_LUA_MODULE([socket], [luasocket])
     AX_LUA_MODULE([lua-utf8], [luautf8])
     AX_LUA_MODULE([pl], [penlight])
+    AX_LUA_MODULE([serpent], [serpent])
     AX_LUA_MODULE([vstruct], [vstruct])
 ], [
     QUE_PROGVAR([luarocks])

core/sile.lua

@@ -218,11 +218,14 @@ function SILE.init ()
       SILE.shaper = SILE.shapers.harfbuzz()
       SILE.outputter = SILE.outputters.debug()
    elseif SILE.backend == "text" then
-      SILE.shaper = SILE.shapers.harfbuzz()
+      SILE.shaper = SILE.shapers.dummy()
       SILE.outputter = SILE.outputters.text()
    elseif SILE.backend == "dummy" then
-      SILE.shaper = SILE.shapers.harfbuzz()
+      SILE.shaper = SILE.shapers.dummy()
       SILE.outputter = SILE.outputters.dummy()
+   elseif SILE.backend == "ast" then
+      SILE.shaper = SILE.shapers.dummy()
+      SILE.outputter = SILE.outputters.ast()
    end
    SILE.pagebuilder = SILE.pagebuilders.base()
    io.stdout:setvbuf("no")
@@ -399,6 +402,7 @@ end
 -- iterate through the tree handling each item in the order encountered.
 -- @tparam table ast SILE content in abstract syntax tree format (a table of strings, functions, or more AST trees).
 function SILE.process (ast)
+   ast = SILE.outputter:preProcess(ast)
    if not ast then
       return
    end
@@ -427,7 +431,7 @@ function SILE.process (ast)
    end
 end
 
-local preloadedinputters = { "xml", "lua", "sil" }
+local preloadedinputters = { "xml", "lua", "ast", "sil" }
 
 local function detectFormat (doc, filename)
    -- Preload default reader types so content detection has something to work with

inputters/ast.lua

@@ -0,0 +1,42 @@
+local base = require("inputters.base")
+local serpent = require("serpent")
+
+local inputter = pl.class(base)
+inputter._name = "ast"
+
+inputter.order = 35
+
+function inputter.appropriate (round, filename, doc)
+   if round == 1 then
+      return filename:match(".ast$")
+   elseif round == 2 then
+      local sniff = doc:sub(1, 100)
+      local promising = sniff:match("^{\n   [^ ]") or sniff:match("command =") or sniff:match("loadstring or load")
+      return promising and inputter.appropriate(3, filename, doc) or false
+   elseif round == 3 then
+      local status, _ = serpent.load(doc, { safe = true })
+      return status and true or false
+   end
+end
+
+function inputter.parse (_, doc)
+   local status, result = serpent.load(doc, { safe = true })
+   if not status then
+      SU.error(result)
+   end
+   return result
+end
+
+function inputter:process (doc)
+   local tree = self:parse(doc)
+   if not tree.type then
+      -- hoping tree is an AST
+      self:requireClass(tree)
+      return SILE.process(tree)
+   else
+      SILE.use(tree, self.options)
+   end
+end
+
+return inputter
+

outputters/ast.lua

@@ -0,0 +1,41 @@
+local base = require("outputters.base")
+local serpent = require("serpent")
+
+local outputter = pl.class(base)
+outputter._name = "ast"
+outputter.extension = "ast"
+
+local outfile
+
+function outputter:_ensureInit ()
+   if not outfile then
+      local fname = self:getOutputFilename()
+      outfile = fname == "-" and io.stdout or io.open(fname, "w+")
+   end
+end
+
+function outputter:preProcess (ast)
+   self:_ensureInit()
+   local serialized = serpent.serialize(
+      ast,
+      {
+         comment = false,
+         compact = true,
+         fatal = true,
+         indent = "   ",
+         metatostring = false,
+         sortkeys = true,
+         sparse = true,
+      }
+   )
+   outfile:write(serialized)
+   -- Only dump the *first* AST we're given dodging SILE generated ones (like folios) later
+   self.preProcess = function () end
+end
+
+function outputter:finish ()
+   self:runHooks("prefinish")
+   outfile:close()
+end
+
+return outputter

outputters/base.lua

@@ -27,6 +27,10 @@ function outputter:runHooks (category, data)
    return data
 end
 
+function outputter.preProcess (_, ast)
+   return ast
+end
+
 function outputter.newPage () end
 
 function outputter:finish ()

shapers/dummy.lua

@@ -0,0 +1,18 @@
+local base = require("shapers.base")
+
+local shaper = pl.class(base)
+shaper._name = "dummy"
+
+function shaper.addShapedGlyphToNnodeValue (_, _, _) end
+
+function shaper.getFace () end
+
+function shaper.createNnodes (_, _)
+   return {}
+end
+
+function shaper.shapeToken (_, _, _)
+   return {}
+end
+
+return shaper

sile.rockspec.in

@@ -43,6 +43,7 @@ dependencies = {
    "luasocket == 3.1.0-1",
    "luautf8 == 0.1.5-2",
    "penlight == 1.14.0-2",
+   "serpent == 0.30-2",
    "vstruct == 2.1.1-1",
 }