WIP: new parser attempt

src/bin/doodle/main.rs

@@ -5,8 +5,9 @@ use std::fs;
 use std::path::PathBuf;
 
 use clap::{Parser, ValueEnum};
-use doodle::decoder::Compiler;
+use doodle::decoder;
 use doodle::read::ReadCtxt;
+use doodle::streamer;
 use doodle::FormatModule;
 
 mod format;
@@ -61,7 +62,7 @@ fn main() -> Result<(), Box<dyn std::error::Error + 'static>> {
                 FormatOutput::Debug => println!("{module:?}"),
                 FormatOutput::Json => serde_json::to_writer(std::io::stdout(), &module).unwrap(),
                 FormatOutput::Rust => {
-                    let program = Compiler::compile_program(&module, &format)?;
+                    let program = decoder::Compiler::compile_program(&module, &format)?;
                     doodle::codegen::print_program(&program);
                 }
             }
@@ -71,7 +72,7 @@ fn main() -> Result<(), Box<dyn std::error::Error + 'static>> {
         Command::File { output, filename } => {
             let mut module = FormatModule::new();
             let format = format::main(&mut module).call();
-            let program = Compiler::compile_program(&module, &format)?;
+            let program = streamer::Compiler::compile_program(&module, &format)?;
 
             let input = fs::read(filename)?;
             let (value, _) = program.run(ReadCtxt::new(&input))?;

src/decoder.rs

@@ -67,7 +67,7 @@ impl Value {
         }
     }
 
-    fn unwrap_usize(self) -> usize {
+    pub fn unwrap_usize(self) -> usize {
         match self {
             Value::U8(n) => usize::from(n),
             Value::U16(n) => usize::from(n),
@@ -76,22 +76,21 @@ impl Value {
         }
     }
 
-    fn unwrap_tuple(self) -> Vec<Value> {
+    pub fn unwrap_tuple(self) -> Vec<Value> {
         match self {
             Value::Tuple(values) => values,
             _ => panic!("value is not a tuple"),
         }
     }
 
-    fn unwrap_bool(self) -> bool {
+    pub fn unwrap_bool(self) -> bool {
         match self {
             Value::Bool(b) => b,
             _ => panic!("value is not a bool"),
         }
     }
 
-    #[allow(dead_code)]
-    fn unwrap_char(self) -> char {
+    pub fn unwrap_char(self) -> char {
         match self {
             Value::Char(c) => c,
             _ => panic!("value is not a char"),
@@ -107,7 +106,7 @@ impl Value {
             .then_some(pattern_scope)
     }
 
-    fn matches_inner(&self, scope: &mut MultiScope<'_>, pattern: &Pattern) -> bool {
+    pub fn matches_inner(&self, scope: &mut dyn ScopeBinding, pattern: &Pattern) -> bool {
         match (pattern, self) {
             (Pattern::Binding(name), head) => {
                 scope.push(name.clone(), head.clone());
@@ -403,7 +402,7 @@ impl Expr {
         self.eval(scope).coerce_mapped_value().clone()
     }
 
-    fn eval_lambda<'a>(&self, scope: &'a Scope<'a>, arg: &Value) -> Value {
+    pub fn eval_lambda<'a>(&self, scope: &'a Scope<'a>, arg: &Value) -> Value {
         match self {
             Expr::Lambda(name, expr) => {
                 let child_scope = SingleScope::new(scope, name, arg);
@@ -451,7 +450,7 @@ pub struct Program {
 }
 
 impl Program {
-    fn new() -> Self {
+    pub fn new() -> Self {
         let decoders = Vec::new();
         Program { decoders }
     }
@@ -696,6 +695,7 @@ pub enum Scope<'a> {
     Multi(&'a MultiScope<'a>),
     Single(SingleScope<'a>),
     Decoder(DecoderScope<'a>),
+    Other(&'a dyn ScopeLookup),
 }
 
 pub struct MultiScope<'a> {
@@ -715,13 +715,24 @@ pub struct DecoderScope<'a> {
     decoder: Decoder,
 }
 
-impl<'a> Scope<'a> {
+pub trait ScopeLookup {
+    fn get_value_by_name(&self, name: &str) -> &Value;
+    fn get_decoder_by_name(&self, name: &str) -> &Decoder;
+    fn get_bindings(&self, bindings: &mut Vec<(Label, ScopeEntry)>);
+}
+
+pub trait ScopeBinding {
+    fn push(&mut self, name: Label, v: Value);
+}
+
+impl<'a> ScopeLookup for Scope<'a> {
     fn get_value_by_name(&self, name: &str) -> &Value {
         match self {
             Scope::Empty => panic!("value not found: {name}"),
             Scope::Multi(multi) => multi.get_value_by_name(name),
             Scope::Single(single) => single.get_value_by_name(name),
             Scope::Decoder(decoder) => decoder.parent.get_value_by_name(name),
+            Scope::Other(other) => other.get_value_by_name(name),
         }
     }
 
@@ -731,19 +742,27 @@ impl<'a> Scope<'a> {
             Scope::Multi(multi) => multi.parent.get_decoder_by_name(name),
             Scope::Single(single) => single.parent.get_decoder_by_name(name),
             Scope::Decoder(decoder) => decoder.get_decoder_by_name(name),
+            Scope::Other(other) => other.get_decoder_by_name(name),
         }
     }
 
-    pub fn get_bindings(&self, bindings: &mut Vec<(Label, ScopeEntry)>) {
+    fn get_bindings(&self, bindings: &mut Vec<(Label, ScopeEntry)>) {
         match self {
             Scope::Empty => {}
             Scope::Multi(multi) => multi.get_bindings(bindings),
             Scope::Single(single) => single.get_bindings(bindings),
             Scope::Decoder(decoder) => decoder.get_bindings(bindings),
+            Scope::Other(other) => other.get_bindings(bindings),
         }
     }
 }
 
+impl<'a> ScopeBinding for MultiScope<'a> {
+    fn push(&mut self, name: Label, v: Value) {
+        self.entries.push((name, v));
+    }
+}
+
 impl<'a> MultiScope<'a> {
     fn new(parent: &'a Scope<'a>) -> MultiScope<'a> {
         let entries = Vec::new();
@@ -759,10 +778,6 @@ impl<'a> MultiScope<'a> {
         Value::Record(self.entries)
     }
 
-    pub fn push(&mut self, name: Label, v: Value) {
-        self.entries.push((name, v));
-    }
-
     fn get_value_by_name(&self, name: &str) -> &Value {
         for (n, v) in self.entries.iter().rev() {
             if n == name {
@@ -807,7 +822,7 @@ impl<'a> SingleScope<'a> {
 }
 
 impl<'a> DecoderScope<'a> {
-    fn new(parent: &'a Scope<'a>, name: &'a str, decoder: Decoder) -> DecoderScope<'a> {
+    pub fn new(parent: &'a Scope<'a>, name: &'a str, decoder: Decoder) -> DecoderScope<'a> {
         DecoderScope {
             parent,
             name,
@@ -1080,7 +1095,7 @@ impl Decoder {
     }
 }
 
-fn value_to_vec_usize(v: &Value) -> Vec<usize> {
+pub fn value_to_vec_usize(v: &Value) -> Vec<usize> {
     let vs = match v {
         Value::Seq(vs) => vs,
         _ => panic!("expected Seq"),
@@ -1094,7 +1109,7 @@ fn value_to_vec_usize(v: &Value) -> Vec<usize> {
         .collect::<Vec<usize>>()
 }
 
-fn make_huffman_codes(lengths: &[usize]) -> Format {
+pub fn make_huffman_codes(lengths: &[usize]) -> Format {
     let max_length = *lengths.iter().max().unwrap();
     let mut bl_count = [0].repeat(max_length + 1);
 

src/error.rs

@@ -1,5 +1,5 @@
 use crate::byte_set::ByteSet;
-use crate::decoder::{Scope, ScopeEntry};
+use crate::decoder::{Scope, ScopeEntry, ScopeLookup};
 use crate::read::ReadCtxt;
 use crate::Label;
 

src/lib.rs

@@ -16,6 +16,7 @@ pub mod byte_set;
 pub mod codegen;
 pub mod decoder;
 pub mod error;
+pub mod streamer;
 
 pub mod output;
 pub mod prelude;

src/output/flat.rs

@@ -1,6 +1,6 @@
 use std::io;
 
-use crate::decoder::{MultiScope, Scope, SingleScope, Value};
+use crate::decoder::{MultiScope, Scope, ScopeBinding, SingleScope, Value};
 use crate::Label;
 use crate::{Format, FormatModule};
 

src/output/tree.rs

@@ -1,6 +1,6 @@
 use std::{borrow::Cow, fmt, io, ops::Deref, rc::Rc};
 
-use crate::decoder::{MultiScope, Scope, SingleScope, Value};
+use crate::decoder::{MultiScope, Scope, ScopeBinding, SingleScope, Value};
 use crate::Label;
 use crate::{DynFormat, Expr, Format, FormatModule};