Add dir

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+- `blockset add` works with directories. PR [#165](https://github.com/datablockset/blockset/pull/165).
+
 ## 0.4.2
 
 - WASI. PR [#158](https://github.com/datablockset/blockset/pull/158).

Cargo.toml

@@ -10,8 +10,9 @@ authors = ["Sergey Shandar"]
 repository = "https://github.com/datablockset/blockset"
 
 [workspace.dependencies]
-io-trait = "0.8.0"
-io-impl = "0.8.1"
-io-test = "0.8.1"
-wasm-bindgen-test = "0.3.41"
-nanvm-lib = "0.0.1"
+blockset-lib = { path = "blockset-lib", version = "0.4.2" }
+io-trait = "0.10.0"
+io-impl = "0.10.0"
+io-test = "0.10.0"
+wasm-bindgen-test = "0.3.42"
+nanvm-lib = "0.0.3"

blockset-lib/src/app/add.rs

@@ -0,0 +1,126 @@
+use core::ops::Deref;
+use std::io::{self, Cursor};
+
+use io_trait::{DirEntry, Io, Metadata};
+use nanvm_lib::{
+    common::{cast::Cast, default::default},
+    js::{
+        any_cast::AnyCast,
+        js_object::Property,
+        js_string::{new_string, JsStringRef},
+        new::New,
+    },
+    mem::{global::GLOBAL, manager::Manager},
+    serializer::to_json,
+};
+
+use crate::{
+    cdt::tree_add::TreeAdd,
+    common::{progress::State, status_line::StatusLine},
+};
+
+use super::{invalid_input, read_to_tree, read_to_tree_file};
+
+pub struct Add<'a, T: Io, S: 'a + TreeAdd, F: Fn(&'a T) -> S> {
+    pub io: &'a T,
+    pub storage: &'a F,
+    pub to_posix_eol: bool,
+    pub display_new: bool,
+    pub status: StatusLine<'a, T>,
+    pub p: State,
+}
+
+pub fn posix_path(s: &str) -> String {
+    s.replace('\\', "/")
+}
+
+fn read_dir_recursive<I: Io>(io: &I, path: &str) -> io::Result<Vec<(String, u64)>> {
+    let mut result: Vec<_> = default();
+    let mut dirs = [path.to_owned()].cast();
+    while let Some(dir) = dirs.pop() {
+        for entry in io.read_dir(dir.as_str())? {
+            let m = entry.metadata()?;
+            if m.is_dir() {
+                dirs.push(entry.path());
+            } else {
+                result.push((posix_path(&entry.path()[path.len() + 1..]), m.len()));
+            }
+        }
+    }
+    Ok(result)
+}
+
+fn str_to_js_string<M: Manager>(m: M, s: impl Deref<Target = str>) -> JsStringRef<M::Dealloc> {
+    new_string(m, s.encode_utf16().collect::<Vec<_>>()).to_ref()
+}
+
+fn property<M: Manager>(
+    m: M,
+    file: impl Deref<Target = str>,
+    hash: impl Deref<Target = str>,
+) -> Property<M::Dealloc> {
+    (
+        str_to_js_string(m, file),
+        str_to_js_string(m, hash).move_to_any(),
+    )
+}
+
+fn dir_to_json<M: Manager>(
+    m: M,
+    list: impl ExactSizeIterator<Item = Property<M::Dealloc>>,
+) -> io::Result<String> {
+    to_json(m.new_js_object(list)).map_err(|_| invalid_input("to_json"))
+}
+
+fn calculate_len(files: &[(String, u64)], state: &mut State) -> usize {
+    // JSON size:
+    // `{` +
+    // `"` + path + `":"` + 45 + `",` = path.len() + 51
+    let (total, json_len) = files.iter().fold((0, 1), |(total, json_len), (path, len)| {
+        (total + len, json_len + path.len() + 51)
+    });
+    state.total += total + json_len as u64;
+    json_len
+}
+
+impl<'a, T: Io, S: 'a + TreeAdd, F: Fn(&'a T) -> S> Add<'a, T, S, F> {
+    pub fn add_file(&mut self, path: &str) -> io::Result<String> {
+        read_to_tree_file(
+            self.to_posix_eol,
+            (self.storage)(self.io),
+            self.io.open(path)?,
+            &mut self.status,
+            self.display_new,
+            self.p,
+        )
+    }
+    fn add_files(&mut self, path: &str, files: Vec<(String, u64)>) -> io::Result<String> {
+        let mut list = Vec::default();
+        for (p, len) in files {
+            let hash = self.add_file((path.to_owned() + "/" + &p).as_str())?;
+            list.push(property(GLOBAL, p, hash));
+            self.p.current += len;
+        }
+        dir_to_json(GLOBAL, list.into_iter())
+    }
+    fn calculate_and_add_files(
+        &mut self,
+        path: &str,
+        files: Vec<(String, u64)>,
+    ) -> io::Result<String> {
+        calculate_len(&files, &mut self.p);
+        self.add_files(path, files)
+    }
+    fn path_to_json(&mut self, path: &str) -> io::Result<String> {
+        self.calculate_and_add_files(path, read_dir_recursive(self.io, path)?)
+    }
+    pub fn add_dir(&mut self, path: &str) -> io::Result<String> {
+        read_to_tree(
+            (self.storage)(self.io),
+            Cursor::new(self.path_to_json(path)?),
+            &mut self.status,
+            self.display_new,
+            self.p,
+        )
+    }
+}

blockset-lib/src/app/add_entry.rs

@@ -0,0 +1,54 @@
+use std::io;
+
+use io_trait::{Io, Metadata};
+
+use crate::{
+    cdt::tree_add::TreeAdd,
+    common::{print::Print, progress::State, status_line::StatusLine},
+};
+
+use super::{
+    add::{posix_path, Add},
+    invalid_input, is_to_posix_eol,
+};
+
+fn add_file_or_dir<'a, T: Io, S: 'a + TreeAdd>(
+    io: &'a T,
+    storage: &'a impl Fn(&'a T) -> S,
+    to_posix_eol: bool,
+    display_new: bool,
+    path: String,
+) -> io::Result<String> {
+    let mut add = Add {
+        io,
+        storage,
+        to_posix_eol,
+        display_new,
+        status: StatusLine::new(io),
+        p: State {
+            total: 0,
+            current: 0,
+        },
+    };
+    if io.metadata(&path)?.is_dir() {
+        add.add_dir(&path)
+    } else {
+        add.p.total = io.metadata(&path)?.len();
+        add.add_file(&path)
+    }
+}
+
+pub fn add_entry<'a, T: Io, S: 'a + TreeAdd>(
+    io: &'a T,
+    a: &mut T::Args,
+    storage: &'a impl Fn(&'a T) -> S,
+    display_new: bool,
+) -> io::Result<()> {
+    let mut path = posix_path(&a.next().ok_or(invalid_input("missing file name"))?);
+    if path.ends_with('/') {
+        path.pop();
+    }
+    let to_posix_eol = is_to_posix_eol(a)?;
+    let k = add_file_or_dir(io, storage, to_posix_eol, display_new, path)?;
+    io.stdout().println([k.as_str()])
+}

blockset-lib/src/app.rs → blockset-lib/src/app/mod.rs

@@ -1,5 +1,10 @@
+mod add;
+mod add_entry;
+
 use std::io::{self, ErrorKind, Read, Write};
 
+use add_entry::add_entry;
+
 use io_trait::Io;
 
 use crate::{
@@ -8,7 +13,7 @@ use crate::{
         base32::{StrEx, ToBase32},
         eol::ToPosixEol,
         print::Print,
-        progress::{self, Progress},
+        progress::{self, Progress, State},
         status_line::{mb, StatusLine},
     },
     forest::{file::FileForest, node_id::ForestNodeId, tree_add::ForestTreeAdd, Forest},
@@ -50,25 +55,40 @@ fn file_read(
     Ok(size == 0)
 }
 
+fn read_to_tree_once(
+    file: &mut (impl Read + Progress),
+    state: &mut StatusLine<'_, impl Io>,
+    display_new: bool,
+    p: State,
+    tree: &mut MainTreeAdd<impl TreeAdd>,
+    new: &mut u64,
+) -> io::Result<bool> {
+    set_progress(
+        state,
+        display_new,
+        *new,
+        State {
+            total: p.total,
+            current: p.current + file.position()?,
+        },
+    )?;
+    file_read(file, tree, new)
+}
+
 fn read_to_tree<T: TreeAdd>(
     s: T,
     mut file: impl Read + Progress,
-    io: &impl Io,
+    state: &mut StatusLine<'_, impl Io>,
     display_new: bool,
+    p: State,
 ) -> io::Result<String> {
     let mut tree = MainTreeAdd::new(s);
-    let mut state = StatusLine::new(io);
     let mut new = 0;
-    loop {
-        let pr = file.progress();
-        set_progress(&mut state, display_new, new, pr?)?;
-        if file_read(&mut file, &mut tree, &mut new)? {
-            return Ok(tree.end()?.0.to_base32());
-        }
-    }
+    while !read_to_tree_once(&mut file, state, display_new, p, &mut tree, &mut new)? {}
+    Ok(tree.end()?.0.to_base32())
 }
 
-fn invalid_input(error: &str) -> io::Error {
+pub fn invalid_input(error: &str) -> io::Error {
     io::Error::new(ErrorKind::InvalidInput, error)
 }
 
@@ -87,30 +107,17 @@ fn read_to_tree_file(
     to_posix_eol: bool,
     s: impl TreeAdd,
     f: impl Read + Progress,
-    io: &impl Io,
+    state: &mut StatusLine<'_, impl Io>,
     display_new: bool,
+    p: State,
 ) -> io::Result<String> {
     if to_posix_eol {
-        read_to_tree(s, ToPosixEol::new(f), io, display_new)
+        read_to_tree(s, ToPosixEol::new(f), state, display_new, p)
     } else {
-        read_to_tree(s, f, io, display_new)
+        read_to_tree(s, f, state, display_new, p)
     }
 }
 
-fn add<'a, T: Io, S: 'a + TreeAdd>(
-    io: &'a T,
-    a: &mut T::Args,
-    storage: impl Fn(&'a T) -> S,
-    display_new: bool,
-) -> io::Result<()> {
-    let stdout = &mut io.stdout();
-    let path = a.next().ok_or(invalid_input("missing file name"))?;
-    let to_posix_eol = is_to_posix_eol(a)?;
-    let f = io.open(&path)?;
-    let k = read_to_tree_file(to_posix_eol, storage(io), f, io, display_new)?;
-    stdout.println([k.as_str()])
-}
-
 fn get_hash(a: &mut impl Iterator<Item = String>) -> io::Result<U224> {
     let b32 = a.next().ok_or(invalid_input("missing hash"))?;
     b32.from_base32::<U224>()
@@ -129,8 +136,8 @@ pub fn run(io: &impl Io) -> io::Result<()> {
     let command = a.next().ok_or(invalid_input("missing command"))?;
     match command.as_str() {
         "validate" => validate(&mut a, stdout),
-        "hash" => add(io, &mut a, |_| (), false),
-        "add" => add(io, &mut a, |io| ForestTreeAdd::new(FileForest(io)), true),
+        "hash" => add_entry(io, &mut a, &|_| (), false),
+        "add" => add_entry(io, &mut a, &|io| ForestTreeAdd::new(FileForest(io)), true),
         "get" => {
             let d = get_hash(&mut a)?;
             let path = a.next().ok_or(invalid_input("missing file name"))?;
@@ -146,6 +153,7 @@ pub fn run(io: &impl Io) -> io::Result<()> {
 mod test {
     use io_test::VirtualIo;
     use io_trait::Io;
+    use std::io::Write;
     use wasm_bindgen_test::wasm_bindgen_test;
 
     use crate::{cdt::node_id::root, common::base32::ToBase32, run, uint::u256::U256};
@@ -367,4 +375,35 @@ mod test {
         let e = run(&mut io);
         assert_eq!(e.unwrap_err().to_string(), "unknown option");
     }
+
+    #[test]
+    #[wasm_bindgen_test]
+    fn test_add_dir() {
+        let f = |a| {
+            let mut io = VirtualIo::new(&["add", a]);
+            io.create_dir("a").unwrap();
+            {
+                let mut f = io.create("a/b.txt").unwrap();
+                f.write_all(b"Hello world!").unwrap();
+            }
+            io.create("c.txt").unwrap();
+            io.create("a/d.txt").unwrap();
+            io.create_dir("a/e").unwrap();
+            io.create("a/e/f.txt").unwrap();
+            let mut a = io.args();
+            a.next().unwrap();
+            run(&mut io).unwrap();
+            // add_dir(&io, "a").unwrap();
+            io.stdout.to_stdout()
+            /*
+            assert_eq!(
+                result,
+                "add-dir: {\"b.txt\":12,\"d.txt\":0,\"e/f.txt\":0}\n"
+            );
+            */
+        };
+        let a = f("a");
+        let b = f("a/");
+        assert_eq!(a, b);
+    }
 }

blockset-lib/src/common/bit_vec.rs

@@ -29,9 +29,16 @@ mod test {
 
     use super::BitVec;
 
+    fn check(v: u32, len: u8, f: fn(v: u32, len: u8) -> BitVec) {
+        let x = f(v, len);
+        assert_eq!(x.value, v as u64);
+        assert_eq!(x.len, len);
+    }
+
     #[test]
     #[wasm_bindgen_test]
     fn test() {
+        check(0b1010, 4, BitVec::new);
         let x = BitVec::new(0b1010, 4);
         assert_eq!(x.value, 0b1010);
         assert_eq!(x.len, 4);