feat: add support for --module-data option in scan command

cli/src/commands/dump.rs

@@ -22,6 +22,7 @@ enum SupportedModules {
     Elf,
     Pe,
     Dotnet,
+    Eml,
 }
 
 #[derive(Debug, Clone, ValueEnum)]

cli/src/commands/mod.rs

@@ -83,6 +83,19 @@ fn external_var_parser(
     Ok((var.to_string(), value))
 }
 
+fn meta_file_value_parser(
+    option: &str,
+) -> Result<(String, PathBuf), anyhow::Error> {
+    let (var, value) = option.split_once('=').ok_or(anyhow!(
+        "the equal sign is missing, use the syntax MODULE=FILE (example: {}=file)",
+        option
+    ))?;
+
+    let value = PathBuf::from(value);
+
+    Ok((var.to_string(), value))
+}
+
 pub fn compile_rules<'a, P>(
     paths: P,
     path_as_namespace: bool,

cli/src/commands/scan.rs

@@ -15,14 +15,16 @@ use superconsole::{Component, Line, Lines, Span};
 use yansi::Color::{Cyan, Red, Yellow};
 use yansi::Paint;
 use yara_x::errors::ScanError;
-use yara_x::{MetaValue, Rule, Rules, ScanResults, Scanner};
+use yara_x::{MetaValue, Rule, Rules, ScanOptions, ScanResults, Scanner};
 
 use crate::commands::{
     compile_rules, external_var_parser, truncate_with_ellipsis,
 };
 use crate::walk::Message;
 use crate::{help, walk};
 
+use super::meta_file_value_parser;
+
 #[derive(Clone, ValueEnum)]
 enum OutputFormats {
     /// Default output format.
@@ -150,6 +152,16 @@ pub fn scan() -> Command {
                 .value_parser(external_var_parser)
                 .action(ArgAction::Append)
         )
+        .arg(
+            arg!(-x --"module-data")
+                .help("Pass FILE's content as extra data to MODULE")
+                .long_help(help::MODULE_DATA_LONG_HELP)
+                .required(false)
+                .value_name("MODULE=FILE")
+                .value_parser(value_parser!(PathBuf))
+                .value_parser(meta_file_value_parser)
+                .action(ArgAction::Append)
+        )
 }
 
 pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
@@ -168,6 +180,13 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
         .get_many::<(String, serde_json::Value)>("define")
         .map(|var| var.cloned().collect());
 
+    let metadata = args
+        .get_many::<(String, PathBuf)>("module-data")
+        .into_iter()
+        .flatten()
+        // collect to eagerly call the parser on each element
+        .collect::<Vec<_>>();
+
     let rules = if compiled_rules {
         if rules_path.len() > 1 {
             bail!(
@@ -247,6 +266,16 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
     let start_time = Instant::now();
     let state = ScanState::new(start_time);
 
+    let all_metadata = {
+        let mut all_metadata = Vec::new();
+        for (module_full_name, metadata_path) in metadata {
+            let meta = std::fs::read(Path::new(metadata_path))?;
+
+            all_metadata.push((module_full_name.to_string(), meta));
+        }
+        all_metadata
+    };
+
     w.walk(
         state,
         // Initialization
@@ -290,8 +319,15 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
                 .unwrap()
                 .push((file_path.clone(), now));
 
+            let scan_options = all_metadata.iter().fold(
+                ScanOptions::new(),
+                |acc, (module_name, meta)| {
+                    acc.set_module_metadata(module_name, meta)
+                },
+            );
+
             let scan_results = scanner
-                .scan_file(&file_path)
+                .scan_file_with_options(file_path.as_path(), scan_options)
                 .with_context(|| format!("scanning {:?}", &file_path));
 
             state

cli/src/help.rs

@@ -24,6 +24,16 @@ Examples:
 --define some_bool=true
 --define some_str=\"foobar\""#;
 
+pub const MODULE_DATA_LONG_HELP: &str = r#"Pass FILE's content as extra data to MODULE
+Some modules require extra data input (besides just the analyzed file) to work. This
+option allows to pass such data to those modules.
+The flag can be used multiple times to pass data to multiple modules.
+The content of the FILE is loaded and interpreted by the module.
+Examples:
+--module-data=mymodule0=./example0.json --module-data=mymodule1=./example1.json
+such invocation will result in passing the content of `example0.json`, `example1.json`
+to `mymodule0`, `mymodule1` respectively."#;
+
 pub const DEPTH_LONG_HELP: &str = r#"Walk directories recursively up to a given depth
 
 This is ignored if <RULES_PATH> is not a directory. When <MAX_DEPTH> is 0 it means

lib/build.rs

@@ -40,6 +40,8 @@ fn generate_module_files(proto_files: Vec<FileDescriptorProto>) {
         }
     }
 
+    let modules = modules;
+
     // Create the modules.rs files, with an entry for each YARA module
     // that has an associated Rust module. Each entry in the file looks
     // like:

lib/src/lib.rs

@@ -59,6 +59,8 @@ pub use scanner::NonMatchingRules;
 pub use scanner::Pattern;
 pub use scanner::Patterns;
 pub use scanner::Rule;
+pub use scanner::ScanError;
+pub use scanner::ScanOptions;
 pub use scanner::ScanResults;
 pub use scanner::Scanner;
 pub use variables::Variable;

lib/src/modules/add_modules.rs

@@ -1,31 +1,31 @@
 // File generated automatically by build.rs. Do not edit.
 {
-#[cfg(feature = "string-module")]
-add_module!(modules, "string", string, "string.String", Some("string"), Some(string::__main__ as MainFn));
-#[cfg(feature = "macho-module")]
-add_module!(modules, "macho", macho, "macho.Macho", Some("macho"), Some(macho::__main__ as MainFn));
-#[cfg(feature = "pe-module")]
-add_module!(modules, "pe", pe, "pe.PE", Some("pe"), Some(pe::__main__ as MainFn));
-#[cfg(feature = "elf-module")]
-add_module!(modules, "elf", elf, "elf.ELF", Some("elf"), Some(elf::__main__ as MainFn));
-#[cfg(feature = "text-module")]
-add_module!(modules, "text", text, "text.Text", Some("text"), Some(text::__main__ as MainFn));
+#[cfg(feature = "console-module")]
+add_module!(modules, "console", console, "console.Console", Some("console"), Some(console::__main__ as MainFn));
 #[cfg(feature = "dotnet-module")]
 add_module!(modules, "dotnet", dotnet, "dotnet.Dotnet", Some("dotnet"), Some(dotnet::__main__ as MainFn));
-#[cfg(feature = "lnk-module")]
-add_module!(modules, "lnk", lnk, "lnk.Lnk", Some("lnk"), Some(lnk::__main__ as MainFn));
+#[cfg(feature = "elf-module")]
+add_module!(modules, "elf", elf, "elf.ELF", Some("elf"), Some(elf::__main__ as MainFn));
 #[cfg(feature = "hash-module")]
 add_module!(modules, "hash", hash, "hash.Hash", Some("hash"), Some(hash::__main__ as MainFn));
+#[cfg(feature = "lnk-module")]
+add_module!(modules, "lnk", lnk, "lnk.Lnk", Some("lnk"), Some(lnk::__main__ as MainFn));
+#[cfg(feature = "macho-module")]
+add_module!(modules, "macho", macho, "macho.Macho", Some("macho"), Some(macho::__main__ as MainFn));
 #[cfg(feature = "magic-module")]
 add_module!(modules, "magic", magic, "magic.Magic", Some("magic"), Some(magic::__main__ as MainFn));
 #[cfg(feature = "math-module")]
 add_module!(modules, "math", math, "math.Math", Some("math"), Some(math::__main__ as MainFn));
+#[cfg(feature = "pe-module")]
+add_module!(modules, "pe", pe, "pe.PE", Some("pe"), Some(pe::__main__ as MainFn));
+#[cfg(feature = "string-module")]
+add_module!(modules, "string", string, "string.String", Some("string"), Some(string::__main__ as MainFn));
 #[cfg(feature = "test_proto2-module")]
 add_module!(modules, "test_proto2", test_proto2, "test_proto2.TestProto2", Some("test_proto2"), Some(test_proto2::__main__ as MainFn));
-#[cfg(feature = "time-module")]
-add_module!(modules, "time", time, "time.Time", Some("time"), Some(time::__main__ as MainFn));
 #[cfg(feature = "test_proto3-module")]
 add_module!(modules, "test_proto3", test_proto3, "test_proto3.TestProto3", Some("test_proto3"), Some(test_proto3::__main__ as MainFn));
-#[cfg(feature = "console-module")]
-add_module!(modules, "console", console, "console.Console", Some("console"), Some(console::__main__ as MainFn));
+#[cfg(feature = "text-module")]
+add_module!(modules, "text", text, "text.Text", Some("text"), Some(text::__main__ as MainFn));
+#[cfg(feature = "time-module")]
+add_module!(modules, "time", time, "time.Time", Some("time"), Some(time::__main__ as MainFn));
 }

lib/src/modules/console.rs

@@ -2,7 +2,7 @@ use crate::modules::prelude::*;
 use crate::modules::protos::console::*;
 
 #[module_main]
-fn main(_data: &[u8]) -> Console {
+fn main(_data: &[u8], _meta: Option<&[u8]>) -> Console {
     // Nothing to do, but we have to return our protobuf
     Console::new()
 }

lib/src/modules/dotnet/mod.rs

@@ -9,8 +9,8 @@ use crate::modules::protos::dotnet::*;
 pub mod parser;
 
 #[module_main]
-fn main(input: &[u8]) -> Dotnet {
-    match parser::Dotnet::parse(input) {
+fn main(data: &[u8], _meta: Option<&[u8]>) -> Dotnet {
+    match parser::Dotnet::parse(data) {
         Ok(dotnet) => dotnet.into(),
         Err(_) => {
             let mut dotnet = Dotnet::new();

lib/src/modules/elf/mod.rs

@@ -19,7 +19,7 @@ pub mod parser;
 mod tests;
 
 #[module_main]
-fn main(data: &[u8]) -> ELF {
+fn main(data: &[u8], _meta: Option<&[u8]>) -> ELF {
     match parser::ElfParser::new().parse(data) {
         Ok(elf) => elf,
         Err(_) => ELF::new(),

lib/src/modules/hash/mod.rs

@@ -29,7 +29,7 @@ thread_local!(
 );
 
 #[module_main]
-fn main(_data: &[u8]) -> Hash {
+fn main(_data: &[u8], _meta: Option<&[u8]>) -> Hash {
     // With every scanned file the cache must be cleared.
     SHA256_CACHE.with(|cache| cache.borrow_mut().clear());
     SHA1_CACHE.with(|cache| cache.borrow_mut().clear());

lib/src/modules/lnk/mod.rs

@@ -17,7 +17,7 @@ use crate::modules::protos::lnk::*;
 pub mod parser;
 
 #[module_main]
-fn main(data: &[u8]) -> Lnk {
+fn main(data: &[u8], _meta: Option<&[u8]>) -> Lnk {
     match parser::LnkParser::new().parse(data) {
         Ok(lnk) => lnk,
         Err(_) => {

lib/src/modules/macho/mod.rs

@@ -367,8 +367,8 @@ fn export_hash(ctx: &mut ScanContext) -> Option<RuntimeString> {
 }
 
 #[module_main]
-fn main(input: &[u8]) -> Macho {
-    match parser::MachO::parse(input) {
+fn main(data: &[u8], _meta: Option<&[u8]>) -> Macho {
+    match parser::MachO::parse(data) {
         Ok(macho) => macho.into(),
         Err(_) => Macho::new(),
     }

lib/src/modules/magic/mod.rs

@@ -34,7 +34,7 @@ thread_local! {
 }
 
 #[module_main]
-fn main(_data: &[u8]) -> Magic {
+fn main(_data: &[u8], _meta: Option<&[u8]>) -> Magic {
     // With every scanned file the cache must be cleared.
     TYPE_CACHE.set(None);
     MIME_TYPE_CACHE.set(None);

lib/src/modules/math.rs

@@ -8,7 +8,7 @@ use crate::modules::prelude::*;
 use crate::modules::protos::math::*;
 
 #[module_main]
-fn main(_data: &[u8]) -> Math {
+fn main(_data: &[u8], _meta: Option<&[u8]>) -> Math {
     // Nothing to do, but we have to return our protobuf
     Math::new()
 }

lib/src/modules/mod.rs

@@ -24,7 +24,7 @@ pub(crate) mod prelude {
 include!("modules.rs");
 
 /// Type of module's main function.
-type MainFn = fn(&[u8]) -> Box<dyn MessageDyn>;
+type MainFn = fn(&[u8], Option<&[u8]>) -> Box<dyn MessageDyn>;
 
 /// Describes a YARA module.
 pub(crate) struct Module {
@@ -134,8 +134,7 @@ pub mod mods {
 
     ```rust
     # use yara_x;
-    # let data = &[];
-    let pe_info = yara_x::mods::invoke::<yara_x::mods::PE>(data);
+    let pe_info = yara_x::mods::invoke::<yara_x::mods::PE>(&[]);
     ```
      */
 
@@ -210,21 +209,37 @@ pub mod mods {
     /// # Example
     /// ```rust
     /// # use yara_x;
-    /// # let data = &[];
-    /// let elf_info = yara_x::mods::invoke::<yara_x::mods::ELF>(data);
+    /// let elf_info = yara_x::mods::invoke::<yara_x::mods::ELF>(&[]);
     /// ```
     pub fn invoke<T: protobuf::MessageFull>(data: &[u8]) -> Option<Box<T>> {
         let module_output = invoke_dyn::<T>(data)?;
         Some(<dyn protobuf::MessageDyn>::downcast_box(module_output).unwrap())
     }
 
+    /// Like [`invoke`], but allows passing metadata to the module.
+    pub fn invoke_with_meta<T: protobuf::MessageFull>(
+        data: &[u8],
+        meta: Option<&[u8]>,
+    ) -> Option<Box<T>> {
+        let module_output = invoke_with_meta_dyn::<T>(data, meta)?;
+        Some(<dyn protobuf::MessageDyn>::downcast_box(module_output).unwrap())
+    }
+
     /// Invoke a YARA module with arbitrary data, but returns a dynamic
     /// structure.
     ///
     /// This function is similar to [`invoke`] but its result is a dynamic-
     /// dispatch version of the structure returned by the YARA module.
     pub fn invoke_dyn<T: protobuf::MessageFull>(
         data: &[u8],
+    ) -> Option<Box<dyn protobuf::MessageDyn>> {
+        invoke_with_meta_dyn::<T>(data, None)
+    }
+
+    /// Like [`invoke_dyn`], but allows passing metadata to the module.
+    pub fn invoke_with_meta_dyn<T: protobuf::MessageFull>(
+        data: &[u8],
+        meta: Option<&[u8]>,
     ) -> Option<Box<dyn protobuf::MessageDyn>> {
         let descriptor = T::descriptor();
         let proto_name = descriptor.full_name();
@@ -233,7 +248,7 @@ pub mod mods {
                 module.root_struct_descriptor.full_name() == proto_name
             })?;
 
-        Some(module.main_fn?(data))
+        Some(module.main_fn?(data, meta))
     }
 
     /// Invoke all YARA modules and return the data produced by them.
@@ -244,8 +259,7 @@ pub mod mods {
     /// # Example
     /// ```rust
     /// # use yara_x;
-    /// # let data = &[];
-    /// let modules_output = yara_x::mods::invoke_all(data);
+    /// let modules_output = yara_x::mods::invoke_all(&[]);
     /// ```
     pub fn invoke_all(data: &[u8]) -> Box<Modules> {
         let mut info = Box::new(Modules::new());

lib/src/modules/modules.rs

@@ -1,29 +1,29 @@
 // File generated automatically by build.rs. Do not edit.
-#[cfg(feature = "string-module")]
-mod string;
-#[cfg(feature = "macho-module")]
-mod macho;
-#[cfg(feature = "pe-module")]
-mod pe;
-#[cfg(feature = "elf-module")]
-mod elf;
-#[cfg(feature = "text-module")]
-mod text;
+#[cfg(feature = "console-module")]
+mod console;
 #[cfg(feature = "dotnet-module")]
 mod dotnet;
-#[cfg(feature = "lnk-module")]
-mod lnk;
+#[cfg(feature = "elf-module")]
+mod elf;
 #[cfg(feature = "hash-module")]
 mod hash;
+#[cfg(feature = "lnk-module")]
+mod lnk;
+#[cfg(feature = "macho-module")]
+mod macho;
 #[cfg(feature = "magic-module")]
 mod magic;
 #[cfg(feature = "math-module")]
 mod math;
+#[cfg(feature = "pe-module")]
+mod pe;
+#[cfg(feature = "string-module")]
+mod string;
 #[cfg(feature = "test_proto2-module")]
 mod test_proto2;
-#[cfg(feature = "time-module")]
-mod time;
 #[cfg(feature = "test_proto3-module")]
 mod test_proto3;
-#[cfg(feature = "console-module")]
-mod console;
+#[cfg(feature = "text-module")]
+mod text;
+#[cfg(feature = "time-module")]
+mod time;