feat: add support for --module-data option in scan command

cli/src/commands/mod.rs

@@ -83,6 +83,19 @@ fn external_var_parser(
     Ok((var.to_string(), value))
 }
 
+fn meta_file_value_parser(
+    option: &str,
+) -> Result<(String, PathBuf), anyhow::Error> {
+    let (var, value) = option.split_once('=').ok_or(anyhow!(
+        "the equal sign is missing, use the syntax MODULE=FILE (example: {}=file)",
+        option
+    ))?;
+
+    let value = PathBuf::from(value);
+
+    Ok((var.to_string(), value))
+}
+
 pub fn compile_rules<'a, P>(
     paths: P,
     external_vars: Option<Vec<(String, Value)>>,

cli/src/commands/scan.rs

@@ -15,14 +15,16 @@ use superconsole::{Component, Line, Lines, Span};
 use yansi::Color::{Cyan, Red, Yellow};
 use yansi::Paint;
 use yara_x::errors::ScanError;
-use yara_x::{MetaValue, Rule, Rules, ScanResults, Scanner};
+use yara_x::{MetaValue, Rule, Rules, ScanOptions, ScanResults, Scanner};
 
 use crate::commands::{
     compile_rules, external_var_parser, truncate_with_ellipsis,
 };
 use crate::walk::Message;
 use crate::{help, walk};
 
+use super::meta_file_value_parser;
+
 #[derive(Clone, ValueEnum)]
 enum OutputFormats {
     /// Default output format.
@@ -88,6 +90,16 @@ pub fn scan() -> Command {
                 .long_help(help::IGNORE_MODULE_LONG_HELP)
                 .action(ArgAction::Append)
         )
+        .arg(
+            arg!(-x --"module-data")
+                .help("Pass FILE's content as extra data to MODULE")
+                .long_help(help::MODULE_DATA_LONG_HELP)
+                .required(false)
+                .value_name("MODULE=FILE")
+                .value_parser(value_parser!(PathBuf))
+                .value_parser(meta_file_value_parser)
+                .action(ArgAction::Append)
+        )
         .arg(
             arg!(-n --"negate")
                 .help("Print non-satisfied rules only")
@@ -153,6 +165,7 @@ pub fn scan() -> Command {
                 .help("Abort scanning after the given number of seconds")
                 .value_parser(value_parser!(u64).range(1..))
         )
+
 }
 
 pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
@@ -171,6 +184,13 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
         .get_many::<(String, serde_json::Value)>("define")
         .map(|var| var.cloned().collect());
 
+    let metadata = args
+        .get_many::<(String, PathBuf)>("module-data")
+        .into_iter()
+        .flatten()
+        // collect to eagerly call the parser on each element
+        .collect::<Vec<_>>();
+
     let rules = if compiled_rules {
         if rules_path.len() > 1 {
             bail!(
@@ -231,6 +251,16 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
     let start_time = Instant::now();
     let state = ScanState::new(start_time);
 
+    let all_metadata = {
+        let mut all_metadata = Vec::new();
+        for (module_full_name, metadata_path) in metadata {
+            let meta = std::fs::read(Path::new(metadata_path))?;
+
+            all_metadata.push((module_full_name.to_string(), meta));
+        }
+        all_metadata
+    };
+
     w.walk(
         state,
         // Initialization
@@ -278,8 +308,15 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
                 .unwrap()
                 .push((file_path.clone(), now));
 
+            let scan_options = all_metadata.iter().fold(
+                ScanOptions::new(),
+                |acc, (module_name, meta)| {
+                    acc.set_module_metadata(module_name, meta)
+                },
+            );
+
             let scan_results = scanner
-                .scan_file(&file_path)
+                .scan_file_with_options(file_path.as_path(), scan_options)
                 .with_context(|| format!("scanning {:?}", &file_path));
 
             state

cli/src/help.rs

@@ -24,6 +24,20 @@ Examples:
 --define some_bool=true
 --define some_str=\"foobar\""#;
 
+pub const MODULE_DATA_LONG_HELP: &str = r#"Pass FILE's content as extra data to MODULE
+
+Some modules require supplementary data to work, in addition to the scanned file. This
+option allows you to provide that extra data. The flag can be used multiple times to
+supply data to different modules. The content of the FILE is loaded and interpreted by
+the respective module.
+
+Examples:
+
+--module-data=mymodule0=./example0.json --module-data=mymodule1=./example1.json
+
+In this example, the contents of example0.json and example1.json will be passed to
+mymodule0 and mymodule1, respectively."#;
+
 pub const DEPTH_LONG_HELP: &str = r#"Walk directories recursively up to a given depth
 
 This is ignored if <RULES_PATH> is not a directory. When <MAX_DEPTH> is 0 it means

lib/src/lib.rs

@@ -59,6 +59,8 @@ pub use scanner::NonMatchingRules;
 pub use scanner::Pattern;
 pub use scanner::Patterns;
 pub use scanner::Rule;
+pub use scanner::ScanError;
+pub use scanner::ScanOptions;
 pub use scanner::ScanResults;
 pub use scanner::Scanner;
 pub use variables::Variable;

lib/src/modules/console.rs

@@ -2,7 +2,7 @@ use crate::modules::prelude::*;
 use crate::modules::protos::console::*;
 
 #[module_main]
-fn main(_data: &[u8]) -> Console {
+fn main(_data: &[u8], _meta: Option<&[u8]>) -> Console {
     // Nothing to do, but we have to return our protobuf
     Console::new()
 }

lib/src/modules/dotnet/mod.rs

@@ -9,8 +9,8 @@ use crate::modules::protos::dotnet::*;
 pub mod parser;
 
 #[module_main]
-fn main(input: &[u8]) -> Dotnet {
-    match parser::Dotnet::parse(input) {
+fn main(data: &[u8], _meta: Option<&[u8]>) -> Dotnet {
+    match parser::Dotnet::parse(data) {
         Ok(dotnet) => dotnet.into(),
         Err(_) => {
             let mut dotnet = Dotnet::new();

lib/src/modules/elf/mod.rs

@@ -19,7 +19,7 @@ pub mod parser;
 mod tests;
 
 #[module_main]
-fn main(data: &[u8]) -> ELF {
+fn main(data: &[u8], _meta: Option<&[u8]>) -> ELF {
     match parser::ElfParser::new().parse(data) {
         Ok(elf) => elf,
         Err(_) => ELF::new(),

lib/src/modules/hash/mod.rs

@@ -29,7 +29,7 @@ thread_local!(
 );
 
 #[module_main]
-fn main(_data: &[u8]) -> Hash {
+fn main(_data: &[u8], _meta: Option<&[u8]>) -> Hash {
     // With every scanned file the cache must be cleared.
     SHA256_CACHE.with(|cache| cache.borrow_mut().clear());
     SHA1_CACHE.with(|cache| cache.borrow_mut().clear());

lib/src/modules/lnk/mod.rs

@@ -17,7 +17,7 @@ use crate::modules::protos::lnk::*;
 pub mod parser;
 
 #[module_main]
-fn main(data: &[u8]) -> Lnk {
+fn main(data: &[u8], _meta: Option<&[u8]>) -> Lnk {
     match parser::LnkParser::new().parse(data) {
         Ok(lnk) => lnk,
         Err(_) => {

lib/src/modules/macho/mod.rs

@@ -367,8 +367,8 @@ fn export_hash(ctx: &mut ScanContext) -> Option<RuntimeString> {
 }
 
 #[module_main]
-fn main(input: &[u8]) -> Macho {
-    match parser::MachO::parse(input) {
+fn main(data: &[u8], _meta: Option<&[u8]>) -> Macho {
+    match parser::MachO::parse(data) {
         Ok(macho) => macho.into(),
         Err(_) => Macho::new(),
     }

lib/src/modules/magic/mod.rs

@@ -34,7 +34,7 @@ thread_local! {
 }
 
 #[module_main]
-fn main(_data: &[u8]) -> Magic {
+fn main(_data: &[u8], _meta: Option<&[u8]>) -> Magic {
     // With every scanned file the cache must be cleared.
     TYPE_CACHE.set(None);
     MIME_TYPE_CACHE.set(None);

lib/src/modules/math.rs

@@ -8,7 +8,7 @@ use crate::modules::prelude::*;
 use crate::modules::protos::math::*;
 
 #[module_main]
-fn main(_data: &[u8]) -> Math {
+fn main(_data: &[u8], _meta: Option<&[u8]>) -> Math {
     // Nothing to do, but we have to return our protobuf
     Math::new()
 }

lib/src/modules/mod.rs

@@ -24,7 +24,7 @@ pub(crate) mod prelude {
 include!("modules.rs");
 
 /// Type of module's main function.
-type MainFn = fn(&[u8]) -> Box<dyn MessageDyn>;
+type MainFn = fn(&[u8], Option<&[u8]>) -> Box<dyn MessageDyn>;
 
 /// Describes a YARA module.
 pub(crate) struct Module {
@@ -134,8 +134,7 @@ pub mod mods {
 
     ```rust
     # use yara_x;
-    # let data = &[];
-    let pe_info = yara_x::mods::invoke::<yara_x::mods::PE>(data);
+    let pe_info = yara_x::mods::invoke::<yara_x::mods::PE>(&[]);
     ```
      */
 
@@ -210,21 +209,37 @@ pub mod mods {
     /// # Example
     /// ```rust
     /// # use yara_x;
-    /// # let data = &[];
-    /// let elf_info = yara_x::mods::invoke::<yara_x::mods::ELF>(data);
+    /// let elf_info = yara_x::mods::invoke::<yara_x::mods::ELF>(&[]);
     /// ```
     pub fn invoke<T: protobuf::MessageFull>(data: &[u8]) -> Option<Box<T>> {
         let module_output = invoke_dyn::<T>(data)?;
         Some(<dyn protobuf::MessageDyn>::downcast_box(module_output).unwrap())
     }
 
+    /// Like [`invoke`], but allows passing metadata to the module.
+    pub fn invoke_with_meta<T: protobuf::MessageFull>(
+        data: &[u8],
+        meta: Option<&[u8]>,
+    ) -> Option<Box<T>> {
+        let module_output = invoke_with_meta_dyn::<T>(data, meta)?;
+        Some(<dyn protobuf::MessageDyn>::downcast_box(module_output).unwrap())
+    }
+
     /// Invoke a YARA module with arbitrary data, but returns a dynamic
     /// structure.
     ///
     /// This function is similar to [`invoke`] but its result is a dynamic-
     /// dispatch version of the structure returned by the YARA module.
     pub fn invoke_dyn<T: protobuf::MessageFull>(
         data: &[u8],
+    ) -> Option<Box<dyn protobuf::MessageDyn>> {
+        invoke_with_meta_dyn::<T>(data, None)
+    }
+
+    /// Like [`invoke_dyn`], but allows passing metadata to the module.
+    pub fn invoke_with_meta_dyn<T: protobuf::MessageFull>(
+        data: &[u8],
+        meta: Option<&[u8]>,
     ) -> Option<Box<dyn protobuf::MessageDyn>> {
         let descriptor = T::descriptor();
         let proto_name = descriptor.full_name();
@@ -233,7 +248,7 @@ pub mod mods {
                 module.root_struct_descriptor.full_name() == proto_name
             })?;
 
-        Some(module.main_fn?(data))
+        Some(module.main_fn?(data, meta))
     }
 
     /// Invoke all YARA modules and return the data produced by them.
@@ -244,8 +259,7 @@ pub mod mods {
     /// # Example
     /// ```rust
     /// # use yara_x;
-    /// # let data = &[];
-    /// let modules_output = yara_x::mods::invoke_all(data);
+    /// let modules_output = yara_x::mods::invoke_all(&[]);
     /// ```
     pub fn invoke_all(data: &[u8]) -> Box<Modules> {
         let mut info = Box::new(Modules::new());

lib/src/modules/pe/mod.rs

@@ -29,8 +29,8 @@ pub mod parser;
 mod rva2off;
 
 #[module_main]
-fn main(input: &[u8]) -> PE {
-    match parser::PE::parse(input) {
+fn main(data: &[u8], _meta: Option<&[u8]>) -> PE {
+    match parser::PE::parse(data) {
         Ok(pe) => pe.into(),
         Err(_) => {
             let mut pe = PE::new();

lib/src/modules/string.rs

@@ -2,7 +2,7 @@ use crate::modules::prelude::*;
 use crate::modules::protos::string::*;
 
 #[module_main]
-fn main(_data: &[u8]) -> String {
+fn main(_data: &[u8], _meta: Option<&[u8]>) -> String {
     // Nothing to do, but we have to return our protobuf
     String::new()
 }

lib/src/modules/test_proto2/mod.rs

@@ -66,7 +66,7 @@ fn to_int(ctx: &ScanContext, string: RuntimeString) -> Option<i64> {
 }
 
 #[module_main]
-fn main(data: &[u8]) -> TestProto2 {
+fn main(data: &[u8], _meta: Option<&[u8]>) -> TestProto2 {
     let mut test = TestProto2::new();
 
     test.set_int32_zero(0);

lib/src/modules/test_proto3/mod.rs

@@ -2,7 +2,7 @@ use crate::modules::prelude::*;
 use crate::modules::protos::test_proto3::TestProto3;
 
 #[module_main]
-fn main(_data: &[u8]) -> TestProto3 {
+fn main(_data: &[u8], _meta: Option<&[u8]>) -> TestProto3 {
     let mut test = TestProto3::new();
 
     test.int32_zero = 0;

lib/src/modules/text.rs

@@ -18,7 +18,7 @@ use lingua::{Language, LanguageDetectorBuilder};
 /// This function must return an instance of the protobuf message indicated
 /// in the `root_message` option in `text.proto`.
 #[module_main]
-fn main(data: &[u8]) -> Text {
+fn main(data: &[u8], _meta: Option<&[u8]>) -> Text {
     // Create an empty instance of the Text protobuf.
     let mut text_proto = Text::new();
 

lib/src/modules/time.rs

@@ -3,7 +3,7 @@ use crate::modules::protos::time::*;
 use std::time::{SystemTime, UNIX_EPOCH};
 
 #[module_main]
-fn main(_data: &[u8]) -> Time {
+fn main(_data: &[u8], _meta: Option<&[u8]>) -> Time {
     // Nothing to do, but we have to return our protobuf
     Time::new()
 }