Skip to content

Commit

Permalink
refactor: change the logic used for specifying additional modules (#83)
Browse files Browse the repository at this point in the history 
This removes the use of the YRX_PROTOC_CONFIG_FILE environment variable, in favor of two new environment variables YRX_EXTRA_PROTOS and YRX_EXTRA_PROTOS_BASE_PATH. The previous mechanism for specifying extra modules was more complex and didn't fit well with Bazel which was the primary use case for this mechanism.

The new YRX_EXTRA_PROTOS accepts a list of space-separated file paths pointing to .proto files. This paths are either absolute or relative to the build.rs file. With YRX_EXTRA_PROTOS_BASE_PATH you can make the paths relative to some other point in the file system.
  • Loading branch information
@plusvic
plusvic authored Feb 22, 2024
1 parent 40d982f commit bd1707f
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 83 deletions.
17 changes: 8 additions & 9 deletions lib/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,20 +143,20 @@ default = [
]

[dependencies]
aho-corasick = { workspace = true, features=["logging"] }
aho-corasick = { workspace = true, features = ["logging"] }
anyhow = { workspace = true }
ariadne = { workspace = true }
array-bytes = { workspace = true }
ascii_tree = { workspace = true}
ascii_tree = { workspace = true }
authenticode-parser = { workspace = true, optional = true }
base64 = { workspace = true }
bincode = { workspace = true }
bitmask = { workspace = true }
bitvec = { workspace = true }
bstr = { workspace = true, features=["serde"] }
bstr = { workspace = true, features = ["serde"] }
crc32fast = { workspace = true, optional = true }
fmmap = { workspace = true }
indexmap = { workspace = true, features=["serde"] }
indexmap = { workspace = true, features = ["serde"] }
intaglio = { workspace = true }
itertools = { workspace = true }
lazy_static = { workspace = true }
Expand All @@ -174,14 +174,14 @@ protobuf = { workspace = true }
rustc-hash = { workspace = true }
regex-syntax = { workspace = true }
regex-automata = { workspace = true }
smallvec = { workspace = true, features=["serde"] }
serde = { workspace = true, features=["rc"] }
smallvec = { workspace = true, features = ["serde"] }
serde = { workspace = true, features = ["rc"] }
serde_json = { workspace = true }
thiserror = { workspace = true }
tlsh-fixed = { workspace = true, optional = true }
uuid = { workspace = true, optional = true, features = ["v4"] }
walrus = { workspace = true }
wasmtime = { workspace = true, features=["cranelift", "parallel-compilation"] }
wasmtime = { workspace = true, features = ["cranelift", "parallel-compilation"] }
yansi = { workspace = true }
yara-x-macros = { workspace = true }
yara-x-parser = { workspace = true }
Expand All @@ -190,12 +190,11 @@ yara-x-proto = { workspace = true }
lingua = { version = "1.6.0", optional = true, default-features = false, features = ["english", "german", "french", "spanish"] }

[build-dependencies]
anyhow = { workspace = true }
globwalk = { workspace = true }
protobuf = { workspace = true }
protobuf-codegen = { workspace = true }
protobuf-parse = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
yara-x-proto = { workspace = true }

[dev-dependencies]
Expand Down
107 changes: 53 additions & 54 deletions lib/build.rs
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
use anyhow::Context;
use std::fs::File;
use std::io::{BufReader, Write};
use std::path::Path;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::{env, fs};

use protobuf_codegen::Codegen;
use protobuf_parse::Parser;
use serde::{Deserialize, Serialize};

use yara_x_proto::exts::module_options as yara_module_options;

#[derive(Serialize, Deserialize)]
struct ProtocConfig {
includes: Vec<String>,
inputs: Vec<String>,
}

fn main() {
println!("cargo:rerun-if-changed=src/modules");
println!("cargo:rerun-if-changed=src/modules/protos");
Expand Down Expand Up @@ -50,53 +44,58 @@ fn main() {
}
}

// If the YRX_PROTOC_CONFIG_FILE environment variable is set, it must
// contain the path to a JSON file that indicates additional include
// directories and input files passed to the `protoc` compiler. The path
// to the JSON file must be either an absolute path or a path relative to
// this `build.rs` file.
// The environment variable `YRX_EXTRA_PROTOS` allows passing a list of
// additional `.proto` files with YARA module definitions, in addition to
// those found in `src/modules/protos`. The value in this variable must be a
// space-separated list of file paths, and the paths must be either absolute
// or relative to the location of this `build.rs` file.
//
// The JSON file must be similar to this:
// If you need to provide a list of paths that are relative to some other
// location in the file system, you can specify a base path using the
// environment variable `YRX_EXTRA_PROTOS_BASE_PATH`. This base path must be
// also absolute or relative to the location of this `build.rs`, and the
// final path for the `.proto` files will be computed by combining the
// relative paths in `YRX_EXTRA_PROTOS` to the base path. For instance,
// if you have:
//
// {
// "includes": [
// "../../vt-protos/protos/tools",
// "../../vt-protos/protos"
// ],
// "inputs": [
// "../../vt-protos/protos/titan.proto",
// "../../vt-protos/protos/filetypes.proto",
// "../../vt-protos/protos/sandbox.proto",
// "../../vt-protos/protos/vtnet.proto",
// "../../vt-protos/protos/submitter.proto",
// "../../vt-protos/protos/analysis.proto",
// "../../vt-protos/protos/tools/net_analysis.proto",
// "../../vt-protos/protos/tools/snort.proto",
// "../../vt-protos/protos/tools/suricata.proto",
// "../../vt-protos/protos/tools/tshark.proto",
// "../../vt-protos/protos/sigma.proto",
// "../../vt-protos/protos/relationships.proto"
// ]
// }
// YRX_EXTRA_PROTOS_BASE_PATH=../../my/dir
// YRX_EXTRA_PROTOS="foo.proto bar.proto qux/qux.proto"
//
// Paths in the "includes" and "inputs" lists must also be absolute or
// relative to this `build.rs` file.
if let Ok(path) = env::var("YRX_PROTOC_CONFIG_FILE") {
let file = File::open(path.as_str())
.unwrap_or_else(|_| panic!("error opening {}", path));

let reader = BufReader::new(file);
let config: ProtocConfig = serde_json::from_reader(reader)
.unwrap_or_else(|_| panic!("invalid config file {}", path));

for path in config.includes {
let path = fs::canonicalize(path).unwrap();
proto_compiler.include(&path);
proto_parser.include(&path);
}

for path in config.inputs {
let path = fs::canonicalize(path).unwrap();
// The final paths will be:
//
// ../../my/dir/foo.proto
// ../../my/dir/bar.proto
// ../../my/dir/qux/qux.proto
//
// All these final paths are relative to this `build.rs` file. Any absolute
// path in `YRX_EXTRA_PROTOS` is not affected by the base path specified in
// `YRX_EXTRA_PROTOS_BASE_PATH`, they remain untouched.
if let Ok(proto_files) = env::var("YRX_EXTRA_PROTOS") {
println!("cargo:warning=YRX_EXTRA_PROTOS={:?}", &proto_files);

for path in proto_files.split(' ').collect::<Vec<_>>() {
let path = if let Ok(base_path) =
env::var("YRX_EXTRA_PROTOS_BASE_PATH")
{
println!(
"cargo:warning=YRX_EXTRA_PROTOS_BASE_PATH={:?}",
&base_path
);
PathBuf::from(base_path).join(path)
} else {
PathBuf::from(path)
};

let path = fs::canonicalize(&path)
.with_context(|| format!("`{:?}`", &path))
.expect("can not read file");

println!("cargo:warning=adding {:?}", &path);

let base_path = path.with_file_name("");

proto_compiler.include(&base_path);
proto_parser.include(&base_path);
proto_compiler.input(&path);
proto_parser.input(&path);
}
Expand Down Expand Up @@ -204,7 +203,7 @@ mod {rust_mod};"#,
{cfg_feature}
add_module!(modules, "{name}", {proto_mod}, "{root_message}", {rust_mod_name}, {main_fn});"#,
)
.unwrap();
.unwrap();
}

write!(add_modules_rs, "}}").unwrap();
Expand Down
20 changes: 0 additions & 20 deletions mod.cfg
View file

This file was deleted.

0 comments on commit bd1707f

Please sign in to comment.