tools/trace-parser: Add Rust trace.dat parser

tests/test_energy_model.py

@@ -383,22 +383,10 @@ def test_estimate_from_trace(self):
             <idle>-0  [000] 0000.0020: cpu_idle:        state=2 cpu_id=2
             """
         )
-
-        with tempfile.TemporaryDirectory() as directory:
-            path = os.path.join(directory, 'trace.txt')
-            with open(path, 'w') as f:
-                f.write(trace_data)
-
-            trace = Trace(path,
-                events=['cpu_idle', 'cpu_frequency'],
-                normalize_time=False,
-                # Disable swap since the folder is going to get removed
-                enable_swap=False,
-                # Parse all the events eagerly since the trace file is going to
-                # be removed.
-                strict_events=True,
-                parser=TxtTraceParser.from_txt_file,
-            )
+        trace = Trace(
+            normalize_time=False,
+            parser=TxtTraceParser.from_string(trace_data),
+        )
 
         energy_df = em.estimate_from_trace(trace)
 

tools/recipes/trace-tools.recipe

@@ -0,0 +1,49 @@
+#! /bin/bash
+
+ALPINE_VERSION=v3.18
+ALPINE_BUILD_DEPENDENCIES=(bash git curl musl-dev gcc)
+CROSS_COMPILATION_BROKEN=0
+
+run_rustup() {
+    export CARGO_HOME="$(readlink -f .)/.cargo"
+    export RUSTUP_HOME="$(readlink -f .)/.rustup"
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain none -y --no-modify-path
+    source $CARGO_HOME/env
+    rustup toolchain install stable --allow-downgrade --profile minimal
+}
+
+build_tracedump() {
+    cd trace-tools &&
+    case $ARCH in
+        x86_64)
+            local triplet="x86_64-unknown-linux-musl"
+            ;;
+        arm64)
+            local triplet="aarch64-unknown-linux-musl"
+            ;;
+        *)
+            echo "Target architecture $ARCH not supported" >&2
+            return 1
+            ;;
+    esac
+    # x86_64 host, non-x86_64 target
+    if [[ "$(arch)" == "x86_64" && "$ARCH" != "x86_64" ]]; then
+        export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-ld
+    fi
+    time RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target="$triplet"
+}
+
+download() {
+    cp -a "$LISA_HOME/tools/trace-parser" .
+    cp "$LISA_HOME/LICENSE.txt" trace-tools/
+}
+
+build() {
+    run_rustup && (build_tracedump)
+}
+
+install() {
+    source "$LISA_HOME/tools/recipes/utils.sh"
+    cp -v trace-tools/trace-dump/target/release/trace-dump "$LISA_ARCH_ASSETS/"
+    install_readme trace-dump trace-tools/ LICENSE.txt
+}

tools/trace-parser/rustfmt.toml

@@ -0,0 +1,3 @@
+imports_granularity="Crate"
+group_imports="StdExternalCrate"
+skip_macro_invocations=["make_closure_coerce", "make_closure_coerce_type", "closure"]

tools/trace-parser/trace-tools/.gitignore

@@ -0,0 +1,4 @@
+debug/
+target/
+Cargo.lock
+**/*.rs.bk

tools/trace-parser/trace-tools/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "trace-tools"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "lib"
+path = "src/lib/lib.rs"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+traceevent = { path = "../traceevent" }
+
+thiserror = "1.0"
+smartstring = {version = "1.0", features = ["serde"]}
+arrow2 = { version = "0.18.0", features = ["io_parquet", "io_parquet_compression"] }
+crossbeam = "0.8"
+serde_json = "1.0"
+
+nom = "7.1"
+bytemuck = "1.13"
+clap = { version = "4.4", features = ["derive"] }
+
+[target.'cfg(target_arch = "x86_64")'.dependencies]
+mimalloc = {version = "0.1", default-features = false }
+
+[profile.release]
+debug = true
+
+# Static build:
+# rustup target add x86_64-unknown-linux-musl
+# RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-musl

tools/trace-parser/trace-tools/fuzz/.gitignore

@@ -0,0 +1,4 @@
+target
+corpus
+artifacts
+coverage

tools/trace-parser/trace-tools/fuzz/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "trace-tools-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2021"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+
+[target.'cfg(target_arch = "x86_64")'.dependencies]
+mimalloc = {version = "0.1", default-features = false }
+
+[dependencies.trace-tools]
+path = ".."
+
+[dependencies.traceevent]
+path = "../../traceevent/"
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[profile.release]
+debug = 1
+
+[[bin]]
+name = "print"
+path = "fuzz_targets/print.rs"
+test = false
+doc = false

tools/trace-parser/trace-tools/fuzz/fuzz_targets/print.rs

@@ -0,0 +1,27 @@
+#![no_main]
+#[cfg(target_arch = "x86_64")]
+use mimalloc::MiMalloc;
+use traceevent::header;
+#[global_allocator]
+#[cfg(target_arch = "x86_64")]
+static GLOBAL: MiMalloc = MiMalloc;
+
+use std::io::Write;
+
+use lib::{check_header, parquet::dump_events, print::print_events};
+use libfuzzer_sys::fuzz_target;
+use traceevent;
+
+fuzz_target!(|data: &[u8]| {
+    // Speedup the test by not writing anything anywhere
+    let mut out = std::io::sink();
+
+    let mut run = move || {
+        let mut reader: traceevent::io::BorrowingCursor<_> = data.into();
+        let header = header::header(&mut reader)?;
+        let res = print_events(&header, reader, &mut out);
+        res
+    };
+
+    let _ = run();
+});

tools/trace-parser/trace-tools/src/bin/trace-dump.rs

@@ -0,0 +1,123 @@
+use std::{error::Error, fs::File, io::Write, path::PathBuf, process::ExitCode};
+
+#[cfg(target_arch = "x86_64")]
+use mimalloc::MiMalloc;
+use traceevent::{header, header::Timestamp};
+#[global_allocator]
+#[cfg(target_arch = "x86_64")]
+static GLOBAL: MiMalloc = MiMalloc;
+
+use clap::{Parser, Subcommand};
+use lib::{
+    check::check_header,
+    parquet::{dump_events, dump_header_metadata},
+    print::print_events,
+};
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+struct Cli {
+    #[arg(long, value_name = "TRACE")]
+    trace: PathBuf,
+
+    #[arg(long, value_name = "ERRORS_JSON")]
+    errors_json: Option<PathBuf>,
+
+    #[command(subcommand)]
+    command: Command,
+}
+
+#[derive(Subcommand)]
+enum Command {
+    HumanReadable {
+        #[arg(long, value_name = "RAW")]
+        raw: bool,
+    },
+    Parquet {
+        #[arg(long, value_name = "EVENTS")]
+        events: Option<Vec<String>>,
+        #[arg(long)]
+        unique_timestamps: bool,
+    },
+    CheckHeader,
+    Metadata,
+}
+
+fn _main() -> Result<(), Box<dyn Error>> {
+    let cli = Cli::parse();
+
+    let path = cli.trace;
+    let file = std::fs::File::open(path)?;
+    let mut reader = unsafe { traceevent::io::MmapFile::new(file) }?;
+    let header = header::header(&mut reader)?;
+
+    // We make the timestamp unique assuming it will be manipulated as an f64 number of seconds by
+    // consumers
+    let conv_ts = |ts| (ts as f64) / 1e9;
+    let make_unique_timestamps = {
+        let mut prev = (0, conv_ts(0));
+        move |mut ts: Timestamp| {
+            ts = std::cmp::max(prev.0, ts);
+            let mut _ts = conv_ts(ts);
+            while prev.1 == _ts {
+                ts += 1;
+                _ts = conv_ts(ts);
+            }
+            prev = (ts, _ts);
+            ts
+        }
+    };
+
+    let stdout = std::io::stdout().lock();
+    let mut out = std::io::BufWriter::with_capacity(1024 * 1024, stdout);
+
+    let res = match cli.command {
+        Command::HumanReadable { raw } => print_events(&header, reader, &mut out, raw),
+        Command::Parquet {
+            events,
+            unique_timestamps,
+        } => {
+            let make_ts: Box<dyn FnMut(_) -> _> = if unique_timestamps {
+                Box::new(make_unique_timestamps)
+            } else {
+                Box::new(|ts| ts)
+            };
+
+            dump_events(&header, reader, make_ts, events)
+        }
+        Command::CheckHeader => check_header(&header, &mut out),
+        Command::Metadata => dump_header_metadata(&header, &mut out),
+    };
+    out.flush()?;
+
+    if let Err(err) = &res {
+        eprintln!("Errors happened while processing the trace: {err}");
+    }
+
+    if let Some(path) = &cli.errors_json {
+        let errors = match &res {
+            Err(err) => err
+                .errors()
+                .into_iter()
+                .map(|err| err.to_string())
+                .collect(),
+            Ok(_) => Vec::new(),
+        };
+        let mut file = File::create(path)?;
+        let json_value = serde_json::json!({
+            "errors": errors,
+        });
+        file.write_all(json_value.to_string().as_bytes())?;
+    }
+    match res {
+        Ok(_) => Ok(()),
+        Err(_) => Err("Errors happened".into()),
+    }
+}
+
+fn main() -> ExitCode {
+    match _main() {
+        Err(_) => ExitCode::from(1),
+        Ok(_) => ExitCode::from(0),
+    }
+}

tools/trace-parser/trace-tools/src/lib/check.rs

@@ -0,0 +1,53 @@
+use std::io::Write;
+
+use traceevent::header::Header;
+
+use crate::error::DynMultiError;
+
+pub fn check_header<W: Write>(header: &Header, mut out: W) -> Result<(), DynMultiError> {
+    for desc in header.event_descs() {
+        writeln!(&mut out, "Checking event \"{}\" format:", desc.name)?;
+
+        let raw_fmt = std::str::from_utf8(desc.raw_fmt()?)?;
+        writeln!(&mut out, "{raw_fmt}")?;
+        match desc.event_fmt() {
+            Err(err) => {
+                writeln!(&mut out, "    Error while parsing event format: {err}")
+            }
+            Ok(fmt) => {
+                let _ = fmt.struct_fmt().map_err(|err| {
+                    writeln!(
+                        &mut out,
+                        "    Error while parsing event struct format: {err}"
+                    )
+                });
+
+                match fmt.print_args() {
+                    Ok(print_args) => {
+                        print_args
+                            .into_iter()
+                            .enumerate()
+                            .try_for_each(|(i, res)| match res {
+                                Err(err) => {
+                                    writeln!(
+                                        &mut out,
+                                        "    Error while compiling printk argument #{i}: {err}"
+                                    )
+                                }
+                                Ok(_) => Ok(()),
+                            })?;
+                        Ok(())
+                    }
+                    Err(err) => {
+                        writeln!(
+                            &mut out,
+                            "    Error while parsing event print format arguments: {err}"
+                        )
+                    }
+                }
+            }
+        }?;
+        writeln!(&mut out)?;
+    }
+    Ok(())
+}