diff --git a/Cargo.lock b/Cargo.lock index cb31e15..d41a338 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + [[package]] name = "backtrace" version = "0.3.69" @@ -121,6 +127,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" + [[package]] name = "cc" version = "1.0.83" @@ -227,6 +239,39 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "darling" version = "0.20.3" @@ -456,6 +501,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + [[package]] name = "joinery" version = "3.1.0" @@ -504,6 +555,15 @@ version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "miette" version = "5.10.0" @@ -555,6 +615,7 @@ dependencies = [ name = "moz-webgpu-cts" version = "0.4.0" dependencies = [ + "camino", "chumsky", "clap", "enumset", @@ -568,7 +629,10 @@ dependencies = [ "miette", "natord", "path-dsl", + "rayon", "regex", + "serde", + "serde_json", "strum", "thiserror", "wax", @@ -654,6 +718,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" version = "1.9.5" @@ -708,6 +792,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + [[package]] name = "same-file" version = "1.0.6" @@ -717,6 +807,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.188" @@ -737,6 +833,17 @@ dependencies = [ "syn 2.0.31", ] +[[package]] +name = "serde_json" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "similar" version = "2.3.0" @@ -840,6 +947,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tardar" +version = "0.1.0" +source = "git+https://github.com/ErichDonGubler/tardar?branch=static-diags#8dddc68b9f1ad730f3a97b1819333e2a6769ccb7" +dependencies = [ + "miette", + "vec1", +] + [[package]] name = "termcolor" version = "1.2.0" @@ -920,6 +1036,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "vec1" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bda7c41ca331fe9a1c278a9e7ee055f4be7f5eb1c2b72f079b4ff8b5fce9d5c" + [[package]] name = "version_check" version = "0.9.4" @@ -939,14 +1061,15 @@ dependencies = [ [[package]] name = "wax" version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d12a78aa0bab22d2f26ed1a96df7ab58e8a93506a3e20adb47c51a93b4e1357" +source = "git+https://github.com/ErichDonGubler/wax?branch=static-miette-diags#b606968c386f98dba23c15f681d8afdc40142b11" dependencies = [ "const_format", "itertools", + "miette", "nom", "pori", "regex", + "tardar", "thiserror", "walkdir", ] diff --git a/moz-webgpu-cts/Cargo.toml b/moz-webgpu-cts/Cargo.toml index ae9a5cf..7fbad8e 100644 --- a/moz-webgpu-cts/Cargo.toml +++ b/moz-webgpu-cts/Cargo.toml @@ -8,6 +8,7 @@ license = "MIT or Apache-2.0" publish = false [dependencies] +camino = "1.1.6" chumsky = { workspace = true } clap = { version = "4.4.2", features = ["derive"] } env_logger = "0.10.0" @@ -20,10 +21,13 @@ log = { workspace = true } miette = { version = "5.10.0", features = ["fancy"] } natord = "1.0.9" path-dsl = "0.6.1" +rayon = "1.8.0" regex = "1.9.5" +serde = { version = "1.0.188", features = ["derive"] } +serde_json = "1.0.107" strum = { version = "0.25.0", features = ["derive"] } thiserror = "1.0.49" -wax = "0.6.0" +wax = { version = "0.6.0", features = ["miette"], git = "https://github.com/ErichDonGubler/wax", branch = "static-miette-diags"} whippit = { version = "0.4.1", path = "../whippit", default-features = false } [dev-dependencies] diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index 74e1332..594771c 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -1,28 +1,39 @@ mod metadata; +mod process_reports; +mod report; mod shared; use self::{ - metadata::{AnalyzeableProps, File, Platform, Subtest, SubtestOutcome, Test, TestOutcome}, - shared::{Expectation, MaybeCollapsed}, + metadata::{ + AnalyzeableProps, BuildProfile, File, Platform, Subtest, SubtestOutcome, Test, TestOutcome, + }, + process_reports::{MaybeDisabled, OutcomesForComparison, TestOutcomes}, + report::{ + ExecutionReport, RunInfo, SubtestExecutionResult, TestExecutionEntry, TestExecutionResult, + }, + shared::{Expectation, MaybeCollapsed, NormalizedExpectationPropertyValue, TestPath}, }; use std::{ collections::{BTreeMap, BTreeSet}, - fmt::Display, + fmt::{Debug, Display}, fs, - io::{self, BufWriter}, + hash::Hash, + io::{self, BufReader, BufWriter}, path::{Path, PathBuf}, process::ExitCode, - sync::Arc, + sync::{mpsc::channel, Arc}, }; -use clap::Parser; +use clap::{Parser, ValueEnum}; use enumset::EnumSetType; +use format::lazy_format; use indexmap::{IndexMap, IndexSet}; -use miette::{miette, Diagnostic, NamedSource, Report, SourceSpan, WrapErr}; +use miette::{miette, Diagnostic, IntoDiagnostic, NamedSource, Report, SourceSpan, WrapErr}; use path_dsl::path; - +use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use regex::Regex; +use strum::IntoEnumIterator; use wax::Glob; use whippit::{metadata::SectionHeader, reexport::chumsky::prelude::Rich}; @@ -37,12 +48,70 @@ struct Cli { #[derive(Debug, Parser)] enum Subcommand { + /// Adjust test expectations in metadata using `wptreport.json` reports from CI runs covering + /// Firefox's implementation of WebGPU. + /// + /// The general usage of this subcommand is to (1) reset expectations according to some + /// heuristic, and then (2) extend expectations from more reports later to accommodate any + /// intermittents that are found. More concretely: + /// + /// 1. Pick a `reset-*` preset (which we'll call `RESET_PRESET`). See docs for `preset` for + /// more details on making this choice. + /// + /// 2. Gather reports into path(s) of your choice. + /// + /// 3. Run `moz-webgpu-cts process-reports --preset=$RESET_PRESET …` against the reports + /// you've gathered to cover all new permanent outcomes. If you are confident you picked the + /// right `RESET_PRESET`, you may delete the reports you provided to this run. + /// + /// 4. As intermittent outcomes are discovered (maybe again), run `moz-webgpu-cts + /// process-reports --preset=merge …` with reports. You may delete the reports after their + /// outcomes have been merged in. + ProcessReports { + /// Direct paths to report files to be processed. + report_paths: Vec, + /// Cross-platform `wax` globs to enumerate report files to be processed. + /// + /// N.B. for Windows users: backslashes are used strictly for escaped characters, and + /// forward slashes (`/`) are the only valid path separator for these globs. + #[clap(long = "glob", value_name = "REPORT_GLOB")] + report_globs: Vec, + /// A heuristic for resolving differences between current metadata and processed reports. + /// + /// When you use this subcommand, you need to use both the `merge` preset and a choice of + /// `reset-*` heuristic. The choice mostly depends on your taste for regressions in + /// intermittent outcomes: + /// + /// * Is your goal is to make changes to Firefox, and make CI pass again? If so, you + /// probably want `reset-contradictory`. + /// + /// * Are you trying to run the `triage` subcommand on a minimized set of expected + /// outcomes? If so, you probably want `reset-all`. + /// + /// `reset-contradictory` changes expectations to match the set of outcomes observed in the + /// provided `reports_*` when they are not a strict subset of expected outcomes in + /// metadata. This is guaranteed to cover new permanent outcomes in metadata, while + /// minimizing changes to current intermittent outcomes in metadata. It may, however, + /// result in some intermittent outcomes not being reset to new permanent outcomes. + /// + /// `reset-all` changes expectations to match reported outcomes _exactly_. Metadata is not + /// even considered. + #[clap(long)] + preset: ReportProcessingPreset, + }, #[clap(name = "fmt")] Format, Triage, ReadTestVariants, } +#[derive(Clone, Copy, Debug, ValueEnum)] +enum ReportProcessingPreset { + Merge, + ResetContradictory, + ResetAll, +} + fn main() -> ExitCode { env_logger::builder() .filter_level(log::LevelFilter::Info) @@ -125,6 +194,467 @@ fn run(cli: Cli) -> ExitCode { } match subcommand { + Subcommand::ProcessReports { + report_globs, + report_paths, + preset, + } => { + let report_globs = { + let mut found_glob_parse_err = false; + let globs = report_globs + .into_iter() + .filter_map(|glob| match Glob::diagnosed(&glob) { + Ok((glob, _diagnostics)) => Some(glob.into_owned().partition()), + Err(diagnostics) => { + found_glob_parse_err = true; + let error_reports = diagnostics + .into_iter() + .filter(|diag| { + // N.B.: There should be at least one of these! + diag.severity() + .map_or(true, |sev| sev == miette::Severity::Error) + }) + .map(Report::new_boxed); + for report in error_reports { + eprintln!("{report:?}"); + } + None + } + }) + .collect::>(); + + if found_glob_parse_err { + log::error!("failed to parse one or more WPT report globs; bailing"); + return ExitCode::FAILURE; + } + + globs + }; + + if report_paths.is_empty() && report_globs.is_empty() { + log::error!("no report paths specified, bailing"); + return ExitCode::FAILURE; + } + + let exec_report_paths = { + let mut found_glob_walk_err = false; + let files = report_globs + .iter() + .flat_map(|(base_path, glob)| { + glob.walk(base_path) + .filter_map(|entry| match entry { + Ok(entry) => Some(entry.into_path()), + Err(e) => { + found_glob_walk_err = true; + let ctx_msg = if let Some(path) = e.path() { + format!( + "failed to enumerate files for glob `{}` at path {}", + glob, + path.display() + ) + } else { + format!("failed to enumerate files for glob `{glob}`") + }; + let e = Report::msg(e).wrap_err(ctx_msg); + eprintln!("{e:?}"); + None + } + }) + .collect::>() // OPT: Can we get rid of this somehow? + }) + .chain(report_paths) + .collect::>(); + + if found_glob_walk_err { + log::error!(concat!( + "failed to enumerate files with WPT report globs, ", + "see above for more details" + )); + return ExitCode::FAILURE; + } + + files + }; + + if exec_report_paths.is_empty() { + log::error!("no WPT report files found, bailing"); + return ExitCode::FAILURE; + } + + log::trace!("working with the following WPT report files: {exec_report_paths:#?}"); + log::info!("working with {} WPT report files", exec_report_paths.len()); + + let meta_files_by_path = { + let raw_meta_files_by_path = match read_metadata() { + Ok(paths) => paths, + Err(AlreadyReportedToCommandline) => return ExitCode::FAILURE, + }; + + log::info!("parsing metadata…"); + let mut found_parse_err = false; + + let files = raw_meta_files_by_path + .into_iter() + .filter_map(|(path, file_contents)| { + match chumsky::Parser::parse(&File::parser(), &*file_contents).into_result() + { + Err(errors) => { + found_parse_err = true; + render_metadata_parse_errors(&path, &file_contents, errors); + None + } + Ok(file) => Some((path, file)), + } + }) + .collect::>(); + + if found_parse_err { + log::error!(concat!( + "found one or more failures while parsing metadata, ", + "see above for more details" + )); + return ExitCode::FAILURE; + } + + files + }; + + let mut outcomes_by_test = IndexMap::::default(); + + log::info!("loading metadata for comparison to reports…"); + for (path, file) in meta_files_by_path { + let File { tests } = file; + for (SectionHeader(name), test) in tests { + let Test { + properties: + AnalyzeableProps { + is_disabled, + expectations, + }, + subtests, + } = test; + + let test_path = TestPath::from_fx_metadata_test( + path.strip_prefix(&gecko_checkout).unwrap(), + &name, + ) + .unwrap(); + + let freak_out_do_nothing = |what: &dyn Display| { + log::error!("hoo boy, not sure what to do yet: {what}") + }; + + let TestOutcomes { + test_outcomes: recorded_test_outcomes, + subtests: recorded_subtests, + } = outcomes_by_test + .entry(test_path.clone().into_owned()) + .or_default(); + + let test_path = &test_path; + let mut reported_dupe_already = false; + + let maybe_disabled = if is_disabled { + MaybeDisabled::Disabled + } else { + MaybeDisabled::Enabled(expectations) + }; + if let Some(_old) = recorded_test_outcomes.metadata.replace(maybe_disabled) { + freak_out_do_nothing( + &lazy_format!( + "duplicate entry for {test_path:?}, discarding previous entries with this and further dupes" + ) + ); + reported_dupe_already = true; + } + + for (SectionHeader(subtest_name), subtest) in subtests { + let Subtest { + properties: + AnalyzeableProps { + is_disabled, + expectations, + }, + } = subtest; + let recorded_subtest_outcomes = + recorded_subtests.entry(subtest_name.clone()).or_default(); + let maybe_disabled = if is_disabled { + MaybeDisabled::Disabled + } else { + MaybeDisabled::Enabled(expectations) + }; + if let Some(_old) = + recorded_subtest_outcomes.metadata.replace(maybe_disabled) + { + if !reported_dupe_already { + freak_out_do_nothing(&lazy_format!( + "duplicate subtest in {test_path:?} named {subtest_name:?}, discarding previous entries with this and further dupes" + )); + } + } + } + } + } + + log::info!("gathering reported test outcomes for reconciliation with metadata…"); + + let (exec_reports_sender, exec_reports_receiver) = channel(); + exec_report_paths + .into_par_iter() + .for_each_with(exec_reports_sender, |sender, path| { + let res = fs::File::open(&path) + .map(BufReader::new) + .map_err(Report::msg) + .wrap_err("failed to open file") + .and_then(|reader| { + serde_json::from_reader::<_, ExecutionReport>(reader) + .into_diagnostic() + .wrap_err("failed to parse JSON") + }) + .wrap_err_with(|| { + format!( + "failed to read WPT execution report from {}", + path.display() + ) + }) + .map(|parsed| (path, parsed)) + .map_err(|e| { + log::error!("{e:?}"); + AlreadyReportedToCommandline + }); + let _ = sender.send(res); + }); + + for res in exec_reports_receiver { + let (_path, exec_report) = match res { + Ok(ok) => ok, + Err(AlreadyReportedToCommandline) => return ExitCode::FAILURE, + }; + + let ExecutionReport { + run_info: + RunInfo { + platform, + build_profile, + }, + entries, + } = exec_report; + + for entry in entries { + let TestExecutionEntry { test_name, result } = entry; + + let test_path = TestPath::from_execution_report(&test_name).unwrap(); + let TestOutcomes { + test_outcomes: recorded_test_outcomes, + subtests: recorded_subtests, + } = outcomes_by_test + .entry(test_path.clone().into_owned()) + .or_default(); + + let (reported_outcome, reported_subtests) = match result { + TestExecutionResult::Complete { outcome, subtests } => (outcome, subtests), + TestExecutionResult::JobMaybeTimedOut { status, subtests } => { + if !status.is_empty() { + log::warn!( + concat!( + "expected an empty `status` field for {:?}, ", + "but found the {:?} status" + ), + test_path, + status, + ) + } + (TestOutcome::Timeout, subtests) + } + }; + + fn accumulate( + recorded: &mut BTreeMap>>, + platform: Platform, + build_profile: BuildProfile, + reported_outcome: Out, + ) where + Out: EnumSetType + Hash, + { + match recorded.entry(platform).or_default().entry(build_profile) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(Expectation::permanent(reported_outcome)); + } + std::collections::btree_map::Entry::Occupied(mut entry) => { + *entry.get_mut() |= reported_outcome + } + } + } + accumulate( + &mut recorded_test_outcomes.reported, + platform, + build_profile, + reported_outcome, + ); + + for reported_subtest in reported_subtests { + let SubtestExecutionResult { + subtest_name: reported_subtest_name, + outcome: reported_outcome, + } = reported_subtest; + + accumulate( + &mut recorded_subtests + .entry(reported_subtest_name.clone()) + .or_default() + .reported, + platform, + build_profile, + reported_outcome, + ); + } + } + } + + log::info!("metadata and reports gathered, now reconciling outcomes…"); + + let mut found_reconciliation_err = false; + let recombined_tests_iter = + outcomes_by_test + .into_iter() + .filter_map(|(test_path, outcomes)| { + fn reconcile( + outcomes: OutcomesForComparison, + preset: ReportProcessingPreset, + ) -> AnalyzeableProps + where + Out: Debug + Default + EnumSetType, + { + let OutcomesForComparison { metadata, reported } = outcomes; + + let metadata = metadata + .map(|maybe_disabled| { + maybe_disabled.map_enabled(|opt| opt.unwrap_or_default()) + }) + .unwrap_or_default(); + + let normalize = NormalizedExpectationPropertyValue::from_fully_expanded; + + let reconciled_expectations = metadata.map_enabled(|metadata| { + let resolve = match preset { + ReportProcessingPreset::ResetAll => return normalize(reported), + ReportProcessingPreset::ResetContradictory => { + |meta: Expectation<_>, rep: Option>| { + rep.filter(|rep| !meta.is_superset(rep)).unwrap_or(meta) + } + } + ReportProcessingPreset::Merge => |meta, rep| match rep { + Some(rep) => meta | rep, + None => meta, + }, + }; + + normalize( + Platform::iter() + .map(|platform| { + let build_profiles = BuildProfile::iter() + .map(|build_profile| { + ( + build_profile, + resolve( + metadata.get(platform, build_profile), + reported + .get(&platform) + .and_then(|rep| { + rep.get(&build_profile) + }) + .copied(), + ), + ) + }) + .collect(); + (platform, build_profiles) + }) + .collect(), + ) + }); + + match reconciled_expectations { + MaybeDisabled::Disabled => AnalyzeableProps { + is_disabled: true, + expectations: Default::default(), + }, + MaybeDisabled::Enabled(expectations) => AnalyzeableProps { + is_disabled: false, + expectations: Some(expectations), + }, + } + } + + let TestOutcomes { + test_outcomes, + subtests: recorded_subtests, + } = outcomes; + + let properties = reconcile(test_outcomes, preset); + + let mut subtests = BTreeMap::new(); + for (subtest_name, subtest) in recorded_subtests { + let subtest_name = SectionHeader(subtest_name); + if subtests.get(&subtest_name).is_some() { + found_reconciliation_err = true; + log::error!("internal error: duplicate test path {test_path:?}"); + } + subtests.insert( + subtest_name, + Subtest { + properties: reconcile(subtest, preset), + }, + ); + } + + if subtests.is_empty() && properties == Default::default() { + None + } else { + Some((test_path, (properties, subtests))) + } + }); + + log::info!( + "outcome reconciliation complete, gathering tests back into new metadata files…" + ); + + let mut files = BTreeMap::::new(); + for (test_path, (properties, subtests)) in recombined_tests_iter { + let name = test_path.test_name().to_string(); + let path = gecko_checkout.join(test_path.rel_metadata_path_fx().to_string()); + let file = files.entry(path).or_default(); + file.tests.insert( + SectionHeader(name), + Test { + properties, + subtests, + }, + ); + } + + log::info!("gathering of new metadata files completed, writing to file system…"); + + for (path, file) in files { + log::debug!("writing new metadata to {}", path.display()); + match write_to_file(&path, metadata::format_file(&file)) { + Ok(()) => (), + Err(AlreadyReportedToCommandline) => { + found_reconciliation_err = true; + } + } + } + + if found_reconciliation_err { + log::error!(concat!( + "one or more errors found while reconciling, ", + "exiting with failure; see above for more details" + )); + return ExitCode::FAILURE; + } + + ExitCode::SUCCESS + } Subcommand::Format => { let raw_test_files_by_path = match read_metadata() { Ok(paths) => paths, @@ -133,9 +663,7 @@ fn run(cli: Cli) -> ExitCode { log::info!("formatting metadata in-place…"); let mut fmt_err_found = false; for (path, file_contents) in raw_test_files_by_path { - match chumsky::Parser::parse(&metadata::File::parser(), &*file_contents) - .into_result() - { + match chumsky::Parser::parse(&File::parser(), &*file_contents).into_result() { Err(errors) => { fmt_err_found = true; render_metadata_parse_errors(&path, &file_contents, errors); diff --git a/moz-webgpu-cts/src/metadata.rs b/moz-webgpu-cts/src/metadata.rs index 106157c..0a5b6e0 100644 --- a/moz-webgpu-cts/src/metadata.rs +++ b/moz-webgpu-cts/src/metadata.rs @@ -15,6 +15,7 @@ use chumsky::{ use enumset::EnumSetType; use format::lazy_format; use joinery::JoinableIterator; +use serde::Deserialize; use strum::{EnumIter, IntoEnumIterator}; use whippit::metadata::{ self, file_parser, @@ -263,7 +264,7 @@ pub enum BuildProfile { Optimized, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct AnalyzeableProps where Out: EnumSetType, @@ -350,7 +351,6 @@ where .collect(); NormalizedExpectationPropertyValue::from_fully_expanded(fully_expanded) - .unwrap() } } }); @@ -565,7 +565,8 @@ where } } -#[derive(Debug, EnumSetType, Hash)] +#[derive(Debug, Deserialize, EnumSetType, Hash)] +#[serde(rename_all = "UPPERCASE")] pub enum TestOutcome { Ok, Timeout, @@ -623,7 +624,8 @@ impl<'a> Properties<'a> for AnalyzeableProps { } } -#[derive(Debug, EnumSetType, Hash)] +#[derive(Debug, Deserialize, EnumSetType, Hash)] +#[serde(rename_all = "UPPERCASE")] pub enum SubtestOutcome { Pass, Fail, diff --git a/moz-webgpu-cts/src/process_reports.rs b/moz-webgpu-cts/src/process_reports.rs new file mode 100644 index 0000000..2ff7fde --- /dev/null +++ b/moz-webgpu-cts/src/process_reports.rs @@ -0,0 +1,62 @@ +use std::collections::BTreeMap; + +use enumset::EnumSetType; + +use crate::{ + metadata::{BuildProfile, Platform, SubtestOutcome, TestOutcome}, + shared::{Expectation, NormalizedExpectationPropertyValue}, +}; + +#[derive(Debug)] +pub(crate) struct OutcomesForComparison +where + Out: EnumSetType, +{ + pub metadata: Option>>>, + pub reported: BTreeMap>>, +} + +impl Default for OutcomesForComparison +where + Out: EnumSetType, +{ + fn default() -> Self { + Self { + metadata: None, + reported: Default::default(), + } + } +} + +#[derive(Debug)] +pub(crate) enum MaybeDisabled { + Disabled, + Enabled(T), +} + +impl Default for MaybeDisabled +where + T: Default, +{ + fn default() -> Self { + Self::Enabled(Default::default()) + } +} + +impl MaybeDisabled { + pub fn map_enabled(self, f: F) -> MaybeDisabled + where + F: FnOnce(T) -> U, + { + match self { + Self::Disabled => MaybeDisabled::Disabled, + Self::Enabled(t) => MaybeDisabled::Enabled(f(t)), + } + } +} + +#[derive(Debug, Default)] +pub(crate) struct TestOutcomes { + pub test_outcomes: OutcomesForComparison, + pub subtests: BTreeMap>, +} diff --git a/moz-webgpu-cts/src/report.rs b/moz-webgpu-cts/src/report.rs new file mode 100644 index 0000000..9097e92 --- /dev/null +++ b/moz-webgpu-cts/src/report.rs @@ -0,0 +1,95 @@ +use serde::{ + de::{Deserializer, Error}, + Deserialize, +}; + +use crate::metadata::{BuildProfile, Platform, SubtestOutcome, TestOutcome}; + +#[derive(Debug, Deserialize)] +pub(crate) struct ExecutionReport { + pub run_info: RunInfo, + #[serde(rename = "results")] + pub entries: Vec, +} + +#[derive(Debug)] +pub(crate) struct RunInfo { + pub platform: Platform, + pub build_profile: BuildProfile, +} + +impl<'de> Deserialize<'de> for RunInfo { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Debug, Deserialize)] + struct ActualRunInfo { + os: String, + processor: String, + win11_2009: bool, + debug: bool, + } + + let ActualRunInfo { + os, + processor, + win11_2009, + debug, + } = ActualRunInfo::deserialize(deserializer)?; + + let platform = match &*os { + "win" => { + if processor == "x86_64" && win11_2009 { + Platform::Windows + } else { + return Err(D::Error::custom("asdf")); + } + } + "mac" => Platform::MacOs, + "linux" => Platform::Linux, + other => return Err(D::Error::custom(format!("unrecognized platform {other:?}"))), + }; + + let build_profile = if debug { + BuildProfile::Debug + } else { + BuildProfile::Optimized + }; + + Ok(RunInfo { + platform, + build_profile, + }) + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct TestExecutionEntry { + #[serde(rename = "test")] + pub test_name: String, + #[serde(flatten)] + pub result: TestExecutionResult, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +pub(crate) enum TestExecutionResult { + Complete { + #[serde(rename = "status")] + outcome: TestOutcome, + subtests: Vec, + }, + JobMaybeTimedOut { + status: String, + subtests: Vec, + }, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct SubtestExecutionResult { + #[serde(rename = "name")] + pub subtest_name: String, + #[serde(rename = "status")] + pub outcome: SubtestOutcome, +} diff --git a/moz-webgpu-cts/src/shared.rs b/moz-webgpu-cts/src/shared.rs index 661816c..a51b8ee 100644 --- a/moz-webgpu-cts/src/shared.rs +++ b/moz-webgpu-cts/src/shared.rs @@ -1,11 +1,17 @@ use std::{ + borrow::Cow, collections::BTreeMap, fmt::{self, Debug, Display, Formatter}, num::NonZeroUsize, ops::{BitOr, BitOrAssign}, + path::Path, }; +use camino::{Utf8Component, Utf8Path}; + use enumset::{EnumSet, EnumSetType}; +use format::lazy_format; +use joinery::JoinableIterator; use strum::IntoEnumIterator; use crate::metadata::{BuildProfile, Platform}; @@ -81,6 +87,13 @@ where pub fn iter(&self) -> impl Iterator { self.inner().iter() } + + pub fn is_superset(&self, rep: &Expectation) -> bool + where + Out: std::fmt::Debug + Default + EnumSetType, + { + self.inner().is_superset(*rep.inner()) + } } impl Display for Expectation @@ -181,7 +194,7 @@ where /// Yes, the type is _gnarly_. Sorry about that. This is some complex domain, okay? 😆😭 /// /// [`AnalyzeableProps`]: crate::metadata::AnalyzeableProps -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct NormalizedExpectationPropertyValue(NormalizedExpectationPropertyValueData) where Out: EnumSetType; @@ -195,6 +208,17 @@ pub type NormalizedExpectationPropertyValueData = MaybeCollapsed< BTreeMap>, >; +impl Default for NormalizedExpectationPropertyValue +where + Out: Default + EnumSetType, +{ + fn default() -> Self { + Self(MaybeCollapsed::Collapsed(MaybeCollapsed::Collapsed( + Default::default(), + ))) + } +} + impl NormalizedExpectationPropertyValue where Out: EnumSetType, @@ -217,12 +241,12 @@ where pub(crate) fn from_fully_expanded( outcomes: BTreeMap>>, - ) -> Option + ) -> Self where Out: Default, { if outcomes.is_empty() { - return None; + return Self::default(); } fn normalize( @@ -269,9 +293,347 @@ where } } - Some(NormalizedExpectationPropertyValue(normalize( - outcomes, - |by_build_profile| normalize(by_build_profile, std::convert::identity), - ))) + NormalizedExpectationPropertyValue(normalize(outcomes, |by_build_profile| { + normalize(by_build_profile, std::convert::identity) + })) } + + pub fn get(&self, platform: Platform, build_profile: BuildProfile) -> Expectation + where + Out: Default, + { + match self.inner() { + MaybeCollapsed::Collapsed(exps) => match exps { + MaybeCollapsed::Collapsed(exps) => *exps, + MaybeCollapsed::Expanded(exps) => { + exps.get(&build_profile).copied().unwrap_or_default() + } + }, + MaybeCollapsed::Expanded(exps) => exps + .get(&platform) + .and_then(|exps| match exps { + MaybeCollapsed::Collapsed(exps) => Some(*exps), + MaybeCollapsed::Expanded(exps) => exps.get(&build_profile).copied(), + }) + .unwrap_or_default(), + } + } +} + +/// A single symbolic path to a test and its metadata. +/// +/// This API is useful as a common representation of a path for [`crate::report::ExecutionReport`]s +/// and [`crate::metadata::File`]s. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) struct TestPath<'a> { + pub scope: TestScope, + /// A relative offset into `scope`. + pub path: Cow<'a, Utf8Path>, + /// The variant of this particular test from this test's source code. If set, you should be + /// able to correlate this with + /// + /// Generally, a test in WPT is _either_ a single test, or a set of test variants. That is, for + /// a given `path`, there will be a single `variant: None`, or multiple tests with `variant: + /// Some(…)`. + pub variant: Option>, +} + +const SCOPE_DIR_FX_PRIVATE_STR: &str = "testing/web-platform/mozilla/meta"; +const SCOPE_DIR_FX_PRIVATE_COMPONENTS: &[&str] = &["testing", "web-platform", "mozilla", "meta"]; +const SCOPE_DIR_FX_PUBLIC_STR: &str = "testing/web-platform/meta"; +const SCOPE_DIR_FX_PUBLIC_COMPONENTS: &[&str] = &["testing", "web-platform", "meta"]; + +impl<'a> TestPath<'a> { + pub fn from_execution_report( + test_url_path: &'a str, + ) -> Result> { + let err = || ExecutionReportPathError { test_url_path }; + let Some((scope, path)) = test_url_path + .strip_prefix("/_mozilla/") + .map(|stripped| (TestScope::FirefoxPrivate, stripped)) + .or_else(|| { + test_url_path + .strip_prefix('/') + .map(|stripped| (TestScope::Public, stripped)) + }) + else { + return Err(err()); + }; + + if path.contains('\\') { + return Err(err()); + } + + let (path, variant) = match path.split('/').next_back() { + Some(path_and_maybe_variants) => match path_and_maybe_variants.find('?') { + Some(query_params_start_idx) => ( + &path[..path.len() - (path_and_maybe_variants.len() - query_params_start_idx)], + Some(&path_and_maybe_variants[query_params_start_idx..]), + ), + None => (path, None), + }, + None => return Err(err()), + }; + + Ok(Self { + scope, + path: Utf8Path::new(path).into(), + variant: variant.map(Into::into), + }) + } + + pub fn from_fx_metadata_test( + rel_meta_file_path: &'a Path, + test_name: &'a str, + ) -> Result> { + let rel_meta_file_path = + Utf8Path::new(rel_meta_file_path.to_str().ok_or(MetadataTestPathError { + rel_meta_file_path, + test_name, + })?); + let err = || MetadataTestPathError { + rel_meta_file_path: rel_meta_file_path.as_std_path(), + test_name, + }; + let rel_meta_file_path = Utf8Path::new( + rel_meta_file_path + .as_str() + .strip_suffix(".ini") + .ok_or(err())?, + ); + + let (scope, path) = { + if let Ok(path) = rel_meta_file_path.strip_prefix(SCOPE_DIR_FX_PRIVATE_STR) { + (TestScope::FirefoxPrivate, path) + } else if let Ok(path) = rel_meta_file_path.strip_prefix(SCOPE_DIR_FX_PUBLIC_STR) { + (TestScope::Public, path) + } else { + return Err(err()); + } + }; + + let (base_name, variant) = Self::split_test_base_name_from_variant(test_name); + + if path.components().next_back() != Some(Utf8Component::Normal(base_name)) { + return Err(err()); + } + + Ok(Self { + scope, + path: Utf8Path::new(path).into(), + variant: variant.map(Into::into), + }) + } + + fn split_test_base_name_from_variant(url_ish_name: &'a str) -> (&'a str, Option<&'a str>) { + match url_ish_name.find('?') { + Some(query_params_start_idx) => ( + &url_ish_name[..url_ish_name.len() - (url_ish_name.len() - query_params_start_idx)], + Some(&url_ish_name[query_params_start_idx..]), + ), + None => (url_ish_name, None), + } + } + + pub fn into_owned(self) -> TestPath<'static> { + let Self { + scope, + path, + variant, + } = self; + + TestPath { + scope: scope.clone(), + path: path.clone().into_owned().into(), + variant: variant.clone().map(|v| v.into_owned().into()), + } + } + + pub(crate) fn test_name(&self) -> impl Display + '_ { + let Self { + path, + variant, + scope: _, + } = self; + let base_name = path.file_name().unwrap(); + + lazy_format!(move |f| { + write!(f, "{base_name}")?; + if let Some(variant) = variant { + write!(f, "{variant}")?; + } + Ok(()) + }) + } + + pub(crate) fn rel_metadata_path_fx(&self) -> impl Display + '_ { + let Self { + path, + variant: _, + scope, + } = self; + + let scope_dir = match scope { + TestScope::Public => SCOPE_DIR_FX_PUBLIC_COMPONENTS, + TestScope::FirefoxPrivate => SCOPE_DIR_FX_PRIVATE_COMPONENTS, + } + .iter() + .join_with(std::path::MAIN_SEPARATOR); + + lazy_format!(move |f| { write!(f, "{scope_dir}{}{path}.ini", std::path::MAIN_SEPARATOR) }) + } +} + +impl Display for TestPath<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Self { + variant, + // These are used by our call to `rel_metadata_path_fx` below: + scope: _, + path: _, + } = self; + write!( + f, + "{}{}", + self.rel_metadata_path_fx(), + lazy_format!(|f| { + match variant { + Some(variant) => write!(f, "{variant}"), + None => Ok(()), + } + }) + ) + } +} + +#[derive(Debug)] +pub struct ExecutionReportPathError<'a> { + test_url_path: &'a str, +} + +impl Display for ExecutionReportPathError<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Self { test_url_path } = self; + write!( + f, + concat!( + "failed to derive test path from execution report's entry ", + "for a test at URL path {:?}" + ), + test_url_path + ) + } +} + +#[derive(Debug)] +pub struct MetadataTestPathError<'a> { + rel_meta_file_path: &'a Path, + test_name: &'a str, +} + +impl Display for MetadataTestPathError<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Self { + rel_meta_file_path, + test_name, + } = self; + write!( + f, + "failed to derive test path from relative metadata path {:?} and test name {:?}", + rel_meta_file_path, test_name + ) + } +} + +/// Symbolically represents a file root from which tests and metadata are based. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) enum TestScope { + /// A public test available at some point in the history of [WPT upstream]. Note that while + /// a test may be public, metadata associated with it is in a private location. + /// + /// [WPT upstream]: https://github.com/web-platform-tests/wpt + Public, + /// A private test specific to Firefox. + FirefoxPrivate, +} + +#[test] +fn parse_test_path() { + assert_eq!( + TestPath::from_fx_metadata_test( + Path::new("testing/web-platform/mozilla/meta/blarg/cts.https.html.ini"), + "cts.https.html?stuff=things" + ) + .unwrap(), + TestPath { + scope: TestScope::FirefoxPrivate, + path: Utf8Path::new("blarg/cts.https.html").into(), + variant: Some("?stuff=things".into()), + } + ); + + assert_eq!( + TestPath::from_fx_metadata_test( + Path::new("testing/web-platform/meta/stuff/things/cts.https.html.ini"), + "cts.https.html" + ) + .unwrap(), + TestPath { + scope: TestScope::Public, + path: Utf8Path::new("stuff/things/cts.https.html").into(), + variant: None, + } + ); +} + +#[test] +fn report_meta_match() { + macro_rules! assert_test_matches_meta { + ($test_run_path:expr, $rel_meta_path:expr, $test_section_header:expr) => { + assert_eq!( + TestPath::from_execution_report($test_run_path).unwrap(), + TestPath::from_fx_metadata_test(Path::new($rel_meta_path), $test_section_header) + .unwrap() + ) + }; + } + + assert_test_matches_meta!( + "/_mozilla/blarg/cts.https.html?stuff=things", + "testing/web-platform/mozilla/meta/blarg/cts.https.html.ini", + "cts.https.html?stuff=things" + ); + + assert_test_matches_meta!( + "/blarg/cts.https.html?stuff=things", + "testing/web-platform/meta/blarg/cts.https.html.ini", + "cts.https.html?stuff=things" + ); +} + +#[test] +fn report_meta_reject() { + macro_rules! assert_test_rejects_meta { + ($test_run_path:expr, $rel_meta_path:expr, $test_section_header:expr) => { + assert_ne!( + TestPath::from_execution_report($test_run_path).unwrap(), + TestPath::from_fx_metadata_test(Path::new($rel_meta_path), $test_section_header) + .unwrap() + ) + }; + } + + assert_test_rejects_meta!( + "/blarg/cts.https.html?stuff=things", + // Wrong: the `mozilla` component shouldn't be after `web-platform` + "testing/web-platform/mozilla/meta/blarg/cts.https.html.ini", + "cts.https.html?stuff=things" + ); + + assert_test_rejects_meta!( + "/_mozilla/blarg/cts.https.html?stuff=things", + // Wrong: missing the `mozilla` component after `web-platform` + "testing/web-platform/meta/blarg/cts.https.html.ini", + "cts.https.html?stuff=things" + ); }