From 96cf3a1b8d3c809ae0035f443821373d472a2963 Mon Sep 17 00:00:00 2001 From: sagudev <16504129+sagudev@users.noreply.github.com> Date: Mon, 29 Jan 2024 18:22:48 +0100 Subject: [PATCH 1/8] Add servo flag --- moz-webgpu-cts/src/main.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index 738aa2b..1ecd874 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -47,6 +47,8 @@ use whippit::{ struct Cli { #[clap(long)] gecko_checkout: Option, + #[clap(long)] + servo: bool, #[clap(subcommand)] subcommand: Subcommand, } @@ -125,6 +127,7 @@ fn run(cli: Cli) -> ExitCode { let Cli { gecko_checkout, subcommand, + servo, } = cli; let gecko_checkout = match gecko_checkout @@ -136,8 +139,11 @@ fn run(cli: Cli) -> ExitCode { }; let read_metadata = || -> Result<_, AlreadyReportedToCommandline> { - let webgpu_cts_meta_parent_dir = - { path!(&gecko_checkout | "testing" | "web-platform" | "mozilla" | "meta" | "webgpu") }; + let webgpu_cts_meta_parent_dir = if !servo { + path!(&gecko_checkout | "testing" | "web-platform" | "mozilla" | "meta" | "webgpu") + } else { + path!(&gecko_checkout | "tests" | "wpt" | "webgpu" | "meta" | "webgpu") + }; let mut found_err = false; let collected = From e4ee6e889ee752250c0303bdbfb3d34af5708704 Mon Sep 17 00:00:00 2001 From: sagudev <16504129+sagudev@users.noreply.github.com> Date: Fri, 16 Feb 2024 17:19:08 +0100 Subject: [PATCH 2/8] Add `FAIL` to TestOutcome --- moz-webgpu-cts/src/main.rs | 10 +++++++++ moz-webgpu-cts/src/metadata.rs | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index 1ecd874..4075fd1 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -932,6 +932,7 @@ fn run(cli: Cli) -> ExitCode { tests_with_runner_errors: TestSet, tests_with_disabled_or_skip: TestSet, tests_with_crashes: TestSet, + tests_with_fails: TestSet, subtests_with_failures_by_test: SubtestByTestSet, subtests_with_timeouts_by_test: SubtestByTestSet, } @@ -1069,6 +1070,14 @@ fn run(cli: Cli) -> ExitCode { // We skip this because this test _should_ contain subtests with // `TIMEOUT` and `NOTRUN`, so we shouldn't actually miss anything. TestOutcome::Timeout => (), + TestOutcome::Fail => receiver(&mut |analysis| { + insert_in_test_set( + &mut analysis.tests_with_fails, + test_name, + expectation, + outcome, + ) + }), TestOutcome::Crash => receiver(&mut |analysis| { insert_in_test_set( &mut analysis.tests_with_crashes, @@ -1267,6 +1276,7 @@ fn run(cli: Cli) -> ExitCode { tests_with_runner_errors, tests_with_disabled_or_skip, tests_with_crashes, + tests_with_fails, subtests_with_failures_by_test, subtests_with_timeouts_by_test, } = analysis; diff --git a/moz-webgpu-cts/src/metadata.rs b/moz-webgpu-cts/src/metadata.rs index 503d1df..54ecf7d 100644 --- a/moz-webgpu-cts/src/metadata.rs +++ b/moz-webgpu-cts/src/metadata.rs @@ -1118,6 +1118,7 @@ where #[serde(rename_all = "UPPERCASE")] pub enum TestOutcome { Ok, + Fail, Timeout, Crash, Error, @@ -1137,6 +1138,7 @@ impl Display for TestOutcome { "{}", match self { Self::Ok => "OK", + Self::Fail => "FAIL", Self::Timeout => "TIMEOUT", Self::Crash => "CRASH", Self::Error => "ERROR", @@ -1155,6 +1157,7 @@ impl<'a> Properties<'a> for TestProps { helper, choice(( keyword("OK").to(TestOutcome::Ok), + keyword("FAIL").to(TestOutcome::Fail), keyword("CRASH").to(TestOutcome::Crash), keyword("TIMEOUT").to(TestOutcome::Timeout), keyword("ERROR").to(TestOutcome::Error), @@ -1629,4 +1632,38 @@ r#" } "### ); + assert_debug_snapshot!( + parser().parse(r#" +[canvas_complex_rgba8unorm_store.https.html] + expected: FAIL + +"#), + @r###" + ParseResult { + output: Some( + ( + "canvas_complex_rgba8unorm_store.https.html", + Test { + properties: TestProps { + is_disabled: false, + expectations: Some( + NormalizedExpectationPropertyValue( + Collapsed( + Collapsed( + [ + Fail, + ], + ), + ), + ), + ), + }, + subtests: {}, + }, + ), + ), + errs: [], + } + "### + ); } From 9ea38cc952b5c6c3c360f35f4c7150d9cf882b09 Mon Sep 17 00:00:00 2001 From: sagudev <16504129+sagudev@users.noreply.github.com> Date: Sun, 18 Feb 2024 09:02:47 +0100 Subject: [PATCH 3/8] pass/fail as test result --- moz-webgpu-cts/src/main.rs | 17 +++++++++-------- moz-webgpu-cts/src/metadata.rs | 31 +++++++++---------------------- 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index 4075fd1..d3b0e1d 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -1070,14 +1070,6 @@ fn run(cli: Cli) -> ExitCode { // We skip this because this test _should_ contain subtests with // `TIMEOUT` and `NOTRUN`, so we shouldn't actually miss anything. TestOutcome::Timeout => (), - TestOutcome::Fail => receiver(&mut |analysis| { - insert_in_test_set( - &mut analysis.tests_with_fails, - test_name, - expectation, - outcome, - ) - }), TestOutcome::Crash => receiver(&mut |analysis| { insert_in_test_set( &mut analysis.tests_with_crashes, @@ -1102,6 +1094,15 @@ fn run(cli: Cli) -> ExitCode { outcome, ) }), + TestOutcome::Pass => (), + TestOutcome::Fail => receiver(&mut |analysis| { + insert_in_test_set( + &mut analysis.tests_with_fails, + test_name, + expectation, + outcome, + ) + }), } } } diff --git a/moz-webgpu-cts/src/metadata.rs b/moz-webgpu-cts/src/metadata.rs index 54ecf7d..bc50541 100644 --- a/moz-webgpu-cts/src/metadata.rs +++ b/moz-webgpu-cts/src/metadata.rs @@ -1118,11 +1118,12 @@ where #[serde(rename_all = "UPPERCASE")] pub enum TestOutcome { Ok, - Fail, Timeout, Crash, Error, Skip, + Pass, + Fail, } impl Default for TestOutcome { @@ -1138,11 +1139,12 @@ impl Display for TestOutcome { "{}", match self { Self::Ok => "OK", - Self::Fail => "FAIL", Self::Timeout => "TIMEOUT", Self::Crash => "CRASH", Self::Error => "ERROR", Self::Skip => "SKIP", + Self::Pass => "PASS", + Self::Fail => "FAIL", } ) } @@ -1157,11 +1159,12 @@ impl<'a> Properties<'a> for TestProps { helper, choice(( keyword("OK").to(TestOutcome::Ok), - keyword("FAIL").to(TestOutcome::Fail), keyword("CRASH").to(TestOutcome::Crash), keyword("TIMEOUT").to(TestOutcome::Timeout), keyword("ERROR").to(TestOutcome::Error), keyword("SKIP").to(TestOutcome::Skip), + keyword("PASS").to(TestOutcome::Pass), + keyword("FAIL").to(TestOutcome::Fail), )), ) .boxed() @@ -1367,24 +1370,6 @@ r#" assert_debug_snapshot!( parser().parse( r#" -[asdf] - # Incorrect; `PASS` isn't a valid test outcome (but it _is_ a valid subtest outcome). - expected: PASS -"# - ), - @r###" - ParseResult { - output: None, - errs: [ - found end of input at 108..112 expected something else, - ], - } - "### - ); - - assert_debug_snapshot!( - parser().parse( -r#" [asdf] [blarg] expected: [PASS, FAIL] @@ -1635,7 +1620,7 @@ r#" assert_debug_snapshot!( parser().parse(r#" [canvas_complex_rgba8unorm_store.https.html] - expected: FAIL + expected: [PASS, FAIL, CRASH] "#), @r###" @@ -1651,6 +1636,8 @@ r#" Collapsed( Collapsed( [ + Crash, + Pass, Fail, ], ), From 38c24132fd734dc7191309c32b0219787fdd1122 Mon Sep 17 00:00:00 2001 From: sagudev <16504129+sagudev@users.noreply.github.com> Date: Sun, 18 Feb 2024 09:02:54 +0100 Subject: [PATCH 4/8] more servo --- moz-webgpu-cts/src/shared.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/moz-webgpu-cts/src/shared.rs b/moz-webgpu-cts/src/shared.rs index 76acee5..d750bea 100644 --- a/moz-webgpu-cts/src/shared.rs +++ b/moz-webgpu-cts/src/shared.rs @@ -338,6 +338,8 @@ const SCOPE_DIR_FX_PRIVATE_STR: &str = "testing/web-platform/mozilla"; const SCOPE_DIR_FX_PRIVATE_COMPONENTS: &[&str] = &["testing", "web-platform", "mozilla"]; const SCOPE_DIR_FX_PUBLIC_STR: &str = "testing/web-platform"; const SCOPE_DIR_FX_PUBLIC_COMPONENTS: &[&str] = &["testing", "web-platform"]; +const SCOPE_DIR_SERVO_PUBLIC_STR: &str = "tests/wpt/webgpu"; +const SCOPE_DIR_SERVO_PUBLIC_COMPONENTS: &[&str] = &["tests", "wpt", "webgpu"]; impl<'a> TestPath<'a> { pub fn from_execution_report( @@ -403,6 +405,8 @@ impl<'a> TestPath<'a> { (TestScope::FirefoxPrivate, path) } else if let Ok(path) = rel_meta_file_path.strip_prefix(SCOPE_DIR_FX_PUBLIC_STR) { (TestScope::Public, path) + } else if let Ok(path) = rel_meta_file_path.strip_prefix(SCOPE_DIR_SERVO_PUBLIC_STR) { + (TestScope::Public, path) } else { return Err(err()); } @@ -492,6 +496,7 @@ impl<'a> TestPath<'a> { } = self; let scope_dir = match scope { + // TODO: servo TestScope::Public => SCOPE_DIR_FX_PUBLIC_COMPONENTS, TestScope::FirefoxPrivate => SCOPE_DIR_FX_PRIVATE_COMPONENTS, } From c20202b6e5175cc5d07d20b07b2004b1e624d384 Mon Sep 17 00:00:00 2001 From: sagudev <16504129+sagudev@users.noreply.github.com> Date: Sun, 18 Feb 2024 15:36:29 +0100 Subject: [PATCH 5/8] rel_metadata_path_fx for servo --- moz-webgpu-cts/src/main.rs | 12 +++++++----- moz-webgpu-cts/src/shared.rs | 16 +++++++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index d3b0e1d..e9420de 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -20,7 +20,7 @@ use std::{ fmt::{self, Debug, Display, Formatter}, fs, hash::Hash, - io::{self, BufReader, BufWriter}, + io::{self, BufReader, BufWriter, Read}, path::{Path, PathBuf}, process::ExitCode, sync::{mpsc::channel, Arc}, @@ -427,11 +427,13 @@ fn run(cli: Cli) -> ExitCode { .into_par_iter() .for_each_with(exec_reports_sender, |sender, path| { let res = fs::File::open(&path) - .map(BufReader::new) .map_err(Report::msg) .wrap_err("failed to open file") - .and_then(|reader| { - serde_json::from_reader::<_, ExecutionReport>(reader) + .and_then(|mut f| { + let mut s = String::new(); + f.read_to_string(&mut s).unwrap(); + let first = s.lines().next().unwrap(); + serde_json::from_str::(first) .into_diagnostic() .wrap_err("failed to parse JSON") }) @@ -713,7 +715,7 @@ fn run(cli: Cli) -> ExitCode { let mut files = BTreeMap::::new(); for (test_path, (properties, subtests)) in recombined_tests_iter { let name = test_path.test_name().to_string(); - let rel_path = Utf8PathBuf::from(test_path.rel_metadata_path_fx().to_string()); + let rel_path = Utf8PathBuf::from(test_path.rel_metadata_path_fx(servo).to_string()); let path = gecko_checkout.join(&rel_path); let file = files.entry(path).or_insert_with(|| File { properties: file_props_by_file diff --git a/moz-webgpu-cts/src/shared.rs b/moz-webgpu-cts/src/shared.rs index d750bea..239fc33 100644 --- a/moz-webgpu-cts/src/shared.rs +++ b/moz-webgpu-cts/src/shared.rs @@ -488,17 +488,23 @@ impl<'a> TestPath<'a> { }) } - pub(crate) fn rel_metadata_path_fx(&self) -> impl Display + '_ { + pub(crate) fn rel_metadata_path_fx(&self, servo: bool) -> impl Display + '_ { let Self { path, variant: _, scope, } = self; - let scope_dir = match scope { - // TODO: servo - TestScope::Public => SCOPE_DIR_FX_PUBLIC_COMPONENTS, - TestScope::FirefoxPrivate => SCOPE_DIR_FX_PRIVATE_COMPONENTS, + let scope_dir = if !servo { + match scope { + TestScope::Public => SCOPE_DIR_FX_PUBLIC_COMPONENTS, + TestScope::FirefoxPrivate => SCOPE_DIR_FX_PRIVATE_COMPONENTS, + } + } else { + match scope { + TestScope::Public => SCOPE_DIR_SERVO_PUBLIC_COMPONENTS, + TestScope::FirefoxPrivate => todo!(), + } } .iter() .chain(&["meta"]) From 48418264730c759c959fe4fd34547f804c8e1920 Mon Sep 17 00:00:00 2001 From: sagudev <16504129+sagudev@users.noreply.github.com> Date: Sun, 18 Feb 2024 17:10:07 +0100 Subject: [PATCH 6/8] strip _webgpu for servo --- moz-webgpu-cts/src/shared.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/moz-webgpu-cts/src/shared.rs b/moz-webgpu-cts/src/shared.rs index 239fc33..13458e1 100644 --- a/moz-webgpu-cts/src/shared.rs +++ b/moz-webgpu-cts/src/shared.rs @@ -351,8 +351,13 @@ impl<'a> TestPath<'a> { .map(|stripped| (TestScope::FirefoxPrivate, stripped)) .or_else(|| { test_url_path - .strip_prefix('/') + .strip_prefix("/_webgpu/") .map(|stripped| (TestScope::Public, stripped)) + .or_else(|| { + test_url_path + .strip_prefix('/') + .map(|stripped| (TestScope::Public, stripped)) + }) }) else { return Err(err()); From 773f118522461d45bfd8dd5183fd8a3712c8965e Mon Sep 17 00:00:00 2001 From: sagudev <16504129+sagudev@users.noreply.github.com> Date: Mon, 15 Apr 2024 19:31:45 +0200 Subject: [PATCH 7/8] Add set-good option --- moz-webgpu-cts/src/main.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index e9420de..73b8b4c 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -30,7 +30,7 @@ use camino::Utf8PathBuf; use clap::{Parser, ValueEnum}; use enumset::EnumSetType; use format::lazy_format; -use indexmap::{IndexMap, IndexSet}; +use indexmap::{Equivalent, IndexMap, IndexSet}; use joinery::JoinableIterator; use miette::{miette, Diagnostic, IntoDiagnostic, NamedSource, Report, SourceSpan, WrapErr}; use path_dsl::path; @@ -106,6 +106,8 @@ enum ReportProcessingPreset { #[value(alias("same-fx"))] Merge, ResetAll, + /// Resets only bad->good + SetGood, } #[derive(Clone, Copy, Debug, Default, ValueEnum)] @@ -628,6 +630,18 @@ fn run(cli: Cli) -> ExitCode { Some(rep) => meta | rep, None => meta, }, + ReportProcessingPreset::SetGood => { + |meta: Expectation<_>, rep: Option>| match rep { + Some(rep) => { + if rep.equivalent(&Expectation::default()) { + rep + } else { + meta + } + } + None => meta, + } + } }; normalize( @@ -681,6 +695,9 @@ fn run(cli: Cli) -> ExitCode { ); return None; } + ReportProcessingPreset::SetGood => { + log::warn!("no good results in {test_path:?}") + } } } From 039200b6100133c7fadab40cfef04771960b4a9f Mon Sep 17 00:00:00 2001 From: sagudev <16504129+sagudev@users.noreply.github.com> Date: Tue, 28 May 2024 15:59:24 +0200 Subject: [PATCH 8/8] ReportProcessingPreset::Set --- moz-webgpu-cts/src/main.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs index 73b8b4c..7edec08 100644 --- a/moz-webgpu-cts/src/main.rs +++ b/moz-webgpu-cts/src/main.rs @@ -106,6 +106,8 @@ enum ReportProcessingPreset { #[value(alias("same-fx"))] Merge, ResetAll, + /// Set's only reported results to new value + Set, /// Resets only bad->good SetGood, } @@ -630,6 +632,10 @@ fn run(cli: Cli) -> ExitCode { Some(rep) => meta | rep, None => meta, }, + ReportProcessingPreset::Set => |meta, rep| match rep { + Some(rep) => rep, + None => meta, + }, ReportProcessingPreset::SetGood => { |meta: Expectation<_>, rep: Option>| match rep { Some(rep) => { @@ -684,7 +690,7 @@ fn run(cli: Cli) -> ExitCode { if test_entry.reported.is_empty() { match preset { - ReportProcessingPreset::Merge => { + ReportProcessingPreset::Merge | ReportProcessingPreset::Set => { log::warn!("no entries found in reports for {test_path:?}") } ReportProcessingPreset::ResetAll