feat: run system commands (#193)

CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## [0.16.0] - 2023-09-15
+
+* Support running external system commands with the syntax below. This is useful for manipulating some external resources during the test.
+  ```
+  system ok
+  echo "Hello, world!"
+  ```
+  The runner will check the exit code of the command, and the output will be ignored. Currently, only `ok` is supported.
+
+  Changes:
+  - (parser) **Breaking change**: Add `Record::System`, and corresponding `TestErrorKind` and `RecordOutput`. Mark `TestErrorKind` and `RecordOutput` as `#[non_exhaustive]`.
+  - (runner) Add `run_command` to `AsyncDB` trait. The default implementation will run the command with `std::process::Command::status`. Implementors can override this method to utilize an asynchronous runtime such as `tokio`.
+
+* fix(runner): fix database name duplication for parallel tests by using the **full path** of the test file (instead of the file name) as the database name.
+
 ## [0.15.3] - 2023-08-02
 
 * fix(bin): fix error context display. To avoid stack backtrace being printed, unset `RUST_BACKTRACE` environment variable, or use pre-built binaries built with stable toolchain instead.

Cargo.toml

@@ -2,7 +2,7 @@
 members = ["sqllogictest", "sqllogictest-bin", "sqllogictest-engines", "tests"]
 
 [workspace.package]
-version = "0.15.3"
+version = "0.16.0"
 edition = "2021"
 homepage = "https://github.com/risinglightdb/sqllogictest-rs"
 keywords = ["sql", "database", "parser", "cli"]

sqllogictest-bin/Cargo.toml

@@ -24,8 +24,8 @@ glob = "0.3"
 itertools = "0.11"
 quick-junit = { version = "0.3" }
 rand = "0.8"
-sqllogictest = { path = "../sqllogictest", version = "0.15" }
-sqllogictest-engines = { path = "../sqllogictest-engines", version = "0.15" }
+sqllogictest = { path = "../sqllogictest", version = "0.16" }
+sqllogictest-engines = { path = "../sqllogictest-engines", version = "0.16" }
 tokio = { version = "1", features = [
     "rt",
     "rt-multi-thread",

sqllogictest-bin/src/engines.rs

@@ -1,4 +1,6 @@
 use std::fmt::Display;
+use std::process::ExitStatus;
+use std::time::Duration;
 
 use async_trait::async_trait;
 use clap::ValueEnum;
@@ -97,14 +99,14 @@ impl std::error::Error for EnginesError {
     }
 }
 
-impl Engines {
-    async fn run(&mut self, sql: &str) -> Result<DBOutput<DefaultColumnType>, anyhow::Error> {
-        Ok(match self {
-            Engines::Postgres(e) => e.run(sql).await?,
-            Engines::PostgresExtended(e) => e.run(sql).await?,
-            Engines::External(e) => e.run(sql).await?,
-        })
-    }
+macro_rules! dispatch_engines {
+    ($impl:expr, $inner:ident, $body:tt) => {{
+        match $impl {
+            Engines::Postgres($inner) => $body,
+            Engines::PostgresExtended($inner) => $body,
+            Engines::External($inner) => $body,
+        }
+    }};
 }
 
 #[async_trait]
@@ -113,6 +115,22 @@ impl AsyncDB for Engines {
     type ColumnType = DefaultColumnType;
 
     async fn run(&mut self, sql: &str) -> Result<DBOutput<Self::ColumnType>, Self::Error> {
-        self.run(sql).await.map_err(EnginesError)
+        dispatch_engines!(self, e, {
+            e.run(sql)
+                .await
+                .map_err(|e| EnginesError(anyhow::Error::from(e)))
+        })
+    }
+
+    fn engine_name(&self) -> &str {
+        dispatch_engines!(self, e, { e.engine_name() })
+    }
+
+    async fn sleep(dur: Duration) {
+        tokio::time::sleep(dur).await
+    }
+
+    async fn run_command(command: std::process::Command) -> std::io::Result<ExitStatus> {
+        Command::from(command).status().await
     }
 }

sqllogictest-engines/Cargo.toml

@@ -19,7 +19,7 @@ postgres-types = { version = "0.2.5", features = ["derive", "with-chrono-0_4"] }
 rust_decimal = { version = "1.30.0", features = ["tokio-pg"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-sqllogictest = { path = "../sqllogictest", version = "0.15" }
+sqllogictest = { path = "../sqllogictest", version = "0.16" }
 thiserror = "1"
 tokio = { version = "1", features = [
     "rt",

sqllogictest-engines/src/external.rs

@@ -1,6 +1,6 @@
 use std::io;
 use std::marker::PhantomData;
-use std::process::Stdio;
+use std::process::{ExitStatus, Stdio};
 use std::time::Duration;
 
 use async_trait::async_trait;
@@ -120,6 +120,10 @@ impl AsyncDB for ExternalDriver {
     async fn sleep(dur: Duration) {
         tokio::time::sleep(dur).await
     }
+
+    async fn run_command(command: std::process::Command) -> std::io::Result<ExitStatus> {
+        Command::from(command).status().await
+    }
 }
 
 struct JsonDecoder<T>(PhantomData<T>);

sqllogictest-engines/src/postgres/extended.rs

@@ -1,4 +1,5 @@
 use std::fmt::Write;
+use std::process::{Command, ExitStatus};
 use std::time::Duration;
 
 use async_trait::async_trait;
@@ -317,4 +318,8 @@ impl sqllogictest::AsyncDB for Postgres<Extended> {
     async fn sleep(dur: Duration) {
         tokio::time::sleep(dur).await
     }
+
+    async fn run_command(command: Command) -> std::io::Result<ExitStatus> {
+        tokio::process::Command::from(command).status().await
+    }
 }

sqllogictest-engines/src/postgres/simple.rs

@@ -1,3 +1,4 @@
+use std::process::{Command, ExitStatus};
 use std::time::Duration;
 
 use async_trait::async_trait;
@@ -64,4 +65,8 @@ impl sqllogictest::AsyncDB for Postgres<Simple> {
     async fn sleep(dur: Duration) {
         tokio::time::sleep(dur).await
     }
+
+    async fn run_command(command: Command) -> std::io::Result<ExitStatus> {
+        tokio::process::Command::from(command).status().await
+    }
 }

sqllogictest/Cargo.toml

@@ -24,3 +24,6 @@ regex = "1.9.1"
 owo-colors = "3.5.0"
 md-5 = "0.10"
 fs-err = "2.9.0"
+
+[dev-dependencies]
+pretty_assertions = "1"

sqllogictest/src/parser.rs

@@ -109,6 +109,14 @@ pub enum Record<T: ColumnType> {
         /// The expected results.
         expected_results: Vec<String>,
     },
+    /// A system command is an external command that is to be executed by the shell. Currently it
+    /// must succeed and the output is ignored.
+    System {
+        loc: Location,
+        conditions: Vec<Condition>,
+        /// The external command.
+        command: String,
+    },
     /// A sleep period.
     Sleep {
         loc: Location,
@@ -239,6 +247,13 @@ impl<T: ColumnType> std::fmt::Display for Record<T> {
                 // query always ends with a blank line
                 writeln!(f)
             }
+            Record::System {
+                loc: _,
+                conditions: _,
+                command,
+            } => {
+                writeln!(f, "system ok\n{command}")
+            }
             Record::Sleep { loc: _, duration } => {
                 write!(f, "sleep {}", humantime::format_duration(*duration))
             }
@@ -612,6 +627,25 @@ fn parse_inner<T: ColumnType>(loc: &Location, script: &str) -> Result<Vec<Record
                     expected_error,
                 });
             }
+            ["system", "ok"] => {
+                // TODO: we don't support asserting error message for system command
+                let mut command = match lines.next() {
+                    Some((_, line)) => line.into(),
+                    None => return Err(ParseErrorKind::UnexpectedEOF.at(loc.next_line())),
+                };
+                for (_, line) in &mut lines {
+                    if line.is_empty() {
+                        break;
+                    }
+                    command += "\n";
+                    command += line;
+                }
+                records.push(Record::System {
+                    loc,
+                    conditions: std::mem::take(&mut conditions),
+                    command,
+                });
+            }
             ["control", res @ ..] => match res {
                 ["sortmode", sort_mode] => match SortMode::try_from_str(sort_mode) {
                     Ok(sort_mode) => records.push(Record::Control(Control::SortMode(sort_mode))),
@@ -739,6 +773,11 @@ mod tests {
         parse_roundtrip::<CustomColumnType>("../tests/custom_type/custom_type.slt")
     }
 
+    #[test]
+    fn test_system_command() {
+        parse_roundtrip::<DefaultColumnType>("../tests/system_command/system_command.slt")
+    }
+
     #[test]
     fn test_fail_unknown_type() {
         let script = "\
@@ -805,18 +844,7 @@ select * from foo;
         let records = normalize_filename(records);
         let reparsed_records = normalize_filename(reparsed_records);
 
-        assert_eq!(
-            records, reparsed_records,
-            "Mismatch in reparsed records\n\
-                    *********\n\
-                    original:\n\
-                    *********\n\
-                    {records:#?}\n\n\
-                    *********\n\
-                    reparsed:\n\
-                    *********\n\
-                    {reparsed_records:#?}\n\n",
-        );
+        pretty_assertions::assert_eq!(records, reparsed_records, "Mismatch in reparsed records");
     }
 
     /// Replaces the actual filename in all Records with
@@ -829,6 +857,7 @@ select * from foo;
                 match &mut record {
                     Record::Include { loc, .. } => normalize_loc(loc),
                     Record::Statement { loc, .. } => normalize_loc(loc),
+                    Record::System { loc, .. } => normalize_loc(loc),
                     Record::Query { loc, .. } => normalize_loc(loc),
                     Record::Sleep { loc, .. } => normalize_loc(loc),
                     Record::Subtest { loc, .. } => normalize_loc(loc),