Merge pull request #609 from tweag/topiary-query-files-crate

Split of Topiary's builtin query files into their own crate

Cargo.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 license = "MIT"
 
 [workspace]
-members = ["topiary", "topiary-cli", "topiary-playground"]
+members = ["topiary", "topiary-cli", "topiary-queries", "topiary-playground"]
 exclude = ["samples"]
 
 [profile.release]

README.md

@@ -123,10 +123,10 @@ This won't work if you are running Topiary from another directory than this
 repository. In order to use Topiary without restriction, **you must set the
 environment variable `TOPIARY_LANGUAGE_DIR` to point to the directory where
 Topiary's language query files (`.scm`) are located**. By default, you should
-set it to `<local path of the topiary repository>/languages`, for example:
+set it to `<local path of the topiary repository>/queries`, for example:
 
 ```sh
-export TOPIARY_LANGUAGE_DIR=/home/me/tools/topiary/languages
+export TOPIARY_LANGUAGE_DIR=/home/me/tools/topiary/queries
 topiary fmt ./projects/helloworld/hello.ml
 ```
 

default.nix

@@ -26,8 +26,9 @@ let
         "Cargo.lock"
         "Cargo.toml"
         "languages.toml"
-        "languages"
+        "queries"
         "topiary"
+        "topiary-queries"
         "topiary-cli"
         "topiary-playground"
         "tests"
@@ -91,17 +92,26 @@ in
     pname = "topiary";
     cargoExtraArgs = "-p topiary-cli";
     postInstall = ''
-      install -Dm444 languages/* -t $out/share/languages
+      install -Dm444 queries/* -t $out/share/queries
     '';
 
     # Set TOPIARY_LANGUAGE_DIR to the Nix store
     # for the build
-    TOPIARY_LANGUAGE_DIR = "${placeholder "out"}/share/languages";
+    TOPIARY_LANGUAGE_DIR = "${placeholder "out"}/share/queries";
 
     # Set TOPIARY_LANGUAGE_DIR to the working directory
     # in a development shell
     shellHook = ''
-      export TOPIARY_LANGUAGE_DIR=$PWD/languages
+      export TOPIARY_LANGUAGE_DIR=$PWD/queries
+    '';
+  });
+
+  topiary-queries = craneLib.buildPackage (commonArgs
+    // {
+    pname = "topiary-queries";
+    cargoExtraArgs = "-p topiary-queries";
+    postInstall = ''
+      install -Dm444 queries/* -t $out/share/queries
     '';
   });
 

flake.nix

@@ -42,7 +42,7 @@
       in
       {
         packages = with code; {
-          inherit topiary-playground;
+          inherit topiary-playground topiary-queries;
           default = topiary-cli;
         };
 

playground.sh

@@ -64,7 +64,7 @@ idempotency() {
 main() {
   local query="${1-}"
   if ! [[ -e "${query}" ]]; then
-    query="languages/${query}.scm"
+    query="queries/${query}.scm"
     [[ -e "${query}" ]] || fail "Couldn't find language query file '${query}'"
   fi
 

queries

@@ -0,0 +1 @@
+topiary-queries/queries/

topiary-cli/Cargo.toml

@@ -40,8 +40,18 @@ tempfile = { workspace = true }
 tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "macros"] }
 toml = { workspace = true }
 topiary = { path = "../topiary" }
+topiary-queries = { path = "../topiary-queries" }
 tree-sitter-facade = { workspace = true }
 
+tree-sitter-json = { workspace = true }
+tree-sitter-rust = { workspace = true }
+tree-sitter-toml = { workspace = true }
+tree-sitter-bash = { workspace = true }
+tree-sitter-nickel = { workspace = true }
+tree-sitter-query = { workspace = true }
+tree-sitter-ocaml = { workspace = true }
+tree-sitter-ocamllex = { workspace = true }
+
 [dev-dependencies]
 assert_cmd = { workspace = true }
 predicates = { workspace = true }

topiary-cli/src/error.rs

@@ -19,6 +19,7 @@ pub enum CLIError {
     IOError(io::Error),
     Generic(Box<dyn error::Error>),
     Multiple,
+    UnsupportedLanguage(String),
 }
 
 /// # Safety
@@ -50,6 +51,7 @@ impl error::Error for TopiaryError {
             Self::Bin(_, Some(CLIError::IOError(error))) => Some(error),
             Self::Bin(_, Some(CLIError::Generic(error))) => error.source(),
             Self::Bin(_, Some(CLIError::Multiple)) => None,
+            Self::Bin(_, Some(CLIError::UnsupportedLanguage(_))) => None,
             Self::Bin(_, None) => None,
         }
     }

topiary-cli/src/io.rs

@@ -1,6 +1,6 @@
 use std::{
     ffi::OsString,
-    fmt,
+    fmt::{self, Display},
     fs::File,
     io::{stdin, stdout, ErrorKind, Read, Result, Write},
     path::{Path, PathBuf},
@@ -11,11 +11,42 @@ use topiary::{Configuration, Language, SupportedLanguage, TopiaryQuery};
 
 use crate::{
     cli::{AtLeastOneInput, ExactlyOneInput, FromStdin},
-    error::{CLIResult, TopiaryError},
+    error::{CLIError, CLIResult, TopiaryError},
     language::LanguageDefinition,
 };
 
-type QuerySource = PathBuf;
+#[derive(Debug, Clone, Hash)]
+pub enum QuerySource {
+    Path(PathBuf),
+    BuiltIn(String),
+}
+
+impl From<PathBuf> for QuerySource {
+    fn from(path: PathBuf) -> Self {
+        QuerySource::Path(path)
+    }
+}
+
+impl From<&PathBuf> for QuerySource {
+    fn from(path: &PathBuf) -> Self {
+        QuerySource::Path(path.clone())
+    }
+}
+
+impl From<&str> for QuerySource {
+    fn from(string: &str) -> Self {
+        QuerySource::BuiltIn(String::from(string))
+    }
+}
+
+impl Display for QuerySource {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            QuerySource::Path(p) => write!(f, "{}", p.to_string_lossy()),
+            QuerySource::BuiltIn(_) => write!(f, "built-in query"),
+        }
+    }
+}
 
 /// Unified interface for input sources. We either have input from:
 /// * Standard input, in which case we need to specify the language and, optionally, query override
@@ -34,7 +65,7 @@ impl From<&ExactlyOneInput> for InputFrom {
             ExactlyOneInput {
                 stdin: Some(FromStdin { language, query }),
                 ..
-            } => InputFrom::Stdin(language.to_owned(), query.to_owned()),
+            } => InputFrom::Stdin(language.to_owned(), query.as_ref().map(|p| p.into())),
 
             ExactlyOneInput {
                 file: Some(path), ..
@@ -51,7 +82,7 @@ impl From<&AtLeastOneInput> for InputFrom {
             AtLeastOneInput {
                 stdin: Some(FromStdin { language, query }),
                 ..
-            } => InputFrom::Stdin(language.to_owned(), query.to_owned()),
+            } => InputFrom::Stdin(language.to_owned(), query.as_ref().map(|p| p.into())),
 
             AtLeastOneInput { files, .. } => InputFrom::Files(files.to_owned()),
         }
@@ -88,10 +119,11 @@ impl<'cfg> InputFile<'cfg> {
     /// Convert our `InputFile` into language definition values that Topiary can consume
     pub async fn to_language_definition(&self) -> CLIResult<LanguageDefinition> {
         let grammar = self.language.grammar().await?;
-        let query = {
-            let contents = tokio::fs::read_to_string(&self.query).await?;
-            TopiaryQuery::new(&grammar, &contents)?
+        let contents = match &self.query {
+            QuerySource::Path(query) => tokio::fs::read_to_string(query).await?,
+            QuerySource::BuiltIn(contents) => contents.to_owned(),
         };
+        let query = TopiaryQuery::new(&grammar, &contents)?;
 
         Ok(LanguageDefinition {
             query,
@@ -111,7 +143,7 @@ impl<'cfg> InputFile<'cfg> {
     }
 
     /// Expose query path for input
-    pub fn query(&self) -> &PathBuf {
+    pub fn query(&self) -> &QuerySource {
         &self.query
     }
 }
@@ -145,7 +177,22 @@ impl<'cfg, 'i> Inputs<'cfg> {
             InputFrom::Stdin(language, query) => {
                 vec![(|| {
                     let language = language.to_language(config);
-                    let query = query.unwrap_or(language.query_file()?);
+                    let query = match query {
+                        // The user specified a query file
+                        Some(p) => p,
+                        // The user did not specify a file, try the default locations
+                        None => match language.query_file() {
+                            Ok(p) => p.into(),
+                            // For some reason, Topiary could not find any
+                            // matching file in a default location. As a final attempt, use try to the the
+                            // builtin ones. Store the error, return that if we
+                            // fail to find anything, because the builtin error might be unexpected.
+                            Err(e) => {
+                                log::warn!("No query files found in any of the expected locations. Falling back to compile-time included files.");
+                                to_query(language).map_err(|_| e)?
+                            }
+                        },
+                    };
 
                     Ok(InputFile {
                         source: InputSource::Stdin,
@@ -159,7 +206,7 @@ impl<'cfg, 'i> Inputs<'cfg> {
                 .into_iter()
                 .map(|path| {
                     let language = Language::detect(&path, config)?;
-                    let query = language.query_file()?;
+                    let query = language.query_file()?.into();
 
                     Ok(InputFile {
                         source: InputSource::Disk(path, None),
@@ -269,3 +316,21 @@ impl<'cfg> TryFrom<&InputFile<'cfg>> for OutputFile {
         }
     }
 }
+
+fn to_query(language: &Language) -> CLIResult<QuerySource> {
+    match language.name.as_str() {
+        "bash" => Ok(topiary_queries::bash().into()),
+        "json" => Ok(topiary_queries::json().into()),
+        "nickel" => Ok(topiary_queries::nickel().into()),
+        "ocaml" => Ok(topiary_queries::ocaml().into()),
+        "ocaml_interface" => Ok(topiary_queries::ocaml_interface().into()),
+        "ocamllex" => Ok(topiary_queries::ocamllex().into()),
+        "rust" => Ok(topiary_queries::rust().into()),
+        "toml" => Ok(topiary_queries::toml().into()),
+        "tree_sitter_query" => Ok(topiary_queries::tree_sitter_query().into()),
+        name => Err(TopiaryError::Bin(
+            format!("The specified language is unsupported: {}", name),
+            Some(CLIError::UnsupportedLanguage(name.to_string())),
+        )),
+    }
+}

topiary-cli/src/language.rs

@@ -52,7 +52,7 @@ impl LanguageDefinitionCache {
                     self,
                     key,
                     input.language(),
-                    input.query().file_name().unwrap().to_string_lossy()
+                    input.query()
                 );
 
                 lang_def.get().to_owned()
@@ -65,7 +65,7 @@ impl LanguageDefinitionCache {
                     self,
                     key,
                     input.language(),
-                    input.query().file_name().unwrap().to_string_lossy()
+                    input.query()
                 );
 
                 let lang_def = Arc::new(input.to_language_definition().await?);

topiary-cli/src/main.rs

@@ -61,7 +61,7 @@ async fn run() -> CLIResult<()> {
                                     "Formatting {}, as {} using {}, to {}",
                                     input.source(),
                                     input.language(),
-                                    input.query().to_string_lossy(),
+                                    input.query(),
                                     output
                                 );
 

topiary-cli/tests/cli-tester.rs

@@ -44,7 +44,7 @@ fn test_fmt_stdin() {
     let mut topiary = Command::cargo_bin("topiary").unwrap();
 
     topiary
-        .env("TOPIARY_LANGUAGE_DIR", "../languages")
+        .env("TOPIARY_LANGUAGE_DIR", "../queries")
         .arg("fmt")
         .arg("--language")
         .arg("json")
@@ -59,12 +59,12 @@ fn test_fmt_stdin_query() {
     let mut topiary = Command::cargo_bin("topiary").unwrap();
 
     topiary
-        .env("TOPIARY_LANGUAGE_DIR", "../languages")
+        .env("TOPIARY_LANGUAGE_DIR", "../queries")
         .arg("fmt")
         .arg("--language")
         .arg("json")
         .arg("--query")
-        .arg("../languages/json.scm")
+        .arg("../queries/json.scm")
         .write_stdin(JSON_INPUT)
         .assert()
         .success()
@@ -97,7 +97,7 @@ fn test_fmt_dir() {
     let mut topiary = Command::cargo_bin("topiary").unwrap();
 
     topiary
-        .env("TOPIARY_LANGUAGE_DIR", "../languages")
+        .env("TOPIARY_LANGUAGE_DIR", "../queries")
         .arg("fmt")
         .arg(json.path().parent().unwrap())
         .assert()
@@ -112,7 +112,7 @@ fn test_fmt_invalid() {
 
     // Can't specify --language with input files
     topiary
-        .env("TOPIARY_LANGUAGE_DIR", "../languages")
+        .env("TOPIARY_LANGUAGE_DIR", "../queries")
         .arg("fmt")
         .arg("--language")
         .arg("json")

topiary-playground/build.rs

@@ -20,11 +20,11 @@ fn to_js_string(path: PathBuf) -> String {
 }
 
 fn main() {
-    println!("cargo:rerun-if-changed=../languages/");
+    println!("cargo:rerun-if-changed=../queries/");
     println!("cargo:rerun-if-changed=../topiary/tests/samples/input/");
 
     // Export test samples and queries as JS files
-    let language_dir = current_dir().unwrap().join("../languages/");
+    let language_dir = current_dir().unwrap().join("../queries/");
     let language_files = fs::read_dir(language_dir).unwrap();
 
     let mut language_map: HashMap<String, String> = HashMap::new();