Support Markdown diff highilighting

build.rs

@@ -352,6 +352,11 @@ fn main() {
             src_dir: "vendor/tree-sitter-zig-src",
             extra_files: vec![],
         },
+        TreeSitterParser {
+            name: "tree-sitter-markdown",
+            src_dir: "vendor/tree-sitter-markdown-src",
+            extra_files: vec!["scanner.cc"],
+        },
     ];
 
     // Only rerun if relevant files in the vendor/ directory change.

sample_files/compare.expected

@@ -1,3 +1,6 @@
+sample_files/Session_before.kt sample_files/Session_after.kt
+46994b58bb24200f82866951013f03ce  -
+
 sample_files/b2_math_before.h sample_files/b2_math_after.h
 637e91c1528894ec0529bc845ef79c88  -
 
@@ -55,12 +58,12 @@ sample_files/haskell_before.hs sample_files/haskell_after.hs
 sample_files/hcl_before.hcl sample_files/hcl_after.hcl
 48b5599f3c6131a7eccaba73254fd599  -
 
-sample_files/helpful_before.el sample_files/helpful_after.el
-bce74573e003cc6b729a63a4bc34c4af  -
-
 sample_files/helpful-unit-test-before.el sample_files/helpful-unit-test-after.el
 79597af48ff80bcf9f5d02d20c51606d  -
 
+sample_files/helpful_before.el sample_files/helpful_after.el
+bce74573e003cc6b729a63a4bc34c4af  -
+
 sample_files/html_before.html sample_files/html_after.html
 5bf6a3d864706e18932c8cd027cb4c0b  -
 
@@ -103,6 +106,9 @@ sample_files/load_before.js sample_files/load_after.js
 sample_files/lua_before.lua sample_files/lua_after.lua
 c3d81271c060bd97dd246c1c5ea6a138  -
 
+sample_files/markdown_before.md sample_files/markdown_after.md
+aca6d38a647626d96bcb0a0cb791eaaa  -
+
 sample_files/metadata_before.clj sample_files/metadata_after.clj
 b71577801352071fd1c6a9079f2d9dbc  -
 
@@ -154,9 +160,6 @@ d88046b43f826a34bbd4a7b9d6bb704d  -
 sample_files/scala_before.scala sample_files/scala_after.scala
 bb18213033492a633e7f978ab9711ae1  -
 
-sample_files/Session_before.kt sample_files/Session_after.kt
-46994b58bb24200f82866951013f03ce  -
-
 sample_files/simple_before.js sample_files/simple_after.js
 b1fe2c184a9a358e314e21aabb0f3cb7  -
 

sample_files/markdown_after.md

@@ -0,0 +1 @@
+# Title of the Document Goes Here

sample_files/markdown_before.md

@@ -0,0 +1 @@
+# Title of the Doc Goes Here

src/parse/guess_language.rs

@@ -47,6 +47,7 @@ pub enum Language {
     Kotlin,
     Lua,
     Make,
+    Markdown,
     Nix,
     OCaml,
     OCamlInterface,
@@ -99,6 +100,7 @@ pub fn language_name(language: Language) -> &'static str {
         Kotlin => "Kotlin",
         Lua => "Lua",
         Make => "Make",
+        Markdown => "Markdown",
         Nix => "Nix",
         OCaml => "OCaml",
         OCamlInterface => "OCaml Interface",
@@ -182,6 +184,7 @@ pub const LANG_EXTENSIONS: &'static [(Language, &[&str])] = &[
     (Kotlin, &["kt", "ktm", "kts"]),
     (Lua, &["lua"]),
     (Make, &["mak", "d", "make", "makefile", "mk", "mkfile"]),
+    (Markdown, &["md", "markdown"]),
     (Nix, &["nix"]),
     (OCaml, &["ml"]),
     (OCamlInterface, &["mli"]),

src/parse/tree_sitter_parser.rs

@@ -68,6 +68,7 @@ extern "C" {
     fn tree_sitter_kotlin() -> ts::Language;
     fn tree_sitter_lua() -> ts::Language;
     fn tree_sitter_make() -> ts::Language;
+    fn tree_sitter_markdown_inline() -> ts::Language;
     fn tree_sitter_nix() -> ts::Language;
     fn tree_sitter_ocaml() -> ts::Language;
     fn tree_sitter_ocaml_interface() -> ts::Language;
@@ -535,6 +536,19 @@ pub fn from_language(language: guess::Language) -> TreeSitterConfig {
                 .unwrap(),
             }
         }
+        Markdown => {
+            let language = unsafe { tree_sitter_markdown_inline() };
+            TreeSitterConfig {
+                language,
+                atom_nodes: vec![].into_iter().collect(),
+                delimiter_tokens: vec![],
+                highlight_query: ts::Query::new(
+                    language,
+                    include_str!("../../vendor/highlights/markdown.scm"),
+                )
+                .unwrap(),
+            }
+        }
         Nix => {
             let language = unsafe { tree_sitter_nix() };
             TreeSitterConfig {

vendor/highlights/markdown.scm

@@ -0,0 +1 @@
+../tree-sitter-markdown/tree-sitter-markdown-inline/queries/highlights.scm

vendor/tree-sitter-markdown-src

@@ -0,0 +1 @@
+tree-sitter-markdown/tree-sitter-markdown-inline/src/

vendor/tree-sitter-markdown/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,20 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+
+**Code example**
+```markdown
+Your example here
+```
+
+**Expected behavior**
+<!-- consider linking to a relevant section in a spec like https://github.github.com/gfm/ -->
+
+**Actual behavior**

vendor/tree-sitter-markdown/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,11 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+<!-- consider linking a relevant section from a spec -->
+<!-- extensions from any spec (rmarkdown, pandoc, ...) are welcome -->

vendor/tree-sitter-markdown/.github/workflows/node.js.yml

@@ -0,0 +1,28 @@
+# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  push:
+    branches: [ "split_parser" ]
+  pull_request:
+    branches: [ "split_parser" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js 14
+      uses: actions/setup-node@v3
+      with:
+        node-version: 14
+        cache: 'npm'
+    - run: npm ci
+    - run: npm run build --if-present
+    - name: Check grammar is compiled correctly
+      run: git diff --exit-code
+    - run: npm test

vendor/tree-sitter-markdown/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+build
+target

vendor/tree-sitter-markdown/CONTRIBUTING.md

@@ -0,0 +1,70 @@
+# Contributing
+
+All contributions are welcome. Specifically, if you found a bug or have a
+suggestion for a new feature or markdown extension, you can always open an
+[issue] or [pull request].
+
+## Issues
+
+If you open an issue please give a short description of your bug or feature
+along with a code example. If there is a relevant spec like the [CommonMark
+Spec][commonmark] or the [GitHub Flavored Markdown Spec][gfm] please link it in
+the issue.
+
+Any feature suggestions are welcome. The grammar should by default only support
+very common syntax, but any extension can be implemented behind a compile time
+flag. (See below)
+
+Some bug reports belong in other repositories if they only concern the
+implementation of the grammar in a specific context like `nvim-treesitter`, but
+you can always open an issue here and I will point you in the right direction.
+
+## Code Overview
+
+Please refer to the [tree-sitter spec] for more details on how to write a tree-
+sitter grammar.
+
+This parse is split into two grammars. One for block structure, which can be
+found in the `tree-sitter-markdown` folder, and one for inline structure, which
+can be found in the `tree-sitter-markdown-inline` folder. Components that are
+parts of either grammar can be found in the `common` folder.
+
+For either of the grammar the most important files are the `grammar.js` which
+defines most nodes and the `src/scanner.cc` which defines nodes that cannot
+be parsed with normal tree-sitter rules. All other files in the `src` subfolder
+are auto-generated by running `tree-sitter generate --no-bindings`. (You need to
+install the [tree-sitter cli tool][tree-sitter-cli] first.)
+
+Some syntactical components can be enabled or disabled by environment variables
+at compile time. The logic for this can be found in the `common/grammar.js`
+file.
+
+Tests are located in the `corpus` subfolder:
+* `spec.txt` is taken from the examples in the [GFM spec][gfm].
+* `failing.txt` are those examples from the spec that do not pass yet.
+* `issues.txt` are test cases covering solved issues.
+* `extension_<>.txt` are covering specific extensions. Some of these are also
+  taken from the GFM spec.
+
+## Pull Requests
+
+I will happily accept any pull requests.
+
+Before submitting any code please check the following:
+
+* You ran `tree-sitter generate --no-bindings` in the `tree-sitter-markdown` or
+  `tree-sitter-markdown-inline` directories respecively after modifying any
+  `grammar.js` file.
+* When running `tree-sitter test` only the cases defined in `failing.txt` or
+  `extension_<>.txt` for not activated extensions fail for **both** gramars.
+* If you implemented new behavior please add tests. (In most cases these belong
+  in a `extension_<>.txt`.)
+* You deleted any auto-generated bindings and files for debugging purposes
+  like `log.html`
+
+[issue]: https://github.com/MDeiml/tree-sitter-markdown/issues/new
+[pull request]: https://github.com/MDeiml/tree-sitter-markdown/compare
+[gfm]: https://github.github.com/gfm/
+[commonmark]: https://spec.commonmark.org/
+[tree-sitter spec]: https://tree-sitter.github.io/tree-sitter/
+[tree-sitter-cli]: https://github.com/tree-sitter/tree-sitter/blob/master/cli/README.md

vendor/tree-sitter-markdown/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "tree-sitter-md"
+description = "markdown grammar for the tree-sitter parsing library"
+version = "0.1.1"
+keywords = ["incremental", "parsing", "markdown"]
+categories = ["parsing", "text-editors"]
+repository = "https://github.com/MDeiml/tree-sitter-markdown"
+edition = "2018"
+license = "MIT"
+
+build = "bindings/rust/build.rs"
+include = [
+  "bindings/rust/*",
+  "tree-sitter-markdown/src/*",
+  "tree-sitter-markdown-inline/src/*",
+  "tree-sitter-markdown/queries/*",
+  "tree-sitter-markdown-inline/queries/*",
+  "benchmark/main.rs",
+]
+
+[lib]
+path = "bindings/rust/lib.rs"
+
+[dependencies]
+tree-sitter = "~0.20"
+
+[build-dependencies]
+cc = "1.0"
+
+[[bin]]
+name = "benchmark"
+path = "benchmark/main.rs"
+
+[profile.release]
+debug = true

vendor/tree-sitter-markdown/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Matthias Deiml
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/tree-sitter-markdown/README.md

@@ -0,0 +1,16 @@
+# tree-sitter-markdown
+A markdown parser for tree-sitter
+
+For now this implements the [CommonMark Spec](https://spec.commonmark.org/). Maybe it will be extended to support [Github flavored markdown](https://github.github.com/gfm/)
+
+## Structure
+
+The parser is spit into two grammars. One for the [block structure](https://spec.commonmark.org/0.30/#blocks-and-inlines) which can be found in `/tree-sitter-markdown` and one for the [inline structure](https://spec.commonmark.org/0.30/#inlines) which is in `/tree-sitter-markdown-inline`.
+Because of this the entire document has to be scanned twice in order to be fully parsed.
+This is motivated by the [parsing strategy section](https://spec.commonmark.org/0.30/#appendix-a-parsing-strategy) of the CommonMark Spec which suggests doing exactly this: Parsing the document twice, first determining the block structure and then parsing any inline content.
+
+It also helps managing complexity, which was a problem with earlier versions of this parser, by allowing block and inline structure to be considered seperately. This was not the case as tree-sitters dynamic precedence can create hard to predict effects.
+
+## Usage
+
+To use the two grammars, first parse the document with the block grammar. Then perform a second parse with the inline grammar using `ts_parser_set_included_ranges` to specify which parts are inline content. These parts are marked as `inline` nodes. Children of those inline nodes should be excluded from these ranges. For an example implementation see `lib.rs` in the `bindings` folder.

vendor/tree-sitter-markdown/benchmark/main.rs

@@ -0,0 +1,21 @@
+use tree_sitter_md::*;
+
+fn main() {
+    let mut parser = MarkdownParser::default();
+    let filename = std::env::args().nth(1).unwrap();
+    let source = std::fs::read(filename).unwrap();
+    let mut tree = parser.parse(&source, None).unwrap();
+    tree.edit(&tree_sitter::InputEdit {
+        start_byte: 0,
+        old_end_byte: 1,
+        new_end_byte: 0,
+        start_position: tree_sitter::Point::new(0, 0),
+        old_end_position: tree_sitter::Point::new(0, 1),
+        new_end_position: tree_sitter::Point::new(0, 0),
+    });
+    reparse(&mut parser, &source[1..], tree);
+}
+
+fn reparse(parser: &mut MarkdownParser, source: &[u8], old_tree: MarkdownTree) {
+    parser.parse(source, Some(&old_tree));
+}

vendor/tree-sitter-markdown/binding.gyp

@@ -0,0 +1,23 @@
+{
+  "targets": [
+    {
+      "target_name": "tree_sitter_markdown_binding",
+      "include_dirs": [
+        "<!(node -e \"require('nan')\")",
+        "tree-sitter-markdown/src",
+        "tree-sitter-markdown-inline/src",
+      ],
+      "sources": [
+        "tree-sitter-markdown/src/parser.c",
+        "tree-sitter-markdown/src/scanner.cc",
+        "tree-sitter-markdown-inline/src/parser.c",
+        "tree-sitter-markdown-inline/src/scanner.cc",
+        "bindings/node/binding.cc"
+      ],
+      "cflags_c": [
+        "-std=c99"
+      ]
+    }
+  ]
+}
+