Performance/parallelise search index generation

README.md

@@ -37,7 +37,7 @@ Current features include:
   arguments, derived types, programs, and modules from the source code.
 - the ability to extract documentation from comments in the source code.
 - LaTeX support in documentation using [MathJax](https://www.mathjax.org/).
-- searchable documentation, using Tipue Search.
+- searchable documentation, using [Lunr Search](https://lunrjs.com).
 - author description and social media (including Github!) links.
 - links to download the source code.
 - links to individual files, both in their raw form or in HTML with syntax

docs/index.rst

@@ -29,8 +29,8 @@ Current features include:
   code,
 - LaTeX support in documentation using `MathJax
   <https://www.mathjax.org/>`__,
-- searchable documentation, using `Tipue Search
-  <http://www.tipue.com/search/>`__,
+- searchable documentation, using `Lunr Search
+  <https://lunrjs.com>`__,
 - author description and social media (including Github!) links,
 - links to download the source code,
 - links to individual files, both in their raw form or in HTML with

ford/output.py

@@ -108,6 +108,8 @@ def relative_url(entity: Union[FortranBase, str], page_url: pathlib.Path) -> str
 
 USER_WRITABLE_ONLY = 0o755
 
+def create_node_wrapper(page_data, node_gen = None):
+    return node_gen.create_node(page_data[0], page_data[1], page_data[2])
 
 class Documentation:
     """
@@ -237,20 +239,25 @@ def __init__(self, settings: ProjectSettings, proj_docs: str, project, pagetree)
 
         if settings.search:
             url = "" if settings.relative else settings.project_url
-            self.tipue = ford.tipue_search.Tipue_Search_JSON_Generator(
-                settings.output_dir, url
-            )
-            self.tipue.create_node(
+            node_generator = ford.tipue_search.Search_Soup_Parser(url)
+            nodes = [node_generator.create_node(
                 self.index.html, "index.html", EntitySettings(category="home")
+            )]
+
+            tasks = self.docs + self.pagetree
+            from functools import partial
+            create_node_wrapper_partial = partial(create_node_wrapper,
+                                                  node_gen = node_generator)
+            from tqdm.contrib.concurrent import process_map as parallel_map
+            extra_nodes = parallel_map(create_node_wrapper_partial,
+                                       [(x.html, x.loc, x.meta) for x in tasks],
+                                       desc = "Creating search index",
+                                       chunksize = 1,
+                                       )
+            nodes.extend(extra_nodes)
+            self.tipue = ford.tipue_search.Search_JSON_Generator(
+                settings.output_dir, nodes
             )
-            jobs = len(self.docs) + len(self.pagetree)
-            for page in (
-                bar := ProgressBar(
-                    "Creating search index", chain(self.docs, self.pagetree), total=jobs
-                )
-            ):
-                bar.set_current(page.loc)
-                self.tipue.create_node(page.html, page.loc, page.meta)
 
     def writeout(self) -> None:
         out_dir: pathlib.Path = self.data["output_dir"]
@@ -286,7 +293,7 @@ def writeout(self) -> None:
         if self.data["graph"]:
             self.graphs.output_graphs(self.njobs)
         if self.data["search"]:
-            copytree(loc / "tipuesearch", out_dir / "tipuesearch")
+            copytree(loc / "search", out_dir / "search")
             self.tipue.print_output()
 
         try:

ford/search/load_search.js

@@ -0,0 +1,62 @@
+ var items = tipuesearch['pages'];
+ var documents = tipuesearch["pages"]
+ var counter = 0
+
+ for (item in documents){
+     documents[item]['id'] = counter;
+     counter = counter +1;
+ }
+
+ var idx = lunr(function () {
+   this.ref('id')
+   this.field('title')
+   this.field('url')
+   this.field('text', { boost: 10 })
+   this.field('tags')
+
+   items.forEach(function (doc) {
+     this.add(doc)
+   }, this)
+})
+
+function lunr_search(term) {
+    document.getElementById('lunrsearchresults').innerHTML = '<ul></ul>';
+    if(term) {
+	document.getElementById('lunrsearchresults').innerHTML = "<p>Search results for '" + term + "'</p>" + document.getElementById('lunrsearchresults').innerHTML;
+	//put results on the screen.
+	var results = idx.search(term);
+	if(results.length>0){
+	    //console.log(idx.search(term));
+	    //if results
+	    for (var i = 0; i < results.length; i++) {
+		// more statements
+		var ref = results[i]['ref'];
+		var url = documents[ref]['url'];
+		var title = documents[ref]['title'];
+		var body = documents[ref]['text'].substring(0,160)+'...';
+		document.querySelectorAll('#lunrsearchresults ul')[0].innerHTML = document.querySelectorAll('#lunrsearchresults ul')[0].innerHTML + "<li class='lunrsearchresult'><a href='" + url + "'><span class='title'>" + title + "</span></a><br /><span class='body'>"+ body +"</span><br /><span class='url'>"+ url +"</span></li>";
+	    }
+	} else {
+	    document.querySelectorAll('#lunrsearchresults ul')[0].innerHTML = "<li class='lunrsearchresult'>No results found...</li>";
+	}
+    }
+    return false;
+}
+
+function getQueryVariable(variable) {
+  var query = window.location.search.substring(1);
+  var vars = query.split('&');
+
+  for (var i = 0; i < vars.length; i++) {
+    var pair = vars[i].split('=');
+
+    if (pair[0] === variable) {
+      return decodeURIComponent(pair[1].replace(/\+/g, '%20'));
+    }
+  }
+}
+
+var searchTerm = getQueryVariable('q');
+if (searchTerm) {
+  lunr_search(searchTerm)
+}

ford/templates/base.html

@@ -19,14 +19,9 @@
     <link href="{{ project_url }}/css/pygments.css" rel="stylesheet">
     <link href="{{ project_url }}/css/font-awesome.min.css" rel="stylesheet">
     <link href="{{ project_url }}/css/local.css" rel="stylesheet">
-    {% if search|lower == 'true' %}
-      <link  href="{{ project_url }}/tipuesearch/tipuesearch.css" rel="stylesheet">
-    {% endif %}
     {% if css %}
       <link href="{{ project_url }}/css/user.css" rel="stylesheet">
     {% endif %}
-
-    <script src="https://code.jquery.com/jquery-3.7.1.min.js" integrity="sha256-/JqT3SQfawRcv/BIHPThkBvs0OEvtFFmqPF/lYI/Cxo=" crossorigin="anonymous"></script>
     <script src="{{ project_url }}/js/svg-pan-zoom.min.js"></script>
   </head>
 
@@ -171,12 +166,5 @@
           <script src="{{ project_url }}/js/MathJax-config/{{ path.basename(mathjax_config) }}"></script>
         {% endif %}
         <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
-
-        {% if search|lower == 'true' %}
-          <script src="{{ project_url }}/tipuesearch/tipuesearch_content.js"></script>
-          <script src="{{ project_url }}/tipuesearch/tipuesearch_set.js"></script>
-          <script src="{{ project_url }}/tipuesearch/tipuesearch.js"></script>
-        {% endif %}
-
   </body>
 </html>

ford/templates/search.html

@@ -3,28 +3,19 @@
 Search Results – {{ project }}
 {% endblock %}
 {% block body %}
-      <div class="row">
-        <div class="col-lg-12">
-			 <h1>Search Results</h1>
-             <div id="tipue_search_content"><div id="tipue_search_loading"></div></div>
-        </div>
-      </div>
-<!--
-    <script>
-      $(document).ready(function() {
-           $('#tipue_search_input').tipuesearch({
-             'mode' : 'static',
-             'show': 10,
-             'newWindow': false,
-             'descriptiveWords': 35,
-         });
-      });
-    </script>
--->
-<script>
-$(document).ready(function() {
-     $('#tipue_search_input').tipuesearch();
-});
-</script>
+<div class="row">
+  <div class="col-lg-12">
+    <h1>Search Results</h1>
+    <div id="lunrsearchresults">
+      <ul></ul>
+    </div>
+  </div>
+</div>
+
+{% if search|lower == 'true' %}
+<script src="https://unpkg.com/lunr/lunr.js"></script>
+<script src="{{ project_url }}/search/search_database.json"></script>
+<script src="{{ project_url }}/search/load_search.js"></script>
+{% endif %}
 {% endblock %}
 

ford/tipue_search.py

@@ -27,7 +27,7 @@
 Tipue Search
 ============
 
-Serializes generated HTML to JSON that can be used by jQuery plugin - Tipue Search.
+Serializes generated HTML to JSON that can be used by Lunr search
 
 Adapted from the Pelican plugin by Talha Mansoor
 https://github.com/getpelican/pelican-plugins/tree/master/tipue_search
@@ -43,12 +43,9 @@
 
 from ford.settings import EntitySettings
 
-
-class Tipue_Search_JSON_Generator:
-    def __init__(self, output_path: os.PathLike, project_url: str):
-        self.output_path = pathlib.Path(output_path)
+class Search_Soup_Parser:
+    def __init__(self, project_url: str):
         self.siteurl = project_url
-        self.json_nodes: List[Dict] = []
         self.only_text = SoupStrainer("div", id="text")
         self.only_title = SoupStrainer("title")
 
@@ -85,17 +82,20 @@ def create_node(self, html, loc, meta: EntitySettings):
         else:
             page_url = loc
 
-        node = {
+        return {
             "title": page_title,
             "text": page_text,
             "tags": page_category,
-            "loc": str(page_url),
+            "url": str(page_url),
         }
 
-        self.json_nodes.append(node)
+class Search_JSON_Generator:
+    def __init__(self, output_path: os.PathLike, nodes: List[Dict]):
+        self.output_path = pathlib.Path(output_path)
+        self.json_nodes = nodes
 
     def print_output(self):
-        path = self.output_path / "tipuesearch" / "tipuesearch_content.js"
+        path = self.output_path / "search" / "search_database.json"
 
         root_node = {"pages": self.json_nodes}
         output = json.dumps(root_node, separators=(",", ":"), ensure_ascii=False)