Use CommonMark in convert-policy.py

.travis.yml

@@ -1,6 +1,4 @@
 language: python
-python:
-  - "2.7"
 
 env:
   global:
@@ -9,7 +7,7 @@ env:
     - SERVER_PUBLIC_KEY="ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDt6Igtp73aTOYXuFb8qLtgs80wWF6cNi3/AItpWAMpX3PymUw7stU7Pi+IoBJz21nfgmxaKp3gfSe2DPNt06l8="
 
 install:
-  - pip install markdown==2.6.11 py-gfm==0.1.4
+  - pip install commonmark==0.9.1
   - npm install
 script:
   - shellcheck deploy.sh

convert-policy.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
+# -*- coding: UTF-8 -*-
 
 import codecs
-import markdown
-from mdx_partial_gfm import PartialGithubFlavoredMarkdownExtension
+import commonmark
 import re
 
 
@@ -35,13 +35,12 @@ def header_text_to_id(header_text):
 
 
 def add_one_header_anchor(line):
-    header_text = line.lstrip('#')
-    header_level = len(line) - len(header_text)
-
-    if header_level <= 2:
+    search = re.search(r'<h([3-6])>(.+)</h([3-6])>', line)
+    if not search:
         return line
 
-    header_text = header_text.lstrip(' ')
+    header_level = search.group(1)
+    header_text = search.group(2)
     header_id = header_text_to_id(header_text)
 
     return '<h{0} id="{1}">{2}<a class="self-link" href="#{1}"></a></h{0}>'.format(header_level, header_id, header_text)
@@ -55,21 +54,19 @@ def rewrite_defs(policy_markdown):
     return re.sub(r'<a id=([^>]*)>[*][*]([^*]*)[*][*]</a>', '<dfn id=\\1>\\2</dfn>', policy_markdown)
 
 
-def fix_nested_lists(policy_markdown):
-    return re.sub(r'^   1[.]', '    1.', policy_markdown, flags=re.MULTILINE)
-
-
-def avoid_link_false_positives(policy_markdown):
-    return re.sub(r'[]] [(]', '] \\(', policy_markdown)
-
-
 def preprocess_markdown(policy_markdown, mapping_pairs):
     result = lower_headers(policy_markdown)
     result = apply_link_mapping(result, mapping_pairs)
-    result = add_header_anchors(result)
     result = rewrite_defs(result)
-    result = fix_nested_lists(result)
-    result = avoid_link_false_positives(result)
+
+    return result
+
+
+def postprocess_html(policy_html, template, title):
+    result = policy_html.replace("&quot;", "\"")
+    result = template.replace("@POLICY_GOES_HERE@", result)
+    result = result.replace("@TITLE_GOES_HERE@", title)
+    result = add_header_anchors(result)
 
     return result
 
@@ -97,11 +94,10 @@ def main():
         (title, policy_markdown) = markdown_title(policy_markdown)
         preprocessed_policy_markdown = preprocess_markdown(policy_markdown, link_mapping_pairs)
 
-        policy_html = markdown.markdown(preprocessed_policy_markdown, extensions=[PartialGithubFlavoredMarkdownExtension()])
+        policy_html = commonmark.commonmark(preprocessed_policy_markdown)
 
-        final_policy_html = template.replace("@POLICY_GOES_HERE@", policy_html)
-        final_policy_html = final_policy_html.replace("@TITLE_GOES_HERE@", title)
+        postprocessed_policy_html = postprocess_html(policy_html, template, title)
 
-        codecs.open("whatwg.org/" + link, "w", encoding="utf-8").write(final_policy_html)
+        codecs.open("whatwg.org/" + link, "w", encoding="utf-8").write(postprocessed_policy_html)
 
 main()