License rules update

etc/scripts/licenses/buildrules.py

@@ -15,6 +15,8 @@
 from licensedcode import cache
 from licensedcode import models
 from licensedcode import match_hash
+from licensedcode import frontmatter
+from license_expression import Licensing
 
 """
 A script to generate license detection rules from a simple text data file.
@@ -172,10 +174,112 @@ def find_rule_base_loc(license_expression):
     )
 
 
+def validate_and_dump_rules(rules, licenses_by_key, licenses_file_path):
+    valid_rules_text, invalid_rules_text = validate_rules_with_details(rules, licenses_by_key)
+    if invalid_rules_text:
+        valid_rules_file = licenses_file_path + ".valid"
+        with open(valid_rules_file, "w") as o:
+            o.write(valid_rules_text)
+
+        invalid_rules_file = licenses_file_path + ".invalid"
+        with open(invalid_rules_file, "w") as o:
+            o.write(invalid_rules_text)
+
+        message = [
+            'Errors while validating rules:',
+            f'See valid rules file: {valid_rules_file}',
+            f'See invalid rules file: {invalid_rules_file}',
+        ]
+        raise models.InvalidRule('\n'.join(message))
+
+
+def validate_rules_with_details(rules, licenses_by_key):
+    """
+    Return a tuple of (text of valid rules, text of invalid rules) in the format
+    expected by this tool. Invalid rules have a YAML comment text added to their
+    metadata section that describes the issue.
+    """
+
+    licensing = Licensing(symbols=licenses_by_key.values())
+
+    valid_rules_texts = []
+    invalid_rules_texts = []
+
+    for rule in rules:
+        error_messages = list(rule.validate(licensing, thorough=True))
+        rule_as_text = dump_skinny_rule(rule, error_messages=error_messages)
+
+        if error_messages:
+            invalid_rules_texts.append(rule_as_text)
+        else:
+            valid_rules_texts.append(rule_as_text)
+
+    valid_rules_text = "\n".join(valid_rules_texts) + start_delimiter
+
+    if invalid_rules_texts:
+        invalid_rules_text = "\n".join(invalid_rules_texts) + start_delimiter
+    else:
+        invalid_rules_text = ""
+
+    return valid_rules_text, invalid_rules_text
+
+
+SKINNY_RULE_TEMPLATE = """\
+{start_delimiter}
+{metadata}
+{end_delimiter}
+{content}
+"""
+
+start_delimiter = "----------------------------------------"
+
+
+def dump_skinny_rule(rule, error_messages=()):
+    """
+    Return a string that dumps the ``rule`` Rule in the input format used by
+    this tool. Add a comment with the ``error_message`` to the metadata if any.
+    """
+    metadata = rule.to_dict()
+    if error_messages:
+        # add missing metadata for sanity
+        if "license_expression" not in metadata:
+            m = {"license_expression": None}
+            m.update(metadata)
+            metadata = m
+
+        if "notes" not in metadata:
+            metadata["notes"] = None
+
+        if "referenced_filenames" not in metadata:
+            metadata["referenced_filenames"] = None
+
+        msgs = "\n".join(f"# {m}" for m in error_messages)
+        end_delimiter = f"{msgs}\n---"
+    else:
+        end_delimiter = "---"
+
+    return frontmatter.dumps_frontmatter(
+        content=rule.text,
+        metadata=metadata,
+        template=SKINNY_RULE_TEMPLATE,
+        start_delimiter=start_delimiter,
+        end_delimiter=end_delimiter)
+
+
 @click.command()
-@click.argument("licenses_file", type=click.Path(), metavar="FILE")
+@click.argument(
+    "licenses_file", type=click.Path(), metavar="FILE",
+)
+@click.option(
+    "-d", "--dump-to-file-on-errors",
+    is_flag=True,
+    default=False,
+    help="On errors, dump the valid rules and the invalid rules in text files "
+        "named after the input FILE with a .valid and a .invalid extension.",
+)
+
 @click.help_option("-h", "--help")
-def cli(licenses_file):
+def cli(licenses_file, dump_to_file_on_errors=False):
     """
         Create rules from a text file with delimited blocks of metadata and texts.
 
@@ -191,6 +295,12 @@ def cli(licenses_file):
             it under the terms of the GNU Lesser General Public License
             version 2.1 as published by the Free Software Foundation;
             ----------------------------------------
+            license_expression: lgpl-2.1
+            relevance: 100
+            is_license_reference: yes
+            ---
+            LGPL-2.1
+            ----------------------------------------
     """
 
     rules_data = load_data(licenses_file)
@@ -213,7 +323,11 @@ def cli(licenses_file):
         rl = models.BasicRule(text=rdata.text, **rdata.data)
         skinny_rules.append(rl)
 
-    models.validate_rules(skinny_rules, licenses_by_key, with_text=True, thorough=True)
+    # these will raise an exception and exit on errors
+    if not dump_to_file_on_errors:
+        models.validate_rules(rules=skinny_rules, licenses_by_key=licenses_by_key, with_text=True, thorough=True)
+    else:
+        validate_and_dump_rules(rules=skinny_rules, licenses_by_key=licenses_by_key, licenses_file_path=licenses_file)
 
     print()
     for rule in skinny_rules:

src/licensedcode/data/licenses/mit.LICENSE

@@ -7,6 +7,11 @@ owner: MIT
 homepage_url: http://opensource.org/licenses/mit-license.php
 notes: Per SPDX.org, this license is OSI certified.
 spdx_license_key: MIT
+other_spdx_license_keys:
+  - LicenseRef-MIT-Bootstrap
+  - LicenseRef-MIT-Discord
+  - LicenseRef-MIT-TC
+  - LicenseRef-MIT-Diehl
 text_urls:
     - http://opensource.org/licenses/mit-license.php
 osi_url: http://www.opensource.org/licenses/MIT

src/licensedcode/data/rules/agpl-3.0-plus_297.RULE

@@ -0,0 +1,18 @@
+---
+license_expression: agpl-3.0-plus
+is_license_notice: yes
+---
+
+The JavaScript code in this page is free software: you can
+    redistribute it and/or modify it under the terms of the GNU Affero
+    General Public License as published by the Free Software
+    Foundation, either version 3 of the License, or (at your option)
+    any later version.  The code is distributed WITHOUT ANY WARRANTY;
+    without even the implied warranty of MERCHANTABILITY or FITNESS
+    FOR A PARTICULAR PURPOSE.  See the GNU GPL for more details.
+
+    As additional permission under GNU AGPL version 3 section 7, you
+    may distribute non-source (e.g., minimized or compacted) forms of
+    that code without the copy of the GNU AGPL normally required by
+    section 4, provided you include this license notice and a URL
+    through which recipients can access the Corresponding Source.

src/licensedcode/data/rules/agpl-3.0_392.RULE

@@ -0,0 +1,9 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+relevance: 100
+referenced_filenames:
+    - LICENSE
+---
+
+The default license for this project is {{AGPL-3.0-only}}(LICENSE).

src/licensedcode/data/rules/agpl-3.0_393.RULE

@@ -0,0 +1,7 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+relevance: 100
+---
+
+license for this project is {{AGPL-3.0-only}}

src/licensedcode/data/rules/agpl-3.0_394.RULE

@@ -0,0 +1,7 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+relevance: 100
+---
+
+license for this project is {{AGPL-3.0}}

src/licensedcode/data/rules/agpl-3.0_395.RULE

@@ -0,0 +1,16 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+---
+
+/// License: AGPLv3
+///
+/// This program is free software: you can redistribute it and/or modify
+/// it under the terms of the GNU Affero General Public License as
+/// published by the Free Software Foundation, either version 3 of the
+/// License.
+///
+/// This program is distributed in the hope that it will be useful,
+/// but WITHOUT ANY WARRANTY; without even the implied warranty of
+/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+/// GNU Affero General Public License for more details.

src/licensedcode/data/rules/agpl-3.0_396.RULE

@@ -0,0 +1,15 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+---
+
+License: AGPLv3
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.

src/licensedcode/data/rules/agpl-3.0_397.RULE

@@ -0,0 +1,13 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+---
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.

src/licensedcode/data/rules/agpl-3.0_398.RULE

@@ -0,0 +1,14 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+---
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.

src/licensedcode/data/rules/agpl-3.0_399.RULE

@@ -0,0 +1,13 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+---
+
+is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License.
+ is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.

src/licensedcode/data/rules/agpl-3.0_400.RULE

@@ -0,0 +1,12 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+---
+
+is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3.
+ is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.

src/licensedcode/data/rules/agpl-3.0_401.RULE

@@ -0,0 +1,19 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+ignorable_urls:
+    - http://www.gnu.org/licenses/
+---
+
+is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.

src/licensedcode/data/rules/agpl-3.0_402.RULE

@@ -0,0 +1,19 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+ignorable_urls:
+    - http://www.gnu.org/licenses/
+---
+
+This program  is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.

src/licensedcode/data/rules/agpl-3.0_403.RULE

@@ -0,0 +1,18 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+ignorable_urls:
+    - http://www.gnu.org/licenses/
+---
+
+This program  is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.

src/licensedcode/data/rules/agpl-3.0_404.RULE

@@ -0,0 +1,19 @@
+---
+license_expression: agpl-3.0
+is_license_notice: yes
+ignorable_urls:
+    - http://www.gnu.org/licenses/
+---
+
+-
+- is free software: you can redistribute it and/or modify
+- it under the terms of the GNU Affero General Public License as published by
+- the Free Software Foundation, either version 3 of the License.
+-
+- is distributed in the hope that it will be useful,
+- but WITHOUT ANY WARRANTY; without even the implied warranty of
+- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+- GNU General Public License for more details.
+-
+- You should have received a copy of the GNU Affero General Public License
+- along with .  If not, see <http://www.gnu.org/licenses/>.