Update rules with required phrases automatically

docs/source/how-to-guides/add_new_license_detection_rule.rst

@@ -73,7 +73,7 @@ More (advanced) rules options:
   be present in the result license detections. These just have the license text and a
   `is_false_positive` flag set to True.
 
-- you can specify key phrases by surrounding one or more words between the `{{`
+- you can specify required phrases by surrounding one or more words between the `{{`
   and `}}` tags. Key phrases are words that **must** be matched/present in order
   for a RULE to be considered a match.
 

etc/scripts/licenses/buildrules.py

@@ -16,6 +16,7 @@
 from licensedcode import models
 from licensedcode import match_hash
 from licensedcode import frontmatter
+from licensedcode.models import get_rule_id_for_text
 from license_expression import Licensing
 
 """
@@ -130,23 +131,6 @@ def load_data(location="00-new-licenses.txt"):
     return rules
 
 
-def rule_exists(text):
-    """
-    Return the matched rule identifier if the text is an existing rule matched
-    exactly, False otherwise.
-    """
-    idx = cache.get_index()
-
-    matches = idx.match(query_string=text)
-    if not matches:
-        return False
-    if len(matches) > 1:
-        return False
-    match = matches[0]
-    if match.matcher == match_hash.MATCH_HASH and match.score() == 100:
-        return match.rule.identifier
-
-
 def all_rule_by_tokens():
     """
     Return a mapping of {tuples of tokens: rule id}, with one item for each
@@ -347,7 +331,7 @@ def cli(licenses_file, dump_to_file_on_errors=False):
 
         text = rule.text
 
-        existing_rule = rule_exists(text)
+        existing_rule = get_rule_id_for_text(text)
         skinny_text = " ".join(text[:80].split()).replace("{", " ").replace("}", " ")
 
         existing_msg = (

etc/scripts/licenses/report_license_rules.py

@@ -62,6 +62,8 @@
     "is_license_reference",
     "is_license_intro",
     "is_license_clue",
+    "is_required_phrase",
+    "skip_for_required_phrase_generation",
     "is_deprecated",
     "has_unknown",
     "only_known_words",

respective.txt

@@ -0,0 +1,3 @@
+ This software is (C) by the respective authors, and licensed under the GPL
+ License.
+

setup-mini.cfg

@@ -158,6 +158,7 @@ console_scripts =
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
+    add-required-phrases = licensedcode.required_phrases:add_required_phrases
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

setup.cfg

@@ -158,6 +158,8 @@ console_scripts =
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
+    add-required-phrases = licensedcode.required_phrases:add_required_phrases
+	gen-new-required-phrases-rules = licensedcode.required_phrases:gen_required_phrases_rules
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

src/licensedcode/cache.py

@@ -9,6 +9,7 @@
 
 import os
 import pickle
+
 from shutil import rmtree
 
 from commoncode.fileutils import create_dir
@@ -185,10 +186,7 @@ def load_or_build(
                     additional_license_plugins=plugin_directories,
                 )
 
-                # save the cache as pickle new tree checksum
-                with open(cache_file, 'wb') as fn:
-                    pickle.dump(license_cache, fn, protocol=PICKLE_PROTOCOL)
-
+                license_cache.dump(cache_file)
                 return license_cache
 
         except lockfile.LockTimeout:
@@ -201,6 +199,13 @@ def has_additional_licenses(self):
         if cache.additional_license_directory or cache.additional_license_plugins:
             return True
 
+    def dump(self, cache_file):
+        """
+        Dump this license cache on disk at ``cache_file``.
+        """
+        with open(cache_file, 'wb') as fn:
+            pickle.dump(self, fn, protocol=PICKLE_PROTOCOL)
+
 
 def build_index(
     licenses_db=None,
@@ -235,9 +240,12 @@ def build_index(
     if not licenses_db:
         # combine the licenses in these additional directories with the licenses in the original DB
         additional_license_dirs = get_license_dirs(additional_dirs=additional_directories)
-        combined_license_directories = [licenses_data_dir] + additional_license_dirs
         # generate a single combined license db with all licenses
-        licenses_db = load_licenses_from_multiple_dirs(license_dirs=combined_license_directories)
+        licenses_db = load_licenses_from_multiple_dirs(
+            builtin_license_data_dir=licenses_data_dir,
+            additional_license_data_dirs=additional_license_dirs,
+            with_deprecated=False,
+        )
 
     # if we have additional directories, extract the rules from them
     additional_rule_dirs = get_rule_dirs(additional_dirs=additional_directories)
@@ -393,7 +401,7 @@ def get_cache(
     Return a LicenseCache either rebuilt, cached or loaded from disk.
 
     If ``index_all_languages`` is True, include texts in all languages when
-    building the license index. Otherwise, only include the English license \
+    building the license index. Otherwise, only include the English license
     texts and rules (the default)
     """
     return populate_cache(
@@ -531,7 +539,7 @@ def validate_spdx_license_keys(license_expression, licensing):
         if not type(key) == str:
             msg = f"Invalid license key: {key} of type {type(key)}, license key should be a string"
             messages.append(msg)
-    
+
         lic = license_db.get(key, None)
         if not lic:
             licenses = load_licenses(with_deprecated=True)

src/licensedcode/data/licenses/dco-1.0.LICENSE

@@ -0,0 +1,39 @@
+---
+key: dco-1.0
+short_name: DCO 1.0
+name: Developer Certificate of Origin 1.0
+category: CLA
+owner: Linux Foundation
+homepage_url: https://developercertificate.org/
+spdx_license_key: LicenseRef-scancode-dco-1.0
+text_urls:
+    - https://developercertificate.org/
+other_urls:
+    - https://github.com/nexB/scancode-toolkit/issues/3038#issuecomment-1317511139
+minimum_coverage: 90
+---
+
+Developer's Certificate of Origin 1.0
+
+By making a contribution to this project, I certify that:
+
+1. The contribution was created in whole or in part by me and I
+   have the right to submit it under the open source license
+   indicated in the file LICENSE; or
+
+2. The contribution is based upon previous work that, to the best
+   of my knowledge, is covered under an appropriate open source
+   license and I have the right under that license to submit that
+   work with modifications, whether created in whole or in part
+   by me, under the same open source license (unless I am
+   permitted to submit under a different license), as indicated
+   in the file LICENSE; or
+
+3. The contribution was provided directly to me by some other
+   person who certified (1), (2) or (3) and I have not modified
+   it.
+
+4. I understand and agree that this project and the contribution
+   are public and that a record of the contribution (including all
+   personal information I submit with it, including my sign-off) is
+   maintained indefinitely and may be redistributed consistent

src/licensedcode/data/licenses/samba-dco-1.0.LICENSE

@@ -0,0 +1,51 @@
+---
+key: samba-dco-1.0
+short_name: Samba Developer's Declaration, Version 1.0
+name: Samba Developer's Declaration, Version 1.0
+category: CLA
+owner: Samba
+homepage_url: https://github.com/samba-team/samba/blob/master/README.contributing
+spdx_license_key: LicenseRef-scancode-samba-dco-1.0
+text_urls:
+    - https://github.com/samba-team/samba/blob/master/README.contributing
+minimum_coverage: 80
+ignorable_urls:
+    - http://www.gnu.org/licenses/gpl-3.0.html
+    - http://www.gnu.org/licenses/lgpl-3.0.html
+---
+
+Samba Developer's Declaration, Version 1.0
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the appropriate
+    version of the GNU General Public License; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the GNU General Public License, in the
+    appropriate version; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a) or (b) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the
+    contribution are public and that a record of the
+    contribution (including all metadata and personal
+    information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed
+    consistent with the Samba Team's policies and the
+    requirements of the GNU GPL where they are relevant.
+
+(e) I am granting this work to this project under the terms of both
+    the GNU General Public License and the GNU Lesser General Public
+    License as published by the Free Software Foundation; either version
+    3 of these Licenses, or (at the option of the project) any later
+    version.
+
+    http://www.gnu.org/licenses/gpl-3.0.html
+    http://www.gnu.org/licenses/lgpl-3.0.html

src/licensedcode/data/rules/agpl-3.0-plus_100.RULE

@@ -9,4 +9,4 @@ ignorable_urls:
 ---
 
 Licensing
-This project is free and open-source software and is distributed under the terms of the GNU Affero General Public License v3 or any later version. It is subject to the license terms in the LICENSE file found in the top-level directory of this distribution and at https://www.gnu.org/licenses/agpl-3.0.md. By submitting a contribution to this project you agree and acknowledge that your contribution may be used under the terms of the GNU Affero General Public License v3 or any later version and you are authorized to submit the contribution on behalf of the copyright holder. No part of this project, including this file, may be copied, modified, propagated, or distributed except according to the terms contained in the LICENSE file.
+This project is free and open-source software and is distributed under the terms of the {{GNU Affero General Public License}} v3 or any later version. It is subject to the license terms in the LICENSE file found in the top-level directory of this distribution and at https://www.gnu.org/licenses/agpl-3.0.md. By submitting a contribution to this project you agree and acknowledge that your contribution may be used under the terms of the {{GNU Affero General Public License}} v3 or any later version and you are authorized to submit the contribution on behalf of the copyright holder. No part of this project, including this file, may be copied, modified, propagated, or distributed except according to the terms contained in the LICENSE file.

src/licensedcode/data/rules/agpl-3.0-plus_106.RULE

@@ -4,4 +4,4 @@ is_license_notice: yes
 relevance: 100
 ---
 
-This program is licensed under the GNU Affero General Public License.
+This program is licensed under the {{GNU Affero General Public License}}.

src/licensedcode/data/rules/agpl-3.0-plus_107.RULE

@@ -4,4 +4,4 @@ is_license_notice: yes
 relevance: 100
 ---
 
-program is licensed under the GNU Affero General Public License.
+program is licensed under the {{GNU Affero General Public License}}.

src/licensedcode/data/rules/agpl-3.0-plus_108.RULE

@@ -8,13 +8,13 @@ referenced_filenames:
 
 License
  is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
+it under the terms of the {{GNU Affero General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+(at your option) any later version}}.
 
  is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
+{{GNU Affero General Public License}} for more details.
 
 Read the LICENSE.txt file for more details.

src/licensedcode/data/rules/agpl-3.0-plus_109.RULE

@@ -7,13 +7,13 @@ referenced_filenames:
 ---
 
 is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
+it under the terms of the {{GNU Affero General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+(at your option) any later version}}.
 
  is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
+{{GNU Affero General Public License}} for more details.
 
 Read the LICENSE.txt file for more details.

src/licensedcode/data/rules/agpl-3.0-plus_11.RULE

@@ -5,8 +5,8 @@ ignorable_urls:
     - http://www.gnu.org/licenses
 ---
 
-is free software: you can redistribute it and/or modify it under the terms of the {{GNU Affero General Public License}} as published by the Free Software Foundation, {{either version 3 of the License, or (at your option) any later version.}}
+is free software: you can redistribute it and/or modify it under the terms of the {{GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version}}.
 
- is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+ is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the {{GNU Affero General Public License}} for more details.
 
-You should have received a copy of the GNU Affero General Public License along with . If not, see http://www.gnu.org/licenses/.
+You should have received a copy of the {{GNU Affero General Public License}} along with . If not, see http://www.gnu.org/licenses/.

src/licensedcode/data/rules/agpl-3.0-plus_110.RULE

@@ -6,11 +6,11 @@ relevance: 100
 
 License
  is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
+it under the terms of the {{GNU Affero General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+(at your option) any later version}}.
 
  is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
+{{GNU Affero General Public License}} for more details.

src/licensedcode/data/rules/agpl-3.0-plus_111.RULE

@@ -5,11 +5,11 @@ relevance: 100
 ---
 
 is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
+it under the terms of the {{GNU Affero General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+(at your option) any later version}}.
 
  is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
+{{GNU Affero General Public License}} for more details.

src/licensedcode/data/rules/agpl-3.0-plus_112.RULE

@@ -7,16 +7,16 @@ ignorable_urls:
     - http://www.gnu.org/licenses/
 ---
 
-License: AGPL-3.0
+{{License: AGPL-3.0}}
     This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as
+    it under the terms of the {{GNU Affero General Public License as
     published by the Free Software Foundation, either version 3 of the
-    License, or (at your option) any later version.
+    License, or (at your option) any later version}}.
     .
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
+    {{GNU Affero General Public License}} for more details.
     .
-    You should have received a copy of the GNU Affero General Public License
+    You should have received a copy of the {{GNU Affero General Public License}}
     along with this program.  If not, see <http://www.gnu.org/licenses/>.

src/licensedcode/data/rules/agpl-3.0-plus_113.RULE

@@ -5,6 +5,6 @@ relevance: 100
 ---
 
 is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published by the
+# under the terms of the {{GNU Affero General Public License as published by the
 # Free Software Foundation; either version 3 of the License, or (at your
-# option) any later version.
+# option) any later version}}.

src/licensedcode/data/rules/agpl-3.0-plus_114.RULE

@@ -4,4 +4,4 @@ is_license_notice: yes
 relevance: 100
 ---
 
-released under AGPL-3.0+ and is free software
+released under {{AGPL-3.0+}} and is free software

src/licensedcode/data/rules/agpl-3.0-plus_116.RULE

@@ -6,4 +6,4 @@ ignorable_urls:
     - http://www.gnu.org/licenses/agpl
 ---
 
-License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+{{License AGPL-3.0 or later}} (http://www.gnu.org/licenses/agpl).