Merge pull request #421 from crocs-muni/fix/fips-search-proper

Fix FIPS aplgo download column parse.
crocs-muni · Jul 4, 2024 · 301c3aa · 301c3aa
2 parents c5b4f3d + 0a62b68
commit 301c3aa
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py
@@ -32,7 +32,9 @@
 FIPS_CMVP_URL = FIPS_BASE_URL + "/projects/cryptographic-module-validation-program"
 FIPS_CAVP_URL = FIPS_BASE_URL + "/projects/Cryptographic-Algorithm-Validation-Program"
 FIPS_MODULE_URL = FIPS_CMVP_URL + "/certificate/{}"
-FIPS_ALG_SEARCH_URL = FIPS_CAVP_URL + "/validation-search?searchMode=validation&page="
+FIPS_ALG_SEARCH_VALIDATION_URL = FIPS_CAVP_URL + "/validation-search?searchMode=validation&page="
+FIPS_ALG_SEARCH_IMPLEMENTATION_URL = FIPS_CAVP_URL + "/validation-search?searchMode=implementation&page="
+FIPS_ALG_SEARCH_URL = FIPS_ALG_SEARCH_VALIDATION_URL
 FIPS_SP_URL = "https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp{}.pdf"
 FIPS_ACTIVE_MODULES_URL = (
     FIPS_CMVP_URL + "/validated-modules/search?SearchMode=Advanced&CertificateStatus=Active&ValidationYear=0"

diff --git a/src/sec_certs/dataset/fips_algorithm.py b/src/sec_certs/dataset/fips_algorithm.py
@@ -96,6 +96,9 @@ def get_number_of_html_pages(html_path: Path) -> int:
     @staticmethod
     def parse_algorithms_from_html(html_path: Path) -> set[FIPSAlgorithm]:
         df = pd.read_html(html_path)[0]
+        for col in df.columns:
+            if "Order by" in col:
+                df.rename(columns={col: col.split("Order by")[0]}, inplace=True)
         df["alg_type"] = df["Validation Number"].map(lambda x: re.sub(r"[0-9\s]", "", x))
         df["alg_number"] = df["Validation Number"].map(lambda x: re.sub(r"[^0-9]", "", x))
         df["alg"] = df.apply(