Merge pull request #239 from laraPPr/clean_up_available_software

clean up the available software script and the tests

.github/workflows/script_module_list.yml

@@ -34,17 +34,22 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+      - name: Make EESSI available
+        uses: eessi/github-action-eessi@v3
       - name: Set up Python
-        uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
+        uses: eessi/github-action-eessi@v3
         with:
           python-version: '3.6'
       - name: Install dependencies
         run: |
+          python -m venv venv
+          . venv/bin/activate
           cd scripts/available_software
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r requirements_tests.txt
       - name: Test with pytest
         run: |
+          . venv/bin/activate
           cd scripts/available_software
           ./test.sh

scripts/available_software/available_software.py

@@ -11,7 +11,6 @@
 @author: Michiel Lachaert (Ghent University)
 @author: Lara Peeters (Ghent University)
 """
-import argparse
 import json
 import os
 import re
@@ -252,7 +251,7 @@ def get_extra_info_eessi(json_data) -> dict:
     for software in modules:
         for mod in modules[software]['versions']:
             if software == "Java":
-                # TODO handle specific naming schema for Java 
+                # TODO handle specific naming schema for Java
                 # code cannot handle "Java/11(@Java/11.0.20)"
                 continue
             base_path = modules[software]['versions'][mod]['targets'][0] + '/modules/all/'
@@ -328,7 +327,6 @@ def generate_software_table_data(software_data: dict, targets: list) -> list:
     @param targets: List with all the target names
     @return: 1D list with all the data for the table
     """
-    #TODO: add same structure as https://github.com/laraPPr/EESSI_docs/blob/test_add_script_generate_software/docs/available_software/overview.md to table
     table_data = [" "] + [target[57:] for target in targets]
 
     for module_name, available in list(software_data.items())[::-1]:
@@ -474,13 +472,16 @@ def generate_markdown_overview(modules: dict) -> None:
 
 # FORMAT OVERVIEW JSON
 # {
-#     "targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"],
-#     "modules": {
-#         "Markov": [1, 0],
-#         "cfd": [1, 1],
-#         "llm": [0, 1],
-#         "science": [1, 1]
-#     }
+#   "targets": [
+#     "/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic",
+#     "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"
+#   ],
+#   "modules": {
+#     "Markov": [1, 0],
+#     "cfd": [1, 1],
+#     "llm": [0, 1],
+#     "science": [1, 1]
+#   }
 # }
 def generate_json_overview_data(modules: dict) -> dict:
     """
@@ -535,16 +536,25 @@ def generate_json_overview(modules: dict, path_data_dir: str) -> str:
 # FORMAT DETAILED JSON:
 #
 # {
-#     "targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"],
-#     "software": {
-#         "cfd": {
-#             "targets": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.e    essi.io/versions/2023.06/software/linux/x86_64/amd/zen2"],
-#             "versions": {
-#                 "2.3.1": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"],
-#                 "2.3.2": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/so    ftware.e    essi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]
-#             }
-#         }
+#   "targets": [
+#     "/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic",
+#     "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"
+#   ],
+#   "software": {
+#     "cfd": {
+#       "targets": [
+#         "/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic",
+#         "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"
+#       ],
+#       "versions": {
+#         "2.3.1": ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"],
+#         "2.3.2": [
+#           "/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic",
+#           "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"
+#         ]
+#       }
 #     }
+#   }
 # }
 
 def generate_json_detailed_data(modules: dict) -> dict:

scripts/available_software/test.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-PYTHONPATH=$PWD:$PYTHONPATH pytest -v -s
+PYTHONPATH=$PWD:$PYTHONPATH python -m pytest -v -s

scripts/available_software/tests/test_data.py

@@ -1,6 +1,9 @@
 import os
 from available_software import modules_eessi, get_unique_software_names
 
+GENERIC = "/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"
+ZEN2 = "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"
+
 
 class TestData:
     # ---------------------------
@@ -24,7 +27,7 @@ def setup_class(cls):
     def test_data_eessi(self):
         sol = modules_eessi()
         assert len(sol) == 2
-        assert len(sol["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"]) == 13
-        assert len(sol["/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]) == 15
-        assert list(get_unique_software_names(sol["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"])) == ["Markov", "cfd", "science"]
-        assert list(get_unique_software_names(sol["/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"])) == ["cfd", "llm", "science"]
+        assert len(sol[GENERIC]) == 13
+        assert len(sol[ZEN2]) == 15
+        assert list(get_unique_software_names(sol[GENERIC])) == ["Markov", "cfd", "science"]
+        assert list(get_unique_software_names(sol[ZEN2])) == ["cfd", "llm", "science"]

scripts/available_software/tests/test_json.py

@@ -6,6 +6,9 @@
 import os
 import json
 
+GENERIC = "/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic"
+ZEN2 = "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"
+
 
 class TestJSON:
     # ---------------------------
@@ -36,7 +39,7 @@ def test_json_generate_simple(self):
         modules = modules_eessi()
         json_data = generate_json_overview_data(modules)
         assert len(json_data.keys()) == 3
-        assert list(json_data["targets"]) == ["/cvmfs/software.eessi.io/versions/2023.06/software/linux/aarch64/generic", "/cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2"]
+        assert list(json_data["targets"]) == [GENERIC, ZEN2]
         assert json_data["modules"] == {
                 "Markov": [1, 0],
                 "cfd": [1, 1],

scripts/available_software/tests/test_module.py

@@ -1,5 +1,5 @@
 import os
-from available_software import module_avail, filter_fn_eessi_modules, module_swap, module_whatis
+from available_software import module_avail, filter_fn_eessi_modules, module_whatis
 
 
 class TestModule: