Feature archdetect rpi

.github/workflows/scorecards.yml

@@ -35,7 +35,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           persist-credentials: false
 

.github/workflows/test-software.eessi.io.yml

@@ -0,0 +1,85 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Check for missing software installations in software.eessi.io
+on: [push, pull_request, workflow_dispatch]
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  check_missing:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        EESSI_VERSION:
+        - 2023.06
+        EESSI_SOFTWARE_SUBDIR_OVERRIDE:
+        - aarch64/generic
+        - aarch64/neoverse_n1
+        - aarch64/neoverse_v1
+        - x86_64/amd/zen2
+        - x86_64/amd/zen3
+        - x86_64/intel/haswell
+        - x86_64/intel/skylake_avx512
+        - x86_64/generic
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+        - name: Mount EESSI CernVM-FS pilot repository
+          uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
+          with:
+              cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
+              cvmfs_http_proxy: DIRECT
+              cvmfs_repositories: software.eessi.io
+
+        - name: Test check_missing_installations.sh script
+          run: |
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
+              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
+              # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
+              # to prevent issues with checks in the Easybuild configuration that use this variable
+              export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
+              module load EasyBuild
+              which eb
+              eb --version
+              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              env | grep ^EESSI | sort
+              echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)"
+              for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do
+                  echo "check missing installations for ${easystack_file}..."
+                  ./check_missing_installations.sh ${easystack_file}
+                  ec=$?
+                  if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
+              done
+
+        - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
+          run: |
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
+              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
+              # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
+              # to prevent issues with checks in the Easybuild configuration that use this variable
+              export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
+              module load EasyBuild
+              which eb
+              eb --version
+              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              env | grep ^EESSI | sort
+              # create dummy easystack file with a single entry (something that is not installed in EESSI)
+              easystack_file="test.yml"
+              echo "easyconfigs:" > ${easystack_file}
+              echo "  - GCC-8.3.0:" >> ${easystack_file}
+              echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):"
+              cat ${easystack_file}
+              # note, check_missing_installations.sh exits 1 if a package was
+              # missing, which is intepreted as false (exit code based, not
+              # boolean logic), hence when the script exits 0 if no package was
+              # missing it is interpreted as true, thus the test did not capture
+              # the missing package
+              if ./check_missing_installations.sh ${easystack_file}; then
+                echo "did NOT capture missing package; test FAILED"
+                exit 1
+              else
+                echo "captured missing package; test PASSED"
+                exit 0
+              fi

.github/workflows/test_eessi_container_script.yml

@@ -22,7 +22,7 @@ jobs:
         #- save
     steps:
         - name: Check out software-layer repository
-          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
         - name: install Apptainer
           run: |
@@ -45,7 +45,7 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
                 outfile=out_listrepos.txt
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
-                grep "EESSI-pilot" ${outfile}
+                grep "EESSI" ${outfile}
 
               # test use of --list-repos with custom repos.cfg
               elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
@@ -57,7 +57,7 @@ jobs:
                 echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
                 echo "repo_version = 20HT.TP" >> cfg/repos.cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
-                grep "EESSI-pilot" ${outfile}
+                grep "EESSI" ${outfile}
 
                 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile2}
@@ -90,15 +90,15 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then
                 outfile=out_readwrite.txt
                 fn="test_${RANDOM}.txt"
-                echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
+                echo "touch /cvmfs/software.eessi.io/${fn}" > test_script.sh
                 chmod u+x test_script.sh
                 export SINGULARITY_BIND="$PWD:/test"
                 ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile}
 
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 # note: must use '--access rw' again here, since touched file is in overlay upper dir
-                ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
-                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
+                ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile}
+                grep "/cvmfs/software.eessi.io/${fn}$" $outfile
 
               # test use of --resume
               elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
@@ -120,12 +120,12 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then
                 outfile=out_save.txt
                 fn="test_${RANDOM}.txt"
-                test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
+                test_cmd="touch /cvmfs/software.eessi.io/${fn}"
                 ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
                 rm -f ${outfile}
 
-                ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
-                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
+                ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile}
+                grep "/cvmfs/software.eessi.io/${fn}$" $outfile
 
                 tar tfv test-save.tar | grep "overlay-upper/${fn}"
 

.github/workflows/test_licenses.yml

@@ -0,0 +1,20 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Test software licenses
+on: [push, pull_request]
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+        - name: set up Python
+          uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+          with:
+            python-version: '3.9'
+
+        - name: Check software licenses
+          run: |
+            python licenses/spdx.py licenses/licenses.json

.github/workflows/tests.yml

@@ -12,17 +12,17 @@ jobs:
       fail-fast: false
     steps:
     - name: checkout
-      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
     - name: set up Python
-      uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64
 
     - name: install Python packages
       run: |
-        pip install archspec
+        pip install archspec==0.2.2
 
     - name: test eessi_software_subdir.py script
       run: |

.github/workflows/tests_archdetect.yml

@@ -13,37 +13,60 @@ jobs:
           - x86_64/intel/skylake_avx512/archspec-linux-6132
           - x86_64/amd/zen2/Azure-CentOS7-7V12
           - x86_64/amd/zen3/Azure-CentOS7-7V73X
-          - ppc64le/power9le/unknown-power9le
           - aarch64/neoverse_n1/Azure-Ubuntu20-Altra
           - aarch64/neoverse_n1/AWS-awslinux-graviton2
           - aarch64/neoverse_v1/AWS-awslinux-graviton3
+          - aarch64/cortex_a72/debian-rpi4
+          # commented out since these targets are currently not supported in software.eessi.io repo
+          # (and some tests assume that the corresponding subdirectory in software layer is there)
+          # - ppc64le/power9le/unknown-power9le
       fail-fast: false
     steps:
     - name: checkout
-      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
-    - name: Enable EESSI
-      uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+    - name: Mount EESSI CernVM-FS pilot repository
+      uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
+      with:
+          cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
+          cvmfs_http_proxy: DIRECT
+          cvmfs_repositories: software.eessi.io
+
     - name: test eessi_archdetect.sh
       run: |
           export EESSI_MACHINE_TYPE=${{matrix.proc_cpuinfo}}
           export EESSI_MACHINE_TYPE=${EESSI_MACHINE_TYPE%%/*}
           export EESSI_PROC_CPUINFO=./tests/archdetect/${{matrix.proc_cpuinfo}}.cpuinfo
+          # check that printing of best match works correctly
           CPU_ARCH=$(./init/eessi_archdetect.sh cpupath)
           if [[ $CPU_ARCH == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.output )" ]]; then
-              echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCH" >&2
+              echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCH"
           else
               echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCH" >&2
               exit 1
           fi
+          # check that $EESSI_SOFTWARE_SUBDIR_OVERRIDE is honored
+          export EESSI_SOFTWARE_SUBDIR_OVERRIDE='dummy/cpu'
+          CPU_ARCH=$(./init/eessi_archdetect.sh cpupath)
+          if [[ $CPU_ARCH == "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
+              echo "Test for picking up on \$EESSI_SOFTWARE_SUBDIR_OVERRIDE PASSED"
+          else
+              echo "Test for picking up on \$EESSI_SOFTWARE_SUBDIR_OVERRIDE FAILED" >&2
+              exit 1
+          fi
+          unset EESSI_SOFTWARE_SUBDIR_OVERRIDE
+          # check that printing of all matches works correctly (-a option for cpupath action)
           CPU_ARCHES=$(./init/eessi_archdetect.sh -a cpupath)
           if [[ $CPU_ARCHES == "$( cat ./tests/archdetect/${{matrix.proc_cpuinfo}}.all.output )" ]]; then
-              echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCHES" >&2
+              echo "Test for ${{matrix.proc_cpuinfo}} PASSED: $CPU_ARCHES"
           else
               echo "Test for ${{matrix.proc_cpuinfo}} FAILED: $CPU_ARCHES" >&2
               exit 1
           fi
-          # Check all those architectures actually exist
-          for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do
-              # Search all EESSI versions as we may drop support at some point 
-              ls -d "$EESSI_PREFIX"/../*/software/linux/"$dir" 
-          done
+          # Check all those architectures actually exist (if this EESSI version has been populated already)
+          if [ -d ${EESSI_PREFIX}/software/linux ]; then
+            for dir in $(echo "$CPU_ARCHES" | tr ':' '\n'); do
+                # Search all EESSI versions as we may drop support at some point
+                ls -d ${EESSI_PREFIX}/software/linux/${dir}
+            done
+          fi

.github/workflows/tests_init.yml

@@ -12,17 +12,17 @@ jobs:
       fail-fast: false
     steps:
     - name: checkout
-      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
     - name: set up Python
-      uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64
 
     - name: install Python packages
       run: |
-        pip install archspec pytest
+        pip install archspec==0.2.2 pytest
 
     - name: unit tests for eessi_software_subdir_for_host.py script
       run:

.github/workflows/tests_readme.yml

@@ -7,8 +7,6 @@ on:
       - init/eessi_defaults
 
   pull_request:
-    branches:
-      - main
     paths:
       - README.md
       - init/eessi_defaults
@@ -19,12 +17,12 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
         - name: Check out software-layer repository
-          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-        - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults
+        - name: verify if README.md is consistent with EESSI_VERSION from init/eessi_defaults
           run: |
             source init/eessi_defaults
-            grep "${EESSI_PILOT_VERSION}" README.md
+            grep "${EESSI_VERSION}" README.md
 
         - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults
           run: |