From 45022b6b47c3046ea3e2d4033197fdd80dc63850 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 8 Mar 2024 17:30:55 +0100 Subject: [PATCH 01/13] Allow for open PRs when checking missing installations in build script --- EESSI-install-software.sh | 2 +- check_missing_installations.sh | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 69de9d1997..75b4f71178 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -237,7 +237,7 @@ else copy_build_log "${eb_last_log}" "${build_logs_dir}" fi - $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} + $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} ${TOPDIR}/${pr_diff} else fatal_error "Easystack file ${easystack_file} not found!" fi diff --git a/check_missing_installations.sh b/check_missing_installations.sh index c902fa8184..623c46c74f 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -10,8 +10,15 @@ TOPDIR=$(dirname $(realpath $0)) -if [ $# -ne 1 ]; then - echo "ERROR: Usage: $0 " >&2 +if [ "$#" -eq 1 ]; then + true +elif [ "$#" -eq 2 ]; then + echo "Using $2 to give create exceptions for PR filtering of easystack" + # Find lines that are added and use from-pr, make them unique, grab the + # PR numbers and use them to construct something we can use within awk + pr_exceptions=$(grep ^+ $2 | grep from-pr | uniq | awk '{print $3}' | xargs -i echo " || /'{}'/") +else + echo "ERROR: Usage: $0 ()" >&2 exit 1 fi easystack=$1 @@ -24,7 +31,10 @@ export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) -grep -v from-pr ${easystack} > ${tmp_easystack} +# Let's use awk so we can allow for exceptions if we are given a PR diff file +awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" +awk_command=${awk_command/\\/} # Strip out the backslash we needed for ! +eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack} source $TOPDIR/scripts/utils.sh @@ -40,6 +50,10 @@ exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" +if [ "$#" -eq 1 ]; then + fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)" +fi + check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" # the above assesses the installed software for each easyconfig provided in From 723fce82b272cccd75a1d1841ef301be419461e6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 11:39:33 +0100 Subject: [PATCH 02/13] Check for unmerged EasyConfig output pattern --- bot/check-build.sh | 143 ++++++++++++++++++++++++++++----------------- 1 file changed, 90 insertions(+), 53 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index ec1ca56bba..e737363f35 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -98,15 +98,15 @@ job_dir=${PWD} job_out="slurm-${SLURM_JOB_ID}.out" [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'" if [[ -f ${job_out} ]]; then - SLURM=1 + SLURM_OUTPUT_FOUND=1 [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" else - SLURM=0 + SLURM_OUTPUT_FOUND=0 [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" fi ERROR=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_error='ERROR: ' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}") [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 @@ -116,7 +116,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi FAILED=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_failed='FAILED: ' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 @@ -126,7 +126,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi MISSING=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_req_missing=' required modules missing:' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}") [[ $? -eq 0 ]] && MISSING=1 || MISSING=0 @@ -136,7 +136,7 @@ if [[ ${SLURM} -eq 1 ]]; then fi NO_MISSING=-1 -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_no_missing='No missing installations' grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}") [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 @@ -147,7 +147,7 @@ fi TGZ=-1 TARBALL= -if [[ ${SLURM} -eq 1 ]]; then +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_tgz_created="\.tar\.gz created!" grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) if [[ $? -eq 0 ]]; then @@ -169,9 +169,27 @@ fi [[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" [[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +# Here, we try to do some additional analysis on the output file +# to see if we can print a more clear 'reason' for the failure +# For now, we only analyse unmerged EasyConfigs as potential cause, but we can easily add checks for other +# specific scenarios below + +# Check for the pattern being added here https://github.com/EESSI/software-layer/pull/493 to the output to +# see if EasyConfigs might have been unmerged, and that's causing a failure +UNMERGED_EASYCONFIG=-1 +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then + gp_unmerged="are you sure all PRs referenced have been merged in EasyBuild" + grep_unmerged=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${gp_unmerged}") + [[ $? -eq 0 ]] && UNMERGED_EASYCONFIG=1 || UNMERGED_EASYCONFIG=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${gp_unmerged}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_unmerged}" +fi + job_result_file=_bot_job${SLURM_JOB_ID}.result -if [[ ${SLURM} -eq 1 ]] && \ +# Default reason: +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \ [[ ${ERROR} -eq 0 ]] && \ [[ ${FAILED} -eq 0 ]] && \ [[ ${MISSING} -eq 0 ]] && \ @@ -180,7 +198,12 @@ if [[ ${SLURM} -eq 1 ]] && \ [[ ! -z ${TARBALL} ]]; then # SUCCESS status="SUCCESS" + reason="" summary=":grin: SUCCESS" +elif [[ ${UNMERGED_EASYCONFIG} -eq 1 ]]; then + status="FAILURE" + reason="EasyConfig not found during missing installation check. Are you sure all PRs referenced have been merged in EasyBuild?" + summary=":cry: FAILURE" else # FAILURE status="FAILURE" @@ -253,14 +276,6 @@ fi # ### -# construct and write complete PR comment details: implements third alternative -comment_template="
__SUMMARY_FMT__
__DETAILS_FMT____ARTEFACTS_FMT__
" -comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" -comment_details_fmt="
_Details_
__DETAILS_LIST__
" -comment_success_item_fmt=":white_check_mark: __ITEM__" -comment_failure_item_fmt=":x: __ITEM__" -comment_artefacts_fmt="
_Artefacts_
__ARTEFACTS_LIST__
" -comment_artefact_details_fmt="
__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
" function print_br_item() { format="${1}" @@ -332,42 +347,65 @@ echo -n "comment_description = " >> ${job_result_file} # - __DETAILS_FMT__ -> variable $comment_details # - __ARTEFACTS_FMT__ -> variable $comment_artefacts +# construct and write complete PR comment details: implements third alternative +comment_template="
__SUMMARY_FMT__
__REASON_FMT____DETAILS_FMT____ARTEFACTS_FMT__
" +comment_success_item_fmt=":white_check_mark: __ITEM__" +comment_failure_item_fmt=":x: __ITEM__" + +# Initialize comment_description +comment_description=${comment_template} + +# Now, start replacing template items one by one +# Replace the summary template (__SUMMARY_FMT__) +comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" +comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} + +# Only add if there is a reason (e.g. no reason for successful runs) +if [[ ! -z ${reason} ]]; then + comment_reason_fmt="
_Reason_
__REASONS__
" + reason_details="${comment_reason_fmt/__REASONS__/${reason}}" + comment_description=${comment_description/__REASON_FMT__/${reason_details}} +else + comment_description=${comment_description/__REASON_FMT__/""} +fi -# first construct comment_details_list, abbreviated CoDeList +# Replace the details template (__DETAILS_FMT__) +# first construct comment_details_list, abbreviated comment_details_list # then use it to set comment_details -CoDeList="" +comment_details_list="" success_msg="job output file ${job_out}" failure_msg="no job output file ${job_out}" -CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_error}" failure_msg="found message matching ${GP_error}" -CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_failed}" failure_msg="found message matching ${GP_failed}" -CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") success_msg="no message matching ${GP_req_missing}" failure_msg="found message matching ${GP_req_missing}" -CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") success_msg="found message(s) matching ${GP_no_missing}" failure_msg="no message matching ${GP_no_missing}" -CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") +comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") success_msg="found message matching ${GP_tgz_created}" failure_msg="no message matching ${GP_tgz_created}" -CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") - -comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" - +comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") +# Now, do the actual repalcement of __DETAILS_FMT__ +comment_details_fmt="
_Details_
__DETAILS_LIST__
" +comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}" +comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} -# first construct comment_artefacts_list, abbreviated CoArList +# first construct comment_artefacts_list # then use it to set comment_artefacts -CoArList="" +comment_artifacts_list="" # TARBALL should only contain a single tarball if [[ ! -z ${TARBALL} ]]; then @@ -427,50 +465,49 @@ if [[ ! -z ${TARBALL} ]]; then software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" - CoArList="" - CoArList="${CoArList}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})" - CoArList="${CoArList}$(print_br_item 'entries: __ITEM__' ${entries})" - CoArList="${CoArList}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" - CoArList="${CoArList}
"
+    comment_artifacts_list=""
+    comment_artifacts_list="${comment_artifacts_list}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})"
+    comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})"
+    comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)"
+    comment_artifacts_list="${comment_artifacts_list}
"
     if [[ ! -z ${modules} ]]; then
         while IFS= read -r mod ; do
-            CoArList="${CoArList}$(print_br_item '__ITEM__' ${mod})"
+            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${mod})"
         done <<< "${modules}"
     else
-        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no module files in tarball')"
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
     fi
-    CoArList="${CoArList}
"
-    CoArList="${CoArList}$(print_br_item 'software under ___ITEM___' ${prefix}/software)"
-    CoArList="${CoArList}
"
+    comment_artifacts_list="${comment_artifacts_list}
"
+    comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)"
+    comment_artifacts_list="${comment_artifacts_list}
"
     if [[ ! -z ${software_pkgs} ]]; then
         while IFS= read -r sw_pkg ; do
-            CoArList="${CoArList}$(print_br_item '__ITEM__' ${sw_pkg})"
+            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${sw_pkg})"
         done <<< "${software_pkgs}"
     else
-        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no software packages in tarball')"
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
     fi
-    CoArList="${CoArList}
"
-    CoArList="${CoArList}$(print_br_item 'other under ___ITEM___' ${prefix})"
-    CoArList="${CoArList}
"
+    comment_artifacts_list="${comment_artifacts_list}
"
+    comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})"
+    comment_artifacts_list="${comment_artifacts_list}
"
     if [[ ! -z ${other_shortened} ]]; then
         while IFS= read -r other ; do
-            CoArList="${CoArList}$(print_br_item '__ITEM__' ${other})"
+            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${other})"
         done <<< "${other_shortened}"
     else
-        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no other files in tarball')"
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
     fi
-    CoArList="${CoArList}
"
+    comment_artifacts_list="${comment_artifacts_list}
" else - CoArList="${CoArList}$(print_dd_item 'No artefacts were created or found.' '')" + comment_artifacts_list="${comment_artifacts_list}$(print_dd_item 'No artefacts were created or found.' '')" fi +comment_artefact_details_fmt="
__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
" comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}" -comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${CoArList}}" -comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" +comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${comment_artifacts_list}}" -# now put all pieces together creating comment_details from comment_template -comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} -comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} +comment_artefacts_fmt="
_Artefacts_
__ARTEFACTS_LIST__
" +comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} echo "${comment_description}" >> ${job_result_file} From f4f277605a17aea824f940ae1de0727b190be47d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 13:22:18 +0100 Subject: [PATCH 03/13] First, try without --from-pr. If that fails, try with --from-pr, but only for items added to the EasyStack file in _this_ PR to software-layer. If that succeeds, the reason must be that one of the EasyBuild PRs wasn't merged yet, and we can add that information to the fail_msg to make it more explicit --- check_missing_installations.sh | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 623c46c74f..fd2c0d34bb 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -31,6 +31,8 @@ export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) +grep -v from-pr ${easystack} > ${tmp_easystack} + # Let's use awk so we can allow for exceptions if we are given a PR diff file awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" awk_command=${awk_command/\\/} # Strip out the backslash we needed for ! @@ -51,7 +53,25 @@ exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" if [ "$#" -eq 1 ]; then - fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)" + # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR + # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg + + # Let's use awk so we can allow for exceptions if we are given a PR diff file + awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" + awk_command=${awk_command/\\/} # Strip out the backslash we needed for ! + eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack} + + msg=">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}," + msg="${msg} allowing for --from-pr's that were added in this PR..." + echo ${msg} + eb_missing_out=$LOCAL_TMPDIR/eb_missing_with_from_pr.out + ${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out} + exit_code=${PIPESTATUS[0]} + + # If now we succeeded, the reason must be that we originally stripped the --from-pr's + if [ "$#" -eq 0 ]; then + fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)" + fi fi check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" From b7891087072ab05b50d93295b0f8990e27b50edf Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 13:24:08 +0100 Subject: [PATCH 04/13] Add CFITSIO as a test example, see if we can get a clear error message printed on a build failure here --- .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 2664d8f417..51e96f1fac 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -10,4 +10,7 @@ easyconfigs: from-pr: 19552 - CDO-2.2.2-gompi-2023b.eb: options: - from-pr: 19792 + from-pr: 19792 + - CFITSIO-4.3.1-GCCcore-13.2.0.eb: + options: + from-pr: 19840 From d178a8f14d6e16f5bcddcd7af0bb7978f647d823 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 13:34:33 +0100 Subject: [PATCH 05/13] Added clarifying comment --- bot/check-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/check-build.sh b/bot/check-build.sh index e737363f35..24c608b04a 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -207,6 +207,8 @@ elif [[ ${UNMERGED_EASYCONFIG} -eq 1 ]]; then else # FAILURE status="FAILURE" + # General failure, we don't know a more specific reason + reason="" summary=":cry: FAILURE" fi From 9123c9c8f483cf5f6b1e931cfe5a4484fc978faa Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 13:37:08 +0100 Subject: [PATCH 06/13] Correct mistake: I meant to _move_ the awk part, not duplicate it --- check_missing_installations.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index fd2c0d34bb..4781ecc72a 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -33,11 +33,6 @@ export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) grep -v from-pr ${easystack} > ${tmp_easystack} -# Let's use awk so we can allow for exceptions if we are given a PR diff file -awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" -awk_command=${awk_command/\\/} # Strip out the backslash we needed for ! -eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack} - source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild From 7e4422335f1c2f3ad682362a060906866b9b90f4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 13:54:07 +0100 Subject: [PATCH 07/13] Now actually check against the exit code of the pipe... --- check_missing_installations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 4781ecc72a..51a328b50c 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -47,7 +47,7 @@ exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" -if [ "$#" -eq 1 ]; then +if [ "$exit_code" -eq 1 ]; then # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg From da749a111320720773d648d4a55caa2ba1a826e3 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Sat, 9 Mar 2024 14:08:27 +0100 Subject: [PATCH 08/13] Make seperate exit code for the run that allows --from-pr --- check_missing_installations.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 51a328b50c..6d84384fd7 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -61,10 +61,10 @@ if [ "$exit_code" -eq 1 ]; then echo ${msg} eb_missing_out=$LOCAL_TMPDIR/eb_missing_with_from_pr.out ${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out} - exit_code=${PIPESTATUS[0]} + exit_code_with_from_pr=${PIPESTATUS[0]} # If now we succeeded, the reason must be that we originally stripped the --from-pr's - if [ "$#" -eq 0 ]; then + if [ "$exit_code_with_from_pr" -eq 0 ]; then fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)" fi fi From a8c2883f0c73af0537a834221c3ae439f44c1b2f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Mon, 11 Mar 2024 15:59:32 +0100 Subject: [PATCH 09/13] Correct typo Co-authored-by: ocaisa --- bot/check-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 24c608b04a..10b1ed6a11 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -400,7 +400,7 @@ comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${succ success_msg="found message matching ${GP_tgz_created}" failure_msg="no message matching ${GP_tgz_created}" comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") -# Now, do the actual repalcement of __DETAILS_FMT__ +# Now, do the actual replacement of __DETAILS_FMT__ comment_details_fmt="
_Details_
__DETAILS_LIST__
" comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}" comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} From 6553e294163fb9e117803a2433acba3c14e135f2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 11 Mar 2024 16:05:16 +0100 Subject: [PATCH 10/13] Skip the if-condition also if pr_exceptions is empty. There is no point in running the eb --missing again in that case, since there were no --from-pr's in this PR to begin with, so unmerged PRs cannot be the reason for failing --- check_missing_installations.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 6d84384fd7..0ee6ec4385 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -47,9 +47,11 @@ exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" -if [ "$exit_code" -eq 1 ]; then +if [ "$exit_code" -eq 1 ] && [ ! -z $pr_exceptions ]; then # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg + # Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as + # well skip this check - unmerged PRs can not be the reason for the non-zero exit code in that scenario # Let's use awk so we can allow for exceptions if we are given a PR diff file awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" From c08831ee060d153bb1da466cfaa00e4103c37e31 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 13 Mar 2024 15:46:24 +0100 Subject: [PATCH 11/13] Update bot/check-build.sh Co-authored-by: Kenneth Hoste --- bot/check-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index 10b1ed6a11..e075518421 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -174,7 +174,7 @@ fi # For now, we only analyse unmerged EasyConfigs as potential cause, but we can easily add checks for other # specific scenarios below -# Check for the pattern being added here https://github.com/EESSI/software-layer/pull/493 to the output to +# Check for the pattern being added here by check_missing_installations.sh to the output to # see if EasyConfigs might have been unmerged, and that's causing a failure UNMERGED_EASYCONFIG=-1 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then From 992b11dc782d391899862f80033f08b1efba4634 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 13 Mar 2024 15:47:37 +0100 Subject: [PATCH 12/13] Remove PR to trigger comment Co-authored-by: Kenneth Hoste --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 2b5844289d..e2d35276df 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -49,6 +49,3 @@ easyconfigs: - libspatialindex-1.9.3-GCCcore-13.2.0.eb: options: from-pr: 19922 - - CFITSIO-4.3.1-GCCcore-13.2.0.eb: - options: - from-pr: 19840 From 8d6493ea05422ee443f10bc9230c3ff3884af998 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 13 Mar 2024 15:48:08 +0100 Subject: [PATCH 13/13] Trigger on any non-zero exit code Co-authored-by: Kenneth Hoste --- check_missing_installations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 0ee6ec4385..7c29228416 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -47,7 +47,7 @@ exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" -if [ "$exit_code" -eq 1 ] && [ ! -z $pr_exceptions ]; then +if [ "$exit_code" -ne 0 ] && [ ! -z $pr_exceptions ]; then # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg # Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as