From 1f7afd15fac490cfd090a9882230acdf7d3fd956 Mon Sep 17 00:00:00 2001 From: valentin-seitz Date: Thu, 14 Dec 2023 16:05:25 +0100 Subject: [PATCH] Add Workflows for generating reference results (#410) * add machine info to reference results * rename script to unify naming * add workflows * make autopep happy and trigger wf once :) * made wf callable * remove push trigger * added some switch case to check for the different names until changes propagate * Fix non terminating timeouts (#421) Co-authored-by: Valentin Seitz * Systests/move to commits (#422) * remove non working gha cache stuff * Move to only commits in the dockerfile --------- Co-authored-by: Valentin Seitz * adopt new naming schema * filter lscpu to hide vulnerabilities * move to consistent naming schema * Adding reference results from precice-tests vm * Make autopep8 * make paths a bit nicer and easier to understand * Update tools/tests/README.md Co-authored-by: Gerasimos Chourdakis * Update tools/tests/README.md Co-authored-by: Gerasimos Chourdakis * introduce variable instead of magic number 10 * added comment on very long rm instructions --------- Co-authored-by: Valentin Seitz Co-authored-by: preCICE Tests VM Co-authored-by: Gerasimos Chourdakis --- .../generate_reference_results_manual.yml | 31 ++++++++ .../generate_reference_results_workflow.yml | 71 +++++++++++++++++++ .../fluid-openfoam_solid-fenics.tar.gz | 3 - .../fluid-openfoam_solid-nutils.tar.gz | 3 - .../fluid-openfoam_solid-openfoam.tar.gz | 3 - .../fluid-openfoam_solid-fenics.tar.gz | 3 + .../fluid-openfoam_solid-nutils.tar.gz | 3 + .../fluid-openfoam_solid-openfoam.tar.gz | 3 + .../reference_results.metadata | 40 ++++++++++- .../fluid-openfoam_solid-calculix.tar.gz | 3 - .../fluid-openfoam_solid-calculix.tar.gz | 3 + perpendicular-flap/reference_results.metadata | 36 +++++++++- tools/tests/README.md | 46 ++++++------ .../component-templates/calculix-adapter.yaml | 4 -- .../component-templates/fenics-adapter.yaml | 4 -- .../component-templates/nutils-adapter.yaml | 4 -- .../component-templates/openfoam-adapter.yaml | 4 -- .../tests/dockerfiles/ubuntu_2204/Dockerfile | 12 ++-- ..._data.py => generate_reference_results.py} | 34 ++++++++- tools/tests/paths.py | 4 +- .../tests/reference_results.metadata.template | 12 ++++ tools/tests/systemtests/Systemtest.py | 15 ++-- tools/tests/tests.yaml | 11 ++- 23 files changed, 277 insertions(+), 75 deletions(-) create mode 100644 .github/workflows/generate_reference_results_manual.yml create mode 100644 .github/workflows/generate_reference_results_workflow.yml delete mode 100644 flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz delete mode 100644 flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz delete mode 100644 flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz create mode 100644 flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz create mode 100644 flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz create mode 100644 flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz delete mode 100644 perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz create mode 100644 perpendicular-flap/reference-results/fluid-openfoam_solid-calculix.tar.gz rename tools/tests/{generate_reference_data.py => generate_reference_results.py} (79%) diff --git a/.github/workflows/generate_reference_results_manual.yml b/.github/workflows/generate_reference_results_manual.yml new file mode 100644 index 000000000..bca10cf4a --- /dev/null +++ b/.github/workflows/generate_reference_results_manual.yml @@ -0,0 +1,31 @@ +name: Generate reference results (manual) +on: + workflow_dispatch: + inputs: + from_ref: + description: 'Use the systemtests + tutorial metadata + reference_version from this ref' + required: true + type: string + commit_msg: + description: 'Commit msg for commit that adds the reference results' + default: "Adding reference results" + type: string + loglevel: + description: 'loglevel used for the systemtests' + default: 'INFO' + required: true + type: choice + options: + - 'DEBUG' + - 'INFO' + - 'WARNING' + - 'ERROR' + - 'CRITICAL' + +jobs: + generate_reference_results_manual: + uses: ./.github/workflows/generate_reference_results_workflow.yml + with: + from_ref: ${{ inputs.from_ref }} + commit_msg: ${{ inputs.commit_msg }} + loglevel: ${{ inputs.loglevel }} \ No newline at end of file diff --git a/.github/workflows/generate_reference_results_workflow.yml b/.github/workflows/generate_reference_results_workflow.yml new file mode 100644 index 000000000..cf79320c6 --- /dev/null +++ b/.github/workflows/generate_reference_results_workflow.yml @@ -0,0 +1,71 @@ +name: Generate reference results workflow +on: + workflow_call: + inputs: + from_ref: + description: 'Use the systemtests + tutorial metadata + reference_version from this ref' + required: true + type: string + commit_msg: + description: 'Commit msg for commit that adds the reference results' + default: "Adding reference results" + type: string + loglevel: + description: 'loglevel used for the systemtests' + default: 'INFO' + required: true + type: string +jobs: + generate_reference_results: + runs-on: [self-hosted, linux, x64, precice-tests-vm] + steps: + - name: Display a quick job summary + run: | + echo "Initiated by: ${{ github.actor }}" + echo "Running generate_reference_results.py --log-level ${{inputs.loglevel}}" + echo "Using Ref: ${{ inputs.from_ref }}" + echo "Commit message on success: ${{ inputs.commit_msg }}" + - name: Move LFS URL to local LFS server + run: | + /home/precice/runners_root/scripts/make_lfs_local.sh + - name: 'Cleanup the folder' + # The first rf -rf ./* removes all non hidden files + # The second rf -rf ./.??* removes all hidden files (but not . and ..) + run: | + ls -la ./ + rm -rf ./* + rm -rf ./.??* + ls -la ./ + - name: Check out Tutorials for systest + uses: actions/checkout@v4 + with: + ref: ${{ inputs.from_ref }} + lfs: true + fetch-depth: 0 + - name: Install Python dependencies + run: | + pip install --user -r tools/tests/requirements.txt + - name: Run tests + run: | + cd tools/tests + test -f generate_reference_results.py && export GENERATE_REF_RESULTS=generate_reference_results.py + test -f generate_reference_data.py && export GENERATE_REF_RESULTS=generate_reference_data.py + echo "Selected $GENERATE_REF_RESULTS to run" + python $GENERATE_REF_RESULTS --log-level=${{inputs.loglevel}} + cd ../../ + - name: Create commit + if: success() + run: | + git checkout ${{ inputs.from_ref }} + git add ./*/*/*.tar.gz + git add ./*/*.tar.gz + git add ./*/*.metadata + git commit -m "${{inputs.commit_msg}}" + git push + - name: Upload artifacts for debugging + if: failure() + uses: actions/upload-artifact@v3 + with: + name: runs + path: | + runs/* diff --git a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz deleted file mode 100644 index fd47ecc01..000000000 --- a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2565ffc51c8d80fab06ccdda33f62efceb6cdc03318998f5adc9ed0ac84acac2 -size 777932 diff --git a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz deleted file mode 100644 index 9c773bcdf..000000000 --- a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8b3157902d3ad78593c5471a3a94dd5559fe6970e4cb54658b22ffd270c00297 -size 532549 diff --git a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz deleted file mode 100644 index ee340cd80..000000000 --- a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c29989e0118ade759e7313330aa8d3d900c660c404eaacd529e979aae6b899c7 -size 498432 diff --git a/flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz new file mode 100644 index 000000000..2aa122746 --- /dev/null +++ b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65d4c461e3f6167846730de14b0778b004a02f2db740f2c263aee4e2bd8b86a6 +size 778312 diff --git a/flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz new file mode 100644 index 000000000..1809fe423 --- /dev/null +++ b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d790bf8a032b1f155699b5a63287124be202d45bcb59cab9b320c3fef86ba56 +size 533210 diff --git a/flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz new file mode 100644 index 000000000..922a4d229 --- /dev/null +++ b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ab9f3f762beec987e0a90ce99e5e3b85f406e4738599fd4b1ad68942ed7f516 +size 498933 diff --git a/flow-over-heated-plate/reference_results.metadata b/flow-over-heated-plate/reference_results.metadata index 640389b49..eba9e358f 100644 --- a/flow-over-heated-plate/reference_results.metadata +++ b/flow-over-heated-plate/reference_results.metadata @@ -5,14 +5,15 @@ This File has been generated by the generate_reference_results.py and should not # Reference Results This file contains an overview of the results over the reference results as well as the arguments used to generate them. +We also include some information on the machine used to generate them ## List of files | name | time | sha256 | |------|------|-------| -| fluid-openfoam_solid-nutils.tar.gz | 2023-09-04 23:41:09 | 8b3157902d3ad78593c5471a3a94dd5559fe6970e4cb54658b22ffd270c00297 | -| fluid-openfoam_solid-fenics.tar.gz | 2023-09-04 23:41:09 | 2565ffc51c8d80fab06ccdda33f62efceb6cdc03318998f5adc9ed0ac84acac2 | -| fluid-openfoam_solid-openfoam.tar.gz | 2023-09-04 23:41:09 | c29989e0118ade759e7313330aa8d3d900c660c404eaacd529e979aae6b899c7 | +| fluid-openfoam_solid-nutils.tar.gz | 2023-12-13 22:22:18 | 7d790bf8a032b1f155699b5a63287124be202d45bcb59cab9b320c3fef86ba56 | +| fluid-openfoam_solid-fenics.tar.gz | 2023-12-13 22:22:18 | 65d4c461e3f6167846730de14b0778b004a02f2db740f2c263aee4e2bd8b86a6 | +| fluid-openfoam_solid-openfoam.tar.gz | 2023-12-13 22:22:18 | 2ab9f3f762beec987e0a90ce99e5e3b85f406e4738599fd4b1ad68942ed7f516 | ## List of arguments used to generate the files @@ -27,3 +28,36 @@ This file contains an overview of the results over the reference results as well | PLATFORM | ubuntu_2204 | | CALULIX_VERSION | 2.20 | | CALULIX_ADAPTER_REF | v2.20.0 | +## Information about the machine + +### uname -a + +Linux precice-tests 5.15.0-89-generic #99-Ubuntu SMP Mon Oct 30 20:42:41 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux + + +### lscpu + +Architecture: x86_64 +CPU op-mode(s): 32-bit, 64-bit +Address sizes: 45 bits physical, 48 bits virtual +Byte Order: Little Endian +CPU(s): 4 +On-line CPU(s) list: 0-3 +Vendor ID: GenuineIntel +Model name: Intel(R) Xeon(R) Gold 6130 CPU @ 2.10GHz +CPU family: 6 +Model: 85 +Thread(s) per core: 1 +Core(s) per socket: 1 +Socket(s): 4 +Stepping: 4 +BogoMIPS: 4199.99 +Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon nopl xtopology tsc_reliable nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 invpcid avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat pku ospke md_clear flush_l1d arch_capabilities +Hypervisor vendor: VMware +Virtualization type: full +L1d cache: 128 KiB (4 instances) +L1i cache: 128 KiB (4 instances) +L2 cache: 4 MiB (4 instances) +L3 cache: 88 MiB (4 instances) +NUMA node(s): 1 +NUMA node0 CPU(s): 0-3 diff --git a/perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz b/perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz deleted file mode 100644 index 63d4135f6..000000000 --- a/perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:596fe1aec3f72fa194c37c485688dea4e308544e3cd773d7c815d6e27c4e55a8 -size 13561235 diff --git a/perpendicular-flap/reference-results/fluid-openfoam_solid-calculix.tar.gz b/perpendicular-flap/reference-results/fluid-openfoam_solid-calculix.tar.gz new file mode 100644 index 000000000..7896d684d --- /dev/null +++ b/perpendicular-flap/reference-results/fluid-openfoam_solid-calculix.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:905c90f4f5c7776c5822b9235e435ae25b9097d2f8bc6392865402b21a525194 +size 13559914 diff --git a/perpendicular-flap/reference_results.metadata b/perpendicular-flap/reference_results.metadata index 1255609aa..6e1e84ac9 100644 --- a/perpendicular-flap/reference_results.metadata +++ b/perpendicular-flap/reference_results.metadata @@ -5,12 +5,13 @@ This File has been generated by the generate_reference_results.py and should not # Reference Results This file contains an overview of the results over the reference results as well as the arguments used to generate them. +We also include some information on the machine used to generate them ## List of files | name | time | sha256 | |------|------|-------| -| fluid-openfoam_solid-calculix.tar.gz | 2023-09-04 23:41:09 | 596fe1aec3f72fa194c37c485688dea4e308544e3cd773d7c815d6e27c4e55a8 | +| fluid-openfoam_solid-calculix.tar.gz | 2023-12-13 22:22:18 | 905c90f4f5c7776c5822b9235e435ae25b9097d2f8bc6392865402b21a525194 | ## List of arguments used to generate the files @@ -25,3 +26,36 @@ This file contains an overview of the results over the reference results as well | PLATFORM | ubuntu_2204 | | CALULIX_VERSION | 2.20 | | CALULIX_ADAPTER_REF | v2.20.0 | +## Information about the machine + +### uname -a + +Linux precice-tests 5.15.0-89-generic #99-Ubuntu SMP Mon Oct 30 20:42:41 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux + + +### lscpu + +Architecture: x86_64 +CPU op-mode(s): 32-bit, 64-bit +Address sizes: 45 bits physical, 48 bits virtual +Byte Order: Little Endian +CPU(s): 4 +On-line CPU(s) list: 0-3 +Vendor ID: GenuineIntel +Model name: Intel(R) Xeon(R) Gold 6130 CPU @ 2.10GHz +CPU family: 6 +Model: 85 +Thread(s) per core: 1 +Core(s) per socket: 1 +Socket(s): 4 +Stepping: 4 +BogoMIPS: 4199.99 +Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon nopl xtopology tsc_reliable nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 invpcid avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat pku ospke md_clear flush_l1d arch_capabilities +Hypervisor vendor: VMware +Virtualization type: full +L1d cache: 128 KiB (4 instances) +L1i cache: 128 KiB (4 instances) +L2 cache: 4 MiB (4 instances) +L3 cache: 88 MiB (4 instances) +NUMA node(s): 1 +NUMA node0 CPU(s): 0-3 diff --git a/tools/tests/README.md b/tools/tests/README.md index 0f9d2cfa5..6f197dbfd 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -38,17 +38,17 @@ gh workflow run run_testsuite_manual.yml -f suites=fenics_test --ref=develop Another example, to use the latest develop branches and enable debug information of the tests: ```shell -gh workflow run run_testsuite_manual.yml -f suites=fenics_test -f build_args="PRECICE_REF:develop,OPENFOAM_ADAPTER_REF:develop,PYTHON_BINDINGS_REF:develop,FENICS_ADAPTER_REF:develop" -f loglevel=DEBUG --ref=develop +gh workflow run run_testsuite_manual.yml -f suites=fenics_test -f build_args="PRECICE_REF:v2.5.0,OPENFOAM_ADAPTER_REF:v1.2.3,PYTHON_BINDINGS_REF:v2.5.0.4,FENICS_ADAPTER_REF:v1.4.0" -f loglevel=DEBUG --ref=develop ``` -where the `*_REF` can also be specific Git commits. +where the `*_REF` should be a specific [commit-ish](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefcommit-ishacommit-ishalsocommittish). Example output: ```text Run cd tools/tests cd tools/tests - python systemtests.py --build_args=PRECICE_REF:develop,OPENFOAM_ADAPTER_REF:develop,PYTHON_BINDINGS_REF:develop,FENICS_ADAPTER_REF:develop --suites=fenics_test --log-level=DEBUG + python systemtests.py --build_args=PRECICE_REF:v2.5.0,OPENFOAM_ADAPTER_REF:v1.2.3,PYTHON_BINDINGS_REF:v2.5.0.4,FENICS_ADAPTER_REF:v1.4.0 --suites=fenics_test --log-level=DEBUG cd ../../ shell: /usr/bin/bash -e {0} INFO: About to run the following systemtest in the directory /home/precice/runners_root/actions-runner-tutorial/_work/tutorials/tutorials/runs: @@ -76,18 +76,9 @@ In this case, building and running seems to work out, but the tests fail because ## Understanding what went wrong -Let's first see how the workflow was triggered. If we expand the `Set up job` step, we can see the inputs provided: - -```text - Inputs - build_args: PRECICE_REF:develop,OPENFOAM_ADAPTER_REF:develop,PYTHON_BINDINGS_REF:develop,FENICS_ADAPTER_REF:develop - loglevel: DEBUG - suites: fenics_test - systests_branch: develop - upload_artifacts: FALSE -``` - -In the summary, we can find the results and more logs as a build artifact. This includes two interesting files: `stdout.log` and `stderr.log`. These include all Docker build steps and the simulation output, as well as the exact git clone command. +The easiest way to debug a systemtest run is first to have a look at the output written into the action on GitHub. +If this does not provide enough hints, the next step is to download the generated `runs` artifact. Note that by default this will only be generated if the systemtests fail. +Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation. ## Adding new tests @@ -138,7 +129,7 @@ Metadata and workflow/script files: - ... - `metadata.yml`: describes each case directory (which participant, which component, which script to run, ...) - `tools/tests/` - - `component-templates/`: jinja2 templates for Docker Compose services, specifying cache system + - `component-templates/`: jinja2 templates for Docker Compose services for the components - `calculix-adapter.yaml` - `fenics-adapter.yaml` - `openfoam-adapter.yaml` @@ -149,7 +140,7 @@ Metadata and workflow/script files: - `docker-compose.field_compare.template.yaml`: Describes how to compare results with fieldcompare (Docker Compose service template) - `components.yaml`: Declares the available components and their parameters/options - `reference_results.metadata.template`: Template for reporting the versions used to generate the reference results - - `reference_versions.yaml`: Versions of components to use for generating the reference results + - `reference_versions.yaml`: List of arguments to use for generating the reference results - `tests.yaml`: Declares the available tests, grouped in test suites User-facing tools: @@ -160,7 +151,7 @@ User-facing tools: - `print_metadata.py`: Prints the metadata of each tutorial that contains a `metadata.yaml` file. - `print_case_combinations.py`: Prints all possible combinations of tutorial cases, using the `metadata.yaml` files. - `build_docker_images.py`: Build the Docker images for each test - - `generate_reference_data.py`: Executes the system tests with the versions defined in `reference_versions.yaml` and generates the reference data archives, with the names described in `tests.yaml`. + - `generate_reference_results.py`: Executes the system tests with the versions defined in `reference_versions.yaml` and generates the reference data archives, with the names described in `tests.yaml`. (should only be used by the CI Pipeline) Implementation scripts: @@ -226,7 +217,7 @@ The components mentioned in the Metadata are defined in the central `components. openfoam-adapter: repository: https://github.com/precice/openfoam-adapter template: component-templates/openfoam-adapter.yaml - build_arguments: # these things mean something to the docker-service + build_arguments: PRECICE_REF: description: Version of preCICE to use default: "main" @@ -247,7 +238,7 @@ openfoam-adapter: This `openfoam-adapter` component has the following attributes: -- `repository`: URL to the Git project (without the `.git` extension), used to fetch the component +- `repository`: URL to the Git projects - `template`: A template for a Docker Compose service of this component - `build_arguments`: Arguments passed to the Docker Compose service (arbitrary) @@ -255,7 +246,7 @@ This `openfoam-adapter` component has the following attributes: Since the docker containers are still a bit mixed in terms of capabilities and support for different build_argument combinations the following rules apply: -- A build_argument ending in **_REF** means that it refers to a git reference (like a branch or commit) beeing used to build the image. +- A build_argument ending in **_REF** means that it refers to a git commit-ish (like a tag or commit) beeing used to build the image. Its important to not use branch names here as we heavily rely on dockers build cache to speedup things. But since the input variable to the docker builder will not change, we might have wrong cache hits. - All other build_arguments are free of rules and up to the container maintainer. ### Component templates @@ -313,8 +304,19 @@ This defines two test suites, namely `openfoam_adapter_pr` and `openfoam_adapter ### Generate Reference Results +#### via GitHub workflow (recommended) + +The preferred way of adding reference results is via the manual triggerable `Generate reference results (manual)` workflow. This takes two inputs: + +- `from_ref`: branch where the new test configuration (e.g added tests, new reference_versions.yaml) is +- `commit_msg`: commit message for adding the reference results into the branch + +The workflow will checkout the `from_ref`, take the status of the systemtests of that branch and execute `python generate_reference_results.py`, upload the LFS objects into the self-hosted LFS server and add a commit with `commit_msg` onto the `from_ref` branch. + +#### manually + In order to generate the reference results edit the `reference_versions.yaml` to match the required `build_arguments` otherwise passed via the cli. -Executing `generate_reference_data.py` will then generate a the following files: +Executing `generate_reference_results.py` will then generate the following files: - all distinct `.tar.gz` defined in the `tests.yaml` - a `reference_results.md` in the tutorial folder describing the arguments used and a sha-1 hash of the `tar.gz` archive. diff --git a/tools/tests/component-templates/calculix-adapter.yaml b/tools/tests/component-templates/calculix-adapter.yaml index 373c6e0f1..038a088b7 100644 --- a/tools/tests/component-templates/calculix-adapter.yaml +++ b/tools/tests/component-templates/calculix-adapter.yaml @@ -5,10 +5,6 @@ build: - {{key}}={{value}} {% endfor %} target: calculix_adapter - cache_from: - - type=gha - cache_to: - - type=gha,mode=min,scope=calculix_adapter depends_on: prepare: condition: service_completed_successfully diff --git a/tools/tests/component-templates/fenics-adapter.yaml b/tools/tests/component-templates/fenics-adapter.yaml index 52177d719..b83a23707 100644 --- a/tools/tests/component-templates/fenics-adapter.yaml +++ b/tools/tests/component-templates/fenics-adapter.yaml @@ -5,10 +5,6 @@ build: - {{key}}={{value}} {% endfor %} target: fenics_adapter - cache_from: - - type=gha - cache_to: - - type=gha,mode=min,scope=fenics_adapter depends_on: prepare: condition: service_completed_successfully diff --git a/tools/tests/component-templates/nutils-adapter.yaml b/tools/tests/component-templates/nutils-adapter.yaml index ee98f1ec1..9963028ab 100644 --- a/tools/tests/component-templates/nutils-adapter.yaml +++ b/tools/tests/component-templates/nutils-adapter.yaml @@ -5,10 +5,6 @@ build: - {{key}}={{value}} {% endfor %} target: nutils_adapter - cache_from: - - type=gha - cache_to: - - type=gha,mode=min,scope=nutils_adapter depends_on: prepare: condition: service_completed_successfully diff --git a/tools/tests/component-templates/openfoam-adapter.yaml b/tools/tests/component-templates/openfoam-adapter.yaml index e3821247a..4657b6e23 100644 --- a/tools/tests/component-templates/openfoam-adapter.yaml +++ b/tools/tests/component-templates/openfoam-adapter.yaml @@ -5,10 +5,6 @@ build: - {{key}}={{value}} {% endfor %} target: openfoam_adapter - cache_from: - - type=gha - cache_to: - - type=gha,mode=min,scope=openfoam_adapter depends_on: prepare: condition: service_completed_successfully diff --git a/tools/tests/dockerfiles/ubuntu_2204/Dockerfile b/tools/tests/dockerfiles/ubuntu_2204/Dockerfile index e31e911a1..f895df384 100644 --- a/tools/tests/dockerfiles/ubuntu_2204/Dockerfile +++ b/tools/tests/dockerfiles/ubuntu_2204/Dockerfile @@ -50,8 +50,9 @@ FROM precice_dependecies as precice ARG PRECICE_REF USER precice WORKDIR /home/precice -RUN git clone --depth 1 https://github.com/precice/precice.git -b ${PRECICE_REF} precice && \ +RUN git clone https://github.com/precice/precice.git precice && \ cd precice && \ + git checkout ${PRECICE_REF} && \ mkdir build && cd build &&\ cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/home/precice/.local/ -DPRECICE_PETScMapping=OFF -DBUILD_TESTING=OFF && \ make all install -j $(nproc) @@ -69,8 +70,10 @@ ARG OPENFOAM_ADAPTER_REF # Build the OpenFOAM adapter USER precice WORKDIR /home/precice -RUN git clone --depth 1 -b ${OPENFOAM_ADAPTER_REF} https://github.com/precice/openfoam-adapter.git &&\ - cd openfoam-adapter && /usr/bin/${OPENFOAM_EXECUTABLE} ./Allwmake -j $(nproc) +RUN git clone https://github.com/precice/openfoam-adapter.git &&\ + cd openfoam-adapter && \ + git checkout ${OPENFOAM_ADAPTER_REF} && \ + /usr/bin/${OPENFOAM_EXECUTABLE} ./Allwmake -j $(nproc) FROM precice_dependecies as python_bindings @@ -117,7 +120,8 @@ RUN wget http://www.dhondt.de/ccx_${CALULIX_VERSION}.src.tar.bz2 && \ ARG CALULIX_ADAPTER_REF WORKDIR /home/precice -RUN git clone --depth 1 --branch ${CALULIX_ADAPTER_REF} https://github.com/precice/calculix-adapter.git && \ +RUN git clone https://github.com/precice/calculix-adapter.git && \ cd calculix-adapter && \ + git checkout ${CALULIX_ADAPTER_REF} &&\ make CXX_VERSION=${CALULIX_VERSION} ADDITIONAL_FFLAGS="-fallow-argument-mismatch" -j $(nproc) && \ ln -s /home/precice/calculix-adapter/bin/ccx_preCICE /home/precice/.local/bin/ccx_preCICE diff --git a/tools/tests/generate_reference_data.py b/tools/tests/generate_reference_results.py similarity index 79% rename from tools/tests/generate_reference_data.py rename to tools/tests/generate_reference_results.py index ea2457984..335b73a62 100644 --- a/tools/tests/generate_reference_data.py +++ b/tools/tests/generate_reference_results.py @@ -10,8 +10,10 @@ import hashlib from jinja2 import Environment, FileSystemLoader import tarfile +import subprocess from datetime import datetime import logging + from paths import PRECICE_TUTORIAL_DIR, PRECICE_TESTS_RUN_DIR, PRECICE_TESTS_DIR, PRECICE_REL_OUTPUT_DIR import time @@ -21,6 +23,31 @@ def create_tar_gz(source_folder: Path, output_filename: Path): tar.add(source_folder, arcname=output_filename.name.replace(".tar.gz", "")) +def get_machine_informations(): + def command_is_avail(command: str): + try: + rc = subprocess.call(['which', command], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except FileNotFoundError: + return False + + return rc == 0 + uname_info = "uname not available on the machine the systemtests were executed." + lscpu_info = "lscpu not available on the machine the systemtests were executed." + if (command_is_avail("uname")): + result = subprocess.run(["uname", "-a"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode == 0: + uname_info = result.stdout + + if (command_is_avail("lscpu") and command_is_avail("grep")): + result_lscpu = subprocess.run(["lscpu"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = subprocess.run(["grep", "-v", "Vulner"], input=result_lscpu.stdout, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode == 0: + lscpu_info = result.stdout + + return (uname_info, lscpu_info) + + def render_reference_results_info( reference_results: List[ReferenceResult], arguments_used: SystemtestArguments, @@ -36,11 +63,14 @@ def sha256sum(filename): 'time': time, 'name': reference_result.path.name, }) - + uname, lscpu = get_machine_informations() render_dict = { 'arguments': arguments_used.arguments, - 'files': files + 'files': files, + 'uname': uname, + 'lscpu': lscpu, } + jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) template = jinja_env.get_template("reference_results.metadata.template") return template.render(render_dict) diff --git a/tools/tests/paths.py b/tools/tests/paths.py index f9a9844a9..8b7f064d2 100644 --- a/tools/tests/paths.py +++ b/tools/tests/paths.py @@ -3,5 +3,5 @@ PRECICE_TESTS_RUN_DIR = Path(__file__).parent.parent.parent / "runs" PRECICE_TOOLS_DIR = Path(__file__).parent.parent PRECICE_TESTS_DIR = Path(__file__).parent -PRECICE_REL_OUTPUT_DIR = "precice_exports" -PRECICE_REL_REFERENCE_DIR = "reference_results" +PRECICE_REL_OUTPUT_DIR = "precice-exports" +PRECICE_REL_REFERENCE_DIR = "reference-results-unpacked" diff --git a/tools/tests/reference_results.metadata.template b/tools/tests/reference_results.metadata.template index d711cd872..d7f189fe2 100644 --- a/tools/tests/reference_results.metadata.template +++ b/tools/tests/reference_results.metadata.template @@ -5,6 +5,7 @@ This File has been generated by the generate_reference_results.py and should not # Reference Results This file contains an overview of the results over the reference results as well as the arguments used to generate them. +We also include some information on the machine used to generate them ## List of files @@ -20,3 +21,14 @@ This file contains an overview of the results over the reference results as well {% for name,value in arguments.items() -%} | {{ name }} | {{ value }} | {% endfor -%} + + +## Information about the machine + +### uname -a + +{{ uname }} + +### lscpu + +{{ lscpu }} \ No newline at end of file diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 6d80663ef..f1a52651a 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -20,6 +20,7 @@ GLOBAL_TIMEOUT = 360 +SHORT_TIMEOUT = 10 def slugify(value, allow_unicode=False): @@ -354,13 +355,12 @@ def _run_field_compare(self): stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) except KeyboardInterrupt as k: process.kill() - # process.send_signal(9) raise KeyboardInterrupt from k except Exception as e: logging.critical( - f"Systemtest {self} had serious issues executin the docker compose command about to kill the docker compose command. Please check the logs! {e}") + f"Systemtest {self} had serious issues executing the docker compose command about to kill the docker compose command. Please check the logs! {e}") process.kill() - stdout, stderr = process.communicate() + process.communicate(timeout=SHORT_TIMEOUT) stdout_data.extend(stdout.decode().splitlines()) stderr_data.extend(stderr.decode().splitlines()) process.poll() @@ -397,7 +397,7 @@ def _build_docker(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate() + stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) except KeyboardInterrupt as k: process.kill() # process.send_signal(9) @@ -405,8 +405,8 @@ def _build_docker(self): except Exception as e: logging.critical( f"systemtest {self} had serious issues building the docker images via the `docker compose build` command. About to kill the docker compose command. Please check the logs! {e}") + process.communicate(timeout=SHORT_TIMEOUT) process.kill() - stdout, stderr = process.communicate() stdout_data.extend(stdout.decode().splitlines()) stderr_data.extend(stderr.decode().splitlines()) @@ -448,9 +448,10 @@ def _run_tutorial(self): raise KeyboardInterrupt from k except Exception as e: logging.critical( - f"Systemtest {self} had serious issues executin the docker compose command about to kill the docker compose command. Please check the logs! {e}") + f"Systemtest {self} had serious issues executing the docker compose command about to kill the docker compose command. Please check the logs! {e}") + process.kill() + stdout, stderr = process.communicate(timeout=SHORT_TIMEOUT) process.kill() - stdout, stderr = process.communicate() stdout_data.extend(stdout.decode().splitlines()) stderr_data.extend(stderr.decode().splitlines()) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 827eae38e..57dc45790 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -5,33 +5,32 @@ test_suites: case_combination: - fluid-openfoam - solid-openfoam - reference_result: ./flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz + reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz openfoam_adapter_release: tutorials: - path: flow-over-heated-plate case_combination: - fluid-openfoam - solid-openfoam - reference_result: ./flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz + reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz fenics_test: tutorials: - path: flow-over-heated-plate case_combination: - fluid-openfoam - solid-fenics - reference_result: ./flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz + reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz nutils_test: tutorials: - path: flow-over-heated-plate case_combination: - fluid-openfoam - solid-nutils - reference_result: ./flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz + reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz calculix_test: tutorials: - path: perpendicular-flap case_combination: - fluid-openfoam - solid-calculix - reference_result: ./perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz - timeout: \ No newline at end of file + reference_result: ./perpendicular-flap/reference-results/fluid-openfoam_solid-calculix.tar.gz \ No newline at end of file