Add Workflows for generating reference results (#410)

* add machine info to reference results * rename script to unify naming * add workflows * make autopep happy and trigger wf once :) * made wf callable * remove push trigger * added some switch case to check for the different names until changes propagate * Fix non terminating timeouts (#421) Co-authored-by: Valentin Seitz <[email protected]> * Systests/move to commits (#422) * remove non working gha cache stuff * Move to only commits in the dockerfile --------- Co-authored-by: Valentin Seitz <[email protected]> * adopt new naming schema * filter lscpu to hide vulnerabilities * move to consistent naming schema * Adding reference results from precice-tests vm * Make autopep8 * make paths a bit nicer and easier to understand * Update tools/tests/README.md Co-authored-by: Gerasimos Chourdakis <[email protected]> * Update tools/tests/README.md Co-authored-by: Gerasimos Chourdakis <[email protected]> * introduce variable instead of magic number 10 * added comment on very long rm instructions --------- Co-authored-by: Valentin Seitz <[email protected]> Co-authored-by: preCICE Tests VM <[email protected]> Co-authored-by: Gerasimos Chourdakis <[email protected]>
precice · Dec 14, 2023 · 1f7afd1 · 1f7afd1
1 parent 622caf7
commit 1f7afd1
Show file tree

Hide file tree

Showing 23 changed files with 277 additions and 75 deletions.
diff --git a/.github/workflows/generate_reference_results_manual.yml b/.github/workflows/generate_reference_results_manual.yml
@@ -0,0 +1,31 @@
+name: Generate reference results (manual)
+on:
+  workflow_dispatch:
+    inputs:
+      from_ref:
+        description: 'Use the systemtests + tutorial metadata + reference_version from this ref'
+        required: true
+        type: string
+      commit_msg:
+        description: 'Commit msg for commit that adds the reference results'
+        default: "Adding reference results"
+        type: string
+      loglevel:
+        description: 'loglevel used for the systemtests'
+        default: 'INFO'
+        required: true
+        type: choice
+        options:
+            - 'DEBUG'
+            - 'INFO'
+            - 'WARNING'
+            - 'ERROR'
+            - 'CRITICAL'
+
+jobs:
+  generate_reference_results_manual:
+    uses: ./.github/workflows/generate_reference_results_workflow.yml
+    with: 
+      from_ref: ${{ inputs.from_ref }}
+      commit_msg: ${{ inputs.commit_msg }}
+      loglevel: ${{ inputs.loglevel }}
diff --git a/.github/workflows/generate_reference_results_workflow.yml b/.github/workflows/generate_reference_results_workflow.yml
@@ -0,0 +1,71 @@
+name: Generate reference results workflow
+on:
+  workflow_call:
+    inputs:
+      from_ref:
+        description: 'Use the systemtests + tutorial metadata + reference_version from this ref'
+        required: true
+        type: string
+      commit_msg:
+        description: 'Commit msg for commit that adds the reference results'
+        default: "Adding reference results"
+        type: string
+      loglevel:
+        description: 'loglevel used for the systemtests'
+        default: 'INFO'
+        required: true
+        type: string
+jobs:
+  generate_reference_results:
+    runs-on: [self-hosted, linux, x64, precice-tests-vm]
+    steps:
+    - name: Display a quick job summary
+      run: |
+        echo "Initiated by: ${{ github.actor }}"
+        echo "Running generate_reference_results.py --log-level ${{inputs.loglevel}}"
+        echo "Using Ref: ${{ inputs.from_ref }}"
+        echo "Commit message on success:  ${{ inputs.commit_msg }}"
+    - name: Move LFS URL to local LFS server
+      run: |
+        /home/precice/runners_root/scripts/make_lfs_local.sh
+    - name: 'Cleanup the folder'
+      # The first rf -rf ./* removes all non hidden files
+      # The second rf -rf ./.??* removes all hidden files (but not . and ..)
+      run: |
+        ls -la ./
+        rm -rf ./*
+        rm -rf ./.??*
+        ls -la ./
+    - name: Check out Tutorials for systest
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.from_ref }}
+        lfs: true
+        fetch-depth: 0
+    - name: Install Python dependencies
+      run: |
+        pip install --user -r tools/tests/requirements.txt
+    - name: Run tests
+      run: |
+        cd tools/tests
+        test -f generate_reference_results.py && export GENERATE_REF_RESULTS=generate_reference_results.py
+        test -f generate_reference_data.py && export GENERATE_REF_RESULTS=generate_reference_data.py
+        echo "Selected $GENERATE_REF_RESULTS to run"
+        python $GENERATE_REF_RESULTS --log-level=${{inputs.loglevel}}
+        cd ../../
+    - name: Create commit
+      if: success()
+      run: |
+        git checkout ${{ inputs.from_ref }}
+        git add ./*/*/*.tar.gz
+        git add ./*/*.tar.gz
+        git add ./*/*.metadata
+        git commit -m "${{inputs.commit_msg}}"
+        git push
+    - name: Upload artifacts for debugging
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: runs
+        path: |
+          runs/*
diff --git a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz
diff --git a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz
diff --git a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz
diff --git a/flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz
diff --git a/flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz
diff --git a/flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz
diff --git a/flow-over-heated-plate/reference_results.metadata b/flow-over-heated-plate/reference_results.metadata
@@ -5,14 +5,15 @@ This File has been generated by the generate_reference_results.py and should not
 # Reference Results
 
 This file contains an overview of the results over the reference results as well as the arguments used to generate them.
+We also include some information on the machine used to generate them
 
 ## List of files
 
 | name | time | sha256 |
 |------|------|-------|
-| fluid-openfoam_solid-nutils.tar.gz | 2023-09-04 23:41:09 | 8b3157902d3ad78593c5471a3a94dd5559fe6970e4cb54658b22ffd270c00297 |
-| fluid-openfoam_solid-fenics.tar.gz | 2023-09-04 23:41:09 | 2565ffc51c8d80fab06ccdda33f62efceb6cdc03318998f5adc9ed0ac84acac2 |
-| fluid-openfoam_solid-openfoam.tar.gz | 2023-09-04 23:41:09 | c29989e0118ade759e7313330aa8d3d900c660c404eaacd529e979aae6b899c7 |
+| fluid-openfoam_solid-nutils.tar.gz | 2023-12-13 22:22:18 | 7d790bf8a032b1f155699b5a63287124be202d45bcb59cab9b320c3fef86ba56 |
+| fluid-openfoam_solid-fenics.tar.gz | 2023-12-13 22:22:18 | 65d4c461e3f6167846730de14b0778b004a02f2db740f2c263aee4e2bd8b86a6 |
+| fluid-openfoam_solid-openfoam.tar.gz | 2023-12-13 22:22:18 | 2ab9f3f762beec987e0a90ce99e5e3b85f406e4738599fd4b1ad68942ed7f516 |
 
 ## List of arguments used to generate the files
 
@@ -27,3 +28,36 @@ This file contains an overview of the results over the reference results as well
 | PLATFORM | ubuntu_2204 |
 | CALULIX_VERSION | 2.20 |
 | CALULIX_ADAPTER_REF | v2.20.0 |
+## Information about the machine
+
+### uname -a
+
+Linux precice-tests 5.15.0-89-generic #99-Ubuntu SMP Mon Oct 30 20:42:41 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
+
+
+### lscpu
+
+Architecture:                       x86_64
+CPU op-mode(s):                     32-bit, 64-bit
+Address sizes:                      45 bits physical, 48 bits virtual
+Byte Order:                         Little Endian
+CPU(s):                             4
+On-line CPU(s) list:                0-3
+Vendor ID:                          GenuineIntel
+Model name:                         Intel(R) Xeon(R) Gold 6130 CPU @ 2.10GHz
+CPU family:                         6
+Model:                              85
+Thread(s) per core:                 1
+Core(s) per socket:                 1
+Socket(s):                          4
+Stepping:                           4
+BogoMIPS:                           4199.99
+Flags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon nopl xtopology tsc_reliable nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 invpcid avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat pku ospke md_clear flush_l1d arch_capabilities
+Hypervisor vendor:                  VMware
+Virtualization type:                full
+L1d cache:                          128 KiB (4 instances)
+L1i cache:                          128 KiB (4 instances)
+L2 cache:                           4 MiB (4 instances)
+L3 cache:                           88 MiB (4 instances)
+NUMA node(s):                       1
+NUMA node0 CPU(s):                  0-3
diff --git a/perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz b/perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz
diff --git a/perpendicular-flap/reference-results/fluid-openfoam_solid-calculix.tar.gz b/perpendicular-flap/reference-results/fluid-openfoam_solid-calculix.tar.gz
diff --git a/perpendicular-flap/reference_results.metadata b/perpendicular-flap/reference_results.metadata
@@ -5,12 +5,13 @@ This File has been generated by the generate_reference_results.py and should not
 # Reference Results
 
 This file contains an overview of the results over the reference results as well as the arguments used to generate them.
+We also include some information on the machine used to generate them
 
 ## List of files
 
 | name | time | sha256 |
 |------|------|-------|
-| fluid-openfoam_solid-calculix.tar.gz | 2023-09-04 23:41:09 | 596fe1aec3f72fa194c37c485688dea4e308544e3cd773d7c815d6e27c4e55a8 |
+| fluid-openfoam_solid-calculix.tar.gz | 2023-12-13 22:22:18 | 905c90f4f5c7776c5822b9235e435ae25b9097d2f8bc6392865402b21a525194 |
 
 ## List of arguments used to generate the files
 
@@ -25,3 +26,36 @@ This file contains an overview of the results over the reference results as well
 | PLATFORM | ubuntu_2204 |
 | CALULIX_VERSION | 2.20 |
 | CALULIX_ADAPTER_REF | v2.20.0 |
+## Information about the machine
+
+### uname -a
+
+Linux precice-tests 5.15.0-89-generic #99-Ubuntu SMP Mon Oct 30 20:42:41 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
+
+
+### lscpu
+
+Architecture:                       x86_64
+CPU op-mode(s):                     32-bit, 64-bit
+Address sizes:                      45 bits physical, 48 bits virtual
+Byte Order:                         Little Endian
+CPU(s):                             4
+On-line CPU(s) list:                0-3
+Vendor ID:                          GenuineIntel
+Model name:                         Intel(R) Xeon(R) Gold 6130 CPU @ 2.10GHz
+CPU family:                         6
+Model:                              85
+Thread(s) per core:                 1
+Core(s) per socket:                 1
+Socket(s):                          4
+Stepping:                           4
+BogoMIPS:                           4199.99
+Flags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon nopl xtopology tsc_reliable nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 avx2 smep bmi2 invpcid avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat pku ospke md_clear flush_l1d arch_capabilities
+Hypervisor vendor:                  VMware
+Virtualization type:                full
+L1d cache:                          128 KiB (4 instances)
+L1i cache:                          128 KiB (4 instances)
+L2 cache:                           4 MiB (4 instances)
+L3 cache:                           88 MiB (4 instances)
+NUMA node(s):                       1
+NUMA node0 CPU(s):                  0-3
diff --git a/tools/tests/README.md b/tools/tests/README.md
@@ -38,17 +38,17 @@ gh workflow run run_testsuite_manual.yml -f suites=fenics_test --ref=develop
 Another example, to use the latest develop branches and enable debug information of the tests:
 
 ```shell
-gh workflow run run_testsuite_manual.yml -f suites=fenics_test -f build_args="PRECICE_REF:develop,OPENFOAM_ADAPTER_REF:develop,PYTHON_BINDINGS_REF:develop,FENICS_ADAPTER_REF:develop" -f loglevel=DEBUG --ref=develop
+gh workflow run run_testsuite_manual.yml -f suites=fenics_test -f build_args="PRECICE_REF:v2.5.0,OPENFOAM_ADAPTER_REF:v1.2.3,PYTHON_BINDINGS_REF:v2.5.0.4,FENICS_ADAPTER_REF:v1.4.0" -f loglevel=DEBUG --ref=develop
 ```
 
-where the `*_REF` can also be specific Git commits.
+where the `*_REF` should be a specific [commit-ish](https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefcommit-ishacommit-ishalsocommittish).
 
 Example output:
 
 ```text
 Run cd tools/tests
   cd tools/tests
-  python systemtests.py --build_args=PRECICE_REF:develop,OPENFOAM_ADAPTER_REF:develop,PYTHON_BINDINGS_REF:develop,FENICS_ADAPTER_REF:develop --suites=fenics_test --log-level=DEBUG
+  python systemtests.py --build_args=PRECICE_REF:v2.5.0,OPENFOAM_ADAPTER_REF:v1.2.3,PYTHON_BINDINGS_REF:v2.5.0.4,FENICS_ADAPTER_REF:v1.4.0 --suites=fenics_test --log-level=DEBUG
   cd ../../
   shell: /usr/bin/bash -e {0}
 INFO: About to run the following systemtest in the directory /home/precice/runners_root/actions-runner-tutorial/_work/tutorials/tutorials/runs:
@@ -76,18 +76,9 @@ In this case, building and running seems to work out, but the tests fail because
 
 ## Understanding what went wrong
 
-Let's first see how the workflow was triggered. If we expand the `Set up job` step, we can see the inputs provided:
-
-```text
- Inputs
-    build_args: PRECICE_REF:develop,OPENFOAM_ADAPTER_REF:develop,PYTHON_BINDINGS_REF:develop,FENICS_ADAPTER_REF:develop
-    loglevel: DEBUG
-    suites: fenics_test
-    systests_branch: develop
-    upload_artifacts: FALSE
-```
-
-In the summary, we can find the results and more logs as a build artifact. This includes two interesting files: `stdout.log` and `stderr.log`. These include all Docker build steps and the simulation output, as well as the exact git clone command.
+The easiest way to debug a systemtest run is first to have a look at the output written into the action on GitHub.
+If this does not provide enough hints, the next step is to download the generated `runs` artifact. Note that by default this will only be generated if the systemtests fail.
+Inside the archive, a test-specific subfolder like `flow-over-heated-plate_fluid-openfoam-solid-fenics_2023-11-19-211723` contains two log files: a `stderr.log` and `stdout.log`. This can be a starting point for a further investigation.
 
 ## Adding new tests
 
@@ -138,7 +129,7 @@ Metadata and workflow/script files:
   - ...
   - `metadata.yml`: describes each case directory (which participant, which component, which script to run, ...)
 - `tools/tests/`
-  - `component-templates/`: jinja2 templates for Docker Compose services, specifying cache system
+  - `component-templates/`: jinja2 templates for Docker Compose services for the components
     - `calculix-adapter.yaml`
     - `fenics-adapter.yaml`
     - `openfoam-adapter.yaml`
@@ -149,7 +140,7 @@ Metadata and workflow/script files:
   - `docker-compose.field_compare.template.yaml`: Describes how to compare results with fieldcompare (Docker Compose service template)
   - `components.yaml`: Declares the available components and their parameters/options
   - `reference_results.metadata.template`: Template for reporting the versions used to generate the reference results
-  - `reference_versions.yaml`: Versions of components to use for generating the reference results
+  - `reference_versions.yaml`: List of arguments to use for generating the reference results
   - `tests.yaml`: Declares the available tests, grouped in test suites
 
 User-facing tools:
@@ -160,7 +151,7 @@ User-facing tools:
   - `print_metadata.py`: Prints the metadata of each tutorial that contains a `metadata.yaml` file.
   - `print_case_combinations.py`: Prints all possible combinations of tutorial cases, using the `metadata.yaml` files.
   - `build_docker_images.py`: Build the Docker images for each test
-  - `generate_reference_data.py`: Executes the system tests with the versions defined in `reference_versions.yaml` and generates the reference data archives, with the names described in `tests.yaml`.
+  - `generate_reference_results.py`: Executes the system tests with the versions defined in `reference_versions.yaml` and generates the reference data archives, with the names described in `tests.yaml`. (should only be used by the CI Pipeline)
 
 Implementation scripts:
 
@@ -226,7 +217,7 @@ The components mentioned in the Metadata are defined in the central `components.
 openfoam-adapter:
   repository: https://github.com/precice/openfoam-adapter
   template: component-templates/openfoam-adapter.yaml
-  build_arguments: # these things mean something to the docker-service
+  build_arguments:
     PRECICE_REF:
       description: Version of preCICE to use
       default: "main"
@@ -247,15 +238,15 @@ openfoam-adapter:
 
 This `openfoam-adapter` component has the following attributes:
 
-- `repository`: URL to the Git project (without the `.git` extension), used to fetch the component
+- `repository`: URL to the Git projects
 - `template`: A template for a Docker Compose service of this component
 - `build_arguments`: Arguments passed to the Docker Compose service (arbitrary)
 
 #### Naming schema for build_arguments
 
 Since the docker containers are still a bit mixed in terms of capabilities and support for different build_argument combinations the following rules apply:
 
-- A build_argument ending in **_REF** means that it refers to a git reference (like a branch or commit) beeing used to build the image.
+- A build_argument ending in **_REF** means that it refers to a git commit-ish (like a tag or commit) beeing used to build the image. Its important to not use branch names here as we heavily rely on dockers build cache to speedup things. But since the input variable to the docker builder will not change, we might have wrong cache hits.
 - All other build_arguments are free of rules and up to the container maintainer.
 
 ### Component templates
@@ -313,8 +304,19 @@ This defines two test suites, namely `openfoam_adapter_pr` and `openfoam_adapter
 
 ### Generate Reference Results
 
+#### via GitHub workflow (recommended)
+
+The preferred way of adding reference results is via the manual triggerable `Generate reference results (manual)` workflow. This takes two inputs:
+
+- `from_ref`: branch where the new test configuration (e.g added tests, new reference_versions.yaml) is
+- `commit_msg`: commit message for adding the reference results into the branch
+
+The workflow will checkout the `from_ref`, take the status of the systemtests of that branch and execute `python generate_reference_results.py`, upload the LFS objects into the self-hosted LFS server and add a commit with `commit_msg` onto the `from_ref` branch.
+
+#### manually
+
 In order to generate the reference results edit the `reference_versions.yaml` to match the required `build_arguments` otherwise passed via the cli.
-Executing `generate_reference_data.py` will then generate a the following files:
+Executing `generate_reference_results.py` will then generate the following files:
 
 - all distinct `.tar.gz` defined in the `tests.yaml`
 - a `reference_results.md` in the tutorial folder describing the arguments used and a sha-1 hash of the `tar.gz` archive.

diff --git a/tools/tests/component-templates/calculix-adapter.yaml b/tools/tests/component-templates/calculix-adapter.yaml
@@ -5,10 +5,6 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: calculix_adapter
-  cache_from:
-  - type=gha
-  cache_to:
-  - type=gha,mode=min,scope=calculix_adapter
 depends_on:    
   prepare:
     condition: service_completed_successfully

diff --git a/tools/tests/component-templates/fenics-adapter.yaml b/tools/tests/component-templates/fenics-adapter.yaml
@@ -5,10 +5,6 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: fenics_adapter
-  cache_from:
-  - type=gha
-  cache_to:
-  - type=gha,mode=min,scope=fenics_adapter
 depends_on:    
   prepare:
     condition: service_completed_successfully

diff --git a/tools/tests/component-templates/nutils-adapter.yaml b/tools/tests/component-templates/nutils-adapter.yaml
@@ -5,10 +5,6 @@ build:
       - {{key}}={{value}}
     {% endfor %}
   target: nutils_adapter
-  cache_from:
-  - type=gha
-  cache_to:
-  - type=gha,mode=min,scope=nutils_adapter
 depends_on:    
   prepare:
     condition: service_completed_successfully