Merge #246

246: Update GPU tests to use latest KernelAbstractions version r=nefrathenrici a=nefrathenrici

Content
- Updates the pipeline.yml to better practices.
- Remove `CUDAKernels`, update to latest version of `KernelAbstractions`
- Adds a function `setup_output` to setup the output array for each test set.

Co-authored-by: nefrathenrici <[email protected]>

.buildkite/JuliaProject.toml

@@ -0,0 +1,5 @@
+[extras]
+CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
+
+[preferences.CUDA_Runtime_jll]
+version = "local"

.buildkite/pipeline.yml

@@ -1,56 +1,43 @@
+agents:
+  queue: central
+  slurm_mem: 8G
+  modules: julia/1.9.3 cuda/11.8 ucx/1.14.1_cuda-11.8 openmpi/4.1.5_cuda-11.8 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
+
 env:
-  JULIA_VERSION: "1.9.2"
-  OPENMPI_VERSION: "4.0.4"
+  JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
   OPENBLAS_NUM_THREADS: 1
+  JULIA_NVTX_CALLBACKS: gc
+  OMPI_MCA_opal_warn_on_missing_libcuda: 0
+  JULIA_CPU_TARGET: 'broadwell;skylake'
 
 steps:
-  - label: "init :computer:"
-    key: "init_cpu_env"
+  - label: "initialize"
+    key: "initialize"
     command:
-      - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project -e 'using Pkg; Pkg.precompile(;strict=true)'"
-      - "julia --project -e 'using Pkg; Pkg.status()'"
-    agents:
-      config: cpu
-      queue: central
-      slurm_ntasks: 1
+      - echo "--- Instantiate project"
+      - julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'
 
-  - label: "init :flower_playing_cards:"
-    key: "init_gpu_env"
-    command:
-      - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project -e 'using Pkg; Pkg.precompile()'"
+      - echo "--- Instantiate test"
+      - julia --project=test -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'
 
-      # force the initialization of the CUDA runtime as it is lazily loaded by default
-      - "julia --project=test -e 'using Pkg; Pkg.develop(path=\".\"); Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
-      - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
-      - "julia --project=test -e 'using CUDA; CUDA.precompile_runtime()'"
-      - "julia --project=test -e 'using Pkg; Pkg.status()'"
+      - julia --project=test -e 'using CUDA; CUDA.precompile_runtime()'
+      - julia --project=test -e 'using Pkg; Pkg.status()'
     agents:
-      config: gpu
-      queue: central
-      slurm_ntasks: 1
-      slurm_gres: "gpu:1"
+      slurm_gpus: 1
+      slurm_cpus_per_task: 8
+    env:
+      JULIA_NUM_PRECOMPILE_TASKS: 8
 
   - wait
 
   - label: ":computer: unit tests"
     key: "cpu_unittests"
     command:
-      - "julia --project=test -e 'using Pkg; Pkg.develop(path=\".\"); Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
       - "julia --project=test --color=yes test/runtests.jl"
-    agents:
-      config: cpu
-      queue: central
-      slurm_ntasks: 1
 
-  - label: ":flower_playing_cards: unit tests"
+  - label: ":flower_playing_cards: GPU unit tests"
     key: "gpu_unittests"
     command:
-      - "julia --project=test -e 'using Pkg; Pkg.develop(path=\".\"); Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
       - "julia --project=test --color=yes test/gpu_tests.jl"
     agents:
-      config: gpu
-      queue: central
-      slurm_ntasks: 1
-      slurm_gres: "gpu:1"
+      slurm_gpus: 1

Project.toml

@@ -16,7 +16,7 @@ Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c"
 CLIMAParameters = "0.7.15"
 DocStringExtensions = "0.8, 0.9"
 ForwardDiff = "0.10"
-KernelAbstractions = "0.8, 0.9"
+KernelAbstractions = "0.9"
 RootSolvers = "0.3, 0.4"
 SpecialFunctions = "1, 2"
 Thermodynamics = "0.9, 0.10, 0.11"

test/Project.toml

@@ -3,7 +3,6 @@ Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 CLIMAParameters = "6eacf6c3-8458-43b9-ae03-caf5306d3d53"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
-CUDAKernels = "72cfdca4-0801-4ab0-bf6a-d52aa10adc57"
 ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d"
 CloudMicrophysics = "6a9e3e04-43cd-43ba-94b9-e8782df3c71b"
 KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"