From e51b7dbe9bcb8dcd27299e60afa8536c66c7434a Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 18 Mar 2022 11:22:13 -0500 Subject: [PATCH 1/5] Add injection prep blurb to README --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index 2307b61..7241a34 100644 --- a/README.md +++ b/README.md @@ -73,3 +73,32 @@ Most subdirectories contain a run.sh script that outlines how to run the problem The case can the be run similar to other MIRGE-Com applications. For examples see the MIRGE-Com [documentation](https://mirgecom.readthedocs.io/en/latest/running/systems.html) + +Notes for running with multispecies: + +Some multispecies tests are in the following directories: +``` +smoke_test_injection_2d +smoke_test_injection_3d +``` + +The 3D multispecies tests are currently the best approximation to the expected Y2 prediction workload. The +general procedure for preparing and running the tests in these directories is as follows: +1. Generate the mesh +``` +cd data&&./make_mesh.sh +``` +2. Choose inert or with combustion: +``` +ln -sf run_params_combustion.yaml run_params.yaml +-or- +ln -sf run_params_inert.yaml run_params.yaml +``` +3. Initialize the case solution (make sure to run this on the same number of ranks as you plan to run): +``` +run_init.sh +``` +4. Run the simulation by restarting from the init'd soln: +``` +run.sh +``` \ No newline at end of file From 5a96f3880900dc0157630a563f72279823c6ae53 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 18 Mar 2022 11:22:40 -0500 Subject: [PATCH 2/5] Make 3d injection smoke test config files --- .../run_params_combustion.yaml | 22 +++++++++++++++++++ ...{run_params.yaml => run_params_inert.yaml} | 0 2 files changed, 22 insertions(+) create mode 100644 smoke_test_injection_3d/run_params_combustion.yaml rename smoke_test_injection_3d/{run_params.yaml => run_params_inert.yaml} (100%) diff --git a/smoke_test_injection_3d/run_params_combustion.yaml b/smoke_test_injection_3d/run_params_combustion.yaml new file mode 100644 index 0000000..f7a0def --- /dev/null +++ b/smoke_test_injection_3d/run_params_combustion.yaml @@ -0,0 +1,22 @@ +nviz: 10 +nrestart: 10 +nhealth: 1 +nstatus: 1 +current_dt: 1.0e-9 +t_final: 2.e-8 +order: 1 +dimen: 3 +mach_inj: 0.8 +nspecies: 7 +#nspecies: 2 +#nspecies: 0 +alpha_sc: 0.1 +s0_sc: -5.0 +kappa_sc: 0.5 +integrator: euler +#health_pres_min: 1 +#health_pres_max: 500000 +health_pres_min: 2.65231e3 +health_pres_max: 2.77231e5 +health_mass_frac_min: -9.e-7 +health_mass_frac_max: 1.0001 diff --git a/smoke_test_injection_3d/run_params.yaml b/smoke_test_injection_3d/run_params_inert.yaml similarity index 100% rename from smoke_test_injection_3d/run_params.yaml rename to smoke_test_injection_3d/run_params_inert.yaml From 0ee196ab99617b5972b02322cc2ede41a366770c Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Sat, 2 Apr 2022 22:11:06 -0700 Subject: [PATCH 3/5] add Lassen script --- smoke_test_injection_3d/lassen.bsub.sh | 47 ++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 smoke_test_injection_3d/lassen.bsub.sh diff --git a/smoke_test_injection_3d/lassen.bsub.sh b/smoke_test_injection_3d/lassen.bsub.sh new file mode 100644 index 0000000..d539422 --- /dev/null +++ b/smoke_test_injection_3d/lassen.bsub.sh @@ -0,0 +1,47 @@ +#!/bin/bash +#BSUB -nnodes 1 # number of nodes +#BSUB -W 120 # walltime in minutes +#BSUB -q pbatch # queue to use + + +set -ex + +# Run this script with 'bsub lassen.bsub.sh' + +# Put any environment activation here, e.g.: +# source ../../config/activate_env.sh + +# OpenCL device selection: +export PYOPENCL_CTX="port:tesla" # Run on Nvidia GPU with pocl +# export PYOPENCL_CTX="port:pthread" # Run on CPU with pocl + +nnodes=$(echo $LSB_MCPU_HOSTS | wc -w) +nnodes=$((nnodes/2-1)) +nproc=$((4*nnodes)) # 4 ranks per node, 1 per GPU + +echo nnodes=$nnodes nproc=$nproc + +# -a 1: 1 task per resource set +# -g 1: 1 GPU per resource set +# -n $nproc: $nproc resource sets +jsrun_cmd="jsrun -g 1 -a 1 -n $nproc" + +# See +# https://mirgecom.readthedocs.io/en/latest/running.html#avoiding-overheads-due-to-caching-of-kernels +# on why this is important +export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch" + +# Fixes https://github.com/illinois-ceesd/mirgecom/issues/292 +# (each rank needs its own POCL cache dir) +export POCL_CACHE_DIR_ROOT="/tmp/$USER/pocl-cache" + +# Print task allocation +$jsrun_cmd js_task_info + +echo "----------------------------" + +# Run application +# -O: switch on optimizations +# POCL_CACHE_DIR=...: each rank needs its own POCL cache dir +$jsrun_cmd bash -c 'POCL_CACHE_DIR=$POCL_CACHE_DIR_ROOT/$$ python -m mpi4py isolator_injection_init.py -i run_params.yaml' +$jsrun_cmd bash -c 'POCL_CACHE_DIR=$POCL_CACHE_DIR_ROOT/$$ python -m mpi4py isolator_injection_run.py -r restart_data/isolator_init-000000 -i run_params.yaml --lazy' From 5d9731af261cdab13314928d4cbc3861f423ec38 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Sun, 3 Apr 2022 09:41:40 -0500 Subject: [PATCH 4/5] run init lazily too --- smoke_test_injection_3d/lassen.bsub.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/smoke_test_injection_3d/lassen.bsub.sh b/smoke_test_injection_3d/lassen.bsub.sh index d539422..d3b70e9 100644 --- a/smoke_test_injection_3d/lassen.bsub.sh +++ b/smoke_test_injection_3d/lassen.bsub.sh @@ -43,5 +43,9 @@ echo "----------------------------" # Run application # -O: switch on optimizations # POCL_CACHE_DIR=...: each rank needs its own POCL cache dir -$jsrun_cmd bash -c 'POCL_CACHE_DIR=$POCL_CACHE_DIR_ROOT/$$ python -m mpi4py isolator_injection_init.py -i run_params.yaml' + +# isolator_injection_init.py must be run with the same config as isolator_injection_run.py, in particular: +# - same numbers of ranks in both cases +# - lazy runs need a lazy init +$jsrun_cmd bash -c 'POCL_CACHE_DIR=$POCL_CACHE_DIR_ROOT/$$ python -m mpi4py isolator_injection_init.py -i run_params.yaml --lazy' $jsrun_cmd bash -c 'POCL_CACHE_DIR=$POCL_CACHE_DIR_ROOT/$$ python -m mpi4py isolator_injection_run.py -r restart_data/isolator_init-000000 -i run_params.yaml --lazy' From 660401040bbf83cf815121dc2eabed2606d6f3ff Mon Sep 17 00:00:00 2001 From: anderson2981 <41346365+anderson2981@users.noreply.github.com> Date: Fri, 8 Apr 2022 14:26:35 -0500 Subject: [PATCH 5/5] Update README.md --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7241a34..c7a6aa3 100644 --- a/README.md +++ b/README.md @@ -101,4 +101,16 @@ run_init.sh 4. Run the simulation by restarting from the init'd soln: ``` run.sh -``` \ No newline at end of file +``` + +For production scale testing see the test_mesh_injection directory. + +```test_mesh_injection/template``` + +There are 4 run sizes here, each with their own meshing. These meshes can be gererated similar to above. On Lassen the user can find pre-generated meshes in + +```/usr/WS1/xpacc/CEESD/Y2_test_meshes``` + +The meshes range in size from 1 million elements (eigthX) to 180 millions elements (oneX). All runs are set to use multi-species with chemistry (nspecies=7). + +A sample Lassen submission script is co-located with the runs, combinedLassenBatch.sh.