Various wave updates: Reformat WW3 inputs; enable PIO for wave restarts; enable WW3 cycling; synchronize UFS restart writes; add uglo_15km WW3 grid

.gitignore

@@ -81,6 +81,7 @@ parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN
 parm/ufs/ufs.configure.*.IN
 parm/ufs/post_itag_gfs
+parm/ufs/ww3_shel.nml.IN
 parm/wafs
 
 # Ignore sorc and logs folders from externals

docs/source/wave.rst

@@ -21,6 +21,8 @@ Here are several regional naming conventions:
 +===========+=======================+
 | glo       | Global domain         |
 +-----------+-----------------------+
+| uglo      | Unstructured global   |
++-----------+-----------------------+
 | ak        | Alaska                |
 +-----------+-----------------------+
 | ao or aoc | Arctic Ocean          |

parm/config/gefs/config.ufs

@@ -512,6 +512,7 @@ fi
 
 # Set the name of the UFS (previously nems) configure template to use
 # Default ufs.configure templates for supported model configurations
+# WW3 restart field variable is different for slow vs fast loop.  Add  WW3_RSTFLDS="ice" for slow loop variables based on coupling scheme. 
 if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
   tmpl_suffix="_esmf"
 fi
@@ -533,9 +534,11 @@ case "${model_list}" in
     ;;
   atm.ocean.ice.wave)
     default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN"
+    WW3_RSTFLDS="ice"
     ;;
   atm.ocean.ice.wave.aero)
     default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN"
+    WW3_RSTFLDS="ice"
     ;;
   *)
     echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}"
@@ -547,6 +550,9 @@ esac
 export ufs_configure_template=${ufs_configure_template:-${default_template:-"/dev/null"}}
 unset model_list default_template
 
+#export wave restart variable: 
+export WW3_RSTFLDS=${WW3_RSTFLDS:-" "}
+
 if [[ ! -r "${ufs_configure_template}" ]]; then
   echo "FATAL ERROR: ${ufs_configure_template} either doesn't exist or is not readable."
   exit 17

parm/config/gefs/config.wave

@@ -23,6 +23,16 @@ export waveMULTIGRID='.false.'        # .true./.false. for multi or shel
 export MESH_WAV="mesh.${waveGRD}.nc"  # Mesh grid for wave model for CMEPS
 export waveesmfGRD=' '                # input grid for multigrid
 
+export WW3_user_histname="false"
+export WW3_historync="false"
+export WW3_restartnc="true"
+export WW3_PIO_FORMAT="pnetcdf"
+export WW3_PIO_IOTASKS=-99
+export WW3_PIO_STRIDE=4
+export WW3_PIO_REARR="box"
+export WW3_PIO_ROOT=-99
+
+
 #Grid dependent variables for various grids
 case "${waveGRD}" in
   "gnh_10m;aoc_9km;gsh_15m")

parm/config/gfs/config.base

@@ -219,7 +219,7 @@ case "${CASE}" in
         export waveGRD='uglo_100km'
         ;;
     "C768" | "C1152")
-        export waveGRD='uglo_m1g16'
+        export waveGRD='uglo_15km'
         ;;
     *)
         echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!"

parm/config/gfs/config.ufs

@@ -605,7 +605,11 @@ if [[ "${skip_ww3}" == "false" ]]; then
     "mx025")
       ntasks_ww3=80
       ;;
-    "uglo_100km")
+    "uglo_15km")
+      ntasks_ww3=1000
+      nthreads_ww3=1
+      ;;
+   "uglo_100km")
       ntasks_ww3=40
       nthreads_ww3=1
       ;;
@@ -630,6 +634,7 @@ fi
 
 # Set the name of the UFS (previously nems) configure template to use
 # Default ufs.configure templates for supported model configurations
+# WW3 restart field variable is different for slow vs fast loop.  Add  WW3_RSTFLDS="ice" for slow loop variables based on coupling scheme.
 if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
   tmpl_suffix="_esmf"
 fi
@@ -651,9 +656,11 @@ case "${model_list}" in
     ;;
   atm.ocean.ice.wave)
     default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN"
+    WW3_RSTFLDS="ice"
     ;;
   atm.ocean.ice.wave.aero)
     default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN"
+    WW3_RSTFLDS="ice"
     ;;
   *)
     echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}"
@@ -665,6 +672,9 @@ esac
 export ufs_configure_template=${ufs_configure_template:-${default_template:-"/dev/null"}}
 unset model_list default_template
 
+# export wave restart variable: 
+export WW3_RSTFLDS=${WW3_RSTFLDS:-" "}
+
 if [[ ! -r "${ufs_configure_template}" ]]; then
   echo "FATAL ERROR: ${ufs_configure_template} either doesn't exist or is not readable."
   exit 17

parm/config/gfs/config.wave

@@ -23,6 +23,17 @@ export waveMULTIGRID='.false.'        # .true./.false. for multi or shel
 export MESH_WAV="mesh.${waveGRD}.nc"  # Mesh grid for wave model for CMEPS
 export waveesmfGRD=' '                # input grid for multigrid
 
+export WW3_user_histname="false"
+export WW3_historync="false"
+export WW3_restartnc="true"
+export WW3_PIO_FORMAT="pnetcdf"
+export WW3_PIO_IOTASKS=-99
+export WW3_PIO_STRIDE=4
+export WW3_PIO_REARR="box"
+export WW3_PIO_ROOT=-99
+
+
+
 #Grid dependent variables for various grids
 case "${waveGRD}" in
   "gnh_10m;aoc_9km;gsh_15m")
@@ -74,6 +85,12 @@ case "${waveGRD}" in
     export wavepostGRD='glo_500'
     export waveuoutpGRD=${waveGRD}
     ;;
+  "uglo_15km")
+    # unstructured 15km grid
+    export waveinterpGRD='glo_15mxt at_10m ep_10m wc_10m glo_30m'
+    export wavepostGRD=''
+    export waveuoutpGRD=${waveGRD}
+    ;; 
   "uglo_100km")
     #unstructured 100km grid
     export waveinterpGRD='glo_200'

parm/wave/glo_15mxt_interp.inp.tmpl

@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'uglo_m1g16'
+  'uglo_15km'
   'glo_15mxt'
 $
  0

sorc/link_workflow.sh

@@ -191,6 +191,7 @@ declare -a ufs_templates=("model_configure.IN" "input_global_nest.nml.IN"
   "ufs.configure.s2swa_esmf.IN"
   "ufs.configure.leapfrog_atm_wav.IN"
   "ufs.configure.leapfrog_atm_wav_esmf.IN"
+  "ww3_shel.nml.IN"
   "post_itag_gfs")
 for file in "${ufs_templates[@]}"; do
   [[ -s "${file}" ]] && rm -f "${file}"

ush/forecast_postdet.sh

@@ -337,10 +337,9 @@ FV3_out() {
 # shellcheck disable=SC2034
 WW3_postdet() {
   echo "SUB ${FUNCNAME[0]}: Linking input data for WW3"
-
   local ww3_grid first_ww3_restart_out ww3_restart_file
   # Copy initial condition files:
-  local restart_date restart_dir
+  local restart_date restart_dir seconds
   if [[ "${RERUN}" == "YES" ]]; then
     restart_date="${RERUN_DATE}"
     restart_dir="${DATArestart}/WW3_RESTART"
@@ -350,27 +349,52 @@ WW3_postdet() {
   fi
 
   echo "Copying WW3 restarts for 'RUN=${RUN}' at '${restart_date}' from '${restart_dir}'"
+
   ww3_restart_file="${restart_dir}/${restart_date:0:8}.${restart_date:8:2}0000.restart.ww3"
   if [[ -s "${ww3_restart_file}" ]]; then
+    #first check if a binary restart exists, if it does set 
+    export WW3_restart_from_binary=true
     ${NCP} "${ww3_restart_file}" "${DATA}/restart.ww3" \
-      || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 )
+          || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 )
     first_ww3_restart_out=$(date --utc -d "${restart_date:0:8} ${restart_date:8:2} + ${restart_interval} hours" +%Y%m%d%H)
+    if [[ "${DOIAU:-NO}" == "YES" ]]; then
+      first_ww3_restart_out=$(date --utc -d "${first_ww3_restart_out:0:8} ${first_ww3_restart_out:8:2} + ${half_window} hours" +%Y%m%d%H)
+    fi 
   else
-    if [[ "${RERUN}" == "YES" ]]; then
-      # In the case of a RERUN, the WW3 restart file is required
-      echo "FATAL ERROR: WW3 restart file '${ww3_restart_file}' not found for RERUN='${RERUN}', ABORT!"
-      exit 1
-    else
-      echo "WARNING: WW3 restart file '${ww3_restart_file}' not found for warm_start='${warm_start}', will start from rest!"
-      first_ww3_restart_out=${model_start_date_current_cycle}
+    # now check to see if a NetCDF restart exists: 
+    seconds=$(to_seconds "${restart_date:8:2}0000")  # convert HHMMSS to seconds
+    ww3_restart_dest_file="ufs.cpld.ww3.r.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}.nc"
+
+    ww3_restart_file="${restart_dir}/${restart_date:0:8}.${restart_date:8:2}0000.restart.ww3.nc"
+    if [[ -s "${ww3_restart_file}" ]]; then  
+      # NetCDF restart exists: 
+      export WW3_restart_from_binary=false
+      ${NCP} "${ww3_restart_file}" "${DATA}/${ww3_restart_dest_file}" \
+             || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 )
+      first_ww3_restart_out=$(date --utc -d "${restart_date:0:8} ${restart_date:8:2} + ${restart_interval} hours" +%Y%m%d%H)
+      if [[ "${DOIAU:-NO}" == "YES" ]]; then
+        first_ww3_restart_out=$(date --utc -d "${first_ww3_restart_out:0:8} ${first_ww3_restart_out:8:2} + ${half_window} hours" +%Y%m%d%H)
+      fi
+    else    
+      if [[ "${RERUN}" == "YES" ]]; then
+        # In the case of a RERUN, the WW3 restart file is required
+        echo "FATAL ERROR: WW3 restart file '${ww3_restart_file}' not found for RERUN='${RERUN}', ABORT!"
+        exit 1
+      else
+        # TODO: See if this will work still with new options... might not
+        echo "WARNING: WW3 restart file '${ww3_restart_file}' not found for warm_start='${warm_start}', will start from rest!"
+        first_ww3_restart_out=${model_start_date_current_cycle}
+      fi
     fi
   fi
 
   # Link restart files
   for (( vdate = first_ww3_restart_out; vdate <= forecast_end_cycle;
          vdate = $(date --utc -d "${vdate:0:8} ${vdate:8:2} + ${restart_interval} hours" +%Y%m%d%H) )); do
-    ww3_restart_file="${vdate:0:8}.${vdate:8:2}0000.restart.ww3"
-    ${NLN} "${DATArestart}/WW3_RESTART/${ww3_restart_file}" "${ww3_restart_file}"
+	 seconds=$(to_seconds "${vdate:8:2}0000")  # convert HHMMSS to seconds
+    ww3_restart_ufs_file="ufs.cpld.ww3.r.${vdate:0:4}-${vdate:4:2}-${vdate:6:2}-${seconds}.nc"
+    ww3_restart_file="${vdate:0:8}.${vdate:8:2}0000.restart.ww3.nc"
+    ${NLN} "${DATArestart}/WW3_RESTART/${ww3_restart_file}" "${ww3_restart_ufs_file}"
   done
 
   # Link output files
@@ -425,7 +449,41 @@ WW3_nml() {
 
 WW3_out() {
   echo "SUB ${FUNCNAME[0]}: Copying output data for WW3"
-  # TODO: Need to add logic to copy restarts from DATArestart/WW3_RESTART to COMOUT_WAVE_RESTART
+
+  # Copy wave namelist from DATA to COMOUT_CONF after the forecast is run (and successfull)
+  ${NCP} "${DATA}/ww3_shel.nml" "${COMOUT_CONF}/ufs.ww3_shel.nml"
+
+  # Copy WW3 restarts at the end of the forecast segment to COM for RUN=gfs|gefs
+  if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then
+    local seconds source_file target_file
+    if [[ "${RUN}" == "gfs" || "${RUN}" == "gefs" ]]; then
+      echo "Copying WW3 restarts for 'RUN=${RUN}' at ${forecast_end_cycle}"
+      target_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.restart.ww3.nc"
+      ${NCP} "${DATArestart}/WW3_RESTART/${target_file}" \
+             "${COMOUT_WAVE_RESTART}/${target_file}"
+    fi
+  fi
+
+  # Copy restarts for next cycle for RUN=gdas|gefs
+  if [[ "${RUN}" == "gdas" || "${RUN}" == "gefs" ]]; then
+    local restart_date
+    restart_date="${model_start_date_next_cycle}"
+    echo "Copying WW3 restarts for 'RUN=${RUN}' at ${restart_date}"
+    target_file="${restart_date:0:8}.${restart_date:8:2}0000.restart.ww3.nc"
+    ${NCP} "${DATArestart}/WW3_RESTART/${target_file}" \
+           "${COMOUT_WAVE_RESTART}/${target_file}"
+  fi
+
+  # Copy restarts for downstream usage in HAFS 
+  if [[ "${RUN}" =~ "gdas" ]]; then
+    local restart_date
+    restart_date="${next_cycle}"
+    echo "Copying WW3 restarts for 'RUN=${RUN}' at ${restart_date}"
+    target_file="${restart_date:0:8}.${restart_date:8:2}0000.restart.ww3.nc"
+    ${NCP} "${DATArestart}/WW3_RESTART/${target_file}" \
+           "${COMOUT_WAVE_RESTART}/${target_file}"
+  fi
+
 }
 
 

ush/forecast_predet.sh

@@ -134,6 +134,29 @@
   # Several model components share DATA/INPUT for input data
   if [[ ! -d "${DATA}/INPUT" ]]; then mkdir -p "${DATA}/INPUT"; fi
 
+  # For CMEPS, CICE, MOM6 and WW3 determine restart writes
+  # Note FV3 has its own restart intervals  
+  cmeps_restart_interval=${restart_interval:-${FHMAX}}
+  # restart_interval = 0 implies write restart at the END of the forecast i.e. at FHMAX
+  # Convert restart interval into an explicit list for FV3
+  if (( cmeps_restart_interval == 0 )); then
+    if [[ "${DOIAU:-NO}" == "YES" ]]; then
+      CMEPS_RESTART_FH=$(( FHMAX + half_window ))
+    else 
+      CMEPS_RESTART_FH=("${FHMAX}")
+    fi
+  else
+    if [[ "${DOIAU:-NO}" == "YES" ]]; then
+      local restart_interval_start=$(( cmeps_restart_interval + half_window ))
+      local restart_interval_end=$(( FHMAX + half_window ))
+    else
+      local restart_interval_start=${cmeps_restart_interval} 
+      local restart_interval_end=${FHMAX}
+    fi
+    CMEPS_RESTART_FH="$(seq -s ' ' "${restart_interval_start}" "${cmeps_restart_interval}" "${restart_interval_end}")"
+  fi
+  # TODO: For GEFS, once cycling waves "self-cycles" and therefore needs to have a restart at 6 hour
+
 }
 
 # shellcheck disable=SC2034
@@ -606,6 +629,12 @@
     #if shel, only 1 waveGRD which is linked to mod_def.ww3
     ${NCP} "${COMIN_WAVE_PREP}/${RUN}wave.mod_def.${waveGRD}" "${DATA}/mod_def.ww3" \
     || ( echo "FATAL ERROR: Failed to copy '${RUN}wave.mod_def.${waveGRD}' from '${COMIN_WAVE_PREP}'"; exit 1 )
+    # copy point weights file for unstructured grid if it exists 
+    local wavpntwghtfile=${FIXgfs}/wave/pnt_wght.${waveGRD}.nc
+    if [[ -f "${wavpntwghtfile}" ]]; then 
+      ${NCP} "${wavpntwghtfile}" "${DATA}/pnt_wght.ww3.nc" \
+      || ( echo "FATAL ERROR: Failed to copy '${wavpntwghtfile}'"; exit 1 )
+    fi
   fi
 
   if [[ "${WW3ICEINP}" == "YES" ]]; then

ush/parsing_model_configure_FV3.sh

@@ -25,6 +25,7 @@ local SHOUR=${model_start_date:8:2}
 local FHROT=${IAU_FHROT:-0}
 local DT_ATMOS=${DELTIM}
 local RESTART_INTERVAL="${FV3_RESTART_FH[*]}"
+local RESTART_FH="${CMEPS_RESTART_FH:-" "}"
 # QUILTING
 local QUILTING_RESTART="${QUILTING_RESTART:-${QUILTING}}"
 local WRITE_GROUP=${WRITE_GROUP:-1}