Merge pull request #453 from nmizukami/cesm-coupling_patch

PIO sync for potentially better robustness for gnu/cray-mpich and a few cleanups
ESCOMP · Mar 7, 2024 · 74e4344 · 74e4344
2 parents 6115b93 + 59f0d53
commit 74e4344
Show file tree

Hide file tree

Showing 15 changed files with 125 additions and 65 deletions.
diff --git a/route/build/Makefile b/route/build/Makefile
@@ -98,6 +98,10 @@ ifeq "$(FC)" "gnu"
   ifeq "$(MODE)" "profile"
     FLAGS = -p -g -Wall -fmax-errors=0 -fbacktrace -fbounds-check -ffpe-trap=zero -ffree-line-length-none $(FLAGS_OMP) -cpp -DVERSION=\"$(VERSION)\" -DBRANCH=\"$(GITBRCH)\" -DHASH=\"$(GITHASH)\"
   endif
+  GNUVER = $(shell gcc -dumpversion | cut -f1 -d.)
+  ifeq ($(shell test $(GNUVER) -ge 10; echo $$?),0)
+    FLAGS += -fallow-argument-mismatch
+  endif
 endif
 
 ifeq "$(FC)" "intel"

diff --git a/route/build/README_cheyenne_build → route/build/README_NCAR_HPC_build b/route/build/README_cheyenne_build → route/build/README_NCAR_HPC_build
@@ -10,79 +10,96 @@
 #                   Currently these are for cheyenne only. Will need to update for Derecho HPC.
 #
 
+# NOTE: 02/19/2024. gcc with cray-mpich may cause the output error for unkonw reasons.  
+#                   gcc with openmpi/5.0.0 is stable 
+#
+
 # For gfortran with MPISERIAL  
-# NM 11/10/2023 this is currently failing to build
+# NM 02/19/2024 this is currently failing to build
 module purge
-module load gnu
+module load gcc
 module load netcdf
+module load ncarcompilers
 
 * tcsh or csh user:
 setenv BLDDIR `pwd`/../
 * bash user:
 export BLDDIR=`pwd`/../
-gmake FC=gnu FC_EXE=gfortran F_MASTER=$BLDDIR NCDF_PATH=$NETCDF EXE=route_runoff isMPISERIAL=yes isPIO=yes MODE=fast
+gmake FC=gnu FC_EXE=gfortran F_MASTER=$BLDDIR NCDF_PATH=$NETCDF isMPISERIAL=yes isPIO=yes MODE=fast EXE=route_runoff 
 
 # For gfortran with MPI and PIO2 libraries
 module purge
-module load gnu
-module load mpt
+module load cmake
+module load gcc
+module load cray-mpich (or openmpi/5.0.0)
+module load craype (if openmpi/5.0.0 is loaded)
 module load netcdf
+module load ncarcompilers
 
 * tcsh or csh user:
 setenv BLDDIR `pwd`/../
 * bash user:
 export BLDDIR=`pwd`/../
-gmake FC=gnu FC_EXE=mpif90 F_MASTER=$BLDDIR NCDF_PATH=$NETCDF EXE=route_runoff MODE=fast
+gmake FC=gnu FC_EXE=mpif90 F_MASTER=$BLDDIR NCDF_PATH=$NETCDF MODE=fast EXE=route_runoff
 
 # For gfortran with MPI, PNETCDF, GPTL (timing) and PIO2 libraries
 module purge
-module load gnu
-module load mpt
+module load cmake
+module load gcc
+module load cray-mpich (or openmpi/5.0.0)
+module load craype (if openmpi/5.0.0 is loaded)
 module load netcdf
-module load pnetcdf
+module load parallel-netcdf
+module load ncarcompilers
 
 * tcsh or csh user:
 setenv BLDDIR `pwd`/../
 * bash user:
 export BLDDIR=`pwd`/../
-gmake FC=gnu FC_EXE=mpif90 F_MASTER=$BLDDIR NCDF_PATH=$NETCDF EXE=route_runoff PNETCDF_PATH=$PNETCDF MODE=fast
+gmake FC=gnu FC_EXE=mpif90 F_MASTER=$BLDDIR NCDF_PATH=$NETCDF PNETCDF_PATH=$PNETCDF MODE=fast EXE=route_runoff
 
-# For gfortran just with MPI (SO WITHOUT ANY OF THE EXTERNAL LIBRARIES)
+# For intel with MPI and PIO2 libraries
 module purge
-module load gnu
-module load mpt
+module load cmake
+module load intel
+module load cray-mpich 
 module load netcdf
+module load ncarcompilers
 
 * tcsh or csh user:
 setenv BLDDIR `pwd`/../
 * bash user:
 export BLDDIR=`pwd`/../
-gmake FC=gnu FC_EXE=mpif90 F_MASTER=$BLDDIR NCDF_PATH=$NETCDF EXE=route_runoff MODE fast
+gmake FC=intel FC_EXE=mpif90 F_MASTER=$BLDDIR NCDF_PATH=$NETCDF MODE=fast EXE=route_runoff
 
-# For intel with MPI, PNETCDF and installed PIO2 libraries
-# EBK 9/30/2021 this is currently failing to build
+# For intel with MPI, PNETCDF, GPTL (timing) and PIO2 libraries
 module purge
+module load cmake
 module load intel
-module load mpt
-module load netcdf-mpi
-module load pnetcdf
-module load pio
+module load cray-mpich 
+module load netcdf
+module load parallel-netcdf
+module load ncarcompilers
 
 * tcsh or csh user:
 setenv BLDDIR `pwd`/../
-gmake FC=intel FC_EXE=mpif90 F_MASTER=$BLDDIR NCDF_PATH=$NETCDF EXE=route_runoff PNETCDF_PATH=$PNETCDF \
-      PIOLIBDIR=$PIO_LIBDIR PIOINCDIR=$PIO_INCDIR
+* bash user:
+export BLDDIR=`pwd`/../
+gmake FC=intel FC_EXE=mpif90 F_MASTER=$BLDDIR NCDF_PATH=$NETCDF PNETCDF_PATH=$PNETCDF MODE=fast EXE=route_runoff
 
-# For intel with MPI, PNETCDF and PIO2 libraries
+# For intel with MPI, PNETCDF and installed PIO2 libraries
+# EBK 9/30/2021 this is currently failing to build
 module purge
+module load cmake
 module load intel
-module load mpt
-module load netcdf
-module load pnetcdf
+module load cray-mpich 
+module load netcdf-mpi
+module load parallel-netcdf
+module load parallelio
+module load ncarcompilers
 
 * tcsh or csh user:
 setenv BLDDIR `pwd`/../
-* bash user:
-export BLDDIR=`pwd`/../
-gmake FC=intel FC_EXE=mpif90 F_MASTER=$BLDDIR NCDF_PATH=$NETCDF EXE=route_runoff PNETCDF_PATH=$PNETCDF MODE=fast
+gmake FC=intel FC_EXE=mpif90 F_MASTER=$BLDDIR NCDF_PATH=$NETCDF EXE=route_runoff PNETCDF_PATH=$PNETCDF \
+      PIOLIBDIR=$PIO_LIBDIR PIOINCDIR=$PIO_INCDIR
 
diff --git a/route/build/cpl/RtmMod.F90 b/route/build/cpl/RtmMod.F90
@@ -66,11 +66,11 @@ SUBROUTINE route_ini(rof_active,flood_active)
     USE globalData,          ONLY: NETOPO_main                 ! data structure: River Network topology (main proc, mainstem)
     USE globalData,          ONLY: RPARAM_trib                 ! data structure: River parameters (other procs, tributary)
     USE globalData,          ONLY: RPARAM_main                 ! data structure: River parameters (main proc, mainstem)
-    USE globalData,          ONLY: pio_netcdf_format
-    USE globalData,          ONLY: pio_typename
     USE globalData,          ONLY: pio_rearranger
     USE globalData,          ONLY: pio_root, pio_stride
     USE globalData,          ONLY: pioSystem
+    USE public_var,          ONLY: pio_netcdf_format
+    USE public_var,          ONLY: pio_typename
     USE init_model_data,     ONLY: init_ntopo_data             !
     USE init_model_data,     ONLY: init_state_data
     USE RtmTimeManager,      ONLY: init_time

diff --git a/route/build/lib/Makefile b/route/build/lib/Makefile
@@ -24,7 +24,7 @@ MPISERLIB    = $(MPISERLIBDIR)/libmpi-serial.a
 MPISERCONF   = $(MPISERLIBDIR)/Makefile.conf
 MPIINC_PATHLIST  = $(CPATH)
 INCLDIR      = -I.
-CMAKE_OPTS   = 
+CMAKE_OPTS   =
 
 all: $(CIMEROOT) $(MCTDIR) $(MPISERIALDIR) $(PIO2DIR)
 
@@ -61,10 +61,12 @@ ifeq "$(isMPISERIAL)" "yes"
   CMAKE_OPTS += -DCMAKE_Fortran_COMPILER=$(FC)
 else
   # Change colon delimited list to space with a -I in front of each
-  MPIINC   = $(subst :, -I,$(MPIINC_PATHLIST))
-  INCLDIR += -I$(MPIINC)
+  ifdef MPIINC_PATHLIST
+    MPIINC   = $(subst :, -I,$(MPIINC_PATHLIST))
+    INCLDIR += -I$(MPIINC)
+  endif
   CPPDEFS += -DHAVE_MPI
-  MPISERIAL = 
+  MPISERIAL =
   CMAKE_OPTS += -DCMAKE_Fortran_COMPILER=mpif90
 endif
 
@@ -85,9 +87,6 @@ CMAKE_OPTS += -Wno-dev -D CMAKE_Fortran_FLAGS:STRING="$(FLAGS) $(CPPDEFS) $(INCL
               -D USER_CMAKE_MODULE_PATH:LIST="$(CIMEROOT)/src/CMake;$(PIO2DIR)/cmake"
 CMAKE_OPTS += -D NetCDF_PATH:PATH=$(NCDF_PATH)
 
-
-
-
 ifdef PNETCDF_PATH
         CMAKE_OPTS += -D PnetCDF_PATH:STRING="$(PNETCDF_PATH)"
         CPPDEFS    += -D_PNETCDF
@@ -100,10 +99,10 @@ ifdef PIO_FILESYSTEM_HINTS
 endif
 CMAKE_OPTS += -D GENF90_PATH=$(PIO2DIR)/scripts
 
+MODEARGS :=
+
 ifeq "$(MODE)" "debug"
-   MODEARGS := --enable-debug
-else
-   MODEARGS :=
+   CMAKE_OPTS += -DCMAKE_BUILD_TYPE=Debug
 endif
 
 $(MPISERCONF):

diff --git a/route/build/src/globalData.f90 b/route/build/src/globalData.f90
@@ -1,5 +1,11 @@
 MODULE globalData
 
+  ! General rules
+  ! This module include data/variables that can be accessed from any other modules
+  ! User do not directly interface the data/variables
+  ! Data/values can be altered throughout the runtime
+  ! See public_var.f90 for difference
+
   USE pio
 
   USE public_var
@@ -133,8 +139,6 @@ MODULE globalData
   integer(i4b),                    public :: nThreads                           ! number of threads
   logical(lgt),                    public :: masterproc                         ! root logical. root processor => true, other => false
   logical(lgt),                    public :: multiProcs                         ! MPI multi-processors logical. => number of processors>1 true, other => false
-  character(len=strLen),           public :: pio_netcdf_format = "64bit_offset" ! netCDF format - use '64bit_offset' for PIO use or 'netCDF-4'
-  character(len=strLen),           public :: pio_typename      = "pnetcdf"      ! netcdf, pnetcdf, netcdf4c, or netcdf4p
   integer(i4b),                    public :: pio_numiotasks    = -99            ! Number of iotasks (ntasks/stride) - see PIO documentation for more information
   integer(i4b),                    public :: pio_rearranger    = 2              ! 0=>PIO_rearr_none 1=> PIO_rearr_box 2=> PIO_rearr_subset
   integer(i4b),                    public :: pio_root          = 1              ! PIO root

diff --git a/route/build/src/histVars_data.f90 b/route/build/src/histVars_data.f90
@@ -26,10 +26,10 @@ MODULE histVars_data
   USE globalData,        ONLY: pid, nNodes
   USE globalData,        ONLY: masterproc
   USE globalData,        ONLY: mpicom_route
-  USE globalData,        ONLY: pio_typename
   USE globalData,        ONLY: pioSystem
   USE globalData,        ONLY: ioDesc_hru_double
   USE globalData,        ONLY: ioDesc_hist_rch_double
+  USE public_var,        ONLY: pio_typename
   USE nr_utils,          ONLY: arth
   USE ncio_utils,        ONLY: get_nc
   USE pio_utils,         ONLY: file_desc_t
@@ -298,7 +298,7 @@ SUBROUTINE read_restart(this, restart_name, ierr, message)
       ! local variable
       character(len=strLen)              :: cmessage              ! error message from subroutines
       real(dp), allocatable              :: array_tmp(:)          ! temp array
-      integer(i4b)                       :: ixRoute, ix1, ix2     ! loop index
+      integer(i4b)                       :: ixRoute               ! loop index
       integer(i4b)                       :: ixFlow, ixVol         ! temporal method index
       logical(lgt)                       :: FileStatus            ! file open or close
       type(file_desc_t)                  :: pioFileDesc           ! pio file handle

diff --git a/route/build/src/historyFile.f90 b/route/build/src/historyFile.f90
@@ -11,8 +11,8 @@ MODULE historyFile
   USE globalData,        ONLY: masterproc
   USE globalData,        ONLY: mpicom_route
   USE globalData,        ONLY: pioSystem
-  USE globalData,        ONLY: pio_netcdf_format
-  USE globalData,        ONLY: pio_typename
+  USE public_var,        ONLY: pio_netcdf_format
+  USE public_var,        ONLY: pio_typename
   USE globalData,        ONLY: version
   USE globalData,        ONLY: gitBranch
   USE globalData,        ONLY: gitHash

diff --git a/route/build/src/init_model_data.f90 b/route/build/src/init_model_data.f90
@@ -308,7 +308,8 @@ SUBROUTINE update_time(finished, ierr, message)
    ierr=0; message='update_time/'
 
    if (simDatetime(1)>=endDatetime) then
-     call close_all()
+     call close_all(ierr, cmessage)
+     if(ierr/=0)then; message=trim(message)//trim(cmessage); return; endif
      finished=.true.;return
    endif
 
@@ -344,13 +345,14 @@ SUBROUTINE init_state_data(pid, nNodes, comm, ierr, message)
   USE public_var, ONLY: dt                     ! simulation time step (seconds)
   USE public_var, ONLY: restart_dir            ! directory containing output data
   USE public_var, ONLY: fname_state_in         ! name of state input file
+  USE public_var, ONLY: accumRunoff            ! routing method ID
   USE public_var, ONLY: impulseResponseFunc    ! IRF routing ID = 1
   USE public_var, ONLY: kinematicWaveTracking  ! KWT routing ID = 2
   USE public_var, ONLY: kinematicWave          ! KW routing ID = 3
   USE public_var, ONLY: muskingumCunge         ! MC routing ID = 4
   USE public_var, ONLY: diffusiveWave          ! DW routing ID = 5
   USE public_var, ONLY: is_lake_sim            ! logical if lakes are activated in simulation
-  USE globalData, ONLY: idxIRF, idxKWT, &
+  USE globalData, ONLY: idxSUM, idxIRF, idxKWT, &
                         idxKW, idxMC, idxDW
   USE globalData, ONLY: nRoutes                ! number of available routing methods
   USE globalData, ONLY: routeMethods           ! ID of active routing method
@@ -408,6 +410,12 @@ SUBROUTINE init_state_data(pid, nNodes, comm, ierr, message)
       RCHFLX_trib(:,:)%BASIN_QR(0)  = 0._dp
       RCHFLX_trib(:,:)%BASIN_QR(1)  = 0._dp
       nRch_root=nRch_mainstem+nTribOutlet+rch_per_proc(0)
+      if (onRoute(accumRunoff)) then
+        do ix = 1,nRch_root
+          RCHFLX_trib(iens,ix)%ROUTE(idxSUM)%REACH_VOL(0:1) = 0._dp
+          RCHFLX_trib(iens,ix)%ROUTE(idxSUM)%REACH_Q        = 0._dp
+        end do
+      end if
       if (onRoute(impulseResponseFunc)) then
         do ix = 1,nRch_root
           RCHFLX_trib(iens,ix)%ROUTE(idxIRF)%REACH_VOL(0:1) = 0._dp
@@ -474,6 +482,12 @@ SUBROUTINE init_state_data(pid, nNodes, comm, ierr, message)
         RCHFLX_trib(:,:)%BASIN_QI     = 0._dp
         RCHFLX_trib(:,:)%BASIN_QR(0)  = 0._dp
         RCHFLX_trib(:,:)%BASIN_QR(1)  = 0._dp
+        if (onRoute(accumRunoff)) then
+          do ix = 1, size(RCHFLX_trib(1,:))
+            RCHFLX_trib(iens,ix)%ROUTE(idxSUM)%REACH_VOL(0:1) = 0._dp
+            RCHFLX_trib(iens,ix)%ROUTE(idxSUM)%REACH_Q        = 0._dp
+          end do
+        end if
         if (onRoute(impulseResponseFunc)) then
           do ix = 1, size(RCHFLX_trib(1,:))
             RCHFLX_trib(iens,ix)%ROUTE(idxIRF)%REACH_VOL(0:1) = 0._dp

diff --git a/route/build/src/kwt_route.f90 b/route/build/src/kwt_route.f90
@@ -481,8 +481,6 @@ subroutine getusq_rch(IENS,JRCH,T0,T1,ixDesire,          & ! input
  !  T_EXIT(:): Vector of times flow particles are expected to exit reach JRCH
  !
  ! ----------------------------------------------------------------------------------------
- USE globalData, ONLY: LKTOPO           ! Lake topology
- USE globalData, ONLY: LAKFLX           ! Lake fluxes
 
  implicit none
  ! Argument variables

diff --git a/route/build/src/network_topo.f90 b/route/build/src/network_topo.f90
@@ -1002,7 +1002,7 @@ SUBROUTINE lakeInlet(nRch, structNTOPO, ierr, message)
    character(*)      , intent(out)               :: message           ! error message
    ! Local variables
    integer(i4b)                                  :: downIndex
-   integer(i4b)                                  :: iRch,jRch         ! loop index
+   integer(i4b)                                  :: iRch              ! loop index
 
    ierr=0; message='lakeInlet/'
 

diff --git a/route/build/src/public_var.f90 b/route/build/src/public_var.f90
@@ -1,7 +1,10 @@
 MODULE public_var
 
-  ! This module include variables that can be accessed from any other modules and values not altered
-  ! except that variables read from control file are populated.
+  ! General rules
+  ! This module include variables that can be accessed from any other modules
+  ! User can set values of the variables through control file (read_control.f90).
+  ! Values should not be altered during the runtime.
+  ! See globalData.f90 for difference
 
   USE nrtype
 
@@ -48,8 +51,10 @@ MODULE public_var
   integer(i4b),parameter,public   :: root=0                 ! root node id
 
   ! I/O related parameters
-  integer(i4b),          public   :: iulog=6                ! logical unit identifier
-  character(len=strLen), public   :: rpntfil='rpointer.rof' ! file name for local restart pointer file
+  integer(i4b),          public   :: iulog=6                            ! logical unit identifier
+  character(len=strLen), public   :: rpntfil='rpointer.rof'             ! file name for local restart pointer file
+  character(len=strLen), public   :: pio_netcdf_format="64bit_offset"   ! netCDF format - use '64bit_offset' for PIO use or 'netCDF-4'
+  character(len=strLen), public   :: pio_typename="pnetcdf"             ! netcdf, pnetcdf, netcdf4c, or netcdf4p
 
   ! ---------- named variables ----------------------------------------------------------------------
 

diff --git a/route/build/src/read_control.f90 b/route/build/src/read_control.f90
@@ -201,6 +201,9 @@ SUBROUTINE read_control(ctl_fname, err, message)
    case('<dname_gageSite>');       dname_gageSite  = trim(cData)                       ! dimension name for gauge site
    case('<dname_gageTime>');       dname_gageTime  = trim(cData)                       ! dimension name for time
    case('<strlen_gageSite>');      read(cData,*,iostat=io_error) strlen_gageSite       ! site name max character length
+   ! IO
+   case('<pio_netcdf_format>');    read(cData,*,iostat=io_error) pio_netcdf_format     ! netCDF format: 64bit_offset (default) for PIO use or netCDF-4
+   case('<pio_netcdf_type>');      read(cData,*,iostat=io_error) pio_typename          ! netCDF type: pnetcdf (default), netcdf, netcdf4c, or netcdf4p
    ! MISCELLANEOUS
    case('<debug>');                read(cData,*,iostat=io_error) debug                 ! print out detailed information throught the probram
    case('<seg_outlet>'   );        read(cData,*,iostat=io_error) idSegOut              ! desired outlet reach id (if -9999 --> route over the entire network)

diff --git a/route/build/src/standalone/model_setup.f90 b/route/build/src/standalone/model_setup.f90
@@ -52,7 +52,6 @@ SUBROUTINE init_data(pid,           & ! input: proc id
                       ierr, message)   ! output: error control
 
    USE public_var,          ONLY: continue_run        ! T-> append output in existing history files. F-> write output in new history file
-   USE globalData,          ONLY: mpicom_route
    USE globalData,          ONLY: version             ! mizuRoute version
    USE globalData,          ONLY: gitBranch           ! git branch
    USE globalData,          ONLY: gitHash             ! git commit hash
@@ -213,7 +212,6 @@ SUBROUTINE inFile_pop(dir_name,         & ! input: name of the directory of the
   integer(i4b)                                      :: nFile            ! number of nc files identified in the text file
   integer(i4b)                                      :: nTime            ! hard coded for now
   logical(lgt)                                      :: existAttr        ! attribute exist or not
-  type(datetime)                                    :: refDatetime      ! reference datetime for each file
   real(dp)                                          :: convTime2sec     ! time conversion to second
   character(len=strLen)                             :: infilename       ! input filename
   character(len=strLen),allocatable                 :: dataLines(:)     ! vector of lines of information (non-comment lines)

diff --git a/route/build/src/write_restart_pio.f90 b/route/build/src/write_restart_pio.f90
@@ -39,9 +39,9 @@ MODULE write_restart_pio
 USE globalData,        ONLY: pid, nNodes
 USE globalData,        ONLY: masterproc
 USE globalData,        ONLY: mpicom_route
-USE globalData,        ONLY: pio_netcdf_format
-USE globalData,        ONLY: pio_typename
 USE globalData,        ONLY: pioSystem
+USE public_var,        ONLY: pio_netcdf_format
+USE public_var,        ONLY: pio_typename
 
 USE globalData,        ONLY: runMode
 USE globalData,        ONLY: rfileout