Supported POINTS partitioning and tested coupled builds tests in CI

.github/build.oasis3-mct.ubuntu22.04

@@ -0,0 +1,81 @@
+#
+# Include file for OASIS3 Makefile for a Linux system using 
+# GNU and OPENMPI. This file is used by the eCLM CI Test 
+# to build the OASIS3-MCT library.
+#
+# This file is based from:
+# https://gitlab.com/cerfacs/oasis3-mct/-/blob/OASIS3-MCT_5.0/util/make_dir/header_examples/make.gfortran_openmpi_linux_openmp
+#
+###############################################################################
+#
+# CHAN  : communication technique used in OASIS3 (MPI1/MPI2)
+CHAN            = MPI1
+#
+# Paths for libraries, object files and binaries
+#
+# COUPLE        : path for oasis3 main directory
+COUPLE          = $(OASIS_ROOT)
+#
+# BUILD_DIR     : dir where the TopMakefileOasis3 is
+BUILD_DIR       = $(OASIS_ROOT)/util/make_dir
+#
+#
+# ARCHDIR       : directory created when compiling
+ARCHDIR         = $(HOME)/.local
+#
+# MPI library ((see the file /etc/modulefiles/mpi/openmpi-x86_64)
+MPIDIR      = /usr/lib/x86_64-linux-gnu/openmpi
+MPIBIN      = /usr/bin
+MPI_INCLUDE = $(MPIDIR)/include
+MPILIB      = -L$(MPIDIR)/lib -lmpi
+#
+# NETCDF library of the system
+NETCDF_INCLUDE1 = /usr/include
+# netcdf.mod and hdf5.mod
+NETCDF_INCLUDE2 = /usr/include
+NETCDF_LIBRARY  =  -L/usr/lib/x86_64-linux-gnu -lnetcdff -lnetcdf
+#
+# Compiling and other commands
+MAKE        = make
+F90         = $(MPIBIN)/mpif90 -I$(MPI_INCLUDE)
+F           = $(F90)
+f90         = $(F90)
+f           = $(F90)
+CC          = $(MPIBIN)/mpicc -I$(MPI_INCLUDE)
+LD          = $(MPIBIN)/mpif90 $(MPILIB)
+AR          = ar
+ARFLAGS     = -ruv
+#
+# CPP keys and compiler options
+#
+CPPDEF    = -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY
+#
+#
+#F90FLAGS_1  = -g -ffree-line-length-0 -fbounds-check -fopenmp
+F90FLAGS_1  = -ffree-line-length-none -fallow-argument-mismatch -fopenmp
+f90FLAGS_1  = $(F90FLAGS_1)
+FFLAGS_1    = $(F90FLAGS_1)
+fFLAGS_1    = $(F90FLAGS_1)
+CCFLAGS_1   =
+LDFLAGS     = -fopenmp
+#
+#
+###################
+#
+# Additional definitions that should not be changed
+#
+FLIBS           = $(NETCDF_LIBRARY)
+# BINDIR        : directory for executables
+BINDIR          = $(ARCHDIR)/bin
+# LIBBUILD      : contains a directory for each library
+LIBBUILD        = $(ARCHDIR)/build/lib
+# INCPSMILE     : includes all *o and *mod for each library
+INCPSMILE       = -I$(LIBBUILD)/psmile.$(CHAN) -I$(LIBBUILD)/scrip  -I$(LIBBUILD)/mct 
+
+F90FLAGS  = $(F90FLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2)
+f90FLAGS  = $(f90FLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2)
+FFLAGS    = $(FFLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2)
+fFLAGS    = $(fFLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2)
+CCFLAGS   = $(CCFLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2)
+#
+#############################################################################

.github/workflows/CI.yml

@@ -5,7 +5,34 @@ on: [push, pull_request]
 
 jobs:
   eclm_build_job:
+    name: ${{ matrix.config.name }}
     runs-on: ubuntu-22.04
+
+    strategy:
+      matrix:
+        config:
+        - {
+            name: "eCLM-Standalone",
+            use_oasis: "False",
+            coup_oas_icon: "False",
+            coup_oas_pfl: "False",
+            use_pdaf: "False"
+          }
+        - {
+            name: "eCLM-ParFlow-ICON",
+            use_oasis: "True",
+            coup_oas_icon: "True",
+            coup_oas_pfl: "True",
+            use_pdaf: "False"
+          }
+        - {
+            name: "eCLM-PDAF",
+            use_oasis: "False",
+            coup_oas_icon: "False",
+            coup_oas_pfl: "False",
+            use_pdaf: "True"
+          }
+
     env:
       CC: mpicc
       FC: mpifort
@@ -24,12 +51,41 @@ jobs:
       - name: Download MPI Fortran compiler
         run: sudo apt-get install gfortran openmpi-bin libopenmpi-dev
 
+      - if: matrix.config.use_oasis == 'True'
+        name: Cache OASIS3-MCT
+        uses: actions/cache@v4
+        id: cache-deps
+        env:
+          cache-name: cache-eCLM-dependencies
+        with:
+          path: "~/.local"
+          key: cache-${{ matrix.config.name }}
+
+      - if: matrix.config.use_oasis == 'True' && steps.cache-deps.outputs.cache-hit != 'true'
+        name: Install OASIS3-MCT
+        run: |
+          git clone https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct.git
+          cd oasis3-mct
+          export OASIS_ROOT=$(pwd)
+          echo "OASIS_ROOT=${OASIS_ROOT}"
+          echo "DEPENDENCIES_DIR=${DEPENDENCIES_DIR}"
+          cd util/make_dir
+          echo "include ${GITHUB_WORKSPACE}/.github/build.oasis3-mct.ubuntu22.04" > make.inc
+          cat make.inc
+          make realclean static-libs -f TopMakefileOasis3
+
       - name: Configure eCLM
         run: |
-          cmake -S src -B $BUILD_DIR                  \
-                -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR   \
-                -DCMAKE_C_COMPILER=$CC                \
-                -DCMAKE_Fortran_COMPILER=$FC
+          cmake -S src -B $BUILD_DIR                                \
+                -DCMAKE_BUILD_TYPE="RELEASE"                        \
+                -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR                 \
+                -DCMAKE_PREFIX_PATH="$HOME/.local"                  \
+                -DCMAKE_C_COMPILER=$CC                              \
+                -DCMAKE_Fortran_COMPILER=$FC                        \
+                -DUSE_OASIS=${{ matrix.config.use_oasis }}          \
+                -DCOUP_OAS_ICON=${{ matrix.config.coup_oas_icon }}  \
+                -DCOUP_OAS_PFL=${{ matrix.config.coup_oas_pfl }}    \
+                -DUSE_PDAF=${{ matrix.config.use_pdaf }}
 
       - name: Build eCLM
         run: cmake --build $BUILD_DIR

cmake/SetBuildOptions.cmake

@@ -36,12 +36,30 @@ elseif(COMPILER STREQUAL "Intel" OR COMPILER STREQUAL "IntelLLVM")
     set(CMAKE_C_FLAGS_RELEASE "-O2 -debug minimal")
     set(CMAKE_Fortran_FLAGS "-free -qno-opt-dynamic-align -ftz -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model source -qopenmp")
     set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
-    set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -debug minimal") 
+    set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -debug minimal")
 else()
     message(FATAL_ERROR "COMPILER='${COMPILER}' is not supported.")
 endif()
 
+if(COUP_OAS_ICON)
+    list(APPEND COUPLED_MODELS "ICON")
+endif()
+if(COUP_OAS_PFL)
+    list(APPEND COUPLED_MODELS "ParFlow")
+endif()
+if(USE_PDAF)
+    list(APPEND COUPLED_MODELS "PDAF")
+endif()
+
+if(COUPLED_MODELS)
+    list(PREPEND COUPLED_MODELS "eCLM")
+    list(JOIN COUPLED_MODELS "-" COUPLING_MODE)
+else()
+    set(COUPLING_MODE "Standalone")
+endif()
+
 message(STATUS " ******* ${CMAKE_PROJECT_NAME} build options ******* ")
 message(STATUS " Build type = '${CMAKE_BUILD_TYPE}'")
 message(STATUS " Compiler = '${COMPILER}'")
-message(STATUS " ********************************** ")
+message(STATUS " Coupling mode = '${COUPLING_MODE}'")
+message(STATUS " ********************************** ")

src/clm5/oasis3/oas_defineMod.F90

@@ -21,6 +21,7 @@ subroutine oas_definitions_init(bounds)
     integer, allocatable :: partition(:)      ! partition descriptor; input to oasis_def_partition
     integer              :: gcell_start       ! starting gridcell index
     integer              :: gcell_previous    ! gridcell index from previous loop iteration
+    integer              :: num_local_gcells  ! total number of gridcells for this process
     integer              :: k, g              ! array/loop indices
     integer              :: grid_id           ! id returned after call to oasis_def_partition
 #ifdef COUP_OAS_ICON
@@ -39,38 +40,56 @@ subroutine oas_definitions_init(bounds)
     ! -----------------------------------------------------------------
     ! ... Define partition
     ! -----------------------------------------------------------------
-    ! partition length = (# elements for partition info) + (max segments ORANGE partition) x (# elements per segment info)
-    !                  = 2 + 200*2 = 402
-    allocate(partition(402))
-    partition(:) = 0; k = 0
-
-    ! Use ORANGE partitioning scheme. This scheme defines an ensemble
-    ! of gridcell segments. See OASIS3-MCT User's guide for more info.
-    partition(1) = 3
-
-    ! Mark 1st segment
-    gcell_start = ldecomp%gdc2glo(bounds%begg)
-    partition(2) = 1
-    gcell_previous = gcell_start
-
-    ! Capture segments by detecting segment boundaries. A boundary is 
-    ! detected when the current and previous gridcells are not consecutive.
-    do g = bounds%begg+1, bounds%endg
-      if (ldecomp%gdc2glo(g) - gcell_previous /= 1) then
-        ! Previous segment complete; its partition params could now be defined
-        partition(3+k) = gcell_start - 1                  ! segment global offset (0-based)
-        partition(4+k) = gcell_previous - gcell_start + 1 ! segment length
-        k = k + 2
-        gcell_start  = ldecomp%gdc2glo(g) ! current gridcell marks the start of a new segment 
-        partition(2) = partition(2) + 1   ! increment number of segments (limited to 200 based from OASIS3-MCT User's guide)
-      end if
-      gcell_previous = ldecomp%gdc2glo(g)
-    enddo
-
-    ! Define partition params for last segment
-    partition(3+k) = gcell_start - 1
-    partition(4+k) = gcell_previous - gcell_start + 1
-    partition(2) = partition(2) + 1
+    if (ldomain%nj == 1) then
+      ! POINTS partitioning scheme
+      ! partition length = 2 + total number of gridpoints allocated to this MPI process
+      num_local_gcells = bounds%endg - bounds%begg + 1
+      allocate(partition(num_local_gcells + 2))
+      partition(:) = 0; k = 0
+
+      ! Use POINTS partitioning scheme. This partition is a list of global indices associated with each process.
+      partition(1) = 4
+      partition(2) = num_local_gcells
+
+      do g = bounds%begg, bounds%endg
+        partition(3+k) = ldecomp%gdc2glo(g)
+        k = k + 1
+      enddo
+    else
+      ! ORANGE partitioning scheme
+      ! partition length = (# elements for partition info) + (max segments ORANGE partition) x (# elements per segment info)
+      !                  = 2 + 200*2 = 402
+      allocate(partition(402))
+      partition(:) = 0; k = 0
+
+      ! Use ORANGE partitioning scheme. This scheme defines an ensemble
+      ! of gridcell segments. See OASIS3-MCT User's guide for more info.
+      partition(1) = 3
+
+      ! Mark 1st segment
+      gcell_start = ldecomp%gdc2glo(bounds%begg)
+      partition(2) = 1
+      gcell_previous = gcell_start
+
+      ! Capture segments by detecting segment boundaries. A boundary is 
+      ! detected when the current and previous gridcells are not consecutive.
+      do g = bounds%begg+1, bounds%endg
+        if (ldecomp%gdc2glo(g) - gcell_previous /= 1) then
+          ! Previous segment complete; its partition params could now be defined
+          partition(3+k) = gcell_start - 1                  ! segment global offset (0-based)
+          partition(4+k) = gcell_previous - gcell_start + 1 ! segment length
+          k = k + 2
+          gcell_start  = ldecomp%gdc2glo(g) ! current gridcell marks the start of a new segment 
+          partition(2) = partition(2) + 1   ! increment number of segments (limited to 200 based from OASIS3-MCT User's guide)
+        end if
+        gcell_previous = ldecomp%gdc2glo(g)
+      enddo
+
+      ! Define partition params for last segment
+      partition(3+k) = gcell_start - 1
+      partition(4+k) = gcell_previous - gcell_start + 1
+      partition(2) = partition(2) + 1
+    end if
 
     call oasis_def_partition(grid_id, partition, ierror, ldomain%ns)
     deallocate(partition)