diff --git a/.github/build.oasis3-mct.ubuntu22.04 b/.github/build.oasis3-mct.ubuntu22.04 new file mode 100644 index 0000000..8c7319f --- /dev/null +++ b/.github/build.oasis3-mct.ubuntu22.04 @@ -0,0 +1,81 @@ +# +# Include file for OASIS3 Makefile for a Linux system using +# GNU and OPENMPI. This file is used by the eCLM CI Test +# to build the OASIS3-MCT library. +# +# This file is based from: +# https://gitlab.com/cerfacs/oasis3-mct/-/blob/OASIS3-MCT_5.0/util/make_dir/header_examples/make.gfortran_openmpi_linux_openmp +# +############################################################################### +# +# CHAN : communication technique used in OASIS3 (MPI1/MPI2) +CHAN = MPI1 +# +# Paths for libraries, object files and binaries +# +# COUPLE : path for oasis3 main directory +COUPLE = $(OASIS_ROOT) +# +# BUILD_DIR : dir where the TopMakefileOasis3 is +BUILD_DIR = $(OASIS_ROOT)/util/make_dir +# +# +# ARCHDIR : directory created when compiling +ARCHDIR = $(HOME)/.local +# +# MPI library ((see the file /etc/modulefiles/mpi/openmpi-x86_64) +MPIDIR = /usr/lib/x86_64-linux-gnu/openmpi +MPIBIN = /usr/bin +MPI_INCLUDE = $(MPIDIR)/include +MPILIB = -L$(MPIDIR)/lib -lmpi +# +# NETCDF library of the system +NETCDF_INCLUDE1 = /usr/include +# netcdf.mod and hdf5.mod +NETCDF_INCLUDE2 = /usr/include +NETCDF_LIBRARY = -L/usr/lib/x86_64-linux-gnu -lnetcdff -lnetcdf +# +# Compiling and other commands +MAKE = make +F90 = $(MPIBIN)/mpif90 -I$(MPI_INCLUDE) +F = $(F90) +f90 = $(F90) +f = $(F90) +CC = $(MPIBIN)/mpicc -I$(MPI_INCLUDE) +LD = $(MPIBIN)/mpif90 $(MPILIB) +AR = ar +ARFLAGS = -ruv +# +# CPP keys and compiler options +# +CPPDEF = -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY +# +# +#F90FLAGS_1 = -g -ffree-line-length-0 -fbounds-check -fopenmp +F90FLAGS_1 = -ffree-line-length-none -fallow-argument-mismatch -fopenmp +f90FLAGS_1 = $(F90FLAGS_1) +FFLAGS_1 = $(F90FLAGS_1) +fFLAGS_1 = $(F90FLAGS_1) +CCFLAGS_1 = +LDFLAGS = -fopenmp +# +# +################### +# +# Additional definitions that should not be changed +# +FLIBS = $(NETCDF_LIBRARY) +# BINDIR : directory for executables +BINDIR = $(ARCHDIR)/bin +# LIBBUILD : contains a directory for each library +LIBBUILD = $(ARCHDIR)/build/lib +# INCPSMILE : includes all *o and *mod for each library +INCPSMILE = -I$(LIBBUILD)/psmile.$(CHAN) -I$(LIBBUILD)/scrip -I$(LIBBUILD)/mct + +F90FLAGS = $(F90FLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2) +f90FLAGS = $(f90FLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2) +FFLAGS = $(FFLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2) +fFLAGS = $(fFLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2) +CCFLAGS = $(CCFLAGS_1) $(CPPDEF) $(INCPSMILE) -I$(NETCDF_INCLUDE1) -I$(NETCDF_INCLUDE2) +# +############################################################################# \ No newline at end of file diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index df4de0e..fbacc61 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -5,7 +5,34 @@ on: [push, pull_request] jobs: eclm_build_job: + name: ${{ matrix.config.name }} runs-on: ubuntu-22.04 + + strategy: + matrix: + config: + - { + name: "eCLM-Standalone", + use_oasis: "False", + coup_oas_icon: "False", + coup_oas_pfl: "False", + use_pdaf: "False" + } + - { + name: "eCLM-ParFlow-ICON", + use_oasis: "True", + coup_oas_icon: "True", + coup_oas_pfl: "True", + use_pdaf: "False" + } + - { + name: "eCLM-PDAF", + use_oasis: "False", + coup_oas_icon: "False", + coup_oas_pfl: "False", + use_pdaf: "True" + } + env: CC: mpicc FC: mpifort @@ -24,12 +51,41 @@ jobs: - name: Download MPI Fortran compiler run: sudo apt-get install gfortran openmpi-bin libopenmpi-dev + - if: matrix.config.use_oasis == 'True' + name: Cache OASIS3-MCT + uses: actions/cache@v4 + id: cache-deps + env: + cache-name: cache-eCLM-dependencies + with: + path: "~/.local" + key: cache-${{ matrix.config.name }} + + - if: matrix.config.use_oasis == 'True' && steps.cache-deps.outputs.cache-hit != 'true' + name: Install OASIS3-MCT + run: | + git clone https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct.git + cd oasis3-mct + export OASIS_ROOT=$(pwd) + echo "OASIS_ROOT=${OASIS_ROOT}" + echo "DEPENDENCIES_DIR=${DEPENDENCIES_DIR}" + cd util/make_dir + echo "include ${GITHUB_WORKSPACE}/.github/build.oasis3-mct.ubuntu22.04" > make.inc + cat make.inc + make realclean static-libs -f TopMakefileOasis3 + - name: Configure eCLM run: | - cmake -S src -B $BUILD_DIR \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ - -DCMAKE_C_COMPILER=$CC \ - -DCMAKE_Fortran_COMPILER=$FC + cmake -S src -B $BUILD_DIR \ + -DCMAKE_BUILD_TYPE="RELEASE" \ + -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ + -DCMAKE_PREFIX_PATH="$HOME/.local" \ + -DCMAKE_C_COMPILER=$CC \ + -DCMAKE_Fortran_COMPILER=$FC \ + -DUSE_OASIS=${{ matrix.config.use_oasis }} \ + -DCOUP_OAS_ICON=${{ matrix.config.coup_oas_icon }} \ + -DCOUP_OAS_PFL=${{ matrix.config.coup_oas_pfl }} \ + -DUSE_PDAF=${{ matrix.config.use_pdaf }} - name: Build eCLM run: cmake --build $BUILD_DIR diff --git a/cmake/SetBuildOptions.cmake b/cmake/SetBuildOptions.cmake index 57b89c0..00f29b7 100644 --- a/cmake/SetBuildOptions.cmake +++ b/cmake/SetBuildOptions.cmake @@ -36,12 +36,30 @@ elseif(COMPILER STREQUAL "Intel" OR COMPILER STREQUAL "IntelLLVM") set(CMAKE_C_FLAGS_RELEASE "-O2 -debug minimal") set(CMAKE_Fortran_FLAGS "-free -qno-opt-dynamic-align -ftz -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model source -qopenmp") set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created") - set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -debug minimal") + set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -debug minimal") else() message(FATAL_ERROR "COMPILER='${COMPILER}' is not supported.") endif() +if(COUP_OAS_ICON) + list(APPEND COUPLED_MODELS "ICON") +endif() +if(COUP_OAS_PFL) + list(APPEND COUPLED_MODELS "ParFlow") +endif() +if(USE_PDAF) + list(APPEND COUPLED_MODELS "PDAF") +endif() + +if(COUPLED_MODELS) + list(PREPEND COUPLED_MODELS "eCLM") + list(JOIN COUPLED_MODELS "-" COUPLING_MODE) +else() + set(COUPLING_MODE "Standalone") +endif() + message(STATUS " ******* ${CMAKE_PROJECT_NAME} build options ******* ") message(STATUS " Build type = '${CMAKE_BUILD_TYPE}'") message(STATUS " Compiler = '${COMPILER}'") -message(STATUS " ********************************** ") +message(STATUS " Coupling mode = '${COUPLING_MODE}'") +message(STATUS " ********************************** ") \ No newline at end of file diff --git a/src/clm5/oasis3/oas_defineMod.F90 b/src/clm5/oasis3/oas_defineMod.F90 index c58b61f..cdae0fc 100644 --- a/src/clm5/oasis3/oas_defineMod.F90 +++ b/src/clm5/oasis3/oas_defineMod.F90 @@ -21,6 +21,7 @@ subroutine oas_definitions_init(bounds) integer, allocatable :: partition(:) ! partition descriptor; input to oasis_def_partition integer :: gcell_start ! starting gridcell index integer :: gcell_previous ! gridcell index from previous loop iteration + integer :: num_local_gcells ! total number of gridcells for this process integer :: k, g ! array/loop indices integer :: grid_id ! id returned after call to oasis_def_partition #ifdef COUP_OAS_ICON @@ -39,38 +40,56 @@ subroutine oas_definitions_init(bounds) ! ----------------------------------------------------------------- ! ... Define partition ! ----------------------------------------------------------------- - ! partition length = (# elements for partition info) + (max segments ORANGE partition) x (# elements per segment info) - ! = 2 + 200*2 = 402 - allocate(partition(402)) - partition(:) = 0; k = 0 - - ! Use ORANGE partitioning scheme. This scheme defines an ensemble - ! of gridcell segments. See OASIS3-MCT User's guide for more info. - partition(1) = 3 - - ! Mark 1st segment - gcell_start = ldecomp%gdc2glo(bounds%begg) - partition(2) = 1 - gcell_previous = gcell_start - - ! Capture segments by detecting segment boundaries. A boundary is - ! detected when the current and previous gridcells are not consecutive. - do g = bounds%begg+1, bounds%endg - if (ldecomp%gdc2glo(g) - gcell_previous /= 1) then - ! Previous segment complete; its partition params could now be defined - partition(3+k) = gcell_start - 1 ! segment global offset (0-based) - partition(4+k) = gcell_previous - gcell_start + 1 ! segment length - k = k + 2 - gcell_start = ldecomp%gdc2glo(g) ! current gridcell marks the start of a new segment - partition(2) = partition(2) + 1 ! increment number of segments (limited to 200 based from OASIS3-MCT User's guide) - end if - gcell_previous = ldecomp%gdc2glo(g) - enddo - - ! Define partition params for last segment - partition(3+k) = gcell_start - 1 - partition(4+k) = gcell_previous - gcell_start + 1 - partition(2) = partition(2) + 1 + if (ldomain%nj == 1) then + ! POINTS partitioning scheme + ! partition length = 2 + total number of gridpoints allocated to this MPI process + num_local_gcells = bounds%endg - bounds%begg + 1 + allocate(partition(num_local_gcells + 2)) + partition(:) = 0; k = 0 + + ! Use POINTS partitioning scheme. This partition is a list of global indices associated with each process. + partition(1) = 4 + partition(2) = num_local_gcells + + do g = bounds%begg, bounds%endg + partition(3+k) = ldecomp%gdc2glo(g) + k = k + 1 + enddo + else + ! ORANGE partitioning scheme + ! partition length = (# elements for partition info) + (max segments ORANGE partition) x (# elements per segment info) + ! = 2 + 200*2 = 402 + allocate(partition(402)) + partition(:) = 0; k = 0 + + ! Use ORANGE partitioning scheme. This scheme defines an ensemble + ! of gridcell segments. See OASIS3-MCT User's guide for more info. + partition(1) = 3 + + ! Mark 1st segment + gcell_start = ldecomp%gdc2glo(bounds%begg) + partition(2) = 1 + gcell_previous = gcell_start + + ! Capture segments by detecting segment boundaries. A boundary is + ! detected when the current and previous gridcells are not consecutive. + do g = bounds%begg+1, bounds%endg + if (ldecomp%gdc2glo(g) - gcell_previous /= 1) then + ! Previous segment complete; its partition params could now be defined + partition(3+k) = gcell_start - 1 ! segment global offset (0-based) + partition(4+k) = gcell_previous - gcell_start + 1 ! segment length + k = k + 2 + gcell_start = ldecomp%gdc2glo(g) ! current gridcell marks the start of a new segment + partition(2) = partition(2) + 1 ! increment number of segments (limited to 200 based from OASIS3-MCT User's guide) + end if + gcell_previous = ldecomp%gdc2glo(g) + enddo + + ! Define partition params for last segment + partition(3+k) = gcell_start - 1 + partition(4+k) = gcell_previous - gcell_start + 1 + partition(2) = partition(2) + 1 + end if call oasis_def_partition(grid_id, partition, ierror, ldomain%ns) deallocate(partition)