From 1a7fc7faa0b96152d79c48bc334aff2e1b5b292d Mon Sep 17 00:00:00 2001 From: Helen Kershaw Date: Thu, 31 Oct 2024 10:20:46 -0400 Subject: [PATCH 01/15] chore: removed unused routine from normal_distribution_mod.f90 fixes #736 --- .../assimilation/normal_distribution_mod.f90 | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/assimilation_code/modules/assimilation/normal_distribution_mod.f90 b/assimilation_code/modules/assimilation/normal_distribution_mod.f90 index 6b0656c62d..b3a7c33290 100644 --- a/assimilation_code/modules/assimilation/normal_distribution_mod.f90 +++ b/assimilation_code/modules/assimilation/normal_distribution_mod.f90 @@ -493,51 +493,6 @@ subroutine set_normal_params_from_ens(ens, num, p) end subroutine set_normal_params_from_ens -!------------------------------------------------------------------------ -subroutine inv_cdf_quadrature_like(quantiles, ens, likelihood, ens_size, cdf, p, x_out) - -interface - function cdf(x, p) - use types_mod, only : r8 - use distribution_params_mod, only : distribution_params_type - real(r8) :: cdf - real(r8), intent(in) :: x - type(distribution_params_type), intent(in) :: p - end function -end interface - -integer, intent(in) :: ens_size -real(r8), intent(in) :: quantiles(ens_size) -real(r8), intent(in) :: ens(ens_size) -real(r8), intent(in) :: likelihood(ens_size) -type(distribution_params_type), intent(in) :: p -real(r8), intent(out) :: x_out(ens_size) - -integer :: i -real(r8) :: quad_like(ens_size + 1), q_ens(ens_size + 1) - -! Assume that the quantiles and the corresponding ens are sorted - -! Get the likelihood for each of the ens_size + 1 intervals -do i = 2, ens_size - quad_like(i) = (likelihood(i - 1) + likelihood(i)) / 2.0_r8 -end do -quad_like(1) = likelihood(1) -quad_like(ens_size + 1) = likelihood(ens_size) - -! Compute the quantiles at the ensemble boundaries for the posterior -q_ens(1) = quad_like(1) * quantiles(1) -do i = 2, ens_size - q_ens(i) = q_ens(i - 1) + quad_like(i) * (quantiles(i) - quantiles(i - 1)) -end do -q_ens(ens_size + 1) = q_ens(ens_size) + & - quad_like(ens_size + 1) * (1.0_r8 - quantiles(ens_size)) - -! Normalize so that this is a posterior cdf -q_ens = q_ens / q_ens(ens_size + 1) - -end subroutine inv_cdf_quadrature_like - !------------------------------------------------------------------------ end module normal_distribution_mod From a43c2cf9f4d34061a1760959268d66002ed13e29 Mon Sep 17 00:00:00 2001 From: Marlee Smith Date: Tue, 5 Nov 2024 09:50:26 -0700 Subject: [PATCH 02/15] Adding info about getting restarts from the gdex; removing info about script to copy restarts --- models/POP/readme.rst | 54 ++++--------------------------------------- 1 file changed, 4 insertions(+), 50 deletions(-) diff --git a/models/POP/readme.rst b/models/POP/readme.rst index 6ece91e7fa..e0fd9b4f34 100644 --- a/models/POP/readme.rst +++ b/models/POP/readme.rst @@ -83,6 +83,8 @@ Summary To use DART and CESM POP2 on NSF NCAR's supercomputer, you will need to complete the following steps. +#. Download an intial ensemble of POP2 restart files from the `NSF NCAR Geoscience + Data Exchange `_ #. Configure the scripts for your specific experiment by editing ``DART_params.csh``. #. Stage your initial ensemble using ``copy_POP_JRA_restarts.py``. @@ -114,53 +116,6 @@ in subdirectories that correspond releases of CESM. For example: contains scripts that should be used with CESM releases 2.1.0-2.1.3. -copy_POP_JRA_restarts.py -~~~~~~~~~~~~~~~~~~~~~~~~ - -This script stages an intial ensemble of POP2 restart files by copying files -from a prior experiment run by *Who Kim*. Thanks Who! - -These restart files can be used as an initial ensemble of model -states. The files are kept in a directory on GLADE that is owned by the Climate -and Global Dynamics (CGD) Ocean Section: - -.. code-block:: - - /glade/campaign/cgd/oce/people/whokim/csm/g210.G_JRA.v14.gx1v7.01 - -Unless you're already a member of the CGD Ocean Section, you must be granted -access to this directory by CISL. Use the `Service Desk -`_ to request permission. If -you're unable to get permission, contact DAReS staff for assistance by emailing -dart@ucar.edu. - -Filepaths beginning with ``/glade/campaign/*`` can't be accessed from NSF NCAR's -supercomputer nodes. You must log on to NSF NCAR's data visualization computer to -copy files from ``/glade/campaign/*``. - -This python script was created by *Dan Amrhein*. Thanks Dan! - -+-------------------------------+-----------------------------------------------------------+ -| Script name | Description | -+===============================+===========================================================+ -| ``copy_POP_JRA_restarts.py`` | This script copies restart files from the | -| | g210.G_JRA.v14.gx1v7.01 experiment that are saved in | -| | campaign storage. You must be granted access to the CGD | -| | Ocean Section campaign storage directory and be logged on | -| | to NSF NCAR's data visualization computer in order to run | -| | this script. The assignment of the ``stagedir`` variable | -| | in this script should match the assignment of the | -| | ``stagedir`` variable in ``DART_params.csh``. | -+-------------------------------+-----------------------------------------------------------+ - -In order to use this script, log in to NSF NCAR's data visualization computer and -use python to run the script. For example: - -.. code-block:: - - $ cd DART/models/POP/shell_scripts/cesm2_1 - $ python copy_POP_JRA_restarts.py - DART_params.csh ~~~~~~~~~~~~~~~ @@ -175,9 +130,8 @@ It is run by the setup scripts. | | that you need to set in order to build and run cases. You | | | must read this file carefully and configure the variables | | | to match your needs. The assignment of the ``stagedir`` | -| | variable in this script should match the assignment of | -| | the ``stagedir`` variable in | -| | ``copy_POP_JRA_restarts.py``. | +| | variable in this script should match the directory path | +| | where the restarts from the GDEX were downloaded/stored. | +---------------------+-----------------------------------------------------------+ Setup scripts From 8b62eac688dca6ed25c619fa5036b2086fe44471 Mon Sep 17 00:00:00 2001 From: Marlee Smith Date: Tue, 5 Nov 2024 09:59:37 -0700 Subject: [PATCH 03/15] Adding info about downloading and using SourceMods --- models/POP/readme.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/models/POP/readme.rst b/models/POP/readme.rst index e0fd9b4f34..690122fdd7 100644 --- a/models/POP/readme.rst +++ b/models/POP/readme.rst @@ -134,6 +134,13 @@ It is run by the setup scripts. | | where the restarts from the GDEX were downloaded/stored. | +---------------------+-----------------------------------------------------------+ +One set of variables to make sure are set correctly in this script are related to +the SourceMods enable POP to recompute the barotropic velocity to prevent the +barotropic solver from crashing. You can find and download the SourceMods available +for POP-DART `here `_. Ensure that the +environment variable use_SourceMods is set to TRUE and the environment variable +SourceModDir matches the directory path where the SourceMods were downloaded/stored. + Setup scripts ~~~~~~~~~~~~~ From 74a241d6e28b8cad18e020e294016e902167bc5a Mon Sep 17 00:00:00 2001 From: Marlee Smith Date: Tue, 5 Nov 2024 10:15:29 -0700 Subject: [PATCH 04/15] Adding doc reference --- models/POP/readme.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/POP/readme.rst b/models/POP/readme.rst index 690122fdd7..614b30f461 100644 --- a/models/POP/readme.rst +++ b/models/POP/readme.rst @@ -1,3 +1,5 @@ +.. _POP: + POP === From 7648e5a88fb1ba32b38a5ab389f1dc76f929463a Mon Sep 17 00:00:00 2001 From: Marlee Smith Date: Tue, 5 Nov 2024 11:38:21 -0700 Subject: [PATCH 05/15] Removing additional bit about copy_POP_JRA_restarts.py --- models/POP/readme.rst | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/models/POP/readme.rst b/models/POP/readme.rst index 614b30f461..f050f6082e 100644 --- a/models/POP/readme.rst +++ b/models/POP/readme.rst @@ -89,7 +89,6 @@ the following steps. Data Exchange `_ #. Configure the scripts for your specific experiment by editing ``DART_params.csh``. -#. Stage your initial ensemble using ``copy_POP_JRA_restarts.py``. #. Run the appropriate DART setup script to create and build the CESM case. If the DART setup script runs to completion, it will print instructions to the @@ -137,7 +136,7 @@ It is run by the setup scripts. +---------------------+-----------------------------------------------------------+ One set of variables to make sure are set correctly in this script are related to -the SourceMods enable POP to recompute the barotropic velocity to prevent the +the SourceMods, which enable POP to recompute the barotropic velocity to prevent the barotropic solver from crashing. You can find and download the SourceMods available for POP-DART `here `_. Ensure that the environment variable use_SourceMods is set to TRUE and the environment variable @@ -213,19 +212,6 @@ integration. Other files needed for assimilation ----------------------------------- -Creating an initial ensemble -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Karspeck et al. (2013) [3]_ find that an ensemble of 1 January model states -selected from a multi-decade free-running integration of POP2 can be used as an -initial ensemble. - -If you have access to CGD's Ocean Section directory on ``/glade/campaign`` you -can use the `copy_POP_JRA_restarts.py`_ script to stage a collection of POP -restart files from Who Kim's multi-century ``g210.G_JRA.v14.gx1v7.01`` -experiment to serve as an initial ensemble. This experiment uses the JRA-55 -dataset for atmospheric forcing (Tsujino et al. 2018 [4]_). - Observation sequence files ~~~~~~~~~~~~~~~~~~~~~~~~~~ From a83943e71c9de63cb50b4b1b86e0d0ecb93ec931 Mon Sep 17 00:00:00 2001 From: Marlee Smith Date: Tue, 5 Nov 2024 15:39:34 -0700 Subject: [PATCH 06/15] Removing an unneeded subroutine broadcast_minmax from the mpi_utilies files (mpi_utilities_mod.f90, mpif08_utilities_mod.f90, null_mpi_utilities_mod.f90); reverting to the original name all_reduce_min_max as this is accurate to the content of the subroutine --- .../modules/utilities/mpi_utilities_mod.f90 | 29 +++---------------- .../utilities/mpif08_utilities_mod.f90 | 29 +++---------------- .../utilities/null_mpi_utilities_mod.f90 | 24 +++------------ 3 files changed, 12 insertions(+), 70 deletions(-) diff --git a/assimilation_code/modules/utilities/mpi_utilities_mod.f90 b/assimilation_code/modules/utilities/mpi_utilities_mod.f90 index 90797018e8..d9bd640a38 100644 --- a/assimilation_code/modules/utilities/mpi_utilities_mod.f90 +++ b/assimilation_code/modules/utilities/mpi_utilities_mod.f90 @@ -12,11 +12,6 @@ !> and allows programs to swap in the null version to compile the !> same source files into a serial program. !> -!> The names of these routines were intentionally picked to be -!> more descriptive to someone who doesn't the MPI interfaces. -!> e.g. MPI_AllReduce() may not immediately tell a user what -!> it does, but broadcast_minmax() is hopefully more helpful. -!> !> If you add any routines or change any arguments in this file !> you must make the same changes in the null version. These two !> modules have the same module name and must have identical @@ -138,9 +133,8 @@ module mpi_utilities_mod task_sync, array_broadcast, send_to, receive_from, iam_task0, & broadcast_send, broadcast_recv, shell_execute, sleep_seconds, & sum_across_tasks, get_dart_mpi_comm, datasize, send_minmax_to, & - get_from_fwd, get_from_mean, broadcast_minmax, broadcast_flag, & - start_mpi_timer, read_mpi_timer, send_sum_to, get_global_max, & - all_reduce_min_max ! deprecated, replace by broadcast_minmax + get_from_fwd, get_from_mean, broadcast_flag, start_mpi_timer, & + read_mpi_timer, send_sum_to, get_global_max, all_reduce_min_max character(len=*), parameter :: source = 'mpi_utilities_mod.f90' @@ -1467,26 +1461,11 @@ end subroutine send_minmax_to !----------------------------------------------------------------------------- -!> cover routine which is deprecated. when all user code replaces this -!> with broadcast_minmax(), remove this. - -subroutine all_reduce_min_max(min_var, max_var, num_elements) - -integer, intent(in) :: num_elements -real(r8), intent(inout) :: min_var(num_elements) -real(r8), intent(inout) :: max_var(num_elements) - -call broadcast_minmax(min_var, max_var, num_elements) - -end subroutine all_reduce_min_max - -!----------------------------------------------------------------------------- - !> Find min and max of each element of an array, put the result on every task. !> Overwrites arrays min_var, max_var with the minimum and maximum for each !> element across all tasks. -subroutine broadcast_minmax(min_var, max_var, num_elements) +subroutine all_reduce_min_max(min_var, max_var, num_elements) integer, intent(in) :: num_elements real(r8), intent(inout) :: min_var(num_elements) @@ -1502,7 +1481,7 @@ subroutine broadcast_minmax(min_var, max_var, num_elements) call mpi_allreduce(MPI_IN_PLACE, min_var, num_elements, datasize, MPI_MIN, get_dart_mpi_comm(), errcode) call mpi_allreduce(MPI_IN_PLACE, max_var, num_elements, datasize, MPI_MAX, get_dart_mpi_comm(), errcode) -end subroutine broadcast_minmax +end subroutine all_reduce_min_max !----------------------------------------------------------------------------- !> Broadcast logical diff --git a/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 b/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 index dee8c618d1..51bc507fe0 100644 --- a/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 +++ b/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 @@ -12,11 +12,6 @@ !> and allows programs to swap in the null version to compile the !> same source files into a serial program. !> -!> The names of these routines were intentionally picked to be -!> more descriptive to someone who doesn't the MPI interfaces. -!> e.g. MPI_AllReduce() may not immediately tell a user what -!> it does, but broadcast_minmax() is hopefully more helpful. -!> !> If you add any routines or change any arguments in this file !> you must make the same changes in the null version. These two !> modules have the same module name and must have identical @@ -138,9 +133,8 @@ module mpi_utilities_mod task_sync, array_broadcast, send_to, receive_from, iam_task0, & broadcast_send, broadcast_recv, shell_execute, sleep_seconds, & sum_across_tasks, get_dart_mpi_comm, datasize, send_minmax_to, & - get_from_fwd, get_from_mean, broadcast_minmax, broadcast_flag, & - start_mpi_timer, read_mpi_timer, send_sum_to, get_global_max, & - all_reduce_min_max ! deprecated, replace by broadcast_minmax + get_from_fwd, get_from_mean, broadcast_flag, start_mpi_timer, & + read_mpi_timer, send_sum_to, get_global_max, all_reduce_min_max character(len=*), parameter :: source = 'mpi_utilities_mod.f90' @@ -1467,26 +1461,11 @@ end subroutine send_minmax_to !----------------------------------------------------------------------------- -!> cover routine which is deprecated. when all user code replaces this -!> with broadcast_minmax(), remove this. - -subroutine all_reduce_min_max(min_var, max_var, num_elements) - -integer, intent(in) :: num_elements -real(r8), intent(inout) :: min_var(num_elements) -real(r8), intent(inout) :: max_var(num_elements) - -call broadcast_minmax(min_var, max_var, num_elements) - -end subroutine all_reduce_min_max - -!----------------------------------------------------------------------------- - !> Find min and max of each element of an array, put the result on every task. !> Overwrites arrays min_var, max_var with the minimum and maximum for each !> element across all tasks. -subroutine broadcast_minmax(min_var, max_var, num_elements) +subroutine all_reduce_min_max(min_var, max_var, num_elements) integer, intent(in) :: num_elements real(r8), intent(inout) :: min_var(num_elements) @@ -1502,7 +1481,7 @@ subroutine broadcast_minmax(min_var, max_var, num_elements) call mpi_allreduce(MPI_IN_PLACE, min_var, num_elements, datasize, MPI_MIN, get_dart_mpi_comm(), errcode) call mpi_allreduce(MPI_IN_PLACE, max_var, num_elements, datasize, MPI_MAX, get_dart_mpi_comm(), errcode) -end subroutine broadcast_minmax +end subroutine all_reduce_min_max !----------------------------------------------------------------------------- !> Broadcast logical diff --git a/assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 b/assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 index cc729c23aa..f2d38eb057 100644 --- a/assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 +++ b/assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 @@ -109,9 +109,8 @@ module mpi_utilities_mod task_sync, array_broadcast, send_to, receive_from, iam_task0, & broadcast_send, broadcast_recv, shell_execute, sleep_seconds, & sum_across_tasks, get_dart_mpi_comm, datasize, send_minmax_to, & - get_from_fwd, get_from_mean, broadcast_minmax, broadcast_flag, & - start_mpi_timer, read_mpi_timer, send_sum_to, get_global_max, & - all_reduce_min_max ! deprecated, replace by broadcast_minmax + get_from_fwd, get_from_mean, broadcast_flag, start_mpi_timer, & + read_mpi_timer, send_sum_to, get_global_max, all_reduce_min_max character(len=*), parameter :: source = 'null_mpi_utilities_mod.f90' @@ -432,32 +431,17 @@ end subroutine send_minmax_to !----------------------------------------------------------------------------- -!> cover routine which is deprecated. when all user code replaces this -!> with broadcast_minmax(), remove this. - -subroutine all_reduce_min_max(min_var, max_var, num_elements) - -integer, intent(in) :: num_elements -real(r8), intent(inout) :: min_var(num_elements) -real(r8), intent(inout) :: max_var(num_elements) - -call broadcast_minmax(min_var, max_var, num_elements) - -end subroutine all_reduce_min_max - -!----------------------------------------------------------------------------- - !> Find min and max of each element of an array across tasks, put the result on every task. !> For this null_mpi_version min_var and max_var are unchanged because there is !> only 1 task. -subroutine broadcast_minmax(min_var, max_var, num_elements) +subroutine all_reduce_min_max(min_var, max_var, num_elements) integer, intent(in) :: num_elements real(r8), intent(inout) :: min_var(num_elements) real(r8), intent(inout) :: max_var(num_elements) -end subroutine broadcast_minmax +end subroutine all_reduce_min_max !----------------------------------------------------------------------------- From 69ff3c9ee3a946c163f0b2de58cd8c6e86104770 Mon Sep 17 00:00:00 2001 From: Marlee Smith Date: Tue, 5 Nov 2024 15:43:57 -0700 Subject: [PATCH 07/15] Changing calls to broadcast_minmax to call all_reduce_min_max in FESOM and mpas_atm model_mods --- models/FESOM/model_mod.f90 | 4 ++-- models/mpas_atm/model_mod.f90 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/models/FESOM/model_mod.f90 b/models/FESOM/model_mod.f90 index 3233cf1492..b6387431f6 100644 --- a/models/FESOM/model_mod.f90 +++ b/models/FESOM/model_mod.f90 @@ -60,7 +60,7 @@ module model_mod use obs_kind_mod, only : get_index_for_quantity -use mpi_utilities_mod, only: my_task_id, broadcast_minmax, task_count +use mpi_utilities_mod, only: my_task_id, all_reduce_min_max, task_count use fesom_modules, only: read_node, read_aux3, read_depth, read_namelist, & nCells => myDim_nod2D, & ! number of surface locations @@ -828,7 +828,7 @@ subroutine pert_model_copies(ens_handle, ens_size, pert_amp, interf_provided) enddo ! get global min/max for each variable -call broadcast_minmax(min_var, max_var, num_variables) +call all_reduce_min_max(min_var, max_var, num_variables) deallocate(within_range) call init_random_seq(random_seq, my_task_id()+1) diff --git a/models/mpas_atm/model_mod.f90 b/models/mpas_atm/model_mod.f90 index 29282f7a8a..839921d457 100644 --- a/models/mpas_atm/model_mod.f90 +++ b/models/mpas_atm/model_mod.f90 @@ -104,7 +104,7 @@ module model_mod QTY_SURFACE_TYPE, & ! for rttov QTY_CLOUD_FRACTION ! for rttov -use mpi_utilities_mod, only: my_task_id, broadcast_minmax +use mpi_utilities_mod, only: my_task_id, all_reduce_min_max use random_seq_mod, only: random_seq_type, init_random_seq, random_gaussian @@ -1812,7 +1812,7 @@ subroutine pert_model_copies(ens_handle, ens_size, pert_amp, interf_provided) enddo ! get global min/max for each variable -call broadcast_minmax(min_var, max_var, num_variables) +call all_reduce_min_max(min_var, max_var, num_variables) deallocate(within_range) call init_random_seq(random_seq, my_task_id()+1) From 55eaccd61fca194464f196c629465b71433bfd59 Mon Sep 17 00:00:00 2001 From: Marlee Smith Date: Tue, 5 Nov 2024 16:16:28 -0700 Subject: [PATCH 08/15] Removing loops in time_manager_mod that are not needed --- .../modules/utilities/time_manager_mod.f90 | 92 ++++++++----------- 1 file changed, 37 insertions(+), 55 deletions(-) diff --git a/assimilation_code/modules/utilities/time_manager_mod.f90 b/assimilation_code/modules/utilities/time_manager_mod.f90 index db35f4fd33..c74230dff4 100644 --- a/assimilation_code/modules/utilities/time_manager_mod.f90 +++ b/assimilation_code/modules/utilities/time_manager_mod.f90 @@ -694,7 +694,6 @@ subroutine set_calendar_type_string(calstring) character(len=len(calstring)) :: str1 character(len=max_calendar_string_length) :: cstring logical :: found_calendar = .false. -integer :: i if ( .not. module_initialized ) call time_manager_init @@ -714,47 +713,34 @@ subroutine set_calendar_type_string(calstring) ! We must check for the gregorian_mars calendar before ! the gregorian calendar for similar reasons. -WhichCalendar : do i = 0, max_type - - if ( cstring == 'NO_CALENDAR' ) then - calendar_type = NO_CALENDAR - found_calendar = .true. - exit WhichCalendar - elseif ( cstring == 'NO CALENDAR' ) then ! allow this as a synonym - calendar_type = NO_CALENDAR - found_calendar = .true. - exit WhichCalendar - elseif ( cstring == 'NONE' ) then ! also allow this - calendar_type = NO_CALENDAR - found_calendar = .true. - exit WhichCalendar - elseif ( cstring == 'THIRTY_DAY_MONTHS' ) then - calendar_type = THIRTY_DAY_MONTHS - found_calendar = .true. - exit WhichCalendar - elseif ( cstring == 'JULIAN' ) then - calendar_type = JULIAN - found_calendar = .true. - exit WhichCalendar - elseif ( cstring == 'NOLEAP' ) then - calendar_type = NOLEAP - found_calendar = .true. - exit WhichCalendar - elseif ( cstring == 'GREGORIAN_MARS' ) then - calendar_type = GREGORIAN_MARS - found_calendar = .true. - exit WhichCalendar - elseif ( cstring == 'SOLAR_MARS' ) then - calendar_type = SOLAR_MARS - found_calendar = .true. - exit WhichCalendar - elseif ( cstring == 'GREGORIAN' ) then - calendar_type = GREGORIAN - found_calendar = .true. - exit WhichCalendar - endif - -enddo WhichCalendar +if ( cstring == 'NO_CALENDAR' ) then + calendar_type = NO_CALENDAR + found_calendar = .true. +elseif ( cstring == 'NO CALENDAR' ) then ! allow this as a synonym + calendar_type = NO_CALENDAR + found_calendar = .true. +elseif ( cstring == 'NONE' ) then ! also allow this + calendar_type = NO_CALENDAR + found_calendar = .true. +elseif ( cstring == 'THIRTY_DAY_MONTHS' ) then + calendar_type = THIRTY_DAY_MONTHS + found_calendar = .true. +elseif ( cstring == 'JULIAN' ) then + calendar_type = JULIAN + found_calendar = .true. +elseif ( cstring == 'NOLEAP' ) then + calendar_type = NOLEAP + found_calendar = .true. +elseif ( cstring == 'GREGORIAN_MARS' ) then + calendar_type = GREGORIAN_MARS + found_calendar = .true. +elseif ( cstring == 'SOLAR_MARS' ) then + calendar_type = SOLAR_MARS + found_calendar = .true. +elseif ( cstring == 'GREGORIAN' ) then + calendar_type = GREGORIAN + found_calendar = .true. +endif if( .not. found_calendar ) then write(errstring,*)'Unknown calendar ',calstring @@ -785,23 +771,19 @@ subroutine get_calendar_string(mystring) ! ! Returns default calendar type for mapping from time to date. -character(len=*), intent(OUT) :: mystring - -integer :: i +character(len=*), intent(out) :: mystring if ( .not. module_initialized ) call time_manager_init -mystring = ' ' +mystring = '' -do i = 0,max_type - if (calendar_type == JULIAN) mystring = 'JULIAN' - if (calendar_type == NOLEAP) mystring = 'NOLEAP' - if (calendar_type == GREGORIAN) mystring = 'GREGORIAN' - if (calendar_type == NO_CALENDAR) mystring = 'NO_CALENDAR' - if (calendar_type == GREGORIAN_MARS) mystring = 'GREGORIAN_MARS' - if (calendar_type == SOLAR_MARS) mystring = 'SOLAR_MARS' - if (calendar_type == THIRTY_DAY_MONTHS) mystring = 'THIRTY_DAY_MONTHS' -enddo +if (calendar_type == JULIAN) mystring = 'JULIAN' +if (calendar_type == NOLEAP) mystring = 'NOLEAP' +if (calendar_type == GREGORIAN) mystring = 'GREGORIAN' +if (calendar_type == NO_CALENDAR) mystring = 'NO_CALENDAR' +if (calendar_type == GREGORIAN_MARS) mystring = 'GREGORIAN_MARS' +if (calendar_type == SOLAR_MARS) mystring = 'SOLAR_MARS' +if (calendar_type == THIRTY_DAY_MONTHS) mystring = 'THIRTY_DAY_MONTHS' if (len_trim(mystring) < 3) then write(errstring,*)'unknown calendar type ', calendar_type From ca2238d83fd580ab6b47fbe4c658b0536560626a Mon Sep 17 00:00:00 2001 From: Marlena Smith <44214771+mjs2369@users.noreply.github.com> Date: Wed, 6 Nov 2024 13:10:49 -0700 Subject: [PATCH 09/15] Update models/POP/readme.rst to have better description of SourceMods Co-authored-by: Helen Kershaw <20047007+hkershaw-brown@users.noreply.github.com> --- models/POP/readme.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/models/POP/readme.rst b/models/POP/readme.rst index f050f6082e..6d8e2c87cc 100644 --- a/models/POP/readme.rst +++ b/models/POP/readme.rst @@ -135,12 +135,13 @@ It is run by the setup scripts. | | where the restarts from the GDEX were downloaded/stored. | +---------------------+-----------------------------------------------------------+ -One set of variables to make sure are set correctly in this script are related to -the SourceMods, which enable POP to recompute the barotropic velocity to prevent the +SourceMods are required to enable POP to recompute the barotropic velocity to prevent the barotropic solver from crashing. You can find and download the SourceMods available -for POP-DART `here `_. Ensure that the -environment variable use_SourceMods is set to TRUE and the environment variable -SourceModDir matches the directory path where the SourceMods were downloaded/stored. +for POP-DART `here `_. + +Put the SourceMods in your home directory: + +~/${cesmtag}/SourceMods Setup scripts ~~~~~~~~~~~~~ From 2e3de8fd452a6cb4ccf33ac3683d60869727581a Mon Sep 17 00:00:00 2001 From: Marlee Smith Date: Wed, 6 Nov 2024 13:57:48 -0700 Subject: [PATCH 10/15] Re-including the reference to Karspeck's paper regarding generating an initial ensemble --- models/POP/readme.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/models/POP/readme.rst b/models/POP/readme.rst index 6d8e2c87cc..1c56c5dc7e 100644 --- a/models/POP/readme.rst +++ b/models/POP/readme.rst @@ -213,6 +213,18 @@ integration. Other files needed for assimilation ----------------------------------- +Creating an initial ensemble +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Karspeck et al. (2013) [3]_ finds that an ensemble of 1 January model states +selected from a multi-decade free-running integration of POP2 can be used as an +initial ensemble. + +You can access a collection of POP restart files from Who Kim's multi-century +``g210.G_JRA.v14.gx1v7.01`` experiment to serve as an initial ensemble in the +`NSF NCAR Geoscience Data Exchange `_. This +experiment uses the JRA-55 dataset for atmospheric forcing (Tsujino et al. 2018 [4]_). + Observation sequence files ~~~~~~~~~~~~~~~~~~~~~~~~~~ From 697792bb3485abb398f6c8ed16125eaf9f1d78e8 Mon Sep 17 00:00:00 2001 From: Marlee Smith Date: Wed, 6 Nov 2024 15:01:12 -0700 Subject: [PATCH 11/15] Removing found_calendar var and corresponding check --- .../modules/utilities/time_manager_mod.f90 | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/assimilation_code/modules/utilities/time_manager_mod.f90 b/assimilation_code/modules/utilities/time_manager_mod.f90 index c74230dff4..cdc88517aa 100644 --- a/assimilation_code/modules/utilities/time_manager_mod.f90 +++ b/assimilation_code/modules/utilities/time_manager_mod.f90 @@ -665,7 +665,8 @@ end function repeat_alarm !========================================================================= subroutine set_calendar_type_integer(mytype) - +!------------------------------------------------------------------------ +! ! Selects calendar for default mapping from time to date - if you know ! the magic integer for the calendar of interest. @@ -684,7 +685,8 @@ end subroutine set_calendar_type_integer subroutine set_calendar_type_string(calstring) - +!------------------------------------------------------------------------ +! ! Selects calendar for default mapping from time to date - given a string. character(len=*), intent(in) :: calstring @@ -693,7 +695,6 @@ subroutine set_calendar_type_string(calstring) character(len=len(calstring)) :: str1 character(len=max_calendar_string_length) :: cstring -logical :: found_calendar = .false. if ( .not. module_initialized ) call time_manager_init @@ -715,34 +716,23 @@ subroutine set_calendar_type_string(calstring) if ( cstring == 'NO_CALENDAR' ) then calendar_type = NO_CALENDAR - found_calendar = .true. elseif ( cstring == 'NO CALENDAR' ) then ! allow this as a synonym calendar_type = NO_CALENDAR - found_calendar = .true. elseif ( cstring == 'NONE' ) then ! also allow this calendar_type = NO_CALENDAR - found_calendar = .true. elseif ( cstring == 'THIRTY_DAY_MONTHS' ) then calendar_type = THIRTY_DAY_MONTHS - found_calendar = .true. elseif ( cstring == 'JULIAN' ) then calendar_type = JULIAN - found_calendar = .true. elseif ( cstring == 'NOLEAP' ) then calendar_type = NOLEAP - found_calendar = .true. elseif ( cstring == 'GREGORIAN_MARS' ) then calendar_type = GREGORIAN_MARS - found_calendar = .true. elseif ( cstring == 'SOLAR_MARS' ) then calendar_type = SOLAR_MARS - found_calendar = .true. elseif ( cstring == 'GREGORIAN' ) then calendar_type = GREGORIAN - found_calendar = .true. -endif - -if( .not. found_calendar ) then +else write(errstring,*)'Unknown calendar ',calstring call error_handler(E_ERR,'set_calendar_type_string',errstring,source) endif From 8f906bd15cf10725b053a747ea198fb9ea9f0c10 Mon Sep 17 00:00:00 2001 From: Marlee Smith Date: Thu, 7 Nov 2024 13:34:16 -0700 Subject: [PATCH 12/15] Removing now redundant script to copy the initial ensemble from a specific directory on glade (ensemble now available on GDEX) --- .../cesm2_1/copy_POP_JRA_restarts.py | 148 ------------------ 1 file changed, 148 deletions(-) delete mode 100644 models/POP/shell_scripts/cesm2_1/copy_POP_JRA_restarts.py diff --git a/models/POP/shell_scripts/cesm2_1/copy_POP_JRA_restarts.py b/models/POP/shell_scripts/cesm2_1/copy_POP_JRA_restarts.py deleted file mode 100644 index d0667833a2..0000000000 --- a/models/POP/shell_scripts/cesm2_1/copy_POP_JRA_restarts.py +++ /dev/null @@ -1,148 +0,0 @@ -from shutil import copyfile, copyfileobj -import getpass -import os -import sys -import glob -from pathlib import Path - -# -----------------------------------------------------# -# Define needed function # -# -----------------------------------------------------# - - -def unzipNsave(old, new): - ''' - Saves *.gz file in old as unzipped file in new - ''' - import gzip - - with gzip.open(old, 'rb') as f_in: - with open(new, 'wb') as f_out: - copyfileobj(f_in, f_out) - - -# -----------------------------------------------------# -# Main body of the script # -# -----------------------------------------------------# - -# Get username # -# -----------------------------------------------------# - -USER = getpass.getuser() - -# Create stage directory if it doesn't exist # -# -----------------------------------------------------# - -case_name = 'g210.G_JRA.v14.gx1v7.01' -stagedir = '/glade/scratch/' + USER + '/' + case_name + '/rest/2010-01-01-00000/' -Path("stagedir").mkdir(parents=True, exist_ok=True) - -# First load from the directory of MONTHLY saves # -# -----------------------------------------------------# - -load_path = '/glade/campaign/cgd/oce/people/whokim/csm/' + case_name + '/rest/' - -years = list(range(41, 202, 10))+list(range(209, 251)) -# For debugging: -# years = list(range(31,32)) -inds = list(range(1, len(years)+1)) - -print('Copying from ' + load_path + ' to ' + stagedir) - -for ii, year in enumerate(years): - - # Zero pad the year and index string so that there are 4 digits - zero_filled_year = str(year).zfill(4) - zero_filled_ii = str(inds[ii]).zfill(4) - - print('Copying year ' + str(year) + ' to ensemble number ' - + zero_filled_ii) - - yearpath = load_path + '/' + zero_filled_year + '-01-01-00000/' - - # Define file paths - popold = yearpath + case_name + '.pop.r.' + zero_filled_year \ - + '-01-01-00000.nc' - ovfold = yearpath + case_name + '.pop.ro.' + zero_filled_year \ - + '-01-01-00000' - ciceold = yearpath + case_name + '.cice.r.' + zero_filled_year \ - + '-01-01-00000.nc' - - popnew = stagedir + case_name + '.pop_' + zero_filled_ii \ - + '.r.2010-01-01-00000.nc' - ovfnew = stagedir + case_name + '.pop_' + zero_filled_ii \ - + '.ro.2010-01-01-00000' - cicenew = stagedir + case_name + '.cice_' + zero_filled_ii \ - + '.r.2010-01-01-00000.nc' - - # Copy over - copyfile(popold, popnew) - copyfile(ovfold, ovfnew) - copyfile(ciceold, cicenew) - - -# Then load the rest from DAILY saves -# Some years (251-262) must be decompressed -# -----------------------------------------------------# - -case_name_dr = 'g210.G_JRA.v14.gx1v7.01.dr' -load_path = '/glade/campaign/cgd/oce/people/whokim/csm/' + case_name_dr + '/rest' - -ly = len(years) -years2 = list(range(251, 272)) - -# Start from where we left off with the daily saves with inds -inds = list(range(ly+1, ly+len(years2)+1)) - -print('Copying from ' + load_path + ' to ' + stagedir) - -for ii, year in enumerate(years2): - - # Zero pad the year and index string so that there are 4 digits - zero_filled_year = str(year).zfill(4) - zero_filled_ii = str(inds[ii]).zfill(4) - - yearpath = load_path + '/' + zero_filled_year + '-01-01-00000/' - - # Define file paths - popold = glob.glob(yearpath + case_name_dr + '.pop.r.' + zero_filled_year - + '-01-01-00000.nc*')[0] - ovfold = glob.glob(yearpath + case_name_dr + '.pop.ro.' + zero_filled_year - + '-01-01-00000*')[0] - ciceold = glob.glob(yearpath + case_name_dr + '.cice.r.' + zero_filled_year - + '-01-01-00000.nc*')[0] - - popnew = stagedir + case_name + '.pop_' + zero_filled_ii \ - + '.r.2010-01-01-00000.nc' - ovfnew = stagedir + case_name + '.pop_' + zero_filled_ii \ - + '.ro.2010-01-01-00000' - cicenew = stagedir + case_name + '.cice_' + zero_filled_ii \ - + '.r.2010-01-01-00000.nc' - - # Copy over, decompressing as necessary - - if popold[-3:] == '.gz': - print('Decompressing and copying year ' + str(year) - + ' to ensemble number ' + zero_filled_ii) - else: - print('Copying year ' + str(year) + ' to ensemble number ' - + zero_filled_ii) - - if popold[-3:] == '.gz': - unzipNsave(popold, popnew) - else: - copyfile(popold, popnew) - - # OVF - if ovfold[-3:] == '.gz': - unzipNsave(ovfold, ovfnew) - else: - copyfile(ovfold, ovfnew) - - # CICE - if ciceold[-3:] == '.gz': - unzipNsave(ciceold, cicenew) - else: - copyfile(ciceold, cicenew) - -print('Completed') From 55a30c2b3d69857e405a7d7d8465d700dac82c10 Mon Sep 17 00:00:00 2001 From: Helen Kershaw Date: Fri, 8 Nov 2024 12:38:30 -0500 Subject: [PATCH 13/15] typo --- models/POP/readme.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/POP/readme.rst b/models/POP/readme.rst index 1c56c5dc7e..3726297166 100644 --- a/models/POP/readme.rst +++ b/models/POP/readme.rst @@ -216,7 +216,7 @@ Other files needed for assimilation Creating an initial ensemble ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Karspeck et al. (2013) [3]_ finds that an ensemble of 1 January model states +Karspeck et al. (2013) [3]_ find that an ensemble of 1 January model states selected from a multi-decade free-running integration of POP2 can be used as an initial ensemble. From d0892c49620319f9764af64687ac869e6a61c18f Mon Sep 17 00:00:00 2001 From: Helen Kershaw Date: Fri, 8 Nov 2024 12:54:12 -0500 Subject: [PATCH 14/15] fix: error message to match the routine name --- assimilation_code/modules/utilities/mpi_utilities_mod.f90 | 2 +- assimilation_code/modules/utilities/mpif08_utilities_mod.f90 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assimilation_code/modules/utilities/mpi_utilities_mod.f90 b/assimilation_code/modules/utilities/mpi_utilities_mod.f90 index d9bd640a38..d091c12638 100644 --- a/assimilation_code/modules/utilities/mpi_utilities_mod.f90 +++ b/assimilation_code/modules/utilities/mpi_utilities_mod.f90 @@ -1475,7 +1475,7 @@ subroutine all_reduce_min_max(min_var, max_var, num_elements) if ( .not. module_initialized ) then write(errstring, *) 'initialize_mpi_utilities() must be called first' - call error_handler(E_ERR,'broadcast_minmax', errstring, source) + call error_handler(E_ERR,'all_reduce_min_max', errstring, source) endif call mpi_allreduce(MPI_IN_PLACE, min_var, num_elements, datasize, MPI_MIN, get_dart_mpi_comm(), errcode) diff --git a/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 b/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 index 51bc507fe0..b2b1791aa2 100644 --- a/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 +++ b/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 @@ -1475,7 +1475,7 @@ subroutine all_reduce_min_max(min_var, max_var, num_elements) if ( .not. module_initialized ) then write(errstring, *) 'initialize_mpi_utilities() must be called first' - call error_handler(E_ERR,'broadcast_minmax', errstring, source) + call error_handler(E_ERR,'all_reduce_min_max', errstring, source) endif call mpi_allreduce(MPI_IN_PLACE, min_var, num_elements, datasize, MPI_MIN, get_dart_mpi_comm(), errcode) From 9159690c2ca0a2cf40596bef1737e2b9cc181d8f Mon Sep 17 00:00:00 2001 From: Helen Kershaw Date: Fri, 8 Nov 2024 13:13:13 -0500 Subject: [PATCH 15/15] bump conf.py and changelog for release --- CHANGELOG.rst | 13 +++++++++++++ conf.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b15c62115b..899a28cba6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,6 +22,19 @@ individual files. The changes are now listed with the most recent at the top. +**November 8 2024 :: POP initial ensemble available from GDEX. Tag v11.8.4** + +Documentation update: + + - POP initial ensemble available from GDEX. + +Bug-fixes: + + - Removed unnecessary loops around calendar types in time_manager_mod. + - Removed unused routine from normal_distribution_mod which case giving compilation warnings. + - Replaced broadcast_minmax calls with all_reduce_min_max. + + **November 7 2024 :: MPI window memory reduction. Tag v11.8.3** - Removes unnecessary copy of state into mpi window. diff --git a/conf.py b/conf.py index 23c29119d6..60e5f6f4cd 100644 --- a/conf.py +++ b/conf.py @@ -21,7 +21,7 @@ author = 'Data Assimilation Research Section' # The full version, including alpha/beta/rc tags -release = '11.8.3' +release = '11.8.4' root_doc = 'index' # -- General configuration ---------------------------------------------------