[Dart-dev] DART/releases Revision: 12758

[dart at ucar.edu]

nancy at ucar.edu
2018-08-03 15:07:12 -0600 (Fri, 03 Aug 2018)
55
pushing the performance fix to the Manhattan release.




Modified: DART/releases/Manhattan/CHANGELOG
===================================================================
--- DART/releases/Manhattan/CHANGELOG	2018-08-03 20:52:09 UTC (rev 12757)
+++ DART/releases/Manhattan/CHANGELOG	2018-08-03 21:07:12 UTC (rev 12758)
@@ -771,9 +771,21 @@
 - Added more functions to the netcdf_utilities_mod.f90
 
 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ MMM DD YYYY :: summary of next changes                       Revision: XXXXX 
++ Aug 03 2018 :: performance fix for distributed mean          Revision: 12758 
 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
+- Important performance fix if model does vertical conversion for localization.
+Results were not wrong but performance was poor if 'distribute_mean = .true.' 
+was selected in the &assim_tools_nml namelist.
+Now distributing the mean runs in close to the non-distributed time and uses
+much less memory for large models. This only impacts models which do a vertical
+conversion of either the observation or state vertical coordinate for localization
+AND which set &assim_tools_nml :: distribute_mean = .true. to use less memory.
+When using a distributed mean "convert_all_obs_verticals_first = .true." should 
+be set.  If your observations will impact most of the model state, then 
+"convert_all_state_verticals_first = .true.' can also be set.
+
+
 # <next few lines under version control, do not edit>
 # $URL$
 # $Revision$

Modified: DART/releases/Manhattan/assimilation_code/modules/assimilation/assim_tools_mod.f90
===================================================================
--- DART/releases/Manhattan/assimilation_code/modules/assimilation/assim_tools_mod.f90	2018-08-03 20:52:09 UTC (rev 12757)
+++ DART/releases/Manhattan/assimilation_code/modules/assimilation/assim_tools_mod.f90	2018-08-03 21:07:12 UTC (rev 12758)
@@ -54,7 +54,7 @@
 
 use mpi_utilities_mod,    only : my_task_id, broadcast_send, broadcast_recv,              & 
                                  sum_across_tasks, task_count, start_mpi_timer,           &
-                                 read_mpi_timer
+                                 read_mpi_timer, task_sync
 
 use adaptive_inflate_mod, only : do_obs_inflate,  do_single_ss_inflate, do_ss_inflate,    &
                                  do_varying_ss_inflate,                                   &
@@ -323,11 +323,7 @@
 integer,                     intent(in)    :: OBS_PRIOR_VAR_START, OBS_PRIOR_VAR_END
 logical,                     intent(in)    :: inflate_only
 
-!>@todo FIXME this routine has a huge amount of local/stack storage.
-!>at some point does it need to be allocated instead?  this routine isn't
-!>called frequently so doing allocate/deallocate isn't a timing issue.  
-!>putting arrays on the stack is fast, but risks running out of stack space 
-!>and dying with strange errors.
+! changed the ensemble sized things here to allocatable
 
 real(r8) :: obs_prior(ens_size), obs_inc(ens_size), increment(ens_size)
 real(r8) :: reg_factor, impact_factor
@@ -339,37 +335,44 @@
 real(r8) :: r_mean, r_var
 real(r8) :: orig_obs_prior_mean(num_groups), orig_obs_prior_var(num_groups)
 real(r8) :: obs_prior_mean(num_groups), obs_prior_var(num_groups)
-real(r8) :: close_obs_dist(obs_ens_handle%my_num_vars)
-real(r8) :: close_state_dist(ens_handle%my_num_vars)
-real(r8) :: last_close_obs_dist(obs_ens_handle%my_num_vars)
-real(r8) :: last_close_state_dist(ens_handle%my_num_vars)
 real(r8) :: diff_sd, outlier_ratio
+real(r8) :: vertvalue_obs_in_localization_coord, whichvert_real
+real(r8), allocatable :: close_obs_dist(:)
+real(r8), allocatable :: close_state_dist(:)
+real(r8), allocatable :: last_close_obs_dist(:)
+real(r8), allocatable :: last_close_state_dist(:)
 
 integer(i8) :: state_index
-integer(i8) :: my_state_indx(ens_handle%my_num_vars)
-integer(i8) :: my_obs_indx(obs_ens_handle%my_num_vars)
+integer(i8), allocatable :: my_state_indx(:)
+integer(i8), allocatable :: my_obs_indx(:)
 
 integer  :: my_num_obs, i, j, owner, owners_index, my_num_state
-integer  :: this_obs_key, obs_mean_index, obs_var_index
+integer :: obs_mean_index, obs_var_index
 integer  :: grp_beg(num_groups), grp_end(num_groups), grp_size, grp_bot, grp_top, group
-integer  :: close_obs_ind(obs_ens_handle%my_num_vars)
-integer  :: close_state_ind(ens_handle%my_num_vars)
-integer  :: last_close_obs_ind(obs_ens_handle%my_num_vars)
-integer  :: last_close_state_ind(ens_handle%my_num_vars)
 integer  :: num_close_obs, obs_index, num_close_states
 integer  :: total_num_close_obs, last_num_close_obs, last_num_close_states
-integer  :: base_obs_kind, base_obs_type, my_obs_kind(obs_ens_handle%my_num_vars)
-integer  :: my_obs_type(obs_ens_handle%my_num_vars)
-integer  :: my_state_kind(ens_handle%my_num_vars), nth_obs
+integer :: base_obs_kind, base_obs_type, nth_obs
 integer  :: num_close_obs_cached, num_close_states_cached
 integer  :: num_close_obs_calls_made, num_close_states_calls_made
-! GSR add new count for only the 'assimilate' type close obs in the tile
 integer  :: localization_unit, secs, days, rev_num_close_obs
+integer :: whichvert_obs_in_localization_coord
+integer :: istatus
+integer, allocatable :: close_obs_ind(:)
+integer, allocatable :: close_state_ind(:)
+integer, allocatable :: last_close_obs_ind(:)
+integer, allocatable :: last_close_state_ind(:)
+integer, allocatable :: my_obs_kind(:)
+integer, allocatable :: my_obs_type(:)