[Dart-dev] DART/branches Revision: 12828

[dart at ucar.edu]

hendric at ucar.edu
2018-09-17 16:45:03 -0600 (Mon, 17 Sep 2018)
120

moved all code to distribute the mean into the
create_mean_window subroutine.  compiles and
gives the correct result.




Modified: DART/branches/rma_distrib/assimilation_code/modules/assimilation/assim_tools_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/assimilation/assim_tools_mod.f90	2018-09-17 20:15:16 UTC (rev 12827)
+++ DART/branches/rma_distrib/assimilation_code/modules/assimilation/assim_tools_mod.f90	2018-09-17 22:45:03 UTC (rev 12828)
@@ -400,6 +400,8 @@
 
 integer, allocatable :: n_close_state_items(:), n_close_obs_items(:)
 
+integer :: group_size = 1
+
 ! timing disabled by default
 timing(:)  = .false.
 t_base(:)  = 0.0_r8
@@ -458,7 +460,7 @@
 !HK make window for mpi one-sided communication
 ! used for vertical conversion in get_close_obs
 ! Need to give create_mean_window the mean copy
-call create_mean_window(ens_handle, ENS_MEAN_COPY, distribute_mean)
+call create_mean_window(ens_handle, ENS_MEAN_COPY, distribute_mean, group_size)
 
 ! filter kinds 1 and 8 return sorted increments, however non-deterministic
 ! inflation can scramble these. the sort is expensive, so help users get better 

Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/distributed_state_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/distributed_state_mod.f90	2018-09-17 20:15:16 UTC (rev 12827)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/distributed_state_mod.f90	2018-09-17 22:45:03 UTC (rev 12828)
@@ -23,7 +23,7 @@
 use window_mod,           only : create_mean_window, create_state_window, free_mean_window, &
                                  free_state_window, mean_ens_handle, data_count, &
                                  NO_WINDOW, MEAN_WINDOW, STATE_WINDOW, &
-                                 mean_win, state_win, current_win
+                                 mean_win, state_win, current_win, group_mean_handle
 use utilities_mod,        only : error_handler, E_ERR
 
 implicit none
@@ -135,21 +135,25 @@
 
 integer :: owner_of_state !> task who owns the state
 integer :: element_index  !> local index of element
-integer :: my_comm        !> local communicator for state_ens_handle
+integer :: my_comm        !> local communicator for group_mean_handle
 
-my_comm= mean_ens_handle%my_communicator
+my_comm= group_mean_handle%my_communicator
 
-if (get_allow_transpose(mean_ens_handle)) then
+if (get_allow_transpose(group_mean_handle)) then
+   print*, 'allow transpose'
    x(1) = mean_ens_handle%vars(my_index, 1)
 else
 
-   call get_var_owner_index(state_ens_handle, my_index, owner_of_state, element_index) ! pe
+   call get_var_owner_index(group_mean_handle, my_index, owner_of_state, element_index) ! pe
 
-   owner_of_state = map_pe_to_task(state_ens_handle, owner_of_state)        ! task
+   owner_of_state = map_pe_to_task(group_mean_handle, owner_of_state)        ! task
 
    if (my_task_id(my_comm) == owner_of_state) then
-      x(1) = mean_ens_handle%copies(1, element_index)
+      x(1) = group_mean_handle%copies(1, element_index)
    else
+      print*, 'rank =', my_task_id(), &
+              ' owner = ', owner_of_state, &
+              ' index = ', element_index
       call get_from_mean(owner_of_state, mean_win, element_index, x(1))
    endif
 

Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90	2018-09-17 20:15:16 UTC (rev 12827)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90	2018-09-17 22:45:03 UTC (rev 12828)
@@ -798,7 +798,6 @@
 endif
 
 if(ens_handle%transpose_type == 3) then
-   print*, my_task_id(), 'allocating space for vars on every task'
    allocate(ens_handle%vars(ens_handle%num_vars,1))
 endif
 

Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/mpi_utilities_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/mpi_utilities_mod.f90	2018-09-17 20:15:16 UTC (rev 12827)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/mpi_utilities_mod.f90	2018-09-17 22:45:03 UTC (rev 12828)
@@ -2124,7 +2124,7 @@
    do i = 0, (task_count()-1)
       call MPI_Barrier(my_local_comm, errcode)
       if(my_task_id() == i) then
-         write(*,'(''WORLD RANK/SIZE:'',I2,''/'',I2,'' GROUP RANK/SIZE:'',I2,''/'',I2,'' SUB GROUP '',I10)') &
+         write(*,'(''WORLD RANK/SIZE:'',I4,''/'',I4,'' GROUP RANK/SIZE:'',I4,''/'',I4,'' SUB GROUP '',I10)') &
             my_task_id(), task_count(), group_rank, group_size, sub_group
             !my_task_id(my_local_comm), task_count(my_local_comm), &
             !my_task_id(my_group_comm), task_count(my_group_comm), &

Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/no_cray_win_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/no_cray_win_mod.f90	2018-09-17 20:15:16 UTC (rev 12827)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/no_cray_win_mod.f90	2018-09-17 22:45:03 UTC (rev 12828)
@@ -25,7 +25,7 @@