[Dart-dev] DART/branches Revision: 12825

[dart at ucar.edu]

hendric at ucar.edu
2018-09-14 13:43:02 -0600 (Fri, 14 Sep 2018)
136

start moving code to distribute the mean into the 
create_mean_window subroutine.  currently this
is giving a invalid memory request.




Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/distributed_state_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/distributed_state_mod.f90	2018-09-13 21:51:26 UTC (rev 12824)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/distributed_state_mod.f90	2018-09-14 19:43:02 UTC (rev 12825)
@@ -81,9 +81,9 @@
 integer(i8),         intent(in)  :: my_index !> index into state vector
 type(ensemble_type), intent(in)  :: ens_handle
 
-if (current_win == MEAN_WINDOW) then
-   call get_mean(x, my_index, ens_handle)
-else if (current_win == STATE_WINDOW) then
+if (      current_win == MEAN_WINDOW ) then
+   call get_mean(x, my_index, ens_handle )
+else if ( current_win == STATE_WINDOW) then
    call get_fwd(x, my_index, ens_handle)
 else
    call error_handler(E_ERR, 'get_state',' No window currently open')
@@ -133,9 +133,12 @@
 integer(i8),         intent(in)  :: my_index !> index into state vector
 type(ensemble_type), intent(in)  :: state_ens_handle
 
-integer                        :: owner_of_state !> task who owns the state
-integer                        :: element_index !> local index of element
+integer :: owner_of_state !> task who owns the state
+integer :: element_index  !> local index of element
+integer :: my_comm        !> local communicator for state_ens_handle
 
+my_comm= mean_ens_handle%my_communicator
+
 if (get_allow_transpose(mean_ens_handle)) then
    x(1) = mean_ens_handle%vars(my_index, 1)
 else
@@ -144,7 +147,7 @@
 
    owner_of_state = map_pe_to_task(state_ens_handle, owner_of_state)        ! task
 
-   if (my_task_id() == owner_of_state) then
+   if (my_task_id(my_comm) == owner_of_state) then
       x(1) = mean_ens_handle%copies(1, element_index)
    else
       call get_from_mean(owner_of_state, mean_win, element_index, x(1))

Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/mpi_utilities_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/mpi_utilities_mod.f90	2018-09-13 21:51:26 UTC (rev 12824)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/mpi_utilities_mod.f90	2018-09-14 19:43:02 UTC (rev 12825)
@@ -2034,7 +2034,6 @@
 ! Note to programmer: openmpi 1.10.0 does not
 ! allow scalars in mpi calls. openmpi 1.10.1 fixes
 ! this.
-!print*, 'mpi_utilities window ', window
 target_disp = (mindex - 1)
 call mpi_win_lock(MPI_LOCK_SHARED, owner, 0, window, errcode)
 call mpi_get(x, 1, datasize, owner, target_disp, 1, datasize, window, errcode)
@@ -2094,6 +2093,7 @@
 !> of all of the dart communicator tasks that are in the same
 !> group, and sub_group is a handle to the group that is the
 !> same across all processors.
+
 subroutine create_groups(my_group_size, my_comm)
 integer, intent(inout) :: my_group_size
 integer, intent(inout), optional :: my_comm

Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/no_cray_win_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/no_cray_win_mod.f90	2018-09-13 21:51:26 UTC (rev 12824)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/no_cray_win_mod.f90	2018-09-14 19:43:02 UTC (rev 12825)
@@ -9,13 +9,14 @@
 
 !> \defgroup window window_mod
 !> @{
-use mpi_utilities_mod,    only : datasize, my_task_id, get_dart_mpi_comm, get_group_comm
+use mpi_utilities_mod,    only : datasize, my_task_id, get_dart_mpi_comm, &
+                                 create_groups, get_group_comm
 use types_mod,            only : r8, i8
 use ensemble_manager_mod, only : ensemble_type, map_pe_to_task, get_var_owner_index, &
                                  copies_in_window, init_ensemble_manager, &
                                  get_allow_transpose, end_ensemble_manager, &
                                  set_num_extra_copies, all_copies_to_all_vars, &
-                                 all_vars_to_all_copies
+                                 my_vars_to_group_copies, all_vars_to_all_copies
 
 use mpi
 
@@ -56,6 +57,7 @@
 real(r8), allocatable :: mean_1d(:)
 
 type(ensemble_type) :: mean_ens_handle
+type(ensemble_type) :: group_mean_handle
 
 contains
 
@@ -101,16 +103,18 @@
 !-------------------------------------------------------------
 !> Using a mean ensemble handle.
 !> 
-subroutine create_mean_window(state_ens_handle, mean_copy, distribute_mean, return_mean_ens_handle)
+recursive subroutine create_mean_window(state_ens_handle, mean_copy, distribute_mean, return_handle)
 
 type(ensemble_type), intent(in)  :: state_ens_handle
 integer,             intent(in)  :: mean_copy