[Dart-dev] DART/branches Revision: 12457

[dart at ucar.edu]

hendric at ucar.edu
2018-03-21 14:23:49 -0600 (Wed, 21 Mar 2018)
66

start of writting code to distribute all copies to all copies.





Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/distributed_state_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/distributed_state_mod.f90	2018-03-20 21:30:21 UTC (rev 12456)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/distributed_state_mod.f90	2018-03-21 20:23:49 UTC (rev 12457)
@@ -15,7 +15,7 @@
 !>    no_cray_win_mod.f90 does not use cray pointers
 !>    cray_win_mod.f90 uses cray pointers
 
-use mpi_utilities_mod,    only : my_task_id, &
+use mpi_utilities_mod,    only : my_task_id, group_task_id, &
                                  get_from_fwd, get_from_mean
 use types_mod,            only : r8, i8
 use ensemble_manager_mod, only : ensemble_type, map_pe_to_task, get_var_owner_index, &
@@ -144,9 +144,10 @@
 
    owner_of_state = map_pe_to_task(state_ens_handle, owner_of_state)        ! task
 
-   if (my_task_id() == owner_of_state) then
+   if (group_task_id() == owner_of_state) then
       x(1) = mean_ens_handle%copies(1, element_index)
    else
+      print*, 'dist mean_win', mean_win
       call get_from_mean(owner_of_state, mean_win, element_index, x(1))
    endif
 

Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90	2018-03-20 21:30:21 UTC (rev 12456)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90	2018-03-21 20:23:49 UTC (rev 12457)
@@ -70,8 +70,9 @@
 !> additional last argument saying this is a group call and not a world one
 
 use mpi_utilities_mod, only : task_count, my_task_id, send_to, receive_from, &
-                              task_sync, broadcast_send, broadcast_recv, &
-                              create_groups, set_group_size, get_group_size
+                              task_sync, broadcast_send, broadcast_recv,     &
+                              create_groups, set_group_size, get_group_size, &
+                              get_group_id, get_group_comm
 
 
 
@@ -93,7 +94,7 @@
           prepare_to_update_copies,   print_ens_handle,         set_current_time,           &
           map_task_to_pe,             map_pe_to_task,           get_current_time,           &
           allocate_single_copy,       put_single_copy,          get_single_copy,            &
-          deallocate_single_copy
+          deallocate_single_copy,     all_copies_to_all_copies
 
 ! version controlled file description for error handling, do not edit
 character(len=*), parameter :: source   = &
@@ -220,7 +221,6 @@
 integer :: transpose_type
 integer :: string1
 
-!>@todo FIXME this didn't used to be first.  why not?
 ! First call to init_ensemble_manager must initialize module and read namelist
 if ( .not. module_initialized ) then
    ! Initialize the module with utilities 
@@ -240,6 +240,12 @@
 
 endif
 
+if(group_size  < 0 .or. group_size > task_count()) then
+   write(string1,'(A,I5)') 'invalid group_size ', group_size
+   call error_handler(E_ERR, 'init_ensemble_manager', &
+                      source, revision, revdate)
+endif
+
 ! FIXME: if we are implementing communication across a group, can we do this here?
 ! or does it need to be for a subcommunicator?
 if (group_size == MPI_GROUPS_NOT_SET) then
@@ -246,12 +252,7 @@
    group_size = task_count()
 endif
 
-if(group_size  < 0 .or. group_size > task_count()) then
-   write(string1,'(A,I5)') 'invalid group_size ', group_size
-   call error_handler(E_ERR, 'init_ensemble_manager', &
-                      source, revision, revdate)
-endif
-
+! FIXME: does this go in set_up_ens_distribution()
 call set_group_size(group_size)
 call create_groups(group_size)
 call task_sync()
@@ -424,7 +425,7 @@
    endif
  
    ! Otherwise, must wait to receive vars and time from storing pe
-      call receive_from(map_pe_to_task(ens_handle, owner), vars, mtime)
+      call receive_from(map_pe_to_task(ens_handle, owner), vars, time = mtime)
 endif
 
 !----- Block of code that must be done by PE that stores the copy IF it is NOT receiver -----
@@ -432,7 +433,9 @@
    ! Send copy to receiving pe
 
    if(present(mtime)) then
-      call send_to(map_pe_to_task(ens_handle, receiving_pe), ens_handle%vars(:, owners_index), ens_handle%time(owners_index))
+      call send_to(map_pe_to_task(ens_handle, receiving_pe), &