[Dart-dev] DART/branches Revision: 12796

[dart at ucar.edu]

hendric at ucar.edu
2018-08-30 15:26:47 -0600 (Thu, 30 Aug 2018)
116

code changes after code review. there is still some
work to be done with accessing and distributing a
data array.




Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90	2018-08-29 20:05:59 UTC (rev 12795)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90	2018-08-30 21:26:47 UTC (rev 12796)
@@ -215,6 +215,7 @@
 !> a node or across more than a node but less than the total number 
 !> of tasks.
 
+! FIXME: JPH, pass in the group size here?
 subroutine init_ensemble_manager(ens_handle, num_copies, &
    num_vars, distribution_type_in, layout_type, transpose_type_in, use_groups)
 
@@ -239,8 +240,8 @@
    module_initialized = .true.
 endif
 
-ens_handle%num_pes = task_count()
-ens_handle%group_size = task_count()  ! FIXME can we avoid having this?
+ens_handle%num_pes         = task_count()
+ens_handle%group_size      = task_count()  ! FIXME can we avoid having this?
 ens_handle%my_communicator = get_dart_mpi_comm()  ! FIXME do we need this?
 
 do_grp = .false.
@@ -253,8 +254,8 @@
    ! if we can, try to use num_pes instead of group size.
    ! if that works, we can remove group_size and just set
    ! num_pes to either task_count or tasks_per_group.
-   ens_handle%num_pes = tasks_per_group
-   ens_handle%group_size = tasks_per_group  ! FIXME can we avoid using this
+   ens_handle%num_pes         = tasks_per_group
+   ens_handle%group_size      = tasks_per_group ! FIXME can we avoid using this
    ens_handle%my_communicator = get_group_comm()  
 endif
 
@@ -271,7 +272,7 @@
 else
    if(distribution_type_in < 1 .or. distribution_type_in > 3) then
        call error_handler(E_ERR, 'init_ensemble_manager', &
-              'only distribution_types 1, 2 and 3 are implemented', source, revision, revdate)
+              'only distribution_type 1 is implemented at the moment', source, revision, revdate)
    endif
    ens_handle%distribution_type = distribution_type_in
 endif
@@ -379,6 +380,8 @@
 if (do_nml_file()) write(nmlfileunit, nml=ensemble_manager_nml)
 if (do_nml_term()) write(     *     , nml=ensemble_manager_nml)
 
+! FIXME: may need a way to set up what the group size is.
+! perhaps get the size from the communicator...
 if (tasks_per_group == MPI_GROUPS_NOT_SET) then
    tasks_per_group = task_count()
 endif
@@ -390,9 +393,9 @@
 endif
 
 ! FIXME: does this go in set_up_ens_distribution()
-call set_group_size(tasks_per_group)
-call create_groups(tasks_per_group)
-call task_sync()
+call set_group_size( tasks_per_group )
+call create_groups(  tasks_per_group )
+call task_sync() !FIXME: JPH - do we need this sync??
    
 end subroutine static_init_ensemble_manager
 
@@ -1205,7 +1208,7 @@
      if(my_pe == recv_pe) then
 
         ! Figure out what piece to receive from each other PE and receive it
-        RECEIVE_FROM_EACH: do sending_pe = 0, num_pes - 1
+        RECEIVE_FROM_EACH: do sending_pe = 0, ens_handle%num_pes - 1
            call get_copy_list(ens_handle, num_copies, sending_pe, copy_list, num_copies_to_receive)
 
            ! Loop to receive for each copy stored on my_pe
@@ -1539,7 +1542,7 @@
 !  then if there are more than 1 group, have each task broadcast their
 !  copies to their corresponding task numbers in each of the other groups.
 
-! 
+ 
 !mean_ens_handle%copies(1,:) = state_ens_handle%copies(mean_copy, :)
 
 ! Short var definitions ens 1
@@ -1592,28 +1595,21 @@
 allocate(copy_list1(1), copy_list2(1), &
          copy_list(1))
    
-! if (use_copy2var_send_loop .eqv. .true. ) then
-! Switched loop index from receiving_pe to sending_pe
-! Aim: to make the communication scale better on Yellowstone, as num_pes >> ens_size
-! For small numbers of tasks (32 or less) the receiving_pe loop may be faster.
-! Namelist option use_copy2var_send_loop can be used to select which
-! communication pattern to use
-!    Default: use sending_pe loop (use_copy2var_send_loop = .true.)
-
 SENDING_PE_LOOP: do sending_pe = 0,  ens_handle1%num_pes- 1
  
+   ! figure out what piece to receive from each other PE and receive it
+   call get_var_list(ens_handle1, num_vars1, sending_pe, var_list, num_vars_to_receive)
+ 
    if (my_pe1 /= sending_pe ) then