[Dart-dev] DART/branches Revision: 12804

[dart at ucar.edu]

hendric at ucar.edu
2018-08-31 10:35:42 -0600 (Fri, 31 Aug 2018)
141

adding some debug statements.  going to test with 
all_copies_to_all_vars and extract the data into
groups from there, as an initial test.




Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90	2018-08-31 15:55:50 UTC (rev 12803)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90	2018-08-31 16:35:42 UTC (rev 12804)
@@ -138,10 +138,10 @@
    integer, allocatable,private :: task_to_pe_list(:), pe_to_task_list(:) ! List of tasks
    ! Flexible my_pe, layout_type which allows different task layouts for different ensemble handles
    integer                      :: my_pe
-   integer                      :: my_group_pe   ! FIXME do we need this and num_pes at the same time?
+   integer                      :: my_group_pe ! FIXME do we need this and num_pes at the same time?
    integer,private              :: layout_type
    integer,private              :: transpose_type
-   integer                      :: num_extras   ! FIXME: this needs to be more general.
+   integer                      :: num_extras ! FIXME: this needs to be more general.
    integer                      :: my_communicator ! FIXME: is this needed?
 end type ensemble_type
 
@@ -217,7 +217,6 @@
 !> a node or across more than a node but less than the total number 
 !> of tasks.
 
-! FIXME: JPH, pass in the group size here?
 subroutine init_ensemble_manager(ens_handle, num_copies, &
    num_vars, distribution_type_in, layout_type, transpose_type_in, use_groups)
 
@@ -257,7 +256,7 @@
    ! if that works, we can remove group_size and just set
    ! num_pes to either task_count or tasks_per_group.
    ens_handle%num_pes         = tasks_per_group
-   ens_handle%group_size      = tasks_per_group ! FIXME can we avoid using this
+   ens_handle%group_size      = tasks_per_group  ! FIXME can we avoid using this
    ens_handle%my_communicator = get_group_comm()  
 endif
 
@@ -274,7 +273,8 @@
 else
    if(distribution_type_in < 1 .or. distribution_type_in > 3) then
        call error_handler(E_ERR, 'init_ensemble_manager', &
-              'only distribution_type 1 is implemented at the moment', source, revision, revdate)
+                          'only distribution_types 1, 2 and 3 are implemented', &
+                          source, revision, revdate)
    endif
    ens_handle%distribution_type = distribution_type_in
 endif
@@ -292,7 +292,9 @@
 endif
 
 ! Check for error: only layout_types 1, 2 and 3 are implemented
-if (ens_handle%layout_type /= 1 .and. ens_handle%layout_type /=2 .and. ens_handle%layout_type /= 3) then
+if (ens_handle%layout_type /= 1 .and. &
+    ens_handle%layout_type /= 2 .and. &
+    ens_handle%layout_type /= 3) then
    call error_handler(E_ERR, 'init_ensemble_manager', &
                       'only layout values 1 (standard), 2 (round-robin) allowed, and 3 (groups) ', &
                       source, revision, revdate)
@@ -381,8 +383,6 @@
 if (do_nml_file()) write(nmlfileunit, nml=ensemble_manager_nml)
 if (do_nml_term()) write(     *     , nml=ensemble_manager_nml)
 
-! FIXME: may need a way to set up what the group size is.
-! perhaps get the size from the communicator...
 if (tasks_per_group == MPI_GROUPS_NOT_SET) then
    tasks_per_group = task_count()
 endif
@@ -394,9 +394,9 @@
 endif
 
 ! FIXME: does this go in set_up_ens_distribution()
-call set_group_size( tasks_per_group )
-call create_groups(  tasks_per_group )
-call task_sync() !FIXME: JPH - do we need this sync??
+call set_group_size(tasks_per_group)
+call create_groups(tasks_per_group)
+call task_sync()
    
 end subroutine static_init_ensemble_manager
 
@@ -1543,9 +1543,6 @@
 !  then if there are more than 1 group, have each task broadcast their
 !  copies to their corresponding task numbers in each of the other groups.
 
- 
-!mean_ens_handle%copies(1,:) = state_ens_handle%copies(mean_copy, :)
-
 ! Short var definitions ens 1
 num_copies1    = ens_handle1%num_copies
 num_vars1      = ens_handle1%num_vars
@@ -1572,6 +1569,7 @@
 max_num_vars2   = get_max_num_vars(ens_handle2)
 
 call task_sync()
+
 print*, 'num_copies1    ', my_task_id(), num_copies1    
 print*, 'num_vars1      ', my_task_id(), num_vars1      
 print*, 'my_num_vars1   ', my_task_id(), my_num_vars1   
@@ -1593,24 +1591,22 @@
 
 allocate(var_list1(my_num_vars1), var_list2(my_num_vars2), &
          var_list(max_num_vars1), transfer_temp(my_num_vars1*group_size/task_count()))
-allocate(copy_list1(1), copy_list2(1), &
-         copy_list(1))