[Dart-dev] DART/branches Revision: 12696

[dart at ucar.edu]

bdobbins at ucar.edu
2018-06-29 09:28:01 -0600 (Fri, 29 Jun 2018)
201
More hackish changes, but now the comms only send the right amount (by doing extra computations before the loop)
Comm time is down (when in synchronous mode), but up in asyncrhonous - not yet sure why



Modified: DART/branches/graph_coloring/assimilation_code/modules/assimilation/assim_graph_tools_mod.f90
===================================================================
--- DART/branches/graph_coloring/assimilation_code/modules/assimilation/assim_graph_tools_mod.f90	2018-06-28 18:52:55 UTC (rev 12695)
+++ DART/branches/graph_coloring/assimilation_code/modules/assimilation/assim_graph_tools_mod.f90	2018-06-29 15:28:01 UTC (rev 12696)
@@ -51,7 +51,7 @@
 
 use mpi_utilities_mod,    only : my_task_id, broadcast_send, broadcast_recv,              & 
                                  sum_across_tasks, task_count, start_mpi_timer,           &
-                                 read_mpi_timer, task_sync
+                                 read_mpi_timer, task_sync, task_sync_filter, task_sync_final
 
 use adaptive_inflate_mod, only : do_obs_inflate,  do_single_ss_inflate,                   &
                                  do_varying_ss_inflate,                                   &
@@ -131,10 +131,12 @@
 !   integer, dimension(:), allocatable :: owner       ! Rank that owns this observation (computed from data)
 end type colors_type
 
-integer, parameter :: max_chunk_size = 8! for now
+integer, parameter :: max_chunk_size = 128! for now
 type chunk_type
     integer :: num_obs
     integer :: owner
+    integer :: comm_obs
+    integer :: comm_size
 
     integer, dimension(max_chunk_size) :: obs_list
 end type chunk_type
@@ -151,6 +153,8 @@
    !type(location_type), dimension(:), allocatable  :: base_obs_loc
 
    real(r8), dimension(:), allocatable :: bcast_buffer
+   integer :: comm_obs
+   integer :: comm_size
 end type chunk_data_type
 
 !============================================================================
@@ -507,7 +511,9 @@
 !integer, dimension(:), allocatable :: obs_list
 type(chunk_type), dimension(:), allocatable :: chunks
 
+integer :: original_chunk_size
 
+
 type(chunk_data_type) chunk_data
 
 integer (kind=8) :: ob_index
@@ -760,6 +766,27 @@
 if (sync_between_timers) then
   call task_sync()
 endif
+
+  !! New functionality - compute the num obs on ALL procs, outside the loop
+  call t_startf('QC_Prep_Loop')
+  do i = 1, size(chunks)
+    original_chunk_size = chunks(i)%num_obs
+    chunks(i)%comm_obs = chunks(i)%num_obs
+    do j = 1, original_chunk_size
+       ob_index = chunks(i)%obs_list(j)
+       owner = chunks(i)%owner
+       owners_index = graph_owners_index(chunks, i, j) 
+       obs_qc = obs_ens_handle%copies(OBS_GLOBAL_QC_COPY, owners_index)
+       call MPI_Bcast(obs_qc, 1, MPI_DOUBLE_PRECISION, owner, MPI_COMM_WORLD, iError) 
+       ! Now everyone has the QC value
+       if (nint(obs_qc) /= 0) then
+           chunks(i)%comm_obs = chunks(i)%comm_obs - 1
+       endif
+     end do
+     chunks(i)%comm_size = (chunks(i)%comm_obs * ens_size * 2) + (chunks(i)%comm_obs * num_groups) + (chunks(i)%comm_obs * 3) + 1 
+  end do
+  call t_stopf('QC_Prep_Loop')
+
 call t_startf('Full Loop')
 
 CHUNK_LOOP: do i = 1, size(chunks)
@@ -776,9 +803,11 @@
    endif
 
     chunk_data%num_obs = chunks(i)%num_obs
+    chunk_data%comm_obs = chunks(i)%comm_obs
+    chunk_data%comm_size = chunks(i)%comm_size
    ! This section is only done by one process - the 'owner' of this chunk:
    if (sync_between_timers) then
-       call task_sync()
+       call task_sync_filter()
    endif
    if (ens_handle%my_pe == chunks(i)%owner) then
       call t_startf('ASSIMILATE:Owned(Compute)')
@@ -807,7 +836,7 @@
 
         ! Only value of 0 for DART QC field should be assimilated
         IF_QC_IS_OKAY: if(nint(chunk_data%obs_qc(j)) ==0 ) then
-           chunk_data%obs_prior(j,:) = obs_ens_handle%copies(1:ens_size, owners_index)
+           chunk_data%obs_prior(:,j) = obs_ens_handle%copies(1:ens_size, owners_index)
 
            do group = 1, num_groups
               grp_bot = grp_beg(group)
@@ -832,7 +861,7 @@
            do group = 1, num_groups
               grp_bot = grp_beg(group)
               grp_top = grp_end(group)
-              call obs_increment(chunk_data%obs_prior(j,grp_bot:grp_top), grp_size, obs(1), obs_err_var, chunk_data%obs_inc(j,grp_bot:grp_top), inflate, my_inflate, my_inflate_sd, chunk_data%net_a(j,group))