[Dart-dev] DART/branches Revision: 11705

[dart at ucar.edu]

hendric at ucar.edu
2017-06-09 10:43:06 -0600 (Fri, 09 Jun 2017)
67
parallel read seems to be working for reading ensemble
members.  




Modified: DART/branches/rma_par_single_file/assimilation_code/modules/io/direct_netcdf_mod.f90
===================================================================
--- DART/branches/rma_par_single_file/assimilation_code/modules/io/direct_netcdf_mod.f90	2017-06-08 16:26:31 UTC (rev 11704)
+++ DART/branches/rma_par_single_file/assimilation_code/modules/io/direct_netcdf_mod.f90	2017-06-09 16:43:06 UTC (rev 11705)
@@ -401,7 +401,7 @@
    if (copies_read >= ens_size) exit
 
    ! what to do if a variable is larger than the memory limit?
-   start_var = 1 ! read first variable first
+   start_var = 1 ! read first variable first !start_pos
    istart    = dart_index ! position in state_ens_handle%copies
 
    my_copy = copies_read + my_pe + 1
@@ -1325,6 +1325,8 @@
 
 !-------------------------------------------------
 !> Find pes for loop indices
+!> if you have M tasks and N ensemble members, only the first
+!> N + extra copy tasks will send, everyone will receive.
 
 
 subroutine get_pe_loops(ens_size, recv_start, recv_end, send_start, send_end)
@@ -1331,13 +1333,12 @@
 
 integer, intent(in)    :: ens_size
 integer, intent(out)   :: recv_start !< for RECIEVING_PE_LOOP
-integer, intent(out)   :: recv_end !< for RECIEVING_PE_LOOP
+integer, intent(out)   :: recv_end   !< for RECIEVING_PE_LOOP
 integer, intent(out)   :: send_start !< for RECEIVE_FROM_EACH_LOOP
-integer, intent(out)   :: send_end !< for RECEIVE_FROM_EACH_LOOP
+integer, intent(out)   :: send_end   !< for RECEIVE_FROM_EACH_LOOP
 
-
 recv_start = 0
-recv_end = task_count() -1
+recv_end   = task_count() -1
 
 send_start  = recv_start
 
@@ -1379,11 +1380,12 @@
 if (state_ens_handle%distribution_type == 1) then
 
    !>@todo MPI vector data type or packing could/should be used here?
+   
+   ! need a contiguous array to send variables with MPI
    allocate(buffer(elm_count))
    buffer = variable_block(start:elm_count*task_count():task_count())
    print*, 'MAP :: size -', map_pe_to_task(state_ens_handle, recv_pe), ':: size(buffer)', size(buffer)
-   call send_to(map_pe_to_task(state_ens_handle, recv_pe), &
-                           buffer)
+   call send_to(map_pe_to_task(state_ens_handle, recv_pe), buffer)
    deallocate(buffer)
 else
 
@@ -1553,7 +1555,7 @@
 
 integer :: get_start_rank
 
-get_start_rank = mod(get_sum_variables_below(variable, domain), task_count())+my_task_id()
+get_start_rank = mod(get_sum_variables_below(variable, domain), task_count())
 
 end function get_start_rank
 

Modified: DART/branches/rma_par_single_file/assimilation_code/modules/io/single_file_io_mod.f90
===================================================================
--- DART/branches/rma_par_single_file/assimilation_code/modules/io/single_file_io_mod.f90	2017-06-08 16:26:31 UTC (rev 11704)
+++ DART/branches/rma_par_single_file/assimilation_code/modules/io/single_file_io_mod.f90	2017-06-09 16:43:06 UTC (rev 11705)
@@ -395,15 +395,16 @@
 
 ! NetCDF IO variables
 integer :: my_ncid, varid, MemDimID, TimeDimID, ret, icopy, ivar, jdim, istart, iend
-integer :: ens_size, member_size, elm_count, time_size, var_size, domain, sending_pe, my_pe
+integer :: member_size, elm_count, time_size, var_size, domain, sending_pe, my_pe
 integer :: ndims, member_dim, dcount, start_pos, end_pos, start_rank, end_rank
 integer :: recv_pe, recv_start, recv_end, send_start, send_end, start_point, end_point
+integer :: ens_size, extra_size
 real(r8), allocatable :: var_block(:)
 character(len=NF90_MAX_NAME) :: fname, dimname, varname, copyname, extraname
 integer, dimension(NF90_MAX_VAR_DIMS) :: dim_lengths
 integer, dimension(NF90_MAX_VAR_DIMS) :: dim_start_point
-logical :: do_perturb, has_member_dim, is_reader
-integer :: ensemble_member, copies_read, my_copy
+logical :: do_perturb, has_member_dim, is_reader, is_extra_copy
+integer :: ens_mem, copies_read, my_copy, start_var, num_state_variables
 ! check whether file_info handle is initialized
 ! if (task_count() > 1) then
 !    call error_handler(E_ERR,'read_singlefile', &
@@ -420,15 +421,17 @@
 ! do this once
 name_handle = get_stage_metadata(file_info)
 
-ens_size = state_ens_handle%num_copies
+ens_size   = state_ens_handle%num_copies - state_ens_handle%num_extras
+extra_size = state_ens_handle%num_extras
 
+! recv_* and send_* are PE's that variables are sent and received
 call get_pe_loops(ens_size, recv_start, recv_end, send_start, send_end)
 
 print*, '****************************************************************************************** '
 write(*,'(6A15)') &