[Dart-dev] DART/branches Revision: 12458

[dart at ucar.edu]

nancy at ucar.edu
2018-03-21 16:03:28 -0600 (Wed, 21 Mar 2018)
372
incremental updates to the ensemble manager - moved num_pes into
the handle, moved the one-time-only initialization into a static
init routine.  wrote down some ideas for how all_copies_to_all_copies
might work.  compiles but obviously can't run.   once we have the
interfaces set, we'll need to update the null_mpi_utilities_mod.f90
so things compile without mpi again.




Modified: DART/branches/rma_distrib/assimilation_code/modules/assimilation/filter_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/assimilation/filter_mod.f90	2018-03-21 20:23:49 UTC (rev 12457)
+++ DART/branches/rma_distrib/assimilation_code/modules/assimilation/filter_mod.f90	2018-03-21 22:03:28 UTC (rev 12458)
@@ -44,12 +44,12 @@
 
 use ensemble_manager_mod,  only : init_ensemble_manager, end_ensemble_manager,                &
                                   ensemble_type, get_copy, get_my_num_copies,                 &
-                                  compute_copy_mean, compute_copy_mean_sd,                    &
+                                  compute_copy_mean, compute_copy_mean_sd, get_my_vars,       &
                                   get_copy_owner_index, get_ensemble_time, set_ensemble_time, &
                                   map_task_to_pe,  map_pe_to_task, prepare_to_update_copies,  &
                                   copies_in_window, set_num_extra_copies, get_allow_transpose, &
                                   allocate_single_copy, allocate_vars, deallocate_single_copy, &
-                                  all_copies_to_all_vars, all_vars_to_all_copies
+                                  all_copies_to_all_vars, all_vars_to_all_copies, get_my_num_vars
                                   
 
 use adaptive_inflate_mod,  only : do_varying_ss_inflate, mean_from_restart, sd_from_restart,  &
@@ -1888,11 +1888,16 @@
 
 character*12 :: task
 integer :: j, i
-integer :: forward_unit
+integer :: forward_unit, nvars
+integer(i8), allocatable :: my_global_varnums(:)
 
+
+nvars = get_my_num_vars(qc_ens_handle)
+if (nvars <= 0) return
+
 write(task, '(i6.6)') my_task_id()
 
-! all tasks open file?
+! all tasks open their own files?
 if(prior_post == PRIOR_DIAG) then
    forward_unit = open_file('prior_forward_ope_errors' // task, 'formatted', 'append')
 else
@@ -1899,13 +1904,20 @@
    forward_unit = open_file('post_forward_ope_errors' // task, 'formatted', 'append')
 endif
 
+! get the global obs numbers, not the local ones
+allocate(my_global_varnums(nvars))
+call get_my_vars(qc_ens_handle, my_global_varnums)
+
 ! qc_ens_handle is a real representing an integer; values /= 0 get written out
+!write(forward_unit, *) 'ensemble #, obs #, global #, istatus'
 do i = 1, ens_size
-   do j = 1, qc_ens_handle%my_num_vars
-      if(nint(qc_ens_handle%copies(i, j)) /= 0) write(forward_unit, *) i, keys(j), nint(qc_ens_handle%copies(i, j))
+   do j = 1, nvars
+      if(nint(qc_ens_handle%copies(i, j)) /= 0) write(forward_unit, *) i, keys(j), my_global_varnums(j), nint(qc_ens_handle%copies(i, j))
    end do
 end do
 
+deallocate(my_global_varnums)
+
 call close_file(forward_unit)
 
 end subroutine verbose_forward_op_output

Modified: DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90
===================================================================
--- DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90	2018-03-21 20:23:49 UTC (rev 12457)
+++ DART/branches/rma_distrib/assimilation_code/modules/utilities/ensemble_manager_mod.f90	2018-03-21 22:03:28 UTC (rev 12458)
@@ -72,7 +72,7 @@
 use mpi_utilities_mod, only : task_count, my_task_id, send_to, receive_from, &
                               task_sync, broadcast_send, broadcast_recv,     &
                               create_groups, set_group_size, get_group_size, &
-                              get_group_id, get_group_comm
+                              get_group_id, get_group_comm, get_dart_mpi_comm
 
 
 
@@ -118,7 +118,7 @@
 !> so my_num_vars will remain expressible as an i4.
   
    integer                      :: num_pes
-   integer                      :: group_size
+   integer                      :: group_size   ! FIXME do we need this and num_pes at the same time?
    integer(i8)                  :: num_vars 
    integer                      :: num_copies, my_num_copies, my_num_vars
    integer,        allocatable  :: my_copies(:)
@@ -136,9 +136,11 @@
    integer, allocatable,private :: task_to_pe_list(:), pe_to_task_list(:) ! List of tasks
    ! Flexible my_pe, layout_type which allows different task layouts for different ensemble handles
    integer                      :: my_pe
+   integer                      :: my_group_pe   ! FIXME do we need this and num_pes at the same time?
    integer,private              :: layout_type
    integer,private              :: transpose_type
    integer                      :: num_extras   ! FIXME: this needs to be more general.
+   integer                      :: my_communicator ! FIXME: is this needed?
 end type ensemble_type
 
 ! FIXME: to separate filter use from plain storage issues,
@@ -158,12 +160,12 @@
 
 ! Module storage for pe information for this process avoids recomputation
 ! FIXME this might need to be per-ensemble as well?
-integer :: num_pes