[Dart-dev] DART/branches Revision: 12587

[dart at ucar.edu]

hendric at ucar.edu
2018-05-17 13:28:49 -0600 (Thu, 17 May 2018)
134

collecting a mask on each individual processor
to select from the use_only_qtys.

updating mkmf and pathnames to be mpi compatable.




Modified: DART/branches/rma_closest_member_tool/assimilation_code/programs/closest_member_tool/closest_member_tool.f90
===================================================================
--- DART/branches/rma_closest_member_tool/assimilation_code/programs/closest_member_tool/closest_member_tool.f90	2018-05-11 21:47:23 UTC (rev 12586)
+++ DART/branches/rma_closest_member_tool/assimilation_code/programs/closest_member_tool/closest_member_tool.f90	2018-05-17 19:28:49 UTC (rev 12587)
@@ -48,9 +48,10 @@
 use state_structure_mod,  only : get_num_domains
 
 use mpi_utilities_mod,    only : initialize_mpi_utilities, task_count,     &
-                                 finalize_mpi_utilities
+                                 finalize_mpi_utilities, my_task_id
 
-use ensemble_manager_mod, only : ensemble_type, init_ensemble_manager, compute_copy_mean_sd
+use ensemble_manager_mod, only : ensemble_type, init_ensemble_manager, compute_copy_mean, &
+                                 get_var_owner_index
 !>@todo needs to destroy ensemble before the end of the program
 
 implicit none
@@ -120,6 +121,9 @@
 character(len=256)              :: string1, my_base, my_desc
 integer                         :: ENS_SD_COPY, ENS_MEAN_COPY 
 
+real(r8), allocatable :: adiff(:), local_diffs(:)
+integer :: copies, vars, owner, owners_index
+
 !> @todo seems like you are using get_state_meta_data just so you can get the kinds.
 !> WRF and MPAS will be doing the vertical conversion also.
 
@@ -213,9 +217,9 @@
 ENS_SD_COPY   = ens_size + 1
 ENS_MEAN_COPY = ens_size + 2
 
-print*, 'compute copy mean sd'
+print*, 'compute copy mean'
 ! Compute mean and spread
-call compute_copy_mean_sd(ens_handle, 1, ens_size, ENS_MEAN_COPY, ENS_SD_COPY)
+call compute_copy_mean(ens_handle, 1, ens_size, ENS_MEAN_COPY)
 
 allocate(mean(model_size), member(model_size))
 allocate(index_list(ens_size), diffs(ens_size))
@@ -251,7 +255,7 @@
                          source,revision,revdate)
    endif
 
-   write(msgstring, *) 'Computing difference based only on items in state vector items of kind:'
+   write(msgstring, *) 'Computing difference based only on items in state vector items of quantity:'
    call error_handler(E_MSG,'',msgstring)
    do i=1, num_kinds
       if (usekind(i)) then
@@ -264,9 +268,6 @@
    allkinds = .true.
 endif
 
-print*, 'exiting early'
-call exit(0)
-
 ! if we are not processing all kinds of state vector items, set up a mask
 ! for the kinds we are.  do this once at the start so we don't replicate
 ! work.  the usekind(number_of_different_kinds) array says whether we are
@@ -273,7 +274,8 @@
 ! going to add differences for this type.   the useindex(state_vector_len)
 ! array is being precomputed here so it's fast to loop over the ensemble
 ! members and only add up differences for the kinds of interest.
-allocate(useindex(model_size))
+!#! allocate(useindex(model_size))
+allocate(useindex(ens_handle%my_num_vars))
 if (.not. allkinds) then
    useindex(:) = .false.
 
@@ -282,19 +284,21 @@
       !> @todo WRF and MPAS are doing vertical conversion here. Does this work
       !> if you have the non-mpi version.
       call get_state_meta_data(ii, loc, stype)
-      if (stype < 1 .or. stype > num_kinds) then
+      if (stype < 0 .or. stype > num_kinds) then
          write(msgstring, *) 'bad KIND from get_state_meta_data, ', stype, ' for index ', ii
-         write(msgstring1, *) 'must be between 1 and ', num_kinds
+         write(msgstring1, *) 'must be between 0 and ', num_kinds
          call error_handler(E_ERR,'closest_member_tool', msgstring, &
                             source,revision,revdate, text2=msgstring1)
 
       endif
-      if (usekind(stype)) then 
-         useindex(i) = .true.
+      call get_var_owner_index(ii, owner, owners_index)
+      if (usekind(stype) .and. my_task_id() == owner) then 
+         useindex(owners_index) = .true.
          j = j + 1
       endif
    enddo
 
+   print*, 'using ', j, ' of ', ens_handle%my_num_vars, ' items in the state vector', my_task_id()
    write(msgstring, *) 'using ', j, ' of ', model_size, ' items in the state vector'
    call error_handler(E_MSG,'closest_member_tool', msgstring)
 else
@@ -302,6 +306,8 @@
    useindex(:) = .true.
 endif
 
+!#! print*, 'task id, useindex', my_task_id(), useindex(:), task_count()
+
 !>@todo FIXME:  do we need 2 versions?  a distributed one where we compute the local diffs