[Dart-dev] DART/branches Revision: 12588

[dart at ucar.edu]

hendric at ucar.edu
2018-05-17 16:33:06 -0600 (Thu, 17 May 2018)
116

adding a mpi_allreduce for an array of number.  this
is currently not working as expected but compiles and
runs.





Modified: DART/branches/rma_closest_member_tool/assimilation_code/modules/utilities/mpi_utilities_mod.f90
===================================================================
--- DART/branches/rma_closest_member_tool/assimilation_code/modules/utilities/mpi_utilities_mod.f90	2018-05-17 19:28:49 UTC (rev 12587)
+++ DART/branches/rma_closest_member_tool/assimilation_code/modules/utilities/mpi_utilities_mod.f90	2018-05-17 22:33:06 UTC (rev 12588)
@@ -89,6 +89,7 @@
    module procedure sum_across_tasks_int4
    module procedure sum_across_tasks_int8
    module procedure sum_across_tasks_real
+   module procedure sum_across_tasks_real_array
 end interface
 
 !   ---- private data for mpi_utilities ----
@@ -1492,7 +1493,38 @@
 
 end subroutine sum_across_tasks_real
 
+!-----------------------------------------------------------------------------
+subroutine sum_across_tasks_real_array(addend, sum)
+ real(r8), intent(in) :: addend(:)
+ real(r8), intent(out) :: sum(:)
 
+ integer :: errcode
+ real(r8), allocatable :: localaddend(:), localsum(:)
+
+! cover routine for MPI all-reduce
+
+if ( .not. module_initialized ) then
+   write(errstring, *) 'initialize_mpi_utilities() must be called first'
+   call error_handler(E_ERR,'sum_across_tasks', errstring, source, revision, revdate)
+endif
+
+allocate(localaddend(size(addend)), localsum(size(addend)))
+localaddend(:) = addend(:)
+
+if (.true.) write(*,*) "PE", myrank, ": Allreduce called"
+
+call MPI_Allreduce(localaddend, localsum, 1, datasize, MPI_SUM, &
+                   0,my_local_comm, errcode)
+if (errcode /= MPI_SUCCESS) then
+   write(errstring, '(a,i8)') 'MPI_Allreduce returned error code ', errcode
+   call error_handler(E_ERR,'sum_across_tasks', errstring, source, revision, revdate)
+endif
+
+sum = localsum
+
+end subroutine sum_across_tasks_real_array
+
+
 !-----------------------------------------------------------------------------
 ! pipe-related utilities
 !-----------------------------------------------------------------------------

Modified: DART/branches/rma_closest_member_tool/assimilation_code/programs/closest_member_tool/closest_member_tool.f90
===================================================================
--- DART/branches/rma_closest_member_tool/assimilation_code/programs/closest_member_tool/closest_member_tool.f90	2018-05-17 19:28:49 UTC (rev 12587)
+++ DART/branches/rma_closest_member_tool/assimilation_code/programs/closest_member_tool/closest_member_tool.f90	2018-05-17 22:33:06 UTC (rev 12588)
@@ -48,7 +48,8 @@
 use state_structure_mod,  only : get_num_domains
 
 use mpi_utilities_mod,    only : initialize_mpi_utilities, task_count,     &
-                                 finalize_mpi_utilities, my_task_id
+                                 finalize_mpi_utilities, my_task_id, &
+                                 sum_across_tasks
 
 use ensemble_manager_mod, only : ensemble_type, init_ensemble_manager, compute_copy_mean, &
                                  get_var_owner_index
@@ -119,10 +120,10 @@
 logical                         :: read_time_from_file = .true.
 type(stage_metadata_type)       :: input_restart_files
 character(len=256)              :: string1, my_base, my_desc
-integer                         :: ENS_SD_COPY, ENS_MEAN_COPY 
+integer                         :: ENS_MEAN_COPY 
 
-real(r8), allocatable :: adiff(:), local_diffs(:)
-integer :: copies, vars, owner, owners_index
+real(r8), allocatable :: adiff(:), local_diffs(:), total_diff(:)
+integer  :: copies, vars, owner, owners_index, addend
 
 !> @todo seems like you are using get_state_meta_data just so you can get the kinds.
 !> WRF and MPAS will be doing the vertical conversion also.
@@ -158,7 +159,7 @@
 
 ! make space for the mean and a single member, plus place to sort list 
 ! for output
-call init_ensemble_manager(ens_handle,      ens_size+2, model_size)
+call init_ensemble_manager(ens_handle,      ens_size+1, model_size)
 call init_ensemble_manager(mean_ens_handle, 1,        model_size)
 
 mean_time         = set_time_missing()
@@ -214,14 +215,12 @@
 
 call read_state(ens_handle, ens_file_info,       .false., member_time)
 !#! call read_state(mean_ens_handle, mean_file_info, .false., mean_time)
-ENS_SD_COPY   = ens_size + 1
-ENS_MEAN_COPY = ens_size + 2
+ENS_MEAN_COPY = ens_size + 1
 
 print*, 'compute copy mean'
 ! Compute mean and spread
 call compute_copy_mean(ens_handle, 1, ens_size, ENS_MEAN_COPY)