[Dart-dev] DART/branches Revision: 12403

[dart at ucar.edu]

hendric at ucar.edu
2018-02-15 15:35:09 -0700 (Thu, 15 Feb 2018)
161
adding back in tests for distributing static data using 'sudo' groups.
test routines are working and would like to use actual groups instead
of logical groups.




Added: DART/branches/rma_static_data/models/POP/distributed_static_data_X_mod.f90
===================================================================
--- DART/branches/rma_static_data/models/POP/distributed_static_data_X_mod.f90	                        (rev 0)
+++ DART/branches/rma_static_data/models/POP/distributed_static_data_X_mod.f90	2018-02-15 22:35:09 UTC (rev 12403)
@@ -0,0 +1,297 @@
+!> Aim: to replace the array phb, with a distributed version
+!> Current thinking: Only distribute the array as much as necessary.
+!> Ideally, if memory was unliimited, every task would have the whole array.
+!>
+!> Splitting in x only, because this is simple, and I can't think
+!> of a reason to split in more than one dimension yet.
+
+module distributed_static_data_mod
+
+use utilities_mod,         only : register_module,          &
+                                  error_handler,            &
+                                  E_ERR, E_MSG, E_ALLMSG,   &
+                                  find_namelist_in_file,    &
+                                  check_namelist_read,      &
+                                  do_nml_file, do_nml_term, &
+                                  nmlfileunit, do_output
+
+use mpi_utilities_mod,     only : datasize, my_task_id,     &
+                                  task_count, send_to,      &
+                                  receive_from , task_sync
+
+use types_mod,             only : r8
+
+use mpi
+
+implicit none
+private
+
+public :: create_window, free_window
+
+!!! only public at the moment for testing
+public :: create_groups, build_my_group, initialize_static_data_space, &
+          distribute_static_data, get_static_data
+
+! grid window
+real(r8), allocatable :: global_static_data(:) !< local static data 
+
+integer  :: sd_window
+
+integer :: group_handle   !< mpi_comm_world group
+integer :: subgroup       !< subgroup for the grid
+integer :: mpi_comm_grid
+integer :: local_rank     !< rank within group
+integer, allocatable :: group_members(:)
+
+! pntecdf variables
+integer(KIND=MPI_OFFSET_KIND) :: x_start, x_length, y_length, num_static_arrays
+integer(KIND=MPI_OFFSET_KIND) :: start(4)
+integer(KIND=MPI_OFFSET_KIND) :: count(4)
+integer(KIND=MPI_OFFSET_KIND) :: stride(4)
+integer(KIND=MPI_OFFSET_KIND) :: my_num_x_vals !< splitting up by we
+
+integer :: array_number = 0 ! starting index for array
+
+!------------------------------------------------------------------
+! namelist variables
+!------------------------------------------------------------------
+integer              :: group_size = 1
+
+namelist /distributed_static_data_nml/  group_size
+
+contains
+
+!-------------------------------------------------------------
+!> create groups for grid
+!> make a communicator for the distributed grid
+subroutine create_groups()
+
+integer ierr ! all MPI errors are fatal anyway
+
+allocate(group_members(group_size)) ! this is module global
+
+call mpi_comm_group(mpi_comm_world, group_handle, ierr)  ! get group handle from mpi_comm_world
+call build_my_group(group_size, group_members) ! create a list of processors in the grid group
+call mpi_group_incl(group_handle, group_size, group_members, subgroup, ierr)
+call mpi_comm_create(mpi_comm_world, subgroup, mpi_comm_grid, ierr)
+call mpi_comm_rank(mpi_comm_grid, local_rank, ierr) ! rank within group
+
+end subroutine create_groups
+
+!-----------------------------------------------------------
+!> build the group to store the grid
+subroutine build_my_group(group_size, group_members)
+
+! need to modify this if your #tasks does not divide by group size
+integer, intent(inout) :: group_size 
+integer, intent(out)   :: group_members(group_size)
+
+integer :: bottom, top !< start and end members of the group
+integer :: i 
+integer :: myrank
+
+myrank = my_task_id()
+
+! integer arithmatic. rouding down to the lowest group size
+bottom = (myrank / group_size ) * group_size