<p><b>dwj07@fsu.edu</b> 2012-01-31 12:24:21 -0700 (Tue, 31 Jan 2012)</p><p><br>
-- BRANCH COMMIT --<br>
<br>
Implementing namelist changes, and registry changes.<br>
<br>
Implementing ad-hoc method of distributing multiple blocks.<br>
<br>
Comments are in place to test the distribution of blocks, and make sure the output from mpas_block_decomp is correct.<br>
<br>
Still need to add explicit block distribution method.<br>
</p><hr noshade><pre><font color="gray">Modified: branches/omp_blocks/block_decomp/namelist.input
===================================================================
--- branches/omp_blocks/block_decomp/namelist.input 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/namelist.input 2012-01-31 19:24:21 UTC (rev 1438)
@@ -1 +1 @@
-link namelist.input.sw
\ No newline at end of file
+link namelist.input.ocean
\ No newline at end of file
Modified: branches/omp_blocks/block_decomp/namelist.input.hyd_atmos
===================================================================
--- branches/omp_blocks/block_decomp/namelist.input.hyd_atmos 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/namelist.input.hyd_atmos 2012-01-31 19:24:21 UTC (rev 1438)
@@ -28,6 +28,13 @@
config_frames_per_outfile = 0
/
+&decomposition
+ config_number_of_blocks = 0
+ config_block_decomp_file_prefix = 'graph.info.part.'
+ config_explicit_proc_decomp = .false.
+ config_proc_decomp_file_prefix = 'graph.info.part.'
+/
+
&restart
config_restart_interval = '1000_00:00:00'
config_do_restart = .false.
Modified: branches/omp_blocks/block_decomp/namelist.input.init_nhyd_atmos
===================================================================
--- branches/omp_blocks/block_decomp/namelist.input.init_nhyd_atmos 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/namelist.input.init_nhyd_atmos 2012-01-31 19:24:21 UTC (rev 1438)
@@ -36,9 +36,15 @@
&io
config_input_name = 'x1.40962.geogrid.nc'
config_output_name = 'x1.40962.init.2010-10-23.nc'
- config_decomp_file_prefix = 'x1.40962.graph.info.part.'
/
+&decomposition
+ config_number_of_blocks = 0
+ config_block_decomp_file_prefix = 'x1.40962.graph.info.part.'
+ config_explicit_proc_decomp = .false.
+ config_proc_decomp_file_prefix = 'graph.info.part.'
+/
+
&restart
config_restart_interval = 3000
config_do_restart = .false.
Modified: branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos
===================================================================
--- branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos 2012-01-31 19:24:21 UTC (rev 1438)
@@ -46,10 +46,16 @@
config_restart_name = 'restart.nc'
config_output_interval = '1_00:00:00'
config_frames_per_outfile = 1
- config_decomp_file_prefix = 'x1.40962.graph.info.part.'
/
config_sfc_update_name = 'sfc_update.nc'
+&decomposition
+ config_number_of_blocks = 0
+ config_block_decomp_file_prefix = 'x1.40962.graph.info.part.'
+ config_explicit_proc_decomp = .false.
+ config_proc_decomp_file_prefix = 'graph.info.part.'
+/
+
&restart
config_restart_interval = '1_00:00:00'
config_do_restart = .false.
Modified: branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos_jw
===================================================================
--- branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos_jw 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos_jw 2012-01-31 19:24:21 UTC (rev 1438)
@@ -44,6 +44,13 @@
config_restart_name = 'restart.nc'
/
+&decomposition
+ config_number_of_blocks = 0
+ config_block_decomp_file_prefix = 'graph.info.part.'
+ config_explicit_proc_decomp = .false.
+ config_proc_decomp_file_prefix = 'graph.info.part.'
+/
+
&restart
config_restart_interval = 3000
config_do_restart = .false.
Modified: branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos_mtn_wave
===================================================================
--- branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos_mtn_wave 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos_mtn_wave 2012-01-31 19:24:21 UTC (rev 1438)
@@ -39,6 +39,13 @@
config_restart_name = 'restart.nc'
/
+&decomposition
+ config_number_of_blocks = 0
+ config_block_decomp_file_prefix = 'graph.info.part.'
+ config_explicit_proc_decomp = .false.
+ config_proc_decomp_file_prefix = 'graph.info.part.'
+/
+
&restart
config_restart_interval = 3000
config_do_restart = .false.
Modified: branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos_squall
===================================================================
--- branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos_squall 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/namelist.input.nhyd_atmos_squall 2012-01-31 19:24:21 UTC (rev 1438)
@@ -33,6 +33,13 @@
config_restart_name = 'restart.nc'
/
+&decomposition
+ config_number_of_blocks = 0
+ config_block_decomp_file_prefix = 'graph.info.part.'
+ config_explicit_proc_decomp = .false.
+ config_proc_decomp_file_prefix = 'graph.info.part.'
+/
+
&restart
config_restart_interval = 3000
config_do_restart = .false.
Modified: branches/omp_blocks/block_decomp/namelist.input.ocean
===================================================================
--- branches/omp_blocks/block_decomp/namelist.input.ocean 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/namelist.input.ocean 2012-01-31 19:24:21 UTC (rev 1438)
@@ -14,6 +14,12 @@
config_output_interval = '20_00:00:00'
config_frames_per_outfile = 1000000
/
+&decomposition
+ config_number_of_blocks = 0
+ config_block_decomp_file_prefix = 'graph.info.part.'
+ config_explicit_proc_decomp = .false.
+ config_proc_decomp_file_prefix = 'graph.info.part.'
+/
&restart
config_do_restart = .false.
config_restart_interval = '120_00:00:00'
Modified: branches/omp_blocks/block_decomp/namelist.input.sw
===================================================================
--- branches/omp_blocks/block_decomp/namelist.input.sw 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/namelist.input.sw 2012-01-31 19:24:21 UTC (rev 1438)
@@ -26,7 +26,12 @@
config_output_interval = '1_00:00:00'
config_frames_per_outfile = 0
/
-
+&decomposition
+ config_number_of_blocks = 0
+ config_block_decomp_file_prefix = 'graph.info.part.'
+ config_explicit_proc_decomp = .false.
+ config_proc_decomp_file_prefix = 'graph.info.part.'
+/
&restart
config_restart_interval = '15_00:00:00'
config_do_restart = .false.
Modified: branches/omp_blocks/block_decomp/src/core_hyd_atmos/Registry
===================================================================
--- branches/omp_blocks/block_decomp/src/core_hyd_atmos/Registry 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/src/core_hyd_atmos/Registry 2012-01-31 19:24:21 UTC (rev 1438)
@@ -27,7 +27,10 @@
namelist character io config_restart_name restart.nc
namelist character io config_output_interval 06:00:00
namelist integer io config_frames_per_outfile 0
-namelist character io config_decomp_file_prefix graph.info.part.
+namelist character decomposition config_block_decomp_file_prefix graph.info.part.
+namelist integer decomposition config_number_of_blocks 0
+namelist logical decomposition config_explicit_proc_decomp .false.
+namelist character decomposition config_proc_decomp_file_prefix graph.info.part.
namelist logical restart config_do_restart false
namelist character restart config_restart_interval none
Modified: branches/omp_blocks/block_decomp/src/core_init_nhyd_atmos/Registry
===================================================================
--- branches/omp_blocks/block_decomp/src/core_init_nhyd_atmos/Registry 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/src/core_init_nhyd_atmos/Registry 2012-01-31 19:24:21 UTC (rev 1438)
@@ -29,8 +29,11 @@
namelist character io config_sfc_update_name sfc_update.nc
namelist character io config_output_name output.nc
namelist character io config_restart_name restart.nc
-namelist character io config_decomp_file_prefix graph.info.part.
namelist integer io config_frames_per_outfile 0
+namelist character decomposition config_block_decomp_file_prefix graph.info.part.
+namelist integer decomposition config_number_of_blocks 0
+namelist logical decomposition config_explicit_proc_decomp .false.
+namelist character decomposition config_proc_decomp_file_prefix graph.info.part.
namelist integer restart config_restart_interval 0
namelist logical restart config_do_restart false
namelist real restart config_restart_time 172800.0
Modified: branches/omp_blocks/block_decomp/src/core_nhyd_atmos/Registry
===================================================================
--- branches/omp_blocks/block_decomp/src/core_nhyd_atmos/Registry 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/src/core_nhyd_atmos/Registry 2012-01-31 19:24:21 UTC (rev 1438)
@@ -45,7 +45,10 @@
namelist character io config_restart_name restart.nc
namelist character io config_output_interval 06:00:00
namelist integer io config_frames_per_outfile 0
-namelist character io config_decomp_file_prefix graph.info.part.
+namelist character decomposition config_block_decomp_file_prefix graph.info.part.
+namelist integer decomposition config_number_of_blocks 0
+namelist logical decomposition config_explicit_proc_decomp .false.
+namelist character decomposition config_proc_decomp_file_prefix graph.info.part.
namelist logical restart config_do_restart false
namelist logical restart config_do_DAcycling false
namelist character restart config_restart_interval none
Modified: branches/omp_blocks/block_decomp/src/core_ocean/Registry
===================================================================
--- branches/omp_blocks/block_decomp/src/core_ocean/Registry 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/src/core_ocean/Registry 2012-01-31 19:24:21 UTC (rev 1438)
@@ -16,7 +16,10 @@
namelist character io config_restart_name restart.nc
namelist character io config_output_interval 24:00:00
namelist integer io config_frames_per_outfile 0
-namelist character io config_decomp_file_prefix graph.info.part.
+namelist character decomposition config_block_decomp_file_prefix graph.info.part.
+namelist integer decomposition config_number_of_blocks 0
+namelist logical decomposition config_explicit_proc_decomp .false.
+namelist character decomposition config_proc_decomp_file_prefix graph.info.part.
namelist logical restart config_do_restart false
namelist character restart config_restart_interval none
namelist character grid config_vert_grid_type isopycnal
Modified: branches/omp_blocks/block_decomp/src/core_sw/Registry
===================================================================
--- branches/omp_blocks/block_decomp/src/core_sw/Registry 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/src/core_sw/Registry 2012-01-31 19:24:21 UTC (rev 1438)
@@ -26,7 +26,10 @@
namelist character io config_restart_name restart.nc
namelist character io config_output_interval 06:00:00
namelist integer io config_frames_per_outfile 0
-namelist character io config_decomp_file_prefix graph.info.part.
+namelist character decomposition config_block_decomp_file_prefix graph.info.part.
+namelist integer decomposition config_number_of_blocks 0
+namelist logical decomposition config_explicit_proc_decomp .false.
+namelist character decomposition config_proc_decomp_file_prefix graph.info.part.
namelist logical restart config_do_restart false
namelist character restart config_restart_interval none
Modified: branches/omp_blocks/block_decomp/src/framework/mpas_block_decomp.F
===================================================================
--- branches/omp_blocks/block_decomp/src/framework/mpas_block_decomp.F 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/src/framework/mpas_block_decomp.F 2012-01-31 19:24:21 UTC (rev 1438)
@@ -2,6 +2,7 @@
use mpas_dmpar
use mpas_hash
+ use mpas_sort
type graph
integer :: nVerticesTotal
@@ -12,115 +13,195 @@
integer, dimension(:,:), pointer :: adjacencyList
end type graph
+ integer :: total_blocks
contains
+ subroutine mpas_block_decomp_cells_for_proc(dminfo, partial_global_graph_info, local_cell_list, block_id, block_start, block_count)!{{{
- subroutine mpas_block_decomp_cells_for_proc(dminfo, partial_global_graph_info, local_cell_list)
-
use mpas_configure
implicit none
- type (dm_info), intent(in) :: dminfo
- type (graph), intent(in) :: partial_global_graph_info
- integer, dimension(:), pointer :: local_cell_list
+ type (dm_info), intent(in) :: dminfo !< Input: domain information
+ type (graph), intent(in) :: partial_global_graph_info !< Input: Global graph information
+ integer, dimension(:), pointer :: local_cell_list !< Output: list of cells this processor owns, ordered by block
+ integer, dimension(:), pointer :: block_id !< Output: list of global block id's this processor owns
+ integer, dimension(:), pointer :: block_start !< Output: offset in local_cell_list for this blocks list of cells
+ integer, dimension(:), pointer :: block_count !< Output: number of cells in blocks
+ integer, dimension(:), pointer :: global_block_list
integer, dimension(:), pointer :: global_cell_list
integer, dimension(:), pointer :: global_start
- integer :: i, j, owner, iunit, istatus
+ integer, dimension(:), allocatable :: local_block_list
+ integer, dimension(:,:), allocatable :: sorted_local_cell_list
+
+ integer :: i, j, global_block_id, local_block_id, owning_proc, iunit, istatus
+ integer :: blocks_per_proc
integer, dimension(:), pointer :: local_nvertices
character (len=256) :: filename
- if (dminfo % nprocs > 1) then
+ if(config_number_of_blocks == 0) then
+ total_blocks = dminfo % nProcs
+ else
+ total_blocks = config_number_of_blocks
+ end if
- allocate(local_nvertices(dminfo % nprocs))
- allocate(global_start(dminfo % nprocs))
- allocate(global_cell_list(partial_global_graph_info % nVerticesTotal))
+ call mpas_get_blocks_per_proc(dminfo, dminfo % my_proc_id, blocks_per_proc)
- if (dminfo % my_proc_id == IO_NODE) then
+ if(total_blocks > 1) then
+ allocate(local_nvertices(dminfo % nprocs))
+ allocate(global_start(dminfo % nprocs))
+ allocate(global_cell_list(partial_global_graph_info % nVerticesTotal))
+ allocate(global_block_list(partial_global_graph_info % nVerticesTotal))
+
+ if (dminfo % my_proc_id == IO_NODE) then
+
+ iunit = 50 + dminfo % my_proc_id
+ if (total_blocks < 10) then
+ write(filename,'(a,i1)') trim(config_block_decomp_file_prefix), total_blocks
+ else if (total_blocks < 100) then
+ write(filename,'(a,i2)') trim(config_block_decomp_file_prefix), total_blocks
+ else if (total_blocks < 1000) then
+ write(filename,'(a,i3)') trim(config_block_decomp_file_prefix), total_blocks
+ else if (total_blocks < 10000) then
+ write(filename,'(a,i4)') trim(config_block_decomp_file_prefix), total_blocks
+ else if (total_blocks < 100000) then
+ write(filename,'(a,i5)') trim(config_block_decomp_file_prefix), total_blocks
+ end if
+
+ open(unit=iunit, file=trim(filename), form='formatted', status='old', iostat=istatus)
+
+ if (istatus /= 0) then
+ write(0,*) 'Could not open block decomposition file for ',total_blocks,' blocks.'
+ write(0,*) 'Filename: ',trim(filename)
+ call mpas_dmpar_abort(dminfo)
+ end if
+
+ local_nvertices(:) = 0
+ do i=1,partial_global_graph_info % nVerticesTotal
+ read(unit=iunit, fmt=*) global_block_id
+ owning_proc = (global_block_id) / blocks_per_proc
+ local_nvertices(owning_proc+1) = local_nvertices(owning_proc+1) + 1
+ end do
+
+ allocate(global_cell_list(partial_global_graph_info % nVerticesTotal))
+
+ global_start(1) = 1
+ do i=2,dminfo % nprocs
+ global_start(i) = global_start(i-1) + local_nvertices(i-1)
+ end do
+
+ rewind(unit=iunit)
+
+ do i=1,partial_global_graph_info % nVerticesTotal
+ read(unit=iunit, fmt=*) global_block_id
+ owning_proc = (global_block_id) / blocks_per_proc
+
+ global_cell_list(global_start(owning_proc+1)) = i
+ global_block_list(global_start(owning_proc+1)) = global_block_id
+ global_start(owning_proc+1) = global_start(owning_proc+1) + 1
+ end do
+
+ global_start(1) = 0
+ do i=2,dminfo % nprocs
+ global_start(i) = global_start(i-1) + local_nvertices(i-1)
+ end do
- iunit = 50 + dminfo % my_proc_id
- if (dminfo % nprocs < 10) then
- write(filename,'(a,i1)') trim(config_decomp_file_prefix), dminfo % nprocs
- else if (dminfo % nprocs < 100) then
- write(filename,'(a,i2)') trim(config_decomp_file_prefix), dminfo % nprocs
- else if (dminfo % nprocs < 1000) then
- write(filename,'(a,i3)') trim(config_decomp_file_prefix), dminfo % nprocs
- else if (dminfo % nprocs < 10000) then
- write(filename,'(a,i4)') trim(config_decomp_file_prefix), dminfo % nprocs
- else if (dminfo % nprocs < 100000) then
- write(filename,'(a,i5)') trim(config_decomp_file_prefix), dminfo % nprocs
- end if
-
- open(unit=iunit, file=trim(filename), form='formatted', status='old', iostat=istatus)
-
- if (istatus /= 0) then
- write(0,*) 'Could not open block decomposition file for ',dminfo % nprocs,' tasks.'
- write(0,*) 'Filename: ',trim(filename)
- call mpas_dmpar_abort(dminfo)
- end if
-
- local_nvertices(:) = 0
- do i=1,partial_global_graph_info % nVerticesTotal
- read(unit=iunit, fmt=*) owner
- local_nvertices(owner+1) = local_nvertices(owner+1) + 1
- end do
-
-! allocate(global_cell_list(partial_global_graph_info % nVerticesTotal))
+ close(unit=iunit)
- global_start(1) = 1
- do i=2,dminfo % nprocs
- global_start(i) = global_start(i-1) + local_nvertices(i-1)
- end do
-
- rewind(unit=iunit)
-
- do i=1,partial_global_graph_info % nVerticesTotal
- read(unit=iunit, fmt=*) owner
- global_cell_list(global_start(owner+1)) = i
- global_start(owner+1) = global_start(owner+1) + 1
- end do
+ call mpas_dmpar_bcast_ints(dminfo, dminfo % nprocs, local_nvertices)
+ allocate(local_cell_list(local_nvertices(dminfo % my_proc_id + 1)))
+ allocate(local_block_list(local_nvertices(dminfo % my_proc_id + 1)))
- global_start(1) = 0
- do i=2,dminfo % nprocs
- global_start(i) = global_start(i-1) + local_nvertices(i-1)
- end do
+ call mpas_dmpar_scatter_ints(dminfo, dminfo % nprocs, local_nvertices(dminfo % my_proc_id + 1), &
+ global_start, local_nvertices, global_cell_list, local_cell_list)
- close(unit=iunit)
+ call mpas_dmpar_scatter_ints(dminfo, dminfo % nprocs, local_nvertices(dminfo % my_proc_id + 1), &
+ global_start, local_nvertices, global_block_list, local_block_list)
- call mpas_dmpar_bcast_ints(dminfo, dminfo % nprocs, local_nvertices)
- allocate(local_cell_list(local_nvertices(dminfo % my_proc_id + 1)))
+ else
- call mpas_dmpar_scatter_ints(dminfo, dminfo % nprocs, local_nvertices(dminfo % my_proc_id + 1), &
- global_start, local_nvertices, global_cell_list, local_cell_list)
+ call mpas_dmpar_bcast_ints(dminfo, dminfo % nprocs, local_nvertices)
+ allocate(local_cell_list(local_nvertices(dminfo % my_proc_id + 1)))
+ allocate(local_block_list(local_nvertices(dminfo % my_proc_id + 1)))
+
+ call mpas_dmpar_scatter_ints(dminfo, dminfo % nprocs, local_nvertices(dminfo % my_proc_id + 1), &
+ global_start, local_nvertices, global_cell_list, local_cell_list)
- else
+ call mpas_dmpar_scatter_ints(dminfo, dminfo % nprocs, local_nvertices(dminfo % my_proc_id + 1), &
+ global_start, local_nvertices, global_block_list, local_block_list)
+ end if
+
+ allocate(sorted_local_cell_list(2, local_nvertices(dminfo % my_proc_id + 1)))
+ allocate(block_id(blocks_per_proc))
+ allocate(block_start(blocks_per_proc))
+ allocate(block_count(blocks_per_proc))
- call mpas_dmpar_bcast_ints(dminfo, dminfo % nprocs, local_nvertices)
- allocate(local_cell_list(local_nvertices(dminfo % my_proc_id + 1)))
+ do i = 1, blocks_per_proc
+ block_start = 0
+ block_count = 0
+ end do
- call mpas_dmpar_scatter_ints(dminfo, dminfo % nprocs, local_nvertices(dminfo % my_proc_id + 1), &
- global_start, local_nvertices, global_cell_list, local_cell_list)
+ do i = 1,local_nvertices(dminfo % my_proc_id +1)
+ call mpas_get_local_block_id(dminfo, local_block_list(i), local_block_id)
+
+ block_id(local_block_id+1) = local_block_list(i)
+
+ sorted_local_cell_list(1, i) = local_block_list(i)
+ sorted_local_cell_list(2, i) = local_cell_list(i)
+
+ block_count(local_block_id+1) = block_count(local_block_id+1) + 1
+ end do
- end if
+ call quicksort(local_nvertices(dminfo % my_proc_id + 1), sorted_local_cell_list)
- deallocate(local_nvertices)
- deallocate(global_start)
- deallocate(global_cell_list)
+ do i = 1, local_nvertices(dminfo % my_proc_id+1)
+ local_cell_list(i) = sorted_local_cell_list(2, i)
+ end do
+
+ do i = 2,blocks_per_proc
+ block_start(i) = block_start(i-1) + block_count(i-1)
+ end do
+
+ !dwj 01/31/12 debugging multiple blocks
+! do i=1,local_nvertices(dminfo % my_proc_id +1)
+! write(*,*) sorted_local_cell_list(1, i), sorted_local_cell_list(2,i)
+! end do
+
+ deallocate(sorted_local_cell_list)
+ deallocate(local_block_list)
+ deallocate(local_nvertices)
+ deallocate(global_start)
+ deallocate(global_cell_list)
+ deallocate(global_block_list)
else
- allocate(local_cell_list(partial_global_graph_info % nVerticesTotal))
- do i=1,size(local_cell_list)
- local_cell_list(i) = i
- end do
- endif
+ allocate(local_cell_list(partial_global_graph_info % nVerticesTotal))
+ allocate(block_id(1))
+ allocate(block_start(1))
+ allocate(block_count(1))
+ block_id(1) = 0
+ block_start(1) = 0
+ block_count(1) = size(local_cell_list)
+ do i=1,size(local_cell_list)
+ local_cell_list(i) = i
+ end do
+ end if
- end subroutine mpas_block_decomp_cells_for_proc
+ !dwj 01/31/12 debugging multiple blocks
+! write(*,*) 'Blocks per proc = ', blocks_per_proc, 'total_blocks = ', total_blocks
+! do i=1,blocks_per_proc
+! write(*,*) block_id(i), block_start(i), block_count(i)
+! end do
- subroutine mpas_block_decomp_partitioned_edge_list(nCells, cellIDList, maxCells, nEdges, cellsOnEdge, edgeIDList, ghostEdgeStart)
+! call mpas_dmpar_abort(dminfo)
+ end subroutine mpas_block_decomp_cells_for_proc!}}}
+
+ subroutine mpas_block_decomp_partitioned_edge_list(nCells, cellIDList, maxCells, nEdges, cellsOnEdge, edgeIDList, ghostEdgeStart)!{{{
+
implicit none
integer, intent(in) :: nCells, maxCells, nEdges
@@ -172,11 +253,10 @@
call mpas_hash_destroy(h)
- end subroutine mpas_block_decomp_partitioned_edge_list
+ end subroutine mpas_block_decomp_partitioned_edge_list!}}}
+ subroutine mpas_block_decomp_all_edges_in_block(maxEdges, nCells, nEdgesOnCell, edgesOnCell, nEdges, edgeList)!{{{
- subroutine mpas_block_decomp_all_edges_in_block(maxEdges, nCells, nEdgesOnCell, edgesOnCell, nEdges, edgeList)
-
implicit none
integer, intent(in) :: maxEdges, nCells
@@ -226,11 +306,10 @@
'Listed fewer edges than expected.'
end if
- end subroutine mpas_block_decomp_all_edges_in_block
+ end subroutine mpas_block_decomp_all_edges_in_block!}}}
+ subroutine mpas_block_decomp_add_halo(dminfo, local_graph_info, local_graph_with_halo)!{{{
- subroutine mpas_block_decomp_add_halo(dminfo, local_graph_info, local_graph_with_halo)
-
implicit none
type (dm_info), intent(in) :: dminfo
@@ -301,6 +380,61 @@
call mpas_hash_destroy(h)
- end subroutine mpas_block_decomp_add_halo
+ end subroutine mpas_block_decomp_add_halo!}}}
+ subroutine mpas_get_blocks_per_proc(dminfo, proc_number, blocks_per_proc)!{{{
+ type(dm_info), intent(in) :: dminfo !< Input: Domain Information
+ integer, intent(in) :: proc_number !< Input: Processor number
+ integer, intent(out) :: blocks_per_proc !< Output: Number of blocks proc_number computes on
+
+ integer :: blocks_per_proc_min, even_blocks, remaining_blocks
+
+ blocks_per_proc_min = total_blocks / dminfo % nProcs
+ remaining_blocks = total_blocks - (blocks_per_proc_min * dminfo % nProcs)
+ even_blocks = total_blocks - remaining_blocks
+
+ blocks_per_proc = blocks_per_proc_min
+
+ if(proc_number .le. remaining_blocks) then
+ block_per_proc = blocks_per_proc + 1
+ endif
+
+ end subroutine mpas_get_blocks_per_proc!}}}
+
+ subroutine mpas_get_local_block_id(dminfo, global_block_number, local_block_number)!{{{
+ type(dm_info), intent(in) :: dminfo !< Input: Domain Information
+ integer, intent(in) :: global_block_number !< Input: Global block id from 0 to config_number_of_blocks-1
+ integer, intent(out) :: local_block_number !< Output: Local block id on owning processor from 0 to blocks_per_proc
+
+ integer :: blocks_per_proc_min, even_blocks, remaining_blocks
+
+ blocks_per_proc_min = total_blocks / dminfo % nProcs
+ remaining_blocks = total_blocks - (blocks_per_proc_min * dminfo % nProcs)
+ even_blocks = total_blocks - remaining_blocks
+
+ if(global_block_number > even_blocks) then
+ local_block_number = blocks_per_proc_min
+ else
+ local_block_number = mod(global_block_number, blocks_per_proc_min)
+ end if
+ end subroutine mpas_get_local_block_id!}}}
+
+ subroutine mpas_get_owning_proc(dminfo, global_block_number, owning_proc)!{{{
+ type(dm_info), intent(in) :: dminfo !< Input: Domain Information
+ integer, intent(in) :: global_block_number !< Input: Global block id from 0 to config_number_of_blocks-1
+ integer, intent(out) :: owning_proc !< Output: Processor number that owns block global_block_number
+
+ integer :: blocks_per_proc_min, even_blocks, remaining_blocks
+
+ blocks_per_proc_min = total_blocks / dminfo % nProcs
+ remaining_blocks = total_blocks - (blocks_per_proc_min * dminfo % nProcs)
+ even_blocks = total_blocks - remaining_blocks
+
+ if(global_block_number > even_blocks) then
+ owning_proc = global_block_number - even_blocks
+ else
+ owning_proc = global_block_number / blocks_per_proc_min
+ end if
+ end subroutine mpas_get_owning_proc!}}}
+
end module mpas_block_decomp
Modified: branches/omp_blocks/block_decomp/src/framework/mpas_grid_types.F
===================================================================
--- branches/omp_blocks/block_decomp/src/framework/mpas_grid_types.F 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/src/framework/mpas_grid_types.F 2012-01-31 19:24:21 UTC (rev 1438)
@@ -167,6 +167,7 @@
#include "block_group_members.inc"
integer :: blockID ! Unique global ID number for this block
+ integer :: localBlockID ! Unique local ID number for this block
type (domain_type), pointer :: domain
Modified: branches/omp_blocks/block_decomp/src/framework/mpas_io_input.F
===================================================================
--- branches/omp_blocks/block_decomp/src/framework/mpas_io_input.F 2012-01-30 21:11:04 UTC (rev 1437)
+++ branches/omp_blocks/block_decomp/src/framework/mpas_io_input.F 2012-01-31 19:24:21 UTC (rev 1438)
@@ -115,6 +115,7 @@
#endif
integer, dimension(:), pointer :: local_cell_list, local_edge_list, local_vertex_list
+ integer, dimension(:), pointer :: block_id, block_start, block_count
integer, dimension(:), pointer :: local_vertlevel_list, needed_vertlevel_list
integer :: nlocal_edges, nlocal_vertices
type (exchange_list), pointer :: send1Halo, recv1Halo
@@ -373,7 +374,7 @@
! Determine which cells are owned by this process
- call mpas_block_decomp_cells_for_proc(domain % dminfo, partial_global_graph_info, local_cell_list)
+ call mpas_block_decomp_cells_for_proc(domain % dminfo, partial_global_graph_info, local_cell_list, block_id, block_start, block_count)
deallocate(partial_global_graph_info % vertexID)
deallocate(partial_global_graph_info % nAdjacent)
</font>
</pre>