<p><b>croesch@ucar.edu</b> 2012-03-30 01:14:02 -0600 (Fri, 30 Mar 2012)</p><p>Seperated halo layer code from mpas_io_input.F and moved it into a new module<br>
<br>
A src/framework/mpas_halo.F<br>
M src/framework/mpas_io_input.F<br>
M src/framework/Makefile<br>
</p><hr noshade><pre><font color="gray">Modified: branches/omp_blocks/halo/src/framework/Makefile
===================================================================
--- branches/omp_blocks/halo/src/framework/Makefile 2012-03-29 16:19:10 UTC (rev 1736)
+++ branches/omp_blocks/halo/src/framework/Makefile 2012-03-30 07:14:02 UTC (rev 1737)
@@ -16,6 +16,7 @@
mpas_sort.o \
mpas_block_decomp.o \
mpas_dmpar.o \
+ mpas_halo.o \
mpas_io_input.o \
mpas_io_output.o \
$(ZOLTANOBJ) \
@@ -26,7 +27,7 @@
framework: $(OBJS)
ar -ru libframework.a $(OBJS)
-mpas_framework.o: mpas_dmpar.o mpas_io_input.o mpas_io_output.o mpas_grid_types.o mpas_configure.o mpas_timer.o
+mpas_framework.o: mpas_dmpar.o mpas_halo.o mpas_io_input.o mpas_io_output.o mpas_grid_types.o mpas_configure.o mpas_timer.o
mpas_configure.o: mpas_dmpar.o
@@ -46,8 +47,10 @@
mpas_block_decomp.o: mpas_grid_types.o mpas_hash.o mpas_configure.o
-mpas_io_input.o: mpas_grid_types.o mpas_dmpar.o mpas_block_decomp.o mpas_sort.o mpas_configure.o mpas_timekeeping.o $(ZOLTANOBJ)
+mpas_halo.o : mpas_grid_types.o mpas_dmpar.o mpas_block_decomp.o mpas_sort.o mpas_configure.o mpas_timekeeping.o $(ZOLTANOBJ)
+mpas_io_input.o: mpas_grid_types.o mpas_dmpar.o mpas_block_decomp.o mpas_sort.o mpas_configure.o mpas_timekeeping.o mpas_halo.o $(ZOLTANOBJ)
+
mpas_io_output.o: mpas_grid_types.o mpas_dmpar.o mpas_sort.o mpas_configure.o
clean:
Added: branches/omp_blocks/halo/src/framework/mpas_halo.F
===================================================================
--- branches/omp_blocks/halo/src/framework/mpas_halo.F (rev 0)
+++ branches/omp_blocks/halo/src/framework/mpas_halo.F 2012-03-30 07:14:02 UTC (rev 1737)
@@ -0,0 +1,335 @@
+module mpas_halo
+
+ use mpas_grid_types
+ use mpas_dmpar
+ use mpas_block_decomp
+ use mpas_sort
+ use mpas_configure
+ use mpas_timekeeping
+ use mpas_hash
+
+#ifdef HAVE_ZOLTAN
+ use mpas_zoltan_interface
+#endif
+
+ contains
+
+
+ subroutine mpas_create_halos(domain, nHalos, block_graph_0Halo, &
+ indexToCellIDField, indexToEdgeIDField, indexToVertexIDField, &
+ cellsOnEdgeField, cellsOnVertexField, &
+ nEdgesOnCellField, edgesOnCellField, verticesOnCellField, &
+ nLocalCells, nReadCells, &
+ maxEdges, vertexDegree, &
+
+ nlocal_edges, nlocal_vertices, &
+ local_edge_list, local_vertex_list, &
+ block_graph_1Halo, block_graph_2Halo, &
+ nOwnCells, nOwnEdges, nOwnVertices, &
+ nCellsCumulative, nEdgesCumulative, nVerticesCumulative, &
+ nCellsHalo, nEdgesHalo, nVerticesHalo, &
+ sendCellList, recvCellList)
+
+ implicit none
+
+ !--- input parameters ---!
+ type (domain_type), pointer :: domain
+ integer, intent(in) :: nHalos
+ type (graph), intent(in) :: block_graph_0Halo
+
+ type (field1dInteger), intent(in) :: indexToCellIDField
+ type (field1dInteger), intent(in) :: indexToEdgeIDField
+ type (field1dInteger), intent(in) :: indexToVertexIDField
+
+ type (field2dInteger), intent(in) :: cellsOnEdgeField
+ type (field2dInteger), intent(in) :: cellsOnVertexField
+
+ type (field1dInteger), intent(in) :: nEdgesOnCellField
+ type (field2dInteger), intent(in) :: edgesOnCellField
+ type (field2dInteger), intent(in) :: verticesOnCellField
+ integer, intent(in) :: nLocalCells
+ integer, intent(in) :: nReadCells
+ integer, intent(in) :: maxEdges
+ integer, intent(in) :: vertexDegree
+
+ !--- output parameters ---!
+ integer, intent(out) :: nlocal_edges, nlocal_vertices
+ integer, dimension(:), pointer :: local_edge_list, local_vertex_list
+ type (graph), intent(out) :: block_graph_1Halo, block_graph_2Halo
+
+ integer, intent(out) :: nOwnCells, nOwnEdges, nOwnVertices
+
+ integer, dimension(nHalos+1), intent(out) :: nCellsCumulative ! own cells, halo 1 cells, halo 2 cells
+ integer, dimension(nHalos+2), intent(out) :: nEdgesCumulative ! own edges, own cell's edges, halo 1 edges, halo 2 edges
+ integer, dimension(nHalos+2), intent(out) :: nVerticesCumulative ! own vertices, own cell's vertices, halo 1 vertices, halo 2 vertices
+
+ integer, dimension(nHalos), intent(out) :: nCellsHalo ! halo 1 cells, halo 2 cells
+ integer, dimension(nHalos+1), intent(out) :: nEdgesHalo ! own cell's edges, halo 1 edges, halo 2 edges
+ integer, dimension(nHalos+1), intent(out) :: nVerticesHalo ! own cell's vertices, halo 1 vertices, halo 2 vertices
+
+ type (exchange_list), pointer :: sendCellList, recvCellList
+
+
+ !--- local variables ---!
+
+ type (exchange_list), pointer :: sendEdgeList, recvEdgeList
+ type (exchange_list), pointer :: sendVertexList, recvVertexList
+
+ type (exchange_list), pointer :: send1Halo, recv1Halo
+ type (exchange_list), pointer :: send2Halo, recv2Halo
+
+ integer, dimension(:,:), pointer :: cellsOnEdge_2Halo
+ integer, dimension(:,:), pointer :: cellsOnVertex_2Halo
+
+
+ integer, dimension(:), pointer :: nEdgesOnCell_2Halo
+ integer, dimension(:,:), pointer :: edgesOnCell_2Halo
+ integer, dimension(:,:), pointer :: verticesOnCell_2Halo
+
+ integer :: i, j, k
+ integer :: cellCount, edgeCount, vertexCount, iEdge, iVertex
+ integer :: ghostEdgeStart, ghostVertexStart
+
+ type (hashtable) :: edgeHash, vertexHash
+
+
+
+
+ ! Get back a graph describing the owned cells plus the cells in the 1-halo
+ call mpas_block_decomp_add_halo(domain % dminfo, block_graph_0Halo, block_graph_1Halo)
+
+ !
+ ! Work out exchange lists for 1-halo and exchange cell information for 1-halo
+ !
+ call mpas_dmpar_get_owner_list(domain % dminfo, &
+ block_graph_0Halo % nVertices, block_graph_1Halo % nVerticesTotal, &
+ block_graph_0Halo % vertexID, block_graph_1Halo % vertexID, &
+ send1Halo, recv1Halo)
+
+ call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % vertexID, block_graph_1Halo % vertexID, &
+ block_graph_0Halo % nVertices, block_graph_1Halo % nVerticesTotal, &
+ send1Halo, recv1Halo)
+
+ call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % nAdjacent, block_graph_1Halo % nAdjacent, &
+ block_graph_0Halo % nVertices, block_graph_1Halo % nVerticesTotal, &
+ send1Halo, recv1Halo)
+
+ call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % adjacencyList, block_graph_1Halo % adjacencyList, &
+ block_graph_0Halo % maxDegree, block_graph_0Halo % nVertices, block_graph_1Halo % nVerticesTotal, &
+ send1Halo, recv1Halo)
+
+ !deallocate(send1Halo, recv1Halo)
+
+
+ !
+ ! Work out exchange lists for 2-halo and exchange cell information for 2-halo
+ !
+ block_graph_1Halo % nVertices = block_graph_1Halo % nVerticesTotal
+ block_graph_1Halo % ghostStart = block_graph_1Halo % nVerticesTotal + 1
+
+ ! Get back a graph describing the owned and 1-halo cells plus the cells in the 2-halo
+ call mpas_block_decomp_add_halo(domain % dminfo, block_graph_1Halo, block_graph_2Halo)
+
+ block_graph_2Halo % nVertices = block_graph_0Halo % nVertices
+ block_graph_2Halo % ghostStart = block_graph_2Halo % nVertices + 1
+
+ nOwnCells = block_graph_2Halo % nVertices
+
+#ifdef HAVE_ZOLTAN
+#ifdef _MPI
+ !! For now, only use Zoltan with MPI
+ !! Zoltan initialization
+ call mpas_zoltan_start()
+
+ !! Zoltan hook for cells
+ call mpas_zoltan_order_loc_hsfc_cells(block_graph_2Halo%nVertices,block_graph_2Halo%VertexID,3,xCell,yCell,zCell)
+#endif
+#endif
+
+
+ call mpas_dmpar_get_owner_list(domain % dminfo, &
+ block_graph_0Halo % nVertices, block_graph_2Halo % nVerticesTotal, &
+ block_graph_0Halo % vertexID, block_graph_2Halo % vertexID, &
+ send2Halo, recv2Halo)
+
+ call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % vertexID, block_graph_2Halo % vertexID, &
+ block_graph_0Halo % nVertices, block_graph_2Halo % nVerticesTotal, &
+ send2Halo, recv2Halo)
+
+ call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % nAdjacent, block_graph_2Halo % nAdjacent, &
+ block_graph_0Halo % nVertices, block_graph_2Halo % nVerticesTotal, &
+ send2Halo, recv2Halo)
+
+ call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % adjacencyList, block_graph_2Halo % adjacencyList, &
+ block_graph_0Halo % maxDegree, block_graph_0Halo % nVertices, block_graph_2Halo % nVerticesTotal, &
+ send2Halo, recv2Halo)
+
+
+ !
+ ! Knowing which cells are in block and the 2-halo, we can exchange lists of which edges are
+ ! on each cell and which vertices are on each cell from the processes that read these
+ ! fields for each cell to the processes that own the cells
+ !
+ allocate(nEdgesOnCell_2Halo(block_graph_2Halo % nVerticesTotal))
+ allocate(edgesOnCell_2Halo(maxEdges, block_graph_2Halo % nVerticesTotal))
+ allocate(verticesOnCell_2Halo(maxEdges, block_graph_2Halo % nVerticesTotal))
+
+ call mpas_dmpar_get_owner_list(domain % dminfo, &
+ size(indexToCellIDField % array), block_graph_2Halo % nVerticesTotal, &
+ indexToCellIDField % array, block_graph_2Halo % vertexID, &
+ sendCellList, recvCellList)
+
+ call mpas_dmpar_alltoall_field(domain % dminfo, nEdgesOnCellField % array, nEdgesOnCell_2Halo, &
+ size(indexToCellIDField % array), nLocalCells, &
+ sendCellList, recvCellList)
+
+ call mpas_dmpar_alltoall_field(domain % dminfo, edgesOnCellField % array, edgesOnCell_2Halo, &
+ maxEdges, nReadCells, block_graph_2Halo % nVerticesTotal, &
+ sendCellList, recvCellList)
+
+ call mpas_dmpar_alltoall_field(domain % dminfo, verticesOnCellField % array, verticesOnCell_2Halo, &
+ maxEdges, nReadCells, block_graph_2Halo % nVerticesTotal, &
+ sendCellList, recvCellList)
+
+
+ !
+ ! Get a list of which edges and vertices are adjacent to cells (including halo cells) in block
+ !
+ call mpas_block_decomp_all_edges_in_block(maxEdges, block_graph_2Halo % nVerticesTotal, block_graph_2Halo % nAdjacent, &
+ edgesOnCell_2Halo, nlocal_edges, local_edge_list)
+ call mpas_block_decomp_all_edges_in_block(maxEdges, block_graph_2Halo % nVerticesTotal, block_graph_2Halo % nAdjacent, &
+ verticesOnCell_2Halo, nlocal_vertices, local_vertex_list)
+
+ call mpas_dmpar_get_owner_list(domain % dminfo, &
+ size(indexToEdgeIDField % array), nlocal_edges, &
+ indexToEdgeIDField % array, local_edge_list, &
+ sendEdgeList, recvEdgeList)
+
+ call mpas_dmpar_get_owner_list(domain % dminfo, &
+ size(indexToVertexIDField % array), nlocal_vertices, &
+ indexToVertexIDField % array, local_vertex_list, &
+ sendVertexList, recvVertexList)
+
+
+ !
+ ! Work out which edges and vertices are owned by this process, and which are ghost
+ !
+ allocate(cellsOnEdge_2Halo(2,nlocal_edges))
+ allocate(cellsOnVertex_2Halo(vertexDegree,nlocal_vertices))
+
+ call mpas_dmpar_alltoall_field(domain % dminfo, cellsOnEdgeField % array, cellsOnEdge_2Halo, &
+ 2, size(cellsOnEdgeField % array, 2), nlocal_edges, &
+ sendEdgeList, recvEdgeList)
+
+ call mpas_dmpar_alltoall_field(domain % dminfo, cellsOnVertexField % array, cellsOnVertex_2Halo, &
+ vertexDegree, size(cellsOnVertexField % array, 2), nlocal_vertices, &
+ sendVertexList, recvVertexList)
+
+
+ !deallocate(sendEdgeList % list)
+ !deallocate(sendEdgeList)
+ !deallocate(recvEdgeList % list)
+ !deallocate(recvEdgeList)
+
+ !deallocate(sendVertexList % list)
+ !deallocate(sendVertexList)
+ !deallocate(recvVertexList % list)
+ !deallocate(recvVertexList)
+
+
+ call mpas_block_decomp_partitioned_edge_list(nOwnCells, &
+ block_graph_2Halo % vertexID(1:nOwnCells), &
+ 2, nlocal_edges, cellsOnEdge_2Halo, local_edge_list, ghostEdgeStart)
+
+ call mpas_block_decomp_partitioned_edge_list(nOwnCells, &
+ block_graph_2Halo % vertexID(1:nOwnCells), &
+ vertexDegree, nlocal_vertices, cellsOnVertex_2Halo, local_vertex_list, ghostVertexStart)
+
+
+ !------- set owned and halo cell indices -------!
+
+ nCellsCumulative(1) = nOwnCells
+ nCellsCumulative(2) = block_graph_1Halo % nVerticesTotal
+ nCellsCumulative(3) = block_graph_2Halo % nVerticesTotal
+
+ !------- determin the perimeter and owned edges of own cells and halos -------!
+
+ nOwnEdges = ghostEdgeStart-1
+ nOwnVertices = ghostVertexStart-1
+
+ ! skip the own edges found at the beginning of local_edge_list
+ call mpas_hash_init(edgeHash)
+ do i=1,nOwnEdges
+ call mpas_hash_insert(edgeHash, local_edge_list(i))
+ end do
+
+ ! skip the own vertices found at the beginning of local_vertex_list
+ call mpas_hash_init(vertexHash)
+ do i=1,nOwnVertices
+ call mpas_hash_insert(vertexHash, local_vertex_list(i))
+ end do
+
+
+ cellCount = 1 !tracks the index of the local cell array
+ edgeCount = nOwnEdges !tracks where to insert the next local edge
+ vertexCount = nOwnVertices !tracks where to insert the next local vertex
+
+ nEdgesCumulative(1) = nOwnEdges
+ nVerticesCumulative(1) = nOwnVertices
+
+ !Order the local_edge_list and local_vertex_list accordingly and set the bounds of each perimeter ----
+ do i = 1, nHalos + 1 ! for the own cells and each halo...
+ do j = cellCount, nCellsCumulative(i)
+
+ ! the number of edges on a cell is same to the number of vertices, and therefore
+ ! nEdgesOnCell_2Halo(j) will be the correct upper bound for both both edges and vertices on cell
+ do k = 1, nEdgesOnCell_2Halo(j)
+ iEdge = edgesOnCell_2Halo(k,j)
+ if (.not. mpas_hash_search(edgeHash, iEdge)) then
+ edgeCount = edgeCount + 1
+ local_edge_list(edgeCount) = iEdge
+ call mpas_hash_insert(edgeHash, iEdge)
+ end if
+
+ iVertex = verticesOnCell_2Halo(k,j)
+ if (.not. mpas_hash_search(vertexHash, iVertex)) then
+ vertexCount = vertexCount + 1
+ local_vertex_list(vertexCount) = iVertex
+ call mpas_hash_insert(vertexHash, iVertex)
+ end if
+ end do
+
+ end do
+
+ cellCount = nCellsCumulative(i) + 1
+ nEdgesCumulative(i+1) = edgeCount
+ nVerticesCumulative(i+1) = vertexCount
+ end do
+
+
+ do i = 1, nHalos
+ nCellsHalo(i) = nCellsCumulative(i+1) - nCellsCumulative(i)
+ end do
+
+ do i = 1, nHalos + 1
+ nEdgesHalo(i) = nEdgesCumulative(i+1) - nEdgesCumulative(i)
+ end do
+
+ do i = 1, nHalos + 1
+ nVerticesHalo(i) = nVerticesCumulative(i+1) - nVerticesCumulative(i)
+ end do
+
+ call mpas_hash_destroy(edgeHash)
+ call mpas_hash_destroy(vertexHash)
+
+
+ deallocate(cellsOnEdge_2Halo)
+ deallocate(cellsOnVertex_2Halo)
+ deallocate(nEdgesOnCell_2Halo)
+ deallocate(edgesOnCell_2Halo)
+ deallocate(verticesOnCell_2Halo)
+
+ end subroutine mpas_create_halos
+
+
+end module mpas_halo
Modified: branches/omp_blocks/halo/src/framework/mpas_io_input.F
===================================================================
--- branches/omp_blocks/halo/src/framework/mpas_io_input.F 2012-03-29 16:19:10 UTC (rev 1736)
+++ branches/omp_blocks/halo/src/framework/mpas_io_input.F 2012-03-30 07:14:02 UTC (rev 1737)
@@ -6,6 +6,7 @@
use mpas_sort
use mpas_configure
use mpas_timekeeping
+ use mpas_halo
#ifdef HAVE_ZOLTAN
@@ -97,14 +98,7 @@
integer, dimension(:), pointer :: nEdgesOnCell_0Halo
integer, dimension(:,:), pointer :: cellsOnCell_0Halo
- integer, dimension(:), pointer :: nEdgesOnCell_2Halo
- integer, dimension(:,:), pointer :: edgesOnCell_2Halo
- integer, dimension(:,:), pointer :: verticesOnCell_2Halo
-
- integer, dimension(:,:), pointer :: cellsOnEdge_2Halo
- integer, dimension(:,:), pointer :: cellsOnVertex_2Halo
-
integer, dimension(:,:), pointer :: cellIDSorted
integer, dimension(:,:), pointer :: edgeIDSorted
integer, dimension(:,:), pointer :: vertexIDSorted
@@ -121,11 +115,9 @@
integer, dimension(:), pointer :: block_id, block_start, block_count
integer, dimension(:), pointer :: local_vertlevel_list, needed_vertlevel_list
integer :: nlocal_edges, nlocal_vertices
- type (exchange_list), pointer :: send1Halo, recv1Halo
- type (exchange_list), pointer :: send2Halo, recv2Halo
+
type (graph) :: partial_global_graph_info
type (graph) :: block_graph_0Halo, block_graph_1Halo, block_graph_2Halo
- integer :: ghostEdgeStart, ghostVertexStart
type (MPAS_Time_type) :: startTime
type (MPAS_Time_type) :: sliceTime
@@ -146,8 +138,7 @@
integer, dimension(:), pointer :: tempIDs
integer :: ntempIDs, offset
- integer :: nHalo, nOwnCells, nOwnEdges, nOwnVertices, cellCount, edgeCount, vertexCount, iEdge, iVertex
- type (hashtable) :: edgeHash, vertexHash
+ integer :: nOwnCells, nOwnEdges, nOwnVertices
if (config_do_restart) then
@@ -472,241 +463,34 @@
block_graph_0Halo % nAdjacent(:) = nEdgesOnCell_0Halo(:)
block_graph_0Halo % adjacencyList(:,:) = cellsOnCell_0Halo(:,:)
- ! Get back a graph describing the owned cells plus the cells in the 1-halo
- call mpas_block_decomp_add_halo(domain % dminfo, block_graph_0Halo, block_graph_1Halo)
-
-
- !
- ! Work out exchange lists for 1-halo and exchange cell information for 1-halo
- !
- call mpas_dmpar_get_owner_list(domain % dminfo, &
- block_graph_0Halo % nVertices, block_graph_1Halo % nVerticesTotal, &
- block_graph_0Halo % vertexID, block_graph_1Halo % vertexID, &
- send1Halo, recv1Halo)
-
- call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % vertexID, block_graph_1Halo % vertexID, &
- block_graph_0Halo % nVertices, block_graph_1Halo % nVerticesTotal, &
- send1Halo, recv1Halo)
-
- call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % nAdjacent, block_graph_1Halo % nAdjacent, &
- block_graph_0Halo % nVertices, block_graph_1Halo % nVerticesTotal, &
- send1Halo, recv1Halo)
-
- call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % adjacencyList, block_graph_1Halo % adjacencyList, &
- block_graph_0Halo % maxDegree, block_graph_0Halo % nVertices, block_graph_1Halo % nVerticesTotal, &
- send1Halo, recv1Halo)
-
-
- !
- ! Work out exchange lists for 2-halo and exchange cell information for 2-halo
- !
- block_graph_1Halo % nVertices = block_graph_1Halo % nVerticesTotal
- block_graph_1Halo % ghostStart = block_graph_1Halo % nVerticesTotal + 1
-
- ! Get back a graph describing the owned and 1-halo cells plus the cells in the 2-halo
- call mpas_block_decomp_add_halo(domain % dminfo, block_graph_1Halo, block_graph_2Halo)
-
- block_graph_2Halo % nVertices = block_graph_0Halo % nVertices
- block_graph_2Halo % ghostStart = block_graph_2Halo % nVertices + 1
+ call mpas_create_halos(domain, nHalos, block_graph_0Halo, &
+ indexToCellIDField, indexToEdgeIDField, indexToVertexIDField, &
+ cellsOnEdgeField, cellsOnVertexField, &
+ nEdgesOnCellField, edgesOnCellField, verticesOnCellField, &
+ size(local_cell_list), nReadCells, &
+ maxEdges, vertexDegree, &
- nOwnCells = block_graph_2Halo % nVertices
+ nlocal_edges, nlocal_vertices, &
+ local_edge_list, local_vertex_list, &
+ block_graph_1Halo, block_graph_2Halo, &
+ nOwnCells, nOwnEdges, nOwnVertices, &
+ nCellsCumulative, nEdgesCumulative, nVerticesCumulative, &
+ nCellsHalo, nEdgesHalo, nVerticesHalo, &
+ sendCellList, recvCellList)
-#ifdef HAVE_ZOLTAN
-#ifdef _MPI
- !! For now, only use Zoltan with MPI
- !! Zoltan initialization
- call mpas_zoltan_start()
- !! Zoltan hook for cells
- call mpas_zoltan_order_loc_hsfc_cells(block_graph_2Halo%nVertices,block_graph_2Halo%VertexID,3,xCell,yCell,zCell)
-#endif
-#endif
-
- call mpas_dmpar_get_owner_list(domain % dminfo, &
- block_graph_0Halo % nVertices, block_graph_2Halo % nVerticesTotal, &
- block_graph_0Halo % vertexID, block_graph_2Halo % vertexID, &
- send2Halo, recv2Halo)
-
- call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % vertexID, block_graph_2Halo % vertexID, &
- block_graph_0Halo % nVertices, block_graph_2Halo % nVerticesTotal, &
- send2Halo, recv2Halo)
-
- call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % nAdjacent, block_graph_2Halo % nAdjacent, &
- block_graph_0Halo % nVertices, block_graph_2Halo % nVerticesTotal, &
- send2Halo, recv2Halo)
-
- call mpas_dmpar_alltoall_field(domain % dminfo, block_graph_0Halo % adjacencyList, block_graph_2Halo % adjacencyList, &
- block_graph_0Halo % maxDegree, block_graph_0Halo % nVertices, block_graph_2Halo % nVerticesTotal, &
- send2Halo, recv2Halo)
-
-
-
- !
- ! Knowing which cells are in block and the 2-halo, we can exchange lists of which edges are
- ! on each cell and which vertices are on each cell from the processes that read these
- ! fields for each cell to the processes that own the cells
- !
- allocate(nEdgesOnCell_2Halo(block_graph_2Halo % nVerticesTotal))
- allocate(edgesOnCell_2Halo(maxEdges, block_graph_2Halo % nVerticesTotal))
- allocate(verticesOnCell_2Halo(maxEdges, block_graph_2Halo % nVerticesTotal))
-
- call mpas_dmpar_get_owner_list(domain % dminfo, &
- size(indexToCellIDField % array), block_graph_2Halo % nVerticesTotal, &
- indexToCellIDField % array, block_graph_2Halo % vertexID, &
- sendCellList, recvCellList)
-
- call mpas_dmpar_alltoall_field(domain % dminfo, nEdgesOnCellField % array, nEdgesOnCell_2Halo, &
- size(indexToCellIDField % array), size(local_cell_list), &
- sendCellList, recvCellList)
-
- call mpas_dmpar_alltoall_field(domain % dminfo, edgesOnCellField % array, edgesOnCell_2Halo, &
- maxEdges, nReadCells, block_graph_2Halo % nVerticesTotal, &
- sendCellList, recvCellList)
-
- call mpas_dmpar_alltoall_field(domain % dminfo, verticesOnCellField % array, verticesOnCell_2Halo, &
- maxEdges, nReadCells, block_graph_2Halo % nVerticesTotal, &
- sendCellList, recvCellList)
-
-
- !
- ! Get a list of which edges and vertices are adjacent to cells (including halo cells) in block
- !
- call mpas_block_decomp_all_edges_in_block(maxEdges, block_graph_2Halo % nVerticesTotal, block_graph_2Halo % nAdjacent, &
- edgesOnCell_2Halo, nlocal_edges, local_edge_list)
- call mpas_block_decomp_all_edges_in_block(maxEdges, block_graph_2Halo % nVerticesTotal, block_graph_2Halo % nAdjacent, &
- verticesOnCell_2Halo, nlocal_vertices, local_vertex_list)
-
- call mpas_dmpar_get_owner_list(domain % dminfo, &
- size(indexToEdgeIDField % array), nlocal_edges, &
- indexToEdgeIDField % array, local_edge_list, &
- sendEdgeList, recvEdgeList)
-
- call mpas_dmpar_get_owner_list(domain % dminfo, &
- size(indexToVertexIDField % array), nlocal_vertices, &
- indexToVertexIDField % array, local_vertex_list, &
- sendVertexList, recvVertexList)
-
-
-
- !
- ! Work out which edges and vertices are owned by this process, and which are ghost
- !
- allocate(cellsOnEdge_2Halo(2,nlocal_edges))
- allocate(cellsOnVertex_2Halo(vertexDegree,nlocal_vertices))
-
- call mpas_dmpar_alltoall_field(domain % dminfo, cellsOnEdgeField % array, cellsOnEdge_2Halo, &
- 2, size(cellsOnEdgeField % array, 2), nlocal_edges, &
- sendEdgeList, recvEdgeList)
-
- call mpas_dmpar_alltoall_field(domain % dminfo, cellsOnVertexField % array, cellsOnVertex_2Halo, &
- vertexDegree, size(cellsOnVertexField % array, 2), nlocal_vertices, &
- sendVertexList, recvVertexList)
-
-
- call mpas_block_decomp_partitioned_edge_list(nOwnCells, &
- block_graph_2Halo % vertexID(1:nOwnCells), &
- 2, nlocal_edges, cellsOnEdge_2Halo, local_edge_list, ghostEdgeStart)
-
- call mpas_block_decomp_partitioned_edge_list(nOwnCells, &
- block_graph_2Halo % vertexID(1:nOwnCells), &
- vertexDegree, nlocal_vertices, cellsOnVertex_2Halo, local_vertex_list, ghostVertexStart)
-
- !------- set owned and halo cell indices -------!
-
- nCellsCumulative(1) = nOwnCells
- nCellsCumulative(2) = block_graph_1Halo % nVerticesTotal
- nCellsCumulative(3) = block_graph_2Halo % nVerticesTotal
-
- !------- determin the perimeter and owned edges of own cells and halos -------!
-
- nOwnEdges = ghostEdgeStart-1
- nOwnVertices = ghostVertexStart-1
-
- ! skip the own edges found at the beginning of local_edge_list
- call mpas_hash_init(edgeHash)
- do i=1,nOwnEdges
- call mpas_hash_insert(edgeHash, local_edge_list(i))
- end do
-
- ! skip the own vertices found at the beginning of local_vertex_list
- call mpas_hash_init(vertexHash)
- do i=1,nOwnVertices
- call mpas_hash_insert(vertexHash, local_vertex_list(i))
- end do
-
- cellCount = 1 !tracks the index of the local cell array
- edgeCount = nOwnEdges !tracks where to insert the next local edge
- vertexCount = nOwnVertices !tracks where to insert the next local vertex
-
- nEdgesCumulative(1) = nOwnEdges
- nVerticesCumulative(1) = nOwnVertices
-
- !Order the local_edge_list and local_vertex_list accordingly and set the bounds of each perimeter ----
- do i = 1, nHalos + 1 ! for the own cells and each halo...
- do j = cellCount, nCellsCumulative(i)
-
- ! the number of edges on a cell is same to the number of vertices, and therefore
- ! nEdgesOnCell_2Halo(j) will be the correct upper bound for both both edges and vertices on cell
- do k = 1, nEdgesOnCell_2Halo(j)
- iEdge = edgesOnCell_2Halo(k,j)
- if (.not. mpas_hash_search(edgeHash, iEdge)) then
- edgeCount = edgeCount + 1
- local_edge_list(edgeCount) = iEdge
- call mpas_hash_insert(edgeHash, iEdge)
- end if
-
- iVertex = verticesOnCell_2Halo(k,j)
- if (.not. mpas_hash_search(vertexHash, iVertex)) then
- vertexCount = vertexCount + 1
- local_vertex_list(vertexCount) = iVertex
- call mpas_hash_insert(vertexHash, iVertex)
- end if
- end do
-
- end do
-
- cellCount = nCellsCumulative(i) + 1
- nEdgesCumulative(i+1) = edgeCount
- nVerticesCumulative(i+1) = vertexCount
- end do
-
- do i = 1, nHalos
- nCellsHalo(i) = nCellsCumulative(i+1) - nCellsCumulative(i)
- end do
-
- do i = 1, nHalos + 1
- nEdgesHalo(i) = nEdgesCumulative(i+1) - nEdgesCumulative(i)
- end do
-
- do i = 1, nHalos + 1
- nVerticesHalo(i) = nVerticesCumulative(i+1) - nVerticesCumulative(i)
- end do
-
- call mpas_hash_destroy(edgeHash)
- call mpas_hash_destroy(vertexHash)
-
-
! At this point, local_edge_list(1:nOwnEdges) contains all of the owned edges for this block
- ! and local_edge_list(ghostEdgeStart:nlocal_edges) contains all of the ghost edges
+ ! and local_edge_list(nOwnEdges+1:nlocal_edges) contains all of the ghost edges
- ! At this point, local_vertex_list(1;ghostVertexStart-1) contains all of the owned vertices for this block
- ! and local_vertex_list(ghostVertexStart:nlocal_vertices) contains all of the ghost vertices
+ ! At this point, local_vertex_list(1:nOwnVertices) contains all of the owned vertices for this block
+ ! and local_vertex_list(nOwnVertices+1:nlocal_vertices) contains all of the ghost vertices
! Also, at this point, block_graph_2Halo % vertexID(1:block_graph_2Halo%nVertices) contains all of the owned
! cells for this block, and block_graph_2Halo % vertexID(block_graph_2Halo%nVertices+1:block_graph_2Halo%nVerticesTotal)
! contains all of the ghost cells
- deallocate(sendEdgeList % list)
- deallocate(sendEdgeList)
- deallocate(recvEdgeList % list)
- deallocate(recvEdgeList)
- deallocate(sendVertexList % list)
- deallocate(sendVertexList)
- deallocate(recvVertexList % list)
- deallocate(recvVertexList)
-
#ifdef HAVE_ZOLTAN
#ifdef _MPI
allocate(xEdge(nlocal_edges))
@@ -1148,7 +932,7 @@
domain % blocklist % mesh % nCellsSolve = nOwnCells
domain % blocklist % mesh % nEdgesSolve = nOwnEdges
- domain % blocklist % mesh % nVerticesSolve = ghostVertexStart-1
+ domain % blocklist % mesh % nVerticesSolve = nOwnVertices
domain % blocklist % mesh % nVertLevelsSolve = domain % blocklist % mesh % nVertLevels ! No vertical decomp yet...
! Link the sendList and recvList pointers in each field type to the appropriate lists
@@ -1156,7 +940,7 @@
! link blocks of fields to eachother
call mpas_create_field_links(domain % blocklist)
-
+
!
! Deallocate fields, graphs, and other memory
!
@@ -1189,11 +973,7 @@
deallocate(cellsOnCell_0Halo)
deallocate(nEdgesOnCell_0Halo)
deallocate(indexToCellID_0Halo)
- deallocate(cellsOnEdge_2Halo)
- deallocate(cellsOnVertex_2Halo)
- deallocate(nEdgesOnCell_2Halo)
- deallocate(edgesOnCell_2Halo)
- deallocate(verticesOnCell_2Halo)
+
deallocate(block_graph_0Halo % vertexID)
deallocate(block_graph_0Halo % nAdjacent)
deallocate(block_graph_0Halo % adjacencyList)
</font>
</pre>