<p><b>mmwolf@sandia.gov</b> 2010-01-06 15:43:15 -0700 (Wed, 06 Jan 2010)</p><p>Added reordering for edges and vertices<br>
</p><hr noshade><pre><font color="gray">Modified: trunk/swmodel/src/module_io_input.F
===================================================================
--- trunk/swmodel/src/module_io_input.F 2010-01-06 22:17:15 UTC (rev 107)
+++ trunk/swmodel/src/module_io_input.F 2010-01-06 22:43:15 UTC (rev 108)
@@ -67,6 +67,8 @@
#ifdef HAVE_ZOLTAN
#ifdef _MPI
type (field1dReal) :: xCellField, yCellField, zCellField
+ type (field1dReal) :: xEdgeField, yEdgeField, zEdgeField
+ type (field1dReal) :: xVertexField, yVertexField, zVertexField
#endif
#endif
@@ -88,6 +90,8 @@
#ifdef HAVE_ZOLTAN
#ifdef _MPI
real (kind=RKIND), dimension(:), pointer :: xCell, yCell, zCell
+ real (kind=RKIND), dimension(:), pointer :: xEdge, yEdge, zEdge
+ real (kind=RKIND), dimension(:), pointer :: xVertex, yVertex, zVertex
#endif
#endif
@@ -188,6 +192,34 @@
allocate(indexToEdgeIDField % array(nReadEdges))
call io_input_field(input_obj, indexToEdgeIDField)
+#ifdef HAVE_ZOLTAN
+#ifdef _MPI
+ ! Edge x-coordinates (in 3d Cartesian space)
+ allocate(xEdgeField % ioinfo)
+ xEdgeField % ioinfo % fieldName = 'xEdge'
+ xEdgeField % ioinfo % start(1) = readEdgeStart
+ xEdgeField % ioinfo % count(1) = nReadEdges
+ allocate(xEdgeField % array(nReadEdges))
+ call io_input_field(input_obj, xEdgeField)
+
+ ! Edge y-coordinates (in 3d Cartesian space)
+ allocate(yEdgeField % ioinfo)
+ yEdgeField % ioinfo % fieldName = 'yEdge'
+ yEdgeField % ioinfo % start(1) = readEdgeStart
+ yEdgeField % ioinfo % count(1) = nReadEdges
+ allocate(yEdgeField % array(nReadEdges))
+ call io_input_field(input_obj, yEdgeField)
+
+ ! Edge z-coordinates (in 3d Cartesian space)
+ allocate(zEdgeField % ioinfo)
+ zEdgeField % ioinfo % fieldName = 'zEdge'
+ zEdgeField % ioinfo % start(1) = readEdgeStart
+ zEdgeField % ioinfo % count(1) = nReadEdges
+ allocate(zEdgeField % array(nReadEdges))
+ call io_input_field(input_obj, zEdgeField)
+#endif
+#endif
+
! Global vertex indices
allocate(indexToVertexIDField % ioinfo)
indexToVertexIDField % ioinfo % fieldName = 'indexToVertexID'
@@ -196,6 +228,34 @@
allocate(indexToVertexIDField % array(nReadVertices))
call io_input_field(input_obj, indexToVertexIDField)
+#ifdef HAVE_ZOLTAN
+#ifdef _MPI
+ ! Vertex x-coordinates (in 3d Cartesian space)
+ allocate(xVertexField % ioinfo)
+ xVertexField % ioinfo % fieldName = 'xVertex'
+ xVertexField % ioinfo % start(1) = readVertexStart
+ xVertexField % ioinfo % count(1) = nReadVertices
+ allocate(xVertexField % array(nReadVertices))
+ call io_input_field(input_obj, xVertexField)
+
+ ! Vertex y-coordinates (in 3d Cartesian space)
+ allocate(yVertexField % ioinfo)
+ yVertexField % ioinfo % fieldName = 'yVertex'
+ yVertexField % ioinfo % start(1) = readVertexStart
+ yVertexField % ioinfo % count(1) = nReadVertices
+ allocate(yVertexField % array(nReadVertices))
+ call io_input_field(input_obj, yVertexField)
+
+ ! Vertex z-coordinates (in 3d Cartesian space)
+ allocate(zVertexField % ioinfo)
+ zVertexField % ioinfo % fieldName = 'zVertex'
+ zVertexField % ioinfo % start(1) = readVertexStart
+ zVertexField % ioinfo % count(1) = nReadVertices
+ allocate(zVertexField % array(nReadVertices))
+ call io_input_field(input_obj, zVertexField)
+#endif
+#endif
+
! Number of cell/edges/vertices adjacent to each cell
allocate(nEdgesOnCellField % ioinfo)
nEdgesOnCellField % ioinfo % fieldName = 'nEdgesOnCell'
@@ -517,6 +577,16 @@
deallocate(recvVertexList % list)
deallocate(recvVertexList)
+#ifdef HAVE_ZOLTAN
+#ifdef _MPI
+ allocate(xEdge(nlocal_edges))
+ allocate(yEdge(nlocal_edges))
+ allocate(zEdge(nlocal_edges))
+ allocate(xVertex(nlocal_vertices))
+ allocate(yVertex(nlocal_vertices))
+ allocate(zVertex(nlocal_vertices))
+#endif
+#endif
!
! Knowing which edges/vertices are owned by this block and which are actually read
@@ -534,6 +604,42 @@
indexToVertexIDField % array, local_vertex_list, &
sendVertexList, recvVertexList)
+#ifdef HAVE_ZOLTAN
+#ifdef _MPI
+ call dmpar_alltoall_field(domain % dminfo, xEdgeField % array, xEdge, &
+ size(xEdgeField % array), nlocal_edges, &
+ sendEdgeList, recvEdgeList)
+ call dmpar_alltoall_field(domain % dminfo, yEdgeField % array, yEdge, &
+ size(yEdgeField % array), nlocal_edges, &
+ sendEdgeList, recvEdgeList)
+ call dmpar_alltoall_field(domain % dminfo, zEdgeField % array, zEdge, &
+ size(zEdgeField % array), nlocal_edges, &
+ sendEdgeList, recvEdgeList)
+
+ call dmpar_alltoall_field(domain % dminfo, xVertexField % array, xVertex, &
+ size(xVertexField % array), nlocal_vertices, &
+ sendVertexList, recvVertexList)
+ call dmpar_alltoall_field(domain % dminfo, yVertexField % array, yVertex, &
+ size(yVertexField % array), nlocal_vertices, &
+ sendVertexList, recvVertexList)
+ call dmpar_alltoall_field(domain % dminfo, zVertexField % array, zVertex, &
+ size(zVertexField % array), nlocal_vertices, &
+ sendVertexList, recvVertexList)
+ !!!!!!!!!!!!!!!!!!
+ !! Reorder edges
+ !!!!!!!!!!!!!!!!!!
+ call zoltanOrderLocHSFC_Edges(ghostEdgeStart-1,local_edge_list,3,xEdge,yEdge,zEdge)
+ !!!!!!!!!!!!!!!!!!
+
+ !!!!!!!!!!!!!!!!!!
+ !! Reorder vertices
+ !!!!!!!!!!!!!!!!!!
+ call zoltanOrderLocHSFC_Verts(ghostVertexStart-1,local_vertex_list,3,xVertex,yVertex,zVertex)
+ !!!!!!!!!!!!!!!!!!
+
+#endif
+#endif
+
!
! Build ownership and exchange lists for vertical levels
! Essentially, process 0 owns all vertical levels when reading and writing,
Modified: trunk/swmodel/src/module_zoltan_interface.F
===================================================================
--- trunk/swmodel/src/module_zoltan_interface.F 2010-01-06 22:17:15 UTC (rev 107)
+++ trunk/swmodel/src/module_zoltan_interface.F 2010-01-06 22:43:15 UTC (rev 108)
@@ -4,11 +4,32 @@
implicit none
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! Data for reordering cells
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
integer :: numCells
integer, dimension(:), pointer :: cellIDs
integer :: geomDim
real (kind=RKIND), dimension(:), pointer :: cellCoordX, cellCoordY, cellCoordZ
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! Data for reordering edges
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ integer :: numEdges
+ integer, dimension(:), pointer :: edgeIDs
+ real (kind=RKIND), dimension(:), pointer :: edgeCoordX, edgeCoordY, edgeCoordZ
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! Data for reordering vertices
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ integer :: numVerts
+ integer, dimension(:), pointer :: vertIDs
+ real (kind=RKIND), dimension(:), pointer :: vertCoordX, vertCoordY, vertCoordZ
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+
contains
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -32,7 +53,6 @@
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
subroutine zoltanOrderLocHSFC_Cells(in_numcells,in_cellIDs,in_geomDim,in_cellX, &
in_cellY, in_cellZ)
-
implicit none
integer :: in_numcells
@@ -48,12 +68,8 @@
integer(ZOLTAN_INT) :: ierr
integer :: numGidEntries, i
- integer(ZOLTAN_INT), allocatable :: global_ids(:)
- integer(ZOLTAN_INT), allocatable :: permIndices(:)
- integer(ZOLTAN_INT), allocatable :: permGIDs(:)
- real(kind=RKIND), allocatable :: permXs(:)
- real(kind=RKIND), allocatable :: permYs(:)
- real(kind=RKIND), allocatable :: permZs(:)
+ integer(ZOLTAN_INT), allocatable :: global_ids(:), permIndices(:),permGIDs(:)
+ real(kind=RKIND), allocatable :: permXs(:),permYs(:),permZs(:)
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -131,7 +147,7 @@
deallocate(permZs)
call Zoltan_Destroy(zz_obj)
- !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
end subroutine zoltanOrderLocHSFC_Cells
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -147,7 +163,6 @@
integer(ZOLTAN_INT), intent(out) :: ierr
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- !! MMW: Should be set to numCells
zqfNumCells = numCells
ierr = ZOLTAN_OK
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -160,13 +175,9 @@
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
subroutine zqfGetCells (data, num_gid_entries, num_lid_entries, global_ids, &
local_ids, wgt_dim, obj_wgts, ierr)
- use mpi
-
integer(ZOLTAN_INT), intent(in) :: data(*)
- integer(ZOLTAN_INT), intent(in) :: num_gid_entries
- integer(ZOLTAN_INT), intent(in) :: num_lid_entries
- integer(ZOLTAN_INT), intent(out) :: global_ids(*)
- integer(ZOLTAN_INT), intent(out) :: local_ids(*)
+ integer(ZOLTAN_INT), intent(in) :: num_gid_entries, num_lid_entries
+ integer(ZOLTAN_INT), intent(out) :: global_ids(*), local_ids(*)
integer(ZOLTAN_INT), intent(in) :: wgt_dim
real(ZOLTAN_FLOAT), intent(out) :: obj_wgts(*)
integer(ZOLTAN_INT), intent(out) :: ierr
@@ -175,9 +186,6 @@
integer :: i
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- !! MMW: cellIDs should have global cell IDs
-
do i= 1, numCells
global_ids(i) = cellIDs(i)
local_ids(i) = i
@@ -185,7 +193,6 @@
ierr = ZOLTAN_OK
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
end subroutine zqfGetCells
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -203,7 +210,6 @@
zqfGeomDim = geomDim
ierr = ZOLTAN_OK
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
end function zqfGeomDim
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -216,18 +222,12 @@
implicit none
integer(ZOLTAN_INT), intent(in) :: data(*)
- integer(ZOLTAN_INT), intent(in) :: num_gid_entries
- integer(ZOLTAN_INT), intent(in) :: num_lid_entries
- integer(ZOLTAN_INT), intent(in) :: global_id
- integer(ZOLTAN_INT), intent(in) :: local_id
+ integer(ZOLTAN_INT), intent(in) :: num_gid_entries, num_lid_entries
+ integer(ZOLTAN_INT), intent(in) :: global_id, local_id
real(ZOLTAN_DOUBLE), intent(out) :: geom_vec(*)
integer(ZOLTAN_INT), intent(out) :: ierr
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- !!geom_vec(1) = 4*( (global_id-1)/32 ) + MODULO(global_id-1,4) + 1;
- !!geom_vec(2) = MODULO(global_id-1,32) /4 + 1;
-
- !! MMW write(*,*) cellCoordX(global_id), cellCoordY(global_id), cellCoordZ(global_id)
!! Assuming geom_dim is 3
geom_vec(1) = cellCoordX(local_id)
geom_vec(2) = cellCoordY(local_id)
@@ -235,8 +235,362 @@
ierr = ZOLTAN_OK
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
end subroutine zqfGetCellGeom
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! The ordering functions should perhaps be refactored so that there
+ !! are not separate functions for cells, edges, and vertices
+ !! Not sure if this is worth it with the additional conditionals that would
+ !! be required.
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ subroutine zoltanOrderLocHSFC_Edges(in_numedges,in_edgeIDs,in_geomDim,in_edgeX, &
+ in_edgeY, in_edgeZ)
+ implicit none
+
+ integer :: in_numedges
+ integer, dimension(:), pointer :: in_edgeIDs
+ integer :: in_geomDim
+ real (kind=RKIND), dimension(:), pointer :: in_edgeX, in_edgeY, in_edgeZ
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! local variables
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ type(Zoltan_Struct), pointer :: zz_obj
+ integer(ZOLTAN_INT) :: ierr
+
+ integer :: numGidEntries, i
+ integer(ZOLTAN_INT), allocatable :: global_ids(:), permIndices(:),permGIDs(:)
+ real(kind=RKIND), allocatable :: permXs(:),permYs(:),permZs(:)
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! Body of subroutine
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ numEdges = in_numedges
+ edgeIDs => in_edgeIDs
+ geomDim = in_geomDim
+ edgeCoordX => in_edgeX
+ edgeCoordY => in_edgeY
+ edgeCoordZ => in_edgeZ
+
+ nullify(zz_obj)
+ zz_obj => Zoltan_Create(MPI_COMM_SELF)
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! General Zoltan Parameters
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ ierr = Zoltan_Set_Param(zz_obj, "ORDER_METHOD", "LOCAL_HSFC")
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! register query functions
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ ierr = Zoltan_Set_Fn(zz_obj, ZOLTAN_NUM_OBJ_FN_TYPE,zqfNumEdges)
+ ierr = Zoltan_Set_Fn(zz_obj, ZOLTAN_OBJ_LIST_FN_TYPE,zqfGetEdges)
+ ierr = Zoltan_Set_Fn(zz_obj, ZOLTAN_NUM_GEOM_FN_TYPE,zqfGeomDim)
+ ierr = Zoltan_Set_Fn(zz_obj, ZOLTAN_GEOM_FN_TYPE, zqfGetEdgeGeom)
+
+ numGidEntries=1
+
+ allocate(global_ids(numEdges))
+ allocate(permIndices(numEdges))
+ allocate(permGIDs(numEdges))
+ allocate(permXs(numEdges))
+ allocate(permYs(numEdges))
+ allocate(permZs(numEdges))
+
+ !! MMW: There might be a way to use edgeIDs directly
+ do i=1,numEdges
+ global_ids(i) = edgeIDs(i)
+ end do
+
+ ierr = Zoltan_Order(zz_obj, numGidEntries, numEdges, global_ids, permIndices);
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! This is necessary for now until we fix a small bug in Zoltan_Order
+ !!!!!!!!!!!!!!!!!!!!!!!!!!
+ do i=1,numEdges
+ permGIDs(i) = global_ids(permIndices(i)+1)
+ permXs(i) = edgeCoordX(permIndices(i)+1)
+ permYs(i) = edgeCoordY(permIndices(i)+1)
+ permZs(i) = edgeCoordZ(permIndices(i)+1)
+ end do
+
+ !!do i=1,numEdges
+ !! write(*,*) global_ids(i), permGIDs(i)
+ !!end do
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! Actually change the ordering of the edges
+ !!!!!!!!!!!!!!!!!!!!!!!!!!
+ do i=1,numEdges
+ edgeIDs(i) = permGIDs(i)
+ edgeCoordX(i) = permXs(i)
+ edgeCoordY(i) = permYs(i)
+ edgeCoordZ(i) = permZs(i)
+ end do
+ !!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ deallocate(global_ids)
+ deallocate(permIndices)
+ deallocate(permGIDs)
+ deallocate(permXs)
+ deallocate(permYs)
+ deallocate(permZs)
+
+ call Zoltan_Destroy(zz_obj)
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ end subroutine zoltanOrderLocHSFC_Edges
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! zoltan query function:
+ !! Returns number of edges
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ integer function zqfNumEdges(data, ierr)
+ ! Local declarations
+ integer(ZOLTAN_INT), intent(in) :: data(*)
+ integer(ZOLTAN_INT), intent(out) :: ierr
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ zqfNumEdges = numEdges
+ ierr = ZOLTAN_OK
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ end function zqfNumEdges
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! zoltan query function:
+ !! Returns lists of Edge IDs
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ subroutine zqfGetEdges (data, num_gid_entries, num_lid_entries, global_ids, &
+ local_ids, wgt_dim, obj_wgts, ierr)
+ integer(ZOLTAN_INT), intent(in) :: data(*)
+ integer(ZOLTAN_INT), intent(in) :: num_gid_entries, num_lid_entries
+ integer(ZOLTAN_INT), intent(out) :: global_ids(*), local_ids(*)
+ integer(ZOLTAN_INT), intent(in) :: wgt_dim
+ real(ZOLTAN_FLOAT), intent(out) :: obj_wgts(*)
+ integer(ZOLTAN_INT), intent(out) :: ierr
+
+ ! local declarations
+ integer :: i
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ do i= 1, numEdges
+ global_ids(i) = edgeIDs(i)
+ local_ids(i) = i
+ end do
+
+ ierr = ZOLTAN_OK
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ end subroutine zqfGetEdges
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! Zoltan Query Function:
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ subroutine zqfGetEdgeGeom(data, num_gid_entries, num_lid_entries, global_id, &
+ local_id, geom_vec, ierr)
+ !use zoltan
+ implicit none
+
+ integer(ZOLTAN_INT), intent(in) :: data(*)
+ integer(ZOLTAN_INT), intent(in) :: num_gid_entries, num_lid_entries
+ integer(ZOLTAN_INT), intent(in) :: global_id, local_id
+ real(ZOLTAN_DOUBLE), intent(out) :: geom_vec(*)
+ integer(ZOLTAN_INT), intent(out) :: ierr
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! Assuming geom_dim is 3
+ geom_vec(1) = edgeCoordX(local_id)
+ geom_vec(2) = edgeCoordY(local_id)
+ geom_vec(3) = edgeCoordZ(local_id)
+
+ ierr = ZOLTAN_OK
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ end subroutine zqfGetEdgeGeom
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+
+
+
+
+
+
+
+
+
+
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ subroutine zoltanOrderLocHSFC_Verts(in_numverts,in_vertIDs,in_geomDim,in_vertX, &
+ in_vertY, in_vertZ)
+ implicit none
+
+ integer :: in_numverts
+ integer, dimension(:), pointer :: in_vertIDs
+ integer :: in_geomDim
+ real (kind=RKIND), dimension(:), pointer :: in_vertX, in_vertY, in_vertZ
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! local variables
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ type(Zoltan_Struct), pointer :: zz_obj
+ integer(ZOLTAN_INT) :: ierr
+
+ integer :: numGidEntries, i
+ integer(ZOLTAN_INT), allocatable :: global_ids(:), permIndices(:),permGIDs(:)
+ real(kind=RKIND), allocatable :: permXs(:),permYs(:),permZs(:)
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! Body of subroutine
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ numVerts = in_numverts
+ vertIDs => in_vertIDs
+ geomDim = in_geomDim
+ vertCoordX => in_vertX
+ vertCoordY => in_vertY
+ vertCoordZ => in_vertZ
+
+ nullify(zz_obj)
+ zz_obj => Zoltan_Create(MPI_COMM_SELF)
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! General Zoltan Parameters
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ ierr = Zoltan_Set_Param(zz_obj, "ORDER_METHOD", "LOCAL_HSFC")
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! register query functions
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ ierr = Zoltan_Set_Fn(zz_obj, ZOLTAN_NUM_OBJ_FN_TYPE,zqfNumVerts)
+ ierr = Zoltan_Set_Fn(zz_obj, ZOLTAN_OBJ_LIST_FN_TYPE,zqfGetVerts)
+ ierr = Zoltan_Set_Fn(zz_obj, ZOLTAN_NUM_GEOM_FN_TYPE,zqfGeomDim)
+ ierr = Zoltan_Set_Fn(zz_obj, ZOLTAN_GEOM_FN_TYPE, zqfGetVertGeom)
+
+ numGidEntries=1
+
+ allocate(global_ids(numVerts))
+ allocate(permIndices(numVerts))
+ allocate(permGIDs(numVerts))
+ allocate(permXs(numVerts))
+ allocate(permYs(numVerts))
+ allocate(permZs(numVerts))
+
+ !! MMW: There might be a way to use vertIDs directly
+ do i=1,numVerts
+ global_ids(i) = vertIDs(i)
+ end do
+
+ ierr = Zoltan_Order(zz_obj, numGidEntries, numVerts, global_ids, permIndices);
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! This is necessary for now until we fix a small bug in Zoltan_Order
+ !!!!!!!!!!!!!!!!!!!!!!!!!!
+ do i=1,numVerts
+ permGIDs(i) = global_ids(permIndices(i)+1)
+ permXs(i) = vertCoordX(permIndices(i)+1)
+ permYs(i) = vertCoordY(permIndices(i)+1)
+ permZs(i) = vertCoordZ(permIndices(i)+1)
+ end do
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! Actually change the ordering of the verts
+ !!!!!!!!!!!!!!!!!!!!!!!!!!
+ do i=1,numVerts
+ vertIDs(i) = permGIDs(i)
+ vertCoordX(i) = permXs(i)
+ vertCoordY(i) = permYs(i)
+ vertCoordZ(i) = permZs(i)
+ end do
+ !!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ deallocate(global_ids)
+ deallocate(permIndices)
+ deallocate(permGIDs)
+ deallocate(permXs)
+ deallocate(permYs)
+ deallocate(permZs)
+
+ call Zoltan_Destroy(zz_obj)
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ end subroutine zoltanOrderLocHSFC_Verts
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! zoltan query function:
+ !! Returns number of verts
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ integer function zqfNumVerts(data, ierr)
+
+ ! Local declarations
+ integer(ZOLTAN_INT), intent(in) :: data(*)
+ integer(ZOLTAN_INT), intent(out) :: ierr
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ zqfNumVerts = numVerts
+ ierr = ZOLTAN_OK
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ end function zqfNumVerts
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! zoltan query function:
+ !! Returns lists of Vert IDs
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ subroutine zqfGetVerts (data, num_gid_entries, num_lid_entries, global_ids, &
+ local_ids, wgt_dim, obj_wgts, ierr)
+
+ integer(ZOLTAN_INT), intent(in) :: data(*)
+ integer(ZOLTAN_INT), intent(in) :: num_gid_entries, num_lid_entries
+ integer(ZOLTAN_INT), intent(out) :: global_ids(*), local_ids(*)
+ integer(ZOLTAN_INT), intent(in) :: wgt_dim
+ real(ZOLTAN_FLOAT), intent(out) :: obj_wgts(*)
+ integer(ZOLTAN_INT), intent(out) :: ierr
+
+ ! local declarations
+ integer :: i
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ do i= 1, numVerts
+ global_ids(i) = vertIDs(i)
+ local_ids(i) = i
+ end do
+
+ ierr = ZOLTAN_OK
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ end subroutine zqfGetVerts
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! Zoltan Query Function:
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ subroutine zqfGetVertGeom(data, num_gid_entries, num_lid_entries, global_id, &
+ local_id, geom_vec, ierr)
+ !use zoltan
+ implicit none
+
+ integer(ZOLTAN_INT), intent(in) :: data(*)
+ integer(ZOLTAN_INT), intent(in) :: num_gid_entries, num_lid_entries
+ integer(ZOLTAN_INT), intent(in) :: global_id, local_id
+ real(ZOLTAN_DOUBLE), intent(out) :: geom_vec(*)
+ integer(ZOLTAN_INT), intent(out) :: ierr
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !! Assuming geom_dim is 3
+ geom_vec(1) = vertCoordX(local_id)
+ geom_vec(2) = vertCoordY(local_id)
+ geom_vec(3) = vertCoordZ(local_id)
+
+ ierr = ZOLTAN_OK
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ end subroutine zqfGetVertGeom
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+
+
+
end module zoltan_interface
</font>
</pre>