<p><b>mpetersen@lanl.gov</b> 2010-10-15 09:11:12 -0600 (Fri, 15 Oct 2010)</p><p>Correct error due to isolated cells on a processor, described in<br>
detail below. Also cleaned up some indentation.<br>
<br>
We have had trouble with our global domains with land, and I have a<br>
bug fix that solves the problem. The error occurs on runs with more<br>
than one processor, where a processor has an isolated ocean cell. For<br>
example:<br>
<br>
1 1 1 L L L 2 2 2<br>
1 1 1 L L 1 2 2 2<br>
1 1 1 L L L 2 2 2<br>
<br>
These are cells owned by processor 1 and 2, and land cells L are shown<br>
but are not really cells. <br>
<br>
In module block_decomp, subroutine block_decomp_add_halo, the section<br>
<br>
do i=1,local_graph_info % nVertices<br>
do j=1,local_graph_info % nAdjacent(i)<br>
if (local_graph_info % adjacencyList(j,i) /= 0) then<br>
if (.not. hash_search(h, local_graph_info % adjacencyList(j,i))) then<br>
call hash_insert(h, local_graph_info % adjacencyList(j,i))<br>
end if<br>
end if<br>
end do<br>
end do <br>
<br>
only adds cells to the hash table if they have neighbors owned by that<br>
processor. Thus isolated cells are not added,<br>
<br>
local_graph_with_halo % nVerticesTotal = hash_size(h)<br>
<br>
is too small, the warning<br>
<br>
block_decomp_add_halo: Somehow we don't have the right number of total cells.<br>
<br>
appears, and we end up reading outside of memory bounds on arrays like<br>
local_graph_with_halo % vertexID.<br>
<br>
My fix is to add all cells to the hash table first, and then add<br>
neighbors of cells.<br>
<br>
lo1-fe> svn diff module_block_decomp.F<br>
Index: module_block_decomp.F<br>
===================================================================<br>
--- module_block_decomp.F (revision 528)<br>
+++ module_block_decomp.F (working copy)<br>
@@ -236,6 +236,10 @@<br>
call hash_init(h)<br>
<br>
do i=1,local_graph_info % nVertices<br>
+ call hash_insert(h, local_graph_info % vertexID(i))<br>
+ end do<br>
+<br>
+ do i=1,local_graph_info % nVertices<br>
do j=1,local_graph_info % nAdjacent(i)<br>
if (local_graph_info % adjacencyList(j,i) /= 0) then<br>
if (.not. hash_search(h, local_graph_info % adjacencyList(j,i))) then<br>
<br>
A more elegant solution would be for the domain decomposition to not<br>
have isolated cells, but kmetis handles this and I don't know how to<br>
change it. Also, users should be able to choose any grid and<br>
decomposition they like without unnecessary errors.<br>
</p><hr noshade><pre><font color="gray">Modified: trunk/mpas/src/framework/module_block_decomp.F
===================================================================
--- trunk/mpas/src/framework/module_block_decomp.F 2010-10-14 23:43:48 UTC (rev 558)
+++ trunk/mpas/src/framework/module_block_decomp.F 2010-10-15 15:11:12 UTC (rev 559)
@@ -147,7 +147,9 @@
do j=1,maxCells
if (cellsOnEdge(j,i) /= 0) exit
end do
-if (j > maxCells) write(0,*) 'Error in block_decomp_partitioned_edge_list: edge/vertex is not adjacent to any valid cells'
+ if (j > maxCells) &
+ write(0,*) 'Error in block_decomp_partitioned_edge_list: ',&
+ 'edge/vertex is not adjacent to any valid cells'
if (hash_search(h, cellsOnEdge(j,i))) then
lastEdge = lastEdge + 1
edgeIDList(lastEdge) = edgeIDListLocal(i)
@@ -155,14 +157,16 @@
ghostEdgeStart = ghostEdgeStart - 1
edgeIDList(ghostEdgeStart) = edgeIDListLocal(i)
end if
-if (ghostEdgeStart <= lastEdge) then
- write(0,*) 'block_decomp_partitioned_edge_list: Somehow we have more edges than we thought we should.'
-end if
+ if (ghostEdgeStart <= lastEdge) then
+ write(0,*) 'block_decomp_partitioned_edge_list: ',&
+ 'Somehow we have more edges than we thought we should.'
+ end if
end do
-if (ghostEdgeStart /= lastEdge + 1) then
- write(0,*) 'block_decomp_partitioned_edge_list: Somehow we didn''t have enough edges to fill edgeIDList.'
-end if
+ if (ghostEdgeStart /= lastEdge + 1) then
+ write(0,*) 'block_decomp_partitioned_edge_list:',&
+ ' Somehow we didn''t have enough edges to fill edgeIDList.'
+ end if
call hash_destroy(h)
@@ -202,10 +206,11 @@
do j=1,nEdgesOnCell(i)
if (.not. hash_search(h, edgesOnCell(j,i))) then
k = k + 1
-if (k > nEdges) then
- write(0,*) 'block_decomp_all_edges_in_block: Trying to add more edges than expected.'
- return
-end if
+ if (k > nEdges) then
+ write(0,*) 'block_decomp_all_edges_in_block: ',&
+ 'Trying to add more edges than expected.'
+ return
+ end if
edgeList(k) = edgesOnCell(j,i)
call hash_insert(h, edgesOnCell(j,i))
end if
@@ -214,9 +219,10 @@
call hash_destroy(h)
-if (k < nEdges) then
- write(0,*) 'block_decomp_all_edges_in_block: Listed fewer edges than expected.'
-end if
+ if (k < nEdges) then
+ write(0,*) 'block_decomp_all_edges_in_block: ',&
+ 'Listed fewer edges than expected.'
+ end if
end subroutine block_decomp_all_edges_in_block
@@ -236,6 +242,10 @@
call hash_init(h)
do i=1,local_graph_info % nVertices
+ call hash_insert(h, local_graph_info % vertexID(i))
+ end do
+
+ do i=1,local_graph_info % nVertices
do j=1,local_graph_info % nAdjacent(i)
if (local_graph_info % adjacencyList(j,i) /= 0) then
if (.not. hash_search(h, local_graph_info % adjacencyList(j,i))) then
@@ -259,8 +269,9 @@
call hash_init(h)
do i=1,local_graph_info % nVertices
-if (hash_search(h, local_graph_info % vertexID(i))) &
- write(0,*) 'block_decomp_add_halo: There appear to be duplicates in vertexID list.'
+ if (hash_search(h, local_graph_info % vertexID(i))) &
+ write(0,*) 'block_decomp_add_halo: ', &
+ 'There appear to be duplicates in vertexID list.'
call hash_insert(h, local_graph_info % vertexID(i))
local_graph_with_halo % vertexID(i) = local_graph_info % vertexID(i)
local_graph_with_halo % nAdjacent(i) = local_graph_info % nAdjacent(i)
@@ -268,8 +279,9 @@
end do
k = local_graph_with_halo % ghostStart
-if (hash_size(h) /= k-1) &
- write(0,*) 'block_decomp_add_halo: Somehow we don''t have the right number of non-ghost cells.'
+ if (hash_size(h) /= k-1) &
+ write(0,*) 'block_decomp_add_halo: ',&
+ 'Somehow we don''t have the right number of non-ghost cells.'
do i=1,local_graph_info % nVertices
do j=1,local_graph_info % nAdjacent(i)
if (local_graph_info % adjacencyList(j,i) /= 0) then
@@ -281,8 +293,9 @@
end if
end do
end do
-if (local_graph_with_halo % nVerticesTotal /= k-1) &
- write(0,*) 'block_decomp_add_halo: Somehow we don''t have the right number of total cells.'
+ if (local_graph_with_halo % nVerticesTotal /= k-1) &
+ write(0,*) 'block_decomp_add_halo: ',&
+ 'Somehow we don''t have the right number of total cells.'
call hash_destroy(h)
Modified: trunk/mpas/src/framework/module_io_input.F
===================================================================
--- trunk/mpas/src/framework/module_io_input.F 2010-10-14 23:43:48 UTC (rev 558)
+++ trunk/mpas/src/framework/module_io_input.F 2010-10-15 15:11:12 UTC (rev 559)
@@ -1127,7 +1127,8 @@
nferr = nf_get_att_real(input_obj % rd_ncid, NF_GLOBAL, attname, attvalue)
end if
if (nferr /= NF_NOERR) then
- write(0,*) 'Warning: Attribute '//trim(attname)//' not found in '//trim(input_obj % filename)
+ write(0,*) 'Warning: Attribute '//trim(attname)//&
+ ' not found in '//trim(input_obj % filename)
if (index(attname, 'sphere_radius') /= 0) then
write(0,*) ' Setting '//trim(attname)//' to 1.0'
attvalue = 1.0
@@ -1151,7 +1152,8 @@
nferr = nf_get_att_text(input_obj % rd_ncid, NF_GLOBAL, attname, attvalue)
if (nferr /= NF_NOERR) then
- write(0,*) 'Warning: Attribute '//trim(attname)//' not found in '//trim(input_obj % filename)
+ write(0,*) 'Warning: Attribute '//trim(attname)//&
+ ' not found in '//trim(input_obj % filename)
if (index(attname, 'on_a_sphere') /= 0) then
write(0,*) ' Setting '//trim(attname)//' to ''YES'''
attvalue = 'YES'
</font>
</pre>