<p><b>dwj07@fsu.edu</b> 2013-02-12 19:11:32 -0700 (Tue, 12 Feb 2013)</p><p><br>
-- BRANCH COMMIT --<br>
<br>
Update script to be more versatile.<br>
</p><hr noshade><pre><font color="gray">Modified: branches/tools/python_scripts/processor_decompositions/README
===================================================================
--- branches/tools/python_scripts/processor_decompositions/README 2013-02-12 23:03:42 UTC (rev 2464)
+++ branches/tools/python_scripts/processor_decompositions/README 2013-02-13 02:11:32 UTC (rev 2465)
@@ -1,25 +1,46 @@
Author: Doug Jacobsen
Date: 02/12/13
-Usage: make_block_graph.py [options]
+Usage: make_partition_files.py [options]
Options:
-h, --help show this help message and exit
-f FILE, --file=FILE Path to grid file
- -g FILE, --graph=FILE
- Path to graph file
+ -m METIS, --metis=METIS
+ Path or name of metis executable
+ -p PROCS, --procs=PROCS
+ Number of processors for decomposition
+ -b BLOCKS, --blocks=BLOCKS
+ Number of blocks for decomposition
+ -w VAR, --weights=VAR
+ Field to weight block partition file on.
-
About:
- This script is intended to take in a grid.nc and graph.info.partn.N file
- where N is the number of blocks a simulation is intended to run with.
+ This script is used to create weighted and unweighted hirearchical
+ decompositions for use with multiple blocks per MPI process in any MPAS core.
+
+ As input, it takes a grid.nc file, the number of blocks to decompose, the
+ number of processor to decompose, and the path to the metis executable or the
+ name of the metis executable if it's in your path.
+
+ Optionally, it takes the name of the variable in grid.nc that should be
+ used to weight the block decomposition file with.
- On output, it generates a block.graph.info file, which provides a graph
- file for blocks.
+ After running the script, several files are generated which are described below:
+ graph.info - Single processor graph decomposition file.
+ (optional) weighted.graph.info - graph.info file used for creating weighted partition files.
+ (weighted.)graph.info.part.BLOCKS - partition file that is weighted or unweighted and has BLOCKS number of partitions.
+ block.graph.info - graph file of blocks. Equivalent to "blocksOnBlock"
+ block.graph.info.part.PROCS - partition file that has PROCS number of partitions.
- This block.graph.info file currently can be passed into metis to generate a
- block.graph.info.part.M file, where M is the number of processors a run
- is intended to be run with.
+ These can be used in MPAS as the block decomposition file
+ (graph.info.part.BLOCKS) and the proc decomposition file
+ (block.graph.info.part.PROCS) to control the topology of blocks on MPI tasks.
- The block.graph.info.part.M file can be used in tandem with the
- graph.info.part.N file to improve performance in runs with multiple blocks.
+ Additional graph.info.part files can be created by running the script
+ multiple times, or my running metis on the graph.info file.
+
+ Additional block.graph.info.part files can be created through the same
+ process, but they can only be used in tandem with the corresponding
+ graph.info.part.BLOCKS file.
+
Deleted: branches/tools/python_scripts/processor_decompositions/make_block_graph.py
===================================================================
--- branches/tools/python_scripts/processor_decompositions/make_block_graph.py 2013-02-12 23:03:42 UTC (rev 2464)
+++ branches/tools/python_scripts/processor_decompositions/make_block_graph.py 2013-02-13 02:11:32 UTC (rev 2465)
@@ -1,90 +0,0 @@
-#!/usr/bin/python
-import sys, os, glob, shutil, numpy, math
-
-from collections import defaultdict
-
-from netCDF4 import *
-from netCDF4 import Dataset as NetCDFFile
-from pylab import *
-
-import matplotlib
-import matplotlib.pyplot as plt
-
-from optparse import OptionParser
-
-parser = OptionParser()
-parser.add_option("-f", "--file", dest="filename", help="Path to grid file", metavar="FILE")
-parser.add_option("-g", "--graph", dest="graphfile", help="Path to graph file", metavar="FILE")
-
-options, args = parser.parse_args()
-
-if not options.filename:
- parser.error("A grid file is required.")
-
-if not options.graphfile:
- parser.error("A graph file is required.")
-
-grid = NetCDFFile(options.filename, 'r')
-
-nCells = len(grid.dimensions['nCells'])
-
-nEdgesOnCell = grid.variables['nEdgesOnCell'][:]
-cellsOnCell = grid.variables['cellsOnCell'][:] - 1
-grid.close()
-
-num_blocks = 0
-owning_block = [0] * nCells
-
-graph = open(options.graphfile, 'r')
-
-i = -1
-for block in iter(lambda: graph.readline(), ""):
- if i >= 0:
- block_arr = block.split()
- owning_block[i] = int(block_arr[0])+1
- num_blocks = max(num_blocks, owning_block[i])
-
- i = i + 1
-
-graph.close()
-
-blocksOnBlock = defaultdict(list)
-nEdges = 0
-
-for i in np.arange(0, nCells):
- for j in np.arange(0, nEdgesOnCell[i]):
- iCell = cellsOnCell[i][j]
- try:
- can_add = True
- for block in blocksOnBlock[owning_block[i]]:
- if block == owning_block[iCell]:
- can_add = False
- except:
- can_add = True
-
- if iCell == -1:
- can_add = False
-
- if owning_block[iCell] == owning_block[i]:
- can_add = False
-
- if owning_block[iCell] <= 0:
- can_add = False
-
- if owning_block[i] <= 0:
- can_add = False
-
- if can_add:
- nEdges = nEdges + 1
- blocksOnBlock[owning_block[i]].append(owning_block[iCell])
-
-del blocksOnBlock[0]
-
-block_graph = open('block.graph.info', 'w+')
-block_graph.write('%s %s</font>
<font color="red">'%(int(num_blocks), int(nEdges/2)))
-for i in np.arange(1, num_blocks+1):
- for block in blocksOnBlock[i]:
- block_graph.write('%s '%int(block))
- block_graph.write('</font>
<font color="gray">')
-
-block_graph.close()
Copied: branches/tools/python_scripts/processor_decompositions/make_partition_files.py (from rev 2464, branches/tools/python_scripts/processor_decompositions/make_block_graph.py)
===================================================================
--- branches/tools/python_scripts/processor_decompositions/make_partition_files.py (rev 0)
+++ branches/tools/python_scripts/processor_decompositions/make_partition_files.py 2013-02-13 02:11:32 UTC (rev 2465)
@@ -0,0 +1,162 @@
+#!/usr/bin/python
+import sys, os, glob, shutil, numpy, math
+
+import subprocess
+
+from collections import defaultdict
+
+from netCDF4 import *
+from netCDF4 import Dataset as NetCDFFile
+from pylab import *
+
+import matplotlib
+import matplotlib.pyplot as plt
+
+from optparse import OptionParser
+
+parser = OptionParser()
+parser.add_option("-f", "--file", dest="filename", help="Path to grid file", metavar="FILE")
+parser.add_option("-m", "--metis", dest="metis_path", help="Path or name of metis executable", metavar="METIS")
+parser.add_option("-p", "--procs", dest="num_procs", help="Number of processors for decomposition", metavar="PROCS")
+parser.add_option("-b", "--blocks", dest="num_blocks", help="Number of blocks for decomposition", metavar="BLOCKS")
+parser.add_option("-w", "--weights", dest="weight_field", help="Field to weight block partition file on.", metavar="VAR")
+
+options, args = parser.parse_args()
+
+if not options.metis_path:
+ parser.error("Path to metis is required.")
+
+if not options.filename:
+ parser.error("A grid file is required.")
+
+if not options.num_procs:
+ parser.error("Number of processors is required.")
+
+if not options.num_blocks:
+ parser.error("Number of blocks is required.")
+
+if not options.weight_field:
+ print "Weight field missing. Defaulting to unweighted graphs."
+ weighted_parts = False
+else:
+ weighted_parts = True
+
+dev_null = open(os.devnull, 'w')
+grid = NetCDFFile(options.filename, 'r')
+
+nCells = len(grid.dimensions['nCells'])
+nEdges = len(grid.dimensions['nEdges'])
+
+nEdgesOnCell = grid.variables['nEdgesOnCell'][:]
+cellsOnCell = grid.variables['cellsOnCell'][:] - 1
+if weighted_parts:
+ try:
+ weights = grid.variables[options.weight_field][:]
+ except:
+ print options.weight_field, ' not found in file. Defaulting to un-weighted partitions.'
+ weighted_parts = False
+grid.close()
+
+num_blocks = 0
+owning_block = [0] * nCells
+
+nEdges = 0
+for i in np.arange(0, nCells):
+ for j in np.arange(0,nEdgesOnCell[i]):
+ if cellsOnCell[i][j] != -1:
+ nEdges = nEdges + 1
+
+nEdges = nEdges/2
+
+graph = open('graph.info', 'w+')
+graph.write('%s %s</font>
<font color="blue">'%(nCells, nEdges))
+if weighted_parts:
+ wgraph = open('weighted.graph.info', 'w+')
+ wgraph.write('%s %s 010</font>
<font color="blue">'%(nCells, nEdges))
+
+for i in np.arange(0, nCells):
+ wgraph.write('%s '%weights[i])
+ for j in np.arange(0,nEdgesOnCell[i]):
+ if weighted_parts:
+ if(cellsOnCell[i][j] >= 0):
+ wgraph.write('%s '%(cellsOnCell[i][j]+1))
+
+ if(cellsOnCell[i][j] >= 0):
+ graph.write('%s '%(cellsOnCell[i][j]+1))
+ graph.write('</font>
<font color="blue">')
+
+ if weighted_parts:
+ wgraph.write('</font>
<font color="blue">')
+graph.close()
+
+if weighted_parts:
+ wgraph.close()
+
+command = "%s"%(options.metis_path)
+if weighted_parts:
+ arg1 = "weighted.graph.info"
+else:
+ arg1 = "graph.info"
+arg2 = "%s"%options.num_blocks
+subprocess.call([command, arg1, arg2], stdout=dev_null, stderr=dev_null)
+
+if weighted_parts:
+ graph = open('weighted.graph.info.part.%s'%options.num_blocks, 'r')
+else:
+ graph = open('graph.info.part.%s'%options.num_blocks, 'r')
+i = -1
+for block in iter(lambda: graph.readline(), ""):
+ if i >= 0:
+ block_arr = block.split()
+ owning_block[i] = int(block_arr[0])+1
+ num_blocks = max(num_blocks, owning_block[i])
+
+ i = i + 1
+graph.close()
+
+blocksOnBlock = defaultdict(list)
+nEdges = 0
+
+for i in np.arange(0, nCells):
+ for j in np.arange(0, nEdgesOnCell[i]):
+ iCell = cellsOnCell[i][j]
+ try:
+ can_add = True
+ for block in blocksOnBlock[owning_block[i]]:
+ if block == owning_block[iCell]:
+ can_add = False
+ except:
+ can_add = True
+
+ if iCell == -1:
+ can_add = False
+
+ if owning_block[iCell] == owning_block[i]:
+ can_add = False
+
+ if owning_block[iCell] <= 0:
+ can_add = False
+
+ if owning_block[i] <= 0:
+ can_add = False
+
+ if can_add:
+ nEdges = nEdges + 1
+ blocksOnBlock[owning_block[i]].append(owning_block[iCell])
+
+del blocksOnBlock[0]
+
+block_graph = open('block.graph.info', 'w+')
+block_graph.write('%s %s</font>
<font color="blue">'%(int(num_blocks), int(nEdges/2)))
+for i in np.arange(1, num_blocks+1):
+ for block in blocksOnBlock[i]:
+ block_graph.write('%s '%int(block))
+ block_graph.write('</font>
<font color="blue">')
+
+block_graph.close()
+
+command = "%s"%(options.metis_path)
+arg1 = "block.graph.info"
+arg2 = "%s"%options.num_procs
+subprocess.call([command, arg1, arg2], stdout=dev_null, stderr=dev_null)
+
</font>
</pre>