<p><b>dwj07@fsu.edu</b> 2011-12-02 14:53:19 -0700 (Fri, 02 Dec 2011)</p><p><br>
-- BRANCH COMMIT --<br>
<br>
Adding some comments.<br>
<br>
Improving compiler compatibility with basin (mainly for gfortran).<br>
</p><hr noshade><pre><font color="gray">Modified: branches/ocean_projects/ocean_test_cases_staging/ocean/baroclinic_channel/basin_src/module_write_netcdf.F
===================================================================
--- branches/ocean_projects/ocean_test_cases_staging/ocean/baroclinic_channel/basin_src/module_write_netcdf.F 2011-12-02 20:57:32 UTC (rev 1234)
+++ branches/ocean_projects/ocean_test_cases_staging/ocean/baroclinic_channel/basin_src/module_write_netcdf.F 2011-12-02 21:53:19 UTC (rev 1235)
@@ -95,7 +95,7 @@
integer, intent(in) :: nVertLevels
integer, intent(in) :: vertexDegree
character (len=16) :: on_a_sphere
- double precision :: sphere_radius
+ real*8 :: sphere_radius
integer :: nferr
Modified: branches/ocean_projects/ocean_test_cases_staging/ocean/hopper_submit_template.sh
===================================================================
--- branches/ocean_projects/ocean_test_cases_staging/ocean/hopper_submit_template.sh 2011-12-02 20:57:32 UTC (rev 1234)
+++ branches/ocean_projects/ocean_test_cases_staging/ocean/hopper_submit_template.sh 2011-12-02 21:53:19 UTC (rev 1235)
@@ -3,8 +3,8 @@
#PBS -q regular
#PBS -l walltime=20:00
#PBS -l mppwidth=num_procs
-#PBD -o run_name.num_procs.out
-#PBD -e run_name.num_procs.err
+#PBD -o stdout.out
+#PBD -e stderr.err
cd working_dir
aprun -n num_procs ocean_model_executable
Modified: branches/ocean_projects/ocean_test_cases_staging/ocean/lobo_submit_template.sh
===================================================================
--- branches/ocean_projects/ocean_test_cases_staging/ocean/lobo_submit_template.sh 2011-12-02 20:57:32 UTC (rev 1234)
+++ branches/ocean_projects/ocean_test_cases_staging/ocean/lobo_submit_template.sh 2011-12-02 21:53:19 UTC (rev 1235)
@@ -3,7 +3,7 @@
#MSUB -l walltime=20:00
#MSUB -l nodes=num_nodes:ppn=procs_per_node
#MSUB -d working_dir
-#MSUB -o run_name.num_procs.out
-#MSUB -e run_name.num_procs.err
+#MSUB -o stdout.out
+#MSUB -e stderr.err
mpirun -np num_procs ./ocean_model.exe
Modified: branches/ocean_projects/ocean_test_cases_staging/ocean/lock_exchange/basin_src/module_write_netcdf.F
===================================================================
--- branches/ocean_projects/ocean_test_cases_staging/ocean/lock_exchange/basin_src/module_write_netcdf.F 2011-12-02 20:57:32 UTC (rev 1234)
+++ branches/ocean_projects/ocean_test_cases_staging/ocean/lock_exchange/basin_src/module_write_netcdf.F 2011-12-02 21:53:19 UTC (rev 1235)
@@ -95,7 +95,7 @@
integer, intent(in) :: nVertLevels
integer, intent(in) :: vertexDegree
character (len=16) :: on_a_sphere
- double precision :: sphere_radius
+ real*8 :: sphere_radius
integer :: nferr
Modified: branches/ocean_projects/ocean_test_cases_staging/ocean/lock_exchange/makeMeshes.sh
===================================================================
--- branches/ocean_projects/ocean_test_cases_staging/ocean/lock_exchange/makeMeshes.sh 2011-12-02 20:57:32 UTC (rev 1234)
+++ branches/ocean_projects/ocean_test_cases_staging/ocean/lock_exchange/makeMeshes.sh 2011-12-02 21:53:19 UTC (rev 1235)
@@ -148,10 +148,12 @@
ln -f -s ${CUR_DIR}/${NAME}_${VERTLEV}levs/lock_exchange_${NAME}_${VERTLEV}levs.grid.nc ${NAME}_${VERTLEV}levs/${PROC}procs/grid.nc
ln -f -s ${CUR_DIR}/${NAME}_${VERTLEV}levs/namelist.input ${NAME}_${VERTLEV}levs/${PROC}procs/namelist.input
+ ## Copy executable to run directory
if [ $# -ge 1 ]; then
cp $1 ${NAME}_${VERTLEV}levs/${PROC}procs/ocean_model.exe
fi
+ ## Copy run information to run directory
if [ $# -ge 2 ]; then
cp $2 ${NAME}_${VERTLEV}levs/${PROC}procs/.
fi
Modified: branches/ocean_projects/ocean_test_cases_staging/ocean/oceanTestCases.sh
===================================================================
--- branches/ocean_projects/ocean_test_cases_staging/ocean/oceanTestCases.sh 2011-12-02 20:57:32 UTC (rev 1234)
+++ branches/ocean_projects/ocean_test_cases_staging/ocean/oceanTestCases.sh 2011-12-02 21:53:19 UTC (rev 1235)
@@ -4,6 +4,12 @@
## Change these variables to customize the run ##
#################################################
+## The repository address below shows how to change the revision number.
+## Be careful with the compile set for an older version, as not all the options
+## that exists now were present in older versions.
+
+# REPOSITORY_ADDRESS="https://svn-mpas-model.cgd.ucar.edu/trunk/mpas -r 995"
+
REPOSITORY_ADDRESS="https://svn-mpas-model.cgd.ucar.edu/trunk/mpas"
COMPILE_SET="ifort-serial"
MACHINE_NAME="lobo"
@@ -25,25 +31,33 @@
CASE_VALID="no"
+## Remove forward slashes from case name, so the case can be tab-compeleted using the directory name
CASE=`echo $CASE | sed "s|/||g"`
+
## Function Definitions ###{{{
+## Checkout and compile mpas.
setup_mpas () { #{{{
rm -f .run_info
rm -f run_paths
touch .run_info
+ ## Check out mpas from repository
echo "Checking out ${REPOSITORY_ADDRESS}"
svn co $REPOSITORY_ADDRESS mpas > /dev/null
cd mpas
+ ## Compile mpas using given compile set
echo "Compiling mpas"
make $COMPILE_SET CORE=ocean > /dev/null &> /dev/null
REV=`svn info | grep "Revision"`
cd ..
+ ## Make a variable containing the make options used to build mpas
MAKE_OPTS=`grep "$COMPILE_SET:" mpas/Makefile -A 11 | tail -n 11 | grep "[a-Z]"`
+ ## Create the run_info file, which will be used later to identify what
+ ## revision the run was performed from
echo "Compile flags:" >> .run_info
grep "$COMPILE_SET:" mpas/Makefile -A 11 | grep "[a-Z]" >> .run_info
echo "" >> .run_info
@@ -52,12 +66,16 @@
echo "$REV" >> .run_info
echo "$MACHINE_NAME" >> .run_info
+ ## Create a header of a Makefile for use in compiling subcode with the same compiler.
echo "${COMPILE_SET}:" > .Makefile.front
echo -e "\t$MAKE_OPTS" >> .Makefile.front
} #}}}
+## Setup test case
setup () { #{{{
echo "Setting up ${CASE} test case"
+
+ ## Copy makefile header to sub directories if needed
if [ -e ${CASE}/basin_src ]; then
cp .Makefile.front ${CASE}/basin_src/Makefile.front
fi
@@ -68,12 +86,15 @@
cp .run_info run_info
cd ${CASE}
+
+ ## Call script to setup run directories, and meshes if needed
./makeMeshes.sh ${CUR_DIR}/mpas/src/ocean_model.exe ${CUR_DIR}/run_info "$PROC_LIST"
cd ${CUR_DIR}
rm run_info
} #}}}
+## Submit test case runs to queue
submit () { #{{{
echo "Submitting ${CASE} test case runs"
RUNS=`cat $CASE/run_paths`
@@ -81,13 +102,17 @@
rm -f start_times_$CASE.sh
mkdir -p $CASE/submits
+ ## Loop over run directories in ./${CASE}
for RUN in $RUNS
do
+ ## Get the name of the current run, and number of processors
+ ## from the run directory
NAME=`echo ${RUN%/*procs}`
NAME=`echo ${NAME##*/}`
PROCS=`echo ${RUN##*/}`
PROCS=`echo ${PROCS%%procs}`
+ ## Determine division of processors based on machine's PPN
if [ $PROCS -lt $MACHINE_PPN ]; then
NODES=1
PPN=${PROCS}
@@ -96,25 +121,33 @@
PPN=${MACHINE_PPN}
fi
+ ## Generate submission script
cat ${CUR_DIR}/${MACHINE_NAME}_submit_template.sh | sed "s/run_name/${CASE}_${NAME}/g" \
| sed "s/num_nodes/${NODES}/g" | sed "s/procs_per_node/${PPN}/g" | sed "s|working_dir|$RUN|g" \
| sed "s/num_procs/$PROCS/g" > $CASE/submits/$NAME.sh
+ ## Submit job, and retain job id for later use
JOB_ID=`${SUBMISSION_CMD} ${CUR_DIR}/${CASE}/submits/$NAME.sh`
JOB_ID=`echo $JOB_ID | grep [0-9]`
+
+ ## Write job id to a file
echo $JOB_ID >> job_ids_${CASE}
+ ## Write start_times and cancel_jobs scripts
echo 'echo "' "$JOB_ID" '" `showstart ' "$JOB_ID | head -n 3 | tail -n 1 | awk '{print" '$6' " }'" '`' >> start_times_${CASE}.sh
echo "canceljob $JOB_ID > /dev/null &> /dev/null" >> cancel_jobs_${CASE}.sh
done
+ ## Finish cancel_jobs script
echo 'echo "All jobs canceled"' >> cancel_jobs_${CASE}.sh
echo "rm -f cancel_jobs_${CASE}.sh start_times_${CASE}.sh job_ids_${CASE}" >> cancel_jobs_${CASE}.sh
+ ## Make scripts executable
chmod +x cancel_jobs_${CASE}.sh
chmod +x start_times_${CASE}.sh
} #}}}
+## Postprocess test case
postprocess () { #{{{
echo "Processing ${CASE} test case data"
rm -f timing_results_${CASE}.txt timing_results2_${CASE}.txt
@@ -125,18 +158,24 @@
RUNS=`cat ${CASE}/run_paths`
+ ## Loop over run directories in ./${CASE}
for RUN in $RUNS
do
+
+ ## Get name, processor counts, and grid spacing from directory name
NAME=`echo ${RUN%/*procs}`
NAME=`echo ${NAME##*/}`
SPACING=`echo ${NAME%m*}`
PROCS=`echo ${RUN##*/}`
PROCS=`echo ${PROCS%%procs}`
+ ## Get final kinetic energy, total run time, and run duration
+ ## from run directory
KE=`cat ${RUN}/stats_max.txt | tail -n 1 | awk '{print $7}'`
- TIME=`grep "total time" ${RUN}/${CASE}_${NAME}.${PROCS}.out | awk '{print $4}' | head -n 1`
+ TIME=`grep "total time" ${RUN}/stdout.out | awk '{print $4}' | head -n 1`
DURATION=`grep "config_run_duration" ${RUN}/namelist.input`
+ ## Determine spacing in kilometers
KM=`expr match "$SPACING" 'k'`
if [ $KM == 0 ]; then
@@ -145,17 +184,22 @@
KM=`echo ${SPACING%k}`
fi
+ ## Build portions of timing_results file
+ ## Written separately so paste will format properly
echo "${NAME}_${PROCS}procs" >> first
echo "${DURATION}" >> second
echo "Final KE: ${KE}" >> third
echo "Total time: ${TIME}" >> fourth
+ ## Write timing_results2 file
echo "${KM}" ${PROCS} ${TIME} >> timing_results2_${CASE}.txt
+ ## Build portions of timing_results3 file
echo "${PROCS} " >> timing_results3_front
echo " ${TIME} ; ... % ${NAME}-${PROCS}procs" >> timing_results3_back
done
+ ## Finish timing_results files, and clean up
echo "]" >> timing_results3_front
echo " ]/86400.;" >> timing_results3_back
@@ -167,12 +211,14 @@
rm -f first second third fourth timing_results3_front timing_results3_back
} #}}}
+## Clean up mpas directory
clean_mpas () { #{{{
echo "Cleaning MPAS"
rm -rf mpas
rm -f .run_info .Makefile.front
} #}}}
+## Clean up test case directory
clean () { #{{{
echo "Cleaning ${CASE} test case"
cd ${CASE}
@@ -187,11 +233,13 @@
#}}}
+## Check to see if mpas has already been built, if not set it up
if [ ! -e mpas/src/ocean_model.exe ]; then
echo "MPAS is not setup properly yet. Setting up MPAS first."
setup_mpas
fi
+## Check to see if pmetis is in path, as it is supposed to be. If not exit
PMETIS_CHECK=`which pmetis | grep "no pmetis in"`
if [ -n "${PMETIS}" ]; then
@@ -199,6 +247,7 @@
exit
fi
+## Verify a valid action and case are passed into script. If not print usage statement, and exit
if [ $# -ge 2 ]; then
if [ $1 == "setup" -o $1 == "submit" -o $1 == "postprocess" -o "clean" ]; then
ACTION_VALID="yes"
@@ -214,6 +263,7 @@
fi
fi
+ ## If case is mpas, action has to be clean
if [ $CASE == "mpas" -a $ACTION != "clean" ]; then
echo "A case of mpas can only have action of clean"
echo ""
@@ -221,6 +271,7 @@
fi
fi
+## Print usage statement
if [ $VALID == "no" ]; then
echo "Invalid usage."
echo "./oceanTestCases.sh [action] [case]"
@@ -232,6 +283,7 @@
exit
fi
+## Handle different actions
if [ $ACTION == "setup" ]; then
setup
elif [ $ACTION == "submit" ]; then
Modified: branches/ocean_projects/ocean_test_cases_staging/ocean/performance/makeMeshes.sh
===================================================================
--- branches/ocean_projects/ocean_test_cases_staging/ocean/performance/makeMeshes.sh 2011-12-02 20:57:32 UTC (rev 1234)
+++ branches/ocean_projects/ocean_test_cases_staging/ocean/performance/makeMeshes.sh 2011-12-02 21:53:19 UTC (rev 1235)
@@ -1,16 +1,23 @@
#!/bin/bash
+#############################################################
+## Configure performance run using the following variables ##
+#############################################################
MESH_LIST="015km_40levs 030km_40levs 060km_40levs 120km_40levs"
-
-OCEAN_MODEL_EXECUTABLE="$1"
MESH_DIR="domains_lobo"
+## If processor list is passed in, use it. Otherwise use the default.
if [ -z "$3" ]; then
PROC_LIST="24 48 96 192 384 768 1536 3072"
else
PROC_LIST="$3"
fi
+##############################################
+## Don't edit below this unless you need to ##
+##############################################
+
+OCEAN_MODEL_EXECUTABLE="$1"
CUR_DIR=`pwd`
rm -f run_paths
@@ -20,6 +27,7 @@
for PROC in $PROC_LIST
do
+ ## Setup run directory
RUN_DIR=${CUR_DIR}/${MESH}/${PROC}procs
mkdir -p $RUN_DIR
</font>
</pre>