[Dart-dev] [6924] DART/trunk/models/cam: Adding setup scripts for CESM1_2_1
nancy at ucar.edu
nancy at ucar.edu
Wed Apr 23 13:05:37 MDT 2014
Revision: 6924
Author: thoar
Date: 2014-04-23 13:05:36 -0600 (Wed, 23 Apr 2014)
Log Message:
-----------
Adding setup scripts for CESM1_2_1
There is a new interp_mod.f90 for cam/se with a bugfix.
Made the 'sed' command more specific when updating the BSUB quantities.
Made it easier to modify the RESTARTDIR in stage_cesm_files, if the
'rest' directory is actually a '.sta' directory, you can make that
change in one place instead of three.
Added code to use the environment variable for MPI_RUN_COMMAND instead
of hardcoding one in CESM_DART_config ... made it more platform-agnostic.
Added the default value for the cam_phis namelist variable.
This value needs to change for different cores, so if it present in
the namelist, we have a hope of changing it automatically.
Modified Paths:
--------------
DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_hybrid
DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_pmo
DART/trunk/models/cam/shell_scripts/CESM_DART_config
DART/trunk/models/cam/work/input.nml
Added Paths:
-----------
DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid
DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_pmo
-------------- next part --------------
Modified: DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_hybrid
===================================================================
--- DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_hybrid 2014-04-23 19:04:21 UTC (rev 6923)
+++ DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_hybrid 2014-04-23 19:05:36 UTC (rev 6924)
@@ -94,8 +94,6 @@
#
# ==============================================================================
-
-
# ==============================================================================
# case options:
#
@@ -113,7 +111,7 @@
# gx1v6 mask. Normally used for CAM with prescribed lnd, ocn and ice.
# ==============================================================================
-setenv case cam_hybrid
+setenv case cam_hybrid_1_1_1
setenv compset F_AMIP_CAM5
setenv resolution f09_f09
setenv cesmtag cesm1_1_1
@@ -123,7 +121,7 @@
# machines and directories:
#
# mach Computer name
-# cesm_datadir Location of some supporting CESM data files.
+# cesmdata Location of some supporting CESM data files.
# cesmroot Location of the CESM code base. This version of the script
# only supports version cesm1_1_1.
# caseroot Will create the CESM case directory here, where the CESM+DART
@@ -143,7 +141,7 @@
# ==============================================================================
setenv mach yellowstone
-setenv cesm_datadir /glade/p/cesm/cseg/inputdata
+setenv cesmdata /glade/p/cesm/cseg/inputdata
setenv cesmroot /glade/p/cesm/cseg/collections/$cesmtag
setenv caseroot /glade/p/work/${USER}/cases/${case}
setenv rundir /glade/scratch/${USER}/${case}/run
@@ -210,8 +208,8 @@
setenv start_day 10
setenv start_tod 00000
-setenv sst_grid ${cesm_datadir}/share/domains/domain.ocn.fv0.9x1.25_gx1v6.130409.nc
-setenv sst_dataset ${cesm_datadir}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
+setenv sst_grid ${cesmdata}/share/domains/domain.ocn.fv0.9x1.25_gx1v6.130409.nc
+setenv sst_dataset ${cesmdata}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
setenv sst_year_start 1850
setenv sst_year_end 2012
@@ -476,6 +474,10 @@
./xmlchange RESUBMIT=$resubmit
./xmlchange PIO_TYPENAME=pnetcdf
+set TEST_MPI = `./xmlquery -valonly MPI_RUN_COMMAND | sed -e 's/MPI_RUN_COMMAND = //'`
+if (${TEST_MPI} == 'UNSET') then
+ ./xmlchange MPI_RUN_COMMAND=mpirun.lsf
+endif
# The river transport model ON is useful only when using an active ocean or
# land surface diagnostics. Setting ROF_GRID, RTM_MODE to 'null' turns off the RTM.
@@ -492,19 +494,21 @@
./xmlchange NCPL_BASE_PERIOD=day
./xmlchange ATM_NCPL=48
+# These are archiving options
+./xmlchange DOUT_S_ROOT=${archdir}
+./xmlchange DOUT_S_SAVE_INT_REST_FILES=FALSE
+./xmlchange DOUT_L_MSROOT="csm/${case}"
+./xmlchange DOUT_L_HTAR=FALSE
+
if ($short_term_archiver == 'off') then
./xmlchange DOUT_S=FALSE
else
./xmlchange DOUT_S=TRUE
- ./xmlchange DOUT_S_ROOT=${archdir}
- ./xmlchange DOUT_S_SAVE_INT_REST_FILES=FALSE
endif
if ($long_term_archiver == 'off') then
./xmlchange DOUT_L_MS=FALSE
else
./xmlchange DOUT_L_MS=TRUE
- ./xmlchange DOUT_L_MSROOT="csm/${case}"
- ./xmlchange DOUT_L_HTAR=FALSE
endif
# level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
@@ -517,6 +521,8 @@
# This creates the EXEROOT and RUNDIR directories.
# ==============================================================================
+echo 'Setting up the case ...'
+
./cesm_setup
if ( $status != 0 ) then
@@ -543,15 +549,14 @@
# NCAR "bluefire", "yellowstone"
set TIMEWALL=`grep BSUB ${case}.run | grep -e '-W' `
set QUEUE=`grep BSUB ${case}.run | grep -e '-q' `
- sed -e "s/$TIMEWALL[3]/$timewall/" \
- -e "s/ptile=[0-9][0-9]*/ptile=$ptile/" \
- -e "s/$QUEUE[3]/$queue/" < ${case}.run >! temp.$$
+ sed -e "/BSUB/s#$TIMEWALL[3]#$timewall#" \
+ -e "/BSUB/s#ptile=[0-9][0-9]*#ptile=$ptile#" \
+ -e "/BSUB/s#$QUEUE[3]#$queue#" < ${case}.run >! temp.$$
${MOVE} temp.$$ ${case}.run
chmod 755 ${case}.run
breaksw
default:
-
breaksw
endsw
@@ -576,7 +581,7 @@
echo "ERROR - No SourceMods for this case."
echo "DART requires modifications to several src files."
echo "These files can be downloaded from:"
- echo "http://www.image.ucar.edu/pub/DART/CESM/DART_SourceMods_cesm1_1_1_24Oct2013.tar"
+ echo "http://www.image.ucar.edu/pub/DART/CESM/DART_SourceMods_cesm1_1_1_23Apr2014.tar"
echo "untar these into your HOME directory - they will create a"
echo "~/cesm_1_1_1 directory with the appropriate SourceMods structure."
exit -4
@@ -652,9 +657,9 @@
# Specifying the following causes CAM to use prescribed aerosols, which is cheaper.
# Prescribed aerosols on the timescales of interest to assimilation are acceptable.
- set chem_datapath = "${cesm_datadir}/atm/cam/chem/trop_mozart_aero"
+ set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
- echo " bndtvghg = '${cesm_datadir}/atm/cam/ggas/ghg_hist_1765-2009_c100902.nc'" >> ${fname}
+ echo " bndtvghg = '${cesmdata}/atm/cam/ggas/ghg_hist_1765-2009_c100902.nc'" >> ${fname}
echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
echo " tracer_cnst_file = 'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
@@ -683,7 +688,7 @@
echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_surf_1850-2010_c20100902_v12.nc'" >> ${fname}
- echo " solar_data_file = '${cesm_datadir}/atm/cam/solar/spectral_irradiance_Lean_1610-2009_ann_c100405.nc'" >> ${fname}
+ echo " solar_data_file = '${cesmdata}/atm/cam/solar/spectral_irradiance_Lean_1610-2009_ann_c100405.nc'" >> ${fname}
# ===========================================================================
set fname = "user_nl_clm${inst_string}"
@@ -748,25 +753,24 @@
set restart_time = $init_time
-
# get the settings for this case from the CESM environment
cd ${caseroot}
source ./Tools/ccsm_getenv || exit -2
cd ${RUNDIR}
echo 'Copying the required CESM files to the run directory to rerun'
-echo 'a previous step. CONTINUE_RUN from env_run.xml is' \$CONTINUE_RUN
-if ( \$CONTINUE_RUN == TRUE ) then
+echo 'a previous step. CONTINUE_RUN from env_run.xml is' \${CONTINUE_RUN}
+if ( \${CONTINUE_RUN} == TRUE ) then
echo 'so files for some later step than the initial one will be restaged.'
- echo "Date to reset files to is: \$restart_time"
+ echo "Date to reset files to is: \${restart_time}"
else
echo 'so files for the initial step of this experiment will be restaged.'
- echo "Date to reset files to is: $init_time"
+ echo "Date to reset files to is: ${init_time}"
endif
echo ''
-if ( \$CONTINUE_RUN == TRUE ) then
+if ( \${CONTINUE_RUN} == TRUE ) then
#----------------------------------------------------------------------
# This block copies over a set of restart files from any previous step of
@@ -774,31 +778,34 @@
# After running this script resubmit the job to rerun.
#----------------------------------------------------------------------
- echo "Staging restart files for run date/time: " \$restart_time
+ echo "Staging restart files for run date/time: " \${restart_time}
# The short term archiver is on, so the files we want should be in one
# of the short term archive 'rest' restart directories. This assumes
# the long term archiver has NOT copied these files to the HPSS yet.
- if ( \$DOUT_S == TRUE ) then
+ if ( \${DOUT_S} == TRUE ) then
# The restarts should be in the short term archive directory. See
# www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/x1631.html#running_ccsm_restart_back
# for more help and information.
- if ( ! -d \$DOUT_S_ROOT/rest/\${restart_time} ) then
+ set RESTARTDIR = \${DOUT_S_ROOT}/rest/\${restart_time}
+ if ( ! -d \${RESTARTDIR} ) then
+
echo "restart file directory not found: "
- echo " \$DOUT_S_ROOT/rest/\${restart_time} "
+ echo " \${RESTARTDIR}"
echo "If the long-term archiver is on, you may have to restore this directory first."
echo "You can also check for either a .sta or a .sta2 hidden subdirectory in"
- echo \$DOUT_S_ROOT
- echo "which may contain the 'rest' directory you need."
+ echo "\${DOUT_S_ROOT}"
+ echo "which may contain the 'rest' directory you need,"
+ echo "and then modify RESTARTDIR in this script."
exit -1
endif
- ${COPY} \$DOUT_S_ROOT/rest/\${restart_time}/* . || exit -1
+ ${COPY} \${RESTARTDIR}/* . || exit -1
else
@@ -857,7 +864,7 @@
${LINK} ${stagedir}/${refcase}.cam\${inst_string}.i.${init_time}.nc cam_initial\${inst_string}.nc
# If you are using the river runoff model, you must specify an initial file here
- # ${LINK} ${stagedir}/${refcase}.rtm\${inst_string}.r.${init_time}.nc .
+ # ${LINK} ${stagedir}/${refcase}.rtm\${inst_string}.r.${init_time}.nc .
@ inst ++
end
Modified: DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_pmo
===================================================================
--- DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_pmo 2014-04-23 19:04:21 UTC (rev 6923)
+++ DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_pmo 2014-04-23 19:05:36 UTC (rev 6924)
@@ -96,8 +96,6 @@
#
# ==============================================================================
-
-
# ==============================================================================
# case options:
#
@@ -114,7 +112,7 @@
# gx1v6 mask. Normally used for CAM with prescribed lnd, ocn and ice.
# ==============================================================================
-setenv case cam_pmo
+setenv case cam_pmo_1_1_1
setenv compset F_AMIP_CAM5
setenv resolution f09_f09
setenv cesmtag cesm1_1_1
@@ -123,7 +121,7 @@
# machines and directories:
#
# mach Computer name
-# cesm_datadir Location of some supporting CESM data files.
+# cesmdata Location of some supporting CESM data files.
# cesmroot Location of the CESM code base. This version of the script
# only supports version cesm1_1_1.
# caseroot Will create the CESM case directory here, where the CESM+DART
@@ -143,7 +141,7 @@
# ==============================================================================
setenv mach yellowstone
-setenv cesm_datadir /glade/p/cesm/cseg/inputdata
+setenv cesmdata /glade/p/cesm/cseg/inputdata
setenv cesmroot /glade/p/cesm/cseg/collections/$cesmtag
setenv caseroot /glade/p/work/${USER}/cases/${case}
setenv rundir /glade/scratch/${USER}/${case}/run
@@ -227,8 +225,8 @@
setenv start_day 10
setenv start_tod 00000
-setenv sst_grid ${cesm_datadir}/share/domains/domain.ocn.fv0.9x1.25_gx1v6.130409.nc
-setenv sst_dataset ${cesm_datadir}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
+setenv sst_grid ${cesmdata}/share/domains/domain.ocn.fv0.9x1.25_gx1v6.130409.nc
+setenv sst_dataset ${cesmdata}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
setenv sst_year_start 1850
setenv sst_year_end 2012
@@ -470,6 +468,10 @@
./xmlchange RESUBMIT=$resubmit
./xmlchange PIO_TYPENAME=pnetcdf
+set TEST_MPI = `./xmlquery -valonly MPI_RUN_COMMAND | sed -e 's/MPI_RUN_COMMAND = //'`
+if (${TEST_MPI} == 'UNSET') then
+ ./xmlchange MPI_RUN_COMMAND=mpirun.lsf
+endif
# The river transport model ON is useful only when using an active ocean or
# land surface diagnostics. Setting ROF_GRID, RTM_MODE to 'null' turns off the RTM.
@@ -486,19 +488,21 @@
./xmlchange NCPL_BASE_PERIOD=day
./xmlchange ATM_NCPL=48
+# These are archiving options
+./xmlchange DOUT_S_ROOT=${archdir}
+./xmlchange DOUT_S_SAVE_INT_REST_FILES=FALSE
+./xmlchange DOUT_L_MSROOT="csm/${case}"
+./xmlchange DOUT_L_HTAR=FALSE
+
if ($short_term_archiver == 'off') then
./xmlchange DOUT_S=FALSE
else
./xmlchange DOUT_S=TRUE
- ./xmlchange DOUT_S_ROOT=${archdir}
- ./xmlchange DOUT_S_SAVE_INT_REST_FILES=FALSE
endif
if ($long_term_archiver == 'off') then
./xmlchange DOUT_L_MS=FALSE
else
./xmlchange DOUT_L_MS=TRUE
- ./xmlchange DOUT_L_MSROOT="csm/${case}"
- ./xmlchange DOUT_L_HTAR=FALSE
endif
# level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
@@ -511,6 +515,8 @@
# This creates the EXEROOT and RUNDIR directories.
# ==============================================================================
+echo 'Setting up the case ...'
+
./cesm_setup
if ( $status != 0 ) then
@@ -537,15 +543,14 @@
# NCAR "bluefire", "yellowstone"
set TIMEWALL=`grep BSUB ${case}.run | grep -e '-W' `
set QUEUE=`grep BSUB ${case}.run | grep -e '-q' `
- sed -e "s/$TIMEWALL[3]/$timewall/" \
- -e "s/ptile=[0-9][0-9]*/ptile=$ptile/" \
- -e "s/$QUEUE[3]/$queue/" < ${case}.run >! temp.$$
+ sed -e "/BSUB/s#$TIMEWALL[3]#$timewall#" \
+ -e "/BSUB/s#ptile=[0-9][0-9]*#ptile=$ptile#" \
+ -e "/BSUB/s#$QUEUE[3]#$queue#" < ${case}.run >! temp.$$
${MOVE} temp.$$ ${case}.run
chmod 755 ${case}.run
breaksw
default:
-
breaksw
endsw
@@ -570,7 +575,7 @@
echo "ERROR - No SourceMods for this case."
echo "DART requires modifications to several src files."
echo "These files can be downloaded from:"
- echo "http://www.image.ucar.edu/pub/DART/CESM/DART_SourceMods_cesm1_1_1_24Oct2013.tar"
+ echo "http://www.image.ucar.edu/pub/DART/CESM/DART_SourceMods_cesm1_1_1_23Apr2014.tar"
echo "untar these into your HOME directory - they will create a"
echo "~/cesm_1_1_1 directory with the appropriate SourceMods structure."
exit -4
@@ -646,9 +651,9 @@
# Specifying the following causes CAM to use prescribed aerosols, which is cheaper.
# Prescribed aerosols on the timescales of interest to assimilation are acceptable.
- set chem_datapath = "${cesm_datadir}/atm/cam/chem/trop_mozart_aero"
+ set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
- echo " bndtvghg = '${cesm_datadir}/atm/cam/ggas/ghg_hist_1765-2009_c100902.nc'" >> ${fname}
+ echo " bndtvghg = '${cesmdata}/atm/cam/ggas/ghg_hist_1765-2009_c100902.nc'" >> ${fname}
echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
echo " tracer_cnst_file = 'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
@@ -677,7 +682,7 @@
echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_surf_1850-2010_c20100902_v12.nc'" >> ${fname}
- echo " solar_data_file = '${cesm_datadir}/atm/cam/solar/spectral_irradiance_Lean_1610-2009_ann_c100405.nc'" >> ${fname}
+ echo " solar_data_file = '${cesmdata}/atm/cam/solar/spectral_irradiance_Lean_1610-2009_ann_c100405.nc'" >> ${fname}
# ===========================================================================
set fname = "user_nl_clm${inst_string}"
@@ -742,25 +747,24 @@
set restart_time = $init_time
-
# get the settings for this case from the CESM environment
cd ${caseroot}
source ./Tools/ccsm_getenv || exit -2
cd ${RUNDIR}
echo 'Copying the required CESM files to the run directory to rerun'
-echo 'a previous step. CONTINUE_RUN from env_run.xml is' \$CONTINUE_RUN
-if ( \$CONTINUE_RUN == TRUE ) then
+echo 'a previous step. CONTINUE_RUN from env_run.xml is' \${CONTINUE_RUN}
+if ( \${CONTINUE_RUN} == TRUE ) then
echo 'so files for some later step than the initial one will be restaged.'
- echo "Date to reset files to is: \$restart_time"
+ echo "Date to reset files to is: \${restart_time}"
else
echo 'so files for the initial step of this experiment will be restaged.'
- echo "Date to reset files to is: $init_time"
+ echo "Date to reset files to is: ${init_time}"
endif
echo ''
-if ( \$CONTINUE_RUN == TRUE ) then
+if ( \${CONTINUE_RUN} == TRUE ) then
#----------------------------------------------------------------------
# This block copies over a set of restart files from any previous step of
@@ -768,31 +772,34 @@
# After running this script resubmit the job to rerun.
#----------------------------------------------------------------------
- echo "Staging restart files for run date/time: " \$restart_time
+ echo "Staging restart files for run date/time: " \${restart_time}
# The short term archiver is on, so the files we want should be in one
# of the short term archive 'rest' restart directories. This assumes
# the long term archiver has NOT copied these files to the HPSS yet.
- if ( \$DOUT_S == TRUE ) then
+ if ( \${DOUT_S} == TRUE ) then
# The restarts should be in the short term archive directory. See
# www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/x1631.html#running_ccsm_restart_back
# for more help and information.
- if ( ! -d \$DOUT_S_ROOT/rest/\${restart_time} ) then
+ set RESTARTDIR = \${DOUT_S_ROOT}/rest/\${restart_time}
+ if ( ! -d \${RESTARTDIR} ) then
+
echo "restart file directory not found: "
- echo " \$DOUT_S_ROOT/rest/\${restart_time} "
+ echo " \${RESTARTDIR}"
echo "If the long-term archiver is on, you may have to restore this directory first."
echo "You can also check for either a .sta or a .sta2 hidden subdirectory in"
- echo \$DOUT_S_ROOT
- echo "which may contain the 'rest' directory you need."
+ echo "\${DOUT_S_ROOT}"
+ echo "which may contain the 'rest' directory you need,"
+ echo "and then modify RESTARTDIR in this script."
exit -1
endif
- ${COPY} \$DOUT_S_ROOT/rest/\${restart_time}/* . || exit -1
+ ${COPY} \${RESTARTDIR}/* . || exit -1
else
Added: DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid
===================================================================
--- DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid (rev 0)
+++ DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid 2014-04-23 19:05:36 UTC (rev 6924)
@@ -0,0 +1,992 @@
+#!/bin/csh -f
+#
+# DART software - Copyright 2004 - 2013 UCAR. This open source software is
+# provided by UCAR, "as is", without charge, subject to all terms of use at
+# http://www.image.ucar.edu/DAReS/DART/DART_download
+#
+# DART $Id$
+
+# ==============================================================================
+#
+# ---------------------
+# Purpose
+# ---------------------
+#
+# This script is designed to set up, stage, and build a multi-instance run
+# of CESM using an F compset where CAM, CLM and CICE are active. The initial state
+# come from a single multi-instance reference case so a CESM hybrid setup
+# is used.
+#
+# DOCN: We are using a single data ocean.
+#
+# Because the atmosphere assimilations typically occur every 6 hours,
+# the methodology here reflects that. All of CESM stops every 6 hours
+# so that a CAM output file would be available for assimilation.
+#
+# CESM/DART requires some modifications to the CESM source code EVEN IF YOU
+# ARE NOT USING DART. The modifications for CAM require a change to the CESM
+# ${CASE}.run script to invoke a DART script that will allow CAM to cycle
+# correctly with the source code modifications. Changing one line will
+# allow you to invoke DART.
+#
+# This script results in a viable setup for a CESM multi-instance experiment.
+# You are STRONGLY encouraged to run the multi-instance CESM a few times and
+# experiment with different settings BEFORE you try to assimilate observations.
+# The data volume is quite large and you should become comfortable using
+# CESM's restart capability to re-stage files in your RUN directory.
+#
+# ${CASEROOT}/CESM_DART_config is automatically run by this script and will
+# augment the CESM case with the required setup and configuration to use DART
+# to perform an assimilation. CESM_DART_config will insert a few dozen
+# lines into the ${case}.run script after it makes a backup copy.
+#
+# This script relies heavily on the information in:
+# http://www.cesm.ucar.edu/models/cesm1.2/cesm/doc/usersguide1_2/book1.html
+#
+# ---------------------
+# How to use this script.
+# ---------------------
+#
+# -- You will have to read and understand the script in its entirety.
+# You will have to modify things outside this script.
+# This script sets up a plain CESM multi-instance run without DART,
+# intentionally. Once it is running, calls to DART can be added.
+#
+# -- Examine the whole script to identify things to change for your experiments.
+#
+# -- Edit this script in the $DART/models/cam/shell_scripts directory
+# or copy it to somewhere where it will be preserved.
+#
+# -- Locate the initial multi-instance files that CESM will need.
+#
+# -- Run this script. When it is executed, it will create:
+# 1) a CESM 'CASE' directory, where the model will be built,
+# 2) a run directory, where each forecast (and assimilation) will take place,
+# 3) a bld directory for the executables.
+# 4) The short term archiver will use a fourth directory for
+# storage of model output until it can be moved to long term storage (HPSS)
+#
+# This script also executes ${CASEROOT}/CESM_DART_config to modify
+# the ${CASEROOT}/${CASE}.run script so that the SourceMods for CAM
+# are effective. CESM_DART_config will also augment the case with all
+# the pieces necessary to run DART when the time comes.
+#
+# -- (if running DART) Edit the DART input.nml that appears in the ${CASEROOT}
+# directory.
+#
+# -- Submit the job using ${CASEROOT}/${CASE}.submit
+#
+# ---------------------
+# Important features
+# ---------------------
+#
+# If you want to change something in your case other than the runtime
+# settings, it is safest to delete everything and start the run from scratch.
+# For the brave, read
+#
+# http://www.cesm.ucar.edu/models/cesm1.2/cesm/doc/usersguide1_2/x1080.html
+#
+# and you may be able to salvage something with
+# ./cesm_setup -clean
+# ./cesm_setup
+# ./${case}.clean_build
+# ./${case}.build
+#
+# ==============================================================================
+
+# ==============================================================================
+# case options:
+#
+# case The value of "case" will be used many ways; directory and file
+# names both locally and on HPSS, and script names; so consider
+# its length and information content.
+# compset Must be one of the CESM standard names, see the CESM documentation
+# for supported strings.
+# resolution Sets the model grid resolution, see the CESM documentation.
+# cesmtag The version of the CESM source code to use when building the code.
+# num_instances The number of ensemble members.
+#
+# BUG 1384 applies here, so ocean and atm/land must be at same resolution.
+# 0.9x1.25_0.9x1.25 (f09_f09). This is roughly a 1 deg grid for atm/lnd,
+# gx1v6 mask. Normally used for CAM with prescribed lnd, ocn and ice.
+# ==============================================================================
+# AMIP_CAM5_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV (F_AMIP_CAM5) (FAMIPC5)
+
+setenv case cam_hybrid_1_2_1
+setenv compset F_AMIP_CAM5
+setenv resolution f09_f09
+setenv cesmtag cesm1_2_1
+setenv num_instances 30
+
+# ==============================================================================
+# machines and directories:
+#
+# mach Computer name
+# cesmdata Location of some supporting CESM data files.
+# cesmroot Location of the CESM code base. This version of the script
+# only supports version cesm1_2_1.
+# caseroot Will create the CESM case directory here, where the CESM+DART
+# configuration files will be stored. This should probably not
+# be in scratch (on yellowstone, your 'work' partition is suggested).
+# This script will delete any existing caseroot, so this script,
+# and other useful things should be kept elsewhere.
+# rundir Will create the CESM run directory here. Will need large
+# amounts of disk space, generally on a scratch partition.
+# exeroot Will create the CESM executable directory here, where the
+# CESM executables will be built. Medium amount of space
+# needed, generally on a scratch partition.
+# archdir Will create the CESM short-term archive directories here.
+# Large, generally on a scratch partition. Files will remain
+# here until the long-term archiver moves it to permanent storage.
+# dartroot Location of the root of _your_ DART installation
+# ==============================================================================
+
+setenv mach yellowstone
+setenv cesmdata /glade/p/cesm/cseg/inputdata
+setenv cesmroot /glade/p/cesm/cseg/collections/$cesmtag
+setenv caseroot /glade/p/work/${USER}/cases/${case}
+setenv rundir /glade/scratch/${USER}/${case}/run
+setenv exeroot /glade/scratch/${USER}/${case}/bld
+setenv archdir /glade/scratch/${USER}/archive/${case}
+setenv dartroot /glade/u/home/${USER}/svn/DART/trunk
+
+# ==============================================================================
+# configure settings:
+#
+# refcase The name of the existing reference case that this run will
+# start from.
+#
+# refyear The specific date/time-of-day in the reference case that this
+# refmon run will start from. (Also see 'runtime settings' below for
+# refday start_year, start_mon, start_day and start_tod.)
+# reftod
+#
+# stagedir The directory location of the reference case files.
+# ==============================================================================
+
+setenv refcase cesm_hybrid
+setenv refyear 2004
+setenv refmon 01
+setenv refday 10
+setenv reftod 00000
+
+# useful combinations of time that we use below
+setenv refdate $refyear-$refmon-$refday
+setenv reftimestamp $refyear-$refmon-$refday-$reftod
+
+setenv stagedir /glade/p/image/CESM_initial_ensemble/rest/${reftimestamp}
+
+# alternative reference case for different times may be available here:
+# setenv stagedir /glade/p/work/raeder/Models/CAM_init/${refcase}_${refdate}
+
+# ==============================================================================
+# runtime settings:
+#
+# start_year generally this is the same as the reference case date, but it can
+# start_month be different if you want to start this run as if it was a different time.
+# start_day
+# start_tod
+#
+# sst_dataset Data ocean file and
+# sst_grid supporting grid file. Must be consistent.
+# sst_year_start Years included in the sst files.
+# sst_year_end
+#
+# short_term_archiver Copies the files from each job step to a 'rest' directory.
+# long_term_archiver Puts the files from all completed steps on tape storage.
+#
+# resubmit How many job steps to run on continue runs (should be 0 initially)
+# stop_option Units for determining the forecast length between assimilations
+# stop_n Number of time units in each forecast
+#
+# If the long-term archiver is off, you get a chance to examine the files before
+# they get moved to long-term storage. You can always submit $CASE.l_archive
+# whenever you want to free up space in the short-term archive directory.
+# ==============================================================================
+
+setenv start_year 2004
+setenv start_month 01
+setenv start_day 10
+setenv start_tod 00000
+
+setenv sst_grid ${cesmdata}/share/domains/domain.ocn.fv0.9x1.25_gx1v6.130409.nc
+setenv sst_dataset ${cesmdata}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
+setenv sst_year_start 1850
+setenv sst_year_end 2012
+
+setenv short_term_archiver on
+setenv long_term_archiver off
+
+setenv resubmit 0
+setenv stop_option nhours
+setenv stop_n 6
+
+# ==============================================================================
+# job settings:
+#
+# queue can be changed during a series by changing the ${case}.run
+# timewall can be changed during a series by changing the ${case}.run
+#
+# TJH: Advancing 30 instances for 6 hours and assimilating took
+# less than 10 minutes on yellowstone using 1800 pes (120 nodes)
+# ==============================================================================
+
+setenv ACCOUNT P8685xxxx
+setenv queue economy
+setenv timewall 0:20
+
+# ==============================================================================
+# standard commands:
+#
+# If you are running on a machine where the standard commands are not in the
+# expected location, add a case for them below.
+# ==============================================================================
+
+set nonomatch # suppress "rm" warnings if wildcard does not match anything
+
+# The FORCE options are not optional.
+# The VERBOSE options are useful for debugging though
+# some systems don't like the -v option to any of the following
+switch ("`hostname`")
+ case be*:
+ # NCAR "bluefire"
+ set MOVE = '/usr/local/bin/mv -fv'
+ set COPY = '/usr/local/bin/cp -fv --preserve=timestamps'
+ set LINK = '/usr/local/bin/ln -fvs'
+ set REMOVE = '/usr/local/bin/rm -fr'
+
+ breaksw
+ default:
+ # NERSC "hopper", NWSC "yellowstone"
+ set MOVE = '/bin/mv -fv'
+ set COPY = '/bin/cp -fv --preserve=timestamps'
+ set LINK = '/bin/ln -fvs'
+ set REMOVE = '/bin/rm -fr'
+
+ breaksw
+endsw
+
+# ==============================================================================
+# ==============================================================================
+# by setting the values above you should be able to execute this script and
+# have it run. however, for running a real experiment there are still many
+# settings below this point - e.g. component namelists, history file options,
+# the processor layout, xml file options, etc - that you will almost certainly
+# want to change before doing a real science run.
+# ==============================================================================
+# ==============================================================================
+
+
+# ==============================================================================
+# Make sure the CESM directories exist.
+# VAR is the shell variable name, DIR is the value
+# ==============================================================================
+
+foreach VAR ( cesmroot dartroot stagedir )
+ set DIR = `eval echo \${$VAR}`
+ if ( ! -d $DIR ) then
+ echo "ERROR: directory '$DIR' not found"
+ echo " In the setup script check the setting of: $VAR "
+ exit -1
+ endif
+end
+
+# ==============================================================================
+# Create the case - this creates the CASEROOT directory.
+#
+# For list of the pre-defined component sets: ./create_newcase -list
+# To create a variant compset, see the CESM documentation and carefully
+# incorporate any needed changes into this script.
+# ==============================================================================
+
+# fatal idea to make caseroot the same dir as where this setup script is
+# since the build process removes all files in the caseroot dir before
+# populating it. try to prevent shooting yourself in the foot.
+
+if ( $caseroot == `dirname $0` ) then
+ echo "ERROR: the setup script should not be located in the caseroot"
+ echo "directory, because all files in the caseroot dir will be removed"
+ echo "before creating the new case. move the script to a safer place."
+ exit -1
+endif
+
+echo "removing old files from ${caseroot}"
+echo "removing old files from ${exeroot}"
+echo "removing old files from ${rundir}"
+${REMOVE} ${caseroot}
+${REMOVE} ${exeroot}
+${REMOVE} ${rundir}
+
+${cesmroot}/scripts/create_newcase -case ${caseroot} -mach ${mach} \
+ -res ${resolution} -compset ${compset}
+
+if ( $status != 0 ) then
+ echo "ERROR: Case could not be created."
+ exit -1
+endif
+
+# preserve a copy of this script as it was run
+set ThisFileName = $0:t
+${COPY} $ThisFileName ${caseroot}/${ThisFileName}.original
+
+# ==============================================================================
+# Record the DARTROOT directory and copy the DART setup script to CASEROOT.
+# CESM_DART_config can be run at some later date if desired, but it presumes
+# to be run from a CASEROOT directory. If CESM_DART_config does not exist locally,
+# then it better exist in the expected part of the DARTROOT tree.
+# ==============================================================================
+
+if ( ! -e CESM_DART_config ) then
+ ${COPY} ${dartroot}/models/cam/shell_scripts/CESM_DART_config .
+endif
+
+if ( -e CESM_DART_config ) then
+ sed -e "s#BOGUS_DART_ROOT_STRING#$dartroot#" < CESM_DART_config >! temp.$$
+ ${MOVE} temp.$$ ${caseroot}/CESM_DART_config
+ chmod 755 ${caseroot}/CESM_DART_config
+else
+ echo "ERROR: the script to configure for data assimilation is not available."
+ echo " CESM_DART_config MUST be present locally or in"
+ echo " ${dartroot}/models/cam/shell_scripts/"
+ exit -2
+endif
+
+# ==============================================================================
+# Configure the case.
+# ==============================================================================
+
+cd ${caseroot}
+
+source ./Tools/ccsm_getenv || exit -2
+
+# MAX_TASKS_PER_NODE comes from $case/Tools/mkbatch.$machine
+@ ptile = $MAX_TASKS_PER_NODE / 2
+@ nthreads = 1
+
+# Save a copy for debug purposes
+foreach FILE ( *xml )
+ if ( ! -e ${FILE}.original ) then
+ ${COPY} $FILE ${FILE}.original
+ endif
+end
+
+if ($num_instances <= 4) then
+
+ # This is only for the purpose of debugging the code.
+ @ atm_tasks = $ptile * $num_instances * 4
+ @ lnd_tasks = $ptile * $num_instances * 4
+ @ ice_tasks = $ptile * $num_instances * 4
+ @ ocn_tasks = $ptile * $num_instances
+ @ cpl_tasks = $ptile * $num_instances
+ @ glc_tasks = $ptile * $num_instances
+ @ rof_tasks = $ptile * $num_instances
+ @ wav_tasks = $ptile * $num_instances
+
+else
+
+ # This works, but a more efficient layout should be used
+ # CAM only; 1 node/instance; no cross node communications
+ # as long as memory is big enough.
+ @ atm_tasks = $ptile * $num_instances * 4
+ @ lnd_tasks = $ptile * $num_instances * 4
+ @ ice_tasks = $ptile * $num_instances * 4
+ @ ocn_tasks = $ptile * $num_instances
+ @ cpl_tasks = $ptile * $num_instances
+ @ glc_tasks = $ptile * $num_instances
+ @ rof_tasks = $ptile * $num_instances
+ @ wav_tasks = $ptile * $num_instances
+
+endif
+
+# echo "task partitioning ... perhaps ... atm // ocn // lnd+ice+glc+rof"
+# presently, all components run 'serially' - one after another.
+# Yellowstone: no large memory nodes, and 15 tasks/node is recommended.
+# Edwards says there's no speed up by running non-active components concurrently,
+# after ATM has run, so just run all components sequentially.
+# BUT, do arrange it so that each member(instance) spans complete nodes:
+# modulo(total pe count / number of instances, 15) == 0.
+
+echo ""
+echo "ATM gets $atm_tasks"
+echo "LND gets $lnd_tasks"
+echo "ICE gets $ice_tasks"
+echo "OCN gets $ocn_tasks"
+echo "CPL gets $cpl_tasks"
+echo "GLC gets $glc_tasks"
+echo "ROF gets $rof_tasks"
+echo "WAV gets $wav_tasks"
+echo ""
+
+./xmlchange NTHRDS_ATM=$nthreads,NTASKS_ATM=$atm_tasks,NINST_ATM=$num_instances
+./xmlchange NTHRDS_LND=$nthreads,NTASKS_LND=$lnd_tasks,NINST_LND=$num_instances
+./xmlchange NTHRDS_ICE=$nthreads,NTASKS_ICE=$ice_tasks,NINST_ICE=$num_instances
+./xmlchange NTHRDS_OCN=$nthreads,NTASKS_OCN=$ocn_tasks,NINST_OCN=1
+./xmlchange NTHRDS_CPL=$nthreads,NTASKS_CPL=$cpl_tasks
+./xmlchange NTHRDS_GLC=$nthreads,NTASKS_GLC=$glc_tasks,NINST_GLC=1
+./xmlchange NTHRDS_ROF=$nthreads,NTASKS_ROF=$rof_tasks,NINST_ROF=1
+./xmlchange NTHRDS_WAV=$nthreads,NTASKS_WAV=$wav_tasks,NINST_WAV=1
+./xmlchange ROOTPE_ATM=0
+./xmlchange ROOTPE_LND=0
+./xmlchange ROOTPE_ICE=0
+./xmlchange ROOTPE_OCN=0
+./xmlchange ROOTPE_CPL=0
+./xmlchange ROOTPE_GLC=0
+./xmlchange ROOTPE_ROF=0
+./xmlchange ROOTPE_WAV=0
+
+# http://www.cesm.ucar.edu/models/cesm1.2/cesm/doc/usersguide1_2/c1096.html#run_start_stop
+# "A hybrid run indicates that CESM is initialized more like a startup, but uses
+# initialization datasets from a previous case. This is somewhat analogous to a
+# branch run with relaxed restart constraints. A hybrid run allows users to bring
+# together combinations of initial/restart files from a previous case (specified
+# by $RUN_REFCASE) at a given model output date (specified by $RUN_REFDATE).
+# Unlike a branch run, the starting date of a hybrid run (specified by $RUN_STARTDATE)
+# can be modified relative to the reference case. In a hybrid run, the model does not
+# continue in a bit-for-bit fashion with respect to the reference case. The resulting
+# climate, however, should be continuous provided that no model source code or
+# namelists are changed in the hybrid run. In a hybrid initialization, the ocean
+# model does not start until the second ocean coupling (normally the second day),
+# and the coupler does a "cold start" without a restart file."
+#
+# TJH:
+# DART's CAM implementation causes a bit more complexity. DART only uses CAM _initial_
+# files, not RESTART files, so there are sourcemods to force a hybrid start for CAM to
+# read initial files - even when CONTINUE_RUN = TRUE.
+
+./xmlchange RUN_TYPE=hybrid
+./xmlchange RUN_STARTDATE=${start_year}-${start_month}-${start_day}
+./xmlchange START_TOD=$start_tod
+./xmlchange RUN_REFCASE=$refcase
+./xmlchange RUN_REFDATE=$refdate
+./xmlchange RUN_REFTOD=$reftod
+./xmlchange BRNCH_RETAIN_CASENAME=FALSE
+./xmlchange GET_REFCASE=FALSE
+./xmlchange EXEROOT=${exeroot}
+./xmlchange RUNDIR=${rundir}
+
+./xmlchange SSTICE_DATA_FILENAME=$sst_dataset
+./xmlchange SSTICE_GRID_FILENAME=$sst_grid
+./xmlchange SSTICE_YEAR_ALIGN=$sst_year_start
+./xmlchange SSTICE_YEAR_START=$sst_year_start
+./xmlchange SSTICE_YEAR_END=$sst_year_end
+
+# Do not change the CALENDAR or the CONTINUE_RUN
+
+./xmlchange CALENDAR=GREGORIAN
+
+./xmlchange STOP_OPTION=$stop_option
+./xmlchange STOP_N=$stop_n
+./xmlchange CONTINUE_RUN=FALSE
+./xmlchange RESUBMIT=$resubmit
+
+./xmlchange PIO_TYPENAME=pnetcdf
+set TEST_MPI = `./xmlquery -valonly MPI_RUN_COMMAND | sed -e 's/MPI_RUN_COMMAND = //'`
+if (${TEST_MPI} == 'UNSET') then
+ ./xmlchange MPI_RUN_COMMAND=mpirun.lsf
+endif
+
+# The river transport model ON is useful only when using an active ocean or
+# land surface diagnostics. Setting ROF_GRID, RTM_MODE to 'null' turns off the RTM.
+# If you turn it ON, you have more work to do ...
+
+./xmlchange ROF_GRID='null'
+./xmlchange RTM_MODE='NULL'
+
+# COUPLING discussion. F compsets are 'tight' coupling.
+# Only change the ATM_NCPL ... everything is based on this one value,
+# including CAM physics and dynamics timesteps.
+# Default values for coupling are preserved in env_run.xml.original
+
+./xmlchange NCPL_BASE_PERIOD=day
+./xmlchange ATM_NCPL=48
+
+# These are archiving options
+./xmlchange DOUT_S_ROOT=${archdir}
+./xmlchange DOUT_S_SAVE_INT_REST_FILES=FALSE
+./xmlchange DOUT_L_MSROOT="csm/${case}"
+./xmlchange DOUT_L_HTAR=FALSE
+
+if ($short_term_archiver == 'off') then
+ ./xmlchange DOUT_S=FALSE
+else
+ ./xmlchange DOUT_S=TRUE
+endif
+if ($long_term_archiver == 'off') then
+ ./xmlchange DOUT_L_MS=FALSE
+else
+ ./xmlchange DOUT_L_MS=TRUE
+endif
+
+# level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
+
+./xmlchange DEBUG=FALSE
+./xmlchange INFO_DBUG=0
+
+# ==============================================================================
+# Update source files.
+# Ideally, using DART would not require any modifications to the model source.
@@ Diff output truncated at 40000 characters. @@
More information about the Dart-dev
mailing list