[Dart-dev] [7743] DART/trunk/models/cam/shell_scripts: These are intended to be the ' entry' level scripts for CAM/DART.
nancy at ucar.edu
nancy at ucar.edu
Fri Mar 20 10:42:03 MDT 2015
Revision: 7743
Author: thoar
Date: 2015-03-20 10:42:02 -0600 (Fri, 20 Mar 2015)
Log Message:
-----------
These are intended to be the 'entry' level scripts for CAM/DART.
They have been tested for FV, SE, and FV-WACCM5. The instructions
on how to modify these for the SE and WACCM cases will be provided
in the model_mod.html. As they stand, they run the FV case by default.
There are 'advanced' versions of these scripts that are intended to
be used by people already familiar with the entry-level scripts.
The advanced scripts are more efficient but are more complicated.
Modified Paths:
--------------
DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid
DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_pmo
-------------- next part --------------
Modified: DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid
===================================================================
--- DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid 2015-03-20 00:17:16 UTC (rev 7742)
+++ DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid 2015-03-20 16:42:02 UTC (rev 7743)
@@ -15,7 +15,8 @@
# This script is designed to set up, stage, and build a multi-instance run
# of CESM using an F compset where CAM, CLM and CICE are active. The initial state
# come from a single multi-instance reference case so a CESM hybrid setup
-# is used.
+# is used. Instructions on what to change to use the SE core or WACCM are
+# outlined in the models/cam/model_mod.html documentation.
#
# DOCN: We are using a single data ocean.
#
@@ -103,12 +104,13 @@
# compset Must be one of the CESM standard names, see the CESM documentation
# for supported strings.
# resolution Sets the model grid resolution, see the CESM documentation.
+# f09_f09 ... FV core at ~ 1 degree
+# BUG 1384 applies here, so ocean and atm/land must be at same resolution.
# cesmtag The version of the CESM source code to use when building the code.
# num_instances The number of ensemble members.
#
-# BUG 1384 applies here, so ocean and atm/land must be at same resolution.
-# 0.9x1.25_0.9x1.25 (f09_f09). This is roughly a 1 deg grid for atm/lnd,
-# gx1v6 mask. Normally used for CAM with prescribed lnd, ocn and ice.
+# Guidelines on what to change for an SE or WACCM run are described in the
+# models/cam/model_mod.html documentation.
# ==============================================================================
# AMIP_CAM5_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV (F_AMIP_CAM5) (FAMIPC5)
@@ -148,7 +150,7 @@
setenv rundir /glade/scratch/${USER}/${case}/run
setenv exeroot /glade/scratch/${USER}/${case}/bld
setenv archdir /glade/scratch/${USER}/archive/${case}
-setenv dartroot /glade/u/home/${USER}/svn/DART/trunk
+setenv dartroot /glade/u/home/${USER}/svn/DART
# ==============================================================================
# configure settings:
@@ -160,9 +162,15 @@
# refmon run will start from. (Also see 'runtime settings' below for
# refday start_year, start_mon, start_day and start_tod.)
# reftod
+# NOTE: all the ref* variables must be treated like strings and have
+# the appropriate number of preceeding zeros
#
# stagedir The directory location of the reference case files.
# ==============================================================================
+# alternative reference case for different times may be available here:
+# or on the HPSS:
+# /CCSM/dart/FV0.9x1.25x30_cesm1_1_1/{Mon}1 for 1-degree FV ensembles
+# setenv stagedir /glade/scratch/${USER}/archive/${refcase}/rest/${reftimestamp}
setenv refcase cesm_hybrid
setenv refyear 2004
@@ -176,9 +184,6 @@
setenv stagedir /glade/p/image/CESM_initial_ensemble/rest/${reftimestamp}
-# alternative reference case for different times may be available here:
-# setenv stagedir /glade/p/work/raeder/Models/CAM_init/${refcase}_${refdate}
-
# ==============================================================================
# runtime settings:
#
@@ -187,11 +192,6 @@
# start_day
# start_tod
#
-# sst_dataset Data ocean file and
-# sst_grid supporting grid file. Must be consistent.
-# sst_year_start Years included in the sst files.
-# sst_year_end
-#
# short_term_archiver Copies the files from each job step to a 'rest' directory.
# long_term_archiver Puts the files from all completed steps on tape storage.
#
@@ -209,12 +209,7 @@
setenv start_day 10
setenv start_tod 00000
-setenv sst_grid ${cesmdata}/share/domains/domain.ocn.fv0.9x1.25_gx1v6.130409.nc
-setenv sst_dataset ${cesmdata}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
-setenv sst_year_start 1850
-setenv sst_year_end 2012
-
-setenv short_term_archiver on
+setenv short_term_archiver off
setenv long_term_archiver off
setenv resubmit 0
@@ -231,7 +226,7 @@
# less than 10 minutes on yellowstone using 1800 pes (120 nodes)
# ==============================================================================
-setenv ACCOUNT NIMGxxxx
+setenv ACCOUNT ZZZZZZZZ
setenv queue economy
setenv timewall 0:20
@@ -325,7 +320,7 @@
exit -1
endif
-# preserve a copy of this script as it was run
+# Preserve a copy of this script as it was run.
set ThisFileName = $0:t
${COPY} $ThisFileName ${caseroot}/${ThisFileName}.original
@@ -341,7 +336,7 @@
endif
if ( -e CESM_DART_config ) then
- sed -e "s#BOGUS_DART_ROOT_STRING#$dartroot#" < CESM_DART_config >! temp.$$
+ sed -e "s#BOGUS_DART_ROOT_STRING#${dartroot}#" < CESM_DART_config >! temp.$$
${MOVE} temp.$$ ${caseroot}/CESM_DART_config
chmod 755 ${caseroot}/CESM_DART_config
else
@@ -380,42 +375,51 @@
endif
end
-if ($num_instances <= 4) then
+# NOTE: If you require bit-for-bit agreement between different runs,
+# in particular, between pmo (single instance) and assimilations (NINST > 1),
+# or if you need to change the number of nodes/member due to changing memory needs,
+# then env_run.xml:BFBFLAG must be set to TRUE, so that the coupler will
+# generate bit-for-bit identical results, regardless of the number of tasks
+# given to it. The time penalty appears to be ~ 0.5% in the forecast.
+# Alternatively, you can set cpl_tasks = same_number in both experiments
- # This is only for the purpose of debugging the code.
- @ atm_tasks = $ptile * $num_instances * 4
- @ lnd_tasks = $ptile * $num_instances * 4
- @ ice_tasks = $ptile * $num_instances * 4
- @ ocn_tasks = $ptile * $num_instances
- @ cpl_tasks = $ptile * $num_instances
- @ glc_tasks = $ptile * $num_instances
- @ rof_tasks = $ptile * $num_instances
- @ wav_tasks = $ptile * $num_instances
+# Task layout:
+# Set the nodes_per_instance below to match your case. If you get 'out of memory'
+# errors OR failures without any messages, try increasing the nodes_per_instance.
+# CAM-FV 1 degree can run on 2 nodes/instance on yellowstone.
+# CAM-SE ne30 (~ 1 degree) needed 5 nodes/instance.
+# By computing task counts like we do below, we guarantee each instance uses
+# a whole number of nodes which is the recommended configuration.
-else
+# Yellowstone: no large memory nodes, and 15 tasks/node is recommended.
+# Edwards says there's no speed up by running non-active components concurrently,
+# after ATM has run, so just run all components sequentially.
+# BUT, do arrange it so that each member(instance) spans complete nodes:
+# modulo(total pe count / number of instances, 15) == 0.
- # This works, but a more efficient layout should be used
- # CAM only; 1 node/instance; no cross node communications
- # as long as memory is big enough.
- @ atm_tasks = $ptile * $num_instances * 4
- @ lnd_tasks = $ptile * $num_instances * 4
- @ ice_tasks = $ptile * $num_instances * 4
- @ ocn_tasks = $ptile * $num_instances
- @ cpl_tasks = $ptile * $num_instances
- @ glc_tasks = $ptile * $num_instances
- @ rof_tasks = $ptile * $num_instances
- @ wav_tasks = $ptile * $num_instances
+@ nodes_per_instance = 2
-endif
+@ atm_tasks = $ptile * $num_instances * $nodes_per_instance
+@ lnd_tasks = $ptile * $num_instances * $nodes_per_instance
+@ ice_tasks = $ptile * $num_instances * $nodes_per_instance
+@ ocn_tasks = $ptile * $num_instances
+@ cpl_tasks = $ptile * $num_instances
+@ glc_tasks = $ptile * $num_instances
+@ rof_tasks = $ptile * $num_instances
+@ wav_tasks = $ptile * $num_instances
-# echo "task partitioning ... perhaps ... atm // ocn // lnd+ice+glc+rof"
-# presently, all components run 'serially' - one after another.
-# Yellowstone: no large memory nodes, and 15 tasks/node is recommended.
-# Edwards says there's no speed up by running non-active components concurrently,
-# after ATM has run, so just run all components sequentially.
-# BUT, do arrange it so that each member(instance) spans complete nodes:
-# modulo(total pe count / number of instances, 15) == 0.
+switch ("`hostname`")
+ case ys*:
+ if ($atm_tasks > 4100) then
+ echo 'WARNING! Running DART on yellowstone with > 4100 tasks can cause filter'
+ echo ' to hang in MPI_finalize. Either reduce your task request,'
+ echo ' or contact DART for a temporary work-around.'
+ exit -9
+ endif
+ breaksw
+endsw
+
echo ""
echo "ATM gets $atm_tasks"
echo "LND gets $lnd_tasks"
@@ -474,19 +478,13 @@
./xmlchange EXEROOT=${exeroot}
./xmlchange RUNDIR=${rundir}
-./xmlchange SSTICE_DATA_FILENAME=$sst_dataset
-./xmlchange SSTICE_GRID_FILENAME=$sst_grid
-./xmlchange SSTICE_YEAR_ALIGN=$sst_year_start
-./xmlchange SSTICE_YEAR_START=$sst_year_start
-./xmlchange SSTICE_YEAR_END=$sst_year_end
+# Do not change the CALENDAR or the value of CONTINUE_RUN in this script.
-# Do not change the CALENDAR or the CONTINUE_RUN
-
./xmlchange CALENDAR=GREGORIAN
+./xmlchange CONTINUE_RUN=FALSE
./xmlchange STOP_OPTION=$stop_option
./xmlchange STOP_N=$stop_n
-./xmlchange CONTINUE_RUN=FALSE
./xmlchange RESUBMIT=$resubmit
./xmlchange PIO_TYPENAME=pnetcdf
@@ -498,7 +496,7 @@
# The river transport model ON is useful only when using an active ocean or
# land surface diagnostics. Setting ROF_GRID, RTM_MODE to 'null' turns off the RTM.
-# If you turn it ON, you have more work to do ...
+# If you turn it ON, you will have to stage initial files etc.
./xmlchange ROF_GRID='null'
./xmlchange RTM_MODE='NULL'
@@ -531,8 +529,9 @@
./xmlchange DOUT_L_MS=TRUE
endif
-# level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
-
+# DEBUG = TRUE implies turning on run and compile time debugging.
+# INFO_DBUG level of debug output, 0=minimum, 1=normal, 2=more, 3=too much.
+# WARNING: CAM-SE fails if DEBUG=TRUE
./xmlchange DEBUG=FALSE
./xmlchange INFO_DBUG=0
@@ -544,65 +543,74 @@
# mods and put in the SourceMods subdirectory found in the 'caseroot' directory.
# ==============================================================================
-if ( -d ~/${cesmtag}/SourceMods ) then
+if ( ! -d ~/${cesmtag}/SourceMods ) then
+ echo "ERROR - No SourceMods for this case."
+ echo "ERROR - No SourceMods for this case."
+ echo "DART requires modifications to several src files."
+ echo "Download the appropriate files for CESM 1_2_1 from:"
+ echo "http://www.image.ucar.edu/pub/DART/CESM"
+ echo "untar these into your HOME directory - they will create a"
+ echo "~/cesm1_2_1 directory with the appropriate SourceMods structure."
+ exit -4
+endif
- # Copy all of the 'generic' SourceMods
+# Copy all of the 'generic' SourceMods
+${COPY} -r ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/ || exit 2
- ${COPY} -r ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/
+# Each CLM version has some SourceMods. Link to the right version.
+# must parse from a variable of the form:
+# CLM_CONFIG_OPTS: -phys clm4_0 -bgc cn
- # Each CLM version has some SourceMods. Link to the right version.
- # must parse from a variable of the form:
- # CLM_CONFIG_OPTS: -phys clm4_0 -bgc cn
+set clm_opts = `echo $CLM_CONFIG_OPTS | sed -e "s/-//"`
- set clm_opts = `echo $CLM_CONFIG_OPTS | sed -e "s/-//"`
+@ iarg = 1
+while ($iarg <= $#clm_opts)
- @ iarg = 1
- while ($iarg <= $#clm_opts)
+ @ iargp1 = $iarg + 1
+ set option = $clm_opts[$iarg]
+ set value = $clm_opts[$iargp1]
- @ iargp1 = $iarg + 1
- set option = $clm_opts[$iarg]
- set value = $clm_opts[$iargp1]
+ switch ( ${option} )
+ case "phys":
+ if ( -e SourceMods/src.clm/src/${value} ) then
+ cd SourceMods/src.clm
+ ${LINK} src/${value}/*/*F90 .
+ cd ../..
+ else
+ echo "No SourceMods for CLM <${value}>."
+ echo "Got the version from CLM_CONFIG_OPTS ... <${CLM_CONFIG_OPTS}>"
+ endif
+ breaksw
+ # case "bgc": no special action needed here at this time
+ # breaksw
- switch ( ${option} )
- case "phys":
- if ( -e SourceMods/src.clm/src/${value} ) then
- cd SourceMods/src.clm
- ${LINK} src/${value}/*/*F90 .
- cd ../..
- else
- echo "No SourceMods for CLM <${value}>."
- echo "Got the version from CLM_CONFIG_OPTS ... <${CLM_CONFIG_OPTS}>"
- endif
- breaksw
- # case "bgc": no special action needed here at this time
- # breaksw
+ default:
+ breaksw
+ endsw
- default:
- breaksw
- endsw
+ @ iarg = $iarg + 2
+end
- @ iarg = $iarg + 2
- end
+# Need to know if we are using WACCM (aka WCCM) for several reasons.
+# Mostly file managment issues.
+set waccm = `echo $CCSM_COMPSET | grep WCCM`
+if ($status == 0) then
+ set waccm = true
+else
+ set waccm = false
+ # WACCM benefits from a modified cd_core.F90, but none of the
+ # other configurations do.
+ echo "Using the default cd_core.F90"
+ ${REMOVE} -v SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/cd_core.F90
+endif
- # Each CAM dynamical core has its own SourceMods
-
- if ( -e SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/*F90 ) then
- cd SourceMods/src.cam
- ${LINK} src/dynamics/${CAM_DYCORE}/*F90 .
- cd ../..
- else
- echo "No SourceMods for CAM dycore <${CAM_DYCORE}>."
- endif
-
+# Each CAM dynamical core has its own SourceMods
+if ( -e SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/*F90 ) then
+ cd SourceMods/src.cam
+ ${LINK} src/dynamics/${CAM_DYCORE}/*F90 .
+ cd ../..
else
- echo "ERROR - No SourceMods for this case."
- echo "ERROR - No SourceMods for this case."
- echo "DART requires modifications to several src files."
- echo "Download the appropriate files for CESM 1_2_1 from:"
- echo "http://www.image.ucar.edu/pub/DART/CESM"
- echo "untar these into your HOME directory - they will create a"
- echo "~/cesm1_2_1 directory with the appropriate SourceMods structure."
- exit -4
+ echo "No SourceMods for CAM dycore <${CAM_DYCORE}>."
endif
# The CESM multi-instance capability is relatively new and still has a few
@@ -718,45 +726,48 @@
echo " nhtfrq = -$stop_n " >> ${fname}
echo " mfilt = 1 " >> ${fname}
+ # CAM forcing files.
# Some of the files specified here are because the default files only
# contain data through 2005 and we are interested in timeframes after that.
# CAM5 does prognostic aerosols by default (which is expensive).
# Specifying the following causes CAM to use prescribed aerosols, which is cheaper.
# Prescribed aerosols on the timescales of interest to assimilation are acceptable.
+ # WACCM5 should not have these set.
- set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
+ if ($waccm == false ) then
- echo " bndtvghg = '${cesmdata}/atm/cam/ggas/ghg_hist_1765-2009_c100902.nc'" >> ${fname}
- echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
- echo " tracer_cnst_file = 'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
+ set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
- echo "aerodep_flx_datapath = '${chem_datapath}/aero'" >> ${fname}
- echo "aerodep_flx_file = 'aerosoldep_rcp4.5_monthly_1849-2104_0.9x1.25_c100407.nc'" >> ${fname}
- echo "aerodep_flx_type = 'CYCLICAL'" >> ${fname}
- echo "aerodep_flx_cycle_yr = 2000" >> ${fname}
+ echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
+ echo " tracer_cnst_file = 'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
- echo " ext_frc_specifier = " >> ${fname}
- echo " 'SO2 -> ${chem_datapath}/emis/ar5_mam3_so2_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'bc_a1 -> ${chem_datapath}/emis/ar5_mam3_bc_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_elev_1850-2010_c20100902_v12.nc'" >> ${fname}
+ echo "aerodep_flx_datapath = '${chem_datapath}/aero'" >> ${fname}
+ echo "aerodep_flx_file = 'aerosoldep_rcp4.5_monthly_1849-2104_0.9x1.25_c100407.nc'" >> ${fname}
+ echo "aerodep_flx_type = 'CYCLICAL'" >> ${fname}
+ echo "aerodep_flx_cycle_yr = 2000" >> ${fname}
- echo " srf_emis_specifier = " >> ${fname}
- echo " 'DMS -> ${chem_datapath}/emis/aerocom_mam3_dms_surf_1849-2010_c20100902.nc'," >> ${fname}
- echo " 'SO2 -> ${chem_datapath}/emis/ar5_mam3_so2_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'SOAG -> ${chem_datapath}/emis/ar5_mam3_soag_1.5_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'bc_a1 -> ${chem_datapath}/emis/ar5_mam3_bc_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_surf_1850-2010_c20100902_v12.nc'" >> ${fname}
+ echo " ext_frc_specifier = " >> ${fname}
+ echo " 'SO2 -> ${chem_datapath}/emis/ar5_mam3_so2_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'bc_a1 -> ${chem_datapath}/emis/ar5_mam3_bc_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_elev_1850-2010_c20100902_v12.nc'" >> ${fname}
+
+ echo " srf_emis_specifier = " >> ${fname}
+ echo " 'DMS -> ${chem_datapath}/emis/aerocom_mam3_dms_surf_1849-2010_c20100902.nc'," >> ${fname}
+ echo " 'SO2 -> ${chem_datapath}/emis/ar5_mam3_so2_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'SOAG -> ${chem_datapath}/emis/ar5_mam3_soag_1.5_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'bc_a1 -> ${chem_datapath}/emis/ar5_mam3_bc_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
+ echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_surf_1850-2010_c20100902_v12.nc'" >> ${fname}
- echo " solar_data_file = '${cesmdata}/atm/cam/solar/spectral_irradiance_Lean_1610-2009_ann_c100405.nc'" >> ${fname}
+ endif
# ===========================================================================
set fname = "user_nl_clm${inst_string}"
@@ -772,9 +783,11 @@
#
# Making a (compact) .h0. file is a good idea, since the clm restart files
# do not have all the metadata required to reconstruct a gridded field.
- # 'TSA' is 2m surface air temperature.
+ # 'TSA' is 2m surface air temperature. This also prevents
+ # having truly empty history files, resulting in ntapes = 0,
+ # which prevents the hybrid-mode model from restarting.
#
- # Every 6 hours
+ # Every stop_n hours
# echo "hist_mfilt = 1" >> ${fname}
# echo "hist_nhtfrq = -$stop_n" >> ${fname}
# Every month
@@ -785,7 +798,7 @@
echo "hist_fincl1 = 'TSA'" >> ${fname}
echo "hist_nhtfrq = -$stop_n" >> ${fname}
echo "hist_mfilt = 1" >> ${fname}
- echo "hist_avgflag_pertape = 'A'" >> ${fname}
+ echo "hist_avgflag_pertape = 'I'" >> ${fname}
# ===========================================================================
set fname = "user_nl_cice${inst_string}"
@@ -973,9 +986,6 @@
echo " files and the files that were staged."
echo ""
echo "2) cd ${caseroot}"
-echo " (on yellowstone) If the ${case}.run script still contains:"
-echo ' #BSUB -R "select[scratch_ok > 0]"'
-echo " around line 9, delete it."
echo ""
echo "3) The case is initially configured to do NO ASSIMILATION by calling"
echo " a script named 'no_assimilate.csh' in the ${case}.run script."
Modified: DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_pmo
===================================================================
--- DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_pmo 2015-03-20 00:17:16 UTC (rev 7742)
+++ DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_pmo 2015-03-20 16:42:02 UTC (rev 7743)
@@ -15,7 +15,8 @@
# This script is designed to set up, stage, and build a single-instance run
# of CESM using an F compset where CAM, CLM and CICE are active. The initial state
# can come from a single multi-instance reference case so a CESM hybrid setup
-# is used.
+# is used. Instructions on what to change to use the SE core or WACCM are
+# outlined in the models/cam/model_mod.html documentation.
#
# DOCN: We are using a single data ocean.
#
@@ -105,11 +106,12 @@
# compset Must be one of the CESM standard names, see the CESM documentation
# for supported strings.
# resolution Sets the model grid resolution, see the CESM documentation.
+# f09_f09 ... FV core at ~ 1 degree
+# BUG 1384 applies here, so ocean and atm/land must be at same resolution.
# cesmtag The version of the CESM source code to use when building the code.
#
-# BUG 1384 applies here, so ocean and atm/land must be at same resolution.
-# 0.9x1.25_0.9x1.25 (f09_f09). This is roughly a 1 deg grid for atm/lnd,
-# gx1v6 mask. Normally used for CAM with prescribed lnd, ocn and ice.
+# Guidelines on what to change for an SE or WACCM run are described in the
+# models/cam/model_mod.html documentation.
# ==============================================================================
# AMIP_CAM5_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV (F_AMIP_CAM5) (FAMIPC5)
@@ -148,7 +150,7 @@
setenv rundir /glade/scratch/${USER}/${case}/run
setenv exeroot /glade/scratch/${USER}/${case}/bld
setenv archdir /glade/scratch/${USER}/archive/${case}
-setenv dartroot /glade/u/home/${USER}/svn/DART/trunk
+setenv dartroot /glade/u/home/${USER}/svn/DART
# ==============================================================================
# configure settings:
@@ -160,23 +162,29 @@
# refmon run will start from. (Also see 'runtime settings' below for
# refday start_year, start_mon, start_day and start_tod.)
# reftod
+# NOTE: all the ref* variables must be treated like strings and have
+# the appropriate number of preceeding zeros
#
-# SingleInstanceRefcase
+# MultiInstanceRefcase
# Filenames from a multi-instance CESM run have an instance number
# in them. Filenames from a single instance CESM run do not.
# Setting this helps the script figure out the right filenames to
-# generate when copying files from the refernenc case:
-# 1 means the restart files have no instance numbers.
-# 0 means the restart files have strings like .0001. in their names.
+# generate when copying files from the reference case:
+# 'true' means the restart files have strings like .0001. in their names.
+# 'false' the restart files have no instance numbers.
#
# TRUTHinstance
-# If SingleInstanceRefcase is 0, this must be set to select the
+# If MultiInstanceRefcase is 'true', this must be set to select the
# specific instance you want to use when selecting one instance from
# a multi-instance run. e.g. If you have a reference case run
# with 80 instances, 1<= TRUTHinstance <= 80. NO LEADING ZEROS.
#
# stagedir The directory location of the reference case files.
# ==============================================================================
+# alternative reference case for different times may be available here:
+# or on the HPSS:
+# /CCSM/dart/FV0.9x1.25x30_cesm1_1_1/{Mon}1 for 1-degree FV ensembles
+# setenv stagedir /glade/scratch/${USER}/archive/${refcase}/rest/${reftimestamp}
setenv refcase cesm_hybrid
setenv refyear 2004
@@ -184,8 +192,8 @@
setenv refday 10
setenv reftod 00000
-setenv SingleInstanceRefcase 0
-setenv TRUTHinstance 23
+setenv MultiInstanceRefcase true
+setenv TRUTHinstance 1
# useful combinations of time that we use below
setenv refdate $refyear-$refmon-$refday
@@ -193,9 +201,6 @@
setenv stagedir /glade/p/image/CESM_initial_ensemble/rest/${reftimestamp}
-# alternative reference case for different times may be available here:
-# setenv stagedir /glade/p/work/raeder/Models/CAM_init/${refcase}_${refdate}
-
# ==============================================================================
# runtime settings:
#
@@ -204,11 +209,6 @@
# start_day
# start_tod
#
-# sst_dataset Data ocean file and
-# sst_grid supporting grid file. Must be consistent.
-# sst_year_start Years included in the sst files.
-# sst_year_end
-#
# short_term_archiver Copies the files from each job step to a 'rest' directory.
# long_term_archiver Puts the files from all completed steps on tape storage.
#
@@ -226,12 +226,7 @@
setenv start_day 10
setenv start_tod 00000
-setenv sst_grid ${cesmdata}/share/domains/domain.ocn.fv0.9x1.25_gx1v6.130409.nc
-setenv sst_dataset ${cesmdata}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
-setenv sst_year_start 1850
-setenv sst_year_end 2012
-
-setenv short_term_archiver on
+setenv short_term_archiver off
setenv long_term_archiver off
setenv resubmit 0
@@ -342,7 +337,7 @@
exit -1
endif
-# preserve a copy of this script as it was run
+# Preserve a copy of this script as it was run.
set ThisFileName = $0:t
${COPY} $ThisFileName ${caseroot}/${ThisFileName}.original
@@ -358,7 +353,7 @@
endif
if ( -e CESM_DART_config ) then
- sed -e "s#BOGUS_DART_ROOT_STRING#$dartroot#" < CESM_DART_config >! temp.$$
+ sed -e "s#BOGUS_DART_ROOT_STRING#${dartroot}#" < CESM_DART_config >! temp.$$
${MOVE} temp.$$ ${caseroot}/CESM_DART_config
chmod 755 ${caseroot}/CESM_DART_config
else
@@ -397,18 +392,39 @@
endif
end
- # This is a decent layout for a single instance run.
- @ atm_tasks = $ptile * 4
- @ lnd_tasks = $ptile * 4
- @ ice_tasks = $ptile * 4
- @ ocn_tasks = $ptile
- @ cpl_tasks = $ptile
- @ glc_tasks = $ptile
- @ rof_tasks = $ptile
- @ wav_tasks = $ptile
+# NOTE: If you require bit-for-bit agreement between different runs,
+# in particular, between pmo (single instance) and assimilations (NINST > 1),
+# or if you need to change the number of nodes/member due to changing memory needs,
+# then env_run.xml:BFBFLAG must be set to TRUE, so that the coupler will
+# generate bit-for-bit identical results, regardless of the number of tasks
+# given to it. The time penalty appears to be ~ 0.5% in the forecast.
+# Alternatively, you can set cpl_tasks = same_number in both experiments
-# echo "task partitioning ... perhaps ... atm // ocn // lnd+ice+glc+rof"
-# presently, all components run 'serially' - one after another.
+# Task layout:
+# Set the nodes_per_instance below to match your case. If you get 'out of memory'
+# errors OR failures without any messages, try increasing the nodes_per_instance.
+# CAM-FV 1 degree can run on 2 nodes/instance on yellowstone.
+# CAM-SE ne30 (~ 1 degree) needed 5 nodes/instance.
+# By computing task counts like we do below, we guarantee each instance uses
+# a whole number of nodes which is the recommended configuration.
+
+# Yellowstone: no large memory nodes, and 15 tasks/node is recommended.
+# Edwards says there's no speed up by running non-active components concurrently,
+# after ATM has run, so just run all components sequentially.
+# BUT, do arrange it so that each member(instance) spans complete nodes:
+# modulo(total pe count / number of instances, 15) == 0.
+
+@ nodes_per_instance = 2
+
+@ atm_tasks = $ptile * $nodes_per_instance
+@ lnd_tasks = $ptile * $nodes_per_instance
+@ ice_tasks = $ptile * $nodes_per_instance
+@ ocn_tasks = $ptile
+@ cpl_tasks = $ptile
+@ glc_tasks = $ptile
+@ rof_tasks = $ptile
+@ wav_tasks = $ptile
+
echo ""
echo "ATM gets $atm_tasks"
echo "LND gets $lnd_tasks"
@@ -467,19 +483,13 @@
./xmlchange EXEROOT=${exeroot}
./xmlchange RUNDIR=${rundir}
-./xmlchange SSTICE_DATA_FILENAME=$sst_dataset
-./xmlchange SSTICE_GRID_FILENAME=$sst_grid
-./xmlchange SSTICE_YEAR_ALIGN=$sst_year_start
-./xmlchange SSTICE_YEAR_START=$sst_year_start
-./xmlchange SSTICE_YEAR_END=$sst_year_end
+# Do not change the CALENDAR or the value of CONTINUE_RUN in this script.
-# Do not change the CALENDAR or the CONTINUE_RUN
-
./xmlchange CALENDAR=GREGORIAN
+./xmlchange CONTINUE_RUN=FALSE
./xmlchange STOP_OPTION=$stop_option
./xmlchange STOP_N=$stop_n
-./xmlchange CONTINUE_RUN=FALSE
./xmlchange RESUBMIT=$resubmit
./xmlchange PIO_TYPENAME=pnetcdf
@@ -491,7 +501,7 @@
# The river transport model ON is useful only when using an active ocean or
# land surface diagnostics. Setting ROF_GRID, RTM_MODE to 'null' turns off the RTM.
-# If you turn it ON, you have more work to do ...
+# If you turn it ON, you will have to stage initial files etc.
./xmlchange ROF_GRID='null'
./xmlchange RTM_MODE='NULL'
@@ -524,8 +534,9 @@
./xmlchange DOUT_L_MS=TRUE
endif
-# level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
-
+# DEBUG = TRUE implies turning on run and compile time debugging.
+# INFO_DBUG level of debug output, 0=minimum, 1=normal, 2=more, 3=too much.
+# WARNING: CAM-SE fails if DEBUG=TRUE
./xmlchange DEBUG=FALSE
./xmlchange INFO_DBUG=0
@@ -537,65 +548,74 @@
# mods and put in the SourceMods subdirectory found in the 'caseroot' directory.
# ==============================================================================
-if ( -d ~/${cesmtag}/SourceMods ) then
+if ( ! -d ~/${cesmtag}/SourceMods ) then
+ echo "ERROR - No SourceMods for this case."
+ echo "ERROR - No SourceMods for this case."
+ echo "DART requires modifications to several src files."
+ echo "Download the appropriate files for CESM 1_2_1 from:"
+ echo "http://www.image.ucar.edu/pub/DART/CESM"
+ echo "untar these into your HOME directory - they will create a"
+ echo "~/cesm1_2_1 directory with the appropriate SourceMods structure."
+ exit -4
+endif
- # Copy all of the 'generic' SourceMods
+# Copy all of the 'generic' SourceMods
+${COPY} -r ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/ || exit 2
- ${COPY} -r ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/
+# Each CLM version has some SourceMods. Link to the right version.
+# must parse from a variable of the form:
+# CLM_CONFIG_OPTS: -phys clm4_0 -bgc cn
- # Each CLM version has some SourceMods. Link to the right version.
- # must parse from a variable of the form:
- # CLM_CONFIG_OPTS: -phys clm4_0 -bgc cn
+set clm_opts = `echo $CLM_CONFIG_OPTS | sed -e "s/-//"`
- set clm_opts = `echo $CLM_CONFIG_OPTS | sed -e "s/-//"`
+@ iarg = 1
+while ($iarg <= $#clm_opts)
- @ iarg = 1
- while ($iarg <= $#clm_opts)
+ @ iargp1 = $iarg + 1
+ set option = $clm_opts[$iarg]
+ set value = $clm_opts[$iargp1]
- @ iargp1 = $iarg + 1
- set option = $clm_opts[$iarg]
- set value = $clm_opts[$iargp1]
+ switch ( ${option} )
+ case "phys":
+ if ( -e SourceMods/src.clm/src/${value} ) then
+ cd SourceMods/src.clm
+ ${LINK} src/${value}/*/*F90 .
+ cd ../..
+ else
+ echo "No SourceMods for CLM <${value}>."
+ echo "Got the version from CLM_CONFIG_OPTS ... <${CLM_CONFIG_OPTS}>"
+ endif
+ breaksw
+ # case "bgc": no special action needed here at this time
+ # breaksw
- switch ( ${option} )
- case "phys":
- if ( -e SourceMods/src.clm/src/${value} ) then
- cd SourceMods/src.clm
- ${LINK} src/${value}/*/*F90 .
- cd ../..
- else
- echo "No SourceMods for CLM <${value}>."
- echo "Got the version from CLM_CONFIG_OPTS ... <${CLM_CONFIG_OPTS}>"
- endif
- breaksw
- # case "bgc": no special action needed here at this time
- # breaksw
+ default:
+ breaksw
+ endsw
- default:
- breaksw
- endsw
+ @ iarg = $iarg + 2
+end
- @ iarg = $iarg + 2
- end
+# Need to know if we are using WACCM (aka WCCM) for several reasons.
+# Mostly file managment issues.
+set waccm = `echo $CCSM_COMPSET | grep WCCM`
+if ($status == 0) then
+ set waccm = true
+else
+ set waccm = false
+ # WACCM benefits from a modified cd_core.F90, but none of the
+ # other configurations do.
+ echo "Using the default cd_core.F90"
+ ${REMOVE} -v SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/cd_core.F90
+endif
- # Each CAM dynamical core has its own SourceMods
-
- if ( -e SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/*F90 ) then
- cd SourceMods/src.cam
- ${LINK} src/dynamics/${CAM_DYCORE}/*F90 .
- cd ../..
- else
- echo "No SourceMods for CAM dycore <${CAM_DYCORE}>."
- endif
-
+# Each CAM dynamical core has its own SourceMods
+if ( -e SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/*F90 ) then
+ cd SourceMods/src.cam
+ ${LINK} src/dynamics/${CAM_DYCORE}/*F90 .
+ cd ../..
else
- echo "ERROR - No SourceMods for this case."
- echo "ERROR - No SourceMods for this case."
- echo "DART requires modifications to several src files."
- echo "Download the appropriate files for CESM 1_2_1 from:"
- echo "http://www.image.ucar.edu/pub/DART/CESM"
- echo "untar these into your HOME directory - they will create a"
- echo "~/cesm1_2_1 directory with the appropriate SourceMods structure."
- exit -4
+ echo "No SourceMods for CAM dycore <${CAM_DYCORE}>."
endif
# The CESM multi-instance capability is relatively new and still has a few
@@ -711,45 +731,48 @@
echo " nhtfrq = -$stop_n " >> ${fname}
echo " mfilt = 1 " >> ${fname}
+ # CAM forcing files.
# Some of the files specified here are because the default files only
# contain data through 2005 and we are interested in timeframes after that.
# CAM5 does prognostic aerosols by default (which is expensive).
# Specifying the following causes CAM to use prescribed aerosols, which is cheaper.
# Prescribed aerosols on the timescales of interest to assimilation are acceptable.
+ # WACCM5 should not have these set.
- set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
+ if ($waccm == false ) then
- echo " bndtvghg = '${cesmdata}/atm/cam/ggas/ghg_hist_1765-2009_c100902.nc'" >> ${fname}
- echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
- echo " tracer_cnst_file = 'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
+ set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
- echo "aerodep_flx_datapath = '${chem_datapath}/aero'" >> ${fname}
- echo "aerodep_flx_file = 'aerosoldep_rcp4.5_monthly_1849-2104_0.9x1.25_c100407.nc'" >> ${fname}
- echo "aerodep_flx_type = 'CYCLICAL'" >> ${fname}
- echo "aerodep_flx_cycle_yr = 2000" >> ${fname}
+ echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
+ echo " tracer_cnst_file = 'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
- echo " ext_frc_specifier = " >> ${fname}
- echo " 'SO2 -> ${chem_datapath}/emis/ar5_mam3_so2_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'bc_a1 -> ${chem_datapath}/emis/ar5_mam3_bc_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_elev_1850-2010_c20100902_v12.nc'" >> ${fname}
+ echo "aerodep_flx_datapath = '${chem_datapath}/aero'" >> ${fname}
+ echo "aerodep_flx_file = 'aerosoldep_rcp4.5_monthly_1849-2104_0.9x1.25_c100407.nc'" >> ${fname}
+ echo "aerodep_flx_type = 'CYCLICAL'" >> ${fname}
+ echo "aerodep_flx_cycle_yr = 2000" >> ${fname}
- echo " srf_emis_specifier = " >> ${fname}
- echo " 'DMS -> ${chem_datapath}/emis/aerocom_mam3_dms_surf_1849-2010_c20100902.nc'," >> ${fname}
- echo " 'SO2 -> ${chem_datapath}/emis/ar5_mam3_so2_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'SOAG -> ${chem_datapath}/emis/ar5_mam3_soag_1.5_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'bc_a1 -> ${chem_datapath}/emis/ar5_mam3_bc_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_surf_1850-2010_c20100902_v12.nc'" >> ${fname}
+ echo " ext_frc_specifier = " >> ${fname}
+ echo " 'SO2 -> ${chem_datapath}/emis/ar5_mam3_so2_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
@@ Diff output truncated at 40000 characters. @@
More information about the Dart-dev
mailing list