[Dart-dev] [7742] DART/trunk/models/cam: These scripts handle the FV case by default.
nancy at ucar.edu
nancy at ucar.edu
Thu Mar 19 18:17:17 MDT 2015
Revision: 7742
Author: thoar
Date: 2015-03-19 18:17:16 -0600 (Thu, 19 Mar 2015)
Log Message:
-----------
These scripts handle the FV case by default.
The input.nml has a separate (commented out) block for SE configurations.
The perfect_model.csh, assimilate.csh and CESM_DART_config can handle
FV, SE, and WACCM configurations with no modification.
The instructions how to modify the CESM*setup* scripts for SE and WACCM
configurations will be put in the model_mod.html to keep them from
invading already-complicated 'simple' versions of the setup scripts.
There is a new SourceMods tarfile
http://www.image.ucar.edu/pub/DART/CESM/DART_SourceMods_cesm1_1_1_19Mar2015.tar
that has a modified lt_archive.sh as well as a src.cam/dynamics/fv/cd_core.F90
that helps stabilize WACCM runs but is not recommended to be used for non-WACCM
cases. The scripts automatically install or remove cd_core.F90 as needed.
Modified Paths:
--------------
DART/trunk/models/cam/model_mod.f90
DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_hybrid
DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_pmo
DART/trunk/models/cam/shell_scripts/CESM_DART_config
DART/trunk/models/cam/shell_scripts/assimilate.csh
DART/trunk/models/cam/shell_scripts/perfect_model.csh
DART/trunk/models/cam/work/input.nml
-------------- next part --------------
Modified: DART/trunk/models/cam/model_mod.f90
===================================================================
--- DART/trunk/models/cam/model_mod.f90 2015-03-19 17:48:00 UTC (rev 7741)
+++ DART/trunk/models/cam/model_mod.f90 2015-03-20 00:17:16 UTC (rev 7742)
@@ -447,6 +447,7 @@
! X_dimid_#d first dimension is 1 larger than # spatial dimensions to accomodate time dimension
! on caminit.nc files.
! These are filled in trans_coord
+
integer :: coord_order
integer, allocatable :: s_dim_3d(:,:), s_dim_2d(:,:), s_dim_1d( :), &
f_dim_3d(:,:), f_dim_2d(:,:), f_dim_1d(:,:), &
@@ -1084,8 +1085,8 @@
call nc_check(nf90_inquire_variable(ncfileid, varid, dimids=f_dimid_3d(1:4,i)), &
'trans_coord', 'inquire_variable'//trim(state_names_3d(i)))
- Alldim3: do j = 1,4 ! time and 3 space
- k = f_dimid_3d(j,i) ! shorthand; the dimid of this fields current dim
+ Alldim3: do j = 1,4 ! time and 3 space
+ k = f_dimid_3d(j,i) ! shorthand; the dimid of this fields current dim
f_dim_3d(j,i) = dim_sizes(k)
! Put the dimensions we want in the state field positions we want.
if (dim_names(k) == 'lev' .or. dim_names(k) == 'ilev') then
@@ -1103,9 +1104,13 @@
! end if
! end do
end do Alldim3
+
if ( s_dim_3d(1,i) == 0 .or. s_dim_3d(2,i) == 0 .or. s_dim_3d(3,i) == 0 ) then
- call error_handler(E_ERR, 'trans_coord', &
- 'num_[lons,lats,levs] was not assigned and = 0' , source, revision, revdate)
+ string1 = 'num_[lons,lats,levs] were not assigned and = 0'
+ string2 = 'This may mean your input.nml model_nml is configured for FV'
+ string3 = 'but the input files are SE.'
+ call error_handler(E_ERR, 'trans_coord', string1, &
+ source, revision, revdate, text2=string2, text3=string3)
end if
end do
Modified: DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_hybrid
===================================================================
--- DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_hybrid 2015-03-19 17:48:00 UTC (rev 7741)
+++ DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_hybrid 2015-03-20 00:17:16 UTC (rev 7742)
@@ -15,12 +15,13 @@
# This script is designed to set up, stage, and build a multi-instance run
# of CESM using an F compset where CAM, CLM and CICE are active. The initial state
# come from a single multi-instance reference case so a CESM hybrid setup
-# is used.
+# is used. Instructions on what to change to use the SE core or WACCM are
+# outlined in the models/cam/model_mod.html documentation.
#
# DOCN: We are using a single data ocean.
#
-# Because the atmosphere assimilations typically occur every 6 hours,
-# the methodology here reflects that. All of CESM stops every 6 hours
+# Because the atmosphere assimilations typically occur multiple times per day,
+# the methodology here reflects that. All of CESM stops every H hours
# so that a CAM output file would be available for assimilation.
#
# CESM/DART requires some modifications to the CESM source code EVEN IF YOU
@@ -56,6 +57,7 @@
#
# -- Edit this script in the $DART/models/cam/shell_scripts directory
# or copy it to somewhere where it will be preserved.
+# A copy will be made, regardless of where it is run.
#
# -- Locate the initial multi-instance files that CESM will need.
#
@@ -103,12 +105,13 @@
# compset Must be one of the CESM standard names, see the CESM documentation
# for supported strings.
# resolution Sets the model grid resolution, see the CESM documentation.
+# f09_f09 ... FV core at ~ 1 degree
+# BUG 1384 applies here, so ocean and atm/land must be at same resolution.
# cesmtag The version of the CESM source code to use when building the code.
# num_instances The number of ensemble members.
#
-# BUG 1384 applies here, so ocean and atm/land must be at same resolution.
-# 0.9x1.25_0.9x1.25 (f09_f09). This is roughly a 1 deg grid for atm/lnd,
-# gx1v6 mask. Normally used for CAM with prescribed lnd, ocn and ice.
+# Guidelines on what to change for an SE or WACCM run are described in the
+# models/cam/model_mod.html documentation.
# ==============================================================================
setenv case cam1_1_1_hybrid
@@ -147,7 +150,7 @@
setenv rundir /glade/scratch/${USER}/${case}/run
setenv exeroot /glade/scratch/${USER}/${case}/bld
setenv archdir /glade/scratch/${USER}/archive/${case}
-setenv dartroot /glade/u/home/${USER}/svn/DART/trunk
+setenv dartroot /glade/u/home/${USER}/svn/DART
# ==============================================================================
# configure settings:
@@ -159,9 +162,15 @@
# refmon run will start from. (Also see 'runtime settings' below for
# refday start_year, start_mon, start_day and start_tod.)
# reftod
+# NOTE: all the ref* variables must be treated like strings and have
+# the appropriate number of preceeding zeros
#
# stagedir The directory location of the reference case files.
# ==============================================================================
+# alternative reference case for different times may be available here:
+# or on the HPSS:
+# /CCSM/dart/FV0.9x1.25x30_cesm1_1_1/{Mon}1 for 1-degree FV ensembles
+# setenv stagedir /glade/scratch/${USER}/archive/${refcase}/rest/${reftimestamp}
setenv refcase cesm_hybrid
setenv refyear 2004
@@ -175,9 +184,6 @@
setenv stagedir /glade/p/image/CESM_initial_ensemble/rest/${reftimestamp}
-# alternative reference case for different times may be available here:
-# setenv stagedir /glade/p/work/raeder/Models/CAM_init/${refcase}_${refdate}
-
# ==============================================================================
# runtime settings:
#
@@ -186,11 +192,6 @@
# start_day
# start_tod
#
-# sst_dataset Data ocean file and
-# sst_grid supporting grid file. Must be consistent.
-# sst_year_start Years included in the sst files.
-# sst_year_end
-#
# short_term_archiver Copies the files from each job step to a 'rest' directory.
# long_term_archiver Puts the files from all completed steps on tape storage.
#
@@ -208,12 +209,7 @@
setenv start_day 10
setenv start_tod 00000
-setenv sst_grid ${cesmdata}/share/domains/domain.ocn.fv0.9x1.25_gx1v6.130409.nc
-setenv sst_dataset ${cesmdata}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
-setenv sst_year_start 1850
-setenv sst_year_end 2012
-
-setenv short_term_archiver on
+setenv short_term_archiver off
setenv long_term_archiver off
setenv resubmit 0
@@ -324,7 +320,7 @@
exit -1
endif
-# preserve a copy of this script as it was run
+# Preserve a copy of this script as it was run.
set ThisFileName = $0:t
${COPY} $ThisFileName ${caseroot}/${ThisFileName}.original
@@ -340,7 +336,7 @@
endif
if ( -e CESM_DART_config ) then
- sed -e "s#BOGUS_DART_ROOT_STRING#$dartroot#" < CESM_DART_config >! temp.$$
+ sed -e "s#BOGUS_DART_ROOT_STRING#${dartroot}#" < CESM_DART_config >! temp.$$
${MOVE} temp.$$ ${caseroot}/CESM_DART_config
chmod 755 ${caseroot}/CESM_DART_config
else
@@ -379,40 +375,43 @@
endif
end
-if ($num_instances <= 4) then
+# NOTE: If you require bit-for-bit agreement between different runs,
+# in particular, between pmo (single instance) and assimilations (NINST > 1),
+# or if you need to change the number of nodes/member due to changing memory needs,
+# then env_run.xml:BFBFLAG must be set to TRUE, so that the coupler will
+# generate bit-for-bit identical results, regardless of the number of tasks
+# given to it. The time penalty appears to be ~ 0.5% in the forecast.
+# Alternatively, you can set cpl_tasks = same_number in both experiments
- # This is only for the purpose of debugging the code.
- @ atm_tasks = $ptile * $num_instances * 4
- @ lnd_tasks = $ptile * $num_instances * 4
- @ ice_tasks = $ptile * $num_instances * 4
- @ ocn_tasks = $ptile * $num_instances
- @ cpl_tasks = $ptile * $num_instances
- @ glc_tasks = $ptile * $num_instances
- @ rof_tasks = $ptile * $num_instances
+# Task layout:
+# Set the nodes_per_instance below to match your case. If you get 'out of memory'
+# errors OR failures without any messages, try increasing the nodes_per_instance.
+# CAM-FV 1 degree can run on 2 nodes/instance on yellowstone.
+# CAM-SE ne30 (~ 1 degree) needed 5 nodes/instance.
+# By computing task counts like we do below, we guarantee each instance uses
+# a whole number of nodes which is the recommended configuration.
-else
+@ nodes_per_instance = 2
- # This works, but a more efficient layout should be used
- # CAM only; 1 node/instance; no cross node communications
- # as long as memory is big enough.
- @ atm_tasks = $ptile * $num_instances * 4
- @ lnd_tasks = $ptile * $num_instances * 4
- @ ice_tasks = $ptile * $num_instances * 4
- @ ocn_tasks = $ptile * $num_instances
- @ cpl_tasks = $ptile * $num_instances
- @ glc_tasks = $ptile * $num_instances
- @ rof_tasks = $ptile * $num_instances
+@ atm_tasks = $ptile * $num_instances * $nodes_per_instance
+@ lnd_tasks = $ptile * $num_instances * $nodes_per_instance
+@ ice_tasks = $ptile * $num_instances * $nodes_per_instance
+@ ocn_tasks = $ptile * $num_instances
+@ cpl_tasks = $ptile * $num_instances
+@ glc_tasks = $ptile * $num_instances
+@ rof_tasks = $ptile * $num_instances
-endif
+switch ("`hostname`")
+ case ys*:
+ if ($atm_tasks > 4100) then
+ echo 'WARNING! Running DART on yellowstone with > 4100 tasks can cause filter'
+ echo ' to hang in MPI_finalize. Either reduce your task request,'
+ echo ' or contact DART for a temporary work-around.'
+ exit -9
+ endif
+ breaksw
+endsw
-# echo "task partitioning ... perhaps ... atm // ocn // lnd+ice+glc+rof"
-# presently, all components run 'serially' - one after another.
-# Yellowstone: no large memory nodes, and 15 tasks/node is recommended.
-# Edwards says there's no speed up by running non-active components concurrently,
-# after ATM has run, so just run all components sequentially.
-# BUT, do arrange it so that each member(instance) spans complete nodes:
-# modulo(total pe count / number of instances, 15) == 0.
-
echo ""
echo "ATM gets $atm_tasks"
echo "LND gets $lnd_tasks"
@@ -468,19 +467,13 @@
./xmlchange EXEROOT=${exeroot}
./xmlchange RUNDIR=${rundir}
-./xmlchange SSTICE_DATA_FILENAME=$sst_dataset
-./xmlchange SSTICE_GRID_FILENAME=$sst_grid
-./xmlchange SSTICE_YEAR_ALIGN=$sst_year_start
-./xmlchange SSTICE_YEAR_START=$sst_year_start
-./xmlchange SSTICE_YEAR_END=$sst_year_end
+# Do not change the CALENDAR or the value of CONTINUE_RUN in this script.
-# Do not change the CALENDAR or the CONTINUE_RUN
-
./xmlchange CALENDAR=GREGORIAN
+./xmlchange CONTINUE_RUN=FALSE
./xmlchange STOP_OPTION=$stop_option
./xmlchange STOP_N=$stop_n
-./xmlchange CONTINUE_RUN=FALSE
./xmlchange RESUBMIT=$resubmit
./xmlchange PIO_TYPENAME=pnetcdf
@@ -525,8 +518,9 @@
./xmlchange DOUT_L_MS=TRUE
endif
-# level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
-
+# DEBUG = TRUE implies turning on run and compile time debugging.
+# INFO_DBUG level of debug output, 0=minimum, 1=normal, 2=more, 3=too much.
+# WARNING: CAM-SE fails if DEBUG=TRUE
./xmlchange DEBUG=FALSE
./xmlchange INFO_DBUG=0
@@ -588,9 +582,7 @@
# mods and put in the SourceMods subdirectory found in the 'caseroot' directory.
# ==============================================================================
-if ( -d ~/${cesmtag}/SourceMods ) then
- ${COPY} -r ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/
-else
+if ( ! -d ~/${cesmtag}/SourceMods ) then
echo "ERROR - No SourceMods for this case."
echo "ERROR - No SourceMods for this case."
echo "DART requires modifications to several src files."
@@ -601,6 +593,29 @@
exit -4
endif
+${COPY} -r ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/ || exit 2
+
+# WACCM benefits from a modified cd_core.F90, but none of the
+# other configurations do.
+
+setenv WACCM `echo $CCSM_COMPSET | grep CCM`
+if ($status == 0) then
+ echo "Using the modified cd_core.F90 because WACCM is being used."
+else
+ echo "Using the default cd_core.F90"
+ ${REMOVE} -v SourceMods/src.cam/dynamics/${CAM_DYCORE}/cd_core.F90
+endif
+
+# Each CAM dynamical core has its own SourceMods
+set numfiles = `ls -1 SourceMods/src.cam/dynamics/${CAM_DYCORE} | wc -l`
+if ( $numfiles > 0 ) then
+ cd SourceMods/src.cam
+ ${LINK} dynamics/${CAM_DYCORE}/*F90 .
+ cd ../..
+else
+ echo "No SourceMods for CAM dycore <${CAM_DYCORE}>."
+endif
+
# The CESM multi-instance capability is relatively new and still has a few
# implementation bugs. These are known problems and will be fixed soon.
# this should be removed when the files are fixed:
@@ -618,10 +633,15 @@
${COPY} ${caseroot}/preview_namelists \
${caseroot}/preview_namelists.original
endif
+ if ( ! -e ${caseroot}/Tools/lt_archive.sh.original ) then
+ ${COPY} ${caseroot}/Tools/lt_archive.sh \
+ ${caseroot}/Tools/lt_archive.sh.original
+ endif
# patch/replace the broken files
- ${COPY} ~/${cesmtag}/clm.buildnml.csh ${caseroot}/Buildconf/.
- ${COPY} ~/${cesmtag}/preview_namelists ${caseroot}/.
+ ${COPY} ~/${cesmtag}/clm.buildnml.csh ${caseroot}/Buildconf/. || exit 2
+ ${COPY} ~/${cesmtag}/preview_namelists ${caseroot}/. || exit 2
+ ${COPY} ~/${cesmtag}/lt_archive.sh ${caseroot}/Tools/. || exit 2
endif
@@ -669,46 +689,17 @@
echo " nhtfrq = -$stop_n " >> ${fname}
echo " mfilt = 1 " >> ${fname}
- # Some of the files specified here are because the default files only
- # contain data through 2005 and we are interested in timeframes after that.
-
# CAM5 does prognostic aerosols by default (which is expensive).
# Specifying the following causes CAM to use prescribed aerosols, which is cheaper.
# Prescribed aerosols on the timescales of interest to assimilation are acceptable.
set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
- echo " bndtvghg = '${cesmdata}/atm/cam/ggas/ghg_hist_1765-2009_c100902.nc'" >> ${fname}
- echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
- echo " tracer_cnst_file = 'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
-
echo "aerodep_flx_datapath = '${chem_datapath}/aero'" >> ${fname}
echo "aerodep_flx_file = 'aerosoldep_rcp4.5_monthly_1849-2104_0.9x1.25_c100407.nc'" >> ${fname}
echo "aerodep_flx_type = 'CYCLICAL'" >> ${fname}
echo "aerodep_flx_cycle_yr = 2000" >> ${fname}
- echo " ext_frc_specifier = " >> ${fname}
- echo " 'SO2 -> ${chem_datapath}/emis/ar5_mam3_so2_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'bc_a1 -> ${chem_datapath}/emis/ar5_mam3_bc_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_elev_1850-2010_c20100902_v12.nc'" >> ${fname}
-
- echo " srf_emis_specifier = " >> ${fname}
- echo " 'DMS -> ${chem_datapath}/emis/aerocom_mam3_dms_surf_1849-2010_c20100902.nc'," >> ${fname}
- echo " 'SO2 -> ${chem_datapath}/emis/ar5_mam3_so2_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'SOAG -> ${chem_datapath}/emis/ar5_mam3_soag_1.5_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'bc_a1 -> ${chem_datapath}/emis/ar5_mam3_bc_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_surf_1850-2010_c20100902_v12.nc'" >> ${fname}
-
- echo " solar_data_file = '${cesmdata}/atm/cam/solar/spectral_irradiance_Lean_1610-2009_ann_c100405.nc'" >> ${fname}
-
# ===========================================================================
set fname = "user_nl_clm${inst_string}"
# ===========================================================================
@@ -723,9 +714,11 @@
#
# Making a (compact) .h0. file is a good idea, since the clm restart files
# do not have all the metadata required to reconstruct a gridded field.
- # 'TSA' is 2m surface air temperature.
+ # 'TSA' is 2m surface air temperature. This also prevents
+ # having truly empty history files, resulting in ntapes = 0,
+ # which prevents the hybrid-mode model from restarting.
#
- # Every 6 hours
+ # Every stop_n hours
# echo "hist_mfilt = 1" >> ${fname}
# echo "hist_nhtfrq = -$stop_n" >> ${fname}
# Every month
@@ -736,7 +729,7 @@
echo "hist_fincl1 = 'TSA'" >> ${fname}
echo "hist_nhtfrq = -$stop_n" >> ${fname}
echo "hist_mfilt = 1" >> ${fname}
- echo "hist_avgflag_pertape = 'A'" >> ${fname}
+ echo "hist_avgflag_pertape = 'I'" >> ${fname}
# ===========================================================================
set fname = "user_nl_cice${inst_string}"
Modified: DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_pmo
===================================================================
--- DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_pmo 2015-03-19 17:48:00 UTC (rev 7741)
+++ DART/trunk/models/cam/shell_scripts/CESM1_1_1_setup_pmo 2015-03-20 00:17:16 UTC (rev 7742)
@@ -15,7 +15,8 @@
# This script is designed to set up, stage, and build a single-instance run
# of CESM using an F compset where CAM, CLM and CICE are active. The initial state
# can come from a single multi-instance reference case so a CESM hybrid setup
-# is used.
+# is used. Instructions on what to change to use the SE core or WACCM are
+# outlined in the models/cam/model_mod.html documentation.
#
# DOCN: We are using a single data ocean.
#
@@ -105,11 +106,12 @@
# compset Must be one of the CESM standard names, see the CESM documentation
# for supported strings.
# resolution Sets the model grid resolution, see the CESM documentation.
+# f09_f09 ... FV core at ~ 1 degree
+# BUG 1384 applies here, so ocean and atm/land must be at same resolution.
# cesmtag The version of the CESM source code to use when building the code.
#
-# BUG 1384 applies here, so ocean and atm/land must be at same resolution.
-# 0.9x1.25_0.9x1.25 (f09_f09). This is roughly a 1 deg grid for atm/lnd,
-# gx1v6 mask. Normally used for CAM with prescribed lnd, ocn and ice.
+# Guidelines on what to change for an SE or WACCM run are described in the
+# models/cam/model_mod.html documentation.
# ==============================================================================
setenv case cam1_1_1_pmo
@@ -147,7 +149,7 @@
setenv rundir /glade/scratch/${USER}/${case}/run
setenv exeroot /glade/scratch/${USER}/${case}/bld
setenv archdir /glade/scratch/${USER}/archive/${case}
-setenv dartroot /glade/u/home/${USER}/svn/DART/trunk
+setenv dartroot /glade/u/home/${USER}/svn/DART
# ==============================================================================
# configure settings:
@@ -159,23 +161,29 @@
# refmon run will start from. (Also see 'runtime settings' below for
# refday start_year, start_mon, start_day and start_tod.)
# reftod
+# NOTE: all the ref* variables must be treated like strings and have
+# the appropriate number of preceeding zeros
#
-# SingleInstanceRefcase
+# MultiInstanceRefcase
# Filenames from a multi-instance CESM run have an instance number
# in them. Filenames from a single instance CESM run do not.
# Setting this helps the script figure out the right filenames to
-# generate when copying files from the refernenc case:
-# 1 means the restart files have no instance numbers.
-# 0 means the restart files have strings like .0001. in their names.
+# generate when copying files from the reference case:
+# 'true' means the restart files have strings like .0001. in their names.
+# 'false' the restart files have no instance numbers.
#
# TRUTHinstance
-# If SingleInstanceRefcase is 0, this must be set to select the
+# If MultiInstanceRefcase is 'true', this must be set to select the
# specific instance you want to use when selecting one instance from
# a multi-instance run. e.g. If you have a reference case run
# with 80 instances, 1<= TRUTHinstance <= 80. NO LEADING ZEROS.
#
# stagedir The directory location of the reference case files.
# ==============================================================================
+# alternative reference case for different times may be available here:
+# or on the HPSS:
+# /CCSM/dart/FV0.9x1.25x30_cesm1_1_1/{Mon}1 for 1-degree FV ensembles
+# setenv stagedir /glade/scratch/${USER}/archive/${refcase}/rest/${reftimestamp}
setenv refcase cesm_hybrid
setenv refyear 2004
@@ -183,8 +191,8 @@
setenv refday 10
setenv reftod 00000
-setenv SingleInstanceRefcase 0
-setenv TRUTHinstance 23
+setenv MultiInstanceRefcase true
+setenv TRUTHinstance 1
# useful combinations of time that we use below
setenv refdate $refyear-$refmon-$refday
@@ -192,9 +200,6 @@
setenv stagedir /glade/p/image/CESM_initial_ensemble/rest/${reftimestamp}
-# alternative reference case for different times may be available here:
-# setenv stagedir /glade/p/work/raeder/Models/CAM_init/${refcase}_${refdate}
-
# ==============================================================================
# runtime settings:
#
@@ -203,11 +208,6 @@
# start_day
# start_tod
#
-# sst_dataset Data ocean file and
-# sst_grid supporting grid file. Must be consistent.
-# sst_year_start Years included in the sst files.
-# sst_year_end
-#
# short_term_archiver Copies the files from each job step to a 'rest' directory.
# long_term_archiver Puts the files from all completed steps on tape storage.
#
@@ -225,12 +225,7 @@
setenv start_day 10
setenv start_tod 00000
-setenv sst_grid ${cesmdata}/share/domains/domain.ocn.fv0.9x1.25_gx1v6.130409.nc
-setenv sst_dataset ${cesmdata}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
-setenv sst_year_start 1850
-setenv sst_year_end 2012
-
-setenv short_term_archiver on
+setenv short_term_archiver off
setenv long_term_archiver off
setenv resubmit 0
@@ -341,7 +336,7 @@
exit -1
endif
-# preserve a copy of this script as it was run
+# Preserve a copy of this script as it was run.
set ThisFileName = $0:t
${COPY} $ThisFileName ${caseroot}/${ThisFileName}.original
@@ -357,7 +352,7 @@
endif
if ( -e CESM_DART_config ) then
- sed -e "s#BOGUS_DART_ROOT_STRING#$dartroot#" < CESM_DART_config >! temp.$$
+ sed -e "s#BOGUS_DART_ROOT_STRING#${dartroot}#" < CESM_DART_config >! temp.$$
${MOVE} temp.$$ ${caseroot}/CESM_DART_config
chmod 755 ${caseroot}/CESM_DART_config
else
@@ -396,17 +391,38 @@
endif
end
- # This is a decent layout for a single instance run.
- @ atm_tasks = $ptile * 4
- @ lnd_tasks = $ptile * 4
- @ ice_tasks = $ptile * 4
- @ ocn_tasks = $ptile
- @ cpl_tasks = $ptile
- @ glc_tasks = $ptile
- @ rof_tasks = $ptile
+# NOTE: If you require bit-for-bit agreement between different runs,
+# in particular, between pmo (single instance) and assimilations (NINST > 1),
+# or if you need to change the number of nodes/member due to changing memory needs,
+# then env_run.xml:BFBFLAG must be set to TRUE, so that the coupler will
+# generate bit-for-bit identical results, regardless of the number of tasks
+# given to it. The time penalty appears to be ~ 0.5% in the forecast.
+# Alternatively, you can set cpl_tasks = same_number in both experiments
-# echo "task partitioning ... perhaps ... atm // ocn // lnd+ice+glc+rof"
-# presently, all components run 'serially' - one after another.
+# Task layout:
+# Set the nodes_per_instance below to match your case. If you get 'out of memory'
+# errors OR failures without any messages, try increasing the nodes_per_instance.
+# CAM-FV 1 degree can run on 2 nodes/instance on yellowstone.
+# CAM-SE ne30 (~ 1 degree) needed 5 nodes/instance.
+# By computing task counts like we do below, we guarantee each instance uses
+# a whole number of nodes which is the recommended configuration.
+
+# Yellowstone: no large memory nodes, and 15 tasks/node is recommended.
+# Edwards says there's no speed up by running non-active components concurrently,
+# after ATM has run, so just run all components sequentially.
+# BUT, do arrange it so that each member(instance) spans complete nodes:
+# modulo(total pe count / number of instances, 15) == 0.
+
+@ nodes_per_instance = 2
+
+@ atm_tasks = $ptile * $nodes_per_instance
+@ lnd_tasks = $ptile * $nodes_per_instance
+@ ice_tasks = $ptile * $nodes_per_instance
+@ ocn_tasks = $ptile
+@ cpl_tasks = $ptile
+@ glc_tasks = $ptile
+@ rof_tasks = $ptile
+
echo ""
echo "ATM gets $atm_tasks"
echo "LND gets $lnd_tasks"
@@ -462,19 +478,13 @@
./xmlchange EXEROOT=${exeroot}
./xmlchange RUNDIR=${rundir}
-./xmlchange SSTICE_DATA_FILENAME=$sst_dataset
-./xmlchange SSTICE_GRID_FILENAME=$sst_grid
-./xmlchange SSTICE_YEAR_ALIGN=$sst_year_start
-./xmlchange SSTICE_YEAR_START=$sst_year_start
-./xmlchange SSTICE_YEAR_END=$sst_year_end
+# Do not change the CALENDAR or the value of CONTINUE_RUN in this script.
-# Do not change the CALENDAR or the CONTINUE_RUN
-
./xmlchange CALENDAR=GREGORIAN
+./xmlchange CONTINUE_RUN=FALSE
./xmlchange STOP_OPTION=$stop_option
./xmlchange STOP_N=$stop_n
-./xmlchange CONTINUE_RUN=FALSE
./xmlchange RESUBMIT=$resubmit
./xmlchange PIO_TYPENAME=pnetcdf
@@ -519,8 +529,9 @@
./xmlchange DOUT_L_MS=TRUE
endif
-# level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
-
+# DEBUG = TRUE implies turning on run and compile time debugging.
+# INFO_DBUG level of debug output, 0=minimum, 1=normal, 2=more, 3=too much.
+# WARNING: CAM-SE fails if DEBUG=TRUE
./xmlchange DEBUG=FALSE
./xmlchange INFO_DBUG=0
@@ -582,9 +593,7 @@
# mods and put in the SourceMods subdirectory found in the 'caseroot' directory.
# ==============================================================================
-if ( -d ~/${cesmtag}/SourceMods ) then
- ${COPY} -r ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/
-else
+if ( ! -d ~/${cesmtag}/SourceMods ) then
echo "ERROR - No SourceMods for this case."
echo "ERROR - No SourceMods for this case."
echo "DART requires modifications to several src files."
@@ -595,6 +604,29 @@
exit -4
endif
+${COPY} -r ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/ || exit 2
+
+# WACCM benefits from a modified cd_core.F90, but none of the
+# other configurations do.
+
+setenv WACCM `echo $CCSM_COMPSET | grep CCM`
+if ($status == 0) then
+ echo "Using the modified cd_core.F90 because WACCM is being used."
+else
+ echo "Using the default cd_core.F90"
+ ${REMOVE} -v SourceMods/src.cam/dynamics/${CAM_DYCORE}/cd_core.F90
+endif
+
+# Each CAM dynamical core has its own SourceMods
+set numfiles = `ls -1 SourceMods/src.cam/dynamics/${CAM_DYCORE} | wc -l`
+if ( $numfiles > 0 ) then
+ cd SourceMods/src.cam
+ ${LINK} dynamics/${CAM_DYCORE}/*F90 .
+ cd ../..
+else
+ echo "No SourceMods for CAM dycore <${CAM_DYCORE}>."
+endif
+
# The CESM multi-instance capability is relatively new and still has a few
# implementation bugs. These are known problems and will be fixed soon.
# this should be removed when the files are fixed:
@@ -612,10 +644,15 @@
${COPY} ${caseroot}/preview_namelists \
${caseroot}/preview_namelists.original
endif
+ if ( ! -e ${caseroot}/Tools/lt_archive.sh.original ) then
+ ${COPY} ${caseroot}/Tools/lt_archive.sh \
+ ${caseroot}/Tools/lt_archive.sh.original
+ endif
# patch/replace the broken files
- ${COPY} ~/${cesmtag}/clm.buildnml.csh ${caseroot}/Buildconf/.
- ${COPY} ~/${cesmtag}/preview_namelists ${caseroot}/.
+ ${COPY} ~/${cesmtag}/clm.buildnml.csh ${caseroot}/Buildconf/. || exit 2
+ ${COPY} ~/${cesmtag}/preview_namelists ${caseroot}/. || exit 2
+ ${COPY} ~/${cesmtag}/lt_archive.sh ${caseroot}/Tools/. || exit 2
endif
@@ -663,46 +700,17 @@
echo " nhtfrq = -$stop_n " >> ${fname}
echo " mfilt = 1 " >> ${fname}
- # Some of the files specified here are because the default files only
- # contain data through 2005 and we are interested in timeframes after that.
-
# CAM5 does prognostic aerosols by default (which is expensive).
# Specifying the following causes CAM to use prescribed aerosols, which is cheaper.
# Prescribed aerosols on the timescales of interest to assimilation are acceptable.
set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
- echo " bndtvghg = '${cesmdata}/atm/cam/ggas/ghg_hist_1765-2009_c100902.nc'" >> ${fname}
- echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
- echo " tracer_cnst_file = 'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'" >> ${fname}
-
echo "aerodep_flx_datapath = '${chem_datapath}/aero'" >> ${fname}
echo "aerodep_flx_file = 'aerosoldep_rcp4.5_monthly_1849-2104_0.9x1.25_c100407.nc'" >> ${fname}
echo "aerodep_flx_type = 'CYCLICAL'" >> ${fname}
echo "aerodep_flx_cycle_yr = 2000" >> ${fname}
- echo " ext_frc_specifier = " >> ${fname}
- echo " 'SO2 -> ${chem_datapath}/emis/ar5_mam3_so2_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'bc_a1 -> ${chem_datapath}/emis/ar5_mam3_bc_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_elev_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_elev_1850-2010_c20100902_v12.nc'" >> ${fname}
-
- echo " srf_emis_specifier = " >> ${fname}
- echo " 'DMS -> ${chem_datapath}/emis/aerocom_mam3_dms_surf_1849-2010_c20100902.nc'," >> ${fname}
- echo " 'SO2 -> ${chem_datapath}/emis/ar5_mam3_so2_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'SOAG -> ${chem_datapath}/emis/ar5_mam3_soag_1.5_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'bc_a1 -> ${chem_datapath}/emis/ar5_mam3_bc_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
- echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_surf_1850-2010_c20100902_v12.nc'" >> ${fname}
-
- echo " solar_data_file = '${cesmdata}/atm/cam/solar/spectral_irradiance_Lean_1610-2009_ann_c100405.nc'" >> ${fname}
-
# ===========================================================================
set fname = "user_nl_clm${inst_string}"
# ===========================================================================
@@ -717,9 +725,11 @@
#
# Making a (compact) .h0. file is a good idea, since the clm restart files
# do not have all the metadata required to reconstruct a gridded field.
- # 'TSA' is 2m surface air temperature.
+ # 'TSA' is 2m surface air temperature. This also prevents
+ # having truly empty history files, resulting in ntapes = 0,
+ # which prevents the hybrid-mode model from restarting.
#
- # Every 6 hours
+ # Every stop_n hours
# echo "hist_mfilt = 1" >> ${fname}
# echo "hist_nhtfrq = -$stop_n" >> ${fname}
# Every month
@@ -730,7 +740,7 @@
echo "hist_fincl1 = 'TSA'" >> ${fname}
echo "hist_nhtfrq = -$stop_n" >> ${fname}
echo "hist_mfilt = 1" >> ${fname}
- echo "hist_avgflag_pertape = 'A'" >> ${fname}
+ echo "hist_avgflag_pertape = 'I'" >> ${fname}
# ===========================================================================
set fname = "user_nl_cice${inst_string}"
@@ -869,12 +879,12 @@
set inst_string = ''
- if ( $SingleInstanceRefcase ) then
+ if ( $MultiInstanceRefcase == 'true') then
+ set true_string = `printf _%04d $TRUTHinstance`
+ echo "Staging initial files from instance $TRUTHinstance for the truth run."
+ else
set true_string = ''
echo "Staging initial files."
- else
- set true_string = `printf _%04d $TRUTHinstance`
- echo "Staging initial files from instance $TRUTHinstance for the truth run."
endif
${LINK} ${stagedir}/${refcase}.clm2\${true_string}.r.${init_time}.nc \
Modified: DART/trunk/models/cam/shell_scripts/CESM_DART_config
===================================================================
--- DART/trunk/models/cam/shell_scripts/CESM_DART_config 2015-03-19 17:48:00 UTC (rev 7741)
+++ DART/trunk/models/cam/shell_scripts/CESM_DART_config 2015-03-20 00:17:16 UTC (rev 7742)
@@ -222,6 +222,20 @@
wq
ex_end
+# If we are using WACCM (i.e. WCCM or WACCM) we have preferred values
+echo "${CCSM_COMPSET}" | grep CCM
+if [[ \$? == 0 ]]; then
+ sed -e "/ vert_normalization_scale_height /c\ vert_normalization_scale_height = 2.5" \
+ -e "/ highest_obs_pressure_Pa /c\ highest_obs_pressure_Pa = 0.0001" \
+ -e "/ highest_state_pressure_Pa /c\ highest_state_pressure_Pa = 0.01" \
+ -e "/ vert_coord /c\ vert_coord = 'log_invP'" \
+ input.nml > input.nml.waccm || exit 1
+ mv input.nml.waccm input.nml || exit 1
+else
+ echo "Apparently not configured for WACCM"
+ echo "CCSM_COMPSET is $CCSM_COMPSET"
+fi
+
exit 0
EndOfText
Modified: DART/trunk/models/cam/shell_scripts/assimilate.csh
===================================================================
--- DART/trunk/models/cam/shell_scripts/assimilate.csh 2015-03-19 17:48:00 UTC (rev 7741)
+++ DART/trunk/models/cam/shell_scripts/assimilate.csh 2015-03-20 00:17:16 UTC (rev 7742)
@@ -379,14 +379,21 @@
# CAM-SE: DART needs a SEMapping_cs_grid.nc file for cubed-sphere grid mapping.
# Use an existing file (given in the namelist), or DART will create one the
# first time it runs. To create one it needs an existing SEMapping.nc file,
-# which should be output from CAM-SE every forecast.
+# which should be output from CAM-SE every forecast. CESM 1_1_1 called this
+# HommeMapping.nc but we require that the DART namelist use 'SEMapping.nc'
+# so we can rename it here.
-if ( $CAM_DYCORE == 'se') then
+if ( $CAM_DYCORE == 'se' || $CAM_DYCORE == 'homme') then
# set the default filenames, and then check the input namelist to
# see if the user has specified a different cs grid filename.
set CS_GRID_FILENAME = 'SEMapping_cs_grid.nc'
- set MAPPING_FILENAME = 'SEMapping.nc'
+ if ( $CAM_DYCORE == 'homme') then
+ set MAPPING_FILENAME = 'HommeMapping.nc'
+ else
+ set MAPPING_FILENAME = 'SEMapping.nc'
+ endif
+
set MYSTRING = `grep cs_grid_file input.nml`
if ($#MYSTRING == 3) then
set MYSTRING = `echo $MYSTRING | sed -e "s#'# #g"`
@@ -395,10 +402,10 @@
# Grid file needs to be in run directory, or cam_to_dart will create one
# based on information from the MAPPING file (which was created by CAM).
- if ( -f ../$CS_GRID_FILENAME) then
+ if ( -f ../$CS_GRID_FILENAME ) then
${LINK} ../$CS_GRID_FILENAME .
else
- ${LINK} ../$MAPPING_FILENAME .
+ ${LINK} ../$MAPPING_FILENAME SEMapping.nc
endif
endif
Modified: DART/trunk/models/cam/shell_scripts/perfect_model.csh
===================================================================
--- DART/trunk/models/cam/shell_scripts/perfect_model.csh 2015-03-19 17:48:00 UTC (rev 7741)
+++ DART/trunk/models/cam/shell_scripts/perfect_model.csh 2015-03-20 00:17:16 UTC (rev 7742)
@@ -164,14 +164,21 @@
# CAM-SE: DART needs a SEMapping_cs_grid.nc file for cubed-sphere grid mapping.
# Use an existing file (given in the namelist), or DART will create one the
# first time it runs. To create one it needs an existing SEMapping.nc file,
-# which should be output from CAM-SE every forecast.
+# which should be output from CAM-SE every forecast. CESM 1_1_1 called this
+# HommeMapping.nc but we require that the DART namelist use 'SEMapping.nc'
+# so we can rename it here.
-if ( $CAM_DYCORE == 'se') then
+if ( $CAM_DYCORE == 'se' || $CAM_DYCORE == 'homme') then
# set the default filenames, and then check the input namelist to
# see if the user has specified a different cs grid filename.
set CS_GRID_FILENAME = 'SEMapping_cs_grid.nc'
- set MAPPING_FILENAME = 'SEMapping.nc'
+ if ( $CAM_DYCORE == 'homme') then
+ set MAPPING_FILENAME = 'HommeMapping.nc'
+ else
+ set MAPPING_FILENAME = 'SEMapping.nc'
+ endif
+
set MYSTRING = `grep cs_grid_file input.nml`
if ($#MYSTRING == 3) then
set MYSTRING = `echo $MYSTRING | sed -e "s#'# #g"`
@@ -183,7 +190,7 @@
if ( -f ../$CS_GRID_FILENAME ) then
${LINK} ../$CS_GRID_FILENAME .
else
- ${LINK} ../$MAPPING_FILENAME .
+ ${LINK} ../$MAPPING_FILENAME SEMapping.nc
endif
endif
Modified: DART/trunk/models/cam/work/input.nml
===================================================================
--- DART/trunk/models/cam/work/input.nml 2015-03-19 17:48:00 UTC (rev 7741)
+++ DART/trunk/models/cam/work/input.nml 2015-03-20 00:17:16 UTC (rev 7742)
@@ -6,6 +6,14 @@
! > Setting up a CAM-SE assimilation
! > Setting up a WACCM assimilation
! > Setting up for perfect_model_obs
+!
+! PLEASE READ
+!
+! https://proxy.subversion.ucar.edu/DAReS/DART/trunk/models/cam/model_mod.html
+!
+! for recommendations on namlist settings for CAM. It has examples on configurations
+! for taking a single state and creating an ensemble, differences between using
+! the FV core or SE core ... variable resolution grids, etc.
! ens_size, num_output_* will be (re)set by the setup script
@@ -61,7 +69,6 @@
inf_sd_initial = 0.6, 0.6,
inf_sd_lower_bound = 0.6, 0.6
/
-! ! inf_flavor is 0:none, 1:obs space, 2: varying state space, 3: fixed state_space
&perfect_model_obs_nml
start_from_restart = .true.
@@ Diff output truncated at 40000 characters. @@
More information about the Dart-dev
mailing list