[Dart-dev] [7743] DART/trunk/models/cam/shell_scripts: These are intended to be the ' entry' level scripts for CAM/DART.

nancy at ucar.edu nancy at ucar.edu
Fri Mar 20 10:42:03 MDT 2015


Revision: 7743
Author:   thoar
Date:     2015-03-20 10:42:02 -0600 (Fri, 20 Mar 2015)
Log Message:
-----------
These are intended to be the 'entry' level scripts for CAM/DART.
They have been tested for FV, SE, and FV-WACCM5. The instructions
on how to modify these for the SE and WACCM cases will be provided
in the model_mod.html. As they stand, they run the FV case by default.

There are 'advanced' versions of these scripts that are intended to
be used by people already familiar with the entry-level scripts.
The advanced scripts are more efficient but are more complicated.

Modified Paths:
--------------
    DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid
    DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_pmo

-------------- next part --------------
Modified: DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid
===================================================================
--- DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid	2015-03-20 00:17:16 UTC (rev 7742)
+++ DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_hybrid	2015-03-20 16:42:02 UTC (rev 7743)
@@ -15,7 +15,8 @@
 # This script is designed to set up, stage, and build a multi-instance run
 # of CESM using an F compset where CAM, CLM and CICE are active. The initial state
 # come from a single multi-instance reference case so a CESM hybrid setup
-# is used.
+# is used. Instructions on what to change to use the SE core or WACCM are
+# outlined in the models/cam/model_mod.html documentation.
 #
 # DOCN: We are using a single data ocean.
 #
@@ -103,12 +104,13 @@
 # compset       Must be one of the CESM standard names, see the CESM documentation
 #               for supported strings.
 # resolution    Sets the model grid resolution, see the CESM documentation.
+#               f09_f09  ... FV core at ~ 1 degree
+#               BUG 1384 applies here, so ocean and atm/land must be at same resolution.
 # cesmtag       The version of the CESM source code to use when building the code.
 # num_instances The number of ensemble members.
 #
-# BUG 1384 applies here, so ocean and atm/land must be at same resolution.
-#   0.9x1.25_0.9x1.25 (f09_f09).  This is roughly a 1 deg grid for atm/lnd,
-#   gx1v6 mask. Normally used for CAM with prescribed lnd, ocn and ice.
+# Guidelines on what to change for an SE or WACCM run are described in the 
+# models/cam/model_mod.html documentation.
 # ==============================================================================
 # AMIP_CAM5_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV (F_AMIP_CAM5) (FAMIPC5)
 
@@ -148,7 +150,7 @@
 setenv rundir       /glade/scratch/${USER}/${case}/run
 setenv exeroot      /glade/scratch/${USER}/${case}/bld
 setenv archdir      /glade/scratch/${USER}/archive/${case}
-setenv dartroot     /glade/u/home/${USER}/svn/DART/trunk
+setenv dartroot     /glade/u/home/${USER}/svn/DART
 
 # ==============================================================================
 # configure settings:
@@ -160,9 +162,15 @@
 # refmon     run will start from.  (Also see 'runtime settings' below for
 # refday     start_year, start_mon, start_day and start_tod.)
 # reftod
+# NOTE:      all the ref* variables must be treated like strings and have
+#            the appropriate number of preceeding zeros 
 #
 # stagedir   The directory location of the reference case files.
 # ==============================================================================
+# alternative reference case for different times may be available here:
+# or on the HPSS:
+# /CCSM/dart/FV0.9x1.25x30_cesm1_1_1/{Mon}1         for 1-degree FV ensembles
+# setenv stagedir /glade/scratch/${USER}/archive/${refcase}/rest/${reftimestamp}
 
 setenv refcase     cesm_hybrid
 setenv refyear     2004
@@ -176,9 +184,6 @@
 
 setenv stagedir /glade/p/image/CESM_initial_ensemble/rest/${reftimestamp}
 
-# alternative reference case for different times may be available here:
-# setenv stagedir /glade/p/work/raeder/Models/CAM_init/${refcase}_${refdate}
-
 # ==============================================================================
 # runtime settings:
 #
@@ -187,11 +192,6 @@
 # start_day
 # start_tod
 #
-# sst_dataset     Data ocean file and
-# sst_grid        supporting grid file. Must be consistent.
-# sst_year_start  Years included in the sst files.
-# sst_year_end
-#
 # short_term_archiver  Copies the files from each job step to a 'rest' directory.
 # long_term_archiver   Puts the files from all completed steps on tape storage.
 #
@@ -209,12 +209,7 @@
 setenv start_day     10
 setenv start_tod     00000
 
-setenv sst_grid    ${cesmdata}/share/domains/domain.ocn.fv0.9x1.25_gx1v6.130409.nc
-setenv sst_dataset ${cesmdata}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
-setenv sst_year_start    1850
-setenv sst_year_end      2012
-
-setenv short_term_archiver on
+setenv short_term_archiver off
 setenv long_term_archiver  off
 
 setenv resubmit            0
@@ -231,7 +226,7 @@
 #      less than 10 minutes on yellowstone using 1800 pes (120 nodes)
 # ==============================================================================
 
-setenv ACCOUNT      NIMGxxxx
+setenv ACCOUNT      ZZZZZZZZ
 setenv queue        economy
 setenv timewall     0:20
 
@@ -325,7 +320,7 @@
    exit -1
 endif
 
-# preserve a copy of this script as it was run
+# Preserve a copy of this script as it was run.
 set ThisFileName = $0:t
 ${COPY} $ThisFileName ${caseroot}/${ThisFileName}.original
 
@@ -341,7 +336,7 @@
 endif
 
 if (   -e CESM_DART_config ) then
-   sed -e "s#BOGUS_DART_ROOT_STRING#$dartroot#" < CESM_DART_config >! temp.$$
+   sed -e "s#BOGUS_DART_ROOT_STRING#${dartroot}#" < CESM_DART_config >! temp.$$
    ${MOVE} temp.$$ ${caseroot}/CESM_DART_config
    chmod 755       ${caseroot}/CESM_DART_config
 else
@@ -380,42 +375,51 @@
    endif
 end
 
-if ($num_instances <= 4) then
+# NOTE: If you require bit-for-bit agreement between different runs,
+#  in particular, between pmo (single instance) and assimilations (NINST > 1),
+#  or if you need to change the number of nodes/member due to changing memory needs,
+#  then env_run.xml:BFBFLAG must be set to TRUE, so that the coupler will
+#  generate bit-for-bit identical results, regardless of the number of tasks
+#  given to it.  The time penalty appears to be ~ 0.5% in the forecast.
+#  Alternatively, you can set cpl_tasks = same_number in both experiments
 
-   # This is only for the purpose of debugging the code.
-   @ atm_tasks = $ptile * $num_instances * 4
-   @ lnd_tasks = $ptile * $num_instances * 4
-   @ ice_tasks = $ptile * $num_instances * 4
-   @ ocn_tasks = $ptile * $num_instances
-   @ cpl_tasks = $ptile * $num_instances
-   @ glc_tasks = $ptile * $num_instances
-   @ rof_tasks = $ptile * $num_instances
-   @ wav_tasks = $ptile * $num_instances
+# Task layout:
+# Set the nodes_per_instance below to match your case.  If you get 'out of memory' 
+# errors OR failures without any messages, try increasing the nodes_per_instance.
+# CAM-FV 1 degree can run on 2 nodes/instance on yellowstone.
+# CAM-SE ne30 (~ 1 degree) needed 5 nodes/instance.
+# By computing task counts like we do below, we guarantee each instance uses 
+# a whole number of nodes which is the recommended configuration.
 
-else
+# Yellowstone: no large memory nodes, and 15 tasks/node is recommended.
+#       Edwards says there's no speed up by running non-active components concurrently,
+#       after ATM has run, so just run all components sequentially.
+#       BUT, do arrange it so that each member(instance) spans complete nodes:
+#       modulo(total pe count / number of instances, 15) == 0.
 
-   # This works, but a more efficient layout should be used
-   # CAM only; 1 node/instance; no cross node communications
-   #           as long as memory is big enough.
-   @ atm_tasks = $ptile * $num_instances * 4
-   @ lnd_tasks = $ptile * $num_instances * 4
-   @ ice_tasks = $ptile * $num_instances * 4
-   @ ocn_tasks = $ptile * $num_instances
-   @ cpl_tasks = $ptile * $num_instances
-   @ glc_tasks = $ptile * $num_instances
-   @ rof_tasks = $ptile * $num_instances
-   @ wav_tasks = $ptile * $num_instances
+@ nodes_per_instance = 2
 
-endif
+@ atm_tasks = $ptile * $num_instances * $nodes_per_instance
+@ lnd_tasks = $ptile * $num_instances * $nodes_per_instance
+@ ice_tasks = $ptile * $num_instances * $nodes_per_instance
+@ ocn_tasks = $ptile * $num_instances
+@ cpl_tasks = $ptile * $num_instances
+@ glc_tasks = $ptile * $num_instances
+@ rof_tasks = $ptile * $num_instances
+@ wav_tasks = $ptile * $num_instances
 
-# echo "task partitioning ... perhaps ... atm // ocn // lnd+ice+glc+rof"
-# presently, all components run 'serially' - one after another.
-# Yellowstone: no large memory nodes, and 15 tasks/node is recommended.
-#              Edwards says there's no speed up by running non-active components concurrently,
-#              after ATM has run, so just run all components sequentially.
-#              BUT, do arrange it so that each member(instance) spans complete nodes:
-#              modulo(total pe count / number of instances, 15) == 0.
 
+switch ("`hostname`")
+   case ys*:
+      if ($atm_tasks > 4100) then
+         echo 'WARNING!   Running DART on yellowstone with > 4100 tasks can cause filter'
+         echo '           to hang in MPI_finalize.  Either reduce your task request,'
+         echo '           or contact DART for a temporary work-around.'
+         exit -9
+      endif
+   breaksw
+endsw
+
 echo ""
 echo "ATM gets $atm_tasks"
 echo "LND gets $lnd_tasks"
@@ -474,19 +478,13 @@
 ./xmlchange EXEROOT=${exeroot}
 ./xmlchange RUNDIR=${rundir}
 
-./xmlchange SSTICE_DATA_FILENAME=$sst_dataset
-./xmlchange SSTICE_GRID_FILENAME=$sst_grid
-./xmlchange SSTICE_YEAR_ALIGN=$sst_year_start
-./xmlchange SSTICE_YEAR_START=$sst_year_start
-./xmlchange SSTICE_YEAR_END=$sst_year_end
+# Do not change the CALENDAR or the value of CONTINUE_RUN in this script.
 
-# Do not change the CALENDAR or the CONTINUE_RUN
-
 ./xmlchange CALENDAR=GREGORIAN
+./xmlchange CONTINUE_RUN=FALSE
 
 ./xmlchange STOP_OPTION=$stop_option
 ./xmlchange STOP_N=$stop_n
-./xmlchange CONTINUE_RUN=FALSE
 ./xmlchange RESUBMIT=$resubmit
 
 ./xmlchange PIO_TYPENAME=pnetcdf
@@ -498,7 +496,7 @@
 
 # The river transport model ON is useful only when using an active ocean or
 # land surface diagnostics. Setting ROF_GRID, RTM_MODE to 'null' turns off the RTM.
-# If you turn it ON, you have more work to do ...
+# If you turn it ON, you will have to stage initial files etc.
 
 ./xmlchange ROF_GRID='null'
 ./xmlchange RTM_MODE='NULL'
@@ -531,8 +529,9 @@
    ./xmlchange DOUT_L_MS=TRUE
 endif
 
-# level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
-
+# DEBUG = TRUE implies turning on run and compile time debugging.
+# INFO_DBUG level of debug output, 0=minimum, 1=normal, 2=more, 3=too much.
+# WARNING: CAM-SE fails if DEBUG=TRUE 
 ./xmlchange DEBUG=FALSE
 ./xmlchange INFO_DBUG=0
 
@@ -544,65 +543,74 @@
 #    mods and put in the SourceMods subdirectory found in the 'caseroot' directory.
 # ==============================================================================
 
-if (    -d     ~/${cesmtag}/SourceMods ) then
+if ( ! -d ~/${cesmtag}/SourceMods ) then
+   echo "ERROR - No SourceMods for this case."
+   echo "ERROR - No SourceMods for this case."
+   echo "DART requires modifications to several src files."
+   echo "Download the appropriate files for CESM 1_2_1 from:"
+   echo "http://www.image.ucar.edu/pub/DART/CESM"
+   echo "untar these into your HOME directory - they will create a"
+   echo "~/cesm1_2_1  directory with the appropriate SourceMods structure."
+   exit -4
+endif
 
-   # Copy all of the 'generic' SourceMods
+# Copy all of the 'generic' SourceMods
+${COPY} -r  ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/   || exit 2
 
-   ${COPY} -r  ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/
+# Each CLM version has some SourceMods. Link to the right version.
+# must parse from a variable of the form:
+# CLM_CONFIG_OPTS: -phys clm4_0 -bgc cn
 
-   # Each CLM version has some SourceMods. Link to the right version.
-   # must parse from a variable of the form:
-   # CLM_CONFIG_OPTS: -phys clm4_0 -bgc cn
+set clm_opts = `echo $CLM_CONFIG_OPTS | sed -e "s/-//"`
 
-   set clm_opts = `echo $CLM_CONFIG_OPTS | sed -e "s/-//"`
+@ iarg = 1
+while ($iarg <= $#clm_opts)
 
-   @ iarg = 1
-   while ($iarg <= $#clm_opts)
+   @ iargp1 = $iarg + 1
+   set option = $clm_opts[$iarg]
+   set  value = $clm_opts[$iargp1]
 
-      @ iargp1 = $iarg + 1
-      set option = $clm_opts[$iarg]
-      set  value = $clm_opts[$iargp1]
+   switch ( ${option} )
+      case "phys":
+         if ( -e    SourceMods/src.clm/src/${value} ) then
+            cd      SourceMods/src.clm
+            ${LINK} src/${value}/*/*F90 .
+            cd      ../..
+         else
+            echo "No SourceMods for CLM <${value}>."
+            echo "Got the version from CLM_CONFIG_OPTS ...  <${CLM_CONFIG_OPTS}>"
+         endif
+      breaksw
+   #  case "bgc":  no special action needed here at this time
+   #  breaksw
 
-      switch ( ${option} )
-         case "phys":
-            if ( -e    SourceMods/src.clm/src/${value} ) then
-               cd      SourceMods/src.clm
-               ${LINK} src/${value}/*/*F90 .
-               cd      ../..
-            else
-               echo "No SourceMods for CLM <${value}>."
-               echo "Got the version from CLM_CONFIG_OPTS ...  <${CLM_CONFIG_OPTS}>"
-            endif
-         breaksw
-      #  case "bgc":  no special action needed here at this time
-      #  breaksw
+      default:
+      breaksw
+   endsw
 
-         default:
-         breaksw
-      endsw
+   @ iarg = $iarg + 2
+end
 
-      @ iarg = $iarg + 2
-   end
+# Need to know if we are using WACCM (aka WCCM) for several reasons.
+# Mostly file managment issues.
+set waccm = `echo $CCSM_COMPSET | grep WCCM`
+if ($status == 0) then
+   set waccm = true
+else
+   set waccm = false
+   # WACCM benefits from a modified cd_core.F90, but none of the
+   # other configurations do. 
+   echo "Using the default cd_core.F90"
+   ${REMOVE} -v SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/cd_core.F90
+endif
 
-   # Each CAM dynamical core has its own SourceMods
-
-   if ( -e    SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/*F90 ) then
-      cd      SourceMods/src.cam
-      ${LINK} src/dynamics/${CAM_DYCORE}/*F90 .
-      cd      ../..
-   else
-      echo "No SourceMods for CAM dycore <${CAM_DYCORE}>."
-   endif
-
+# Each CAM dynamical core has its own SourceMods
+if ( -e    SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/*F90 ) then
+   cd      SourceMods/src.cam
+   ${LINK} src/dynamics/${CAM_DYCORE}/*F90 .
+   cd      ../..
 else
-   echo "ERROR - No SourceMods for this case."
-   echo "ERROR - No SourceMods for this case."
-   echo "DART requires modifications to several src files."
-   echo "Download the appropriate files for CESM 1_2_1 from:"
-   echo "http://www.image.ucar.edu/pub/DART/CESM"
-   echo "untar these into your HOME directory - they will create a"
-   echo "~/cesm1_2_1  directory with the appropriate SourceMods structure."
-   exit -4
+   echo "No SourceMods for CAM dycore <${CAM_DYCORE}>."
 endif
 
 # The CESM multi-instance capability is relatively new and still has a few
@@ -718,45 +726,48 @@
    echo " nhtfrq        = -$stop_n "                      >> ${fname}
    echo " mfilt         = 1 "                             >> ${fname}
 
+   # CAM forcing files.
    # Some of the files specified here are because the default files only
    # contain data through 2005 and we are interested in timeframes after that.
 
    # CAM5 does prognostic aerosols by default (which is expensive).
    # Specifying the following causes CAM to use prescribed aerosols, which is cheaper.
    # Prescribed aerosols on the timescales of interest to assimilation are acceptable.
+   # WACCM5 should not have these set.
 
-   set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
+   if ($waccm == false ) then
 
-   echo " bndtvghg              = '${cesmdata}/atm/cam/ggas/ghg_hist_1765-2009_c100902.nc'" >> ${fname}
-   echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'"               >> ${fname}
-   echo " tracer_cnst_file      =  'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'"               >> ${fname}
+    set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
 
-   echo "aerodep_flx_datapath   = '${chem_datapath}/aero'"                                      >> ${fname}
-   echo "aerodep_flx_file       = 'aerosoldep_rcp4.5_monthly_1849-2104_0.9x1.25_c100407.nc'"    >> ${fname}
-   echo "aerodep_flx_type       = 'CYCLICAL'"                                                   >> ${fname}
-   echo "aerodep_flx_cycle_yr   = 2000"                                                         >> ${fname}
+    echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'"               >> ${fname}
+    echo " tracer_cnst_file      =  'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'"               >> ${fname}
 
-   echo " ext_frc_specifier     = "                                                             >> ${fname}
-   echo " 'SO2    -> ${chem_datapath}/emis/ar5_mam3_so2_elev_1850-2010_c20100902_v12.nc',"      >> ${fname}
-   echo " 'bc_a1  -> ${chem_datapath}/emis/ar5_mam3_bc_elev_1850-2010_c20100902_v12.nc',"       >> ${fname}
-   echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_elev_1850-2010_c20100902_v12.nc',"       >> ${fname}
-   echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_elev_1850-2010_c20100902_v12.nc'"    >> ${fname}
+    echo "aerodep_flx_datapath   = '${chem_datapath}/aero'"                                      >> ${fname}
+    echo "aerodep_flx_file       = 'aerosoldep_rcp4.5_monthly_1849-2104_0.9x1.25_c100407.nc'"    >> ${fname}
+    echo "aerodep_flx_type       = 'CYCLICAL'"                                                   >> ${fname}
+    echo "aerodep_flx_cycle_yr   = 2000"                                                         >> ${fname}
 
-   echo " srf_emis_specifier    = "                                                             >> ${fname}
-   echo " 'DMS    -> ${chem_datapath}/emis/aerocom_mam3_dms_surf_1849-2010_c20100902.nc',"      >> ${fname}
-   echo " 'SO2    -> ${chem_datapath}/emis/ar5_mam3_so2_surf_1850-2010_c20100902_v12.nc',"      >> ${fname}
-   echo " 'SOAG   -> ${chem_datapath}/emis/ar5_mam3_soag_1.5_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
-   echo " 'bc_a1  -> ${chem_datapath}/emis/ar5_mam3_bc_surf_1850-2010_c20100902_v12.nc',"       >> ${fname}
-   echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_surf_1850-2010_c20100902_v12.nc',"       >> ${fname}
-   echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_surf_1850-2010_c20100902_v12.nc'"    >> ${fname}
+    echo " ext_frc_specifier     = "                                                             >> ${fname}
+    echo " 'SO2    -> ${chem_datapath}/emis/ar5_mam3_so2_elev_1850-2010_c20100902_v12.nc',"      >> ${fname}
+    echo " 'bc_a1  -> ${chem_datapath}/emis/ar5_mam3_bc_elev_1850-2010_c20100902_v12.nc',"       >> ${fname}
+    echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
+    echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
+    echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_elev_1850-2010_c20100902_v12.nc',"       >> ${fname}
+    echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
+    echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_elev_1850-2010_c20100902_v12.nc'"    >> ${fname}
+ 
+    echo " srf_emis_specifier    = "                                                             >> ${fname}
+    echo " 'DMS    -> ${chem_datapath}/emis/aerocom_mam3_dms_surf_1849-2010_c20100902.nc',"      >> ${fname}
+    echo " 'SO2    -> ${chem_datapath}/emis/ar5_mam3_so2_surf_1850-2010_c20100902_v12.nc',"      >> ${fname}
+    echo " 'SOAG   -> ${chem_datapath}/emis/ar5_mam3_soag_1.5_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
+    echo " 'bc_a1  -> ${chem_datapath}/emis/ar5_mam3_bc_surf_1850-2010_c20100902_v12.nc',"       >> ${fname}
+    echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
+    echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
+    echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_surf_1850-2010_c20100902_v12.nc',"       >> ${fname}
+    echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
+    echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_surf_1850-2010_c20100902_v12.nc'"    >> ${fname}
 
-   echo " solar_data_file = '${cesmdata}/atm/cam/solar/spectral_irradiance_Lean_1610-2009_ann_c100405.nc'" >> ${fname}
+   endif
 
    # ===========================================================================
    set fname = "user_nl_clm${inst_string}"
@@ -772,9 +783,11 @@
    #
    # Making a (compact) .h0. file is a good idea, since the clm restart files
    # do not have all the metadata required to reconstruct a gridded field.
-   # 'TSA' is 2m surface air temperature.
+   # 'TSA' is 2m surface air temperature.  This also prevents
+   # having truly empty history files, resulting in ntapes = 0,
+   # which prevents the hybrid-mode model from restarting.
    #
-   # Every 6 hours
+   # Every stop_n hours
    # echo "hist_mfilt  = 1"              >> ${fname}
    # echo "hist_nhtfrq = -$stop_n"       >> ${fname}
    # Every month
@@ -785,7 +798,7 @@
    echo "hist_fincl1 = 'TSA'"          >> ${fname}
    echo "hist_nhtfrq = -$stop_n"       >> ${fname}
    echo "hist_mfilt  = 1"              >> ${fname}
-   echo "hist_avgflag_pertape = 'A'"   >> ${fname}
+   echo "hist_avgflag_pertape = 'I'"   >> ${fname}
 
    # ===========================================================================
    set fname = "user_nl_cice${inst_string}"
@@ -973,9 +986,6 @@
 echo "   files and the files that were staged."
 echo ""
 echo "2) cd ${caseroot}"
-echo "   (on yellowstone) If the ${case}.run script still contains:"
-echo '   #BSUB -R "select[scratch_ok > 0]"'
-echo "   around line 9, delete it."
 echo ""
 echo "3) The case is initially configured to do NO ASSIMILATION by calling"
 echo "   a script named 'no_assimilate.csh' in the ${case}.run script."

Modified: DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_pmo
===================================================================
--- DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_pmo	2015-03-20 00:17:16 UTC (rev 7742)
+++ DART/trunk/models/cam/shell_scripts/CESM1_2_1_setup_pmo	2015-03-20 16:42:02 UTC (rev 7743)
@@ -15,7 +15,8 @@
 # This script is designed to set up, stage, and build a single-instance run
 # of CESM using an F compset where CAM, CLM and CICE are active. The initial state
 # can come from a single multi-instance reference case so a CESM hybrid setup
-# is used.
+# is used. Instructions on what to change to use the SE core or WACCM are
+# outlined in the models/cam/model_mod.html documentation.
 #
 # DOCN: We are using a single data ocean.
 #
@@ -105,11 +106,12 @@
 # compset       Must be one of the CESM standard names, see the CESM documentation
 #               for supported strings.
 # resolution    Sets the model grid resolution, see the CESM documentation.
+#               f09_f09  ... FV core at ~ 1 degree
+#               BUG 1384 applies here, so ocean and atm/land must be at same resolution.
 # cesmtag       The version of the CESM source code to use when building the code.
 #
-# BUG 1384 applies here, so ocean and atm/land must be at same resolution.
-#   0.9x1.25_0.9x1.25 (f09_f09).  This is roughly a 1 deg grid for atm/lnd,
-#   gx1v6 mask. Normally used for CAM with prescribed lnd, ocn and ice.
+# Guidelines on what to change for an SE or WACCM run are described in the 
+# models/cam/model_mod.html documentation.
 # ==============================================================================
 # AMIP_CAM5_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV (F_AMIP_CAM5) (FAMIPC5)
 
@@ -148,7 +150,7 @@
 setenv rundir       /glade/scratch/${USER}/${case}/run
 setenv exeroot      /glade/scratch/${USER}/${case}/bld
 setenv archdir      /glade/scratch/${USER}/archive/${case}
-setenv dartroot     /glade/u/home/${USER}/svn/DART/trunk
+setenv dartroot     /glade/u/home/${USER}/svn/DART
 
 # ==============================================================================
 # configure settings:
@@ -160,23 +162,29 @@
 # refmon     run will start from.  (Also see 'runtime settings' below for
 # refday     start_year, start_mon, start_day and start_tod.)
 # reftod
+# NOTE:      all the ref* variables must be treated like strings and have
+#            the appropriate number of preceeding zeros 
 #
-# SingleInstanceRefcase
+# MultiInstanceRefcase
 #            Filenames from a multi-instance CESM run have an instance number
 #            in them.  Filenames from a single instance CESM run do not.
 #            Setting this helps the script figure out the right filenames to
-#            generate when copying files from the refernenc case:
-#               1 means the restart files have no instance numbers.
-#               0 means the restart files have strings like .0001. in their names.
+#            generate when copying files from the reference case:
+#              'true' means the restart files have strings like .0001. in their names.
+#              'false' the restart files have no instance numbers.
 #
 # TRUTHinstance
-#            If SingleInstanceRefcase is 0, this must be set to select the
+#            If MultiInstanceRefcase is 'true', this must be set to select the
 #            specific instance you want to use when selecting one instance from
 #            a multi-instance run.  e.g. If you have a reference case run
 #            with 80 instances, 1<= TRUTHinstance <= 80. NO LEADING ZEROS.
 #
 # stagedir   The directory location of the reference case files.
 # ==============================================================================
+# alternative reference case for different times may be available here:
+# or on the HPSS:
+# /CCSM/dart/FV0.9x1.25x30_cesm1_1_1/{Mon}1         for 1-degree FV ensembles
+# setenv stagedir /glade/scratch/${USER}/archive/${refcase}/rest/${reftimestamp}
 
 setenv refcase     cesm_hybrid
 setenv refyear     2004
@@ -184,8 +192,8 @@
 setenv refday      10
 setenv reftod      00000
 
-setenv SingleInstanceRefcase 0
-setenv TRUTHinstance 23
+setenv MultiInstanceRefcase true
+setenv TRUTHinstance 1
 
 # useful combinations of time that we use below
 setenv refdate      $refyear-$refmon-$refday
@@ -193,9 +201,6 @@
 
 setenv stagedir /glade/p/image/CESM_initial_ensemble/rest/${reftimestamp}
 
-# alternative reference case for different times may be available here:
-# setenv stagedir /glade/p/work/raeder/Models/CAM_init/${refcase}_${refdate}
-
 # ==============================================================================
 # runtime settings:
 #
@@ -204,11 +209,6 @@
 # start_day
 # start_tod
 #
-# sst_dataset     Data ocean file and
-# sst_grid        supporting grid file. Must be consistent.
-# sst_year_start  Years included in the sst files.
-# sst_year_end
-#
 # short_term_archiver  Copies the files from each job step to a 'rest' directory.
 # long_term_archiver   Puts the files from all completed steps on tape storage.
 #
@@ -226,12 +226,7 @@
 setenv start_day     10
 setenv start_tod     00000
 
-setenv sst_grid    ${cesmdata}/share/domains/domain.ocn.fv0.9x1.25_gx1v6.130409.nc
-setenv sst_dataset ${cesmdata}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
-setenv sst_year_start    1850
-setenv sst_year_end      2012
-
-setenv short_term_archiver on
+setenv short_term_archiver off
 setenv long_term_archiver  off
 
 setenv resubmit            0
@@ -342,7 +337,7 @@
    exit -1
 endif
 
-# preserve a copy of this script as it was run
+# Preserve a copy of this script as it was run.
 set ThisFileName = $0:t
 ${COPY} $ThisFileName ${caseroot}/${ThisFileName}.original
 
@@ -358,7 +353,7 @@
 endif
 
 if (   -e CESM_DART_config ) then
-   sed -e "s#BOGUS_DART_ROOT_STRING#$dartroot#" < CESM_DART_config >! temp.$$
+   sed -e "s#BOGUS_DART_ROOT_STRING#${dartroot}#" < CESM_DART_config >! temp.$$
    ${MOVE} temp.$$ ${caseroot}/CESM_DART_config
    chmod 755       ${caseroot}/CESM_DART_config
 else
@@ -397,18 +392,39 @@
    endif
 end
 
-   # This is a decent layout for a single instance run.
-   @ atm_tasks = $ptile * 4
-   @ lnd_tasks = $ptile * 4
-   @ ice_tasks = $ptile * 4
-   @ ocn_tasks = $ptile
-   @ cpl_tasks = $ptile
-   @ glc_tasks = $ptile
-   @ rof_tasks = $ptile
-   @ wav_tasks = $ptile
+# NOTE: If you require bit-for-bit agreement between different runs,
+#  in particular, between pmo (single instance) and assimilations (NINST > 1),
+#  or if you need to change the number of nodes/member due to changing memory needs,
+#  then env_run.xml:BFBFLAG must be set to TRUE, so that the coupler will
+#  generate bit-for-bit identical results, regardless of the number of tasks
+#  given to it.  The time penalty appears to be ~ 0.5% in the forecast.
+#  Alternatively, you can set cpl_tasks = same_number in both experiments
 
-# echo "task partitioning ... perhaps ... atm // ocn // lnd+ice+glc+rof"
-# presently, all components run 'serially' - one after another.
+# Task layout:
+# Set the nodes_per_instance below to match your case.  If you get 'out of memory' 
+# errors OR failures without any messages, try increasing the nodes_per_instance.
+# CAM-FV 1 degree can run on 2 nodes/instance on yellowstone.
+# CAM-SE ne30 (~ 1 degree) needed 5 nodes/instance.
+# By computing task counts like we do below, we guarantee each instance uses 
+# a whole number of nodes which is the recommended configuration.
+
+# Yellowstone: no large memory nodes, and 15 tasks/node is recommended.
+#       Edwards says there's no speed up by running non-active components concurrently,
+#       after ATM has run, so just run all components sequentially.
+#       BUT, do arrange it so that each member(instance) spans complete nodes:
+#       modulo(total pe count / number of instances, 15) == 0.
+
+@ nodes_per_instance = 2
+
+@ atm_tasks = $ptile * $nodes_per_instance
+@ lnd_tasks = $ptile * $nodes_per_instance
+@ ice_tasks = $ptile * $nodes_per_instance
+@ ocn_tasks = $ptile
+@ cpl_tasks = $ptile
+@ glc_tasks = $ptile
+@ rof_tasks = $ptile
+@ wav_tasks = $ptile
+
 echo ""
 echo "ATM gets $atm_tasks"
 echo "LND gets $lnd_tasks"
@@ -467,19 +483,13 @@
 ./xmlchange EXEROOT=${exeroot}
 ./xmlchange RUNDIR=${rundir}
 
-./xmlchange SSTICE_DATA_FILENAME=$sst_dataset
-./xmlchange SSTICE_GRID_FILENAME=$sst_grid
-./xmlchange SSTICE_YEAR_ALIGN=$sst_year_start
-./xmlchange SSTICE_YEAR_START=$sst_year_start
-./xmlchange SSTICE_YEAR_END=$sst_year_end
+# Do not change the CALENDAR or the value of CONTINUE_RUN in this script.
 
-# Do not change the CALENDAR or the CONTINUE_RUN
-
 ./xmlchange CALENDAR=GREGORIAN
+./xmlchange CONTINUE_RUN=FALSE
 
 ./xmlchange STOP_OPTION=$stop_option
 ./xmlchange STOP_N=$stop_n
-./xmlchange CONTINUE_RUN=FALSE
 ./xmlchange RESUBMIT=$resubmit
 
 ./xmlchange PIO_TYPENAME=pnetcdf
@@ -491,7 +501,7 @@
 
 # The river transport model ON is useful only when using an active ocean or
 # land surface diagnostics. Setting ROF_GRID, RTM_MODE to 'null' turns off the RTM.
-# If you turn it ON, you have more work to do ...
+# If you turn it ON, you will have to stage initial files etc.
 
 ./xmlchange ROF_GRID='null'
 ./xmlchange RTM_MODE='NULL'
@@ -524,8 +534,9 @@
    ./xmlchange DOUT_L_MS=TRUE
 endif
 
-# level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
-
+# DEBUG = TRUE implies turning on run and compile time debugging.
+# INFO_DBUG level of debug output, 0=minimum, 1=normal, 2=more, 3=too much.
+# WARNING: CAM-SE fails if DEBUG=TRUE 
 ./xmlchange DEBUG=FALSE
 ./xmlchange INFO_DBUG=0
 
@@ -537,65 +548,74 @@
 #    mods and put in the SourceMods subdirectory found in the 'caseroot' directory.
 # ==============================================================================
 
-if (    -d     ~/${cesmtag}/SourceMods ) then
+if ( ! -d ~/${cesmtag}/SourceMods ) then
+   echo "ERROR - No SourceMods for this case."
+   echo "ERROR - No SourceMods for this case."
+   echo "DART requires modifications to several src files."
+   echo "Download the appropriate files for CESM 1_2_1 from:"
+   echo "http://www.image.ucar.edu/pub/DART/CESM"
+   echo "untar these into your HOME directory - they will create a"
+   echo "~/cesm1_2_1  directory with the appropriate SourceMods structure."
+   exit -4
+endif
 
-   # Copy all of the 'generic' SourceMods
+# Copy all of the 'generic' SourceMods
+${COPY} -r  ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/   || exit 2
 
-   ${COPY} -r  ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/
+# Each CLM version has some SourceMods. Link to the right version.
+# must parse from a variable of the form:
+# CLM_CONFIG_OPTS: -phys clm4_0 -bgc cn
 
-   # Each CLM version has some SourceMods. Link to the right version.
-   # must parse from a variable of the form:
-   # CLM_CONFIG_OPTS: -phys clm4_0 -bgc cn
+set clm_opts = `echo $CLM_CONFIG_OPTS | sed -e "s/-//"`
 
-   set clm_opts = `echo $CLM_CONFIG_OPTS | sed -e "s/-//"`
+@ iarg = 1
+while ($iarg <= $#clm_opts)
 
-   @ iarg = 1
-   while ($iarg <= $#clm_opts)
+   @ iargp1 = $iarg + 1
+   set option = $clm_opts[$iarg]
+   set  value = $clm_opts[$iargp1]
 
-      @ iargp1 = $iarg + 1
-      set option = $clm_opts[$iarg]
-      set  value = $clm_opts[$iargp1]
+   switch ( ${option} )
+      case "phys":
+         if ( -e    SourceMods/src.clm/src/${value} ) then
+            cd      SourceMods/src.clm
+            ${LINK} src/${value}/*/*F90 .
+            cd      ../..
+         else
+            echo "No SourceMods for CLM <${value}>."
+            echo "Got the version from CLM_CONFIG_OPTS ...  <${CLM_CONFIG_OPTS}>"
+         endif
+      breaksw
+   #  case "bgc":  no special action needed here at this time
+   #  breaksw
 
-      switch ( ${option} )
-         case "phys":
-            if ( -e    SourceMods/src.clm/src/${value} ) then
-               cd      SourceMods/src.clm
-               ${LINK} src/${value}/*/*F90 .
-               cd      ../..
-            else
-               echo "No SourceMods for CLM <${value}>."
-               echo "Got the version from CLM_CONFIG_OPTS ...  <${CLM_CONFIG_OPTS}>"
-            endif
-         breaksw
-      #  case "bgc":  no special action needed here at this time
-      #  breaksw
+      default:
+      breaksw
+   endsw
 
-         default:
-         breaksw
-      endsw
+   @ iarg = $iarg + 2
+end
 
-      @ iarg = $iarg + 2
-   end
+# Need to know if we are using WACCM (aka WCCM) for several reasons.
+# Mostly file managment issues.
+set waccm = `echo $CCSM_COMPSET | grep WCCM`
+if ($status == 0) then
+   set waccm = true
+else
+   set waccm = false
+   # WACCM benefits from a modified cd_core.F90, but none of the
+   # other configurations do. 
+   echo "Using the default cd_core.F90"
+   ${REMOVE} -v SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/cd_core.F90
+endif
 
-   # Each CAM dynamical core has its own SourceMods
-
-   if ( -e    SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/*F90 ) then
-      cd      SourceMods/src.cam
-      ${LINK} src/dynamics/${CAM_DYCORE}/*F90 .
-      cd      ../..
-   else
-      echo "No SourceMods for CAM dycore <${CAM_DYCORE}>."
-   endif
-
+# Each CAM dynamical core has its own SourceMods
+if ( -e    SourceMods/src.cam/src/dynamics/${CAM_DYCORE}/*F90 ) then
+   cd      SourceMods/src.cam
+   ${LINK} src/dynamics/${CAM_DYCORE}/*F90 .
+   cd      ../..
 else
-   echo "ERROR - No SourceMods for this case."
-   echo "ERROR - No SourceMods for this case."
-   echo "DART requires modifications to several src files."
-   echo "Download the appropriate files for CESM 1_2_1 from:"
-   echo "http://www.image.ucar.edu/pub/DART/CESM"
-   echo "untar these into your HOME directory - they will create a"
-   echo "~/cesm1_2_1  directory with the appropriate SourceMods structure."
-   exit -4
+   echo "No SourceMods for CAM dycore <${CAM_DYCORE}>."
 endif
 
 # The CESM multi-instance capability is relatively new and still has a few
@@ -711,45 +731,48 @@
    echo " nhtfrq        = -$stop_n "                      >> ${fname}
    echo " mfilt         = 1 "                             >> ${fname}
 
+   # CAM forcing files.
    # Some of the files specified here are because the default files only
    # contain data through 2005 and we are interested in timeframes after that.
 
    # CAM5 does prognostic aerosols by default (which is expensive).
    # Specifying the following causes CAM to use prescribed aerosols, which is cheaper.
    # Prescribed aerosols on the timescales of interest to assimilation are acceptable.
+   # WACCM5 should not have these set.
 
-   set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
+   if ($waccm == false ) then
 
-   echo " bndtvghg              = '${cesmdata}/atm/cam/ggas/ghg_hist_1765-2009_c100902.nc'" >> ${fname}
-   echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'"               >> ${fname}
-   echo " tracer_cnst_file      =  'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'"               >> ${fname}
+    set chem_datapath = "${cesmdata}/atm/cam/chem/trop_mozart_aero"
 
-   echo "aerodep_flx_datapath   = '${chem_datapath}/aero'"                                      >> ${fname}
-   echo "aerodep_flx_file       = 'aerosoldep_rcp4.5_monthly_1849-2104_0.9x1.25_c100407.nc'"    >> ${fname}
-   echo "aerodep_flx_type       = 'CYCLICAL'"                                                   >> ${fname}
-   echo "aerodep_flx_cycle_yr   = 2000"                                                         >> ${fname}
+    echo " prescribed_ozone_file = 'ozone_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'"               >> ${fname}
+    echo " tracer_cnst_file      =  'oxid_1.9x2.5_L26_1850-2015_rcp45_c101108.nc'"               >> ${fname}
 
-   echo " ext_frc_specifier     = "                                                             >> ${fname}
-   echo " 'SO2    -> ${chem_datapath}/emis/ar5_mam3_so2_elev_1850-2010_c20100902_v12.nc',"      >> ${fname}
-   echo " 'bc_a1  -> ${chem_datapath}/emis/ar5_mam3_bc_elev_1850-2010_c20100902_v12.nc',"       >> ${fname}
-   echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_elev_1850-2010_c20100902_v12.nc',"       >> ${fname}
-   echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_elev_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_elev_1850-2010_c20100902_v12.nc'"    >> ${fname}
+    echo "aerodep_flx_datapath   = '${chem_datapath}/aero'"                                      >> ${fname}
+    echo "aerodep_flx_file       = 'aerosoldep_rcp4.5_monthly_1849-2104_0.9x1.25_c100407.nc'"    >> ${fname}
+    echo "aerodep_flx_type       = 'CYCLICAL'"                                                   >> ${fname}
+    echo "aerodep_flx_cycle_yr   = 2000"                                                         >> ${fname}
 
-   echo " srf_emis_specifier    = "                                                             >> ${fname}
-   echo " 'DMS    -> ${chem_datapath}/emis/aerocom_mam3_dms_surf_1849-2010_c20100902.nc',"      >> ${fname}
-   echo " 'SO2    -> ${chem_datapath}/emis/ar5_mam3_so2_surf_1850-2010_c20100902_v12.nc',"      >> ${fname}
-   echo " 'SOAG   -> ${chem_datapath}/emis/ar5_mam3_soag_1.5_surf_1850-2010_c20100902_v12.nc'," >> ${fname}
-   echo " 'bc_a1  -> ${chem_datapath}/emis/ar5_mam3_bc_surf_1850-2010_c20100902_v12.nc',"       >> ${fname}
-   echo " 'num_a1 -> ${chem_datapath}/emis/ar5_mam3_num_a1_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'num_a2 -> ${chem_datapath}/emis/ar5_mam3_num_a2_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'pom_a1 -> ${chem_datapath}/emis/ar5_mam3_oc_surf_1850-2010_c20100902_v12.nc',"       >> ${fname}
-   echo " 'so4_a1 -> ${chem_datapath}/emis/ar5_mam3_so4_a1_surf_1850-2010_c20100902_v12.nc',"   >> ${fname}
-   echo " 'so4_a2 -> ${chem_datapath}/emis/ar5_mam3_so4_a2_surf_1850-2010_c20100902_v12.nc'"    >> ${fname}
+    echo " ext_frc_specifier     = "                                                             >> ${fname}
+    echo " 'SO2    -> ${chem_datapath}/emis/ar5_mam3_so2_elev_1850-2010_c20100902_v12.nc',"      >> ${fname}

@@ Diff output truncated at 40000 characters. @@


More information about the Dart-dev mailing list