[Dart-dev] [5984] DART/branches/development/models: The setup and assimilate scripts are now VERY similar across all

nancy at ucar.edu nancy at ucar.edu
Tue Mar 5 10:50:18 MST 2013


Revision: 5984
Author:   thoar
Date:     2013-03-05 10:50:17 -0700 (Tue, 05 Mar 2013)
Log Message:
-----------
The setup and assimilate scripts are now VERY similar across all
model components.
The CLM setup script could use a block specifying the pe layout
for a 30 member ensemble ... 

The save intermediate restart options for CESM are untested
and some of the model components do not have a tested 'st_archive.sh'
but will soon. Since it is not required ... it is better to save
what we have now.

Modified Paths:
--------------
    DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh
    DART/branches/development/models/POP/shell_scripts/assimilate.csh
    DART/branches/development/models/cam/shell_scripts/CESM_setup_startup.csh
    DART/branches/development/models/cam/shell_scripts/assimilate.csh
    DART/branches/development/models/clm/shell_scripts/CESM1_1_setup.csh
    DART/branches/development/models/clm/shell_scripts/assimilate.csh
    DART/branches/development/models/clm/work/input.nml

-------------- next part --------------
Modified: DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh
===================================================================
--- DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh	2013-03-05 17:40:05 UTC (rev 5983)
+++ DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh	2013-03-05 17:50:17 UTC (rev 5984)
@@ -13,7 +13,8 @@
 # This script is designed to configure and build a multi-instance CESM model
 # that has POP as active components
 # and will use DART to assimilate observations at regular intervals.
-# This script does not build DART.
+# This script does not build DART. It works best if the appropriate DART
+# executables have been built, however.
 #
 # This script relies heavily on the information in:
 # http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/book1.html
@@ -47,13 +48,7 @@
 # For the brave, read
 #
 # http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/x1142.html
-#
-# and you may be able to salvage something with
-# ./cesm_setup -clean
-# ./cesm_setup
-# ./${case}.clean_build
-# ./${case}.build
-#
+
 # ==============================================================================
 # ====  Set case options
 # ==============================================================================
@@ -63,11 +58,11 @@
 #    script names; so consider it's length and information content.
 # num_instances:  Number of ensemble members
 
-setenv case                 pop_hybrid_opt
+setenv case                 pop_test
 setenv compset              GIAF
 setenv cesmtag              cesm1_1_1
 setenv resolution           T62_gx1v6
-setenv num_instances        30
+setenv num_instances        4
 
 # ==============================================================================
 # define machines and directories
@@ -75,9 +70,9 @@
 # mach            Computer name
 # cesm_datadir    Root path of the public CESM data files
 # cesmroot        Location of the cesm code base
-#                 For cesm1_1 on yellowstone
+#                    i.e. cesm1_1_1 on yellowstone
 # DARTroot        Location of DART code tree.
-#                    Executables, scripts and input in $DARTroot/models/dev/...
+#                    Executables, scripts and input in $DARTroot/models/POP/...
 # caseroot        Your (future) cesm case directory, where this CESM+DART will be built.
 #                    Preferably not a frequently scrubbed location.
 #                    This script will delete any existing caseroot, so this script,
@@ -99,7 +94,7 @@
 setenv DARTroot     /glade/u/home/${USER}/svn/DART/dev
 
 # ==============================================================================
-# configure settings
+# configure settings ... run_startdate format is yyyy-mm-dd
 # ==============================================================================
 
 setenv stream_year_first 2004
@@ -131,7 +126,7 @@
 # TJH: How many T62_gx1v6 POP instances can fit on 1 node?
 # ==============================================================================
 
-setenv ACCOUNT      P86850054
+setenv ACCOUNT      P8685nnnn
 setenv timewall     0:30
 setenv queue        regular
 setenv ptile        15
@@ -157,6 +152,7 @@
       set   COPY = '/bin/cp -fv --preserve=timestamps'
       set   LINK = '/bin/ln -fvs'
       set REMOVE = '/bin/rm -fr'
+      set nonomatch
 
    breaksw
 endsw
@@ -191,7 +187,9 @@
 
 # Save a copy for debug purposes
 foreach FILE ( *xml )
-   ${COPY} $FILE ${FILE}.original
+   if ( ~ -e        ${FILE}.original ) then
+      ${COPY} $FILE ${FILE}.original
+   endif
 end
 
 # Try to lay out the tasks in some sort of reasonable way.
@@ -202,46 +200,46 @@
    # This is only for the purpose of debugging the code.
    # A more efficient layout must be done when running a full assimilation.
 
-   @ atm_pes = $ptile
    @ cpl_pes = $ptile
-   @ ice_pes = $ptile
+   @ atm_pes = $ptile * $num_instances
+   @ ice_pes = $ptile * $num_instances
    @ lnd_pes = $ptile
+   @ rof_pes = $ptile
    @ glc_pes = $ptile
-   @ rof_pes = $ptile
-   @ ocn_pes = $ptile * 4
+   @ ocn_pes = $ptile * $num_instances
 
 else
 
    # layout for 30 members.
    # Made in conjunction with Mike Levy, David Bailey and Jim Edwards.
 
+   @ cpl_pes = 450
    @ atm_pes = 450
-   @ cpl_pes = 450
    @ ice_pes = 450
    @ lnd_pes = 450
+   @ rof_pes = 450
    @ glc_pes = 450
-   @ rof_pes = 450
    @ ocn_pes = 1800
 
 endif
 
 echo "task partitioning ... ice and ocean run at the same time."
 echo ""
-echo "ATM  gets $atm_pes"
-echo "CPL  gets $cpl_pes"
-echo "ICE  gets $ice_pes"
-echo "LND  gets $lnd_pes"
-echo "GLC  gets $glc_pes"
-echo "DROF gets $rof_pes"
-echo "OCN  gets $ocn_pes"
+echo "CPL gets $cpl_pes"
+echo "ATM gets $atm_pes"
+echo "ICE gets $ice_pes"
+echo "LND gets $lnd_pes"
+echo "ROF gets $rof_pes"
+echo "GLC gets $glc_pes"
+echo "OCN gets $ocn_pes"
 echo ""
 
+./xmlchange ROOTPE_CPL=0,NTHRDS_CPL=1,NTASKS_CPL=$cpl_pes
 ./xmlchange ROOTPE_ATM=0,NTHRDS_ATM=1,NTASKS_ATM=$atm_pes,NINST_ATM=$num_instances
 ./xmlchange ROOTPE_ICE=0,NTHRDS_ICE=1,NTASKS_ICE=$ice_pes,NINST_ICE=$num_instances
-./xmlchange ROOTPE_GLC=0,NTHRDS_GLC=1,NTASKS_GLC=$glc_pes,NINST_GLC=1
 ./xmlchange ROOTPE_LND=0,NTHRDS_LND=1,NTASKS_LND=$lnd_pes,NINST_LND=1
 ./xmlchange ROOTPE_ROF=0,NTHRDS_ROF=1,NTASKS_ROF=$rof_pes,NINST_ROF=1
-./xmlchange ROOTPE_CPL=0,NTHRDS_CPL=1,NTASKS_CPL=$cpl_pes
+./xmlchange ROOTPE_GLC=0,NTHRDS_GLC=1,NTASKS_GLC=$glc_pes,NINST_GLC=1
 ./xmlchange              NTHRDS_OCN=1,NTASKS_OCN=$ocn_pes,NINST_OCN=$num_instances
 ./xmlchange ROOTPE_OCN=$ice_pes
 
@@ -267,20 +265,13 @@
 ./xmlchange BRNCH_RETAIN_CASENAME=FALSE
 ./xmlchange GET_REFCASE=FALSE
 ./xmlchange EXEROOT=${exeroot}
+./xmlchange PIO_TYPENAME=pnetcdf
 
-# The streams files were generated with a NO_LEAP calendar in mind.
-# We need to test these with a GREGORIAN calendar.
+# DOUT_S     is to turn on/off the short-term archiving
+# DOUT_L_MS  is to store to the HPSS (formerly "MSS")
 
-./xmlchange CALENDAR=GREGORIAN
-
-./xmlchange STOP_OPTION=$stop_option
-./xmlchange STOP_N=$stop_n
-./xmlchange CONTINUE_RUN=FALSE
-./xmlchange RESUBMIT=$resubmit
-./xmlchange PIO_TYPENAME=pnetcdf
-
+./xmlchange DOUT_S_ROOT=${archdir}
 ./xmlchange DOUT_S=TRUE
-./xmlchange DOUT_S_ROOT=${archdir}
 ./xmlchange DOUT_S_SAVE_INT_REST_FILES=FALSE
 ./xmlchange DOUT_L_MS=FALSE
 ./xmlchange DOUT_L_MSROOT="csm/${case}"
@@ -292,6 +283,22 @@
 ./xmlchange DATM_CPLHIST_YR_START=$refyear
 ./xmlchange DATM_CPLHIST_YR_END=$refyear
 
+# The streams files were generated with a NO_LEAP calendar in mind.
+# We need to test these with a GREGORIAN calendar.
+
+./xmlchange CALENDAR=GREGORIAN
+./xmlchange STOP_OPTION=$stop_option
+./xmlchange STOP_N=$stop_n
+./xmlchange CONTINUE_RUN=FALSE
+./xmlchange RESUBMIT=$resubmit
+
+# The river transport model ON is useful only when using an active ocean or
+# land surface diagnostics. Setting ROF_GRID to 'null' turns off the RTM.
+# TJH - guidance needed from Alicia
+
+# ./xmlchange ROF_GRID='null'
+# ./xmlchange ROF_GRID='r05'
+
 # level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
 
 ./xmlchange DEBUG=FALSE
@@ -326,7 +333,9 @@
    set instance  = `printf %04d $inst`
    set instance2 = `printf %02d $inst`
 
+   # ===========================================================================
    set fname = "user_nl_datm_$instance"
+   # ===========================================================================
 
    echo "streams = 'datm.streams.txt.CPLHIST3HrWx.Solar_$instance             $stream_year_align $stream_year_first $stream_year_last'," >> $fname
    echo "          'datm.streams.txt.CPLHIST3HrWx.nonSolarNonPrecip_$instance $stream_year_align $stream_year_first $stream_year_last'"  >> $fname
@@ -340,15 +349,23 @@
    echo "restfils = 'unset'"               >> $fname
    echo "restfilm = 'unset'"               >> $fname
 
+   # ===========================================================================
+   set fname = "user_nl_cice_$instance"
+   # ===========================================================================
    # CICE Namelists
    # this is only used for a hybrid start, else rpointers are used.
-   echo "ice_ic = 'b40.20th.005_ens"$instance2".cice.r.2004-01-01-00000.nc'" >> user_nl_cice_$instance
 
+   echo "ice_ic = 'b40.20th.005_ens"$instance2".cice.r.2004-01-01-00000.nc'" >> $fname
+
+   # ===========================================================================
+   set fname = "user_nl_pop2_$instance"
+   # ===========================================================================
+
    # POP Namelists
    # init_ts_suboption = 'data_assim'   for non bit-for-bit restarting (assimilation mode)
    # init_ts_suboption = 'null'         for 'perfect' restarting/forecasting
 
-   echo "init_ts_suboption = 'data_assim'" >> user_nl_pop2_$instance
+   echo "init_ts_suboption = 'data_assim'" >> $fname
 
    @ inst ++
 end
@@ -470,15 +487,19 @@
 
 set stagedir = /glade/p/work/aliciak/DART_IC/CCSM4_ensembles/rest/2004-01-01-00000
 
+echo ''
 echo "Copying the restart files from ${stagedir}"
+echo 'into the CESM run directory.'
+echo ''
 
-@ i = 1
-while ($i <= $num_instances)
-   set n4 = `printf %04d $i`
-   set n2 = `printf %02d $i`
+@ inst = 1
+while ($inst <= $num_instances)
+   set n4 = `printf %04d $inst`
+   set n2 = `printf %02d $inst`
 
    echo ''
-   echo "Staging restarts for instance $i of $num_instances"
+   echo "Staging restarts for instance $inst of $num_instances"
+
    #AK: Note that the pop ocean must have the .hdr file to describe the
    #    metadata for binary restarts.
    ${COPY} ${stagedir}/rpointer.ocn_${n4}.restart                       ${rundir}
@@ -489,10 +510,13 @@
    ${COPY} ${stagedir}/b40.20th.005_ens${n2}.pop.ro.2004-01-01-00000    ${rundir}
    ${COPY} ${stagedir}/b40.20th.005_ens${n2}.cice.r.2004-01-01-00000.nc ${rundir}
 
-   @ i ++
-
+   @ inst ++
 end
 
+if (  -e   ${stagedir}/prior_inflate_restart* ) then
+   ${COPY} ${stagedir}/prior_inflate_restart* ${rundir}/.
+endif
+
 # ==============================================================================
 # Edit the run script to reflect project, queue, and wallclock
 # ==============================================================================
@@ -501,7 +525,9 @@
 echo 'Updating the run script to set wallclock and queue.'
 echo ''
 
-${COPY} ${case}.run ${case}.run.orig
+if ( ~ -e  ${case}.run.original ) then
+   ${COPY} ${case}.run ${case}.run.original
+endif
 
 source Tools/ccsm_getenv
 set BATCH = `echo $BATCHSUBMIT | sed 's/ .*$//'`
@@ -510,7 +536,7 @@
       # NCAR "bluefire", "yellowstone"
       set TIMEWALL=`grep BSUB ${case}.run | grep -e '-W' `
       set    QUEUE=`grep BSUB ${case}.run | grep -e '-q' `
-      sed -e "s/ptile=32/ptile=$ptile/" \
+      sed -e "s/ptile=[0-9][0-9]*/ptile=$ptile/" \
           -e "s/$TIMEWALL[3]/$timewall/" \
           -e "s/$QUEUE[3]/$queue/" < ${case}.run >! temp.$$
       ${MOVE} temp.$$  ${case}.run
@@ -588,33 +614,37 @@
    cat                add_to_run.txt >> temp.$$
    tail -n $lastlines ${case}.run    >> temp.$$
 
-   ${MOVE} temp.$$ ${case}.run
+   ${MOVE}   temp.$$ ${case}.run
+   ${REMOVE} add_to_run.txt
 
+else
+   echo "ERROR in grep of ${case}.run: aborting"
+   echo "status was ${STATUSCHECK}"
+   exit -6
 endif
 
-chmod 0744 ${case}.run
+chmod 0755 ${case}.run
 
 # ==============================================================================
 # Stage the required parts of DART in the CASEROOT directory.
 # ==============================================================================
 
-# AK: the standard CESM short-term archiving script may need to be altered
+# The standard CESM short-term archiving script may need to be altered
 # to archive addtional or subsets of things, or to reduce the amount of
-# data that is sent to the long-term archive.  Put a version of st_archive.sh
-# in  ${DARTroot}/models/POP/shell_scripts when/if necessary
-#
-if ( ~ -e  Tools/st_archive.sh.orig ) then
-   ${MOVE} Tools/st_archive.sh      Tools/st_archive.sh.orig
-else
-   echo "a Tools/st_archive.sh backup copy already exists"
+# data that is sent to the long-term archive.
+
+if ( ~ -e  Tools/st_archive.sh.original ) then
+   ${COPY} Tools/st_archive.sh Tools/st_archive.sh.original
 endif
 
-# AK note: you must use this assimilate.csh script.
-#          It has hardcoded variables that point to the location of
-#          the observations sequence files and the DART working
-#          directory and the PE layout for DART.
+# NOTE: the assimilate.csh script and input.nml must be modified for your
+#       situation. The script has variables that point to the location of
+#       the observations sequence files and the DART working directory
+#       and may be customized for a more efficient PE layout for DART.
+#       If you are running this, you know what to do with input.nml.
+#       If you don't, you should give up now. Really.
 
-# ${COPY} ${DARTroot}/models/POP/shell_scripts/st_archive.sh   Tools/ TJH DEBUG
+${COPY} ${DARTroot}/models/POP/shell_scripts/st_archive.sh   Tools/
 ${COPY} ${DARTroot}/models/POP/shell_scripts/assimilate.csh  assimilate.csh
 ${COPY} ${DARTroot}/models/POP/work/input.nml                input.nml
 
@@ -627,7 +657,7 @@
    if ( $status != 0 ) then
       echo "ERROR: ${DARTroot}/models/POP/work/${FILE} not copied to ${exeroot}"
       echo "ERROR: ${DARTroot}/models/POP/work/${FILE} not copied to ${exeroot}"
-      exit -3
+      exit -7
    endif
 end
 

Modified: DART/branches/development/models/POP/shell_scripts/assimilate.csh
===================================================================
--- DART/branches/development/models/POP/shell_scripts/assimilate.csh	2013-03-05 17:40:05 UTC (rev 5983)
+++ DART/branches/development/models/POP/shell_scripts/assimilate.csh	2013-03-05 17:50:17 UTC (rev 5984)
@@ -1,4 +1,4 @@
-#!/bin/tcsh
+#!/bin/csh
 #
 # DART software - Copyright 2004 - 2011 UCAR. This open source software is
 # provided by UCAR, "as is", without charge, subject to all terms of use at
@@ -19,8 +19,7 @@
       # the VERBOSE options are useful for debugging.
       set   MOVE = '/usr/local/bin/mv -fv'
       set   COPY = '/usr/local/bin/cp -fv --preserve=timestamps'
-      set  FLINK = '/usr/local/bin/ln -fvs'
-      set   LINK = '/usr/local/bin/ln -vs'
+      set   LINK = '/usr/local/bin/ln -fvs'
       set REMOVE = '/usr/local/bin/rm -fr'
 
       set BASEOBSDIR = /glade/proj3/image/Observations/WOD09
@@ -34,8 +33,7 @@
       # the VERBOSE options are useful for debugging.
       set   MOVE = 'mv -fv'
       set   COPY = 'cp -fv --preserve=timestamps'
-      set  FLINK = 'ln -fvs'
-      set   LINK = 'ln -vs'
+      set   LINK = 'ln -fvs'
       set REMOVE = 'rm -fr'
 
       set BASEOBSDIR = /glade/p/image/Observations/WOD09
@@ -47,8 +45,7 @@
       # NERSC "hopper"
       set   MOVE = 'mv -fv'
       set   COPY = 'cp -fv --preserve=timestamps'
-      set  FLINK = 'ln -fvs'
-      set   LINK = 'ln -vs'
+      set   LINK = 'ln -fvs'
       set REMOVE = 'rm -fr'
 
       set BASEOBSDIR = /scratch/scratchdirs/nscollin/ACARS
@@ -73,7 +70,7 @@
 
 #-------------------------------------------------------------------------
 # Determine time of model state ... from file name of first member
-# of the form "./${CASE}.pop.$ensemble_member.r.2000-01-06-00000.nc"
+# of the form "./${CASE}.pop_${ensemble_member}.r.2000-01-06-00000.nc"
 #
 # Piping stuff through 'bc' strips off any preceeding zeros.
 #-------------------------------------------------------------------------
@@ -95,22 +92,19 @@
 
 #-----------------------------------------------------------------------------
 # Get observation sequence file ... or die right away.
-# Cannot specify -f on the link command and still check status.
 # The observation file names have a time that matches the stopping time of POP.
 #-----------------------------------------------------------------------------
 
-set YYYYMM   = `printf %04d%02d ${OCN_YEAR} ${OCN_MONTH}`
-set OBSFNAME = `printf obs_seq.0Z.%04d%02d%02d ${OCN_YEAR} ${OCN_MONTH} ${OCN_DAY}`
+set YYYYMMDD = `printf %04d%02d%02d ${OCN_YEAR} ${OCN_MONTH} ${OCN_DAY}`
+set YYYYMM   = `printf %04d%02d     ${OCN_YEAR} ${OCN_MONTH}`
+set OBSFNAME = obs_seq.0Z.${YYYYMMDD}
 set OBS_FILE = ${BASEOBSDIR}/${YYYYMM}/${OBSFNAME}
 
-${REMOVE}           obs_seq.out
-${LINK} ${OBS_FILE} obs_seq.out
-
-set lnstat = $status
-if ($lnstat != 0) then
+if (  -e   ${OBS_FILE} ) then
+   ${LINK} ${OBS_FILE} obs_seq.out
+else
    echo "ERROR ... no observation file $OBS_FILE"
    echo "ERROR ... no observation file $OBS_FILE"
-   echo "ERROR ... ln died with status $lnstat"
    exit -1
 endif
 
@@ -125,13 +119,12 @@
 else
    echo "ERROR ... DART required file ${CASEROOT}/input.nml not found ... ERROR"
    echo "ERROR ... DART required file ${CASEROOT}/input.nml not found ... ERROR"
-   exit -1
+   exit -2
 endif
 
-# Since the obs sequence files are small, modify the DART input.nml such
-# that the num_output_obs_members matches the ensemble size.
-#
-# g;num_output_state_members ;s;= .*;= $ensemble_size;
+# Modify the DART input.nml such that
+# the DART ensemble size matches the CESM number of instances
+# WARNING: the output files contain ALL enemble members ==> BIG
 
 ex input.nml <<ex_end
 g;ens_size ;s;= .*;= $ensemble_size;
@@ -166,7 +159,7 @@
    else
       echo "ERROR: no sampling error correction file for this ensemble size."
       echo "ERROR: looking for ${SAMP_ERR_FILE}"
-      exit -2
+      exit -3
    endif
 else
    echo "Sampling Error Correction not requested for this assimilation."
@@ -251,11 +244,11 @@
       # If one exists, use it as input for this assimilation
       if ( $nfiles > 0 ) then
          set latest = `cat latestfile`
-         ${FLINK} $latest ${PRIOR_INF_IFNAME}
+         ${LINK} $latest ${PRIOR_INF_IFNAME}
       else
          echo "ERROR: Requested PRIOR inflation but specified no incoming inflation file."
          echo "ERROR: expected something like ../${PRIOR_INF_OFNAME}.YYYY-MM-DD-SSSSS"
-         exit 4
+         exit -4
       endif
 
    endif
@@ -278,11 +271,11 @@
       # If one exists, use it as input for this assimilation
       if ( $nfiles > 0 ) then
          set latest = `cat latestfile`
-         ${FLINK} $latest ${POSTE_INF_IFNAME}
+         ${LINK} $latest ${POSTE_INF_IFNAME}
       else
          echo "ERROR: Requested POSTERIOR inflation but specified no incoming inflation file."
          echo "ERROR: expected something like ../${POSTE_INF_OFNAME}.YYYY-MM-DD-SSSSS"
-         exit 6
+         exit -5
       endif
    endif
 else
@@ -290,22 +283,20 @@
 endif
 
 #=========================================================================
-# Block 4: convert N POP restart files to DART initial conditions file(s)
+# Block 4: Convert N POP restart files to DART initial condition files.
 # pop_to_dart is serial code, we can do all of these at the same time
-# and just wait for them to finish IFF it were not for the fact we'd have
-# to have unique namelists for all of them.
+# as long as we can have unique namelists for each of them.
 #
-# At the end of the block, we have DART restart files  filter_ics.[1-N]
-# that came from pointer files ../rpointer.ocn.[1-N].restart
+# At the end of the block, we have DART initial condition files  filter_ics.[1-N]
+# that came from pointer files ../rpointer.ocn_[1-N].restart
 #
-# DART namelist settings appropriate/required:
+# REQUIRED DART namelist settings:
 # &filter_nml:           restart_in_file_name    = 'filter_ics'
 #                        restart_out_file_name   = 'filter_restart'
 # &ensemble_manager_nml: single_restart_file_in  = '.false.'
 # &pop_to_dart_nml:      pop_to_dart_output_file = 'dart_ics',
 # &dart_to_pop_nml:      dart_to_pop_input_file  = 'dart_restart',
-# &dart_to_pop_nml:      advance_time_present    = .false.
-#
+#                        advance_time_present    = .false.
 #=========================================================================
 
 echo "`date` -- BEGIN POP TO DART"
@@ -328,8 +319,8 @@
    sed -e "s/dart_ics/..\/${DART_IC_FILENAME}/" \
        -e "s/dart_restart/..\/${DART_RESTART_FILE}/" < ../input.nml >! input.nml
 
-   ${FLINK} $OCN_RESTART_FILENAME pop.r.nc
-   ${FLINK}     $OCN_NML_FILENAME pop_in
+   ${LINK} $OCN_RESTART_FILENAME pop.r.nc
+   ${LINK}     $OCN_NML_FILENAME pop_in
 
    echo "starting pop_to_dart for member ${member} at "`date`
    ${EXEROOT}/pop_to_dart >! output.${member}.pop_to_dart &
@@ -344,7 +335,7 @@
 if ($status != 0) then
    echo "ERROR ... DART died in 'pop_to_dart' ... ERROR"
    echo "ERROR ... DART died in 'pop_to_dart' ... ERROR"
-   exit -7
+   exit -6
 endif
 
 echo "`date` -- END POP-TO-DART for all ${ensemble_size} members."
@@ -366,7 +357,7 @@
 # &filter_nml:           first_obs_seconds      = -1,
 # &filter_nml:           last_obs_days          = -1,
 # &filter_nml:           last_obs_seconds       = -1,
-# &ensemble_manager_nml: single_restart_file_in = '.false.'
+# &ensemble_manager_nml: single_restart_file_in = .false.
 #
 #=========================================================================
 
@@ -374,13 +365,27 @@
 # Lots of ways to get the filename
 
 set OCN_RESTART_FILENAME = `head -1 ../rpointer.ocn_0001.restart`
+
 ${LINK} ../$OCN_RESTART_FILENAME pop.r.nc
 ${LINK} ../pop2_in_0001          pop_in
 
+# On yellowstone, you can explore task layouts with the following:
+if ( $?LSB_PJL_TASK_GEOMETRY ) then
+   setenv ORIGINAL_LAYOUT "${LSB_PJL_TASK_GEOMETRY}"
+
+   # setenv GEOMETRY_32_1NODE \
+   #    "{(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31)}";
+   # setenv LSB_PJL_TASK_GEOMETRY "${GEOMETRY_32_1NODE}"
+endif
+
 echo "`date` -- BEGIN FILTER"
-${LAUNCHCMD} ${EXEROOT}/filter || exit -2
+${LAUNCHCMD} ${EXEROOT}/filter || exit -7
 echo "`date` -- END FILTER"
 
+if ( $?LSB_PJL_TASK_GEOMETRY ) then
+   setenv LSB_PJL_TASK_GEOMETRY "${ORIGINAL_LAYOUT}"
+endif
+
 ${MOVE} Prior_Diag.nc      ../Prior_Diag.${OCN_DATE_EXT}.nc
 ${MOVE} Posterior_Diag.nc  ../Posterior_Diag.${OCN_DATE_EXT}.nc
 ${MOVE} obs_seq.final      ../obs_seq.${OCN_DATE_EXT}.final
@@ -402,17 +407,14 @@
 #=========================================================================
 # Block 6: Update the POP restart files ... simultaneously ...
 #
-# Each member will do its job in its own directory.
-# That way, we can do N of them simultaneously.
-# The namelist already reflects the right filenames.
-# The right filenames are already linked from the pop_to_dart conversion.
+# Each member will do its job in its own directory, which already exists
+# and has the required input files remaining from 'Block 4'
 #=========================================================================
 
 echo "`date` -- BEGIN DART TO POP"
 set member = 1
 while ( $member <= $ensemble_size )
 
-   set m4 = `printf %04d $member`
    cd member_${member}
 
    echo "starting dart_to_pop for member ${member} at "`date`

Modified: DART/branches/development/models/cam/shell_scripts/CESM_setup_startup.csh
===================================================================
--- DART/branches/development/models/cam/shell_scripts/CESM_setup_startup.csh	2013-03-05 17:40:05 UTC (rev 5983)
+++ DART/branches/development/models/cam/shell_scripts/CESM_setup_startup.csh	2013-03-05 17:50:17 UTC (rev 5984)
@@ -13,7 +13,8 @@
 # This script is designed to configure and build a multi-instance CESM model
 # that has CAM, CLM, and CICE as active components over a single data ocean,
 # and will use DART to assimilate observations at regular intervals.
-# This script does not build DART.
+# This script does not build DART. It works best if the appropriate DART
+# executables have been built, however.
 #
 # This script relies heavily on the information in:
 # http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/book1.html
@@ -47,30 +48,7 @@
 # For the brave, read
 #
 # http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/x1142.html
-#
-# ==============================================================================
-# === IMPORTANT modifications to the distribution code before ANYTHING
-# ==============================================================================
 
-# Had to edit the following to remove the LSB_PJL_... word too long error
-# cesm1_1_beta04/scripts/ccsm_utils/Machines/mkbatch.bluefire .
-# On bluefire for cesm1_1_beta04 it's not enough to use the hard-wired
-# SourceMods below; mkbatch.bluefire cannot be modified via SourceMods.
-# So the cesm1_1_beta04 version in Tim's directory must be used.
-# As long as OMP_NUM_THREADS == 1 ... the default is fine.
-# This may also be a problem if the number of nodes is increased, e.g.
-# if the regular memory nodes are used, or if the resolution is increased,
-# because the number of entries in BIND_THRD_GEOMETRY will increase.
-#
-# The cesm1_1_beta04 lt_archive script did not create parent dirs
-# if they did not already exist.  To fix the script, edit:
-#   ${cesmroot}/scripts/ccsm_utils/Tools/lt_archive.csh
-# and change 'mkdir' to 'mkdir -p'.  This is fixed in more recent
-# versions of the code.
-#
-# For these reasons, it is required that you use ~thoar/cesm1_1_beta04
-# as the source of the CESM code.
-
 # ==============================================================================
 # ====  Set case options
 # ==============================================================================
@@ -92,7 +70,7 @@
 # mach            Computer name
 # cesm_datadir    Root path of the public CESM data files
 # cesmroot        Location of the cesm code base
-#                 For cesm1_1 on yellowstone
+#                    i.e. cesm1_1_1 on yellowstone
 # DARTroot        Location of DART code tree.
 #                    Executables, scripts and input in $DARTroot/models/cam/...
 # caseroot        Your (future) cesm case directory, where this CESM+DART will be built.
@@ -118,7 +96,7 @@
 # ==============================================================================
 # configure settings ... run_startdate format is yyyy-mm-dd
 # ==============================================================================
-#
+
 setenv run_startdate 2008-10-31
 setenv sst_dataset ${cesm_datadir}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
 setenv year_start  1850
@@ -128,7 +106,6 @@
 # runtime settings --  How many assimilation steps will be done after this one
 #
 # stop_option   Units for determining the forecast length between assimilations
-#               Changing stop_option requires changes to user_nl_cam below.
 # stop_n        Number of time units in the forecast
 # ==============================================================================
 
@@ -150,8 +127,8 @@
 #         from running for long periods.
 # ==============================================================================
 
-setenv ACCOUNT      P86850054
-setenv timewall     0:30
+setenv ACCOUNT      P8685nnnn
+setenv timewall     0:40
 setenv queue        regular
 setenv ptile        15
 
@@ -176,6 +153,7 @@
       set   COPY = '/bin/cp -fv --preserve=timestamps'
       set   LINK = '/bin/ln -fvs'
       set REMOVE = '/bin/rm -fr'
+      set nonomatch
 
    breaksw
 endsw
@@ -210,7 +188,9 @@
 
 # Save a copy for debug purposes
 foreach FILE ( *xml )
-   ${COPY} $FILE ${FILE}.original
+   if ( ~ -e        ${FILE}.original ) then
+      ${COPY} $FILE ${FILE}.original
+   endif
 end
 
 # num_tasks_per_instance = #tasks_node / #instances_node
@@ -279,9 +259,13 @@
 ./xmlchange RUN_TYPE=startup
 ./xmlchange RUN_STARTDATE=$run_startdate
 ./xmlchange EXEROOT=${exeroot}
+./xmlchange PIO_TYPENAME=pnetcdf
 
+# DOUT_S     is to turn on/off the short-term archiving
+# DOUT_L_MS  is to store to the HPSS (formerly "MSS")
+
 ./xmlchange DOUT_S_ROOT=${archdir}
-./xmlchange DOUT_S=FALSE
+./xmlchange DOUT_S=TRUE
 ./xmlchange DOUT_S_SAVE_INT_REST_FILES=TRUE
 ./xmlchange DOUT_L_MS=FALSE
 ./xmlchange DOUT_L_MSROOT="csm/${case}"
@@ -293,15 +277,12 @@
 ./xmlchange SSTICE_YEAR_END=$year_end
 
 # Do not change the CALENDAR or the CONTINUE_RUN
-# DOUT_S     is to turn on/off the short-term archiving
-# DOUT_L_MS  is to store to the HPSS (formerly "MSS")
 
 ./xmlchange CALENDAR=GREGORIAN
 ./xmlchange STOP_OPTION=$stop_option
 ./xmlchange STOP_N=$stop_n
 ./xmlchange CONTINUE_RUN=FALSE
 ./xmlchange RESUBMIT=$resubmit
-./xmlchange PIO_TYPENAME=pnetcdf
 
 # The river transport model ON is useful only when using an active ocean or
 # land surface diagnostics. Setting ROF_GRID to 'null' turns off the RTM.
@@ -408,7 +389,7 @@
    # ===========================================================================
 
    echo  " ice_ic = 'ice_restart_${instance}.nc' " >> ${fname}
-  
+
    @ inst ++
 end
 
@@ -427,6 +408,8 @@
 
 # none needed for CAM
 
+./preview_namelists
+
 # ==============================================================================
 # Update source files if need be
 #    Ideally, using DART will not require any source mods.
@@ -446,7 +429,7 @@
    echo "http://www.image.ucar.edu/pub/DART/CESM/DART_SourceMods_cesm1_1_1.tar"
    echo "untar these into your HOME directory - they will create a"
    echo "~/cesm_1_1_1  directory with the appropriate SourceMods structure."
-   exit -3
+   exit -4
 endif
 
 # ==============================================================================
@@ -461,15 +444,15 @@
 
 if ( $status != 0 ) then
    echo "ERROR: Case could not be built."
-   exit -4
+   exit -5
 endif
 
 # ==============================================================================
 # Stage the restarts now that the run directory exists
+#
+# CAM5 ... /glade/p/work/raeder/Exp/Pincus_Cam5/archive/Exp3/rest/2008-11-01-00000
 # ==============================================================================
 
-# CAM5 ... /glade/p/work/raeder/Exp/Pincus_Cam5/archive/Exp3/rest/2008-11-01-00000
-
 set stagedir = /glade/p/work/raeder/Exp/Pincus_Cam5/archive/Exp3/rest/2008-11-01-00000
 
 echo ''
@@ -477,14 +460,18 @@
 echo 'into the CESM run directory.'
 echo ''
 
-@ n = 1
-while ($n <= $num_instances)
-   set instance  = `printf %04d $n`
-   echo "Staging restarts for instance $n of $num_instances"
-   ${COPY} ${stagedir}/cam_initial_${n}.nc ${rundir}/cam_initial_${instance}.nc
-   ${COPY} ${stagedir}/clm_restart_${n}.nc ${rundir}/clm_restart_${instance}.nc
-   ${COPY} ${stagedir}/ice_restart_${n}.nc ${rundir}/ice_restart_${instance}.nc
-   @ n++
+@ inst = 1
+while ($inst <= $num_instances)
+   set instance  = `printf %04d $inst`
+
+   echo ''
+   echo "Staging restarts for instance $inst of $num_instances"
+
+   ${COPY} ${stagedir}/cam_initial_${inst}.nc ${rundir}/cam_initial_${instance}.nc
+   ${COPY} ${stagedir}/clm_restart_${inst}.nc ${rundir}/clm_restart_${instance}.nc
+   ${COPY} ${stagedir}/ice_restart_${inst}.nc ${rundir}/ice_restart_${instance}.nc
+
+   @ inst ++
 end
 
 if (  -e   ${stagedir}/prior_inflate_restart* ) then
@@ -492,27 +479,6 @@
 endif
 
 # ==============================================================================
-# IMPORTANT: All resubmits must be type 'startup'.
-# Change Tools/ccsm_postrun.csh line 83 to CONTINUE_RUN -val FALSE'
-# ==============================================================================
-
-echo ''
-echo 'Changing Tools/ccsm_postrun_setup	such that all the resubmits are "startup",'
-echo 'which means CONTINUE_RUN should be FALSE in ccsm_postrun_setup'
-echo ''
-
-if ( ~ -e  Tools/cesm_postrun_setup.original ) then
-   ${COPY} Tools/cesm_postrun_setup Tools/cesm_postrun_setup.original
-endif
-
-ex Tools/cesm_postrun_setup <<ex_end
-/use COMP_RUN_BARRIERS as surrogate for timing run logical/
-/CONTINUE_RUN/
-s;TRUE;FALSE;
-wq
-ex_end
-
-# ==============================================================================
 # Edit the run script to reflect project, queue, and wallclock
 # ==============================================================================
 
@@ -531,7 +497,7 @@
       # NCAR "bluefire", "yellowstone"
       set TIMEWALL=`grep BSUB ${case}.run | grep -e '-W' `
       set    QUEUE=`grep BSUB ${case}.run | grep -e '-q' `
-      sed -e "s/ptile=32/ptile=$ptile/" \
+      sed -e "s/ptile=[0-9][0-9]*/ptile=$ptile/" \
           -e "s/$TIMEWALL[3]/$timewall/" \
           -e "s/$QUEUE[3]/$queue/" < ${case}.run >! temp.$$
       ${MOVE} temp.$$  ${case}.run
@@ -542,8 +508,24 @@
    breaksw
 endsw
 
-chmod 0755 ${case}.run
+# ==============================================================================
+# IMPORTANT: All resubmits must be type 'startup'.
+# ==============================================================================
 
+echo ''
+echo 'Changing the run script such that all the resubmits are "startup",'
+echo 'which means CONTINUE_RUN should be FALSE'
+echo ''
+
+ex ${case}.run <<ex_end
+/use COMP_RUN_BARRIERS as surrogate for timing run logical/
+/CONTINUE_RUN/
+s;TRUE;FALSE;
+/CONTINUE_RUN/
+s;TRUE;FALSE;
+wq
+ex_end
+
 # ==============================================================================
 # The *.run script must be modified to call the DART assimilate script.
 # The modifications are contained in a "here" document that MUST NOT
@@ -617,25 +599,33 @@
 else
    echo "ERROR in grep of ${case}.run: aborting"
    echo "status was ${STATUSCHECK}"
-   exit -5
+   exit -6
 endif
 
+chmod 0755 ${case}.run
+
 # ==============================================================================
 # Stage the required parts of DART in the CASEROOT directory.
 # ==============================================================================
 
-# The cesm1_1_beta04 release had an error in that it did not
-# provide the lt_archive.sh script, and the one in the repos
-# did not have the -p flag, which is a good idea. So, for now ...
+# The standard CESM short-term archiving script may need to be altered
+# to archive addtional or subsets of things, or to reduce the amount of
+# data that is sent to the long-term archive.
 
 if ( ~ -e  Tools/st_archive.sh.original ) then
    ${COPY} Tools/st_archive.sh Tools/st_archive.sh.original
 endif
 
-# ${COPY} ${DARTroot}/models/cam/shell_scripts/st_archive.sh  Tools/
+# NOTE: the assimilate.csh script and input.nml must be modified for your
+#       situation. The script has variables that point to the location of
+#       the observations sequence files and the DART working directory
+#       and may be customized for a more efficient PE layout for DART.
+#       If you are running this, you know what to do with input.nml.
+#       If you don't, you should give up now. Really.
 
-${COPY} ${DARTroot}/models/cam/shell_scripts/assimilate.csh  .
-${COPY} ${DARTroot}/models/cam/work/input.nml                .
+${COPY} ${DARTroot}/models/cam/shell_scripts/st_archive.sh   Tools/
+${COPY} ${DARTroot}/models/cam/shell_scripts/assimilate.csh  assimilate.csh
+${COPY} ${DARTroot}/models/cam/work/input.nml                input.nml
 
 # ==============================================================================
 # Stage the DART executables in the CESM execution root directory: EXEROOT
@@ -646,7 +636,7 @@
    if ( $status != 0 ) then
       echo "ERROR: ${DARTroot}/models/cam/work/${FILE} not copied to ${exeroot}"
       echo "ERROR: ${DARTroot}/models/cam/work/${FILE} not copied to ${exeroot}"
-      exit -6
+      exit -7
    endif
 end
 
@@ -667,8 +657,8 @@
 #    echo "You can make one with ${DARTroot}/models/cam/work/fill_inflation_restart"
 #    echo ''
 # endif
-# 
-# 
+#
+#
 # only warn people if a precomputed final_full for this number of instances
 # does not already exist.
 # if (! -f ${DARTroot}/system_simulation/final_full_precomputed_tables/final_full.${num_instances}) then
@@ -690,9 +680,9 @@
 echo "Time to check the case."
 echo ''
 echo "cd into ${caseroot}"
-echo 'Modify what you like in input.nml, make sure the observation directory'
-echo 'names set in assimilate.csh match those on your system, and submit'
-echo 'the CESM job by running:'
-echo "./$case.submit"
+echo "Modify what you like in input.nml, make sure the observation directory"
+echo "names set in assimilate.csh match those on your system, and submit"
+echo "the CESM job by running:"
+echo "./${case}.submit"
 echo ''
 

Modified: DART/branches/development/models/cam/shell_scripts/assimilate.csh
===================================================================
--- DART/branches/development/models/cam/shell_scripts/assimilate.csh	2013-03-05 17:40:05 UTC (rev 5983)
+++ DART/branches/development/models/cam/shell_scripts/assimilate.csh	2013-03-05 17:50:17 UTC (rev 5984)
@@ -86,7 +86,6 @@
 
 #-----------------------------------------------------------------------------
 # Get observation sequence file ... or die right away.
-# Cannot specify -f on the link command and still check status.
 # The observation file names have a time that matches the stopping time of CAM.
 #-----------------------------------------------------------------------------
 
@@ -370,9 +369,9 @@
 if ( $?LSB_PJL_TASK_GEOMETRY ) then
    setenv ORIGINAL_LAYOUT "${LSB_PJL_TASK_GEOMETRY}"
 
-   # setenv NANCY_GEOMETRY_54_1NODE \
-   #    "{(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53)}";
-   # setenv LSB_PJL_TASK_GEOMETRY "${NANCY_GEOMETRY_54_1NODE}"
+   # setenv GEOMETRY_32_1NODE \
+   #    "{(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31)}";
+   # setenv LSB_PJL_TASK_GEOMETRY "${GEOMETRY_32_1NODE}"
 endif
 
 echo "`date` -- BEGIN FILTER"
@@ -451,14 +450,10 @@
 
    set n4 = `printf %04d $member`
 
-   set LND_POINTER_FILENAME = `printf rpointer.lnd_%04d ${member}`
-   set ICE_POINTER_FILENAME = `printf rpointer.ice_%04d ${member}`
-   # the ATM pointer file points to a restart file - which we cannot use
+   set LND_RESTART_FILENAME = `printf ${MYCASE}.clm2_%04d.r.${ATM_DATE_EXT}.nc ${member}`
+   set ICE_RESTART_FILENAME = `printf ${MYCASE}.cice_%04d.r.${ATM_DATE_EXT}.nc ${member}`
+   set ATM_INITIAL_FILENAME = `printf ${MYCASE}.cam_%04d.i.${ATM_DATE_EXT}.nc  ${member}`
 
-   set LND_RESTART_FILENAME = `head -n 1 ${LND_POINTER_FILENAME}`
-   set ICE_RESTART_FILENAME = `head -n 1 ${ICE_POINTER_FILENAME}`
-   set ATM_INITIAL_FILENAME = `printf ${MYCASE}.cam_%04d.i.${ATM_DATE_EXT}.nc ${member}`
-
    ${LINK} ${LND_RESTART_FILENAME} clm_restart_${n4}.nc || exit -9
    ${LINK} ${ICE_RESTART_FILENAME} ice_restart_${n4}.nc || exit -9
    ${LINK} ${ATM_INITIAL_FILENAME} cam_initial_${n4}.nc || exit -9
@@ -469,6 +464,9 @@
 
 #-------------------------------------------------------------------------
 # We have to communicate the current model time to the env_run.xml script
+# CAM is running in a 'perpetual startup' mode, so it constantly needs new
+# start time information. The other models are running 'restart', so they
+# keep track of it internally.
 #-------------------------------------------------------------------------
 
 cd ${CASEROOT}
@@ -479,20 +477,21 @@
 ./xmlchange RUN_STARTDATE=${YYYYMMDD}
 ./xmlchange START_TOD=${SSSSS}
 
-cd ${RUNDIR}
-
+#-------------------------------------------------------------------------
 # we (DART) do not need these files, and CESM does not need them either
 # to continue a run.  if we remove them here they do not get moved to
 # the short-term archiver.
+#-------------------------------------------------------------------------
 
-${REMOVE} ../*.rs.*
-${REMOVE} ../*.rh0.*
-${REMOVE} ../*.rs1.*
-${REMOVE} ../PET*ESMF_LogFile
+${REMOVE} ${RUNDIR}/*.rs.*
+${REMOVE} ${RUNDIR}/*.rh0.*
+${REMOVE} ${RUNDIR}/*.rs1.*
+${REMOVE} ${RUNDIR}/PET*ESMF_LogFile
 
 #-------------------------------------------------------------------------
 # Cleanup
 #-------------------------------------------------------------------------
+
 echo "finished assimilate script at "`date`
 echo "`date` -- END ASSIMILATE"
 

Modified: DART/branches/development/models/clm/shell_scripts/CESM1_1_setup.csh
===================================================================
--- DART/branches/development/models/clm/shell_scripts/CESM1_1_setup.csh	2013-03-05 17:40:05 UTC (rev 5983)
+++ DART/branches/development/models/clm/shell_scripts/CESM1_1_setup.csh	2013-03-05 17:50:17 UTC (rev 5984)
@@ -11,9 +11,10 @@
 # ---------------------
 #
 # This script is designed to configure and build a multi-instance CESM model
-# that has CAM, CLM, and CICE as active components over a single data ocean,
+# that has CLM running under a set of data atmospheres (from a DART/CAM assim).
 # and will use DART to assimilate observations at regular intervals.
-# This script does not build DART.
+# This script does not build DART. It works best if the appropriate DART
+# executables have been built, however.
 #
 # This script relies heavily on the information in:
 # http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/book1.html
@@ -31,8 +32,11 @@
 # -- Examine the whole script to identify things to change for your experiments.

@@ Diff output truncated at 40000 characters. @@


More information about the Dart-dev mailing list