[Dart-dev] [5961] DART/branches/development/models/POP/shell_scripts/ CESM1_1_setup_hybrid.csh: Assimilates ...

nancy at ucar.edu nancy at ucar.edu
Fri Feb 15 16:01:35 MST 2013


Revision: 5961
Author:   thoar
Date:     2013-02-15 16:01:34 -0700 (Fri, 15 Feb 2013)
Log Message:
-----------
Assimilates ... need to work on the short-term archiver only.

Might be one cleaner assimilate.csh:dart_to_pop section.

Modified Paths:
--------------
    DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh

-------------- next part --------------
Modified: DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh
===================================================================
--- DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh	2013-02-14 23:44:10 UTC (rev 5960)
+++ DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh	2013-02-15 23:01:34 UTC (rev 5961)
@@ -50,7 +50,9 @@
 #
 # and you may be able to salvage something with
 # ./cesm_setup -clean
-# ./${CASENAME}.clean_build
+# ./cesm_setup
+# ./${case}.clean_build
+# ./${case}.build
 #
 # ==============================================================================
 # ====  Set case options
@@ -61,11 +63,11 @@
 #    script names; so consider it's length and information content.
 # num_instances:  Number of ensemble members
 
-setenv case                 pop_hybrid
+setenv case                 pop_hybrid_opt
 setenv compset              GIAF
 setenv cesmtag              cesm1_1_1
 setenv resolution           T62_gx1v6
-setenv num_instances        4
+setenv num_instances        30
 
 # ==============================================================================
 # define machines and directories
@@ -131,7 +133,7 @@
 
 setenv ACCOUNT      P86850054
 setenv timewall     0:30
-setenv queue        premium
+setenv queue        regular
 setenv ptile        15
 
 # ==============================================================================
@@ -178,7 +180,7 @@
 
    if ( $status != 0 ) then
       echo "ERROR: Case could not be created."
-      exit 1
+      exit -1
    endif
 
 # ==============================================================================
@@ -192,34 +194,56 @@
    ${COPY} $FILE ${FILE}.original
 end
 
-# This is only for the purpose of debugging the code.
-# A more efficient layout must be done when running a full assimilation.
+# Try to lay out the tasks in some sort of reasonable way.
+# The ocean root pe is the only one that is not zero.
 
-# 1/8 cpl, 1/8 cice, 1/8 atm, 1/8 lnd, 4/8 ocean
+if ($num_instances == 4) then
 
-@ total_nt = 128
-@ atm_pes  = $total_nt / 8
-@ cpl_pes  = $total_nt / 8
-@ ice_pes  = $total_nt / 8
-@ lnd_pes  = $total_nt / 8
-@ ocn_pes  = $total_nt - ($cpl_pes + $ice_pes + $lnd_pes + $atm_pes)
+   # This is only for the purpose of debugging the code.
+   # A more efficient layout must be done when running a full assimilation.
 
-echo "task partitioning ..."
+   @ atm_pes = $ptile
+   @ cpl_pes = $ptile
+   @ ice_pes = $ptile
+   @ lnd_pes = $ptile
+   @ glc_pes = $ptile
+   @ rof_pes = $ptile
+   @ ocn_pes = $ptile * 4
+
+else
+
+   # layout for 30 members.
+   # Made in conjunction with Mike Levy, David Bailey and Jim Edwards.
+
+   @ atm_pes = 450
+   @ cpl_pes = 450
+   @ ice_pes = 450
+   @ lnd_pes = 450
+   @ glc_pes = 450
+   @ rof_pes = 450
+   @ ocn_pes = 1800
+
+endif
+
+echo "task partitioning ... ice and ocean run at the same time."
 echo ""
-echo "LND gets $lnd_pes"
-echo "ATM gets $atm_pes"
-echo "ICE gets $ice_pes"
-echo "OCN gets $ocn_pes"
-echo "GLC gets $ocn_pes"
-echo "CPL gets $cpl_pes"
+echo "ATM  gets $atm_pes"
+echo "CPL  gets $cpl_pes"
+echo "ICE  gets $ice_pes"
+echo "LND  gets $lnd_pes"
+echo "GLC  gets $glc_pes"
+echo "DROF gets $rof_pes"
+echo "OCN  gets $ocn_pes"
 echo ""
 
-./xmlchange NTHRDS_ATM=1,NTASKS_ATM=$atm_pes,NINST_ATM=$num_instances
-./xmlchange NTHRDS_ICE=1,NTASKS_ICE=$ice_pes,NINST_ICE=$num_instances
-./xmlchange NTHRDS_OCN=1,NTASKS_OCN=$ocn_pes,NINST_OCN=$num_instances
-./xmlchange NTHRDS_GLC=1,NTASKS_GLC=$ocn_pes,NINST_GLC=1
-./xmlchange NTHRDS_LND=1,NTASKS_LND=$lnd_pes,NINST_LND=1
-./xmlchange NTHRDS_CPL=1,NTASKS_CPL=$cpl_pes
+./xmlchange ROOTPE_ATM=0,NTHRDS_ATM=1,NTASKS_ATM=$atm_pes,NINST_ATM=$num_instances
+./xmlchange ROOTPE_ICE=0,NTHRDS_ICE=1,NTASKS_ICE=$ice_pes,NINST_ICE=$num_instances
+./xmlchange ROOTPE_GLC=0,NTHRDS_GLC=1,NTASKS_GLC=$glc_pes,NINST_GLC=1
+./xmlchange ROOTPE_LND=0,NTHRDS_LND=1,NTASKS_LND=$lnd_pes,NINST_LND=1
+./xmlchange ROOTPE_ROF=0,NTHRDS_ROF=1,NTASKS_ROF=$rof_pes,NINST_ROF=1
+./xmlchange ROOTPE_CPL=0,NTHRDS_CPL=1,NTASKS_CPL=$cpl_pes
+./xmlchange              NTHRDS_OCN=1,NTASKS_OCN=$ocn_pes,NINST_OCN=$num_instances
+./xmlchange ROOTPE_OCN=$ice_pes
 
 # http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/c1158.html#run_start_stop
 # "A hybrid run indicates that CESM is initialized more like a startup, but uses
@@ -242,15 +266,20 @@
 ./xmlchange RUN_REFTOD=$run_reftod
 ./xmlchange BRNCH_RETAIN_CASENAME=FALSE
 ./xmlchange GET_REFCASE=FALSE
-./xmlchange CALENDAR=GREGORIAN
 ./xmlchange EXEROOT=${exeroot}
 
+# The streams files were generated with a NO_LEAP calendar in mind.
+# We need to test these with a GREGORIAN calendar.
+
+./xmlchange CALENDAR=GREGORIAN
+
 ./xmlchange STOP_OPTION=$stop_option
 ./xmlchange STOP_N=$stop_n
 ./xmlchange CONTINUE_RUN=FALSE
 ./xmlchange RESUBMIT=$resubmit
+./xmlchange PIO_TYPENAME=pnetcdf
 
-./xmlchange DOUT_S=FALSE
+./xmlchange DOUT_S=TRUE
 ./xmlchange DOUT_S_ROOT=${archdir}
 ./xmlchange DOUT_S_SAVE_INT_REST_FILES=FALSE
 ./xmlchange DOUT_L_MS=FALSE
@@ -265,14 +294,9 @@
 
 # level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
 
-./xmlchange DEBUG=TRUE
-./xmlchange INFO_DBUG=3
+./xmlchange DEBUG=FALSE
+./xmlchange INFO_DBUG=0
 
-# The river transport model ON is useful only when using an active ocean or
-# land surface diagnostics. Setting ROF_GRID to 'null' turns off the RTM.
-
-#./xmlchange ROF_GRID='null'
-
 # ==============================================================================
 # Set up the case.
 # This creates the EXEROOT and RUNDIR directories.
@@ -282,7 +306,7 @@
 
 if ( $status != 0 ) then
    echo "ERROR: Case could not be set up."
-   exit 2
+   exit -2
 endif
 
 # ==============================================================================
@@ -324,7 +348,7 @@
    # init_ts_suboption = 'data_assim'   for non bit-for-bit restarting (assimilation mode)
    # init_ts_suboption = 'null'         for 'perfect' restarting/forecasting
 
-   echo "init_ts_suboption = 'null'" >> user_nl_pop2_$instance
+   echo "init_ts_suboption = 'data_assim'" >> user_nl_pop2_$instance
 
    @ inst ++
 end
@@ -380,8 +404,7 @@
    @ filename_index = $#name_parse - 1
    @ instance_index = $#name_parse
    set streamname = $name_parse[$filename_index]
-   set   instance = $name_parse[$instance_index]
-   set          n = `printf %d $instance`
+   set    n = `echo $name_parse[$instance_index] | sed 's/0*//'`
 
    if (-e $DARTroot/models/POP/shell_scripts/user_$streamname*template) then
 
@@ -395,7 +418,7 @@
    else
       echo "DIED Looking for a DART stream txt template for $FNAME"
       echo "DIED Looking for a DART stream txt template for $FNAME"
-      exit 3
+      exit -3
    endif
 
 end
@@ -411,10 +434,17 @@
 #    mods and put in the SourceMods subdirectory found in the 'case' directory.
 # ==============================================================================
 
-if ( -d ~/${cesmtag}/SourceMods ) then
+if (    -d     ~/${cesmtag}/SourceMods ) then
    ${COPY} -r  ~/${cesmtag}/SourceMods/* ${caseroot}/SourceMods/
 else
-   echo "FYI - No SourceMods for this case"
+   echo "ERROR - No SourceMods for this case."
+   echo "ERROR - No SourceMods for this case."
+   echo "DART requires modifications to several src.pop2/ files."
+   echo "These files can be downloaded from:"
+   echo "http://www.image.ucar.edu/pub/DART/CESM/DART_SourceMods_cesm1_1_1.tar"
+   echo "untar these into your HOME directory - they will create a"
+   echo "~/cesm_1_1_1  directory with the appropriate SourceMods structure."
+   exit -4
 endif
 
 # ==============================================================================
@@ -429,14 +459,16 @@
 
 if ( $status != 0 ) then
    echo "ERROR: Case could not be built."
-   exit 4
+   exit -5
 endif
 
 # ==============================================================================
 # Stage the restarts now that the run directory exists
+#
+# set stagedir = /glade/scratch/thoar/DART_POP_RESTARTS/2004-01-01-00000
 # ==============================================================================
 
-set stagedir = /glade/scratch/thoar/DART_POP_RESTARTS/2004-01-01-00000
+set stagedir = /glade/p/work/aliciak/DART_IC/CCSM4_ensembles/rest/2004-01-01-00000
 
 echo "Copying the restart files from ${stagedir}"
 
@@ -445,6 +477,7 @@
    set n4 = `printf %04d $i`
    set n2 = `printf %02d $i`
 
+   echo ''
    echo "Staging restarts for instance $i of $num_instances"
    #AK: Note that the pop ocean must have the .hdr file to describe the
    #    metadata for binary restarts.
@@ -456,9 +489,6 @@
    ${COPY} ${stagedir}/b40.20th.005_ens${n2}.pop.ro.2004-01-01-00000    ${rundir}
    ${COPY} ${stagedir}/b40.20th.005_ens${n2}.cice.r.2004-01-01-00000.nc ${rundir}
 
-   #${COPY} ${stagedir}/rpointer.atm_${n4}                               ${rundir}
-   echo ""
-
    @ i ++
 
 end
@@ -478,16 +508,12 @@
 switch ( $BATCH )
    case bsub*:
       # NCAR "bluefire", "yellowstone"
-      sed s/ptile=32/ptile=$ptile/ < ${case}.run >! temp.$$
-      ${MOVE} temp.$$  ${case}.run
-
       set TIMEWALL=`grep BSUB ${case}.run | grep -e '-W' `
-      sed s/$TIMEWALL[3]/$timewall/ < ${case}.run >! temp.$$
+      set    QUEUE=`grep BSUB ${case}.run | grep -e '-q' `
+      sed -e "s/ptile=32/ptile=$ptile/" \
+          -e "s/$TIMEWALL[3]/$timewall/" \
+          -e "s/$QUEUE[3]/$queue/" < ${case}.run >! temp.$$
       ${MOVE} temp.$$  ${case}.run
-
-      set QUEUE=`grep BSUB ${case}.run | grep -e '-q' `
-      sed s/$QUEUE[3]/$queue/ < ${case}.run >! temp.$$
-      ${MOVE} temp.$$  ${case}.run
    breaksw
 
    default:
@@ -513,13 +539,12 @@
 # -------------------------------------------------------------------------
 # START OF DART: if CESM finishes correctly (pirated from ccsm_postrun.csh);
 # perform an assimilation with DART.
-# -------------------------------------------------------------------------
 
 set CplLogFile = `ls -1t cpl.log* | head -n 1`
 if ($CplLogFile == "") then
    echo 'ERROR: Model did not complete - no cpl.log file present - exiting.'
    echo 'ERROR: Assimilation will not be attempted.'
-   exit 1
+   exit -1
 endif
 
 grep 'SUCCESSFUL TERMINATION' $CplLogFile
@@ -530,12 +555,12 @@
       echo "`date` -- DART HAS FINISHED"
    else
       echo "`date` -- DART FILTER ERROR - ABANDON HOPE"
-      exit 3
+      exit -3
    endif
 else
    echo 'ERROR: Model did not complete successfully - exiting.'
    echo 'ERROR: Assimilation will not be attempted.'
-   exit 2
+   exit -2
 endif
 
 # END OF DART BLOCK
@@ -563,7 +588,7 @@
    cat                add_to_run.txt >> temp.$$
    tail -n $lastlines ${case}.run    >> temp.$$
 
-#  ${MOVE} temp.$$ ${case}.run    TJH DEBUG
+   ${MOVE} temp.$$ ${case}.run
 
 endif
 
@@ -602,7 +627,7 @@
    if ( $status != 0 ) then
       echo "ERROR: ${DARTroot}/models/POP/work/${FILE} not copied to ${exeroot}"
       echo "ERROR: ${DARTroot}/models/POP/work/${FILE} not copied to ${exeroot}"
-      exit 3
+      exit -3
    endif
 end
 
@@ -611,23 +636,24 @@
 # ==============================================================================
 
 echo ''
-echo 'Time to check the case.'
+echo "Time to check the case."
+echo ''
 echo "cd into ${caseroot}"
-echo 'Modify what you like in input.nml, make sure the observation directory'
-echo 'names set in assimilate.csh match those on your system, and submit'
-echo 'the CESM job by running:'
+echo "Modify what you like in input.nml, make sure the observation directory"
+echo "names set in assimilate.csh match those on your system, and submit"
+echo "the CESM job by running:"
 echo "./${case}.submit"
 echo ''
+echo "For continued submissions after the initial (hybrid) startup,"
+echo "make the following changes to the env_run variables:"
+echo ''
+echo "  ./xmlchange -file env_run.xml -id STOP_N        -val 1"
+echo "  ./xmlchange -file env_run.xml -id CONTINUE_RUN  -val TRUE"
+echo "  ./xmlchange -file env_run.xml -id RESUBMIT      -val <your_favorite_number>"
+echo ''
+echo "Check the streams listed in the streams text files.  If more or different"
+echo 'dates need to be added, then do this in the $CASEROOT/user_*files*'
+echo "then invoke 'preview_namelists' so you can check the information in the"
+echo "CaseDocs or ${rundir} directories."
+echo ''
 
-# ==============================================================================
-# for continued submissions after the initial startup one,
-# make the following changes to the env_run variables:
-#
-#    ./xmlchange -file env_run.xml -id STOP_N        -val 1
-#    ./xmlchange -file env_run.xml -id RESUBMIT      -val <your_favorite_number>
-#
-# Check the streams listed in the streams text files.  If more or different
-# dates need to be added, then do this in the $CASEROOT/user_*files*
-# then invoke "preview_namelists" for this information to be moved to
-# the CaseDocs and the executable directory.
-# ==============================================================================


More information about the Dart-dev mailing list