[Dart-dev] [5984] DART/branches/development/models: The setup and assimilate scripts are now VERY similar across all
nancy at ucar.edu
nancy at ucar.edu
Tue Mar 5 10:50:18 MST 2013
Revision: 5984
Author: thoar
Date: 2013-03-05 10:50:17 -0700 (Tue, 05 Mar 2013)
Log Message:
-----------
The setup and assimilate scripts are now VERY similar across all
model components.
The CLM setup script could use a block specifying the pe layout
for a 30 member ensemble ...
The save intermediate restart options for CESM are untested
and some of the model components do not have a tested 'st_archive.sh'
but will soon. Since it is not required ... it is better to save
what we have now.
Modified Paths:
--------------
DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh
DART/branches/development/models/POP/shell_scripts/assimilate.csh
DART/branches/development/models/cam/shell_scripts/CESM_setup_startup.csh
DART/branches/development/models/cam/shell_scripts/assimilate.csh
DART/branches/development/models/clm/shell_scripts/CESM1_1_setup.csh
DART/branches/development/models/clm/shell_scripts/assimilate.csh
DART/branches/development/models/clm/work/input.nml
-------------- next part --------------
Modified: DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh
===================================================================
--- DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh 2013-03-05 17:40:05 UTC (rev 5983)
+++ DART/branches/development/models/POP/shell_scripts/CESM1_1_setup_hybrid.csh 2013-03-05 17:50:17 UTC (rev 5984)
@@ -13,7 +13,8 @@
# This script is designed to configure and build a multi-instance CESM model
# that has POP as active components
# and will use DART to assimilate observations at regular intervals.
-# This script does not build DART.
+# This script does not build DART. It works best if the appropriate DART
+# executables have been built, however.
#
# This script relies heavily on the information in:
# http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/book1.html
@@ -47,13 +48,7 @@
# For the brave, read
#
# http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/x1142.html
-#
-# and you may be able to salvage something with
-# ./cesm_setup -clean
-# ./cesm_setup
-# ./${case}.clean_build
-# ./${case}.build
-#
+
# ==============================================================================
# ==== Set case options
# ==============================================================================
@@ -63,11 +58,11 @@
# script names; so consider it's length and information content.
# num_instances: Number of ensemble members
-setenv case pop_hybrid_opt
+setenv case pop_test
setenv compset GIAF
setenv cesmtag cesm1_1_1
setenv resolution T62_gx1v6
-setenv num_instances 30
+setenv num_instances 4
# ==============================================================================
# define machines and directories
@@ -75,9 +70,9 @@
# mach Computer name
# cesm_datadir Root path of the public CESM data files
# cesmroot Location of the cesm code base
-# For cesm1_1 on yellowstone
+# i.e. cesm1_1_1 on yellowstone
# DARTroot Location of DART code tree.
-# Executables, scripts and input in $DARTroot/models/dev/...
+# Executables, scripts and input in $DARTroot/models/POP/...
# caseroot Your (future) cesm case directory, where this CESM+DART will be built.
# Preferably not a frequently scrubbed location.
# This script will delete any existing caseroot, so this script,
@@ -99,7 +94,7 @@
setenv DARTroot /glade/u/home/${USER}/svn/DART/dev
# ==============================================================================
-# configure settings
+# configure settings ... run_startdate format is yyyy-mm-dd
# ==============================================================================
setenv stream_year_first 2004
@@ -131,7 +126,7 @@
# TJH: How many T62_gx1v6 POP instances can fit on 1 node?
# ==============================================================================
-setenv ACCOUNT P86850054
+setenv ACCOUNT P8685nnnn
setenv timewall 0:30
setenv queue regular
setenv ptile 15
@@ -157,6 +152,7 @@
set COPY = '/bin/cp -fv --preserve=timestamps'
set LINK = '/bin/ln -fvs'
set REMOVE = '/bin/rm -fr'
+ set nonomatch
breaksw
endsw
@@ -191,7 +187,9 @@
# Save a copy for debug purposes
foreach FILE ( *xml )
- ${COPY} $FILE ${FILE}.original
+ if ( ~ -e ${FILE}.original ) then
+ ${COPY} $FILE ${FILE}.original
+ endif
end
# Try to lay out the tasks in some sort of reasonable way.
@@ -202,46 +200,46 @@
# This is only for the purpose of debugging the code.
# A more efficient layout must be done when running a full assimilation.
- @ atm_pes = $ptile
@ cpl_pes = $ptile
- @ ice_pes = $ptile
+ @ atm_pes = $ptile * $num_instances
+ @ ice_pes = $ptile * $num_instances
@ lnd_pes = $ptile
+ @ rof_pes = $ptile
@ glc_pes = $ptile
- @ rof_pes = $ptile
- @ ocn_pes = $ptile * 4
+ @ ocn_pes = $ptile * $num_instances
else
# layout for 30 members.
# Made in conjunction with Mike Levy, David Bailey and Jim Edwards.
+ @ cpl_pes = 450
@ atm_pes = 450
- @ cpl_pes = 450
@ ice_pes = 450
@ lnd_pes = 450
+ @ rof_pes = 450
@ glc_pes = 450
- @ rof_pes = 450
@ ocn_pes = 1800
endif
echo "task partitioning ... ice and ocean run at the same time."
echo ""
-echo "ATM gets $atm_pes"
-echo "CPL gets $cpl_pes"
-echo "ICE gets $ice_pes"
-echo "LND gets $lnd_pes"
-echo "GLC gets $glc_pes"
-echo "DROF gets $rof_pes"
-echo "OCN gets $ocn_pes"
+echo "CPL gets $cpl_pes"
+echo "ATM gets $atm_pes"
+echo "ICE gets $ice_pes"
+echo "LND gets $lnd_pes"
+echo "ROF gets $rof_pes"
+echo "GLC gets $glc_pes"
+echo "OCN gets $ocn_pes"
echo ""
+./xmlchange ROOTPE_CPL=0,NTHRDS_CPL=1,NTASKS_CPL=$cpl_pes
./xmlchange ROOTPE_ATM=0,NTHRDS_ATM=1,NTASKS_ATM=$atm_pes,NINST_ATM=$num_instances
./xmlchange ROOTPE_ICE=0,NTHRDS_ICE=1,NTASKS_ICE=$ice_pes,NINST_ICE=$num_instances
-./xmlchange ROOTPE_GLC=0,NTHRDS_GLC=1,NTASKS_GLC=$glc_pes,NINST_GLC=1
./xmlchange ROOTPE_LND=0,NTHRDS_LND=1,NTASKS_LND=$lnd_pes,NINST_LND=1
./xmlchange ROOTPE_ROF=0,NTHRDS_ROF=1,NTASKS_ROF=$rof_pes,NINST_ROF=1
-./xmlchange ROOTPE_CPL=0,NTHRDS_CPL=1,NTASKS_CPL=$cpl_pes
+./xmlchange ROOTPE_GLC=0,NTHRDS_GLC=1,NTASKS_GLC=$glc_pes,NINST_GLC=1
./xmlchange NTHRDS_OCN=1,NTASKS_OCN=$ocn_pes,NINST_OCN=$num_instances
./xmlchange ROOTPE_OCN=$ice_pes
@@ -267,20 +265,13 @@
./xmlchange BRNCH_RETAIN_CASENAME=FALSE
./xmlchange GET_REFCASE=FALSE
./xmlchange EXEROOT=${exeroot}
+./xmlchange PIO_TYPENAME=pnetcdf
-# The streams files were generated with a NO_LEAP calendar in mind.
-# We need to test these with a GREGORIAN calendar.
+# DOUT_S is to turn on/off the short-term archiving
+# DOUT_L_MS is to store to the HPSS (formerly "MSS")
-./xmlchange CALENDAR=GREGORIAN
-
-./xmlchange STOP_OPTION=$stop_option
-./xmlchange STOP_N=$stop_n
-./xmlchange CONTINUE_RUN=FALSE
-./xmlchange RESUBMIT=$resubmit
-./xmlchange PIO_TYPENAME=pnetcdf
-
+./xmlchange DOUT_S_ROOT=${archdir}
./xmlchange DOUT_S=TRUE
-./xmlchange DOUT_S_ROOT=${archdir}
./xmlchange DOUT_S_SAVE_INT_REST_FILES=FALSE
./xmlchange DOUT_L_MS=FALSE
./xmlchange DOUT_L_MSROOT="csm/${case}"
@@ -292,6 +283,22 @@
./xmlchange DATM_CPLHIST_YR_START=$refyear
./xmlchange DATM_CPLHIST_YR_END=$refyear
+# The streams files were generated with a NO_LEAP calendar in mind.
+# We need to test these with a GREGORIAN calendar.
+
+./xmlchange CALENDAR=GREGORIAN
+./xmlchange STOP_OPTION=$stop_option
+./xmlchange STOP_N=$stop_n
+./xmlchange CONTINUE_RUN=FALSE
+./xmlchange RESUBMIT=$resubmit
+
+# The river transport model ON is useful only when using an active ocean or
+# land surface diagnostics. Setting ROF_GRID to 'null' turns off the RTM.
+# TJH - guidance needed from Alicia
+
+# ./xmlchange ROF_GRID='null'
+# ./xmlchange ROF_GRID='r05'
+
# level of debug output, 0=minimum, 1=normal, 2=more, 3=too much, valid values: 0,1,2,3 (integer)
./xmlchange DEBUG=FALSE
@@ -326,7 +333,9 @@
set instance = `printf %04d $inst`
set instance2 = `printf %02d $inst`
+ # ===========================================================================
set fname = "user_nl_datm_$instance"
+ # ===========================================================================
echo "streams = 'datm.streams.txt.CPLHIST3HrWx.Solar_$instance $stream_year_align $stream_year_first $stream_year_last'," >> $fname
echo " 'datm.streams.txt.CPLHIST3HrWx.nonSolarNonPrecip_$instance $stream_year_align $stream_year_first $stream_year_last'" >> $fname
@@ -340,15 +349,23 @@
echo "restfils = 'unset'" >> $fname
echo "restfilm = 'unset'" >> $fname
+ # ===========================================================================
+ set fname = "user_nl_cice_$instance"
+ # ===========================================================================
# CICE Namelists
# this is only used for a hybrid start, else rpointers are used.
- echo "ice_ic = 'b40.20th.005_ens"$instance2".cice.r.2004-01-01-00000.nc'" >> user_nl_cice_$instance
+ echo "ice_ic = 'b40.20th.005_ens"$instance2".cice.r.2004-01-01-00000.nc'" >> $fname
+
+ # ===========================================================================
+ set fname = "user_nl_pop2_$instance"
+ # ===========================================================================
+
# POP Namelists
# init_ts_suboption = 'data_assim' for non bit-for-bit restarting (assimilation mode)
# init_ts_suboption = 'null' for 'perfect' restarting/forecasting
- echo "init_ts_suboption = 'data_assim'" >> user_nl_pop2_$instance
+ echo "init_ts_suboption = 'data_assim'" >> $fname
@ inst ++
end
@@ -470,15 +487,19 @@
set stagedir = /glade/p/work/aliciak/DART_IC/CCSM4_ensembles/rest/2004-01-01-00000
+echo ''
echo "Copying the restart files from ${stagedir}"
+echo 'into the CESM run directory.'
+echo ''
-@ i = 1
-while ($i <= $num_instances)
- set n4 = `printf %04d $i`
- set n2 = `printf %02d $i`
+@ inst = 1
+while ($inst <= $num_instances)
+ set n4 = `printf %04d $inst`
+ set n2 = `printf %02d $inst`
echo ''
- echo "Staging restarts for instance $i of $num_instances"
+ echo "Staging restarts for instance $inst of $num_instances"
+
#AK: Note that the pop ocean must have the .hdr file to describe the
# metadata for binary restarts.
${COPY} ${stagedir}/rpointer.ocn_${n4}.restart ${rundir}
@@ -489,10 +510,13 @@
${COPY} ${stagedir}/b40.20th.005_ens${n2}.pop.ro.2004-01-01-00000 ${rundir}
${COPY} ${stagedir}/b40.20th.005_ens${n2}.cice.r.2004-01-01-00000.nc ${rundir}
- @ i ++
-
+ @ inst ++
end
+if ( -e ${stagedir}/prior_inflate_restart* ) then
+ ${COPY} ${stagedir}/prior_inflate_restart* ${rundir}/.
+endif
+
# ==============================================================================
# Edit the run script to reflect project, queue, and wallclock
# ==============================================================================
@@ -501,7 +525,9 @@
echo 'Updating the run script to set wallclock and queue.'
echo ''
-${COPY} ${case}.run ${case}.run.orig
+if ( ~ -e ${case}.run.original ) then
+ ${COPY} ${case}.run ${case}.run.original
+endif
source Tools/ccsm_getenv
set BATCH = `echo $BATCHSUBMIT | sed 's/ .*$//'`
@@ -510,7 +536,7 @@
# NCAR "bluefire", "yellowstone"
set TIMEWALL=`grep BSUB ${case}.run | grep -e '-W' `
set QUEUE=`grep BSUB ${case}.run | grep -e '-q' `
- sed -e "s/ptile=32/ptile=$ptile/" \
+ sed -e "s/ptile=[0-9][0-9]*/ptile=$ptile/" \
-e "s/$TIMEWALL[3]/$timewall/" \
-e "s/$QUEUE[3]/$queue/" < ${case}.run >! temp.$$
${MOVE} temp.$$ ${case}.run
@@ -588,33 +614,37 @@
cat add_to_run.txt >> temp.$$
tail -n $lastlines ${case}.run >> temp.$$
- ${MOVE} temp.$$ ${case}.run
+ ${MOVE} temp.$$ ${case}.run
+ ${REMOVE} add_to_run.txt
+else
+ echo "ERROR in grep of ${case}.run: aborting"
+ echo "status was ${STATUSCHECK}"
+ exit -6
endif
-chmod 0744 ${case}.run
+chmod 0755 ${case}.run
# ==============================================================================
# Stage the required parts of DART in the CASEROOT directory.
# ==============================================================================
-# AK: the standard CESM short-term archiving script may need to be altered
+# The standard CESM short-term archiving script may need to be altered
# to archive addtional or subsets of things, or to reduce the amount of
-# data that is sent to the long-term archive. Put a version of st_archive.sh
-# in ${DARTroot}/models/POP/shell_scripts when/if necessary
-#
-if ( ~ -e Tools/st_archive.sh.orig ) then
- ${MOVE} Tools/st_archive.sh Tools/st_archive.sh.orig
-else
- echo "a Tools/st_archive.sh backup copy already exists"
+# data that is sent to the long-term archive.
+
+if ( ~ -e Tools/st_archive.sh.original ) then
+ ${COPY} Tools/st_archive.sh Tools/st_archive.sh.original
endif
-# AK note: you must use this assimilate.csh script.
-# It has hardcoded variables that point to the location of
-# the observations sequence files and the DART working
-# directory and the PE layout for DART.
+# NOTE: the assimilate.csh script and input.nml must be modified for your
+# situation. The script has variables that point to the location of
+# the observations sequence files and the DART working directory
+# and may be customized for a more efficient PE layout for DART.
+# If you are running this, you know what to do with input.nml.
+# If you don't, you should give up now. Really.
-# ${COPY} ${DARTroot}/models/POP/shell_scripts/st_archive.sh Tools/ TJH DEBUG
+${COPY} ${DARTroot}/models/POP/shell_scripts/st_archive.sh Tools/
${COPY} ${DARTroot}/models/POP/shell_scripts/assimilate.csh assimilate.csh
${COPY} ${DARTroot}/models/POP/work/input.nml input.nml
@@ -627,7 +657,7 @@
if ( $status != 0 ) then
echo "ERROR: ${DARTroot}/models/POP/work/${FILE} not copied to ${exeroot}"
echo "ERROR: ${DARTroot}/models/POP/work/${FILE} not copied to ${exeroot}"
- exit -3
+ exit -7
endif
end
Modified: DART/branches/development/models/POP/shell_scripts/assimilate.csh
===================================================================
--- DART/branches/development/models/POP/shell_scripts/assimilate.csh 2013-03-05 17:40:05 UTC (rev 5983)
+++ DART/branches/development/models/POP/shell_scripts/assimilate.csh 2013-03-05 17:50:17 UTC (rev 5984)
@@ -1,4 +1,4 @@
-#!/bin/tcsh
+#!/bin/csh
#
# DART software - Copyright 2004 - 2011 UCAR. This open source software is
# provided by UCAR, "as is", without charge, subject to all terms of use at
@@ -19,8 +19,7 @@
# the VERBOSE options are useful for debugging.
set MOVE = '/usr/local/bin/mv -fv'
set COPY = '/usr/local/bin/cp -fv --preserve=timestamps'
- set FLINK = '/usr/local/bin/ln -fvs'
- set LINK = '/usr/local/bin/ln -vs'
+ set LINK = '/usr/local/bin/ln -fvs'
set REMOVE = '/usr/local/bin/rm -fr'
set BASEOBSDIR = /glade/proj3/image/Observations/WOD09
@@ -34,8 +33,7 @@
# the VERBOSE options are useful for debugging.
set MOVE = 'mv -fv'
set COPY = 'cp -fv --preserve=timestamps'
- set FLINK = 'ln -fvs'
- set LINK = 'ln -vs'
+ set LINK = 'ln -fvs'
set REMOVE = 'rm -fr'
set BASEOBSDIR = /glade/p/image/Observations/WOD09
@@ -47,8 +45,7 @@
# NERSC "hopper"
set MOVE = 'mv -fv'
set COPY = 'cp -fv --preserve=timestamps'
- set FLINK = 'ln -fvs'
- set LINK = 'ln -vs'
+ set LINK = 'ln -fvs'
set REMOVE = 'rm -fr'
set BASEOBSDIR = /scratch/scratchdirs/nscollin/ACARS
@@ -73,7 +70,7 @@
#-------------------------------------------------------------------------
# Determine time of model state ... from file name of first member
-# of the form "./${CASE}.pop.$ensemble_member.r.2000-01-06-00000.nc"
+# of the form "./${CASE}.pop_${ensemble_member}.r.2000-01-06-00000.nc"
#
# Piping stuff through 'bc' strips off any preceeding zeros.
#-------------------------------------------------------------------------
@@ -95,22 +92,19 @@
#-----------------------------------------------------------------------------
# Get observation sequence file ... or die right away.
-# Cannot specify -f on the link command and still check status.
# The observation file names have a time that matches the stopping time of POP.
#-----------------------------------------------------------------------------
-set YYYYMM = `printf %04d%02d ${OCN_YEAR} ${OCN_MONTH}`
-set OBSFNAME = `printf obs_seq.0Z.%04d%02d%02d ${OCN_YEAR} ${OCN_MONTH} ${OCN_DAY}`
+set YYYYMMDD = `printf %04d%02d%02d ${OCN_YEAR} ${OCN_MONTH} ${OCN_DAY}`
+set YYYYMM = `printf %04d%02d ${OCN_YEAR} ${OCN_MONTH}`
+set OBSFNAME = obs_seq.0Z.${YYYYMMDD}
set OBS_FILE = ${BASEOBSDIR}/${YYYYMM}/${OBSFNAME}
-${REMOVE} obs_seq.out
-${LINK} ${OBS_FILE} obs_seq.out
-
-set lnstat = $status
-if ($lnstat != 0) then
+if ( -e ${OBS_FILE} ) then
+ ${LINK} ${OBS_FILE} obs_seq.out
+else
echo "ERROR ... no observation file $OBS_FILE"
echo "ERROR ... no observation file $OBS_FILE"
- echo "ERROR ... ln died with status $lnstat"
exit -1
endif
@@ -125,13 +119,12 @@
else
echo "ERROR ... DART required file ${CASEROOT}/input.nml not found ... ERROR"
echo "ERROR ... DART required file ${CASEROOT}/input.nml not found ... ERROR"
- exit -1
+ exit -2
endif
-# Since the obs sequence files are small, modify the DART input.nml such
-# that the num_output_obs_members matches the ensemble size.
-#
-# g;num_output_state_members ;s;= .*;= $ensemble_size;
+# Modify the DART input.nml such that
+# the DART ensemble size matches the CESM number of instances
+# WARNING: the output files contain ALL enemble members ==> BIG
ex input.nml <<ex_end
g;ens_size ;s;= .*;= $ensemble_size;
@@ -166,7 +159,7 @@
else
echo "ERROR: no sampling error correction file for this ensemble size."
echo "ERROR: looking for ${SAMP_ERR_FILE}"
- exit -2
+ exit -3
endif
else
echo "Sampling Error Correction not requested for this assimilation."
@@ -251,11 +244,11 @@
# If one exists, use it as input for this assimilation
if ( $nfiles > 0 ) then
set latest = `cat latestfile`
- ${FLINK} $latest ${PRIOR_INF_IFNAME}
+ ${LINK} $latest ${PRIOR_INF_IFNAME}
else
echo "ERROR: Requested PRIOR inflation but specified no incoming inflation file."
echo "ERROR: expected something like ../${PRIOR_INF_OFNAME}.YYYY-MM-DD-SSSSS"
- exit 4
+ exit -4
endif
endif
@@ -278,11 +271,11 @@
# If one exists, use it as input for this assimilation
if ( $nfiles > 0 ) then
set latest = `cat latestfile`
- ${FLINK} $latest ${POSTE_INF_IFNAME}
+ ${LINK} $latest ${POSTE_INF_IFNAME}
else
echo "ERROR: Requested POSTERIOR inflation but specified no incoming inflation file."
echo "ERROR: expected something like ../${POSTE_INF_OFNAME}.YYYY-MM-DD-SSSSS"
- exit 6
+ exit -5
endif
endif
else
@@ -290,22 +283,20 @@
endif
#=========================================================================
-# Block 4: convert N POP restart files to DART initial conditions file(s)
+# Block 4: Convert N POP restart files to DART initial condition files.
# pop_to_dart is serial code, we can do all of these at the same time
-# and just wait for them to finish IFF it were not for the fact we'd have
-# to have unique namelists for all of them.
+# as long as we can have unique namelists for each of them.
#
-# At the end of the block, we have DART restart files filter_ics.[1-N]
-# that came from pointer files ../rpointer.ocn.[1-N].restart
+# At the end of the block, we have DART initial condition files filter_ics.[1-N]
+# that came from pointer files ../rpointer.ocn_[1-N].restart
#
-# DART namelist settings appropriate/required:
+# REQUIRED DART namelist settings:
# &filter_nml: restart_in_file_name = 'filter_ics'
# restart_out_file_name = 'filter_restart'
# &ensemble_manager_nml: single_restart_file_in = '.false.'
# &pop_to_dart_nml: pop_to_dart_output_file = 'dart_ics',
# &dart_to_pop_nml: dart_to_pop_input_file = 'dart_restart',
-# &dart_to_pop_nml: advance_time_present = .false.
-#
+# advance_time_present = .false.
#=========================================================================
echo "`date` -- BEGIN POP TO DART"
@@ -328,8 +319,8 @@
sed -e "s/dart_ics/..\/${DART_IC_FILENAME}/" \
-e "s/dart_restart/..\/${DART_RESTART_FILE}/" < ../input.nml >! input.nml
- ${FLINK} $OCN_RESTART_FILENAME pop.r.nc
- ${FLINK} $OCN_NML_FILENAME pop_in
+ ${LINK} $OCN_RESTART_FILENAME pop.r.nc
+ ${LINK} $OCN_NML_FILENAME pop_in
echo "starting pop_to_dart for member ${member} at "`date`
${EXEROOT}/pop_to_dart >! output.${member}.pop_to_dart &
@@ -344,7 +335,7 @@
if ($status != 0) then
echo "ERROR ... DART died in 'pop_to_dart' ... ERROR"
echo "ERROR ... DART died in 'pop_to_dart' ... ERROR"
- exit -7
+ exit -6
endif
echo "`date` -- END POP-TO-DART for all ${ensemble_size} members."
@@ -366,7 +357,7 @@
# &filter_nml: first_obs_seconds = -1,
# &filter_nml: last_obs_days = -1,
# &filter_nml: last_obs_seconds = -1,
-# &ensemble_manager_nml: single_restart_file_in = '.false.'
+# &ensemble_manager_nml: single_restart_file_in = .false.
#
#=========================================================================
@@ -374,13 +365,27 @@
# Lots of ways to get the filename
set OCN_RESTART_FILENAME = `head -1 ../rpointer.ocn_0001.restart`
+
${LINK} ../$OCN_RESTART_FILENAME pop.r.nc
${LINK} ../pop2_in_0001 pop_in
+# On yellowstone, you can explore task layouts with the following:
+if ( $?LSB_PJL_TASK_GEOMETRY ) then
+ setenv ORIGINAL_LAYOUT "${LSB_PJL_TASK_GEOMETRY}"
+
+ # setenv GEOMETRY_32_1NODE \
+ # "{(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31)}";
+ # setenv LSB_PJL_TASK_GEOMETRY "${GEOMETRY_32_1NODE}"
+endif
+
echo "`date` -- BEGIN FILTER"
-${LAUNCHCMD} ${EXEROOT}/filter || exit -2
+${LAUNCHCMD} ${EXEROOT}/filter || exit -7
echo "`date` -- END FILTER"
+if ( $?LSB_PJL_TASK_GEOMETRY ) then
+ setenv LSB_PJL_TASK_GEOMETRY "${ORIGINAL_LAYOUT}"
+endif
+
${MOVE} Prior_Diag.nc ../Prior_Diag.${OCN_DATE_EXT}.nc
${MOVE} Posterior_Diag.nc ../Posterior_Diag.${OCN_DATE_EXT}.nc
${MOVE} obs_seq.final ../obs_seq.${OCN_DATE_EXT}.final
@@ -402,17 +407,14 @@
#=========================================================================
# Block 6: Update the POP restart files ... simultaneously ...
#
-# Each member will do its job in its own directory.
-# That way, we can do N of them simultaneously.
-# The namelist already reflects the right filenames.
-# The right filenames are already linked from the pop_to_dart conversion.
+# Each member will do its job in its own directory, which already exists
+# and has the required input files remaining from 'Block 4'
#=========================================================================
echo "`date` -- BEGIN DART TO POP"
set member = 1
while ( $member <= $ensemble_size )
- set m4 = `printf %04d $member`
cd member_${member}
echo "starting dart_to_pop for member ${member} at "`date`
Modified: DART/branches/development/models/cam/shell_scripts/CESM_setup_startup.csh
===================================================================
--- DART/branches/development/models/cam/shell_scripts/CESM_setup_startup.csh 2013-03-05 17:40:05 UTC (rev 5983)
+++ DART/branches/development/models/cam/shell_scripts/CESM_setup_startup.csh 2013-03-05 17:50:17 UTC (rev 5984)
@@ -13,7 +13,8 @@
# This script is designed to configure and build a multi-instance CESM model
# that has CAM, CLM, and CICE as active components over a single data ocean,
# and will use DART to assimilate observations at regular intervals.
-# This script does not build DART.
+# This script does not build DART. It works best if the appropriate DART
+# executables have been built, however.
#
# This script relies heavily on the information in:
# http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/book1.html
@@ -47,30 +48,7 @@
# For the brave, read
#
# http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/x1142.html
-#
-# ==============================================================================
-# === IMPORTANT modifications to the distribution code before ANYTHING
-# ==============================================================================
-# Had to edit the following to remove the LSB_PJL_... word too long error
-# cesm1_1_beta04/scripts/ccsm_utils/Machines/mkbatch.bluefire .
-# On bluefire for cesm1_1_beta04 it's not enough to use the hard-wired
-# SourceMods below; mkbatch.bluefire cannot be modified via SourceMods.
-# So the cesm1_1_beta04 version in Tim's directory must be used.
-# As long as OMP_NUM_THREADS == 1 ... the default is fine.
-# This may also be a problem if the number of nodes is increased, e.g.
-# if the regular memory nodes are used, or if the resolution is increased,
-# because the number of entries in BIND_THRD_GEOMETRY will increase.
-#
-# The cesm1_1_beta04 lt_archive script did not create parent dirs
-# if they did not already exist. To fix the script, edit:
-# ${cesmroot}/scripts/ccsm_utils/Tools/lt_archive.csh
-# and change 'mkdir' to 'mkdir -p'. This is fixed in more recent
-# versions of the code.
-#
-# For these reasons, it is required that you use ~thoar/cesm1_1_beta04
-# as the source of the CESM code.
-
# ==============================================================================
# ==== Set case options
# ==============================================================================
@@ -92,7 +70,7 @@
# mach Computer name
# cesm_datadir Root path of the public CESM data files
# cesmroot Location of the cesm code base
-# For cesm1_1 on yellowstone
+# i.e. cesm1_1_1 on yellowstone
# DARTroot Location of DART code tree.
# Executables, scripts and input in $DARTroot/models/cam/...
# caseroot Your (future) cesm case directory, where this CESM+DART will be built.
@@ -118,7 +96,7 @@
# ==============================================================================
# configure settings ... run_startdate format is yyyy-mm-dd
# ==============================================================================
-#
+
setenv run_startdate 2008-10-31
setenv sst_dataset ${cesm_datadir}/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2011_c110307.nc
setenv year_start 1850
@@ -128,7 +106,6 @@
# runtime settings -- How many assimilation steps will be done after this one
#
# stop_option Units for determining the forecast length between assimilations
-# Changing stop_option requires changes to user_nl_cam below.
# stop_n Number of time units in the forecast
# ==============================================================================
@@ -150,8 +127,8 @@
# from running for long periods.
# ==============================================================================
-setenv ACCOUNT P86850054
-setenv timewall 0:30
+setenv ACCOUNT P8685nnnn
+setenv timewall 0:40
setenv queue regular
setenv ptile 15
@@ -176,6 +153,7 @@
set COPY = '/bin/cp -fv --preserve=timestamps'
set LINK = '/bin/ln -fvs'
set REMOVE = '/bin/rm -fr'
+ set nonomatch
breaksw
endsw
@@ -210,7 +188,9 @@
# Save a copy for debug purposes
foreach FILE ( *xml )
- ${COPY} $FILE ${FILE}.original
+ if ( ~ -e ${FILE}.original ) then
+ ${COPY} $FILE ${FILE}.original
+ endif
end
# num_tasks_per_instance = #tasks_node / #instances_node
@@ -279,9 +259,13 @@
./xmlchange RUN_TYPE=startup
./xmlchange RUN_STARTDATE=$run_startdate
./xmlchange EXEROOT=${exeroot}
+./xmlchange PIO_TYPENAME=pnetcdf
+# DOUT_S is to turn on/off the short-term archiving
+# DOUT_L_MS is to store to the HPSS (formerly "MSS")
+
./xmlchange DOUT_S_ROOT=${archdir}
-./xmlchange DOUT_S=FALSE
+./xmlchange DOUT_S=TRUE
./xmlchange DOUT_S_SAVE_INT_REST_FILES=TRUE
./xmlchange DOUT_L_MS=FALSE
./xmlchange DOUT_L_MSROOT="csm/${case}"
@@ -293,15 +277,12 @@
./xmlchange SSTICE_YEAR_END=$year_end
# Do not change the CALENDAR or the CONTINUE_RUN
-# DOUT_S is to turn on/off the short-term archiving
-# DOUT_L_MS is to store to the HPSS (formerly "MSS")
./xmlchange CALENDAR=GREGORIAN
./xmlchange STOP_OPTION=$stop_option
./xmlchange STOP_N=$stop_n
./xmlchange CONTINUE_RUN=FALSE
./xmlchange RESUBMIT=$resubmit
-./xmlchange PIO_TYPENAME=pnetcdf
# The river transport model ON is useful only when using an active ocean or
# land surface diagnostics. Setting ROF_GRID to 'null' turns off the RTM.
@@ -408,7 +389,7 @@
# ===========================================================================
echo " ice_ic = 'ice_restart_${instance}.nc' " >> ${fname}
-
+
@ inst ++
end
@@ -427,6 +408,8 @@
# none needed for CAM
+./preview_namelists
+
# ==============================================================================
# Update source files if need be
# Ideally, using DART will not require any source mods.
@@ -446,7 +429,7 @@
echo "http://www.image.ucar.edu/pub/DART/CESM/DART_SourceMods_cesm1_1_1.tar"
echo "untar these into your HOME directory - they will create a"
echo "~/cesm_1_1_1 directory with the appropriate SourceMods structure."
- exit -3
+ exit -4
endif
# ==============================================================================
@@ -461,15 +444,15 @@
if ( $status != 0 ) then
echo "ERROR: Case could not be built."
- exit -4
+ exit -5
endif
# ==============================================================================
# Stage the restarts now that the run directory exists
+#
+# CAM5 ... /glade/p/work/raeder/Exp/Pincus_Cam5/archive/Exp3/rest/2008-11-01-00000
# ==============================================================================
-# CAM5 ... /glade/p/work/raeder/Exp/Pincus_Cam5/archive/Exp3/rest/2008-11-01-00000
-
set stagedir = /glade/p/work/raeder/Exp/Pincus_Cam5/archive/Exp3/rest/2008-11-01-00000
echo ''
@@ -477,14 +460,18 @@
echo 'into the CESM run directory.'
echo ''
-@ n = 1
-while ($n <= $num_instances)
- set instance = `printf %04d $n`
- echo "Staging restarts for instance $n of $num_instances"
- ${COPY} ${stagedir}/cam_initial_${n}.nc ${rundir}/cam_initial_${instance}.nc
- ${COPY} ${stagedir}/clm_restart_${n}.nc ${rundir}/clm_restart_${instance}.nc
- ${COPY} ${stagedir}/ice_restart_${n}.nc ${rundir}/ice_restart_${instance}.nc
- @ n++
+@ inst = 1
+while ($inst <= $num_instances)
+ set instance = `printf %04d $inst`
+
+ echo ''
+ echo "Staging restarts for instance $inst of $num_instances"
+
+ ${COPY} ${stagedir}/cam_initial_${inst}.nc ${rundir}/cam_initial_${instance}.nc
+ ${COPY} ${stagedir}/clm_restart_${inst}.nc ${rundir}/clm_restart_${instance}.nc
+ ${COPY} ${stagedir}/ice_restart_${inst}.nc ${rundir}/ice_restart_${instance}.nc
+
+ @ inst ++
end
if ( -e ${stagedir}/prior_inflate_restart* ) then
@@ -492,27 +479,6 @@
endif
# ==============================================================================
-# IMPORTANT: All resubmits must be type 'startup'.
-# Change Tools/ccsm_postrun.csh line 83 to CONTINUE_RUN -val FALSE'
-# ==============================================================================
-
-echo ''
-echo 'Changing Tools/ccsm_postrun_setup such that all the resubmits are "startup",'
-echo 'which means CONTINUE_RUN should be FALSE in ccsm_postrun_setup'
-echo ''
-
-if ( ~ -e Tools/cesm_postrun_setup.original ) then
- ${COPY} Tools/cesm_postrun_setup Tools/cesm_postrun_setup.original
-endif
-
-ex Tools/cesm_postrun_setup <<ex_end
-/use COMP_RUN_BARRIERS as surrogate for timing run logical/
-/CONTINUE_RUN/
-s;TRUE;FALSE;
-wq
-ex_end
-
-# ==============================================================================
# Edit the run script to reflect project, queue, and wallclock
# ==============================================================================
@@ -531,7 +497,7 @@
# NCAR "bluefire", "yellowstone"
set TIMEWALL=`grep BSUB ${case}.run | grep -e '-W' `
set QUEUE=`grep BSUB ${case}.run | grep -e '-q' `
- sed -e "s/ptile=32/ptile=$ptile/" \
+ sed -e "s/ptile=[0-9][0-9]*/ptile=$ptile/" \
-e "s/$TIMEWALL[3]/$timewall/" \
-e "s/$QUEUE[3]/$queue/" < ${case}.run >! temp.$$
${MOVE} temp.$$ ${case}.run
@@ -542,8 +508,24 @@
breaksw
endsw
-chmod 0755 ${case}.run
+# ==============================================================================
+# IMPORTANT: All resubmits must be type 'startup'.
+# ==============================================================================
+echo ''
+echo 'Changing the run script such that all the resubmits are "startup",'
+echo 'which means CONTINUE_RUN should be FALSE'
+echo ''
+
+ex ${case}.run <<ex_end
+/use COMP_RUN_BARRIERS as surrogate for timing run logical/
+/CONTINUE_RUN/
+s;TRUE;FALSE;
+/CONTINUE_RUN/
+s;TRUE;FALSE;
+wq
+ex_end
+
# ==============================================================================
# The *.run script must be modified to call the DART assimilate script.
# The modifications are contained in a "here" document that MUST NOT
@@ -617,25 +599,33 @@
else
echo "ERROR in grep of ${case}.run: aborting"
echo "status was ${STATUSCHECK}"
- exit -5
+ exit -6
endif
+chmod 0755 ${case}.run
+
# ==============================================================================
# Stage the required parts of DART in the CASEROOT directory.
# ==============================================================================
-# The cesm1_1_beta04 release had an error in that it did not
-# provide the lt_archive.sh script, and the one in the repos
-# did not have the -p flag, which is a good idea. So, for now ...
+# The standard CESM short-term archiving script may need to be altered
+# to archive addtional or subsets of things, or to reduce the amount of
+# data that is sent to the long-term archive.
if ( ~ -e Tools/st_archive.sh.original ) then
${COPY} Tools/st_archive.sh Tools/st_archive.sh.original
endif
-# ${COPY} ${DARTroot}/models/cam/shell_scripts/st_archive.sh Tools/
+# NOTE: the assimilate.csh script and input.nml must be modified for your
+# situation. The script has variables that point to the location of
+# the observations sequence files and the DART working directory
+# and may be customized for a more efficient PE layout for DART.
+# If you are running this, you know what to do with input.nml.
+# If you don't, you should give up now. Really.
-${COPY} ${DARTroot}/models/cam/shell_scripts/assimilate.csh .
-${COPY} ${DARTroot}/models/cam/work/input.nml .
+${COPY} ${DARTroot}/models/cam/shell_scripts/st_archive.sh Tools/
+${COPY} ${DARTroot}/models/cam/shell_scripts/assimilate.csh assimilate.csh
+${COPY} ${DARTroot}/models/cam/work/input.nml input.nml
# ==============================================================================
# Stage the DART executables in the CESM execution root directory: EXEROOT
@@ -646,7 +636,7 @@
if ( $status != 0 ) then
echo "ERROR: ${DARTroot}/models/cam/work/${FILE} not copied to ${exeroot}"
echo "ERROR: ${DARTroot}/models/cam/work/${FILE} not copied to ${exeroot}"
- exit -6
+ exit -7
endif
end
@@ -667,8 +657,8 @@
# echo "You can make one with ${DARTroot}/models/cam/work/fill_inflation_restart"
# echo ''
# endif
-#
-#
+#
+#
# only warn people if a precomputed final_full for this number of instances
# does not already exist.
# if (! -f ${DARTroot}/system_simulation/final_full_precomputed_tables/final_full.${num_instances}) then
@@ -690,9 +680,9 @@
echo "Time to check the case."
echo ''
echo "cd into ${caseroot}"
-echo 'Modify what you like in input.nml, make sure the observation directory'
-echo 'names set in assimilate.csh match those on your system, and submit'
-echo 'the CESM job by running:'
-echo "./$case.submit"
+echo "Modify what you like in input.nml, make sure the observation directory"
+echo "names set in assimilate.csh match those on your system, and submit"
+echo "the CESM job by running:"
+echo "./${case}.submit"
echo ''
Modified: DART/branches/development/models/cam/shell_scripts/assimilate.csh
===================================================================
--- DART/branches/development/models/cam/shell_scripts/assimilate.csh 2013-03-05 17:40:05 UTC (rev 5983)
+++ DART/branches/development/models/cam/shell_scripts/assimilate.csh 2013-03-05 17:50:17 UTC (rev 5984)
@@ -86,7 +86,6 @@
#-----------------------------------------------------------------------------
# Get observation sequence file ... or die right away.
-# Cannot specify -f on the link command and still check status.
# The observation file names have a time that matches the stopping time of CAM.
#-----------------------------------------------------------------------------
@@ -370,9 +369,9 @@
if ( $?LSB_PJL_TASK_GEOMETRY ) then
setenv ORIGINAL_LAYOUT "${LSB_PJL_TASK_GEOMETRY}"
- # setenv NANCY_GEOMETRY_54_1NODE \
- # "{(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53)}";
- # setenv LSB_PJL_TASK_GEOMETRY "${NANCY_GEOMETRY_54_1NODE}"
+ # setenv GEOMETRY_32_1NODE \
+ # "{(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31)}";
+ # setenv LSB_PJL_TASK_GEOMETRY "${GEOMETRY_32_1NODE}"
endif
echo "`date` -- BEGIN FILTER"
@@ -451,14 +450,10 @@
set n4 = `printf %04d $member`
- set LND_POINTER_FILENAME = `printf rpointer.lnd_%04d ${member}`
- set ICE_POINTER_FILENAME = `printf rpointer.ice_%04d ${member}`
- # the ATM pointer file points to a restart file - which we cannot use
+ set LND_RESTART_FILENAME = `printf ${MYCASE}.clm2_%04d.r.${ATM_DATE_EXT}.nc ${member}`
+ set ICE_RESTART_FILENAME = `printf ${MYCASE}.cice_%04d.r.${ATM_DATE_EXT}.nc ${member}`
+ set ATM_INITIAL_FILENAME = `printf ${MYCASE}.cam_%04d.i.${ATM_DATE_EXT}.nc ${member}`
- set LND_RESTART_FILENAME = `head -n 1 ${LND_POINTER_FILENAME}`
- set ICE_RESTART_FILENAME = `head -n 1 ${ICE_POINTER_FILENAME}`
- set ATM_INITIAL_FILENAME = `printf ${MYCASE}.cam_%04d.i.${ATM_DATE_EXT}.nc ${member}`
-
${LINK} ${LND_RESTART_FILENAME} clm_restart_${n4}.nc || exit -9
${LINK} ${ICE_RESTART_FILENAME} ice_restart_${n4}.nc || exit -9
${LINK} ${ATM_INITIAL_FILENAME} cam_initial_${n4}.nc || exit -9
@@ -469,6 +464,9 @@
#-------------------------------------------------------------------------
# We have to communicate the current model time to the env_run.xml script
+# CAM is running in a 'perpetual startup' mode, so it constantly needs new
+# start time information. The other models are running 'restart', so they
+# keep track of it internally.
#-------------------------------------------------------------------------
cd ${CASEROOT}
@@ -479,20 +477,21 @@
./xmlchange RUN_STARTDATE=${YYYYMMDD}
./xmlchange START_TOD=${SSSSS}
-cd ${RUNDIR}
-
+#-------------------------------------------------------------------------
# we (DART) do not need these files, and CESM does not need them either
# to continue a run. if we remove them here they do not get moved to
# the short-term archiver.
+#-------------------------------------------------------------------------
-${REMOVE} ../*.rs.*
-${REMOVE} ../*.rh0.*
-${REMOVE} ../*.rs1.*
-${REMOVE} ../PET*ESMF_LogFile
+${REMOVE} ${RUNDIR}/*.rs.*
+${REMOVE} ${RUNDIR}/*.rh0.*
+${REMOVE} ${RUNDIR}/*.rs1.*
+${REMOVE} ${RUNDIR}/PET*ESMF_LogFile
#-------------------------------------------------------------------------
# Cleanup
#-------------------------------------------------------------------------
+
echo "finished assimilate script at "`date`
echo "`date` -- END ASSIMILATE"
Modified: DART/branches/development/models/clm/shell_scripts/CESM1_1_setup.csh
===================================================================
--- DART/branches/development/models/clm/shell_scripts/CESM1_1_setup.csh 2013-03-05 17:40:05 UTC (rev 5983)
+++ DART/branches/development/models/clm/shell_scripts/CESM1_1_setup.csh 2013-03-05 17:50:17 UTC (rev 5984)
@@ -11,9 +11,10 @@
# ---------------------
#
# This script is designed to configure and build a multi-instance CESM model
-# that has CAM, CLM, and CICE as active components over a single data ocean,
+# that has CLM running under a set of data atmospheres (from a DART/CAM assim).
# and will use DART to assimilate observations at regular intervals.
-# This script does not build DART.
+# This script does not build DART. It works best if the appropriate DART
+# executables have been built, however.
#
# This script relies heavily on the information in:
# http://www.cesm.ucar.edu/models/cesm1.1/cesm/doc/usersguide/book1.html
@@ -31,8 +32,11 @@
# -- Examine the whole script to identify things to change for your experiments.
@@ Diff output truncated at 40000 characters. @@
More information about the Dart-dev
mailing list