[Dart-dev] [6871] DART/trunk/models: The archivers now archive three times per 'month' : on the 10th, 20th, and
nancy at ucar.edu
nancy at ucar.edu
Fri Apr 4 12:23:35 MDT 2014
Revision: 6871
Author: thoar
Date: 2014-04-04 12:23:35 -0600 (Fri, 04 Apr 2014)
Log Message:
-----------
The archivers now archive three times per 'month': on the 10th, 20th, and
the 'last' day of the month. The short-term archive directory is removed only
if the reinstatement move (at the end) was successful (as per Alice Bertini).
Works with CESM1_2_1 and CESM1_1_1
Modified Paths:
--------------
DART/trunk/models/CESM/shell_scripts/st_archive.sh
DART/trunk/models/POP/shell_scripts/st_archive.sh
DART/trunk/models/cam/shell_scripts/st_archive.sh
DART/trunk/models/clm/shell_scripts/st_archive.sh
-------------- next part --------------
Modified: DART/trunk/models/CESM/shell_scripts/st_archive.sh
===================================================================
--- DART/trunk/models/CESM/shell_scripts/st_archive.sh 2014-04-04 18:03:04 UTC (rev 6870)
+++ DART/trunk/models/CESM/shell_scripts/st_archive.sh 2014-04-04 18:23:35 UTC (rev 6871)
@@ -42,13 +42,11 @@
#
#result is returned in $inst_suffix
get_inst_suffix() {
-# echo "get_inst_suffixa $1 $2 ${inst_suffix}"
if [ $2 -eq 1 ]; then # only one instance of this component
inst_suffix=""
else # multiple instances of this component
inst_suffix=`printf _%04d $1`
fi
-# echo "get_inst_suffixb $1 $2 ${inst_suffix}"
}
echo ""
@@ -66,7 +64,7 @@
mkdir -p ${sta} 2> /dev/null
if [ $? -ne 0 ]; then
- echo "st_archive.sh: error, unable to create short-term archive directory .sta"
+ echo "st_archive.sh: error, unable to create short-term archive directory ${sta}"
echo "st_archive.sh: exiting"
exit 1
fi
@@ -110,6 +108,11 @@
export NINST_GLC=1
fi
+if [ -z "$NINST_WAV" ]; then
+ echo "st_archive.sh: warning, NINST_WAV not set -- using 1 instance"
+ export NINST_WAV=1
+fi
+
#create directory for restart files
set ${CASE}.cpl.r.*
cplfile=`ls -rt $* 2> /dev/null | tail -1`
@@ -136,12 +139,12 @@
mv $* ${sta}/rest/${dname}
set cpl.log.*; dispose ifiles_n ${sta}/cpl/logs $*
-set ccsm*.log.*; dispose ifiles_n ${sta}/cpl/logs $*
+set cesm*.log.*; dispose ifiles_n ${sta}/cpl/logs $*
set ${CASE}.cpl.r.*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/cpl/rest $*
set ${CASE}.cpl.h* ; dispose ifiles_n ${sta}/cpl/hist $*
-# TJH FIXME possible tweaking - remove assimilate_dir/member_* directories? anything else?
+# DART assimilation-related files
set assimilate_???/*/*dart_log.*; dispose ifiles_n ${sta}/dart/logs $*
set assimilate_???/*/output.*; dispose ifiles_n ${sta}/dart/logs $*
set *dart_log.*; dispose ifiles_n ${sta}/dart/logs $*
@@ -151,8 +154,12 @@
set *obs_seq.*.out; dispose ifiles_n ${sta}/dart/hist $*
set *obs_seq.*.final; dispose ifiles_n ${sta}/dart/hist $*
set *obs_seq.*.perfect; dispose ifiles_n ${sta}/dart/hist $*
-set *pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
-set *po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set cam_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set cam_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set clm_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set clm_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set pop_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set pop_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
IDX=1
@@ -305,6 +312,21 @@
IDX=`expr $IDX + 1`
done
+
+IDX=1
+while [ $IDX -le $NINST_WAV ]
+do
+ get_inst_suffix $IDX $NINST_WAV
+ set wav${inst_suffix}.log.*; dispose ifiles_n ${sta}/wav/logs $*
+ set ${CASE}.ww3${inst_suffix}.r.[0-9]*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/wav/rest $*
+ set ${CASE}.ww3${inst_suffix}.h*; dispose ifiles_n ${sta}/wav/hist $*
+ set ${CASE}.ww3${inst_suffix}.i.*; dispose ifiles_y ${sta}/wav/init $*
+ set ${CASE}.dwav${inst_suffix}.r.* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/wav/rest $*
+ set ${CASE}.dwav${inst_suffix}.rs* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/wav/rest $*
+ set ${CASE}.dwav${inst_suffix}.h.* ; dispose ifiles_n ${sta}/wav/hist $*
+ IDX=`expr $IDX + 1`
+done
+
# changes for DART assimilation runs. stopping the model every day
# (or every 6 hours) results in too much output to be archived.
# keep every Nth restart, as well as the previous restart until
@@ -328,7 +350,6 @@
# copy back the required files for next restart
cp ${sta}/rest/${dname}/* .
-
# now possibly delete the current contents of the restart dir so
# it won't be picked up by the long term archiver. the diagnostic
# files are saved for all times, but these are the restart files
@@ -359,17 +380,19 @@
touch ${sta}/rest/${dname}_removed
fi
-# rm the .hv files because they have no unique time stamp.
-rm -rf ${sta}/ocn/hist/${CASE}.pop.*.hv.nc
-
-
# end of DART changes
# make files visible again in the archive directory so they
# are eligible again for the long term archiver to save.
-mv ${sta}/* ${DOUT_S_ROOT}
-rm -fr ${sta}
+if mv ${sta}/* ${DOUT_S_ROOT}; then
+ rm -fr ${sta}
+else
+ echo "st_archive.sh: error, final move command unsuccessful"
+ echo " some short-term archive data may be in ${sta}"
+ echo "st_archive.sh: exiting"
+ exit 1
+fi
echo "st_archive.sh: short-term archiving completed successfully"
Modified: DART/trunk/models/POP/shell_scripts/st_archive.sh
===================================================================
--- DART/trunk/models/POP/shell_scripts/st_archive.sh 2014-04-04 18:03:04 UTC (rev 6870)
+++ DART/trunk/models/POP/shell_scripts/st_archive.sh 2014-04-04 18:23:35 UTC (rev 6871)
@@ -42,13 +42,11 @@
#
#result is returned in $inst_suffix
get_inst_suffix() {
-# echo "get_inst_suffixa $1 $2 ${inst_suffix}"
if [ $2 -eq 1 ]; then # only one instance of this component
inst_suffix=""
else # multiple instances of this component
inst_suffix=`printf _%04d $1`
fi
-# echo "get_inst_suffixb $1 $2 ${inst_suffix}"
}
echo ""
@@ -66,7 +64,7 @@
mkdir -p ${sta} 2> /dev/null
if [ $? -ne 0 ]; then
- echo "st_archive.sh: error, unable to create short-term archive directory"
+ echo "st_archive.sh: error, unable to create short-term archive directory ${sta}"
echo "st_archive.sh: exiting"
exit 1
fi
@@ -110,6 +108,11 @@
export NINST_GLC=1
fi
+if [ -z "$NINST_WAV" ]; then
+ echo "st_archive.sh: warning, NINST_WAV not set -- using 1 instance"
+ export NINST_WAV=1
+fi
+
#create directory for restart files
set ${CASE}.cpl.r.*
cplfile=`ls -rt $* 2> /dev/null | tail -1`
@@ -136,12 +139,12 @@
mv $* ${sta}/rest/${dname}
set cpl.log.*; dispose ifiles_n ${sta}/cpl/logs $*
-set ccsm*.log.*; dispose ifiles_n ${sta}/cpl/logs $*
+set cesm*.log.*; dispose ifiles_n ${sta}/cpl/logs $*
set ${CASE}.cpl.r.*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/cpl/rest $*
set ${CASE}.cpl.h* ; dispose ifiles_n ${sta}/cpl/hist $*
-# TJH FIXME possible tweaking - remove assimilate_dir/member_* directories? anything else?
+# DART assimilation-related files
set assimilate_???/*/*dart_log.*; dispose ifiles_n ${sta}/dart/logs $*
set assimilate_???/*/output.*; dispose ifiles_n ${sta}/dart/logs $*
set *dart_log.*; dispose ifiles_n ${sta}/dart/logs $*
@@ -151,8 +154,12 @@
set *obs_seq.*.out; dispose ifiles_n ${sta}/dart/hist $*
set *obs_seq.*.final; dispose ifiles_n ${sta}/dart/hist $*
set *obs_seq.*.perfect; dispose ifiles_n ${sta}/dart/hist $*
-set *pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
-set *po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set cam_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set cam_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set clm_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set clm_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set pop_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set pop_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
IDX=1
@@ -270,17 +277,19 @@
get_inst_suffix $IDX $NINST_OCN
set ocn${inst_suffix}.log.*; dispose ifiles_n ${sta}/ocn/logs $*
set ${CASE}.pop${inst_suffix}.r.*.hdr; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.r.*0000; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.r.*0000.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.r.*[0-9]; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.r.*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.rh.ecosys.*.hdr; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.ecosys.*0000; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.ecosys.*0000.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.ecosys.*[0-9]; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.ecosys.*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.rh.*.hdr; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.*0000; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.*0000.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.*[0-9]; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.[0-9]*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.ro.*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.d?*; dispose ifiles_n ${sta}/ocn/hist $*
set ${CASE}.pop${inst_suffix}.h*; dispose ifiles_n ${sta}/ocn/hist $*
+
set ${CASE}.docn${inst_suffix}.r.* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.docn${inst_suffix}.rs* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.docn${inst_suffix}.h.* ; dispose ifiles_n ${sta}/ocn/hist $*
@@ -303,12 +312,88 @@
IDX=`expr $IDX + 1`
done
-#copy back the required files for next restart
+
+IDX=1
+while [ $IDX -le $NINST_WAV ]
+do
+ get_inst_suffix $IDX $NINST_WAV
+ set wav${inst_suffix}.log.*; dispose ifiles_n ${sta}/wav/logs $*
+ set ${CASE}.ww3${inst_suffix}.r.[0-9]*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/wav/rest $*
+ set ${CASE}.ww3${inst_suffix}.h*; dispose ifiles_n ${sta}/wav/hist $*
+ set ${CASE}.ww3${inst_suffix}.i.*; dispose ifiles_y ${sta}/wav/init $*
+ set ${CASE}.dwav${inst_suffix}.r.* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/wav/rest $*
+ set ${CASE}.dwav${inst_suffix}.rs* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/wav/rest $*
+ set ${CASE}.dwav${inst_suffix}.h.* ; dispose ifiles_n ${sta}/wav/hist $*
+ IDX=`expr $IDX + 1`
+done
+
+# changes for DART assimilation runs. stopping the model every day
+# (or every 6 hours) results in too much output to be archived.
+# keep every Nth restart, as well as the previous restart until
+# we know the next step has completed successfully.
+
+sta2=${DOUT_S_ROOT}/.sta2
+mkdir -p ${sta2} 2> /dev/null
+
+if [ $? -ne 0 ]; then
+ echo "st_archive.sh: error, unable to create short-term archive directory .sta2"
+ echo "st_archive.sh: exiting"
+ exit 1
+fi
+
+# delete previous contents, save last successful step for possible restart
+rm -fr ${sta2}/*
+mkdir -p ${sta2}/${dname} 2> /dev/null
+cp ${sta}/rest/${dname}/* ${sta2}/${dname}
+echo "st_archive.sh: copied last successful step into ${sta2}/${dname} for safekeeping"
+
+# copy back the required files for next restart
cp ${sta}/rest/${dname}/* .
-mv ${sta}/* ${DOUT_S_ROOT}
-rm -fr ${sta}
+# now possibly delete the current contents of the restart dir so
+# it won't be picked up by the long term archiver. the diagnostic
+# files are saved for all times, but these are the restart files
+# needed to start a new model advance. if you try to save every
+# set of restart files you will be archiving a very very very
+# large amount of data.
+# dname: YYYY-MM-DD-SSSSS
+# col: 1234567890123456
+
+ year=`echo $dname | cut -b1-4`
+month=`echo $dname | cut -b6-7`
+ day=`echo $dname | cut -b9-10`
+ secs=`echo $dname | cut -b12-16`
+
+# if you want to save more often (or less) alter the test here
+# using the time variables from immediately above.
+
+# approximately the last day of each month: all months except feb have 30 days,
+# and all febs have a 28th. save the 10th, 20th, and "last" day of each month.
+if [[ $month == 02 ]]; then lastday=28; else lastday=30; fi
+
+if [[ $secs == 00000 && ($day == 10 || $day == 20 || $day == $lastday) ]]; then
+ echo "st_archive: PRESERVING contents of restart ${dname}"
+else
+ echo "st_archive: DELETING contents of restart ${dname}"
+ rm -rf ${sta}/rest/${dname}
+ touch ${sta}/rest/${dname}_removed
+fi
+
+# end of DART changes
+
+
+# make files visible again in the archive directory so they
+# are eligible again for the long term archiver to save.
+if mv ${sta}/* ${DOUT_S_ROOT}; then
+ rm -fr ${sta}
+else
+ echo "st_archive.sh: error, final move command unsuccessful"
+ echo " some short-term archive data may be in ${sta}"
+ echo "st_archive.sh: exiting"
+ exit 1
+fi
+
echo "st_archive.sh: short-term archiving completed successfully"
exit 0
Modified: DART/trunk/models/cam/shell_scripts/st_archive.sh
===================================================================
--- DART/trunk/models/cam/shell_scripts/st_archive.sh 2014-04-04 18:03:04 UTC (rev 6870)
+++ DART/trunk/models/cam/shell_scripts/st_archive.sh 2014-04-04 18:23:35 UTC (rev 6871)
@@ -42,13 +42,11 @@
#
#result is returned in $inst_suffix
get_inst_suffix() {
-# echo "get_inst_suffixa $1 $2 ${inst_suffix}"
if [ $2 -eq 1 ]; then # only one instance of this component
inst_suffix=""
else # multiple instances of this component
inst_suffix=`printf _%04d $1`
fi
-# echo "get_inst_suffixb $1 $2 ${inst_suffix}"
}
echo ""
@@ -66,7 +64,7 @@
mkdir -p ${sta} 2> /dev/null
if [ $? -ne 0 ]; then
- echo "st_archive.sh: error, unable to create short-term archive directory"
+ echo "st_archive.sh: error, unable to create short-term archive directory ${sta}"
echo "st_archive.sh: exiting"
exit 1
fi
@@ -110,6 +108,11 @@
export NINST_GLC=1
fi
+if [ -z "$NINST_WAV" ]; then
+ echo "st_archive.sh: warning, NINST_WAV not set -- using 1 instance"
+ export NINST_WAV=1
+fi
+
#create directory for restart files
set ${CASE}.cpl.r.*
cplfile=`ls -rt $* 2> /dev/null | tail -1`
@@ -136,7 +139,7 @@
mv $* ${sta}/rest/${dname}
set cpl.log.*; dispose ifiles_n ${sta}/cpl/logs $*
-set ccsm*.log.*; dispose ifiles_n ${sta}/cpl/logs $*
+set cesm*.log.*; dispose ifiles_n ${sta}/cpl/logs $*
set ${CASE}.cpl.r.*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/cpl/rest $*
set ${CASE}.cpl.h* ; dispose ifiles_n ${sta}/cpl/hist $*
@@ -151,8 +154,12 @@
set *obs_seq.*.out; dispose ifiles_n ${sta}/dart/hist $*
set *obs_seq.*.final; dispose ifiles_n ${sta}/dart/hist $*
set *obs_seq.*.perfect; dispose ifiles_n ${sta}/dart/hist $*
-set *pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
-set *po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set cam_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set cam_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set clm_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set clm_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set pop_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set pop_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
IDX=1
@@ -270,17 +277,19 @@
get_inst_suffix $IDX $NINST_OCN
set ocn${inst_suffix}.log.*; dispose ifiles_n ${sta}/ocn/logs $*
set ${CASE}.pop${inst_suffix}.r.*.hdr; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.r.*0000; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.r.*0000.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.r.*[0-9]; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.r.*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.rh.ecosys.*.hdr; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.ecosys.*0000; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.ecosys.*0000.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.ecosys.*[0-9]; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.ecosys.*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.rh.*.hdr; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.*0000; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.*0000.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.*[0-9]; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.[0-9]*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.ro.*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.d?*; dispose ifiles_n ${sta}/ocn/hist $*
set ${CASE}.pop${inst_suffix}.h*; dispose ifiles_n ${sta}/ocn/hist $*
+
set ${CASE}.docn${inst_suffix}.r.* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.docn${inst_suffix}.rs* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.docn${inst_suffix}.h.* ; dispose ifiles_n ${sta}/ocn/hist $*
@@ -303,12 +312,94 @@
IDX=`expr $IDX + 1`
done
-#copy back the required files for next restart
+
+IDX=1
+while [ $IDX -le $NINST_WAV ]
+do
+ get_inst_suffix $IDX $NINST_WAV
+ set wav${inst_suffix}.log.*; dispose ifiles_n ${sta}/wav/logs $*
+ set ${CASE}.ww3${inst_suffix}.r.[0-9]*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/wav/rest $*
+ set ${CASE}.ww3${inst_suffix}.h*; dispose ifiles_n ${sta}/wav/hist $*
+ set ${CASE}.ww3${inst_suffix}.i.*; dispose ifiles_y ${sta}/wav/init $*
+ set ${CASE}.dwav${inst_suffix}.r.* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/wav/rest $*
+ set ${CASE}.dwav${inst_suffix}.rs* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/wav/rest $*
+ set ${CASE}.dwav${inst_suffix}.h.* ; dispose ifiles_n ${sta}/wav/hist $*
+ IDX=`expr $IDX + 1`
+done
+
+# changes for DART assimilation runs. stopping the model every day
+# (or every 6 hours) results in too much output to be archived.
+# keep every Nth restart, as well as the previous restart until
+# we know the next step has completed successfully.
+
+sta2=${DOUT_S_ROOT}/.sta2
+mkdir -p ${sta2} 2> /dev/null
+
+if [ $? -ne 0 ]; then
+ echo "st_archive.sh: error, unable to create short-term archive directory .sta2"
+ echo "st_archive.sh: exiting"
+ exit 1
+fi
+
+# delete previous contents, save last successful step for possible restart
+rm -fr ${sta2}/*
+mkdir -p ${sta2}/${dname} 2> /dev/null
+cp ${sta}/rest/${dname}/* ${sta2}/${dname}
+echo "st_archive.sh: copied last successful step into ${sta2}/${dname} for safekeeping"
+
+# copy back the required files for next restart
cp ${sta}/rest/${dname}/* .
-mv ${sta}/* ${DOUT_S_ROOT}
-rm -fr ${sta}
+# now possibly delete the current contents of the restart dir so
+# it won't be picked up by the long term archiver. the diagnostic
+# files are saved for all times, but these are the restart files
+# needed to start a new model advance. if you try to save every
+# set of restart files you will be archiving a very very very
+# large amount of data.
+# dname: YYYY-MM-DD-SSSSS
+# col: 1234567890123456
+
+ year=`echo $dname | cut -b1-4`
+month=`echo $dname | cut -b6-7`
+ day=`echo $dname | cut -b9-10`
+ secs=`echo $dname | cut -b12-16`
+
+# if you want to save more often (or less) alter the test here
+# using the time variables from immediately above.
+
+# approximately the last day of each month: all months except feb have 30 days,
+# and all febs have a 28th. save the 10th, 20th, and "last" day of each month.
+if [[ $month == 02 ]]; then lastday=28; else lastday=30; fi
+
+if [[ $secs == 00000 && ($day == 10 || $day == 20 || $day == $lastday) ]]; then
+ echo "st_archive: PRESERVING contents of restart ${dname}"
+else
+ echo "st_archive: DELETING contents of restart ${dname}"
+ rm -rf ${sta}/rest/${dname}
+ touch ${sta}/rest/${dname}_removed
+fi
+
+# end of DART changes
+
+
+# make files visible again in the archive directory so they
+# are eligible again for the long term archiver to save.
+if mv ${sta}/* ${DOUT_S_ROOT}; then
+ rm -fr ${sta}
+else
+ echo "st_archive.sh: error, final move command unsuccessful"
+ echo " some short-term archive data may be in ${sta}"
+ echo "st_archive.sh: exiting"
+ exit 1
+fi
+
echo "st_archive.sh: short-term archiving completed successfully"
exit 0
+
+# <next few lines under version control, do not edit>
+# $URL$
+# $Revision$
+# $Date$
+
Modified: DART/trunk/models/clm/shell_scripts/st_archive.sh
===================================================================
--- DART/trunk/models/clm/shell_scripts/st_archive.sh 2014-04-04 18:03:04 UTC (rev 6870)
+++ DART/trunk/models/clm/shell_scripts/st_archive.sh 2014-04-04 18:23:35 UTC (rev 6871)
@@ -42,13 +42,11 @@
#
#result is returned in $inst_suffix
get_inst_suffix() {
-# echo "get_inst_suffixa $1 $2 ${inst_suffix}"
if [ $2 -eq 1 ]; then # only one instance of this component
inst_suffix=""
else # multiple instances of this component
inst_suffix=`printf _%04d $1`
fi
-# echo "get_inst_suffixb $1 $2 ${inst_suffix}"
}
echo ""
@@ -66,7 +64,7 @@
mkdir -p ${sta} 2> /dev/null
if [ $? -ne 0 ]; then
- echo "st_archive.sh: error, unable to create short-term archive directory"
+ echo "st_archive.sh: error, unable to create short-term archive directory ${sta}"
echo "st_archive.sh: exiting"
exit 1
fi
@@ -110,6 +108,11 @@
export NINST_GLC=1
fi
+if [ -z "$NINST_WAV" ]; then
+ echo "st_archive.sh: warning, NINST_WAV not set -- using 1 instance"
+ export NINST_WAV=1
+fi
+
#create directory for restart files
set ${CASE}.cpl.r.*
cplfile=`ls -rt $* 2> /dev/null | tail -1`
@@ -136,12 +139,12 @@
mv $* ${sta}/rest/${dname}
set cpl.log.*; dispose ifiles_n ${sta}/cpl/logs $*
-set ccsm*.log.*; dispose ifiles_n ${sta}/cpl/logs $*
+set cesm*.log.*; dispose ifiles_n ${sta}/cpl/logs $*
set ${CASE}.cpl.r.*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/cpl/rest $*
set ${CASE}.cpl.h* ; dispose ifiles_n ${sta}/cpl/hist $*
-# TJH FIXME possible tweaking - remove assimilate_dir/member_* directories? anything else?
+# DART assimilation-related files
set assimilate_???/*/*dart_log.*; dispose ifiles_n ${sta}/dart/logs $*
set assimilate_???/*/output.*; dispose ifiles_n ${sta}/dart/logs $*
set *dart_log.*; dispose ifiles_n ${sta}/dart/logs $*
@@ -151,8 +154,12 @@
set *obs_seq.*.out; dispose ifiles_n ${sta}/dart/hist $*
set *obs_seq.*.final; dispose ifiles_n ${sta}/dart/hist $*
set *obs_seq.*.perfect; dispose ifiles_n ${sta}/dart/hist $*
-set *pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
-set *po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set cam_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set cam_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set clm_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set clm_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set pop_*pr*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
+set pop_*po*inflate_restart*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_n ${sta}/dart/rest $*
IDX=1
@@ -270,17 +277,19 @@
get_inst_suffix $IDX $NINST_OCN
set ocn${inst_suffix}.log.*; dispose ifiles_n ${sta}/ocn/logs $*
set ${CASE}.pop${inst_suffix}.r.*.hdr; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.r.*0000; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.r.*0000.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.r.*[0-9]; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.r.*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.rh.ecosys.*.hdr; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.ecosys.*0000; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.ecosys.*0000.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.ecosys.*[0-9]; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.ecosys.*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.rh.*.hdr; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.*0000; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
- set ${CASE}.pop${inst_suffix}.rh.*0000.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.*[0-9]; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.[0-9]*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
+ set ${CASE}.pop${inst_suffix}.rh.*.nc; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.ro.*; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.pop${inst_suffix}.d?*; dispose ifiles_n ${sta}/ocn/hist $*
set ${CASE}.pop${inst_suffix}.h*; dispose ifiles_n ${sta}/ocn/hist $*
+
set ${CASE}.docn${inst_suffix}.r.* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.docn${inst_suffix}.rs* ; latest=`ls -rt $* 2> /dev/null | tail -1`; mv $latest ${sta}/rest/${dname} 2> /dev/null; dispose ifiles_y ${sta}/ocn/rest $*
set ${CASE}.docn${inst_suffix}.h.* ; dispose ifiles_n ${sta}/ocn/hist $*
@@ -303,12 +312,88 @@
IDX=`expr $IDX + 1`
done
-#copy back the required files for next restart
+
+IDX=1
+while [ $IDX -le $NINST_WAV ]
+do
+ get_inst_suffix $IDX $NINST_WAV
@@ Diff output truncated at 40000 characters. @@
More information about the Dart-dev
mailing list