[Dart-dev] [3448] DART/trunk/models/cam/full_experiment:
New scripts and revised scripts for archiving data to the mass store (ms).
nancy at ucar.edu
nancy at ucar.edu
Tue Jul 8 13:00:28 MDT 2008
An HTML attachment was scrubbed...
URL: http://mailman.ucar.edu/pipermail/dart-dev/attachments/20080708/8ae08ebb/attachment-0001.html
-------------- next part --------------
Modified: DART/trunk/models/cam/full_experiment/auto_diag2ms_LSF.csh
===================================================================
--- DART/trunk/models/cam/full_experiment/auto_diag2ms_LSF.csh 2008-07-08 18:54:15 UTC (rev 3447)
+++ DART/trunk/models/cam/full_experiment/auto_diag2ms_LSF.csh 2008-07-08 19:00:28 UTC (rev 3448)
@@ -44,13 +44,15 @@
set diag_name = diagnostics.tar
set saved = saved_diagnostics
-set write_pass = da$$
+set write_pass = $$
if ($?LS_SUBCWD) then
cd $LS_SUBCWD
endif
-echo 'auto_diag2ms_LSF starts in' >! $saved
+touch $saved
+echo '------------------------------------------------------------' >> $saved
+echo 'auto_diag2ms_LSF starts in' >> $saved
pwd >> $saved
date >> $saved
@@ -69,20 +71,62 @@
set ms_dir = /RAEDER/DAI/${exp_dir}/$case/${obs_seq}
# IBM tar requires 1 entry/line in list of things to exclude from tar
-echo DART >! tar_excl_list
-echo CAM >> tar_excl_list
-echo CLM >> tar_excl_list
+echo DART >! tar_excl_list
+echo CAM >> tar_excl_list
+echo CLM >> tar_excl_list
# batch* are the files into which DART,CAM,CLM are archived by auto_re2ms_LSF.csh,
# which is run at the same time as this script.
-echo 'batch*' >> tar_excl_list
-echo $saved >> tar_excl_list
+echo 'batch*' >> tar_excl_list
+echo $saved >> tar_excl_list
-tar -c -v -f $diag_name -X tar_excl_list * >>& $saved
+## Added to make mean easily accessible in the form of a CAM initial file
+## (Don't forget to save/bundle a CLM initial file with these cam_init_[obs_#]Hhh.nc files)
+echo 'cam_analyses.tar' >> tar_excl_list
+
+#-----------------------------
+# Stuff the Posterior mean fields into CAM initial files.
+# Tar together with a CLM initial file.
+# Arguments to mean2cam_init.csh are
+# set ms_file = $1 script searches for local Posterior_Diag.nc first, so give a dummy MS name.
+# set local_dir = $2
+# set kind = $3
+# set dim = $4
+# set element1 = $5
+# set yrmoday = $6 set to $obs_seq instead of yyyymmdd, since obs_seq is easily available
+
+if (-e ../../mean2cam_init.csh) then
+ # mean2cam_init.csh needs CAM initial files to receive the analyses from Posterior_Diag.nc.
+ # They should have been saved during the assimilation and be living in the exp/obs_#### directory.
+ ls -l cam_init_memb*
+ if ($status != 0) then
+ echo "cam_init_memb# not available; exiting" >>& $saved
+ exit
+ endif
+
+ ../../mean2cam_init.csh no_MS '.' Posterior copy 1 ${obs_seq}H
+
+ tar -c -f cam_analyses.tar cam_init_${obs_seq}H* clm_init_*
+
+ msrcp -pe 1000 -pr $proj_num -wpwd $write_pass -comment "write password $write_pass" \
+ cam_analyses.tar mss:${ms_dir}/cam_analyses.tar >>& $saved &
+ rm c[al]m_init_*.nc
+else
+ echo "NO mean2cam_init.csh, so no CAM initial file format analyses created"
+endif
+
+#-----------------------------
+
+if (! -e $diag_name && ! -e ${diag_name}.gz) tar -c -v -f $diag_name -X tar_excl_list * >>& $saved
rm tar_excl_list
if ($compress == true) then
- gzip $diag_name
- set diag_name = ${diag_name}.gz
+ if (-e $diag_name) then
+ gzip $diag_name >>& $saved
+ set diag_name = ${diag_name}.gz
+ else
+ echo "$diag_name does not exist at gzip" >> $saved
+ exit
+ endif
endif
echo "files will be written to ${ms_dir}/${diag_name}" >> $saved
@@ -91,17 +135,6 @@
msrcp -pe 1000 -pr $proj_num -wpwd $write_pass -comment "write password $write_pass" \
${diag_name} mss:${ms_dir}/${diag_name} >>& $saved
-
-# check to see if it's okay to remove diagnostics.tar and P*.nc (leave obs_seq.final)
-# Better check would be to add before the gzip
-# tar -t -f diagnostics.tar > table
-# foreach name (Posterior_Diag.nc Prior_Diag.nc obs_seq.final)
-# grep $name table
-# if ($status != 0) then
-# set tarok = false
-# endif
-# end
-#
set list = `ls -l $diag_name`
set local_size = $list[5]
set list = `msls -l ${ms_dir}/${diag_name}`
@@ -110,8 +143,8 @@
if ($local_size == $ms_size) then
echo "Archived files with write password $write_pass" >> $saved
- echo "msrcp of $ms_dir/$obs_seq succeeded; REMOVING $diag_name and P*.nc " >> $saved
- rm $diag_name P*.nc
+ echo "msrcp of $ms_dir/$obs_seq succeeded; REMOVING $diag_name and P*Diag.nc " >> $saved
+ rm $diag_name P*Diag.nc
else
echo "msrcp of ${ms_dir}/$obs_seq failed; " >> $saved
echo "NOT removing $diag_name and P*.nc" >> $saved
Deleted: DART/trunk/models/cam/full_experiment/auto_re2ms.csh
===================================================================
--- DART/trunk/models/cam/full_experiment/auto_re2ms.csh 2008-07-08 18:54:15 UTC (rev 3447)
+++ DART/trunk/models/cam/full_experiment/auto_re2ms.csh 2008-07-08 19:00:28 UTC (rev 3448)
@@ -1,259 +0,0 @@
-#!/bin/csh
-#
-# Data Assimilation Research Testbed -- DART
-# Copyright 2004-2007, Data Assimilation Research Section
-# University Corporation for Atmospheric Research
-# Licensed under the GPL -- www.gpl.org/licenses/gpl.html
-#
-# <next few lines under version control, do not edit>
-# $URL$
-# $Id$
-# $Revision$
-# $Date$
-
-# script for copying 1 day/obs_seq of restart files to the mass store.
-# CAM,CLM, and possibly filter_ic for each ensemble member are lumped together
-# so that we can retrieve a subset of the ensemble members for a new experiment.
-# Then it lumps together ensemble members into batches to reduce the number of files.
-
-# change for lightning?
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:/usr/local/dcs/lib
-echo $LD_LIBRARY_PATH
-
-# set echo verbose
-
-
-if ($#argv < 2) then
- echo "usage; from case/experiment/obs_seq directory; "
- echo " auto_re2ms.csh num_ens_members num_per_batch compress(optnl)"
- echo " Compresses caminput and clminput files."
- echo " tars ensemble members together into batches"
- exit
-else if ($#argv == 2) then
- set comp = ' '
-else if ($#argv == 3) then
- set comp = .cmp
-endif
-
-set num_ens = $1
-set num_per_batch = $2
-
-# fix this for your local system accounting
-set proj_num = NNNNN
-set ret_period = 1000
-set write_pass = da$$
-echo "with write password $write_pass" >! saved_restart
-
-if ($num_per_batch > $num_ens) exit "restart2ms; num_per_batch > num_ens"
-
-# Parse parts of the path name for construction of MSS name.
-set direct = `pwd`
-set obs_seq = $direct:t
-
-cd ..
-set direct = `pwd`
-set exp_dir = $direct:t
-
-cd ..
-set direct = `pwd`
-set case = $direct:t
-
-cd ${exp_dir}/${obs_seq}
-
-# New section for archiving initial ensembles from a spin-up
-# set months = (Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)
-# echo $obs_seq >! obs_seq_name
-
-# # This section implemented for set of ICs 2x per MONTH.
-# set STRING = "1,$ s#_# #g"
-# set parts = `sed -e "$STRING" obs_seq_name`
-# \rm obs_seq_name
-# set nick = $parts[2]
-# # This assumes 2 obs_seqs / month
-# @ month = ($nick / 2) + 1
-# @ month = $month % 12
-# @ day = (($nick % 2) * 14) + 1
-# set date = $months[$month]_$day
-
-# # For ICs every 5 days
-# set STRING = "1,$ s#-# #g"
-# set parts = `sed -e "$STRING" obs_seq_name`
-# \rm obs_seq_name
-# set month = $parts[1]
-# set day = $parts[2]
-# set date = $months[$month]_$day
-# END of spin-up section
-
-# orig
-set ms_root = /RAEDER/DAI/$case/${exp_dir}/${obs_seq}/${1}x${2}
-# A year of ICs goes somewhere else
-# 2x monthly from somewhere besides CAM_init
-# set ms_root = /RAEDER/DAI/CAM_init/$case/${exp_dir}/${date}/${1}x${2}
-# 5 day series
-# set ms_root = /RAEDER/DAI/CAM_init/$case/${date}/${1}x${2}
-set ms_dir = mss:$ms_root
-echo "files will be written to ${ms_root}/batch#" >> saved_restart
-echo "files will be written to ${ms_root}/batch#"
-
-# Figure out how many files to divide ensemble members among
-@ nbatch = $num_ens / $num_per_batch
-if ($num_ens % $num_per_batch != 0 ) @ nbatch++
-
-set opts = "-pe $ret_period -pr $proj_num -wpwd $write_pass"
-set p1 = ' -comment "write password '
-set p2 = ' "'
-set msrcp_opts = "$opts $p1 $write_pass $p2"
-echo $msrcp_opts >> saved_restart
-
-set ok_to_remove = true
-
-# Tally up and list the extra (non-filter) ic files
-# Add more if blocks as needed.
-set DART_list = ' '
-set num_files = 0
-set do_filter = false
- if (-e DART/assim_tools_ics) then
- set DART_list = ($DART_list assim_tools_ics)
- @ num_files++
- endif
- if (-e DART/prior_inf_ic) then
- set DART_list = ($DART_list prior_inf_ic)
- @ num_files++
- endif
- if (-e DART/post_inf_ic) then
- set DART_list = ($DART_list post_inf_ic)
- @ num_files++
- endif
- if (-e DART/filter_ic) then
- set DART_list = ($DART_list filter_ic)
- @ num_files++
- else if (-e DART/filter_ic*0$num_ens || -e DART/filter_ic*.$num_ens) then
- set do_filter = true
- else
- echo 'NOT ENOUGH ENSEMBLE MEMBERS IN .../DART' >> saved_restart
- exit
- endif
-
-echo "DART_list = ($DART_list) " >> saved_restart
-echo "DART_list = ($DART_list) "
-
-# convert wordlist into a single character string (insert ,s)
-set n = 1
-if ($n <= $num_files) then
- set DART_files = $DART_list[1]
- while ($n < $num_files)
- @ n++
- set DART_files = "$DART_files,$DART_list[$n]"
- end
-else
- set DART_files = 'none'
-endif
-
-if ($DART_files != 'none' && do_filter == false) then
- tar -c -f ic_files.tar DART/{$DART_files}
- msrcp $msrcp_opts ic_files.tar ${ms_dir}/DART/ic_files.tar &
-endif
-
-# Do this here, manually, or within tar; -z not available on tempest
-# also, not effective to compress filter_ic files (big and dense)
-set fil = CAM/caminput_1.*
-if ($comp != ' ' && $fil:e != 'gz') then
- gzip -r CAM
- gzip -r CLM
- echo "compressed CAM and CLM" >> saved_restart
-endif
-
-set batch = 1
-while($batch <= $nbatch)
-
- @ base = (($batch - 1) * $num_per_batch) + 1
- @ member = $base
- if ($batch == 1 && (-e CAM/caminput_0.nc || -e CAM/caminput_0.nc.gz)) \
- set member = 0
-# BUT will there be a filter_ic.0000 ?
- echo base and member $base $member >> saved_restart
- echo base and member $base $member
-
- # create the tar file using the first ensemble member of this batch
- if ($do_filter == true) then
- # if (-e DART/assim_tools_ics && $member <= $base) then
- if ($DART_files != 'none' && $member <= $base) then
-# won't work for $member < $base?
- tar -c -f batch${batch}${comp} CAM/caminput_$member.nc* \
- CLM/clminput_$member.nc* DART/filter_ic*[.0]$member DART/{$DART_files}
- echo "tar -c -f batch${batch}${comp} CAM/caminput_$member.nc* \
- CLM/clminput_$member.nc* DART/filter_ic*[.0]$member DART/{$DART_files} "
- else
- tar -c -f batch${batch}${comp} CAM/caminput_$member.nc* \
- CLM/clminput_$member.nc* DART/filter_ic*[.0]$member
- echo "tar -c -f batch${batch}${comp} CAM/caminput_$member.nc* \
- CLM/clminput_$member.nc* DART/filter_ic*[.0]$member "
- endif
- else
- tar -c -f batch${batch}${comp} CAM/caminput_$member.nc* CLM/clminput_$member.nc*
- endif
- # tack on additional ens members until this batch is complete
- set n = 2
- while ($n <= $num_per_batch)
- @ member++
-# This looks wrong; $n will never be big enough, unless something crazy happens.
-# if ($n > $num_ens) then
- if ($member > $num_ens) then
- @ batch = $nbatch + 1
- @ n = $num_per_batch + 1
- else
- if ($do_filter == true) then
- tar -r -f batch${batch}${comp} CAM/caminput_$member.nc* \
- CLM/clminput_$member.nc* DART/filter_ic*[.0]$member
- echo "tar -r -f batch${batch}${comp} CAM/caminput_$member.nc* \
- CLM/clminput_$member.nc* DART/filter_ic*[.0]$member "
- else
- tar -r -f batch${batch}${comp} CAM/caminput_$member.nc* CLM/clminput_$member.nc*
- endif
- endif
- @ n++
- end
-
- msrcp $msrcp_opts batch${batch}${comp} ${ms_dir}/batch${batch}${comp}
- if ($batch < $nbatch) rm batch${batch}${comp}
-
- @ batch++
-end
-# Correct batch back to value of last batch, for use below.
-@ batch = $batch - 1
-
-msrcp $msrcp_opts batch*${comp} ${ms_dir}
-
-wait
-
-# Check to see if it's okay to remove DART directory
-if ($do_filter != true) then
- set list = `ls -l DART/filter_ic`
- set local_size = $list[5]
- set list = `msls -l ${ms_root}/DART/filter_ic`
- set ms_size = $list[5]
- echo "local_size ms_size = $local_size $ms_size" >> saved_restart
- if ($local_size != $ms_size) set ok_to_remove = false
-endif
-
-# Check to see if it's okay to remove CAM/CLM directories
-set list = `ls -l batch${batch}${comp}`
-set local_size = $list[5]
-set list = `msls -l ${ms_root}/batch${batch}${comp}`
-set ms_size = $list[5]
-echo " CAM/CLM(/DART) local_size ms_size = $local_size $ms_size" >> saved_restart
-if ($local_size != $ms_size) set ok_to_remove = false
-
-
-if ($ok_to_remove == true) then
- echo "Archived files with write password $write_pass" >> saved_restart
- echo "msrcp of ${ms_root} succeeded; REMOVING $obs_seq DART,CAM,CLM" >> saved_restart
- rm -rf DART CAM CLM
- rm batch${batch}${comp}
-else
- echo "msrcp of ${ms_root} failed; NOT removing $obs_seq DART,CAM,CLM" >> saved_restart
-endif
-
-chmod 444 saved_restart
-
-exit
Modified: DART/trunk/models/cam/full_experiment/auto_re2ms_LSF.csh
===================================================================
--- DART/trunk/models/cam/full_experiment/auto_re2ms_LSF.csh 2008-07-08 18:54:15 UTC (rev 3447)
+++ DART/trunk/models/cam/full_experiment/auto_re2ms_LSF.csh 2008-07-08 19:00:28 UTC (rev 3448)
@@ -25,8 +25,9 @@
#
#BSUB -J restart2ms
#BSUB -o restart2ms.%J.log
-#BSUB -q standby
-#BSUB -W 1:00
+#BSUB -P [project number]
+#BSUB -q share
+#BSUB -W 2:00
#BSUB -n 1
#xxxx -x
#xxxx -R "span[ptile=1]"
@@ -36,6 +37,11 @@
pwd
cd $LS_SUBCWD
+set comp = ''
+set ret_period = 1000
+set proj_num = #####
+set write_pass = $$
+
set num_ens = 1
while (-e CAM/caminput_${num_ens}.nc && ! -z CAM/caminput_${num_ens}.nc)
@ num_ens++
@@ -55,10 +61,7 @@
echo "size_element = $size_element"
# This assumes compression factor of .8, from CAM and CLM only
# (2^31 - 1) / .8 = 2680000000
-# orig;
@ test_element = 26500000 / $size_element
-# gz
-# @ test_element = 21000000 / $size_element
set div = 1
set size_file = ${num_ens}
echo div, size_file are $div $size_file
@@ -77,8 +80,189 @@
echo " for num_ens = $num_ens"
-../../auto_re2ms.csh $num_ens $num_per_batch comp
+touch saved_restart
+echo "------------------------------------------------" >> saved_restart
+echo "with write password $write_pass" >> saved_restart
+if ($num_per_batch > $num_ens) exit "restart2ms; num_per_batch > num_ens"
+
+# Parse parts of the path name for construction of MSS name.
+set direct = `pwd`
+set obs_seq = $direct:t
+
+cd ..
+set direct = `pwd`
+set exp_dir = $direct:t
+
+cd ..
+set direct = `pwd`
+set case = $direct:t
+
+cd ${exp_dir}/${obs_seq}
+
+set ms_root = /RAEDER/DAI/$case/${exp_dir}/${obs_seq}/${num_ens}x${num_per_batch}
+set ms_dir = mss:$ms_root
+echo "files will be written to ${ms_root}/batch#" >> saved_restart
+echo "files will be written to ${ms_root}/batch#"
+
+# Figure out how many files to divide ensemble members among
+@ nbatch = $num_ens / $num_per_batch
+if ($num_ens % $num_per_batch != 0 ) @ nbatch++
+
+set opts = "-pe $ret_period -pr $proj_num -wpwd $write_pass -comment "
+set p1 = "write password $write_pass"
+echo $opts $p1 >> saved_restart
+
+set ok_to_remove = true
+
+# Tally up and list the extra (non-ensemble) ic files
+# Add more 'if' blocks as needed.
+set DART_list = ' '
+set num_files = 0
+set do_filter = false
+if (-e DART/prior_inf_ic) then
+ set DART_list = ($DART_list prior_inf_ic)
+ @ num_files++
+endif
+if (-e DART/post_inf_ic) then
+ set DART_list = ($DART_list post_inf_ic)
+ @ num_files++
+endif
+if (-e DART/filter_ic) then
+ set DART_list = ($DART_list filter_ic)
+ @ num_files++
+else if (-e DART/filter_ic*0$num_ens || -e DART/filter_ic*.$num_ens) then
+ set do_filter = true
+else
+ echo 'NOT ENOUGH ENSEMBLE MEMBERS IN .../DART' >> saved_restart
+ exit
+endif
+echo "DART_list = ($DART_list) " >> saved_restart
+echo "DART_list = ($DART_list) "
+
+# Convert wordlist into a single character string (insert ,s)
+set n = 1
+if ($n <= $num_files) then
+ set DART_files = $DART_list[1]
+ while ($n < $num_files)
+ @ n++
+ set DART_files = "$DART_files,$DART_list[$n]"
+ end
+else
+ set DART_files = 'none'
+endif
+
+if ($DART_files != 'none' && $do_filter == false) then
+ tar -c -f ic_files.tar DART/{$DART_files}
+ msrcp $opts "$p1" ic_files.tar ${ms_dir}/DART/ic_files.tar &
+endif
+
+# Compress here, instead of within tar; -z not available on some machines.
+# It's not effective to compress filter_ic files (big and dense)
+set fil = CAM/caminput_1.*
+if ($comp != '' && $fil:e != 'gz') then
+ gzip -r CAM
+ gzip -r CLM
+ echo "compressed CAM and CLM" >> saved_restart
+endif
+
+set batch = 1
+while($batch <= $nbatch)
+
+ @ base = (($batch - 1) * $num_per_batch) + 1
+ @ member = $base
+ if ($batch == 1 && (-e CAM/caminput_0.nc || -e CAM/caminput_0.nc.gz)) set member = 0
+# BUT will there be a filter_ic.0000 ?
+ echo base and member $base $member >> saved_restart
+ echo base and member $base $member
+
+# create the tar file using the first ensemble member of this batch
+ if ($do_filter == true) then
+ if ($DART_files != 'none' && $member <= $base) then
+# won't work for $member < $base?
+ tar -c -f batch${batch}${comp} CAM/caminput_$member.nc* \
+ CLM/clminput_$member.nc* DART/filter_ic*[.0]$member DART/{$DART_files}
+ echo "tar -c -f batch${batch}${comp} CAM/caminput_$member.nc* \
+ CLM/clminput_$member.nc* DART/filter_ic*[.0]$member DART/{$DART_files} "
+ else
+ tar -c -f batch${batch}${comp} CAM/caminput_$member.nc* \
+ CLM/clminput_$member.nc* DART/filter_ic*[.0]$member
+ echo "tar -c -f batch${batch}${comp} CAM/caminput_$member.nc* \
+ CLM/clminput_$member.nc* DART/filter_ic*[.0]$member "
+ endif
+ else
+ tar -c -f batch${batch}${comp} CAM/caminput_$member.nc* CLM/clminput_$member.nc*
+ endif
+ # tack on additional ens members until this batch is complete
+ set n = 2
+ while ($n <= $num_per_batch)
+ @ member++
+ if ($member > $num_ens) then
+ @ batch = $nbatch + 1
+ @ n = $num_per_batch + 1
+ else
+ if ($do_filter == true) then
+ tar -r -f batch${batch}${comp} CAM/caminput_$member.nc* \
+ CLM/clminput_$member.nc* DART/filter_ic*[.0]$member
+ echo "tar -r -f batch${batch}${comp} CAM/caminput_$member.nc* \
+ CLM/clminput_$member.nc* DART/filter_ic*[.0]$member "
+ else
+ tar -r -f batch${batch}${comp} CAM/caminput_$member.nc* CLM/clminput_$member.nc*
+ endif
+ endif
+ @ n++
+ end
+
+# Do all at once, outside loop, for efficient MS access
+ msrcp $opts "$p1" batch${batch}${comp} ${ms_dir}/batch${batch}${comp}
+ if ($batch < $nbatch) rm batch${batch}${comp}
+
+ @ batch++
+end
+#
+# Correct batch back to value of last batch, for use below.
+@ batch = $batch - 1
+
+# msrcp $opts "$p1" batch*${comp} ${ms_dir}/batch*${comp}
+# $opts and "p1" contents are being interpretted as files,
+# maybe because of the multi-file copy batch*...
+# ls -l batch*${comp}
+# echo "msrcp $opts $p1 batch*${comp} ${ms_dir}" >> saved_restart
+# msrcp $opts "$p1" batch*${comp} ${ms_dir}
+
+wait
+
+# Check to see if it's okay to remove DART directory
+if ($do_filter != true) then
+ set list = `ls -l DART/filter_ic`
+ set local_size = $list[5]
+ set list = `msls -l ${ms_root}/DART/filter_ic`
+ set ms_size = $list[5]
+ echo "local_size ms_size = $local_size $ms_size" >> saved_restart
+ if ($local_size != $ms_size) set ok_to_remove = false
+endif
+
+# Check to see if it's okay to remove CAM/CLM directories
+set list = `ls -l batch${batch}${comp}`
+set local_size = $list[5]
+set list = `msls -l ${ms_root}/batch${batch}${comp}`
+set ms_size = $list[5]
+echo " CAM/CLM(/DART) local_size ms_size = $local_size $ms_size" >> saved_restart
+if ($local_size != $ms_size) set ok_to_remove = false
+
+if ($ok_to_remove == true) then
+ echo "Archived files with write password $write_pass" >> saved_restart
+ echo "msrcp of ${ms_root} succeeded; REMOVING $obs_seq DART,CAM,CLM" >> saved_restart
+ rm -rf DART CAM CLM
+ # rm batch${batch}${comp}
+ rm batch*${comp}
+else
+ echo "msrcp of ${ms_root} failed; NOT removing $obs_seq DART,CAM,CLM" >> saved_restart
+endif
+
+chmod 444 saved_restart
+#================================================
+
echo "finished with auto_re2ms at " `date`
exit
Deleted: DART/trunk/models/cam/full_experiment/diag2ms.csh
===================================================================
--- DART/trunk/models/cam/full_experiment/diag2ms.csh 2008-07-08 18:54:15 UTC (rev 3447)
+++ DART/trunk/models/cam/full_experiment/diag2ms.csh 2008-07-08 19:00:28 UTC (rev 3448)
@@ -1,65 +0,0 @@
-#!/bin/csh
-#
-# Data Assimilation Research Testbed -- DART
-# Copyright 2004-2007, Data Assimilation Research Section
-# University Corporation for Atmospheric Research
-# Licensed under the GPL -- www.gpl.org/licenses/gpl.html
-#
-# <next few lines under version control, do not edit>
-# $URL$
-# $Id$
-# $Revision$
-# $Date$
-
-# script for copying 1 day/obs_seq of output diagnostics to mass store.
-
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:/usr/local/dcs/lib
-echo $LD_LIBRARY_PATH
-
-# set echo verbose
-
-set compress = false
-
-# echo "usage; from case/experiment/obs_seq directory; "
-# echo " diag2ms compress(optnl)"
-# echo " optionally compresses diagnostic files."
-# exit
-
-if ($#argv == 1) then
- set compress = true
-endif
-
-set direct = `pwd`
-set obs_seq = $direct:t
-
-cd ..
-set direct = `pwd`
-set case = $direct:t
-
-cd ..
-set direct = `pwd`
-set exp_dir = $direct:t
-
-cd $case/${obs_seq}
-set ms_dir = /RAEDER/DAI/${exp_dir}/$case/${obs_seq}
-echo files will be written to ${ms_dir}/diagnostics.tar
-
-tar cv -f diagnostics.tar --exclude saved [^CD]* >& saved
-if ($compress == true) then
- gzip diagnostics.tar
- msrcp -pe 365 diagnostics.tar.gz mss:${ms_dir}/diagnostics.tar.gz
-else
- msrcp -pe 365 diagnostics.tar mss:${ms_dir}/diagnostics.tar
-endif
-
-msls -l "${ms_dir}/diagnostics.tar*"
-if ($status == 0) then
- rm diagnostics.tar.gz
- echo ${ms_dir} was copied to mass store >> saved
-else
- echo ${ms_dir} was NOT copied to mass store
-endif
-
-echo "msrcp done; check files on mass store, and then delete local copies"
-
-exit
Deleted: DART/trunk/models/cam/full_experiment/diags2ms_LSF.csh
===================================================================
--- DART/trunk/models/cam/full_experiment/diags2ms_LSF.csh 2008-07-08 18:54:15 UTC (rev 3447)
+++ DART/trunk/models/cam/full_experiment/diags2ms_LSF.csh 2008-07-08 19:00:28 UTC (rev 3448)
@@ -1,77 +0,0 @@
-#!/bin/csh
-#
-# Data Assimilation Research Testbed -- DART
-# Copyright 2004-2007, Data Assimilation Research Section
-# University Corporation for Atmospheric Research
-# Licensed under the GPL -- www.gpl.org/licenses/gpl.html
-#
-# <next few lines under version control, do not edit>
-# $URL$
-# $Id$
-# $Revision$
-# $Date$
-
-# script for copying multiple day/obs_seq of diagnostics files to mass store.
-# For lack of better bookkeeping, the diag2ms.csh script needs to be copied
-# to $HOME ... at some point, this may be streamlined by integrating diag2ms.csh
-# into this script. TJH 9/30/2005
-
-#### LSF options for BSUB
-### -J job name (master script job.csh presumes filter.xxxx.log)
-### -o output listing filename
-### -P account number
-### -q queue
-### -n number of tasks (processors)
-### -x exclusive use of node
-### -R "span[ptile=(num procs you want on each node)]"
-#
-#BSUB -J diags2ms
-#BSUB -o diags2ms.%J.log
-#BSUB -q standby
-#BSUB -n 1
-#xxxx -x
-#xxxx -R "span[ptile=1]"
-
-set compress = true
-set obs_root = 01_
-set obs_seq_first = 1
-set obs_seq_last = 58
-
-
-date
-echo 'diags2ms starts in'
-pwd
-cd $LS_SUBCWD
-
-
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:/usr/local/dcs/lib
-echo $LD_LIBRARY_PATH
-
-# set echo verbose
-
-# if ($#argv < 2) then
-# echo "usage; from case/experiment directory; "
-# echo " diags2ms obs_seq_first obs_seq_last compress(optnl)"
-# echo " Optionally Compresses tar files."
-# echo " ASSUMES obs_root of 01_; change for other months/roots"
-# exit
-# else if ($#argv == 3) then
-# set compress = true
-# endif
-
-set obs_seq = $obs_seq_first
-while ($obs_seq <= $obs_seq_last)
- set obs = ${obs_root}
- if ($obs_seq < 10) set obs = ${obs_root}0
- set obs = $obs$obs_seq
- cd $obs
- if ($compress == true) then
- ~/diag2ms.csh $compress
- else
- ~/diag2ms.csh
- endif
- cd ..
- @ obs_seq++
-end
-
-exit
Added: DART/trunk/models/cam/full_experiment/mean2cam_init.csh
===================================================================
--- DART/trunk/models/cam/full_experiment/mean2cam_init.csh (rev 0)
+++ DART/trunk/models/cam/full_experiment/mean2cam_init.csh 2008-07-08 19:00:28 UTC (rev 3448)
@@ -0,0 +1,88 @@
+#!/bin/csh
+
+# Extract the mean copy for each field from each of the timeslots on
+# a series of files (Posterior usually) and put each timeslot into
+# CAM initial file format.
+# What about the CLM initial files? Can't average some fields on them.
+# Does it make a difference to short forecasts?
+
+# set echo verbose
+
+# P[oste]rior_core.nc files will have copies 1,2,83,84, but ordered as 83,84,1,2
+# ncks will call copy 83 'copy 1', so those files will require
+# dim_val = 3
+
+set ms_file = $1
+set local_dir = $2
+set kind = $3
+set dim = $4
+set element1 = $5
+set yrmoday = $6
+
+set num_times = 4
+set hours = (06 12 18 24)
+set ens_member = 1
+
+if (! -d $local_dir) mkdir $local_dir
+cd $local_dir
+
+if (! -e ${kind}.nc) then
+ if (! -e ${kind}_Diag.nc) then
+ if (! -e diagnostics.tar) then
+ if (! -e diagnostics.tar.gz) then
+ echo "starting msrcp at `date`"
+ msrcp mss:$ms_file .
+ endif
+ echo "starting gunzip at `date`"
+# ? skip this and add -z to tar -x below? That would short-circuit some of
+# the failure recovery I've built in, but in this context I can't use it.
+ gunzip diagnostics.tar.gz
+ endif
+ echo "starting tar -x at `date`"
+ tar -x -f diagnostics.tar ${kind}_Diag.nc
+ if ($status == 0) rm diagnostics.tar &
+ endif
+
+ set time = 1
+ while ($time <= $num_times)
+ set out_name = cam_init_${yrmoday}$hours[$time].nc
+ # Template into which we'll stuff the mean fields
+# cp ../caminput.nc $out_name
+# This could be a mv, if it's reliable enough
+ cp cam_init_memb${ens_member}_H$hours[$time].nc $out_name
+
+ # ncks makes the time/record dimension of the out_file to be the largest
+ # of the sizes of the in_ and out_file, if out_file pre-exists.
+ # Prevent this by extracting one timeslot to a new file.
+ # (This had -x -v copy in my first successful test, but that's redundant(?)
+ # with the ncwa, where'd I'd like to handle all the 'copy' purging.
+ ncks -F -A -d ${dim},${element1} -d time,$time ${kind}_Diag.nc time_copy_slab.nc
+ if ($status != 0) then
+ echo "timeslot $time not found on ${kind}_Diag.nc"
+ exit
+ endif
+ # Exclude the copy dimension and variable from new file,
+ # along with mis-dimensioned P0
+ # ncwa will change the rank; use it to remove dimension 'copy'
+ ncwa -x -v P0,CopyMetaData -a ${dim} time_copy_slab.nc avgd_copy_out.nc
+ # Permute the remaining dimensions into CAM; (time, lev, [s]lat, [s]lon)
+ # from DART (time, lat, [s]lon, [s]lev)
+ # Requires 3 calls to handle the 3 (staggered) grids
+ ncpdq -a lev,lat,lon avgd_copy_out.nc re-orderedT.nc
+ ncpdq -a lev,slat,lon re-orderedT.nc re-orderedUS.nc
+ ncpdq -a lev,lat,slon re-orderedUS.nc re-orderedVS.nc
+ # Finally, stuff the contents into the CAM initial file template
+ # -A causes variables to be overwritten if there is a name "conflict",
+ # without querying the user/script.
+ # -x -v $dim removes the *variable* called copy. (On blueice) it didn't work
+ # to try to remove it in the ncwa command.
+ ncks -A -x -v ${dim} re-orderedVS.nc $out_name
+
+ @ time++
+ rm time_copy_slab.nc avgd_copy_out.nc re-order*
+ end
+ cd ..
+
+endif
+
+exit
Property changes on: DART/trunk/models/cam/full_experiment/mean2cam_init.csh
___________________________________________________________________
Name: svn:executable
+ *
Added: DART/trunk/models/cam/full_experiment/obs_seq2ms.csh
===================================================================
--- DART/trunk/models/cam/full_experiment/obs_seq2ms.csh (rev 0)
+++ DART/trunk/models/cam/full_experiment/obs_seq2ms.csh 2008-07-08 19:00:28 UTC (rev 3448)
@@ -0,0 +1,152 @@
+#!/usr/local/bin/tcsh
+# Doesn't let [^is]* work #!/bin/csh
+
+#BSUB -J obs_seq2ms
+#BSUB -o obs_seq2ms.%J.log
+#BSUB -P 93300315
+#BSUB -q share
+#BSUB -W 3:00
+#BSUB -n 1
+#BSUB -R "span[ptile=1]"
+#
+##=============================================================================
+# Script for archiving months of obs_seq.final files for easier access than diagnostics.tar.gz
+
+if ($?LS_SUBCWD) then
+ cd $LS_SUBCWD
+endif
+
+set obs_seq_first = 1
+set mo_first = 8
+set mo_last = 8
+set input_root = obs_
+set digits = 4
+# set input_root = 08_
+# set digits = 2
+set saved = saved_obs_seq_${mo_first}-${mo_last}
+
+set remove_finals = true
+
+set year = 2006
+
+set months = (Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)
+set mo_days = (31 28 31 30 31 30 31 31 30 31 30 31)
+
+# fix this for your local system accounting
+set proj_num = 93300315
+set ret_period = 1000
+set write_pass = da$$
+echo "with write password $write_pass" >! $saved
+
+#-----------------------------------------------------------------------------
+# Parse parts of the path name for construction of MSS name.
+set direct = `pwd`
+set exp_dir = $direct:t
+
+cd ..
+set direct = `pwd`
+set case = $direct:t
+
+cd ${exp_dir}
+
+set ms_root = /RAEDER/DAI/$case/${exp_dir}
+set ms_dir = mss:$ms_root
+echo "files will be written to ${ms_root}/MM_obs_seq.tar" >> $saved
+
+set opts = "-pe $ret_period -pr $proj_num -wpwd $write_pass -comment "
+set p1 = "write password $write_pass"
+echo $opts $p1 >> $saved
+#
+#-----------------------------------------
+# Figure out how many months to archive
+if ($mo_last >= $mo_first) then
+ @ mo_num = ($mo_last - $mo_first) + 1
+else
+ @ mo_num = $mo_last + (12 - $mo_first) + 1
+endif
+
+#-----------------------------------------
+
+set ok_to_remove = true
+
+set mo = 1
+set mo_cal = $mo_first
+@ obs_seq_n = $obs_seq_first - 1
+echo "starting mo loop with mo mo_cal mo_num obs_seq_n = $mo $mo_cal $mo_num $obs_seq_n" >> $saved
+# set echo verbose
+while($mo <= $mo_num)
+ if ($mo_cal == 13) then
+ @ year++
+ set mo_cal = 1
+ endif
+
+ @ leap = $year % 4
+ if ($leap == 0) set mo_days[2] = 29
+
+ @ obs_seq_1 = $obs_seq_n + 1
+ @ obs_seq_n = ($obs_seq_1 + $mo_days[${mo_cal}]) - 1
+
+ # create the tar file using the first obs_seq/day of this month
+ set obs_seq = $obs_seq_1
+ set input_dir = `printf "%s%0${digits}d" ${input_root} $obs_seq`
+
+ set out_file = $months[${mo_cal}]_obs_seq.tar
+ echo "out_file and input_dir = "$out_file $input_dir >> $saved
+
+ tar -c -f $out_file ${input_dir}/{obs_seq.final,input.nml}
+
+ # tack on additional ens members until this month is complete
+ @ obs_seq++
+ echo "starting obs_seq loop " >> $saved
+
+ while ($obs_seq <= $obs_seq_n)
+ set input_dir = `printf "%s%0${digits}d" ${input_root} $obs_seq`
+ if ( -e ${input_dir}/obs_seq.final) then
+ tar -r -f $out_file ${input_dir}/{obs_seq.final,input.nml}
+ else
+ # finish up
+ echo "obs_seq $obs_seq does not exist; finishing" >> $saved
+ @ mo = $mo_num + 1
+ @ obs_seq = $obs_seq_n + 1
+ if ($mo_cal != $mo_last) then
+ echo "DID NOT FINISH ARCHIVE; $mo_cal is last month done" >> $saved
+ endif
+ endif
+ @ obs_seq++
+ end
+
+ msrcp $opts "$p1" $out_file ${ms_dir}/$out_file
+
+# Check to see if it's okay to remove obs_seq.final, etc
+ set list = `ls -l $out_file`
+ set local_size = $list[5]
+ set list = `msls -l ${ms_root}/$out_file`
+ set ms_size = $list[5]
+ echo "for $months[${mo_cal}] local_size, ms_size = $local_size $ms_size" >> $saved
+ if ($local_size == $ms_size) then
+ rm $out_file &
+ echo "msrcp of ${ms_root}/$out_file SUCCEEDED " >> $saved
+ echo "Archived obs_seq.final files with write password $write_pass" >> $saved
+ if ($remove_finals == true) then
+ set obs_seq = $obs_seq_1
+ while ($obs_seq <= $obs_seq_n)
+ set input_dir = `printf "%s%0${digits}d" ${input_root} $obs_seq`
+ rm ${input_dir}/[^is]* &
+ @ obs_seq++
+ end
+ echo "REMOVED $out_file, ./obs_seq.final " >> $saved
+ endif
+ else
+ echo "msrcp failed; NOT removing $months[${mo_cal}]: $obs_seq_1 - $obs_seq_n " >> $saved
+ set mo = $mo_num + 1
+ endif
+
+ @ mo++
+ @ mo_cal++
+end
+
+wait
+
+chmod 444 $saved
+
+exit
Property changes on: DART/trunk/models/cam/full_experiment/obs_seq2ms.csh
___________________________________________________________________
Name: svn:executable
+ *
More information about the Dart-dev
mailing list