[Dart-dev] [4019] DART/trunk/observations/AIRS: The scripts used to generate multiple days of AIRS data files, with
nancy at ucar.edu
nancy at ucar.edu
Thu Aug 27 14:19:52 MDT 2009
An HTML attachment was scrubbed...
URL: http://mailman.ucar.edu/pipermail/dart-dev/attachments/20090827/4098b79d/attachment.html
-------------- next part --------------
Added: DART/trunk/observations/AIRS/shell_scripts/README
===================================================================
--- DART/trunk/observations/AIRS/shell_scripts/README (rev 0)
+++ DART/trunk/observations/AIRS/shell_scripts/README 2009-08-27 20:19:52 UTC (rev 4019)
@@ -0,0 +1,7 @@
+
+These scripts are intended to help download the original AIRS hdf
+data files, convert them in bulk, and merge the resulting obs_seq files.
+
+In most cases, they're intended to be copied over to the ../work directory
+and then customized for the particular time period and local directory names.
+
Added: DART/trunk/observations/AIRS/shell_scripts/download.sh
===================================================================
--- DART/trunk/observations/AIRS/shell_scripts/download.sh (rev 0)
+++ DART/trunk/observations/AIRS/shell_scripts/download.sh 2009-08-27 20:19:52 UTC (rev 4019)
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# download the requested tar files from the NCAR mass store.
+
+# set the first and last days. can roll over
+# month and year boundaries now!
+let start_year=2006
+let start_month=10
+let start_day=1
+
+let end_year=2007
+let end_month=1
+let end_day=31
+
+# end of things you should have to set in this script
+
+# convert the start and stop times to gregorian days, so we can
+# compute total number of days including rolling over month and
+# year boundaries. make sure all values have leading 0s if they
+# are < 10. do the end time first so we can use the same values
+# to set the initial day while we are doing the total day calc.
+mon2=`printf %02d $end_month`
+day2=`printf %02d $end_day`
+end_d=(`echo ${end_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+mon2=`printf %02d $start_month`
+day2=`printf %02d $start_day`
+start_d=(`echo ${start_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+curday=(`echo ${start_year}${mon2}${day2}00 0 | ./advance_time`)
+
+# how many total days are going to be converted (for the loop counter)
+let totaldays=${end_d[0]}-${start_d[0]}+1
+
+# loop over each day
+let d=1
+while (( d <= totaldays)) ; do
+
+ # parse out the parts from a string which is YYYYMMDDHH
+ year=${curday:0:4}
+ month=${curday:4:2}
+ day=${curday:6:2}
+
+
+ echo getting ${year}${month}${day}.tar from mass store
+ msrcp mss:/MIJEONG/AIRS/V5/L2/${year}${month}/${year}${month}${day}.tar .
+
+
+ # advance the day; the output is YYYYMMDD00
+ curday=(`echo ${year}${month}${day}00 +1d | ./advance_time`)
+
+ # advance the loop counter
+ let d=d+1
+
+done
+
+exit 0
+
Property changes on: DART/trunk/observations/AIRS/shell_scripts/download.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: DART/trunk/observations/AIRS/shell_scripts/mergeit.sh
===================================================================
--- DART/trunk/observations/AIRS/shell_scripts/mergeit.sh (rev 0)
+++ DART/trunk/observations/AIRS/shell_scripts/mergeit.sh 2009-08-27 20:19:52 UTC (rev 4019)
@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# merge the files into "daily" files which start at 03:01Z
+# and end at 03:00Z the following day. (the name of the file
+# is the first day.)
+
+# set the first and last days to be merged. can roll over
+# month and year boundaries now! note that for the end day,
+# you need at least the first 40ish files from the following day
+# for the merge to have the right data (from 0Z to 3Z) available.
+
+let start_year=2006
+let start_month=10
+let start_day=1
+
+let end_year=2007
+let end_month=1
+let end_day=31
+
+# end of things you should have to set in this script
+
+# convert the start and stop times to gregorian days, so we can
+# compute total number of days including rolling over month and
+# year boundaries. make sure all values have leading 0s if they
+# are < 10. do the end time first so we can use the same values
+# to set the initial day while we are doing the total day calc.
+
+# these outputs from advance time (with the -g flag) are
+# 2 integers: gregorian_day_number seconds
+# and since we don't set hours, minutes, or seconds, the second
+# number is always 0 and uninteresting for us.
+mon2=`printf %02d $end_month`
+day2=`printf %02d $end_day`
+end_d=(`echo ${end_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+mon2=`printf %02d $start_month`
+day2=`printf %02d $start_day`
+start_d=(`echo ${start_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+# these are a string in the format YYYYMMDDHH
+# do them here to prime the loop below which first takes them apart.
+curday=(`echo ${start_year}${mon2}${day2}00 0 | ./advance_time`)
+nextday=(`echo ${start_year}${mon2}${day2}00 +1d | ./advance_time`)
+
+# how many total days are going to be merged (for the loop counter)
+# (pull out the first of the 2 numbers which are output from advance_time)
+let totaldays=${end_d[0]}-${start_d[0]}+1
+
+# loop over each day
+let d=1
+while (( d <= totaldays)) ; do
+
+ # parse out the parts from a string which is YYYYMMDDHH
+ # both for the current day and tomorrow
+ cyear=${curday:0:4}
+ cmonth=${curday:4:2}
+ cday=${curday:6:2}
+ nyear=${nextday:0:4}
+ nmonth=${nextday:4:2}
+ nday=${nextday:6:2}
+
+ # compute the equivalent gregorian days here.
+ g=(`echo ${cyear}${cmonth}${cday}00 0 -g | ./advance_time`)
+ greg1=${g[0]}
+ let greg2=greg1+1
+
+ echo starting AIRS obs merge ${cyear}${cmonth}${cday}
+ echo gregorian: $greg
+
+ # all of todays data plus the first 40 of tomorrows
+ ls AIRS.${cyear}.${cmonth}.${cday}.*.out > olist
+ ls AIRS.${nyear}.${nmonth}.${nday}.0[0123]?.out >> olist
+
+ sed -e "s/YYYY/${cyear}/g" \
+ -e "s/MM/${cmonth}/g" \
+ -e "s/DD/${cday}/g" \
+ -e "s/GREG1/${greg1}/g" \
+ -e "s/GREG2/${greg2}/g" < ./input.nml.template > input.nml
+
+ # do the merge here
+ ./obs_sequence_tool
+
+
+ # advance the day; the output is YYYYMMDD00
+ curday=nextday
+ nextday=(`echo ${year}${month}${day}00 +1d | ./advance_time`)
+
+ # advance the loop counter
+ let d=d+1
+
+done
+
+exit 0
+
Property changes on: DART/trunk/observations/AIRS/shell_scripts/mergeit.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: DART/trunk/observations/AIRS/shell_scripts/oneday_down.sh
===================================================================
--- DART/trunk/observations/AIRS/shell_scripts/oneday_down.sh (rev 0)
+++ DART/trunk/observations/AIRS/shell_scripts/oneday_down.sh 2009-08-27 20:19:52 UTC (rev 4019)
@@ -0,0 +1,113 @@
+#!/bin/bash
+
+# this version gets the tar file from the mass store first.
+# unpack one day of tar files at a time, convert them into
+# individual obs_seq files. this program also does the merge
+# of the 240 individual daily swaths into a single obs_seq file.
+
+# this program should be started from the work directory.
+# it assumes ../data, ../tars, the output dir, etc
+# exist relative to starting from AIRS/work.
+
+# set the first and last days to be converted. can roll over
+# month and year boundaries now!
+let start_year=2006
+let start_month=10
+let start_day=1
+
+let end_year=2007
+let end_month=1
+let end_day=31
+
+# relative to work dir
+output_dir=../output.thin
+
+# whether to download the tar file from the mass store or not
+# set to one of: true or false
+download=true
+
+# end of things you should have to set in this script
+
+# convert the start and stop times to gregorian days, so we can
+# compute total number of days including rolling over month and
+# year boundaries. make sure all values have leading 0s if they
+# are < 10. do the end time first so we can use the same values
+# to set the initial day while we are doing the total day calc.
+mon2=`printf %02d $end_month`
+day2=`printf %02d $end_day`
+end_d=(`echo ${end_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+mon2=`printf %02d $start_month`
+day2=`printf %02d $start_day`
+start_d=(`echo ${start_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+curday=(`echo ${start_year}${mon2}${day2}00 0 | ./advance_time`)
+
+# how many total days are going to be converted (for the loop counter)
+let totaldays=${end_d[0]}-${start_d[0]}+1
+
+# loop over each day
+let d=1
+while (( d <= totaldays)) ; do
+
+ # parse out the parts from a string which is YYYYMMDDHH
+ year=${curday:0:4}
+ month=${curday:4:2}
+ day=${curday:6:2}
+
+ # compute the equivalent gregorian day here.
+ g=(`echo ${year}${month}${day}00 0 -g | ./advance_time`)
+ greg=${g[0]}
+
+ echo starting AIRS to obs ${year}${month}${day}
+ echo gregorian: $greg
+
+ # download the tar file from the mss first
+ if [[ "$download" = "true" ]]; then
+ echo getting ${year}${month}${day}.tar from mass store
+ (cd ../tars; msrcp mss:/MIJEONG/AIRS/V5/L2/${year}${month}/${year}${month}${day}.tar . )
+ fi
+
+ # assume the original collection of data (hdf files, one per swath)
+ # are in ../tars and that the filenames inside the tar files are named
+ # YYYYMM/YYYYMMDD/*.hdf
+ (cd ../data; tar -xvf ../tars/${year}${month}${day}.tar >> tarlog)
+
+ # construct the input list of files for the converter.
+ # cd there first in a subshell so the ls just contains simple file names
+ (cd ../data/${year}${month}/${year}${month}${day}; ls AIR*hdf > flist)
+
+ # get back to work dir and edit a template file to set the
+ # values that change in the namelists.
+ sed -e "s/YYYY/${year}/g" \
+ -e "s/MM/${month}/g" \
+ -e "s/DD/${day}/g" \
+ -e "s/GREG/${greg}/g" < ./input.nml.template > input.nml
+
+ # actually make the obs_seq files, one per input. these still need to
+ # be merged if you want daily files.
+ ./convert_airs_L2
+
+ # do the merge now
+ ls ${output_dir}/AIRS.${year}.${month}.${day}.*.out > olist
+ ./obs_sequence_tool
+
+ # start local mods
+ # ok, this is a local mod - to try to keep from running out of disk space
+ remote_dir=/gpfs/ptmp/dart/Obs_sets/AIRS_24_subx4_ascii/${year}${month}/
+ cp -f ${output_dir}/AIRS.${year}${month}${day}.out $remote_dir
+ # and clean up so we don't run out of disk space
+ (cd ../data/${year}${month}/${year}${month}${day}; rm AIR*hdf)
+ (cd ${output_dir}; rm AIRS.${year}.${month}.${day}.*.out)
+ (cd ../tars; rm ${year}${month}${day}.tar)
+ # end local mods
+
+ # advance the day; the output is YYYYMMDD00
+ curday=(`echo ${year}${month}${day}00 +1d | ./advance_time`)
+
+ # advance the loop counter
+ let d=d+1
+
+done
+
+exit 0
Property changes on: DART/trunk/observations/AIRS/shell_scripts/oneday_down.sh
___________________________________________________________________
Name: svn:executable
+ *
More information about the Dart-dev
mailing list