[Dart-dev] [4019] DART/trunk/observations/AIRS: The scripts used to generate multiple days of AIRS data files, with

nancy at ucar.edu nancy at ucar.edu
Thu Aug 27 14:19:52 MDT 2009


An HTML attachment was scrubbed...
URL: http://mailman.ucar.edu/pipermail/dart-dev/attachments/20090827/4098b79d/attachment.html 
-------------- next part --------------
Added: DART/trunk/observations/AIRS/shell_scripts/README
===================================================================
--- DART/trunk/observations/AIRS/shell_scripts/README	                        (rev 0)
+++ DART/trunk/observations/AIRS/shell_scripts/README	2009-08-27 20:19:52 UTC (rev 4019)
@@ -0,0 +1,7 @@
+
+These scripts are intended to help download the original AIRS hdf
+data files, convert them in bulk, and merge the resulting obs_seq files.
+
+In most cases, they're intended to be copied over to the ../work directory
+and then customized for the particular time period and local directory names.
+

Added: DART/trunk/observations/AIRS/shell_scripts/download.sh
===================================================================
--- DART/trunk/observations/AIRS/shell_scripts/download.sh	                        (rev 0)
+++ DART/trunk/observations/AIRS/shell_scripts/download.sh	2009-08-27 20:19:52 UTC (rev 4019)
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# download the requested tar files from the NCAR mass store.
+
+# set the first and last days. can roll over
+# month and year boundaries now!
+let start_year=2006
+let start_month=10
+let start_day=1
+
+let end_year=2007
+let end_month=1
+let end_day=31
+
+# end of things you should have to set in this script
+
+# convert the start and stop times to gregorian days, so we can
+# compute total number of days including rolling over month and
+# year boundaries.  make sure all values have leading 0s if they
+# are < 10.  do the end time first so we can use the same values
+# to set the initial day while we are doing the total day calc.
+mon2=`printf %02d $end_month`
+day2=`printf %02d $end_day`
+end_d=(`echo ${end_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+mon2=`printf %02d $start_month`
+day2=`printf %02d $start_day`
+start_d=(`echo ${start_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+curday=(`echo ${start_year}${mon2}${day2}00 0 | ./advance_time`)
+
+# how many total days are going to be converted (for the loop counter)
+let totaldays=${end_d[0]}-${start_d[0]}+1
+
+# loop over each day
+let d=1
+while (( d <= totaldays)) ; do
+
+  # parse out the parts from a string which is YYYYMMDDHH
+  year=${curday:0:4}
+  month=${curday:4:2}
+  day=${curday:6:2}
+
+
+  echo getting ${year}${month}${day}.tar from mass store
+  msrcp mss:/MIJEONG/AIRS/V5/L2/${year}${month}/${year}${month}${day}.tar .
+
+
+  # advance the day; the output is YYYYMMDD00
+  curday=(`echo ${year}${month}${day}00 +1d | ./advance_time`)
+
+  # advance the loop counter
+  let d=d+1
+ 
+done
+
+exit 0
+


Property changes on: DART/trunk/observations/AIRS/shell_scripts/download.sh
___________________________________________________________________
Name: svn:executable
   + *

Added: DART/trunk/observations/AIRS/shell_scripts/mergeit.sh
===================================================================
--- DART/trunk/observations/AIRS/shell_scripts/mergeit.sh	                        (rev 0)
+++ DART/trunk/observations/AIRS/shell_scripts/mergeit.sh	2009-08-27 20:19:52 UTC (rev 4019)
@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# merge the files into "daily" files which start at 03:01Z
+# and end at 03:00Z the following day.  (the name of the file
+# is the first day.)
+
+# set the first and last days to be merged.  can roll over
+# month and year boundaries now!   note that for the end day,
+# you need at least the first 40ish files from the following day
+# for the merge to have the right data (from 0Z to 3Z) available.
+
+let start_year=2006
+let start_month=10
+let start_day=1
+
+let end_year=2007
+let end_month=1
+let end_day=31
+
+# end of things you should have to set in this script
+
+# convert the start and stop times to gregorian days, so we can
+# compute total number of days including rolling over month and
+# year boundaries.  make sure all values have leading 0s if they
+# are < 10.  do the end time first so we can use the same values
+# to set the initial day while we are doing the total day calc.
+
+# these outputs from advance time (with the -g flag) are
+# 2 integers: gregorian_day_number seconds
+# and since we don't set hours, minutes, or seconds, the second
+# number is always 0 and uninteresting for us.
+mon2=`printf %02d $end_month`
+day2=`printf %02d $end_day`
+end_d=(`echo ${end_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+mon2=`printf %02d $start_month`
+day2=`printf %02d $start_day`
+start_d=(`echo ${start_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+# these are a string in the format YYYYMMDDHH
+# do them here to prime the loop below which first takes them apart.
+curday=(`echo ${start_year}${mon2}${day2}00 0 | ./advance_time`)
+nextday=(`echo ${start_year}${mon2}${day2}00 +1d | ./advance_time`)
+
+# how many total days are going to be merged (for the loop counter)
+# (pull out the first of the 2 numbers which are output from advance_time)
+let totaldays=${end_d[0]}-${start_d[0]}+1
+
+# loop over each day
+let d=1
+while (( d <= totaldays)) ; do
+
+  # parse out the parts from a string which is YYYYMMDDHH
+  # both for the current day and tomorrow
+  cyear=${curday:0:4}
+  cmonth=${curday:4:2}
+  cday=${curday:6:2}
+  nyear=${nextday:0:4}
+  nmonth=${nextday:4:2}
+  nday=${nextday:6:2}
+
+  # compute the equivalent gregorian days here.
+  g=(`echo ${cyear}${cmonth}${cday}00 0 -g | ./advance_time`)
+  greg1=${g[0]}
+  let greg2=greg1+1
+
+  echo starting AIRS obs merge ${cyear}${cmonth}${cday}
+  echo gregorian: $greg
+
+  # all of todays data plus the first 40 of tomorrows
+  ls AIRS.${cyear}.${cmonth}.${cday}.*.out > olist
+  ls AIRS.${nyear}.${nmonth}.${nday}.0[0123]?.out >> olist
+
+  sed -e "s/YYYY/${cyear}/g"    \
+      -e "s/MM/${cmonth}/g"     \
+      -e "s/DD/${cday}/g"       \
+      -e "s/GREG1/${greg1}/g"  \
+      -e "s/GREG2/${greg2}/g"    < ./input.nml.template > input.nml
+
+  # do the merge here
+  ./obs_sequence_tool
+  
+
+  # advance the day; the output is YYYYMMDD00
+  curday=nextday
+  nextday=(`echo ${year}${month}${day}00 +1d | ./advance_time`)
+
+  # advance the loop counter
+  let d=d+1
+ 
+done
+
+exit 0
+


Property changes on: DART/trunk/observations/AIRS/shell_scripts/mergeit.sh
___________________________________________________________________
Name: svn:executable
   + *

Added: DART/trunk/observations/AIRS/shell_scripts/oneday_down.sh
===================================================================
--- DART/trunk/observations/AIRS/shell_scripts/oneday_down.sh	                        (rev 0)
+++ DART/trunk/observations/AIRS/shell_scripts/oneday_down.sh	2009-08-27 20:19:52 UTC (rev 4019)
@@ -0,0 +1,113 @@
+#!/bin/bash
+
+# this version gets the tar file from the mass store first.
+# unpack one day of tar files at a time, convert them into
+# individual obs_seq files.  this program also does the merge
+# of the 240 individual daily swaths into a single obs_seq file.
+
+# this program should be started from the work directory.
+# it assumes ../data, ../tars, the output dir, etc
+# exist relative to starting from AIRS/work.
+
+# set the first and last days to be converted.  can roll over
+# month and year boundaries now!
+let start_year=2006
+let start_month=10
+let start_day=1
+
+let end_year=2007
+let end_month=1
+let end_day=31
+
+# relative to work dir
+output_dir=../output.thin
+
+# whether to download the tar file from the mass store or not 
+# set to one of: true or false
+download=true
+
+# end of things you should have to set in this script
+
+# convert the start and stop times to gregorian days, so we can
+# compute total number of days including rolling over month and
+# year boundaries.  make sure all values have leading 0s if they
+# are < 10.  do the end time first so we can use the same values
+# to set the initial day while we are doing the total day calc.
+mon2=`printf %02d $end_month`
+day2=`printf %02d $end_day`
+end_d=(`echo ${end_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+mon2=`printf %02d $start_month`
+day2=`printf %02d $start_day`
+start_d=(`echo ${start_year}${mon2}${day2}00 0 -g | ./advance_time`)
+
+curday=(`echo ${start_year}${mon2}${day2}00 0 | ./advance_time`)
+
+# how many total days are going to be converted (for the loop counter)
+let totaldays=${end_d[0]}-${start_d[0]}+1
+
+# loop over each day
+let d=1
+while (( d <= totaldays)) ; do
+
+  # parse out the parts from a string which is YYYYMMDDHH
+  year=${curday:0:4}
+  month=${curday:4:2}
+  day=${curday:6:2}
+
+  # compute the equivalent gregorian day here.
+  g=(`echo ${year}${month}${day}00 0 -g | ./advance_time`)
+  greg=${g[0]}
+
+  echo starting AIRS to obs ${year}${month}${day}
+  echo gregorian: $greg
+
+  # download the tar file from the mss first
+  if [[ "$download" = "true" ]]; then
+    echo getting ${year}${month}${day}.tar from mass store
+    (cd ../tars; msrcp mss:/MIJEONG/AIRS/V5/L2/${year}${month}/${year}${month}${day}.tar . )
+  fi
+
+  # assume the original collection of data (hdf files, one per swath)
+  # are in ../tars and that the filenames inside the tar files are named
+  # YYYYMM/YYYYMMDD/*.hdf
+  (cd ../data; tar -xvf ../tars/${year}${month}${day}.tar >> tarlog)
+  
+  # construct the input list of files for the converter.
+  # cd there first in a subshell so the ls just contains simple file names
+  (cd ../data/${year}${month}/${year}${month}${day}; ls AIR*hdf > flist)
+
+  # get back to work dir and edit a template file to set the 
+  # values that change in the namelists.
+  sed -e "s/YYYY/${year}/g" \
+      -e "s/MM/${month}/g"  \
+      -e "s/DD/${day}/g"    \
+      -e "s/GREG/${greg}/g" < ./input.nml.template > input.nml
+
+  # actually make the obs_seq files, one per input.  these still need to
+  # be merged if you want daily files.
+  ./convert_airs_L2
+  
+  # do the merge now
+  ls ${output_dir}/AIRS.${year}.${month}.${day}.*.out > olist
+  ./obs_sequence_tool
+
+  # start local mods
+  # ok, this is a local mod - to try to keep from running out of disk space
+  remote_dir=/gpfs/ptmp/dart/Obs_sets/AIRS_24_subx4_ascii/${year}${month}/
+  cp -f ${output_dir}/AIRS.${year}${month}${day}.out $remote_dir
+  # and clean up so we don't run out of disk space
+  (cd ../data/${year}${month}/${year}${month}${day}; rm AIR*hdf)
+  (cd ${output_dir}; rm AIRS.${year}.${month}.${day}.*.out)
+  (cd ../tars; rm ${year}${month}${day}.tar)
+  # end local mods
+
+  # advance the day; the output is YYYYMMDD00
+  curday=(`echo ${year}${month}${day}00 +1d | ./advance_time`)
+
+  # advance the loop counter
+  let d=d+1
+ 
+done
+
+exit 0


Property changes on: DART/trunk/observations/AIRS/shell_scripts/oneday_down.sh
___________________________________________________________________
Name: svn:executable
   + *


More information about the Dart-dev mailing list