[Dart-dev] DART/branches Revision: 12574

[dart at ucar.edu]

nancy at ucar.edu
2018-05-03 11:15:02 -0600 (Thu, 03 May 2018)
397
clean up the conversion script a bit; use a better
variable name for the download directory instead of
just the time string, which is used for other things.

commit a script that runs from the command line or
from a batch system that converts multiple days of
data at a time.  on cheyenne doing 36 days at a time
on one compute node this runs frightenly fast (less
than 10 minutes/month of obs).




Modified: DART/branches/rma_trunk/observations/obs_converters/gps/shell_scripts/gpsro_to_obsseq.csh
===================================================================
--- DART/branches/rma_trunk/observations/obs_converters/gps/shell_scripts/gpsro_to_obsseq.csh	2018-05-03 15:36:03 UTC (rev 12573)
+++ DART/branches/rma_trunk/observations/obs_converters/gps/shell_scripts/gpsro_to_obsseq.csh	2018-05-03 17:15:02 UTC (rev 12574)
@@ -76,7 +76,7 @@
 #
 #   The option -q (quiet) suppresses the progress output which can fill
 #   your log files quickly if you're downloading lots of files.
-#   
+#
 #   The option -O file (direct output to given file) lets you rename the
 #   file or put it in a different location easily if you are only downloading
 #   a single file.
@@ -121,6 +121,7 @@
 setenv DART_WORK_DIR  $DART_DIR/observations/obs_converters/gps/work
 setenv CONV_PROG      convert_cosmic_gps_cdf
 setenv DATE_PROG      advance_time
+setenv DOWNLOAD_DIR   rawdata
 
 # options on wget command:
 set wget_cmd = "wget -q --http-user=$cdaac_user --http-passwd=$cdaac_pw" 
@@ -183,7 +184,7 @@
 
 if ( ! -d $datadir ) then
   echo 'creating data processing directory: ' $datadir
-  mkdir $datadir
+  mkdir -p $datadir
 endif
 if ( ! -e $datadir/input.nml ) then
   echo 'data processing directory does not contain an input.nml'
@@ -224,8 +225,11 @@
 # the work dir to the data processing directory.
 cp -f ./$DATE_PROG ./$CONV_PROG $datadir
 
+########################################################################
+# DOWNLOAD SECTION
+########################################################################
+
 if ( $downld == 'yes' ) then
-   echo 'if not already there, changing dir to data proc directory'
    cd $datadir
    echo 'current dir now ' `pwd`
 
@@ -239,23 +243,21 @@
       echo 'starting raw file download at' `date`
    endif
    
-   set yyyy = `echo $datea | cut -b1-4`
-   
-   if ( ! -d $datea ) then
-     echo 'year/month/day directory not found: ' $datea
-     echo 'creating now.'
-     mkdir $datea
+   if ( ! -d $DOWNLOAD_DIR ) then
+     echo 'creating new download directory: ' $DOWNLOAD_DIR
+     mkdir -p $DOWNLOAD_DIR
    else
      echo 'existing directory found, cleaning up before new download'
-     rm -fr $datea
-     mkdir $datea
+     rm -fr $DOWNLOAD_DIR
+     mkdir -p $DOWNLOAD_DIR
    endif
    
-   cd $datea
+   cd $DOWNLOAD_DIR
    ln -sf $DART_WORK_DIR/input.nml .
    
    set jyyyydd = `echo $datea 0 -j | ../$DATE_PROG` 
-   set mday = `printf %03d $jyyyydd[2]`
+   set yyyy    = `echo $datea | cut -b1-4`
+   set mday    = `printf %03d $jyyyydd[2]`
    echo 'downloading tar file of obs for date: ' $datea ', which is julian day: ' $jyyyydd
 
    # do this loop once for each satellite source you've selected
@@ -271,7 +273,7 @@
          echo no data available/downloaded for satellite $sat on $yyyy $mday
          rm $day_file
       else
-      tar -xf $day_file --strip-components=3      # remove long path
+         tar -xf $day_file --strip-components=3    # remove long path
       endif
    end
    rm input.nml dart_log.*
@@ -291,8 +293,11 @@
    echo 'not downloading data; assume it is already on local disk'
 endif
 
+########################################################################
+# CONVERSION SECTION
+########################################################################
+
 if ( $convert == 'yes') then
-   echo 'if not already there, changing dir to data proc directory'
    cd $datadir
    echo 'current dir now ' `pwd`
    
@@ -300,13 +305,13 @@
       echo 'starting gpsro conversion at ' `date`
    endif