[Dart-dev] [4501] DART/trunk/observations/gps: Changes from Ryan Torn to remove the observation window and time
nancy at ucar.edu
nancy at ucar.edu
Fri Sep 24 14:11:43 MDT 2010
Revision: 4501
Author: nancy
Date: 2010-09-24 14:11:43 -0600 (Fri, 24 Sep 2010)
Log Message:
-----------
Changes from Ryan Torn to remove the observation window and time
override code. This removes 'obs_window' from the input.nml namelist,
and the executable no longer needs to read a time from the stdin.
I also updated the documentation and moved all the scripting into
a new shell_scripts directory to be more consistent with other
parts of dart, and to make it easier to see how the scripts are
intended to be used.
The one not-backward-compatible part of this change is that you will
need to remove 'obs_window' from your gps input namelist before you
run again.
Modified Paths:
--------------
DART/trunk/observations/gps/convert_cosmic_gps_cdf.f90
DART/trunk/observations/gps/gps.html
DART/trunk/observations/gps/work/input.nml
DART/trunk/observations/gps/work/path_names_convert_cosmic_gps_cdf
Added Paths:
-----------
DART/trunk/observations/gps/shell_scripts/
DART/trunk/observations/gps/shell_scripts/README
DART/trunk/observations/gps/shell_scripts/convert_script.csh
DART/trunk/observations/gps/shell_scripts/cosmic_download.csh
DART/trunk/observations/gps/shell_scripts/cosmic_to_obsseq.csh
DART/trunk/observations/gps/shell_scripts/download_script.csh
Removed Paths:
-------------
DART/trunk/observations/gps/cosmic/input.nml
DART/trunk/observations/gps/work/README
DART/trunk/observations/gps/work/convert_script.csh
DART/trunk/observations/gps/work/cosmic_download.csh
DART/trunk/observations/gps/work/cosmic_to_obsseq.csh
DART/trunk/observations/gps/work/download_script.csh
-------------- next part --------------
Modified: DART/trunk/observations/gps/convert_cosmic_gps_cdf.f90
===================================================================
--- DART/trunk/observations/gps/convert_cosmic_gps_cdf.f90 2010-09-17 17:23:34 UTC (rev 4500)
+++ DART/trunk/observations/gps/convert_cosmic_gps_cdf.f90 2010-09-24 20:11:43 UTC (rev 4501)
@@ -20,26 +20,27 @@
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-use types_mod, only : r8, metadatalength
-use time_manager_mod, only : time_type, set_calendar_type, GREGORIAN, set_time,&
- increment_time, get_time, set_date, operator(-), &
- print_date
-use utilities_mod, only : initialize_utilities, find_namelist_in_file, &
- check_namelist_read, nmlfileunit, do_nml_file, &
- get_next_filename, error_handler, E_ERR, E_MSG, &
- nc_check, find_textfile_dims, do_nml_term
-use location_mod, only : VERTISHEIGHT, set_location
-use obs_sequence_mod, only : obs_sequence_type, obs_type, read_obs_seq, &
- static_init_obs_sequence, init_obs, destroy_obs, &
- write_obs_seq, init_obs_sequence, get_num_obs, &
- insert_obs_in_seq, destroy_obs_sequence, &
- set_copy_meta_data, set_qc_meta_data, set_qc, &
- set_obs_values, set_obs_def, insert_obs_in_seq
-use obs_def_mod, only : obs_def_type, set_obs_def_time, set_obs_def_kind, &
- set_obs_def_error_variance, set_obs_def_location, &
- set_obs_def_key
-use obs_def_gps_mod, only : set_gpsro_ref
-use obs_kind_mod, only : GPSRO_REFRACTIVITY
+use types_mod, only : r8, metadatalength
+use time_manager_mod, only : time_type, set_calendar_type, GREGORIAN, set_time,&
+ increment_time, get_time, set_date, operator(-), &
+ print_date
+use utilities_mod, only : initialize_utilities, find_namelist_in_file, &
+ check_namelist_read, nmlfileunit, do_nml_file, &
+ get_next_filename, error_handler, E_ERR, E_MSG, &
+ nc_check, find_textfile_dims, do_nml_term
+use location_mod, only : VERTISHEIGHT, set_location
+use obs_sequence_mod, only : obs_sequence_type, obs_type, read_obs_seq, &
+ static_init_obs_sequence, init_obs, destroy_obs, &
+ write_obs_seq, init_obs_sequence, get_num_obs, &
+ insert_obs_in_seq, destroy_obs_sequence, &
+ set_copy_meta_data, set_qc_meta_data, set_qc, &
+ set_obs_values, set_obs_def, insert_obs_in_seq
+use obs_def_mod, only : obs_def_type, set_obs_def_time, set_obs_def_kind, &
+ set_obs_def_error_variance, set_obs_def_location, &
+ set_obs_def_key
+use obs_def_gps_mod, only : set_gpsro_ref
+use obs_kind_mod, only : GPSRO_REFRACTIVITY
+use obs_utilities_mod, only : add_obs_to_seq
use netcdf
@@ -60,10 +61,9 @@
character (len=80) :: name
character (len=19) :: datestr
character (len=6) :: subset
-integer :: rcode, ncid, varid, nlevels, k, nfiles, num_new_obs, &
- aday, asec, dday, dsec, oday, osec, &
- iyear, imonth, iday, ihour, imin, isec, &
- glat, glon, zloc, obs_num, io, iunit, nobs, filenum, dummy
+integer :: rcode, ncid, varid, nlevels, k, nfiles, num_new_obs, oday, osec, &
+ iyear, imonth, iday, ihour, imin, isec, glat, glon, zloc, obs_num, &
+ io, iunit, nobs, filenum, dummy
logical :: file_exist, first_obs, did_obs, from_list = .false.
real(r8) :: hght_miss, refr_miss, azim_miss, oerr, &
qc, lato, lono, hghto, refro, azimo, wght, nx, ny, &
@@ -75,7 +75,7 @@
type(obs_def_type) :: obs_def
type(obs_sequence_type) :: obs_seq
type(obs_type) :: obs, prev_obs
-type(time_type) :: time_obs, time_anal
+type(time_type) :: time_obs, time_anal, prev_time
!------------------------------------------------------------------------
! Declare namelist parameters
@@ -84,43 +84,22 @@
integer, parameter :: nmaxlevels = 200 ! max number of observation levels
logical :: local_operator = .true. ! see html file for more on non/local
-logical :: overwrite_time = .false. ! careful - see note below
real(r8) :: obs_levels(nmaxlevels) = -1.0_r8
-real(r8) :: obs_window = 12.0 ! accept obs within +/- hours from anal time
real(r8) :: ray_ds = 5000.0_r8 ! delta stepsize (m) along ray, nonlocal op
real(r8) :: ray_htop = 15000.0_r8 ! max height (m) for nonlocal op
character(len=128) :: gpsro_netcdf_file = 'cosmic_gps_input.nc'
character(len=128) :: gpsro_netcdf_filelist = 'cosmic_gps_input_list'
character(len=128) :: gpsro_out_file = 'obs_seq.gpsro'
-namelist /convert_cosmic_gps_nml/ obs_levels, local_operator, obs_window, &
- ray_ds, ray_htop, gpsro_netcdf_file, &
- gpsro_netcdf_filelist, gpsro_out_file
+namelist /convert_cosmic_gps_nml/ obs_levels, local_operator, ray_ds, &
+ ray_htop, gpsro_netcdf_file, &
+ gpsro_netcdf_filelist, gpsro_out_file
-
-! 'overwrite_time' replaces the actual observation times with the
-! analysis time for all obs. this is intentionally not in the namelist
-! because we think observations should preserve the original times in all
-! cases. but one of our users had a special request to overwrite the time
-! with the synoptic time, so there is code in this converter to do that
-! if this option is set to .true. but we still don't encourage its use.
-
! initialize some values
obs_num = 1
qc = 0.0_r8
-
-print*,'Enter the target assimilation time (yyyy-mm-dd_hh:mm:ss)'
-read*,datestr
-
+first_obs = .true.
call set_calendar_type(GREGORIAN)
-read(datestr(1:4), fmt='(i4)') iyear
-read(datestr(6:7), fmt='(i2)') imonth
-read(datestr(9:10), fmt='(i2)') iday
-read(datestr(12:13), fmt='(i2)') ihour
-read(datestr(15:16), fmt='(i2)') imin
-read(datestr(18:19), fmt='(i2)') isec
-time_anal = set_date(iyear, imonth, iday, ihour, imin, isec)
-call get_time(time_anal, asec, aday)
! read the necessary parameters from input.nml
call initialize_utilities()
@@ -149,17 +128,6 @@
end if
end do
-! should error check the window some
-if (obs_window <= 0.0_r8 .or. obs_window > 24.0_r8) then
- call error_handler(E_ERR, 'convert_cosmic_gps_cdf', &
- 'Bad value for obs_window (hours)', &
- source, revision, revdate)
-else
- ! convert to seconds
- obs_window = obs_window * 3600.0_r8
-endif
-
-
! cannot have both a single filename and a list; the namelist must
! shut one off.
if (gpsro_netcdf_file /= '' .and. gpsro_netcdf_filelist /= '') then
@@ -184,14 +152,15 @@
inquire(file=gpsro_out_file, exist=file_exist)
if ( file_exist ) then
-print *, "found existing obs_seq file, appending to ", trim(gpsro_out_file)
+ print *, "found existing obs_seq file, appending to ", trim(gpsro_out_file)
call read_obs_seq(gpsro_out_file, 0, 0, num_new_obs, obs_seq)
else
-print *, "no existing obs_seq file, creating ", trim(gpsro_out_file)
-print *, "max entries = ", num_new_obs
+ print *, "no existing obs_seq file, creating ", trim(gpsro_out_file)
+ print *, "max entries = ", num_new_obs
call init_obs_sequence(obs_seq, num_copies, num_qc, num_new_obs)
+
do k = 1, num_copies
meta_data = 'COSMIC GPS observation'
call set_copy_meta_data(obs_seq, k, meta_data)
@@ -232,23 +201,6 @@
time_obs = set_date(iyear, imonth, iday, ihour, imin, isec)
call get_time(time_obs, osec, oday)
- ! time1-time2 is always positive no matter the relative magnitudes
- call get_time(time_anal-time_obs,dsec,dday)
- if ( real(dsec+dday*86400) > obs_window ) then
- call error_handler(E_MSG, 'convert_cosmic_gps_cdf: ', &
- 'Input file '//trim(next_infile), &
- source, revision, revdate)
- write(msgstring, '(A,F8.4,A)') 'Ignored because obs time > ', &
- obs_window / 3600.0, ' hours from analysis time'
- call error_handler(E_MSG, '', msgstring, &
- source, revision, revdate)
- call print_date(time_obs, ' observation time: ')
- call print_date(time_anal, ' window center time: ')
-
- filenum = filenum + 1
- cycle fileloop
- end if
-
call nc_check( nf90_inq_dimid(ncid, "MSL_alt", varid), 'inq dimid MSL_alt')
call nc_check( nf90_inquire_dimension(ncid, varid, name, nobs), 'inq dim MSL_alt')
@@ -347,11 +299,7 @@
call set_gpsro_ref(obs_num, nx, ny, nz, rfict, ray_ds, ray_htop, subset)
call set_obs_def_location(obs_def,set_location(lono,lato,hghto,VERTISHEIGHT))
call set_obs_def_kind(obs_def, GPSRO_REFRACTIVITY)
- if (overwrite_time) then ! generally do not want to do this here.
- call set_obs_def_time(obs_def, set_time(asec, aday))
- else
- call set_obs_def_time(obs_def, set_time(osec, oday))
- endif
+ call set_obs_def_time(obs_def, set_time(osec, oday))
call set_obs_def_error_variance(obs_def, oerr * oerr)
call set_obs_def_key(obs_def, obs_num)
call set_obs_def(obs, obs_def)
@@ -360,22 +308,10 @@
call set_obs_values(obs, obs_val)
qc_val(1) = qc
call set_qc(obs, qc_val)
-
- ! first one, insert with no prev. otherwise, since all times will be the
- ! same for this column, insert with the prev obs as the starting point.
- ! (this code used to call append, but calling insert makes it work even if
- ! the input files are processed out of strict time order, which one message
- ! i got seemed to indicate was happening...)
- if (first_obs) then
- call insert_obs_in_seq(obs_seq, obs)
- first_obs = .false.
- !print *, 'inserting first obs'
- else
- call insert_obs_in_seq(obs_seq, obs, prev_obs)
- !print *, 'inserting other obs'
- endif
+
+ call add_obs_to_seq(obs_seq, obs, time_obs, prev_obs, prev_time, first_obs)
+
obs_num = obs_num+1
- prev_obs = obs
if (.not. did_obs) did_obs = .true.
@@ -397,12 +333,12 @@
! minor stab at cleanup, in the off chance this will someday get turned
! into a subroutine in a module. probably not all that needs to be done,
! but a start.
-!print *, 'calling destroy_obs'
call destroy_obs(obs)
- call destroy_obs(prev_obs)
-print *, 'skipping destroy_seq'
- ! get core dumps here, not sure why?
- !if (get_num_obs(obs_seq) > 0) call destroy_obs_sequence(obs_seq)
+! if obs == prev_obs then you can't delete the same obs twice.
+! buf if they differ, then it's a leak. for now, don't delete prev
+! since the program is exiting here anyway.
+ !call destroy_obs(prev_obs)
+ if (get_num_obs(obs_seq) > 0) call destroy_obs_sequence(obs_seq)
endif
! END OF MAIN ROUTINE
Deleted: DART/trunk/observations/gps/cosmic/input.nml
===================================================================
--- DART/trunk/observations/gps/cosmic/input.nml 2010-09-17 17:23:34 UTC (rev 4500)
+++ DART/trunk/observations/gps/cosmic/input.nml 2010-09-24 20:11:43 UTC (rev 4501)
@@ -1,59 +0,0 @@
-
-
-! local operator does not do ray-path reconstruction
-! obs levels are in kilometers
-&convert_cosmic_gps_nml
- local_operator = .false.,
- obs_window = 12.0,
- obs_levels = 0.22, 0.55, 1.1, 1.8, 2.7, 3.7, 4.9,
- 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0,
- ray_ds = 5000.0,
- ray_htop = 13000.0
- /
-
-
-! ryan's original levels for wrf:
-! obs_levels = 0.3, 0.6, 0.9, 1.2, 1.5, 1.8, 2.1, 2.4, 2.7,
-! 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0,
-! 7.5, 8.0, 8.5, 9.0, 9.5, 10.0, 11.0, 12.0,
-! 13.0, 14.0, 15.0, 16.0, 17.0
-
-
-&preprocess_nml
- input_obs_kind_mod_file = '../../../obs_kind/DEFAULT_obs_kind_mod.F90',
- output_obs_kind_mod_file = '../../../obs_kind/obs_kind_mod.f90',
- input_obs_def_mod_file = '../../../obs_def/DEFAULT_obs_def_mod.F90',
- output_obs_def_mod_file = '../../../obs_def/obs_def_mod.f90',
- input_files = '../../../obs_def/obs_def_gps_mod.f90' /
-
-&obs_kind_nml
- /
-
-&location_nml
- /
-
-&utilities_nml
- nmlfilename = 'convert.nml'
- /
-
-&obs_sequence_nml
- write_binary_obs_sequence = .false. /
-
-&obs_sequence_tool_nml
- num_input_files = 1,
- filename_seq = '../obs_seq2007010106',
- filename_out = '../unused',
- print_only = .true.,
- gregorian_cal = .true.,
- first_obs_days = -1,
- first_obs_seconds = -1,
- last_obs_days = -1,
- last_obs_seconds = -1,
-/
-! obs_types =
-! keep_types =
-! min_lat =
-! max_lat =
-! min_lon =
-! max_lon =
-
Modified: DART/trunk/observations/gps/gps.html
===================================================================
--- DART/trunk/observations/gps/gps.html 2010-09-17 17:23:34 UTC (rev 4500)
+++ DART/trunk/observations/gps/gps.html 2010-09-24 20:11:43 UTC (rev 4501)
@@ -8,40 +8,58 @@
<BODY>
<A NAME="TOP"></A>
+<div>
<center>
+<strong><big><big><big>DART Project Documentation</big></big></big></strong> <br />
+<table width=80% border=0 summary="" cellpadding=0>
+<tr>
+<td align=center valign=middle>
+<a href="../../index.html">DART Documentation Main Index</a> </td>
+<td align=center valign=middle>
+<img src="../../doc/html/Dartboard9.png" alt="DART project logo" height=50 width=100> </td>
+<td align=center valign=middle>
+<small><small>Subversion info for this html file: <br />
+<!-- version tag follows, do not edit -->$Id$ </small></small> </td>
+</tr>
+</table> <br />
<A HREF="#Overview">OVERVIEW</A> /
<A HREF="#DataSources">DATA SOURCES</A> /
<A HREF="#Programs">PROGRAMS</A> /
-<A HREF="#Modules">MODULES USES</A> /
<A HREF="#Namelist">NAMELIST</A> /
+<A HREF="#Modules">MODULES USES</A> /
<A HREF="#KnownBugs">KNOWN BUGS</A> /
<A HREF="#FuturePlans">FUTURE PLANS</A> /
<A HREF="#Legalese">TERMS OF USE</A>
</center>
+</div>
<H1>GPS Observations</H1>
-<!-- version tag follows, do not edit --><P>$Id$</P>
<P>
-GPS Radio Occultation data is being returned
+GPS Radio Occultation data are being returned
from a series of satellites as part of the
<a href="http://www.cosmic.ucar.edu">COSMIC</a>
project.
-The programs in this directory help to extract the
+The programs in this directory extract the
data from the distribution files and put them into
DART observation sequence (obs_seq) file format.
<br>
<br>
-The COSMIC satellites measure the deviation and phase delay
-in the
+The COSMIC satellites measure the phase delay caused
+by deviation of the
straight-line path of the GPS satellite signal as
it passes through the Earth's atmosphere when the
GPS and COSMIC
satellites rise and set relative to each other.
-From this deviation and phase delay, vertical profiles of
-temperature and moisture can be computed.
+This deviation results from changes in the angle of refraction of light
+as it passes through regions of varying density of atmosphere.
+These changes are a result of variations in the temperature,
+pressure, and moisture content.
+Vertical profiles of temperature and moisture can be derived as
+the signal passes through more and more atmosphere
+until it is obscured by the earth's horizon.
There are thousands of observations each day
-distributed around the globe, including areas which
+distributed around the globe, including in areas which
previously were poorly observed.
</P>
@@ -69,8 +87,8 @@
Currently each vertical profile is stored in a separate file,
and there are between 1000-2000 profiles/day, so converting a day's
worth of observations involves downloading many individual files.
-There are plans in place to bundle these profiles together in
-a tar file to make it easier to download the raw data.
+There are plans in place by the data providers to bundle these
+profiles together in a tar file to make it simpler to download the raw data.
</P>
<!--==================================================================-->
@@ -80,39 +98,31 @@
<H2>PROGRAMS</H2>
<P>
-The data is distributed in
+The data are distributed in
<a href="http://www.unidata.ucar.edu/software/netcdf">netCDF</a>
file format. DART requires all observations to be in a proprietary
format often called DART "obs_seq" format.
-The files in this directory, a combination
-of C shell scripts and a Fortran source executable,
+The files in this directory (a combination
+of C shell scripts and a Fortran source executable)
do this data conversion.
<br>
<br>
-Optional namelist interface
-<A HREF="#Namelist"> <em class=code>&convert_cosmic_gps_nml</em> </A>
-may be read from file <em class=file>input.nml</em>.
-<br>
-<br>
-The work directory contains several scripts, including one which downloads
-the raw data files a day at a time (cosmic_download.csh), and one
-which executes the conversion program (convert_script.csh). These
-scripts make 6 hour files by default, but have options for other
-times. The input files are downloaded a day at a time. Be aware
-that each profile is stored in a separate netcdf file, and there
+The shell_scripts directory contains several example scripts, including one
+which downloads the raw data files a day at a time (download_script.csh),
+and one which executes the conversion program (convert_script.csh).
+These scripts make 6 hour files by default, but have options for other times.
+Each profile is stored in a separate netcdf file and there
are between 1000-2000 files/day, so the download process can be
lengthy. You probably want to download as a separate preprocess step
-and do not use the options to automatically delete the input files.
+and do not use the script options to automatically delete the input files.
Keep the files around until you are sure you are satisified with the
output files and then delete them by hand.
<br>
<br>
-The conversion executable, convert_cosmic_gps_cdf, reads the namelist
-from the file 'input.nml', but also reads an analysis time from the
-terminal or the standard input unit. This makes it simpler to convert
-multiple files without editing the namelist. The program prompts for
-the time string with the required time format,
-<em class=code>yyyy-mm-dd_hh:mm:ss</em> .
+The conversion executable <em class=file>convert_cosmic_gps_cdf</em>,
+reads the namelist
+<A HREF="#Namelist"> <em class=code>&convert_cosmic_gps_nml</em> </A>
+from the file <em class=file>input.nml</em>.
<br>
<br>
The namelist lets you select from one of two different forward
@@ -127,26 +137,18 @@
the conversion, and the type of operator is stored in the observation,
so at runtime the corresponding forward operator will be used to
compute the expected observation value.
+<br>
+<br>
+The namelist also lets you specify at what heights you want
+observations to be extracted. The raw data is very dense in
+the vertical; using all values would not results in a set of
+independent observations. The current source code interpolates
+the vertical profile into 200 bins, and then reinterpolates
+again onto the requested vertical heights. There are plans to
+revise the code to avoid the initial vertical binning step.
</P>
<!--==================================================================-->
-
-<A NAME="Modules"></A>
-<HR>
-<H2>MODULES USED</H2>
-<PRE>
-types_mod
-time_manager_mod
-utilities_mod
-location_mod
-obs_sequence_mod
-obs_def_mod
-obs_def_gps_mod
-obs_kind_mod
-netcdf
-</PRE>
-
-<!--==================================================================-->
<!--=================== DESCRIPTION OF A NAMELIST ====================-->
<!--==================================================================-->
@@ -155,14 +157,11 @@
<H2>NAMELIST</H2>
<P>We adhere to the F90 standard of starting a namelist with an ampersand
'&' and terminating with a slash '/' for all our namelist input.
-Consider yourself forewarned that character strings that contain a '/' must be
-enclosed in quotes to prevent them from prematurely terminating the namelist.
-The namelist declaration (i.e. what follows) has a different syntax, naturally.
</P>
<div class=namelist>
<pre>
<em class=call>namelist / convert_cosmic_gps_nml / </em>
- obs_levels, local_operator, obs_window, &
+ obs_levels, local_operator, &
ray_ds, ray_htop, gpsro_netcdf_file, &
gpsro_netcdf_filelist, gpsro_out_file
</pre>
@@ -196,14 +195,6 @@
ray-path reconstruction.
Default: .true.</TD></TR>
-<TR><!--contents--><TD valign=top>obs_window</TD>
- <!-- type --><TD valign=top>real(r8)</TD>
- <!--descript--><TD>Accept and convert observations if they are within
- plus or minus this number of hours from the analysis
- time (which is read as an input from the terminal or
- the standard input unit).
- Default: 12.0</TD></TR>
-
<TR><!--contents--><TD valign=top>ray_ds</TD>
<!-- type --><TD valign=top>real(r8)</TD>
<!--descript--><TD>For the non-local operator only, the delta stepsize,
@@ -243,20 +234,6 @@
new data will be inserted into that file.
Default: 'obs_seq.gpsro'</TD></TR>
-<TR><!--contents--><TD valign=top>overwrite_time</TD>
- <!-- type --><TD valign=top>logical</TD>
- <!--descript--><TD>This item is NOT in the standard namelist, but code
- exists in the program to support the functionality
- if the source is edited and this variable is added
- to the namelist. If set to
- .true., the output file will be created with all
- observations marked with the specified analysis time
- instead of the actual observation collection time.
- This is generally not what you want, but might be
- useful if you are binning observations by time to do
- thinning or super-ob'ing data.
- Default: .false.</TD></TR>
-
</TABLE>
</div>
@@ -264,6 +241,24 @@
<!--==================================================================-->
+<A NAME="Modules"></A>
+<HR>
+<H2>MODULES USED</H2>
+<PRE>
+types_mod
+time_manager_mod
+utilities_mod
+location_mod
+obs_sequence_mod
+obs_def_mod
+obs_def_gps_mod
+obs_kind_mod
+obs_utilities_mod
+netcdf
+</PRE>
+
+<!--==================================================================-->
+
<A NAME="KnownBugs"></A>
<HR>
<H2>KNOWN BUGS</H2>
@@ -273,8 +268,7 @@
important to get all the observations within a particular time window
files with filenames from a few minutes before and after the window
should be converted.
-Times really outside the window can be excluded either by setting
-the 'obs_window' namelist variable, or can be trimmed later with the
+Times really outside the window can be excluded in a separate step using the
<a href="../../obs_sequence/obs_sequence_tool.html">obs_sequence_tool</a>.
</P>
@@ -286,7 +280,12 @@
<HR>
<H2>FUTURE PLANS</H2>
<P>
-none
+The current code bins the data into 200 vertical bins as a first step,
+and then reinterpolates again onto the requested vertical heights.
+The original intent was to smooth out any noise in the dense profiles
+before interpolating onto the final heights, but plots have shown that
+the original data is smooth and can be used directly. There are plans
+to remove the intermediate binning step.
</P>
<!--==================================================================-->
@@ -298,7 +297,7 @@
<H2>Terms of Use</H2>
<P>
-DART software - Copyright © 2004 - 2010 UCAR.<br>
+DART software - Copyright © 2004 - 2010 UCAR.<br>
This open source software is provided by UCAR, "as is",<br>
without charge, subject to all terms of use at<br>
<a href="http://www.image.ucar.edu/DAReS/DART/DART_download">
Copied: DART/trunk/observations/gps/shell_scripts/README (from rev 4500, DART/trunk/observations/gps/work/README)
===================================================================
--- DART/trunk/observations/gps/shell_scripts/README (rev 0)
+++ DART/trunk/observations/gps/shell_scripts/README 2010-09-24 20:11:43 UTC (rev 4501)
@@ -0,0 +1,72 @@
+# DART software - Copyright \xA9 2004 - 2010 UCAR. This open source software is
+# provided by UCAR, "as is", without charge, subject to all terms of use at
+# http://www.image.ucar.edu/DAReS/DART/DART_download
+#
+# DART $Id$
+
+The intent for the scripts in this directory are:
+
+Summary:
+1) cd to the ../work directory and run ./quickbuild.csh to compile everything.
+2) Edit cosmic_download.csh and cosmic_to_obsseq.csh once to set DART_DIR.
+3) Edit ./download_script.csh to set the cosmic name, pw, and dates. Run it.
+4) Edit ./convert_script.csh to set the cosmic name, pw, and dates. Run it.
+5) For additional days repeat steps 3 and 4.
+
+
+More details:
+
+1) quickbuild.csh:
+
+Make sure your $DART/mkmf/mkmf.template is one that matches the
+platform and compiler for your system. It should be the same as
+how you have it set to build the other DART executables.
+
+Run quickbuild.csh and it should compile all the executables needed
+to do the GPS conversion into DART obs_sequence files.
+
+
+2) download_script.csh (which calls cosmic_download.csh):
+
+Edit cosmic_download.csh once to set the DART_DIR to where you have
+downloaded the DART distribution. (There are a few additional options
+in this script, but the distribution version should be good for most users.)
+After this you should be able to ignore this script.
+
+Edit and run the download_script.csh. You will need to set your
+cosmic web site user name and password, and set the days for which you
+want to download data.
+
+
+3) convert_script.csh (which calls cosmic_to_obsseq.csh):
+
+Edit cosmic_to_obsseq.csh once to set the DART_DIR to where you have
+downloaded the DART distribution. (There are a few additional options
+in this script, but the distribution version should be good for most users.)
+After this you should be able to ignore this script.
+
+Edit and run the convert_script.csh. You will need to set your
+cosmic web site user name and password, and set the days for which you
+want to convert data into DART obs_sequence files.
+
+There are options on the convert_script.csh to bypass step 2 and do both
+the download and convert in a single step. For one or two days this
+might be a good option, but it is common to have problems during the download.
+It is easier to restart the download separately from the conversion
+if you're using the separate download script. Also, the conversion needs
+3 hours from the following day to create a file which has all obs within
+the window (usually 3 hours) from the center time (usually 0Z). So if the
+download is done as a part of the script it must download 2 days, do the
+conversion, and then move on to the next day. The script is currently
+not smart enough to avoid redownloading data, so if you are converting
+multiple consecutive days it will redownload the next day's data anyway
+and will not realize you already have it. It cannot simply look for the
+existance of a directory since that complicates restarting failed or
+partial downloads. It is also risky to automatically delete the data files
+in the script, since if there are errors (full file system, etc) that
+don't cause the script to exit, you will delete the data files and not
+have the converted obs files. This is all only a consideration because
+currently it is slow to download the data. If that becomes faster, then
+most of the discussion in this paragraph is moot.
+
+
Copied: DART/trunk/observations/gps/shell_scripts/convert_script.csh (from rev 4500, DART/trunk/observations/gps/work/convert_script.csh)
===================================================================
--- DART/trunk/observations/gps/shell_scripts/convert_script.csh (rev 0)
+++ DART/trunk/observations/gps/shell_scripts/convert_script.csh 2010-09-24 20:11:43 UTC (rev 4501)
@@ -0,0 +1,56 @@
+#!/bin/csh
+#
+# DART software - Copyright \xA9 2004 - 2010 UCAR. This open source software is
+# provided by UCAR, "as is", without charge, subject to all terms of use at
+# http://www.image.ucar.edu/DAReS/DART/DART_download
+#
+# $Id$
+#
+# Main script:
+# generate multiple days of gps observations
+#
+# calls the cosmic_to_obsseq script with 4 args:
+#
+# - the date in YYYYMMDD format
+# - the working directory location
+# - whether to download the data automatically from the cosmic web site
+# (downloading data requires signing up for a username/password to
+# access the site, and then setting the username and password here
+# before running this script.) set to 'no' if the data has already been
+# downloaded separately before now. set to 'both' to download both the
+# current day plus the next day (needed to get the first N hours of the
+# following day for assimilation windows centered on midnight). set to
+# 'next' if the current day's data is already downloaded.
+# - whether to delete the data automatically from the local disk after the
+# conversion is done. values are 'no', 'curr', or 'both'. 'curr' deletes
+# the current day but leaves the following day. 'both' deletes both
+# the current and next day's data.
+#
+
+setenv cosmic_user xxx
+setenv cosmic_pw yyy
+
+# assumes all data predownloaded, and will be deleted afterwards
+# by hand.
+./cosmic_to_obsseq.csh 20070930 ../cosmic no no
+./cosmic_to_obsseq.csh 20071001 ../cosmic no no
+./cosmic_to_obsseq.csh 20071002 ../cosmic no no
+./cosmic_to_obsseq.csh 20071003 ../cosmic no no
+./cosmic_to_obsseq.csh 20071004 ../cosmic no no
+./cosmic_to_obsseq.csh 20071005 ../cosmic no no
+./cosmic_to_obsseq.csh 20071006 ../cosmic no no
+./cosmic_to_obsseq.csh 20071007 ../cosmic no no
+
+## example of using both, curr and next for on-demand download and cleanup.
+#./cosmic_to_obsseq.csh 20071001 ../cosmic both curr
+#./cosmic_to_obsseq.csh 20071002 ../cosmic next curr
+#./cosmic_to_obsseq.csh 20071003 ../cosmic next curr
+#./cosmic_to_obsseq.csh 20071004 ../cosmic next both
+
+exit 0
+
+# <next few lines under version control, do not edit>
+# $URL$
+# $Revision$
+# $Date$
+
Copied: DART/trunk/observations/gps/shell_scripts/cosmic_download.csh (from rev 4500, DART/trunk/observations/gps/work/cosmic_download.csh)
===================================================================
--- DART/trunk/observations/gps/shell_scripts/cosmic_download.csh (rev 0)
+++ DART/trunk/observations/gps/shell_scripts/cosmic_download.csh 2010-09-24 20:11:43 UTC (rev 4501)
@@ -0,0 +1,155 @@
+#!/bin/csh
+#
+# DART software - Copyright \xA9 2004 - 2010 UCAR. This open source software is
+# provided by UCAR, "as is", without charge, subject to all terms of use at
+# http://www.image.ucar.edu/DAReS/DART/DART_download
+#
+# $Id$
+#
+########################################################################
+#
+# cosmic_download.csh - script that downloads COSMIC observations.
+# then you can run cosmic_to_obsseq.csh with a third argument of
+# 'no' so it does not re-download the same files.
+#
+# requires 2 args:
+# $1 - analysis date (yyyymmdd format)
+# $2 - base observation directory
+#
+# and update the 3 env settings at the top to match your system.
+#
+# created May 2009, nancy collins ncar/cisl
+# converted from cosmic_to_obsseq.csh
+# updated Aug 2009, nancy collins ncar/cisl
+#
+# from the cosmic web site about the use of 'wget' to download
+# the many files needed to do this process:
+# -------
+# Hints for using wget for fetching CDAAC files from CDAAC:
+#
+# Here is one recipe for fetching all cosmic real time atmPrf files for one day:
+#
+# wget -nd -np -r -l 10 -w 2 --http-user=xxxx --http-passwd=xxxx http://cosmic-io.cosmic.ucar.edu/cdaac/login/cosmicrt/level2/atmPrf/2009.007/
+#
+# The option -np (no parents) is important. Without it, all manner of
+# files from throughout the site will be loaded, I think due to the
+# links back to the main page which are everywhere.
+#
+# The option -r (recursive fetch) is necessary unless there is just
+# one file you want to fetch.
+#
+# The option -l 10 (limit of recursive depth to 10 levels) is necessary
+# in order to get around the default 5 level depth.
+#
+# The option -nd dumps all fetched files into your current directory.
+# Otherwise a directory hierarchy will be created:
+# cosmic-io.cosmic.ucar.edu/cdaac/login/cosmic/level2/atmPrf/2006.207
+#
+# The option -w 2 tells wget to wait two seconds between each file fetch
+# so as to have pity on our poor web server.
+# -------
+#
+########################################################################
+
+# should only have to set the DART_DIR and the rest should be in the
+# right place relative to it.
+setenv DART_DIR /home/user/DART
+
+setenv DART_WORK_DIR ${DART_DIR}/observations/gps/work
+setenv DATE_PROG advance_time
+
+set chatty=yes
+set downld=yes
+
+set datea = ${1} # target date, YYYYMMDD
+set datadir = ${2} # where to process the files
+
+set assim_freq = 6 # hours, sets centers of windows.
+set download_window = 3 # window half-width (some users choose 2 hours)
+set gps_repository_path = 'http://cosmic-io.cosmic.ucar.edu/cdaac/login'
+set wget_cmd = 'wget -q -nd -np -r -l 10 -w 1'
+
+# i've done this wrong enough times and wasted a lot of download time,
+# so do a bunch of bullet-proofing here before going on.
+
+# verify the dirs all exist, the input.nml is in place.
+if ( ! -d ${DART_WORK_DIR} ) then
+ echo 'work directory not found: ' ${DART_WORK_DIR}
+ exit
+endif
+
+echo 'current dir is ' `pwd`
+if ( `pwd` != ${DART_WORK_DIR} ) then
+ echo 'if not already there, changing directory to work dir.'
+ cd ${DART_WORK_DIR}
+ echo 'current dir now ' `pwd`
+endif
+
+if ( ! -d ${datadir} ) then
+ echo 'data processing directory not found: ' ${datadir}
+ echo 'creating now.'
+ mkdir ${datadir}
+ ls -ld ${datadir}
+endif
+
+if ( ! -e ${datadir}/${DATE_PROG} ) then
+ echo 'data processing directory does not contain the time converter'
+ echo 'copying from work dir to data proc dir'
+ echo `pwd`/${DATE_PROG} '->' ${datadir}/${DATE_PROG}
+ cp -f ./${DATE_PROG} ${datadir}
+else
+ echo 'using time conversion program already found in data proc dir'
+endif
+
+echo 'changing dir to data proc directory'
+cd ${datadir}
+echo 'current dir now ' `pwd`
+
+ if ( ! $?cosmic_user || ! $?cosmic_pw ) then
+ echo "You must setenv cosmic_user to your username for the cosmic web site"
+ echo "and setenv cosmic_pw to your password. (or export cosmic_user=val for"
+ echo "ksh/bash users) then rerun this script. "
+ exit -1
+ endif
+
+if ( $chatty == 'yes' ) then
+ echo 'starting raw file download at' `date`
+endif
+
+set get = "${wget_cmd} --http-user=${cosmic_user} --http-passwd=${cosmic_pw}"
+
+set yyyy = `echo $datea | cut -b1-4`
+
+if ( ! -d ${datea} ) then
+ echo 'year/month/day directory not found: ' ${datea}
+ echo 'creating now.'
+ mkdir ${datea}
+endif
+
+cd ${datea}
+ln -sf ../input.nml .
+
+echo $datea
+set jyyyydd = `echo $datea 0 -j | ../${DATE_PROG}`
+echo $jyyyydd
+@ mday = $jyyyydd[2] + 1000 ; set mday = `echo $mday | cut -b2-4`
+${get} ${gps_repository_path}/cosmic/level2/atmPrf/${yyyy}.${mday}/
+rm -f *.html *.txt
+${get} ${gps_repository_path}/champ/level2/atmPrf/${yyyy}.${mday}/
+rm -f *.html *.txt input.nml
+
+if ( $chatty == 'yes' ) then
+ # the ls arg list line gets too long in some cases
+ echo `/bin/ls | grep _nc | wc -l` 'raw files'
+ echo 'all raw files download at ' `date`
+endif
+
+cd ..
+
+exit 0
+
+# <next few lines under version control, do not edit>
+# $URL$
+# $Revision$
+# $Date$
+
Copied: DART/trunk/observations/gps/shell_scripts/cosmic_to_obsseq.csh (from rev 4500, DART/trunk/observations/gps/work/cosmic_to_obsseq.csh)
===================================================================
--- DART/trunk/observations/gps/shell_scripts/cosmic_to_obsseq.csh (rev 0)
+++ DART/trunk/observations/gps/shell_scripts/cosmic_to_obsseq.csh 2010-09-24 20:11:43 UTC (rev 4501)
@@ -0,0 +1,213 @@
+#!/bin/csh
+#
+# DART software - Copyright \xA9 2004 - 2010 UCAR. This open source software is
+# provided by UCAR, "as is", without charge, subject to all terms of use at
+# http://www.image.ucar.edu/DAReS/DART/DART_download
+#
+# $Id$
+#
+########################################################################
+#
+# cosmic_to_obsseq.csh - script that downloads COSMIC observations
+# and converts them to a DART observation sequence file.
+#
+# requires 3 args:
+# $1 - analysis date (yyyymmdd format)
+# $2 - base observation directory
+# $3 - yes to download COSMIC data (calls cosmic_download.csh)
+# $4 - yes to delete raw COSMIC data when finished
+#
+# update the DART_DIR setting at the top to match your system.
+#
+# created June 2008, Ryan Torn NCAR/MMM
+# updated Nov 2008, nancy collins ncar/cisl
+# updated Aug 2009, nancy collins ncar/cisl
+#
+########################################################################
+
+# should only have to set the DART_DIR and the rest should be in the
+# right place relative to this directory.
+setenv DART_DIR /home/user/dart
+
+setenv DART_WORK_DIR ${DART_DIR}/observations/gps/work
+setenv CONV_PROG convert_cosmic_gps_cdf
+setenv DATE_PROG advance_time
+
+set chatty=yes
+
+set datea = ${1} # target date, YYYYMMDD
+set datadir = ${2} # where to process the files
+set downld = ${3} # download? 'both' (both days) 'next' (next day)
+set cleanup = ${4} # delete COSMIC files at end? 'both' or 'curr'
+
+set assim_freq = 6 # hours, sets centers of windows.
+set download_window = 3 # window half-width (some users choose 2 hours)
+
+# i've done this wrong enough times and wasted a lot of download time,
+# so do a bunch of bullet-proofing here before going on.
+
+# verify the dirs all exist, the input.nml is in place.
+if ( ! -d ${DART_WORK_DIR} ) then
+ echo 'work directory not found: ' ${DART_WORK_DIR}
+ exit 1
+endif
+
+echo 'current dir is ' `pwd`
+if ( `pwd` != ${DART_WORK_DIR} ) then
+ echo 'if not already there, changing directory to work dir.'
+ cd ${DART_WORK_DIR}
+ echo 'current dir now ' `pwd`
+endif
+
+if ( ! -d ${datadir} ) then
+ echo 'data processing directory not found: ' ${datadir}
+ echo 'creating now.'
+ mkdir ${datadir}
+ ls -ld ${datadir}
+endif
+if ( ! -e ${datadir}/input.nml ) then
+ echo 'data processing directory does not contain an input.nml'
+ echo 'copying from work dir to data proc dir'
+ echo `pwd`/input.nml '->' ${datadir}/input.nml
+ cp -f ./input.nml ${datadir}
+else
+ echo 'data processing directory already contains an input.nml'
+ echo 'which will be used for this run.'
+ diff -q ./input.nml ${datadir}/input.nml
+endif
+
+if ( ! -e ${datadir}/${DATE_PROG} ) then
+ echo 'data processing directory does not contain the time converter'
+ echo 'copying from work dir to data proc dir'
+ echo `pwd`/${DATE_PROG} '->' ${datadir}/${DATE_PROG}
+ cp -f ./${DATE_PROG} ${datadir}
+else
+ echo 'using time conversion program already found in data proc dir'
+endif
+
+if ( ! -e ${datadir}/${CONV_PROG} ) then
+ echo 'data processing directory does not contain the data converter'
+ echo 'copying from work dir to data proc dir'
+ echo `pwd`/${CONV_PROG} '->' ${datadir}/${CONV_PROG}
+ cp -f ./${CONV_PROG} ${datadir}
+else
+ echo 'using data conversion program already found in data proc dir'
+endif
+
+set date2 = `echo $datea +1d -f ccyymmdd | ${DATE_PROG}`
+
+if ( $downld == 'both' || $downld == 'next' ) then
+
+ if ( $downld == 'both' ) ./cosmic_download.csh ${datea} ${datadir}
+
+ ./cosmic_download.csh ${date2} ${datadir}
+
+endif
+
+echo 'changing dir to data proc directory'
+cd ${datadir}
+echo 'current dir now ' `pwd`
+
+# save original
+set dates = $datea
+
+@ hh = $assim_freq + 100 ; set hh = `echo $hh | cut -b2-3`
+set datea = `echo $datea | cut -b1-8`${hh}
@@ Diff output truncated at 40000 characters. @@
More information about the Dart-dev
mailing list