[Dart-dev] [7989] DART/trunk/observations/MADIS: sort the times before adding the observations to the output obs_seq file.
nancy at ucar.edu
nancy at ucar.edu
Fri May 15 15:54:12 MDT 2015
Revision: 7989
Author: nancy
Date: 2015-05-15 15:54:11 -0600 (Fri, 15 May 2015)
Log Message:
-----------
sort the times before adding the observations to the output obs_seq file.
this speeds the converters by a huge amount. updated the path_names files
to include the sort module. also fixed an apparent bug in the satwnd
converter when it was trying to identify duplicate observations. the wrong
indices were being used for pressure and band when traversing the arrays.
Modified Paths:
--------------
DART/trunk/observations/MADIS/convert_madis_acars.f90
DART/trunk/observations/MADIS/convert_madis_mesonet.f90
DART/trunk/observations/MADIS/convert_madis_satwnd.f90
DART/trunk/observations/MADIS/work/path_names_convert_madis_acars
DART/trunk/observations/MADIS/work/path_names_convert_madis_mesonet
DART/trunk/observations/MADIS/work/path_names_convert_madis_satwnd
-------------- next part --------------
Modified: DART/trunk/observations/MADIS/convert_madis_acars.f90
===================================================================
--- DART/trunk/observations/MADIS/convert_madis_acars.f90 2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/convert_madis_acars.f90 2015-05-15 21:54:11 UTC (rev 7989)
@@ -41,6 +41,7 @@
acars_rel_hum_error
use dewpoint_obs_err_mod, only : dewpt_error_from_rh_and_temp, &
rh_error_from_dewpt_and_temp
+use sort_mod, only : index_sort
use obs_kind_mod, only : ACARS_U_WIND_COMPONENT, ACARS_V_WIND_COMPONENT, &
ACARS_TEMPERATURE, ACARS_SPECIFIC_HUMIDITY, &
ACARS_DEWPOINT, ACARS_RELATIVE_HUMIDITY
@@ -77,6 +78,7 @@
real(r8), allocatable :: lat(:), lon(:), palt(:), tair(:), relh(:), tdew(:), &
wdir(:), wspd(:), latu(:), lonu(:), palu(:)
integer, allocatable :: qc_palt(:), qc_tair(:), qc_relh(:), qc_tdew(:), qc_wdir(:), qc_wspd(:)
+integer, allocatable :: indu(:), sorted_indu(:)
type(obs_sequence_type) :: obs_seq
type(obs_type) :: obs, prev_obs
@@ -105,6 +107,7 @@
allocate(latu(nobs)) ; allocate(lonu(nobs))
allocate(palu(nobs)) ; allocate(tdew(nobs))
allocate(tobu(nobs))
+allocate(indu(nobs)) ; allocate(sorted_indu(nobs))
nvars = 3
if (include_specific_humidity) nvars = nvars + 1
@@ -174,14 +177,11 @@
qc = 1.0_r8
nused = 0
-obsloop: do n = 1, nobs
+obsloop1: do n = 1, nobs
- ! compute time of observation
- time_obs = increment_time(comp_day0, tobs(n))
-
! check the lat/lon values to see if they are ok
- if ( lat(n) > 90.0_r8 .or. lat(n) < -90.0_r8 ) cycle obsloop
- if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop
+ if ( lat(n) > 90.0_r8 .or. lat(n) < -90.0_r8 ) cycle obsloop1
+ if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop1
! change lon from -180 to 180 into 0-360
if ( lon(n) < 0.0_r8 ) lon(n) = lon(n) + 360.0_r8
@@ -191,10 +191,29 @@
if ( lon(n) == lonu(i) .and. &
lat(n) == latu(i) .and. &
tobs(n) == tobu(i) .and. &
- palt(n) == palu(i) ) cycle obsloop
+ palt(n) == palu(i) ) cycle obsloop1
end do
- if ( palt(n) == palt_miss .or. qc_palt(n) /= 0 ) cycle obsloop
+ nused = nused + 1
+ indu(nused) = n
+ latu(nused) = lat(n)
+ lonu(nused) = lon(n)
+ palu(nused) = palt(n)
+ tobu(nused) = tobs(n)
+
+end do obsloop1
+
+call index_sort(tobu, sorted_indu, nused)
+
+obsloop2: do i = 1, nused
+
+ ! get the next unique observation in sorted time order
+ n = indu(sorted_indu(i))
+
+ ! compute time of observation
+ time_obs = increment_time(comp_day0, tobs(n))
+
+ if ( palt(n) == palt_miss .or. qc_palt(n) /= 0 ) cycle obsloop2
pres = pres_alt_to_pres(palt(n))
! extract actual time of observation in file into oday, osec.
@@ -314,14 +333,8 @@
100 continue
- nused = nused + 1
- latu(nused) = lat(n)
- lonu(nused) = lon(n)
- palu(nused) = palt(n)
- tobu(nused) = tobs(n)
+end do obsloop2
-end do obsloop
-
! if we added any obs to the sequence, write it now.
if ( get_num_obs(obs_seq) > 0 ) call write_obs_seq(obs_seq, acars_out_file)
Modified: DART/trunk/observations/MADIS/convert_madis_mesonet.f90
===================================================================
--- DART/trunk/observations/MADIS/convert_madis_mesonet.f90 2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/convert_madis_mesonet.f90 2015-05-15 21:54:11 UTC (rev 7989)
@@ -34,7 +34,7 @@
use types_mod, only : r8, missing_r8
use utilities_mod, only : nc_check, initialize_utilities, finalize_utilities
use time_manager_mod, only : time_type, set_calendar_type, set_date, &
- increment_time, get_time, operator(-), GREGORIAN
+ increment_time, get_time, operator(-), GREGORIAN
use location_mod, only : VERTISSURFACE
use obs_sequence_mod, only : obs_sequence_type, obs_type, read_obs_seq, &
static_init_obs_sequence, init_obs, write_obs_seq, &
@@ -51,8 +51,9 @@
LAND_SFC_TEMPERATURE, LAND_SFC_SPECIFIC_HUMIDITY, &
LAND_SFC_DEWPOINT, LAND_SFC_RELATIVE_HUMIDITY, &
LAND_SFC_ALTIMETER
-use obs_utilities_mod, only : getvar_real, get_or_fill_QC, add_obs_to_seq, &
- create_3d_obs, getvar_int, getdimlen, set_missing_name
+use sort_mod, only : index_sort
+use obs_utilities_mod, only : getvar_real, get_or_fill_QC, add_obs_to_seq, &
+ create_3d_obs, getvar_int, getdimlen, set_missing_name
use netcdf
@@ -73,23 +74,26 @@
integer, parameter :: num_copies = 1, & ! number of copies in sequence
num_qc = 1 ! number of QC entries
-integer :: ncid, nobs, nvars, n, i, oday, osec, nused
+integer :: ncid, nobs, nvars, n, i, oday, osec, nused
logical :: file_exist, first_obs
real(r8) :: alti_miss, tair_miss, tdew_miss, wdir_miss, wspd_miss, uwnd, &
vwnd, palt, qobs, qsat, rh, oerr, pres, qerr, qc
-integer, allocatable :: tobs(:), tobu(:)
+integer, allocatable :: tobs(:), tobu(:), indu(:), sorted_indu(:)
real(r8), allocatable :: lat(:), lon(:), elev(:), alti(:), tair(:), &
tdew(:), wdir(:), wspd(:), latu(:), lonu(:)
integer, allocatable :: qc_alti(:), qc_tair(:), qc_tdew(:), qc_wdir(:), qc_wspd(:)
-
+
type(obs_sequence_type) :: obs_seq
type(obs_type) :: obs, prev_obs
type(time_type) :: comp_day0, time_obs, prev_time
+!----------------------------------------------------------------
+
call initialize_utilities('convert_madis_mesonet')
+
! put the reference date into DART format
call set_calendar_type(GREGORIAN)
comp_day0 = set_date(1970, 1, 1, 0, 0, 0)
@@ -109,6 +113,7 @@
allocate(tair(nobs)) ; allocate(tdew(nobs))
allocate(wdir(nobs)) ; allocate(wspd(nobs))
allocate(tobs(nobs)) ; allocate(tobu(nobs))
+allocate(indu(nobs)) ; allocate(sorted_indu(nobs))
nvars = 4
if (include_specific_humidity) nvars = nvars + 1
@@ -178,25 +183,41 @@
qc = 1.0_r8
nused = 0
-obsloop: do n = 1, nobs
+obsloop1: do n = 1, nobs
- ! compute time of observation
- time_obs = increment_time(comp_day0, tobs(n))
-
! check the lat/lon values to see if they are ok
- if ( lat(n) > 90.0_r8 .or. lat(n) < -90.0_r8 ) cycle obsloop
- if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop
+ if ( lat(n) > 90.0_r8 .or. lat(n) < -90.0_r8 ) cycle obsloop1
+ if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop1
! change lon from -180 to 180 into 0-360
if ( lon(n) < 0.0_r8 ) lon(n) = lon(n) + 360.0_r8
- ! Check for duplicate observations
- do i = 1, nused
+ ! check for duplicate observations
+ do i=1, nused
if ( lon(n) == lonu(i) .and. &
lat(n) == latu(i) .and. &
- tobs(n) == tobu(i) ) cycle obsloop
- end do
+ tobs(n) == tobu(i) ) cycle obsloop1
+ enddo
+ nused = nused + 1
+ indu(nused) = n
+ latu(nused) = lat(n)
+ lonu(nused) = lon(n)
+ tobu(nused) = tobs(n)
+
+enddo obsloop1
+
+! sort into time order
+call index_sort(tobu, sorted_indu, nused)
+
+
+obsloop2: do i = 1, nused
+
+ n = indu(sorted_indu(i))
+
+ ! compute time of observation
+ time_obs = increment_time(comp_day0, tobs(n))
+
palt = pres_alt_to_pres(elev(n)) * 0.01_r8
! extract actual time of observation in file into oday, osec.
@@ -334,19 +355,15 @@
100 continue
- nused = nused + 1
- latu(nused) = lat(n)
- lonu(nused) = lon(n)
- tobu(nused) = tobs(n)
+end do obsloop2
-end do obsloop
-
! if we added any obs to the sequence, write it now.
if ( get_num_obs(obs_seq) > 0 ) call write_obs_seq(obs_seq, surface_out_file)
! end of main program
call finalize_utilities()
+
end program convert_madis_mesonet
! <next few lines under version control, do not edit>
Modified: DART/trunk/observations/MADIS/convert_madis_satwnd.f90
===================================================================
--- DART/trunk/observations/MADIS/convert_madis_satwnd.f90 2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/convert_madis_satwnd.f90 2015-05-15 21:54:11 UTC (rev 7989)
@@ -45,6 +45,7 @@
use meteor_mod, only : wind_dirspd_to_uv
use obs_err_mod, only : sat_wind_error, sat_wv_wind_error
use obs_kind_mod, only : SAT_U_WIND_COMPONENT, SAT_V_WIND_COMPONENT
+use sort_mod, only : index_sort
use obs_utilities_mod, only : getvar_real, get_or_fill_QC, add_obs_to_seq, &
create_3d_obs, getvar_int, getdimlen
@@ -77,10 +78,11 @@
real(r8), allocatable :: lat(:), lon(:), latu(:), lonu(:), &
- pres(:), prsu(:), tobs(:), tobu(:), &
+ pres(:), prsu(:), tobs(:), &
wdir(:), wspd(:)
integer, allocatable :: band(:), bndu(:)
integer, allocatable :: qc_wdir(:), qc_wspd(:)
+integer, allocatable :: indu(:), sorted_indu(:), tobu(:)
type(obs_sequence_type) :: obs_seq
type(obs_type) :: obs, prev_obs
@@ -122,6 +124,7 @@
allocate(tobs(nobs)) ; allocate(tobu(nobs))
allocate(band(nobs)) ; allocate(bndu(nobs))
allocate(wdir(nobs)) ; allocate(wspd(nobs))
+allocate(indu(nobs)) ; allocate(sorted_indu(nobs))
allocate(qc_wdir(nobs)) ; allocate(qc_wspd(nobs))
! read in the data arrays
@@ -171,34 +174,52 @@
qc = 1.0_r8
nused = 0
-obsloop: do n = 1, nobs
+obsloop1: do n = 1, nobs
- ! compute time of observation
- time_obs = increment_time(comp_day0, nint(tobs(n)))
-
! check the lat/lon values to see if they are ok
- if ( lat(n) > 90.0_r8 .or. lat(n) < -90.0_r8 ) cycle obsloop
- if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop
+ if ( lat(n) > 90.0_r8 .or. lat(n) < -90.0_r8 ) cycle obsloop1
+ if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop1
+ ! change lon from -180 to 180 into 0-360
if ( lon(n) < 0.0_r8 ) lon(n) = lon(n) + 360.0_r8
! Check for duplicate observations
do i = 1, nused
if ( lon(n) == lonu(i) .and. &
lat(n) == latu(i) .and. &
- prsu(n) == pres(i) .and. &
+ pres(n) == prsu(i) .and. &
band(n) == bndu(i) .and. &
- tobs(n) == tobu(i) ) cycle obsloop
+ nint(tobs(n)) == tobu(i) ) cycle obsloop1
end do
! if selecting only certain bands, cycle if not wanted
if (.not. allbands) then
- if (.not. iruse .and. band(n) == 1) cycle obsloop
- if (.not. visuse .and. band(n) == 2) cycle obsloop
+ if (.not. iruse .and. band(n) == 1) cycle obsloop1
+ if (.not. visuse .and. band(n) == 2) cycle obsloop1
if (.not. wvuse .and. &
- (band(n) == 3 .or. band(n) == 5 .or. band(n) == 7)) cycle obsloop
+ (band(n) == 3 .or. band(n) == 5 .or. band(n) == 7)) cycle obsloop1
endif
+ nused = nused + 1
+ indu(nused) = n
+ latu(nused) = lat(n)
+ lonu(nused) = lon(n)
+ prsu(nused) = pres(n)
+ bndu(nused) = band(n)
+ tobu(nused) = nint(tobs(n))
+
+end do obsloop1
+
+call index_sort(tobu, sorted_indu, nused)
+
+obsloop2: do i = 1, nused
+
+ ! get the next unique observation in sorted time order
+ n = indu(sorted_indu(i))
+
+ ! compute time of observation
+ time_obs = increment_time(comp_day0, nint(tobs(n)))
+
! extract actual time of observation in file into oday, osec.
call get_time(time_obs, osec, oday)
@@ -211,7 +232,7 @@
! perform sanity checks on observation errors and values
if ( oerr == missing_r8 .or. wdir(n) < 0.0_r8 .or. wdir(n) > 360.0_r8 .or. &
- wspd(n) < 0.0_r8 .or. wspd(n) > 120.0_r8 ) cycle obsloop
+ wspd(n) < 0.0_r8 .or. wspd(n) > 120.0_r8 ) cycle obsloop2
call create_3d_obs(lat(n), lon(n), pres(n), VERTISPRESSURE, uwnd, &
SAT_U_WIND_COMPONENT, oerr, oday, osec, qc, obs)
@@ -223,15 +244,8 @@
endif
- nused = nused + 1
- latu(nused) = lat(n)
- lonu(nused) = lon(n)
- prsu(nused) = pres(n)
- band(nused) = bndu(n)
- tobu(nused) = tobs(n)
+end do obsloop2
-end do obsloop
-
! need to wait to close file because in the loop it queries the
! report types.
call nc_check( nf90_close(ncid) , &
Modified: DART/trunk/observations/MADIS/work/path_names_convert_madis_acars
===================================================================
--- DART/trunk/observations/MADIS/work/path_names_convert_madis_acars 2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/work/path_names_convert_madis_acars 2015-05-15 21:54:11 UTC (rev 7989)
@@ -1,16 +1,17 @@
+assim_model/assim_model_mod.f90
+common/types_mod.f90
+location/threed_sphere/location_mod.f90
+models/template/model_mod.f90
+mpi_utilities/null_mpi_utilities_mod.f90
+obs_def/obs_def_mod.f90
+obs_kind/obs_kind_mod.f90
+obs_sequence/obs_sequence_mod.f90
observations/MADIS/convert_madis_acars.f90
+observations/MADIS/meteor_mod.f90
+observations/obs_error/dewpoint_obs_err_mod.f90
observations/obs_error/ncep_obs_err_mod.f90
-observations/obs_error/dewpoint_obs_err_mod.f90
-observations/MADIS/meteor_mod.f90
observations/utilities/obs_utilities_mod.f90
-location/threed_sphere/location_mod.f90
-obs_sequence/obs_sequence_mod.f90
-obs_kind/obs_kind_mod.f90
-obs_def/obs_def_mod.f90
-assim_model/assim_model_mod.f90
-models/template/model_mod.f90
-common/types_mod.f90
random_seq/random_seq_mod.f90
+sort/sort_mod.f90
+time_manager/time_manager_mod.f90
utilities/utilities_mod.f90
-time_manager/time_manager_mod.f90
-mpi_utilities/null_mpi_utilities_mod.f90
Modified: DART/trunk/observations/MADIS/work/path_names_convert_madis_mesonet
===================================================================
--- DART/trunk/observations/MADIS/work/path_names_convert_madis_mesonet 2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/work/path_names_convert_madis_mesonet 2015-05-15 21:54:11 UTC (rev 7989)
@@ -1,16 +1,17 @@
+assim_model/assim_model_mod.f90
+common/types_mod.f90
+location/threed_sphere/location_mod.f90
+models/template/model_mod.f90
+mpi_utilities/null_mpi_utilities_mod.f90
+obs_def/obs_def_mod.f90
+obs_kind/obs_kind_mod.f90
+obs_sequence/obs_sequence_mod.f90
observations/MADIS/convert_madis_mesonet.f90
+observations/MADIS/meteor_mod.f90
+observations/obs_error/dewpoint_obs_err_mod.f90
observations/obs_error/ncep_obs_err_mod.f90
-observations/obs_error/dewpoint_obs_err_mod.f90
-observations/MADIS/meteor_mod.f90
observations/utilities/obs_utilities_mod.f90
-location/threed_sphere/location_mod.f90
-obs_sequence/obs_sequence_mod.f90
-obs_kind/obs_kind_mod.f90
-obs_def/obs_def_mod.f90
-assim_model/assim_model_mod.f90
-models/template/model_mod.f90
-common/types_mod.f90
random_seq/random_seq_mod.f90
+sort/sort_mod.f90
+time_manager/time_manager_mod.f90
utilities/utilities_mod.f90
-time_manager/time_manager_mod.f90
-mpi_utilities/null_mpi_utilities_mod.f90
Modified: DART/trunk/observations/MADIS/work/path_names_convert_madis_satwnd
===================================================================
--- DART/trunk/observations/MADIS/work/path_names_convert_madis_satwnd 2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/work/path_names_convert_madis_satwnd 2015-05-15 21:54:11 UTC (rev 7989)
@@ -1,15 +1,16 @@
+assim_model/assim_model_mod.f90
+common/types_mod.f90
+location/threed_sphere/location_mod.f90
+models/template/model_mod.f90
+mpi_utilities/null_mpi_utilities_mod.f90
+obs_def/obs_def_mod.f90
+obs_kind/obs_kind_mod.f90
+obs_sequence/obs_sequence_mod.f90
observations/MADIS/convert_madis_satwnd.f90
+observations/MADIS/meteor_mod.f90
observations/obs_error/ncep_obs_err_mod.f90
-observations/MADIS/meteor_mod.f90
observations/utilities/obs_utilities_mod.f90
-location/threed_sphere/location_mod.f90
-obs_sequence/obs_sequence_mod.f90
-obs_kind/obs_kind_mod.f90
-obs_def/obs_def_mod.f90
-assim_model/assim_model_mod.f90
-models/template/model_mod.f90
-common/types_mod.f90
random_seq/random_seq_mod.f90
+sort/sort_mod.f90
+time_manager/time_manager_mod.f90
utilities/utilities_mod.f90
-time_manager/time_manager_mod.f90
-mpi_utilities/null_mpi_utilities_mod.f90
More information about the Dart-dev
mailing list