[Dart-dev] [7989] DART/trunk/observations/MADIS: sort the times before adding the observations to the output obs_seq file.

nancy at ucar.edu nancy at ucar.edu
Fri May 15 15:54:12 MDT 2015


Revision: 7989
Author:   nancy
Date:     2015-05-15 15:54:11 -0600 (Fri, 15 May 2015)
Log Message:
-----------
sort the times before adding the observations to the output obs_seq file.
this speeds the converters by a huge amount.  updated the path_names files
to include the sort module.  also fixed an apparent bug in the satwnd
converter when it was trying to identify duplicate observations.  the wrong
indices were being used for pressure and band when traversing the arrays.

Modified Paths:
--------------
    DART/trunk/observations/MADIS/convert_madis_acars.f90
    DART/trunk/observations/MADIS/convert_madis_mesonet.f90
    DART/trunk/observations/MADIS/convert_madis_satwnd.f90
    DART/trunk/observations/MADIS/work/path_names_convert_madis_acars
    DART/trunk/observations/MADIS/work/path_names_convert_madis_mesonet
    DART/trunk/observations/MADIS/work/path_names_convert_madis_satwnd

-------------- next part --------------
Modified: DART/trunk/observations/MADIS/convert_madis_acars.f90
===================================================================
--- DART/trunk/observations/MADIS/convert_madis_acars.f90	2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/convert_madis_acars.f90	2015-05-15 21:54:11 UTC (rev 7989)
@@ -41,6 +41,7 @@
                               acars_rel_hum_error
 use dewpoint_obs_err_mod, only : dewpt_error_from_rh_and_temp, &
                                  rh_error_from_dewpt_and_temp
+use          sort_mod, only : index_sort
 use      obs_kind_mod, only : ACARS_U_WIND_COMPONENT, ACARS_V_WIND_COMPONENT, &
                               ACARS_TEMPERATURE, ACARS_SPECIFIC_HUMIDITY, &
                               ACARS_DEWPOINT, ACARS_RELATIVE_HUMIDITY
@@ -77,6 +78,7 @@
 real(r8), allocatable :: lat(:), lon(:), palt(:), tair(:), relh(:), tdew(:), &
                          wdir(:), wspd(:), latu(:), lonu(:), palu(:)
 integer,  allocatable :: qc_palt(:), qc_tair(:), qc_relh(:), qc_tdew(:), qc_wdir(:), qc_wspd(:)
+integer,  allocatable :: indu(:), sorted_indu(:)
 
 type(obs_sequence_type) :: obs_seq
 type(obs_type)          :: obs, prev_obs
@@ -105,6 +107,7 @@
 allocate(latu(nobs))  ;  allocate(lonu(nobs))
 allocate(palu(nobs))  ;  allocate(tdew(nobs))
 allocate(tobu(nobs))
+allocate(indu(nobs))  ;  allocate(sorted_indu(nobs))
 
 nvars = 3
 if (include_specific_humidity) nvars = nvars + 1
@@ -174,14 +177,11 @@
 qc = 1.0_r8
 
 nused = 0
-obsloop: do n = 1, nobs
+obsloop1: do n = 1, nobs
 
-  ! compute time of observation
-  time_obs = increment_time(comp_day0, tobs(n))
-
   ! check the lat/lon values to see if they are ok
-  if ( lat(n) >  90.0_r8 .or. lat(n) <  -90.0_r8 ) cycle obsloop
-  if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop
+  if ( lat(n) >  90.0_r8 .or. lat(n) <  -90.0_r8 ) cycle obsloop1
+  if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop1
 
   ! change lon from -180 to 180 into 0-360
   if ( lon(n) < 0.0_r8 )  lon(n) = lon(n) + 360.0_r8
@@ -191,10 +191,29 @@
     if ( lon(n)  == lonu(i) .and. &
          lat(n)  == latu(i) .and. &
          tobs(n) == tobu(i) .and. & 
-         palt(n) == palu(i) ) cycle obsloop
+         palt(n) == palu(i) ) cycle obsloop1
   end do
 
-  if ( palt(n) == palt_miss .or. qc_palt(n) /= 0 ) cycle obsloop  
+  nused = nused + 1
+  indu(nused) = n
+  latu(nused) =  lat(n)
+  lonu(nused) =  lon(n)
+  palu(nused) = palt(n)
+  tobu(nused) = tobs(n)
+
+end do obsloop1
+
+call index_sort(tobu, sorted_indu, nused)
+
+obsloop2: do i = 1, nused
+
+  ! get the next unique observation in sorted time order
+  n = indu(sorted_indu(i))
+
+  ! compute time of observation
+  time_obs = increment_time(comp_day0, tobs(n))
+
+  if ( palt(n) == palt_miss .or. qc_palt(n) /= 0 ) cycle obsloop2
   pres = pres_alt_to_pres(palt(n))
 
   ! extract actual time of observation in file into oday, osec.
@@ -314,14 +333,8 @@
 
 100 continue
 
-  nused = nused + 1
-  latu(nused) =  lat(n)
-  lonu(nused) =  lon(n)
-  palu(nused) = palt(n)
-  tobu(nused) = tobs(n)
+end do obsloop2
 
-end do obsloop
-
 ! if we added any obs to the sequence, write it now.
 if ( get_num_obs(obs_seq) > 0 )  call write_obs_seq(obs_seq, acars_out_file)
 

Modified: DART/trunk/observations/MADIS/convert_madis_mesonet.f90
===================================================================
--- DART/trunk/observations/MADIS/convert_madis_mesonet.f90	2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/convert_madis_mesonet.f90	2015-05-15 21:54:11 UTC (rev 7989)
@@ -34,7 +34,7 @@
 use         types_mod, only : r8, missing_r8
 use     utilities_mod, only : nc_check, initialize_utilities, finalize_utilities
 use  time_manager_mod, only : time_type, set_calendar_type, set_date, &
-                                  increment_time, get_time, operator(-), GREGORIAN
+                              increment_time, get_time, operator(-), GREGORIAN
 use      location_mod, only : VERTISSURFACE
 use  obs_sequence_mod, only : obs_sequence_type, obs_type, read_obs_seq, &
                               static_init_obs_sequence, init_obs, write_obs_seq, & 
@@ -51,8 +51,9 @@
                               LAND_SFC_TEMPERATURE, LAND_SFC_SPECIFIC_HUMIDITY, & 
                               LAND_SFC_DEWPOINT, LAND_SFC_RELATIVE_HUMIDITY, &
                               LAND_SFC_ALTIMETER                
-use  obs_utilities_mod, only : getvar_real, get_or_fill_QC, add_obs_to_seq, &
-                               create_3d_obs, getvar_int, getdimlen, set_missing_name
+use          sort_mod, only : index_sort
+use obs_utilities_mod, only : getvar_real, get_or_fill_QC, add_obs_to_seq, &
+                              create_3d_obs, getvar_int, getdimlen, set_missing_name
 
 use           netcdf
 
@@ -73,23 +74,26 @@
 integer, parameter :: num_copies = 1,   &   ! number of copies in sequence
                       num_qc     = 1        ! number of QC entries
 
-integer :: ncid, nobs, nvars, n, i, oday, osec, nused
+integer  :: ncid, nobs, nvars, n, i, oday, osec, nused
 logical  :: file_exist, first_obs
 real(r8) :: alti_miss, tair_miss, tdew_miss, wdir_miss, wspd_miss, uwnd, &
             vwnd, palt, qobs, qsat, rh, oerr, pres, qerr, qc
 
-integer,  allocatable :: tobs(:), tobu(:)
+integer,  allocatable :: tobs(:), tobu(:), indu(:), sorted_indu(:)
 real(r8), allocatable :: lat(:), lon(:), elev(:), alti(:), tair(:), & 
                          tdew(:), wdir(:), wspd(:), latu(:), lonu(:)
 integer,  allocatable :: qc_alti(:), qc_tair(:), qc_tdew(:), qc_wdir(:), qc_wspd(:)
-
+  
 type(obs_sequence_type) :: obs_seq
 type(obs_type)          :: obs, prev_obs
 type(time_type)         :: comp_day0, time_obs, prev_time
 
 
+!----------------------------------------------------------------
+
 call initialize_utilities('convert_madis_mesonet')
 
+
 ! put the reference date into DART format
 call set_calendar_type(GREGORIAN)
 comp_day0 = set_date(1970, 1, 1, 0, 0, 0)
@@ -109,6 +113,7 @@
 allocate(tair(nobs))  ;  allocate(tdew(nobs))
 allocate(wdir(nobs))  ;  allocate(wspd(nobs))
 allocate(tobs(nobs))  ;  allocate(tobu(nobs))
+allocate(indu(nobs))  ;  allocate(sorted_indu(nobs))
 
 nvars = 4
 if (include_specific_humidity) nvars = nvars + 1
@@ -178,25 +183,41 @@
 qc = 1.0_r8
 
 nused = 0
-obsloop: do n = 1, nobs
+obsloop1: do n = 1, nobs
 
-  ! compute time of observation
-  time_obs = increment_time(comp_day0, tobs(n))
-
   ! check the lat/lon values to see if they are ok
-  if ( lat(n) >  90.0_r8 .or. lat(n) <  -90.0_r8 ) cycle obsloop
-  if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop
+  if ( lat(n) >  90.0_r8 .or. lat(n) <  -90.0_r8 ) cycle obsloop1
+  if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop1
 
   ! change lon from -180 to 180 into 0-360
   if ( lon(n) < 0.0_r8 )  lon(n) = lon(n) + 360.0_r8
 
-  ! Check for duplicate observations
-  do i = 1, nused
+  ! check for duplicate observations
+  do i=1, nused
     if ( lon(n) == lonu(i) .and. &
          lat(n) == latu(i) .and. &
-        tobs(n) == tobu(i) ) cycle obsloop
-  end do
+        tobs(n) == tobu(i) ) cycle obsloop1
+  enddo
 
+  nused = nused + 1
+  indu(nused) =  n
+  latu(nused) =  lat(n)
+  lonu(nused) =  lon(n)
+  tobu(nused) = tobs(n)
+
+enddo obsloop1
+
+! sort into time order
+call index_sort(tobu, sorted_indu, nused)
+
+
+obsloop2: do i = 1, nused
+
+  n = indu(sorted_indu(i))
+
+  ! compute time of observation
+  time_obs = increment_time(comp_day0, tobs(n))
+
   palt = pres_alt_to_pres(elev(n)) * 0.01_r8
 
   ! extract actual time of observation in file into oday, osec.
@@ -334,19 +355,15 @@
 
 100 continue
 
-  nused = nused + 1
-  latu(nused) =  lat(n)
-  lonu(nused) =  lon(n)
-  tobu(nused) = tobs(n)
+end do obsloop2
 
-end do obsloop
-
 ! if we added any obs to the sequence, write it now.
 if ( get_num_obs(obs_seq) > 0 )  call write_obs_seq(obs_seq, surface_out_file)
 
 ! end of main program
 call finalize_utilities()
 
+
 end program convert_madis_mesonet
 
 ! <next few lines under version control, do not edit>

Modified: DART/trunk/observations/MADIS/convert_madis_satwnd.f90
===================================================================
--- DART/trunk/observations/MADIS/convert_madis_satwnd.f90	2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/convert_madis_satwnd.f90	2015-05-15 21:54:11 UTC (rev 7989)
@@ -45,6 +45,7 @@
 use        meteor_mod, only : wind_dirspd_to_uv
 use       obs_err_mod, only : sat_wind_error, sat_wv_wind_error
 use      obs_kind_mod, only : SAT_U_WIND_COMPONENT, SAT_V_WIND_COMPONENT
+use          sort_mod, only : index_sort
 use obs_utilities_mod, only : getvar_real, get_or_fill_QC, add_obs_to_seq, &
                               create_3d_obs, getvar_int, getdimlen
 
@@ -77,10 +78,11 @@
 
 
 real(r8), allocatable :: lat(:), lon(:), latu(:), lonu(:), &
-                          pres(:), prsu(:), tobs(:), tobu(:), &
+                          pres(:), prsu(:), tobs(:), &
                           wdir(:), wspd(:)
 integer,  allocatable :: band(:), bndu(:)
 integer,  allocatable :: qc_wdir(:), qc_wspd(:)
+integer,  allocatable :: indu(:), sorted_indu(:), tobu(:)
 
 type(obs_sequence_type) :: obs_seq
 type(obs_type)          :: obs, prev_obs
@@ -122,6 +124,7 @@
 allocate(tobs(nobs))  ;  allocate(tobu(nobs))
 allocate(band(nobs))  ;  allocate(bndu(nobs))
 allocate(wdir(nobs))  ;  allocate(wspd(nobs))
+allocate(indu(nobs))  ;  allocate(sorted_indu(nobs))
 allocate(qc_wdir(nobs)) ;  allocate(qc_wspd(nobs))
 
 ! read in the data arrays
@@ -171,34 +174,52 @@
 qc = 1.0_r8
 
 nused = 0
-obsloop: do n = 1, nobs
+obsloop1: do n = 1, nobs
 
-  ! compute time of observation
-  time_obs = increment_time(comp_day0, nint(tobs(n)))
-
   ! check the lat/lon values to see if they are ok
-  if ( lat(n) >  90.0_r8 .or. lat(n) <  -90.0_r8 ) cycle obsloop
-  if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop
+  if ( lat(n) >  90.0_r8 .or. lat(n) <  -90.0_r8 ) cycle obsloop1
+  if ( lon(n) > 180.0_r8 .or. lon(n) < -180.0_r8 ) cycle obsloop1
 
+  ! change lon from -180 to 180 into 0-360
   if ( lon(n) < 0.0_r8 )  lon(n) = lon(n) + 360.0_r8
 
   ! Check for duplicate observations
   do i = 1, nused
     if ( lon(n) == lonu(i) .and. &
          lat(n) == latu(i) .and. &
-        prsu(n) == pres(i) .and. &
+        pres(n) == prsu(i) .and. &
         band(n) == bndu(i) .and. &
-        tobs(n) == tobu(i) ) cycle obsloop
+        nint(tobs(n)) == tobu(i) ) cycle obsloop1
   end do
 
   ! if selecting only certain bands, cycle if not wanted
   if (.not. allbands) then
-     if (.not. iruse  .and. band(n) == 1) cycle obsloop
-     if (.not. visuse .and. band(n) == 2) cycle obsloop
+     if (.not. iruse  .and. band(n) == 1) cycle obsloop1
+     if (.not. visuse .and. band(n) == 2) cycle obsloop1
      if (.not. wvuse  .and. &
-         (band(n) == 3  .or.  band(n) == 5  .or. band(n) == 7)) cycle obsloop
+         (band(n) == 3  .or.  band(n) == 5  .or. band(n) == 7)) cycle obsloop1
   endif
 
+  nused = nused + 1
+  indu(nused) = n
+  latu(nused) =  lat(n)
+  lonu(nused) =  lon(n)
+  prsu(nused) = pres(n)
+  bndu(nused) = band(n)
+  tobu(nused) = nint(tobs(n))
+
+end do obsloop1
+
+call index_sort(tobu, sorted_indu, nused)
+
+obsloop2: do i = 1, nused
+
+  ! get the next unique observation in sorted time order
+  n = indu(sorted_indu(i))
+
+  ! compute time of observation
+  time_obs = increment_time(comp_day0, nint(tobs(n)))
+
   ! extract actual time of observation in file into oday, osec.
   call get_time(time_obs, osec, oday)
 
@@ -211,7 +232,7 @@
 
    !  perform sanity checks on observation errors and values
    if ( oerr == missing_r8 .or. wdir(n) < 0.0_r8 .or. wdir(n) > 360.0_r8 .or. &
-      wspd(n) < 0.0_r8 .or. wspd(n) > 120.0_r8 )  cycle obsloop
+      wspd(n) < 0.0_r8 .or. wspd(n) > 120.0_r8 )  cycle obsloop2
 
       call create_3d_obs(lat(n), lon(n), pres(n), VERTISPRESSURE, uwnd, &
                          SAT_U_WIND_COMPONENT, oerr, oday, osec, qc, obs)
@@ -223,15 +244,8 @@
 
   endif
 
-  nused = nused + 1
-  latu(nused) =  lat(n)
-  lonu(nused) =  lon(n)
-  prsu(nused) = pres(n)
-  band(nused) = bndu(n)
-  tobu(nused) = tobs(n)
+end do obsloop2
 
-end do obsloop
-
 ! need to wait to close file because in the loop it queries the
 ! report types.
 call nc_check( nf90_close(ncid) , &

Modified: DART/trunk/observations/MADIS/work/path_names_convert_madis_acars
===================================================================
--- DART/trunk/observations/MADIS/work/path_names_convert_madis_acars	2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/work/path_names_convert_madis_acars	2015-05-15 21:54:11 UTC (rev 7989)
@@ -1,16 +1,17 @@
+assim_model/assim_model_mod.f90
+common/types_mod.f90
+location/threed_sphere/location_mod.f90
+models/template/model_mod.f90
+mpi_utilities/null_mpi_utilities_mod.f90
+obs_def/obs_def_mod.f90
+obs_kind/obs_kind_mod.f90
+obs_sequence/obs_sequence_mod.f90
 observations/MADIS/convert_madis_acars.f90
+observations/MADIS/meteor_mod.f90
+observations/obs_error/dewpoint_obs_err_mod.f90
 observations/obs_error/ncep_obs_err_mod.f90
-observations/obs_error/dewpoint_obs_err_mod.f90
-observations/MADIS/meteor_mod.f90
 observations/utilities/obs_utilities_mod.f90
-location/threed_sphere/location_mod.f90
-obs_sequence/obs_sequence_mod.f90
-obs_kind/obs_kind_mod.f90
-obs_def/obs_def_mod.f90
-assim_model/assim_model_mod.f90
-models/template/model_mod.f90
-common/types_mod.f90
 random_seq/random_seq_mod.f90
+sort/sort_mod.f90
+time_manager/time_manager_mod.f90
 utilities/utilities_mod.f90
-time_manager/time_manager_mod.f90
-mpi_utilities/null_mpi_utilities_mod.f90

Modified: DART/trunk/observations/MADIS/work/path_names_convert_madis_mesonet
===================================================================
--- DART/trunk/observations/MADIS/work/path_names_convert_madis_mesonet	2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/work/path_names_convert_madis_mesonet	2015-05-15 21:54:11 UTC (rev 7989)
@@ -1,16 +1,17 @@
+assim_model/assim_model_mod.f90
+common/types_mod.f90
+location/threed_sphere/location_mod.f90
+models/template/model_mod.f90
+mpi_utilities/null_mpi_utilities_mod.f90
+obs_def/obs_def_mod.f90
+obs_kind/obs_kind_mod.f90
+obs_sequence/obs_sequence_mod.f90
 observations/MADIS/convert_madis_mesonet.f90
+observations/MADIS/meteor_mod.f90
+observations/obs_error/dewpoint_obs_err_mod.f90
 observations/obs_error/ncep_obs_err_mod.f90
-observations/obs_error/dewpoint_obs_err_mod.f90
-observations/MADIS/meteor_mod.f90
 observations/utilities/obs_utilities_mod.f90
-location/threed_sphere/location_mod.f90
-obs_sequence/obs_sequence_mod.f90
-obs_kind/obs_kind_mod.f90
-obs_def/obs_def_mod.f90
-assim_model/assim_model_mod.f90
-models/template/model_mod.f90
-common/types_mod.f90
 random_seq/random_seq_mod.f90
+sort/sort_mod.f90
+time_manager/time_manager_mod.f90
 utilities/utilities_mod.f90
-time_manager/time_manager_mod.f90
-mpi_utilities/null_mpi_utilities_mod.f90

Modified: DART/trunk/observations/MADIS/work/path_names_convert_madis_satwnd
===================================================================
--- DART/trunk/observations/MADIS/work/path_names_convert_madis_satwnd	2015-05-15 21:04:29 UTC (rev 7988)
+++ DART/trunk/observations/MADIS/work/path_names_convert_madis_satwnd	2015-05-15 21:54:11 UTC (rev 7989)
@@ -1,15 +1,16 @@
+assim_model/assim_model_mod.f90
+common/types_mod.f90
+location/threed_sphere/location_mod.f90
+models/template/model_mod.f90
+mpi_utilities/null_mpi_utilities_mod.f90
+obs_def/obs_def_mod.f90
+obs_kind/obs_kind_mod.f90
+obs_sequence/obs_sequence_mod.f90
 observations/MADIS/convert_madis_satwnd.f90
+observations/MADIS/meteor_mod.f90
 observations/obs_error/ncep_obs_err_mod.f90
-observations/MADIS/meteor_mod.f90
 observations/utilities/obs_utilities_mod.f90
-location/threed_sphere/location_mod.f90
-obs_sequence/obs_sequence_mod.f90
-obs_kind/obs_kind_mod.f90
-obs_def/obs_def_mod.f90
-assim_model/assim_model_mod.f90
-models/template/model_mod.f90
-common/types_mod.f90
 random_seq/random_seq_mod.f90
+sort/sort_mod.f90
+time_manager/time_manager_mod.f90
 utilities/utilities_mod.f90
-time_manager/time_manager_mod.f90
-mpi_utilities/null_mpi_utilities_mod.f90


More information about the Dart-dev mailing list