;---ncl-talk: 7/21/2017
;---
;---I have txt files containing 35 different variables which I'm trying to save 
;---in netcdf format. The current problem is that I need to specify every variable 
;---on its own and it is not elegant (attached). Later it will be problem since 
;---columns will be rearranged in random order from time to time.
;---
;---------------------------------------------------------------------------------
;                     LOCAL FUNCTIONS and PROCEDURES
;---------------------------------------------------------------------------------
undef("create_time")
function create_time(TIMESTAMP[*]:string)
; 
; Create a udunits time variable for netCDF
; Parse: TIMESTAMP = "2017-06-08 22:53:30" 
; Delimeters: - :"
;
local delim, yyyy, mm, dd, hh, mn, sc, units
begin
   delim= "- :" + str_get_dq()                           ; eliminate "
   yyyy = toint( str_get_field(TIMESTAMP,1,delim) )
   mm   = toint( str_get_field(TIMESTAMP,2,delim) )
   dd   = toint( str_get_field(TIMESTAMP,3,delim) )
   hh   = toint( str_get_field(TIMESTAMP,4,delim) )
   mn   = toint( str_get_field(TIMESTAMP,5,delim) )
   sc   = toint( str_get_field(TIMESTAMP,6,delim) )

  ;time_units= "seconds since 2017-01-01 00:00:00"       ; reference time is arbitrary
   time_units= "seconds since 2017-06-01 00:00:00"      

   time = cd_inv_calendar(yyyy,mm,dd,hh,mn,sc,time_units, 0)

   time!0    = "time"
   time&time =  time
   return(time)
end

;-----------------

undef("create_var_nc")
procedure create_var_nc(ncdf[1]:file, x[*]:numeric, time[*]:numeric, varName[1]:string, units[1]:string)
; 
; procedure used because nothing is returned. Several 'actions' are performed
; objective: create a time series variable for netCDF
; [1] assign attributes aand a 'time' coordinate variable to variable
; [2] write variable to netCDF file
;
begin
   x!0    = "time"           ; name dimension
   x&time =  time            ; associate values with the dimension name
   x@units=  units

   ncdf->$varName$ = x       ; write variable data object to netCDF
end

;---------------------------------------------------------------------------------
;                                 MAIN
;---------------------------------------------------------------------------------
;             Golden Rule of Data Processing: Look at your data!
;---------------------------------------------------------------------------------

;---Input text file

   dirTxt = "./"
   filTxt = "TOA5_STUP_10s_5.dat"            ; four header lines
   pthTxt = dirTxt + filTxt

;---Read entire text file as "string": Extract information
;   line 1  (index 0) : header: general information
;   line 2  (index 1) : header: column (variable) names
;   line 3  (index 2) : header: column (variable) units
;   line 4  (index 3) : header: "Smp"                  
;   line 5: (index 4:): data:   numeric values from line 5 onward    

   dat_str = asciiread (pthTxt, -1, "string")
   nrow_str= dimsizes(dat_str)                     ; total # lines;  includes 4 header lines
   print(dat_str)
   print("-----")

;---Good programming practice
;---Make sure the number of column variable names; variable units; numeric values are the same

   delim  = "," + str_get_dq()                      ; elminate " also
   nfldn  = str_fields_count(dat_str(1), delim)     ; number of column/variable names  (2nd line)
   nfldu  = str_fields_count(dat_str(2), delim)     ; number of column/variable units  (3rd line)
   nfldv  = str_fields_count(dat_str(4), delim)     ; number of column/variable values (5th line)

   print("nfldn="+nfldn+"  nfldu="+nfldu+"  nfldv="+nfldv)
   print("-----")

   if (nfldn.ne.nfldu .or. nfldn.ne.nfldv) then  ; error/safety check
       print("number of data variables, units and values are different")
       exit
   end if
   nvar = nfldn                                  ; convenience: # variables:  nvar=nflds=nfldu-nfldv
                                                 ; nvar includes the TIMESTAMP variable

;---Parse the variable (column) names and units. This includes the  TIMESTAMP variable.

   varName  = new( nvar, "string", "No_FillValue") 
   varUnit  = new( nvar, "string", "No_FillValue") 
   do nv=0,nvar-1
      varName(nv)   = str_get_field(dat_str(1) , (nv+1), delim)
      varUnit(nv)   = str_get_field(dat_str(2) , (nv+1), delim)
   end do

   print("name="+varName+"  units= "+varUnit)
   print("-----")

;---Explicitly extract the TIMESTAMP. Treat it 'special' 
;   This is the first variable (column/field).
;   TIMESTAMP has 6 fields (parts). 

   TIMESTAMP       = str_get_field(dat_str(4:), 1, delim)   ; [*]:string
   TIMESTAMP@units = "yyyy-mm-dd hh:mn:sc"
   ntim            = dimsizes(TIMESTAMP)
   nSTAMP          = 6   ; yyyy,mm,dd,hh,mn,sc          

   print(TIMESTAMP)
   print("-----")

;---Read numeric values: skip the 4 header lines
;   Account for multi-part TIMESTAMP
;   readAsciiTable uses 'automatic' parsing
;   Caveat: It handle the negative data values correctly. 
;           However, it will treat the - part of "yyyy-mm-dd hh:mn:sc"  
;           as a negative sign and not as a separator. These will be ignored.
;           Look at the output from 'printVarSummary(data)'

   data   = readAsciiTable(pthTxt, nvar+nSTAMP-1 , "float", 4)     ; (ntim,nvar)
   delete(data@_FillValue)                  ; no missing data
                                            ; or, set to appropriate value; data@_FillValue= ...
   data!0 = "time"                          ; not necessary but 'nice'                         
   data!1 = "variable"                      ; all elements including 6 TIMESTAMP elements
   printVarSummary(data)

   print(data)
   print("-----")

   varName := varName(1:)                   ; eliminate TIMESTAMP; := syntax to overwrite
   varUnit := varUnit(1:)
   nvar     = nvar-1                        ; no TIMESTAMP

   print("name="+varName+"  units= "+varUnit)
   print("-----")

;---netCDF file

   dirNC  = "./"
   filNC  = str_get_field(filTxt, 1, ".")+".nc"  ; use Txt root name 
   pthNC  = dirNC + filNC
   system("/bin/rm -f "+pthNC)              ; if pthNC exists, remove

;---Use netCDF4:  TIMESTAMP is a string variable. nc3 does not have string type

   setfileoption("nc","Format","NetCDF4")   ; nc4 allows type 'string'
   ncdf   = addfile(pthNC, "c")

;---Create global attributes of the file (optional)
   fAtt               = True                ; assign file attributes
   fAtt@title         = "TOA: Text to netCDF"
   fAtt@Source        = filTxt  
   fAtt@Source_Line0  = str_sub_str(dat_str(0),str_get_dq()," ") ; eliminate "
   fAtt@NCL           = "version: "+get_ncl_version()
   fAtt@Conventions   = "None"
   fAtt@creation_date = systemfunc ("date")
   fileattdef( ncdf, fAtt )                      ; copy file attributes

;---time: make Unlimited; recommended for most applications

   filedimdef(ncdf,"time",-1,True) 
   time            = create_time(TIMESTAMP)  ; time(time)

;---time: Assign 'time' meta data to TIMESTAMP

   TIMESTAMP!0     = "time"                  ; time!0
   TIMESTAMP&time  =  time

   ncdf->time      = time
   ncdf->TIMESTAMP = TIMESTAMP

; Loop over data variables (columns): Write variable and units to netCDF
; Note: the TIMESTAMP is skipped.

   do nv=0,nvar-1
      print("nv="+nv+"  "+varName(nv)+"  "+ varUnit(nv))
      create_var_nc(ncdf, data(:,nv+nSTAMP), time,  varName(nv), varUnit(nv))
   end do