[ncl-talk] large NetCDF file output problem
Vollaro, David
dvollaro at albany.edu
Wed Feb 25 13:12:48 MST 2015
Hi,
I am trying to write to a NetCDF output file and am running into what I believe is a size limitation. When
I write out fewer times this problem goes away. I searched for the error and believe that is linked to an
output size issue. I tried changing the NetCDF format following http://www.ncl.ucar.edu/FAQ/#file_io_004
However, I still get the same error:
ncendef: ncid 589824: NetCDF: One or more variable sizes violate format constraints
fatal:NetCDF: Operation not allowed in define mode: error attempting to write variable (u) to file (/jm13/djv/XUVTest.nc)
fatal:["Execute.c":8128]:Execute: Error occurred at or near line 164 in file netcdf-size-tst.ncl
(0) u
ncredef: ncid 589824: NetCDF: Operation not allowed in define mode
ncendef: ncid 589824: NetCDF: One or more variable sizes violate format constraints
fatal:NetCDF: Operation not allowed in define mode: error attempting to write variable (lev) to file (/jm13/djv/XUVTest.nc)
fatal:["Execute.c":8128]:Execute: Error occurred at or near line 166 in file netcdf-size-tst.ncl
(0) v
ncredef: ncid 589824: NetCDF: Operation not allowed in define mode
ncendef: ncid 589824: NetCDF: One or more variable sizes violate format constraints
fatal:NetCDF: Operation not allowed in define mode: error attempting to write variable (time) to file (/jm13/djv/XUVTest.nc)
fatal:["Execute.c":8128]:Execute: Error occurred at or near line 168 in file netcdf-size-tst.ncl
(0) t
ncclose: ncid 589824: NetCDF: One or more variable sizes violate format constraints
----------------------------------------------------------------------------------------------------
I am running version 6.1.2. I have attached the script below and it is called with a parameter:
ncl ityp=3 netcdf-size-tst.ncl
I have tried "NetCDF3" "NetCDF4" "LargeFile" "64BitOffset" and "NetCDF4Classic" to no avail.
Any help would be appreciated! Thanks
Dave Vollaro
----------------------------------------------------------------------------------------------------
test script
----------------------------------------------------------------------------------------------------
load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl"
load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl"
load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl"
load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl"
;load "$NCARG_ROOT/lib/ncarg/nclscripts/contrib/ut_string.ncl"
begin
; netcdf-size-tst.ncl
; Modified by djv 2/13/15
; This program will test size constraints in writing out NETCDF output in ncl.
; Will address below errors:
; ncendef:ncid 327680: NetCDF:One or more variable sizes violate format constraints
; ncvarput: ncid 327680: NetCDF: Operation not allowed in define mode
;
; Will test various setfileoption "Format" options:
; ie "NetCDF3" "NetCDF4" "LargeFile" "64BitOffset" "NetCDF4Classic"
print("Starting time:")
sys_cmd = "date "
system(sys_cmd)
sys_cmd = "sleep 1 "
system(sys_cmd)
;***********************************************
; set USER DEF VARS
;***********************************************
ilon = 512 ;num lons of input GG grid
jlat = 256 ;num lats of input GG grid
odatpth = "/jm13/djv/"
ofile_type = new((/5/),"string")
ofile_type = (/"NetCDF3","NetCDF4","LargeFile","64BitOffset","NetCDF4Classic"/)
print(ofile_type)
ntimes_out=30
year = 1979
;***********************************************
; LEVEL EXTRACT RANGE
;***********************************************
parm = True
parm at blev = 1000 ;bottommost plev to extract both=999 extract all levs
parm at tlev = 50 ;topmost plev to extract both=999 extract all levs
;***********************************************
; DATA SOURCE
;***********************************************
url = "http://ramadda.atmos.albany.edu:8080/repository/opendap/Top/"
filename_u = url+ "UAlbany+Repository/ERA-Interim/"+year+"/u."+year+".nc/entry.das"
filename_v = url+ "UAlbany+Repository/ERA-Interim/"+year+"/v."+year+".nc/entry.das"
filename_t = url+ "UAlbany+Repository/ERA-Interim/"+year+"/t."+year+".nc/entry.das"
exists = isfilepresent(filename_t)
if(.not.exists) then
print("OPeNDAP isfilepresent test unsuccessful.")
print("File doesn't exist or NCL doesn't have OPeNDAP ability on this system")
else
print("OPeNDAP isfilepresent test successful.")
ufile_nc = addfile(filename_u,"r")
vfile_nc = addfile(filename_v,"r")
tfile_nc = addfile(filename_t,"r")
vars = getfilevarnames(ufile_nc)
print(vars)
end if
sys_cmd = "ncdump -h "+filename_u
system(sys_cmd)
namesu = getfilevarnames(ufile_nc) ;Get the variable names in the
namesv = getfilevarnames(vfile_nc) ;Get the variable names in the
namest = getfilevarnames(tfile_nc) ;Get the variable names in the
print(namesu) ;GRIB file and print them out.
print(namesv) ;GRIB file and print them out.
print(namest) ;GRIB file and print them out.
;***********************************************
; level range selection processing section
;***********************************************
levname = str_match(namesu,"lev") ;finds lev array name (top2bot)
levdims = getfilevardimsizes (vfile_nc,levname) ;
LEV = ufile_nc->$levname$ ;read in level data
if (parm at blev.eq. 999.and.parm at tlev.eq. 999) then
lev0=0
lev1=levdims-1
else
lev0 = ind_nearest_coord(parm at blev,LEV,0)
lev1 = ind_nearest_coord(parm at tlev,LEV,0)
end if
nlev_out = lev1 - lev0 + 1
print (parm at blev+" "+lev0+" "+parm at tlev+" "+lev1+" nlevout="+nlev_out)
print (LEV(lev0:lev1))
;***********************************************
; check for existing output files and remove
;***********************************************
ncout = odatpth+"XUVTest.nc"
rmcmd = "'rm' " + ncout
system (rmcmd) ;remove any pre-exist ncout fil
ncdf_out = addfile(ncout ,"c") ;create output NC file
setfileoption(ncdf_out,"DefineMode",True)
setfileoption("nc","Format",ofile_type(ityp))
print("NCOUT "+ofile_type(ityp))
;===================================================================
; create global attributes of the file
;===================================================================
fAtt = True ; assign file attributes
fAtt at title = "NCL Efficient Approach to netCDF Creation"
fAtt at source_file = "original-file.nc"
fAtt at Conventions = "None"
fAtt at creation_date = systemfunc ("date")
fileattdef( ncdf_out, fAtt ) ; copy file attributes
iout=0
;******************************************************************************
; data processing loop (U/V/T(tim,lev,lat,lon)
;******************************************************************************
do i = 0,ntimes_out-1
print (i+" ")
vname = str_match(namesv,"v") ;finds lev array name (top2bot)
V = vfile_nc->v(i,lev0:lev1,:,:) ; read in 3D dataset for 1time
uname = str_match(namesu,"u") ;finds lev array name (top2bot)
U = ufile_nc->u(i,lev0:lev1,:,:) ; read in 3D dataset for 1time
tname = str_match(namest,"t") ;finds lev array name (top2bot)
T = tfile_nc->t(i,lev0:lev1,:,:) ; read in 3D dataset for 1time
if (i .eq. 0 ) then
;===================================================================
; define dimension characteristics of NETCDF output file
;===================================================================
dimNames = (/"time", "level", "latitude", "longitude"/)
dimSizes = (/ ntimes_out,nlev_out,jlat,ilon/)
dimUnlim = (/ False, False, False, False/)
filedimdef(ncdf_out,dimNames,dimSizes,dimUnlim)
;===================================================================
; predefine the the dimensionality of the variables to be written out
;===================================================================
filevardef(ncdf_out, "u" ,typeof(U),dimNames)
filevardef(ncdf_out, "v" ,typeof(V),dimNames)
filevardef(ncdf_out, "t" ,typeof(T),dimNames)
uNew = new ( (/ntimes_out,nlev_out,jlat,ilon/), typeof(U) )
vNew = new ( (/ntimes_out,nlev_out,jlat,ilon/), typeof(V) )
tNew = new ( (/ntimes_out,nlev_out,jlat,ilon/), typeof(T) )
end if
uNew(iout,:,:,:) = U(lev0:lev1,:,:)
vNew(iout,:,:,:) = V(lev0:lev1,:,:)
tNew(iout,:,:,:) = T(lev0:lev1,:,:)
iout=iout+1
end do
;==========End time processing loop ================================
end
print("write output")
ncdf_out->$uname$ = (/uNew/)
print("u")
ncdf_out->$vname$ = (/vNew/)
print("v")
ncdf_out->$tname$ = (/tNew/)
print("t")
exit
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://mailman.ucar.edu/pipermail/ncl-talk/attachments/20150225/a7f8a55f/attachment.html
More information about the ncl-talk
mailing list