[ncl-talk] fatal:NclMalloc Failed:[errno=12] Segmentation fault
Amal Inge
amalingenieur89 at gmail.com
Mon Dec 3 15:29:00 MST 2018
Rick this is the end of ncl -x
+ ;----------------------------------------------------------------------
+ ; This is the main code
+ ;----------------------------------------------------------------------
+ begin
+ ;---Name of shapefile containing USA outlines
+ shp_fname = "GSAs_simplified.shp"
+
+ ;---Rough area we are interested in. Everything outside this will be
masked.
+ minlon = -08.0
+ maxlon = 45.0
+ minlat = 29.00
+ maxlat = 48.0
+
+ ;---Read in zonal winds
+ a = addfile("dox_med_1.4-101.nc","r")
+ dox = a->dox
+ printVarSummary(dox)
+ dox_dims = dimsizes(dox)
+ ntim = dox_dims(0)
+ nlev = dox_dims(1)
+ nlat = dox_dims(2)
+ nlon = dox_dims(3)
+
+ ;---Create a new mask using a shapefile of USA
+ opt = True
+ opt at return_mask = True
+ ; opt at debug = True
+ opt at minlon = minlon ; Makes the shapefile masking
+ opt at maxlon = maxlon ; go faster.
+ opt at minlat = minlat
+ opt at maxlat = maxlat
+ opt at shape_var = "SECT_COD"
+ opt at shape_names = ("GSA01")
+ pmask2d = shapefile_mask_data(dox(0,0,:,:),shp_fname,opt)
+
+ ;---Mask original 4D variable against a 4D pmask array
+ pmask4d = conform_dims(dox_dims,pmask2d,(/2,3/))
+ dox_masked = mask(dox,pmask4d,1)
+
+ copy_VarMeta(dox,dox_masked) ; copy attributes and coordinate
arrays
+ dox_masked!1 = "lev" ; rename three of the dimensions
+ dox_masked!2 = "lat"
+ dox_masked!3 = "lon"
+
+ printVarSummary(dox)
+ printVarSummary(dox_masked)
+ printMinMax(dox,0)
+ printMinMax(dox_masked,0)
+
+ wks = gsn_open_wks("eps","plot_dox01_0-209")
+
+ cmap = read_colormap_file("cmp_b2r")
+
+ res = True ; plot mods desired
+ res at gsnMaximize = True
+ res at gsnDraw = False
+ res at gsnFrame = False
+
+
+ res at cnFillOn = True ; turn on color fill
+ res at cnFillPalette = cmap(::-1,:) ; reverse color map
+ res at cnLinesOn = False ; turn off contour lines
+ res at cnLineLabelsOn = False ; turn off contour line labels
+ res at lbLabelBarOn = False ; will turn on in panel
+ res at cnLevelSelectionMode = "ManualLevels" ; manually set the
contour levels with the following 3 resources
+ res at cnMinLevelValF = 0. ; set the minimum contour
level
+ res at cnMaxLevelValF = 300. ; set the maximum contour
level
+ res at cnLevelSpacingF = 10
+
+ res at mpOutlineOn = False
+ res at mpDataBaseVersion = "MediumRes"
+ res at mpMinLonF = minlon
+ res at mpMaxLonF = maxlon
+ res at mpMinLatF = minlat
+ res at mpMaxLatF = maxlat
+ res at mpCenterLonF = (minlon+maxlon)/2.
+ res at pmTickMarkDisplayMode = "Always" ; nicer tickmarks
+ res at pmTitleZone = 4 ; moves title down
+
+ res at gsnAddCyclic = False ; Regional data
+
+ ;---Be sure to use same levels for both plots
+ res at gsnRightString = "mmol/m3"
+ res at gsnLeftString = "Dissolved_Molecular_Oxygen"
+
+ lnres = True
+ pres = True
+ pres at gsnMaximize = True
+ pres at gsnPanelLabelBar = True
+
+ ;---Comment out the plotting for now because it's time consuming.
+ ;; do nt=0,dox_dims(0)-1
+ ;; res at tiMainString = "Original data, nt = " + nt
+ ;; plot_orig = gsn_csm_contour_map(wks,dox(nt,0,:,:),res)
+ ;; res at tiMainString = "Masked data, nt = " + nt
+ ;; plot_mask = gsn_csm_contour_map(wks,dox_masked(nt,0,:,:),res)
+ ;; id_orig =
gsn_add_shapefile_polylines(wks,plot_orig,shp_fname,lnres)
+ ;; id_mask =
gsn_add_shapefile_polylines(wks,plot_mask,shp_fname,lnres)
+ ;;
+ ;; gsn_panel(wks,(/plot_orig,plot_mask/),(/2,1/),pres)
+ ;; end do
+
+ ;---Open file for writing
+ diro = "./" ; Output directory
+ filo = "dox01_med_1.4-101.nc"
+ system("/bin/rm -f " + diro + filo) ; remove if exists
+ foutput = addfile (diro + filo, "c") ; open output file
+ setfileoption(foutput,"DefineMode",True)
+
+ ;---Write global file attributes
+ fAtt = True ; assign file attributes
+ fAtt at title = "NCL Efficient Approach to netCDF Creation"
+ fAtt at source_file = "dox_med_1.4-101.nc"
+ fAtt at Conventions = "None"
+ fAtt at creation_date = systemfunc ("date")
+
+ fileattdef( foutput, fAtt ) ; copy file attributes
+
+ ;---Write file dimension names and sizes
+ dimNames = (/"time","lev", "lat", "lon"/)
+ dimSizes = (/ -1 , nlev, nlat, nlon /)
+ dimUnlim = (/ True , False, False, False/)
+ filedimdef(foutput,dimNames,dimSizes,dimUnlim)
+
+ filevardef(foutput, "time"
,typeof(dox_masked&time),getvardims(dox_masked&time))
+ filevardef(foutput, "lev"
,typeof(dox_masked&lev),getvardims(dox_masked&lev))
+ filevardef(foutput, "lat"
,typeof(dox_masked&lat),getvardims(dox_masked&lat))
+ filevardef(foutput, "lon"
,typeof(dox_masked&lon),getvardims(dox_masked&lon))
+ filevardef(foutput, "dox",typeof(dox_masked),getvardims(dox_masked))
+ ; different
from name on script
+ ;---Write variable attributes
+ filevarattdef(foutput,"dox",dox_masked) ;
copy T attributes
+ filevarattdef(foutput,"time" ,dox_masked&time) ; copy time
attributes
+ filevarattdef(foutput,"lev" ,dox_masked&lev) ; copy lev
attributes
+ filevarattdef(foutput,"lat" ,dox_masked&lat) ; copy lat
attributes
+ filevarattdef(foutput,"lon" ,dox_masked&lon) ; copy lon
attributes
+
+ setfileoption(foutput,"DefineMode",False)
+
+ ;---Write variable values
+ foutput->time = (/dox_masked&time/)
+ foutput->lev = (/dox_masked&lev/)
+ foutput->lat = (/dox_masked&lat/)
+ foutput->lon = (/dox_masked&lon/)
+ foutput->dox = (/dox_masked/)
+ delete(dox)
+ end
Variable: dox
Type: float
Total Size: 2624014800 bytes
656003700 values
Number of Dimensions: 4
Dimensions and sizes: [time | 215] x [depth | 18] x [latitude | 253] x
[longitude | 670]
Coordinates:
time: [915148800..1477958400]
depth: [1.472102..101.7803]
latitude: [30.1875..45.9375]
longitude: [-5.5625..36.25]
Number Of Attributes: 6
standard_name :
mole_concentration_of_dissolved_molecular_oxygen_in_sea_water
long_name : Mole concentration of Dissolved Molecular Oxygen in sea
water
units : millimol m-3
_FillValue : 1e+20
missing_value : 1e+20
_ChunkSizes : ( 1, 22, 84, 161 )
fatal:NclMalloc Failed:[errno=12]
Segmentation fault
Could you advice me where can i delete some variables? maybe it could be
better because the script works with data of one depth i think it will
never work with heavy data 4D
Le lun. 3 déc. 2018 à 23:19, Amal Inge <amalingenieur89 at gmail.com> a écrit :
> hi kevin, it was just a mistake because i tried many files.
> i correct it but same error i get
>
> Le lun. 3 déc. 2018 à 22:59, Kevin Hallock <hallock at ucar.edu> a écrit :
>
>> Hello,
>>
>> I noticed something else in your dox.ncl script that could potentially
>> cause problems:
>>
>> a = addfile("dox_med_1.4-101.nc","r")
>> ...
>> diro = "./" ; Output directory
>> filo = "dox_med_1.4-101.nc"
>> system("/bin/rm -f " + diro + filo) ; remove if exists
>> foutput = addfile (diro + filo, "c") ; open output file
>> ...
>> foutput->dox = (/dox_masked/)
>>
>> The possible issue I see with this is that the script reads from the file
>> “dox_med_1.4-101.nc”, deletes the file, recreates the file, and then
>> writes several variables back onto the file. If any errors occur before all
>> of the desired variables are written to foutput, then you could end up with
>> an incomplete input data file when you run this script in the future. If
>> it’s absolutely necessary to use the same filename, you may want to instead
>> use a temporary filename for “filo” and then rename the temporary file at
>> the end of your script:
>>
>> filo = "tmp_dox_med_1.4-101.nc"
>> foutput = addfile (diro + filo, "c") ; open output file
>> ...
>> foutput->dox = (/dox_masked/)
>> system("/bin/mv " + diro + filo + " " + diro + "dox_med_1.4-101.nc")
>>
>>
>> I don’t think this is the cause of the memory allocation error, but I
>> thought it might be worth mentioning anyway.
>>
>> I hope this helps,
>> Kevin
>>
>>
>> On Dec 3, 2018, at 2:28 PM, Rick Brownrigg <brownrig at ucar.edu> wrote:
>>
>> Hi,
>>
>> Errno=12 is "Cannot allocate memory".
>>
>> That dox variable looks to be about 2.6GB, and I can't quite tell, but it
>> looks like you may be creating another 2 variables of comparable size(?)
>> How much memory does you machine have? Can you tell how far the script
>> gets executed (i.e., perhaps put some print statements at strategic places
>> in the script to see how far it gets).
>>
>> Rick
>>
>> On Mon, Dec 3, 2018 at 2:18 PM Amal Inge <amalingenieur89 at gmail.com>
>> wrote:
>>
>>> Dear NCL experts,
>>> i tried to run the script attached but i had this error: fatal:NclMalloc
>>> Failed:[errno=12] Segmentation fault
>>> what should be the solution please knowing that i have enough space in
>>> the memory ?
>>> many thanks
>>>
>>>
>>> netcdf dox_med_1.4-101 {
>>> dimensions:
>>> longitude = 670 ;
>>> latitude = 253 ;
>>> depth = 18 ;
>>> time = UNLIMITED ; // (215 currently)
>>> variables:
>>> float longitude(longitude) ;
>>> longitude:standard_name = "longitude" ;
>>> longitude:long_name = "longitude" ;
>>> longitude:units = "degrees_east" ;
>>> longitude:axis = "X" ;
>>> float latitude(latitude) ;
>>> latitude:standard_name = "latitude" ;
>>> latitude:long_name = "latitude" ;
>>> latitude:units = "degrees_north" ;
>>> latitude:axis = "Y" ;
>>> float depth(depth) ;
>>> depth:standard_name = "depth" ;
>>> depth:long_name = "depth" ;
>>> depth:units = "m" ;
>>> depth:positive = "down" ;
>>> depth:axis = "Z" ;
>>> depth:_CoordinateAxisType = "Height" ;
>>> depth:_CoordinateZisPositive = "down" ;
>>> double time(time) ;
>>> time:standard_name = "time" ;
>>> time:long_name = "time" ;
>>> time:units = "seconds since 1970-01-01 00:00:00" ;
>>> time:calendar = "standard" ;
>>> time:axis = "T" ;
>>> float dox(time, depth, latitude, longitude) ;
>>> dox:standard_name =
>>> "mole_concentration_of_dissolved_molecular_oxygen_in_sea_water" ;
>>> dox:long_name = "Mole concentration of Dissolved Molecular
>>> Oxygen in sea water" ;
>>> dox:units = "millimol m-3" ;
>>> dox:_FillValue = 1.e+20f ;
>>> dox:missing_value = 1.e+20f ;
>>> dox:_ChunkSizes = 1, 22, 84, 161 ;
>>>
>>> // global attributes:
>>> :CDI = "Climate Data Interface version 1.8.1 (
>>> http://mpimet.mpg.de/cdi)" ;
>>> :Conventions = "CF-1.0" ;
>>> :source = "3DVAR-OGSTM-BFM" ;
>>> :institution = "OGS (Istituto Nazionale di Oceanografia e di
>>> Geofisica Sperimentale) , Sgonico (Trieste) - Italy" ;
>>> :references = "Please check in CMEMS catalogue the INFO section
>>> for product MEDSEA_ANALYSIS_FORECAST_BIO_006_008 -
>>> http://marine.copernicus.eu/" ;
>>> :comment = "Please check in CMEMS catalogue the INFO section for
>>> product MEDSEA_ANALYSIS_FORECAST_BIO_006_008 -
>>> http://marine.copernicus.eu/" ;
>>> :contact = "servicedesk.cmems at mercator-ocean.eu" ;
>>> :bulletin_date = "2017-12-01" ;
>>> :bulletin_type = "analysis" ;
>>> :field_type = "monthly_mean_beginning_at_time_field" ;
>>> :title = "Net Primary Production and Dissolved Oxygen (3D) -
>>> Monthly Mean" ;
>>> :History = "Translated to CF-1.0 Conventions by Netcdf-Java CDM
>>> (CFGridWriter2)\n",
>>> "Original Dataset = sv03-med-ogs-bio-rean-m; Translation
>>> Date = 2018-12-03T14:34:09.553Z" ;
>>> :geospatial_lat_min = 30.1875 ;
>>> :geospatial_lat_max = 45.9375 ;
>>> :geospatial_lon_min = -5.5625 ;
>>> :geospatial_lon_max = 36.25 ;
>>> :CDO = "Climate Data Operators version 1.8.1 (
>>> http://mpimet.mpg.de/cdo)" ;
>>>
>>> _______________________________________________
>>> ncl-talk mailing list
>>> ncl-talk at ucar.edu
>>> List instructions, subscriber options, unsubscribe:
>>> http://mailman.ucar.edu/mailman/listinfo/ncl-talk
>>>
>> _______________________________________________
>> ncl-talk mailing list
>> ncl-talk at ucar.edu
>> List instructions, subscriber options, unsubscribe:
>> http://mailman.ucar.edu/mailman/listinfo/ncl-talk
>>
>>
>>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.ucar.edu/pipermail/ncl-talk/attachments/20181203/9a07722b/attachment.html>
More information about the ncl-talk
mailing list