[ncl-talk] fatal:NclMalloc Failed:[errno=12] Segmentation fault

Amal Inge amalingenieur89 at gmail.com
Mon Dec 3 15:29:00 MST 2018


Rick this is the end of ncl -x

+ ;----------------------------------------------------------------------
+ ; This is the main code
+ ;----------------------------------------------------------------------
+ begin
+ ;---Name of shapefile containing USA outlines
+    shp_fname = "GSAs_simplified.shp"
+
+ ;---Rough area we are interested in. Everything outside this will be
masked.
+    minlon =  -08.0
+    maxlon =  45.0
+    minlat =  29.00
+    maxlat =  48.0
+
+ ;---Read in zonal winds
+    a = addfile("dox_med_1.4-101.nc","r")
+    dox = a->dox
+    printVarSummary(dox)
+    dox_dims = dimsizes(dox)
+    ntim = dox_dims(0)
+    nlev = dox_dims(1)
+    nlat = dox_dims(2)
+    nlon = dox_dims(3)
+
+ ;---Create a new mask using a shapefile of USA
+    opt             = True
+    opt at return_mask = True
+ ;  opt at debug       = True
+    opt at minlon      = minlon     ; Makes the shapefile masking
+    opt at maxlon      = maxlon     ; go faster.
+    opt at minlat      = minlat
+    opt at maxlat      = maxlat
+    opt at shape_var   = "SECT_COD"
+    opt at shape_names = ("GSA01")
+    pmask2d  = shapefile_mask_data(dox(0,0,:,:),shp_fname,opt)
+
+ ;---Mask original 4D variable against a 4D pmask array
+    pmask4d    = conform_dims(dox_dims,pmask2d,(/2,3/))
+    dox_masked = mask(dox,pmask4d,1)
+
+    copy_VarMeta(dox,dox_masked)       ; copy attributes and coordinate
arrays
+    dox_masked!1 = "lev"               ; rename three of the dimensions
+    dox_masked!2 = "lat"
+    dox_masked!3 = "lon"
+
+    printVarSummary(dox)
+    printVarSummary(dox_masked)
+    printMinMax(dox,0)
+    printMinMax(dox_masked,0)
+
+    wks = gsn_open_wks("eps","plot_dox01_0-209")
+
+    cmap = read_colormap_file("cmp_b2r")
+
+    res                       = True     ; plot mods desired
+    res at gsnMaximize           = True
+    res at gsnDraw               = False
+    res at gsnFrame              = False
+
+
+    res at cnFillOn              = True     ; turn on color fill
+    res at cnFillPalette         = cmap(::-1,:)   ; reverse color map
+    res at cnLinesOn             = False    ; turn off contour lines
+    res at cnLineLabelsOn        = False    ; turn off contour line labels
+    res at lbLabelBarOn          = False    ; will turn on in panel
+    res at cnLevelSelectionMode  = "ManualLevels"    ; manually set the
contour levels with the following 3 resources
+    res at cnMinLevelValF        = 0.            ; set the minimum contour
level
+    res at cnMaxLevelValF        = 300.            ; set the maximum contour
level
+    res at cnLevelSpacingF       = 10
+
+    res at mpOutlineOn           = False
+    res at mpDataBaseVersion     = "MediumRes"
+    res at mpMinLonF             = minlon
+    res at mpMaxLonF             = maxlon
+    res at mpMinLatF             = minlat
+    res at mpMaxLatF             = maxlat
+    res at mpCenterLonF          = (minlon+maxlon)/2.
+    res at pmTickMarkDisplayMode = "Always"           ; nicer tickmarks
+    res at pmTitleZone           = 4                  ; moves title down
+
+    res at gsnAddCyclic          =  False             ; Regional data
+
+ ;---Be sure to use same levels for both plots
+    res at gsnRightString = "mmol/m3"
+    res at gsnLeftString  = "Dissolved_Molecular_Oxygen"
+
+    lnres = True
+    pres = True
+    pres at gsnMaximize       = True
+    pres at gsnPanelLabelBar  = True
+
+ ;---Comment out the plotting for now because it's time consuming.
+ ;;   do nt=0,dox_dims(0)-1
+ ;;     res at tiMainString = "Original data, nt = " + nt
+ ;;     plot_orig = gsn_csm_contour_map(wks,dox(nt,0,:,:),res)
+ ;;     res at tiMainString = "Masked data, nt = " + nt
+ ;;     plot_mask = gsn_csm_contour_map(wks,dox_masked(nt,0,:,:),res)
+ ;;     id_orig =
gsn_add_shapefile_polylines(wks,plot_orig,shp_fname,lnres)
+ ;;     id_mask =
gsn_add_shapefile_polylines(wks,plot_mask,shp_fname,lnres)
+ ;;
+ ;;     gsn_panel(wks,(/plot_orig,plot_mask/),(/2,1/),pres)
+ ;;   end do
+
+ ;---Open file for writing
+    diro = "./"                     ; Output directory
+    filo = "dox01_med_1.4-101.nc"
+    system("/bin/rm -f " + diro + filo)    ; remove if exists
+    foutput  = addfile (diro + filo, "c")  ; open output file
+    setfileoption(foutput,"DefineMode",True)
+
+ ;---Write global file attributes
+    fAtt               = True            ; assign file attributes
+    fAtt at title         = "NCL Efficient Approach to netCDF Creation"
+    fAtt at source_file   =  "dox_med_1.4-101.nc"
+    fAtt at Conventions   = "None"
+    fAtt at creation_date = systemfunc ("date")
+
+    fileattdef( foutput, fAtt )            ; copy file attributes
+
+ ;---Write file dimension names and sizes
+    dimNames = (/"time","lev", "lat", "lon"/)
+    dimSizes = (/ -1   ,  nlev,  nlat,  nlon /)
+    dimUnlim = (/ True , False, False, False/)
+    filedimdef(foutput,dimNames,dimSizes,dimUnlim)
+
+    filevardef(foutput, "time"
,typeof(dox_masked&time),getvardims(dox_masked&time))
+    filevardef(foutput, "lev"
,typeof(dox_masked&lev),getvardims(dox_masked&lev))
+    filevardef(foutput, "lat"
,typeof(dox_masked&lat),getvardims(dox_masked&lat))
+    filevardef(foutput, "lon"
,typeof(dox_masked&lon),getvardims(dox_masked&lon))
+    filevardef(foutput, "dox",typeof(dox_masked),getvardims(dox_masked))
+                                                               ; different
from name on script
+ ;---Write variable attributes
+    filevarattdef(foutput,"dox",dox_masked)                           ;
copy T attributes
+    filevarattdef(foutput,"time" ,dox_masked&time)         ; copy time
attributes
+    filevarattdef(foutput,"lev"  ,dox_masked&lev)          ; copy lev
attributes
+    filevarattdef(foutput,"lat"  ,dox_masked&lat)          ; copy lat
attributes
+    filevarattdef(foutput,"lon"  ,dox_masked&lon)          ; copy lon
attributes
+
+    setfileoption(foutput,"DefineMode",False)
+
+ ;---Write variable values
+    foutput->time   = (/dox_masked&time/)
+    foutput->lev    = (/dox_masked&lev/)
+    foutput->lat    = (/dox_masked&lat/)
+    foutput->lon    = (/dox_masked&lon/)
+    foutput->dox = (/dox_masked/)
+ delete(dox)
+ end

Variable: dox
Type: float
Total Size: 2624014800 bytes
            656003700 values
Number of Dimensions: 4
Dimensions and sizes:    [time | 215] x [depth | 18] x [latitude | 253] x
[longitude | 670]
Coordinates:
            time: [915148800..1477958400]
            depth: [1.472102..101.7803]
            latitude: [30.1875..45.9375]
            longitude: [-5.5625..36.25]
Number Of Attributes: 6
  standard_name :
mole_concentration_of_dissolved_molecular_oxygen_in_sea_water
  long_name :    Mole concentration of Dissolved Molecular Oxygen in sea
water
  units :    millimol m-3
  _FillValue :    1e+20
  missing_value :    1e+20
  _ChunkSizes :    ( 1, 22, 84, 161 )
fatal:NclMalloc Failed:[errno=12]
Segmentation fault

Could you advice me where can i delete some variables? maybe it could be
better  because the script works with data of one depth i think it will
never work with heavy data 4D

Le lun. 3 déc. 2018 à 23:19, Amal Inge <amalingenieur89 at gmail.com> a écrit :

> hi kevin, it was just a mistake because i tried many files.
> i correct it but same error i get
>
> Le lun. 3 déc. 2018 à 22:59, Kevin Hallock <hallock at ucar.edu> a écrit :
>
>> Hello,
>>
>> I noticed something else in your dox.ncl script that could potentially
>> cause problems:
>>
>>    a = addfile("dox_med_1.4-101.nc","r")
>>    ...
>>    diro = "./"                     ; Output directory
>>    filo = "dox_med_1.4-101.nc"
>>    system("/bin/rm -f " + diro + filo)    ; remove if exists
>>    foutput  = addfile (diro + filo, "c")  ; open output file
>>    ...
>>    foutput->dox = (/dox_masked/)
>>
>> The possible issue I see with this is that the script reads from the file
>> “dox_med_1.4-101.nc”, deletes the file, recreates the file, and then
>> writes several variables back onto the file. If any errors occur before all
>> of the desired variables are written to foutput, then you could end up with
>> an incomplete input data file when you run this script in the future. If
>> it’s absolutely necessary to use the same filename, you may want to instead
>> use a temporary filename for “filo” and then rename the temporary file at
>> the end of your script:
>>
>>    filo = "tmp_dox_med_1.4-101.nc"
>>    foutput  = addfile (diro + filo, "c")  ; open output file
>>    ...
>>    foutput->dox = (/dox_masked/)
>>    system("/bin/mv " + diro + filo + " " + diro + "dox_med_1.4-101.nc")
>>
>>
>> I don’t think this is the cause of the memory allocation error, but I
>> thought it might be worth mentioning anyway.
>>
>> I hope this helps,
>> Kevin
>>
>>
>> On Dec 3, 2018, at 2:28 PM, Rick Brownrigg <brownrig at ucar.edu> wrote:
>>
>> Hi,
>>
>> Errno=12 is "Cannot allocate memory".
>>
>> That dox variable looks to be about 2.6GB, and I can't quite tell, but it
>> looks like you may be creating another 2 variables of comparable size(?)
>> How much memory does you machine have?  Can you tell how far the script
>> gets executed (i.e., perhaps put some print statements at strategic places
>> in the script to see how far it gets).
>>
>> Rick
>>
>> On Mon, Dec 3, 2018 at 2:18 PM Amal Inge <amalingenieur89 at gmail.com>
>> wrote:
>>
>>> Dear NCL experts,
>>> i tried to run the script attached but i had this error: fatal:NclMalloc
>>> Failed:[errno=12] Segmentation fault
>>> what should be the solution please knowing that i have enough space in
>>> the memory ?
>>> many thanks
>>>
>>>
>>> netcdf dox_med_1.4-101 {
>>> dimensions:
>>>     longitude = 670 ;
>>>     latitude = 253 ;
>>>     depth = 18 ;
>>>     time = UNLIMITED ; // (215 currently)
>>> variables:
>>>     float longitude(longitude) ;
>>>         longitude:standard_name = "longitude" ;
>>>         longitude:long_name = "longitude" ;
>>>         longitude:units = "degrees_east" ;
>>>         longitude:axis = "X" ;
>>>     float latitude(latitude) ;
>>>         latitude:standard_name = "latitude" ;
>>>         latitude:long_name = "latitude" ;
>>>         latitude:units = "degrees_north" ;
>>>         latitude:axis = "Y" ;
>>>     float depth(depth) ;
>>>         depth:standard_name = "depth" ;
>>>         depth:long_name = "depth" ;
>>>         depth:units = "m" ;
>>>         depth:positive = "down" ;
>>>         depth:axis = "Z" ;
>>>         depth:_CoordinateAxisType = "Height" ;
>>>         depth:_CoordinateZisPositive = "down" ;
>>>     double time(time) ;
>>>         time:standard_name = "time" ;
>>>         time:long_name = "time" ;
>>>         time:units = "seconds since 1970-01-01 00:00:00" ;
>>>         time:calendar = "standard" ;
>>>         time:axis = "T" ;
>>>     float dox(time, depth, latitude, longitude) ;
>>>         dox:standard_name =
>>> "mole_concentration_of_dissolved_molecular_oxygen_in_sea_water" ;
>>>         dox:long_name = "Mole concentration of Dissolved Molecular
>>> Oxygen in sea water" ;
>>>         dox:units = "millimol m-3" ;
>>>         dox:_FillValue = 1.e+20f ;
>>>         dox:missing_value = 1.e+20f ;
>>>         dox:_ChunkSizes = 1, 22, 84, 161 ;
>>>
>>> // global attributes:
>>>         :CDI = "Climate Data Interface version 1.8.1 (
>>> http://mpimet.mpg.de/cdi)" ;
>>>         :Conventions = "CF-1.0" ;
>>>         :source = "3DVAR-OGSTM-BFM" ;
>>>         :institution = "OGS (Istituto Nazionale di Oceanografia e di
>>> Geofisica Sperimentale) , Sgonico (Trieste) - Italy" ;
>>>         :references = "Please check in CMEMS catalogue the INFO section
>>> for product MEDSEA_ANALYSIS_FORECAST_BIO_006_008 -
>>> http://marine.copernicus.eu/" ;
>>>         :comment = "Please check in CMEMS catalogue the INFO section for
>>> product MEDSEA_ANALYSIS_FORECAST_BIO_006_008 -
>>> http://marine.copernicus.eu/" ;
>>>         :contact = "servicedesk.cmems at mercator-ocean.eu" ;
>>>         :bulletin_date = "2017-12-01" ;
>>>         :bulletin_type = "analysis" ;
>>>         :field_type = "monthly_mean_beginning_at_time_field" ;
>>>         :title = "Net Primary Production and Dissolved Oxygen (3D) -
>>> Monthly Mean" ;
>>>         :History = "Translated to CF-1.0 Conventions by Netcdf-Java CDM
>>> (CFGridWriter2)\n",
>>>             "Original Dataset = sv03-med-ogs-bio-rean-m; Translation
>>> Date = 2018-12-03T14:34:09.553Z" ;
>>>         :geospatial_lat_min = 30.1875 ;
>>>         :geospatial_lat_max = 45.9375 ;
>>>         :geospatial_lon_min = -5.5625 ;
>>>         :geospatial_lon_max = 36.25 ;
>>>         :CDO = "Climate Data Operators version 1.8.1 (
>>> http://mpimet.mpg.de/cdo)" ;
>>>
>>> _______________________________________________
>>> ncl-talk mailing list
>>> ncl-talk at ucar.edu
>>> List instructions, subscriber options, unsubscribe:
>>> http://mailman.ucar.edu/mailman/listinfo/ncl-talk
>>>
>> _______________________________________________
>> ncl-talk mailing list
>> ncl-talk at ucar.edu
>> List instructions, subscriber options, unsubscribe:
>> http://mailman.ucar.edu/mailman/listinfo/ncl-talk
>>
>>
>>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.ucar.edu/pipermail/ncl-talk/attachments/20181203/9a07722b/attachment.html>


More information about the ncl-talk mailing list