[ncl-talk] Speed up reading arrays?
Tabish Ansari
tabishumaransari at gmail.com
Tue Jan 16 09:58:37 MST 2018
Hi
I'm reading in some wrf data from multiple simulations and storing the
relevant variables as 1d arrays. This process is taking several minutes. Is
there I can changes something structurally in the algorithm to make this
faster?
Here are the lines of code:
print("reading wrf directories...")
DATADir1 = "/data2/tabish/control-run-so4-ECMWF/"
DATADir2 = "/data2/tabish/best-guess_run/"
DATADir3 = "/data2/tabish/what-if_run/"
;DATADir4 = "/data2/tabish/NH3-sensitivity_run/"
FILES1 = systemfunc (" ls -1 " + DATADir1 + "wrfout_d03_2014-* ")
FILES2 = systemfunc (" ls -1 " + DATADir2 + "wrfout_d03_2014-* ")
FILES3 = systemfunc (" ls -1 " + DATADir3 + "wrfout_d03_2014-* ")
;FILES4 = systemfunc (" ls -1 " + DATADir4 + "wrfout_d03_2014-* ")
a = addfiles(FILES1,"r")
b = addfiles(FILES2,"r")
c = addfiles(FILES3,"r")
;d = addfiles(FILES4,"r")
times = wrf_times_c(a[:]->Times,0)
;print(times)
;times_crop = times(16:135)
I=79
J=144
so4_sim = a[:]->so4_a01(:,0,J,I) + a[:]->so4_a02(:,0,J,I)
+a[:]->so4_a03(:,0,J,I) +a[:]->so4_a04(:,0,J,I) +a[:]->so4_a05(:,0,J,I)
+a[:]->so4_a06(:,0,J,I)
no3_sim = a[:]->no3_a01(:,0,J,I) + a[:]->no3_a02(:,0,J,I)
+a[:]->no3_a03(:,0,J,I) +a[:]->no3_a04(:,0,J,I) +a[:]->no3_a05(:,0,J,I)
+a[:]->no3_a06(:,0,J,I)
nh4_sim = a[:]->nh4_a01(:,0,J,I) + a[:]->nh4_a02(:,0,J,I)
+a[:]->nh4_a03(:,0,J,I) +a[:]->nh4_a04(:,0,J,I) +a[:]->nh4_a05(:,0,J,I)
+a[:]->nh4_a06(:,0,J,I)
chl_sim = a[:]->cl_a01(:,0,J,I) + a[:]->cl_a02(:,0,J,I)
+a[:]->cl_a03(:,0,J,I) +a[:]->cl_a04(:,0,J,I) +a[:]->cl_a05(:,0,J,I)
+a[:]->cl_a06(:,0,J,I)
oc_sim = a[:]->oc_a01(:,0,J,I) + a[:]->oc_a02(:,0,J,I)
+a[:]->oc_a03(:,0,J,I) +a[:]->oc_a04(:,0,J,I) +a[:]->oc_a05(:,0,J,I)
+a[:]->oc_a06(:,0,J,I)
bc_sim = a[:]->bc_a01(:,0,J,I) + a[:]->bc_a02(:,0,J,I)
+a[:]->bc_a03(:,0,J,I) +a[:]->bc_a04(:,0,J,I) +a[:]->bc_a05(:,0,J,I)
+a[:]->bc_a06(:,0,J,I)
co_sim = a[:]->co(:,0,J,I)
no2_sim = a[:]->no2(:,0,J,I)
so2_sim = a[:]->so2(:,0,J,I)
o3_sim = a[:]->o3(:,0,J,I)
so4_sim_260 = a[:]->so4_a01(:,3,J,I) + a[:]->so4_a02(:,3,J,I)
+a[:]->so4_a03(:,3,J,I) +a[:]->so4_a04(:,3,J,I) +a[:]->so4_a05(:,3,J,I)
+a[:]->so4_a06(:,3,J,I)
no3_sim_260 = a[:]->no3_a01(:,3,J,I) + a[:]->no3_a02(:,3,J,I)
+a[:]->no3_a03(:,3,J,I) +a[:]->no3_a04(:,3,J,I) +a[:]->no3_a05(:,3,J,I)
+a[:]->no3_a06(:,3,J,I)
nh4_sim_260 = a[:]->nh4_a01(:,3,J,I) + a[:]->nh4_a02(:,3,J,I)
+a[:]->nh4_a03(:,3,J,I) +a[:]->nh4_a04(:,3,J,I) +a[:]->nh4_a05(:,3,J,I)
+a[:]->nh4_a06(:,3,J,I)
chl_sim_260 = a[:]->cl_a01(:,3,J,I) + a[:]->cl_a02(:,3,J,I)
+a[:]->cl_a03(:,3,J,I) +a[:]->cl_a04(:,3,J,I) +a[:]->cl_a05(:,3,J,I)
+a[:]->cl_a06(:,3,J,I)
oc_sim_260 = a[:]->oc_a01(:,3,J,I) + a[:]->oc_a02(:,3,J,I)
+a[:]->oc_a03(:,3,J,I) +a[:]->oc_a04(:,3,J,I) +a[:]->oc_a05(:,3,J,I)
+a[:]->oc_a06(:,3,J,I)
so4_sim_b = b[:]->so4_a01(:,0,J,I) + b[:]->so4_a02(:,0,J,I)
+b[:]->so4_a03(:,0,J,I) +b[:]->so4_a04(:,0,J,I) +b[:]->so4_a05(:,0,J,I)
+b[:]->so4_a06(:,0,J,I)
no3_sim_b = b[:]->no3_a01(:,0,J,I) + b[:]->no3_a02(:,0,J,I)
+b[:]->no3_a03(:,0,J,I) +b[:]->no3_a04(:,0,J,I) +b[:]->no3_a05(:,0,J,I)
+b[:]->no3_a06(:,0,J,I)
nh4_sim_b = b[:]->nh4_a01(:,0,J,I) + b[:]->nh4_a02(:,0,J,I)
+b[:]->nh4_a03(:,0,J,I) +b[:]->nh4_a04(:,0,J,I) +b[:]->nh4_a05(:,0,J,I)
+b[:]->nh4_a06(:,0,J,I)
chl_sim_b = b[:]->cl_a01(:,0,J,I) + b[:]->cl_a02(:,0,J,I)
+b[:]->cl_a03(:,0,J,I) +b[:]->cl_a04(:,0,J,I) +b[:]->cl_a05(:,0,J,I)
+b[:]->cl_a06(:,0,J,I)
oc_sim_b = b[:]->oc_a01(:,0,J,I) + b[:]->oc_a02(:,0,J,I)
+b[:]->oc_a03(:,0,J,I) +b[:]->oc_a04(:,0,J,I) +b[:]->oc_a05(:,0,J,I)
+b[:]->oc_a06(:,0,J,I)
bc_sim_b = b[:]->bc_a01(:,0,J,I) + b[:]->bc_a02(:,0,J,I)
+b[:]->bc_a03(:,0,J,I) +b[:]->bc_a04(:,0,J,I) +b[:]->bc_a05(:,0,J,I)
+b[:]->bc_a06(:,0,J,I)
co_sim_b = b[:]->co(:,0,J,I)
no2_sim_b = b[:]->no2(:,0,J,I)
so2_sim_b = b[:]->so2(:,0,J,I)
o3_sim_b = b[:]->o3(:,0,J,I)
so4_sim_260_b = b[:]->so4_a01(:,3,J,I) + b[:]->so4_a02(:,3,J,I)
+b[:]->so4_a03(:,3,J,I) +b[:]->so4_a04(:,3,J,I) +b[:]->so4_a05(:,3,J,I)
+b[:]->so4_a06(:,3,J,I)
no3_sim_260_b = b[:]->no3_a01(:,3,J,I) + b[:]->no3_a02(:,3,J,I)
+b[:]->no3_a03(:,3,J,I) +b[:]->no3_a04(:,3,J,I) +b[:]->no3_a05(:,3,J,I)
+b[:]->no3_a06(:,3,J,I)
nh4_sim_260_b = b[:]->nh4_a01(:,3,J,I) + b[:]->nh4_a02(:,3,J,I)
+b[:]->nh4_a03(:,3,J,I) +b[:]->nh4_a04(:,3,J,I) +b[:]->nh4_a05(:,3,J,I)
+b[:]->nh4_a06(:,3,J,I)
chl_sim_260_b = b[:]->cl_a01(:,3,J,I) + b[:]->cl_a02(:,3,J,I)
+b[:]->cl_a03(:,3,J,I) +b[:]->cl_a04(:,3,J,I) +b[:]->cl_a05(:,3,J,I)
+b[:]->cl_a06(:,3,J,I)
oc_sim_260_b = b[:]->oc_a01(:,3,J,I) + b[:]->oc_a02(:,3,J,I)
+b[:]->oc_a03(:,3,J,I) +b[:]->oc_a04(:,3,J,I) +b[:]->oc_a05(:,3,J,I)
+b[:]->oc_a06(:,3,J,I)
so4_sim_c = c[:]->so4_a01(:,0,J,I) + c[:]->so4_a02(:,0,J,I)
+c[:]->so4_a03(:,0,J,I) +c[:]->so4_a04(:,0,J,I) +c[:]->so4_a05(:,0,J,I)
+c[:]->so4_a06(:,0,J,I)
no3_sim_c = c[:]->no3_a01(:,0,J,I) + c[:]->no3_a02(:,0,J,I)
+c[:]->no3_a03(:,0,J,I) +c[:]->no3_a04(:,0,J,I) +c[:]->no3_a05(:,0,J,I)
+c[:]->no3_a06(:,0,J,I)
nh4_sim_c = c[:]->nh4_a01(:,0,J,I) + c[:]->nh4_a02(:,0,J,I)
+c[:]->nh4_a03(:,0,J,I) +c[:]->nh4_a04(:,0,J,I) +c[:]->nh4_a05(:,0,J,I)
+c[:]->nh4_a06(:,0,J,I)
chl_sim_c = c[:]->cl_a01(:,0,J,I) + c[:]->cl_a02(:,0,J,I)
+c[:]->cl_a03(:,0,J,I) +c[:]->cl_a04(:,0,J,I) +c[:]->cl_a05(:,0,J,I)
+c[:]->cl_a06(:,0,J,I)
oc_sim_c = c[:]->oc_a01(:,0,J,I) + c[:]->oc_a02(:,0,J,I)
+c[:]->oc_a03(:,0,J,I) +c[:]->oc_a04(:,0,J,I) +c[:]->oc_a05(:,0,J,I)
+c[:]->oc_a06(:,0,J,I)
bc_sim_c = c[:]->bc_a01(:,0,J,I) + c[:]->bc_a02(:,0,J,I)
+c[:]->bc_a03(:,0,J,I) +c[:]->bc_a04(:,0,J,I) +c[:]->bc_a05(:,0,J,I)
+c[:]->bc_a06(:,0,J,I)
co_sim_c = c[:]->co(:,0,J,I)
no2_sim_c = c[:]->no2(:,0,J,I)
so2_sim_c = c[:]->so2(:,0,J,I)
o3_sim_c = c[:]->o3(:,0,J,I)
so4_sim_260_c = c[:]->so4_a01(:,3,J,I) + c[:]->so4_a02(:,3,J,I)
+c[:]->so4_a03(:,3,J,I) +c[:]->so4_a04(:,3,J,I) +c[:]->so4_a05(:,3,J,I)
+c[:]->so4_a06(:,3,J,I)
no3_sim_260_c = c[:]->no3_a01(:,3,J,I) + c[:]->no3_a02(:,3,J,I)
+c[:]->no3_a03(:,3,J,I) +c[:]->no3_a04(:,3,J,I) +c[:]->no3_a05(:,3,J,I)
+c[:]->no3_a06(:,3,J,I)
nh4_sim_260_c = c[:]->nh4_a01(:,3,J,I) + c[:]->nh4_a02(:,3,J,I)
+c[:]->nh4_a03(:,3,J,I) +c[:]->nh4_a04(:,3,J,I) +c[:]->nh4_a05(:,3,J,I)
+c[:]->nh4_a06(:,3,J,I)
chl_sim_260_c = c[:]->cl_a01(:,3,J,I) + c[:]->cl_a02(:,3,J,I)
+c[:]->cl_a03(:,3,J,I) +c[:]->cl_a04(:,3,J,I) +c[:]->cl_a05(:,3,J,I)
+c[:]->cl_a06(:,3,J,I)
oc_sim_260_c = c[:]->oc_a01(:,3,J,I) + c[:]->oc_a02(:,3,J,I)
+c[:]->oc_a03(:,3,J,I) +c[:]->oc_a04(:,3,J,I) +c[:]->oc_a05(:,3,J,I)
+c[:]->oc_a06(:,3,J,I)
;so4_sim_d = d[:]->so4_a01(:,0,J,I) + d[:]->so4_a02(:,0,J,I)
+d[:]->so4_a03(:,0,J,I) +d[:]->so4_a04(:,0,J,I) +d[:]->so4_a05(:,0,J,I)
+d[:]->so4_a06(:,0,J,I)
;no3_sim_d = d[:]->no3_a01(:,0,J,I) + d[:]->no3_a02(:,0,J,I)
+d[:]->no3_a03(:,0,J,I) +d[:]->no3_a04(:,0,J,I) +d[:]->no3_a05(:,0,J,I)
+d[:]->no3_a06(:,0,J,I)
;nh4_sim_d = d[:]->nh4_a01(:,0,J,I) + d[:]->nh4_a02(:,0,J,I)
+d[:]->nh4_a03(:,0,J,I) +d[:]->nh4_a04(:,0,J,I) +d[:]->nh4_a05(:,0,J,I)
+d[:]->nh4_a06(:,0,J,I)
;so4_sim_260_d = d[:]->so4_a01(:,3,J,I) + d[:]->so4_a02(:,3,J,I)
+d[:]->so4_a03(:,3,J,I) +d[:]->so4_a04(:,3,J,I) +d[:]->so4_a05(:,3,J,I)
+d[:]->so4_a06(:,3,J,I)
;no3_sim_260_d = d[:]->no3_a01(:,3,J,I) + d[:]->no3_a02(:,3,J,I)
+d[:]->no3_a03(:,3,J,I) +d[:]->no3_a04(:,3,J,I) +d[:]->no3_a05(:,3,J,I)
+d[:]->no3_a06(:,3,J,I)
;nh4_sim_260_d = d[:]->nh4_a01(:,3,J,I) + d[:]->nh4_a02(:,3,J,I)
+d[:]->nh4_a03(:,3,J,I) +d[:]->nh4_a04(:,3,J,I) +d[:]->nh4_a05(:,3,J,I)
+d[:]->nh4_a06(:,3,J,I)
print("Stored relevant pollutant arrays.")
Please note that I've commented out DataDir4 and its related arrays. When I
try to load that too, I get a segmentation fault.
I just need those arrays for further processing and ultimately for a
combined time-series plot. Is there anyway to speed this up, and probably
also read in DataDir4 arrays without a seg fault?
Thanks in advance!
Tabish
Tabish U Ansari
PhD student, Lancaster Environment Center
Lancaster Univeristy
Bailrigg, Lancaster,
LA1 4YW, United Kingdom
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.ucar.edu/pipermail/ncl-talk/attachments/20180116/8eb7a130/attachment.html>
More information about the ncl-talk
mailing list