<p><b>dwj07@fsu.edu</b> 2013-03-29 09:31:15 -0600 (Fri, 29 Mar 2013)</p><p><br>
-- BRANCH COMMIT --<br>
<br>
Changing out singles for masters around timers.<br>
Calling MPI_init_thread rahter than MPI_init<br>
Adding cppflag MPAS_OPENMP.<br>
</p><hr noshade><pre><font color="gray">Modified: branches/ocean_projects/openmp_elements/Makefile
===================================================================
--- branches/ocean_projects/openmp_elements/Makefile 2013-03-29 14:40:23 UTC (rev 2678)
+++ branches/ocean_projects/openmp_elements/Makefile 2013-03-29 15:31:15 UTC (rev 2679)
@@ -278,7 +278,7 @@
ifeq "$(TAU)" "true"
LINKER=tau_f90.sh
- CPPINCLUDES += -DMPAS_TAU
+ override CPPFLAGS += -DMPAS_TAU
TAU_MESSAGE="TAU Hooks are on."
else
LINKER=$(FC)
@@ -289,6 +289,7 @@
FFLAGS+=$(OPENMP_FLAG)
CFLAGS+=$(OPENMP_FLAG)
LDFLAGS+=$(OPENMP_FLAG)
+ override CPPFLAGS += -DMPAS_OPENMP
OPENMP_MESSAGE="OpenMP is on."
else
OPENMP_MESSAGE="OpenMP is off."
Modified: branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_diagnostics.F
===================================================================
--- branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_diagnostics.F 2013-03-29 14:40:23 UTC (rev 2678)
+++ branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_diagnostics.F 2013-03-29 15:31:15 UTC (rev 2679)
@@ -407,9 +407,9 @@
! For an isopycnal model, density should remain constant.
! For zlevel, calculate in-situ density
if (config_vert_coord_movement.ne.'isopycnal') then
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("equation of state", .false., diagEOSTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
! compute in-place density
call ocn_equation_of_state_rho(s, grid, 0, 'relative', err)
@@ -417,9 +417,9 @@
! compute rhoDisplaced, the potential density referenced to the top layer
call ocn_equation_of_state_rho(s, grid, 1, 'relative', err)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("equation of state", diagEOSTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
endif
!
@@ -695,9 +695,9 @@
integer, dimension(:), pointer :: maxLevelEdgeTop, nEdgesOnEdge
integer, dimension(:,:), pointer :: cellsOnEdge, edgesOnEdge
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("ocn_fuperp")
- !$OMP END SINGLE
+ !$OMP END MASTER
u => s % u % array
uBcl => s % uBcl % array
@@ -731,9 +731,9 @@
end do
!$OMP END DO
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("ocn_fuperp")
- !$OMP END SINGLE
+ !$OMP END MASTER
end subroutine ocn_fuperp!}}}
@@ -760,9 +760,9 @@
real (kind=RKIND), dimension(:,:), pointer :: h_edge, u
integer, dimension(:), pointer :: maxLevelEdgeTop
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("ocn_filter_btr_mode_u")
- !$OMP END SINGLE
+ !$OMP END MASTER
u => s % u % array
h_edge => s % h_edge % array
@@ -790,9 +790,9 @@
enddo ! iEdge
!$OMP END DO
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("ocn_filter_btr_mode_u")
- !$OMP END SINGLE
+ !$OMP END MASTER
end subroutine ocn_filter_btr_mode_u!}}}
@@ -821,9 +821,9 @@
integer, dimension(:), pointer :: maxLevelEdgeTop
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("ocn_filter_btr_mode_tend_u")
- !$OMP END SINGLE
+ !$OMP END MASTER
tend_u => tend % u % array
h_edge => s % h_edge % array
@@ -851,9 +851,9 @@
enddo ! iEdge
!$OMP END DO
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("ocn_filter_btr_mode_tend_u")
- !$OMP END SINGLE
+ !$OMP END MASTER
end subroutine ocn_filter_btr_mode_tend_u!}}}
Modified: branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_tendency.F
===================================================================
--- branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_tendency.F 2013-03-29 14:40:23 UTC (rev 2678)
+++ branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_tendency.F 2013-03-29 15:31:15 UTC (rev 2679)
@@ -96,9 +96,9 @@
integer :: err
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("ocn_tend_h")
- !$OMP END SINGLE
+ !$OMP END MASTER
uTransport => s % uTransport % array
wTop => s % wTop % array
@@ -124,24 +124,24 @@
!
! QC Comment (3/15/12): need to make sure that uTranport is the right
! transport velocity here.
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("hadv", .false., thickHadvTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call ocn_thick_hadv_tend(grid, uTransport, h_edge, tend_h, err)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("hadv", thickHadvTimer)
!
! height tendency: vertical advection term -d/dz(hw)
!
call mpas_timer_start("vadv", .false., thickVadvTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call ocn_thick_vadv_tend(grid, wtop, tend_h, err)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("vadv", thickVadvTimer)
call mpas_timer_stop("ocn_tend_h")
- !$OMP END SINGLE
+ !$OMP END MASTER
end subroutine ocn_tend_h!}}}
@@ -176,9 +176,9 @@
integer :: err
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("ocn_tend_u")
- !$OMP END SINGLE
+ !$OMP END MASTER
u => s % u % array
rho => s % rho % array
@@ -212,34 +212,34 @@
! velocity tendency: nonlinear Coriolis term and grad of kinetic energy
!
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("coriolis", .false., velCorTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call ocn_vel_coriolis_tend(grid, Vor_edge, h_edge, u, ke, tend_u, err)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("coriolis", velCorTimer)
!
! velocity tendency: vertical advection term -w du/dz
!
call mpas_timer_start("vadv", .false., velVadvTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call ocn_vel_vadv_tend(grid, u, h_edge, wtop, tend_u, err)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("vadv", velVadvTimer)
!
! velocity tendency: pressure gradient
!
call mpas_timer_start("pressure grad", .false., velPgradTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
if (config_pressure_gradient_type.eq.'MontgomeryPotential') then
call ocn_vel_pressure_grad_tend(grid, MontPot, zMid, rho, tend_u, err)
else
call ocn_vel_pressure_grad_tend(grid, pressure, zMid, rho, tend_u, err)
end if
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("pressure grad", velPgradTimer)
!
@@ -248,9 +248,9 @@
! strictly only valid for config_mom_del2 == constant
!
call mpas_timer_start("hmix", .false., velHmixTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call ocn_vel_hmix_tend(grid, divergence, vorticity, viscosity, tend_u, scratch, err)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("hmix", velHmixTimer)
!
@@ -258,19 +258,18 @@
!
call mpas_timer_start("forcings", .false., velForceTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call ocn_vel_forcing_tend(grid, u, u_src, ke_edge, h_edge, tend_u, err)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("forcings", velForceTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
!
! velocity tendency: vertical mixing d/dz( nu_v du/dz))
!
- !$OMP END SINGLE
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("ocn_tend_u")
- !$OMP END SINGLE
+ !$OMP END MASTER
end subroutine ocn_tend_u!}}}
@@ -303,9 +302,9 @@
integer :: err, iEdge, k
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("ocn_tend_tracer")
- !$OMP END SINGLE
+ !$OMP END MASTER
uTransport => s % uTransport % array
h => s % h % array
@@ -348,38 +347,38 @@
!
! Monotonoic Advection, or standard advection
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("adv", .false., tracerHadvTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call mpas_ocn_tracer_advection_tend(tracers, uh, wTop, h, h, dt, grid, tend_h, tend_tr, scratch)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("adv", tracerHadvTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
!
! tracer tendency: del2 horizontal tracer diffusion, div(h \kappa_2 </font>
<font color="gray">abla \phi)
!
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("hmix", .false., tracerHmixTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call ocn_tracer_hmix_tend(grid, h_edge, tracers, tend_tr, scratch, err)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("hmix", tracerHmixTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
! add restoring to T and S in top model layer
!
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("restoring", .false., tracerRestoringTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call ocn_restoring_tend(grid, h, s%index_temperature, s%index_salinity, tracers, tend_tr, err)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("restoring", tracerRestoringTimer)
call mpas_timer_stop("ocn_tend_tracer")
- !$OMP END SINGLE
+ !$OMP END MASTER
!$OMP SINGLE
call mpas_deallocate_scratch_field(scratch % uh)
Modified: branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_time_integration_rk4.F
===================================================================
--- branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_time_integration_rk4.F 2013-03-29 14:40:23 UTC (rev 2678)
+++ branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_time_integration_rk4.F 2013-03-29 15:31:15 UTC (rev 2679)
@@ -94,9 +94,9 @@
real (kind=RKIND), dimension(:), allocatable:: A,C,uTemp
real (kind=RKIND), dimension(:,:), allocatable:: tracersTemp
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_setup_provis_states(domain % blocklist)
- !$OMP END SINGLE
+ !$OMP END MASTER
!
! Initialize time_levs(2) with state at current time
@@ -120,9 +120,9 @@
end do
!$OMP END DO
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_copy_state(block % provis, block % state % time_levs(1) % state)
- !$OMP END SINGLE
+ !$OMP END MASTER
block => block % next
end do
@@ -137,16 +137,16 @@
rk_substep_weights(3) = dt
rk_substep_weights(4) = 0.
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("RK4-main loop")
- !$OMP END SINGLE
+ !$OMP END MASTER
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! BEGIN RK loop
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
do rk_step = 1, 4
! --- update halos for diagnostic variables
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("RK4-diagnostic halo update")
call mpas_dmpar_exch_halo_field(domain % blocklist % provis % Vor_edge)
if (config_mom_del4 > 0.0) then
@@ -154,13 +154,13 @@
call mpas_dmpar_exch_halo_field(domain % blocklist % provis % vorticity)
end if
call mpas_timer_stop("RK4-diagnostic halo update")
- !$OMP END SINGLE
+ !$OMP END MASTER
! --- compute tendencies
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("RK4-tendency computations")
- !$OMP END SINGLE
+ !$OMP END MASTER
block => domain % blocklist
do while (associated(block))
! advection of u uses u, while advection of h and tracers use uTransport.
@@ -179,7 +179,7 @@
call ocn_tend_tracer(block % tend, block % provis, block % diagnostics, block % mesh, block % scratch, dt)
block => block % next
end do
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("RK4-tendency computations")
! --- update halos for prognostic variables
@@ -193,7 +193,7 @@
! --- compute next substep state
call mpas_timer_start("RK4-update diagnostic variables")
- !$OMP END SINGLE
+ !$OMP END MASTER
if (rk_step < 4) then
block => domain % blocklist
do while (associated(block))
@@ -242,15 +242,15 @@
block => block % next
end do
end if
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("RK4-update diagnostic variables")
- !$OMP END SINGLE
+ !$OMP END MASTER
!--- accumulate update (for RK4)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("RK4-RK4 accumulate update")
- !$OMP END SINGLE
+ !$OMP END MASTER
block => domain % blocklist
do while (associated(block))
!$OMP WORKSHARE
@@ -273,24 +273,24 @@
block => block % next
end do
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("RK4-RK4 accumulate update")
- !$OMP END SINGLE
+ !$OMP END MASTER
end do
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! END RK loop
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("RK4-main loop")
- !$OMP END SINGLE
+ !$OMP END MASTER
!
! A little clean up at the end: rescale tracer fields and compute diagnostics for new state
!
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("RK4-cleaup phase")
- !$OMP END SINGLE
+ !$OMP END MASTER
! Rescale tracers
block => domain % blocklist
@@ -306,9 +306,9 @@
block => block % next
end do
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("RK4-implicit vert mix")
- !$OMP END SINGLE
+ !$OMP END MASTER
block => domain % blocklist
do while(associated(block))
! Call ocean diagnostic solve in preparation for vertical mixing. Note
@@ -327,14 +327,14 @@
! this leads to lack of volume conservation. It is required because halo updates in RK4 are only
! conducted on tendencies, not on the velocity and tracer fields. So this update is required to
! communicate the change due to implicit vertical mixing across the boundary.
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("RK4-implicit vert mix halos")
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(2) % state % u)
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(2) % state % tracers)
call mpas_timer_stop("RK4-implicit vert mix halos")
call mpas_timer_stop("RK4-implicit vert mix")
call mpas_timer_stop("RK4-implicit vert mix")
- !$OMP END SINGLE
+ !$OMP END MASTER
block => domain % blocklist
do while (associated(block))
@@ -384,11 +384,11 @@
block => block % next
end do
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("RK4-cleaup phase")
call mpas_deallocate_provis_states(domain % blocklist)
- !$OMP END SINGLE
+ !$OMP END MASTER
end subroutine ocn_time_integrator_rk4!}}}
Modified: branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_time_integration_split.F
===================================================================
--- branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_time_integration_split.F 2013-03-29 14:40:23 UTC (rev 2678)
+++ branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_time_integration_split.F 2013-03-29 15:31:15 UTC (rev 2679)
@@ -100,7 +100,7 @@
real (kind=RKIND), dimension(:), allocatable:: uTemp
real (kind=RKIND), dimension(:,:), allocatable:: tracersTemp
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se timestep", .false., timer_main)
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -109,7 +109,7 @@
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
call mpas_timer_start("se prep", .false., timer_prep)
- !$OMP END SINGLE
+ !$OMP END MASTER
block => domain % blocklist
do while (associated(block))
@@ -163,9 +163,9 @@
block => block % next
end do
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("se prep", timer_prep)
- !$OMP END SINGLE
+ !$OMP END MASTER
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! BEGIN large iteration loop
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -176,7 +176,7 @@
do split_explicit_step = 1, config_n_ts_iter
! --- update halos for diagnostic variables
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se halo diag", .false., timer_halo_diagnostic)
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(2) % state % Vor_edge)
if (config_mom_del4 > 0.0) then
@@ -184,7 +184,7 @@
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(2) % state % vorticity)
end if
call mpas_timer_stop("se halo diag", timer_halo_diagnostic)
- !$OMP END SINGLE
+ !$OMP END MASTER
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
@@ -193,9 +193,9 @@
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! compute velocity tendencies, T(u*,w*,p*)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se bcl vel", .false., timer_bcl_vel)
- !$OMP END SINGLE
+ !$OMP END MASTER
block => domain % blocklist
do while (associated(block))
@@ -209,9 +209,9 @@
block % state % time_levs(stage1_tend_time) % state % u % array, &
block % state % time_levs(stage1_tend_time) % state % wTop % array, err)
- call ocn_tend_u(block % tend, block % state % time_levs(stage1_tend_time) % state, block % diagnostics, block % mesh, block % scratch)
+ call ocn_tend_u(block % tend, block % state % time_levs(stage1_tend_time) % state, block % diagnostics, block % mesh, block % scratch)
- block => block % next
+ block => block % next
end do
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -284,15 +284,15 @@
block => block % next
end do
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se halo ubcl", .false., timer_halo_ubcl)
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(2) % state % uBcl)
call mpas_timer_stop("se halo ubcl", timer_halo_ubcl)
- !$OMP END SINGLE
+ !$OMP END MASTER
end do ! do j=1,config_n_bcl_iter
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("se bcl vel", timer_bcl_vel)
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! END baroclinic iterations on linear Coriolis term
@@ -306,7 +306,7 @@
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
call mpas_timer_start("se btr vel", .false., timer_btr_vel)
- !$OMP END SINGLE
+ !$OMP END MASTER
oldBtrSubcycleTime = 1
newBtrSubcycleTime = 2
@@ -427,11 +427,11 @@
end do ! block
! boundary update on uBtrNew
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se halo ubtr", .false., timer_halo_ubtr)
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle)
call mpas_timer_stop("se halo ubtr", timer_halo_ubtr)
- !$OMP END SINGLE
+ !$OMP END MASTER
endif ! config_btr_gam1_uWt1>1.0e-12
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -537,11 +537,11 @@
end do ! block
! boundary update on SSHnew
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se halo ssh", .false., timer_halo_ssh)
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle)
call mpas_timer_stop("se halo ssh", timer_halo_ssh)
- !$OMP END SINGLE
+ !$OMP END MASTER
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! Barotropic subcycle: VELOCITY CORRECTOR STEP
@@ -593,16 +593,17 @@
block => block % next
end do ! block
+ !$OMP BARRIER
!$OMP SINGLE
call mpas_deallocate_scratch_field(domain % blocklist % scratch % uTempEdges)
!$OMP END SINGLE
! boundary update on uBtrNew
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se halo ubtr", .false., timer_halo_ubtr)
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle)
call mpas_timer_stop("se halo ubtr", timer_halo_ubtr)
- !$OMP END SINGLE
+ !$OMP END MASTER
end do !do BtrCorIter=1,config_n_btr_cor_iter
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -706,11 +707,11 @@
end do ! block
! boundary update on SSHnew
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se halo ssh", .false., timer_halo_ssh)
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle)
call mpas_timer_stop("se halo ssh", timer_halo_ssh)
- !$OMP END SINGLE
+ !$OMP END MASTER
endif ! config_btr_solve_SSH2
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -766,11 +767,11 @@
! boundary update on F
!$OMP BARRIER
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se halo F", .false., timer_halo_f)
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(1) % state % FBtr)
call mpas_timer_stop("se halo F", timer_halo_f)
- !$OMP END SINGLE
+ !$OMP END MASTER
! Check that you can compute SSH using the total sum or the individual increments
! over the barotropic subcycles.
@@ -839,9 +840,9 @@
endif ! split_explicit
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("se btr vel", timer_btr_vel)
- !$OMP END SINGLE
+ !$OMP END MASTER
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
@@ -866,11 +867,11 @@
end do
! update halo for thickness and tracer tendencies
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se halo h", .false., timer_halo_h)
call mpas_dmpar_exch_halo_field(domain % blocklist % tend % h)
call mpas_timer_stop("se halo h", timer_halo_h)
- !$OMP END SINGLE
+ !$OMP END MASTER
block => domain % blocklist
do while (associated(block))
@@ -880,11 +881,11 @@
end do
! update halo for thickness and tracer tendencies
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se halo tracers", .false., timer_halo_tracers)
call mpas_dmpar_exch_halo_field(domain % blocklist % tend % tracers)
call mpas_timer_stop("se halo tracers", timer_halo_tracers)
- !$OMP END SINGLE
+ !$OMP END MASTER
block => domain % blocklist
do while (associated(block))
@@ -1010,9 +1011,9 @@
! END large iteration loop
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se implicit vert mix")
- !$OMP END SINGLE
+ !$OMP END MASTER
block => domain % blocklist
do while(associated(block))
@@ -1033,13 +1034,13 @@
! this leads to lack of volume conservation. It is required because halo updates in stage 3 are only
! conducted on tendencies, not on the velocity and tracer fields. So this update is required to
! communicate the change due to implicit vertical mixing across the boundary.
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("se implicit vert mix halos")
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(2) % state % u)
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(2) % state % tracers)
call mpas_timer_stop("se implicit vert mix halos")
call mpas_timer_stop("se implicit vert mix")
- !$OMP END SINGLE
+ !$OMP END MASTER
call mpas_timer_stop("se implicit vert mix")
block => domain % blocklist
@@ -1065,7 +1066,7 @@
+ block % state % time_levs(2) % state % uBolusGM % array(:,:)
!$OMP END WORKSHARE
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_reconstruct(block % mesh, block % state % time_levs(2) % state % u % array, &
block % state % time_levs(2) % state % uReconstructX % array, &
block % state % time_levs(2) % state % uReconstructY % array, &
@@ -1073,9 +1074,9 @@
block % state % time_levs(2) % state % uReconstructZonal % array, &
block % state % time_levs(2) % state % uReconstructMeridional % array &
)
- !$OMP END SINGLE NOWAIT
+ !$OMP END MASTER
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_reconstruct(block % mesh, block % mesh % u_src % array, &
block % state % time_levs(2) % state % uSrcReconstructX % array, &
block % state % time_levs(2) % state % uSrcReconstructY % array, &
@@ -1083,16 +1084,16 @@
block % state % time_levs(2) % state % uSrcReconstructZonal % array, &
block % state % time_levs(2) % state % uSrcReconstructMeridional % array &
)
- !$OMP END SINGLE
+ !$OMP END MASTER
!TDR
call ocn_time_average_accumulate(block % state % time_levs(2) % state, block % state % time_levs(1) % state)
block => block % next
end do
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("se timestep", timer_main)
- !$OMP END SINGLE
+ !$OMP END MASTER
end subroutine ocn_time_integrator_split!}}}
Modified: branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_tracer_advection_std.F
===================================================================
--- branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_tracer_advection_std.F 2013-03-29 14:40:23 UTC (rev 2678)
+++ branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_tracer_advection_std.F 2013-03-29 15:31:15 UTC (rev 2679)
@@ -61,18 +61,18 @@
real (kind=RKIND), dimension(:,:), intent(in) :: verticalCellSize !< Input: Distance between vertical interfaces of a cell
type (mesh_type), intent(in) :: grid !< Input: Grid information
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("tracer-hadv", .false.)
- !$OMP END SINGLE
+ !$OMP END MASTER
call mpas_ocn_tracer_advection_std_hadv_tend(tracers, uh, grid, tend)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("tracer-hadv")
call mpas_timer_start("tracer-vadv", .false.)
- !$OMP END SINGLE
+ !$OMP END MASTER
call mpas_ocn_tracer_advection_std_vadv_tend(tracers, w, verticalCellSize, grid, tend)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("tracer-vadv")
- !$OMP END SINGLE
+ !$OMP END MASTER
end subroutine mpas_ocn_tracer_advection_std_tend!}}}
Modified: branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_tracer_hmix.F
===================================================================
--- branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_tracer_hmix.F 2013-03-29 14:40:23 UTC (rev 2678)
+++ branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_tracer_hmix.F 2013-03-29 15:31:15 UTC (rev 2679)
@@ -127,18 +127,18 @@
if(.not.tracerHmixOn) return
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("del2", .false., del2Timer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call ocn_tracer_hmix_del2_tend(grid, h_edge, tracers, tend, err1)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("del2", del2Timer)
call mpas_timer_start("del4", .false., del4Timer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call ocn_tracer_hmix_del4_tend(grid, h_edge, tracers, tend, scratch, err2)
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_stop("del4", del4Timer)
- !$OMP END SINGLE
+ !$OMP END MASTER
err = ior(err1, err2)
Modified: branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_vel_hmix_del4.F
===================================================================
--- branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_vel_hmix_del4.F 2013-03-29 14:40:23 UTC (rev 2678)
+++ branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_vel_hmix_del4.F 2013-03-29 15:31:15 UTC (rev 2679)
@@ -159,14 +159,12 @@
edgeSignOnVertex => grid % edgeSignOnVertex % array
edgeSignOnCell => grid % edgeSignOnCell % array
!DWJ 01/29/13 OMP DEL4
- !$OMP MASTER
+ !$OMP SINGLE
call mpas_allocate_scratch_field(scratch % delsq_u, .true.)
call mpas_allocate_scratch_field(scratch % delsq_divergence, .true.)
call mpas_allocate_scratch_field(scratch % delsq_vorticity, .true.)
- !$OMP END MASTER
+ !$OMP END SINGLE
- !$OMP BARRIER
-
delsq_u => scratch % delsq_u % array
delsq_divergence => scratch % delsq_divergence % array
delsq_vorticity => scratch % delsq_vorticity % array
Modified: branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_vmix.F
===================================================================
--- branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_vmix.F 2013-03-29 14:40:23 UTC (rev 2678)
+++ branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_vmix.F 2013-03-29 15:31:15 UTC (rev 2679)
@@ -511,7 +511,7 @@
real (KIND=RKIND) :: m
integer i
- call mpas_timer_start("tridiagonal_solve")
+ !call mpas_timer_start("tridiagonal_solve")
! Use work variables for b and r
bTemp(1) = b(1)
@@ -530,7 +530,7 @@
x(i) = (rTemp(i) - c(i)*x(i+1))/bTemp(i)
end do
- call mpas_timer_stop("tridiagonal_solve")
+ !call mpas_timer_stop("tridiagonal_solve")
end subroutine tridiagonal_solve!}}}
@@ -558,7 +558,7 @@
real (KIND=RKIND) :: m
integer i,j
- call mpas_timer_start("tridiagonal_solve_mult")
+ !call mpas_timer_start("tridiagonal_solve_mult")
! Use work variables for b and r
bTemp(1) = b(1)
@@ -585,7 +585,7 @@
end do
end do
- call mpas_timer_stop("tridiagonal_solve_mult")
+ !call mpas_timer_stop("tridiagonal_solve_mult")
end subroutine tridiagonal_solve_mult!}}}
Modified: branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_vmix_coefs_rich.F
===================================================================
--- branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_vmix_coefs_rich.F 2013-03-29 14:40:23 UTC (rev 2678)
+++ branches/ocean_projects/openmp_elements/src/core_ocean/mpas_ocn_vmix_coefs_rich.F 2013-03-29 15:31:15 UTC (rev 2679)
@@ -142,12 +142,12 @@
rhoDisplaced => s % rhoDisplaced % array
tracers => s % tracers % array
- !$OMP SINGLE
+ !$OMP MASTER
call mpas_timer_start("eos rich", .false., richEOSTimer)
call ocn_equation_of_state_rho(s, grid, 0,'relative', err)
call ocn_equation_of_state_rho(s, grid, 1,'relative', err)
call mpas_timer_stop("eos rich", richEOSTimer)
- !$OMP END SINGLE
+ !$OMP END MASTER
call ocn_vmix_get_rich_numbers(grid, indexT, indexS, u, h, h_edge, &
rho, rhoDisplaced, tracers, RiTopOfEdge, RiTopOfCell, scratch, err1)
@@ -460,7 +460,7 @@
end do
!$OMP END DO
- !$OMP SINGLE
+ !$OMP MASTER
! interpolate drhoTopOfCell to drhoTopOfEdge
drhoTopOfEdge = 0.0
do iEdge=1,nEdges
@@ -516,13 +516,14 @@
/ (du2TopOfCell(k,iCell) + 1e-20)
end do
end do
- !$OMP END SINGLE
+ !$OMP END MASTER
+ !$OMP BARRIER
!$OMP SINGLE
- call mpas_allocate_scratch_field(scratch % drhoTopOfCell)
- call mpas_allocate_scratch_field(scratch % drhoTopOfEdge)
- call mpas_allocate_scratch_field(scratch % du2TopOfCell)
- call mpas_allocate_scratch_field(scratch % du2TopOfEdge)
+ call mpas_deallocate_scratch_field(scratch % drhoTopOfCell)
+ call mpas_deallocate_scratch_field(scratch % drhoTopOfEdge)
+ call mpas_deallocate_scratch_field(scratch % du2TopOfCell)
+ call mpas_deallocate_scratch_field(scratch % du2TopOfEdge)
! deallocate(drhoTopOfCell, drhoTopOfEdge, &
! du2TopOfCell, du2TopOfEdge)
!$OMP END SINGLE
Modified: branches/ocean_projects/openmp_elements/src/framework/mpas_dmpar.F
===================================================================
--- branches/ocean_projects/openmp_elements/src/framework/mpas_dmpar.F 2013-03-29 14:40:23 UTC (rev 2678)
+++ branches/ocean_projects/openmp_elements/src/framework/mpas_dmpar.F 2013-03-29 15:31:15 UTC (rev 2679)
@@ -118,7 +118,11 @@
dminfo % comm = mpi_comm
dminfo % using_external_comm = .true.
else
+#ifdef MPAS_OPENMP
+ call MPI_Init_thread(MPI_THREAD_SINGLE, mpi_ierr)
+#else
call MPI_Init(mpi_ierr)
+#endif
dminfo % comm = MPI_COMM_WORLD
dminfo % using_external_comm = .false.
end if
</font>
</pre>