<p><b>dwj07@fsu.edu</b> 2012-09-05 08:53:12 -0600 (Wed, 05 Sep 2012)</p><p><br>
-- BRANCH COMMIT --<br>
<br>
Fixing timers to work with multi-threaded applications.<br>
</p><hr noshade><pre><font color="gray">Modified: branches/omp_blocks/openmp_test/src/core_sw_elements/mpas_sw_time_integration.F
===================================================================
--- branches/omp_blocks/openmp_test/src/core_sw_elements/mpas_sw_time_integration.F 2012-09-04 20:15:16 UTC (rev 2140)
+++ branches/omp_blocks/openmp_test/src/core_sw_elements/mpas_sw_time_integration.F 2012-09-05 14:53:12 UTC (rev 2141)
@@ -30,20 +30,24 @@
if (trim(config_time_integration) == 'RK4') then
call sw_rk4(domain, dt)
else
- !$omp single
+ !$omp barrier
+ !$omp master
write(0,*) 'Unknown time integration option '//trim(config_time_integration)
write(0,*) 'Currently, only ''RK4'' is supported.'
- !$omp end single
+ !$omp end master
+ !$omp barrier
stop
end if
block => domain % blocklist
- !$omp single
+ !$omp barrier
+ !$omp master
do while (associated(block))
block % state % time_levs(2) % state % xtime % scalar = timeStamp
block => block % next
end do
- !$omp end single
+ !$omp end master
+ !$omp barrier
end subroutine sw_timestep
@@ -73,10 +77,12 @@
real (kind=RKIND), dimension(4) :: rk_weights, rk_substep_weights
- !$omp single
call mpas_timer_start('computations')
+ !$omp barrier
+ !$omp master
call mpas_setup_provis_states(domain % blocklist)
- !$omp end single
+ !$omp end master
+ !$omp barrier
!
! Initialize time_levs(2) with state at current time
@@ -101,9 +107,11 @@
end do
!$omp end do
- !$omp single
+ !$omp barrier
+ !$omp master
call mpas_copy_state(block % provis, block % state % time_levs(1) % state)
- !$omp end single
+ !$omp end master
+ !$omp barrier
block => block % next
end do
@@ -117,9 +125,7 @@
rk_substep_weights(2) = dt/2.
rk_substep_weights(3) = dt
rk_substep_weights(4) = 0.
- !$omp single
call mpas_timer_stop('computations')
- !$omp end single
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! BEGIN RK loop
@@ -127,56 +133,49 @@
do rk_step = 1, 4
! --- update halos for diagnostic variables
+ call mpas_timer_start('communications')
!$omp barrier
- !$omp single
- call mpas_timer_start('communications')
+ !$omp master
call mpas_dmpar_exch_halo_field(domain % blocklist % provis % pv_edge)
if (config_h_mom_eddy_visc4 > 0.0) then
call mpas_dmpar_exch_halo_field(domain % blocklist % provis % divergence)
call mpas_dmpar_exch_halo_field(domain % blocklist % provis % vorticity)
end if
+ !$omp end master
+ !$omp barrier
call mpas_timer_stop('communications')
call mpas_timer_start('computations')
- !$omp end single
! --- compute tendencies
block => domain % blocklist
do while (associated(block))
- !$omp single
call mpas_timer_start('sw_compute_tend')
- !$omp end single
call sw_compute_tend(block % tend, block % provis, block % mesh)
- !$omp single
call mpas_timer_stop('sw_compute_tend')
call mpas_timer_start('sw_compute_scalar_tend')
- !$omp end single
call sw_compute_scalar_tend(block % tend, block % provis, block % mesh)
- !$omp single
call mpas_timer_stop('sw_compute_scalar_tend')
call mpas_timer_start('sw_enforce_boundary')
- !$omp end single
call sw_enforce_boundary_edge(block % tend, block % mesh)
- !$omp single
call mpas_timer_stop('sw_enforce_boundary')
- !$omp end single
block => block % next
end do
! --- update halos for prognostic variables
- !$omp barrier
-
- !$omp single
call mpas_timer_stop('computations')
call mpas_timer_start('communications')
+ !$omp barrier
+ !$omp master
call mpas_dmpar_exch_halo_field(domain % blocklist % tend % u)
call mpas_dmpar_exch_halo_field(domain % blocklist % tend % h)
call mpas_dmpar_exch_halo_field(domain % blocklist % tend % tracers)
+ !$omp end master
+ !$omp barrier
call mpas_timer_stop('communications')
call mpas_timer_start('computations')
- !$omp end single
! --- compute next substep state
@@ -205,20 +204,15 @@
block % provis % u % array(:,:) = block % state % time_levs(1) % state % u % array(:,:)
!$omp end workshare
end if
- !$omp single
call mpas_timer_start('sw_compute_solve_diagnostics')
- !$omp end single
call sw_compute_solve_diagnostics(dt, block % provis, block % mesh)
- !$omp single
call mpas_timer_stop('sw_compute_solve_diagnostics')
- !$omp end single
block => block % next
end do
end if
!--- accumulate update (for RK4)
!$omp barrier
-
block => domain % blocklist
do while (associated(block))
!$omp workshare
@@ -240,11 +234,7 @@
block => block % next
end do
-
- !$omp single
- call mpas_timer_stop('computations')
- !$omp end single
-
+ call mpas_timer_stop('computations')
end do
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! END RK loop
@@ -254,9 +244,8 @@
!
! A little clean up at the end: decouple new scalar fields and compute diagnostics for new state
!
- !$omp single
+ !$omp barrier
call mpas_timer_start('computations')
- !$omp end single
block => domain % blocklist
do while (associated(block))
!$omp do private(iCell, k)
@@ -278,7 +267,7 @@
call sw_compute_solve_diagnostics(dt, block % state % time_levs(2) % state, block % mesh)
!$omp barrier
- !$omp single
+ !$omp master
call mpas_reconstruct(block % mesh, block % state % time_levs(2) % state % u % array, &
block % state % time_levs(2) % state % uReconstructX % array, &
block % state % time_levs(2) % state % uReconstructY % array, &
@@ -286,16 +275,18 @@
block % state % time_levs(2) % state % uReconstructZonal % array, &
block % state % time_levs(2) % state % uReconstructMeridional % array &
)
- !$omp end single
+ !$omp end master
+ !$omp barrier
block => block % next
end do
!$omp barrier
- !$omp single
+ !$omp master
call mpas_deallocate_provis_states(domain % blocklist)
+ !$omp end master
+ !$omp barrier
call mpas_timer_stop('computations')
- !$omp end single
end subroutine sw_rk4
</font>
</pre>