<p><b>dwj07@fsu.edu</b> 2012-09-18 14:02:55 -0600 (Tue, 18 Sep 2012)</p><p><br>
-- BRANCH COMMIT --<br>
<br>
Adding OpenMP to split explicit.<br>
</p><hr noshade><pre><font color="gray">Modified: branches/omp_blocks/openmp_test/src/core_ocean_blocks/mpas_ocn_time_integration_split.F
===================================================================
--- branches/omp_blocks/openmp_test/src/core_ocean_blocks/mpas_ocn_time_integration_split.F 2012-09-18 17:04:21 UTC (rev 2163)
+++ branches/omp_blocks/openmp_test/src/core_ocean_blocks/mpas_ocn_time_integration_split.F 2012-09-18 20:02:55 UTC (rev 2164)
@@ -88,6 +88,7 @@
eoe, oldBtrSubcycleTime, newBtrSubcycleTime, uPerpTime, BtrCorIter, &
n_bcl_iter(config_n_ts_iter)
type (block_type), pointer :: block
+ type (block_type) :: block_d
real (kind=RKIND) :: uhSum, hSum, flux, sshEdge, &
CoriolisTerm, uCorr, temp, temp_h, coef, FBtr_coeff, sshCell1, sshCell2
integer :: num_tracers, ucorr_coef, err
@@ -109,47 +110,51 @@
call mpas_timer_start("se prep", .false., timer_prep)
block => domain % blocklist
do while (associated(block))
+ block_d = block
+ !$omp task firstprivate(block_d)
! Initialize * variables that are used to compute baroclinic tendencies below.
- do iEdge=1,block % mesh % nEdges
- do k=1,block % mesh % nVertLevels !maxLevelEdgeTop % array(iEdge)
+ do iEdge=1,block_d % mesh % nEdges
+ do k=1,block_d % mesh % nVertLevels !maxLevelEdgeTop % array(iEdge)
! The baroclinic velocity needs be recomputed at the beginning of a
! timestep because the implicit vertical mixing is conducted on the
! total u. We keep uBtr from the previous timestep.
- block % state % time_levs(1) % state % uBcl % array(k,iEdge) &
- = block % state % time_levs(1) % state % u % array(k,iEdge) &
- - block % state % time_levs(1) % state % uBtr % array( iEdge)
+ block_d % state % time_levs(1) % state % uBcl % array(k,iEdge) &
+ = block_d % state % time_levs(1) % state % u % array(k,iEdge) &
+ - block_d % state % time_levs(1) % state % uBtr % array( iEdge)
- block % state % time_levs(2) % state % u % array(k,iEdge) &
- = block % state % time_levs(1) % state % u % array(k,iEdge)
+ block_d % state % time_levs(2) % state % u % array(k,iEdge) &
+ = block_d % state % time_levs(1) % state % u % array(k,iEdge)
- block % state % time_levs(2) % state % uBcl % array(k,iEdge) &
- = block % state % time_levs(1) % state % uBcl % array(k,iEdge)
+ block_d % state % time_levs(2) % state % uBcl % array(k,iEdge) &
+ = block_d % state % time_levs(1) % state % uBcl % array(k,iEdge)
- block % state % time_levs(2) % state % h_edge % array(k,iEdge) &
- = block % state % time_levs(1) % state % h_edge % array(k,iEdge)
+ block_d % state % time_levs(2) % state % h_edge % array(k,iEdge) &
+ = block_d % state % time_levs(1) % state % h_edge % array(k,iEdge)
end do
end do
- block % state % time_levs(2) % state % ssh % array(:) &
- = block % state % time_levs(1) % state % ssh % array(:)
+ block_d % state % time_levs(2) % state % ssh % array(:) &
+ = block_d % state % time_levs(1) % state % ssh % array(:)
- do iCell=1,block % mesh % nCells
- do k=1,block % mesh % maxLevelCell % array(iCell)
+ do iCell=1,block_d % mesh % nCells
+ do k=1,block_d % mesh % maxLevelCell % array(iCell)
- block % state % time_levs(2) % state % h % array(k,iCell) &
- = block % state % time_levs(1) % state % h % array(k,iCell)
+ block_d % state % time_levs(2) % state % h % array(k,iCell) &
+ = block_d % state % time_levs(1) % state % h % array(k,iCell)
- block % state % time_levs(2) % state % tracers % array(:,k,iCell) &
- = block % state % time_levs(1) % state % tracers % array(:,k,iCell)
+ block_d % state % time_levs(2) % state % tracers % array(:,k,iCell) &
+ = block_d % state % time_levs(1) % state % tracers % array(:,k,iCell)
end do
end do
+ !$omp end task
block => block % next
end do
+ !$omp taskwait
call mpas_timer_stop("se prep", timer_prep)
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -181,12 +186,16 @@
block => domain % blocklist
do while (associated(block))
+ block_d = block
+ !$omp task firstprivate(block_d)
if (.not.config_implicit_vertical_mix) then
- call ocn_vmix_coefs(block % mesh, block % state % time_levs(2) % state, block % diagnostics, err)
+ call ocn_vmix_coefs(block_d % mesh, block_d % state % time_levs(2) % state, block_d % diagnostics, err)
end if
- call ocn_tend_u(block % tend, block % state % time_levs(2) % state, block % diagnostics, block % mesh)
+ call ocn_tend_u(block_d % tend, block_d % state % time_levs(2) % state, block_d % diagnostics, block_d % mesh)
+ !$omp end task
block => block % next
end do
+ !$omp taskwait
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! BEGIN baroclinic iterations on linear Coriolis term
@@ -202,58 +211,63 @@
block => domain % blocklist
do while (associated(block))
- allocate(uTemp(block % mesh % nVertLevels))
+ block_d = block
+ !$omp task firstprivate(block_d) private(uTemp)
+ allocate(uTemp(block_d % mesh % nVertLevels))
+
! Put f*uBcl^{perp} in uNew as a work variable
- call ocn_fuperp(block % state % time_levs(2) % state , block % mesh)
+ call ocn_fuperp(block_d % state % time_levs(2) % state , block_d % mesh)
- do iEdge=1,block % mesh % nEdges
- cell1 = block % mesh % cellsOnEdge % array(1,iEdge)
- cell2 = block % mesh % cellsOnEdge % array(2,iEdge)
+ do iEdge=1,block_d % mesh % nEdges
+ cell1 = block_d % mesh % cellsOnEdge % array(1,iEdge)
+ cell2 = block_d % mesh % cellsOnEdge % array(2,iEdge)
uTemp = 0.0 ! could put this after with uTemp(maxleveledgetop+1:nvertlevels)=0
- do k=1,block % mesh % maxLevelEdgeTop % array(iEdge)
+ do k=1,block_d % mesh % maxLevelEdgeTop % array(iEdge)
! uBclNew = uBclOld + dt*(-f*uBclPerp + T(u*,w*,p*) + g*grad(SSH*) )
! Here uNew is a work variable containing -fEdge(iEdge)*uBclPerp(k,iEdge)
- uTemp(k) = block % state % time_levs(1) % state % uBcl % array(k,iEdge) &
- + dt * (block % tend % u % array (k,iEdge) &
- + block % state % time_levs(2) % state % u % array (k,iEdge) & ! this is f*uBcl^{perp}
- + split * gravity * ( block % state % time_levs(2) % state % ssh % array(cell2) &
- - block % state % time_levs(2) % state % ssh % array(cell1) ) &
- /block % mesh % dcEdge % array(iEdge) )
+ uTemp(k) = block_d % state % time_levs(1) % state % uBcl % array(k,iEdge) &
+ + dt * (block_d % tend % u % array (k,iEdge) &
+ + block_d % state % time_levs(2) % state % u % array (k,iEdge) & ! this is f*uBcl^{perp}
+ + split * gravity * ( block_d % state % time_levs(2) % state % ssh % array(cell2) &
+ - block_d % state % time_levs(2) % state % ssh % array(cell1) ) &
+ /block_d % mesh % dcEdge % array(iEdge) )
enddo
! hSum is initialized outside the loop because on land boundaries
! maxLevelEdgeTop=0, but I want to initialize hSum with a
! nonzero value to avoid a NaN.
- uhSum = block % state % time_levs(2) % state % h_edge % array(1,iEdge) * uTemp(1)
- hSum = block % state % time_levs(2) % state % h_edge % array(1,iEdge)
+ uhSum = block_d % state % time_levs(2) % state % h_edge % array(1,iEdge) * uTemp(1)
+ hSum = block_d % state % time_levs(2) % state % h_edge % array(1,iEdge)
- do k=2,block % mesh % maxLevelEdgeTop % array(iEdge)
- uhSum = uhSum + block % state % time_levs(2) % state % h_edge % array(k,iEdge) * uTemp(k)
- hSum = hSum + block % state % time_levs(2) % state % h_edge % array(k,iEdge)
+ do k=2,block_d % mesh % maxLevelEdgeTop % array(iEdge)
+ uhSum = uhSum + block_d % state % time_levs(2) % state % h_edge % array(k,iEdge) * uTemp(k)
+ hSum = hSum + block_d % state % time_levs(2) % state % h_edge % array(k,iEdge)
enddo
- block % state % time_levs(1) % state % GBtrForcing % array(iEdge) = split*uhSum/hSum/dt
+ block_d % state % time_levs(1) % state % GBtrForcing % array(iEdge) = split*uhSum/hSum/dt
- do k=1,block % mesh % maxLevelEdgeTop % array(iEdge)
+ do k=1,block_d % mesh % maxLevelEdgeTop % array(iEdge)
! These two steps are together here:
!{\bf u}'_{k,n+1} = {\bf u}'_{k,n} - \Delta t {\overline {\bf G}}
!{\bf u}'_{k,n+1/2} = \frac{1}{2}\left({\bf u}^{'}_{k,n} +{\bf u}'_{k,n+1}\right)
! so that uBclNew is at time n+1/2
- block % state % time_levs(2) % state % uBcl % array(k,iEdge) &
+ block_d % state % time_levs(2) % state % uBcl % array(k,iEdge) &
= 0.5*( &
- block % state % time_levs(1) % state % uBcl % array(k,iEdge) &
- + uTemp(k) - dt * block % state % time_levs(1) % state % GBtrForcing % array(iEdge))
+ block_d % state % time_levs(1) % state % uBcl % array(k,iEdge) &
+ + uTemp(k) - dt * block_d % state % time_levs(1) % state % GBtrForcing % array(iEdge))
enddo
enddo ! iEdge
deallocate(uTemp)
+ !$omp end task
block => block % next
end do
+ !$omp taskwait
call mpas_timer_start("se halo ubcl", .false., timer_halo_ubcl)
call mpas_dmpar_exch_halo_field(domain % blocklist % state % time_levs(2) % state % uBcl)
@@ -282,61 +296,72 @@
block => domain % blocklist
do while (associated(block))
+ block_d = block
+ !$omp task firstprivate(block_d)
+
! For Split_Explicit unsplit, simply set uBtrNew=0, uBtrSubcycle=0, and uNew=uBclNew
- block % state % time_levs(2) % state % uBtr % array(:) = 0.0
+ block_d % state % time_levs(2) % state % uBtr % array(:) = 0.0
- block % state % time_levs(2) % state % u % array(:,:) = block % state % time_levs(2) % state % uBcl % array(:,:)
+ block_d % state % time_levs(2) % state % u % array(:,:) = block_d % state % time_levs(2) % state % uBcl % array(:,:)
- do iEdge=1,block % mesh % nEdges
- do k=1,block % mesh % nVertLevels
+ do iEdge=1,block_d % mesh % nEdges
+ do k=1,block_d % mesh % nVertLevels
! uTranport = uBcl + uBolus
! This is u used in advective terms for h and tracers
! in tendency calls in stage 3.
- block % state % time_levs(2) % state % uTransport % array(k,iEdge) &
- = block % mesh % edgeMask % array(k,iEdge) &
- *( block % state % time_levs(2) % state % uBcl % array(k,iEdge) &
- + block % state % time_levs(1) % state % uBolusGM % array(k,iEdge) )
+ block_d % state % time_levs(2) % state % uTransport % array(k,iEdge) &
+ = block_d % mesh % edgeMask % array(k,iEdge) &
+ *( block_d % state % time_levs(2) % state % uBcl % array(k,iEdge) &
+ + block_d % state % time_levs(1) % state % uBolusGM % array(k,iEdge) )
enddo
end do ! iEdge
+
+ !$omp end task
block => block % next
end do ! block
+ !$omp taskwait
elseif (trim(config_time_integration) == 'split_explicit') then
! Initialize variables for barotropic subcycling
block => domain % blocklist
do while (associated(block))
+ block_d = block
+ !$omp task firstprivate(block_d)
if (config_filter_btr_mode) then
- block % state % time_levs(1) % state % GBtrForcing % array(:) = 0.0
+ block_d % state % time_levs(1) % state % GBtrForcing % array(:) = 0.0
endif
- do iCell=1,block % mesh % nCells
+ do iCell=1,block_d % mesh % nCells
! sshSubcycleOld = sshOld
- block % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(iCell) &
- = block % state % time_levs(1) % state % ssh % array(iCell)
+ block_d % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(iCell) &
+ = block_d % state % time_levs(1) % state % ssh % array(iCell)
end do
- do iEdge=1,block % mesh % nEdges
+ do iEdge=1,block_d % mesh % nEdges
! uBtrSubcycleOld = uBtrOld
- block % state % time_levs(oldBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
- = block % state % time_levs(1) % state % uBtr % array(iEdge)
+ block_d % state % time_levs(oldBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
+ = block_d % state % time_levs(1) % state % uBtr % array(iEdge)
! uBtrNew = BtrOld This is the first for the summation
- block % state % time_levs(2) % state % uBtr % array(iEdge) &
- = block % state % time_levs(1) % state % uBtr % array(iEdge)
+ block_d % state % time_levs(2) % state % uBtr % array(iEdge) &
+ = block_d % state % time_levs(1) % state % uBtr % array(iEdge)
! FBtr = 0
- block % state % time_levs(1) % state % FBtr % array(iEdge) = 0.0
+ block_d % state % time_levs(1) % state % FBtr % array(iEdge) = 0.0
end do
+ !$omp end task
+
block => block % next
end do ! block
+ !$omp taskwait
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! BEGIN Barotropic subcycle loop
@@ -351,34 +376,38 @@
block => domain % blocklist
do while (associated(block))
+ block_d = block
- do iEdge=1,block % mesh % nEdges
+ !$omp task firstprivate(block_d)
+ do iEdge=1,block_d % mesh % nEdges
- cell1 = block % mesh % cellsOnEdge % array(1,iEdge)
- cell2 = block % mesh % cellsOnEdge % array(2,iEdge)
+ cell1 = block_d % mesh % cellsOnEdge % array(1,iEdge)
+ cell2 = block_d % mesh % cellsOnEdge % array(2,iEdge)
! Compute the barotropic Coriolis term, -f*uPerp
CoriolisTerm = 0.0
- do i = 1,block % mesh % nEdgesOnEdge % array(iEdge)
- eoe = block % mesh % edgesOnEdge % array(i,iEdge)
+ do i = 1,block_d % mesh % nEdgesOnEdge % array(iEdge)
+ eoe = block_d % mesh % edgesOnEdge % array(i,iEdge)
CoriolisTerm = CoriolisTerm &
- + block % mesh % weightsOnEdge % array(i,iEdge) &
- * block % state % time_levs(uPerpTime) % state % uBtrSubcycle % array(eoe) &
- * block % mesh % fEdge % array(eoe)
+ + block_d % mesh % weightsOnEdge % array(i,iEdge) &
+ * block_d % state % time_levs(uPerpTime) % state % uBtrSubcycle % array(eoe) &
+ * block_d % mesh % fEdge % array(eoe)
end do
! uBtrNew = uBtrOld + dt/J*(-f*uBtroldPerp - g*grad(SSH) + G)
- block % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
- = (block % state % time_levs(oldBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
+ block_d % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
+ = (block_d % state % time_levs(oldBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
+ dt / config_n_btr_subcycles * (CoriolisTerm - gravity &
- * (block % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell2) &
- - block % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell1) ) &
- / block % mesh % dcEdge % array(iEdge) &
- + block % state % time_levs(1) % state % GBtrForcing % array(iEdge))) * block % mesh % edgeMask % array(1, iEdge)
+ * (block_d % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell2) &
+ - block_d % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell1) ) &
+ / block_d % mesh % dcEdge % array(iEdge) &
+ + block_d % state % time_levs(1) % state % GBtrForcing % array(iEdge))) * block_d % mesh % edgeMask % array(1, iEdge)
end do
+ !$omp end task
block => block % next
end do ! block
+ !$omp taskwait
! boundary update on uBtrNew
call mpas_timer_start("se halo ubtr", .false., timer_halo_ubtr)
@@ -391,9 +420,11 @@
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
block => domain % blocklist
do while (associated(block))
+ block_d = block
+
+ !$omp task firstprivate(block_d)
+ block_d % tend % ssh % array(:) = 0.0
- block % tend % ssh % array(:) = 0.0
-
if (config_btr_solve_SSH2) then
! If config_btr_solve_SSH2=.true., then do NOT accumulate FBtr in this SSH predictor
! section, because it will be accumulated in the SSH corrector section.
@@ -408,36 +439,38 @@
! config_btr_gam1_uWt1=0.5 flux = 1/2*(uBtrNew+uBtrOld)*H
! config_btr_gam1_uWt1= 0 flux = uBtrOld*H
! mrp 120201 efficiency: could we combine the following edge and cell loops?
- do iEdge=1,block % mesh % nEdges
- cell1 = block % mesh % cellsOnEdge % array(1,iEdge)
- cell2 = block % mesh % cellsOnEdge % array(2,iEdge)
+ do iEdge=1,block_d % mesh % nEdges
+ cell1 = block_d % mesh % cellsOnEdge % array(1,iEdge)
+ cell2 = block_d % mesh % cellsOnEdge % array(2,iEdge)
- sshEdge = 0.5 * (block % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell1) &
- + block % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell2) )
- hSum = sshEdge + block % mesh % referenceBottomDepthTopOfCell % array (block % mesh % maxLevelEdgeTop % array(iEdge)+1)
+ sshEdge = 0.5 * (block_d % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell1) &
+ + block_d % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell2) )
+ hSum = sshEdge + block_d % mesh % referenceBottomDepthTopOfCell % array (block_d % mesh % maxLevelEdgeTop % array(iEdge)+1)
- flux = ((1.0-config_btr_gam1_uWt1) * block % state % time_levs(oldBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
- + config_btr_gam1_uWt1 * block % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge)) &
+ flux = ((1.0-config_btr_gam1_uWt1) * block_d % state % time_levs(oldBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
+ + config_btr_gam1_uWt1 * block_d % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge)) &
* hSum
- block % tend % ssh % array(cell1) = block % tend % ssh % array(cell1) - flux * block % mesh % dvEdge % array(iEdge)
- block % tend % ssh % array(cell2) = block % tend % ssh % array(cell2) + flux * block % mesh % dvEdge % array(iEdge)
+ block_d % tend % ssh % array(cell1) = block_d % tend % ssh % array(cell1) - flux * block_d % mesh % dvEdge % array(iEdge)
+ block_d % tend % ssh % array(cell2) = block_d % tend % ssh % array(cell2) + flux * block_d % mesh % dvEdge % array(iEdge)
- block % state % time_levs(1) % state % FBtr % array(iEdge) = block % state % time_levs(1) % state % FBtr % array(iEdge) &
+ block_d % state % time_levs(1) % state % FBtr % array(iEdge) = block_d % state % time_levs(1) % state % FBtr % array(iEdge) &
+ FBtr_coeff*flux
end do
! SSHnew = SSHold + dt/J*(-div(Flux))
- do iCell=1,block % mesh % nCells
+ do iCell=1,block_d % mesh % nCells
- block % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(iCell) &
- = block % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(iCell) &
- + dt/config_n_btr_subcycles * block % tend % ssh % array(iCell) / block % mesh % areaCell % array (iCell)
+ block_d % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(iCell) &
+ = block_d % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(iCell) &
+ + dt/config_n_btr_subcycles * block_d % tend % ssh % array(iCell) / block_d % mesh % areaCell % array (iCell)
end do
+ !$omp end task
block => block % next
end do ! block
+ !$omp taskwait
! boundary update on SSHnew
call mpas_timer_start("se halo ssh", .false., timer_halo_ssh)
@@ -452,35 +485,40 @@
block => domain % blocklist
do while (associated(block))
- do iEdge=1,block % mesh % nEdges
- cell1 = block % mesh % cellsOnEdge % array(1,iEdge)
- cell2 = block % mesh % cellsOnEdge % array(2,iEdge)
+ block_d = block
+
+ !$omp task firstprivate(block_d)
+ do iEdge=1,block_d % mesh % nEdges
+ cell1 = block_d % mesh % cellsOnEdge % array(1,iEdge)
+ cell2 = block_d % mesh % cellsOnEdge % array(2,iEdge)
! Compute the barotropic Coriolis term, -f*uPerp
CoriolisTerm = 0.0
- do i = 1,block % mesh % nEdgesOnEdge % array(iEdge)
- eoe = block % mesh % edgesOnEdge % array(i,iEdge)
- CoriolisTerm = CoriolisTerm + block % mesh % weightsOnEdge % array(i,iEdge) &
- * block % state % time_levs(uPerpTime) % state % uBtrSubcycle % array(eoe) &
- * block % mesh % fEdge % array(eoe)
+ do i = 1,block_d % mesh % nEdgesOnEdge % array(iEdge)
+ eoe = block_d % mesh % edgesOnEdge % array(i,iEdge)
+ CoriolisTerm = CoriolisTerm + block_d % mesh % weightsOnEdge % array(i,iEdge) &
+ * block_d % state % time_levs(uPerpTime) % state % uBtrSubcycle % array(eoe) &
+ * block_d % mesh % fEdge % array(eoe)
end do
! In this final solve for velocity, SSH is a linear
! combination of SSHold and SSHnew.
- sshCell1 = (1-config_btr_gam2_SSHWt1)*block % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell1) &
- + config_btr_gam2_SSHWt1 *block % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(cell1)
- sshCell2 = (1-config_btr_gam2_SSHWt1)*block % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell2) &
- + config_btr_gam2_SSHWt1 *block % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(cell2)
+ sshCell1 = (1-config_btr_gam2_SSHWt1)*block_d % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell1) &
+ + config_btr_gam2_SSHWt1 *block_d % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(cell1)
+ sshCell2 = (1-config_btr_gam2_SSHWt1)*block_d % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell2) &
+ + config_btr_gam2_SSHWt1 *block_d % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(cell2)
! uBtrNew = uBtrOld + dt/J*(-f*uBtroldPerp - g*grad(SSH) + G)
- block % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
- = (block % state % time_levs(oldBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
- + dt/config_n_btr_subcycles *(CoriolisTerm - gravity *(sshCell2 - sshCell1) /block % mesh % dcEdge % array(iEdge) &
- + block % state % time_levs(1) % state % GBtrForcing % array(iEdge))) * block % mesh % edgeMask % array(1,iEdge)
+ block_d % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
+ = (block_d % state % time_levs(oldBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
+ + dt/config_n_btr_subcycles *(CoriolisTerm - gravity *(sshCell2 - sshCell1) /block_d % mesh % dcEdge % array(iEdge) &
+ + block_d % state % time_levs(1) % state % GBtrForcing % array(iEdge))) * block_d % mesh % edgeMask % array(1,iEdge)
end do
+ !$omp end task
block => block % next
end do ! block
+ !$omp taskwait
! boundary update on uBtrNew
call mpas_timer_start("se halo ubtr", .false., timer_halo_ubtr)
@@ -495,45 +533,49 @@
block => domain % blocklist
do while (associated(block))
- block % tend % ssh % array(:) = 0.0
+ block_d = block
+ !$omp task firstprivate(block_d)
+ block_d % tend % ssh % array(:) = 0.0
! config_btr_gam3_uWt2 sets the forward weighting of velocity in the SSH computation
! config_btr_gam3_uWt2= 1 flux = uBtrNew*H
! config_btr_gam3_uWt2=0.5 flux = 1/2*(uBtrNew+uBtrOld)*H
! config_btr_gam3_uWt2= 0 flux = uBtrOld*H
! mrp 120201 efficiency: could we combine the following edge and cell loops?
- do iEdge=1,block % mesh % nEdges
- cell1 = block % mesh % cellsOnEdge % array(1,iEdge)
- cell2 = block % mesh % cellsOnEdge % array(2,iEdge)
+ do iEdge=1,block_d % mesh % nEdges
+ cell1 = block_d % mesh % cellsOnEdge % array(1,iEdge)
+ cell2 = block_d % mesh % cellsOnEdge % array(2,iEdge)
! SSH is a linear combination of SSHold and SSHnew.
- sshCell1 = (1-config_btr_gam2_SSHWt1)*block % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell1) &
- + config_btr_gam2_SSHWt1 *block % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(cell1)
- sshCell2 = (1-config_btr_gam2_SSHWt1)*block % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell2) &
- + config_btr_gam2_SSHWt1 *block % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(cell2)
+ sshCell1 = (1-config_btr_gam2_SSHWt1)*block_d % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell1) &
+ + config_btr_gam2_SSHWt1 *block_d % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(cell1)
+ sshCell2 = (1-config_btr_gam2_SSHWt1)*block_d % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(cell2) &
+ + config_btr_gam2_SSHWt1 *block_d % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(cell2)
sshEdge = 0.5 * (sshCell1 + sshCell2)
- hSum = sshEdge + block % mesh % referenceBottomDepthTopOfCell % array (block % mesh % maxLevelEdgeTop % array(iEdge)+1)
+ hSum = sshEdge + block_d % mesh % referenceBottomDepthTopOfCell % array (block_d % mesh % maxLevelEdgeTop % array(iEdge)+1)
- flux = ((1.0-config_btr_gam3_uWt2) * block % state % time_levs(oldBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
- + config_btr_gam3_uWt2 * block % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge)) &
+ flux = ((1.0-config_btr_gam3_uWt2) * block_d % state % time_levs(oldBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge) &
+ + config_btr_gam3_uWt2 * block_d % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge)) &
* hSum
- block % tend % ssh % array(cell1) = block % tend % ssh % array(cell1) - flux * block % mesh % dvEdge % array(iEdge)
- block % tend % ssh % array(cell2) = block % tend % ssh % array(cell2) + flux * block % mesh % dvEdge % array(iEdge)
+ block_d % tend % ssh % array(cell1) = block_d % tend % ssh % array(cell1) - flux * block_d % mesh % dvEdge % array(iEdge)
+ block_d % tend % ssh % array(cell2) = block_d % tend % ssh % array(cell2) + flux * block_d % mesh % dvEdge % array(iEdge)
- block % state % time_levs(1) % state % FBtr % array(iEdge) = block % state % time_levs(1) % state % FBtr % array(iEdge) + flux
+ block_d % state % time_levs(1) % state % FBtr % array(iEdge) = block_d % state % time_levs(1) % state % FBtr % array(iEdge) + flux
end do
! SSHnew = SSHold + dt/J*(-div(Flux))
- do iCell=1,block % mesh % nCells
- block % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(iCell) &
- = block % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(iCell) &
- + dt/config_n_btr_subcycles * block % tend % ssh % array(iCell) / block % mesh % areaCell % array (iCell)
+ do iCell=1,block_d % mesh % nCells
+ block_d % state % time_levs(newBtrSubcycleTime) % state % sshSubcycle % array(iCell) &
+ = block_d % state % time_levs(oldBtrSubcycleTime) % state % sshSubcycle % array(iCell) &
+ + dt/config_n_btr_subcycles * block_d % tend % ssh % array(iCell) / block_d % mesh % areaCell % array (iCell)
end do
+ !$omp end task
block => block % next
end do ! block
+ !$omp taskwait
! boundary update on SSHnew
call mpas_timer_start("se halo ssh", .false., timer_halo_ssh)
@@ -547,20 +589,24 @@
block => domain % blocklist
do while (associated(block))
+ block_d = block
+ !$omp task firstprivate(block_d)
! uBtrNew = uBtrNew + uBtrSubcycleNEW
! This accumulates the sum.
! If the Barotropic Coriolis iteration is limited to one, this could
! be merged with the above code.
- do iEdge=1,block % mesh % nEdges
+ do iEdge=1,block_d % mesh % nEdges
- block % state % time_levs(2) % state % uBtr % array(iEdge) &
- = block % state % time_levs(2) % state % uBtr % array(iEdge) &
- + block % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge)
+ block_d % state % time_levs(2) % state % uBtr % array(iEdge) &
+ = block_d % state % time_levs(2) % state % uBtr % array(iEdge) &
+ + block_d % state % time_levs(newBtrSubcycleTime) % state % uBtrSubcycle % array(iEdge)
end do ! iEdge
+ !$omp end task
block => block % next
end do ! block
+ !$omp taskwait
! advance time pointers
oldBtrSubcycleTime = mod(oldBtrSubcycleTime,2)+1
@@ -574,17 +620,21 @@
! Normalize Barotropic subcycle sums: ssh, uBtr, and F
block => domain % blocklist
do while (associated(block))
-
- do iEdge=1,block % mesh % nEdges
- block % state % time_levs(1) % state % FBtr % array(iEdge) = block % state % time_levs(1) % state % FBtr % array(iEdge) &
+ block_d = block
+
+ !$omp task firstprivate(block_d)
+ do iEdge=1,block_d % mesh % nEdges
+ block_d % state % time_levs(1) % state % FBtr % array(iEdge) = block_d % state % time_levs(1) % state % FBtr % array(iEdge) &
/ (config_n_btr_subcycles*config_btr_subcycle_loop_factor)
- block % state % time_levs(2) % state % uBtr % array(iEdge) = block % state % time_levs(2) % state % uBtr % array(iEdge) &
+ block_d % state % time_levs(2) % state % uBtr % array(iEdge) = block_d % state % time_levs(2) % state % uBtr % array(iEdge) &
/ (config_n_btr_subcycles*config_btr_subcycle_loop_factor + 1)
end do
+ !$omp end task
block => block % next
end do ! block
+ !$omp taskwait
! boundary update on F
@@ -599,8 +649,10 @@
! be removed later.
block => domain % blocklist
do while (associated(block))
+ block_d = block
- allocate(uTemp(block % mesh % nVertLevels))
+ !$omp task firstprivate(block_d) private(uTemp)
+ allocate(uTemp(block_d % mesh % nVertLevels))
! Correction velocity uCorr = (Flux - Sum(h u*))/H
! or, for the full latex version:
@@ -614,37 +666,37 @@
ucorr_coef = 0
endif
- do iEdge=1,block % mesh % nEdges
+ do iEdge=1,block_d % mesh % nEdges
! velocity for uCorrection is uBtr + uBcl + uBolus
uTemp(:) &
- = block % state % time_levs(2) % state % uBtr % array( iEdge) &
- + block % state % time_levs(2) % state % uBcl % array(:,iEdge) &
- + block % state % time_levs(1) % state % uBolusGM % array(:,iEdge)
+ = block_d % state % time_levs(2) % state % uBtr % array( iEdge) &
+ + block_d % state % time_levs(2) % state % uBcl % array(:,iEdge) &
+ + block_d % state % time_levs(1) % state % uBolusGM % array(:,iEdge)
! hSum is initialized outside the loop because on land boundaries
! maxLevelEdgeTop=0, but I want to initialize hSum with a
! nonzero value to avoid a NaN.
- uhSum = block % state % time_levs(2) % state % h_edge % array(1,iEdge) * uTemp(1)
- hSum = block % state % time_levs(2) % state % h_edge % array(1,iEdge)
+ uhSum = block_d % state % time_levs(2) % state % h_edge % array(1,iEdge) * uTemp(1)
+ hSum = block_d % state % time_levs(2) % state % h_edge % array(1,iEdge)
- do k=2,block % mesh % maxLevelEdgeTop % array(iEdge)
- uhSum = uhSum + block % state % time_levs(2) % state % h_edge % array(k,iEdge) * uTemp(k)
- hSum = hSum + block % state % time_levs(2) % state % h_edge % array(k,iEdge)
+ do k=2,block_d % mesh % maxLevelEdgeTop % array(iEdge)
+ uhSum = uhSum + block_d % state % time_levs(2) % state % h_edge % array(k,iEdge) * uTemp(k)
+ hSum = hSum + block_d % state % time_levs(2) % state % h_edge % array(k,iEdge)
enddo
- uCorr = ucorr_coef*(( block % state % time_levs(1) % state % FBtr % array(iEdge) - uhSum)/hSum)
+ uCorr = ucorr_coef*(( block_d % state % time_levs(1) % state % FBtr % array(iEdge) - uhSum)/hSum)
- do k=1,block % mesh % nVertLevels
+ do k=1,block_d % mesh % nVertLevels
! uTranport = uBtr + uBcl + uBolus + uCorrection
! This is u used in advective terms for h and tracers
! in tendency calls in stage 3.
- block % state % time_levs(2) % state % uTransport % array(k,iEdge) &
- = block % mesh % edgeMask % array(k,iEdge) &
- *( block % state % time_levs(2) % state % uBtr % array( iEdge) &
- + block % state % time_levs(2) % state % uBcl % array(k,iEdge) &
- + block % state % time_levs(1) % state % uBolusGM % array(k,iEdge) &
+ block_d % state % time_levs(2) % state % uTransport % array(k,iEdge) &
+ = block_d % mesh % edgeMask % array(k,iEdge) &
+ *( block_d % state % time_levs(2) % state % uBtr % array( iEdge) &
+ + block_d % state % time_levs(2) % state % uBcl % array(k,iEdge) &
+ + block_d % state % time_levs(1) % state % uBolusGM % array(k,iEdge) &
+ uCorr )
enddo
@@ -653,8 +705,11 @@
deallocate(uTemp)
+ !$omp end task
+
block => block % next
end do ! block
+ !$omp taskwait
endif ! split_explicit
@@ -675,11 +730,15 @@
! dwj: 02/22/12 splitting thickness and tracer tendency computations and halo updates to allow monotonic advection.
block => domain % blocklist
do while (associated(block))
- call ocn_wtop(block % state % time_levs(1) % state,block % state % time_levs(2) % state, block % mesh)
+ block_d = block
+ !$omp task firstprivate(block_d)
+ call ocn_wtop(block_d % state % time_levs(1) % state,block_d % state % time_levs(2) % state, block_d % mesh)
- call ocn_tend_h(block % tend, block % state % time_levs(2) % state, block % mesh)
+ call ocn_tend_h(block_d % tend, block_d % state % time_levs(2) % state, block_d % mesh)
+ !$omp end task
block => block % next
end do
+ !$omp taskwait
! update halo for thickness and tracer tendencies
call mpas_timer_start("se halo h", .false., timer_halo_h)
@@ -688,10 +747,14 @@
block => domain % blocklist
do while (associated(block))
- call ocn_tend_scalar(block % tend, block % state % time_levs(2) % state, block % diagnostics, block % mesh, dt)
+ block_d = block
+ !$omp task firstprivate(block_d)
+ call ocn_tend_scalar(block_d % tend, block_d % state % time_levs(2) % state, block_d % diagnostics, block_d % mesh, dt)
+ !$omp end task
block => block % next
end do
+ !$omp taskwait
! update halo for thickness and tracer tendencies
call mpas_timer_start("se halo tracers", .false., timer_halo_tracers)
@@ -700,7 +763,10 @@
block => domain % blocklist
do while (associated(block))
+ block_d = block
+ !$omp task firstprivate(block_d)
+
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
! If iterating, reset variables for next iteration
@@ -713,49 +779,49 @@
! Only need T & S for earlier iterations,
! then all the tracers needed the last time through.
- do iCell=1,block % mesh % nCells
+ do iCell=1,block_d % mesh % nCells
! sshNew is a pointer, defined above.
- do k=1,block % mesh % maxLevelCell % array(iCell)
+ do k=1,block_d % mesh % maxLevelCell % array(iCell)
! this is h_{n+1}
temp_h &
- = block % state % time_levs(1) % state % h % array(k,iCell) &
- + dt* block % tend % h % array(k,iCell)
+ = block_d % state % time_levs(1) % state % h % array(k,iCell) &
+ + dt* block_d % tend % h % array(k,iCell)
! this is h_{n+1/2}
- block % state % time_levs(2) % state % h % array(k,iCell) &
+ block_d % state % time_levs(2) % state % h % array(k,iCell) &
= 0.5*( &
- block % state % time_levs(1) % state % h % array(k,iCell) &
+ block_d % state % time_levs(1) % state % h % array(k,iCell) &
+ temp_h)
do i=1,2
! This is Phi at n+1
temp = ( &
- block % state % time_levs(1) % state % tracers % array(i,k,iCell) &
- * block % state % time_levs(1) % state % h % array(k,iCell) &
- + dt * block % tend % tracers % array(i,k,iCell)) &
+ block_d % state % time_levs(1) % state % tracers % array(i,k,iCell) &
+ * block_d % state % time_levs(1) % state % h % array(k,iCell) &
+ + dt * block_d % tend % tracers % array(i,k,iCell)) &
/ temp_h
! This is Phi at n+1/2
- block % state % time_levs(2) % state % tracers % array(i,k,iCell) &
+ block_d % state % time_levs(2) % state % tracers % array(i,k,iCell) &
= 0.5*( &
- block % state % time_levs(1) % state % tracers % array(i,k,iCell) &
+ block_d % state % time_levs(1) % state % tracers % array(i,k,iCell) &
+ temp )
end do
end do
end do ! iCell
- do iEdge=1,block % mesh % nEdges
+ do iEdge=1,block_d % mesh % nEdges
- do k=1,block % mesh % nVertLevels
+ do k=1,block_d % mesh % nVertLevels
! u = uBtr + uBcl
! here uBcl is at time n+1/2
! This is u used in next iteration or step
- block % state % time_levs(2) % state % u % array(k,iEdge) &
- = block % mesh % edgeMask % array(k,iEdge) &
- *( block % state % time_levs(2) % state % uBtr % array( iEdge) &
- + block % state % time_levs(2) % state % uBcl % array(k,iEdge) )
+ block_d % state % time_levs(2) % state % u % array(k,iEdge) &
+ = block_d % mesh % edgeMask % array(k,iEdge) &
+ *( block_d % state % time_levs(2) % state % uBtr % array( iEdge) &
+ + block_d % state % time_levs(2) % state % uBcl % array(k,iEdge) )
enddo
@@ -763,7 +829,7 @@
! mrp 110512 I really only need this to compute h_edge, density, pressure, and SSH
! I can par this down later.
- call ocn_diagnostic_solve(dt, block % state % time_levs(2) % state, block % mesh)
+ call ocn_diagnostic_solve(dt, block_d % state % time_levs(2) % state, block_d % mesh)
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
@@ -775,21 +841,21 @@
!TDR: should we move this code into a subroutine called "compute_final_values_at_nplus1"?
!TDR: this could be within a contains statement in this routine
- do iCell=1,block % mesh % nCells
- do k=1,block % mesh % maxLevelCell % array(iCell)
+ do iCell=1,block_d % mesh % nCells
+ do k=1,block_d % mesh % maxLevelCell % array(iCell)
! this is h_{n+1}
- block % state % time_levs(2) % state % h % array(k,iCell) &
- = block % state % time_levs(1) % state % h % array(k,iCell) &
- + dt* block % tend % h % array(k,iCell)
+ block_d % state % time_levs(2) % state % h % array(k,iCell) &
+ = block_d % state % time_levs(1) % state % h % array(k,iCell) &
+ + dt* block_d % tend % h % array(k,iCell)
! This is Phi at n+1
- do i=1,block % state % time_levs(1) % state % num_tracers
- block % state % time_levs(2) % state % tracers % array(i,k,iCell) &
- = (block % state % time_levs(1) % state % tracers % array(i,k,iCell) &
- * block % state % time_levs(1) % state % h % array(k,iCell) &
- + dt * block % tend % tracers % array(i,k,iCell)) &
- / block % state % time_levs(2) % state % h % array(k,iCell)
+ do i=1,block_d % state % time_levs(1) % state % num_tracers
+ block_d % state % time_levs(2) % state % tracers % array(i,k,iCell) &
+ = (block_d % state % time_levs(1) % state % tracers % array(i,k,iCell) &
+ * block_d % state % time_levs(1) % state % h % array(k,iCell) &
+ + dt * block_d % tend % tracers % array(i,k,iCell)) &
+ / block_d % state % time_levs(2) % state % h % array(k,iCell)
enddo
end do
@@ -804,19 +870,22 @@
! note that uBcl is recomputed at the beginning of the next timestep due to Imp Vert mixing,
! so uBcl does not have to be recomputed here.
- do iEdge=1,block % mesh % nEdges
- do k=1,block % mesh % maxLevelEdgeTop % array(iEdge)
- block % state % time_levs(2) % state % u % array(k,iEdge) &
- = block % state % time_levs(2) % state % uBtr % array( iEdge) &
- +2*block % state % time_levs(2) % state % uBcl % array(k,iEdge) &
- - block % state % time_levs(1) % state % uBcl % array(k,iEdge)
+ do iEdge=1,block_d % mesh % nEdges
+ do k=1,block_d % mesh % maxLevelEdgeTop % array(iEdge)
+ block_d % state % time_levs(2) % state % u % array(k,iEdge) &
+ = block_d % state % time_levs(2) % state % uBtr % array( iEdge) &
+ +2*block_d % state % time_levs(2) % state % uBcl % array(k,iEdge) &
+ - block_d % state % time_levs(1) % state % uBcl % array(k,iEdge)
end do
end do ! iEdges
endif ! split_explicit_step
+ !$omp end task
+
block => block % next
end do
+ !$omp taskwait
@@ -829,9 +898,13 @@
call mpas_timer_start("se implicit vert mix")
block => domain % blocklist
do while(associated(block))
- call ocn_vmix_implicit(dt, block % mesh, block % diagnostics, block % state % time_levs(2) % state, err)
+ block_d = block
+ !$omp task firstprivate(block_d)
+ call ocn_vmix_implicit(dt, block_d % mesh, block_d % diagnostics, block_d % state % time_levs(2) % state, err)
+ !$omp end task
block => block % next
end do
+ !$omp taskwait
! Update halo on u and tracers, which were just updated for implicit vertical mixing. If not done,
! this leads to lack of volume conservation. It is required because halo updates in stage 3 are only
@@ -847,48 +920,54 @@
block => domain % blocklist
do while (associated(block))
+ block_d = block
+ !$omp task firstprivate(block_d)
+
if (config_test_case == 1) then ! For case 1, wind field should be fixed
- block % state % time_levs(2) % state % u % array(:,:) = block % state % time_levs(1) % state % u % array(:,:)
+ block_d % state % time_levs(2) % state % u % array(:,:) = block_d % state % time_levs(1) % state % u % array(:,:)
end if
if (config_prescribe_velocity) then
- block % state % time_levs(2) % state % u % array(:,:) = block % state % time_levs(1) % state % u % array(:,:)
+ block_d % state % time_levs(2) % state % u % array(:,:) = block_d % state % time_levs(1) % state % u % array(:,:)
end if
if (config_prescribe_thickness) then
- block % state % time_levs(2) % state % h % array(:,:) = block % state % time_levs(1) % state % h % array(:,:)
+ block_d % state % time_levs(2) % state % h % array(:,:) = block_d % state % time_levs(1) % state % h % array(:,:)
end if
- call ocn_diagnostic_solve(dt, block % state % time_levs(2) % state, block % mesh)
+ call ocn_diagnostic_solve(dt, block_d % state % time_levs(2) % state, block_d % mesh)
! Compute velocity transport, used in advection terms of h and tracer tendency
- block % state % time_levs(2) % state % uTransport % array(:,:) &
- = block % state % time_levs(2) % state % u % array(:,:) &
- + block % state % time_levs(2) % state % uBolusGM % array(:,:)
+ block_d % state % time_levs(2) % state % uTransport % array(:,:) &
+ = block_d % state % time_levs(2) % state % u % array(:,:) &
+ + block_d % state % time_levs(2) % state % uBolusGM % array(:,:)
- call mpas_reconstruct(block % mesh, block % state % time_levs(2) % state % u % array, &
- block % state % time_levs(2) % state % uReconstructX % array, &
- block % state % time_levs(2) % state % uReconstructY % array, &
- block % state % time_levs(2) % state % uReconstructZ % array, &
- block % state % time_levs(2) % state % uReconstructZonal % array, &
- block % state % time_levs(2) % state % uReconstructMeridional % array &
+ call mpas_reconstruct(block_d % mesh, block_d % state % time_levs(2) % state % u % array, &
+ block_d % state % time_levs(2) % state % uReconstructX % array, &
+ block_d % state % time_levs(2) % state % uReconstructY % array, &
+ block_d % state % time_levs(2) % state % uReconstructZ % array, &
+ block_d % state % time_levs(2) % state % uReconstructZonal % array, &
+ block_d % state % time_levs(2) % state % uReconstructMeridional % array &
)
!TDR
- call mpas_reconstruct(block % mesh, block % mesh % u_src % array, &
- block % state % time_levs(2) % state % uSrcReconstructX % array, &
- block % state % time_levs(2) % state % uSrcReconstructY % array, &
- block % state % time_levs(2) % state % uSrcReconstructZ % array, &
- block % state % time_levs(2) % state % uSrcReconstructZonal % array, &
- block % state % time_levs(2) % state % uSrcReconstructMeridional % array &
+ call mpas_reconstruct(block_d % mesh, block_d % mesh % u_src % array, &
+ block_d % state % time_levs(2) % state % uSrcReconstructX % array, &
+ block_d % state % time_levs(2) % state % uSrcReconstructY % array, &
+ block_d % state % time_levs(2) % state % uSrcReconstructZ % array, &
+ block_d % state % time_levs(2) % state % uSrcReconstructZonal % array, &
+ block_d % state % time_levs(2) % state % uSrcReconstructMeridional % array &
)
!TDR
- call ocn_time_average_accumulate(block % state % time_levs(2) % state, block % state % time_levs(1) % state)
+ call ocn_time_average_accumulate(block_d % state % time_levs(2) % state, block_d % state % time_levs(1) % state)
+ !$omp end task
+
block => block % next
end do
+ !$omp taskwait
call mpas_timer_stop("se timestep", timer_main)
</font>
</pre>