CCPP tendencies bugfixes, global restart reproducibility, halo boundary update in dycore

atmos_model.F90

@@ -99,13 +99,13 @@ module atmos_model_mod
                               IPD_interstitial => GFS_interstitial
 use IPD_driver,         only: IPD_initialize, IPD_initialize_rst
 use CCPP_driver,        only: CCPP_step, non_uniform_blocks
+
+use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper
 #else
 use IPD_driver,         only: IPD_initialize, IPD_initialize_rst, IPD_step
 use physics_abstraction_layer, only: time_vary_step, radiation_step1, physics_step1, physics_step2
 #endif
 
-use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper
-
 use FV3GFS_io_mod,      only: FV3GFS_restart_read, FV3GFS_restart_write, &
                               FV3GFS_IPD_checksum,                       &
                               FV3GFS_diag_register, FV3GFS_diag_output,  &
@@ -291,16 +291,16 @@ subroutine update_atmos_radiation_physics (Atmos)
 #ifdef CCPP
       call CCPP_step (step="time_vary", nblks=Atm_block%nblks, ierr=ierr)
       if (ierr/=0)  call mpp_error(FATAL, 'Call to CCPP time_vary step failed')
+
+!--- call stochastic physics pattern generation / cellular automata
+      call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr)
+      if (ierr/=0)  call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed')
+
 #else
       Func1d => time_vary_step
       call IPD_step (IPD_Control, IPD_Data(:), IPD_Diag, IPD_Restart, IPD_func1d=Func1d)
 #endif
 
-!--- call stochastic physics pattern generation / cellular automata
-    call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr) 
-    if (ierr/=0)  call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed')
-
-
 !--- if coupled, assign coupled fields
 
       if( IPD_Control%cplflx .or. IPD_Control%cplwav ) then
@@ -625,14 +625,15 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
 #ifdef CCPP
    call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, &
                         IPD_Interstitial, commglobal, mpp_npes(), Init_parm)
-#else
-   call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, Init_parm)
-#endif
 
 !--- Initialize stochastic physics pattern generation / cellular automata for first time step
    call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr)
    if (ierr/=0)  call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed')
 
+#else
+   call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, Init_parm)
+#endif
+
    Atmos%Diag => IPD_Diag
 
    Atm(mygrid)%flagstruct%do_skeb = IPD_Control%do_skeb

ccpp/CMakeLists.txt

@@ -120,11 +120,13 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel")
     endif (LEGACY_INTEL)
   elseif (${CMAKE_BUILD_TYPE} MATCHES "Release")
     # Specify aggressive optimization flags (to be overwritten for individual files in ccpp-physics' CMakeLists.txt)
-    if (SIMDMULTIARCH)
-      set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512")
-    else (SIMDMULTIARCH)
-      set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX2")
-    endif (SIMDMULTIARCH)
+    if (AVX2)
+      if (SIMDMULTIARCH)
+        set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512")
+      else (SIMDMULTIARCH)
+        set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX2")
+      endif (SIMDMULTIARCH)
+    endif (AVX2)
     set (CMAKE_Fortran_FLAGS  "${CMAKE_Fortran_FLAGS} -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3")
   endif (${CMAKE_BUILD_TYPE} MATCHES "Debug")
   set (CMAKE_Fortran_FLAGS_DEFAULT_PREC "-i4 -real-size 64")

ccpp/driver/CCPP_driver.F90

@@ -93,7 +93,7 @@ subroutine CCPP_step (step, nblks, ierr)
         end do
       end do
 
-   else if (trim(step)=="physics_init") then
+    else if (trim(step)=="physics_init") then
 
       ! Since the physics init steps are independent of the blocking structure,
       ! we can use cdata_domain here. Since we don't use threading on the outside,
@@ -107,7 +107,7 @@ subroutine CCPP_step (step, nblks, ierr)
         return
       end if
 
-   else if (trim(step)=="time_vary") then
+    else if (trim(step)=="time_vary") then
 
       ! Since the time_vary steps only use data structures for all blocks (except the
       ! CCPP-internal variables ccpp_error_flag and ccpp_error_message, which are defined
@@ -123,8 +123,8 @@ subroutine CCPP_step (step, nblks, ierr)
         return
       end if
 
-   ! Radiation and stochastic physics
-   else if (trim(step)=="radiation" .or. trim(step)=="physics" .or. trim(step)=="stochastics") then
+    ! Radiation and stochastic physics
+    else if (trim(step)=="radiation" .or. trim(step)=="physics" .or. trim(step)=="stochastics") then
 
       ! Set number of threads available to physics schemes to one,
       ! because threads are used on the outside for blocking
@@ -162,8 +162,8 @@ subroutine CCPP_step (step, nblks, ierr)
 !$OMP end parallel
       if (ierr/=0) return
 
-   ! Finalize
-   else if (trim(step)=="finalize") then
+    ! Finalize
+    else if (trim(step)=="finalize") then
 
       ! Loop over blocks, don't use threading on the outside but allowing threading
       ! inside the finalization, similar to what is done for the initialization

ccpp/suites/suite_FV3_GFS_2017_couplednsst.xml

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<suite name="FV3_GFS_2017_couplednsst" lib="ccppphys" ver="4">
+  <!-- <init></init> -->
+  <group name="time_vary">
+    <subcycle loop="1">
+      <scheme>GFS_time_vary_pre</scheme>
+      <scheme>GFS_rrtmg_setup</scheme>
+      <scheme>GFS_rad_time_vary</scheme>
+      <scheme>GFS_phys_time_vary</scheme>
+    </subcycle>
+  </group>
+  <group name="radiation">
+    <subcycle loop="1">
+      <scheme>GFS_suite_interstitial_rad_reset</scheme>
+      <scheme>GFS_rrtmg_pre</scheme>
+      <scheme>rrtmg_sw_pre</scheme>
+      <scheme>rrtmg_sw</scheme>
+      <scheme>rrtmg_sw_post</scheme>
+      <scheme>rrtmg_lw_pre</scheme>
+      <scheme>rrtmg_lw</scheme>
+      <scheme>rrtmg_lw_post</scheme>
+      <scheme>GFS_rrtmg_post</scheme>
+    </subcycle>
+  </group>
+  <group name="physics">
+    <subcycle loop="1">
+      <scheme>GFS_suite_interstitial_phys_reset</scheme>
+      <scheme>GFS_suite_stateout_reset</scheme>
+      <scheme>get_prs_fv3</scheme>
+      <scheme>GFS_suite_interstitial_1</scheme>
+      <scheme>GFS_surface_generic_pre</scheme>
+      <scheme>GFS_surface_composites_pre</scheme>
+      <scheme>dcyc2t3</scheme>
+      <scheme>GFS_surface_composites_inter</scheme>
+      <scheme>GFS_suite_interstitial_2</scheme>
+    </subcycle>
+    <!-- Surface iteration loop -->
+    <subcycle loop="2">
+      <scheme>sfc_diff</scheme>
+      <scheme>GFS_surface_loop_control_part1</scheme>
+      <scheme>lsm_noah</scheme>
+      <scheme>sfc_nst_pre</scheme>
+      <scheme>sfc_nst</scheme>
+      <scheme>sfc_nst_post</scheme>
+      <scheme>sfc_cice</scheme>
+      <scheme>sfc_sice</scheme>
+      <scheme>GFS_surface_loop_control_part2</scheme>
+    </subcycle>
+    <!-- End of surface iteration loop -->
+    <subcycle loop="1">
+      <scheme>GFS_surface_composites_post</scheme>
+      <scheme>sfc_diag</scheme>
+      <scheme>sfc_diag_post</scheme>
+      <scheme>GFS_surface_generic_post</scheme>
+      <scheme>GFS_PBL_generic_pre</scheme>
+      <scheme>hedmf</scheme>
+      <scheme>GFS_PBL_generic_post</scheme>
+      <scheme>GFS_GWD_generic_pre</scheme>
+      <scheme>cires_ugwp</scheme>
+      <scheme>cires_ugwp_post</scheme>
+      <scheme>GFS_GWD_generic_post</scheme>
+      <scheme>rayleigh_damp</scheme>
+      <scheme>GFS_suite_stateout_update</scheme>
+      <scheme>ozphys</scheme>
+      <scheme>get_phi_fv3</scheme>
+      <scheme>GFS_suite_interstitial_3</scheme>
+      <scheme>GFS_DCNV_generic_pre</scheme>
+      <scheme>samfdeepcnv</scheme>
+      <scheme>GFS_DCNV_generic_post</scheme>
+      <scheme>GFS_SCNV_generic_pre</scheme>
+      <scheme>samfshalcnv</scheme>
+      <scheme>GFS_SCNV_generic_post</scheme>
+      <scheme>GFS_suite_interstitial_4</scheme>
+      <scheme>cnvc90</scheme>
+      <scheme>GFS_MP_generic_pre</scheme>
+      <scheme>zhaocarr_gscond</scheme>
+      <scheme>zhaocarr_precpd</scheme>
+      <scheme>GFS_MP_generic_post</scheme>
+      <scheme>maximum_hourly_diagnostics</scheme>
+    </subcycle>
+  </group>
+  <group name="stochastics">
+    <subcycle loop="1">
+      <scheme>GFS_stochastics</scheme>
+    </subcycle>
+  </group>
+  <!-- <finalize></finalize> -->
+</suite>

ccpp/suites/suite_FV3_GFS_cpld_rasmgshocnsst.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<suite name="FV3_GFS_cpld_rasmgshocnsst" lib="ccppphys" ver="3">
+<suite name="FV3_GFS_cpld_rasmgshocnsst" lib="ccppphys" ver="4">
   <!-- <init></init> -->
   <group name="time_vary">
     <subcycle loop="1">

gfsphysics/GFS_layer/GFS_diagnostics.F90

@@ -2445,7 +2445,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%name = 'dt3dt_nophys'
       ExtDiag(idx)%desc = 'temperature tendency due to non-physics processes'
       ExtDiag(idx)%unit = 'K s-1'
-      ExtDiag(idx)%mod_name = 'gfs_phys'
+      ExtDiag(idx)%mod_name = 'gfs_dyn'
       ExtDiag(idx)%time_avg = .TRUE.
       allocate (ExtDiag(idx)%data(nblks))
       do nb = 1,nblks
@@ -2626,7 +2626,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%name = 'du3dt_nophys'
       ExtDiag(idx)%desc = 'u momentum tendency due to non-physics processes'
       ExtDiag(idx)%unit = 'm s-2'
-      ExtDiag(idx)%mod_name = 'gfs_phys'
+      ExtDiag(idx)%mod_name = 'gfs_dyn'
       ExtDiag(idx)%time_avg = .TRUE.
       allocate (ExtDiag(idx)%data(nblks))
       do nb = 1,nblks
@@ -2638,7 +2638,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%name = 'dv3dt_nophys'
       ExtDiag(idx)%desc = 'v momentum tendency due to non-physics processes'
       ExtDiag(idx)%unit = 'm s-2'
-      ExtDiag(idx)%mod_name = 'gfs_phys'
+      ExtDiag(idx)%mod_name = 'gfs_dyn'
       ExtDiag(idx)%time_avg = .TRUE.
       allocate (ExtDiag(idx)%data(nblks))
       do nb = 1,nblks
@@ -2785,7 +2785,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%name = 'dq3dt_nophys'
       ExtDiag(idx)%desc = 'water vapor specific humidity tendency due to non-physics processes'
       ExtDiag(idx)%unit = 'kg kg-1 s-1'
-      ExtDiag(idx)%mod_name = 'gfs_phys'
+      ExtDiag(idx)%mod_name = 'gfs_dyn'
       ExtDiag(idx)%time_avg = .TRUE.
       allocate (ExtDiag(idx)%data(nblks))
       do nb = 1,nblks
@@ -2797,7 +2797,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%name = 'dq3dt_o3nophys'
       ExtDiag(idx)%desc = 'ozone concentration tendency due to non-physics processes'
       ExtDiag(idx)%unit = 'kg kg-1 s-1'
-      ExtDiag(idx)%mod_name = 'gfs_phys'
+      ExtDiag(idx)%mod_name = 'gfs_dyn'
       ExtDiag(idx)%time_avg = .TRUE.
       allocate (ExtDiag(idx)%data(nblks))
       do nb = 1,nblks

gfsphysics/GFS_layer/GFS_typedefs.F90

@@ -1551,9 +1551,9 @@ module GFS_typedefs
 #ifdef CCPP
     real (kind=kind_phys), pointer :: TRAIN  (:,:)   => null()  !< accumulated stratiform T tendency (K s-1)
 #endif
-#ifdef CCPP
+!#ifdef CCPP
     real (kind=kind_phys), pointer :: cldfra  (:,:)   => null()  !< instantaneous 3D cloud fraction
-#endif
+!#endif
     !--- MP quantities for 3D diagnositics 
     real (kind=kind_phys), pointer :: refl_10cm(:,:) => null()  !< instantaneous refl_10cm 
 !

io/FV3GFS_io.F90

@@ -1077,16 +1077,16 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain)
 
         if (Sfcprop(nb)%lakefrac(ix) > zero) then
           Sfcprop(nb)%oceanfrac(ix) = zero ! lake & ocean don't coexist in a cell
-          if (Sfcprop(nb)%fice(ix) < Model%min_lakeice) then
-             Sfcprop(nb)%fice(ix) = zero
-             if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0
-          endif
+!         if (Sfcprop(nb)%fice(ix) < Model%min_lakeice) then
+!            Sfcprop(nb)%fice(ix) = zero
+!            if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0
+!         endif
         else
           Sfcprop(nb)%oceanfrac(ix) = one - Sfcprop(nb)%landfrac(ix)
-          if (Sfcprop(nb)%fice(ix) < Model%min_seaice) then
-             Sfcprop(nb)%fice(ix) = zero
-             if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0
-          endif
+!         if (Sfcprop(nb)%fice(ix) < Model%min_seaice) then
+!            Sfcprop(nb)%fice(ix) = zero
+!            if (Sfcprop(nb)%slmsk(ix) == 2) Sfcprop(nb)%slmsk(ix) = 0
+!         endif
         endif
         !
         !--- NSSTM variables
@@ -1357,28 +1357,47 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain)
         enddo
       enddo
     else
+      if( Model%phour < 1.e-7) then
 !$omp parallel do default(shared) private(nb, ix, tem)
-      do nb = 1, Atm_block%nblks
-        do ix = 1, Atm_block%blksz(nb)
+        do nb = 1, Atm_block%nblks
+          do ix = 1, Atm_block%blksz(nb)
       !--- specify tsfcl/zorll/zorli from existing variable tsfco/zorlo
-!         Sfcprop(nb)%tsfcl(ix) = Sfcprop(nb)%tsfco(ix)
-!         Sfcprop(nb)%zorll(ix) = Sfcprop(nb)%zorlo(ix)
-!         Sfcprop(nb)%zorli(ix) = Sfcprop(nb)%zorlo(ix)
-!         Sfcprop(nb)%zorl(ix)  = Sfcprop(nb)%zorlo(ix)
-!         Sfcprop(nb)%tsfc(ix)  = Sfcprop(nb)%tsfco(ix)
-          if (Sfcprop(nb)%slmsk(ix) == 1) then
-            Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorll(ix) 
-            Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix)
-          else
-            tem = one - Sfcprop(nb)%fice(ix)
-            Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorli(ix) * Sfcprop(nb)%fice(ix) &
-                                 + Sfcprop(nb)%zorlo(ix) * tem
-
-            Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tisfc(ix) * Sfcprop(nb)%fice(ix) &
-                                 + Sfcprop(nb)%tsfco(ix) * tem
-          endif
+!           Sfcprop(nb)%tsfcl(ix) = Sfcprop(nb)%tsfco(ix)
+!           Sfcprop(nb)%zorll(ix) = Sfcprop(nb)%zorlo(ix)
+!           Sfcprop(nb)%zorli(ix) = Sfcprop(nb)%zorlo(ix)
+!           Sfcprop(nb)%zorl(ix)  = Sfcprop(nb)%zorlo(ix)
+            if (Sfcprop(nb)%slmsk(ix) == 1) then
+              Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorll(ix) 
+              Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix)
+            else
+              tem = one - Sfcprop(nb)%fice(ix)
+              Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorli(ix) * Sfcprop(nb)%fice(ix) &
+                                   + Sfcprop(nb)%zorlo(ix) * tem
+              Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tisfc(ix) * Sfcprop(nb)%fice(ix) &
+                                   + Sfcprop(nb)%tsfco(ix) * tem
+            endif
+          enddo
         enddo
-      enddo
+      else
+!$omp parallel do default(shared) private(nb, ix, tem)
+        do nb = 1, Atm_block%nblks
+          do ix = 1, Atm_block%blksz(nb)
+      !--- specify tsfcl/zorll/zorli from existing variable tsfco/zorlo
+            Sfcprop(nb)%tsfc(ix)  = Sfcprop(nb)%tsfco(ix)
+            if (Sfcprop(nb)%slmsk(ix) == 1) then
+              Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorll(ix)
+              Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix)
+            else
+              tem = one - Sfcprop(nb)%fice(ix)
+              Sfcprop(nb)%zorl(ix) = Sfcprop(nb)%zorli(ix) * Sfcprop(nb)%fice(ix) &
+                                   + Sfcprop(nb)%zorlo(ix) * tem
+              if (Sfcprop(nb)%fice(ix) > min(Model%min_seaice,Model%min_lakeice)) then
+                Sfcprop(nb)%tsfc(ix) = Sfcprop(nb)%tsfcl(ix)
+              endif
+            endif
+          enddo
+        enddo
+      endif
     endif ! if (Model%frac_grid)
 
 !#ifdef CCPP