Make stochastic physics independent of fv3atm and its submodules

.github/CODEOWNERS

@@ -0,0 +1,7 @@
+# This is a comment.
+# Each line is a file pattern followed by one or more owners.
+# Order is important; the last matching pattern takes the most
+# precedence.
+
+# The components are owned by their respective teams
+/FV3/ @noaa-emc/fv3atm-team

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,56 @@
+---
+name: Bug report
+about: Create a report to fix bugs
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+## Description
+Provide a clear and concise description of what the bug is.
+Also give a description of how to fix the bug.
+
+
+### To Reproduce:
+What compilers/machines are you seeing this with?
+Give explicit steps to reproduce the behavior.
+1. do this
+2. then that
+3. then, oops, look at the bug
+
+
+## Additional context
+Add any other context about the problem here.
+Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example:
+- needs to be fixed also in noaa-emc/nems/issues/<issue_number>
+- needed for noaa-emc/fv3atm/pull/<pr_number>
+
+
+## Output
+
+**Screenshots**
+If applicable, drag and drop screenshots to help explain your problem.
+
+**output logs**
+If applicable, include relevant output logs.
+Either drag and drop the entire log file here (if a long log) or
+
+```
+paste the code here (if a short section of log)
+```
+
+## Testing:
+
+1. Have you tested the code changes? On what platforms?
+
+2. Have you run regression test in ufs-weather-model or ufs-s2s-model with code changes?
+- Will the baseline results change? 
+- If the baseline results change, is it expected? Please give brief explanation.
+
+## Dependent PRs:
+
+Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example:
+- required to support noaa-emc/GFDL_atmos_cubed_sphere/issues/<issue_number>
+- ncar/ccpp-physics/pull/<pr_number>
+- associated ufs-weather-model/pull/<pr_number>

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,32 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+## Description
+Provide a clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+## Solution
+Add a clear and concise description of what solution you provide.
+
+## Alternatives
+If applicable, add a description of any alternative solutions or features you've considered.
+
+## Testing:
+
+1. Have you tested the code changes? On what platforms?
+2. Have you run regression test in ufs-weather-model or ufs-s2s-model with code changes?
+- Will the baseline results change?
+- If yes, please give brief explanation of your code updates on the regression test.
+
+## Dependent PRs:
+
+Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example:
+- required to support noaa-emc/GFDL_atmos_cubed_sphere/issues/<issue_number>
+- ncar/ccpp-physics/pull/<pr_number>
+- associated ufs-weather-model/pull/<pr_number>
+

.github/pull_request_template.md

@@ -0,0 +1,37 @@
+## Description
+
+(Instructions: this, and all subsequent sections of text should be removed and filled in as appropriate.)
+Provide a detailed description of what this PR does.  
+What bug does it fix, or what feature does it add?  
+Is a change of answers expected from this PR?  
+
+
+
+### Issue(s) addressed
+
+Link the issues to be closed with this PR, whether in this repository, or in another repository.
+(Remember, issues should always be created before starting work on a PR branch!)
+- fixes #<issue_number>
+- fixes noaa-emc/fv3atm/issues/<issue_number>
+
+
+
+## Testing
+
+How were these changes tested?  
+What compilers / HPCs was it tested with?  
+Are the changes covered by regression tests? (If not, why? Do new tests need to be added?)  
+Have the ufs-weather-model regression test been run? On what platform?  
+- Will the code updates change regression test baseline? If yes, why? Please show the baseline directory below.
+- Please commit the regression test log files in your ufs-weather-model branch
+
+
+## Dependencies
+
+If testing this branch requires non-default branches in other repositories, list them.
+Those branches should have matching names (ideally)
+
+Do PRs in upstream repositories need to be merged first?
+If so add the "waiting for other repos" label and list the upstream PRs
+- waiting on noaa-emc/nems/pull/<pr_number>
+- waiting on noaa-emc/fv3atm/pull/<pr_number>

CMakeLists.txt

@@ -129,6 +129,7 @@ target_link_libraries(fv3atm PUBLIC ipd)
 target_link_libraries(fv3atm PUBLIC io)
 target_link_libraries(fv3atm PUBLIC fv3dycore)
 target_link_libraries(fv3atm PUBLIC stochastic_physics)
+target_link_libraries(fv3atm PUBLIC stochastic_physics_wrapper)
 
 target_link_libraries(fv3atm PUBLIC
                       nemsio::nemsio w3emc::w3emc_d w3nco::w3nco_d sp::sp_d bacio::bacio_4)
@@ -142,7 +143,7 @@ target_link_libraries(fv3atm PUBLIC esmf)
 ### Install
 ###############################################################################
 install(
-  TARGETS fv3atm fv3dycore io ipd gfsphysics ${CCPP_LIBRARIES} cpl stochastic_physics
+  TARGETS fv3atm fv3dycore io ipd gfsphysics ${CCPP_LIBRARIES} cpl stochastic_physics stochastic_physics_wrapper
   EXPORT fv3atm-config
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)

atmos_model.F90

@@ -104,9 +104,7 @@ module atmos_model_mod
 use physics_abstraction_layer, only: time_vary_step, radiation_step1, physics_step1, physics_step2
 #endif
 
-use stochastic_physics, only: init_stochastic_physics,         &
-                              run_stochastic_physics
-use stochastic_physics_sfc, only: run_stochastic_physics_sfc
+use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper
 
 use FV3GFS_io_mod,      only: FV3GFS_restart_read, FV3GFS_restart_write, &
                               FV3GFS_IPD_checksum,                       &
@@ -251,26 +249,17 @@ module atmos_model_mod
 ! </INOUT>
 
 subroutine update_atmos_radiation_physics (Atmos)
-#ifdef OPENMP
-    use omp_lib
-#endif
 !-----------------------------------------------------------------------
   type (atmos_data_type), intent(in) :: Atmos
 !--- local variables---
     integer :: nb, jdat(8), rc
     procedure(IPD_func0d_proc), pointer :: Func0d => NULL()
     procedure(IPD_func1d_proc), pointer :: Func1d => NULL()
-    integer :: nthrds
+    !
 #ifdef CCPP
     integer :: ierr
 #endif
 
-#ifdef OPENMP
-    nthrds = omp_get_max_threads()
-#else
-    nthrds = 1
-#endif
-
     if (mpp_pe() == mpp_root_pe() .and. debug) write(6,*) "statein driver"
 !--- get atmospheric state from the dynamic core
     call set_atmosphere_pelist()
@@ -307,24 +296,7 @@ subroutine update_atmos_radiation_physics (Atmos)
 #endif
 
 !--- call stochastic physics pattern generation / cellular automata
-    if (IPD_Control%do_sppt .OR. IPD_Control%do_shum .OR. IPD_Control%do_skeb .OR. IPD_Control%do_sfcperts) then
-       call run_stochastic_physics(IPD_Control, IPD_Data(:)%Grid, IPD_Data(:)%Coupling, nthrds)
-    end if
-
-    if(IPD_Control%do_ca)then
-       if(IPD_Control%ca_sgs)then
-          call cellular_automata_sgs(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, &
-            IPD_Control%nca,IPD_Control%ncells,IPD_Control%nlives,IPD_Control%nfracseed,&
-            IPD_Control%nseed,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,&
-            IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1))
-       endif
-       if(IPD_Control%ca_global)then
-          call cellular_automata_global(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, &
-            IPD_Control%nca_g,IPD_Control%ncells_g,IPD_Control%nlives_g,IPD_Control%nfracseed,&
-            IPD_Control%nseed_g,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,&
-            IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1),IPD_Control%nsmooth,IPD_Control%ca_amplitude)
-      endif
-    endif
+    call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block)
 
 !--- if coupled, assign coupled fields
       if( IPD_Control%cplflx .or. IPD_Control%cplwav ) then
@@ -649,45 +621,11 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
    call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, Init_parm)
 #endif
 
-   if (IPD_Control%do_sppt .OR. IPD_Control%do_shum .OR. IPD_Control%do_skeb .OR. IPD_Control%do_sfcperts) then
-      ! Initialize stochastic physics
-      call init_stochastic_physics(IPD_Control, Init_parm, mpp_npes(), nthrds)
-      if(IPD_Control%me == IPD_Control%master) print *,'do_skeb=',IPD_Control%do_skeb
-   end if
+!--- Initialize stochastic physics pattern generation / cellular automata for first time step
+   call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block)
 
    Atmos%Diag => IPD_Diag
 
-   if (IPD_Control%do_sfcperts) then
-      ! Get land surface perturbations here (move to GFS_time_vary
-      ! step if wanting to update each time-step)
-      call run_stochastic_physics_sfc(IPD_Control, IPD_Data(:)%Grid, IPD_Data(:)%Coupling)
-   end if
-
-   ! Initialize cellular automata
-   if(IPD_Control%do_ca)then
-      ! DH* The current implementation of cellular_automata assumes that all blocksizes are the
-      ! same - abort if this is not the case, otherwise proceed with Atm_block%blksz(1) below
-      if (.not. minval(Atm_block%blksz)==maxval(Atm_block%blksz)) then
-         call mpp_error(FATAL, 'Logic errror: cellular_automata not compatible with non-uniform blocksizes')
-      end if
-      ! *DH
-      if(IPD_Control%do_ca)then
-       if(IPD_Control%ca_sgs)then
-          call cellular_automata_sgs(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, &
-            IPD_Control%nca,IPD_Control%ncells,IPD_Control%nlives,IPD_Control%nfracseed,&
-            IPD_Control%nseed,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,&
-            IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1))
-       endif
-       if(IPD_Control%ca_global)then
-          call cellular_automata_global(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, &
-            IPD_Control%nca_g,IPD_Control%ncells_g,IPD_Control%nlives_g,IPD_Control%nfracseed,&
-            IPD_Control%nseed_g,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,&
-            IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1),IPD_Control%nsmooth,IPD_Control%ca_amplitude)
-       endif
-
-    endif
-   endif
-
    Atm(mygrid)%flagstruct%do_skeb = IPD_Control%do_skeb
 
 !  initialize the IAU module

makefile

@@ -12,7 +12,7 @@ ifeq ($(strip $(exist)),)
 endif
 
 
-FFLAGS   += -I$(FMS_DIR) -Igfsphysics -Iipd -Icpl -Iio -Iatmos_cubed_sphere -Iccpp/driver -I../stochastic_physics
+FFLAGS   += -I$(FMS_DIR) -Igfsphysics -Iipd -Icpl -Iio -Iatmos_cubed_sphere -Iccpp/driver -Istochastic_physics
 CPPDEFS  += -DESMF_VERSION_MAJOR=$(ESMF_VERSION_MAJOR)
 
 # Flag to CCPP build for 32bit dynamics
@@ -41,8 +41,9 @@ libs:
 	$(MAKE) -C io                  $(MAKE_OPTS) FMS_DIR=$(FMS_DIR)
 	$(MAKE) -C atmos_cubed_sphere  $(MAKE_OPTS) FMS_DIR=$(FMS_DIR)
 	$(MAKE) -C ../stochastic_physics  $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) 32BIT=N  # force gfs physics to 64bit
+	$(MAKE) -C stochastic_physics  $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) 32BIT=N  # force gfs physics to 64bit
 
-$(FV3_EXE): atmos_model.o coupler_main.o ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a ../stochastic_physics/libstochastic_physics.a cpl/libfv3cpl.a
+$(FV3_EXE): atmos_model.o coupler_main.o ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a stochastic_physics/libstochastic_physics_wrapper.a ../stochastic_physics/libstochastic_physics.a cpl/libfv3cpl.a
 	$(LD) -o $@ $^ $(NCEPLIBS) $(LDFLAGS)
 
 else
@@ -87,7 +88,7 @@ endif
 ifneq (,$(findstring CCPP,$(CPPDEFS)))
 esmf_make_fragment:
 	@rm -rf nems_dir; mkdir nems_dir
-	@cp $(FV3CAP_LIB) ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a cpl/libfv3cpl.a ../stochastic_physics/libstochastic_physics.a nems_dir
+	@cp $(FV3CAP_LIB) ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a cpl/libfv3cpl.a stochastic_physics/libstochastic_physics_wrapper.a ../stochastic_physics/libstochastic_physics.a nems_dir
 	@cp fv3gfs_cap_mod.mod nems_dir
 	@echo "# ESMF self-describing build dependency makefile fragment" > fv3.mk
 	@echo "# src location $(PWD)" >> fv3.mk
@@ -97,7 +98,7 @@ esmf_make_fragment:
 	#@echo "ESMF_DEP_INCPATH   = $(PWD)/nems_dir" >> fv3.mk
 	@echo "ESMF_DEP_INCPATH   = $(PWD) $(addprefix $(PWD)/, nems_dir ccpp/driver atmos_cubed_sphere io gfsphysics cpl ipd ../stochastic_physics)" >> fv3.mk
 	@echo "ESMF_DEP_CMPL_OBJS ="                 >> fv3.mk
-	@echo "ESMF_DEP_LINK_OBJS = $(addprefix $(PWD)/nems_dir/, libfv3cap.a libccppdriver.a libfv3core.a libfv3io.a libipd.a libgfsphys.a libfv3cpl.a libstochastic_physics.a) $(SIONLIB_LINK_FLAGS)" >> fv3.mk
+	@echo "ESMF_DEP_LINK_OBJS = $(addprefix $(PWD)/nems_dir/, libfv3cap.a libccppdriver.a libfv3core.a libfv3io.a libipd.a libgfsphys.a libfv3cpl.a libstochastic_physics_wrapper.a libstochastic_physics.a) $(SIONLIB_LINK_FLAGS)" >> fv3.mk
 	@echo "ESMF_DEP_SHRD_PATH ="                 >> fv3.mk
 	@echo "ESMF_DEP_SHRD_LIBS ="                 >> fv3.mk
 	@echo
@@ -143,6 +144,7 @@ clean:
 	(cd ccpp/driver         && make clean)
 	(cd ipd                 && make clean)
 	(cd ../stochastic_physics  && make clean)
+	(cd stochastic_physics  && make clean)
 	(cd io                  && make clean)
 	(cd atmos_cubed_sphere  && make clean)
 	(cd cpl                 && make clean)

stochastic_physics/CMakeLists.txt

@@ -1,62 +1,19 @@
-
-if(32BIT)
-remove_definitions(-DOVERLOAD_R4)
-remove_definitions(-DOVERLOAD_R8)
-message ("Force 64 bits in stochastic_physics")
-if(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
-    if(REPRO)
-        string (REPLACE "-i4 -real-size 32" "-i4 -real-size 64" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}")
-    else()
-        string (REPLACE "-i4 -real-size 32" "-i4 -real-size 64 -no-prec-div -no-prec-sqrt" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}")
-    endif()
-elseif(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
-    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8")
-endif()
-endif()
-
 add_library(
-    stochastic_physics
+    stochastic_physics_wrapper
 
-    ../../stochastic_physics/plumes.f90
-
-    ../../stochastic_physics/stochy_gg_def.f
-    ../../stochastic_physics/stochy_resol_def.f
-    ../../stochastic_physics/stochy_layout_lag.f
-    ../../stochastic_physics/four_to_grid_stochy.F
-    ../../stochastic_physics/glats_stochy.f
-    ../../stochastic_physics/sumfln_stochy.f
-    ../../stochastic_physics/gozrineo_stochy.f
-    ../../stochastic_physics/num_parthds_stochy.f
-    ../../stochastic_physics/get_ls_node_stochy.f
-    ../../stochastic_physics/get_lats_node_a_stochy.f
-    ../../stochastic_physics/setlats_a_stochy.f
-    ../../stochastic_physics/setlats_lag_stochy.f
-    ../../stochastic_physics/epslon_stochy.f
-    ../../stochastic_physics/getcon_lag_stochy.f
-    ../../stochastic_physics/pln2eo_stochy.f
-    ../../stochastic_physics/dozeuv_stochy.f
-    ../../stochastic_physics/dezouv_stochy.f
-
-    ../../stochastic_physics/spectral_layout.F90
-    ../../stochastic_physics/getcon_spectral.F90
-    ../../stochastic_physics/stochy_namelist_def.F90
-    ../../stochastic_physics/compns_stochy.F90
-    ../../stochastic_physics/stochy_internal_state_mod.F90
-    ../../stochastic_physics/stochastic_physics.F90
-    ../../stochastic_physics/stochy_patterngenerator.F90
-    ../../stochastic_physics/stochy_data_mod.F90
-    ../../stochastic_physics/get_stochy_pattern.F90
-    ../../stochastic_physics/initialize_spectral_mod.F90
-    ../../stochastic_physics/cellular_automata_global.F90
-    ../../stochastic_physics/cellular_automata_sgs.F90
-    ../../stochastic_physics/update_ca.F90
+    ./stochastic_physics_wrapper.F90
 )
 
+target_include_directories(stochastic_physics_wrapper PRIVATE ${CMAKE_BINARY_DIR}/stochastic_physics)
 if(CCPP)
-  target_include_directories(stochastic_physics PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src)
-  target_include_directories(stochastic_physics PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/physics)
+  target_include_directories(stochastic_physics_wrapper PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src)
+  target_include_directories(stochastic_physics_wrapper PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/physics)
 endif()
-target_link_libraries(stochastic_physics PRIVATE fms)
-target_link_libraries(stochastic_physics PRIVATE gfsphysics)
-target_link_libraries(stochastic_physics PRIVATE fv3dycore)
-target_link_libraries(stochastic_physics PRIVATE sp::sp_d)
+# DH* is there a better way to do this?
+target_include_directories(stochastic_physics_wrapper PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/../mod>)
+# *DH
+target_link_libraries(stochastic_physics_wrapper fms)
+target_link_libraries(stochastic_physics_wrapper stochastic_physics)
+target_link_libraries(stochastic_physics_wrapper gfsphysics)
+target_link_libraries(stochastic_physics_wrapper fv3dycore)
+