Merge pull request ESCOMP#237 from uturuncoglu/feature/esmx

Enable build with ESMX and introduce export_all
samsrabin · Jul 10, 2023 · ae827c3 · ae827c3
2 parents fe5a578 + 39bc5b1
commit ae827c3
Show file tree

Hide file tree

Showing 15 changed files with 192 additions and 61 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,6 +2,12 @@ cmake_minimum_required(VERSION 3.10)
 include(ExternalProject)
 include(FetchContent)
 
+set(DISABLE_FoX OFF CACHE BOOL "Disable FoX library to process XML files.")
+message("DISABLE_FoX = ${DISABLE_FoX}")
+if(DISABLE_FoX)
+  add_definitions(-DDISABLE_FoX)
+endif()
+
 if (DEFINED CIMEROOT)
   message("Using CIME in ${CIMEROOT} with compiler ${COMPILER}")
   include(${CASEROOT}/Macros.cmake)
@@ -23,7 +29,9 @@ else()
   set(BLD_STANDALONE TRUE)
   project(NUOPC_DATA_MODELS LANGUAGES Fortran VERSION 0.1)
   list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
-  set(FOX_ROOT ${CMAKE_SOURCE_DIR}/fox)
+  if(NOT DISABLE_FoX)
+    set(FOX_ROOT ${CMAKE_SOURCE_DIR}/fox)
+  endif()
 endif()
 message("CMAKE_MODULE_PATH is ${CMAKE_MODULE_PATH}, CMAKE_Fortran_COMPILER is ${CMAKE_Fortran_COMPILER}")
 enable_language(Fortran)
@@ -62,29 +70,32 @@ endif()
 add_subdirectory(streams)
 add_subdirectory(dshr)
 
-if(IS_DIRECTORY "${FOX_ROOT}")
-  message(STATUS "FoX library is already checked out!")
-  message(STATUS "FoX source dir: ${FOX_ROOT}")
-else()
-  FetchContent_Declare(fox
-    GIT_REPOSITORY https://github.com/ESMCI/fox.git
-    GIT_TAG 4.1.2.1
-    SOURCE_DIR ${FOX_ROOT}
-    BINARY_DIR ${FOX_ROOT}/..
-  )
-  FetchContent_GetProperties(fox)
-  if(NOT fox_POPULATED)
-    FetchContent_Populate(fox)
-    message(STATUS "FoX source dir: ${fox_SOURCE_DIR}")
-    message(STATUS "FoX binary dir: ${fox_BINARY_DIR}")
+if(NOT DISABLE_FoX)
+  if(IS_DIRECTORY "${FOX_ROOT}")
+    message(STATUS "FoX library is already checked out!")
+    message(STATUS "FoX source dir: ${FOX_ROOT}")
+  else()
+    FetchContent_Declare(fox
+      GIT_REPOSITORY https://github.com/ESMCI/fox.git
+      GIT_TAG 4.1.2.1
+      SOURCE_DIR ${FOX_ROOT}
+      BINARY_DIR ${FOX_ROOT}/..
+    )
+    FetchContent_GetProperties(fox)
+    if(NOT fox_POPULATED)
+      FetchContent_Populate(fox)
+      message(STATUS "FoX source dir: ${fox_SOURCE_DIR}")
+      message(STATUS "FoX binary dir: ${fox_BINARY_DIR}")
+    endif()
   endif()
+  add_subdirectory(fox)
+
+  target_include_directories(streams PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/fox/include>
+                                            $<INSTALL_INTERFACE:mod>)
+  target_include_directories(dshr PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/fox/include>
+                                         $<INSTALL_INTERFACE:mod>)
 endif()
-add_subdirectory(fox)
 
-target_include_directories(streams PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/fox/include>
-                                          $<INSTALL_INTERFACE:mod>)
-target_include_directories(dshr PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/fox/include>
-                                       $<INSTALL_INTERFACE:mod>)
 target_include_directories(dshr PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/streams>
                                        $<INSTALL_INTERFACE:mod>)
 
@@ -112,6 +123,9 @@ foreach(COMP datm dice dlnd docn drof dwav)
 endforeach(COMP)
 
 foreach(DEPS streams dshr cdeps_share FoX_dom FoX_wxml FoX_sax FoX_common FoX_utils FoX_fsys)
+  if(DISABLE_FoX AND ${DEPS} MATCHES "^FoX")
+     continue()
+  endif()
   if(NOT BLD_STANDALONE AND ${DEPS} STREQUAL "cdeps_share")
      continue()
   endif()

diff --git a/datm/CMakeLists.txt b/datm/CMakeLists.txt
@@ -25,4 +25,17 @@ add_dependencies(datm dshr streams)
 target_include_directories (datm PRIVATE ${ESMF_F90COMPILEPATHS})
 target_include_directories (datm PRIVATE ${CMAKE_SOURCE_DIR})
 target_include_directories (datm PRIVATE ${PIO_Fortran_INCLUDE_DIR})
-target_include_directories (datm PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../fox/include)
+if(NOT DISABLE_FoX)
+  target_include_directories (datm PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../fox/include)
+endif()
+
+if(BLD_STANDALONE)
+  # ESMX requires mod files
+  foreach (SRC ${SRCFILES})
+    string(REGEX REPLACE "[.]F90$" ".mod" MOD ${SRC})
+    if (NOT DEFINED CIMEROOT AND MOD STREQUAL atm_comp_nuopc.mod)
+      set(MOD cdeps_datm_comp.mod)
+    endif()
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${MOD}" DESTINATION include)
+  endforeach ()
+endif()
diff --git a/datm/atm_comp_nuopc.F90 b/datm/atm_comp_nuopc.F90
@@ -136,6 +136,7 @@ module cdeps_datm_comp
   integer                      :: nx_global                           ! global nx
   integer                      :: ny_global                           ! global ny
   logical                      :: skip_restart_read = .false.         ! true => skip restart read in continuation run
+  logical                      :: export_all = .false.                ! true => export all fields, do not check connected or not
 
   ! linked lists
   type(fldList_type) , pointer :: fldsImport => null()
@@ -222,7 +223,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     ! local variables
     integer           :: nu         ! unit number
     integer           :: ierr       ! error code
-    integer           :: bcasttmp(9)
+    integer           :: bcasttmp(10)
     type(ESMF_VM)     :: vm
     character(len=*),parameter :: subname=trim(modName) // ':(InitializeAdvertise) '
     character(*)    ,parameter :: F00 = "('(" // trim(modName) // ") ',8a)"
@@ -247,7 +248,8 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
          anomaly_forcing, &
          skip_restart_read, &
          flds_presndep, &
-         flds_preso3
+         flds_preso3, &
+         export_all
 
     rc = ESMF_SUCCESS
 
@@ -284,7 +286,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
        if(flds_co2)          bcasttmp(7) = 1
        if(flds_wiso)         bcasttmp(8) = 1
        if(skip_restart_read) bcasttmp(9) = 1
-
+       if(export_all)        bcasttmp(10) = 1
     end if
     call ESMF_GridCompGet(gcomp, vm=vm, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
@@ -305,7 +307,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
     call ESMF_VMBroadcast(vm, restfilm, CL, main_task, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
-    call ESMF_VMBroadcast(vm, bcasttmp, 9, main_task, rc=rc)
+    call ESMF_VMBroadcast(vm, bcasttmp, 10, main_task, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
     nx_global         = bcasttmp(1)
     ny_global         = bcasttmp(2)
@@ -316,6 +318,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     flds_co2          = (bcasttmp(7) == 1)
     flds_wiso         = (bcasttmp(8) == 1)
     skip_restart_read = (bcasttmp(9) == 1)
+    export_all        = (bcasttmp(10) == 1)
 
     ! write namelist input to standard out
     if (my_task == main_task) then
@@ -335,6 +338,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
        write(logunit,F02)' flds_co2       = ',flds_co2
        write(logunit,F02)' flds_wiso      = ',flds_wiso
        write(logunit,F02)' skip_restart_read = ',skip_restart_read
+       write(logunit,F02)' export_all     = ',export_all
     end if
 
     ! Validate sdat datamode
@@ -433,10 +437,10 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
     ! NUOPC_Realize "realizes" a previously advertised field in the importState and exportState
     ! by replacing the advertised fields with the newly created fields of the same name.
     call dshr_fldlist_realize( exportState, fldsExport, flds_scalar_name, flds_scalar_num, model_mesh, &
-         subname//':datmExport', rc=rc)
+         subname//':datmExport', export_all, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
     call dshr_fldlist_realize( importState, fldsImport, flds_scalar_name, flds_scalar_num, model_mesh, &
-         subname//':datmImport', rc=rc)
+         subname//':datmImport', .false., rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
     ! Get the time to interpolate the stream data to
@@ -770,6 +774,7 @@ subroutine datm_init_dfields(rc)
       call ESMF_StateGet(exportState, itemNameList=lfieldnames, rc=rc)
       if (chkerr(rc,__LINE__,u_FILE_u)) return
       do n = 1, fieldCount
+         call ESMF_LogWrite(trim(subname)//': field name = '//trim(lfieldnames(n)), ESMF_LOGMSG_INFO)
          call ESMF_StateGet(exportState, itemName=trim(lfieldnames(n)), field=lfield, rc=rc)
          if (chkerr(rc,__LINE__,u_FILE_u)) return
          call ESMF_FieldGet(lfield, rank=rank, rc=rc)

diff --git a/dice/CMakeLists.txt b/dice/CMakeLists.txt
@@ -19,4 +19,17 @@ add_dependencies(dice dshr streams)
 target_include_directories (dice PRIVATE ${ESMF_F90COMPILEPATHS})
 target_include_directories (dice PRIVATE "${CMAKE_SOURCE_DIR}")
 target_include_directories (dice PRIVATE "${PIO_Fortran_INCLUDE_DIR}")
-target_include_directories (dice PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../fox/include)
+if(NOT DISABLE_FoX)
+  target_include_directories (dice PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../fox/include)
+endif()
+
+if(BLD_STANDALONE)
+  # ESMX requires mod files
+  foreach (SRC ${SRCFILES})
+    string(REGEX REPLACE "[.]F90$" ".mod" MOD ${SRC})
+    if (NOT DEFINED CIMEROOT AND MOD STREQUAL ice_comp_nuopc.mod)
+      set(MOD cdeps_dice_comp.mod)
+    endif()
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${MOD}" DESTINATION include)
+  endforeach ()
+endif()
diff --git a/dice/ice_comp_nuopc.F90 b/dice/ice_comp_nuopc.F90
@@ -85,6 +85,7 @@ module cdeps_dice_comp
   character(CL)                :: restfilm = nullstr                  ! model restart file namelist
   integer                      :: nx_global
   integer                      :: ny_global
+  logical                      :: export_all = .false.                ! true => export all fields, do not check connected or not
 
   ! linked lists
   type(fldList_type) , pointer :: fldsImport => null()
@@ -183,7 +184,8 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
 
     namelist / dice_nml /  datamode, &
          model_meshfile, model_maskfile, &
-         restfilm, nx_global, ny_global, flux_swpf, flux_Qmin, flux_Qacc, flux_Qacc0
+         restfilm, nx_global, ny_global, &
+         flux_swpf, flux_Qmin, flux_Qacc, flux_Qacc0, export_all
 
     rc = ESMF_SUCCESS
 
@@ -212,7 +214,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
        end if
 
        ! write namelist input to standard out
-       write(logunit,F00)' datamode = ',trim(datamode)
+       write(logunit,F00)' datamode       = ',trim(datamode)
        write(logunit,F00)' model_meshfile = ',trim(model_meshfile)
        write(logunit,F00)' model_maskfile = ',trim(model_maskfile)
        write(logunit,F01)' nx_global  = ',nx_global
@@ -222,10 +224,12 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
        write(logunit,F02)' flux_Qacc  = ',flux_Qacc
        write(logunit,F03)' flux_Qacc0 = ',flux_Qacc0
        write(logunit,F00)' restfilm = ',trim(restfilm)
+       write(logunit,F02)' export_all = ',export_all
        bcasttmp = 0
        bcasttmp(1) = nx_global
        bcasttmp(2) = ny_global
        if(flux_Qacc) bcasttmp(3) = 1
+       if(export_all) bcasttmp(4) = 1
        rbcasttmp(1) = flux_swpf
        rbcasttmp(2) = flux_Qmin
        rbcasttmp(3) = flux_Qacc0
@@ -251,12 +255,12 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     nx_global = bcasttmp(1)
     ny_global = bcasttmp(2)
     flux_Qacc = (bcasttmp(3) == 1)
+    export_all= (bcasttmp(4) == 1)
 
     flux_swpf  = rbcasttmp(1)
     flux_Qmin  = rbcasttmp(2)
     flux_Qacc0 = rbcasttmp(3)
 
-
     ! Validate datamode
     if ( trim(datamode) == 'ssmi' .or. trim(datamode) == 'ssmi_iaf') then
        if (my_task == main_task) write(logunit,*) ' dice datamode = ',trim(datamode)
@@ -327,10 +331,10 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
     ! NUOPC_Realize "realizes" a previously advertised field in the importState and exportState
     ! by replacing the advertised fields with the newly created fields of the same name.
     call dshr_fldlist_realize( exportState, fldsExport, flds_scalar_name, flds_scalar_num, model_mesh, &
-         subname//':diceExport', rc=rc)
+         subname//':diceExport', export_all, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
     call dshr_fldlist_realize( importState, fldsImport, flds_scalar_name, flds_scalar_num, model_mesh, &
-         subname//':diceImport', rc=rc)
+         subname//':diceImport', .false., rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
     ! for single column, the target point might not be a point where the ice/ocn mask is > 0

diff --git a/dlnd/CMakeLists.txt b/dlnd/CMakeLists.txt
@@ -17,4 +17,17 @@ add_dependencies(dlnd dshr streams)
 target_include_directories (dlnd PRIVATE ${ESMF_F90COMPILEPATHS})
 target_include_directories (dlnd PRIVATE "${CMAKE_SOURCE_DIR}")
 target_include_directories (dlnd PRIVATE "${PIO_Fortran_INCLUDE_DIR}")
-target_include_directories (dlnd PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../fox/include)
+if(NOT DISABLE_FoX)
+  target_include_directories (dlnd PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../fox/include)
+endif()
+
+if(BLD_STANDALONE)
+  # ESMX requires mod files
+  foreach (SRC ${SRCFILES})
+    string(REGEX REPLACE "[.]F90$" ".mod" MOD ${SRC})
+    if (NOT DEFINED CIMEROOT AND MOD STREQUAL lnd_comp_nuopc.mod)
+      set(MOD cdeps_dlnd_comp.mod)
+    endif()
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${MOD}" DESTINATION include)
+  endforeach ()
+endif()
diff --git a/dlnd/lnd_comp_nuopc.F90 b/dlnd/lnd_comp_nuopc.F90
@@ -81,6 +81,8 @@ module cdeps_dlnd_comp
   integer                  :: nx_global                           ! global nx dimension of model mesh
   integer                  :: ny_global                           ! global ny dimension of model mesh
   logical                  :: skip_restart_read = .false.         ! true => skip restart read in continuation
+  logical                  :: export_all = .false.                ! true => export all fields, do not check connected or not
+
   ! linked lists
   type(fldList_type) , pointer :: fldsExport => null()
   type(dfield_type)  , pointer :: dfields    => null()
@@ -166,7 +168,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     type(ESMF_VM) :: vm
     character(CL) :: cvalue
     integer       :: nu         ! unit number
-    integer       :: bcasttmp(3)
+    integer       :: bcasttmp(4)
     integer       :: ierr       ! error code
     character(len=*) , parameter :: subname=trim(modName)//':(InitializeAdvertise) '
     character(*)     , parameter :: F00 = "('(" // trim(modName) // ") ',8a)"
@@ -175,7 +177,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     !-------------------------------------------------------------------------------
 
     namelist / dlnd_nml / datamode, model_meshfile, model_maskfile, &
-         nx_global, ny_global, restfilm, skip_restart_read
+         nx_global, ny_global, restfilm, skip_restart_read, export_all
 
     rc = ESMF_SUCCESS
 
@@ -206,7 +208,9 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
        bcasttmp(1) = nx_global
        bcasttmp(2) = ny_global
        if(skip_restart_read) bcasttmp(3) = 1
+       if(export_all) bcasttmp(4) = 1
     end if
+
     call ESMF_GridCompGet(gcomp, vm=vm, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
@@ -223,6 +227,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     nx_global = bcasttmp(1)
     ny_global = bcasttmp(2)
     skip_restart_read = (bcasttmp(3) == 1)
+    export_all = (bcasttmp(4) == 1)
 
     ! write namelist input to standard out
     if (my_task == main_task) then
@@ -233,6 +238,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
        write(logunit,F01)' ny_global         = ',ny_global
        write(logunit,F00)' restfilm          = ',trim(restfilm)
        write(logunit,F02)' skip_restart_read = ',skip_restart_read
+       write(logunit,F02)' export_all        = ',export_all
     endif
 
     ! Validate sdat datamode
@@ -289,7 +295,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
 
     ! Realize the actively coupled fields, now that a mesh is established and
     ! initialize dfields data type (to map streams to export state fields)
-    call dlnd_comp_realize(importState, exportState, rc=rc)
+    call dlnd_comp_realize(importState, exportState, export_all, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
     ! Read restart if necessary
@@ -459,11 +465,12 @@ subroutine dlnd_comp_advertise(importState, exportState, rc)
   end subroutine dlnd_comp_advertise
 
   !===============================================================================
-  subroutine dlnd_comp_realize(importState, exportState, rc)
+  subroutine dlnd_comp_realize(importState, exportState, export_all, rc)
 
     ! input/output variables
     type(ESMF_State) , intent(inout) :: importState
     type(ESMF_State) , intent(inout) :: exportState
+    logical          , intent(in)    :: export_all
     integer          , intent(out)   :: rc
 
     ! local variables
@@ -478,7 +485,7 @@ subroutine dlnd_comp_realize(importState, exportState, rc)
     ! -------------------------------------
 
     call dshr_fldlist_realize( exportState, fldsExport, flds_scalar_name, flds_scalar_num,  model_mesh, &
-         subname//':dlndExport', rc=rc)
+         subname//':dlndExport', export_all, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
   end subroutine dlnd_comp_realize

diff --git a/docn/CMakeLists.txt b/docn/CMakeLists.txt
@@ -23,4 +23,17 @@ add_dependencies(docn dshr streams)
 target_include_directories (docn PRIVATE ${ESMF_F90COMPILEPATHS})
 target_include_directories (docn PRIVATE ${CMAKE_SOURCE_DIR})
 target_include_directories (docn PRIVATE ${PIO_Fortran_INCLUDE_DIR})
-target_include_directories (docn PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../fox/include)
+if(NOT DISABLE_FoX)
+  target_include_directories (docn PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../fox/include)
+endif()
+
+if(BLD_STANDALONE)
+  # ESMX requires mod files
+  foreach (SRC ${SRCFILES})
+    string(REGEX REPLACE "[.]F90$" ".mod" MOD ${SRC})
+    if (NOT DEFINED CIMEROOT AND MOD STREQUAL ocn_comp_nuopc.mod)
+      set(MOD cdeps_docn_comp.mod)
+    endif()
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${MOD}" DESTINATION include)
+  endforeach ()
+endif()