From 96a2554bf20bb340f24288406f1d0d02eb9bbb5b Mon Sep 17 00:00:00 2001 From: Dusan Jovic <48258889+DusanJovic-NOAA@users.noreply.github.com> Date: Mon, 27 Dec 2021 09:05:45 -0500 Subject: [PATCH] Use 'model set run clock' routine in FV3 NUOPC cap. (#450) * Add 'SetRunClock' specialization routine to FV3 NUOPC cap, which sets fv3 model clock. FV3 cap does not keep local copy of clock (clock_fv3) as saved module variable anymore. * Minor code cleanup. --- ccpp/data/GFS_typedefs.F90 | 24 +-- fv3_cap.F90 | 402 ++++++++++--------------------------- module_fcst_grid_comp.F90 | 103 +++++++--- module_fv3_config.F90 | 4 +- 4 files changed, 191 insertions(+), 342 deletions(-) diff --git a/ccpp/data/GFS_typedefs.F90 b/ccpp/data/GFS_typedefs.F90 index 280f9aa76..14387929f 100644 --- a/ccpp/data/GFS_typedefs.F90 +++ b/ccpp/data/GFS_typedefs.F90 @@ -3740,46 +3740,46 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & if(gwd_opt==1) then if(me==master) & - write(0,*) 'FLAG: gwd_opt==1 so gwd not generic' + write(*,*) 'FLAG: gwd_opt==1 so gwd not generic' Model%flag_for_gwd_generic_tend=.false. elseif(me==master) then - write(0,*) 'NO FLAG: gwd is generic' + write(*,*) 'NO FLAG: gwd is generic' endif if(satmedmf .and. isatmedmf==0) then if(me==master) & - write(0,*) 'FLAG: satmedmf and isatedmf=0 so pbl not generic' + write(*,*) 'FLAG: satmedmf and isatedmf=0 so pbl not generic' Model%flag_for_pbl_generic_tend=.false. elseif(satmedmf .and. isatmedmf==1) then if(me==master) & - write(0,*) 'FLAG: satmedmf and isatedmf=1 so pbl not generic' + write(*,*) 'FLAG: satmedmf and isatedmf=1 so pbl not generic' Model%flag_for_pbl_generic_tend=.false. else if(hybedmf) then if(me==master) & - write(0,*) 'FLAG: hybedmf so pbl not generic' + write(*,*) 'FLAG: hybedmf so pbl not generic' Model%flag_for_pbl_generic_tend=.false. else if(do_mynnedmf) then if(me==master) & - write(0,*) 'FLAG: do_mynnedmf so pbl not generic' + write(*,*) 'FLAG: do_mynnedmf so pbl not generic' Model%flag_for_pbl_generic_tend=.false. elseif(me==master) then - write(0,*) 'NO FLAG: pbl is generic' + write(*,*) 'NO FLAG: pbl is generic' endif if(imfshalcnv == Model%imfshalcnv_gf) then if(me==master) & - write(0,*) 'FLAG: imfshalcnv_gf so scnv not generic' + write(*,*) 'FLAG: imfshalcnv_gf so scnv not generic' Model%flag_for_scnv_generic_tend=.false. elseif(me==master) then - write(0,*) 'NO FLAG: scnv is generic' + write(*,*) 'NO FLAG: scnv is generic' endif if(imfdeepcnv == Model%imfdeepcnv_gf) then if(me==master) & - write(0,*) 'FLAG: imfdeepcnv_gf so dcnv not generic' + write(*,*) 'FLAG: imfdeepcnv_gf so dcnv not generic' Model%flag_for_dcnv_generic_tend=.false. elseif(me==master) then - write(0,*) 'NO FLAG: dcnv is generic' + write(*,*) 'NO FLAG: dcnv is generic' endif ! @@ -5250,7 +5250,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & endif if (me == Model%master) & - write(0,*) ' num_p3d=', Model%num_p3d, ' num_p2d=', Model%num_p2d, & + write(*,*) ' num_p3d=', Model%num_p3d, ' num_p2d=', Model%num_p2d, & ' crtrh=', Model%crtrh, ' npdf3d=', Model%npdf3d, & ' pdfcld=', Model%pdfcld, ' shcnvcw=', Model%shcnvcw, & ' cnvcld=', Model%cnvcld, ' ncnvcld3d=',Model%ncnvcld3d, & diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 06e3edae9..7b7eb44e7 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -21,14 +21,14 @@ module fv3gfs_cap_mod label_RealizeProvided, & label_Advance, & label_CheckImport, & + label_SetRunClock, & label_TimestampExport, & label_Finalize, & NUOPC_ModelGet ! use module_fv3_config, only: quilting, output_fh, & nfhout, nfhout_hf, nsout, dt_atmos, & - calendar, calendar_type, & - force_date_from_configure, & + calendar, & cplprint_flag,output_1st_tstep_rst, & first_kdt @@ -60,7 +60,6 @@ module fv3gfs_cap_mod ! !----------------------------------------------------------------------- ! - type(ESMF_Clock),save :: clock_fv3 type(ESMF_GridComp) :: fcstComp type(ESMF_State) :: fcstState @@ -136,6 +135,14 @@ subroutine SetServices(gcomp, rc) specPhaseLabel="phase2", specRoutine=ModelAdvance_phase2, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + ! specializations to set fv3 cap run clock (model clock) + call ESMF_MethodRemove(gcomp, label=label_SetRunClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call NUOPC_CompSpecialize(gcomp, specLabel=label_SetRunClock, & + specRoutine=ModelSetRunClock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + ! specializations required to support 'inline' run sequences call NUOPC_CompSpecialize(gcomp, specLabel=label_CheckImport, & specPhaseLabel="phase1", specRoutine=fv3_checkimport, rc=rc) @@ -171,24 +178,19 @@ subroutine InitializeAdvertise(gcomp, rc) character(240) :: msgString logical :: isPresent, isSet type(ESMF_VM) :: vm, fcstVM - type(ESMF_Time) :: currTime, startTime, stopTime - type(ESMF_TimeInterval) :: RunDuration, timeStep, rsthour, IAU_offsetTI + type(ESMF_Time) :: currTime, startTime + type(ESMF_TimeInterval) :: timeStep, rsthour type(ESMF_Config) :: cf type(ESMF_RegridMethod_Flag) :: regridmethod - type(ESMF_TimeInterval) :: earthStep - real(ESMF_KIND_R8) :: medAtmCouplingIntervalSec - type(ESMF_Clock) :: fv3Clock - type(ESMF_TimeInterval) :: fv3Step - integer,dimension(6) :: date, date_init - integer :: i, j, k, io_unit, urc, ist + integer :: i, j, k, urc, ist integer :: noutput_fh, nfh, nfh2 integer :: petcount integer :: num_output_file integer :: nfhmax_hf real :: nfhmax real :: output_startfh, outputfh, outputfh2(2) - logical :: opened, loutput_fh, lfreq + logical :: loutput_fh, lfreq character(ESMF_MAXSTR) :: name integer,dimension(:), allocatable :: petList, fcstPetList, originPetList, targetPetList character(len=esmf_maxstr),allocatable :: fcstItemNameList(:) @@ -197,13 +199,19 @@ subroutine InitializeAdvertise(gcomp, rc) integer :: isrcTermProcessing character(len=*),parameter :: subname='(fv3_cap:InitializeAdvertise)' - real(kind=8) :: MPI_Wtime, timewri, timeis, timerhs + real(kind=8) :: MPI_Wtime, timeis, timerhs ! !------------------------------------------------------------------------ ! rc = ESMF_SUCCESS timeis = MPI_Wtime() + call ESMF_GridCompGet(gcomp,name=name,vm=vm,rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_VMGet(vm, petCount=petcount, localpet=mype, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + ! query for importState and exportState call NUOPC_ModelGet(gcomp, driverClock=clock, importState=importState, exportState=exportState, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -230,18 +238,6 @@ subroutine InitializeAdvertise(gcomp, rc) write(msgString,'(A,i6)') trim(subname)//' dbug = ',dbug call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO, rc=rc) - call ESMF_GridCompGet(gcomp,name=name,vm=vm,rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_VMGet(vm, petCount=petcount, localpet=mype, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - -! print *,'in fv3_cap,initAdvertize,name=',trim(name),'petcount=',petcount,'mype=',mype -! -! create an instance clock for fv3 - clock_fv3 = ESMF_ClockCreate(clock, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! !------------------------------------------------------------------------ ! get config variables ! @@ -267,8 +263,8 @@ subroutine InitializeAdvertise(gcomp, rc) noutput_fh = ESMF_ConfigGetLen(config=CF, label ='output_fh:',rc=rc) - if(mype == 0) print *,'af nems config,quilting=',quilting,'calendar=', trim(calendar),' iau_offset=',iau_offset, & - 'noutput_fh=',noutput_fh + if(mype == 0) print *,'af nems config,quilting=',quilting,' calendar=', trim(calendar),' iau_offset=',iau_offset, & + ' noutput_fh=',noutput_fh ! nfhout = 0 ; nfhmax_hf = 0 ; nfhout_hf = 0 ; nsout = 0 if ( quilting ) then @@ -284,9 +280,8 @@ subroutine InitializeAdvertise(gcomp, rc) label ='isrcTermProcessing:',rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if(mype == 0) print *,'af nems config,quilting=',quilting,'write_groups=', & - write_groups,wrttasks_per_group,'calendar=',trim(calendar),'calendar_type=',calendar_type, & - 'isrcTermProcessing=', isrcTermProcessing + if(mype == 0) print *,'af nems config,quilting=',quilting,' write_groups=', & + write_groups,wrttasks_per_group,' isrcTermProcessing=', isrcTermProcessing ! call ESMF_ConfigGetAttribute(config=CF,value=num_files, & label ='num_files:',rc=rc) @@ -340,92 +335,10 @@ subroutine InitializeAdvertise(gcomp, rc) call ESMF_ConfigGetAttribute(config=CF, value=dt_atmos, label ='dt_atmos:', rc=rc) call ESMF_ConfigGetAttribute(config=CF, value=nfhmax, label ='nhours_fcst:',rc=rc) if(mype == 0) print *,'af nems config,dt_atmos=',dt_atmos,'nfhmax=',nfhmax - call ESMF_TimeIntervalSet(timeStep,s=dt_atmos,rc=rc) - call ESMF_ClockSet(clock_fv3,timeStep=timeStep, rc=rc) -! -!------------------------------------------------------------------------ -! may need to set currTime for restart -! - call ESMF_ClockGet(clock_fv3, currTime=currTime, StartTime=startTime, & - RunDuration=RunDuration, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - stopTime = startTime + RunDuration - -! *** read restart time from restart file - do i=751,899 - inquire(i, opened=opened) - if(.not. opened)then - io_unit = i - exit - endif - enddo -! - date = 0 ; date_init = 0 - force_date_from_configure = .true. -! - open(unit=io_unit, file=trim('INPUT/coupler.res'),status="old",err=998 ) - read (io_unit,*,err=999) calendar_type - read (io_unit,*) date_init - read (io_unit,*) date - close(io_unit) - force_date_from_configure = .false. -! - if(date(1) == 0 .and. date_init(1) /= 0) date = date_init - if(mype == 0) print *,'bf clock_fv3,date=',date,'date_init=',date_init - - call ESMF_VMbroadcast(vm, date, 6, 0) - call ESMF_TimeSet(time=currTime,yy=date(1),mm=date(2),dd=date(3),h=date(4), & - m=date(5),s=date(6),rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -999 continue -998 continue -! if(mype==0) print *,'final date =',date,'date_init=',date_init - -!reset currTime in clock - call ESMF_ClockSet(clock_fv3, currTime=currTime, startTime=startTime, & - stopTime=stopTime, timeStep=timeStep, rc=rc) + call ESMF_TimeIntervalSet(timeStep, s=dt_atmos, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - ! - !Under NUOPC, the EARTH driver clock is a separate instance from the - ! - fv3 clock. However, the fv3 clock may have been reset from restart - ! - therefore the EARTH driver clock must also be adjusted. - ! - Affected: currTime, timeStep - call ESMF_ClockGet(clock, timeStep=earthStep, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) call ESMF_Finalize(endflag=ESMF_END_ABORT) - - if (earthStep > (stopTime-currTime)) earthStep = stopTime - currTime - call ESMF_ClockSet(clock, currTime=currTime, timeStep=earthStep, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) call ESMF_Finalize(endflag=ESMF_END_ABORT) - - ! Set fv3 component clock as copy of EARTH clock. - call NUOPC_CompSetClock(gcomp, clock, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) call ESMF_Finalize(endflag=ESMF_END_ABORT) - - ! Read in the FV3 coupling interval - call ESMF_ConfigGetAttribute(config=CF, value=medAtmCouplingIntervalSec, & - label="atm_coupling_interval_sec:", default=-1.0_ESMF_KIND_R8, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) & - call ESMF_Finalize(endflag=ESMF_END_ABORT) - - if (medAtmCouplingIntervalSec > 0._ESMF_KIND_R8) then ! The coupling time step is provided - call ESMF_TimeIntervalSet(fv3Step, s_r8=medAtmCouplingIntervalSec, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) & - call ESMF_Finalize(endflag=ESMF_END_ABORT) - call ESMF_GridCompGet(gcomp, clock=fv3Clock, rc=RC) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) & - call ESMF_Finalize(endflag=ESMF_END_ABORT) - call ESMF_ClockSet(fv3Clock, timestep=fv3Step, rc=RC) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) & - call ESMF_Finalize(endflag=ESMF_END_ABORT) - - endif - first_kdt = 1 if( output_1st_tstep_rst) then rsthour = currTime - StartTime @@ -467,7 +380,7 @@ subroutine InitializeAdvertise(gcomp, rc) ! call fcst Initialize (including creating fcstgrid and fcst fieldbundle) call ESMF_GridCompInitialize(fcstComp, exportState=fcstState, & - clock=clock_fv3, userRc=urc, rc=rc) + clock=clock, userRc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return @@ -520,9 +433,6 @@ subroutine InitializeAdvertise(gcomp, rc) return endif enddo -! -! set up ESMF time interval at center of iau window - call ESMF_TimeIntervalSet(IAU_offsetTI, h=iau_offset, rc=rc) ! k = num_pes_fcst timerhs = MPI_Wtime() @@ -568,7 +478,7 @@ subroutine InitializeAdvertise(gcomp, rc) ! call into wrtComp(i) Initialize call ESMF_GridCompInitialize(wrtComp(i), importState=wrtstate(i), & - clock=clock_fv3, phase=1, userRc=urc, rc=rc) + clock=clock, phase=1, userRc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return @@ -621,8 +531,6 @@ subroutine InitializeAdvertise(gcomp, rc) if (i==1) then ! this is a Store() for the first wrtComp -> must do the Store() - timewri = MPI_Wtime() - call ESMF_FieldBundleRegridStore(fcstFB(j), wrtFB(j,i), & regridMethod=regridmethod, routehandle=routehandle(j,i), & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, & @@ -886,87 +794,18 @@ subroutine ModelAdvance(gcomp, rc) type(ESMF_GridComp) :: gcomp integer, intent(out) :: rc - ! local variables - type(ESMF_Clock) :: clock - type(ESMF_Time) :: currTime, startTime, stopTime - ! type(ESMF_TimeInterval) :: timeStep - - character(len=*),parameter :: subname='(fv3_cap:ModelAdvance)' - character(240) :: msgString - ! character(240) :: startTime_str, currTime_str, stopTime_str, timeStep_str - !----------------------------------------------------------------------------- rc = ESMF_SUCCESS if (profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance: ") - ! Because of the way that the internal Clock was set in SetClock(), - ! its timeStep is likely smaller than the parent timeStep. As a consequence - ! the time interval covered by a single parent timeStep will result in - ! multiple calls to the ModelAdvance() routine. Every time the currTime - ! will come in by one internal timeStep advanced. This goes until the - ! stopTime of the internal Clock has been reached. - - call ESMF_ClockPrint(clock_fv3, options="currTime", & - preString="------>Advancing FV3 from: ", unit=msgString, rc=rc) + call ModelAdvance_phase1(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ModelAdvance_phase2(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -!----------------------------------------------------------------------- -!*** Use the internal Clock set by NUOPC layer for FV3 but update stopTime -!----------------------------------------------------------------------- - - ! Component internal Clock gets updated per NUOPC rules - call ESMF_GridCompGet(gcomp, clock=clock, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - ! The stopTime will be updated to be the next coupling time - call ESMF_ClockGet(clock, currTime=currTime, stopTime=stopTime, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - ! Set the coupling time to be stopTime in Clock that FV3 core uses - call ESMF_ClockSet(clock_fv3, currTime=currTime, stopTime=stopTime, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_ClockPrint(clock_fv3, options="currTime", & - preString="entering FV3_ADVANCE with clock_fv3 current: ", & - unit=msgString) - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ClockPrint(clock_fv3, options="startTime", & - preString="entering FV3_ADVANCE with clock_fv3 start: ", & - unit=msgString) - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ClockPrint(clock_fv3, options="stopTime", & - preString="entering FV3_ADVANCE with clock_fv3 stop: ", & - unit=msgString) - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - - ! call ESMF_ClockGet(clock_fv3, startTime=startTime, currTime=currTime, & - ! timeStep=timeStep, stopTime=stopTime, rc=rc) - ! if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - ! call ESMF_TimeGet(startTime, timestring=startTime_str, rc=rc) - ! call ESMF_TimeGet(currTime, timestring=currTime_str, rc=rc) - ! call ESMF_TimeGet(stopTime, timestring=stopTime_str, rc=rc) - ! call ESMF_TimeIntervalGet(timeStep, timestring=timeStep_str, rc=rc) - -! -!----------------------------------------------------------------------------- -!*** integration loop - - integrate: do while(.NOT.ESMF_ClockIsStopTime(clock_fv3, rc=rc)) - - call ModelAdvance_phase1(gcomp, rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ModelAdvance_phase2(gcomp, rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - enddo integrate -! if (profile_memory) call ESMF_VMLogMemInfo("Leaving FV3 ModelAdvance: ") end subroutine ModelAdvance @@ -979,102 +818,41 @@ subroutine ModelAdvance_phase1(gcomp, rc) ! local variables type(ESMF_Clock) :: clock - type(ESMF_TimeInterval) :: timeStep - type(ESMF_Time) :: startTime, stopTime, currTime - integer :: urc logical :: fcstpe character(len=*),parameter :: subname='(fv3_cap:ModelAdvance_phase1)' character(240) :: msgString - integer :: date(6) - !----------------------------------------------------------------------------- rc = ESMF_SUCCESS if(profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance_phase1: ") - fcstpe = .false. - if( mype < num_pes_fcst ) fcstpe = .true. - - ! Expecting to be called by NUOPC run method exactly once for every coupling - ! step. - ! Also expecting the coupling step to be identical to the timeStep for - ! clock_fv3. - - call ESMF_ClockPrint(clock_fv3, options="currTime", & - preString="------>Advancing FV3 phase1 from: ", unit=msgString, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - -!----------------------------------------------------------------------- -!*** Use the internal Clock set by NUOPC layer for FV3 but update stopTime -!----------------------------------------------------------------------- - - ! Component internal Clock gets updated per NUOPC rules call ESMF_GridCompGet(gcomp, clock=clock, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - ! The stopTime will be updated to be the next external coupling time - call ESMF_ClockGet(clock, currTime=currTime, stopTime=stopTime, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - ! Set the FV3-OCN coupling time to be stopTime in Clock that FV3 core uses - !call ESMF_ClockSet(clock_fv3, currTime=currTime, stopTime=stopTime, rc=rc) - call ESMF_ClockSet(clock_fv3, stopTime=stopTime, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_ClockPrint(clock_fv3, options="currTime", & - preString="entering FV3_ADVANCE phase1 with clock_fv3 current: ", & + call ESMF_ClockPrint(clock, options="currTime", & + preString="entering FV3_ADVANCE phase1 with clock current: ", & unit=msgString) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ClockPrint(clock_fv3, options="startTime", & - preString="entering FV3_ADVANCE phase1 with clock_fv3 start: ", & + call ESMF_ClockPrint(clock, options="startTime", & + preString="entering FV3_ADVANCE phase1 with clock start: ", & unit=msgString) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ClockPrint(clock_fv3, options="stopTime", & - preString="entering FV3_ADVANCE phase1 with clock_fv3 stop: ", & + call ESMF_ClockPrint(clock, options="stopTime", & + preString="entering FV3_ADVANCE phase1 with clock stop: ", & unit=msgString) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ClockGet(clock_fv3, startTime=startTime, currTime=currTime, & - timeStep=timeStep, stopTime=stopTime, rc=rc) + call ESMF_GridCompRun(fcstComp, exportState=fcstState, clock=clock, phase=1, userRc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - -! if(mype==0) print *,'total steps=', nint((stopTime-startTime)/timeStep) -! if(mype==lead_wrttask(1)) print *,'on wrt lead,total steps=', nint((stopTime-startTime)/timeStep) - call ESMF_TimeGet(time=stopTime,yy=date(1),mm=date(2),dd=date(3),h=date(4), & - m=date(5),s=date(6),rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! if(mype==0) print *,'af clock,stop date=',date -! if(mype==lead_wrttask(1)) print *,'on wrt lead,af clock,stop date=',date - call ESMF_TimeIntervalGet(timeStep,yy=date(1),mm=date(2),d=date(3),h=date(4), & - m=date(5),s=date(6),rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! if(mype==0) print *,'af clock,timestep date=',date -! if(mype==lead_wrttask(1)) print *,'on wrt lead,af clock,timestep date=',date - - call ESMF_LogWrite('Model Advance phase1: before fcstcomp run ', ESMF_LOGMSG_INFO, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_GridCompRun(fcstComp, exportState=fcstState, clock=clock_fv3, & - phase=1, userRc=urc, rc=rc) - if (rc /= ESMF_SUCCESS) then - if(mype==0) print *,'after fcstComp phase1 rc=',rc - endif - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return - call ESMF_LogWrite('Model Advance phase1: after fcstcomp run ', ESMF_LOGMSG_INFO, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - ! assign import_data called during phase=1 if( dbug > 0 .or. cplprint_flag ) then - call diagnose_cplFields(gcomp, clock_fv3, fcstpe, cplprint_flag, dbug, 'import') + fcstpe = .false. + if( mype < num_pes_fcst ) fcstpe = .true. + call diagnose_cplFields(gcomp, clock, fcstpe, cplprint_flag, dbug, 'import') endif if (profile_memory) call ESMF_VMLogMemInfo("Leaving FV3 ModelAdvance_phase1: ") @@ -1090,7 +868,7 @@ subroutine ModelAdvance_phase2(gcomp, rc) ! local variables type(ESMF_Time) :: currTime type(ESMF_TimeInterval) :: timeStep - type(ESMF_Time) :: startTime, stopTime + type(ESMF_Time) :: startTime type(ESMF_TimeInterval) :: time_elapsed integer :: na, i, urc @@ -1099,53 +877,45 @@ subroutine ModelAdvance_phase2(gcomp, rc) character(len=*),parameter :: subname='(fv3_cap:ModelAdvance_phase2)' character(240) :: msgString - real(kind=8) :: MPI_Wtime - real(kind=8) :: timewri, timerhi, timerh + + type(ESMF_Clock) :: clock, clock_out !----------------------------------------------------------------------------- rc = ESMF_SUCCESS + if(profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance_phase2: ") - fcstpe = .false. - if( mype < num_pes_fcst ) fcstpe = .true. -! - timewri = MPI_Wtime() - call ESMF_LogWrite('Model Advance phase2: before fcstComp run phase2', ESMF_LOGMSG_INFO, rc=rc) + call ESMF_GridCompGet(gcomp, clock=clock, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_GridCompRun(fcstComp, exportState=fcstState, clock=clock_fv3, & - phase=2, userRc=urc, rc=rc) - + call ESMF_GridCompRun(fcstComp, exportState=fcstState, clock=clock, phase=2, userRc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return - call ESMF_LogWrite('Model Advance phase2: after fcstComp run phase2', ESMF_LOGMSG_INFO, rc=rc) + clock_out = ESMF_ClockCreate(clock, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_ClockAdvance(clock = clock_fv3, rc = RC) + call ESMF_ClockAdvance(clock_out, rc = RC) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_ClockGet(clock_fv3, startTime=startTime, currTime=currTime, & - timeStep=timeStep, stopTime=stopTime, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - time_elapsed = currtime - starttime - na = nint(time_elapsed/timeStep) - call ESMF_TimeIntervalGet(time_elapsed, s=nfseconds, rc=rc) -! - if(mype==0) print *,'n fv3_cap,in model run, advance,na=',na - !------------------------------------------------------------------------------- !*** if it is output time, call data transfer and write grid comp run if( quilting ) then + call ESMF_ClockGet(clock_out, startTime=startTime, currTime=currTime, & + timeStep=timeStep, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + time_elapsed = currTime - startTime + na = nint(time_elapsed/timeStep) + call ESMF_TimeIntervalGet(time_elapsed, s=nfseconds, rc=rc) + output: if (ANY(nint(output_fh(:)*3600.0) == nfseconds)) then ! if (mype == 0 .or. mype == lead_wrttask(1)) print *,' aft fcst run output time=',nfseconds, & 'FBcount=',FBcount,'na=',na - timerhi = MPI_Wtime() call ESMF_VMEpochEnter(epoch=ESMF_VMEpoch_Buffer, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -1165,11 +935,7 @@ subroutine ModelAdvance_phase2(gcomp, rc) call ESMF_LogWrite('Model Advance: before wrtcomp run ', ESMF_LOGMSG_INFO, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - timerhi = MPI_Wtime() - call ESMF_GridCompRun(wrtComp(n_group), importState=wrtState(n_group), clock=clock_fv3,userRc=urc,rc=rc) - - timerh = MPI_Wtime() - + call ESMF_GridCompRun(wrtComp(n_group), importState=wrtState(n_group), clock=clock_out, userRc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return @@ -1186,28 +952,63 @@ subroutine ModelAdvance_phase2(gcomp, rc) endif ! quilting -!jw check clock - call ESMF_ClockPrint(clock_fv3, options="currTime", & - preString="leaving FV3_ADVANCE phase2 with clock_fv3 current: ", & + call ESMF_ClockPrint(clock, options="currTime", & + preString="leaving FV3_ADVANCE phase2 with clock current: ", & unit=msgString) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ClockPrint(clock_fv3, options="startTime", & - preString="leaving FV3_ADVANCE phase2 with clock_fv3 start: ", & + call ESMF_ClockPrint(clock, options="startTime", & + preString="leaving FV3_ADVANCE phase2 with clock start: ", & unit=msgString) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) - call ESMF_ClockPrint(clock_fv3, options="stopTime", & - preString="leaving FV3_ADVANCE phase2 with clock_fv3 stop: ", & + call ESMF_ClockPrint(clock, options="stopTime", & + preString="leaving FV3_ADVANCE phase2 with clock stop: ", & unit=msgString) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO) if( dbug > 0 .or. cplprint_flag ) then - call diagnose_cplFields(gcomp, clock_fv3, fcstpe, cplprint_flag, dbug, 'export') + fcstpe = .false. + if( mype < num_pes_fcst ) fcstpe = .true. + call diagnose_cplFields(gcomp, clock_out, fcstpe, cplprint_flag, dbug, 'export') end if if(profile_memory) call ESMF_VMLogMemInfo("Leaving FV3 ModelAdvance_phase2: ") end subroutine ModelAdvance_phase2 +!----------------------------------------------------------------------------- + + subroutine ModelSetRunClock(gcomp, rc) + + type(ESMF_GridComp) :: gcomp + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: dclock, mclock + type(ESMF_TimeInterval) :: dtimestep, mtimestep + type(ESMF_Time) :: mcurrtime, mstoptime + +!----------------------------------------------------------------------------- + + rc = ESMF_SUCCESS + + call NUOPC_ModelGet(gcomp, driverClock=dclock, modelClock=mclock, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_ClockGet(dclock, timeStep=dtimestep, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_ClockGet(mclock, currTime=mcurrtime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_TimeIntervalSet(mtimestep,s=dt_atmos,rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + mstoptime = mcurrtime + dtimestep + + call ESMF_ClockSet(mclock, timeStep=mtimestep, stopTime=mstoptime, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + end subroutine ModelSetRunClock + !----------------------------------------------------------------------------- subroutine fv3_checkimport(gcomp, rc) @@ -1230,6 +1031,8 @@ subroutine fv3_checkimport(gcomp, rc) character(esmf_maxstr) :: msgString integer :: date(6) + rc = ESMF_SUCCESS + ! query the Component for its clock call ESMF_GridCompGet(gcomp, clock=clock, importState=importState, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -1346,6 +1149,7 @@ subroutine ModelFinalize(gcomp, rc) rc = ESMF_SUCCESS ! call ESMF_GridCompGet(gcomp,vm=vm,rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! !*** finalize grid comps if( quilting ) then diff --git a/module_fcst_grid_comp.F90 b/module_fcst_grid_comp.F90 index 7d1544984..b3ef3f491 100644 --- a/module_fcst_grid_comp.F90 +++ b/module_fcst_grid_comp.F90 @@ -64,9 +64,8 @@ module module_fcst_grid_comp use module_fv3_io_def, only: num_pes_fcst, num_files, filename_base, & nbdlphys, iau_offset use module_fv3_config, only: dt_atmos, fcst_mpi_comm, fcst_ntasks, & - quilting, calendar, calendar_type, & - cplprint_flag, force_date_from_configure, & - restart_endfcst + quilting, calendar, & + cplprint_flag, restart_endfcst use get_stochy_pattern_mod, only: write_stoch_restart_atm ! @@ -87,6 +86,7 @@ module module_fcst_grid_comp !----- coupled model data ----- + integer :: calendar_type = -99 integer :: date_init(6) integer :: numLevels = 0 integer :: numSoilLayers = 0 @@ -196,6 +196,8 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) type(time_type) :: Time_init, Time, Time_step, Time_end, & Time_restart, Time_step_restart type(time_type) :: iautime + integer :: io_unit, calendar_type_res, date_res(6), date_init_res(6) + ! !----------------------------------------------------------------------- !*********************************************************************** @@ -236,28 +238,24 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) call constants_init call sat_vapor_pres_init -! - if ( force_date_from_configure ) then - - select case( uppercase(trim(calendar)) ) - case( 'JULIAN' ) - calendar_type = JULIAN - case( 'GREGORIAN' ) - calendar_type = GREGORIAN - case( 'NOLEAP' ) - calendar_type = NOLEAP - case( 'THIRTY_DAY' ) - calendar_type = THIRTY_DAY_MONTHS - case( 'NO_CALENDAR' ) - calendar_type = NO_CALENDAR - case default - call mpp_error ( FATAL, 'fcst_initialize: calendar must be one of '// & - 'JULIAN|GREGORIAN|NOLEAP|THIRTY_DAY|NO_CALENDAR.' ) - end select - endif -! - call set_calendar_type (calendar_type ) + select case( uppercase(trim(calendar)) ) + case( 'JULIAN' ) + calendar_type = JULIAN + case( 'GREGORIAN' ) + calendar_type = GREGORIAN + case( 'NOLEAP' ) + calendar_type = NOLEAP + case( 'THIRTY_DAY' ) + calendar_type = THIRTY_DAY_MONTHS + case( 'NO_CALENDAR' ) + calendar_type = NO_CALENDAR + case default + call mpp_error ( FATAL, 'fcst_initialize: calendar must be one of '// & + 'JULIAN|GREGORIAN|NOLEAP|THIRTY_DAY|NO_CALENDAR.' ) + end select + + call set_calendar_type (calendar_type) ! !----------------------------------------------------------------------- !*** set atmos time @@ -266,7 +264,6 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) call ESMF_ClockGet(clock, CurrTime=CurrTime, StartTime=StartTime, & StopTime=StopTime, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - RunDuration = StopTime - CurrTime date_init = 0 call ESMF_TimeGet (StartTime, & @@ -284,10 +281,9 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) H=date(4), M =date(5), S =date(6), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if (mype == 0) write(*,'(A,6I5)') 'CurrTime =',date - Time = set_date (date(1), date(2), date(3), & date(4), date(5), date(6)) + if (mype == 0) write(*,'(A,6I5)') 'CurrTime =',date date_end=0 call ESMF_TimeGet (StopTime, & @@ -298,7 +294,58 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) Time_end = set_date (date_end(1), date_end(2), date_end(3), & date_end(4), date_end(5), date_end(6)) if (mype == 0) write(*,'(A,6I5)') 'StopTime =',date_end -! + +!------------------------------------------------------------------------ +! If this is a restarted run ('INPUT/coupler.res' file exists), +! compare date and date_init to the values in 'coupler.res' + + if (mype == 0) then + inquire(FILE='INPUT/coupler.res', EXIST=fexist) + if (fexist) then ! file exists, this is a restart run + + call ESMF_UtilIOUnitGet(unit=io_unit, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + open(unit=io_unit, file='INPUT/coupler.res', status='old', action='read', err=998) + read (io_unit,*,err=999) calendar_type_res + read (io_unit,*) date_init_res + read (io_unit,*) date_res + close(io_unit) + + if(date_res(1) == 0 .and. date_init_res(1) /= 0) date_res = date_init_res + + if(mype == 0) write(*,'(A,6(I4))') 'INPUT/coupler.res: date_init=',date_init_res + if(mype == 0) write(*,'(A,6(I4))') 'INPUT/coupler.res: date =',date_res + + if (calendar_type /= calendar_type_res) then + write(0,'(A)') 'fcst_initialize ERROR: calendar_type /= calendar_type_res' + write(0,'(A,6(I4))')' calendar_type = ', calendar_type + write(0,'(A,6(I4))')' calendar_type_res = ', calendar_type_res + call ESMF_Finalize(endflag=ESMF_END_ABORT) + endif + + if (.not. ALL(date_init.EQ.date_init_res)) then + write(0,'(A)') 'fcst_initialize ERROR: date_init /= date_init_res' + write(0,'(A,6(I4))')' date_init = ', date_init + write(0,'(A,6(I4))')' date_init_res = ', date_init_res + call ESMF_Finalize(endflag=ESMF_END_ABORT) + endif + + if (.not. ALL(date.EQ.date_res)) then + write(0,'(A)') 'fcst_initialize ERROR: date /= date_res' + write(0,'(A,6(I4))')' date = ', date + write(0,'(A,6(I4))')' date_res = ', date_res + call ESMF_Finalize(endflag=ESMF_END_ABORT) + endif + + 999 continue + 998 continue + + endif ! fexist + endif ! mype == 0 + + RunDuration = StopTime - CurrTime + CALL ESMF_TimeIntervalGet(RunDuration, S=Run_length, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! diff --git a/module_fv3_config.F90 b/module_fv3_config.F90 index 5a27967a0..64522ec8e 100644 --- a/module_fv3_config.F90 +++ b/module_fv3_config.F90 @@ -2,7 +2,7 @@ module module_fv3_config !------------------------------------------------------------------------ ! -!*** fv3 configure variablse from model_configure +!*** fv3 configure variables from model_configure ! ! revision history ! 01/2017 Jun Wang Initial code @@ -19,12 +19,10 @@ module module_fv3_config ! logical :: cplprint_flag logical :: quilting, output_1st_tstep_rst - logical :: force_date_from_configure logical :: restart_endfcst ! real,dimension(:),allocatable :: output_fh character(esmf_maxstr),dimension(:),allocatable :: filename_base character(17) :: calendar=' ' - integer :: calendar_type = -99 ! end module module_fv3_config