diff --git a/parm/atm/berror/hybvar_bump.yaml b/parm/atm/berror/hybvar_bump.yaml index ba39a78c0..879441e46 100644 --- a/parm/atm/berror/hybvar_bump.yaml +++ b/parm/atm/berror/hybvar_bump.yaml @@ -10,7 +10,7 @@ components: active variables: &active_vars [psi,chi,t,ps,rh,liq_wat,o3mr] bump: io: - data directory: !ENV &staticb_dir ${DATA}/berror + data directory: &staticb_dir $(DATA)/berror drivers: multivariate strategy: specific_univariate read local nicas: true @@ -95,17 +95,17 @@ components: covariance model: ensemble members from template: template: - datetime: '{{BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' filetype: fms restart state variables: *3dvars datapath: ens/mem%mem%/ - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' - nmembers: !ENV ${NMEM_ENKF} + nmembers: $(NMEM_ENKF) zero padding: 3 # localization: # localization method: SABER diff --git a/parm/atm/berror/hybvar_gsibec.yaml b/parm/atm/berror/hybvar_gsibec.yaml index ce381d079..33df8ca16 100644 --- a/parm/atm/berror/hybvar_gsibec.yaml +++ b/parm/atm/berror/hybvar_gsibec.yaml @@ -8,10 +8,10 @@ components: specific_humidity,cloud_liquid_ice,cloud_liquid_water, mole_fraction_of_ozone_in_air] output variables: *control_vars - gsi akbk: !ENV &akbk ${DATA}/fv3jedi/akbk.nc4 - gsi error covariance file: !ENV &gsiberr ${DATA}/berror/gsi-coeffs-gfs-global.nc4 -# gsi error covariance file: !ENV &gsiberr ${DATA}/berror/global_berror.f77 - gsi berror namelist file: !ENV &gsibnml ${DATA}/berror/gfs_gsi_global.nml + gsi akbk: &akbk $(DATA)/fv3jedi/akbk.nc4 + gsi error covariance file: &gsiberr $(DATA)/berror/gsi-coeffs-gfs-global.nc4 +# gsi error covariance file: &gsiberr $(DATA)/berror/global_berror.f77 + gsi berror namelist file: &gsibnml $(DATA)/berror/gfs_gsi_global.nml processor layout x direction: &layout_gsib_x 3 processor layout y direction: &layout_gsib_y 2 debugging mode: false @@ -35,17 +35,17 @@ components: covariance model: ensemble members from template: template: - datetime: '{{BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' filetype: fms restart state variables: *3dvars_anal datapath: ens/mem%mem% - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' - nmembers: !ENV ${NMEM_ENKF} + nmembers: $(NMEM_ENKF) zero padding: 3 # localization: # localization method: SABER diff --git a/parm/atm/berror/staticb_bump.yaml b/parm/atm/berror/staticb_bump.yaml index 27c207d1d..8cabf7060 100644 --- a/parm/atm/berror/staticb_bump.yaml +++ b/parm/atm/berror/staticb_bump.yaml @@ -8,7 +8,7 @@ saber blocks: active variables: &active_vars [psi,chi,t,ps,rh,liq_wat,o3mr] bump: io: - data directory: !ENV &staticb_dir ${DATA}/berror + data directory: &staticb_dir $(DATA)/berror drivers: multivariate strategy: specific_univariate read local nicas: true diff --git a/parm/atm/berror/staticb_gsibec.yaml b/parm/atm/berror/staticb_gsibec.yaml index 66b98b535..ab82efe43 100644 --- a/parm/atm/berror/staticb_gsibec.yaml +++ b/parm/atm/berror/staticb_gsibec.yaml @@ -6,10 +6,10 @@ saber central block: specific_humidity,cloud_liquid_ice,cloud_liquid_water, mole_fraction_of_ozone_in_air] output variables: *control_vars - gsi akbk: !ENV &akbk ${DATA}/fv3jedi/akbk.nc4 - gsi error covariance file: !ENV &gsiberr ${DATA}/berror/gsi-coeffs-gfs-global.nc4 -# gsi error covariance file: !ENV &gsiberr ${DATA}/berror/global_berror.f77 - gsi berror namelist file: !ENV &gsibnml ${DATA}/berror/gfs_gsi_global.nml + gsi akbk: &akbk $(DATA)/fv3jedi/akbk.nc4 + gsi error covariance file: &gsiberr $(DATA)/berror/gsi-coeffs-gfs-global.nc4 +# gsi error covariance file: &gsiberr $(DATA)/berror/global_berror.f77 + gsi berror namelist file: &gsibnml $(DATA)/berror/gfs_gsi_global.nml processor layout x direction: &layout_gsib_x 3 processor layout y direction: &layout_gsib_y 2 debugging mode: false diff --git a/parm/atm/hofx/hofx4d.yaml b/parm/atm/hofx/hofx4d.yaml index 0bd231398..14b6c0e66 100644 --- a/parm/atm/hofx/hofx4d.yaml +++ b/parm/atm/hofx/hofx4d.yaml @@ -1,35 +1,35 @@ -window begin: '{{ATM_WINDOW_BEGIN}}' +window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' window length: $(ATM_WINDOW_LENGTH) forecast length: $(ATM_WINDOW_LENGTH) geometry: fms initialization: - namelist filename: !ENV ${DATA}/fv3jedi/fmsmpp.nml - field table filename: !ENV ${DATA}/fv3jedi/field_table - akbk: !ENV ${DATA}/fv3jedi/akbk.nc4 + namelist filename: $(DATA)/fv3jedi/fmsmpp.nml + field table filename: $(DATA)/fv3jedi/field_table + akbk: $(DATA)/fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: !ENV ${DATA}/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml initial condition: - datapath: !ENV ${DATA}/bkg + datapath: $(DATA)/bkg filetype: fms restart - datetime: '{{ATM_WINDOW_BEGIN}}' - filename_core: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{ATM_BEGIN_YYYYmmddHHMMSS}}.coupler.res' + datetime: '{{ ATM_WINDOW_BEGIN | to_isotime }}' + filename_core: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ ATM_WINDOW_BEGIN | to_fv3time }}.coupler.res' state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] model: name: PSEUDO - datapath: !ENV ${DATA}/bkg + datapath: $(DATA)/bkg filetype: fms restart - datetime: '{{ATM_WINDOW_BEGIN}}' + datetime: '{{ ATM_WINDOW_BEGIN | to_isotime }}' filename is datetime templated: true filename_core: '%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc' filename_trcr: '%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc' diff --git a/parm/atm/hofx/hofx_nomodel.yaml b/parm/atm/hofx/hofx_nomodel.yaml index b50d23417..f980d53be 100644 --- a/parm/atm/hofx/hofx_nomodel.yaml +++ b/parm/atm/hofx/hofx_nomodel.yaml @@ -1,26 +1,26 @@ -window begin: '{{ATM_WINDOW_BEGIN}}' +window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' window length: $(ATM_WINDOW_LENGTH) geometry: fms initialization: - namelist filename: !ENV ${DATA}/fv3jedi/fmsmpp.nml - field table filename: !ENV ${DATA}/fv3jedi/field_table - akbk: !ENV ${DATA}/fv3jedi/akbk.nc4 + namelist filename: $(DATA)/fv3jedi/fmsmpp.nml + field table filename: $(DATA)/fv3jedi/field_table + akbk: $(DATA)/fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: !ENV ${DATA}/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml state: - datapath: !ENV ${DATA}/bkg + datapath: $(DATA)/bkg filetype: fms restart - datetime: '{{BKG_ISOTIME}}' - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + datetime: '{{ current_cycle | to_isotime }}' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] diff --git a/parm/atm/lgetkf/lgetkf.yaml b/parm/atm/lgetkf/lgetkf.yaml index 226dbee55..b8a6f9886 100644 --- a/parm/atm/lgetkf/lgetkf.yaml +++ b/parm/atm/lgetkf/lgetkf.yaml @@ -4,32 +4,33 @@ geometry: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) field metadata override: ./fv3jedi/gfs-restart.yaml -window begin: '$(ATM_WINDOW_BEGIN)' -window length: PT6H +window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' +window length: $(ATM_WINDOW_LENGTH) +increment variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr] background: - datetime: '{{BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' members from template: template: - datetime: '{{BKG_ISOTIME}}' + datetime: '{{ current_cycle | to_isotime }}' filetype: fms restart state variables: [ua,va,t,DZ,delp,ps,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] datapath: bkg/mem%mem% - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' pattern: '%mem%' - nmembers: !ENV ${NMEM_ENKF} + nmembers: $(NMEM_ENKF) zero padding: 3 observations: !INC ${OBS_LIST} @@ -41,6 +42,7 @@ driver: save prior mean: false save posterior mean increment: false save posterior ensemble increments: true + do posterior observer: false local ensemble DA: solver: GETKF diff --git a/parm/atm/obs/config/aircraft.yaml b/parm/atm/obs/config/aircraft.yaml index 460af86b9..4920a90db 100644 --- a/parm/atm/obs/config/aircraft.yaml +++ b/parm/atm/obs/config/aircraft.yaml @@ -3,7 +3,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}aircraft.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)aircraft.${{ current_cycle | to_YMDH }}.nc4 obsgrouping: group variables: ["stationIdentification"] sort variable: "pressure" @@ -11,7 +11,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_aircraft_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_aircraft_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [windEastward, windNorthward, airTemperature, specificHumidity] diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index 0096f2fd0..df6c38528 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}amsua_n19.${PDY}${cyc}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)amsua_n19.{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_amsua_n19_${PDY}${cyc}.nc4 + obsfile: $(DATA)/diags/diag_amsua_n19_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -20,16 +20,16 @@ obs operator: obs options: Sensor_ID: amsua_n19 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias.${gPDY}${gcyc}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias.${PDY}${cyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &amsua_n19_tlapse !ENV ${DATA}/obs/${GPREFIX}amsua_n19.tlapse.${gPDY}${gcyc}.txt + tlapse: &amsua_n19_tlapse $(DATA)/obs/$(GPREFIX)amsua_n19.tlapse.txt - name: lapse_rate tlapse: *amsua_n19_tlapse - name: emissivity @@ -46,11 +46,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias_cov.${gPDY}${gcyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias_cov.${PDY}${cyc}.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.nc4 obs filters: - filter: BlackList filter variables: diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml index 0d9e863d5..28b19e692 100644 --- a/parm/atm/obs/config/atms_n20.yaml +++ b/parm/atm/obs/config/atms_n20.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}atms_n20.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)atms_n20.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_atms_n20_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_atms_n20_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightness_temperature] @@ -20,16 +20,16 @@ obs operator: obs options: Sensor_ID: atms_n20 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}atms_n20.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}atms_n20.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &atms_n20_tlapse !ENV ${DATA}/obs/${GPREFIX}atms_n20.tlapse.${GDATE}.txt + tlapse: &atms_n20_tlapse $(DATA)/obs/$(GPREFIX)atms_n20.tlapse.txt - name: lapse_rate tlapse: *atms_n20_tlapse - name: emissivity @@ -46,11 +46,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}atms_n20.satbias_cov.${GDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)atms_n20.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}atms_n20.satbias_cov.${CDATE}.nc4 + output file: $(DATA)/bc/$(APREFIX)atms_n20.satbias_cov.nc4 obs filters: - filter: BlackList filter variables: diff --git a/parm/atm/obs/config/cris-fsr_n20.yaml b/parm/atm/obs/config/cris-fsr_n20.yaml index 3d2a43313..579439862 100644 --- a/parm/atm/obs/config/cris-fsr_n20.yaml +++ b/parm/atm/obs/config/cris-fsr_n20.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}cris-fsr_n20.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)cris-fsr_n20.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_cris-fsr_n20_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_cris-fsr_n20_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightness_temperature] @@ -51,16 +51,16 @@ obs operator: obs options: Sensor_ID: cris-fsr_n20 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_n20.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_n20.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_n20.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_n20.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &cris-fsr_n20_tlapse !ENV ${DATA}/obs/${GPREFIX}cris-fsr_n20.tlapse.${GDATE}.txt + tlapse: &cris-fsr_n20_tlapse $(DATA)/obs/$(GPREFIX)cris-fsr_n20.tlapse.txt - name: lapse_rate tlapse: *cris-fsr_n20_tlapse - name: emissivity @@ -77,11 +77,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_n20.satbias_cov.${GDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_n20.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_n20.satbias_cov.${CDATE}.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_n20.satbias_cov.nc4 # #obs filters: ## Wavenumber Check diff --git a/parm/atm/obs/config/cris-fsr_npp.yaml b/parm/atm/obs/config/cris-fsr_npp.yaml index a3943264d..55409f454 100644 --- a/parm/atm/obs/config/cris-fsr_npp.yaml +++ b/parm/atm/obs/config/cris-fsr_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}cris-fsr_npp.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)cris-fsr_npp.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_cris-fsr_npp_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_cris-fsr_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightness_temperature] @@ -51,16 +51,16 @@ obs operator: obs options: Sensor_ID: cris-fsr_npp EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_npp.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_npp.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_npp.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_npp.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &cris-fsr_npp_tlapse !ENV ${DATA}/obs/${GPREFIX}cris-fsr_npp.tlapse.${GDATE}.txt + tlapse: &cris-fsr_npp_tlapse $(DATA)/obs/$(GPREFIX)cris-fsr_npp.tlapse.txt - name: lapse_rate tlapse: *cris-fsr_npp_tlapse - name: emissivity @@ -77,11 +77,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}cris-fsr_npp.satbias_cov.${GDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)cris-fsr_npp.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}cris-fsr_npp.satbias_cov.${CDATE}.nc4 + output file: $(DATA)/bc/$(APREFIX)cris-fsr_npp.satbias_cov.nc4 #obs filters: ## Wavenumber Check #- filter: BlackList diff --git a/parm/atm/obs/config/lgetkf_amsua_n19.yaml b/parm/atm/obs/config/lgetkf_amsua_n19.yaml index 77dff58ae..d5fe9691a 100644 --- a/parm/atm/obs/config/lgetkf_amsua_n19.yaml +++ b/parm/atm/obs/config/lgetkf_amsua_n19.yaml @@ -6,11 +6,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ./obs/${OPREFIX}amsua_n19.${PDY}${cyc}.nc4 + obsfile: ./obs/$(OPREFIX)amsua_n19.{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ./diags/diag_amsua_n19_${PDY}${cyc}.nc4 + obsfile: ./diags/diag_amsua_n19_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [brightnessTemperature] @@ -27,14 +27,14 @@ obs operator: obs error: covariance model: diagonal obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias.${gPDY}${gcyc}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias.${PDY}${cyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias.nc4 variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &amsua_n19_tlapse !ENV ${DATA}/obs/${GPREFIX}amsua_n19.tlapse.${gPDY}${gcyc}.txt + tlapse: &amsua_n19_tlapse $(DATA)/obs/$(GPREFIX)amsua_n19.tlapse.txt - name: lapse_rate tlapse: *amsua_n19_tlapse - name: emissivity @@ -51,11 +51,11 @@ obs bias: step size: 1.0e-4 largest analysis variance: 10000.0 prior: - input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias_cov.${gPDY}${gcyc}.nc4 + input file: $(DATA)/obs/$(GPREFIX)amsua_n19.satbias_cov.nc4 inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias_cov.${PDY}${cyc}.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n19.satbias_cov.nc4 obs filters: - filter: Bounds Check filter variables: diff --git a/parm/atm/obs/config/lgetkf_sondes.yaml b/parm/atm/obs/config/lgetkf_sondes.yaml index 837b54a48..4f2358097 100644 --- a/parm/atm/obs/config/lgetkf_sondes.yaml +++ b/parm/atm/obs/config/lgetkf_sondes.yaml @@ -6,11 +6,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ./obs/${OPREFIX}sondes.${PDY}${cyc}.nc4 + obsfile: ./obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ./diags/diag_sondes_lgetkf_${PDY}${cyc}.nc4 + obsfile: ./diags/diag_sondes_lgetkf_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [windEastward, windNorthward, airTemperature] diff --git a/parm/atm/obs/config/omi_aura.yaml b/parm/atm/obs/config/omi_aura.yaml index e5346e322..79973780c 100644 --- a/parm/atm/obs/config/omi_aura.yaml +++ b/parm/atm/obs/config/omi_aura.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}omi_aura.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)omi_aura.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_omi_aura_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_omi_aura_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [ozoneTotal] diff --git a/parm/atm/obs/config/ompsnp_npp.yaml b/parm/atm/obs/config/ompsnp_npp.yaml index cae446044..5ba772fb3 100644 --- a/parm/atm/obs/config/ompsnp_npp.yaml +++ b/parm/atm/obs/config/ompsnp_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ompsnp_npp.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ompsnp_npp.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ompsnp_npp_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ompsnp_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [ozoneTotal] diff --git a/parm/atm/obs/config/ompstc8_npp.yaml b/parm/atm/obs/config/ompstc8_npp.yaml index c3aee74f4..e3b11ff64 100644 --- a/parm/atm/obs/config/ompstc8_npp.yaml +++ b/parm/atm/obs/config/ompstc8_npp.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ompstc8_npp.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ompstc8_npp.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ompstc8_npp_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ompstc8_npp_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [ozoneTotal] diff --git a/parm/atm/obs/config/satwind.yaml b/parm/atm/obs/config/satwind.yaml index ebef654b4..853cbce77 100644 --- a/parm/atm/obs/config/satwind.yaml +++ b/parm/atm/obs/config/satwind.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}satwind.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)satwind.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_satwind_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_satwind_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [windEastward, windNorthward] diff --git a/parm/atm/obs/config/sfc.yaml b/parm/atm/obs/config/sfc.yaml index 32a3f1acb..31fd74c0c 100644 --- a/parm/atm/obs/config/sfc.yaml +++ b/parm/atm/obs/config/sfc.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}sfc.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)sfc.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_sfc_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_sfc_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [stationPressure] diff --git a/parm/atm/obs/config/sfcship.yaml b/parm/atm/obs/config/sfcship.yaml index c5bf1b304..c3530f404 100644 --- a/parm/atm/obs/config/sfcship.yaml +++ b/parm/atm/obs/config/sfcship.yaml @@ -3,12 +3,12 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}sfcship.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)sfcship.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File overwrite: true - obsfile: !ENV ${DATA}/diags/diag_sfcship_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_sfcship_${{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [stationPressure, airTemperature, specificHumidity] diff --git a/parm/atm/obs/config/sondes.yaml b/parm/atm/obs/config/sondes.yaml index 3131a1bf4..e9331f7bd 100644 --- a/parm/atm/obs/config/sondes.yaml +++ b/parm/atm/obs/config/sondes.yaml @@ -3,7 +3,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}sondes.${PDY}${cyc}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4 obsgrouping: group variables: ["stationIdentification"] sort variable: "pressure" @@ -11,7 +11,7 @@ obs space: obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_sondes_${PDY}${cyc}.nc4 + obsfile: $(DATA)/diags/diag_sondes_{{ current_cycle | to_YMDH }}.nc4 io pool: max pool size: 1 simulated variables: [stationPressure, airTemperature, windEastward, windNorthward, specificHumidity] diff --git a/parm/atm/obs/config/ssmis_f17.yaml b/parm/atm/obs/config/ssmis_f17.yaml index 12d96ee47..5054951ef 100644 --- a/parm/atm/obs/config/ssmis_f17.yaml +++ b/parm/atm/obs/config/ssmis_f17.yaml @@ -4,22 +4,22 @@ obs operator: obs options: Sensor_ID: ssmis_f17 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs space: name: ssmis_f17 obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ssmis_f17.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ssmis_f17.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ssmis_f17_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ssmis_f17_${{ current_cycle | to_YMDH }}.nc4 simulated variables: [brightness_temperature] channels: 1-24 obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}ssmis_f17.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}ssmis_f17.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)ssmis_f17.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)ssmis_f17.satbias.nc4 variational bc: predictors: - name: constant @@ -36,7 +36,7 @@ obs bias: - name: sine_of_latitude - name: lapse_rate order: 2 - tlapse: &ssmis_f17_tlapse !ENV ${DATA}/obs/${GPREFIX}ssmis_f17.tlapse.${GDATE}.txt + tlapse: &ssmis_f17_tlapse $(DATA)/obs/$(GPREFIX)ssmis_f17.tlapse.txt - name: lapse_rate tlapse: *ssmis_f17_tlapse - name: emissivity diff --git a/parm/atm/obs/config/ssmis_f18.yaml b/parm/atm/obs/config/ssmis_f18.yaml index 41bdf450a..2c757268a 100644 --- a/parm/atm/obs/config/ssmis_f18.yaml +++ b/parm/atm/obs/config/ssmis_f18.yaml @@ -4,22 +4,22 @@ obs operator: obs options: Sensor_ID: ssmis_f18 EndianType: little_endian - CoefficientPath: !ENV ${DATA}/crtm/ + CoefficientPath: $(DATA)/crtm/ obs space: name: ssmis_f18 obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}ssmis_f18.${CDATE}.nc4 + obsfile: $(DATA)/obs/$(OPREFIX)ssmis_f18.${{ current_cycle | to_YMDH }}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/diag_ssmis_f18_${CDATE}.nc4 + obsfile: $(DATA)/diags/diag_ssmis_f18_${{ current_cycle | to_YMDH }}.nc4 simulated variables: [brightness_temperature] channels: 1-24 obs bias: - input file: !ENV ${DATA}/obs/${GPREFIX}ssmis_f18.satbias.${GDATE}.nc4 - output file: !ENV ${DATA}/bc/${APREFIX}ssmis_f18.satbias.${CDATE}.nc4 + input file: $(DATA)/obs/$(GPREFIX)ssmis_f18.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)ssmis_f18.satbias.nc4 variational bc: predictors: - name: constant @@ -36,7 +36,7 @@ obs bias: - name: sine_of_latitude - name: lapse_rate order: 2 - tlapse: &ssmis_f18_tlapse !ENV ${DATA}/obs/${GPREFIX}ssmis_f18.tlapse.${GDATE}.txt + tlapse: &ssmis_f18_tlapse $(DATA)/obs/$(GPREFIX)ssmis_f18.tlapse.txt - name: lapse_rate tlapse: *ssmis_f18_tlapse - name: emissivity diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index 2ea6542b0..4563f45ac 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -1,6 +1,6 @@ cost function: cost type: 3D-Var - window begin: '{{ATM_WINDOW_BEGIN}}' + window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' window length: $(ATM_WINDOW_LENGTH) analysis variables: &3dvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] geometry: @@ -9,8 +9,8 @@ cost function: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) @@ -18,12 +18,12 @@ cost function: background: datapath: ./bkg filetype: fms restart - datetime: '{{BKG_ISOTIME}}' - filename_core: '{{BKG_YYYYmmddHHMMSS}}.fv_core.res.nc' - filename_trcr: '{{BKG_YYYYmmddHHMMSS}}.fv_tracer.res.nc' - filename_sfcd: '{{BKG_YYYYmmddHHMMSS}}.sfc_data.nc' - filename_sfcw: '{{BKG_YYYYmmddHHMMSS}}.fv_srf_wnd.res.nc' - filename_cplr: '{{BKG_YYYYmmddHHMMSS}}.coupler.res' + datetime: '{{ current_cycle | to_isotime }}' + filename_core: '{{ current_cycle | to_fv3time }}.fv_core.res.nc' + filename_trcr: '{{ current_cycle | to_fv3time }}.fv_tracer.res.nc' + filename_sfcd: '{{ current_cycle | to_fv3time }}.sfc_data.nc' + filename_sfcw: '{{ current_cycle | to_fv3time }}.fv_srf_wnd.res.nc' + filename_cplr: '{{ current_cycle | to_fv3time }}.coupler.res' state variables: [ua,va,t,DZ,delp,ps,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] @@ -42,8 +42,8 @@ variational: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) @@ -59,8 +59,8 @@ variational: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) @@ -81,8 +81,8 @@ final: field table filename: ./fv3jedi/field_table akbk: ./fv3jedi/akbk.nc4 layout: - - !ENV ${layout_x} - - !ENV ${layout_y} + - $(layout_x) + - $(layout_y) npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) diff --git a/test/atm/CMakeLists.txt b/test/atm/CMakeLists.txt index e429c321a..117e2ea8e 100644 --- a/test/atm/CMakeLists.txt +++ b/test/atm/CMakeLists.txt @@ -25,26 +25,6 @@ if (BUILD_GDASBUNDLE) ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}" ) - # test for ush/run_jedi_exe.py 3dhofx - add_test(NAME test_gdasapp_run_jedi_exe_3dhofx - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_3dhofx.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - - # test for ush/run_jedi_exe.py 3dvar - add_test(NAME test_gdasapp_run_jedi_exe_3dvar - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_3dvar.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - - # test for ush/run_jedi_exe.py 3denvar - add_test(NAME test_gdasapp_run_jedi_exe_3denvar - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_3denvar.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - - # test for ush/run_jedi_exe.py letkf - add_test(NAME test_gdasapp_run_jedi_exe_letkf - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/run_jedi_exe_letkf.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) - # Test exgdas scripts from the global-worflow if (WORKFLOW_TESTS) add_subdirectory(global-workflow) diff --git a/test/atm/global-workflow/CMakeLists.txt b/test/atm/global-workflow/CMakeLists.txt index c31571c58..850983f97 100644 --- a/test/atm/global-workflow/CMakeLists.txt +++ b/test/atm/global-workflow/CMakeLists.txt @@ -1,13 +1,37 @@ # test for creating an experiment directory within the global-workflow file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) +# test for ush/run_jedi_exe.py 3dhofx +add_test(NAME test_gdasapp_run_jedi_exe_3dhofx + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3dhofx.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + +# test for ush/run_jedi_exe.py 3dvar +add_test(NAME test_gdasapp_run_jedi_exe_3dvar + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3dvar.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + +# test for ush/run_jedi_exe.py 3denvar +add_test(NAME test_gdasapp_run_jedi_exe_3denvar + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_3denvar.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + +# test for ush/run_jedi_exe.py letkf +add_test(NAME test_gdasapp_run_jedi_exe_letkf + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/run_jedi_exe_letkf.sh + ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + add_test(NAME test_gdasapp_setup_atm_cycled_exp COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/setup_workflow_exp.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_var_prep - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_prep.sh +add_test(NAME test_gdasapp_atm_jjob_var_init + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_init.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) @@ -16,13 +40,13 @@ add_test(NAME test_gdasapp_atm_jjob_var_run ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_var_post - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_post.sh +add_test(NAME test_gdasapp_atm_jjob_var_final + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_var_final.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_ens_prep - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_prep.sh +add_test(NAME test_gdasapp_atm_jjob_ens_init + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_init.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) @@ -31,7 +55,7 @@ add_test(NAME test_gdasapp_atm_jjob_ens_run ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) -add_test(NAME test_gdasapp_atm_jjob_ens_post - COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_post.sh +add_test(NAME test_gdasapp_atm_jjob_ens_final + COMMAND ${PROJECT_SOURCE_DIR}/test/atm/global-workflow/jjob_ens_final.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/atm/global-workflow/testrun) diff --git a/test/atm/global-workflow/config.atmanal b/test/atm/global-workflow/config.atmanl similarity index 76% rename from test/atm/global-workflow/config.atmanal rename to test/atm/global-workflow/config.atmanl index 0ee19fb54..d7039cd8d 100755 --- a/test/atm/global-workflow/config.atmanal +++ b/test/atm/global-workflow/config.atmanl @@ -1,9 +1,9 @@ #!/bin/bash -x -########## config.atmanal ########## +########## config.atmanl ########## # configuration common to all atm analysis tasks -echo "BEGIN: config.atmanal" +echo "BEGIN: config.atmanl" export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/atm/obs/config/ export OBS_LIST=@OBS_LIST@ @@ -19,7 +19,13 @@ export R2D2_BC_DUMP='oper_gdas' export R2D2_ARCH_DB='local' export INTERP_METHOD='barycentric' +export layout_x=1 +export layout_y=1 + export io_layout_x=1 export io_layout_y=1 -echo "END: config.atmanal" +export JEDIEXE=${HOMEgfs}/exec/fv3jedi_var.x +export crtm_VERSION="2.3.0" + +echo "END: config.atmanl" diff --git a/test/atm/global-workflow/config.base.emc.dyn b/test/atm/global-workflow/config.base.emc.dyn index 47d8bbc96..1c7239931 100755 --- a/test/atm/global-workflow/config.base.emc.dyn +++ b/test/atm/global-workflow/config.base.emc.dyn @@ -40,7 +40,7 @@ export FIXreg2grb2=$HOMEgfs/fix/reg2grb2 export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops export COMINsyn="@COMINsyn@" -export DMPDIR="@DMPDIR@" +export DMPDIR=@DUMPDIR@ # USER specific paths export HOMEDIR="@HOMEDIR@" @@ -109,7 +109,7 @@ export DUMP_SUFFIX="" if [[ "$CDATE" -ge "2019092100" && "$CDATE" -le "2019110700" ]]; then export DUMP_SUFFIX="p" # Use dumps from NCO GFS v15.3 parallel fi -export DATAROOT="@DATAPATH@/RUNDIR/${jobid}" # TODO: set via prod_envir in Ops +export DATAROOT="@DATAPATH@/RUNDIR" # TODO: set via prod_envir in Ops export RUNDIR="${DATAROOT}" # TODO: Should be removed; use DATAROOT instead export ARCDIR="$NOSCRUB/archive/$PSLOT" export ICSDIR="@ICSDIR@" @@ -119,8 +119,12 @@ export ATARDIR="@ATARDIR@" export envir=${envir:-"prod"} export NET="gfs" export RUN=${RUN:-${CDUMP:-"gfs"}} -export COMIN_OBS=${DMPDIR}/${CDUMP}.${PDY}/$cyc/atmos -export COMIN_GES_OBS=${DMPDIR}/${CDUMP}.${PDY}/$cyc/atmos +CDUMP_OBS="${CDUMP}" +if [ "${CDUMP_OBS}" = "enkfgdas" ]; then + CDUMP_OBS="gdas" +fi +export COMIN_OBS=${DMPDIR}/${CDUMP_OBS}.${PDY}/$cyc/atmos +export COMIN_GES_OBS=${DMPDIR}/${CDUMP_OBS}.${PDY}/$cyc/atmos export COMINatmos=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos export COMOUTatmos=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos export COMINwave=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/wave diff --git a/test/atm/global-workflow/config.yaml b/test/atm/global-workflow/config.yaml index b406303a7..f3c699f8b 100644 --- a/test/atm/global-workflow/config.yaml +++ b/test/atm/global-workflow/config.yaml @@ -4,5 +4,6 @@ base: DO_JEDIENS: "YES" UTILROOT: /apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2 DATAPATH: "@bindir@/test/atm/global-workflow/testrun" -atmanal: + DUMPDIR: "@dumpdir@" +atmanl: OBS_LIST: "@srcdir@/test/atm/global-workflow/gdas_prototype.yaml" diff --git a/test/atm/global-workflow/gdas_prototype.yaml b/test/atm/global-workflow/gdas_prototype.yaml index fd867bcd0..18d64e54e 100644 --- a/test/atm/global-workflow/gdas_prototype.yaml +++ b/test/atm/global-workflow/gdas_prototype.yaml @@ -1,6 +1,6 @@ observers: - !INC ${OBS_YAML_DIR}/amsua_n19.yaml -##- !INC ${OBS_YAML_DIR}/sondes.yaml +- !INC ${OBS_YAML_DIR}/sondes.yaml ##- !INC ${OBS_YAML_DIR}/atms_n20.yaml ##- !INC ${OBS_YAML_DIR}/aircraft.yaml ##- !INC ${OBS_YAML_DIR}/satwind.yaml diff --git a/test/atm/global-workflow/jjob_var_post.sh b/test/atm/global-workflow/jjob_ens_final.sh similarity index 65% rename from test/atm/global-workflow/jjob_var_post.sh rename to test/atm/global-workflow/jjob_ens_final.sh index 815220e88..b1197631d 100755 --- a/test/atm/global-workflow/jjob_var_post.sh +++ b/test/atm/global-workflow/jjob_ens_final.sh @@ -12,19 +12,27 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS -export CDUMP=gdas +export RUN=enkfgdas +export CDUMP=enkfgdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} export jobid=$pid export COMROOT=$DATAROOT +export NMEM_ENKF=3 export ACCOUNT=da-cpu +export DO_JEDIENS="YES" + +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_POST + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_prep.sh b/test/atm/global-workflow/jjob_ens_init.sh similarity index 61% rename from test/atm/global-workflow/jjob_ens_prep.sh rename to test/atm/global-workflow/jjob_ens_init.sh index b1b0a3c3f..fd27980be 100755 --- a/test/atm/global-workflow/jjob_ens_prep.sh +++ b/test/atm/global-workflow/jjob_ens_init.sh @@ -12,7 +12,8 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS -export CDUMP=gdas +export RUN=enkfgdas +export CDUMP=enkfgdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} @@ -21,6 +22,11 @@ export COMROOT=$DATAROOT export NMEM_ENKF=3 export ACCOUNT=da-cpu +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') @@ -28,23 +34,29 @@ machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"' GDATE=`date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - 6 hours"` gPDY=$(echo $GDATE | cut -c1-8) gcyc=$(echo $GDATE | cut -c9-10) -mkdir -p $ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/atmos/ + +mkdir -p $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ +flist="satbias tlapse" +for file in $flist; do + cp -r $GDASAPP_TESTDATA/lowres/gdas.$gPDY/$gcyc/atmos/*${file}* $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ +done # Copy tiled ges and atmf006 files to ROTDIR +mkdir -p $ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/atmos/ for imem in $(seq 1 $NMEM_ENKF); do memchar="mem"$(printf %03i $imem) - source=$GDASAPP_TESTDATA/lowres/enkfgdas.$gPDY/$gcyc/atmos/$memchar - target=$ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/atmos/$memchar + source=$GDASAPP_TESTDATA/lowres/enkfgdas.$gPDY/$gcyc/$memchar/atmos + target=$ROTDIR/gdas_test/enkfgdas.$gPDY/$gcyc/$memchar/atmos mkdir -p $target rm -rf $target/RESTART ln -fs $source/RESTART $target/ - ln -fs $source/gdas.t${gcyc}z.atmf006.nc $target/ + ln -fs $source/enkfgdas.t${gcyc}z.atmf006.nc $target/ done if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_PREP + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_run.sh b/test/atm/global-workflow/jjob_ens_run.sh index 35f7d5ec7..d3a5899ab 100755 --- a/test/atm/global-workflow/jjob_ens_run.sh +++ b/test/atm/global-workflow/jjob_ens_run.sh @@ -15,7 +15,8 @@ export gPDY=20210323 export gcyc=12 export GDATE=${gPDY}${gcyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS -export CDUMP=gdas +export RUN=enkfgdas +export CDUMP=enkfgdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} @@ -25,13 +26,18 @@ export NMEM_ENKF=3 export ACCOUNT=da-cpu export DO_JEDIENS="YES" +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine = 'HERA' -o $machine = 'ORION' ]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=debug --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN + sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=debug --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN else - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_RUN + ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN fi diff --git a/test/atm/global-workflow/jjob_ens_post.sh b/test/atm/global-workflow/jjob_var_final.sh similarity index 70% rename from test/atm/global-workflow/jjob_ens_post.sh rename to test/atm/global-workflow/jjob_var_final.sh index 06c8b81b3..c3e61e7a0 100755 --- a/test/atm/global-workflow/jjob_ens_post.sh +++ b/test/atm/global-workflow/jjob_var_final.sh @@ -12,21 +12,25 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS +export RUN=gdas export CDUMP=gdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg export pid=${pid:-$$} export jobid=$pid export COMROOT=$DATAROOT -export NMEM_ENKF=3 export ACCOUNT=da-cpu -export DO_JEDIENS="YES" + +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST + ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ENSANAL_POST + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_var_prep.sh b/test/atm/global-workflow/jjob_var_init.sh similarity index 75% rename from test/atm/global-workflow/jjob_var_prep.sh rename to test/atm/global-workflow/jjob_var_init.sh index c4d2a89ee..c0b875502 100755 --- a/test/atm/global-workflow/jjob_var_prep.sh +++ b/test/atm/global-workflow/jjob_var_init.sh @@ -12,6 +12,7 @@ export PDY=20210323 export cyc=18 export CDATE=${PDY}${cyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS +export RUN=gdas export CDUMP=gdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg @@ -20,6 +21,11 @@ export jobid=$pid export COMROOT=$DATAROOT export ACCOUNT=da-cpu +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') @@ -29,13 +35,13 @@ gPDY=$(echo $GDATE | cut -c1-8) gcyc=$(echo $GDATE | cut -c9-10) mkdir -p $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ -flist="abias atmf006 RESTART" +flist="abias atmf006 satbias tlapse RESTART" for file in $flist; do cp -r $GDASAPP_TESTDATA/lowres/gdas.$gPDY/$gcyc/atmos/*${file}* $ROTDIR/gdas_test/gdas.$gPDY/$gcyc/atmos/ done if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP + ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE else - sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_PREP + sbatch -n 1 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_var_run.sh b/test/atm/global-workflow/jjob_var_run.sh index 30da673b8..eb44fa707 100755 --- a/test/atm/global-workflow/jjob_var_run.sh +++ b/test/atm/global-workflow/jjob_var_run.sh @@ -15,6 +15,7 @@ export gPDY=20210323 export gcyc=12 export GDATE=${gPDY}${gcyc} export ROTDIR=$bindir/test/atm/global-workflow/testrun/ROTDIRS +export RUN=gdas export CDUMP=gdas export DATAROOT=$bindir/test/atm/global-workflow/testrun/RUNDIR export COMIN_GES=${bindir}/test/atm/bkg @@ -23,11 +24,16 @@ export jobid=$pid export COMROOT=$DATAROOT export ACCOUNT=da-cpu +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # detemine machine from config.base machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') if [ $machine != 'HERA' ]; then - ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN + ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN else - sbatch -n 6 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGDAS_GLOBAL_ATMOS_ANALYSIS_RUN + sbatch -n 6 --account=$ACCOUNT --qos=debug --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN fi diff --git a/test/atm/run_jedi_exe_3denvar.sh b/test/atm/global-workflow/run_jedi_exe_3denvar.sh similarity index 89% rename from test/atm/run_jedi_exe_3denvar.sh rename to test/atm/global-workflow/run_jedi_exe_3denvar.sh index 3f716dc42..faa6d09dc 100755 --- a/test/atm/run_jedi_exe_3denvar.sh +++ b/test/atm/global-workflow/run_jedi_exe_3denvar.sh @@ -30,6 +30,13 @@ elif [ "$machine" = "orion" ]; then gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" fi +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # Create test run directory mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3denvar cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3denvar diff --git a/test/atm/run_jedi_exe_3dhofx.sh b/test/atm/global-workflow/run_jedi_exe_3dhofx.sh similarity index 88% rename from test/atm/run_jedi_exe_3dhofx.sh rename to test/atm/global-workflow/run_jedi_exe_3dhofx.sh index 84a9b057a..f55fb6415 100755 --- a/test/atm/run_jedi_exe_3dhofx.sh +++ b/test/atm/global-workflow/run_jedi_exe_3dhofx.sh @@ -30,6 +30,13 @@ elif [ "$machine" = "orion" ]; then gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" fi +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # Create test run directory mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_hofx3d cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_hofx3d diff --git a/test/atm/run_jedi_exe_3dvar.sh b/test/atm/global-workflow/run_jedi_exe_3dvar.sh similarity index 89% rename from test/atm/run_jedi_exe_3dvar.sh rename to test/atm/global-workflow/run_jedi_exe_3dvar.sh index ffc50816e..d333db3cc 100755 --- a/test/atm/run_jedi_exe_3dvar.sh +++ b/test/atm/global-workflow/run_jedi_exe_3dvar.sh @@ -30,6 +30,13 @@ elif [ "$machine" = "orion" ]; then gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" fi +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # Create test run directory mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3dvar cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_3dvar diff --git a/test/atm/run_jedi_exe_letkf.sh b/test/atm/global-workflow/run_jedi_exe_letkf.sh similarity index 89% rename from test/atm/run_jedi_exe_letkf.sh rename to test/atm/global-workflow/run_jedi_exe_letkf.sh index 6039ebcf6..287b67641 100755 --- a/test/atm/run_jedi_exe_letkf.sh +++ b/test/atm/global-workflow/run_jedi_exe_letkf.sh @@ -30,6 +30,13 @@ elif [ "$machine" = "orion" ]; then gdasfix="/work2/noaa/da/cmartin/GDASApp/fix" fi +# Setup python path for workflow utilities and tasks +export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +echo $HOMEgfs +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + # Create test run directory mkdir -p ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_letkf cd ${bindir}/test/atm/global-workflow/testrun/gdas_single_test_letkf diff --git a/test/atm/global-workflow/setup_workflow_exp.sh b/test/atm/global-workflow/setup_workflow_exp.sh index bedd46ab8..8d18fc6e5 100755 --- a/test/atm/global-workflow/setup_workflow_exp.sh +++ b/test/atm/global-workflow/setup_workflow_exp.sh @@ -24,12 +24,13 @@ rm -rf $comrot $expdir config # copy config.yaml to local config cp -r $configdir config cp $srcdir/test/atm/global-workflow/config.base.emc.dyn config/ -cp $srcdir/test/atm/global-workflow/config.atmanal config/ +cp $srcdir/test/atm/global-workflow/config.atmanl config/ cp $srcdir/test/atm/global-workflow/config.yaml . # update paths in config.yaml sed -i -e "s~@bindir@~${bindir}~g" config.yaml sed -i -e "s~@srcdir@~${srcdir}~g" config.yaml +sed -i -e "s~@dumpdir@~${GDASAPP_TESTDATA}/lowres~g" config.yaml # run the script ln -sf $srcdir/../../workflow/setup_expt.py . @@ -50,4 +51,8 @@ $srcdir/../../workflow/setup_expt.py cycled --idate $idate \ --expdir $expdir \ --yaml $expdir/../config.yaml +echo " " +echo "$expdir/../config.yaml is" +cat $expdir/../config.yaml + exit $? diff --git a/test/soca/CMakeLists.txt b/test/soca/CMakeLists.txt index c9e3efe66..cbbdd3bb6 100644 --- a/test/soca/CMakeLists.txt +++ b/test/soca/CMakeLists.txt @@ -87,7 +87,7 @@ add_test(NAME test_gdasapp_soca_ana_prep WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/3dvar) set_tests_properties(test_gdasapp_soca_ana_prep PROPERTIES - ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}") + ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_SOURCE_DIR}/../../ush/python/pygw/src:$ENV{PYTHONPATH}") # Test exgdas scripts from the global-worflow if (WORKFLOW_TESTS) diff --git a/ush/run_jedi_exe.py b/ush/run_jedi_exe.py index 6bebd4d01..4c4693f6e 100755 --- a/ush/run_jedi_exe.py +++ b/ush/run_jedi_exe.py @@ -7,6 +7,8 @@ import subprocess import sys import yaml +import pygw +from pygw.yaml_file import parse_j2yaml, save_as_yaml def export_envar(yamlfile, bashout): @@ -77,6 +79,8 @@ def run_jedi_exe(yamlconfig): gprefix = executable_subconfig['dump'] + ".t" + str(gcyc) + "z." comin = executable_subconfig.get('gdas_fix_root', './') comin_ges_ens = os.path.join(comin, 'cases', 'enkfgdas.' + str(gPDY), str(gcyc)) + dump = executable_subconfig['dump'] + output_file = os.path.join(workdir, f"gdas_{app_mode}.yaml") single_exec = True var_config = { @@ -112,6 +116,8 @@ def run_jedi_exe(yamlconfig): 'valid_time': f"{valid_time.strftime('%Y-%m-%dT%H:%M:%SZ')}", 'window_begin': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}", 'prev_valid_time': f"{prev_cycle.strftime('%Y-%m-%dT%H:%M:%SZ')}", + 'previous_cycle': f"{prev_cycle}", + 'current_cycle': f"{valid_time}", 'atm_window_length': executable_subconfig['atm_window_length'], 'ATM_WINDOW_LENGTH': f"PT{assim_freq}H", 'ATM_WINDOW_BEGIN': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}", @@ -127,24 +133,52 @@ def run_jedi_exe(yamlconfig): executable_subconfig['atm_window_length']), 'BKG_TSTEP': executable_subconfig.get('forecast_step', 'PT6H'), 'INTERP_METHOD': executable_subconfig.get('interp_method', 'barycentric'), + 'output_file': os.path.join(workdir, f"gdas_{app_mode}.yaml"), + 'dump': f"{dump}", + 'output_file': f"{output_file}", } - output_file = os.path.join(workdir, f"gdas_{app_mode}.yaml") + # set some environment variables os.environ['PARMgfs'] = os.path.join(all_config_dict['GDASApp home'], 'parm') for key, value in var_config.items(): os.environ[key] = str(value) # generate YAML for executable based on input config logging.info(f'Using yamlconfig {yamlconfig}') - genYAML(yamlconfig, output=output_file) + + local_dict = { + 'npx_ges': f"{int(os.environ['CASE'][1:]) + 1}", + 'npy_ges': f"{int(os.environ['CASE'][1:]) + 1}", + 'npz_ges': f"{int(os.environ['LEVS']) - 1}", + 'npz': f"{int(os.environ['LEVS']) - 1}", + 'npx_anl': f"{int(os.environ['CASE_ANL'][1:]) + 1}", + 'npy_anl': f"{int(os.environ['CASE_ANL'][1:]) + 1}", + 'npz_anl': f"{int(os.environ['LEVS']) - 1}", + 'NMEM_ENKF': f"{int(os.environ['NMEM_ENKF'])}", + 'ATM_WINDOW_BEGIN': window_begin, + 'ATM_WINDOW_LENGTH': f"PT{assim_freq}H", + 'BKG_TSTEP': executable_subconfig.get('forecast_step', 'PT6H'), + 'OPREFIX': f"{dump}.t{cyc}z.", # TODO: CDUMP is being replaced by RUN + 'APREFIX': f"{dump}.t{cyc}z.", # TODO: CDUMP is being replaced by RUN + 'GPREFIX': f"gdas.t{gcyc}z.", + 'DATA': os.path.join(workdir), + 'layout_x': str(executable_subconfig['layout_x']), + 'layout_y': str(executable_subconfig['layout_y']), + 'previous_cycle': prev_cycle, + 'current_cycle': valid_time, + } + + varda_yaml = parse_j2yaml(all_config_dict['template'], local_dict) + save_as_yaml(varda_yaml, output_file) + logging.info(f'Wrote YAML file to {output_file}') # use R2D2 to stage backgrounds, obs, bias correction files, etc. if app_mode in ['variational', 'hofx']: - ufsda.stage.gdas_single_cycle(var_config) + ufsda.stage.gdas_single_cycle(var_config, local_dict) # stage ensemble backgrouns for letkf if app_mode in ['letkf']: ufsda.stage.background_ens(var_config) - ufsda.stage.atm_obs(var_config) - ufsda.stage.bias_obs(var_config) + ufsda.stage.atm_obs(var_config, local_dict) + ufsda.stage.bias_obs(var_config, local_dict) # link additional fix files needed (CRTM, fieldmetadata, etc.) gdasfix = executable_subconfig['gdas_fix_root'] diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index b6863ab5f..0be2e9562 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -12,7 +12,7 @@ import logging import glob import numpy as np -from pygw.yaml_file import YAMLFile +from pygw.yaml_file import YAMLFile, parse_j2yaml import ufsda.soca_utils __all__ = ['atm_background', 'atm_obs', 'bias_obs', 'background', 'background_ens', 'fv3jedi', 'obs', 'berror', 'gdas_fix', 'gdas_single_cycle'] @@ -146,7 +146,7 @@ def atm_background(config): ufsda.r2d2.fetch(r2d2_config) -def atm_obs(config): +def atm_obs(config, local_dict): # fetch atm analysis obs r2d2_config = { 'start': config['prev_valid_time'], @@ -159,22 +159,25 @@ def atm_obs(config): r2d2_config = NiceDict(r2d2_config) # get list of obs to process and their output files obs_list_yaml = config['OBS_LIST'] - obs_list_config = YAMLFile(path=obs_list_yaml) + obs_list_config = Configuration(obs_list_yaml) + obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) + for ob in obs_list_config['observers']: + ob_config = parse_j2yaml(ob, local_dict) # first get obs r2d2_config.pop('file_type', None) r2d2_config['type'] = 'ob' r2d2_config['provider'] = 'ncdiag' r2d2_config['start'] = config['ATM_WINDOW_BEGIN'] r2d2_config['end'] = r2d2_config['start'] - ob_basename = os.path.basename(ob['obs space']['obsdatain']['engine']['obsfile']) + ob_basename = os.path.basename(ob_config['obs space']['obsdatain']['engine']['obsfile']) target_file = os.path.join(os.environ['COMOUT'], ob_basename) r2d2_config['target_file_fmt'] = target_file - r2d2_config['obs_types'] = [ob['obs space']['name']] + r2d2_config['obs_types'] = [ob_config['obs space']['name']] ufsda.r2d2.fetch(r2d2_config) -def bias_obs(config): +def bias_obs(config, local_dict): # fetch bias files r2d2_config = { 'start': config['prev_valid_time'], @@ -187,12 +190,15 @@ def bias_obs(config): r2d2_config = NiceDict(r2d2_config) # get list of obs to process and their output files obs_list_yaml = config['OBS_LIST'] - obs_list_config = YAMLFile(path=obs_list_yaml) + obs_list_config = Configuration(obs_list_yaml) + obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) + for ob in obs_list_config['observers']: + ob_config = parse_j2yaml(ob, local_dict) r2d2_config.pop('file_type', None) - r2d2_config['obs_types'] = [ob['obs space']['name']] + r2d2_config['obs_types'] = [ob_config['obs space']['name']] # get bias files if needed - if 'obs bias' in ob.keys(): + if 'obs bias' in ob_config.keys(): r2d2_config['type'] = 'bc' r2d2_config['provider'] = 'gsi' r2d2_config['start'] = config['prev_valid_time'] @@ -200,7 +206,7 @@ def bias_obs(config): # fetch satbias r2d2_config['file_type'] = 'satbias' - target_file = ob['obs bias']['input file'] + target_file = ob_config['obs bias']['input file'] ob_basename = os.path.basename(target_file) target_file = os.path.join(os.environ['COMOUT'], ob_basename) r2d2_config['target_file_fmt'] = target_file @@ -230,7 +236,7 @@ def bias_obs(config): ufsda.r2d2.fetch(r2d2_config) -def gdas_single_cycle(config): +def gdas_single_cycle(config, local_dict): # grab backgrounds first r2d2_config = { 'start': config['prev_valid_time'], @@ -267,8 +273,9 @@ def gdas_single_cycle(config): obs_list_yaml = config['OBS_LIST'] obs_list_config = Configuration(obs_list_yaml) obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) + for ob in obs_list_config['observers']: - ob_config = YAMLFile(path=ob) + ob_config = parse_j2yaml(ob, local_dict) # first get obs r2d2_config.pop('file_type', None) r2d2_config['type'] = 'ob'