Add support for global aerosol DA in JCB (#33)

Add in templates so that global aerosol DA can use JCB to generate YAML
configuration for the variational application + B matrix supporting
executables.

algorithm/aero/aero_convert_background.yaml.j2

@@ -0,0 +1,48 @@
+input geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_ges }}
+  npy: {{ aero_npy_ges }}
+  npz: {{ aero_npz_ges }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+output geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_anl }}
+  npy: {{ aero_npy_anl }}
+  npz: {{ aero_npz_anl }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+states:
+- input:
+    datetime: '{{ aero_background_error_time_iso }}'
+    filetype: fms restart
+    state variables: [mass_fraction_of_sulfate_in_air,
+                        mass_fraction_of_hydrophobic_black_carbon_in_air,
+                        mass_fraction_of_hydrophilic_black_carbon_in_air,
+                        mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                        mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                        mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                        mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                        mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                        mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                        mass_fraction_of_sea_salt004_in_air]
+    datapath: {{ aero_background_path }}
+    filename_core: '{{ aero_background_error_time_fv3 }}.fv_core.res.nc'
+    filename_trcr: '{{ aero_background_error_time_fv3 }}.fv_tracer.res.nc'
+    filename_cplr: '{{ aero_background_error_time_fv3 }}.coupler.res'
+  output:
+    filetype: fms restart
+    datapath: {{ aero_background_path }}
+    filename_core: 'anlres.fv_core.res.nc'
+    filename_trcr: 'anlres.fv_tracer.res.nc'
+    filename_cplr: 'anlres.coupler.res'

algorithm/aero/aero_gen_bmatrix_diagb.yaml.j2

@@ -0,0 +1,81 @@
+geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_anl }}
+  npy: {{ aero_npy_anl }}
+  npz: {{ aero_npz_anl }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+date: '{{ aero_background_error_time_iso }}'
+background:
+  datetime: '{{ aero_background_error_time_iso }}'
+  filetype: fms restart
+  state variables: [mass_fraction_of_sulfate_in_air,
+                    mass_fraction_of_hydrophobic_black_carbon_in_air,
+                    mass_fraction_of_hydrophilic_black_carbon_in_air,
+                    mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                    mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                    mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                    mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                    mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                    mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                    mass_fraction_of_sea_salt004_in_air]
+  datapath: {{ aero_background_path }}
+  filename_core: '{{ aero_background_error_time_fv3 }}.anlres.fv_core.res.nc'
+  filename_trcr: '{{ aero_background_error_time_fv3 }}.anlres.fv_tracer.res.nc'
+  filename_cplr: '{{ aero_background_error_time_fv3 }}.anlres.coupler.res'
+background error:
+  filetype: fms restart
+  datapath: {{ aero_standard_deviation_path }}
+  filename_trcr: stddev.fv_tracer.res.nc
+  filename_cplr: stddev.coupler.res
+
+climate background error:
+  geometry:
+    fms initialization:
+        namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+        field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+    akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+    layout:
+    - {{ aero_layout_x }}
+    - {{ aero_layout_y }}
+    npx: {{ aero_npx_clim_b }}
+    npy: {{ aero_npy_clim_b }}
+    npz: {{ aero_npz_clim_b }}
+    field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+  climate background error stddev:
+    filetype: fms restart
+    skip coupler file: true
+    datapath: {{ aero_climatological_b_path }}
+    filename_trcr: stddev.fv_tracer.res.nc
+    filename_cplr: stddev.coupler.res
+  diagb weight: {{ aero_diagb_weight }}
+  staticb rescaling factor: {{ aero_diagb_static_rescale_factor }}
+
+variables:
+  name:
+  - mass_fraction_of_sulfate_in_air
+  - mass_fraction_of_hydrophobic_black_carbon_in_air
+  - mass_fraction_of_hydrophilic_black_carbon_in_air
+  - mass_fraction_of_hydrophobic_organic_carbon_in_air
+  - mass_fraction_of_hydrophilic_organic_carbon_in_air
+  - mass_fraction_of_dust001_in_air
+  - mass_fraction_of_dust002_in_air
+  - mass_fraction_of_dust003_in_air
+  - mass_fraction_of_dust004_in_air
+  - mass_fraction_of_dust005_in_air
+  - mass_fraction_of_sea_salt001_in_air
+  - mass_fraction_of_sea_salt002_in_air
+  - mass_fraction_of_sea_salt003_in_air
+  - mass_fraction_of_sea_salt004_in_air
+
+rescale: {{ aero_diagb_rescale_factor }}    # rescales the filtered std. dev. by "rescale"
+number of halo points: {{ aero_diagb_n_halo }}
+number of neighbors: {{ aero_diagb_n_neighbors }}
+simple smoothing:
+  horizontal iterations: {{ aero_diagb_smooth_horiz_iter }}
+  vertical iterations: {{ aero_diagb_smooth_vert_iter }}

algorithm/aero/aero_gen_bmatrix_diffusion.yaml.j2

@@ -0,0 +1,48 @@
+geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_anl }}
+  npy: {{ aero_npy_anl }}
+  npz: {{ aero_npz_anl }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+date: '{{ aero_background_error_time_iso }}'
+background:
+  datetime: '{{ aero_background_error_time_iso }}'
+  filetype: fms restart
+  state variables: [mass_fraction_of_sulfate_in_air,
+                    mass_fraction_of_hydrophobic_black_carbon_in_air,
+                    mass_fraction_of_hydrophilic_black_carbon_in_air,
+                    mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                    mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                    mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                    mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                    mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                    mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                    mass_fraction_of_sea_salt004_in_air]
+  datapath: {{ aero_background_path }}
+  filename_core: '{{ aero_background_error_time_fv3 }}.anlres.fv_core.res.nc'
+  filename_trcr: '{{ aero_background_error_time_fv3 }}.anlres.fv_tracer.res.nc'
+  filename_cplr: '{{ aero_background_error_time_fv3 }}.anlres.coupler.res'
+background error:
+  covariance model: SABER
+  saber central block:
+    saber block name: diffusion
+    calibration:
+      normalization:
+        iterations: {{ aero_diffusion_iter }}
+      groups:
+      - horizontal:
+          fixed value: {{ aero_diffusion_horiz_len }}
+        write:
+          filepath: "{{ aero_berror_diffusion_directory }}/diffusion_hz"
+      - vertical:
+          levels: {{ aero_npz_ges }}
+          fixed value: {{ aero_diffusion_fixed_val }}
+          as gaussian: true
+        write:
+          filepath: "{{ aero_berror_diffusion_directory }}/diffusion_vt"

model/aero/aero_3dfgat_outer_loop_1.yaml.j2

@@ -0,0 +1 @@
+aero_3dvar_outer_loop_1.yaml.j2

model/aero/aero_3dfgat_outer_loop_2.yaml.j2

@@ -0,0 +1 @@
+aero_3dvar_outer_loop_2.yaml.j2

model/aero/aero_3dvar_outer_loop_1.yaml.j2

@@ -0,0 +1,17 @@
+- ninner: 35
+  gradient norm reduction: 1e-10
+  test: on
+  geometry:
+    fms initialization:
+      namelist filename: {{aero_fv3jedi_files_path}}/fmsmpp.nml
+      field table filename: {{aero_fv3jedi_files_path}}/field_table
+    akbk: {{aero_fv3jedi_files_path}}/akbk.nc4
+    layout:
+    - {{aero_layout_x}}
+    - {{aero_layout_y}}
+    npx: {{aero_npx_anl}}
+    npy: {{aero_npy_anl}}
+    npz: {{aero_npz_anl}}
+    field metadata override: {{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml
+  diagnostics:
+    departures: bkgmob

model/aero/aero_3dvar_outer_loop_2.yaml.j2

@@ -0,0 +1,17 @@
+- ninner: 35
+  gradient norm reduction: 1e-10
+  test: on
+  geometry:
+    fms initialization:
+      namelist filename: {{aero_fv3jedi_files_path}}/fmsmpp.nml
+      field table filename: {{aero_fv3jedi_files_path}}/field_table
+    akbk: {{aero_fv3jedi_files_path}}/akbk.nc4
+    layout:
+    - {{aero_layout_x}}
+    - {{aero_layout_y}}
+    npx: {{aero_npx_anl}}
+    npy: {{aero_npy_anl}}
+    npz: {{aero_npz_anl}}
+    field metadata override: {{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml
+  diagnostics:
+    departures: bkgmob1

model/aero/aero_background.yaml.j2

@@ -0,0 +1,10 @@
+datapath: {{ aero_background_path }}
+filetype: fms restart
+datetime: '{{ aero_background_time_iso }}'
+filename is datetime templated: true
+filename_core: '%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc'
+filename_trcr: '%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc'
+filename_cplr: '%yyyy%mm%dd.%hh%MM%ss.coupler.res'
+state variables: [t,delp,sphum,so4,bc1,bc2,oc1,oc2,
+                    dust1,dust2,dust3,dust4,dust5,
+                    seas1,seas2,seas3,seas4]

model/aero/aero_background_error_static_diffusion.yaml.j2

@@ -0,0 +1,33 @@
+covariance model: SABER
+saber central block:
+  saber block name: diffusion 
+  read:
+    groups:
+    - variables: [mass_fraction_of_sulfate_in_air,
+                       mass_fraction_of_hydrophobic_black_carbon_in_air,
+                       mass_fraction_of_hydrophilic_black_carbon_in_air,
+                       mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                       mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                       mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                       mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                       mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                       mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                       mass_fraction_of_sea_salt004_in_air]
+      horizontal:
+        filepath: "{{aero_berror_data_directory}}/diffusion_hz"
+      vertical:
+        levels: {{aero_npz_anl}} 
+        filepath: "{{aero_berror_data_directory}}/diffusion_vt" 
+saber outer blocks:
+- saber block name: StdDev
+  read:
+    model file:
+      datetime: "{{ aero_cycle_time_iso }}"
+      set datetime on read: true
+      filetype: fms restart
+      psinfile: true
+      datapath: "{{aero_berror_data_directory}}"
+      filename_core: '{{ aero_cycle_time_fv3 }}.stddev.fv_core.res.nc'
+      filename_trcr: '{{ aero_cycle_time_fv3 }}.stddev.fv_tracer.res.nc'
+      filename_cplr: '{{ aero_cycle_time_fv3 }}.stddev.coupler.res'
+      date: "{{ aero_cycle_time_iso }}" 

model/aero/aero_final_increment_cubed_sphere.yaml.j2

@@ -0,0 +1,23 @@
+geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{aero_layout_x}}
+  - {{aero_layout_y}}
+  npx: {{aero_npx_anl}}
+  npy: {{aero_npy_anl}}
+  npz: {{aero_npz_anl}}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+output:
+  state component:
+    datapath: {{ aero_analysis_path }}
+    prefix: aeroinc
+    filetype: fms restart
+    filename_core: '{{ aero_cycle_time_fv3 }}.fv_core.res.nc'
+    filename_trcr: '{{ aero_cycle_time_fv3 }}.fv_tracer.res.nc'
+    filename_cplr: '{{ aero_cycle_time_fv3 }}.coupler.res'
+    state variables: [t,delp,sphum,so4,bc1,bc2,oc1,oc2,
+                      dust1,dust2,dust3,dust4,dust5,
+                      seas1,seas2,seas3,seas4]

model/aero/aero_final_increment_gaussian.yaml.j2

@@ -0,0 +1,18 @@
+grid type: regular gaussian
+local interpolator type: atlas interpolator
+interpolation method:
+  type: finite-element
+number of latitude gridpoints: {{ aero_npy_ges - 1 }}
+variables to output: [mass_fraction_of_sulfate_in_air,
+                       mass_fraction_of_hydrophobic_black_carbon_in_air,
+                       mass_fraction_of_hydrophilic_black_carbon_in_air,
+                       mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                       mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                       mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                       mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                       mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                       mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                       mass_fraction_of_sea_salt004_in_air]
+all model levels: true
+datapath: {{ aero_analysis_path }}
+prefix: aeroinc_gauss

model/aero/aero_geometry_background.yaml.j2

@@ -0,0 +1,11 @@
+fms initialization:
+  namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+  field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+layout:
+- {{aero_layout_x}}
+- {{aero_layout_y}}
+npx: {{aero_npx_ges}}
+npy: {{aero_npy_ges}}
+npz: {{aero_npz_ges}}
+field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"

model/aero/aero_model_pseudo.yaml.j2

@@ -0,0 +1,13 @@
+name: PSEUDO
+datapath: {{ aero_background_path}}
+filetype: fms restart
+filename is datetime templated: true
+filename_core: '%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc'
+filename_trcr: '%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc'
+filename_sfcd: '%yyyy%mm%dd.%hh%MM%ss.sfc_data.nc'
+filename_sfcw: '%yyyy%mm%dd.%hh%MM%ss.fv_srf_wnd.res.nc'
+filename_cplr: '%yyyy%mm%dd.%hh%MM%ss.coupler.res'
+tstep: {{aero_forecast_timestep}}
+model variables: [t,delp,sphum,so4,bc1,bc2,oc1,oc2,
+                      dust1,dust2,dust3,dust4,dust5,
+                      seas1,seas2,seas3,seas4]

observations/aero/viirs_n20_aod.yaml.j2

@@ -0,0 +1,63 @@
+-
+
+  # Observation Space (I/O)
+  # -----------------------
+  obs space:
+    name: viirs_n20_aod
+    obsdatain:
+      engine:
+        type: H5File
+        obsfile: "{{aero_obsdatain_path}}/{{aero_obsdatain_prefix}}{{observation_from_jcb}}{{aero_obsdatain_suffix}}"
+    obsdataout:
+      engine:
+        type: H5File
+        obsfile: "{{aero_obsdataout_path}}/{{aero_obsdataout_prefix}}{{observation_from_jcb}}{{aero_obsdataout_suffix}}"
+    io pool:
+      max pool size: 1
+    simulated variables: [aerosolOpticalDepth]
+    channels: 4
+
+  # Observation Operator
+  # --------------------
+  get values:
+    interpolation method: barycentric
+    time interpolation: linear
+  obs operator:
+    name: AodCRTM
+    Absorbers: [H2O,O3]
+    obs options:
+      Sensor_ID: &{{observation_from_jcb}}_sensor_id v.viirs-m_j1
+      EndianType: little_endian
+      AerosolOption: aerosols_gocart_default
+      CoefficientPath: "{{crtm_coefficient_path}}"
+  obs error:
+    covariance model: diagonal
+
+  # Observation Bias Correction (VarBC)
+  # -----------------------------------
+
+  # Observation Filters (QC)
+  # ------------------------  
+  obs filters:
+    - filter: PreQC
+      maxvalue: 1
+    - filter: Domain Check
+      where:
+      - variable:
+          name: latitude@MetaData
+        minvalue: -60
+        maxvalue: 60
+    - filter: Bounds Check
+      filter variables:
+      - name: aerosolOpticalDepth
+        channels: 4
+      minvalue: 0
+      maxvalue: 4.9
+      action:
+        name: reject
+    - filter: Background Check
+      channels: 4
+      threshold: 3.0
+      action:
+        name: inflate error
+        inflation factor: 3.0