First pass at subsetter.py

examples/subset/airnow_wrfchem.yaml

@@ -0,0 +1,114 @@
+analysis:
+  start_time: '2019-09-05-06:00:00' #UTC
+  end_time:   '2019-09-06-06:00:00' #UTC
+  output_dir: $HOME/Data/melodies_monet_output
+  debug: True
+
+
+model:
+  # Regional Atmospheric Chemistry Model/Earth System Research Lab
+  WRF-Chem:
+    # files: $HOME/Data/wrfchem_example/racm_esrl/wrfout_d01_2019*
+    files: $HOME/Data/wrfchem_example/racm_esrl/subset_wrfout_d01_2019*
+    required_vars:
+      - Times
+      - HGT
+      - PSFC
+      - P
+      - PB
+      - PH
+      - PHB
+      - XTIME
+      - XLAT
+      - XLONG
+      - T
+      - QVAPOR
+    mod_type: 'wrfchem'
+    mod_kwargs:
+      mech: 'racm_esrl_vcp'
+    radius_of_influence: 12000 # meters
+    mapping: # model species name : obs species name
+      AirNow:
+        PM2_5_DRY: "PM2.5"
+        o3: "OZONE"
+    projection: ~
+    plot_kwargs:
+      color: 'magenta'
+      marker: 's'
+      linestyle: '-'
+
+
+obs:
+  AirNow:
+    use_airnow: True
+    filename: $HOME/Data/surface_obs/AIRNOW_20190901_20190930.nc
+    obs_type: pt_sfc
+    variables:
+      PM2.5:
+        unit_scale: 1
+        unit_scale_method: '*'
+        nan_value: -1.0
+        ylabel_plot: 'PM2.5 (ug/m3)'
+        ty_scale: 2.0
+        vmin_plot: 0.0
+        vmax_plot: 22.0
+        vdiff_plot: 15.0
+        nlevels_plot: 23
+      OZONE:
+        unit_scale: 1
+        unit_scale_method: '*'
+        nan_value: -1.0
+        ylabel_plot: 'Ozone (ppbv)'
+        vmin_plot: 15.0
+        vmax_plot: 55.0
+        vdiff_plot: 20.0
+        nlevels_plot: 21
+
+
+plots:
+
+  plot_timeseries_group:
+    type: 'timeseries'
+    fig_kwargs:
+      figsize: [12, 6]
+    default_plot_kwargs:
+      linewidth: 2.0
+      markersize: 10
+    text_kwargs:
+      fontsize: 24
+    domain_type: ['all','state_name','epa_region']
+    domain_name: ['CONUS','CA','R9']
+    data: ['AirNow_WRF-Chem']
+    data_proc:
+      rem_obs_nan: True
+      ts_select_time: 'time_local'
+      ts_avg_window: 'H'
+      set_axis: True
+
+  plot_overlay_group:
+    type: 'spatial_overlay'
+    fig_kwargs:
+      states: True
+      figsize: [10, 5]
+    text_kwargs:
+      fontsize: 16
+    domain_type: ['all','state_name','epa_region']
+    domain_name: ['CONUS','CA','R9']
+    data: ['AirNow_WRF-Chem']
+    data_proc:
+      rem_obs_nan: True
+      set_axis: True
+
+  plot_bias_group:
+    type: 'spatial_bias'
+    fig_kwargs:
+      states: True
+      figsize: [10, 5]
+    text_kwargs:
+      fontsize: 16
+    domain_type: ['all','state_name','epa_region']
+    domain_name: ['CONUS','CA','R9']
+    data: ['AirNow_WRF-Chem']
+    data_proc:
+      rem_obs_nan: True
+      set_axis: True

examples/subset/carma.yaml

@@ -0,0 +1,85 @@
+analysis:
+  start_time: '2019-06'
+  end_time:   '2019-08'
+  freq: 'MS'
+  output_dir: $HOME/Plots/CARMA
+
+obs:
+  MOD08_M3:
+    data_format: gridded_eos
+    datadir: $HOME/Data/MOD08_M3
+    filestr: MOD08_M3.AYYYYDDD.061.*.hdf
+    # filestr: MOD08_M3.AYYYYDDD.061.*.nc
+    variables:
+      AOD_550_Dark_Target_Deep_Blue_Combined_Mean_Mean:
+        fillvalue: -9999
+        scale: 0.001
+        units: none
+        plot_params:
+          name: 'Terra MODIS AOD 550 nm  YYYY M_ABBR'
+          range_min: 0.0
+          range_max: 0.7
+          nlevel: 15
+
+model:
+  MAM4:
+    # datadir: /glade/campaign/acom/acom-climate/tilmes/CARMA/...
+    data_format: cesm
+    datadir: $HOME/Data/CARMA/FCnudged_f09.mam.mar27.2000_2021.001/atm/hist
+    filestr: FCnudged_f09.mam.mar27.2000_2021.001.cam.h0.YYYY-MM.nc
+    variables:
+      AODdnMODE1:
+        plot_params:
+          name: 'MAM4 Accumulation Mode AOD 550 nm  YYYY M_ABBR'
+          range_min: 0.0
+          range_max: 0.7
+          nlevel: 15
+      AODdnMODE2:
+        plot_params:
+          name: 'MAM4 Aitken Mode AOD 550 nm  YYYY M_ABBR'
+          range_min: 0.0
+          range_max: 0.1
+          nlevel: 11
+      AODdnMODE3:
+        plot_params:
+          name: 'MAM4 Coarse Mode AOD 550 nm  YYYY M_ABBR'
+          range_min: 0.0
+          range_max: 0.7
+          nlevel: 15
+      AODVIS:
+        plot_params:
+          name: 'MAM4 AOD 550 nm  YYYY M_ABBR'
+          range_min: 0.0
+          range_max: 0.7
+          nlevel: 15
+    combine:
+      AODVIS:
+        - AODdnMODE1
+        - AODdnMODE2
+        - AODdnMODE3
+    mapping:
+      MOD08_M3:
+        AODVIS: AOD_550_Dark_Target_Deep_Blue_Combined_Mean_Mean
+
+  CARMA:
+    # datadir: /glade/scratch/tilmes/archive/
+    datadir: $HOME/Data/CARMA/FCnudged_f09.carma_trop_stratv05.2000_2020.001/atm/hist
+    data_format: cesm
+    filestr: FCnudged_f09.carma_trop_stratv05.2000_2020.001.cam.h0.YYYY-MM.nc
+    variables:
+      AODVIS:
+        plot_params:
+          name: 'CARMA AOD 550 nm  YYYY M_ABBR'
+          range_min: 0.0
+          range_max: 0.7
+          nlevel: 15
+    mapping:
+      MOD08_M3:
+        AODVIS: AOD_550_Dark_Target_Deep_Blue_Combined_Mean_Mean
+
+
+# plots:
+#     spatial:
+#         monthly_climo:
+
+#    time_series:

examples/subset/run.py

@@ -0,0 +1,13 @@
+from melodies_monet import driver
+
+an = driver.analysis()
+an.control = 'airnow_wrfchem.yaml'
+an.read_control()
+
+an.open_models()
+
+an.open_obs()
+
+an.pair_data()
+
+an.plotting()

scripts/subsetter.py

@@ -0,0 +1,73 @@
+# Copyright (C) 2022 National Center for Atmospheric Research and National Oceanic and Atmospheric Administration
+# SPDX-License-Identifier: Apache-2.0
+#
+import os
+import sys
+import subprocess
+import argparse
+import logging
+import yaml
+from glob import glob
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--control', type=str,
+    default='control.yaml',
+    help='yaml control file')
+parser.add_argument('--logfile', type=str,
+    default=sys.stdout,
+    help='log file (default: stdout)')
+parser.add_argument('--outdir', type=str,
+    default=None,
+    help='output directory')
+parser.add_argument('--compress_level', type=int,
+    default=1,
+    help='NetCDF compression level')
+parser.add_argument('--debug', action='store_true',
+    help='set logging level to debug')
+args = parser.parse_args()
+
+# Setup logging
+logging_level = logging.DEBUG if args.debug else logging.INFO
+logging.basicConfig(stream=args.logfile, level=logging_level)
+
+# Read YAML control
+with open(args.control, 'r') as f:
+    control = yaml.safe_load(f)
+
+logging.debug(control)
+
+# set compression level
+level_str = str(args.compress_level)
+
+for model in control['model']:
+    logging.info('processing:' + model)
+
+    var_str = '-v '
+
+    if 'required_vars' in control['model'][model]:
+        for var in control['model'][model]['required_vars']:
+            var_str += var + ','
+
+    for var in control['model'][model]['variables']:
+        var_str += var + ','
+
+    var_str = var_str.strip(',')
+    logging.info(var_str)
+
+    file_str = os.path.expandvars(control['model'][model]['files'])
+
+    files = sorted(glob(file_str))
+    print(files)
+
+    for file_in in files:
+        file_subdirs = list(os.path.split(file_in))
+        file_subdirs[-1] = 'subset_' + file_subdirs[-1]
+        if args.outdir is not None:
+            file_out = os.path.join(args.outdir, file_subdirs[-1])
+        else:
+            file_out = os.path.join(*tuple(file_subdirs))
+        command = f'ncks -O -L {level_str} {var_str} {file_in} {file_out}'
+        logging.info(command)
+        # os.system(command)
+        subprocess.run(command.split())
+