From acf1273f913d16e6bedee5b9067758a9ff451797 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Thu, 1 Jun 2023 14:39:58 +0000 Subject: [PATCH 1/2] saving working version of driver --- melodies_monet/driver.py | 153 +++++++++++++++++++++++++++++++-------- 1 file changed, 123 insertions(+), 30 deletions(-) diff --git a/melodies_monet/driver.py b/melodies_monet/driver.py index e55f9a74..06219096 100644 --- a/melodies_monet/driver.py +++ b/melodies_monet/driver.py @@ -160,7 +160,7 @@ def open_obs(self, time_interval=None): except ValueError: print('something happened opening file') - def open_sat_obs(self): + def open_sat_obs(self,time_interval=None): """Methods to opens satellite data observations. Uses in-house python code to open and load observations. Alternatively may use the satpy reader. @@ -171,11 +171,16 @@ def open_sat_obs(self): Fills the object class associated with the equivalent label (self.label) with satellite observation dataset read in from the associated file (self.file) by the satellite file reader """ + from .util import time_interval_subset as tsub + try: if self.label == 'omps_l3': self.obj = mio.sat._omps_l3_mm.read_OMPS_l3(self.file) elif self.label == 'omps_nm': - self.obj = mio.sat._omps_nadir_mm.read_OMPS_nm(self.file) + if time_interval is not None: + flst = tsub.subset_OMPS_l2(self.file,time_interval) + else: flst = self.file + self.obj = mio.sat._omps_nadir_mm.read_OMPS_nm(flst) elif self.label == 'mopitt_l3': print('Reading MOPITT') self.obj = mio.sat._mopitt_l3_mm.read_mopittdataset(self.file, 'column') @@ -308,9 +313,7 @@ def glob_files(self): # add option to read list of files from text file if 'txt' in self.file_str: with open(self.file_str,'r') as f: - self.files = f.read().split(' \n')[:-1] - - # self.files = sort(self.file_str) + self.files = f.read().split() if self.file_vert_str is not None: self.files_vert = sort(glob(self.file_vert_str)) @@ -337,6 +340,7 @@ def open_model_files(self, time_interval=None): ------- None """ + from .util import time_interval_subset as tsub print(self.model.lower()) self.glob_files() @@ -388,11 +392,27 @@ def open_model_files(self, time_interval=None): self.obj = mio.models._cesm_se_mm.open_mfdataset(self.files,**self.mod_kwargs) #self.obj, self.obj_scrip = read_cesm_se.open_mfdataset(self.files,**self.mod_kwargs) #self.obj.monet.scrip = self.obj_scrip + # MEB: addition for personal local copy to work with development UFS-RAQMS + # not intended for public versions + elif 'fv3raqms' in self.model.lower(): + if len(self.files) > 1: + self.obj = mio.models.fv3raqms.open_mfdataset(self.files,**self.mod_kwargs) + else: + self.obj = mio.models.fv3raqms.open_dataset(self.files) + self.obj = self.obj.rename({'sfcp':'surfpres_pa','dpm':'dp_pa','pdash':'pres_pa_mid'}) + self.obj['surfpres_pa'] *= 100 + self.obj['dp_pa'] *= 100 elif 'raqms' in self.model.lower(): + if time_interval is not None: + # fill filelist with subset + file_sublist = tsub.subset_model_filelist(self.files,'%m_%d_%Y_%H','6H',time_interval) + else: + # fill filelist with all files + file_sublist = self.files if len(self.files) > 1: - self.obj = mio.raqms.open_mfdataset(self.files,**self.mod_kwargs) + self.obj = mio.raqms.open_mfdataset(file_sublist,**self.mod_kwargs) else: - self.obj = mio.raqms.open_dataset(self.files,**self.mod_kwargs) + self.obj = mio.raqms.open_dataset(file_sublist,**self.mod_kwargs) else: print('**** Reading Unspecified model output. Take Caution...') if len(self.files) > 1: @@ -462,6 +482,11 @@ def __init__(self): self.debug = False self.save = None self.read = None + '''options for regrid from obsgrid to user-specified. + Defaults to 1x1degree, single timestep''' + self.usergrid_ntime = 1 + self.usergrid_nlon = 360 + self.usergrid_nlat = 180 def __repr__(self): return ( @@ -542,7 +567,12 @@ def read_control(self, control=None): self.time_intervals \ = [[time_stamps[n], time_stamps[n+1]] for n in range(len(time_stamps)-1)] - + if 'usergrid_ntime' in self.control_dict['analysis'].keys(): + self.usergrid_ntime = self.control_dict['analysis']['usergrid_ntime'] + if 'usergrid_nlat' in self.control_dict['analysis'].keys(): + self.usergrid_nlat = self.control_dict['analysis']['usergrid_nlat'] + if 'usergrid_nlon' in self.control_dict['analysis'].keys(): + self.usergrid_nlon = self.control_dict['analysis']['usergrid_nlon'] # Enable Dask progress bars? (default: false) enable_dask_progress_bars = self.control_dict["analysis"].get( "enable_dask_progress_bars", False) @@ -601,8 +631,13 @@ def read_analysis(self): read_saved_data(analysis=self,filenames=self.read[attr]['filenames'], method='pkl', attr=attr) elif self.read[attr]['method']=='netcdf': read_saved_data(analysis=self,filenames=self.read[attr]['filenames'], method='netcdf', attr=attr) + if attr == 'paired': + # initialize model/obs attributes, since needed for plotting and stats + self.open_models(load_files=False) + self.open_obs(load_files=False) + - def open_models(self, time_interval=None): + def open_models(self, time_interval=None,load_files=True): """Open all models listed in the input yaml file and create a :class:`model` object for each of them, populating the :attr:`models` dict. @@ -610,7 +645,8 @@ def open_models(self, time_interval=None): __________ time_interval (optional, default None) : [pandas.Timestamp, pandas.Timestamp] If not None, restrict models to datetime range spanned by time interval [start, end]. - + load_files (optional, default True): boolean + If False, only populate :attr: dict with yaml file parameters and do not open model files. Returns ------- None @@ -631,6 +667,7 @@ def open_models(self, time_interval=None): m.radius_of_influence = self.control_dict['model'][mod]['radius_of_influence'] else: m.radius_of_influence = 1e6 + # this initial_file/last_file stuff should be removed. Should move to internal thing for satellite data mapping. if 'initial_file' in self.control_dict['model'][mod].keys(): m.initial_file = self.control_dict['model'][mod]['initial_file'] else: m.initial_file = False @@ -670,10 +707,11 @@ def open_models(self, time_interval=None): raise ValueError( '"Scrip_file" must be provided for unstructured grid output!' ) # open the model - m.open_model_files(time_interval=time_interval) + if load_files: + m.open_model_files(time_interval=time_interval) self.models[m.label] = m - def open_obs(self, time_interval=None): + def open_obs(self, time_interval=None, load_files=True): """Open all observations listed in the input yaml file and create an :class:`observation` instance for each of them, populating the :attr:`obs` dict. @@ -682,8 +720,9 @@ def open_obs(self, time_interval=None): __________ time_interval (optional, default None) : [pandas.Timestamp, pandas.Timestamp] If not None, restrict obs to datetime range spanned by time interval [start, end]. - - + load_files (optional, default True): boolean + If False, only populate :attr: dict with yaml file parameters and do not open obs files. + Returns ------- None @@ -700,11 +739,12 @@ def open_obs(self, time_interval=None): o.debug = self.control_dict['obs'][obs]['debug'] if 'variables' in self.control_dict['obs'][obs].keys(): o.variable_dict = self.control_dict['obs'][obs]['variables'] - if o.obs_type == 'pt_sfc': - o.open_obs(time_interval=time_interval) - elif o.obs_type in ['sat_swath_sfc', 'sat_swath_clm', 'sat_grid_sfc',\ - 'sat_grid_clm', 'sat_swath_prof']: - o.open_sat_obs() + if load_files: + if o.obs_type == 'pt_sfc': + o.open_obs(time_interval=time_interval) + elif o.obs_type in ['sat_swath_sfc', 'sat_swath_clm', 'sat_grid_sfc',\ + 'sat_grid_clm', 'sat_swath_prof']: + o.open_sat_obs(time_interval=time_interval) self.obs[o.label] = o def pair_data(self, time_interval=None): @@ -793,6 +833,7 @@ def pair_data(self, time_interval=None): paired_data = paired_data.where((paired_data.o3vmr > 0)) p = pair() + p.type = obs.obs_type p.obs = obs.label p.model = mod.label p.model_vars = keys @@ -809,6 +850,7 @@ def pair_data(self, time_interval=None): # combine model and observations into paired dataset obs_dat['o3vmr'] = (['time','x','y'],model_obsgrid.sel(time=slice(self.start_time.date(),self.end_time.date())).data) p = pair() + p.type = obs.obs_type p.obs = obs.label p.model = mod.label p.model_vars = keys @@ -826,6 +868,57 @@ def concat_pairs(self): """ pass + def regrid_paired(self,pair_label): + '''Re-grid observation and model pair data to specified grid + ''' + from .util import grid_util + import numpy as np + import pandas as pd + import xarray as xr + + paired_ds_dims = self.paired[pair_label].obj.dims + obs_time = pd.to_datetime(self.paired[pair_label].obj['time']) + + grid,edge,time_stamps = grid_util.generate_uniform_grid(paired_ds_dims,self.control_dict['analysis']['start_time'], + self.control_dict['analysis']['end_time'], + obs_time,self.usergrid_ntime,self.usergrid_nlat,self.usergrid_nlon) + pair_ds = self.paired[pair_label].obj + vlst = list(pair_ds.variables.keys()) + # remove lat,lon,time from variable listing to keep only paired variables + vlst.remove('latitude') + vlst.remove('longitude') + vlst.remove('time') + + lons,lats = np.meshgrid(grid['longitude'],grid['latitude']) + usergridded = xr.Dataset({},coords={'latitude':(['x','y'],lats), + 'longitude':(['x','y'],lons), + 'time':(['time'],pd.to_datetime(grid['time'],unit='s'))}) + for v in vlst: + # initialize count and data arrays + count_grid = np.zeros((self.usergrid_ntime, self.usergrid_nlat, self.usergrid_nlon), + dtype=np.int32) + data_grid = np.zeros((self.usergrid_ntime, self.usergrid_nlat, self.usergrid_nlon), + dtype=np.float32) + grid_util.update_data_grid(edge['time_edges'], edge['lat_edges'], edge['lon_edges'], + time_stamps.flatten(), pair_ds['latitude'].data.flatten(), pair_ds['longitude'].data.flatten(), + pair_ds[v].data.flatten(),count_grid, data_grid) + usergridded['not_norm_{}'.format(v)] = (['time','x','y'],data_grid) + # normalize data + grid_util.normalize_data_grid(count_grid, data_grid) + print(data_grid.shape) + print(usergridded.dims) + usergridded[v] = (['time','x','y'],data_grid) + usergridded['counts_{}'.format(v)] = (['time','x','y'],count_grid) + p = pair() + p.type = self.paired[pair_label].type + p.obs = self.paired[pair_label].obs + p.model = self.paired[pair_label].model + p.model_vars = self.paired[pair_label].model_vars + p.obs_vars = self.paired[pair_label].obs_vars + p.obj = usergridded + label = '{}_grid'.format(pair_label) + self.paired[label] = p + ### TODO: Create the plotting driver (most complicated one) # def plotting(self): def plotting(self): @@ -844,8 +937,8 @@ def plotting(self): ------- None """ - obs_to_pair = list(self.models[(list(self.models.keys()))[0]].mapping.keys())[0] - if self.obs[obs_to_pair].obs_type.lower() == 'pt_sfc': + pair_keys = list(self.paired.keys()) + if self.paired[pair_keys[0]].type.lower() == 'pt_sfc': from .plots import surfplots as splots,savefig else: from .plots import satplots as splots,savefig @@ -868,7 +961,7 @@ def plotting(self): # first get the observational obs labels pair1 = self.paired[list(self.paired.keys())[0]] obs_vars = pair1.obs_vars - + obs_type = pair1.type # loop through obs variables for obsvar in obs_vars: # Loop also over the domain types. So can easily create several overview and zoomed in plots. @@ -889,22 +982,22 @@ def plotting(self): if obsvar == modvar: modvar = modvar + '_new' - # for pt_sft data, convert to pandas dataframe, format, and trim - if self.obs[obs_to_pair].obs_type.lower() == 'pt_sfc': + # for pt_sfc data, convert to pandas dataframe, format, and trim + if obs_type == 'pt_sfc': # convert to dataframe pairdf_all = p.obj.to_dataframe(dim_order=["time", "x"]) # Select only the analysis time window. pairdf_all = pairdf_all.loc[self.start_time : self.end_time] # keep data in xarray, fix formatting, and trim - elif self.obs[obs_to_pair].obs_type.lower() in ["sat_swath_sfc", "sat_swath_clm", + elif obs_type in ["sat_swath_sfc", "sat_swath_clm", "sat_grid_sfc", "sat_grid_clm", "sat_swath_prof"]: # convert index to time; setup for sat_swath_clm - if 'time' not in p.obj.dims and self.obs[obs_to_pair].obs_type.lower() == 'sat_swath_clm': + if 'time' not in p.obj.dims and obs_type == 'sat_swath_clm': pairdf_all = p.obj.swap_dims({'x':'time'}) # squash lat/lon dimensions into single dimension - elif self.obs[obs_to_pair].obs_type.lower() == 'sat_grid_clm': + elif obs_type == 'sat_grid_clm': pairdf_all = p.obj.stack(ll=['x','y']) pairdf_all = pairdf_all.rename_dims({'ll':'y'}) else: @@ -975,13 +1068,13 @@ def plotting(self): pairdf_all.query(domain_type + ' == ' + '"' + domain_name + '"', inplace=True) # Drop NaNs if using pandas - if self.obs[obs_to_pair].obs_type.lower() == 'pt_sfc': + if obs_type == 'pt_sfc': if grp_dict['data_proc']['rem_obs_nan'] == True: # I removed drop=True in reset_index in order to keep 'time' as a column. pairdf = pairdf_all.reset_index().dropna(subset=[modvar, obsvar]) else: pairdf = pairdf_all.reset_index().dropna(subset=[modvar]) - elif self.obs[obs_to_pair].obs_type.lower() in ["sat_swath_sfc", "sat_swath_clm", + elif obs_type in ["sat_swath_sfc", "sat_swath_clm", "sat_grid_sfc", "sat_grid_clm", "sat_swath_prof"]: # xarray doesn't need nan drop because its math operations seem to ignore nans @@ -1055,7 +1148,7 @@ def plotting(self): vmin = None vmax = None # Select time to use as index. - if self.obs[obs_to_pair].obs_type.lower() == 'pt_sfc': + if obs_type == 'pt_sfc': pairdf = pairdf.set_index(grp_dict['data_proc']['ts_select_time']) a_w = grp_dict['data_proc']['ts_avg_window'] if p_index == 0: From af0d8da7ffa27d9f18d2a48c5a2dcb24f5f2fc12 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Fri, 16 Jun 2023 20:44:29 +0000 Subject: [PATCH 2/2] time-chunking implementation and various omps-nm pairing fixes --- ...net-example-omps_nm_raqms-time_chunk.ipynb | 125 ++++++++++++++++++ examples/jupyter_notebooks/raqms-files.txt | 124 +++++++++++++++++ examples/yaml/control_omps_nm-raqms.yaml | 85 ++++++++++++ melodies_monet/driver.py | 113 ++++------------ melodies_monet/util/satellite_utilities.py | 36 ++--- melodies_monet/util/time_interval_subset.py | 33 +++++ 6 files changed, 413 insertions(+), 103 deletions(-) create mode 100644 examples/jupyter_notebooks/Monet-example-omps_nm_raqms-time_chunk.ipynb create mode 100644 examples/jupyter_notebooks/raqms-files.txt create mode 100644 examples/yaml/control_omps_nm-raqms.yaml create mode 100644 melodies_monet/util/time_interval_subset.py diff --git a/examples/jupyter_notebooks/Monet-example-omps_nm_raqms-time_chunk.ipynb b/examples/jupyter_notebooks/Monet-example-omps_nm_raqms-time_chunk.ipynb new file mode 100644 index 00000000..1d049578 --- /dev/null +++ b/examples/jupyter_notebooks/Monet-example-omps_nm_raqms-time_chunk.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "51def9a8-dcf7-489c-a9e6-bc78886a7917", + "metadata": {}, + "source": [ + "## Implementation of processing over time intervals\n", + "\n", + "Testing of Melodies-Monet OMPS Nadir Mapper L2 pairing with time interval chunking" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64e61a73-ace6-4041-a194-fe26e7a6b126", + "metadata": {}, + "outputs": [], + "source": [ + "from melodies_monet import driver" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17e4ebe7-7b5e-4628-919d-7275dd088478", + "metadata": {}, + "outputs": [], + "source": [ + "an = driver.analysis()\n", + "an.control = '../yaml/control_omps_nm-raqms.yaml'\n", + "an.read_control()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2bc2dc2d-5297-4827-aa3e-90052ea4b64c", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "base_prefix = an.save['paired']['prefix']#.copy()\n", + "for t in an.time_intervals:\n", + " an.open_models(time_interval=t)\n", + " an.open_obs(time_interval=t)\n", + " \n", + " an.pair_data()\n", + " \n", + " # adjust saved name for file to include time interval bounds\n", + " an.save['paired']['prefix'] = base_prefix+'_'+t[0].strftime('%Y%m%d')+'_'+t[1].strftime('%Y%m%d')\n", + " an.save_analysis()" + ] + }, + { + "cell_type": "markdown", + "id": "a6ba95ad-f73a-4b68-8ae5-7858eee6c970", + "metadata": {}, + "source": [ + "Notes regarding satellite pairing methods:\n", + "- some additional development needed to deal with time chuncking. Some OMPS NM orbit files cross the day. This doesn't cause issues if it is within the specified time_interval, but data past the time_interval in the file will be dropped and currently will not be read in during the next time_interval\n", + "\n", + "- Satellite pairing bilinearly interpolates model data in time to satellite observation times. When observations are before (after) the first (last) model file, time interpolation is nearest-neighbor and only the first (last) file is used. Right now the time-interpolation does not take into account if processing is being done over time chunks. Impact should be minimal, as observations have been filtered to be within time_intervals and the model file subsetter should be selecting all the necessary files." + ] + }, + { + "cell_type": "markdown", + "id": "8ddb4e95-1db0-4f01-b433-2d279b1d25fe", + "metadata": {}, + "source": [ + "### Read in saved paired data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce25d669-166a-42a0-861e-ceef59b26d34", + "metadata": {}, + "outputs": [], + "source": [ + "an = driver.analysis()\n", + "an.control = 'control_omps_nm-raqms.yaml'\n", + "an.read_control()\n", + "an.read_analysis()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f41b0217-c3cf-4cde-996e-d6149cea5aa5", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "#an.paired['omps_nm_raqms'].obj = an.paired['omps_nm_raqms'].obj.swap_dims({'time':'x'})\n", + "an.paired['omps_nm_raqms'].obj['o3vmr'].values[np.isnan(an.paired['omps_nm_raqms'].obj['ozone_column'].values)] = np.nan\n", + "an.paired['omps_nm_raqms'].obj['o3vmr'].values[an.paired['omps_nm_raqms'].obj['o3vmr'].values < 50] = np.nan\n", + "an.paired['omps_nm_raqms'].obj['ozone_column'].values[np.isnan(an.paired['omps_nm_raqms'].obj['o3vmr'].values)] = np.nan\n", + "an.plotting()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dev_monet", + "language": "python", + "name": "develop_monet" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/jupyter_notebooks/raqms-files.txt b/examples/jupyter_notebooks/raqms-files.txt new file mode 100644 index 00000000..2b1eb7fe --- /dev/null +++ b/examples/jupyter_notebooks/raqms-files.txt @@ -0,0 +1,124 @@ +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190801/uwhyb_08_01_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190801/uwhyb_08_02_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190801/uwhyb_08_02_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190801/uwhyb_08_02_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190802/uwhyb_08_02_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190802/uwhyb_08_03_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190802/uwhyb_08_03_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190802/uwhyb_08_03_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190803/uwhyb_08_03_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190803/uwhyb_08_04_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190803/uwhyb_08_04_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190803/uwhyb_08_04_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190804/uwhyb_08_04_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190804/uwhyb_08_05_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190804/uwhyb_08_05_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190804/uwhyb_08_05_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190805/uwhyb_08_05_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190805/uwhyb_08_06_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190805/uwhyb_08_06_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190805/uwhyb_08_06_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190806/uwhyb_08_06_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190806/uwhyb_08_07_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190806/uwhyb_08_07_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190806/uwhyb_08_07_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190807/uwhyb_08_07_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190807/uwhyb_08_08_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190807/uwhyb_08_08_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190807/uwhyb_08_08_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190808/uwhyb_08_08_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190808/uwhyb_08_09_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190808/uwhyb_08_09_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190808/uwhyb_08_09_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190809/uwhyb_08_09_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190809/uwhyb_08_10_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190809/uwhyb_08_10_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190809/uwhyb_08_10_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190810/uwhyb_08_10_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190810/uwhyb_08_11_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190810/uwhyb_08_11_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190810/uwhyb_08_11_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190811/uwhyb_08_11_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190811/uwhyb_08_12_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190811/uwhyb_08_12_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190811/uwhyb_08_12_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190812/uwhyb_08_12_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190812/uwhyb_08_13_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190812/uwhyb_08_13_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190812/uwhyb_08_13_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190813/uwhyb_08_13_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190813/uwhyb_08_14_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190813/uwhyb_08_14_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190813/uwhyb_08_14_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190814/uwhyb_08_14_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190814/uwhyb_08_15_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190814/uwhyb_08_15_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190814/uwhyb_08_15_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190815/uwhyb_08_15_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190815/uwhyb_08_16_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190815/uwhyb_08_16_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190815/uwhyb_08_16_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190816/uwhyb_08_16_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190816/uwhyb_08_17_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190816/uwhyb_08_17_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190816/uwhyb_08_17_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190817/uwhyb_08_17_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190817/uwhyb_08_18_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190817/uwhyb_08_18_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190817/uwhyb_08_18_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190818/uwhyb_08_18_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190818/uwhyb_08_19_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190818/uwhyb_08_19_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190818/uwhyb_08_19_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190819/uwhyb_08_19_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190819/uwhyb_08_20_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190819/uwhyb_08_20_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190819/uwhyb_08_20_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190820/uwhyb_08_20_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190820/uwhyb_08_21_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190820/uwhyb_08_21_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190820/uwhyb_08_21_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190821/uwhyb_08_21_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190821/uwhyb_08_22_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190821/uwhyb_08_22_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190821/uwhyb_08_22_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190822/uwhyb_08_22_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190822/uwhyb_08_23_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190822/uwhyb_08_23_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190822/uwhyb_08_23_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190823/uwhyb_08_23_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190823/uwhyb_08_24_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190823/uwhyb_08_24_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190823/uwhyb_08_24_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190824/uwhyb_08_24_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190824/uwhyb_08_25_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190824/uwhyb_08_25_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190824/uwhyb_08_25_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190825/uwhyb_08_25_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190825/uwhyb_08_26_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190825/uwhyb_08_26_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190825/uwhyb_08_26_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190826/uwhyb_08_26_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190826/uwhyb_08_27_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190826/uwhyb_08_27_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190826/uwhyb_08_27_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190827/uwhyb_08_27_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190827/uwhyb_08_28_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190827/uwhyb_08_28_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190827/uwhyb_08_28_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190828/uwhyb_08_28_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190828/uwhyb_08_29_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190828/uwhyb_08_29_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190828/uwhyb_08_29_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190829/uwhyb_08_29_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190829/uwhyb_08_30_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190829/uwhyb_08_30_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190829/uwhyb_08_30_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190830/uwhyb_08_30_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190830/uwhyb_08_31_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190830/uwhyb_08_31_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190830/uwhyb_08_31_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190831/uwhyb_08_31_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190831/uwhyb_09_01_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190831/uwhyb_09_01_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190831/uwhyb_09_01_2019_12Z.chem.assim.nc diff --git a/examples/yaml/control_omps_nm-raqms.yaml b/examples/yaml/control_omps_nm-raqms.yaml new file mode 100644 index 00000000..f772460b --- /dev/null +++ b/examples/yaml/control_omps_nm-raqms.yaml @@ -0,0 +1,85 @@ +analysis: + start_time: '2019-08-01-00:00:00' #UTC + end_time: '2019-09-01-00:00:00' #UTC + time_interval: '5D' + debug: True + output_dir: /ships19/aqda/mbruckner/monet_plots + save: + paired: + method: 'netcdf' + prefix: 'firex_omps' + data: 'all' + read: + paired: + method: 'netcdf' + filenames: {'omps_nm_raqms':['firex_omps_201908*_20190*_omps_nm_raqms.nc4']} +model: + raqms: # model label + files: /ships19/aqda/mbruckner/MELODIES-MONET-1/examples/jupyter_notebooks/raqms-files.txt + mod_type: 'raqms' + apply_ak: True # for satellite comparison, applies averaging kernels/apriori when true. Default to False + radius_of_influence: 120000 #meters + variables: #Opt + o3vmr: # specifying to switch units to ppbv + need: True + mapping: #model species name : obs species name + omps_nm: + o3vmr: ozone_column #The mapping tables need to contain the same species for all models. + plot_kwargs: #Opt + color: 'purple' + marker: '+' + linestyle: 'dotted' +obs: + omps_nm: # obs label + filename: /ships19/aqda/mbruckner/OMPS-NPP/O3-daily/2019/nadir_mapper/OMPS-NPP_NMTO3-L2_v2.1_2019m08*t*.h5 + obs_type: sat_swath_clm + variables: #Opt + + +plots: + plot_grp1: + type: 'timeseries' # plot type + fig_kwargs: #Opt to define figure options + figsize: [12,6] # figure size if multiple plots + default_plot_kwargs: # Opt to define defaults for all plots. Model kwargs overwrite these. + linewidth: 2.0 + markersize: 10. + text_kwargs: #Opt + fontsize: 18. + domain_type: ['all'] #List of domain types: 'all' or any domain in obs file. (e.g., airnow: epa_region, state_name, siteid, etc.) + domain_name: ['Global'] #List of domain names. If domain_type = all domain_name is used in plot title. + data: ['omps_nm_raqms'] # make this a list of pairs in obs_model where the obs is the obs label and model is the model_label + data_proc: + rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN. + ts_select_time: 'time' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local' + ts_avg_window: 'min' # Options: None for no averaging or list pandas resample rule (e.g., 'H', 'D') + set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs. + plot_grp2: + type: 'taylor' # plot type + fig_kwargs: #Opt to define figure options + figsize: [8,8] # figure size if multiple plots + default_plot_kwargs: # Opt to define defaults for all plots. Model kwargs overwrite these. + linewidth: 2.0 + markersize: 10. + text_kwargs: #Opt + fontsize: 16. + domain_type: ['all'] #List of domain types: 'all' or any domain in obs file. (e.g., airnow: epa_region, state_name, siteid, etc.) + domain_name: ['Global'] # of domain names. If domain_type = all domain_name is used in plot title. + data: ['omps_nm_raqms'] # make this a list of pairs in obs_model where the obs is the obs label and model is the model_label + data_proc: + rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN. + set_axis: True #If select True, add ty_scale for each variable in obs. + plot_grp3: + type: 'gridded_spatial_bias' #'gridded_spatial_bias' # plot type + fig_kwargs: #For all spatial plots, specify map_kwargs here too. + states: True + figsize: [10, 5] # figure size + text_kwargs: #Opt + fontsize: 16. + #label: '$\\Delta$ DU' + domain_type: ['all'] #List of domain types: 'all' or any domain in obs file. (e.g., airnow: epa_region, state_name, siteid, etc.) + domain_name: ['Global'] #List of domain names. If domain_type = all domain_name is used in plot title. + data: ['omps_nm_raqms'] # make this a list of pairs in obs_model where the obs is the obs label and model is the model_label + data_proc: + rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN. + set_axis: True #If select True, add vdiff_plot for each variable in obs. \ No newline at end of file diff --git a/melodies_monet/driver.py b/melodies_monet/driver.py index 31917ff5..8c3437a1 100644 --- a/melodies_monet/driver.py +++ b/melodies_monet/driver.py @@ -182,7 +182,17 @@ def open_sat_obs(self,time_interval=None): if time_interval is not None: flst = tsub.subset_OMPS_l2(self.file,time_interval) else: flst = self.file + self.obj = mio.sat._omps_nadir_mm.read_OMPS_nm(flst) + + # couple of changes to move to reader + self.obj = self.obj.swap_dims({'x':'time'}) # indexing needs + self.obj = self.obj.sortby('time') # enforce time in order. + # restrict observation data to time_interval if using + # additional development to deal with files crossing intervals needed (eg situtations where orbit start at 23hrs, ends next day). + if time_interval is not None: + self.obj = self.obj.sel(time=slice(time_interval[0],time_interval[-1])) + elif self.label == 'mopitt_l3': print('Reading MOPITT') self.obj = mio.sat._mopitt_l3_mm.read_mopittdataset(self.file, 'column') @@ -426,14 +436,6 @@ def open_model_files(self, time_interval=None): #self.obj.monet.scrip = self.obj_scrip # MEB: addition for personal local copy to work with development UFS-RAQMS # not intended for public versions - elif 'fv3raqms' in self.model.lower(): - if len(self.files) > 1: - self.obj = mio.models.fv3raqms.open_mfdataset(self.files,**self.mod_kwargs) - else: - self.obj = mio.models.fv3raqms.open_dataset(self.files) - self.obj = self.obj.rename({'sfcp':'surfpres_pa','dpm':'dp_pa','pdash':'pres_pa_mid'}) - self.obj['surfpres_pa'] *= 100 - self.obj['dp_pa'] *= 100 elif 'raqms' in self.model.lower(): if time_interval is not None: # fill filelist with subset @@ -478,6 +480,10 @@ def mask_and_scale(self): self.obj[v].data += scale elif d['unit_scale_method'] == '-': self.obj[v].data += -1 * scale + if self.obj[v].units == 'ppv': + print('changing units for {}'.format(v)) + self.obj[v].values *= 1e9 + self.obj[v].attrs['units'] = 'ppbv' class analysis: """The analysis class. @@ -509,12 +515,7 @@ def __init__(self): self.debug = False self.save = None self.read = None - '''options for regrid from obsgrid to user-specified. - Defaults to 1x1degree, single timestep''' - self.usergrid_ntime = 1 - self.usergrid_nlon = 360 - self.usergrid_nlat = 180 - + def __repr__(self): return ( f"{type(self).__name__}(\n" @@ -597,12 +598,7 @@ def read_control(self, control=None): self.time_intervals \ = [[time_stamps[n], time_stamps[n+1]] for n in range(len(time_stamps)-1)] - if 'usergrid_ntime' in self.control_dict['analysis'].keys(): - self.usergrid_ntime = self.control_dict['analysis']['usergrid_ntime'] - if 'usergrid_nlat' in self.control_dict['analysis'].keys(): - self.usergrid_nlat = self.control_dict['analysis']['usergrid_nlat'] - if 'usergrid_nlon' in self.control_dict['analysis'].keys(): - self.usergrid_nlon = self.control_dict['analysis']['usergrid_nlon'] + # Enable Dask progress bars? (default: false) enable_dask_progress_bars = self.control_dict["analysis"].get( "enable_dask_progress_bars", False) @@ -697,13 +693,6 @@ def open_models(self, time_interval=None,load_files=True): m.radius_of_influence = self.control_dict['model'][mod]['radius_of_influence'] else: m.radius_of_influence = 1e6 - # this initial_file/last_file stuff should be removed. Should move to internal thing for satellite data mapping. - if 'initial_file' in self.control_dict['model'][mod].keys(): - m.initial_file = self.control_dict['model'][mod]['initial_file'] - else: m.initial_file = False - if 'last_file' in self.control_dict['model'][mod].keys(): - m.last_file = self.control_dict['model'][mod]['last_file'] - else: m.last_file = False if 'mod_kwargs' in self.control_dict['model'][mod].keys(): m.mod_kwargs = self.control_dict['model'][mod]['mod_kwargs'] @@ -879,16 +868,18 @@ def pair_data(self, time_interval=None): if obs.label == 'omps_nm': from .util import satellite_utilities as sutil + + #necessary observation index things + #the along track coordinate dim sometimes needs to be time and other times an unassigned 'x' + obs.obj = obs.obj.swap_dims({'time':'x'}) if mod.apply_ak == True: - keys.append('pres_pa_mid') - keys.append('surfpres_pa') - model_obj = mod.obj[keys] - paired_data = sutil.omps_nm_pairing_apriori(model_obj,obs.obj) + model_obj = mod.obj[keys+['pres_pa_mid','surfpres_pa']] + + paired_data = sutil.omps_nm_pairing_apriori(model_obj,obs.obj,keys) else: - keys.append('dp_pa') - model_obj = mod.obj[keys] + model_obj = mod.obj[keys+['dp_pa']] paired_data = sutil.omps_nm_pairing(model_obj,obs.obj,keys) - + paired_data = paired_data.where((paired_data.o3vmr > 0)) p = pair() p.type = obs.obs_type @@ -926,57 +917,6 @@ def concat_pairs(self): """ pass - def regrid_paired(self,pair_label): - '''Re-grid observation and model pair data to specified grid - ''' - from .util import grid_util - import numpy as np - import pandas as pd - import xarray as xr - - paired_ds_dims = self.paired[pair_label].obj.dims - obs_time = pd.to_datetime(self.paired[pair_label].obj['time']) - - grid,edge,time_stamps = grid_util.generate_uniform_grid(paired_ds_dims,self.control_dict['analysis']['start_time'], - self.control_dict['analysis']['end_time'], - obs_time,self.usergrid_ntime,self.usergrid_nlat,self.usergrid_nlon) - pair_ds = self.paired[pair_label].obj - vlst = list(pair_ds.variables.keys()) - # remove lat,lon,time from variable listing to keep only paired variables - vlst.remove('latitude') - vlst.remove('longitude') - vlst.remove('time') - - lons,lats = np.meshgrid(grid['longitude'],grid['latitude']) - usergridded = xr.Dataset({},coords={'latitude':(['x','y'],lats), - 'longitude':(['x','y'],lons), - 'time':(['time'],pd.to_datetime(grid['time'],unit='s'))}) - for v in vlst: - # initialize count and data arrays - count_grid = np.zeros((self.usergrid_ntime, self.usergrid_nlat, self.usergrid_nlon), - dtype=np.int32) - data_grid = np.zeros((self.usergrid_ntime, self.usergrid_nlat, self.usergrid_nlon), - dtype=np.float32) - grid_util.update_data_grid(edge['time_edges'], edge['lat_edges'], edge['lon_edges'], - time_stamps.flatten(), pair_ds['latitude'].data.flatten(), pair_ds['longitude'].data.flatten(), - pair_ds[v].data.flatten(),count_grid, data_grid) - usergridded['not_norm_{}'.format(v)] = (['time','x','y'],data_grid) - # normalize data - grid_util.normalize_data_grid(count_grid, data_grid) - print(data_grid.shape) - print(usergridded.dims) - usergridded[v] = (['time','x','y'],data_grid) - usergridded['counts_{}'.format(v)] = (['time','x','y'],count_grid) - p = pair() - p.type = self.paired[pair_label].type - p.obs = self.paired[pair_label].obs - p.model = self.paired[pair_label].model - p.model_vars = self.paired[pair_label].model_vars - p.obs_vars = self.paired[pair_label].obs_vars - p.obj = usergridded - label = '{}_grid'.format(pair_label) - self.paired[label] = p - ### TODO: Create the plotting driver (most complicated one) # def plotting(self): def plotting(self): @@ -1057,7 +997,9 @@ def plotting(self): "sat_grid_sfc", "sat_grid_clm", "sat_swath_prof"]: # convert index to time; setup for sat_swath_clm + if 'time' not in p.obj.dims and obs_type == 'sat_swath_clm': + pairdf_all = p.obj.swap_dims({'x':'time'}) # squash lat/lon dimensions into single dimension elif obs_type == 'sat_grid_clm': @@ -1066,6 +1008,7 @@ def plotting(self): else: pairdf_all = p.obj # Select only the analysis time window. + print(pairdf_all.dims) pairdf_all = pairdf_all.sel(time=slice(self.start_time,self.end_time)) # Determine the default plotting colors. diff --git a/melodies_monet/util/satellite_utilities.py b/melodies_monet/util/satellite_utilities.py index c58b1f10..e51de81c 100644 --- a/melodies_monet/util/satellite_utilities.py +++ b/melodies_monet/util/satellite_utilities.py @@ -12,7 +12,7 @@ def omps_l3_daily_o3_pairing(model_data,obs_data,ozone_ppbv_varname): # factor for converting ppv profiles to DU column # also requires conversion of dp from Pa to hPa - du_fac = 1.0e4*6.023e23/28.97/9.8/2.687e19 + du_fac = 1.0e-5*6.023e23/28.97/9.8/2.687e19 column = (du_fac*(model_data['dp_pa']/100.)*model_data[ozone_ppbv_varname]).sum('z') # initialize regrid and apply to column data @@ -87,7 +87,7 @@ def space_and_time_pairing(model_data,obs_data,pair_variables): ds[j][:,tindex,:] += np.expand_dims(tfac1.values,axis=1)*interm_var.values return ds -def omps_nm_pairing(model_data,obs_data,pair_variables): +def omps_nm_pairing(model_data,obs_data,ozone_ppbv_varname): 'Pairs UFS-RAQMS ozone mixing ratio with OMPS nadir mapper retrievals. Calculates column without applying apriori' import xarray as xr @@ -96,37 +96,37 @@ def omps_nm_pairing(model_data,obs_data,pair_variables): print('pairing without applying averaging kernel') - du_fac = 1.0e4*6.023e23/28.97/9.8/2.687e19 # conversion factor; moves model from ppv to dobson - + du_fac = 1.0e-5*6.023e23/28.97/9.8/2.687e19 # conversion factor; moves model from ppbv to dobson + pair_variables = ['dp_pa',ozone_ppbv_varname] paired_ds = space_and_time_pairing(model_data,obs_data,pair_variables) # calculate ozone column, no averaging kernel or apriori applied. col = np.nansum(du_fac*(paired_ds['dp_pa']/100.)*paired_ds['o3vmr'],axis=0) # new dimensions will be (satellite_x, satellite_y) - ds = xr.Dataset({'o3vmr': (['x','y'],col), - 'ozone_column':(['x','y'],obs_data.ozone_column.values) + ds = xr.Dataset({ozone_ppbv_varname[0]: (['time','y'],col), + 'ozone_column':(['time','y'],obs_data.ozone_column.values) }, coords={ - 'longitude':(['x','y'],obs_data['longitude'].values), - 'latitude':(['x','y'],obs_data['latitude'].values), - 'time':(['x'],obs_data.time.values), + 'longitude':(['time','y'],obs_data['longitude'].values), + 'latitude':(['time','y'],obs_data['latitude'].values), + 'time':(['time'],obs_data.time.values), }) return ds -def omps_nm_pairing_apriori(model_data,obs_data): +def omps_nm_pairing_apriori(model_data,obs_data,ozone_ppbv_varname): 'Pairs UFS-RAQMS data with OMPS nm. Applies satellite apriori column to model observations.' import xarray as xr import pandas as pd - du_fac = 1.0e4*6.023e23/28.97/9.8/2.687e19 # conversion factor; moves model from ppv to dobson + du_fac = 1.0e-5*6.023e23/28.97/9.8/2.687e19 # conversion factor; moves model from ppv to dobson print('pairing with averaging kernel application') # Grab necessary shape information - nf,nz_m,nx_m,ny_m = model_data['o3vmr'].shape + nf,nz_m,nx_m,ny_m = model_data[ozone_ppbv_varname[0]].shape nx,ny = obs_data.ozone_column.shape ## initialize intermediates for use in calcluating column pressure_temp = np.zeros((nz_m,nx,ny)) @@ -139,7 +139,7 @@ def omps_nm_pairing_apriori(model_data,obs_data): if len(tindex): # regrid spatially (model lat/lon to satellite swath lat/lon) regridr = xe.Regridder(model_data.isel(time=f),obs_data[['latitude','longitude']].sel(x=tindex),'bilinear') - regrid_oz = regridr(model_data['o3vmr'][f]) + regrid_oz = regridr(model_data[ozone_ppbv_varname[0]][f]) regrid_p = regridr(model_data['pres_pa_mid'][f]) # this one should be pressure variable (for the interpolation). sfp = regridr(model_data['surfpres_pa'][f]) # fixes for observations before/after model time range. @@ -208,12 +208,12 @@ def omps_nm_pairing_apriori(model_data,obs_data): ap = obs_data.apriori[:,:,i].values oz = oz + ap*(1-eff) + (eff)*(add) - ds = xr.Dataset({'o3vmr': (['x','y'],oz), - 'ozone_column':(['x','y'],obs_data.ozone_column.values) + ds = xr.Dataset({ozone_ppbv_varname[0]: (['time','y'],oz), + 'ozone_column':(['time','y'],obs_data.ozone_column.values) }, coords={ - 'longitude':(['x','y'],obs_data['longitude'].values), - 'latitude':(['x','y'],obs_data['latitude'].values), - 'time':(['x'],obs_data.time.values), + 'longitude':(['time','y'],obs_data['longitude'].values), + 'latitude':(['time','y'],obs_data['latitude'].values), + 'time':(['time'],obs_data.time.values), }) return ds diff --git a/melodies_monet/util/time_interval_subset.py b/melodies_monet/util/time_interval_subset.py new file mode 100644 index 00000000..23abb924 --- /dev/null +++ b/melodies_monet/util/time_interval_subset.py @@ -0,0 +1,33 @@ +def subset_model_filelist(all_files,timeformat,timestep,timeinterval): + '''Subset model filelist to within a given time interval. + Filename requirements: + - individual files for each timestep + - time must be in filename + ''' + import pandas as pd + subset_interval = pd.date_range(start=timeinterval[0],end=timeinterval[-1],freq=timestep) + interval_files = [] + for i in subset_interval: + flst = [fs for fs in all_files if i.strftime(timeformat) in fs] + if len(flst) == 1: + interval_files.append(flst[0]) + elif len(flst) >1: + print('More than 1 file for {} in listing'.format(i.strftime(timeformat))) + return interval_files + +def subset_OMPS_l2(file_path,timeinterval): + '''Dependent on filenaming convention + ''' + import pandas as pd + from glob import glob + import fnmatch + all_files = glob(file_path) + interval_files = [] + subset_interval = pd.date_range(start=timeinterval[0],end=timeinterval[-1],freq='D',inclusive='left') + + for i in subset_interval: + fst = fnmatch.filter(all_files,'*OMPS-NPP_NMTO3-L2_v*_{}*_o*'.format(i.strftime('%Ym%m%d'))) + fst.sort() + for j in fst: + interval_files.append(j) + return interval_files \ No newline at end of file