diff --git a/examples/jupyter_notebooks/Monet-example-omps_nm_raqms-time_chunk.ipynb b/examples/jupyter_notebooks/Monet-example-omps_nm_raqms-time_chunk.ipynb new file mode 100644 index 00000000..1d049578 --- /dev/null +++ b/examples/jupyter_notebooks/Monet-example-omps_nm_raqms-time_chunk.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "51def9a8-dcf7-489c-a9e6-bc78886a7917", + "metadata": {}, + "source": [ + "## Implementation of processing over time intervals\n", + "\n", + "Testing of Melodies-Monet OMPS Nadir Mapper L2 pairing with time interval chunking" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64e61a73-ace6-4041-a194-fe26e7a6b126", + "metadata": {}, + "outputs": [], + "source": [ + "from melodies_monet import driver" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17e4ebe7-7b5e-4628-919d-7275dd088478", + "metadata": {}, + "outputs": [], + "source": [ + "an = driver.analysis()\n", + "an.control = '../yaml/control_omps_nm-raqms.yaml'\n", + "an.read_control()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2bc2dc2d-5297-4827-aa3e-90052ea4b64c", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "base_prefix = an.save['paired']['prefix']#.copy()\n", + "for t in an.time_intervals:\n", + " an.open_models(time_interval=t)\n", + " an.open_obs(time_interval=t)\n", + " \n", + " an.pair_data()\n", + " \n", + " # adjust saved name for file to include time interval bounds\n", + " an.save['paired']['prefix'] = base_prefix+'_'+t[0].strftime('%Y%m%d')+'_'+t[1].strftime('%Y%m%d')\n", + " an.save_analysis()" + ] + }, + { + "cell_type": "markdown", + "id": "a6ba95ad-f73a-4b68-8ae5-7858eee6c970", + "metadata": {}, + "source": [ + "Notes regarding satellite pairing methods:\n", + "- some additional development needed to deal with time chuncking. Some OMPS NM orbit files cross the day. This doesn't cause issues if it is within the specified time_interval, but data past the time_interval in the file will be dropped and currently will not be read in during the next time_interval\n", + "\n", + "- Satellite pairing bilinearly interpolates model data in time to satellite observation times. When observations are before (after) the first (last) model file, time interpolation is nearest-neighbor and only the first (last) file is used. Right now the time-interpolation does not take into account if processing is being done over time chunks. Impact should be minimal, as observations have been filtered to be within time_intervals and the model file subsetter should be selecting all the necessary files." + ] + }, + { + "cell_type": "markdown", + "id": "8ddb4e95-1db0-4f01-b433-2d279b1d25fe", + "metadata": {}, + "source": [ + "### Read in saved paired data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce25d669-166a-42a0-861e-ceef59b26d34", + "metadata": {}, + "outputs": [], + "source": [ + "an = driver.analysis()\n", + "an.control = 'control_omps_nm-raqms.yaml'\n", + "an.read_control()\n", + "an.read_analysis()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f41b0217-c3cf-4cde-996e-d6149cea5aa5", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "#an.paired['omps_nm_raqms'].obj = an.paired['omps_nm_raqms'].obj.swap_dims({'time':'x'})\n", + "an.paired['omps_nm_raqms'].obj['o3vmr'].values[np.isnan(an.paired['omps_nm_raqms'].obj['ozone_column'].values)] = np.nan\n", + "an.paired['omps_nm_raqms'].obj['o3vmr'].values[an.paired['omps_nm_raqms'].obj['o3vmr'].values < 50] = np.nan\n", + "an.paired['omps_nm_raqms'].obj['ozone_column'].values[np.isnan(an.paired['omps_nm_raqms'].obj['o3vmr'].values)] = np.nan\n", + "an.plotting()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dev_monet", + "language": "python", + "name": "develop_monet" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/jupyter_notebooks/raqms-files.txt b/examples/jupyter_notebooks/raqms-files.txt new file mode 100644 index 00000000..2b1eb7fe --- /dev/null +++ b/examples/jupyter_notebooks/raqms-files.txt @@ -0,0 +1,124 @@ +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190801/uwhyb_08_01_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190801/uwhyb_08_02_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190801/uwhyb_08_02_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190801/uwhyb_08_02_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190802/uwhyb_08_02_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190802/uwhyb_08_03_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190802/uwhyb_08_03_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190802/uwhyb_08_03_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190803/uwhyb_08_03_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190803/uwhyb_08_04_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190803/uwhyb_08_04_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190803/uwhyb_08_04_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190804/uwhyb_08_04_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190804/uwhyb_08_05_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190804/uwhyb_08_05_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190804/uwhyb_08_05_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190805/uwhyb_08_05_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190805/uwhyb_08_06_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190805/uwhyb_08_06_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190805/uwhyb_08_06_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190806/uwhyb_08_06_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190806/uwhyb_08_07_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190806/uwhyb_08_07_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190806/uwhyb_08_07_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190807/uwhyb_08_07_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190807/uwhyb_08_08_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190807/uwhyb_08_08_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190807/uwhyb_08_08_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190808/uwhyb_08_08_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190808/uwhyb_08_09_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190808/uwhyb_08_09_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190808/uwhyb_08_09_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190809/uwhyb_08_09_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190809/uwhyb_08_10_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190809/uwhyb_08_10_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190809/uwhyb_08_10_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190810/uwhyb_08_10_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190810/uwhyb_08_11_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190810/uwhyb_08_11_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190810/uwhyb_08_11_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190811/uwhyb_08_11_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190811/uwhyb_08_12_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190811/uwhyb_08_12_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190811/uwhyb_08_12_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190812/uwhyb_08_12_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190812/uwhyb_08_13_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190812/uwhyb_08_13_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190812/uwhyb_08_13_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190813/uwhyb_08_13_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190813/uwhyb_08_14_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190813/uwhyb_08_14_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190813/uwhyb_08_14_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190814/uwhyb_08_14_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190814/uwhyb_08_15_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190814/uwhyb_08_15_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190814/uwhyb_08_15_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190815/uwhyb_08_15_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190815/uwhyb_08_16_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190815/uwhyb_08_16_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190815/uwhyb_08_16_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190816/uwhyb_08_16_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190816/uwhyb_08_17_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190816/uwhyb_08_17_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190816/uwhyb_08_17_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190817/uwhyb_08_17_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190817/uwhyb_08_18_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190817/uwhyb_08_18_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190817/uwhyb_08_18_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190818/uwhyb_08_18_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190818/uwhyb_08_19_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190818/uwhyb_08_19_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190818/uwhyb_08_19_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190819/uwhyb_08_19_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190819/uwhyb_08_20_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190819/uwhyb_08_20_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190819/uwhyb_08_20_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190820/uwhyb_08_20_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190820/uwhyb_08_21_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190820/uwhyb_08_21_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190820/uwhyb_08_21_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190821/uwhyb_08_21_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190821/uwhyb_08_22_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190821/uwhyb_08_22_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190821/uwhyb_08_22_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190822/uwhyb_08_22_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190822/uwhyb_08_23_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190822/uwhyb_08_23_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190822/uwhyb_08_23_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190823/uwhyb_08_23_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190823/uwhyb_08_24_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190823/uwhyb_08_24_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190823/uwhyb_08_24_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190824/uwhyb_08_24_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190824/uwhyb_08_25_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190824/uwhyb_08_25_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190824/uwhyb_08_25_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190825/uwhyb_08_25_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190825/uwhyb_08_26_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190825/uwhyb_08_26_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190825/uwhyb_08_26_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190826/uwhyb_08_26_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190826/uwhyb_08_27_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190826/uwhyb_08_27_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190826/uwhyb_08_27_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190827/uwhyb_08_27_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190827/uwhyb_08_28_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190827/uwhyb_08_28_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190827/uwhyb_08_28_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190828/uwhyb_08_28_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190828/uwhyb_08_29_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190828/uwhyb_08_29_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190828/uwhyb_08_29_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190829/uwhyb_08_29_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190829/uwhyb_08_30_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190829/uwhyb_08_30_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190829/uwhyb_08_30_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190830/uwhyb_08_30_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190830/uwhyb_08_31_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190830/uwhyb_08_31_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190830/uwhyb_08_31_2019_12Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190831/uwhyb_08_31_2019_18Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190831/uwhyb_09_01_2019_00Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190831/uwhyb_09_01_2019_06Z.chem.assim.nc +/ships19/aqda/lenzen/CalNex/UW-Hybrid/Mission_1X1/Netcdf4/190831/uwhyb_09_01_2019_12Z.chem.assim.nc diff --git a/examples/yaml/control_omps_nm-raqms.yaml b/examples/yaml/control_omps_nm-raqms.yaml new file mode 100644 index 00000000..f772460b --- /dev/null +++ b/examples/yaml/control_omps_nm-raqms.yaml @@ -0,0 +1,85 @@ +analysis: + start_time: '2019-08-01-00:00:00' #UTC + end_time: '2019-09-01-00:00:00' #UTC + time_interval: '5D' + debug: True + output_dir: /ships19/aqda/mbruckner/monet_plots + save: + paired: + method: 'netcdf' + prefix: 'firex_omps' + data: 'all' + read: + paired: + method: 'netcdf' + filenames: {'omps_nm_raqms':['firex_omps_201908*_20190*_omps_nm_raqms.nc4']} +model: + raqms: # model label + files: /ships19/aqda/mbruckner/MELODIES-MONET-1/examples/jupyter_notebooks/raqms-files.txt + mod_type: 'raqms' + apply_ak: True # for satellite comparison, applies averaging kernels/apriori when true. Default to False + radius_of_influence: 120000 #meters + variables: #Opt + o3vmr: # specifying to switch units to ppbv + need: True + mapping: #model species name : obs species name + omps_nm: + o3vmr: ozone_column #The mapping tables need to contain the same species for all models. + plot_kwargs: #Opt + color: 'purple' + marker: '+' + linestyle: 'dotted' +obs: + omps_nm: # obs label + filename: /ships19/aqda/mbruckner/OMPS-NPP/O3-daily/2019/nadir_mapper/OMPS-NPP_NMTO3-L2_v2.1_2019m08*t*.h5 + obs_type: sat_swath_clm + variables: #Opt + + +plots: + plot_grp1: + type: 'timeseries' # plot type + fig_kwargs: #Opt to define figure options + figsize: [12,6] # figure size if multiple plots + default_plot_kwargs: # Opt to define defaults for all plots. Model kwargs overwrite these. + linewidth: 2.0 + markersize: 10. + text_kwargs: #Opt + fontsize: 18. + domain_type: ['all'] #List of domain types: 'all' or any domain in obs file. (e.g., airnow: epa_region, state_name, siteid, etc.) + domain_name: ['Global'] #List of domain names. If domain_type = all domain_name is used in plot title. + data: ['omps_nm_raqms'] # make this a list of pairs in obs_model where the obs is the obs label and model is the model_label + data_proc: + rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN. + ts_select_time: 'time' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local' + ts_avg_window: 'min' # Options: None for no averaging or list pandas resample rule (e.g., 'H', 'D') + set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs. + plot_grp2: + type: 'taylor' # plot type + fig_kwargs: #Opt to define figure options + figsize: [8,8] # figure size if multiple plots + default_plot_kwargs: # Opt to define defaults for all plots. Model kwargs overwrite these. + linewidth: 2.0 + markersize: 10. + text_kwargs: #Opt + fontsize: 16. + domain_type: ['all'] #List of domain types: 'all' or any domain in obs file. (e.g., airnow: epa_region, state_name, siteid, etc.) + domain_name: ['Global'] # of domain names. If domain_type = all domain_name is used in plot title. + data: ['omps_nm_raqms'] # make this a list of pairs in obs_model where the obs is the obs label and model is the model_label + data_proc: + rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN. + set_axis: True #If select True, add ty_scale for each variable in obs. + plot_grp3: + type: 'gridded_spatial_bias' #'gridded_spatial_bias' # plot type + fig_kwargs: #For all spatial plots, specify map_kwargs here too. + states: True + figsize: [10, 5] # figure size + text_kwargs: #Opt + fontsize: 16. + #label: '$\\Delta$ DU' + domain_type: ['all'] #List of domain types: 'all' or any domain in obs file. (e.g., airnow: epa_region, state_name, siteid, etc.) + domain_name: ['Global'] #List of domain names. If domain_type = all domain_name is used in plot title. + data: ['omps_nm_raqms'] # make this a list of pairs in obs_model where the obs is the obs label and model is the model_label + data_proc: + rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN. + set_axis: True #If select True, add vdiff_plot for each variable in obs. \ No newline at end of file diff --git a/melodies_monet/driver.py b/melodies_monet/driver.py index c7ca3f95..8c3437a1 100644 --- a/melodies_monet/driver.py +++ b/melodies_monet/driver.py @@ -162,28 +162,53 @@ def open_obs(self, time_interval=None): except ValueError: print('something happened opening file') - def open_sat_obs(self): + def open_sat_obs(self,time_interval=None): """Methods to opens satellite data observations. Uses in-house python code to open and load observations. Alternatively may use the satpy reader. + + Returns + ------- + type + Fills the object class associated with the equivalent label (self.label) with satellite observation + dataset read in from the associated file (self.file) by the satellite file reader """ + from .util import time_interval_subset as tsub + try: - if extension in {'.nc', '.ncf', '.netcdf', '.nc4'}: - if len(files) > 1: - self.obj = xr.open_mfdataset(files) - else: - self.obj = xr.open_dataset(files[0]) - elif extension in ['.ict', '.icarrt']: - assert len(files) == 1, "monetio.icarrt.add_data can only read one file" - self.obj = mio.icarrt.add_data(files[0]) - else: - raise ValueError(f'extension {extension!r} currently unsupported') - except Exception as e: - print('something happened opening file:', e) - return + if self.label == 'omps_l3': + self.obj = mio.sat._omps_l3_mm.read_OMPS_l3(self.file) + elif self.label == 'omps_nm': + if time_interval is not None: + flst = tsub.subset_OMPS_l2(self.file,time_interval) + else: flst = self.file - self.mask_and_scale() # mask and scale values from the control values - self.filter_obs() + self.obj = mio.sat._omps_nadir_mm.read_OMPS_nm(flst) + + # couple of changes to move to reader + self.obj = self.obj.swap_dims({'x':'time'}) # indexing needs + self.obj = self.obj.sortby('time') # enforce time in order. + # restrict observation data to time_interval if using + # additional development to deal with files crossing intervals needed (eg situtations where orbit start at 23hrs, ends next day). + if time_interval is not None: + self.obj = self.obj.sel(time=slice(time_interval[0],time_interval[-1])) + + elif self.label == 'mopitt_l3': + print('Reading MOPITT') + self.obj = mio.sat._mopitt_l3_mm.read_mopittdataset(self.file, 'column') + elif self.label == 'modis_l2': + from monetio import modis_l2 + print('Reading MODIS L2') + self.obj = modis_l2.read_mfdataset( + self.file, self.variable_dict, debug=self.debug) + elif self.label == 'tropomi_l2_no2': + from monetio import tropomi_l2_no2 + print('Reading TROPOMI L2 NO2') + self.obj = tropomi_l2_no2.read_trpdataset( + self.file, self.variable_dict, debug=self.debug) + else: print('file reader not implemented for {} observation'.format(self.label)) + except ValueError: + print('something happened opening file') def filter_obs(self): """Filter observations based on filter_dict. @@ -218,29 +243,7 @@ def filter_obs(self): self.obj = self.obj.where(self.obj[column] != filter_vals,drop=True) else: raise ValueError(f'Filter operation {filter_op!r} is not supported') - - try: - if self.label == 'omps_l3': - self.obj = mio.sat._omps_l3_mm.read_OMPS_l3(self.file) - elif self.label == 'omps_nm': - self.obj = mio.sat._omps_nadir_mm.read_OMPS_nm(self.file) - elif self.label == 'mopitt_l3': - print('Reading MOPITT') - self.obj = mio.sat._mopitt_l3_mm.read_mopittdataset(self.file, 'column') - elif self.label == 'modis_l2': - from monetio import modis_l2 - print('Reading MODIS L2') - self.obj = modis_l2.read_mfdataset( - self.file, self.variable_dict, debug=self.debug) - elif self.label == 'tropomi_l2_no2': - from monetio import tropomi_l2_no2 - print('Reading TROPOMI L2 NO2') - self.obj = tropomi_l2_no2.read_trpdataset( - self.file, self.variable_dict, debug=self.debug) - else: print('file reader not implemented for {} observation'.format(self.label)) - except ValueError: - print('something happened opening file') - + def mask_and_scale(self): """Mask and scale observations, including unit conversions and setting detection limits. @@ -352,9 +355,7 @@ def glob_files(self): # add option to read list of files from text file if 'txt' in self.file_str: with open(self.file_str,'r') as f: - self.files = f.read().split(' \n')[:-1] - - # self.files = sort(self.file_str) + self.files = f.read().split() if self.file_vert_str is not None: self.files_vert = sort(glob(self.file_vert_str)) @@ -381,6 +382,7 @@ def open_model_files(self, time_interval=None): ------- None """ + from .util import time_interval_subset as tsub print(self.model.lower()) self.glob_files() @@ -432,11 +434,19 @@ def open_model_files(self, time_interval=None): self.obj = mio.models._cesm_se_mm.open_mfdataset(self.files,**self.mod_kwargs) #self.obj, self.obj_scrip = read_cesm_se.open_mfdataset(self.files,**self.mod_kwargs) #self.obj.monet.scrip = self.obj_scrip + # MEB: addition for personal local copy to work with development UFS-RAQMS + # not intended for public versions elif 'raqms' in self.model.lower(): + if time_interval is not None: + # fill filelist with subset + file_sublist = tsub.subset_model_filelist(self.files,'%m_%d_%Y_%H','6H',time_interval) + else: + # fill filelist with all files + file_sublist = self.files if len(self.files) > 1: - self.obj = mio.raqms.open_mfdataset(self.files,**self.mod_kwargs) + self.obj = mio.raqms.open_mfdataset(file_sublist,**self.mod_kwargs) else: - self.obj = mio.raqms.open_dataset(self.files,**self.mod_kwargs) + self.obj = mio.raqms.open_dataset(file_sublist,**self.mod_kwargs) else: print('**** Reading Unspecified model output. Take Caution...') if len(self.files) > 1: @@ -470,6 +480,10 @@ def mask_and_scale(self): self.obj[v].data += scale elif d['unit_scale_method'] == '-': self.obj[v].data += -1 * scale + if self.obj[v].units == 'ppv': + print('changing units for {}'.format(v)) + self.obj[v].values *= 1e9 + self.obj[v].attrs['units'] = 'ppbv' class analysis: """The analysis class. @@ -501,7 +515,7 @@ def __init__(self): self.debug = False self.save = None self.read = None - + def __repr__(self): return ( f"{type(self).__name__}(\n" @@ -584,7 +598,7 @@ def read_control(self, control=None): self.time_intervals \ = [[time_stamps[n], time_stamps[n+1]] for n in range(len(time_stamps)-1)] - + # Enable Dask progress bars? (default: false) enable_dask_progress_bars = self.control_dict["analysis"].get( "enable_dask_progress_bars", False) @@ -643,8 +657,13 @@ def read_analysis(self): read_saved_data(analysis=self,filenames=self.read[attr]['filenames'], method='pkl', attr=attr) elif self.read[attr]['method']=='netcdf': read_saved_data(analysis=self,filenames=self.read[attr]['filenames'], method='netcdf', attr=attr) + if attr == 'paired': + # initialize model/obs attributes, since needed for plotting and stats + self.open_models(load_files=False) + self.open_obs(load_files=False) + - def open_models(self, time_interval=None): + def open_models(self, time_interval=None,load_files=True): """Open all models listed in the input yaml file and create a :class:`model` object for each of them, populating the :attr:`models` dict. @@ -652,7 +671,8 @@ def open_models(self, time_interval=None): __________ time_interval (optional, default None) : [pandas.Timestamp, pandas.Timestamp] If not None, restrict models to datetime range spanned by time interval [start, end]. - + load_files (optional, default True): boolean + If False, only populate :attr: dict with yaml file parameters and do not open model files. Returns ------- None @@ -673,12 +693,6 @@ def open_models(self, time_interval=None): m.radius_of_influence = self.control_dict['model'][mod]['radius_of_influence'] else: m.radius_of_influence = 1e6 - if 'initial_file' in self.control_dict['model'][mod].keys(): - m.initial_file = self.control_dict['model'][mod]['initial_file'] - else: m.initial_file = False - if 'last_file' in self.control_dict['model'][mod].keys(): - m.last_file = self.control_dict['model'][mod]['last_file'] - else: m.last_file = False if 'mod_kwargs' in self.control_dict['model'][mod].keys(): m.mod_kwargs = self.control_dict['model'][mod]['mod_kwargs'] @@ -737,10 +751,11 @@ def open_models(self, time_interval=None): m.proj = ccrs.Projection(proj_in) # open the model - m.open_model_files(time_interval=time_interval) + if load_files: + m.open_model_files(time_interval=time_interval) self.models[m.label] = m - def open_obs(self, time_interval=None): + def open_obs(self, time_interval=None, load_files=True): """Open all observations listed in the input yaml file and create an :class:`observation` instance for each of them, populating the :attr:`obs` dict. @@ -749,8 +764,9 @@ def open_obs(self, time_interval=None): __________ time_interval (optional, default None) : [pandas.Timestamp, pandas.Timestamp] If not None, restrict obs to datetime range spanned by time interval [start, end]. - - + load_files (optional, default True): boolean + If False, only populate :attr: dict with yaml file parameters and do not open obs files. + Returns ------- None @@ -769,11 +785,12 @@ def open_obs(self, time_interval=None): o.debug = self.control_dict['obs'][obs]['debug'] if 'variables' in self.control_dict['obs'][obs].keys(): o.variable_dict = self.control_dict['obs'][obs]['variables'] - if o.obs_type == 'pt_sfc': - o.open_obs(time_interval=time_interval) - elif o.obs_type in ['sat_swath_sfc', 'sat_swath_clm', 'sat_grid_sfc',\ - 'sat_grid_clm', 'sat_swath_prof']: - o.open_sat_obs() + if load_files: + if o.obs_type == 'pt_sfc': + o.open_obs(time_interval=time_interval) + elif o.obs_type in ['sat_swath_sfc', 'sat_swath_clm', 'sat_grid_sfc',\ + 'sat_grid_clm', 'sat_swath_prof']: + o.open_sat_obs(time_interval=time_interval) self.obs[o.label] = o def pair_data(self, time_interval=None): @@ -851,18 +868,21 @@ def pair_data(self, time_interval=None): if obs.label == 'omps_nm': from .util import satellite_utilities as sutil + + #necessary observation index things + #the along track coordinate dim sometimes needs to be time and other times an unassigned 'x' + obs.obj = obs.obj.swap_dims({'time':'x'}) if mod.apply_ak == True: - keys.append('pres_pa_mid') - keys.append('surfpres_pa') - model_obj = mod.obj[keys] - paired_data = sutil.omps_nm_pairing_apriori(model_obj,obs.obj) + model_obj = mod.obj[keys+['pres_pa_mid','surfpres_pa']] + + paired_data = sutil.omps_nm_pairing_apriori(model_obj,obs.obj,keys) else: - keys.append('dp_pa') - model_obj = mod.obj[keys] + model_obj = mod.obj[keys+['dp_pa']] paired_data = sutil.omps_nm_pairing(model_obj,obs.obj,keys) - + paired_data = paired_data.where((paired_data.o3vmr > 0)) p = pair() + p.type = obs.obs_type p.obs = obs.label p.model = mod.label p.model_vars = keys @@ -879,6 +899,7 @@ def pair_data(self, time_interval=None): # combine model and observations into paired dataset obs_dat['o3vmr'] = (['time','x','y'],model_obsgrid.sel(time=slice(self.start_time.date(),self.end_time.date())).data) p = pair() + p.type = obs.obs_type p.obs = obs.label p.model = mod.label p.model_vars = keys @@ -915,8 +936,8 @@ def plotting(self): None """ import matplotlib.pyplot as plt - obs_to_pair = list(self.models[(list(self.models.keys()))[0]].mapping.keys())[0] - if self.obs[obs_to_pair].obs_type.lower() == 'pt_sfc': + pair_keys = list(self.paired.keys()) + if self.paired[pair_keys[0]].type.lower() == 'pt_sfc': from .plots import surfplots as splots,savefig else: from .plots import satplots as splots,savefig @@ -943,7 +964,7 @@ def plotting(self): # first get the observational obs labels pair1 = self.paired[list(self.paired.keys())[0]] obs_vars = pair1.obs_vars - + obs_type = pair1.type # loop through obs variables for obsvar in obs_vars: # Loop also over the domain types. So can easily create several overview and zoomed in plots. @@ -964,27 +985,30 @@ def plotting(self): if obsvar == modvar: modvar = modvar + '_new' - # for pt_sft data, convert to pandas dataframe, format, and trim - if self.obs[obs_to_pair].obs_type.lower() == 'pt_sfc': + # for pt_sfc data, convert to pandas dataframe, format, and trim + if obs_type == 'pt_sfc': # convert to dataframe pairdf_all = p.obj.to_dataframe(dim_order=["time", "x"]) # Select only the analysis time window. pairdf_all = pairdf_all.loc[self.start_time : self.end_time] # keep data in xarray, fix formatting, and trim - elif self.obs[obs_to_pair].obs_type.lower() in ["sat_swath_sfc", "sat_swath_clm", + elif obs_type in ["sat_swath_sfc", "sat_swath_clm", "sat_grid_sfc", "sat_grid_clm", "sat_swath_prof"]: # convert index to time; setup for sat_swath_clm - if 'time' not in p.obj.dims and self.obs[obs_to_pair].obs_type.lower() == 'sat_swath_clm': + + if 'time' not in p.obj.dims and obs_type == 'sat_swath_clm': + pairdf_all = p.obj.swap_dims({'x':'time'}) # squash lat/lon dimensions into single dimension - elif self.obs[obs_to_pair].obs_type.lower() == 'sat_grid_clm': + elif obs_type == 'sat_grid_clm': pairdf_all = p.obj.stack(ll=['x','y']) pairdf_all = pairdf_all.rename_dims({'ll':'y'}) else: pairdf_all = p.obj # Select only the analysis time window. + print(pairdf_all.dims) pairdf_all = pairdf_all.sel(time=slice(self.start_time,self.end_time)) # Determine the default plotting colors. @@ -1090,13 +1114,13 @@ def plotting(self): pairdf_all = pairdf_all.loc[pairdf_all[grp_var].isin(grp_select[grp_var].values)] # Drop NaNs if using pandas - if self.obs[obs_to_pair].obs_type.lower() == 'pt_sfc': + if obs_type == 'pt_sfc': if grp_dict['data_proc']['rem_obs_nan'] == True: # I removed drop=True in reset_index in order to keep 'time' as a column. pairdf = pairdf_all.reset_index().dropna(subset=[modvar, obsvar]) else: pairdf = pairdf_all.reset_index().dropna(subset=[modvar]) - elif self.obs[obs_to_pair].obs_type.lower() in ["sat_swath_sfc", "sat_swath_clm", + elif obs_type in ["sat_swath_sfc", "sat_swath_clm", "sat_grid_sfc", "sat_grid_clm", "sat_swath_prof"]: # xarray doesn't need nan drop because its math operations seem to ignore nans @@ -1170,7 +1194,7 @@ def plotting(self): vmin = None vmax = None # Select time to use as index. - if self.obs[obs_to_pair].obs_type.lower() == 'pt_sfc': + if obs_type == 'pt_sfc': pairdf = pairdf.set_index(grp_dict['data_proc']['ts_select_time']) a_w = grp_dict['data_proc']['ts_avg_window'] if p_index == 0: diff --git a/melodies_monet/util/satellite_utilities.py b/melodies_monet/util/satellite_utilities.py index c58b1f10..e51de81c 100644 --- a/melodies_monet/util/satellite_utilities.py +++ b/melodies_monet/util/satellite_utilities.py @@ -12,7 +12,7 @@ def omps_l3_daily_o3_pairing(model_data,obs_data,ozone_ppbv_varname): # factor for converting ppv profiles to DU column # also requires conversion of dp from Pa to hPa - du_fac = 1.0e4*6.023e23/28.97/9.8/2.687e19 + du_fac = 1.0e-5*6.023e23/28.97/9.8/2.687e19 column = (du_fac*(model_data['dp_pa']/100.)*model_data[ozone_ppbv_varname]).sum('z') # initialize regrid and apply to column data @@ -87,7 +87,7 @@ def space_and_time_pairing(model_data,obs_data,pair_variables): ds[j][:,tindex,:] += np.expand_dims(tfac1.values,axis=1)*interm_var.values return ds -def omps_nm_pairing(model_data,obs_data,pair_variables): +def omps_nm_pairing(model_data,obs_data,ozone_ppbv_varname): 'Pairs UFS-RAQMS ozone mixing ratio with OMPS nadir mapper retrievals. Calculates column without applying apriori' import xarray as xr @@ -96,37 +96,37 @@ def omps_nm_pairing(model_data,obs_data,pair_variables): print('pairing without applying averaging kernel') - du_fac = 1.0e4*6.023e23/28.97/9.8/2.687e19 # conversion factor; moves model from ppv to dobson - + du_fac = 1.0e-5*6.023e23/28.97/9.8/2.687e19 # conversion factor; moves model from ppbv to dobson + pair_variables = ['dp_pa',ozone_ppbv_varname] paired_ds = space_and_time_pairing(model_data,obs_data,pair_variables) # calculate ozone column, no averaging kernel or apriori applied. col = np.nansum(du_fac*(paired_ds['dp_pa']/100.)*paired_ds['o3vmr'],axis=0) # new dimensions will be (satellite_x, satellite_y) - ds = xr.Dataset({'o3vmr': (['x','y'],col), - 'ozone_column':(['x','y'],obs_data.ozone_column.values) + ds = xr.Dataset({ozone_ppbv_varname[0]: (['time','y'],col), + 'ozone_column':(['time','y'],obs_data.ozone_column.values) }, coords={ - 'longitude':(['x','y'],obs_data['longitude'].values), - 'latitude':(['x','y'],obs_data['latitude'].values), - 'time':(['x'],obs_data.time.values), + 'longitude':(['time','y'],obs_data['longitude'].values), + 'latitude':(['time','y'],obs_data['latitude'].values), + 'time':(['time'],obs_data.time.values), }) return ds -def omps_nm_pairing_apriori(model_data,obs_data): +def omps_nm_pairing_apriori(model_data,obs_data,ozone_ppbv_varname): 'Pairs UFS-RAQMS data with OMPS nm. Applies satellite apriori column to model observations.' import xarray as xr import pandas as pd - du_fac = 1.0e4*6.023e23/28.97/9.8/2.687e19 # conversion factor; moves model from ppv to dobson + du_fac = 1.0e-5*6.023e23/28.97/9.8/2.687e19 # conversion factor; moves model from ppv to dobson print('pairing with averaging kernel application') # Grab necessary shape information - nf,nz_m,nx_m,ny_m = model_data['o3vmr'].shape + nf,nz_m,nx_m,ny_m = model_data[ozone_ppbv_varname[0]].shape nx,ny = obs_data.ozone_column.shape ## initialize intermediates for use in calcluating column pressure_temp = np.zeros((nz_m,nx,ny)) @@ -139,7 +139,7 @@ def omps_nm_pairing_apriori(model_data,obs_data): if len(tindex): # regrid spatially (model lat/lon to satellite swath lat/lon) regridr = xe.Regridder(model_data.isel(time=f),obs_data[['latitude','longitude']].sel(x=tindex),'bilinear') - regrid_oz = regridr(model_data['o3vmr'][f]) + regrid_oz = regridr(model_data[ozone_ppbv_varname[0]][f]) regrid_p = regridr(model_data['pres_pa_mid'][f]) # this one should be pressure variable (for the interpolation). sfp = regridr(model_data['surfpres_pa'][f]) # fixes for observations before/after model time range. @@ -208,12 +208,12 @@ def omps_nm_pairing_apriori(model_data,obs_data): ap = obs_data.apriori[:,:,i].values oz = oz + ap*(1-eff) + (eff)*(add) - ds = xr.Dataset({'o3vmr': (['x','y'],oz), - 'ozone_column':(['x','y'],obs_data.ozone_column.values) + ds = xr.Dataset({ozone_ppbv_varname[0]: (['time','y'],oz), + 'ozone_column':(['time','y'],obs_data.ozone_column.values) }, coords={ - 'longitude':(['x','y'],obs_data['longitude'].values), - 'latitude':(['x','y'],obs_data['latitude'].values), - 'time':(['x'],obs_data.time.values), + 'longitude':(['time','y'],obs_data['longitude'].values), + 'latitude':(['time','y'],obs_data['latitude'].values), + 'time':(['time'],obs_data.time.values), }) return ds diff --git a/melodies_monet/util/time_interval_subset.py b/melodies_monet/util/time_interval_subset.py new file mode 100644 index 00000000..23abb924 --- /dev/null +++ b/melodies_monet/util/time_interval_subset.py @@ -0,0 +1,33 @@ +def subset_model_filelist(all_files,timeformat,timestep,timeinterval): + '''Subset model filelist to within a given time interval. + Filename requirements: + - individual files for each timestep + - time must be in filename + ''' + import pandas as pd + subset_interval = pd.date_range(start=timeinterval[0],end=timeinterval[-1],freq=timestep) + interval_files = [] + for i in subset_interval: + flst = [fs for fs in all_files if i.strftime(timeformat) in fs] + if len(flst) == 1: + interval_files.append(flst[0]) + elif len(flst) >1: + print('More than 1 file for {} in listing'.format(i.strftime(timeformat))) + return interval_files + +def subset_OMPS_l2(file_path,timeinterval): + '''Dependent on filenaming convention + ''' + import pandas as pd + from glob import glob + import fnmatch + all_files = glob(file_path) + interval_files = [] + subset_interval = pd.date_range(start=timeinterval[0],end=timeinterval[-1],freq='D',inclusive='left') + + for i in subset_interval: + fst = fnmatch.filter(all_files,'*OMPS-NPP_NMTO3-L2_v*_{}*_o*'.format(i.strftime('%Ym%m%d'))) + fst.sort() + for j in fst: + interval_files.append(j) + return interval_files \ No newline at end of file