From b3381d9c085d55845cd436158749490e9279c545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 16 May 2021 10:51:25 +0200 Subject: [PATCH 1/4] ENH: - implement a general OnDemandImport class and migrate existing unique classes in yt.utilities.on_demand_import - extend usage of the new on-demand-import mechanism to second class optional deps --- .../construction_data_containers.py | 2 +- yt/data_objects/data_containers.py | 14 +- yt/data_objects/particle_trajectories.py | 2 +- yt/data_objects/profiles.py | 2 +- .../selection_objects/cut_region.py | 6 +- .../selection_objects/spheroids.py | 4 +- yt/fields/xray_emission_fields.py | 2 +- yt/frontends/amrvac/io.py | 2 +- .../amrvac/tests/test_read_amrvac_namelist.py | 2 +- yt/frontends/arepo/data_structures.py | 2 +- yt/frontends/arepo/io.py | 2 +- yt/frontends/chombo/data_structures.py | 2 +- yt/frontends/eagle/data_structures.py | 2 +- yt/frontends/enzo/data_structures.py | 2 +- yt/frontends/enzo/io.py | 2 +- yt/frontends/enzo_e/data_structures.py | 2 +- yt/frontends/enzo_e/io.py | 2 +- yt/frontends/enzo_e/tests/test_outputs.py | 2 +- yt/frontends/fits/data_structures.py | 30 +- yt/frontends/fits/misc.py | 8 +- yt/frontends/gadget/data_structures.py | 2 +- yt/frontends/gadget/io.py | 2 +- yt/frontends/gadget_fof/data_structures.py | 2 +- yt/frontends/gadget_fof/io.py | 2 +- yt/frontends/gdf/data_structures.py | 2 +- yt/frontends/gdf/io.py | 2 +- yt/frontends/gizmo/data_structures.py | 2 +- yt/frontends/halo_catalog/data_structures.py | 2 +- yt/frontends/halo_catalog/io.py | 2 +- yt/frontends/http_stream/data_structures.py | 2 +- yt/frontends/http_stream/io.py | 2 +- yt/frontends/moab/data_structures.py | 2 +- yt/frontends/open_pmd/data_structures.py | 2 +- yt/frontends/open_pmd/fields.py | 2 +- yt/frontends/owls/data_structures.py | 2 +- yt/frontends/owls/owls_ion_tables.py | 2 +- yt/frontends/owls_subfind/data_structures.py | 2 +- yt/frontends/owls_subfind/io.py | 2 +- yt/frontends/ramses/data_structures.py | 2 +- yt/frontends/ramses/tests/test_outputs.py | 2 +- yt/frontends/sdf/data_structures.py | 2 +- yt/frontends/swift/data_structures.py | 2 +- yt/frontends/swift/io.py | 2 +- yt/frontends/swift/tests/test_outputs.py | 2 +- yt/frontends/ytdata/data_structures.py | 2 +- yt/frontends/ytdata/io.py | 2 +- yt/frontends/ytdata/utilities.py | 2 +- yt/funcs.py | 2 +- yt/geometry/geometry_handler.py | 2 +- yt/loaders.py | 2 +- yt/sample_data/api.py | 6 +- yt/startup_tasks.py | 23 +- yt/testing.py | 22 +- yt/utilities/amr_kdtree/amr_kdtree.py | 2 +- .../answer_testing/testing_utilities.py | 2 +- yt/utilities/command_line.py | 12 +- yt/utilities/file_handler.py | 16 +- .../conversion/conversion_athena.py | 2 +- .../grid_data_format/tests/test_writer.py | 2 +- yt/utilities/grid_data_format/writer.py | 2 +- yt/utilities/io_handler.py | 2 +- yt/utilities/minimal_representation.py | 2 +- yt/utilities/on_demand_imports.py | 724 ++---------------- .../parallel_analysis_interface.py | 5 +- yt/utilities/rpdb.py | 5 +- yt/utilities/tests/test_cosmology.py | 2 +- yt/visualization/fits_image.py | 36 +- yt/visualization/fixed_resolution.py | 2 +- yt/visualization/fixed_resolution_filters.py | 4 +- yt/visualization/geo_plot_utils.py | 2 +- yt/visualization/mapserver/pannable_map.py | 2 +- yt/visualization/tests/test_fits_image.py | 6 +- .../tests/test_geo_projections.py | 8 +- .../volume_rendering/image_handling.py | 2 +- .../tests/test_off_axis_SPH.py | 9 +- 75 files changed, 230 insertions(+), 822 deletions(-) diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index 76e164e6384..b1c4c91ecd2 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -2595,7 +2595,7 @@ def export_sketchfab( @parallel_root_only def _upload_to_sketchfab(self, data, files): - from yt.utilities.on_demand_imports import _requests as requests + from yt.utilities.on_demand_imports import requests SKETCHFAB_DOMAIN = "sketchfab.com" SKETCHFAB_API_URL = f"https://api.{SKETCHFAB_DOMAIN}/v2/models" diff --git a/yt/data_objects/data_containers.py b/yt/data_objects/data_containers.py index 923f15f8910..389044897da 100644 --- a/yt/data_objects/data_containers.py +++ b/yt/data_objects/data_containers.py @@ -23,7 +23,7 @@ YTSpatialFieldUnitError, ) from yt.utilities.object_registries import data_object_registry -from yt.utilities.on_demand_imports import _firefly as firefly +from yt.utilities.on_demand_imports import Firefly from yt.utilities.parameter_file_storage import ParameterFileStore @@ -518,7 +518,7 @@ def to_dataframe(self, fields): >>> dd = ds.all_data() >>> df = dd.to_dataframe([("gas", "density"), ("gas", "temperature")]) """ - from yt.utilities.on_demand_imports import _pandas as pd + from yt.utilities.on_demand_imports import pandas as pd data = {} fields = self._determine_fields(fields) @@ -794,8 +794,8 @@ def create_firefly_object( ## for safety, in case someone passes a float just cast it default_decimation_factor = int(default_decimation_factor) - ## initialize a firefly reader instance - reader = firefly.data_reader.Reader( + ## initialize a Firefly reader instance + reader = Firefly.data_reader.Reader( JSONdir=JSONdir, clean_JSONdir=True, **kwargs ) @@ -853,8 +853,8 @@ def create_firefly_object( tracked_filter_flags = np.ones(len(tracked_names)) tracked_colormap_flags = np.ones(len(tracked_names)) - ## create a firefly ParticleGroup for this particle type - pg = firefly.data_reader.ParticleGroup( + ## create a Firefly ParticleGroup for this particle type + pg = Firefly.data_reader.ParticleGroup( UIname=ptype, coordinates=self[ptype, "relative_particle_position"].in_units( coordinate_units @@ -866,7 +866,7 @@ def create_firefly_object( decimation_factor=default_decimation_factor, ) - ## bind this particle group to the firefly reader object + ## bind this particle group to the Firefly reader object reader.addParticleGroup(pg) return reader diff --git a/yt/data_objects/particle_trajectories.py b/yt/data_objects/particle_trajectories.py index 5d988d6d77a..93024e8d532 100644 --- a/yt/data_objects/particle_trajectories.py +++ b/yt/data_objects/particle_trajectories.py @@ -6,7 +6,7 @@ from yt.units.yt_array import array_like_field from yt.utilities.exceptions import YTIllDefinedParticleData from yt.utilities.lib.particle_mesh_operations import CICSample_3 -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from yt.utilities.parallel_tools.parallel_analysis_interface import parallel_root_only diff --git a/yt/data_objects/profiles.py b/yt/data_objects/profiles.py index e704c32edca..6fb5434f035 100644 --- a/yt/data_objects/profiles.py +++ b/yt/data_objects/profiles.py @@ -594,7 +594,7 @@ def to_dataframe(self, fields=None, only_used=False): >>> df1 = p.to_dataframe() >>> df2 = p.to_dataframe(fields=("gas", "density"), only_used=True) """ - from yt.utilities.on_demand_imports import _pandas as pd + from yt.utilities.on_demand_imports import pandas as pd idxs, masked, fields = self._export_prep(fields, only_used) pdata = {self.x_field[-1]: self.x[idxs]} diff --git a/yt/data_objects/selection_objects/cut_region.py b/yt/data_objects/selection_objects/cut_region.py index 812f515c5d9..681425c58f9 100644 --- a/yt/data_objects/selection_objects/cut_region.py +++ b/yt/data_objects/selection_objects/cut_region.py @@ -11,7 +11,7 @@ from yt.funcs import iter_fields, validate_object, validate_sequence from yt.geometry.selection_routines import points_in_cells from yt.utilities.exceptions import YTIllDefinedCutRegion -from yt.utilities.on_demand_imports import _scipy +from yt.utilities.on_demand_imports import scipy class YTCutRegion(YTSelectionContainer3D): @@ -196,7 +196,7 @@ def _part_ind_KDTree(self, ptype): dx_loc = dx[lvl_mask] pos_loc = pos[lvl_mask] - grid_tree = _scipy.spatial.cKDTree(pos_loc, boxsize=1) + grid_tree = scipy.spatial.cKDTree(pos_loc, boxsize=1) # Compute closest cell for all remaining particles dist, icell = grid_tree.query( @@ -238,7 +238,7 @@ def _part_ind(self, ptype): # implementation. Else, fall back onto the direct # brute-force algorithm. try: - _scipy.spatial.KDTree + scipy.spatial.KDTree return self._part_ind_KDTree(ptype) except ImportError: return self._part_ind_brute_force(ptype) diff --git a/yt/data_objects/selection_objects/spheroids.py b/yt/data_objects/selection_objects/spheroids.py index 728d698a823..a23a9722573 100644 --- a/yt/data_objects/selection_objects/spheroids.py +++ b/yt/data_objects/selection_objects/spheroids.py @@ -17,7 +17,7 @@ from yt.utilities.exceptions import YTEllipsoidOrdering, YTException, YTSphereTooSmall from yt.utilities.logger import ytLogger as mylog from yt.utilities.math_utils import get_rotation_matrix -from yt.utilities.on_demand_imports import _miniball +from yt.utilities.on_demand_imports import miniball class YTSphere(YTSelectionContainer3D): @@ -108,7 +108,7 @@ def __init__(self, points, ds=None, field_parameters=None, data_source=None): f"Not enough points. Expected at least 2, got {len(points)}" ) mylog.debug("Building minimal sphere around points.") - mb = _miniball.Miniball(points) + mb = miniball.Miniball(points) if not mb.is_valid(): raise YTException("Could not build valid sphere around points.") diff --git a/yt/fields/xray_emission_fields.py b/yt/fields/xray_emission_fields.py index 60532aec855..2a49dd1f878 100644 --- a/yt/fields/xray_emission_fields.py +++ b/yt/fields/xray_emission_fields.py @@ -12,7 +12,7 @@ BilinearFieldInterpolator, UnilinearFieldInterpolator, ) -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py data_version = {"cloudy": 2, "apec": 3} diff --git a/yt/frontends/amrvac/io.py b/yt/frontends/amrvac/io.py index 5fbc4d0c866..e1fd5524588 100644 --- a/yt/frontends/amrvac/io.py +++ b/yt/frontends/amrvac/io.py @@ -11,7 +11,7 @@ from yt.geometry.selection_routines import GridSelector from yt.utilities.io_handler import BaseIOHandler -from yt.utilities.on_demand_imports import _f90nml as f90nml +from yt.utilities.on_demand_imports import f90nml def read_amrvac_namelist(parfiles): diff --git a/yt/frontends/amrvac/tests/test_read_amrvac_namelist.py b/yt/frontends/amrvac/tests/test_read_amrvac_namelist.py index d3d11d5158f..e40390806a9 100644 --- a/yt/frontends/amrvac/tests/test_read_amrvac_namelist.py +++ b/yt/frontends/amrvac/tests/test_read_amrvac_namelist.py @@ -4,7 +4,7 @@ from yt.frontends.amrvac.api import read_amrvac_namelist from yt.testing import requires_module -from yt.utilities.on_demand_imports import _f90nml as f90nml +from yt.utilities.on_demand_imports import f90nml test_dir = os.path.dirname(os.path.abspath(__file__)) blast_wave_parfile = os.path.join(test_dir, "sample_parfiles", "bw_3d.par") diff --git a/yt/frontends/arepo/data_structures.py b/yt/frontends/arepo/data_structures.py index 66702ec3cdc..13e58c3e42d 100644 --- a/yt/frontends/arepo/data_structures.py +++ b/yt/frontends/arepo/data_structures.py @@ -2,7 +2,7 @@ from yt.frontends.gadget.api import GadgetHDF5Dataset from yt.funcs import mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from .fields import ArepoFieldInfo diff --git a/yt/frontends/arepo/io.py b/yt/frontends/arepo/io.py index 0af4b993016..24402aeb83a 100644 --- a/yt/frontends/arepo/io.py +++ b/yt/frontends/arepo/io.py @@ -1,7 +1,7 @@ import numpy as np from yt.frontends.gadget.api import IOHandlerGadgetHDF5 -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py class IOHandlerArepoHDF5(IOHandlerGadgetHDF5): diff --git a/yt/frontends/chombo/data_structures.py b/yt/frontends/chombo/data_structures.py index 5b06188ac23..ae965571856 100644 --- a/yt/frontends/chombo/data_structures.py +++ b/yt/frontends/chombo/data_structures.py @@ -13,7 +13,7 @@ from yt.geometry.grid_geometry_handler import GridIndex from yt.utilities.file_handler import HDF5FileHandler, warn_h5py from yt.utilities.lib.misc_utilities import get_box_grids_level -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from yt.utilities.parallel_tools.parallel_analysis_interface import parallel_root_only from .fields import ( diff --git a/yt/frontends/eagle/data_structures.py b/yt/frontends/eagle/data_structures.py index 92046e4e3fa..c18fe0edcea 100644 --- a/yt/frontends/eagle/data_structures.py +++ b/yt/frontends/eagle/data_structures.py @@ -6,7 +6,7 @@ from yt.fields.field_info_container import FieldInfoContainer from yt.frontends.gadget.data_structures import GadgetHDF5Dataset from yt.frontends.owls.fields import OWLSFieldInfo -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from .fields import EagleNetworkFieldInfo diff --git a/yt/frontends/enzo/data_structures.py b/yt/frontends/enzo/data_structures.py index 114dc5240aa..58773af872b 100644 --- a/yt/frontends/enzo/data_structures.py +++ b/yt/frontends/enzo/data_structures.py @@ -16,7 +16,7 @@ from yt.geometry.geometry_handler import YTDataChunk from yt.geometry.grid_geometry_handler import GridIndex from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py, _libconf as libconf +from yt.utilities.on_demand_imports import h5py, libconf from .fields import EnzoFieldInfo diff --git a/yt/frontends/enzo/io.py b/yt/frontends/enzo/io.py index 9e6112bfedc..a25d43b7f99 100644 --- a/yt/frontends/enzo/io.py +++ b/yt/frontends/enzo/io.py @@ -5,7 +5,7 @@ from yt.geometry.selection_routines import GridSelector from yt.utilities.io_handler import BaseIOHandler from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py _convert_mass = ("particle_mass", "mass") diff --git a/yt/frontends/enzo_e/data_structures.py b/yt/frontends/enzo_e/data_structures.py index ac15e937a38..8ae0ab427a2 100644 --- a/yt/frontends/enzo_e/data_structures.py +++ b/yt/frontends/enzo_e/data_structures.py @@ -19,7 +19,7 @@ from yt.geometry.grid_geometry_handler import GridIndex from yt.utilities.cosmology import Cosmology from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py, _libconf as libconf +from yt.utilities.on_demand_imports import h5py, libconf class EnzoEGrid(AMRGridPatch): diff --git a/yt/frontends/enzo_e/io.py b/yt/frontends/enzo_e/io.py index f9a370de349..7e083ad9122 100644 --- a/yt/frontends/enzo_e/io.py +++ b/yt/frontends/enzo_e/io.py @@ -2,7 +2,7 @@ from yt.utilities.exceptions import YTException from yt.utilities.io_handler import BaseIOHandler -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py class EnzoEIOHandler(BaseIOHandler): diff --git a/yt/frontends/enzo_e/tests/test_outputs.py b/yt/frontends/enzo_e/tests/test_outputs.py index c5046594739..418ae2db0dd 100644 --- a/yt/frontends/enzo_e/tests/test_outputs.py +++ b/yt/frontends/enzo_e/tests/test_outputs.py @@ -10,7 +10,7 @@ data_dir_load, requires_ds, ) -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py _fields = ( ("gas", "density"), diff --git a/yt/frontends/fits/data_structures.py b/yt/frontends/fits/data_structures.py index e06b0c3e9fb..3d77544674f 100644 --- a/yt/frontends/fits/data_structures.py +++ b/yt/frontends/fits/data_structures.py @@ -27,7 +27,7 @@ from yt.utilities.decompose import decompose_array, get_psize from yt.utilities.file_handler import FITSFileHandler from yt.utilities.io_handler import io_registry -from yt.utilities.on_demand_imports import NotAModule, _astropy +from yt.utilities.on_demand_imports import astropy from .fields import FITSFieldInfo, WCSFITSFieldInfo, YTFITSFieldInfo @@ -88,7 +88,7 @@ def _determine_image_units(self, bunit): try: try: # First let AstroPy attempt to figure the unit out - u = 1.0 * _astropy.units.Unit(bunit, format="fits") + u = 1.0 * astropy.units.Unit(bunit, format="fits") u = YTQuantity.from_astropy(u).units except ValueError: try: @@ -133,7 +133,7 @@ def _detect_output_fields(self): for i, fits_file in enumerate(self.dataset._handle._fits_files): for j, hdu in enumerate(fits_file): if ( - isinstance(hdu, _astropy.pyfits.BinTableHDU) + isinstance(hdu, astropy.io.fits.BinTableHDU) or hdu.header["naxis"] == 0 ): continue @@ -274,14 +274,10 @@ def check_fits_valid(filename): ext = filename.rsplit(".", 1)[0].rsplit(".", 1)[-1] if ext.upper() not in ("FITS", "FTS"): return None - elif isinstance(_astropy.pyfits, NotAModule): - raise RuntimeError( - "This appears to be a FITS file, but AstroPy is not installed." - ) try: with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=UserWarning, append=True) - fileh = _astropy.pyfits.open(filename) + fileh = astropy.io.fits.open(filename) header, _ = find_primary_header(fileh) if header["naxis"] >= 2: return fileh @@ -355,24 +351,24 @@ def __init__( self.nan_mask = nan_mask self._handle = FITSFileHandler(self.filenames[0]) if isinstance( - self.filenames[0], _astropy.pyfits.hdu.image._ImageBaseHDU - ) or isinstance(self.filenames[0], _astropy.pyfits.HDUList): + self.filenames[0], astropy.io.fits.hdu.image._ImageBaseHDU + ) or isinstance(self.filenames[0], astropy.io.fits.HDUList): fn = f"InMemoryFITSFile_{uuid.uuid4().hex}" else: fn = self.filenames[0] self._handle._fits_files.append(self._handle) if self.num_files > 1: for fits_file in auxiliary_files: - if isinstance(fits_file, _astropy.pyfits.hdu.image._ImageBaseHDU): - f = _astropy.pyfits.HDUList([fits_file]) - elif isinstance(fits_file, _astropy.pyfits.HDUList): + if isinstance(fits_file, astropy.io.fits.hdu.image._ImageBaseHDU): + f = astropy.io.fits.HDUList([fits_file]) + elif isinstance(fits_file, astropy.io.fits.HDUList): f = fits_file else: if os.path.exists(fits_file): fn = fits_file else: fn = os.path.join(ytcfg.get("yt", "test_data_dir"), fits_file) - f = _astropy.pyfits.open( + f = astropy.io.fits.open( fn, memmap=True, do_not_scale_image_data=True, ignore_blank=True ) self._handle._fits_files.append(f) @@ -502,9 +498,9 @@ def _determine_structure(self): ] def _determine_wcs(self): - wcs = _astropy.pywcs.WCS(header=self.primary_header) + wcs = astropy.wcs.WCS(header=self.primary_header) if self.naxis == 4: - self.wcs = _astropy.pywcs.WCS(naxis=3) + self.wcs = astropy.wcs.WCS(naxis=3) self.wcs.wcs.crpix = wcs.wcs.crpix[:3] self.wcs.wcs.cdelt = wcs.wcs.cdelt[:3] self.wcs.wcs.crval = wcs.wcs.crval[:3] @@ -873,7 +869,7 @@ def _determine_structure(self): self.naxis = 2 def _determine_wcs(self): - self.wcs = _astropy.pywcs.WCS(naxis=2) + self.wcs = astropy.wcs.WCS(naxis=2) self.events_info = {} for k, v in self.primary_header.items(): if k.startswith("TTYP"): diff --git a/yt/frontends/fits/misc.py b/yt/frontends/fits/misc.py index 29d5381de2c..363a630279d 100644 --- a/yt/frontends/fits/misc.py +++ b/yt/frontends/fits/misc.py @@ -7,7 +7,7 @@ from yt.fields.derived_field import ValidateSpatial from yt.units.yt_array import YTArray, YTQuantity from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _astropy +from yt.utilities.on_demand_imports import astropy def _make_counts(emin, emax): @@ -24,8 +24,8 @@ def _counts(field, data): else: sigma = None if sigma is not None and sigma > 0.0: - kern = _astropy.conv.Gaussian2DKernel(x_stddev=sigma) - img[:, :, 0] = _astropy.conv.convolve(img[:, :, 0], kern) + kern = astropy.convolution.Gaussian2DKernel(x_stddev=sigma) + img[:, :, 0] = astropy.convolution.convolve(img[:, :, 0], kern) return data.ds.arr(img, "counts/pixel") return _counts @@ -209,7 +209,7 @@ class PlotWindowWCS: """ def __init__(self, pw): - WCSAxes = _astropy.wcsaxes.WCSAxes + WCSAxes = astropy.visualization.wcsaxes.WCSAxes if pw.oblique: raise NotImplementedError("WCS axes are not implemented for oblique plots.") diff --git a/yt/frontends/gadget/data_structures.py b/yt/frontends/gadget/data_structures.py index 1e4f8057eca..6828a6a852d 100644 --- a/yt/frontends/gadget/data_structures.py +++ b/yt/frontends/gadget/data_structures.py @@ -14,7 +14,7 @@ from yt.utilities.cosmology import Cosmology from yt.utilities.fortran_utils import read_record from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from .definitions import ( SNAP_FORMAT_2_OFFSET, diff --git a/yt/frontends/gadget/io.py b/yt/frontends/gadget/io.py index 7c27d8f13d5..61cc570f6fe 100644 --- a/yt/frontends/gadget/io.py +++ b/yt/frontends/gadget/io.py @@ -7,7 +7,7 @@ from yt.units.yt_array import uconcatenate # type: ignore from yt.utilities.lib.particle_kdtree_tools import generate_smoothing_length from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from .definitions import SNAP_FORMAT_2_OFFSET, gadget_hdf5_ptypes diff --git a/yt/frontends/gadget_fof/data_structures.py b/yt/frontends/gadget_fof/data_structures.py index bd357f4bfb4..375d23ea405 100644 --- a/yt/frontends/gadget_fof/data_structures.py +++ b/yt/frontends/gadget_fof/data_structures.py @@ -16,7 +16,7 @@ from yt.geometry.particle_geometry_handler import ParticleIndex from yt.utilities.cosmology import Cosmology from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py class GadgetFOFParticleIndex(ParticleIndex): diff --git a/yt/frontends/gadget_fof/io.py b/yt/frontends/gadget_fof/io.py index a88db213075..8c71a5e7962 100644 --- a/yt/frontends/gadget_fof/io.py +++ b/yt/frontends/gadget_fof/io.py @@ -4,7 +4,7 @@ from yt.funcs import mylog from yt.utilities.io_handler import BaseIOHandler -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py class IOHandlerGadgetFOFHDF5(BaseIOHandler): diff --git a/yt/frontends/gdf/data_structures.py b/yt/frontends/gdf/data_structures.py index 91e8edde86c..8cd2c4cf6b8 100644 --- a/yt/frontends/gdf/data_structures.py +++ b/yt/frontends/gdf/data_structures.py @@ -13,7 +13,7 @@ from yt.utilities.exceptions import YTGDFUnknownGeometry from yt.utilities.lib.misc_utilities import get_box_grids_level from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from .fields import GDFFieldInfo diff --git a/yt/frontends/gdf/io.py b/yt/frontends/gdf/io.py index c4f9ecfb26c..4dd4283d5ef 100644 --- a/yt/frontends/gdf/io.py +++ b/yt/frontends/gdf/io.py @@ -3,7 +3,7 @@ from yt.funcs import mylog from yt.geometry.selection_routines import GridSelector from yt.utilities.io_handler import BaseIOHandler -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py def _grid_dname(grid_id): diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index 63af45c8db2..1225aec63ec 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -1,7 +1,7 @@ import os from yt.frontends.gadget.data_structures import GadgetHDF5Dataset -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from .fields import GizmoFieldInfo diff --git a/yt/frontends/halo_catalog/data_structures.py b/yt/frontends/halo_catalog/data_structures.py index b2d713cc9ff..7872f61805f 100644 --- a/yt/frontends/halo_catalog/data_structures.py +++ b/yt/frontends/halo_catalog/data_structures.py @@ -16,7 +16,7 @@ from yt.frontends.ytdata.data_structures import SavedDataset from yt.funcs import parse_h5_attr from yt.geometry.particle_geometry_handler import ParticleIndex -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from .fields import YTHaloCatalogFieldInfo, YTHaloCatalogHaloFieldInfo diff --git a/yt/frontends/halo_catalog/io.py b/yt/frontends/halo_catalog/io.py index c4315d0cc5d..5c86f2d0d9a 100644 --- a/yt/frontends/halo_catalog/io.py +++ b/yt/frontends/halo_catalog/io.py @@ -6,7 +6,7 @@ from yt.funcs import parse_h5_attr from yt.units.yt_array import uvstack # type: ignore from yt.utilities.io_handler import BaseIOHandler -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py class IOHandlerYTHaloCatalog(BaseIOHandler): diff --git a/yt/frontends/http_stream/data_structures.py b/yt/frontends/http_stream/data_structures.py index 5507f91bb40..0abc8d064e7 100644 --- a/yt/frontends/http_stream/data_structures.py +++ b/yt/frontends/http_stream/data_structures.py @@ -6,7 +6,7 @@ from yt.data_objects.static_output import ParticleDataset, ParticleFile from yt.frontends.sph.fields import SPHFieldInfo from yt.geometry.particle_geometry_handler import ParticleIndex -from yt.utilities.on_demand_imports import _requests as requests +from yt.utilities.on_demand_imports import requests class HTTPParticleFile(ParticleFile): diff --git a/yt/frontends/http_stream/io.py b/yt/frontends/http_stream/io.py index a13001bc15b..1f0e8d0ae4b 100644 --- a/yt/frontends/http_stream/io.py +++ b/yt/frontends/http_stream/io.py @@ -2,7 +2,7 @@ from yt.funcs import mylog from yt.utilities.io_handler import BaseIOHandler -from yt.utilities.on_demand_imports import _requests as requests +from yt.utilities.on_demand_imports import requests class IOHandlerHTTPStream(BaseIOHandler): diff --git a/yt/frontends/moab/data_structures.py b/yt/frontends/moab/data_structures.py index 55ec5dd8017..658bcf8dcbe 100644 --- a/yt/frontends/moab/data_structures.py +++ b/yt/frontends/moab/data_structures.py @@ -8,7 +8,7 @@ from yt.funcs import setdefaultattr from yt.geometry.unstructured_mesh_handler import UnstructuredIndex from yt.utilities.file_handler import HDF5FileHandler -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from .fields import MoabFieldInfo, PyneFieldInfo diff --git a/yt/frontends/open_pmd/data_structures.py b/yt/frontends/open_pmd/data_structures.py index 3830b551eb3..686a483a6c1 100644 --- a/yt/frontends/open_pmd/data_structures.py +++ b/yt/frontends/open_pmd/data_structures.py @@ -16,7 +16,7 @@ from yt.geometry.grid_geometry_handler import GridIndex from yt.utilities.file_handler import HDF5FileHandler, warn_h5py from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py ompd_known_versions = [Version(_) for _ in ("1.0.0", "1.0.1", "1.1.0")] opmd_required_attributes = ["openPMD", "basePath"] diff --git a/yt/frontends/open_pmd/fields.py b/yt/frontends/open_pmd/fields.py index 1edfe86fd2b..ae18f29b9e6 100644 --- a/yt/frontends/open_pmd/fields.py +++ b/yt/frontends/open_pmd/fields.py @@ -7,7 +7,7 @@ from yt.frontends.open_pmd.misc import is_const_component, parse_unit_dimension from yt.units.yt_array import YTQuantity from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from yt.utilities.physical_constants import mu_0, speed_of_light diff --git a/yt/frontends/owls/data_structures.py b/yt/frontends/owls/data_structures.py index 498edd265fd..76021dae079 100644 --- a/yt/frontends/owls/data_structures.py +++ b/yt/frontends/owls/data_structures.py @@ -3,7 +3,7 @@ import yt.units from yt.frontends.gadget.data_structures import GadgetHDF5Dataset from yt.utilities.definitions import sec_conversion -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from .fields import OWLSFieldInfo diff --git a/yt/frontends/owls/owls_ion_tables.py b/yt/frontends/owls/owls_ion_tables.py index 2e99e09c012..20bd000d626 100644 --- a/yt/frontends/owls/owls_ion_tables.py +++ b/yt/frontends/owls/owls_ion_tables.py @@ -1,6 +1,6 @@ import numpy as np -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py def h5rd(fname, path, dtype=None): diff --git a/yt/frontends/owls_subfind/data_structures.py b/yt/frontends/owls_subfind/data_structures.py index 55eee574423..c4a95f9b6b5 100644 --- a/yt/frontends/owls_subfind/data_structures.py +++ b/yt/frontends/owls_subfind/data_structures.py @@ -10,7 +10,7 @@ from yt.geometry.particle_geometry_handler import ParticleIndex from yt.utilities.exceptions import YTException from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from .fields import OWLSSubfindFieldInfo diff --git a/yt/frontends/owls_subfind/io.py b/yt/frontends/owls_subfind/io.py index bfdb5e0fbe9..794e72d1be1 100644 --- a/yt/frontends/owls_subfind/io.py +++ b/yt/frontends/owls_subfind/io.py @@ -2,7 +2,7 @@ from yt.funcs import mylog from yt.utilities.io_handler import BaseIOHandler -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py _pos_names = ["CenterOfMass", "CentreOfMass"] diff --git a/yt/frontends/ramses/data_structures.py b/yt/frontends/ramses/data_structures.py index ebfcfefd957..9decc89e652 100644 --- a/yt/frontends/ramses/data_structures.py +++ b/yt/frontends/ramses/data_structures.py @@ -15,7 +15,7 @@ from yt.geometry.oct_geometry_handler import OctreeIndex from yt.utilities.cython_fortran_utils import FortranFile as fpu from yt.utilities.lib.cosmology_time import friedman -from yt.utilities.on_demand_imports import _f90nml as f90nml +from yt.utilities.on_demand_imports import f90nml from yt.utilities.physical_constants import kb, mp from .definitions import ( diff --git a/yt/frontends/ramses/tests/test_outputs.py b/yt/frontends/ramses/tests/test_outputs.py index 49a6ce2a6de..8be12cb8eb0 100644 --- a/yt/frontends/ramses/tests/test_outputs.py +++ b/yt/frontends/ramses/tests/test_outputs.py @@ -21,7 +21,7 @@ data_dir_load, requires_ds, ) -from yt.utilities.on_demand_imports import _f90nml as f90nml +from yt.utilities.on_demand_imports import f90nml _fields = ( ("gas", "temperature"), diff --git a/yt/frontends/sdf/data_structures.py b/yt/frontends/sdf/data_structures.py index c716a042bc7..d3c9a7ca090 100644 --- a/yt/frontends/sdf/data_structures.py +++ b/yt/frontends/sdf/data_structures.py @@ -6,7 +6,7 @@ from yt.funcs import setdefaultattr from yt.geometry.particle_geometry_handler import ParticleIndex from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _requests as requests +from yt.utilities.on_demand_imports import requests from yt.utilities.sdf import HTTPSDFRead, SDFIndex, SDFRead from .fields import SDFFieldInfo diff --git a/yt/frontends/swift/data_structures.py b/yt/frontends/swift/data_structures.py index 666511d348b..c396f826a40 100644 --- a/yt/frontends/swift/data_structures.py +++ b/yt/frontends/swift/data_structures.py @@ -7,7 +7,7 @@ from yt.frontends.sph.fields import SPHFieldInfo from yt.funcs import only_on_root from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py class SwiftDataset(SPHDataset): diff --git a/yt/frontends/swift/io.py b/yt/frontends/swift/io.py index 1f14c360ae7..b1fd4346571 100644 --- a/yt/frontends/swift/io.py +++ b/yt/frontends/swift/io.py @@ -1,7 +1,7 @@ import numpy as np from yt.frontends.sph.io import IOHandlerSPH -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py class IOHandlerSwift(IOHandlerSPH): diff --git a/yt/frontends/swift/tests/test_outputs.py b/yt/frontends/swift/tests/test_outputs.py index fecdafc71de..abfc6a0a7f2 100644 --- a/yt/frontends/swift/tests/test_outputs.py +++ b/yt/frontends/swift/tests/test_outputs.py @@ -3,7 +3,7 @@ from yt import load from yt.frontends.swift.api import SwiftDataset from yt.testing import ParticleSelectionComparison, assert_almost_equal, requires_file -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py keplerian_ring = "KeplerianRing/keplerian_ring_0020.hdf5" EAGLE_6 = "EAGLE_6/eagle_0005.hdf5" diff --git a/yt/frontends/ytdata/data_structures.py b/yt/frontends/ytdata/data_structures.py index 5af309c52d3..194867d67e2 100644 --- a/yt/frontends/ytdata/data_structures.py +++ b/yt/frontends/ytdata/data_structures.py @@ -25,7 +25,7 @@ from yt.units.yt_array import YTQuantity, uconcatenate # type: ignore from yt.utilities.exceptions import GenerationInProgress, YTFieldTypeNotFound from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from yt.utilities.parallel_tools.parallel_analysis_interface import parallel_root_only from yt.utilities.tree_container import TreeContainer diff --git a/yt/frontends/ytdata/io.py b/yt/frontends/ytdata/io.py index 9104f3207e8..59490d68718 100644 --- a/yt/frontends/ytdata/io.py +++ b/yt/frontends/ytdata/io.py @@ -4,7 +4,7 @@ from yt.geometry.selection_routines import GridSelector from yt.units.yt_array import uvstack # type: ignore from yt.utilities.io_handler import BaseIOHandler -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py class IOHandlerYTNonspatialhdf5(BaseIOHandler): diff --git a/yt/frontends/ytdata/utilities.py b/yt/frontends/ytdata/utilities.py index 49e572bac71..cf4a087f55e 100644 --- a/yt/frontends/ytdata/utilities.py +++ b/yt/frontends/ytdata/utilities.py @@ -1,6 +1,6 @@ from yt.units.yt_array import YTArray from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py def save_as_dataset(ds, filename, data, field_types=None, extra_attrs=None): diff --git a/yt/funcs.py b/yt/funcs.py index 14d7c57659c..6eb4207d70a 100644 --- a/yt/funcs.py +++ b/yt/funcs.py @@ -31,7 +31,7 @@ from yt.units import YTArray, YTQuantity from yt.utilities.exceptions import YTInvalidWidthError from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _requests as requests +from yt.utilities.on_demand_imports import requests # Some functions for handling sequences and other types diff --git a/yt/geometry/geometry_handler.py b/yt/geometry/geometry_handler.py index fa6e6a42233..f9c102bd4b4 100644 --- a/yt/geometry/geometry_handler.py +++ b/yt/geometry/geometry_handler.py @@ -10,7 +10,7 @@ from yt.utilities.exceptions import YTFieldNotFound from yt.utilities.io_handler import io_registry from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from yt.utilities.parallel_tools.parallel_analysis_interface import ( ParallelAnalysisInterface, parallel_root_only, diff --git a/yt/loaders.py b/yt/loaders.py index e63069d2202..0b4cf59102c 100644 --- a/yt/loaders.py +++ b/yt/loaders.py @@ -28,7 +28,7 @@ output_type_registry, simulation_time_series_registry, ) -from yt.utilities.on_demand_imports import _pooch as pooch +from yt.utilities.on_demand_imports import pooch # --- Loaders for known data formats --- diff --git a/yt/sample_data/api.py b/yt/sample_data/api.py index 2e77133d5a4..adde7b29fd3 100644 --- a/yt/sample_data/api.py +++ b/yt/sample_data/api.py @@ -12,11 +12,7 @@ import pkg_resources from yt.funcs import mylog -from yt.utilities.on_demand_imports import ( - _pandas as pd, - _pooch as pooch, - _requests as requests, -) +from yt.utilities.on_demand_imports import pandas as pd, pooch, requests num_exp = re.compile(r"\d*(\.\d*)?") byte_unit_exp = re.compile(r"[KMGT]?B") diff --git a/yt/startup_tasks.py b/yt/startup_tasks.py index 211bc143739..66c8ae808d4 100644 --- a/yt/startup_tasks.py +++ b/yt/startup_tasks.py @@ -15,21 +15,24 @@ signal_print_traceback, ) from yt.utilities import rpdb +from yt.utilities.on_demand_imports import mpi4py exe_name = os.path.basename(sys.executable) # At import time, we determined whether or not we're being run in parallel. def turn_on_parallelism(): + """ + Activate parallel capabilities. + + Raises + ------ + ImportError if mpi4py isn't installed. + """ parallel_capable = False - try: - # we import this to check if mpi4py is installed - from mpi4py import MPI # NOQA - except ImportError as e: - mylog.error( - "Warning: Attempting to turn on parallelism, " - "but mpi4py import failed. Try pip install mpi4py." - ) - raise e - # Now we have to turn on the parallelism from the perspective of the + + # check that mpi4py is correctly installed + mpi4py.MPI + + # Now we have to turn on the parallelism from the perspective of the # parallel_analysis_interface from yt.utilities.parallel_tools.parallel_analysis_interface import ( enable_parallelism, diff --git a/yt/testing.py b/yt/testing.py index c84e097e1e9..371be2a7f04 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -901,19 +901,17 @@ def requires_module_pytest(*module_names): So that it can be later renamed to `requires_module`. """ - import pytest + from importlib import import_module - from yt.utilities import on_demand_imports as odi + import pytest def deco(func): - required_modules = { - name: getattr(odi, f"_{name}")._module for name in module_names - } - missing = [ - name - for name, mod in required_modules.items() - if isinstance(mod, odi.NotAModule) - ] + missing = [] + for name in module_names: + try: + import_module(name) + except ImportError: + missing.append(name) # note that order between these two decorators matters @pytest.mark.skipif( @@ -1182,7 +1180,7 @@ def run_nose( import sys from yt.utilities.logger import ytLogger as mylog - from yt.utilities.on_demand_imports import _nose + from yt.utilities.on_demand_imports import nose orig_level = mylog.getEffectiveLevel() mylog.setLevel(50) @@ -1217,7 +1215,7 @@ def run_nose( ) os.chdir(yt_dir) try: - _nose.run(argv=nose_argv) + nose.run(argv=nose_argv) finally: os.chdir(initial_dir) mylog.setLevel(orig_level) diff --git a/yt/utilities/amr_kdtree/amr_kdtree.py b/yt/utilities/amr_kdtree/amr_kdtree.py index 546a7c93cdf..18be314913f 100644 --- a/yt/utilities/amr_kdtree/amr_kdtree.py +++ b/yt/utilities/amr_kdtree/amr_kdtree.py @@ -12,7 +12,7 @@ from yt.utilities.lib.amr_kdtools import Node from yt.utilities.lib.partitioned_grid import PartitionedGrid from yt.utilities.math_utils import periodic_position -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from yt.utilities.parallel_tools.parallel_analysis_interface import ( ParallelAnalysisInterface, ) diff --git a/yt/utilities/answer_testing/testing_utilities.py b/yt/utilities/answer_testing/testing_utilities.py index 6f7729b85b1..b4a91d717a8 100644 --- a/yt/utilities/answer_testing/testing_utilities.py +++ b/yt/utilities/answer_testing/testing_utilities.py @@ -14,7 +14,7 @@ from yt.data_objects.selection_objects.region import YTRegion from yt.data_objects.static_output import Dataset from yt.loaders import load, load_simulation -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from yt.visualization.volume_rendering.scene import Scene diff --git a/yt/utilities/command_line.py b/yt/utilities/command_line.py index 4fd72128d2d..656e0f4d756 100644 --- a/yt/utilities/command_line.py +++ b/yt/utilities/command_line.py @@ -27,6 +27,7 @@ from yt.loaders import load from yt.utilities.exceptions import YTFieldNotParseable, YTUnidentifiedDataType from yt.utilities.metadata import get_metadata +from yt.utilities.on_demand_imports import bottle from yt.visualization.plot_window import ProjectionPlot, SlicePlot # isort: off @@ -807,14 +808,7 @@ def __call__(self, args): p.set_cmap("all", args.cmap) PannableMapServer(p.data_source, args.field, args.takelog, args.cmap) - try: - import bottle - except ImportError as e: - raise ImportError( - "The mapserver functionality requires the bottle " - "package to be installed. Please install using `pip " - "install bottle`." - ) from e + bottle.debug(True) if args.host is not None: colonpl = args.host.find(":") @@ -1300,7 +1294,7 @@ class YTUploadFileCmd(YTCommand): name = "upload" def __call__(self, args): - from yt.utilities.on_demand_imports import _requests as requests + from yt.utilities.on_demand_imports import requests fs = iter(FileStreamer(open(args.file, "rb"))) upload_url = ytcfg.get("yt", "curldrop_upload_url") diff --git a/yt/utilities/file_handler.py b/yt/utilities/file_handler.py index 436e52663f5..bf484914099 100644 --- a/yt/utilities/file_handler.py +++ b/yt/utilities/file_handler.py @@ -1,6 +1,6 @@ from contextlib import contextmanager -from yt.utilities.on_demand_imports import NotAModule, _h5py as h5py +from yt.utilities.on_demand_imports import NotAModule, h5py def valid_hdf5_signature(fn): @@ -53,14 +53,14 @@ def close(self): class FITSFileHandler(HDF5FileHandler): def __init__(self, filename): - from yt.utilities.on_demand_imports import _astropy + from yt.utilities.on_demand_imports import astropy - if isinstance(filename, _astropy.pyfits.hdu.image._ImageBaseHDU): - self.handle = _astropy.pyfits.HDUList(filename) - elif isinstance(filename, _astropy.pyfits.HDUList): + if isinstance(filename, astropy.io.fits.hdu.image._ImageBaseHDU): + self.handle = astropy.io.fits.HDUList(filename) + elif isinstance(filename, astropy.io.fits.HDUList): self.handle = filename else: - self.handle = _astropy.pyfits.open( + self.handle = astropy.io.fits.open( filename, memmap=True, do_not_scale_image_data=True, ignore_blank=True ) self._fits_files = [] @@ -94,7 +94,7 @@ def warn_netcdf(fn): # data model used by netCDF-3. netcdf4_classic = valid_hdf5_signature(fn) and fn.endswith((".nc", ".nc4")) needs_netcdf = classic or netcdf4_classic - from yt.utilities.on_demand_imports import _netCDF4 as netCDF4 + from yt.utilities.on_demand_imports import netCDF4 if needs_netcdf and isinstance(netCDF4.Dataset, NotAModule): raise RuntimeError( @@ -109,7 +109,7 @@ def __init__(self, filename): @contextmanager def open_ds(self, **kwargs): - from yt.utilities.on_demand_imports import _netCDF4 as netCDF4 + from yt.utilities.on_demand_imports import netCDF4 ds = netCDF4.Dataset(self.filename, mode="r", **kwargs) yield ds diff --git a/yt/utilities/grid_data_format/conversion/conversion_athena.py b/yt/utilities/grid_data_format/conversion/conversion_athena.py index 84dfa6f3d12..4cc0c105c91 100644 --- a/yt/utilities/grid_data_format/conversion/conversion_athena.py +++ b/yt/utilities/grid_data_format/conversion/conversion_athena.py @@ -3,7 +3,7 @@ import numpy as np from yt.utilities.grid_data_format.conversion.conversion_abc import Converter -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py translation_dict = {} translation_dict["density"] = "density" diff --git a/yt/utilities/grid_data_format/tests/test_writer.py b/yt/utilities/grid_data_format/tests/test_writer.py index a6bf9f4094e..d80fccd5cf7 100644 --- a/yt/utilities/grid_data_format/tests/test_writer.py +++ b/yt/utilities/grid_data_format/tests/test_writer.py @@ -6,7 +6,7 @@ from yt.loaders import load from yt.testing import assert_equal, fake_random_ds, requires_module from yt.utilities.grid_data_format.writer import write_to_gdf -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py TEST_AUTHOR = "yt test runner" TEST_COMMENT = "Testing write_to_gdf" diff --git a/yt/utilities/grid_data_format/writer.py b/yt/utilities/grid_data_format/writer.py index a637a66b12f..f37364d62f3 100644 --- a/yt/utilities/grid_data_format/writer.py +++ b/yt/utilities/grid_data_format/writer.py @@ -7,7 +7,7 @@ from yt import __version__ as yt_version from yt.funcs import iter_fields from yt.utilities.exceptions import YTGDFAlreadyExists -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from yt.utilities.parallel_tools.parallel_analysis_interface import ( communication_system, parallel_objects, diff --git a/yt/utilities/io_handler.py b/yt/utilities/io_handler.py index a9bd252aab4..cad551fcb1d 100644 --- a/yt/utilities/io_handler.py +++ b/yt/utilities/io_handler.py @@ -7,7 +7,7 @@ import numpy as np from yt.geometry.selection_routines import GridSelector -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py io_registry = {} diff --git a/yt/utilities/minimal_representation.py b/yt/utilities/minimal_representation.py index 918038dfc0d..36b38f44ed4 100644 --- a/yt/utilities/minimal_representation.py +++ b/yt/utilities/minimal_representation.py @@ -8,7 +8,7 @@ from yt.funcs import compare_dicts, is_sequence from yt.units.yt_array import YTArray, YTQuantity -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py def _sanitize_list(flist): diff --git a/yt/utilities/on_demand_imports.py b/yt/utilities/on_demand_imports.py index 8d520dac6f2..a56ec7afa1d 100644 --- a/yt/utilities/on_demand_imports.py +++ b/yt/utilities/on_demand_imports.py @@ -1,4 +1,11 @@ import sys +from importlib import import_module +from typing import Optional + +if sys.version_info >= (3, 8): + from functools import cached_property +else: + from yt._maintenance.backports import cached_property class NotAModule: @@ -21,659 +28,76 @@ def __call__(self, *args, **kwargs): raise self.error -class NotCartopy(NotAModule): +class OnDemandImport: """ - A custom class to return error messages dependent on system installation - for cartopy imports. + Implement a safely importable drop-in replacement for a module that + might not be installed, and fail *only* on the first attempt to use it. """ - def __init__(self, pkg_name): - self.pkg_name = pkg_name - if any(s in sys.version for s in ("Anaconda", "Continuum")): - # the conda-based installs of cartopy don't have issues with the - # GEOS library, so the error message for users with conda can be - # relatively short. Discussion related to this is in - # yt-project/yt#1966 - self.error = ImportError( - "This functionality requires the %s " - "package to be installed." % self.pkg_name - ) - else: - self.error = ImportError( - "This functionality requires the %s " - "package to be installed. Try installing proj4 and " - "geos with your package manager and building shapely " - "and cartopy from source with: \n \n " - "python -m pip install --no-binary :all: shapely cartopy \n \n" - "For further instruction please refer to the " - "yt documentation." % self.pkg_name - ) - - -class netCDF4_imports: - _name = "netCDF4" - _Dataset = None - - def __init__(self): - # this ensures the import ordering between netcdf4 and h5py. If h5py is - # imported first, can get file lock errors on some systems (including travis-ci) - # so we need to do this before initializing h5py_imports()! - # similar to this issue https://github.com/pydata/xarray/issues/2560 - try: - import netCDF4 # noqa F401 - except ImportError: - pass - - @property - def Dataset(self): - if self._Dataset is None: - try: - from netCDF4 import Dataset - except ImportError: - Dataset = NotAModule(self._name) - self._Dataset = Dataset - return self._Dataset - - -_netCDF4 = netCDF4_imports() - - -class astropy_imports: - _name = "astropy" - _pyfits = None - - @property - def pyfits(self): - if self._pyfits is None: - try: - import astropy.io.fits as pyfits - - self.log - except ImportError: - pyfits = NotAModule(self._name) - self._pyfits = pyfits - return self._pyfits - - _pywcs = None - - @property - def pywcs(self): - if self._pywcs is None: - try: - import astropy.wcs as pywcs - - self.log - except ImportError: - pywcs = NotAModule(self._name) - self._pywcs = pywcs - return self._pywcs - - _log = None - - @property - def log(self): - if self._log is None: - try: - from astropy import log - - if log.exception_logging_enabled(): - log.disable_exception_logging() - except ImportError: - log = NotAModule(self._name) - self._log = log - return self._log - - _units = None - - @property - def units(self): - if self._units is None: - try: - from astropy import units - - self.log - except ImportError: - units = NotAModule(self._name) - self._units = units - return self._units - - _conv = None - - @property - def conv(self): - if self._conv is None: - try: - import astropy.convolution as conv - - self.log - except ImportError: - conv = NotAModule(self._name) - self._conv = conv - return self._conv - - _time = None - - @property - def time(self): - if self._time is None: - try: - import astropy.time as time - - self.log - except ImportError: - time = NotAModule(self._name) - self._time = time - return self._time - - _wcsaxes = None - - @property - def wcsaxes(self): - if self._wcsaxes is None: - try: - import astropy.visualization.wcsaxes as wcsaxes - - self.log - except ImportError: - wcsaxes = NotAModule(self._name) - self._wcsaxes = wcsaxes - return self._wcsaxes - - _version = None - - @property - def __version__(self): - if self._version is None: - try: - import astropy - - version = astropy.__version__ - except ImportError: - version = NotAModule(self._name) - self._version = version - return self._version - - -_astropy = astropy_imports() - - -class cartopy_imports: - _name = "cartopy" - - _crs = None - - @property - def crs(self): - if self._crs is None: - try: - import cartopy.crs as crs - except ImportError: - crs = NotCartopy(self._name) - self._crs = crs - return self._crs - - _version = None - - @property - def __version__(self): - if self._version is None: - try: - import cartopy - - version = cartopy.__version__ - except ImportError: - version = NotCartopy(self._name) - self._version = version - return self._version - - -_cartopy = cartopy_imports() - - -class pooch_imports: - _name = "pooch" - _module = None - - def __init__(self): - try: - import pooch as myself - - self._module = myself - except ImportError: - self._module = NotAModule(self._name) - - def __getattr__(self, attr): - return getattr(self._module, attr) - - -_pooch = pooch_imports() - - -class scipy_imports: - _name = "scipy" - _integrate = None - - @property - def integrate(self): - if self._integrate is None: - try: - import scipy.integrate as integrate - except ImportError: - integrate = NotAModule(self._name) - self._integrate = integrate - return self._integrate - - _stats = None - - @property - def stats(self): - if self._stats is None: - try: - import scipy.stats as stats - except ImportError: - stats = NotAModule(self._name) - self._stats = stats - return self._stats - - _optimize = None - - @property - def optimize(self): - if self._optimize is None: - try: - import scipy.optimize as optimize - except ImportError: - optimize = NotAModule(self._name) - self._optimize = optimize - return self._optimize - - _interpolate = None - - @property - def interpolate(self): - if self._interpolate is None: - try: - import scipy.interpolate as interpolate - except ImportError: - interpolate = NotAModule(self._name) - self._interpolate = interpolate - return self._interpolate - - _special = None - - @property - def special(self): - if self._special is None: - try: - import scipy.special as special - except ImportError: - special = NotAModule(self._name) - self._special = special - return self._special - - _signal = None - - @property - def signal(self): - if self._signal is None: - try: - import scipy.signal as signal - except ImportError: - signal = NotAModule(self._name) - self._signal = signal - return self._signal - - _spatial = None - - @property - def spatial(self): - if self._spatial is None: - try: - import scipy.spatial as spatial - except ImportError: - spatial = NotAModule(self._name) - self._spatial = spatial - return self._spatial - - _ndimage = None - - @property - def ndimage(self): - if self._ndimage is None: - try: - import scipy.ndimage as ndimage - except ImportError: - ndimage = NotAModule(self._name) - self._ndimage = ndimage - return self._ndimage - - -_scipy = scipy_imports() - + def __init__(self, name: str, instructions: Optional[str] = None): + self._name = name + self._submodules = {} + msg = f"Failed to import an optional dependency from yt ({self._name})." + if instructions: + msg += f"\n{instructions}" + self._error = ImportError(msg) -class h5py_imports: - _name = "h5py" - _err = None - _File = None - - @property - def File(self): - if self._err: - raise self._err - if self._File is None: - try: - from h5py import File - except ImportError: - File = NotAModule(self._name) - self._File = File - return self._File - - _Group = None - - @property - def Group(self): - if self._err: - raise self._err - if self._Group is None: - try: - from h5py import Group - except ImportError: - Group = NotAModule(self._name) - self._Group = Group - return self._Group - - _Dataset = None - - @property - def Dataset(self): - if self._err: - raise self._err - if self._Dataset is None: - try: - from h5py import Dataset - except ImportError: - Dataset = NotAModule(self._name) - self._Dataset = Dataset - return self._Dataset - - ___version__ = None - - @property - def __version__(self): - if self._err: - raise self._err - if self.___version__ is None: - try: - from h5py import __version__ - except ImportError: - __version__ = NotAModule(self._name) - self.___version__ = __version__ - return self.___version__ - - _get_config = None - - @property - def get_config(self): - if self._err: - raise self._err - if self._get_config is None: - try: - from h5py import get_config - except ImportError: - get_config = NotAModule(self._name) - self._get_config = get_config - return self._get_config - - _h5f = None - - @property - def h5f(self): - if self._err: - raise self._err - if self._h5f is None: - try: - import h5py.h5f as h5f - except ImportError: - h5f = NotAModule(self._name) - self._h5f = h5f - return self._h5f - - _h5p = None - - @property - def h5p(self): - if self._err: - raise self._err - if self._h5p is None: - try: - import h5py.h5p as h5p - except ImportError: - h5p = NotAModule(self._name) - self._h5p = h5p - return self._h5p - - _h5d = None - - @property - def h5d(self): - if self._err: - raise self._err - if self._h5d is None: - try: - import h5py.h5d as h5d - except ImportError: - h5d = NotAModule(self._name) - self._h5d = h5d - return self._h5d - - _h5s = None - - @property - def h5s(self): - if self._err: - raise self._err - if self._h5s is None: - try: - import h5py.h5s as h5s - except ImportError: - h5s = NotAModule(self._name) - self._h5s = h5s - return self._h5s - - _version = None - - @property - def version(self): - if self._err: - raise self._err - if self._version is None: - try: - import h5py.version as version - except ImportError: - version = NotAModule(self._name) - self._version = version - return self._version - - -_h5py = h5py_imports() - - -class nose_imports: - _name = "nose" - _run = None - - @property - def run(self): - if self._run is None: - try: - from nose import run - except ImportError: - run = NotAModule(self._name) - self._run = run - return self._run - - -_nose = nose_imports() - - -class libconf_imports: - _name = "libconf" - _load = None - - @property - def load(self): - if self._load is None: - try: - from libconf import load - except ImportError: - load = NotAModule(self._name) - self._load = load - return self._load - - -_libconf = libconf_imports() - - -class yaml_imports: - _name = "yaml" - _load = None - _FullLoader = None - - @property - def load(self): - if self._load is None: - try: - from yaml import load - except ImportError: - load = NotAModule(self._name) - self._load = load - return self._load - - @property - def FullLoader(self): - if self._FullLoader is None: - try: - from yaml import FullLoader - except ImportError: - FullLoader = NotAModule(self._name) - self._FullLoader = FullLoader - return self._FullLoader - - -_yaml = yaml_imports() - - -class NotMiniball(NotAModule): - def __init__(self, pkg_name): - super().__init__(pkg_name) - str = ( - "This functionality requires the %s package to be installed. " - "Installation instructions can be found at " - "https://github.com/weddige/miniball or alternatively you can " - "install via `python -m pip install MiniballCpp`." - ) - self.error = ImportError(str % self.pkg_name) - - -class miniball_imports: - _name = "miniball" - _Miniball = None - - @property - def Miniball(self): - if self._Miniball is None: - try: - from miniball import Miniball - except ImportError: - Miniball = NotMiniball(self._name) - self._Miniball = Miniball - return self._Miniball - - -_miniball = miniball_imports() - - -class f90nml_imports: - _name = "f90nml" - _module = None - - def __init__(self): + @cached_property + def module(self): try: - import f90nml as myself - - self._module = myself - except ImportError: - self._module = NotAModule(self._name) - - def __getattr__(self, attr): - return getattr(self._module, attr) - - -_f90nml = f90nml_imports() - - -class requests_imports: - _name = "requests" - _module = None - - def __init__(self): + return import_module(self._name) + except ImportError as exc: + raise self._error from exc + + def __getattr__(self, key): + # the recursive nature of this method is intended to support + # arbitrary levels of nesting for submodules that need to be + # explicitly imported on their own, like scipy.spatial or astropy.io.fits + if key in self._submodules: + return self._submodules[key] try: - import requests as myself - - self._module = myself - except ImportError: - self._module = NotAModule(self._name) - - def __getattr__(self, attr): - return getattr(self._module, attr) - - -_requests = requests_imports() - - -class pandas_imports: - _name = "pandas" - _module = None - - def __init__(self): - try: - import pandas as myself - - self._module = myself - except ImportError: - self._module = NotAModule(self._name) - - def __getattr__(self, attr): - return getattr(self._module, attr) - - -_pandas = pandas_imports() - - -class firefly_imports: - _name = "firefly" - _data_reader = None - _server = None - - @property - def data_reader(self): - if self._data_reader is None: - try: - import Firefly.data_reader as data_reader - except ImportError: - data_reader = NotAModule(self._name) - self._data_reader = data_reader - return self._data_reader - - @property - def server(self): - if self._server is None: - try: - import Firefly.server as server - except ImportError: - server = NotAModule(self._name) - self._server = server - return self._server - - -_firefly = firefly_imports() + return getattr(self.module, key) + except AttributeError: + self._submodules[key] = OnDemandImport(f"{self._name}.{key}") + return self._submodules[key] + + +if any(s in sys.version for s in ("Anaconda", "Continuum")): + # conda distributions of cartopy don't have issues with the GEOS library + # so we can keep the error message short + # see https://github.com/yt-project/yt/pull/1966 + _cartopy_instructions = None +else: + _cartopy_instructions = ( + "Try installing proj4 and geos with your package manager " + "and building shapely and cartopy from source via Pypi with:\n" + "`python -m pip install --no-binary :all: shapely cartopy`\n" + "For further instruction please refer to the yt documentation." + ) + +optional_dependencies = { + "astropy": None, + "bottle": None, + "cartopy": _cartopy_instructions, + "Firefly": None, + "f90nml": None, + # Breaking alphabetical ordering on purpose here. + # Importing h5py before netCDF4 can lead to file lock errors + # on some systems (including travis-ci) + # see https://github.com/pydata/xarray/issues/2560 + "netCDF4": None, + "h5py": None, + "libconf": None, + "miniball": "This package can be installed from PyPI with\n`python3 -m pip install MiniballCpp`", + "mpi4py": None, + "nose": None, + "pandas": None, + "pooch": None, + "requests": None, + "scipy": None, + "yaml": None, +} + +_module = sys.modules[__name__] +for package_name, instructions in optional_dependencies.items(): + setattr(_module, package_name, OnDemandImport(package_name, instructions)) diff --git a/yt/utilities/parallel_tools/parallel_analysis_interface.py b/yt/utilities/parallel_tools/parallel_analysis_interface.py index d50f5e5224a..424ee122d78 100644 --- a/yt/utilities/parallel_tools/parallel_analysis_interface.py +++ b/yt/utilities/parallel_tools/parallel_analysis_interface.py @@ -19,6 +19,7 @@ from yt.utilities.exceptions import YTNoDataInObjectError from yt.utilities.lib.quad_tree import QuadTree, merge_quadtrees from yt.utilities.logger import ytLogger as mylog +from yt.utilities.on_demand_imports import mpi4py # We default to *no* parallelism unless it gets turned on, in which case this # will be changed. @@ -92,9 +93,9 @@ def enable_parallelism(suppress_logging: bool = False, communicator=None) -> boo """ global parallel_capable, MPI try: - from mpi4py import MPI as _MPI + _MPI = mpi4py.MPI except ImportError: - mylog.error("Could not enable parallelism: mpi4py is not installed") + mylog.error("mpi4py was not found. Disabling parallel computation") return False MPI = _MPI exe_name = os.path.basename(sys.executable) diff --git a/yt/utilities/rpdb.py b/yt/utilities/rpdb.py index 5bbcd25852b..73575bb9655 100644 --- a/yt/utilities/rpdb.py +++ b/yt/utilities/rpdb.py @@ -8,6 +8,7 @@ from xmlrpc.server import SimpleXMLRPCServer from yt.config import ytcfg +from yt.utilities.on_demand_imports import mpi4py class PdbXMLRPCServer(SimpleXMLRPCServer): @@ -48,13 +49,11 @@ def rpdb_excepthook(exc_type, exc, tb): server.serve_forever() server.server_close() if size > 1: - from mpi4py import MPI - # This COMM_WORLD is okay. We want to barrierize here, while waiting # for shutdown from the rest of the parallel group. If you are running # with --rpdb it is assumed you know what you are doing and you won't # let this get out of hand. - MPI.COMM_WORLD.Barrier() + mpi4py.MPI.COMM_WORLD.Barrier() class pdb_handler: diff --git a/yt/utilities/tests/test_cosmology.py b/yt/utilities/tests/test_cosmology.py index ae069d68dae..288275118bc 100644 --- a/yt/utilities/tests/test_cosmology.py +++ b/yt/utilities/tests/test_cosmology.py @@ -12,7 +12,7 @@ from yt.units.yt_array import YTArray, YTQuantity from yt.utilities.answer_testing.framework import data_dir_load from yt.utilities.cosmology import Cosmology -from yt.utilities.on_demand_imports import _yaml as yaml +from yt.utilities.on_demand_imports import yaml local_dir = os.path.dirname(os.path.abspath(__file__)) diff --git a/yt/visualization/fits_image.py b/yt/visualization/fits_image.py index 7aa8bc3c4e5..646087d9d14 100644 --- a/yt/visualization/fits_image.py +++ b/yt/visualization/fits_image.py @@ -12,7 +12,7 @@ from yt.units import dimensions from yt.units.unit_object import Unit # type: ignore from yt.units.yt_array import YTArray, YTQuantity -from yt.utilities.on_demand_imports import _astropy +from yt.utilities.on_demand_imports import astropy from yt.utilities.parallel_tools.parallel_analysis_interface import parallel_root_only from yt.visualization.fixed_resolution import FixedResolutionBuffer, ParticleImageBuffer from yt.visualization.particle_plots import ParticleAxisAlignedDummyDataSource @@ -172,10 +172,10 @@ def __init__( "weight_field", ] - if isinstance(data, _astropy.pyfits.PrimaryHDU): - data = _astropy.pyfits.HDUList([data]) + if isinstance(data, astropy.io.fits.PrimaryHDU): + data = astropy.io.fits.HDUList([data]) - if isinstance(data, _astropy.pyfits.HDUList): + if isinstance(data, astropy.io.fits.HDUList): self.hdulist = data for hdu in data: self.fields.append(hdu.header["btype"]) @@ -189,14 +189,14 @@ def __init__( key = " " else: key = name[-1] - w = _astropy.pywcs.WCS( + w = astropy.wcs.WCS( header=self.hdulist[0].header, key=key, naxis=self.dimensionality ) setattr(self, "wcs" + key.strip().lower(), w) return - self.hdulist = _astropy.pyfits.HDUList() + self.hdulist = astropy.io.fits.HDUList() if hasattr(data, "keys"): img_data = data @@ -269,9 +269,9 @@ def __init__( self.shape = this_img.shape this_img = np.asarray(this_img.T) if is_first: - hdu = _astropy.pyfits.PrimaryHDU(this_img) + hdu = astropy.io.fits.PrimaryHDU(this_img) else: - hdu = _astropy.pyfits.ImageHDU(this_img) + hdu = astropy.io.fits.ImageHDU(this_img) hdu.name = name hdu.header["btype"] = name hdu.header["bunit"] = re.sub("()", "", self.field_units[name]) @@ -294,7 +294,7 @@ def __init__( self.dimensionality = len(self.shape) if wcs is None: - w = _astropy.pywcs.WCS( + w = astropy.wcs.WCS( header=self.hdulist[0].header, naxis=self.dimensionality ) # FRBs and covering grids are special cases where @@ -501,7 +501,7 @@ def convolve(self, field, kernel, **kwargs): """ if self.dimensionality == 3: raise RuntimeError("Convolution currently only works for 2D FITSImageData!") - conv = _astropy.conv + conv = astropy.convolution if field not in self.keys(): raise KeyError(f"{field} not an image!") idx = self.fields.index(field) @@ -626,7 +626,7 @@ def writeto(self, fileobj, fields=None, overwrite=False, **kwargs): if fields is None: hdus = self.hdulist else: - hdus = _astropy.pyfits.HDUList() + hdus = astropy.io.fits.HDUList() for field in fields: hdus.append(self.hdulist[field]) hdus.writeto(fileobj, overwrite=overwrite, **kwargs) @@ -698,7 +698,7 @@ def pop(self, key): im = self.hdulist.pop(idx) self.field_units.pop(key) self.fields.remove(key) - f = _astropy.pyfits.PrimaryHDU(im.data, header=im.header) + f = astropy.io.fits.PrimaryHDU(im.data, header=im.header) return FITSImageData(f, current_time=f.header["TIME"], unit_header=f.header) def close(self): @@ -715,7 +715,7 @@ def from_file(cls, filename): filename : string The name of the file to open. """ - f = _astropy.pyfits.open(filename, lazy_load_hdus=False) + f = astropy.io.fits.open(filename, lazy_load_hdus=False) return cls(f, current_time=f[0].header["TIME"], unit_header=f[0].header) @classmethod @@ -741,10 +741,10 @@ def from_images(cls, image_list): raise RuntimeError("Images do not have the same shape!") for hdu in fid.hdulist: if len(data) == 0: - data.append(_astropy.pyfits.PrimaryHDU(hdu.data, header=hdu.header)) + data.append(astropy.io.fits.PrimaryHDU(hdu.data, header=hdu.header)) else: - data.append(_astropy.pyfits.ImageHDU(hdu.data, header=hdu.header)) - data = _astropy.pyfits.HDUList(data) + data.append(astropy.io.fits.ImageHDU(hdu.data, header=hdu.header)) + data = astropy.io.fits.HDUList(data) return cls( data, current_time=first_image.current_time, @@ -807,7 +807,7 @@ def create_sky_wcs( units = [str(unit) for unit in old_wcs.wcs.cunit] new_dx = (YTQuantity(-deltas[0], units[0]) * scaleq).in_units("deg") new_dy = (YTQuantity(deltas[1], units[1]) * scaleq).in_units("deg") - new_wcs = _astropy.pywcs.WCS(naxis=naxis) + new_wcs = astropy.wcs.WCS(naxis=naxis) cdelt = [new_dx.v, new_dy.v] cunit = ["deg"] * 2 if naxis == 3: @@ -903,7 +903,7 @@ def construct_image(ds, axis, data_source, center, image_res, width, length_unit ) else: frb = None - w = _astropy.pywcs.WCS(naxis=2) + w = astropy.wcs.WCS(naxis=2) w.wcs.crpix = crpix w.wcs.cdelt = cdelt w.wcs.crval = crval diff --git a/yt/visualization/fixed_resolution.py b/yt/visualization/fixed_resolution.py index db365e340f9..56690616c05 100644 --- a/yt/visualization/fixed_resolution.py +++ b/yt/visualization/fixed_resolution.py @@ -12,7 +12,7 @@ add_points_to_greyscale_image, ) from yt.utilities.lib.pixelization_routines import pixelize_cylinder -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py from .fixed_resolution_filters import apply_filter, filter_registry from .volume_rendering.api import off_axis_projection diff --git a/yt/visualization/fixed_resolution_filters.py b/yt/visualization/fixed_resolution_filters.py index d69dc8fafdd..976ed2ae419 100644 --- a/yt/visualization/fixed_resolution_filters.py +++ b/yt/visualization/fixed_resolution_filters.py @@ -48,7 +48,7 @@ def __init__(self, nbeam=30, sigma=2.0): self.sigma = sigma def apply(self, buff): - from yt.utilities.on_demand_imports import _scipy + from yt.utilities.on_demand_imports import scipy hnbeam = self.nbeam // 2 sigma = self.sigma @@ -61,7 +61,7 @@ def apply(self, buff): g2d /= g2d.max() npm, nqm = np.shape(buff) - spl = _scipy.signal.convolve(buff, g2d) + spl = scipy.signal.convolve(buff, g2d) return spl[hnbeam : npm + hnbeam, hnbeam : nqm + hnbeam] diff --git a/yt/visualization/geo_plot_utils.py b/yt/visualization/geo_plot_utils.py index db6716e42e8..08903fd7c55 100644 --- a/yt/visualization/geo_plot_utils.py +++ b/yt/visualization/geo_plot_utils.py @@ -36,7 +36,7 @@ def cartopy_importer(transform_name): r"""Convenience function to import cartopy projection types""" def _func(*args, **kwargs): - from yt.utilities.on_demand_imports import _cartopy as cartopy + from yt.utilities.on_demand_imports import cartopy return getattr(cartopy.crs, transform_name)(*args, **kwargs) diff --git a/yt/visualization/mapserver/pannable_map.py b/yt/visualization/mapserver/pannable_map.py index 86bec605a8f..e2d99b7ad80 100644 --- a/yt/visualization/mapserver/pannable_map.py +++ b/yt/visualization/mapserver/pannable_map.py @@ -1,11 +1,11 @@ import os from functools import wraps -import bottle import numpy as np from yt.fields.derived_field import ValidateSpatial from yt.utilities.lib.misc_utilities import get_color_bounds +from yt.utilities.on_demand_imports import bottle from yt.utilities.png_writer import write_png_to_string from yt.visualization.fixed_resolution import FixedResolutionBuffer from yt.visualization.image_writer import apply_colormap diff --git a/yt/visualization/tests/test_fits_image.py b/yt/visualization/tests/test_fits_image.py index db5dfc6fc41..2ffc426a9c1 100644 --- a/yt/visualization/tests/test_fits_image.py +++ b/yt/visualization/tests/test_fits_image.py @@ -6,7 +6,7 @@ from yt.loaders import load from yt.testing import fake_random_ds, requires_file, requires_module -from yt.utilities.on_demand_imports import _astropy +from yt.utilities.on_demand_imports import astropy from yt.visualization.fits_image import ( FITSImageData, FITSOffAxisProjection, @@ -194,8 +194,8 @@ def test_fits_image(): fid7.convolve("density", (3.0, "cm")) sigma = 3.0 / fid7.wcs.wcs.cdelt[0] - kernel = _astropy.conv.Gaussian2DKernel(x_stddev=sigma) - data_conv = _astropy.conv.convolve(fid4["density"].data.d, kernel) + kernel = astropy.convolution.Gaussian2DKernel(x_stddev=sigma) + data_conv = astropy.convolution.convolve(fid4["density"].data.d, kernel) assert_allclose(data_conv, fid7["density"].data.d) pfid = FITSParticleProjection(ds, "x", ("io", "particle_mass")) diff --git a/yt/visualization/tests/test_geo_projections.py b/yt/visualization/tests/test_geo_projections.py index f40ef9fe826..e5e2bb12c49 100644 --- a/yt/visualization/tests/test_geo_projections.py +++ b/yt/visualization/tests/test_geo_projections.py @@ -70,7 +70,7 @@ def tearDown(self): @requires_module("cartopy") def test_geo_projection_setup(self): - from yt.utilities.on_demand_imports import _cartopy as cartopy + from yt.utilities.on_demand_imports import cartopy axis = "altitude" self.slc = yt.SlicePlot(self.ds, axis, ("stream", "Density"), origin="native") @@ -86,7 +86,7 @@ def test_geo_projection_setup(self): @requires_module("cartopy") def test_geo_projections(self): - from yt.utilities.on_demand_imports import _cartopy as cartopy + from yt.utilities.on_demand_imports import cartopy self.slc = yt.SlicePlot( self.ds, "altitude", ("stream", "Density"), origin="native" @@ -110,7 +110,7 @@ def test_geo_projections(self): @requires_module("cartopy") def test_projection_object(self): - from yt.utilities.on_demand_imports import _cartopy as cartopy + from yt.utilities.on_demand_imports import cartopy shortlist = ["Orthographic", "PlateCarree", "Mollweide"] @@ -130,7 +130,7 @@ def test_projection_object(self): @requires_module("cartopy") def test_nondefault_transform(self): - from yt.utilities.on_demand_imports import _cartopy as cartopy + from yt.utilities.on_demand_imports import cartopy axis = "altitude" self.ds.coordinates.data_transform[axis] = "Miller" diff --git a/yt/visualization/volume_rendering/image_handling.py b/yt/visualization/volume_rendering/image_handling.py index 4c7ba8d23fa..2e6c37b5566 100644 --- a/yt/visualization/volume_rendering/image_handling.py +++ b/yt/visualization/volume_rendering/image_handling.py @@ -1,7 +1,7 @@ import numpy as np from yt.funcs import mylog -from yt.utilities.on_demand_imports import _h5py as h5py +from yt.utilities.on_demand_imports import h5py def export_rgba( diff --git a/yt/visualization/volume_rendering/tests/test_off_axis_SPH.py b/yt/visualization/volume_rendering/tests/test_off_axis_SPH.py index 1799a9d5508..b1efc35785f 100644 --- a/yt/visualization/volume_rendering/tests/test_off_axis_SPH.py +++ b/yt/visualization/volume_rendering/tests/test_off_axis_SPH.py @@ -2,12 +2,9 @@ from yt.testing import assert_almost_equal, fake_sph_orientation_ds, requires_module from yt.utilities.lib.pixelization_routines import pixelize_sph_kernel_projection -from yt.utilities.on_demand_imports import _scipy +from yt.utilities.on_demand_imports import scipy from yt.visualization.volume_rendering import off_axis_projection as OffAP -spatial = _scipy.spatial -ndimage = _scipy.ndimage - def test_no_rotation(): """Determines if a projection processed through @@ -270,7 +267,7 @@ def test_center_3(): @requires_module("scipy") def find_compare_maxima(expected_maxima, buf, resolution, width): buf_ndarray = buf.ndarray_view() - max_filter_buf = ndimage.filters.maximum_filter(buf_ndarray, size=5) + max_filter_buf = scipy.ndimage.filters.maximum_filter(buf_ndarray, size=5) maxima = np.isclose(max_filter_buf, buf_ndarray, rtol=1e-09) # ignore contributions from zones of no smoothing @@ -299,7 +296,7 @@ def find_compare_maxima(expected_maxima, buf, resolution, width): x_coord /= x_scaling_factor y_coord /= y_scaling_factor if ( - spatial.distance.euclidean( + scipy.spatial.distance.euclidean( [x_coord, y_coord], [expected_maxima[0][i], expected_maxima[1][i]] ) < pixel_tolerance From e226035bbc9a7e29f6223ee16b41e08e18ea9593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 17 May 2021 22:57:04 +0200 Subject: [PATCH 2/4] DOC: add documentation on how to manage soft dependencies --- doc/source/developing/index.rst | 1 + .../managing_optional_dependencies.rst | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 doc/source/developing/managing_optional_dependencies.rst diff --git a/doc/source/developing/index.rst b/doc/source/developing/index.rst index 44af92b09bf..e22845a3196 100644 --- a/doc/source/developing/index.rst +++ b/doc/source/developing/index.rst @@ -27,3 +27,4 @@ and contributing code! creating_frontend external_analysis deprecating_features + managing_optional_dependencies diff --git a/doc/source/developing/managing_optional_dependencies.rst b/doc/source/developing/managing_optional_dependencies.rst new file mode 100644 index 00000000000..e4b41dac96a --- /dev/null +++ b/doc/source/developing/managing_optional_dependencies.rst @@ -0,0 +1,38 @@ +How to manage soft (optional) dependencies +------------------------------------------ + +We might sometimes rely on heavy external libraries to support some features +outside of yt's core. Typically, many frontends require HDF5 support, provided +by the ``h5py`` library, but many users do not need it and shouldn't have to +install such a large library to get yt. + +A mechanism to support soft-dependencies is implemented in the +``yt/utilities/on_demand_imports.py`` module. Existing soft dependencies are +listed in a ``optional_dependencies`` dictionary using package names for keys. +The dictionary is then processed to generate drop-in replacement for the actual +packages, which behave as normal instances of the packages they mirror: + +.. code-block:: python + + from yt.utilities.on_demand_imports import h5py + + + def process(file): + with h5py.File(file, mode="r") as fh: + # perform interesting operations + ... + +In case the package is missing, an ``ImportError`` will be raised at runtime if +and only if the code using it is run. In the example above, this means that as +long as the ``process`` function is not run, no error will be raised. An +implication is that soft dependencies cannot be used inconditionally in the +global scope of a module, because it is run at yt's importtime, but should +rather be restricted to lower level scopes, inside of functions or properly +guarded conditional blocks. + +Note that, mocking an external package using the ``OnDemandImport`` class also +delays any actual import to the first time it is used (this is how we can afford +to fail only at the conditions we have to). A direct implication is that objects +defined as such can safely be imported at the top level of a module, where the +bulk of import statements live, without affecting import durations in any +significant way. From dbbdae4aa5d051b1a3a492936c9556bcfcfae490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 26 Nov 2021 17:59:31 +0100 Subject: [PATCH 3/4] TYP: add type annotation --- yt/utilities/on_demand_imports.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yt/utilities/on_demand_imports.py b/yt/utilities/on_demand_imports.py index a56ec7afa1d..fc7bf6bcab6 100644 --- a/yt/utilities/on_demand_imports.py +++ b/yt/utilities/on_demand_imports.py @@ -1,6 +1,7 @@ import sys from importlib import import_module -from typing import Optional +from types import ModuleType +from typing import Dict, Optional, Union if sys.version_info >= (3, 8): from functools import cached_property @@ -36,7 +37,7 @@ class OnDemandImport: def __init__(self, name: str, instructions: Optional[str] = None): self._name = name - self._submodules = {} + self._submodules: Dict[str, Union[ModuleType, "OnDemandImport"]] = {} msg = f"Failed to import an optional dependency from yt ({self._name})." if instructions: msg += f"\n{instructions}" From 605c1b5554a150416f27ebf7e2b1c2c687777198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 26 Nov 2021 18:05:52 +0100 Subject: [PATCH 4/4] RFC: don't setup module attributes dynamically at runtime --- yt/utilities/on_demand_imports.py | 49 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/yt/utilities/on_demand_imports.py b/yt/utilities/on_demand_imports.py index fc7bf6bcab6..1416100ed48 100644 --- a/yt/utilities/on_demand_imports.py +++ b/yt/utilities/on_demand_imports.py @@ -76,29 +76,28 @@ def __getattr__(self, key): "For further instruction please refer to the yt documentation." ) -optional_dependencies = { - "astropy": None, - "bottle": None, - "cartopy": _cartopy_instructions, - "Firefly": None, - "f90nml": None, - # Breaking alphabetical ordering on purpose here. - # Importing h5py before netCDF4 can lead to file lock errors - # on some systems (including travis-ci) - # see https://github.com/pydata/xarray/issues/2560 - "netCDF4": None, - "h5py": None, - "libconf": None, - "miniball": "This package can be installed from PyPI with\n`python3 -m pip install MiniballCpp`", - "mpi4py": None, - "nose": None, - "pandas": None, - "pooch": None, - "requests": None, - "scipy": None, - "yaml": None, -} +astropy = OnDemandImport("astropy", None) +bottle = OnDemandImport("bottle", None) +cartopy = OnDemandImport("cartopy", _cartopy_instructions) +Firefly = OnDemandImport("Firefly", None) +f90nml = OnDemandImport("f90nml", None) -_module = sys.modules[__name__] -for package_name, instructions in optional_dependencies.items(): - setattr(_module, package_name, OnDemandImport(package_name, instructions)) +# Breaking alphabetical ordering on purpose here. +# Importing h5py before netCDF4 can lead to file lock errors +# on some systems (including travis-ci) +# see https://github.com/pydata/xarray/issues/2560 +netCDF4 = OnDemandImport("netCDF4", None) +h5py = OnDemandImport("h5py", None) +libconf = OnDemandImport("libconf", None) +miniball = OnDemandImport( + "miniball", + "This package can be installed from PyPI with" + "`python3 -m pip install MiniballCpp`", +) +mpi4py = OnDemandImport("mpi4py", None) +nose = OnDemandImport("nose", None) +pandas = OnDemandImport("pandas", None) +pooch = OnDemandImport("pooch", None) +requests = OnDemandImport("requests", None) +scipy = OnDemandImport("scipy", None) +yaml = OnDemandImport("yaml", None)