From 5d53943e5288e938a95733ef54f6987c25a710ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 17 May 2021 09:46:51 +0200 Subject: [PATCH] refactor: extend usage of the new on-demand-import mechanism to second class optional deps --- yt/data_objects/data_containers.py | 15 +++--------- yt/startup_tasks.py | 23 +++++++++++-------- yt/utilities/command_line.py | 10 ++------ yt/utilities/on_demand_imports.py | 15 +++--------- .../parallel_analysis_interface.py | 4 ++-- yt/utilities/rpdb.py | 5 ++-- yt/visualization/mapserver/pannable_map.py | 2 +- 7 files changed, 26 insertions(+), 48 deletions(-) diff --git a/yt/data_objects/data_containers.py b/yt/data_objects/data_containers.py index 31b0d5d59a8..b3389dde5d3 100644 --- a/yt/data_objects/data_containers.py +++ b/yt/data_objects/data_containers.py @@ -22,6 +22,7 @@ YTSpatialFieldUnitError, ) from yt.utilities.object_registries import data_object_registry +from yt.utilities.on_demand_imports import firefly_api from yt.utilities.parameter_file_storage import ParameterFileStore @@ -782,18 +783,8 @@ def create_firefly_object( >>> reader.particleGroups[0].decimation_factor=100 >>> reader.dumpToJSON() """ - - ## attempt to import firefly_api - try: - from firefly_api.particlegroup import ParticleGroup - from firefly_api.reader import Reader - except ImportError as e: - raise ImportError( - "Can't find firefly_api, ensure it " - "is in your python path or install it with " - "'$ pip install firefly_api'. It is also available " - "on github at github.com/agurvich/firefly_api" - ) from e + Reader = firefly_api.reader.Reader + ParticleGroup = firefly_api.particlegroup.ParticleGroup ## handle default arguments fields_to_include = [] if fields_to_include is None else fields_to_include diff --git a/yt/startup_tasks.py b/yt/startup_tasks.py index f99ecf1086d..e7c7cb4ac0f 100644 --- a/yt/startup_tasks.py +++ b/yt/startup_tasks.py @@ -14,21 +14,24 @@ signal_print_traceback, ) from yt.utilities import rpdb +from yt.utilities.on_demand_imports import mpi4py exe_name = os.path.basename(sys.executable) # At import time, we determined whether or not we're being run in parallel. def turn_on_parallelism(): + """ + Activate parallel capabilities. + + Raises + ------ + ImportError if mpi4py isn't installed. + """ parallel_capable = False - try: - # we import this to check if mpi4py is installed - from mpi4py import MPI # NOQA - except ImportError as e: - mylog.error( - "Warning: Attempting to turn on parallelism, " - "but mpi4py import failed. Try pip install mpi4py." - ) - raise e - # Now we have to turn on the parallelism from the perspective of the + + # check that mpi4py is correctly installed + mpi4py.MPI + + # Now we have to turn on the parallelism from the perspective of the # parallel_analysis_interface from yt.utilities.parallel_tools.parallel_analysis_interface import ( enable_parallelism, diff --git a/yt/utilities/command_line.py b/yt/utilities/command_line.py index b8abeae5afe..cc4515e6ece 100644 --- a/yt/utilities/command_line.py +++ b/yt/utilities/command_line.py @@ -35,6 +35,7 @@ YTUnidentifiedDataType, ) from yt.utilities.metadata import get_metadata +from yt.utilities.on_demand_imports import bottle from yt.visualization.plot_window import ProjectionPlot, SlicePlot # isort: off @@ -1055,14 +1056,7 @@ def __call__(self, args): p.set_cmap("all", args.cmap) PannableMapServer(p.data_source, args.field, args.takelog, args.cmap) - try: - import bottle - except ImportError as e: - raise ImportError( - "The mapserver functionality requires the bottle " - "package to be installed. Please install using `pip " - "install bottle`." - ) from e + bottle.debug(True) if args.host is not None: colonpl = args.host.find(":") diff --git a/yt/utilities/on_demand_imports.py b/yt/utilities/on_demand_imports.py index 06489a26f4e..677ec4c92f1 100644 --- a/yt/utilities/on_demand_imports.py +++ b/yt/utilities/on_demand_imports.py @@ -60,18 +60,6 @@ def __getattr__(self, key): return self._submodules[key] -# TEMP: these require special treatments -__special_cases = [ - # the following are currently not using the on-demand-imports mechanism - # and replace it with unnecessary special treatments - "bottle", - "firefly_api", - "mpi4py", - "resource", - "yt_astro_analysis", -] - - if any(s in sys.version for s in ("Anaconda", "Continuum")): # conda distributions of cartopy don't have issues with the GEOS library # so we can keep the error message short @@ -87,7 +75,9 @@ def __getattr__(self, key): targets = [ "astropy", + "bottle", ("cartopy", _cartopy_instructions), + "firefly_api", "f90nml", # Breaking alphabetical ordering on purpose here. # Importing h5py before netCDF4 can lead to file lock errors @@ -100,6 +90,7 @@ def __getattr__(self, key): "miniball", "This package can be installed from Pypi with\n`python -m pip install MiniballCpp`", ), + "mpi4py", "nose", "pandas", "pooch", diff --git a/yt/utilities/parallel_tools/parallel_analysis_interface.py b/yt/utilities/parallel_tools/parallel_analysis_interface.py index a2f3abacf2f..913c8021fdb 100644 --- a/yt/utilities/parallel_tools/parallel_analysis_interface.py +++ b/yt/utilities/parallel_tools/parallel_analysis_interface.py @@ -18,6 +18,7 @@ from yt.utilities.exceptions import YTNoDataInObjectError from yt.utilities.lib.quad_tree import QuadTree, merge_quadtrees from yt.utilities.logger import ytLogger as mylog +from yt.utilities.on_demand_imports import mpi4py # We default to *no* parallelism unless it gets turned on, in which case this # will be changed. @@ -86,12 +87,11 @@ def enable_parallelism(suppress_logging=False, communicator=None): """ global parallel_capable, MPI try: - from mpi4py import MPI as _MPI + MPI = mpi4py.MPI except ImportError: mylog.info("mpi4py was not found. Disabling parallel computation") parallel_capable = False return - MPI = _MPI exe_name = os.path.basename(sys.executable) # if no communicator specified, set to COMM_WORLD diff --git a/yt/utilities/rpdb.py b/yt/utilities/rpdb.py index 5bbcd25852b..73575bb9655 100644 --- a/yt/utilities/rpdb.py +++ b/yt/utilities/rpdb.py @@ -8,6 +8,7 @@ from xmlrpc.server import SimpleXMLRPCServer from yt.config import ytcfg +from yt.utilities.on_demand_imports import mpi4py class PdbXMLRPCServer(SimpleXMLRPCServer): @@ -48,13 +49,11 @@ def rpdb_excepthook(exc_type, exc, tb): server.serve_forever() server.server_close() if size > 1: - from mpi4py import MPI - # This COMM_WORLD is okay. We want to barrierize here, while waiting # for shutdown from the rest of the parallel group. If you are running # with --rpdb it is assumed you know what you are doing and you won't # let this get out of hand. - MPI.COMM_WORLD.Barrier() + mpi4py.MPI.COMM_WORLD.Barrier() class pdb_handler: diff --git a/yt/visualization/mapserver/pannable_map.py b/yt/visualization/mapserver/pannable_map.py index 9ec6cb343f9..a3eac20018f 100644 --- a/yt/visualization/mapserver/pannable_map.py +++ b/yt/visualization/mapserver/pannable_map.py @@ -1,10 +1,10 @@ import os from functools import wraps -import bottle import numpy as np from yt.utilities.lib.misc_utilities import get_color_bounds +from yt.utilities.on_demand_imports import bottle from yt.utilities.png_writer import write_png_to_string from yt.visualization.fixed_resolution import FixedResolutionBuffer from yt.visualization.image_writer import apply_colormap