diff --git a/doc/cmdline.rst b/doc/cmdline.rst index c5744547..d4b79c20 100644 --- a/doc/cmdline.rst +++ b/doc/cmdline.rst @@ -82,7 +82,14 @@ Install the package on your system. .. option:: --deps Which dependencies to install. One of ``all``, ``production``, ``develop``, - or ``none``. Default ``all``. + or ``none``. ``all`` and ``develop`` install the extras ``test``, ``docs``, + and ``dev``. Default ``all``. + +.. option:: --extras + + Which named extra features to install dependencies for. Specify ``all`` to + install all optional dependencies, or a comma-separated list of extras. + Default depends on ``--deps``. .. option:: --user diff --git a/doc/pyproject_toml.rst b/doc/pyproject_toml.rst index 2fa18ed3..f727af74 100644 --- a/doc/pyproject_toml.rst +++ b/doc/pyproject_toml.rst @@ -67,12 +67,18 @@ requires "configparser; python_version == '2.7'" ] -dev-requires - Packages that are required for development. This field is in the same format - as ``requires``. +requires-extra + Lists of packages needed for every optional feature. The requirements + are specified in the same format as for ``requires``. The requirements of + the two reserved extras ``test`` and ``doc`` as well as the extra ``dev`` + are installed by ``flit install``. For example: + + .. code-block:: toml + + [tool.flit.metadata.requires-extra] + test = ["pytest>=2.7.3", "pytest-cov"] + doc = ["sphinx"] - These are not (yet) encoded in the wheel, but are used when doing - ``flit install``. description-file A path (relative to the .toml file) to a file containing a longer description of your package to show on PyPI. This should be written in `reStructuredText diff --git a/flit/__init__.py b/flit/__init__.py index c3535a1d..e4554ff3 100644 --- a/flit/__init__.py +++ b/flit/__init__.py @@ -60,7 +60,12 @@ def main(argv=None): ) add_shared_install_options(parser_install) parser_install.add_argument('--deps', choices=['all', 'production', 'develop', 'none'], default='all', - help="Which set of dependencies to install") + help="Which set of dependencies to install. If --deps=develop, the extras dev, doc, and test are installed" + ) + parser_install.add_argument('--extras', default=(), type=lambda l: l.split(',') if l else (), + help="Install the dependencies of these (comma separated) extras additionally to the ones implied by --deps. " + "--extras=all can be useful in combination with --deps=production, --deps=none precludes using --extras" + ) parser_installfrom = subparsers.add_parser('installfrom', help="Download and install a package using flit from source" @@ -107,7 +112,8 @@ def main(argv=None): from .install import Installer try: Installer(args.ini_file, user=args.user, python=args.python, - symlink=args.symlink, deps=args.deps, pth=args.pth_file).install() + symlink=args.symlink, deps=args.deps, extras=args.extras, + pth=args.pth_file).install() except (common.NoDocstringError, common.NoVersionError) as e: sys.exit(e.args[0]) elif args.subcmd == 'installfrom': diff --git a/flit/common.py b/flit/common.py index b73ebc96..063a39d1 100644 --- a/flit/common.py +++ b/flit/common.py @@ -227,20 +227,31 @@ class Metadata: requires_external = () provides_extra = () - metadata_version="2.1" - - # this is part of metadata spec 2, we are using it for installation but it - # doesn't actually get written to the metadata file - dev_requires = () + metadata_version = "2.1" def __init__(self, data): self.name = data.pop('name') self.version = data.pop('version') self.author_email = data.pop('author_email') self.summary = data.pop('summary') + requires_extra = data.pop('requires_extra', {}) + dev_requires = data.pop('dev_requires', None) + if dev_requires is not None: + if 'dev' in requires_extra: + raise ValueError('Ambiguity: Encountered dev-requires together with its replacement requires-extra.dev.') + log.warning('“dev-requires = ...” is obsolete. Use “requires-extra = {"dev" = ...}” instead.') + requires_extra.setdefault('dev', []).extend(dev_requires) + explicit_extras = data.pop('provides_extra', ()) + self.provides_extra = list(set(explicit_extras) | requires_extra.keys()) for k, v in data.items(): assert hasattr(self, k), "data does not have attribute '{}'".format(k) setattr(self, k, v) + if requires_extra: + self.requires_dist = list(self.requires_dist) + [ + '{}; extra == "{}"'.format(d, e) + for e, ds in requires_extra.items() + for d in ds + ] def _normalise_name(self, n): return n.lower().replace('-', '_') diff --git a/flit/inifile.py b/flit/inifile.py index 1920bd2b..5e953058 100644 --- a/flit/inifile.py +++ b/flit/inifile.py @@ -34,6 +34,7 @@ class ConfigError(ValueError): 'dist-name', 'entry-points-file', 'description-file', + 'requires-extra', } | metadata_list_fields metadata_required_fields = { @@ -231,6 +232,16 @@ def _prep_metadata(md_sect, path): if not all(isinstance(a, str) for a in value): raise ConfigError('Expected a list of strings for {} field' .format(key)) + elif key == 'requires-extra': + if not isinstance(value, dict): + raise ConfigError('Expected a dict for requires-extra field, found {!r}' + .format(value)) + if not all(isinstance(e, list) for e in value.values()): + raise ConfigError('Expected a dict of lists for requires-extra field') + for e, reqs in value.items(): + if not all(isinstance(a, str) for a in reqs): + raise ConfigError('Expected a string list for requires-extra. (extra {})' + .format(e)) else: if not isinstance(value, str): raise ConfigError('Expected a string for {} field, found {!r}' diff --git a/flit/install.py b/flit/install.py index 502c4bf5..c880c57c 100644 --- a/flit/install.py +++ b/flit/install.py @@ -84,17 +84,24 @@ def __str__(self): return ("Installing packages as root is not recommended. " "To allow this, set FLIT_ROOT_INSTALL=1 and try again.") +class DependencyError(Exception): + def __str__(self): + return 'To install dependencies for extras, you cannot set deps=none.' + class Installer(object): def __init__(self, ini_path, user=None, python=sys.executable, - symlink=False, deps='all', pth=False): + symlink=False, deps='all', extras=(), pth=False): self.ini_path = ini_path self.python = python self.symlink = symlink self.pth = pth self.deps = deps + self.extras = extras if deps != 'none' and os.environ.get('FLIT_NO_NETWORK', ''): self.deps = 'none' log.warning('Not installing dependencies, because FLIT_NO_NETWORK is set') + if deps == 'none' and extras: + raise DependencyError() self.ini_info = inifile.read_pkg_ini(ini_path) self.module = common.Module(self.ini_info['module'], ini_path.parent) @@ -184,6 +191,20 @@ def _record_installed_directory(self, path): for f in files: self.installed_files.append(os.path.join(dirpath, f)) + @property + def extra_reqs(self): + return self.ini_info['metadata'].get('requires_extra', {}) + + def _extras_to_install(self): + extras_to_install = set(self.extras) + if self.deps == 'all' or 'all' in extras_to_install: + extras_to_install |= set(self.extra_reqs.keys()) + # We don’t remove 'all' from the set because there might be an extra called “all”. + elif self.deps == 'develop': + extras_to_install |= {'dev', 'doc', 'test'} + log.info("Extras to install for deps %r: %s", self.deps, extras_to_install) + return extras_to_install + def install_requirements(self): """Install requirements of a package with pip. @@ -196,8 +217,9 @@ def install_requirements(self): return if self.deps in ('all', 'production'): requirements.extend(self.ini_info['metadata'].get('requires_dist', [])) - if self.deps in ('all', 'develop'): - requirements.extend(self.ini_info['metadata'].get('dev_requires', [])) + + for extra in self._extras_to_install(): + requirements.extend(self.extra_reqs.get(extra, [])) # there aren't any requirements, so return if len(requirements) == 0: @@ -311,8 +333,11 @@ def install_with_pip(self): renamed_whl = os.path.join(td, wb.wheel_filename) os.rename(temp_whl, renamed_whl) + extras = self._extras_to_install() + whl_with_extras = '{}[{}]'.format(renamed_whl, ','.join(extras)) \ + if extras else renamed_whl - cmd = [self.python, '-m', 'pip', 'install', renamed_whl] + cmd = [self.python, '-m', 'pip', 'install', whl_with_extras] if self.user: cmd.append('--user') if self.deps == 'none': diff --git a/pyproject.toml b/pyproject.toml index 276ef754..41b8e917 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,18 @@ classifiers=["Intended Audience :: Developers", "Topic :: Software Development :: Libraries :: Python Modules", ] +[tool.flit.metadata.requires-extra] +test = [ + "testpath", + "responses", + "pytest>=2.7.3", + "pytest-warnings", + "pytest-cov", +] +doc = [ + "pygments-github-lexers", # TOML highlighting +] + [tool.flit.metadata.urls] Documentation = "https://flit.readthedocs.io/en/latest/" diff --git a/tests/samples/dev_requires_with_empty_lines.ini b/tests/samples/dev_requires_with_empty_lines.ini deleted file mode 100644 index 31653d24..00000000 --- a/tests/samples/dev_requires_with_empty_lines.ini +++ /dev/null @@ -1,7 +0,0 @@ -[metadata] -module=module1 -author=Sir Robin -author-email=robin@camelot.uk -home-page=http://github.com/sirrobin/module1 -dev-requires= - foo diff --git a/tests/samples/extras-dev-conflict.toml b/tests/samples/extras-dev-conflict.toml new file mode 100644 index 00000000..0fe249d2 --- /dev/null +++ b/tests/samples/extras-dev-conflict.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["flit"] + +[tool.flit.metadata] +module = "module1" +author = "Sir Robin" +author-email = "robin@camelot.uk" +home-page = "http://github.com/sirrobin/module1" +description-file = "EG_README.rst" +dev-requires = ["apackage"] + +[tool.flit.metadata.requires-extra] +dev = ["anotherpackage"] diff --git a/tests/samples/extras.toml b/tests/samples/extras.toml new file mode 100644 index 00000000..16d95a5b --- /dev/null +++ b/tests/samples/extras.toml @@ -0,0 +1,14 @@ +[build-system] +requires = ["flit"] + +[tool.flit.metadata] +module = "module1" +author = "Sir Robin" +author-email = "robin@camelot.uk" +home-page = "http://github.com/sirrobin/module1" +description-file = "EG_README.rst" +requires = ["toml"] + +[tool.flit.metadata.requires-extra] +test = ["pytest"] +custom = ["requests"] diff --git a/tests/test_common.py b/tests/test_common.py index 9b6b70d8..e78355c2 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,4 +1,3 @@ -import os from pathlib import Path from unittest import TestCase import pytest @@ -7,18 +6,18 @@ check_version, normalize_file_permissions ) -samples_dir = os.path.join(os.path.dirname(__file__), 'samples') +samples_dir = Path(__file__).parent / 'samples' class ModuleTests(TestCase): def test_package_importable(self): i = Module('package1', samples_dir) - assert i.path == Path(samples_dir, 'package1') - assert i.file == Path(samples_dir, 'package1', '__init__.py') + assert i.path == samples_dir / 'package1' + assert i.file == samples_dir / 'package1' / '__init__.py' assert i.is_package def test_module_importable(self): i = Module('module1', samples_dir) - assert i.path == Path(samples_dir, 'module1.py') + assert i.path == samples_dir / 'module1.py' assert not i.is_package def test_missing_name(self): diff --git a/tests/test_inifile.py b/tests/test_inifile.py index c7d3e5fb..0d7bab41 100644 --- a/tests/test_inifile.py +++ b/tests/test_inifile.py @@ -1,11 +1,11 @@ import logging -import pathlib +from pathlib import Path import pytest -from flit.inifile import read_pkg_ini, ConfigError, flatten_entrypoints +from flit.inifile import read_pkg_ini, ConfigError, flatten_entrypoints, _prep_metadata -samples_dir = pathlib.Path(__file__).parent / 'samples' +samples_dir = Path(__file__).parent / 'samples' def test_invalid_classifier(): with pytest.raises(ConfigError): @@ -18,13 +18,9 @@ def test_classifiers_with_space(): """ read_pkg_ini(samples_dir / 'classifiers_with_space.ini') -@pytest.mark.parametrize(('filename', 'key', 'expected'), [ - ('requires_with_empty_lines.ini', 'requires_dist', ['foo', 'bar']), - ('dev_requires_with_empty_lines.ini', 'dev_requires', ['foo']), -]) -def test_requires_with_empty_lines(filename, key, expected): - ini_info = read_pkg_ini(samples_dir / filename) - assert ini_info['metadata'][key] == expected +def test_requires_with_empty_lines(): + ini_info = read_pkg_ini(samples_dir / 'requires_with_empty_lines.ini') + assert ini_info['metadata']['requires_dist'] == ['foo', 'bar'] def test_missing_entrypoints(): with pytest.raises(FileNotFoundError): @@ -56,3 +52,13 @@ def test_bad_description_extension(caplog): assert info['metadata']['description_content_type'] is None assert any((r.levelno == logging.WARN and "Unknown extension" in r.msg) for r in caplog.records) + +@pytest.mark.parametrize(('erroneous', 'match'), [ + ({'requires-extra': None}, r'Expected a dict for requires-extra field'), + ({'requires-extra': dict(dev=None)}, r'Expected a dict of lists for requires-extra field'), + ({'requires-extra': dict(dev=[1])}, r'Expected a string list for requires-extra'), +]) +def test_faulty_requires_extra(erroneous, match): + metadata = {'module': 'mymod', 'author': '', 'author-email': ''} + with pytest.raises(ConfigError, match=match): + _prep_metadata(dict(metadata, **erroneous), None) diff --git a/tests/test_install.py b/tests/test_install.py index 1ccee245..9c248d06 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -5,10 +5,11 @@ from unittest import TestCase, SkipTest from unittest.mock import patch +import pytest from testpath import assert_isfile, assert_isdir, assert_islink, MockCommand from flit import install -from flit.install import Installer, _requires_dist_to_pip_requirement +from flit.install import Installer, _requires_dist_to_pip_requirement, DependencyError samples_dir = pathlib.Path(__file__).parent / 'samples' @@ -117,6 +118,43 @@ def test_install_requires(self): assert len(calls) == 1 assert calls[0]['argv'][1:5] == ['-m', 'pip', 'install', '-r'] + def test_extras_error(self): + with pytest.raises(DependencyError): + Installer(samples_dir / 'requires-requests.toml', + user=False, deps='none', extras='dev') + +@pytest.mark.parametrize(('deps', 'extras', 'installed'), [ + ('none', [], set()), + ('develop', [], {'pytest;'}), # TODO: why not also normal reqs, i.e. toml? + ('production', [], {'toml;'}), + ('all', [], {'toml;', 'pytest;', 'requests;'}), +]) +def test_install_requires_extra(deps, extras, installed): + it = InstallTests() + try: + it.setUp() + ins = Installer(samples_dir / 'extras.toml', python='mock_python', + user=False, deps=deps, extras=extras) + + cmd = MockCommand('mock_python') + get_reqs = ( + "#!{python}\n" + "import sys\n" + "with open({recording_file!r}, 'wb') as w, open(sys.argv[-1], 'rb') as r:\n" + " w.write(r.read())" + ).format(python=sys.executable, recording_file=cmd.recording_file) + cmd.content = get_reqs + + with cmd as mock_py: + ins.install_requirements() + with open(mock_py.recording_file) as f: + str_deps = f.read() + deps = str_deps.split('\n') if str_deps else [] + + assert set(deps) == installed + finally: + it.tearDown() + def test_requires_dist_to_pip_requirement(): rd = 'pathlib2 (>=2.3); python_version == "2.7"' assert _requires_dist_to_pip_requirement(rd) == \ diff --git a/tests/test_metadata.py b/tests/test_metadata.py new file mode 100644 index 00000000..51870f9b --- /dev/null +++ b/tests/test_metadata.py @@ -0,0 +1,30 @@ +from pathlib import Path + +import pytest + +from flit.common import Metadata +from flit.inifile import read_pkg_ini + +samples_dir = Path(__file__).parent / 'samples' + +def test_extras(): + info = read_pkg_ini(samples_dir / 'extras.toml') + assert info['metadata']['requires_extra']['test'] == ['pytest'] + assert info['metadata']['requires_extra']['custom'] == ['requests'] + +def test_extras_dev_conflict(): + info = read_pkg_ini(samples_dir / 'extras-dev-conflict.toml') + with pytest.raises(ValueError, match=r'Ambiguity'): + Metadata(dict(name=info['module'], version='0.0', summary='', **info['metadata'])) + +def test_extras_dev_warning(caplog): + info = read_pkg_ini(samples_dir / 'extras-dev-conflict.toml') + info['metadata']['requires_extra'] = {} + meta = Metadata(dict(name=info['module'], version='0.0', summary='', **info['metadata'])) + assert '“dev-requires = ...” is obsolete' in caplog.text + assert set(meta.requires_dist) == {'apackage; extra == "dev"'} + +def test_extra_conditions(): + info = read_pkg_ini(samples_dir / 'extras.toml') + meta = Metadata(dict(name=info['module'], version='0.0', summary='', **info['metadata'])) + assert set(meta.requires_dist) == {'toml', 'pytest; extra == "test"', 'requests; extra == "custom"'}