From 53a3cae97642b55e77e5187d0175d15603447998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Eustace?= Date: Fri, 26 Jul 2019 22:52:14 +0200 Subject: [PATCH] Add support for url dependencies --- docs/docs/versions.md | 18 ++ poetry/console/commands/add.py | 6 +- poetry/console/commands/init.py | 56 +++-- poetry/json/schemas/poetry-schema.json | 39 +++ poetry/packages/__init__.py | 1 + poetry/packages/dependency.py | 3 + poetry/packages/package.py | 6 +- poetry/packages/url_dependency.py | 40 +++ poetry/puzzle/provider.py | 62 ++++- poetry/repositories/legacy_repository.py | 3 +- poetry/repositories/pypi_repository.py | 145 +---------- poetry/utils/inspector.py | 236 ++++++++++++++++++ tests/conftest.py | 26 ++ tests/console/commands/test_add.py | 80 ++++++ tests/console/conftest.py | 20 ++ .../fixtures/with-url-dependency.test | 35 +++ tests/installation/test_installer.py | 15 ++ .../with_url_dependency/pyproject.toml | 24 ++ .../with_url_dependency/__init__.py | 0 tests/masonry/builders/test_builder.py | 17 ++ 20 files changed, 660 insertions(+), 172 deletions(-) create mode 100644 poetry/packages/url_dependency.py create mode 100644 poetry/utils/inspector.py create mode 100644 tests/installation/fixtures/with-url-dependency.test create mode 100644 tests/masonry/builders/fixtures/with_url_dependency/pyproject.toml create mode 100644 tests/masonry/builders/fixtures/with_url_dependency/with_url_dependency/__init__.py diff --git a/docs/docs/versions.md b/docs/docs/versions.md index a4e616c30be..f0fd3934563 100644 --- a/docs/docs/versions.md +++ b/docs/docs/versions.md @@ -113,6 +113,24 @@ my-package = { path = "../my-package/dist/my-package-0.1.0.tar.gz" } You can install path dependencies in editable/development mode. Just pass `--develop my-package` (repeatable as much as you want) to the `install` command. + + +### `url` dependencies + +To depend on a library located on a remote archive, +you can use the `url` property: + +```toml +[tool.poetry.dependencies] +# directory +my-package = { url = "https://example.com/my-package-0.1.0.tar.gz" } +``` + +with the corresponding `add` call: + +```bash +poetry add https://example.com/my-package-0.1.0.tar.gz +``` ### Python restricted dependencies diff --git a/poetry/console/commands/add.py b/poetry/console/commands/add.py index 9c44d66b2e7..09bf0fd09bc 100644 --- a/poetry/console/commands/add.py +++ b/poetry/console/commands/add.py @@ -76,7 +76,11 @@ def handle(self): for key in poetry_content[section]: if key.lower() == name.lower(): pair = self._parse_requirements([name])[0] - if "git" in pair or pair.get("version") == "latest": + if ( + "git" in pair + or "url" in pair + or pair.get("version") == "latest" + ): continue raise ValueError("Package {} is already present".format(name)) diff --git a/poetry/console/commands/init.py b/poetry/console/commands/init.py index 8d0756784ec..efc52c25a73 100644 --- a/poetry/console/commands/init.py +++ b/poetry/console/commands/init.py @@ -14,6 +14,8 @@ from poetry.utils._compat import Path from poetry.utils._compat import OrderedDict +from poetry.utils._compat import urlparse +from poetry.utils.helpers import temporary_directory from .command import Command from .env_command import EnvCommand @@ -149,6 +151,7 @@ def handle(self): " - A git url with a revision (https://github.com/sdispater/poetry.git@develop)\n" " - A file path (../my-package/my-package.whl)\n" " - A directory (../my-package/)\n" + " - An url (https://example.com/packages/my-package-0.1.0.tar.gz)\n" ) help_displayed = False if self.confirm(question, True): @@ -211,6 +214,7 @@ def _determine_requirements( constraint = self._parse_requirements([package])[0] if ( "git" in constraint + or "url" in constraint or "path" in constraint or "version" in constraint ): @@ -276,7 +280,7 @@ def _determine_requirements( requires = self._parse_requirements(requires) result = [] for requirement in requires: - if "git" in requirement or "path" in requirement: + if "git" in requirement or "url" in requirement or "path" in requirement: result.append(requirement) continue elif "version" not in requirement: @@ -343,28 +347,42 @@ def _parse_requirements( extras = [e.strip() for e in extras_m.group(1).split(",")] requirement, _ = requirement.split("[") - if requirement.startswith(("git+https://", "git+ssh://")): - url = requirement.lstrip("git+") - rev = None - if "@" in url: - url, rev = url.split("@") + url_parsed = urlparse.urlparse(requirement) + if url_parsed.scheme and url_parsed.netloc: + # Url + if url_parsed.scheme in ["git+https", "git+ssh"]: + url = requirement.lstrip("git+") + rev = None + if "@" in url: + url, rev = url.split("@") + + pair = OrderedDict( + [("name", url.split("/")[-1].rstrip(".git")), ("git", url)] + ) + if rev: + pair["rev"] = rev - pair = OrderedDict( - [("name", url.split("/")[-1].rstrip(".git")), ("git", url)] - ) - if rev: - pair["rev"] = rev + if extras: + pair["extras"] = extras - if extras: - pair["extras"] = extras + package = Provider.get_package_from_vcs( + "git", url, reference=pair.get("rev") + ) + pair["name"] = package.name + result.append(pair) - package = Provider.get_package_from_vcs( - "git", url, reference=pair.get("rev") - ) - pair["name"] = package.name - result.append(pair) + continue + elif url_parsed.scheme in ["http", "https"]: + package = Provider.get_package_from_url(requirement) - continue + pair = OrderedDict( + [("name", package.name), ("url", package.source_url)] + ) + if extras: + pair["extras"] = extras + + result.append(pair) + continue elif (os.path.sep in requirement or "/" in requirement) and cwd.joinpath( requirement ).exists(): diff --git a/poetry/json/schemas/poetry-schema.json b/poetry/json/schemas/poetry-schema.json index 86305516f75..c02c93aff79 100644 --- a/poetry/json/schemas/poetry-schema.json +++ b/poetry/json/schemas/poetry-schema.json @@ -210,6 +210,9 @@ { "$ref": "#/definitions/path-dependency" }, + { + "$ref": "#/definitions/url-dependency" + }, { "$ref": "#/definitions/multiple-constraints-dependency" } @@ -394,6 +397,42 @@ } } }, + "url-dependency": { + "type": "object", + "required": [ + "url" + ], + "additionalProperties": false, + "properties": { + "url": { + "type": "string", + "description": "The url to the file." + }, + "python": { + "type": "string", + "description": "The python versions for which the dependency should be installed." + }, + "platform": { + "type": "string", + "description": "The platform(s) for which the dependency should be installed." + }, + "markers": { + "type": "string", + "description": "The PEP 508 compliant environment markers for which the dependency should be installed." + }, + "optional": { + "type": "boolean", + "description": "Whether the dependency is optional or not." + }, + "extras": { + "type": "array", + "description": "The required extras for this dependency.", + "items": { + "type": "string" + } + } + } + }, "multiple-constraints-dependency": { "type": "array", "minItems": 1, diff --git a/poetry/packages/__init__.py b/poetry/packages/__init__.py index 1a4b2153c81..27f78e48066 100644 --- a/poetry/packages/__init__.py +++ b/poetry/packages/__init__.py @@ -20,6 +20,7 @@ from .utils.utils import is_url from .utils.utils import path_to_url from .utils.utils import strip_extras +from .url_dependency import URLDependency from .vcs_dependency import VCSDependency diff --git a/poetry/packages/dependency.py b/poetry/packages/dependency.py index 459aab2f897..26d917f6109 100644 --- a/poetry/packages/dependency.py +++ b/poetry/packages/dependency.py @@ -171,6 +171,9 @@ def is_file(self): def is_directory(self): return False + def is_url(self): + return False + def accepts(self, package): # type: (poetry.packages.Package) -> bool """ Determines if the given package matches this dependency. diff --git a/poetry/packages/package.py b/poetry/packages/package.py index 981c5e18b00..648a476cfd4 100644 --- a/poetry/packages/package.py +++ b/poetry/packages/package.py @@ -18,8 +18,8 @@ from .dependency import Dependency from .directory_dependency import DirectoryDependency from .file_dependency import FileDependency +from .url_dependency import URLDependency from .vcs_dependency import VCSDependency -from .utils.utils import convert_markers from .utils.utils import create_nested_marker AUTHOR_REGEX = re.compile(r"(?u)^(?P[- .,\w\d'’\"()]+)(?: <(?P.+?)>)?$") @@ -111,7 +111,7 @@ def pretty_string(self): @property def full_pretty_version(self): - if self.source_type in ["file", "directory"]: + if self.source_type in ["file", "directory", "url"]: return "{} {}".format(self._pretty_version, self.source_url) if self.source_type not in ["hg", "git"]: @@ -314,6 +314,8 @@ def add_dependency( base=self.root_dir, develop=constraint.get("develop", True), ) + elif "url" in constraint: + dependency = URLDependency(name, constraint["url"], category=category) else: version = constraint["version"] diff --git a/poetry/packages/url_dependency.py b/poetry/packages/url_dependency.py new file mode 100644 index 00000000000..f128dc49366 --- /dev/null +++ b/poetry/packages/url_dependency.py @@ -0,0 +1,40 @@ +from poetry.utils._compat import urlparse + +from .dependency import Dependency + + +class URLDependency(Dependency): + def __init__( + self, + name, + url, # type: str + category="main", # type: str + optional=False, # type: bool + ): + self._url = url + + parsed = urlparse.urlparse(url) + if not parsed.scheme or not parsed.netloc: + raise ValueError("{} does not seem like a valid url".format(url)) + + super(URLDependency, self).__init__( + name, "*", category=category, optional=optional, allows_prereleases=True + ) + + @property + def url(self): + return self._url + + @property + def base_pep_508_name(self): # type: () -> str + requirement = self.pretty_name + + if self.extras: + requirement += "[{}]".format(",".join(self.extras)) + + requirement += " @ {}".format(self._url) + + return requirement + + def is_url(self): # type: () -> bool + return True diff --git a/poetry/puzzle/provider.py b/poetry/puzzle/provider.py index b795bcaa83f..75c30195b96 100644 --- a/poetry/puzzle/provider.py +++ b/poetry/puzzle/provider.py @@ -17,6 +17,7 @@ from poetry.packages import FileDependency from poetry.packages import Package from poetry.packages import PackageCollection +from poetry.packages import URLDependency from poetry.packages import VCSDependency from poetry.packages import dependency_from_pep_508 @@ -30,10 +31,13 @@ from poetry.utils._compat import PY35 from poetry.utils._compat import Path from poetry.utils._compat import OrderedDict +from poetry.utils._compat import urlparse from poetry.utils.helpers import parse_requires from poetry.utils.helpers import safe_rmtree +from poetry.utils.helpers import temporary_directory from poetry.utils.env import EnvManager from poetry.utils.env import EnvCommandError +from poetry.utils.inspector import Inspector from poetry.utils.setup_reader import SetupReader from poetry.utils.toml_file import TomlFile @@ -63,6 +67,7 @@ def __init__( self._package = package self._pool = pool self._io = io + self._inspector = Inspector() self._python_constraint = package.python_constraint self._search_for = {} self._is_debugging = self._io.is_debug() or self._io.is_very_verbose() @@ -127,6 +132,8 @@ def search_for(self, dependency): # type: (Dependency) -> List[Package] packages = self.search_for_file(dependency) elif dependency.is_directory(): packages = self.search_for_directory(dependency) + elif dependency.is_url(): + packages = self.search_for_url(dependency) else: constraint = dependency.constraint @@ -234,18 +241,18 @@ def search_for_file(self, dependency): # type: (FileDependency) -> List[Package @classmethod def get_package_from_file(cls, file_path): # type: (Path) -> Package - if file_path.suffix == ".whl": - meta = pkginfo.Wheel(str(file_path)) - else: - # Assume sdist - meta = pkginfo.SDist(str(file_path)) + info = Inspector().inspect(file_path) + if not info["name"]: + raise RuntimeError( + "Unable to determine the package name of {}".format(file_path) + ) - package = Package(meta.name, meta.version) + package = Package(info["name"], info["version"]) package.source_type = "file" package.source_url = file_path.as_posix() - package.description = meta.summary - for req in meta.requires_dist: + package.description = info["summary"] + for req in info["requires_dist"]: dep = dependency_from_pep_508(req) for extra in dep.in_extras: if extra not in package.extras: @@ -256,8 +263,8 @@ def get_package_from_file(cls, file_path): # type: (Path) -> Package if not dep.is_optional(): package.requires.append(dep) - if meta.requires_python: - package.python_versions = meta.requires_python + if info["requires_python"]: + package.python_versions = info["requires_python"] return package @@ -428,6 +435,40 @@ def get_package_from_directory( return package + def search_for_url(self, dependency): # type: (URLDependency) -> List[Package] + package = self.get_package_from_url(dependency.url) + + if dependency.name != package.name: + # For now, the dependency's name must match the actual package's name + raise RuntimeError( + "The dependency name for {} does not match the actual package's name: {}".format( + dependency.name, package.name + ) + ) + + for extra in dependency.extras: + if extra in package.extras: + for dep in package.extras[extra]: + dep.activate() + + package.requires += package.extras[extra] + + return [package] + + @classmethod + def get_package_from_url(cls, url): # type: (str) -> Package + with temporary_directory() as temp_dir: + temp_dir = Path(temp_dir) + file_name = os.path.basename(urlparse.urlparse(url).path) + Inspector().download(url, temp_dir / file_name) + + package = cls.get_package_from_file(temp_dir / file_name) + + package.source_type = "url" + package.source_url = url + + return package + def incompatibilities_for( self, package ): # type: (DependencyPackage) -> List[Incompatibility] @@ -495,6 +536,7 @@ def complete_package( if not package.is_root() and package.source_type not in { "directory", "file", + "url", "git", }: package = DependencyPackage( diff --git a/poetry/repositories/legacy_repository.py b/poetry/repositories/legacy_repository.py index 910cac368cb..dea6f5bd640 100644 --- a/poetry/repositories/legacy_repository.py +++ b/poetry/repositories/legacy_repository.py @@ -39,6 +39,7 @@ from poetry.semver import VersionRange from poetry.utils._compat import Path from poetry.utils.helpers import canonicalize_name +from poetry.utils.inspector import Inspector from poetry.utils.patterns import wheel_file_re from poetry.version.markers import InvalidMarker @@ -163,8 +164,8 @@ def __init__( self._name = name self._url = url.rstrip("/") self._auth = auth + self._inspector = Inspector() self._cache_dir = Path(CACHE_DIR) / "cache" / "repositories" / name - self._cache = CacheManager( { "default": "releases", diff --git a/poetry/repositories/pypi_repository.py b/poetry/repositories/pypi_repository.py index 1179d424173..cbca5190d3c 100644 --- a/poetry/repositories/pypi_repository.py +++ b/poetry/repositories/pypi_repository.py @@ -1,13 +1,8 @@ import logging import os -import tarfile -import zipfile -import pkginfo -from bz2 import BZ2File from collections import defaultdict -from gzip import GzipFile from typing import Dict from typing import List from typing import Union @@ -40,6 +35,7 @@ from poetry.utils._compat import to_str from poetry.utils.helpers import parse_requires from poetry.utils.helpers import temporary_directory +from poetry.utils.inspector import Inspector from poetry.utils.patterns import wheel_file_re from poetry.utils.setup_reader import SetupReader from poetry.version.markers import InvalidMarker @@ -76,6 +72,7 @@ def __init__(self, url="https://pypi.org/", disable_cache=False, fallback=True): self._session = CacheControl( session(), cache=FileCache(str(release_cache_dir / "_http")) ) + self._inspector = Inspector() super(PyPiRepository, self).__init__() @@ -456,30 +453,14 @@ def _get_info_from_wheel( "Downloading wheel: {}".format(urlparse.urlparse(url).path.rsplit("/")[-1]), level="debug", ) - info = {"summary": "", "requires_python": None, "requires_dist": None} filename = os.path.basename(urlparse.urlparse(url).path.rsplit("/")[-1]) with temporary_directory() as temp_dir: - filepath = os.path.join(temp_dir, filename) - self._download(url, filepath) - - try: - meta = pkginfo.Wheel(filepath) - except ValueError: - # Unable to determine dependencies - # Assume none - return info - - if meta.summary: - info["summary"] = meta.summary or "" - - info["requires_python"] = meta.requires_python - - if meta.requires_dist: - info["requires_dist"] = meta.requires_dist + filepath = Path(temp_dir) / filename + self._download(url, str(filepath)) - return info + return self._inspector.inspect_wheel(filepath) def _get_info_from_sdist( self, url @@ -488,7 +469,6 @@ def _get_info_from_sdist( "Downloading sdist: {}".format(urlparse.urlparse(url).path.rsplit("/")[-1]), level="debug", ) - info = {"summary": "", "requires_python": None, "requires_dist": None} filename = os.path.basename(urlparse.urlparse(url).path) @@ -496,120 +476,7 @@ def _get_info_from_sdist( filepath = Path(temp_dir) / filename self._download(url, str(filepath)) - try: - meta = pkginfo.SDist(str(filepath)) - if meta.summary: - info["summary"] = meta.summary - - if meta.requires_python: - info["requires_python"] = meta.requires_python - - if meta.requires_dist: - info["requires_dist"] = list(meta.requires_dist) - - return info - except ValueError: - # Unable to determine dependencies - # We pass and go deeper - pass - - # Still not dependencies found - # So, we unpack and introspect - suffix = filepath.suffix - gz = None - if suffix == ".zip": - tar = zipfile.ZipFile(str(filepath)) - else: - if suffix == ".bz2": - gz = BZ2File(str(filepath)) - suffixes = filepath.suffixes - if len(suffixes) > 1 and suffixes[-2] == ".tar": - suffix = ".tar.bz2" - else: - gz = GzipFile(str(filepath)) - suffix = ".tar.gz" - - tar = tarfile.TarFile(str(filepath), fileobj=gz) - - try: - tar.extractall(os.path.join(temp_dir, "unpacked")) - finally: - if gz: - gz.close() - - tar.close() - - unpacked = Path(temp_dir) / "unpacked" - sdist_dir = unpacked / Path(filename).name.rstrip(suffix) - - # Checking for .egg-info at root - eggs = list(sdist_dir.glob("*.egg-info")) - if eggs: - egg_info = eggs[0] - - requires = egg_info / "requires.txt" - if requires.exists(): - with requires.open(encoding="utf-8") as f: - info["requires_dist"] = parse_requires(f.read()) - - return info - - # Searching for .egg-info in sub directories - eggs = list(sdist_dir.glob("**/*.egg-info")) - if eggs: - egg_info = eggs[0] - - requires = egg_info / "requires.txt" - if requires.exists(): - with requires.open(encoding="utf-8") as f: - info["requires_dist"] = parse_requires(f.read()) - - return info - - # Still nothing, try reading (without executing it) - # the setup.py file. - try: - setup_info = self._inspect_sdist_with_setup(sdist_dir) - - for key, value in info.items(): - if value: - continue - - info[key] = setup_info[key] - - return info - except Exception as e: - self._log( - "An error occurred when reading setup.py or setup.cfg: {}".format( - str(e) - ), - "warning", - ) - return info - - def _inspect_sdist_with_setup(self, sdist_dir): - info = {"requires_python": None, "requires_dist": None} - - result = SetupReader.read_from_directory(sdist_dir) - requires = "" - for dep in result["install_requires"]: - requires += dep + "\n" - - if result["extras_require"]: - requires += "\n" - - for extra_name, deps in result["extras_require"].items(): - requires += "[{}]\n".format(extra_name) - - for dep in deps: - requires += dep + "\n" - - requires += "\n" - - info["requires_dist"] = parse_requires(requires) - info["requires_python"] = result["python_requires"] - - return info + return self._inspector.inspect_sdist(filepath) def _download(self, url, dest): # type: (str, str) -> None r = get(url, stream=True) diff --git a/poetry/utils/inspector.py b/poetry/utils/inspector.py new file mode 100644 index 00000000000..446c4747142 --- /dev/null +++ b/poetry/utils/inspector.py @@ -0,0 +1,236 @@ +from typing import Dict +from typing import List +from typing import Union + +import logging +import os +import tarfile +import zipfile +from bz2 import BZ2File +from gzip import GzipFile + +import pkginfo +from requests import get + +from ._compat import Path +from .helpers import parse_requires +from .setup_reader import SetupReader +from .toml_file import TomlFile + +logger = logging.getLogger(__name__) + + +class Inspector: + """ + A class to download and inspect remote packages. + """ + + @classmethod + def download(cls, url, dest): # type: (str, Path) -> None + r = get(url, stream=True) + r.raise_for_status() + + with open(str(dest), "wb") as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + + def inspect(self, file_path): # type: (Path) -> Dict[str, Union[str, List[str]]] + if file_path.suffix == ".whl": + return self.inspect_wheel(file_path) + + return self.inspect_sdist(file_path) + + def inspect_wheel( + self, file_path + ): # type: (Path) -> Dict[str, Union[str, List[str]]] + info = { + "name": "", + "version": "", + "summary": "", + "requires_python": None, + "requires_dist": None, + } + + try: + meta = pkginfo.Wheel(str(file_path)) + except ValueError: + # Unable to determine dependencies + # Assume none + return info + + if meta.name: + info["name"] = meta.name + + if meta.version: + info["version"] = meta.version + + if meta.summary: + info["summary"] = meta.summary or "" + + info["requires_python"] = meta.requires_python + + if meta.requires_dist: + info["requires_dist"] = meta.requires_dist + + return info + + def inspect_sdist( + self, file_path + ): # type: (Path) -> Dict[str, Union[str, List[str]]] + info = { + "name": "", + "version": "", + "summary": "", + "requires_python": None, + "requires_dist": None, + } + + try: + meta = pkginfo.SDist(str(file_path)) + if meta.name: + info["name"] = meta.name + + if meta.version: + info["version"] = meta.version + + if meta.summary: + info["summary"] = meta.summary + + if meta.requires_python: + info["requires_python"] = meta.requires_python + + if meta.requires_dist: + info["requires_dist"] = list(meta.requires_dist) + + return info + except ValueError: + # Unable to determine dependencies + # We pass and go deeper + pass + + # Still not dependencies found + # So, we unpack and introspect + suffix = file_path.suffix + gz = None + if suffix == ".zip": + tar = zipfile.ZipFile(str(file_path)) + else: + if suffix == ".bz2": + gz = BZ2File(str(file_path)) + suffixes = file_path.suffixes + if len(suffixes) > 1 and suffixes[-2] == ".tar": + suffix = ".tar.bz2" + else: + gz = GzipFile(str(file_path)) + suffix = ".tar.gz" + + tar = tarfile.TarFile(str(file_path), fileobj=gz) + + try: + tar.extractall(os.path.join(str(file_path.parent), "unpacked")) + finally: + if gz: + gz.close() + + tar.close() + + unpacked = file_path.parent / "unpacked" + elements = list(unpacked.glob("*")) + if len(elements) == 1 and elements[0].is_dir(): + sdist_dir = elements[0] + else: + sdist_dir = unpacked / file_path.name.rstrip(suffix) + + pyproject = TomlFile(sdist_dir / "pyproject.toml") + if pyproject.exists(): + from poetry.poetry import Poetry + + pyproject_content = pyproject.read() + if "tool" in pyproject_content and "poetry" in pyproject_content["tool"]: + package = Poetry.create(sdist_dir).package + return { + "name": package.name, + "version": package.version.text, + "summary": package.description, + "requires_dist": [dep.to_pep_508() for dep in package.requires], + "requires_python": package.python_versions, + } + + # Checking for .egg-info at root + eggs = list(sdist_dir.glob("*.egg-info")) + if eggs: + egg_info = eggs[0] + + requires = egg_info / "requires.txt" + if requires.exists(): + with requires.open(encoding="utf-8") as f: + info["requires_dist"] = parse_requires(f.read()) + + return info + + # Searching for .egg-info in sub directories + eggs = list(sdist_dir.glob("**/*.egg-info")) + if eggs: + egg_info = eggs[0] + + requires = egg_info / "requires.txt" + if requires.exists(): + with requires.open(encoding="utf-8") as f: + info["requires_dist"] = parse_requires(f.read()) + + return info + + # Still nothing, try reading (without executing it) + # the setup.py file. + try: + setup_info = self._inspect_sdist_with_setup(sdist_dir) + + for key, value in info.items(): + if value: + continue + + info[key] = setup_info[key] + + return info + except Exception as e: + logger.warning( + "An error occurred when reading setup.py or setup.cfg: {}".format( + str(e) + ) + ) + return info + + def _inspect_sdist_with_setup( + self, sdist_dir + ): # type: (Path) -> Dict[str, Union[str, List[str]]] + info = { + "name": None, + "version": None, + "summary": "", + "requires_python": None, + "requires_dist": None, + } + + result = SetupReader.read_from_directory(sdist_dir) + requires = "" + for dep in result["install_requires"]: + requires += dep + "\n" + + if result["extras_require"]: + requires += "\n" + + for extra_name, deps in result["extras_require"].items(): + requires += "[{}]\n".format(extra_name) + + for dep in deps: + requires += dep + "\n" + + requires += "\n" + + info["name"] = result["name"] + info["version"] = result["version"] + info["requires_dist"] = parse_requires(requires) + info["requires_python"] = result["python_requires"] + + return info diff --git a/tests/conftest.py b/tests/conftest.py index d642415014b..1ab630fef0b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,8 @@ import urlparse from poetry.config import Config +from poetry.utils._compat import PY2 +from poetry.utils._compat import WINDOWS from poetry.utils._compat import Path from poetry.utils.toml_file import TomlFile @@ -43,10 +45,28 @@ def mock_clone(_, source, dest): / parts.path.lstrip("/").rstrip(".git") ) + if dest.exists(): + shutil.rmtree(str(dest)) + shutil.rmtree(str(dest)) shutil.copytree(str(folder), str(dest)) +def mock_download(self, url, dest): + parts = urlparse.urlparse(url) + + fixtures = Path(__file__).parent / "fixtures" + fixture = fixtures / parts.path.lstrip("/") + + if dest.exists(): + os.unlink(str(dest)) + + if PY2 and WINDOWS: + shutil.copyfile(str(fixture), str(dest)) + else: + os.symlink(str(fixture), str(dest)) + + @pytest.fixture def tmp_dir(): dir_ = tempfile.mkdtemp(prefix="poetry_") @@ -75,6 +95,12 @@ def git_mock(mocker): p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24" +@pytest.fixture(autouse=True) +def download_mock(mocker): + # Patch download to not download anything but to just copy from fixtures + mocker.patch("poetry.utils.inspector.Inspector.download", new=mock_download) + + @pytest.fixture def http(): httpretty.enable() diff --git a/tests/console/commands/test_add.py b/tests/console/commands/test_add.py index 3daf0ee7631..2a50000bf51 100644 --- a/tests/console/commands/test_add.py +++ b/tests/console/commands/test_add.py @@ -446,6 +446,86 @@ def test_add_constraint_with_extras_option(app, repo, installer): } +def test_add_url_constraint_wheel(app, repo, installer, mocker): + p = mocker.patch("poetry.utils._compat.Path.cwd") + p.return_value = Path(__file__) / ".." + + command = app.find("add") + tester = CommandTester(command) + + repo.add_package(get_package("pendulum", "1.4.4")) + + tester.execute( + "https://poetry.eustace.io/distributions/demo-0.1.0-py2.py3-none-any.whl" + ) + + expected = """\ + +Updating dependencies +Resolving dependencies... + +Writing lock file + + +Package operations: 2 installs, 0 updates, 0 removals + + - Installing pendulum (1.4.4) + - Installing demo (0.1.0 https://poetry.eustace.io/distributions/demo-0.1.0-py2.py3-none-any.whl) +""" + + assert expected == tester.io.fetch_output() + + assert len(installer.installs) == 2 + + content = app.poetry.file.read()["tool"]["poetry"] + + assert "demo" in content["dependencies"] + assert content["dependencies"]["demo"] == { + "url": "https://poetry.eustace.io/distributions/demo-0.1.0-py2.py3-none-any.whl" + } + + +def test_add_url_constraint_wheel_with_extras(app, repo, installer, mocker): + command = app.find("add") + tester = CommandTester(command) + + repo.add_package(get_package("pendulum", "1.4.4")) + repo.add_package(get_package("cleo", "0.6.5")) + repo.add_package(get_package("tomlkit", "0.5.5")) + + tester.execute( + "https://poetry.eustace.io/distributions/demo-0.1.0-py2.py3-none-any.whl[foo,bar]" + ) + + expected = """\ + +Updating dependencies +Resolving dependencies... + +Writing lock file + + +Package operations: 4 installs, 0 updates, 0 removals + + - Installing cleo (0.6.5) + - Installing pendulum (1.4.4) + - Installing tomlkit (0.5.5) + - Installing demo (0.1.0 https://poetry.eustace.io/distributions/demo-0.1.0-py2.py3-none-any.whl) +""" + + assert expected == tester.io.fetch_output() + + assert len(installer.installs) == 4 + + content = app.poetry.file.read()["tool"]["poetry"] + + assert "demo" in content["dependencies"] + assert content["dependencies"]["demo"] == { + "url": "https://poetry.eustace.io/distributions/demo-0.1.0-py2.py3-none-any.whl", + "extras": ["foo", "bar"], + } + + def test_add_constraint_with_python(app, repo, installer): command = app.find("add") tester = CommandTester(command) diff --git a/tests/console/conftest.py b/tests/console/conftest.py index 602d2adfd33..e08fc15f0cc 100644 --- a/tests/console/conftest.py +++ b/tests/console/conftest.py @@ -17,6 +17,8 @@ from poetry.packages import Locker as BaseLocker from poetry.repositories import Pool from poetry.repositories import Repository as BaseRepository +from poetry.utils._compat import PY2 +from poetry.utils._compat import WINDOWS from poetry.utils._compat import Path from poetry.utils.toml_file import TomlFile from poetry.repositories.exceptions import PackageNotFound @@ -43,6 +45,21 @@ def mock_clone(self, source, dest): shutil.copytree(str(folder), str(dest)) +def mock_download(self, url, dest): + parts = urlparse.urlparse(url) + + fixtures = Path(__file__).parent.parent / "fixtures" + fixture = fixtures / parts.path.lstrip("/") + + if dest.exists(): + shutil.rmtree(str(dest)) + + if PY2 and WINDOWS: + shutil.copyfile(str(fixture), str(dest)) + else: + os.symlink(str(fixture), str(dest)) + + @pytest.fixture def installed(): return BaseRepository() @@ -68,6 +85,9 @@ def setup(mocker, installer, installed, config): p = mocker.patch("poetry.vcs.git.Git.rev_parse") p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24" + # Patch download to not download anything but to just copy from fixtures + mocker.patch("poetry.utils.inspector.Inspector.download", new=mock_download) + # Setting terminal width environ = dict(os.environ) os.environ["COLUMNS"] = "80" diff --git a/tests/installation/fixtures/with-url-dependency.test b/tests/installation/fixtures/with-url-dependency.test new file mode 100644 index 00000000000..c61ac815ad4 --- /dev/null +++ b/tests/installation/fixtures/with-url-dependency.test @@ -0,0 +1,35 @@ +[[package]] +name = "demo" +version = "0.1.0" +description = "" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.source] +type = "url" +reference = "" +url = "https://poetry.eustace.io/distributions/demo-0.1.0-py2.py3-none-any.whl" + +[package.dependencies] +pendulum = ">=1.4.4" + +[package.extras] +bar = ["tomlkit"] +foo = ["cleo"] + +[[package]] +name = "pendulum" +version = "1.4.4" +description = "" +category = "main" +optional = false +python-versions = "*" + +[metadata] +python-versions = "*" +content-hash = "123456789" + +[metadata.hashes] +demo = [] +pendulum = [] diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index b6c6a69b22e..a82615ad82a 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -1499,3 +1499,18 @@ def test_installer_can_install_dependencies_from_forced_source( assert len(installer.installer.installs) == 1 assert len(installer.installer.updates) == 0 assert len(installer.installer.removals) == 0 + + +def test_run_installs_with_url_file(installer, locker, repo, package): + url = "https://poetry.eustace.io/distributions/demo-0.1.0-py2.py3-none-any.whl" + package.add_dependency("demo", {"url": url}) + + repo.add_package(get_package("pendulum", "1.4.4")) + + installer.run() + + expected = fixture("with-url-dependency") + + assert locker.written_data == expected + + assert len(installer.installer.installs) == 2 diff --git a/tests/masonry/builders/fixtures/with_url_dependency/pyproject.toml b/tests/masonry/builders/fixtures/with_url_dependency/pyproject.toml new file mode 100644 index 00000000000..9db0956d95b --- /dev/null +++ b/tests/masonry/builders/fixtures/with_url_dependency/pyproject.toml @@ -0,0 +1,24 @@ +[tool.poetry] +name = "with-url-dependency" +version = "1.2.3" +description = "Some description." +authors = [ + "Sébastien Eustace " +] +license = "MIT" + +homepage = "https://poetry.eustace.io/" +repository = "https://github.com/sdispater/poetry" +documentation = "https://poetry.eustace.io/docs" + +keywords = ["packaging", "dependency", "poetry"] + +classifiers = [ + "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: Libraries :: Python Modules" +] + +# Requirements +[tool.poetry.dependencies] +python = "^3.6" +demo = { url = "https://poetry.eustace.io/distributions/demo-0.1.0-py2.py3-none-any.whl" } diff --git a/tests/masonry/builders/fixtures/with_url_dependency/with_url_dependency/__init__.py b/tests/masonry/builders/fixtures/with_url_dependency/with_url_dependency/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/masonry/builders/test_builder.py b/tests/masonry/builders/test_builder.py index cddcc9c9cef..6b43e5280b8 100644 --- a/tests/masonry/builders/test_builder.py +++ b/tests/masonry/builders/test_builder.py @@ -132,3 +132,20 @@ def test_metadata_with_vcs_dependencies(): requires_dist = metadata["Requires-Dist"] assert "cleo @ git+https://github.com/sdispater/cleo.git@master" == requires_dist + + +def test_metadata_with_url_dependencies(): + builder = Builder( + Poetry.create(Path(__file__).parent / "fixtures" / "with_url_dependency"), + NullEnv(), + NullIO(), + ) + + metadata = Parser().parsestr(builder.get_metadata_content()) + + requires_dist = metadata["Requires-Dist"] + + assert ( + "demo @ https://poetry.eustace.io/distributions/demo-0.1.0-py2.py3-none-any.whl" + == requires_dist + )