Merge pull request conda#529 from jacksmith15/feat/460/configure-pypi…

…-repo-in-environment-file

Configure private pip repositories in the environment file

README.md

@@ -153,15 +153,36 @@ The default category is `main`.
 `conda-lock` can also lock the `dependencies.pip` section of
 [environment.yml][envyaml], using a vendored copy of [Poetry's][poetry] dependency solver.
 
+
 ### private pip repositories
-Right now `conda-lock` only supports [legacy](https://warehouse.pypa.io/api-reference/legacy.html) pypi repos with basic auth. Most self-hosted repositories like Nexus, Artifactory etc. use this. To use this feature, add your private repo into Poetry's config _including_ the basic auth in the url:
 
-```bash
-poetry config repositories.foo https://username:password@foo.repo/simple/
+Right now `conda-lock` only supports [legacy](https://warehouse.pypa.io/api-reference/legacy.html) pypi repos with basic auth. Most self-hosted repositories like Nexus, Artifactory etc. use this. You can configure private pip repositories in a similar way to channels, for example:
+
+```yaml
+channels:
+  - conda-forge
+pip-repositories:
+  - http://$PIP_USER:$PIP_PASSWORD@private-pypi.org/api/pypi/simple
+dependencies:
+  - python=3.11
+  - requests=2.26
+  - pip:
+    - fake-private-package==1.0.0
 ```
 
-The private repo will be used in addition to `pypi.org`. For projects using `pyproject.toml`, it is possible to [disable `pypi.org` entirely](#disabling-pypiorg).
+See [the related docs for private channels](./docs/authenticated_channels.md#what_gets_stored) to understand the rules regarding environment variable substitution.
+
+Alternatively, you can use the `poetry` configuration file format to configure private PyPi repositories. The configuration file should be named `config.toml` and have the following format:
+
+```toml
+[repositories.example]
+url = "https://username:password@example.repo/simple"
+```
+
+The location of this file can be determined with `python -c 'from conda_lock._vendor.poetry.locations import CONFIG_DIR; print(CONFIG_DIR)'`
+
 
+Private repositories will be used in addition to `pypi.org`. For projects using `pyproject.toml`, it is possible to [disable `pypi.org` entirely](#disabling-pypiorg).
 ### --dev-dependencies/--no-dev-dependencies
 
 By default conda-lock will include dev dependencies in the specification of the lock (if the files that the lock

conda_lock/conda_lock.py

@@ -73,6 +73,7 @@
 from conda_lock.lookup import set_lookup_location
 from conda_lock.models.channel import Channel
 from conda_lock.models.lock_spec import LockSpecification
+from conda_lock.models.pip_repository import PipRepository
 from conda_lock.pypi_solver import solve_pypi
 from conda_lock.src_parser import make_lock_spec
 from conda_lock.virtual_package import (
@@ -715,6 +716,7 @@ def _solve_for_arch(
     spec: LockSpecification,
     platform: str,
     channels: List[Channel],
+    pip_repositories: List[PipRepository],
     update_spec: Optional[UpdateSpecification] = None,
     strip_auth: bool = False,
 ) -> List[LockedDependency]:
@@ -756,6 +758,7 @@ def _solve_for_arch(
             conda_locked={dep.name: dep for dep in conda_deps.values()},
             python_version=conda_deps["python"].version,
             platform=platform,
+            pip_repositories=pip_repositories,
             allow_pypi_requests=spec.allow_pypi_requests,
             strip_auth=strip_auth,
         )
@@ -820,6 +823,7 @@ def create_lockfile_from_spec(
             spec=spec,
             platform=platform,
             channels=[*spec.channels, virtual_package_channel],
+            pip_repositories=spec.pip_repositories,
             update_spec=update_spec,
             strip_auth=strip_auth,
         )

conda_lock/models/lock_spec.py

@@ -5,11 +5,12 @@
 
 from typing import Dict, List, Optional, Union
 
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, Field, validator
 from typing_extensions import Literal
 
 from conda_lock.models import StrictModel
 from conda_lock.models.channel import Channel
+from conda_lock.models.pip_repository import PipRepository
 from conda_lock.virtual_package import FakeRepoData
 
 
@@ -61,6 +62,7 @@ class LockSpecification(BaseModel):
     # TODO: Should we store the auth info in here?
     channels: List[Channel]
     sources: List[pathlib.Path]
+    pip_repositories: List[PipRepository] = Field(default_factory=list)
     virtual_package_repo: Optional[FakeRepoData] = None
     allow_pypi_requests: bool = True
 
@@ -84,6 +86,8 @@ def content_hash_for_platform(self, platform: str) -> str:
                 )
             ],
         }
+        if self.pip_repositories:
+            data["pip_repositories"] = [repo.json() for repo in self.pip_repositories]
         if self.virtual_package_repo is not None:
             vpr_data = self.virtual_package_repo.all_repodata
             data["virtual_package_hash"] = {
@@ -103,3 +107,12 @@ def validate_channels(cls, v: List[Union[Channel, str]]) -> List[Channel]:
             if e.url == "nodefaults":
                 raise ValueError("nodefaults channel is not allowed, ref #418")
         return typing.cast(List[Channel], v)
+
+    @validator("pip_repositories", pre=True)
+    def validate_pip_repositories(
+        cls, value: List[Union[PipRepository, str]]
+    ) -> List[PipRepository]:
+        for index, repository in enumerate(value):
+            if isinstance(repository, str):
+                value[index] = PipRepository.from_string(repository)
+        return typing.cast(List[PipRepository], value)

conda_lock/models/pip_repository.py

@@ -0,0 +1,48 @@
+from hashlib import sha256
+from urllib.parse import urlparse, urlunparse
+
+from pydantic import BaseModel, ConfigDict
+
+
+class PipRepository(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    url: str
+
+    @classmethod
+    def from_string(cls, url: str) -> "PipRepository":
+        return PipRepository(url=url)
+
+    @property
+    def base_url(self) -> str:
+        """The base URL of the pip repository, without a URL path."""
+        full_url = urlparse(self.url)
+        return full_url.scheme + "://" + full_url.netloc
+
+    @property
+    def stripped_base_url(self) -> str:
+        """The base URL of the pip repository, without any basic auth."""
+        base_url = urlparse(self.base_url)
+        return urlunparse(base_url._replace(netloc=base_url.netloc.split("@", 1)[-1]))
+
+    @property
+    def name(self) -> str:
+        """Poetry solver requires a name for each repository.
+
+        We use this to match solver results back to their relevant
+        repository.
+        """
+        sha = sha256()
+        sha.update(self.url.encode("utf-8"))
+        return sha.hexdigest()
+
+    def normalize_solver_url(self, solver_url: str) -> str:
+        """Normalize the URL returned by Poetry's solver.
+
+        Poetry doesn't return URLs with URL-based basic Auth, because it gets converted to
+        header-based Basic Auth. Because of this, we have to add the auth back in here.
+        """
+        if not solver_url.startswith(self.stripped_base_url):
+            # The resolved package URL is at a different host to the repository
+            return solver_url
+        return solver_url.replace(self.stripped_base_url, self.base_url, 1)

conda_lock/pypi_solver.py

@@ -2,8 +2,9 @@
 import sys
 
 from pathlib import Path
+from posixpath import expandvars
 from typing import TYPE_CHECKING, Dict, List, Optional
-from urllib.parse import urldefrag, urlsplit, urlunsplit
+from urllib.parse import urldefrag, urlparse, urlsplit, urlunsplit
 
 from clikit.api.io.flags import VERY_VERBOSE
 from clikit.io import ConsoleIO, NullIO
@@ -32,6 +33,7 @@
 )
 from conda_lock.lookup import conda_name_to_pypi_name
 from conda_lock.models import lock_spec
+from conda_lock.models.pip_repository import PipRepository
 
 
 if TYPE_CHECKING:
@@ -188,6 +190,7 @@ def get_requirements(
     platform: str,
     pool: Pool,
     env: Env,
+    pip_repositories: Optional[List[PipRepository]] = None,
     strip_auth: bool = False,
 ) -> List[LockedDependency]:
     """Extract distributions from Poetry package plan, ignoring uninstalls
@@ -196,10 +199,17 @@ def get_requirements(
     """
     chooser = Chooser(pool, env=env)
     requirements: List[LockedDependency] = []
+
+    repositories_by_name = {
+        repository.name: repository for repository in pip_repositories or []
+    }
+
     for op in result:
         if not isinstance(op, Uninstall) and not op.skipped:
             # Take direct references verbatim
             source: Optional[DependencySource] = None
+            source_repository = repositories_by_name.get(op.package.source_reference)
+
             if op.package.source_type == "url":
                 url, fragment = urldefrag(op.package.source_url)
                 hash_type, hash = fragment.split("=")
@@ -221,6 +231,9 @@ def get_requirements(
                     hashes[link.hash_name] = link.hash
                 hash = HashModel.parse_obj(hashes)
 
+            if source_repository:
+                url = source_repository.normalize_solver_url(url)
+
             requirements.append(
                 LockedDependency(
                     name=op.package.name,
@@ -245,6 +258,7 @@ def solve_pypi(
     conda_locked: Dict[str, LockedDependency],
     python_version: str,
     platform: str,
+    pip_repositories: Optional[List[PipRepository]] = None,
     allow_pypi_requests: bool = True,
     verbose: bool = False,
     strip_auth: bool = False,
@@ -283,7 +297,9 @@ def solve_pypi(
     for dep in dependencies:
         dummy_package.add_dependency(dep)
 
-    pool = _prepare_repositories_pool(allow_pypi_requests)
+    pool = _prepare_repositories_pool(
+        allow_pypi_requests, pip_repositories=pip_repositories
+    )
 
     installed = Repository()
     locked = Repository()
@@ -334,7 +350,14 @@ def solve_pypi(
     with s.use_environment(env):
         result = s.solve(use_latest=to_update)
 
-    requirements = get_requirements(result, platform, pool, env, strip_auth=strip_auth)
+    requirements = get_requirements(
+        result,
+        platform,
+        pool,
+        env,
+        pip_repositories=pip_repositories,
+        strip_auth=strip_auth,
+    )
 
     # use PyPI names of conda packages to walking the dependency tree and propagate
     # categories from explicit to transitive dependencies
@@ -361,7 +384,9 @@ def solve_pypi(
     return {dep.name: dep for dep in requirements}
 
 
-def _prepare_repositories_pool(allow_pypi_requests: bool) -> Pool:
+def _prepare_repositories_pool(
+    allow_pypi_requests: bool, pip_repositories: Optional[List[PipRepository]] = None
+) -> Pool:
     """
     Prepare the pool of repositories to solve pip dependencies
 
@@ -373,6 +398,12 @@ def _prepare_repositories_pool(allow_pypi_requests: bool) -> Pool:
     factory = Factory()
     config = factory.create_config()
     repos = [
+        factory.create_legacy_repository(
+            {"name": pip_repository.name, "url": expandvars(pip_repository.url)},
+            config,
+        )
+        for pip_repository in pip_repositories or []
+    ] + [
         factory.create_legacy_repository(
             {"name": source[0], "url": source[1]["url"]}, config
         )

conda_lock/src_parser/__init__.py

@@ -6,6 +6,7 @@
 from conda_lock.common import ordered_union
 from conda_lock.models.channel import Channel
 from conda_lock.models.lock_spec import Dependency, LockSpecification
+from conda_lock.models.pip_repository import PipRepository
 from conda_lock.src_parser.aggregation import aggregate_lock_specs
 from conda_lock.src_parser.environment_yaml import (
     parse_environment_file,
@@ -76,6 +77,7 @@ def make_lock_spec(
     src_files: List[pathlib.Path],
     virtual_package_repo: FakeRepoData,
     channel_overrides: Optional[Sequence[str]] = None,
+    pip_repository_overrides: Optional[Sequence[str]] = None,
     platform_overrides: Optional[Sequence[str]] = None,
     required_categories: Optional[AbstractSet[str]] = None,
 ) -> LockSpecification:
@@ -98,6 +100,15 @@ def make_lock_spec(
         else aggregated_lock_spec.channels
     )
 
+    pip_repositories = (
+        [
+            PipRepository.from_string(repo_override)
+            for repo_override in pip_repository_overrides
+        ]
+        if pip_repository_overrides
+        else aggregated_lock_spec.pip_repositories
+    )
+
     if required_categories is None:
         dependencies = aggregated_lock_spec.dependencies
     else:
@@ -118,6 +129,7 @@ def dep_has_category(d: Dependency, categories: AbstractSet[str]) -> bool:
     return LockSpecification(
         dependencies=dependencies,
         channels=channels,
+        pip_repositories=pip_repositories,
         sources=aggregated_lock_spec.sources,
         virtual_package_repo=virtual_package_repo,
         allow_pypi_requests=aggregated_lock_spec.allow_pypi_requests,

conda_lock/src_parser/aggregation.py

@@ -1,12 +1,13 @@
 import logging
 
 from itertools import chain
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, TypeVar
 
 from conda_lock.common import ordered_union
 from conda_lock.errors import ChannelAggregationError
 from conda_lock.models.channel import Channel
 from conda_lock.models.lock_spec import Dependency, LockSpecification
+from conda_lock.models.pip_repository import PipRepository
 
 
 logger = logging.getLogger(__name__)
@@ -44,10 +45,20 @@ def aggregate_lock_specs(
     except ValueError as e:
         raise ChannelAggregationError(*e.args)
 
+    try:
+        # For discussion see
+        # <https://github.com/conda/conda-lock/pull/529#issuecomment-1766060611>
+        pip_repositories = unify_package_sources(
+            [lock_spec.pip_repositories for lock_spec in lock_specs]
+        )
+    except ValueError as e:
+        raise ChannelAggregationError(*e.args)
+
     return LockSpecification(
         dependencies=dependencies,
         # Ensure channel are correctly ordered
         channels=channels,
+        pip_repositories=pip_repositories,
         # uniquify metadata, preserving order
         sources=ordered_union(lock_spec.sources for lock_spec in lock_specs),
         allow_pypi_requests=all(
@@ -56,7 +67,12 @@ def aggregate_lock_specs(
     )
 
 
-def unify_package_sources(collections: List[List[Channel]]) -> List[Channel]:
+PackageSource = TypeVar("PackageSource", Channel, PipRepository)
+
+
+def unify_package_sources(
+    collections: List[List[PackageSource]],
+) -> List[PackageSource]:
     """Unify the package sources from multiple lock specs.
 
     To be able to merge the lock specs, the package sources must be compatible between

conda_lock/src_parser/environment_yaml.py

@@ -120,6 +120,8 @@ def parse_environment_file(
     except ValueError:
         pass
 
+    pip_repositories: List[str] = env_yaml_data.get("pip-repositories", [])
+
     # These extension fields are nonstandard
     category: str = env_yaml_data.get("category") or "main"
 
@@ -132,5 +134,6 @@ def parse_environment_file(
     return LockSpecification(
         dependencies=dep_map,
         channels=channels,  # type: ignore
+        pip_repositories=pip_repositories,  # type: ignore
         sources=[environment_file],
     )