From 801110f98a82450098d226570c7d6b5693fd2ede Mon Sep 17 00:00:00 2001 From: Christian Roth Date: Sun, 29 Jan 2023 21:08:06 +0100 Subject: [PATCH 1/6] fix #247: support auth for private pypi packages --- conda_lock/conda_lock.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index 4390ef09..b83d4cf7 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -914,8 +914,11 @@ def _add_auth_to_line(line: str, auth: Dict[str, str]) -> str: def _add_auth_to_lockfile(lockfile: str, auth: Dict[str, str]) -> str: + # do not substitute in comments, but do substitute in pip installable packages + # with the pattern: # pip package @ url. + pkg_pattern = re.compile(r"(^[^#@].*|^# pip .*)") lockfile_with_auth = "\n".join( - _add_auth_to_line(line, auth) if line[0] not in ("#", "@") else line + _add_auth_to_line(line, auth) if pkg_pattern.match(line) else line for line in lockfile.strip().split("\n") ) if lockfile.endswith("\n"): From 94c4c426c36589bff1b70147fc49e425b1f8e33c Mon Sep 17 00:00:00 2001 From: Christian Roth Date: Wed, 8 Feb 2023 21:20:06 +0100 Subject: [PATCH 2/6] test adding auth to package from pypi index --- tests/test-lockfile-with-auth/test.lock | 2 ++ tests/test-stripped-lockfile/test.lock | 2 ++ tests/test_conda_lock.py | 1 + 3 files changed, 5 insertions(+) diff --git a/tests/test-lockfile-with-auth/test.lock b/tests/test-lockfile-with-auth/test.lock index 242f0c15..c0c74fdc 100644 --- a/tests/test-lockfile-with-auth/test.lock +++ b/tests/test-lockfile-with-auth/test.lock @@ -1,6 +1,8 @@ # The following domains require authentication: # - a.mychannel.cloud # - c.mychannel.cloud +# - d.mychannel.cloud http://username_a:password_a@a.mychannel.cloud/mypackage http://b.mychannel.cloud/mypackage http://username_c:password_c@c.mychannel.cloud/mypackage +# pip mypackage @ http://username_d:password_d@d.mychannel.cloud/mypackage diff --git a/tests/test-stripped-lockfile/test.lock b/tests/test-stripped-lockfile/test.lock index aaa83a4e..d5fce44e 100644 --- a/tests/test-stripped-lockfile/test.lock +++ b/tests/test-stripped-lockfile/test.lock @@ -1,6 +1,8 @@ # The following domains require authentication: # - a.mychannel.cloud # - c.mychannel.cloud +# - d.mychannel.cloud http://a.mychannel.cloud/mypackage http://b.mychannel.cloud/mypackage http://c.mychannel.cloud/mypackage +# pip mypackage @ http://d.mychannel.cloud/mypackage diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 1df8c9b0..12845d0c 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -1474,6 +1474,7 @@ def auth_(): return { "a.mychannel.cloud": "username_a:password_a", "c.mychannel.cloud": "username_c:password_c", + "d.mychannel.cloud": "username_d:password_d", } From 9e45cdf39364294d861f70426bb90dcd1a89e27f Mon Sep 17 00:00:00 2001 From: Christian Roth Date: Wed, 8 Feb 2023 22:19:57 +0100 Subject: [PATCH 3/6] support stripping auth from pypi packages --- conda_lock/conda_lock.py | 19 ++++++++++--------- tests/test-lockfile/no-auth.lock | 1 + tests/test-lockfile/test.lock | 1 + tests/test-stripped-lockfile/no-auth.lock | 1 + 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index 16a64e8a..f1f27732 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -91,10 +91,14 @@ DEFAULT_FILES = [pathlib.Path("environment.yml")] # Captures basic auth credentials, if they exists, in the second capture group. -AUTH_PATTERN = re.compile(r"^(https?:\/\/)(.*:.*@)?(.*)") +AUTH_PATTERN = re.compile(r"^(# pip .* @ )?(https?:\/\/)(.*:.*@)?(.*)") + +# Do not substitute in comments, but do substitute in pip installable packages +# with the pattern: # pip package @ url. +PKG_PATTERN = re.compile(r"(^[^#@].*|^# pip .*)") # Captures the domain in the second group. -DOMAIN_PATTERN = re.compile(r"^(https?:\/\/)?([^\/]+)(.*)") +DOMAIN_PATTERN = re.compile(r"^(# pip .* @ )?(https?:\/\/)?([^\/]+)(.*)") # Captures the platform in the first group. PLATFORM_PATTERN = re.compile(r"^# platform: (.*)$") @@ -913,11 +917,8 @@ def _add_auth_to_line(line: str, auth: Dict[str, str]) -> str: def _add_auth_to_lockfile(lockfile: str, auth: Dict[str, str]) -> str: - # do not substitute in comments, but do substitute in pip installable packages - # with the pattern: # pip package @ url. - pkg_pattern = re.compile(r"(^[^#@].*|^# pip .*)") lockfile_with_auth = "\n".join( - _add_auth_to_line(line, auth) if pkg_pattern.match(line) else line + _add_auth_to_line(line, auth) if PKG_PATTERN.match(line) else line for line in lockfile.strip().split("\n") ) if lockfile.endswith("\n"): @@ -933,17 +934,17 @@ def _add_auth(lockfile: str, auth: Dict[str, str]) -> Iterator[pathlib.Path]: def _strip_auth_from_line(line: str) -> str: - return AUTH_PATTERN.sub(r"\1\3", line) + return AUTH_PATTERN.sub(r"\1\2\4", line) def _extract_domain(line: str) -> str: - return DOMAIN_PATTERN.sub(r"\2", line) + return DOMAIN_PATTERN.sub(r"\3", line) def _strip_auth_from_lockfile(lockfile: str) -> str: lockfile_lines = lockfile.strip().split("\n") stripped_lockfile_lines = tuple( - _strip_auth_from_line(line) if line[0] not in ("#", "@") else line + _strip_auth_from_line(line) if PKG_PATTERN.match(line) else line for line in lockfile_lines ) stripped_domains = sorted( diff --git a/tests/test-lockfile/no-auth.lock b/tests/test-lockfile/no-auth.lock index c8e30114..46d14367 100644 --- a/tests/test-lockfile/no-auth.lock +++ b/tests/test-lockfile/no-auth.lock @@ -1,3 +1,4 @@ http://a.mychannel.cloud/mypackage http://b.mychannel.cloud/mypackage http://c.mychannel.cloud/mypackage +# pip mypackage @ http://d.mychannel.cloud/mypackage diff --git a/tests/test-lockfile/test.lock b/tests/test-lockfile/test.lock index a54e2d72..2a5981bd 100644 --- a/tests/test-lockfile/test.lock +++ b/tests/test-lockfile/test.lock @@ -1,3 +1,4 @@ http://user:password@a.mychannel.cloud/mypackage http://b.mychannel.cloud/mypackage http://user:password@c.mychannel.cloud/mypackage +# pip mypackage @ http://user:password@d.mychannel.cloud/mypackage diff --git a/tests/test-stripped-lockfile/no-auth.lock b/tests/test-stripped-lockfile/no-auth.lock index c8e30114..46d14367 100644 --- a/tests/test-stripped-lockfile/no-auth.lock +++ b/tests/test-stripped-lockfile/no-auth.lock @@ -1,3 +1,4 @@ http://a.mychannel.cloud/mypackage http://b.mychannel.cloud/mypackage http://c.mychannel.cloud/mypackage +# pip mypackage @ http://d.mychannel.cloud/mypackage From 4fda0274a6719a1fd3cfe8ce7ced27270ce6e2fd Mon Sep 17 00:00:00 2001 From: Christian Roth Date: Thu, 16 Feb 2023 20:41:02 +0100 Subject: [PATCH 4/6] sync comments to match regex changes Co-authored-by: Ben Mares --- conda_lock/conda_lock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index f1f27732..0f41d31c 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -90,14 +90,14 @@ logger = logging.getLogger(__name__) DEFAULT_FILES = [pathlib.Path("environment.yml")] -# Captures basic auth credentials, if they exists, in the second capture group. +# Captures basic auth credentials, if they exists, in the third capture group. AUTH_PATTERN = re.compile(r"^(# pip .* @ )?(https?:\/\/)(.*:.*@)?(.*)") # Do not substitute in comments, but do substitute in pip installable packages # with the pattern: # pip package @ url. PKG_PATTERN = re.compile(r"(^[^#@].*|^# pip .*)") -# Captures the domain in the second group. +# Captures the domain in the third group. DOMAIN_PATTERN = re.compile(r"^(# pip .* @ )?(https?:\/\/)?([^\/]+)(.*)") # Captures the platform in the first group. From caf8e847097c0d763303e8895a145dea3b2bfc28 Mon Sep 17 00:00:00 2001 From: Christian Roth Date: Thu, 16 Feb 2023 20:53:40 +0100 Subject: [PATCH 5/6] unittests strip/auth/extract helper functions for pip dependencies --- tests/test_conda_lock.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 12845d0c..8537e2c8 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -1393,6 +1393,10 @@ def invoke_install(*extra_args: str): "http://user:password@conda.mychannel.cloud/mypackage", "http://conda.mychannel.cloud/mypackage", ), + ( + "# pip mypackage @ https://username1:password1@pypi.mychannel.cloud/simple", + "# pip mypackage @ https://pypi.mychannel.cloud/simple", + ), ), ) def test__strip_auth_from_line(line: str, stripped: str): @@ -1404,6 +1408,10 @@ def test__strip_auth_from_line(line: str, stripped: str): ( ("https://conda.mychannel.cloud/mypackage", "conda.mychannel.cloud"), ("http://conda.mychannel.cloud/mypackage", "conda.mychannel.cloud"), + ( + "# pip mypackage @ https://pypi.mychannel.cloud/simple", + "pypi.mychannel.cloud", + ), ), ) def test__extract_domain(line: str, stripped: str): @@ -1463,6 +1471,14 @@ def test__strip_auth_from_lockfile(lockfile: str, stripped_lockfile: str): }, "https://username1:password1@conda.mychannel.cloud/channel1/mypackage", ), + ( + "# pip mypackage @ https://pypi.mychannel.cloud/simple", + { + "pypi.mychannel.cloud": "username:password", + "pypi.mychannel.cloud/simple": "username1:password1", + }, + "# pip mypackage @ https://username1:password1@pypi.mychannel.cloud/simple", + ), ), ) def test__add_auth_to_line(line: str, auth: Dict[str, str], line_with_auth: str): From 0e7bd6725c1be38266f48c994ef58fde8e3b446e Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 16 Feb 2023 22:00:45 +0100 Subject: [PATCH 6/6] Add a few more test cases --- tests/test_conda_lock.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 769d56ea..8486976f 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -1395,6 +1395,10 @@ def invoke_install(*extra_args: str): "# pip mypackage @ https://username1:password1@pypi.mychannel.cloud/simple", "# pip mypackage @ https://pypi.mychannel.cloud/simple", ), + ( + "# pip mypackage @ https://pypi.mychannel.cloud/simple", + "# pip mypackage @ https://pypi.mychannel.cloud/simple", + ), ), ) def test__strip_auth_from_line(line: str, stripped: str): @@ -1477,6 +1481,14 @@ def test__strip_auth_from_lockfile(lockfile: str, stripped_lockfile: str): }, "# pip mypackage @ https://username1:password1@pypi.mychannel.cloud/simple", ), + ( + "# pip mypackage @ https://pypi.otherchannel.cloud/simple", + { + "pypi.mychannel.cloud": "username:password", + "pypi.mychannel.cloud/simple": "username1:password1", + }, + "# pip mypackage @ https://pypi.otherchannel.cloud/simple", + ), ), ) def test__add_auth_to_line(line: str, auth: Dict[str, str], line_with_auth: str):