From 8eae768c927a44c21fc9186bd4e888b9c904df3e Mon Sep 17 00:00:00 2001 From: Keewis Date: Thu, 7 Jan 2021 02:07:16 +0100 Subject: [PATCH 1/5] use conda.api instead of parallel calls to the conda binary --- ci/min_deps_check.py | 63 ++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 95c6ff22aa2..f622a5cdd6f 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -2,14 +2,15 @@ publication date. Compare it against requirements/py36-min-all-deps.yml to verify the policy on obsolete dependencies is being followed. Print a pretty report :) """ -import subprocess +import itertools import sys -from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta from typing import Dict, Iterator, Optional, Tuple +import conda.api import yaml +CHANNELS = ["conda-forge", "defaults"] IGNORE_DEPS = { "black", "coveralls", @@ -89,30 +90,32 @@ def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]: Return map of {(major version, minor version): publication date} """ - stdout = subprocess.check_output( - ["conda", "search", pkg, "--info", "-c", "defaults", "-c", "conda-forge"] + + def metadata(entry): + name = entry.name + filename = entry.fn + version = entry.version + filename_version = filename[len(name) :].split("-")[1] + + if version != filename_version: + raise RuntimeError( + f"{entry.name}: version != filename version: {version} vs {filename_version}" + ) + + time = datetime.fromtimestamp(entry.timestamp) + major, minor = map(int, version.split(".")[:2]) + + return (major, minor), time + + records = sorted( + metadata(entry) + for entry in conda.api.SubdirData.query_all(pkg, channels=CHANNELS) ) - out = {} # type: Dict[Tuple[int, int], datetime] - major = None - minor = None - - for row in stdout.decode("utf-8").splitlines(): - label, _, value = row.partition(":") - label = label.strip() - if label == "file name": - value = value.strip()[len(pkg) :] - smajor, sminor = value.split("-")[1].split(".")[:2] - major = int(smajor) - minor = int(sminor) - if label == "timestamp": - assert major is not None - assert minor is not None - ts = datetime.strptime(value.split()[0].strip(), "%Y-%m-%d") - - if (major, minor) in out: - out[major, minor] = min(out[major, minor], ts) - else: - out[major, minor] = ts + + out = { + version: min(time for _, time in group) + for version, group in itertools.groupby(records, key=lambda x: x[0]) + } # Hardcoded fix to work around incorrect dates in conda if pkg == "python": @@ -200,12 +203,10 @@ def fmt_version(major: int, minor: int, patch: int = None) -> str: def main() -> None: fname = sys.argv[1] - with ThreadPoolExecutor(8) as ex: - futures = [ - ex.submit(process_pkg, pkg, major, minor, patch) - for pkg, major, minor, patch in parse_requirements(fname) - ] - rows = [f.result() for f in futures] + rows = [ + process_pkg(pkg, major, minor, patch) + for pkg, major, minor, patch in parse_requirements(fname) + ] print("Package Required Policy Status") print("------------- -------------------- -------------------- ------") From 904483a3fea7df4660e56cc929ad3a8c77b6675b Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 8 Jan 2021 01:29:39 +0100 Subject: [PATCH 2/5] don't select releases without release dates --- ci/min_deps_check.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 5848bd69853..89cf4e86178 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -104,20 +104,19 @@ def metadata(entry): f"{entry.name}: version != filename version: {version} vs {filename_version}" ) - time = datetime.fromtimestamp(entry.timestamp) + time = datetime.fromtimestamp(entry.timestamp) if entry.timestamp != 0 else None major, minor = map(int, version.split(".")[:2]) return (major, minor), time - records = sorted( - metadata(entry) - for entry in conda.api.SubdirData.query_all(pkg, channels=CHANNELS) - ) + raw_data = conda.api.SubdirData.query_all(pkg, channels=CHANNELS) + records = sorted([metadata(entry) for entry in raw_data], key=lambda x: x[0]) - out = { - version: min(time for _, time in group) + release_dates = { + version: [time for _, time in group if time is not None] for version, group in itertools.groupby(records, key=lambda x: x[0]) } + out = {version: min(dates) for version, dates in release_dates.items() if dates} # Hardcoded fix to work around incorrect dates in conda if pkg == "python": From 897d9fd5cdb425e5201b37d6b66dac4056ca0412 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 8 Jan 2021 01:32:30 +0100 Subject: [PATCH 3/5] update the format to be wide enough to fit matplotlib-base --- ci/min_deps_check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 89cf4e86178..3ba443d42b6 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -209,9 +209,9 @@ def main() -> None: for pkg, major, minor, patch in parse_requirements(fname) ] - print("Package Required Policy Status") - print("------------- -------------------- -------------------- ------") - fmt = "{:13} {:7} ({:10}) {:7} ({:10}) {}" + print("Package Required Policy Status") + print("----------------- -------------------- -------------------- ------") + fmt = "{:17} {:7} ({:10}) {:7} ({:10}) {}" for row in rows: print(fmt.format(*row)) From 95f83fc9ab4a280221810aa7700fa8a3a889ed23 Mon Sep 17 00:00:00 2001 From: Keewis Date: Fri, 8 Jan 2021 01:34:31 +0100 Subject: [PATCH 4/5] don't verify the version using the filename --- ci/min_deps_check.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 3ba443d42b6..0e8fb68d403 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -94,15 +94,7 @@ def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]: """ def metadata(entry): - name = entry.name - filename = entry.fn version = entry.version - filename_version = filename[len(name) :].split("-")[1] - - if version != filename_version: - raise RuntimeError( - f"{entry.name}: version != filename version: {version} vs {filename_version}" - ) time = datetime.fromtimestamp(entry.timestamp) if entry.timestamp != 0 else None major, minor = map(int, version.split(".")[:2]) From 3a7d72918a0fe7bf1ab2a1d660a4f49ec0f01329 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 9 Jan 2021 15:05:06 +0100 Subject: [PATCH 5/5] filter invalid / missing dates before retrieving the metadata --- ci/min_deps_check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/min_deps_check.py b/ci/min_deps_check.py index 0e8fb68d403..5bb4c155410 100755 --- a/ci/min_deps_check.py +++ b/ci/min_deps_check.py @@ -96,17 +96,17 @@ def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]: def metadata(entry): version = entry.version - time = datetime.fromtimestamp(entry.timestamp) if entry.timestamp != 0 else None + time = datetime.fromtimestamp(entry.timestamp) major, minor = map(int, version.split(".")[:2]) return (major, minor), time raw_data = conda.api.SubdirData.query_all(pkg, channels=CHANNELS) - records = sorted([metadata(entry) for entry in raw_data], key=lambda x: x[0]) + data = sorted(metadata(entry) for entry in raw_data if entry.timestamp != 0) release_dates = { version: [time for _, time in group if time is not None] - for version, group in itertools.groupby(records, key=lambda x: x[0]) + for version, group in itertools.groupby(data, key=lambda x: x[0]) } out = {version: min(dates) for version, dates in release_dates.items() if dates}