pkg: parse local versions (#13933)

* pkg: parse local versions

* offline

* str

* manifest

* ci

(cherry picked from commit 47833df)

.actions/setup_tools.py

@@ -22,7 +22,7 @@
 import urllib.request
 from datetime import datetime
 from importlib.util import module_from_spec, spec_from_file_location
-from itertools import groupby
+from itertools import chain, groupby
 from types import ModuleType
 from typing import List
 
@@ -45,7 +45,7 @@ def _load_py_module(name: str, location: str) -> ModuleType:
 def load_requirements(
     path_dir: str, file_name: str = "base.txt", comment_char: str = "#", unfreeze: bool = True
 ) -> List[str]:
-    """Load requirements from a file.
+    """Loading requirements from a file.
 
     >>> path_req = os.path.join(_PROJECT_ROOT, "requirements")
     >>> load_requirements(path_req)  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
@@ -438,3 +438,33 @@ def _download_frontend(root: str = _PROJECT_ROOT):
     # If installing from source without internet connection, we don't want to break the installation
     except Exception:
         print("The Lightning UI downloading has failed!")
+
+
+def _adjust_require_versions(source_dir: str = "src", req_dir: str = "requirements") -> None:
+    """Parse the base requirements and append  as version adjustments if needed `pkg>=X1.Y1.Z1,==X2.Y2.*`."""
+    reqs = load_requirements(req_dir, file_name="base.txt")
+    for i, req in enumerate(reqs):
+        pkg_name = req[: min(req.index(c) for c in ">=" if c in req)]
+        ver_ = parse_version_from_file(os.path.join(source_dir, pkg_name))
+        if not ver_:
+            continue
+        ver2 = ".".join(ver_.split(".")[:2] + ["*"])
+        reqs[i] = f"{req}, =={ver2}"
+
+    with open(os.path.join(req_dir, "base.txt"), "w") as fp:
+        fp.writelines([ln + os.linesep for ln in reqs])
+
+
+def _load_aggregate_requirements(req_dir: str = "requirements", freeze_requirements: bool = False) -> None:
+    """Load all base requirements from all particular packages and prune duplicates."""
+    requires = [
+        load_requirements(d, file_name="base.txt", unfreeze=not freeze_requirements)
+        for d in glob.glob(os.path.join(req_dir, "*"))
+        if os.path.isdir(d)
+    ]
+    if not requires:
+        return None
+    # TODO: add some smarter version aggregation per each package
+    requires = list(chain(*requires))
+    with open(os.path.join(req_dir, "base.txt"), "w") as fp:
+        fp.writelines([ln + os.linesep for ln in requires])

requirements/base.txt

@@ -0,0 +1,2 @@
+pytorch_lightning>=1.6.5
+lightning_app>=0.5.2

setup.py

@@ -60,7 +60,9 @@
 # http://blog.ionelmc.ro/2014/05/25/python-packaging/
 _PATH_ROOT = os.path.dirname(__file__)
 _PATH_SRC = os.path.join(_PATH_ROOT, "src")
+_PATH_REQUIRE = os.path.join(_PATH_ROOT, "requirements")
 _PATH_SETUP = os.path.join(_PATH_SRC, _REAL_PKG_NAME or "lightning", "__setup__.py")
+_FREEZE_REQUIREMENTS = bool(int(os.environ.get("FREEZE_REQUIREMENTS", 0)))
 
 
 # Hardcode the env variable from time of package creation, otherwise it fails during installation
@@ -89,11 +91,19 @@ def _load_py_module(name: str, location: str) -> ModuleType:
 # engineer specific practices
 if __name__ == "__main__":
     _SETUP_TOOLS = _load_py_module(name="setup_tools", location=os.path.join(".actions", "setup_tools.py"))
+
+    if _PACKAGE_NAME == "lightning":  # install just the meta package
+        _SETUP_TOOLS._adjust_require_versions(_PATH_SRC, _PATH_REQUIRE)
+    elif _PACKAGE_NAME not in _PACKAGE_MAPPING:  # install everything
+        _SETUP_TOOLS._load_aggregate_requirements(_PATH_REQUIRE, _FREEZE_REQUIREMENTS)
+
     if _PACKAGE_NAME not in _PACKAGE_MAPPING:
         _SETUP_TOOLS.set_version_today(os.path.join(_PATH_SRC, "lightning", "__version__.py"))
+
     for lit_name, pkg_name in _PACKAGE_MAPPING.items():
         # fixme: if we run creation of meta pkg against stable we shall pull the source
         _SETUP_TOOLS.create_meta_package(os.path.join(_PATH_ROOT, "src"), pkg_name, lit_name)
+
     _SETUP_MODULE = _load_py_module(name="pkg_setup", location=_PATH_SETUP)
     _SETUP_MODULE._adjust_manifest(pkg_name=_REAL_PKG_NAME)
     setup(**_SETUP_MODULE._setup_args(pkg_name=_REAL_PKG_NAME))

src/lightning/__setup__.py

@@ -1,7 +1,5 @@
-import glob
 import os.path
 from importlib.util import module_from_spec, spec_from_file_location
-from itertools import chain
 from types import ModuleType
 from typing import Any, Dict
 
@@ -10,6 +8,7 @@
 _PROJECT_ROOT = "."
 _SOURCE_ROOT = os.path.join(_PROJECT_ROOT, "src")
 _PACKAGE_ROOT = os.path.join(_SOURCE_ROOT, "lightning")
+_PATH_REQUIREMENTS = os.path.join("requirements")
 _FREEZE_REQUIREMENTS = bool(int(os.environ.get("FREEZE_REQUIREMENTS", 0)))
 
 
@@ -22,6 +21,9 @@ def _load_py_module(name: str, location: str) -> ModuleType:
     return py
 
 
+_SETUP_TOOLS = _load_py_module("setup_tools", os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py"))
+
+
 def _adjust_manifest(**kwargs: Any) -> None:
     # todo: consider rather aggregation of particular manifest adjustments
     manifest_path = os.path.join(_PROJECT_ROOT, "MANIFEST.in")
@@ -31,6 +33,7 @@ def _adjust_manifest(**kwargs: Any) -> None:
     if kwargs["pkg_name"] == "lightning":
         lines += [
             "recursive-include src/lightning *.md",
+            "include requirements/base.txt",
             # fixme: this is strange, this shall work with setup find package - include
             "prune src/lightning_app",
             "prune src/pytorch_lightning",
@@ -47,29 +50,17 @@ def _adjust_manifest(**kwargs: Any) -> None:
 
 
 def _setup_args(**kwargs: Any) -> Dict[str, Any]:
-    _path_setup_tools = os.path.join(_PROJECT_ROOT, ".actions", "setup_tools.py")
-    _setup_tools = _load_py_module("setup_tools", _path_setup_tools)
     _about = _load_py_module("about", os.path.join(_PACKAGE_ROOT, "__about__.py"))
     _version = _load_py_module("version", os.path.join(_PACKAGE_ROOT, "__version__.py"))
-    _long_description = _setup_tools.load_readme_description(
+    _long_description = _SETUP_TOOLS.load_readme_description(
         _PROJECT_ROOT, homepage=_about.__homepage__, version=_version.version
     )
-    if kwargs["pkg_name"] == "lightning":
-        _include_pkgs = ["lightning", "lightning.*"]
-        # todo: generate this list automatically with parsing feature pkg versions
-        _requires = ["pytorch-lightning>=1.6.5, <1.7.0", "lightning-app>=0.5.2, <0.6.0"]
-    else:
-        _include_pkgs = ["*"]
-        _requires = [
-            _setup_tools.load_requirements(d, unfreeze=not _FREEZE_REQUIREMENTS)
-            for d in glob.glob(os.path.join("requirements", "*"))
-            if os.path.isdir(d)
-        ]
-        _requires = list(chain(*_requires))
+    _include_pkgs = ["lightning", "lightning.*"] if kwargs["pkg_name"] == "lightning" else ["*"]
+
     # TODO: consider invaliding some additional arguments from packages, for example if include data or safe to zip
 
     # TODO: remove this once lightning-ui package is ready as a dependency
-    _setup_tools._download_frontend(_PROJECT_ROOT)
+    _SETUP_TOOLS._download_frontend(_PROJECT_ROOT)
 
     return dict(
         name="lightning",
@@ -94,7 +85,7 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]:
             ],
         },
         setup_requires=[],
-        install_requires=_requires,
+        install_requires=_SETUP_TOOLS.load_requirements(_PATH_REQUIREMENTS, unfreeze=True),
         extras_require={},  # todo: consider porting all other packages extras with prefix
         project_urls={
             "Bug Tracker": "https://github.com/Lightning-AI/lightning/issues",