Add ability for drivers to require specific versions of distros; fix …

…the sqlite:// ZODB URI resolver.

.github/workflows/tests.yml

@@ -87,14 +87,12 @@ jobs:
       matrix:
         # XXX: PyPy 3.9 is failing to build zope.interface 6.0!
         # Need to take this up with the zope.interface repo.
-        python-version: [3.7, 3.8, 3.9, '3.10', '3.11']
+        python-version: [3.8, 3.9, '3.10', '3.11']
         os: [ubuntu-latest, macos-latest]
         exclude:
           - os: macos-latest
             python-version: pypy-3.9
-          # Get an error importing _bz2 on macos/3.7
-          - os: macos-latest
-            python-version: 3.7
+
     steps:
       - name: checkout
         uses: actions/checkout@v3

CHANGES.rst

@@ -6,15 +6,19 @@
 ==================
 
 - Drop support for Python versions that are end of life, including
-  everything less than 3.7.
+  everything less than 3.8.
 - Add the "Requires Python" metadata to prevent installation on Python
-  < 3.7.
+  < 3.8.
 - Add support for Python 3.11.
 - Bump tested database drivers to their latest versions, with the
   exception of ``mysql-connector-python``; this driver is only tested
   at version 8.0.31 as there are known incompatibilities with 8.0.32
   (which is currently the latest version).
-
+- pg8000: Require 1.29.0. See :issue:`495`.
+- Fix the SQLite ZODB URI resolver. The ``data_dir`` query parameter
+  replaces the ``path`` query parameter.
+- Remove the (local) runtime (install) dependency on
+  ``setuptools`` / ``pkg_resources``. This was undeclared.
 
 3.5.0 (2022-09-16)
 ==================

appveyor.yml

@@ -72,12 +72,6 @@ environment:
       PYTHON_EXE: python
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
 
-    - PYTHON: "C:\\Python37-x64"
-      PYTHON_VERSION: "3.7.x"
-      PYTHON_ARCH: "64"
-      PYTHON_EXE: python
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
-
     ## 32-bit, wheel only (no testing)
 
     - PYTHON: "C:\\Python38"
@@ -86,11 +80,6 @@ environment:
       PYTHON_EXE: python
       GWHEEL_ONLY: true
 
-    - PYTHON: "C:\\Python37"
-      PYTHON_VERSION: "3.7.x"
-      PYTHON_ARCH: "32"
-      PYTHON_EXE: python
-      GWHEEL_ONLY: true
 
 services:
   - mysql

scripts/releases/make-manylinux

@@ -98,7 +98,7 @@ if [ -d /RelStorage -a -d /opt/python ]; then
     rm -rf wheelhouse
     mkdir wheelhouse
 
-    for variant in `ls -d /opt/python/cp{37,38,39,310,311}*`; do
+    for variant in `ls -d /opt/python/cp{38,39,310,311}*`; do
         # XXX: Cython 3.0b3 doesn't compile correctly on Python 3.12; the long object
         # has changed.
         echo "Building $variant"

setup.py

@@ -41,7 +41,7 @@ def _dummy_cythonize(extensions, **_kwargs):
 def read_file(*path):
     base_dir = os.path.dirname(__file__)
     file_path = (base_dir, ) + tuple(path)
-    with open(os.path.join(*file_path)) as f:
+    with open(os.path.join(*file_path), 'rt', encoding='utf-8') as f:
         result = f.read()
     return result
 
@@ -101,13 +101,13 @@ def read_file(*path):
     license="ZPL 2.1",
     platforms=["any"],
     description="A backend for ZODB that stores pickles in a relational database.",
-    python_requires=">=3.7",
+    # 3.8: importlib.metadata
+    python_requires=">=3.8",
     classifiers=[
         "Intended Audience :: Developers",
         "License :: OSI Approved :: Zope Public License",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
@@ -271,11 +271,13 @@ def read_file(*path):
             'mysqlclient >= 2.0.0',
             # mysql-connector-python; one of two pure-python versions
             # XXX: >= 8.0.32 has issues with binary values!
+            # This requirement is repeated in the driver class.
             'mysql-connector-python == 8.0.31; python_version == "3.10"',
 
             # postgresql
             # pure-python
             # pg8000
+            # This requirement is repeated in the driver class.
             'pg8000 >= 1.29.0; python_version == "3.11"',
             # CFFI, runs on all implementations.
             'psycopg2cffi >= 2.7.4; python_version == "3.11" or platform_python_implementation == "PyPy"',

src/relstorage/adapters/drivers.py

@@ -19,6 +19,12 @@
 
 import importlib
 import sys
+from importlib import metadata
+
+from packaging.version import parse as parse_version
+from packaging.version import InvalidVersion
+from packaging.requirements import Requirement
+
 
 from zope.interface import directlyProvides
 from zope.interface import implementer
@@ -73,6 +79,49 @@ class DriverNotImportableError(DriverNotAvailableError,
                                ImportError):
     "When the module can't be imported."
 
+
+class _FalseReason:
+    def __init__(self, exc):
+        if isinstance(exc, str):
+            self.message = exc
+        else:
+            self.message = "%s: %s" % (type(exc).__name__, exc)
+
+    def __bool__(self):
+        return False
+
+    def __str__(self):
+        return self.message
+
+
+def _has_requirement(requirement):
+    # A requirement is a named distribution (the thing you install
+    # from PyPI) and (an optional but important for this use)
+    # a version specifier. This specifier lets you
+    # specify ranges and exclusions.
+    #
+    # This, this checks that a distribution of the correct version
+    # is installed.
+    #
+    # *requirement* is a ``packaging.requirements.Requirement`` object.
+    dist_name = requirement.name
+    try:
+        installed_ver_str = metadata.version(dist_name)
+    except ImportError as ex:
+        return _FalseReason(ex)
+
+    try:
+        installed_ver = parse_version(installed_ver_str)
+    except InvalidVersion as ex:
+        return _FalseReason(ex)
+
+    if installed_ver not in requirement.specifier:
+        return _FalseReason('Requirement %s not met with package: %s' % (
+            requirement, installed_ver
+        ))
+    return True
+
+
 class AbstractModuleDriver(object):
     """
     Base implementation of a driver, based on a module, as used in DBAPI.
@@ -95,11 +144,21 @@ class AbstractModuleDriver(object):
     #: Can this module be used on PyPy?
     AVAILABLE_ON_PYPY = True
 
-    #: Set this to false if your subclass can do static checks
+    #: Set this to a false value if your subclass can do static checks
     #: at import time to determine it should not be used.
     #: Helpful for things like Python version detection.
     STATIC_AVAILABLE = True
 
+    #: Set this to a sequence of strings of requirements ``("pg8000 >= 1.29",)``
+    #: Creating an instance will validate that the requirements
+    #: are met (packages with correct versions are installed).
+    #:
+    #: Do this only when a requirement cannot be specified in
+    #: setup.py as an installation requirement.
+    #:
+    #: .. versionadded:: NEXT
+    REQUIREMENTS = ()
+
     #: Priority of this driver, when available. Lower is better.
     #: (That is, first choice should have value 1, and second choice value
     #: 2, and so on.)
@@ -139,20 +198,7 @@ class AbstractModuleDriver(object):
     supports_64bit_unsigned_id = True
 
     def __init__(self):
-        if PYPY and not self.AVAILABLE_ON_PYPY:
-            raise self.DriverNotAvailableError(self.__name__)
-        if not self.STATIC_AVAILABLE:
-            raise self.DriverNotAvailableError(self.__name__)
-        try:
-            self.driver_module = mod = self.get_driver_module()
-        except ImportError as ex:
-            logger.debug(
-                "Attempting to load driver named %r from %r failed; if no driver was specified, "
-                "or the driver was set to 'auto', there may be more drivers to attempt.",
-                self.__name__, self.MODULE_NAME,
-                exc_info=True)
-            raise DriverNotImportableError(self.__name__, reason=str(ex)) from ex
-
+        self.driver_module = mod = self._check_preconditions()
 
         self.disconnected_exceptions = (mod.OperationalError,
                                         mod.InterfaceError,
@@ -168,6 +214,32 @@ def __init__(self):
         self._connect = mod.connect
         self.priority = self.PRIORITY if not PYPY else self.PRIORITY_PYPY
 
+    def _check_preconditions(self):
+        if PYPY and not self.AVAILABLE_ON_PYPY:
+            raise self.DriverNotAvailableError(self.__name__, reason="Not available on PyPy")
+        if not self.STATIC_AVAILABLE:
+            raise self.DriverNotAvailableError(self.__name__, reason=self.STATIC_AVAILABLE)
+
+        # Check that it can be imported. Cannnot do this with
+        # a Requirement because the distro name may not match the importable name.
+        try:
+            mod = self.get_driver_module()
+        except ImportError as ex:
+            logger.debug(
+                "Attempting to load driver named %r from %r failed; if no driver was specified, "
+                "or the driver was set to 'auto', there may be more drivers to attempt.",
+                self.__name__, self.MODULE_NAME,
+                exc_info=True)
+            raise DriverNotImportableError(self.__name__, reason=str(ex)) from ex
+
+        # It can be imported. Verify versions.
+        for req in self.REQUIREMENTS:
+            has_it = _has_requirement(Requirement(req))
+            if not has_it:
+                raise self.DriverNotAvailableError(self.__name__, reason=has_it)
+        return mod
+
+
     def connect(self, *args, **kwargs):
         return self._connect(*args, **kwargs)
 

src/relstorage/adapters/interfaces.py

@@ -248,7 +248,8 @@ class DriverNotAvailableError(Exception):
     driver_options = None
 
     #: The underlying reason string, for example, from an import error
-    #: if such is available.
+    #: if such is available. This can be an arbitrary object; if it
+    #: is not None, its ``str()`` value is included in our own.
     reason = None
 
     def __init__(self, driver_name, driver_options=None, reason=None):
@@ -278,7 +279,7 @@ def _format_drivers(self):
         return ' Options: %s.' % (formatted,)
 
     def __str__(self):
-        if self.reason:
+        if self.reason is not None:
             reason = ' (reason=%s)' % (self.reason,)
         else:
             reason = ''

src/relstorage/adapters/mysql/drivers/mysqlconnector.py

@@ -43,6 +43,10 @@ class PyMySQLConnectorDriver(AbstractMySQLDriver):
     MODULE_NAME = 'mysql.connector'
     PRIORITY = 4
     PRIORITY_PYPY = 2
+    REQUIREMENTS = (
+        'mysql-connector-python == 8.0.31',
+    )
+
 
     USE_PURE = True
 

src/relstorage/adapters/postgresql/drivers/pg8000.py

@@ -62,7 +62,10 @@ def compiler_class(self):
 @implementer(IDBDriver)
 class PG8000Driver(AbstractPostgreSQLDriver):
     __name__ = 'pg8000'
-    MODULE_NAME = __name__
+    MODULE_NAME = __name__ # The thing to import
+    REQUIREMENTS = (
+        'pg8000 >= 1.29.0',
+    )
     PRIORITY = 3
     PRIORITY_PYPY = 2
 

src/relstorage/adapters/tests/test_drivers.py

@@ -19,6 +19,7 @@
 
 
 import unittest
+from unittest.mock import patch as Patch
 
 from zope.interface import implementer
 
@@ -133,3 +134,58 @@ def test_enter_critical_phase_exit_on_rollback(self):
         driver.enter_critical_phase_until_transaction_end(conn, None)
         conn.rollback()
         self.assertFalse(driver.is_in_critical_phase(conn, None))
+
+
+class TestAbstractModuleDriver(unittest.TestCase):
+
+    def test_requirements(self):
+        import zope.interface
+        from packaging.version import InvalidVersion
+        from .. import drivers
+
+
+        class Driver(drivers.AbstractModuleDriver):
+            __name__ = 'Testing'
+            MODULE_NAME = 'zope.interface'
+            REQUIREMENTS = (
+                # Guaranteed to have this. We didn't put a version
+                # on it, so it will always work.
+                MODULE_NAME,
+            )
+
+        d = Driver.__new__(Driver)
+        mod = d._check_preconditions()
+        self.assertIs(mod, zope.interface)
+
+        # Multiple
+        d.REQUIREMENTS = (
+            'zope.interface > 1, != 2',
+            'ZODB'
+        )
+        mod = d._check_preconditions()
+        self.assertIs(mod, zope.interface)
+
+        # Version mismatch
+        d.REQUIREMENTS = (
+            'ZODB == 1',
+        )
+        with self.assertRaisesRegex(
+            drivers.DriverNotAvailableError,
+            "DriverNotAvailableError: Driver 'Testing' is not available "
+            r"\(reason=Requirement ZODB==1 not met with package:"
+        ):
+            d._check_preconditions()
+
+        # Not installed
+        d.REQUIREMENTS = ( 'relstorage.this.is.not.a.distro',)
+        with self.assertRaisesRegex(
+            drivers.DriverNotAvailableError,
+            'package metadata'
+        ):
+            d._check_preconditions()
+
+        # Fake in unparseable version
+        d.REQUIREMENTS = ('ZODB',)
+        with Patch.object(drivers, 'parse_version', side_effect=InvalidVersion):
+            with self.assertRaisesRegex(drivers.DriverNotAvailableError, "InvalidVersion"):
+                d._check_preconditions()