FIX: patch wdm for Chrome 115+ and freeze it to 3.8.6

CHANGELOG.md

@@ -99,7 +99,49 @@ TODOs:
 - can we force order of how `selene.*` is rendered on autocomplete? via `__all__`...
 - deprecate `have.js_returned` in favour of `have.script_returned`
 
-## 2.0.0rc2 (to be released on 13.04.2023)
+## 2.0.0rc3 (to be released on 21.07.2023)
+
+### HOTFIX webdriver_manager after changes in google chromedrivers APIs
+
+This hotfix is really hot, so might break something. Use it on your own risk.
+If something went wrong, roll back to 2.0.0rc2.
+
+We also froze webdriver_manager version to 3.8.6, so it will not be updated automatically and our hotfix will not be broken :D. Let's see how it goes further... One day we hope to remove hotfix and unfreeze webdriver_manager version.
+
+Should work for new versions of Chrome from v115 out of the box. 
+
+If you use webdriver_manager on your own, you can do the following trick to patch it with the fix:
+
+```python
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from webdriver_manager.chrome import ChromeDriverManager
+from webdriver_manager.core.utils import ChromeType
+
+from selene import support
+
+chrome_driver = webdriver.Chrome(
+    service=Service(
+        support._extensions.webdriver_manager.patch._to_find_chromedrivers_from_115(
+            ChromeDriverManager(chrome_type=ChromeType.GOOGLE)
+        ).install()
+    )
+)
+```
+
+Notice underscore prefixes in module and patch function names at `_extensions.webdriver_manager.patch._to_find_chromedrivers_from_115`. Use it on your own risk, as it is marked as private and experimental;).
+
+Remember that currently on macOS the fix itself might not be enough, for Chrome versions less than 117, you probably will have to install [Chrome for Testing](https://github.com/GoogleChromeLabs/chrome-for-testing#what-is-chrome-for-testing) browser instead of Chrome and fix it with `xattr -cr 'Google Chrome for Testing.app'` command. An alternative to installing Chrome for Testing, can be setting binary location manually via:
+
+```python
+from selene import browser
+from selenium import webdriver
+
+browser.config.driver_options = webdriver.ChromeOptions()
+browser.config.driver_options.binary_location = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
+```
+
+## 2.0.0rc2 (released on 13.04.2023)
 
 ### Driver is guessed by config.driver_options too
 

README.md

@@ -549,7 +549,7 @@ TBD
 
 <!-- References -->
 [selenide]: http://selenide.org/
-[latest-recommended-version]: https://pypi.org/project/selene/2.0.0b17/
+[latest-recommended-version]: https://pypi.org/project/selene/2.0.0rc2/
 [brunch-ver-1]: https://github.com/yashaka/selene/tree/1.x
 [selene-stable]: https://pypi.org/project/selene/1.0.2/
 [python-37]: https://www.python.org/downloads/release/python-370/

examples/run_local_in_chrome_with_options_for_headless/test_todomvc.py

@@ -5,7 +5,7 @@
 
 def test_complete_task():
     options = webdriver.ChromeOptions()
-    options.add_argument('--headless')
+    options.add_argument('--headless=new')
     # additional options:
     options.add_argument('--no-sandbox')
     options.add_argument('--disable-gpu')

examples/run_local_in_default_chrome/test_todomvc.py

@@ -19,7 +19,7 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-from selene import browser, by, have
+from selene import browser, have
 
 
 def test_completes_todo():

examples/run_local_in_firefox_via_set_config_driver_options/test_todomvc.py

@@ -19,8 +19,8 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-from selene import browser, by, have
 from selenium.webdriver import FirefoxOptions
+from selene import browser, have
 
 
 def test_completes_todo():

examples/run_local_in_firefox_via_set_config_name/test_todomvc.py

@@ -19,7 +19,7 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-from selene import browser, by, have
+from selene import browser, have
 
 
 def test_completes_todo():

examples/run_local_in_safari_via_set_config_driver/test_todomvc.py

@@ -27,8 +27,7 @@
 
 
 def test_completes_todo():
-    webdriver.Op
-    # browser.config.driver = webdriver.Safari(service=SafariService())
+    browser.config.driver = webdriver.Safari(service=SafariService())
 
     browser.open('http://todomvc.com/examples/emberjs/')
     browser.should(have.title_containing('TodoMVC'))

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "selene"
-version = "2.0.0rc2"
+version = "2.0.0rc3"
 description = "User-oriented browser tests in Python (Selenide port)"
 authors = ["Iakiv Kramarenko <yashaka@gmail.com>"]
 license = "MIT"
@@ -49,7 +49,7 @@ Changelog = "https://github.com/yashaka/selene/releases"
 python = "^3.7"
 selenium = ">=4.4.3"
 future = "*"
-webdriver-manager = ">=3.8.5"
+webdriver-manager = "==3.8.6"
 typing-extensions = "==4.3.0"
 
 [tool.poetry.dev-dependencies]

selene/core/configuration.py

@@ -34,6 +34,7 @@
 from selenium.webdriver.common.options import BaseOptions
 from selenium.webdriver.common.service import Service
 
+from selene import support
 from selene.common import fp, helpers
 from selene.common.data_structures import persistent
 from selene.common.fp import F
@@ -72,30 +73,27 @@ def install_and_build_chrome():
         # TODO: consider simplifying the logic... to much of ifs
         #       probably all ifs were already before calling this function
         #       see example of simplification in install_and_build_firefox
-        if config.driver_options:
-            if isinstance(config.driver_options, ChromeOptions):
-                return Chrome(
-                    service=config.driver_service
-                    or ChromeService(
-                        ChromeDriverManager(chrome_type=ChromeType.GOOGLE).install()
-                    ),
-                    options=config.driver_options,
-                )
-            else:
-                raise ValueError(
-                    f'Default config.build_driver_strategy ("driver factory"), '
-                    f'if config.driver_name is set to "chrome", - '
-                    f'expects only instance of ChromeOptions or None'
-                    f'in config.driver_options,'
-                    f'but got: {config.driver_options}'
-                )
-        else:
-            return Chrome(
-                service=config.driver_service
-                or ChromeService(
-                    ChromeDriverManager(chrome_type=ChromeType.GOOGLE).install()
-                )
+        if config.driver_options and not isinstance(
+            config.driver_options, ChromeOptions
+        ):
+            raise ValueError(
+                f'Default config.build_driver_strategy ("driver factory"), '
+                f'if config.driver_name is set to "chrome", - '
+                f'expects only instance of ChromeOptions or None'
+                f'in config.driver_options,'
+                f'but got: {config.driver_options}'
+            )
+
+        driver_manager = (
+            support._extensions.webdriver_manager.patch._to_find_chromedrivers_from_115(
+                ChromeDriverManager(chrome_type=ChromeType.GOOGLE)
             )
+        )
+
+        return Chrome(
+            service=config.driver_service or ChromeService(driver_manager.install()),
+            options=config.driver_options,
+        )
 
     def install_and_build_firefox():
         return Firefox(

selene/support/__init__.py

@@ -1,3 +1,3 @@
 # TODO: consider renaming support to _support to emphasize its experimental nature
 
-from . import _logging
+from . import _logging, _extensions

selene/support/_extensions/__init__.py

@@ -0,0 +1 @@
+from . import webdriver_manager

selene/support/_extensions/webdriver_manager/__init__.py

@@ -0,0 +1 @@
+from . import patch

selene/support/_extensions/webdriver_manager/patch.py

@@ -0,0 +1,179 @@
+# MIT License
+#
+# Copyright (c) 2023 Iakiv Kramarenko
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+from webdriver_manager.chrome import ChromeDriverManager
+
+
+def _to_find_chromedrivers_from_115(driver_manager: ChromeDriverManager):
+    """
+    Fixes webdriver_manager issue with latest chrome versions (>= 115.0.5763.0)
+    See https://github.com/SergeyPirogov/webdriver_manager/issues/536
+
+    Fix is based on simple ideas from:
+    * https://github.com/SergeyPirogov/webdriver_manager/issues/536#issuecomment-1641266654
+    * https://github.com/SergeyPirogov/webdriver_manager/issues/536#issuecomment-1641396604
+
+    It monkey patches all driver_manager inner objects (including nested ones)
+    and methods
+
+    But also for newer versions of Chrome browsers – it patches PATTERN dict
+    from wdm utils.
+    This PATTERN patching is the most risky part of the fix,
+    because it changes the whole library behavior,
+    not just the object passed to this function.
+    Let's keep fingers crossed;p
+    """
+
+    from webdriver_manager.core import logger as wdm_logger
+    from packaging import version
+    from webdriver_manager.core import utils as wdm_utils
+    from webdriver_manager.core.utils import ChromeType
+
+    driver_utils = driver_manager.driver
+    http_client = driver_utils._http_client
+
+    def chrome_apis_url(endpoint):
+        return f'https://googlechromelabs.github.io/chrome-for-testing/{endpoint}'
+
+    good_binary_version = None
+    good_binary_url = None
+    installed_browser_version = driver_utils.get_browser_version_from_os()
+
+    if not installed_browser_version:
+        wdm_logger.log(
+            'Failed to get version of Chrome installed at your OS '
+            f'(detected os type: {driver_utils.os_type}).'
+            f'Going to install the chromedriver binary '
+            f'matching latest known stable version of Chrome...'
+        )
+        last_known_good_versions_with_downloads = http_client.get(
+            chrome_apis_url('last-known-good-versions-with-downloads.json')
+        ).json()
+        stable_channel = last_known_good_versions_with_downloads.get(
+            'channels', {}
+        ).get('Stable', {})
+
+        last_known_good_version = (stable_channel.get('version', {})) or None
+        platform_and_url_pairs = stable_channel.get('downloads', {}).get(
+            'chromedriver', []
+        )
+        url_where_platform_is_os_type = next(
+            iter(
+                pair.get('url', None)
+                for pair in platform_and_url_pairs
+                if pair.get('platform', None) == driver_utils.get_os_type()
+            ),
+            None,
+        )
+        wdm_logger.log(
+            f'latest known stable version of Chrome: {last_known_good_version}'
+        )
+
+        good_binary_version = last_known_good_version
+        good_binary_url = url_where_platform_is_os_type
+
+    if installed_browser_version:
+        if version.parse(installed_browser_version) >= version.parse('115.0.5763.0'):
+            # patching wdm_utils.PATTERN
+            # we need all 4 sub-versions not just 3 of them
+            wdm_utils.PATTERN[ChromeType.GOOGLE] = r"\d+\.\d+\.\d+.\d+"
+            # retaking version from os after patched pattern
+            good_binary_version = wdm_utils.get_browser_version_from_os(
+                driver_utils.get_browser_type()
+            )
+            # let's reset _browser_version to the new 4-sub-versions value
+            # from here, we assume that
+            # "good" binary version is the same as "good" browser version
+            driver_utils._browser_version = good_binary_version
+
+            known_good_versions = (
+                http_client.get(
+                    chrome_apis_url('known-good-versions-with-downloads.json')
+                )
+                .json()
+                .get('versions', [])
+            )
+
+            matched_version_downloads_chromedriver_per_platform: list = next(
+                iter(
+                    info.get('downloads', {}).get('chromedriver', [])
+                    for info in known_good_versions
+                    if info.get('version', None) == good_binary_version
+                ),
+                [],
+            )
+
+            good_binary_url = next(
+                iter(
+                    info.get('url', None)
+                    for info in matched_version_downloads_chromedriver_per_platform
+                    if info.get('platform')
+                    == driver_utils.get_os_type().replace('_', '-')
+                ),
+                None,
+            )
+
+    if good_binary_url:
+        # it happened that we found good binary url on our own
+        # let's monkey patch WDM classes and objects they know it too ;P
+
+        # now we provide exactly correct and ready for download url
+        driver_utils._url = good_binary_url
+
+        # let's just return it as is
+        driver_utils.get_driver_download_url = lambda: driver_utils._url
+
+        # just in case...
+        driver_utils._version = good_binary_version
+
+        # new endpoints provide file for download differently,
+        # old wdm filename logic just does not work
+        # so let's patch it too
+        class PatchedFile(wdm_utils.File):
+            filename = good_binary_url.split('/')[-1]  # type: ignore
+
+        # and use PatchedFile on download...
+        def download_file(url: str):
+            wdm_logger.log(f"About to download new driver from {url}")
+            response = http_client.get(url)
+            return PatchedFile(response)
+
+        # – by download manager
+        # (we safely patch only current object, not the whole class)
+        driver_manager._download_manager.download_file = download_file
+
+        # similar story with processing filenames logic...
+        def get_binary(files, driver_name):
+            try:
+                return next(
+                    file
+                    for file in files
+                    if driver_name in file.split('/')[-1].split('.')[0]
+                )
+            except Exception as e:
+                raise Exception(
+                    f"Can't find binary for {driver_name} among {files}"
+                ) from e
+
+        # safely patching only exactly our manager object's get_binary
+        driver_manager.driver_cache._DriverCache__get_binary = get_binary
+
+    return driver_manager

tests/acceptance/custom_driver_management_via_callable/test_adding_todos.py

@@ -26,7 +26,7 @@
 from webdriver_manager.chrome import ChromeDriverManager
 from webdriver_manager.core.utils import ChromeType
 
-from selene import by, have, Browser, Config
+from selene import by, have, Browser, Config, support
 import pytest
 
 from selene.support.webdriver import WebHelper
@@ -49,7 +49,9 @@ def create_chrome():
 
             return webdriver.Chrome(
                 service=ChromeService(
-                    ChromeDriverManager(chrome_type=ChromeType.GOOGLE).install()
+                    support._extensions.webdriver_manager.patch._to_find_chromedrivers_from_115(
+                        ChromeDriverManager(chrome_type=ChromeType.GOOGLE)
+                    ).install()
                 ),
                 options=webdriver.ChromeOptions(),
             )