Delay engine and book downloads until needed for a test (#1035)

* Delay downloading engines until needed Also, - Since the Sjeng site seems unreliable recently, skip the test if the download fails. - Use a local logger instead of the one from lichess_bot. * Delay downloading the opening book until needed * Use exceptions for engine download errors * Skip tests when engine dowloads fail * Spacing * Skip tests that can't be run * Add time for downloading engines * Download stockfish in test_homemade() In case the tests are run in a different order. * Download stockfish in lichess_org_simulator() Since stockfish is used as the opponent in all engine tests, make sure it exists in case the tests are run in a different order.
lichess-bot-devs · Oct 14, 2024 · 4909a8e · 4909a8e
1 parent 0444a00
commit 4909a8e
Show file tree

Hide file tree

Showing 2 changed files with 54 additions and 25 deletions.
diff --git a/test_bot/test_bot.py b/test_bot/test_bot.py
@@ -41,6 +41,7 @@ def download_sf() -> None:
     archive_link = f"https://github.com/official-stockfish/Stockfish/releases/download/sf_16/{sf_base}.{archive_ext}"
 
     response = requests.get(archive_link, allow_redirects=True)
+    response.raise_for_status()
     archive_name = f"./TEMP/sf_zip.{archive_ext}"
     with open(archive_name, "wb") as file:
         file.write(response.content)
@@ -61,8 +62,10 @@ def download_lc0() -> None:
     """Download Leela Chess Zero 0.29.0."""
     if os.path.exists("./TEMP/lc0.exe"):
         return
+
     response = requests.get("https://github.com/LeelaChessZero/lc0/releases/download/v0.29.0/lc0-v0.29.0-windows-cpu-dnnl.zip",
                             allow_redirects=True)
+    response.raise_for_status()
     with open("./TEMP/lc0_zip.zip", "wb") as file:
         file.write(response.content)
     with zipfile.ZipFile("./TEMP/lc0_zip.zip", "r") as zip_ref:
@@ -73,7 +76,9 @@ def download_sjeng() -> None:
     """Download Sjeng."""
     if os.path.exists("./TEMP/sjeng.exe"):
         return
+
     response = requests.get("https://sjeng.org/ftp/Sjeng112.zip", allow_redirects=True)
+    response.raise_for_status()
     with open("./TEMP/sjeng_zip.zip", "wb") as file:
         file.write(response.content)
     with zipfile.ZipFile("./TEMP/sjeng_zip.zip", "r") as zip_ref:
@@ -82,14 +87,10 @@ def download_sjeng() -> None:
 
 
 os.makedirs("TEMP", exist_ok=True)
-download_sf()
-if platform == "win32":
-    download_lc0()
-    download_sjeng()
 logging_level = logging.DEBUG
 testing_log_file_name = None
 lichess_bot.logging_configurer(logging_level, testing_log_file_name, None, False)
-lichess_bot.logger.info("Downloaded engines")
+logger = logging.getLogger(__name__)
 
 
 def lichess_org_simulator(opponent_path: str,
@@ -113,6 +114,13 @@ def lichess_org_simulator(opponent_path: str,
     wtime = start_time
     btime = start_time
 
+    if opponent_path == stockfish_path:
+        try:
+            download_sf()
+        except Exception:
+            logger.exception("Could not download the Stockfish chess engine")
+            pytest.skip("Could not download the Stockfish chess engine")
+
     engine = chess.engine.SimpleEngine.popen_uci(opponent_path)
     engine.configure({"Skill Level": 0, "Move Overhead": 1000, "Use NNUE": False}
                      if opponent_path == stockfish_path else {})
@@ -168,7 +176,7 @@ def run_bot(raw_config: CONFIG_DICT_TYPE, logging_level: int, opponent_path: str
     """
     config.insert_default_values(raw_config)
     CONFIG = config.Configuration(raw_config)
-    lichess_bot.logger.info(lichess_bot.intro())
+    logger.info(lichess_bot.intro())
     manager = Manager()
     board_queue: Queue[chess.Board] = manager.Queue()
     clock_queue: Queue[tuple[datetime.timedelta, datetime.timedelta, datetime.timedelta]] = manager.Queue()
@@ -179,7 +187,7 @@ def run_bot(raw_config: CONFIG_DICT_TYPE, logging_level: int, opponent_path: str
     username = user_profile["username"]
     if user_profile.get("title") != "BOT":
         return False
-    lichess_bot.logger.info(f"Welcome {username}!")
+    logger.info(f"Welcome {username}!")
     lichess_bot.disable_restart()
 
     results: Queue[bool] = manager.Queue()
@@ -200,32 +208,36 @@ def run_bot(raw_config: CONFIG_DICT_TYPE, logging_level: int, opponent_path: str
     return result
 
 
-@pytest.mark.timeout(150, method="thread")
+@pytest.mark.timeout(180, method="thread")
 def test_sf() -> None:
     """Test lichess-bot with Stockfish (UCI)."""
     if platform != "linux" and platform != "win32":
-        assert True
-        return
+        pytest.skip("Platform must be Linux or Windows.")
     with open("./config.yml.default") as file:
         CONFIG = yaml.safe_load(file)
     CONFIG["token"] = ""
     CONFIG["engine"]["dir"] = "./TEMP/"
     CONFIG["engine"]["name"] = f"sf{file_extension}"
     CONFIG["engine"]["uci_options"]["Threads"] = 1
     CONFIG["pgn_directory"] = "TEMP/sf_game_record"
+    logger.info("Downloading Stockfish")
+    try:
+        download_sf()
+    except Exception:
+        logger.exception("Could not download the Stockfish chess engine")
+        pytest.skip("Could not download the Stockfish chess engine")
     win = run_bot(CONFIG, logging_level)
-    lichess_bot.logger.info("Finished Testing SF")
+    logger.info("Finished Testing SF")
     assert win
     assert os.path.isfile(os.path.join(CONFIG["pgn_directory"],
                                        "bo vs b - zzzzzzzz.pgn"))
 
 
-@pytest.mark.timeout(150, method="thread")
+@pytest.mark.timeout(180, method="thread")
 def test_lc0() -> None:
     """Test lichess-bot with Leela Chess Zero (UCI)."""
     if platform != "win32":
-        assert True
-        return
+        pytest.skip("Platform must be Windows.")
     with open("./config.yml.default") as file:
         CONFIG = yaml.safe_load(file)
     CONFIG["token"] = ""
@@ -236,19 +248,24 @@ def test_lc0() -> None:
     CONFIG["engine"]["uci_options"].pop("Hash", None)
     CONFIG["engine"]["uci_options"].pop("Move Overhead", None)
     CONFIG["pgn_directory"] = "TEMP/lc0_game_record"
+    logger.info("Downloading LC0")
+    try:
+        download_lc0()
+    except Exception:
+        logger.exception("Could not download the LC0 chess engine")
+        pytest.skip("Could not download the LC0 chess engine")
     win = run_bot(CONFIG, logging_level)
-    lichess_bot.logger.info("Finished Testing LC0")
+    logger.info("Finished Testing LC0")
     assert win
     assert os.path.isfile(os.path.join(CONFIG["pgn_directory"],
                                        "bo vs b - zzzzzzzz.pgn"))
 
 
-@pytest.mark.timeout(150, method="thread")
+@pytest.mark.timeout(180, method="thread")
 def test_sjeng() -> None:
     """Test lichess-bot with Sjeng (XBoard)."""
     if platform != "win32":
-        assert True
-        return
+        pytest.skip("Platform must be Windows.")
     with open("./config.yml.default") as file:
         CONFIG = yaml.safe_load(file)
     CONFIG["token"] = ""
@@ -258,27 +275,39 @@ def test_sjeng() -> None:
     CONFIG["engine"]["name"] = "sjeng.exe"
     CONFIG["engine"]["ponder"] = False
     CONFIG["pgn_directory"] = "TEMP/sjeng_game_record"
+    logger.info("Downloading Sjeng")
+    try:
+        download_sjeng()
+    except Exception:
+        logger.exception("Could not download the Sjeng chess engine")
+        pytest.skip("Could not download the Sjeng chess engine")
     win = run_bot(CONFIG, logging_level)
-    lichess_bot.logger.info("Finished Testing Sjeng")
+    logger.info("Finished Testing Sjeng")
     assert win
     assert os.path.isfile(os.path.join(CONFIG["pgn_directory"],
                                        "bo vs b - zzzzzzzz.pgn"))
 
 
-@pytest.mark.timeout(150, method="thread")
+@pytest.mark.timeout(180, method="thread")
 def test_homemade() -> None:
     """Test lichess-bot with a homemade engine running Stockfish (Homemade)."""
     if platform != "linux" and platform != "win32":
-        assert True
-        return
+        pytest.skip("Platform must be Linux or Windows.")
+
+    try:
+        download_sf()
+    except Exception:
+        logger.exception("Could not download the Stockfish chess engine")
+        pytest.skip("Could not download the Stockfish chess engine")
+
     with open("./config.yml.default") as file:
         CONFIG = yaml.safe_load(file)
     CONFIG["token"] = ""
     CONFIG["engine"]["name"] = f"Stockfish{test_suffix}"
     CONFIG["engine"]["protocol"] = "homemade"
     CONFIG["pgn_directory"] = "TEMP/homemade_game_record"
     win = run_bot(CONFIG, logging_level)
-    lichess_bot.logger.info("Finished Testing Homemade")
+    logger.info("Finished Testing Homemade")
     assert win
     assert os.path.isfile(os.path.join(CONFIG["pgn_directory"],
                                        "bo vs b - zzzzzzzz.pgn"))
@@ -309,7 +338,7 @@ def engine_path(CONFIG: CONFIG_DICT_TYPE) -> str:
     CONFIG["pgn_directory"] = "TEMP/bug_game_record"
 
     win = run_bot(CONFIG, logging_level, engine_path(CONFIG))
-    lichess_bot.logger.info("Finished Testing buggy engine")
+    logger.info("Finished Testing buggy engine")
     assert win
     assert os.path.isfile(os.path.join(CONFIG["pgn_directory"],
                                        "bo vs b - zzzzzzzz.pgn"))
diff --git a/test_bot/test_external_moves.py b/test_bot/test_external_moves.py
@@ -111,7 +111,6 @@ def download_opening_book() -> None:
 
 
 os.makedirs("TEMP", exist_ok=True)
-download_opening_book()
 
 
 def get_online_move_wrapper(li: LICHESS_TYPE, board: chess.Board, game: Game, online_moves_cfg: Configuration,
@@ -124,6 +123,7 @@ def test_external_moves() -> None:
     """Test that the code for external moves works properly."""
     li = MockLichess()
     game = get_game()
+    download_opening_book()
     online_cfg, online_cfg_2, draw_or_resign_cfg, polyglot_cfg = get_configs()
 
     starting_fen = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"