Lichess Puzzles (#217)

* Add command line script that allows parsing lichess puzzles

* Add recursive python path

* Added model loading in train_cli.py
Fixed model export
Set kernels for alpha-vile-large

* Update get_validation_data

* Added processes commad-line param

* Check for additional_loaders not None

DeepCrazyhouse/src/domain/neural_net/architectures/pytorch/alpha_vile.py

@@ -46,9 +46,29 @@ def get_alpha_vile_model(args, model_size='normal'):
 
     kernels = [3] * depth
     end_idx = int(len(kernels) * kernel_5_ratio + 0.5)
-    for idx in range(end_idx):
-        kernels[idx] = 5
-    random.shuffle(kernels)
+
+    if model_size == 'large':
+        kernels[1] = 5
+        kernels[6] = 5
+        kernels[7] = 5
+        kernels[9] = 5
+        kernels[10] = 5
+        kernels[14] = 5
+        kernels[18] = 5
+        kernels[19] = 5
+        kernels[23] = 5
+        kernels[25] = 5
+        kernels[26] = 5
+        kernels[27] = 5
+        kernels[28] = 5
+        kernels[29] = 5
+        kernels[33] = 5
+        kernels[34] = 5
+        kernels[35] = 5
+    else:
+        for idx in range(end_idx):
+            kernels[idx] = 5
+        random.shuffle(kernels)
 
     use_transformers = [False] * len(kernels)
     if nb_transformers > 0:

DeepCrazyhouse/src/preprocessing/pgn_converter_util.py

@@ -69,14 +69,15 @@ def get_planes_from_pgn(params):
     return metadata, game_idx, results[0], results[1], results[2], results[3], results[4]
 
 
-def get_planes_from_game(game, mate_in_one=False):
+def get_planes_from_move_sequence(board: chess.Board, y_init, all_moves, mate_in_one=False):
     """
     Returns all plane descriptions of a given game and their corresponding target values:
     - the game outcome (-1, 0, 1)
     - the next move which will be played in each position
 
-    :param game: Game handle which is a python-chess object
-    (e.g. mv_hist_len = 8 means that the current position and the 7 previous positions are exported)
+    :param board: Board object which is a python-chess object
+    :param y_init: Evaluation of the initial board position
+    :param all_moves: List of all moves to be applied to the position
     :param mate_in_one: Decide weather only to export the position before the last mate-in-one move
                         (this option is for evaluation and DEBUG purposes)
     :return: x - the position description of all moves in the game
@@ -95,24 +96,11 @@ def get_planes_from_game(game, mate_in_one=False):
     y_policy = []
     plys_to_end = []  # save the number of plys until the end of the game for each position that was considered
     phase_vector = []  # save all phases that occurred during the game
-    board = game.board()  # get the initial board state
-    # update the y value accordingly
-    if board.turn == chess.WHITE:
-        y_init = 1
-    else:
-        y_init = -1
-    if game.headers["Result"] == "0-1":
-        y_init *= -1
-    elif game.headers["Result"] == "1/2-1/2":
-        y_init = 0
 
-    all_moves = []  # Extract all moves first and save them into a list
-    for move in game.main_line():
-        all_moves.append(move)
     # Iterate through all moves (except the last one) and play them on a board.
     # you don't want to push the last move on the board because you had no movement policy to learn from in this case
     # The moves get pushed at the end of the for-loop and is only used in the next loop.
-    # Therefore we can iterate over 'all' moves
+    # Therefore, we can iterate over 'all' moves
     for plys, move in enumerate(all_moves):
         board_occ = 0  # by default the positions hasn't occurred before
         fen = board.fen()
@@ -166,3 +154,47 @@ def get_planes_from_game(game, mate_in_one=False):
         y_policy = np.stack(y_policy, axis=0)
 
     return x, y_value, y_policy, plys_to_end, phase_vector
+
+
+def get_planes_from_game(game, mate_in_one=False):
+    """
+    Returns all plane descriptions of a given game and their corresponding target values:
+    - the game outcome (-1, 0, 1)
+    - the next move which will be played in each position
+
+    :param game: Game handle which is a python-chess object
+    (e.g. mv_hist_len = 8 means that the current position and the 7 previous positions are exported)
+    :param mate_in_one: Decide weather only to export the position before the last mate-in-one move
+                        (this option is for evaluation and DEBUG purposes)
+    :return: x - the position description of all moves in the game
+             y_value - the target values of the scene description. Here the game outcome.
+                  returns -1 if the current player lost, +1 if the current player won, 0 for draw
+             y_policy - the policy vector one-hot encoded indicating the next move the player current player chose
+              in this position
+             plys_to_end - array of how many plys to the end of the game for each position.
+              This can be used to apply discounting
+             phase_vector - array of the game phase of each position
+    """
+
+    board = game.board()  # get the initial board state
+    # update the y value accordingly
+    if board.turn == chess.WHITE:
+        y_init = 1
+    else:
+        y_init = -1
+    if game.headers["Result"] == "0-1":
+        y_init *= -1
+    elif game.headers["Result"] == "1/2-1/2":
+        y_init = 0
+
+    all_moves = []  # Extract all moves first and save them into a list
+    for move in game.main_line():
+        all_moves.append(move)
+
+    try:
+        return get_planes_from_move_sequence(board, y_init, all_moves, mate_in_one)
+    except Exception:
+        print("game.headers:")
+        print(game.headers)
+        print("game", game)
+

DeepCrazyhouse/src/preprocessing/pgn_to_planes_converter.py

@@ -571,51 +571,8 @@ def export_pgn_batch(self, cur_part, game_idx_start, game_idx_end, pgn_sel, nb_w
             synchronizer=zarr.ThreadSynchronizer(),
             compression=compressor,
         )
-        # export the images
-        zarr_file.create_dataset(
-            name="x",
-            data=x,
-            shape=x.shape,
-            dtype=np.int16,
-            chunks=(128, x.shape[1], x.shape[2], x.shape[3]),
-            synchronizer=zarr.ThreadSynchronizer(),
-            compression=compressor,
-        )
-        # create the label arrays and copy the labels data in them
-        zarr_file.create_dataset(
-            name="y_value", shape=y_value.shape, dtype=np.int16, data=y_value, synchronizer=zarr.ThreadSynchronizer()
-        )
-        zarr_file.create_dataset(
-            name="y_policy",
-            shape=y_policy.shape,
-            dtype=np.int16,
-            data=y_policy,
-            chunks=(128, y_policy.shape[1]),
-            synchronizer=zarr.ThreadSynchronizer(),
-            compression=compressor,
-        )
-        zarr_file.create_dataset(
-            name="plys_to_end",
-            shape=plys_to_end.shape,
-            dtype=np.int16,
-            data=plys_to_end,
-            synchronizer=zarr.ThreadSynchronizer()
-        )
-        zarr_file.create_dataset(
-            name="phase_vector",
-            shape=phase_vector.shape,
-            dtype=np.int16,
-            data=phase_vector,
-            synchronizer=zarr.ThreadSynchronizer()
-        )
-        zarr_file.create_dataset(
-            name="start_indices",
-            shape=start_indices.shape,
-            dtype=np.int32,
-            data=start_indices,
-            synchronizer=zarr.ThreadSynchronizer(),
-            compression=compressor,
-        )
+        export_main_data(zarr_file, compressor, start_indices, x, y_value, y_policy, plys_to_end, phase_vector)
+
         zarr_file.create_group("/parameters")  # export the parameter settings and statistics of the file
         zarr_file.create_dataset(
             name="/parameters/pgn_name",
@@ -692,6 +649,55 @@ def export_pgn_batch(self, cur_part, game_idx_start, game_idx_end, pgn_sel, nb_w
         return True
 
 
+def export_main_data(zarr_file, compressor, start_indices, x, y_value, y_policy, plys_to_end, phase_vector):
+    """Exports the main data entries into the zarr-file."""
+    # export the images
+    zarr_file.create_dataset(
+        name="x",
+        data=x,
+        shape=x.shape,
+        dtype=np.int16,
+        chunks=(128, x.shape[1], x.shape[2], x.shape[3]),
+        synchronizer=zarr.ThreadSynchronizer(),
+        compression=compressor,
+    )
+    # create the label arrays and copy the labels data in them
+    zarr_file.create_dataset(
+        name="y_value", shape=y_value.shape, dtype=np.int16, data=y_value, synchronizer=zarr.ThreadSynchronizer()
+    )
+    zarr_file.create_dataset(
+        name="y_policy",
+        shape=y_policy.shape,
+        dtype=np.int16,
+        data=y_policy,
+        chunks=(128, y_policy.shape[1]),
+        synchronizer=zarr.ThreadSynchronizer(),
+        compression=compressor,
+    )
+    zarr_file.create_dataset(
+        name="plys_to_end",
+        shape=plys_to_end.shape,
+        dtype=np.int16,
+        data=plys_to_end,
+        synchronizer=zarr.ThreadSynchronizer()
+    )
+    zarr_file.create_dataset(
+        name="phase_vector",
+        shape=phase_vector.shape,
+        dtype=np.int16,
+        data=phase_vector,
+        synchronizer=zarr.ThreadSynchronizer()
+    )
+    zarr_file.create_dataset(
+        name="start_indices",
+        shape=start_indices.shape,
+        dtype=np.int32,
+        data=start_indices,
+        synchronizer=zarr.ThreadSynchronizer(),
+        compression=compressor,
+    )
+
+
 def export_pgn_to_datasetfile():
     """ Converts the pgn file of the games selected to a dataset file"""
     PGN2PlanesConverter(