Updating to v0.9.91

- Nested parallelism only applies to protein extraction, other parallel functions use the classic `apply_async` method
- Corrected bug that didn't erase alignment log files


Former-commit-id: 61aefad

captus/align.py

@@ -618,7 +618,7 @@ def align(full_command, args):
         )
         reclaimed_bytes = 0
         files_to_delete = list(out_dir.resolve().rglob("*.mafft.log"))
-        files_to_delete = list(out_dir.resolve().rglob("*.muscle.log"))
+        files_to_delete += list(out_dir.resolve().rglob("*.muscle.log"))
         files_to_delete += list(out_dir.resolve().rglob("*.clipkit.log"))
         for del_file in files_to_delete:
             reclaimed_bytes += del_file.stat().st_size

captus/extract.py

@@ -35,7 +35,8 @@
                    format_dep_msg, has_valid_ext, make_output_dir, make_tmp_dir_within,
                    mmseqs_path_version, python_library_check, quit_with_error, red,
                    remove_formatting, scipio_path_version, set_ram, set_threads, successful_exit,
-                   tqdm_parallel_async_run, tqdm_serial_run, yaml_perl_get_version)
+                   tqdm_parallel_async_run, tqdm_parallel_nested_run, tqdm_serial_run,
+                   yaml_perl_get_version)
 from .version import __version__
 
 
@@ -460,8 +461,8 @@ def extract(full_command, args):
                     tqdm_serial_run(scipio_coding, scipio_params, d_msg, f_msg,
                                     "extraction", args.show_less)
                 else:
-                    tqdm_parallel_async_run(scipio_coding, scipio_params, d_msg, f_msg,
-                                            "extraction", prot_concurrent, args.show_less)
+                    tqdm_parallel_nested_run(scipio_coding, scipio_params, d_msg, f_msg,
+                                             "extraction", prot_concurrent, args.show_less)
                 log.log("")
 
             if blat_params:

captus/misc.py

@@ -15,6 +15,7 @@
 
 import argparse
 import importlib
+import multiprocessing
 import os
 import platform
 import re
@@ -118,6 +119,35 @@ def tqdm_parallel_async_run(
     Run a function in parallel asynchronous mode updating a tqdm progress bar
     Keep in mind that the function referred as 'function_name' cannot be nested within another
     """
+    def update(function_message):
+        log.log(function_message, print_to_screen=False)
+        if not show_less:
+            tqdm.write(function_message)
+        pbar.update()
+
+    start = time.time()
+    log.log(bold(f"{description_msg}:"))
+    process = multiprocessing.Pool(threads)
+    tqdm_cols = min(shutil.get_terminal_size().columns, 120)
+    pbar = tqdm(total=len(params_list), ncols=tqdm_cols, unit=unit)
+    for i in range(pbar.total):
+        process.apply_async(function, params_list[i], callback=update)
+    process.close()
+    process.join()
+    pbar.close()
+    log.log(bold(
+        f" \u2514\u2500\u2192 {finished_msg} for {len(params_list)} {unit}(s)"
+        f" [{elapsed_time(time.time() - start)}]"
+    ))
+
+
+def tqdm_parallel_nested_run(
+    function, params_list, description_msg, finished_msg, unit, threads, show_less=False
+    ):
+    """
+    Run a function in parallel allowing children processes to span their own
+    parallel processes
+    """
     start = time.time()
     log.log(bold(f"{description_msg}:"))
     tqdm_cols = min(shutil.get_terminal_size().columns, 120)

captus/version.py

@@ -14,4 +14,4 @@
 not, see <http://www.gnu.org/licenses/>.
 """
 
-__version__ = '0.9.90'
+__version__ = '0.9.91'