diff --git a/captus/align.py b/captus/align.py index 09499c2..1206a29 100644 --- a/captus/align.py +++ b/captus/align.py @@ -618,7 +618,7 @@ def align(full_command, args): ) reclaimed_bytes = 0 files_to_delete = list(out_dir.resolve().rglob("*.mafft.log")) - files_to_delete = list(out_dir.resolve().rglob("*.muscle.log")) + files_to_delete += list(out_dir.resolve().rglob("*.muscle.log")) files_to_delete += list(out_dir.resolve().rglob("*.clipkit.log")) for del_file in files_to_delete: reclaimed_bytes += del_file.stat().st_size diff --git a/captus/extract.py b/captus/extract.py index 420b8a8..009b639 100644 --- a/captus/extract.py +++ b/captus/extract.py @@ -35,7 +35,8 @@ format_dep_msg, has_valid_ext, make_output_dir, make_tmp_dir_within, mmseqs_path_version, python_library_check, quit_with_error, red, remove_formatting, scipio_path_version, set_ram, set_threads, successful_exit, - tqdm_parallel_async_run, tqdm_serial_run, yaml_perl_get_version) + tqdm_parallel_async_run, tqdm_parallel_nested_run, tqdm_serial_run, + yaml_perl_get_version) from .version import __version__ @@ -460,8 +461,8 @@ def extract(full_command, args): tqdm_serial_run(scipio_coding, scipio_params, d_msg, f_msg, "extraction", args.show_less) else: - tqdm_parallel_async_run(scipio_coding, scipio_params, d_msg, f_msg, - "extraction", prot_concurrent, args.show_less) + tqdm_parallel_nested_run(scipio_coding, scipio_params, d_msg, f_msg, + "extraction", prot_concurrent, args.show_less) log.log("") if blat_params: diff --git a/captus/misc.py b/captus/misc.py index b53fa6d..fe84070 100644 --- a/captus/misc.py +++ b/captus/misc.py @@ -15,6 +15,7 @@ import argparse import importlib +import multiprocessing import os import platform import re @@ -118,6 +119,35 @@ def tqdm_parallel_async_run( Run a function in parallel asynchronous mode updating a tqdm progress bar Keep in mind that the function referred as 'function_name' cannot be nested within another """ + def update(function_message): + log.log(function_message, print_to_screen=False) + if not show_less: + tqdm.write(function_message) + pbar.update() + + start = time.time() + log.log(bold(f"{description_msg}:")) + process = multiprocessing.Pool(threads) + tqdm_cols = min(shutil.get_terminal_size().columns, 120) + pbar = tqdm(total=len(params_list), ncols=tqdm_cols, unit=unit) + for i in range(pbar.total): + process.apply_async(function, params_list[i], callback=update) + process.close() + process.join() + pbar.close() + log.log(bold( + f" \u2514\u2500\u2192 {finished_msg} for {len(params_list)} {unit}(s)" + f" [{elapsed_time(time.time() - start)}]" + )) + + +def tqdm_parallel_nested_run( + function, params_list, description_msg, finished_msg, unit, threads, show_less=False + ): + """ + Run a function in parallel allowing children processes to span their own + parallel processes + """ start = time.time() log.log(bold(f"{description_msg}:")) tqdm_cols = min(shutil.get_terminal_size().columns, 120) diff --git a/captus/version.py b/captus/version.py index d4f4d17..9a87972 100644 --- a/captus/version.py +++ b/captus/version.py @@ -14,4 +14,4 @@ not, see . """ -__version__ = '0.9.90' +__version__ = '0.9.91'