Skip to content

Commit

Permalink
Improve Dataset.load_from_files prog. bar status
Browse files Browse the repository at this point in the history
  • Loading branch information
sergioburdisso committed Feb 22, 2020
1 parent 2029c37 commit 2a700d5
Showing 1 changed file with 8 additions and 2 deletions.
10 changes: 8 additions & 2 deletions pyss3/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -1535,11 +1535,15 @@ def load_from_files(data_path, folder_label=True, as_single_doc=False):
if not folder_label:

files = listdir(data_path)
for file in tqdm(files, desc=" Category files",
leave=True, disable=Print.is_quiet()):
progress_bar = tqdm(
total=len(files), desc="Loading documents",
disable=Print.is_quiet()
)
for file in files:
file_path = path.join(data_path, file)
if path.isfile(file_path):
cat = path.splitext(file)[0]
progress_bar.set_description_str("Loading '%s' documents" % cat)

with open(file_path, "r", encoding=ENCODING) as fcat:
docs = (fcat.readlines()
Expand All @@ -1549,6 +1553,8 @@ def load_from_files(data_path, folder_label=True, as_single_doc=False):
y_data.extend([cat] * len(docs))

cat_info[cat] = len(docs)
progress_bar.update(1)
progress_bar.close()
else:
folders = listdir(data_path)
for icat, cat in enumerate(folders):
Expand Down

0 comments on commit 2a700d5

Please sign in to comment.