Skip to content

Commit

Permalink
Update readme and ruff
Browse files Browse the repository at this point in the history
Signed-off-by: Elron Bandel <elron.bandel@ibm.com>
  • Loading branch information
elronbandel committed Jan 28, 2024
1 parent e0bca34 commit c23f04a
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 4 deletions.
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,14 @@ In the dynamic landscape of generative NLP, traditional text processing pipeline

https://github.com/IBM/unitxt/assets/23455264/baef9131-39d4-4164-90b2-05da52919fdf

### 🦄 Currently on Unitxt Catalog

![NLP Tasks](https://img.shields.io/badge/NLP_tasks-21-blue)
![Dataset Cards](https://img.shields.io/badge/Dataset_Cards-377-blue)
![Templates](https://img.shields.io/badge/Templates-143-blue)
![Formats](https://img.shields.io/badge/Formats-7-blue)
![Metrics](https://img.shields.io/badge/Metrics-47-blue)

### 🦄 Run Unitxt Exploration Dashboard

To launch unitxt graphical user interface run:
Expand Down
12 changes: 8 additions & 4 deletions src/unitxt/catalog.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
import os
import re
from collections import Counter
from pathlib import Path
from typing import Optional
from collections import Counter

import requests

from .artifact import Artifact, Artifactory, reset_artifacts_cache, Artifactories
from .artifact import Artifact, Artifactories, Artifactory, reset_artifacts_cache
from .logging_utils import get_logger
from .version import version
from .text_utils import print_dict
from .version import version

logger = get_logger()
COLLECTION_SEPARATOR = "."
Expand Down Expand Up @@ -146,12 +147,14 @@ def get_local_catalogs_paths():
result.append(artifactory.location)
return result


def count_files_recursively(folder):
file_count = 0
for _, _, files in os.walk(folder):
file_count += len(files)
return file_count


def local_catalog_summary(catalog_path):
result = {}

Expand All @@ -161,9 +164,10 @@ def local_catalog_summary(catalog_path):

return result


def summary():
result = Counter()
for local_catalog_path in get_local_catalogs_paths():
result += Counter(local_catalog_summary(local_catalog_path))
result += Counter(local_catalog_summary(local_catalog_path))
print_dict(result)
return result

0 comments on commit c23f04a

Please sign in to comment.