From 3e64c1e78d5dbc9ef6899625584eb4249d4c18b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Radek=20Je=C5=BEek?= Date: Fri, 19 Jan 2024 09:34:10 +0100 Subject: [PATCH] feat: add lm_eval extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Radek Ježek --- .github/workflows/main.yml | 2 +- examples/extensions/lm_eval/__init__.py | 9 + examples/extensions/lm_eval/lm_eval_cli.py | 33 + examples/extensions/lm_eval/lm_eval_model.py | 46 + poetry.lock | 863 +++++++++++++++++- pyproject.toml | 8 + src/genai/extensions/lm_eval/__init__.py | 0 src/genai/extensions/lm_eval/__main__.py | 30 + src/genai/extensions/lm_eval/model.py | 305 +++++++ .../TestLMEval.test_generate_until.yaml | 183 ++++ .../TestLMEval.test_loglikelihood.yaml | 230 +++++ ...ihood_raises_for_invalid_tokenization.yaml | 184 ++++ tests/integration/extensions/test_lm_eval.py | 87 ++ 13 files changed, 1977 insertions(+), 3 deletions(-) create mode 100644 examples/extensions/lm_eval/__init__.py create mode 100644 examples/extensions/lm_eval/lm_eval_cli.py create mode 100644 examples/extensions/lm_eval/lm_eval_model.py create mode 100644 src/genai/extensions/lm_eval/__init__.py create mode 100644 src/genai/extensions/lm_eval/__main__.py create mode 100644 src/genai/extensions/lm_eval/model.py create mode 100644 tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_generate_until.yaml create mode 100644 tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_loglikelihood.yaml create mode 100644 tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_loglikelihood_raises_for_invalid_tokenization.yaml create mode 100644 tests/integration/extensions/test_lm_eval.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 78fb1ba5..8bfaff28 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: - name: Install dependencies run: | if [[ $(python -c "import sys; print(sys.version_info[:2] >= (3, 12))") == "True" ]]; then - poetry install --no-interaction -E llama-index -E localserver -E langchain + poetry install --no-interaction -E llama-index -E localserver -E langchain -E lm-eval else poetry install --no-interaction --all-extras fi diff --git a/examples/extensions/lm_eval/__init__.py b/examples/extensions/lm_eval/__init__.py new file mode 100644 index 00000000..11c14782 --- /dev/null +++ b/examples/extensions/lm_eval/__init__.py @@ -0,0 +1,9 @@ +""" +lm_eval + +.. admonition:: Before you start + :class: important + + To use the following extension, first install it by running + :bash:`pip install 'ibm-generative-ai[lm_eval]'`. +""" diff --git a/examples/extensions/lm_eval/lm_eval_cli.py b/examples/extensions/lm_eval/lm_eval_cli.py new file mode 100644 index 00000000..fba6fe88 --- /dev/null +++ b/examples/extensions/lm_eval/lm_eval_cli.py @@ -0,0 +1,33 @@ +""" +lm-evaluation-harness CLI usage + +The recommended way to run benchmarks is through CLI. +In your python environment with 'ibm-generative-ai[lm-eval]' installed: + +Example:: + + python -m genai.extensions.lm_eval \\ + --model="ibm_genai" \\ + --model_args="model_id=tiiuae/falcon-40b,temperature=0" \\ + --task="hellaswag" \\ + --num_fewshot=10 \\ + --output_path="falcon-40b_hellaswag.json" +""" + +import subprocess + +subprocess.run( + [ + "python", + "-m", + "genai.extensions.lm_eval", + "--model=ibm_genai", + "--model_args=model_id=tiiuae/falcon-40b,temperature=0", + "--task=hellaswag", + "--num_fewshot=10", + "--limit=10", # WARNING: only for debug purposes, remove for full testing dataset + ], + check=True, + text=True, + capture_output=False, +) diff --git a/examples/extensions/lm_eval/lm_eval_model.py b/examples/extensions/lm_eval/lm_eval_model.py new file mode 100644 index 00000000..3bc6200f --- /dev/null +++ b/examples/extensions/lm_eval/lm_eval_model.py @@ -0,0 +1,46 @@ +""" +lm-evaluation-harness advanced usage + +Use lm-evaluation extension from code to have additional control over concurrency or execution options + +Note: + This is for advanced usage only, use CLI in most cases (lm_eval_cli example) +""" + +import logging +from pprint import pprint + +from dotenv import load_dotenv +from lm_eval import simple_evaluate + +from genai import Client, Credentials +from genai.extensions.lm_eval.model import IBMGenAILMEval +from genai.schema import TextGenerationParameters + +load_dotenv() + +logging.getLogger("httpx").setLevel(logging.WARN) +logging.getLogger("genai").setLevel(logging.WARN) + +task_name = "arc_challenge" +model_id = "tiiuae/falcon-40b" +num_fewshot = 25 +limit = 10 # WARNING: only for debug purposes, set None for full testing dataset + +client = Client( + credentials=Credentials.from_env(), + config={"api_client_config": {"transport_options": {"retries": 999}}}, +) +model = IBMGenAILMEval( + client=client, + model_id=model_id, + show_progressbar=True, + parameters=TextGenerationParameters(temperature=0), +) +results = simple_evaluate(model, tasks=[task_name], num_fewshot=num_fewshot, log_samples=False, limit=limit) + +# add info about the model and few shot config +# "model_kwargs": model_kwargs, +results["config"] = {"model": model_id, "use_cache": False, "limit": limit, "model_kwargs": model.dump_parameters()} + +pprint(results) diff --git a/poetry.lock b/poetry.lock index 66101220..e8a972f9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,16 @@ # This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +[[package]] +name = "absl-py" +version = "2.1.0" +description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." +optional = true +python-versions = ">=3.7" +files = [ + {file = "absl-py-2.1.0.tar.gz", hash = "sha256:7820790efbb316739cde8b4e19357243fc3608a152024288513dd968d7d959ff"}, + {file = "absl_py-2.1.0-py3-none-any.whl", hash = "sha256:526a04eadab8b4ee719ce68f204172ead1027549089702d99b9059f129ff1308"}, +] + [[package]] name = "accelerate" version = "0.28.0" @@ -435,6 +446,70 @@ files = [ {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, ] +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = true +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + [[package]] name = "cfgv" version = "3.4.0" @@ -446,6 +521,17 @@ files = [ {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, ] +[[package]] +name = "chardet" +version = "5.2.0" +description = "Universal encoding detector for Python 3" +optional = true +python-versions = ">=3.7" +files = [ + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, +] + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -702,6 +788,25 @@ graphql = ["graphql-core (>=3.2.3,<4.0.0)"] http = ["httpx"] validation = ["openapi-spec-validator (>=0.2.8,<0.7.0)", "prance (>=0.18.2)"] +[[package]] +name = "dataproperty" +version = "1.0.1" +description = "Python library for extract property from data." +optional = true +python-versions = ">=3.7" +files = [ + {file = "DataProperty-1.0.1-py3-none-any.whl", hash = "sha256:0b8b07d4fb6453fcf975b53d35dea41f3cfd69c9d79b5010c3cf224ff0407a7a"}, + {file = "DataProperty-1.0.1.tar.gz", hash = "sha256:723e5729fa6e885e127a771a983ee1e0e34bb141aca4ffe1f0bfa7cde34650a4"}, +] + +[package.dependencies] +mbstrdecoder = ">=1.0.0,<2" +typepy = {version = ">=1.2.0,<2", extras = ["datetime"]} + +[package.extras] +logging = ["loguru (>=0.4.1,<1)"] +test = ["pytest (>=6.0.1)", "pytest-md-report (>=0.3)", "tcolorpy (>=0.1.2)"] + [[package]] name = "datasets" version = "2.18.0" @@ -921,6 +1026,42 @@ files = [ dnspython = ">=2.0.0" idna = ">=2.0.0" +[[package]] +name = "evaluate" +version = "0.4.1" +description = "HuggingFace community-driven open-source library of evaluation" +optional = true +python-versions = ">=3.7.0" +files = [ + {file = "evaluate-0.4.1-py3-none-any.whl", hash = "sha256:3ff079ab09572c0a2c1e6d749887c19f6783ab993320412cd39f6fe501d28510"}, + {file = "evaluate-0.4.1.tar.gz", hash = "sha256:d721d9f2059ced79770d8a0509e954fbd1bbac96a8f9160e29888d8073cda3d9"}, +] + +[package.dependencies] +datasets = ">=2.0.0" +dill = "*" +fsspec = {version = ">=2021.05.0", extras = ["http"]} +huggingface-hub = ">=0.7.0" +multiprocess = "*" +numpy = ">=1.17" +packaging = "*" +pandas = "*" +requests = ">=2.19.0" +responses = "<0.19" +tqdm = ">=4.62.1" +xxhash = "*" + +[package.extras] +dev = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "black (>=22.0,<23.0)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "jiwer", "mauve-text", "nltk", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] +docs = ["s3fs"] +evaluator = ["scipy (>=1.7.1)", "transformers"] +quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "isort (>=5.0.0)", "pyyaml (>=5.3.1)"] +template = ["cookiecutter", "gradio (>=3.0.0)"] +tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)"] +tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] +tests = ["Werkzeug (>=1.0.1)", "absl-py", "accelerate", "bert-score (>=0.3.6)", "cer (>=1.2.0)", "charcut (>=1.1.1)", "jiwer", "mauve-text", "nltk", "pytest", "pytest-datadir", "pytest-xdist", "requests-file (>=1.5.1)", "rouge-score (>=0.1.2)", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1,<=2.10)", "texttable (>=1.6.3)", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "torch", "transformers", "trectools", "unidecode (>=1.3.4)"] +torch = ["torch"] + [[package]] name = "exceptiongroup" version = "1.2.0" @@ -1469,6 +1610,20 @@ files = [ {file = "joblib-1.3.2.tar.gz", hash = "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1"}, ] +[[package]] +name = "jsonlines" +version = "4.0.0" +description = "Library with helpers for the jsonlines file format" +optional = true +python-versions = ">=3.8" +files = [ + {file = "jsonlines-4.0.0-py3-none-any.whl", hash = "sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55"}, + {file = "jsonlines-4.0.0.tar.gz", hash = "sha256:0c6d2c09117550c089995247f605ae4cf77dd1533041d366351f6f298822ea74"}, +] + +[package.dependencies] +attrs = ">=19.2.0" + [[package]] name = "jsonpatch" version = "1.33" @@ -1676,6 +1831,187 @@ files = [ httpx = ">=0.20.0" pydantic = ">=1.10" +[[package]] +name = "lm-eval" +version = "0.4.2" +description = "A framework for evaluating language models" +optional = true +python-versions = ">=3.8" +files = [ + {file = "lm_eval-0.4.2-py3-none-any.whl", hash = "sha256:8f3fe50ec92bc98ea62ad96236facc78d7e371db85dbc980627299e5750c8db4"}, + {file = "lm_eval-0.4.2.tar.gz", hash = "sha256:ddc2a1ea7041cbadf45afd118ebdecd707b1f14446fad51df212d2bf3defb4d2"}, +] + +[package.dependencies] +accelerate = ">=0.21.0" +datasets = ">=2.16.0" +dill = "*" +evaluate = ">=0.4.0" +jsonlines = "*" +more-itertools = "*" +numexpr = "*" +peft = ">=0.2.0" +pybind11 = ">=2.6.2" +pytablewriter = "*" +rouge-score = ">=0.0.4" +sacrebleu = ">=1.5.0" +scikit-learn = ">=0.24.1" +sqlitedict = "*" +torch = ">=1.8" +tqdm-multiprocess = "*" +transformers = ">=4.1" +word2number = "*" +zstandard = "*" + +[package.extras] +all = ["lm-eval[anthropic]", "lm-eval[dev]", "lm-eval[gptq]", "lm-eval[hf-transfer]", "lm-eval[ifeval]", "lm-eval[mamba]", "lm-eval[math]", "lm-eval[multilingual]", "lm-eval[openai]", "lm-eval[promptsource]", "lm-eval[sentencepiece]", "lm-eval[testing]", "lm-eval[vllm]", "lm-eval[wandb]", "lm-eval[zeno]"] +anthropic = ["anthropic"] +dev = ["mypy", "pre-commit", "pytest", "pytest-cov", "pytest-xdist"] +gptq = ["auto-gptq[triton] (>=0.6.0)"] +hf-transfer = ["hf-transfer"] +ifeval = ["immutabledict", "langdetect"] +mamba = ["causal-conv1d (==1.0.2)", "mamba-ssm"] +math = ["antlr4-python3-runtime (==4.11)", "sympy (>=1.12)"] +multilingual = ["jieba (>=0.42.1)", "nagisa (>=0.2.7)", "pycountry"] +neuronx = ["optimum[neuronx]"] +openai = ["openai (==1.3.9)", "tiktoken"] +optimum = ["optimum[openvino]"] +promptsource = ["promptsource (>=0.2.3)"] +sentencepiece = ["protobuf (>=4.22.1)", "sentencepiece (>=0.1.98)"] +testing = ["pytest", "pytest-cov", "pytest-xdist"] +vllm = ["vllm (==0.3.2)"] +wandb = ["numpy", "pandas", "wandb (>=0.16.3)"] +zeno = ["pandas", "zeno-client"] + +[[package]] +name = "lxml" +version = "5.2.0" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = true +python-versions = ">=3.6" +files = [ + {file = "lxml-5.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c54f8d6160080831a76780d850302fdeb0e8d0806f661777b0714dfb55d9a08a"}, + {file = "lxml-5.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0e95ae029396382a0d2e8174e4077f96befcd4a2184678db363ddc074eb4d3b2"}, + {file = "lxml-5.2.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5810fa80e64a0c689262a71af999c5735f48c0da0affcbc9041d1ef5ef3920be"}, + {file = "lxml-5.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae69524fd6a68b288574013f8fadac23cacf089c75cd3fc5b216277a445eb736"}, + {file = "lxml-5.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fadda215e32fe375d65e560b7f7e2a37c7f9c4ecee5315bb1225ca6ac9bf5838"}, + {file = "lxml-5.2.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:f1f164e4cc6bc646b1fc86664c3543bf4a941d45235797279b120dc740ee7af5"}, + {file = "lxml-5.2.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:3603a8a41097daf7672cae22cc4a860ab9ea5597f1c5371cb21beca3398b8d6a"}, + {file = "lxml-5.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b3b4bb89a785f4fd60e05f3c3a526c07d0d68e3536f17f169ca13bf5b5dd75a5"}, + {file = "lxml-5.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1effc10bf782f0696e76ecfeba0720ea02c0c31d5bffb7b29ba10debd57d1c3d"}, + {file = "lxml-5.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b03531f6cd6ce4b511dcece060ca20aa5412f8db449274b44f4003f282e6272f"}, + {file = "lxml-5.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7fac15090bb966719df06f0c4f8139783746d1e60e71016d8a65db2031ca41b8"}, + {file = "lxml-5.2.0-cp310-cp310-win32.whl", hash = "sha256:92bb37c96215c4b2eb26f3c791c0bf02c64dd251effa532b43ca5049000c4478"}, + {file = "lxml-5.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:b0181c22fdb89cc19e70240a850e5480817c3e815b1eceb171b3d7a3aa3e596a"}, + {file = "lxml-5.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ada8ce9e6e1d126ef60d215baaa0c81381ba5841c25f1d00a71cdafdc038bd27"}, + {file = "lxml-5.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3cefb133c859f06dab2ae63885d9f405000c4031ec516e0ed4f9d779f690d8e3"}, + {file = "lxml-5.2.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ede2a7a86a977b0c741654efaeca0af7860a9b1ae39f9268f0936246a977ee0"}, + {file = "lxml-5.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d46df6f0b1a0cda39d12c5c4615a7d92f40342deb8001c7b434d7c8c78352e58"}, + {file = "lxml-5.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2259243ee734cc736e237719037efb86603c891fd363cc7973a2d0ac8a0e3f"}, + {file = "lxml-5.2.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c53164f29ed3c3868787144e8ea8a399ffd7d8215f59500a20173593c19e96eb"}, + {file = "lxml-5.2.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:371aab9a397dcc76625ad3b02fa9b21be63406d69237b773156e7d1fc2ce0cae"}, + {file = "lxml-5.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e08784288a179b59115b5e57abf6d387528b39abb61105fe17510a199a277a40"}, + {file = "lxml-5.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c232726f7b6df5143415a06323faaa998ef8abbe1c0ed00d718755231d76f08"}, + {file = "lxml-5.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e4366e58c0508da4dee4c7c70cee657e38553d73abdffa53abbd7d743711ee11"}, + {file = "lxml-5.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c84dce8fb2e900d4fb094e76fdad34a5fd06de53e41bddc1502c146eb11abd74"}, + {file = "lxml-5.2.0-cp311-cp311-win32.whl", hash = "sha256:0947d1114e337dc2aae2fa14bbc9ed5d9ca1a0acd6d2f948df9926aef65305e9"}, + {file = "lxml-5.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:1eace37a9f4a1bef0bb5c849434933fd6213008ec583c8e31ee5b8e99c7c8500"}, + {file = "lxml-5.2.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f2cb157e279d28c66b1c27e0948687dc31dc47d1ab10ce0cd292a8334b7de3d5"}, + {file = "lxml-5.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:53c0e56f41ef68c1ce4e96f27ecdc2df389730391a2fd45439eb3facb02d36c8"}, + {file = "lxml-5.2.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:703d60e59ab45c17485c2c14b11880e4f7f0eab07134afa9007573fa5a779a5a"}, + {file = "lxml-5.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eaf5e308a5e50bc0548c4fdca0117a31ec9596f8cfc96592db170bcecc71a957"}, + {file = "lxml-5.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af64df85fecd3cf3b2e792f0b5b4d92740905adfa8ce3b24977a55415f1a0c40"}, + {file = "lxml-5.2.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:df7dfbdef11702fd22c2eaf042d7098d17edbc62d73f2199386ad06cbe466f6d"}, + {file = "lxml-5.2.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7250030a7835bfd5ba6ca7d1ad483ec90f9cbc29978c5e75c1cc3e031d3c4160"}, + {file = "lxml-5.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:be5faa2d5c8c8294d770cfd09d119fb27b5589acc59635b0cf90f145dbe81dca"}, + {file = "lxml-5.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:347ec08250d5950f5b016caa3e2e13fb2cb9714fe6041d52e3716fb33c208663"}, + {file = "lxml-5.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:dc7b630c4fb428b8a40ddd0bfc4bc19de11bb3c9b031154f77360e48fe8b4451"}, + {file = "lxml-5.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ae550cbd7f229cdf2841d9b01406bcca379a5fb327b9efb53ba620a10452e835"}, + {file = "lxml-5.2.0-cp312-cp312-win32.whl", hash = "sha256:7c61ce3cdd6e6c9f4003ac118be7eb3036d0ce2afdf23929e533e54482780f74"}, + {file = "lxml-5.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:f90c36ca95a44d2636bbf55a51ca30583b59b71b6547b88d954e029598043551"}, + {file = "lxml-5.2.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:1cce2eaad7e38b985b0f91f18468dda0d6b91862d32bec945b0e46e2ffe7222e"}, + {file = "lxml-5.2.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:60a3983d32f722a8422c01e4dc4badc7a307ca55c59e2485d0e14244a52c482f"}, + {file = "lxml-5.2.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60847dfbdfddf08a56c4eefe48234e8c1ab756c7eda4a2a7c1042666a5516564"}, + {file = "lxml-5.2.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bbe335f0d1a86391671d975a1b5e9b08bb72fba6b567c43bdc2e55ca6e6c086"}, + {file = "lxml-5.2.0-cp36-cp36m-manylinux_2_28_aarch64.whl", hash = "sha256:3ac7c8a60b8ad51fe7bca99a634dd625d66492c502fd548dc6dc769ce7d94b6a"}, + {file = "lxml-5.2.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:73e69762cf740ac3ae81137ef9d6f15f93095f50854e233d50b29e7b8a91dbc6"}, + {file = "lxml-5.2.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:281ee1ffeb0ab06204dfcd22a90e9003f0bb2dab04101ad983d0b1773bc10588"}, + {file = "lxml-5.2.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:ba3a86b0d5a5c93104cb899dff291e3ae13729c389725a876d00ef9696de5425"}, + {file = "lxml-5.2.0-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:356f8873b1e27b81793e30144229adf70f6d3e36e5cb7b6d289da690f4398953"}, + {file = "lxml-5.2.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:2a34e74ffe92c413f197ff4967fb1611d938ee0691b762d062ef0f73814f3aa4"}, + {file = "lxml-5.2.0-cp36-cp36m-win32.whl", hash = "sha256:6f0d2b97a5a06c00c963d4542793f3e486b1ed3a957f8c19f6006ed39d104bb0"}, + {file = "lxml-5.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:35e39c6fd089ad6674eb52d93aa874d6027b3ae44d2381cca6e9e4c2e102c9c8"}, + {file = "lxml-5.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5f6e4e5a62114ae76690c4a04c5108d067442d0a41fd092e8abd25af1288c450"}, + {file = "lxml-5.2.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93eede9bcc842f891b2267c7f0984d811940d1bc18472898a1187fe560907a99"}, + {file = "lxml-5.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ad364026c2cebacd7e01d1138bd53639822fefa8f7da90fc38cd0e6319a2699"}, + {file = "lxml-5.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f06e4460e76468d99cc36d5b9bc6fc5f43e6662af44960e13e3f4e040aacb35"}, + {file = "lxml-5.2.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:ca3236f31d565555139d5b00b790ed2a98ac6f0c4470c4032f8b5e5a5dba3c1a"}, + {file = "lxml-5.2.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:a9b67b850ab1d304cb706cf71814b0e0c3875287083d7ec55ee69504a9c48180"}, + {file = "lxml-5.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5261c858c390ae9a19aba96796948b6a2d56649cbd572968970dc8da2b2b2a42"}, + {file = "lxml-5.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e8359fb610c8c444ac473cfd82dae465f405ff807cabb98a9b9712bbd0028751"}, + {file = "lxml-5.2.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:f9e27841cddfaebc4e3ffbe5dbdff42891051acf5befc9f5323944b2c61cef16"}, + {file = "lxml-5.2.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:641a8da145aca67671205f3e89bfec9815138cf2fe06653c909eab42e486d373"}, + {file = "lxml-5.2.0-cp37-cp37m-win32.whl", hash = "sha256:931a3a13e0f574abce8f3152b207938a54304ccf7a6fd7dff1fdb2f6691d08af"}, + {file = "lxml-5.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:246c93e2503c710cf02c7e9869dc0258223cbefe5e8f9ecded0ac0aa07fd2bf8"}, + {file = "lxml-5.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:11acfcdf5a38cf89c48662123a5d02ae0a7d99142c7ee14ad90de5c96a9b6f06"}, + {file = "lxml-5.2.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:200f70b5d95fc79eb9ed7f8c4888eef4e274b9bf380b829d3d52e9ed962e9231"}, + {file = "lxml-5.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba4d02aed47c25be6775a40d55c5774327fdedba79871b7c2485e80e45750cb2"}, + {file = "lxml-5.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e283b24c14361fe9e04026a1d06c924450415491b83089951d469509900d9f32"}, + {file = "lxml-5.2.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:03e3962d6ad13a862dacd5b3a3ea60b4d092a550f36465234b8639311fd60989"}, + {file = "lxml-5.2.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:6e45fd5213e5587a610b7e7c8c5319a77591ab21ead42df46bb342e21bc1418d"}, + {file = "lxml-5.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:27877732946843f4b6bfc56eb40d865653eef34ad2edeed16b015d5c29c248df"}, + {file = "lxml-5.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4d16b44ad0dd8c948129639e34c8d301ad87ebc852568ace6fe9a5ad9ce67ee1"}, + {file = "lxml-5.2.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:b8f842df9ba26135c5414e93214e04fe0af259bb4f96a32f756f89467f7f3b45"}, + {file = "lxml-5.2.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c74e77df9e36c8c91157853e6cd400f6f9ca7a803ba89981bfe3f3fc7e5651ef"}, + {file = "lxml-5.2.0-cp38-cp38-win32.whl", hash = "sha256:1459a998c10a99711ac532abe5cc24ba354e4396dafef741c7797f8830712d56"}, + {file = "lxml-5.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:a00f5931b7cccea775123c3c0a2513aee58afdad8728550cc970bff32280bdd2"}, + {file = "lxml-5.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ddda5ba8831f258ac7e6364be03cb27aa62f50c67fd94bc1c3b6247959cc0369"}, + {file = "lxml-5.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:56835b9e9a7767202fae06310c6b67478963e535fe185bed3bf9af5b18d2b67e"}, + {file = "lxml-5.2.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25fef8794f0dc89f01bdd02df6a7fec4bcb2fbbe661d571e898167a83480185e"}, + {file = "lxml-5.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32d44af078485c4da9a7ec460162392d49d996caf89516fa0b75ad0838047122"}, + {file = "lxml-5.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f354d62345acdf22aa3e171bd9723790324a66fafe61bfe3873b86724cf6daaa"}, + {file = "lxml-5.2.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:6a7e0935f05e1cf1a3aa1d49a87505773b04f128660eac2a24a5594ea6b1baa7"}, + {file = "lxml-5.2.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:75a4117b43694c72a0d89f6c18a28dc57407bde4650927d4ef5fd384bdf6dcc7"}, + {file = "lxml-5.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:57402d6cdd8a897ce21cf8d1ff36683583c17a16322a321184766c89a1980600"}, + {file = "lxml-5.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:56591e477bea531e5e1854f5dfb59309d5708669bc921562a35fd9ca5182bdcd"}, + {file = "lxml-5.2.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7efbce96719aa275d49ad5357886845561328bf07e1d5ab998f4e3066c5ccf15"}, + {file = "lxml-5.2.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a3c39def0965e8fb5c8d50973e0c7b4ce429a2fa730f3f9068a7f4f9ce78410b"}, + {file = "lxml-5.2.0-cp39-cp39-win32.whl", hash = "sha256:5188f22c00381cb44283ecb28c8d85c2db4a3035774dd851876c8647cb809c27"}, + {file = "lxml-5.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:ed1fe80e1fcdd1205a443bddb1ad3c3135bb1cd3f36cc996a1f4aed35960fbe8"}, + {file = "lxml-5.2.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d2b339fb790fc923ae2e9345c8633e3d0064d37ea7920c027f20c8ae6f65a91f"}, + {file = "lxml-5.2.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06036d60fccb21e22dd167f6d0e422b9cbdf3588a7e999a33799f9cbf01e41a5"}, + {file = "lxml-5.2.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a1611fb9de0a269c05575c024e6d8cdf2186e3fa52b364e3b03dcad82514d57"}, + {file = "lxml-5.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:05fc3720250d221792b6e0d150afc92d20cb10c9cdaa8c8f93c2a00fbdd16015"}, + {file = "lxml-5.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:11e41ffd3cd27b0ca1c76073b27bd860f96431d9b70f383990f1827ca19f2f52"}, + {file = "lxml-5.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0382e6a3eefa3f6699b14fa77c2eb32af2ada261b75120eaf4fc028a20394975"}, + {file = "lxml-5.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:be5c8e776ecbcf8c1bce71a7d90e3a3680c9ceae516cac0be08b47e9fac0ca43"}, + {file = "lxml-5.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da12b4efc93d53068888cb3b58e355b31839f2428b8f13654bd25d68b201c240"}, + {file = "lxml-5.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f46f8033da364bacc74aca5e319509a20bb711c8a133680ca5f35020f9eaf025"}, + {file = "lxml-5.2.0-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:50a26f68d090594477df8572babac64575cd5c07373f7a8319c527c8e56c0f99"}, + {file = "lxml-5.2.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:57cbadf028727705086047994d2e50124650e63ce5a035b0aa79ab50f001989f"}, + {file = "lxml-5.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:8aa11638902ac23f944f16ce45c9f04c9d5d57bb2da66822abb721f4efe5fdbb"}, + {file = "lxml-5.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b7150e630b879390e02121e71ceb1807f682b88342e2ea2082e2c8716cf8bd93"}, + {file = "lxml-5.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4add722393c99da4d51c8d9f3e1ddf435b30677f2d9ba9aeaa656f23c1b7b580"}, + {file = "lxml-5.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd0f25a431cd16f70ec1c47c10b413e7ddfe1ccaaddd1a7abd181e507c012374"}, + {file = "lxml-5.2.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:883e382695f346c2ea3ad96bdbdf4ca531788fbeedb4352be3a8fcd169fc387d"}, + {file = "lxml-5.2.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:80cc2b55bb6e35d3cb40936b658837eb131e9f16357241cd9ba106ae1e9c5ecb"}, + {file = "lxml-5.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:59ec2948385336e9901008fdf765780fe30f03e7fdba8090aafdbe5d1b7ea0cd"}, + {file = "lxml-5.2.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ddbea6e58cce1a640d9d65947f1e259423fc201c9cf9761782f355f53b7f3097"}, + {file = "lxml-5.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52d6cdea438eb7282c41c5ac00bd6d47d14bebb6e8a8d2a1c168ed9e0cacfbab"}, + {file = "lxml-5.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c556bbf88a8b667c849d326dd4dd9c6290ede5a33383ffc12b0ed17777f909d"}, + {file = "lxml-5.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:947fa8bf15d1c62c6db36c6ede9389cac54f59af27010251747f05bddc227745"}, + {file = "lxml-5.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e6cb8f7a332eaa2d876b649a748a445a38522e12f2168e5e838d1505a91cdbb7"}, + {file = "lxml-5.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:16e65223f34fd3d65259b174f0f75a4bb3d9893698e5e7d01e54cd8c5eb98d85"}, + {file = "lxml-5.2.0.tar.gz", hash = "sha256:21dc490cdb33047bc7f7ad76384f3366fa8f5146b86cc04c4af45de901393b90"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html-clean = ["lxml-html-clean"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=3.0.10)"] + [[package]] name = "m2r" version = "0.3.1" @@ -1778,6 +2114,23 @@ dev = ["marshmallow[tests]", "pre-commit (>=3.5,<4.0)", "tox"] docs = ["alabaster (==0.7.16)", "autodocsumm (==0.2.12)", "sphinx (==7.2.6)", "sphinx-issues (==4.0.0)", "sphinx-version-warning (==1.1.2)"] tests = ["pytest", "pytz", "simplejson"] +[[package]] +name = "mbstrdecoder" +version = "1.1.3" +description = "mbstrdecoder is a Python library for multi-byte character string decoder" +optional = true +python-versions = ">=3.7" +files = [ + {file = "mbstrdecoder-1.1.3-py3-none-any.whl", hash = "sha256:d66c1ed3f2dc4e7c5d87cd44a75be10bc5af4250f95b38bbaedd7851308ce938"}, + {file = "mbstrdecoder-1.1.3.tar.gz", hash = "sha256:dcfd2c759322eb44fe193a9e0b1b86c5b87f3ec5ea8e1bb43b3e9ae423f1e8fe"}, +] + +[package.dependencies] +chardet = ">=3.0.4,<6" + +[package.extras] +test = ["Faker (>=1.0.2)", "pytest (>=6.0.1)", "pytest-md-report (>=0.1)"] + [[package]] name = "mistune" version = "0.8.4" @@ -1789,6 +2142,17 @@ files = [ {file = "mistune-0.8.4.tar.gz", hash = "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e"}, ] +[[package]] +name = "more-itertools" +version = "10.2.0" +description = "More routines for operating on iterables, beyond itertools" +optional = true +python-versions = ">=3.8" +files = [ + {file = "more-itertools-10.2.0.tar.gz", hash = "sha256:8fccb480c43d3e99a00087634c06dd02b0d50fbf088b380de5a41a015ec239e1"}, + {file = "more_itertools-10.2.0-py3-none-any.whl", hash = "sha256:686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684"}, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -1868,7 +2232,7 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, - {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, + {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] [[package]] @@ -2088,6 +2452,47 @@ files = [ [package.dependencies] setuptools = "*" +[[package]] +name = "numexpr" +version = "2.10.0" +description = "Fast numerical expression evaluator for NumPy" +optional = true +python-versions = ">=3.9" +files = [ + {file = "numexpr-2.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1af6dc6b3bd2e11a802337b352bf58f30df0b70be16c4f863b70a3af3a8ef95e"}, + {file = "numexpr-2.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c66dc0188358cdcc9465b6ee54fd5eef2e83ac64b1d4ba9117c41df59bf6fca"}, + {file = "numexpr-2.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83f1e7a7f7ee741b8dcd20c56c3f862a3a3ec26fa8b9fcadb7dcd819876d2f35"}, + {file = "numexpr-2.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f0b045e1831953a47cc9fabae76a6794c69cbb60921751a5cf2d555034c55bf"}, + {file = "numexpr-2.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1d8eb88b0ae3d3c609d732a17e71096779b2bf47b3a084320ffa93d9f9132786"}, + {file = "numexpr-2.10.0-cp310-cp310-win32.whl", hash = "sha256:629b66cc1b750671e7fb396506b3f9410612e5bd8bc1dd55b5a0a0041d839f95"}, + {file = "numexpr-2.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:78e0a8bc4417c3dedcbae3c473505b69080535246edc977c7dccf3ec8454a685"}, + {file = "numexpr-2.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a602692cd52ce923ce8a0a90fb1d6cf186ebe8706eed83eee0de685e634b9aa9"}, + {file = "numexpr-2.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:745b46a1fb76920a3eebfaf26e50bc94a9c13b5aee34b256ab4b2d792dbaa9ca"}, + {file = "numexpr-2.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10789450032357afaeda4ac4d06da9542d1535c13151e8d32b49ae1a488d1358"}, + {file = "numexpr-2.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4feafc65ea3044b8bf8f305b757a928e59167a310630c22b97a57dff07a56490"}, + {file = "numexpr-2.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:937d36c6d3cf15601f26f84f0f706649f976491e9e0892d16cd7c876d77fa7dc"}, + {file = "numexpr-2.10.0-cp311-cp311-win32.whl", hash = "sha256:03d0ba492e484a5a1aeb24b300c4213ed168f2c246177be5733abb4e18cbb043"}, + {file = "numexpr-2.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:6b5f8242c075477156d26b3a6b8e0cd0a06d4c8eb68d907bde56dd3c9c683e92"}, + {file = "numexpr-2.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b276e2ba3e87ace9a30fd49078ad5dcdc6a1674d030b1ec132599c55465c0346"}, + {file = "numexpr-2.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb5e12787101f1216f2cdabedc3417748f2e1f472442e16bbfabf0bab2336300"}, + {file = "numexpr-2.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05278bad96b5846d712eba58b44e5cec743bdb3e19ca624916c921d049fdbcf6"}, + {file = "numexpr-2.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6cdf9e64c5b3dbb61729edb505ea75ee212fa02b85c5b1d851331381ae3b0e1"}, + {file = "numexpr-2.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e3a973265591b0a875fd1151c4549e468959c7192821aac0bb86937694a08efa"}, + {file = "numexpr-2.10.0-cp312-cp312-win32.whl", hash = "sha256:416e0e9f0fc4cced67767585e44cb6b301728bdb9edbb7c534a853222ec62cac"}, + {file = "numexpr-2.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:748e8d4cde22d9a5603165293fb293a4de1a4623513299416c64fdab557118c2"}, + {file = "numexpr-2.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc3506c30c03b082da2cadef43747d474e5170c1f58a6dcdf882b3dc88b1e849"}, + {file = "numexpr-2.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:efa63ecdc9fcaf582045639ddcf56e9bdc1f4d9a01729be528f62df4db86c9d6"}, + {file = "numexpr-2.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96a64d0dd8f8e694da3f8582d73d7da8446ff375f6dd239b546010efea371ac3"}, + {file = "numexpr-2.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d47bb567e330ebe86781864219a36cbccb3a47aec893bd509f0139c6b23e8104"}, + {file = "numexpr-2.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c7517b774d309b1f0896c89bdd1ddd33c4418a92ecfbe5e1df3ac698698f6fcf"}, + {file = "numexpr-2.10.0-cp39-cp39-win32.whl", hash = "sha256:04e8620e7e676504201d4082e7b3ee2d9b561d1cb9470b47a6104e10c1e2870e"}, + {file = "numexpr-2.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:56d0d96b130f7cd4d78d0017030d6a0e9d9fc2a717ac51d4cf4860b39637e86a"}, + {file = "numexpr-2.10.0.tar.gz", hash = "sha256:c89e930752639df040539160326d8f99a84159bbea41943ab8e960591edaaef0"}, +] + +[package.dependencies] +numpy = ">=1.19.3" + [[package]] name = "numpy" version = "1.26.4" @@ -2488,6 +2893,50 @@ files = [ {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] +[[package]] +name = "pathvalidate" +version = "3.2.0" +description = "pathvalidate is a Python library to sanitize/validate a string such as filenames/file-paths/etc." +optional = true +python-versions = ">=3.7" +files = [ + {file = "pathvalidate-3.2.0-py3-none-any.whl", hash = "sha256:cc593caa6299b22b37f228148257997e2fa850eea2daf7e4cc9205cef6908dee"}, + {file = "pathvalidate-3.2.0.tar.gz", hash = "sha256:5e8378cf6712bff67fbe7a8307d99fa8c1a0cb28aa477056f8fc374f0dff24ad"}, +] + +[package.extras] +docs = ["Sphinx (>=2.4)", "sphinx-rtd-theme (>=1.2.2)", "urllib3 (<2)"] +test = ["Faker (>=1.0.8)", "allpairspy (>=2)", "click (>=6.2)", "pytest (>=6.0.1)", "pytest-discord (>=0.1.4)", "pytest-md-report (>=0.4.1)"] + +[[package]] +name = "peft" +version = "0.10.0" +description = "Parameter-Efficient Fine-Tuning (PEFT)" +optional = true +python-versions = ">=3.8.0" +files = [ + {file = "peft-0.10.0-py3-none-any.whl", hash = "sha256:d5249c97e818d3e31f92553c73c2953acd0ec12649b8b749afff7152cbc86cbb"}, + {file = "peft-0.10.0.tar.gz", hash = "sha256:36a7628c15f88d37abb26cfc74c22468f9037ee02e9c9b65de943cfe7c672049"}, +] + +[package.dependencies] +accelerate = ">=0.21.0" +huggingface-hub = ">=0.17.0" +numpy = ">=1.17" +packaging = ">=20.0" +psutil = "*" +pyyaml = "*" +safetensors = "*" +torch = ">=1.13.0" +tqdm = "*" +transformers = "*" + +[package.extras] +dev = ["black", "hf-doc-builder", "ruff (>=0.2.1,<0.3.0)"] +docs-specific = ["black", "hf-doc-builder"] +quality = ["black", "hf-doc-builder", "ruff (>=0.2.1,<0.3.0)"] +test = ["black", "datasets", "diffusers (<0.21.0)", "hf-doc-builder", "parameterized", "pytest", "pytest-cov", "pytest-xdist", "ruff (>=0.2.1,<0.3.0)", "scipy"] + [[package]] name = "pillow" version = "10.2.0" @@ -2621,6 +3070,25 @@ tomli = ">=1.2.2" [package.extras] poetry-plugin = ["poetry (>=1.0,<2.0)"] +[[package]] +name = "portalocker" +version = "2.8.2" +description = "Wraps the portalocker recipe for easy usage" +optional = true +python-versions = ">=3.8" +files = [ + {file = "portalocker-2.8.2-py3-none-any.whl", hash = "sha256:cfb86acc09b9aa7c3b43594e19be1345b9d16af3feb08bf92f23d4dce513a28e"}, + {file = "portalocker-2.8.2.tar.gz", hash = "sha256:2b035aa7828e46c58e9b31390ee1f169b98e1066ab10b9a6a861fe7e25ee4f33"}, +] + +[package.dependencies] +pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} + +[package.extras] +docs = ["sphinx (>=1.7.1)"] +redis = ["redis"] +tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)", "types-redis"] + [[package]] name = "pre-commit" version = "3.6.2" @@ -2726,6 +3194,31 @@ files = [ {file = "pyarrow_hotfix-0.6.tar.gz", hash = "sha256:79d3e030f7ff890d408a100ac16d6f00b14d44a502d7897cd9fc3e3a534e9945"}, ] +[[package]] +name = "pybind11" +version = "2.12.0" +description = "Seamless operability between C++11 and Python" +optional = true +python-versions = ">=3.6" +files = [ + {file = "pybind11-2.12.0-py3-none-any.whl", hash = "sha256:df8d60b94f9e714d81013db233393d430ebf9f3551642b82291cf1b14d1afdbd"}, + {file = "pybind11-2.12.0.tar.gz", hash = "sha256:5e3c557a84b06b969247630407fc4d985bed157b4253b13153b8e8e165e0c3dc"}, +] + +[package.extras] +global = ["pybind11-global (==2.12.0)"] + +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = true +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + [[package]] name = "pydantic" version = "2.6.4" @@ -2890,6 +3383,42 @@ cov = ["coverage", "coverage-python-version"] doc = ["recommonmark", "sphinx", "sphinx-git", "sphinx-rtd-theme", "sphinxcontrib-apidoc"] test = ["pytest", "pytest-cov"] +[[package]] +name = "pytablewriter" +version = "1.2.0" +description = "pytablewriter is a Python library to write a table in various formats: AsciiDoc / CSV / Elasticsearch / HTML / JavaScript / JSON / LaTeX / LDJSON / LTSV / Markdown / MediaWiki / NumPy / Excel / Pandas / Python / reStructuredText / SQLite / TOML / TSV / YAML." +optional = true +python-versions = ">=3.7" +files = [ + {file = "pytablewriter-1.2.0-py3-none-any.whl", hash = "sha256:4a30e2bb4bf5bc1069b1d2b2bc41947577c4517ab0875b23a5b194d296f543d8"}, + {file = "pytablewriter-1.2.0.tar.gz", hash = "sha256:0204a4bb684a22140d640f2599f09e137bcdc18b3dd49426f4a555016e246b46"}, +] + +[package.dependencies] +DataProperty = ">=1.0.1,<2" +mbstrdecoder = ">=1.0.0,<2" +pathvalidate = ">=2.3.0,<4" +setuptools = ">=38.3.0" +tabledata = ">=1.3.1,<2" +tcolorpy = ">=0.0.5,<1" +typepy = {version = ">=1.3.2,<2", extras = ["datetime"]} + +[package.extras] +all = ["PyYAML (>=3.11,<7)", "SimpleSQLite (>=1.3.2,<2)", "XlsxWriter (>=0.9.6,<4)", "dominate (>=2.1.5,<3)", "elasticsearch (>=8.0.1,<9)", "loguru (>=0.4.1,<1)", "pandas (>=0.25.3,<3)", "pytablereader (>=0.31.3,<2)", "pytablewriter-altcol-theme (>=0.1.0,<1)", "pytablewriter-altrow-theme (>=0.2.0,<1)", "simplejson (>=3.8.1,<4)", "toml (>=0.9.3,<1)", "xlwt"] +docs = ["PyYAML (>=3.11,<7)", "SimpleSQLite (>=1.3.2,<2)", "Sphinx (>=2.4)", "XlsxWriter (>=0.9.6,<4)", "dominate (>=2.1.5,<3)", "elasticsearch (>=8.0.1,<9)", "loguru (>=0.4.1,<1)", "pandas (>=0.25.3,<3)", "pytablereader (>=0.31.3,<2)", "pytablewriter-altcol-theme (>=0.1.0,<1)", "pytablewriter-altrow-theme (>=0.2.0,<1)", "simplejson (>=3.8.1,<4)", "sphinx-rtd-theme (>=1.2.2)", "toml (>=0.9.3,<1)", "xlwt"] +es = ["elasticsearch (>=8.0.1,<9)"] +es8 = ["elasticsearch (>=8.0.1,<9)"] +excel = ["XlsxWriter (>=0.9.6,<4)", "xlwt"] +from = ["pytablereader (>=0.31.3,<2)"] +html = ["dominate (>=2.1.5,<3)"] +logging = ["loguru (>=0.4.1,<1)"] +pandas = ["pandas (>=0.25.3,<3)"] +sqlite = ["SimpleSQLite (>=1.3.2,<2)"] +test = ["PyYAML (>=3.11,<7)", "SimpleSQLite (>=1.3.2,<2)", "XlsxWriter (>=0.9.6,<4)", "beautifulsoup4 (>=4.10)", "dominate (>=2.1.5,<3)", "elasticsearch (>=8.0.1,<9)", "loguru (>=0.4.1,<1)", "pandas (>=0.25.3,<3)", "pytablereader (>=0.31.3,<2)", "pytablereader[excel,sqlite] (>=0.31.3)", "pytablewriter-altcol-theme (>=0.1.0,<1)", "pytablewriter-altrow-theme (>=0.2.0,<1)", "pytest (>=6.0.1)", "pytest-md-report (>=0.4.1)", "simplejson (>=3.8.1,<4)", "sqliteschema (>=1.3.0)", "tablib (>=3.2.0)", "toml (>=0.9.3,<1)", "xlwt"] +theme = ["pytablewriter-altcol-theme (>=0.1.0,<1)", "pytablewriter-altrow-theme (>=0.2.0,<1)"] +toml = ["toml (>=0.9.3,<1)"] +yaml = ["PyYAML (>=3.11,<7)"] + [[package]] name = "pytest" version = "8.1.1" @@ -3071,6 +3600,29 @@ files = [ {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, ] +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +optional = true +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + [[package]] name = "pyyaml" version = "6.0.1" @@ -3253,6 +3805,40 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "responses" +version = "0.18.0" +description = "A utility library for mocking out the `requests` Python library." +optional = true +python-versions = ">=3.7" +files = [ + {file = "responses-0.18.0-py3-none-any.whl", hash = "sha256:15c63ad16de13ee8e7182d99c9334f64fd81f1ee79f90748d527c28f7ca9dd51"}, + {file = "responses-0.18.0.tar.gz", hash = "sha256:380cad4c1c1dc942e5e8a8eaae0b4d4edf708f4f010db8b7bcfafad1fcd254ff"}, +] + +[package.dependencies] +requests = ">=2.0,<3.0" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=4.6)", "pytest-cov", "pytest-localserver", "types-mock", "types-requests"] + +[[package]] +name = "rouge-score" +version = "0.1.2" +description = "Pure python implementation of ROUGE-1.5.5." +optional = true +python-versions = ">=3.7" +files = [ + {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, +] + +[package.dependencies] +absl-py = "*" +nltk = "*" +numpy = "*" +six = ">=1.14.0" + [[package]] name = "ruamel-yaml" version = "0.18.6" @@ -3356,6 +3942,30 @@ files = [ {file = "ruff-0.3.4.tar.gz", hash = "sha256:f0f4484c6541a99862b693e13a151435a279b271cff20e37101116a21e2a1ad1"}, ] +[[package]] +name = "sacrebleu" +version = "2.4.1" +description = "Hassle-free computation of shareable, comparable, and reproducible BLEU, chrF, and TER scores" +optional = true +python-versions = ">=3.6" +files = [ + {file = "sacrebleu-2.4.1-py3-none-any.whl", hash = "sha256:d24a783598ea5cfa2bb461cd377a5e3f76fa38a7df170bf99069fbd4c8157d25"}, + {file = "sacrebleu-2.4.1.tar.gz", hash = "sha256:294f5cb46c18a1ee6d066899b7cb6ad9393eaf824d362baded5551930e95bd4e"}, +] + +[package.dependencies] +colorama = "*" +lxml = "*" +numpy = ">=1.17" +portalocker = "*" +regex = "*" +tabulate = ">=0.8.9" + +[package.extras] +dev = ["lxml-stubs", "mypy", "pytest", "types-tabulate", "wheel"] +ja = ["ipadic (>=1.0,<2.0)", "mecab-python3 (>=1.0.5,<=1.0.6)"] +ko = ["mecab-ko (>=1.0.0,<=1.0.1)", "mecab-ko-dic (>=1.0,<2.0)"] + [[package]] name = "safetensors" version = "0.4.2" @@ -3488,6 +4098,90 @@ tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] testing = ["h5py (>=3.7.0)", "huggingface_hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools_rust (>=1.5.2)"] torch = ["safetensors[numpy]", "torch (>=1.10)"] +[[package]] +name = "scikit-learn" +version = "1.4.1.post1" +description = "A set of python modules for machine learning and data mining" +optional = true +python-versions = ">=3.9" +files = [ + {file = "scikit-learn-1.4.1.post1.tar.gz", hash = "sha256:93d3d496ff1965470f9977d05e5ec3376fb1e63b10e4fda5e39d23c2d8969a30"}, + {file = "scikit_learn-1.4.1.post1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c540aaf44729ab5cd4bd5e394f2b375e65ceaea9cdd8c195788e70433d91bbc5"}, + {file = "scikit_learn-1.4.1.post1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4310bff71aa98b45b46cd26fa641309deb73a5d1c0461d181587ad4f30ea3c36"}, + {file = "scikit_learn-1.4.1.post1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f43dd527dabff5521af2786a2f8de5ba381e182ec7292663508901cf6ceaf6e"}, + {file = "scikit_learn-1.4.1.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c02e27d65b0c7dc32f2c5eb601aaf5530b7a02bfbe92438188624524878336f2"}, + {file = "scikit_learn-1.4.1.post1-cp310-cp310-win_amd64.whl", hash = "sha256:629e09f772ad42f657ca60a1a52342eef786218dd20cf1369a3b8d085e55ef8f"}, + {file = "scikit_learn-1.4.1.post1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6145dfd9605b0b50ae72cdf72b61a2acd87501369a763b0d73d004710ebb76b5"}, + {file = "scikit_learn-1.4.1.post1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1afed6951bc9d2053c6ee9a518a466cbc9b07c6a3f9d43bfe734192b6125d508"}, + {file = "scikit_learn-1.4.1.post1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce03506ccf5f96b7e9030fea7eb148999b254c44c10182ac55857bc9b5d4815f"}, + {file = "scikit_learn-1.4.1.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ba516fcdc73d60e7f48cbb0bccb9acbdb21807de3651531208aac73c758e3ab"}, + {file = "scikit_learn-1.4.1.post1-cp311-cp311-win_amd64.whl", hash = "sha256:78cd27b4669513b50db4f683ef41ea35b5dddc797bd2bbd990d49897fd1c8a46"}, + {file = "scikit_learn-1.4.1.post1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a1e289f33f613cefe6707dead50db31930530dc386b6ccff176c786335a7b01c"}, + {file = "scikit_learn-1.4.1.post1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:0df87de9ce1c0140f2818beef310fb2e2afdc1e66fc9ad587965577f17733649"}, + {file = "scikit_learn-1.4.1.post1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:712c1c69c45b58ef21635360b3d0a680ff7d83ac95b6f9b82cf9294070cda710"}, + {file = "scikit_learn-1.4.1.post1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1754b0c2409d6ed5a3380512d0adcf182a01363c669033a2b55cca429ed86a81"}, + {file = "scikit_learn-1.4.1.post1-cp312-cp312-win_amd64.whl", hash = "sha256:1d491ef66e37f4e812db7e6c8286520c2c3fc61b34bf5e59b67b4ce528de93af"}, + {file = "scikit_learn-1.4.1.post1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:aa0029b78ef59af22cfbd833e8ace8526e4df90212db7ceccbea582ebb5d6794"}, + {file = "scikit_learn-1.4.1.post1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:14e4c88436ac96bf69eb6d746ac76a574c314a23c6961b7d344b38877f20fee1"}, + {file = "scikit_learn-1.4.1.post1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7cd3a77c32879311f2aa93466d3c288c955ef71d191503cf0677c3340ae8ae0"}, + {file = "scikit_learn-1.4.1.post1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a3ee19211ded1a52ee37b0a7b373a8bfc66f95353af058a210b692bd4cda0dd"}, + {file = "scikit_learn-1.4.1.post1-cp39-cp39-win_amd64.whl", hash = "sha256:234b6bda70fdcae9e4abbbe028582ce99c280458665a155eed0b820599377d25"}, +] + +[package.dependencies] +joblib = ">=1.2.0" +numpy = ">=1.19.5,<2.0" +scipy = ">=1.6.0" +threadpoolctl = ">=2.0.0" + +[package.extras] +benchmark = ["matplotlib (>=3.3.4)", "memory-profiler (>=0.57.0)", "pandas (>=1.1.5)"] +docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.3.4)", "memory-profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)", "sphinx (>=6.0.0)", "sphinx-copybutton (>=0.5.2)", "sphinx-gallery (>=0.15.0)", "sphinx-prompt (>=1.3.0)", "sphinxext-opengraph (>=0.4.2)"] +examples = ["matplotlib (>=3.3.4)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)"] +tests = ["black (>=23.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.3)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.19.12)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.0.272)", "scikit-image (>=0.17.2)"] + +[[package]] +name = "scipy" +version = "1.12.0" +description = "Fundamental algorithms for scientific computing in Python" +optional = true +python-versions = ">=3.9" +files = [ + {file = "scipy-1.12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:78e4402e140879387187f7f25d91cc592b3501a2e51dfb320f48dfb73565f10b"}, + {file = "scipy-1.12.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:f5f00ebaf8de24d14b8449981a2842d404152774c1a1d880c901bf454cb8e2a1"}, + {file = "scipy-1.12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e53958531a7c695ff66c2e7bb7b79560ffdc562e2051644c5576c39ff8efb563"}, + {file = "scipy-1.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e32847e08da8d895ce09d108a494d9eb78974cf6de23063f93306a3e419960c"}, + {file = "scipy-1.12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4c1020cad92772bf44b8e4cdabc1df5d87376cb219742549ef69fc9fd86282dd"}, + {file = "scipy-1.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:75ea2a144096b5e39402e2ff53a36fecfd3b960d786b7efd3c180e29c39e53f2"}, + {file = "scipy-1.12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:408c68423f9de16cb9e602528be4ce0d6312b05001f3de61fe9ec8b1263cad08"}, + {file = "scipy-1.12.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5adfad5dbf0163397beb4aca679187d24aec085343755fcdbdeb32b3679f254c"}, + {file = "scipy-1.12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3003652496f6e7c387b1cf63f4bb720951cfa18907e998ea551e6de51a04467"}, + {file = "scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b8066bce124ee5531d12a74b617d9ac0ea59245246410e19bca549656d9a40a"}, + {file = "scipy-1.12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8bee4993817e204d761dba10dbab0774ba5a8612e57e81319ea04d84945375ba"}, + {file = "scipy-1.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:a24024d45ce9a675c1fb8494e8e5244efea1c7a09c60beb1eeb80373d0fecc70"}, + {file = "scipy-1.12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e7e76cc48638228212c747ada851ef355c2bb5e7f939e10952bc504c11f4e372"}, + {file = "scipy-1.12.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f7ce148dffcd64ade37b2df9315541f9adad6efcaa86866ee7dd5db0c8f041c3"}, + {file = "scipy-1.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c39f92041f490422924dfdb782527a4abddf4707616e07b021de33467f917bc"}, + {file = "scipy-1.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7ebda398f86e56178c2fa94cad15bf457a218a54a35c2a7b4490b9f9cb2676c"}, + {file = "scipy-1.12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:95e5c750d55cf518c398a8240571b0e0782c2d5a703250872f36eaf737751338"}, + {file = "scipy-1.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:e646d8571804a304e1da01040d21577685ce8e2db08ac58e543eaca063453e1c"}, + {file = "scipy-1.12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:913d6e7956c3a671de3b05ccb66b11bc293f56bfdef040583a7221d9e22a2e35"}, + {file = "scipy-1.12.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba1b0c7256ad75401c73e4b3cf09d1f176e9bd4248f0d3112170fb2ec4db067"}, + {file = "scipy-1.12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:730badef9b827b368f351eacae2e82da414e13cf8bd5051b4bdfd720271a5371"}, + {file = "scipy-1.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6546dc2c11a9df6926afcbdd8a3edec28566e4e785b915e849348c6dd9f3f490"}, + {file = "scipy-1.12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:196ebad3a4882081f62a5bf4aeb7326aa34b110e533aab23e4374fcccb0890dc"}, + {file = "scipy-1.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:b360f1b6b2f742781299514e99ff560d1fe9bd1bff2712894b52abe528d1fd1e"}, + {file = "scipy-1.12.0.tar.gz", hash = "sha256:4bf5abab8a36d20193c698b0f1fc282c1d083c94723902c447e5d2f1780936a3"}, +] + +[package.dependencies] +numpy = ">=1.22.4,<1.29.0" + +[package.extras] +dev = ["click", "cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pydevtool", "rich-click", "ruff", "types-psutil", "typing_extensions"] +doc = ["jupytext", "matplotlib (>2)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-design (>=0.2.0)"] +test = ["asv", "gmpy2", "hypothesis", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + [[package]] name = "semver" version = "3.0.2" @@ -4010,6 +4704,16 @@ postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] pymysql = ["pymysql"] sqlcipher = ["sqlcipher3_binary"] +[[package]] +name = "sqlitedict" +version = "2.1.0" +description = "Persistent dict in Python, backed up by sqlite3 and pickle, multithread-safe." +optional = true +python-versions = "*" +files = [ + {file = "sqlitedict-2.1.0.tar.gz", hash = "sha256:03d9cfb96d602996f1d4c2db2856f1224b96a9c431bdd16e78032a72940f9e8c"}, +] + [[package]] name = "starlette" version = "0.27.0" @@ -4042,6 +4746,25 @@ files = [ [package.dependencies] mpmath = ">=0.19" +[[package]] +name = "tabledata" +version = "1.3.3" +description = "tabledata is a Python library to represent tabular data. Used for pytablewriter/pytablereader/SimpleSQLite/etc." +optional = true +python-versions = ">=3.7" +files = [ + {file = "tabledata-1.3.3-py3-none-any.whl", hash = "sha256:4abad1c996d8607e23b045b44dc0c5f061668f3c37585302c5f6c84c93a89962"}, + {file = "tabledata-1.3.3.tar.gz", hash = "sha256:c90daaba9a408e4397934b3ff2f6c06797d5289676420bf520c741ad43e6ff91"}, +] + +[package.dependencies] +DataProperty = ">=1.0.1,<2" +typepy = ">=1.2.0,<2" + +[package.extras] +logging = ["loguru (>=0.4.1,<1)"] +test = ["pytablewriter (>=0.46)", "pytest"] + [[package]] name = "tabulate" version = "0.9.0" @@ -4056,6 +4779,20 @@ files = [ [package.extras] widechars = ["wcwidth"] +[[package]] +name = "tcolorpy" +version = "0.1.4" +description = "tcolopy is a Python library to apply true color for terminal text." +optional = true +python-versions = ">=3.7" +files = [ + {file = "tcolorpy-0.1.4-py3-none-any.whl", hash = "sha256:d0926480aa5012f34877d69fc3b670f207dc165674e68ad07458fa6ee5b12724"}, + {file = "tcolorpy-0.1.4.tar.gz", hash = "sha256:f0dceb1cb95e554cee63024b3cd2fd8d4628c568773de2d1e6b4f0478461901c"}, +] + +[package.extras] +test = ["pytest (>=6.0.1)", "pytest-md-report (>=0.4.1)"] + [[package]] name = "tenacity" version = "8.2.3" @@ -4070,6 +4807,17 @@ files = [ [package.extras] doc = ["reno", "sphinx", "tornado (>=4.5)"] +[[package]] +name = "threadpoolctl" +version = "3.4.0" +description = "threadpoolctl" +optional = true +python-versions = ">=3.8" +files = [ + {file = "threadpoolctl-3.4.0-py3-none-any.whl", hash = "sha256:8f4c689a65b23e5ed825c8436a92b818aac005e0f3715f6a1664d7c7ee29d262"}, + {file = "threadpoolctl-3.4.0.tar.gz", hash = "sha256:f11b491a03661d6dd7ef692dd422ab34185d982466c49c8f98c8f716b5c93196"}, +] + [[package]] name = "tiktoken" version = "0.6.0" @@ -4349,6 +5097,24 @@ notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] +[[package]] +name = "tqdm-multiprocess" +version = "0.0.11" +description = "Easy multiprocessing with tqdm and logging redirected to main process." +optional = true +python-versions = ">=3.6" +files = [ + {file = "tqdm-multiprocess-0.0.11.tar.gz", hash = "sha256:a74002a1222ea9cbe8cdc9bd460108c6009be359621fbee9b92d0515d4d180f7"}, + {file = "tqdm_multiprocess-0.0.11-py3-none-any.whl", hash = "sha256:3ebdf03e7a675150fa0bbceaa9c3c64b8cb556e9ffafa4fe6c078e51820524aa"}, +] + +[package.dependencies] +colorama = "*" +tqdm = "*" + +[package.extras] +dev = ["twine"] + [[package]] name = "transformers" version = "4.39.1" @@ -4447,6 +5213,27 @@ build = ["cmake (>=3.20)", "lit"] tests = ["autopep8", "flake8", "isort", "numpy", "pytest", "scipy (>=1.7.1)", "torch"] tutorials = ["matplotlib", "pandas", "tabulate", "torch"] +[[package]] +name = "typepy" +version = "1.3.2" +description = "typepy is a Python library for variable type checker/validator/converter at a run time." +optional = true +python-versions = ">=3.7" +files = [ + {file = "typepy-1.3.2-py3-none-any.whl", hash = "sha256:d5d1022a424132622993800f1d2cd16cfdb691ac4e3b9c325f0fcb37799db1ae"}, + {file = "typepy-1.3.2.tar.gz", hash = "sha256:b69fd48b9f50cdb3809906eef36b855b3134ff66c8893a4f8580abddb0b39517"}, +] + +[package.dependencies] +mbstrdecoder = ">=1.0.0,<2" +packaging = {version = "*", optional = true, markers = "extra == \"datetime\""} +python-dateutil = {version = ">=2.8.0,<3.0.0", optional = true, markers = "extra == \"datetime\""} +pytz = {version = ">=2018.9", optional = true, markers = "extra == \"datetime\""} + +[package.extras] +datetime = ["packaging", "python-dateutil (>=2.8.0,<3.0.0)", "pytz (>=2018.9)"] +test = ["packaging", "pytest (>=6.0.1)", "python-dateutil (>=2.8.0,<3.0.0)", "pytz (>=2018.9)", "tcolorpy"] + [[package]] name = "typing-extensions" version = "4.10.0" @@ -4585,6 +5372,16 @@ files = [ {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, ] +[[package]] +name = "word2number" +version = "1.1" +description = "Convert number words eg. three hundred and forty two to numbers (342)." +optional = true +python-versions = "*" +files = [ + {file = "word2number-1.1.zip", hash = "sha256:70e27a5d387f67b04c71fbb7621c05930b19bfd26efd6851e6e0f9969dcde7d0"}, +] + [[package]] name = "wrapt" version = "1.16.0" @@ -4899,13 +5696,75 @@ files = [ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +[[package]] +name = "zstandard" +version = "0.22.0" +description = "Zstandard bindings for Python" +optional = true +python-versions = ">=3.8" +files = [ + {file = "zstandard-0.22.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:275df437ab03f8c033b8a2c181e51716c32d831082d93ce48002a5227ec93019"}, + {file = "zstandard-0.22.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ac9957bc6d2403c4772c890916bf181b2653640da98f32e04b96e4d6fb3252a"}, + {file = "zstandard-0.22.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe3390c538f12437b859d815040763abc728955a52ca6ff9c5d4ac707c4ad98e"}, + {file = "zstandard-0.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1958100b8a1cc3f27fa21071a55cb2ed32e9e5df4c3c6e661c193437f171cba2"}, + {file = "zstandard-0.22.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93e1856c8313bc688d5df069e106a4bc962eef3d13372020cc6e3ebf5e045202"}, + {file = "zstandard-0.22.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1a90ba9a4c9c884bb876a14be2b1d216609385efb180393df40e5172e7ecf356"}, + {file = "zstandard-0.22.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3db41c5e49ef73641d5111554e1d1d3af106410a6c1fb52cf68912ba7a343a0d"}, + {file = "zstandard-0.22.0-cp310-cp310-win32.whl", hash = "sha256:d8593f8464fb64d58e8cb0b905b272d40184eac9a18d83cf8c10749c3eafcd7e"}, + {file = "zstandard-0.22.0-cp310-cp310-win_amd64.whl", hash = "sha256:f1a4b358947a65b94e2501ce3e078bbc929b039ede4679ddb0460829b12f7375"}, + {file = "zstandard-0.22.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:589402548251056878d2e7c8859286eb91bd841af117dbe4ab000e6450987e08"}, + {file = "zstandard-0.22.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a97079b955b00b732c6f280d5023e0eefe359045e8b83b08cf0333af9ec78f26"}, + {file = "zstandard-0.22.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:445b47bc32de69d990ad0f34da0e20f535914623d1e506e74d6bc5c9dc40bb09"}, + {file = "zstandard-0.22.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33591d59f4956c9812f8063eff2e2c0065bc02050837f152574069f5f9f17775"}, + {file = "zstandard-0.22.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:888196c9c8893a1e8ff5e89b8f894e7f4f0e64a5af4d8f3c410f0319128bb2f8"}, + {file = "zstandard-0.22.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:53866a9d8ab363271c9e80c7c2e9441814961d47f88c9bc3b248142c32141d94"}, + {file = "zstandard-0.22.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4ac59d5d6910b220141c1737b79d4a5aa9e57466e7469a012ed42ce2d3995e88"}, + {file = "zstandard-0.22.0-cp311-cp311-win32.whl", hash = "sha256:2b11ea433db22e720758cba584c9d661077121fcf60ab43351950ded20283440"}, + {file = "zstandard-0.22.0-cp311-cp311-win_amd64.whl", hash = "sha256:11f0d1aab9516a497137b41e3d3ed4bbf7b2ee2abc79e5c8b010ad286d7464bd"}, + {file = "zstandard-0.22.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6c25b8eb733d4e741246151d895dd0308137532737f337411160ff69ca24f93a"}, + {file = "zstandard-0.22.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f9b2cde1cd1b2a10246dbc143ba49d942d14fb3d2b4bccf4618d475c65464912"}, + {file = "zstandard-0.22.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a88b7df61a292603e7cd662d92565d915796b094ffb3d206579aaebac6b85d5f"}, + {file = "zstandard-0.22.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:466e6ad8caefb589ed281c076deb6f0cd330e8bc13c5035854ffb9c2014b118c"}, + {file = "zstandard-0.22.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1d67d0d53d2a138f9e29d8acdabe11310c185e36f0a848efa104d4e40b808e4"}, + {file = "zstandard-0.22.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:39b2853efc9403927f9065cc48c9980649462acbdf81cd4f0cb773af2fd734bc"}, + {file = "zstandard-0.22.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8a1b2effa96a5f019e72874969394edd393e2fbd6414a8208fea363a22803b45"}, + {file = "zstandard-0.22.0-cp312-cp312-win32.whl", hash = "sha256:88c5b4b47a8a138338a07fc94e2ba3b1535f69247670abfe422de4e0b344aae2"}, + {file = "zstandard-0.22.0-cp312-cp312-win_amd64.whl", hash = "sha256:de20a212ef3d00d609d0b22eb7cc798d5a69035e81839f549b538eff4105d01c"}, + {file = "zstandard-0.22.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d75f693bb4e92c335e0645e8845e553cd09dc91616412d1d4650da835b5449df"}, + {file = "zstandard-0.22.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:36a47636c3de227cd765e25a21dc5dace00539b82ddd99ee36abae38178eff9e"}, + {file = "zstandard-0.22.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68953dc84b244b053c0d5f137a21ae8287ecf51b20872eccf8eaac0302d3e3b0"}, + {file = "zstandard-0.22.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2612e9bb4977381184bb2463150336d0f7e014d6bb5d4a370f9a372d21916f69"}, + {file = "zstandard-0.22.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23d2b3c2b8e7e5a6cb7922f7c27d73a9a615f0a5ab5d0e03dd533c477de23004"}, + {file = "zstandard-0.22.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d43501f5f31e22baf822720d82b5547f8a08f5386a883b32584a185675c8fbf"}, + {file = "zstandard-0.22.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a493d470183ee620a3df1e6e55b3e4de8143c0ba1b16f3ded83208ea8ddfd91d"}, + {file = "zstandard-0.22.0-cp38-cp38-win32.whl", hash = "sha256:7034d381789f45576ec3f1fa0e15d741828146439228dc3f7c59856c5bcd3292"}, + {file = "zstandard-0.22.0-cp38-cp38-win_amd64.whl", hash = "sha256:d8fff0f0c1d8bc5d866762ae95bd99d53282337af1be9dc0d88506b340e74b73"}, + {file = "zstandard-0.22.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2fdd53b806786bd6112d97c1f1e7841e5e4daa06810ab4b284026a1a0e484c0b"}, + {file = "zstandard-0.22.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:73a1d6bd01961e9fd447162e137ed949c01bdb830dfca487c4a14e9742dccc93"}, + {file = "zstandard-0.22.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9501f36fac6b875c124243a379267d879262480bf85b1dbda61f5ad4d01b75a3"}, + {file = "zstandard-0.22.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48f260e4c7294ef275744210a4010f116048e0c95857befb7462e033f09442fe"}, + {file = "zstandard-0.22.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:959665072bd60f45c5b6b5d711f15bdefc9849dd5da9fb6c873e35f5d34d8cfb"}, + {file = "zstandard-0.22.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d22fdef58976457c65e2796e6730a3ea4a254f3ba83777ecfc8592ff8d77d303"}, + {file = "zstandard-0.22.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a7ccf5825fd71d4542c8ab28d4d482aace885f5ebe4b40faaa290eed8e095a4c"}, + {file = "zstandard-0.22.0-cp39-cp39-win32.whl", hash = "sha256:f058a77ef0ece4e210bb0450e68408d4223f728b109764676e1a13537d056bb0"}, + {file = "zstandard-0.22.0-cp39-cp39-win_amd64.whl", hash = "sha256:e9e9d4e2e336c529d4c435baad846a181e39a982f823f7e4495ec0b0ec8538d2"}, + {file = "zstandard-0.22.0.tar.gz", hash = "sha256:8226a33c542bcb54cd6bd0a366067b610b41713b64c9abec1bc4533d69f51e70"}, +] + +[package.dependencies] +cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} + +[package.extras] +cffi = ["cffi (>=1.11)"] + [extras] huggingface = ["transformers"] langchain = ["langchain-core", "pyyaml"] llama-index = ["llama-index-core"] +lm-eval = ["lm-eval", "tqdm"] localserver = ["fastapi", "uvicorn"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "17b3ace4f09a40328857d83d3434cbc108b10a60238d18158084a445decdc61b" +content-hash = "94af6903cabc9b9618a91edcde7962c2dc39cf2b38f137afe2c0ee8b30638165" diff --git a/pyproject.toml b/pyproject.toml index f404b085..ec91a2ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,8 @@ llama-index-core = { version = "^0.10.0", optional = true } uvicorn = { version = "^0.22.0", optional = true } fastapi = { version = "^0.100.0", optional = true } deprecated = "^1.2.14" +lm-eval = { version = "^0.4.2", optional = true } +tqdm = { version = "^4.66.1", optional = true } [tool.black] # left for IDE compatibility (pycharm) line-length = 120 @@ -114,10 +116,12 @@ pytest-httpx = "^0.30.0" langchain = ["langchain-core", "pyyaml"] huggingface = ["datasets", "transformers"] llama-index = ["llama-index-core"] +lm-eval = ["lm-eval", "tqdm"] localserver = ["uvicorn", "fastapi"] [tool.pytest.ini_options] addopts = "--cov --cov-report term-missing --cov-fail-under 80 -v" +testpaths = ["tests"] markers = [ "unit", "integration", @@ -151,6 +155,10 @@ warn_required_dynamic_aliases = true help = "Install dependencies for the SDK core and it's extensions" cmd = "poetry install --all-extras --without dev" +[tool.poe.tasks.lm_eval] +help = "Run lm_eval with support for genai models" +cmd = "python -m genai.extensions.lm_eval" + [tool.poe.tasks.install-dev] help = "Install dependencies and related tooling for development" sequence = [{ cmd = "poetry install --all-extras" }, { cmd = "pre-commit install" }] diff --git a/src/genai/extensions/lm_eval/__init__.py b/src/genai/extensions/lm_eval/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/genai/extensions/lm_eval/__main__.py b/src/genai/extensions/lm_eval/__main__.py new file mode 100644 index 00000000..0e9a44f5 --- /dev/null +++ b/src/genai/extensions/lm_eval/__main__.py @@ -0,0 +1,30 @@ +import logging +import signal + +from genai import handle_shutdown_event +from genai.extensions.lm_eval.model import initialize_model + +try: + # load dotenv if installed + from dotenv import load_dotenv + + load_dotenv() +except ImportError: + ... + + +try: + from lm_eval.__main__ import cli_evaluate +except ImportError: + raise ImportError("Could not import lm_eval: Please install ibm-generative-ai[lm-eval] extension.") # noqa: B904 + + +initialize_model() + +signal.signal(signal.SIGINT, handle_shutdown_event) +signal.signal(signal.SIGTERM, handle_shutdown_event) + +logging.getLogger("httpx").setLevel(logging.WARN) +logging.getLogger("genai").setLevel(logging.WARN) + +cli_evaluate() diff --git a/src/genai/extensions/lm_eval/model.py b/src/genai/extensions/lm_eval/model.py new file mode 100644 index 00000000..10b07046 --- /dev/null +++ b/src/genai/extensions/lm_eval/model.py @@ -0,0 +1,305 @@ +import json +from collections import defaultdict +from typing import Any, Iterator, NamedTuple, Optional, Type, cast + +from genai import Client, Credentials +from genai.schema import ( + BaseTokens, + DecodingMethod, + TextGenerationParameters, + TextGenerationReturnOptions, + TextTokenizationParameters, + TextTokenizationReturnOptions, +) +from genai.text.generation import CreateExecutionOptions as TextGenerationExecutionOptions +from genai.text.tokenization import CreateExecutionOptions as TokenizationExecutionOptions + +try: + import lm_eval.utils + from lm_eval.api.instance import Instance + from lm_eval.api.model import LM + from lm_eval.api.registry import register_model + from lm_eval.models.utils import Grouper +except ImportError: + raise ImportError("Could not import lm_eval: Please install ibm-generative-ai[lm-eval] extension.") # noqa: B904 +try: + from tqdm import tqdm +except ImportError: + raise ImportError("Could not import tqdm: Please install ibm-generative-ai[lm-eval] extension.") # noqa: B904 + + +class LogLikelihoodResult(NamedTuple): + log_likelihood: float + is_greedy: bool + + +def initialize_model(): + pass # model is registered by importing this module + + +@register_model("ibm_genai") +class IBMGenAILMEval(LM): + """ + Implementation of LM model interface for evaluating GenAI model with the lm_eval framework. + + See https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/model_guide.md for reference. + """ + + DEFAULT_TOKENIZATION_EXECUTION_OPTIONS = TokenizationExecutionOptions( + batch_size=100, + concurrency_limit=5, + ) + DEFAULT_GENERATION_EXECUTION_OPTIONS = TextGenerationExecutionOptions() + DEFAULT_NUM_RETRIES = 6 # Increased number of retries for robustness, because evaluation typically runs for hours + + @classmethod + def create_from_arg_string( + cls: Type["IBMGenAILMEval"], + arg_string: str, + additional_config: Optional[dict] = None, + ) -> "IBMGenAILMEval": + """Allow the user to specify model parameters (TextGenerationParameters) in CLI arguments.""" + additional_config = {} if additional_config is None else additional_config + args = lm_eval.utils.simple_parse_args_string(arg_string) + model_id = args.pop("model_id", None) + if model_id is None: + raise ValueError("'model_id' is required, please pass it in 'model_args'") + parameters = TextGenerationParameters.model_validate(args) + return cls(client=Client(credentials=Credentials.from_env()), model_id=model_id, parameters=parameters) + + def __init__( + self, + client: Optional[Client] = None, + model_id: Optional[str] = None, + parameters: Optional[TextGenerationParameters] = None, + show_progressbar: Optional[bool] = True, + tokenization_execution_options: Optional[TokenizationExecutionOptions] = None, + generation_execution_options: Optional[TextGenerationExecutionOptions] = None, + ): + super().__init__() + self._client = client or Client( + credentials=Credentials.from_env(), + config={"api_client_config": {"transport_options": {"retries": self.DEFAULT_NUM_RETRIES}}}, + ) + self._model_id = model_id + self._parameters = parameters or TextGenerationParameters() + self._show_progressbar = show_progressbar + + for opts, name in [ + (tokenization_execution_options, "tokenization"), + (generation_execution_options, "generation"), + ]: + if opts and opts.ordered is False: + raise ValueError(f"Ordering is not configurable for evaluation ({name}_execution_options).") + + self._tokenization_execution_options = ( + tokenization_execution_options or self.DEFAULT_TOKENIZATION_EXECUTION_OPTIONS + ) + self._generation_execution_options = generation_execution_options or self.DEFAULT_GENERATION_EXECUTION_OPTIONS + + def dump_parameters(self): + return self._parameters.model_dump() + + def _tokenize(self, inputs: list[str]) -> Iterator[list[str]]: + pb = tqdm(desc="Tokenizing requests", total=len(inputs), disable=not self._show_progressbar) + for response in self._client.text.tokenization.create( + model_id=self._model_id, + input=inputs, + parameters=TextTokenizationParameters(return_options=TextTokenizationReturnOptions(tokens=True)), + execution_options=self._tokenization_execution_options, + ): + pb.update(len(response.results)) + for result in response.results: + yield result.tokens + pb.close() + + def _check_last_token_is_stop_token(self, response_tokens: list[str], context_tokens: list[str]) -> bool: + """ + Check whether tokens from context and response are the same. + Only last token can differ, in case or stop sequence () + + Returns: + True if only last token differs, False if all tokens are the same + Raises: + RuntimeError: if some other tokens differ than the last one + RuntimeError: if last token differs but context token is substring of response token. + Loglikelihood of second part of token is not defined + + """ + context_length = len(context_tokens) + if response_tokens[: context_length - 1] == context_tokens[:-1]: + if response_tokens[-1].startswith(context_tokens[-1]): + raise RuntimeError( + f"The context sent to loglikelihood evaluation ends with a token that is substring of the " + f"continuation token:\n" + f"context_tokens={context_tokens}\n" + f"response_tokens={response_tokens[:context_length]}\n" + "This is not allowed as it would skew the results. Please check your data." + ) + return response_tokens[:context_length][-1] != context_tokens[-1] + raise RuntimeError( + f"There is an unexpected difference between tokenizer and model tokens:\n" + f"context_tokens={context_tokens}\n" + f"response_tokens={response_tokens[:context_length]}" + ) + + def _check_model_logprobs_support(self): + input_tokens = ( + list( + self._client.text.generation.create( + model_id=self._model_id, + inputs=["The best ice cream flavor is:"], + parameters=self._log_likelihood_parameters, + execution_options=self._generation_execution_options, + ) + )[0] + .results[0] + .input_tokens + ) + + if all(token.logprob is None for token in input_tokens): + raise RuntimeError(f"Model {self._model_id} is not supported: does not return logprobs for input tokens") + + def _get_log_likelihood(self, input_tokens: list[BaseTokens], context_tokens: list[str]) -> LogLikelihoodResult: + response_tokens: list[str] = [token.text for token in input_tokens] + context_length = len(context_tokens) + + if self._check_last_token_is_stop_token(response_tokens, context_tokens): + context_length -= 1 + + return LogLikelihoodResult( + log_likelihood=sum(token.logprob for token in input_tokens[context_length:]), + is_greedy=all(token.rank == 1 for token in input_tokens[context_length:]), + ) + + @property + def _log_likelihood_parameters(self): + return TextGenerationParameters.model_validate( + { + **self._parameters.model_dump(), + "max_new_tokens": 1, # 0 is treated like "unlimited" + "return_options": TextGenerationReturnOptions( + input_tokens=True, + token_logprobs=True, + token_ranks=True, + ), + } + ) + + def loglikelihood(self, requests: list[Instance]) -> list[tuple[float, bool]]: + """ + Args: + requests: Each request contains Instance.args : Tuple[str, str] containing: + 1. an input string to the LM and + 2. a target string on which the loglikelihood of the LM producing this target, + conditioned on the input, will be returned. + Returns: + tuple (loglikelihood, is_greedy) for each request according to the input order: + loglikelihood: probability of generating the target string conditioned on the input + is_greedy: True if and only if the target string would be generated by greedy sampling from the LM + """ + self._check_model_logprobs_support() + + requests = [request.args for request in requests] + results: list[LogLikelihoodResult] = [] + + contexts_tokenized = list(self._tokenize([context for context, _ in requests])) + generation_inputs = [context + continuation for context, continuation in requests] + + pb = tqdm(desc="Running text generation", total=len(contexts_tokenized), disable=not self._show_progressbar) + for response, context_tokens in zip( + self._client.text.generation.create( + model_id=self._model_id, + inputs=generation_inputs, + parameters=self._log_likelihood_parameters, + execution_options=self._generation_execution_options, + ), + contexts_tokenized, + ): + pb.update(len(response.results)) + for result in response.results: + results.append(self._get_log_likelihood(result.input_tokens, context_tokens)) + pb.close() + return cast(list[tuple[float, bool]], results) + + def loglikelihood_rolling(self, requests: list[Instance]) -> list[tuple[float, bool]]: + """ + Used to evaluate perplexity on a data distribution. + + Args: + requests: Each request contains Instance.args : tuple[str] containing an input string to the model whose + entire loglikelihood, conditioned on purely the EOT token, will be calculated. + Returns: + tuple (loglikelihood,) for each request according to the input order: + loglikelihood: solely the probability of producing each piece of text given no starting input. + """ + + self._check_model_logprobs_support() + + generation_inputs = [request.args[0] for request in requests] + results: list[LogLikelihoodResult] = [] + for response in zip( + self._client.text.generation.create( + model_id=self._model_id, + inputs=generation_inputs, + parameters=self._log_likelihood_parameters, + execution_options=self._generation_execution_options, + ), + ): + for result in response.results: + results.append(self._get_log_likelihood(result.input_tokens, [])) + + return cast(list[tuple[float, bool]], results) + + def generate_until(self, requests: list[Instance]) -> list[str]: + """ + From official model_guide: https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/model_guide.md: + + Each request contains Instance.args : Tuple[str, dict] containing: + 1. an input string to the LM and + 2. a dictionary of keyword arguments used to control generation parameters. + Using this input and these generation parameters, text will be sampled from the language model + + ( + typically until a maximum output length or specific stopping string sequences--for example, + {"until": ["\n\n", "."], "max_gen_toks": 128} + ). + The generated input+output text from the model will then be returned. + """ + # group requests by their args (e.g. temperature, do_sample, etc.) + grouper = Grouper(requests, lambda request: json.dumps(request.args[1], sort_keys=True)) + results: dict[str, list[str]] = defaultdict(list) + + pb = tqdm(desc="Running text generation", total=len(requests), disable=not self._show_progressbar) + + for key, requests_group in grouper.get_grouped().items(): + generation_parameters: dict[str, Any] = requests_group[0].args[1] + inputs = [request.args[0] for request in requests_group] + + # Process parameters + do_sample = generation_parameters.pop("do_sample", False) + decoding_method = DecodingMethod.SAMPLE if do_sample else DecodingMethod.GREEDY + until = generation_parameters.pop("until") + stop_sequences = [until] if isinstance(until, str) else until + max_new_tokens = generation_parameters.pop("max_gen_toks", None) + temperature = generation_parameters.pop("temperature", None) + + parameters = TextGenerationParameters.model_validate( + { + **self._parameters.model_dump(), + "decoding_method": decoding_method, + "stop_sequences": stop_sequences, + "temperature": temperature, + "max_new_tokens": max_new_tokens, + } + ) + + for response in self._client.text.generation.create( + model_id=self._model_id, inputs=inputs, parameters=parameters + ): + results[key].extend(result.generated_text for result in response.results) + pb.update(len(response.results)) + + pb.close() + + return grouper.get_original(results) diff --git a/tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_generate_until.yaml b/tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_generate_until.yaml new file mode 100644 index 00000000..6b568daa --- /dev/null +++ b/tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_generate_until.yaml @@ -0,0 +1,183 @@ +interactions: +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + method: GET + uri: https://api.com/v2/text/generation/limits?version=2023-11-22 + response: + body: + string: '{"result":{"concurrency":{"limit":10,"remaining":10}}}' + headers: + cache-control: + - private + content-length: + - '54' + content-type: + - application/json; charset=utf-8 + content-version: + - '2023-11-22' + date: + - Tue, 02 Apr 2024 16:28:50 GMT + keep-alive: + - timeout=72 + set-cookie: + - 2eef5f4c257f6bca76e8da5586743beb=85b373fa0b9c193f95d2f0987342a678; path=/; + HttpOnly; Secure; SameSite=None + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"input": "Here are three sentences. My favorite color is ", "model_id": + "google/flan-t5-xl", "parameters": {"decoding_method": "greedy", "max_new_tokens": + 1000, "stop_sequences": ["."], "temperature": 1.0}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '207' + content-type: + - application/json + method: POST + uri: https://api.com/v2/text/generation?version=2024-03-19 + response: + body: + string: '{"id":"731c0441-2f1e-49bd-852d-f6771a0d8d64","model_id":"google/flan-t5-xl","created_at":"2024-04-02T16:28:50.829Z","results":[{"generated_text":"My + favorite color is blue.","generated_token_count":6,"input_token_count":11,"stop_reason":"stop_sequence","stop_sequence":"."}]}' + headers: + content-length: + - '275' + content-type: + - application/json; charset=utf-8 + content-version: + - '2024-03-19' + date: + - Tue, 02 Apr 2024 16:28:50 GMT + keep-alive: + - timeout=72 + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"input": "Here are three sentences. When I''m bored, I ", "model_id": + "google/flan-t5-xl", "parameters": {"decoding_method": "greedy", "max_new_tokens": + 1000, "stop_sequences": ["."], "temperature": 1.0}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '204' + content-type: + - application/json + method: POST + uri: https://api.com/v2/text/generation?version=2024-03-19 + response: + body: + string: '{"id":"c9ea47b0-4fc0-4880-9b3a-548c985717df","model_id":"google/flan-t5-xl","created_at":"2024-04-02T16:28:51.036Z","results":[{"generated_text":"I + like to read books.","generated_token_count":6,"input_token_count":14,"stop_reason":"stop_sequence","stop_sequence":"."}]}' + headers: + content-length: + - '270' + content-type: + - application/json; charset=utf-8 + content-version: + - '2024-03-19' + date: + - Tue, 02 Apr 2024 16:28:51 GMT + keep-alive: + - timeout=72 + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + method: GET + uri: https://api.com/v2/text/generation/limits?version=2023-11-22 + response: + body: + string: '{"result":{"concurrency":{"limit":10,"remaining":10}}}' + headers: + cache-control: + - private + content-length: + - '54' + content-type: + - application/json; charset=utf-8 + content-version: + - '2023-11-22' + date: + - Tue, 02 Apr 2024 16:28:54 GMT + keep-alive: + - timeout=72 + set-cookie: + - 2eef5f4c257f6bca76e8da5586743beb=e7012ef98dc1d6cddd80c399165de22f; path=/; + HttpOnly; Secure; SameSite=None + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"input": "Here are three sentences. I''m happy because ", "model_id": + "google/flan-t5-xl", "parameters": {"decoding_method": "greedy", "max_new_tokens": + 1000, "stop_sequences": ["."], "temperature": 0.0}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '204' + content-type: + - application/json + method: POST + uri: https://api.com/v2/text/generation?version=2024-03-19 + response: + body: + string: '{"id":"c1c972ce-58f2-4952-8df7-83019430442a","model_id":"google/flan-t5-xl","created_at":"2024-04-02T16:28:54.447Z","results":[{"generated_text":"I + got a new job.","generated_token_count":7,"input_token_count":12,"stop_reason":"stop_sequence","stop_sequence":"."}]}' + headers: + content-length: + - '265' + content-type: + - application/json; charset=utf-8 + content-version: + - '2024-03-19' + date: + - Tue, 02 Apr 2024 16:28:54 GMT + keep-alive: + - timeout=72 + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_loglikelihood.yaml b/tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_loglikelihood.yaml new file mode 100644 index 00000000..79e7b891 --- /dev/null +++ b/tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_loglikelihood.yaml @@ -0,0 +1,230 @@ +interactions: +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + method: GET + uri: https://api.com/v2/text/generation/limits?version=2023-11-22 + response: + body: + string: '{"result":{"concurrency":{"limit":10,"remaining":10}}}' + headers: + cache-control: + - private + content-length: + - '54' + content-type: + - application/json; charset=utf-8 + content-version: + - '2023-11-22' + date: + - Tue, 02 Apr 2024 16:30:51 GMT + keep-alive: + - timeout=72 + set-cookie: + - 2eef5f4c257f6bca76e8da5586743beb=85b373fa0b9c193f95d2f0987342a678; path=/; + HttpOnly; Secure; SameSite=None + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"input": "The best ice cream flavor is:", "model_id": "tiiuae/falcon-40b", + "parameters": {"max_new_tokens": 1, "return_options": {"generated_tokens": false, + "input_text": false, "input_tokens": true, "token_logprobs": true, "token_ranks": + true}}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '247' + content-type: + - application/json + method: POST + uri: https://api.com/v2/text/generation?version=2024-03-19 + response: + body: + string: "{\"id\":\"51b58fa3-e8bd-477d-b2e3-e1d7de92e85e\",\"model_id\":\"tiiuae/falcon-40b\",\"created_at\":\"2024-04-02T16:30:51.515Z\",\"results\":[{\"generated_text\":\" + Chocolate\",\"generated_token_count\":1,\"input_token_count\":7,\"stop_reason\":\"max_tokens\",\"input_tokens\":[{\"text\":\"The\",\"logprob\":null},{\"text\":\"\u0120best\",\"logprob\":-5.19140625,\"rank\":7},{\"text\":\"\u0120ice\",\"logprob\":-7.53125,\"rank\":193},{\"text\":\"\u0120cream\",\"logprob\":-0.77978515625,\"rank\":1},{\"text\":\"\u0120flavor\",\"logprob\":-5.53515625,\"rank\":35},{\"text\":\"\u0120is\",\"logprob\":-1.68359375,\"rank\":2},{\"text\":\":\",\"logprob\":-3.53515625,\"rank\":8}],\"seed\":1656840585}]}" + headers: + content-length: + - '592' + content-type: + - application/json; charset=utf-8 + content-version: + - '2024-03-19' + date: + - Tue, 02 Apr 2024 16:30:51 GMT + keep-alive: + - timeout=72 + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"input": ["Classify the following tweet: ''No this is my first job'' into + ''complaint'' or ''no complaint'':", "Classify the following tweet: ''Please + just give me my money back.'' into ''complaint'' or ''no complaint'':"], "model_id": + "tiiuae/falcon-40b", "parameters": {"return_options": {"tokens": true}}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '297' + content-type: + - application/json + method: POST + uri: https://api.com/v2/text/tokenization?version=2024-01-10 + response: + body: + string: "{\"model_id\":\"tiiuae/falcon-40b\",\"created_at\":\"2024-04-02T16:30:51.916Z\",\"results\":[{\"token_count\":27,\"tokens\":[\"Class\",\"ify\",\"\u0120the\",\"\u0120following\",\"\u0120tweet\",\":\",\"\u0120\",\"'\",\"No\",\"\u0120this\",\"\u0120is\",\"\u0120my\",\"\u0120first\",\"\u0120job\",\"'\",\"\u0120into\",\"\u0120\",\"'\",\"compl\",\"aint\",\"'\",\"\u0120or\",\"\u0120\",\"'\",\"no\",\"\u0120complaint\",\"':\"]},{\"token_count\":28,\"tokens\":[\"Class\",\"ify\",\"\u0120the\",\"\u0120following\",\"\u0120tweet\",\":\",\"\u0120\",\"'\",\"Please\",\"\u0120just\",\"\u0120give\",\"\u0120me\",\"\u0120my\",\"\u0120money\",\"\u0120back\",\".'\",\"\u0120into\",\"\u0120\",\"'\",\"compl\",\"aint\",\"'\",\"\u0120or\",\"\u0120\",\"'\",\"no\",\"\u0120complaint\",\"':\"]}]}" + headers: + content-length: + - '531' + content-type: + - application/json; charset=utf-8 + content-version: + - '2024-01-10' + date: + - Tue, 02 Apr 2024 16:30:51 GMT + keep-alive: + - timeout=72 + set-cookie: + - 2eef5f4c257f6bca76e8da5586743beb=8037b965638f6f2fcbf28b6380782b2a; path=/; + HttpOnly; Secure; SameSite=None + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + method: GET + uri: https://api.com/v2/text/generation/limits?version=2023-11-22 + response: + body: + string: '{"result":{"concurrency":{"limit":10,"remaining":10}}}' + headers: + cache-control: + - private + content-length: + - '54' + content-type: + - application/json; charset=utf-8 + content-version: + - '2023-11-22' + date: + - Tue, 02 Apr 2024 16:30:52 GMT + keep-alive: + - timeout=72 + set-cookie: + - 2eef5f4c257f6bca76e8da5586743beb=6e96b2939356f29ea99dc5efdefbb9c6; path=/; + HttpOnly; Secure; SameSite=None + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"input": "Classify the following tweet: ''No this is my first job'' into + ''complaint'' or ''no complaint'':no complaint", "model_id": "tiiuae/falcon-40b", + "parameters": {"max_new_tokens": 1, "return_options": {"generated_tokens": false, + "input_text": false, "input_tokens": true, "token_logprobs": true, "token_ranks": + true}}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '321' + content-type: + - application/json + method: POST + uri: https://api.com/v2/text/generation?version=2024-03-19 + response: + body: + string: "{\"id\":\"a4146a19-0857-4e5b-a347-5b84fe98968c\",\"model_id\":\"tiiuae/falcon-40b\",\"created_at\":\"2024-04-02T16:30:52.504Z\",\"results\":[{\"generated_text\":\"\\n\",\"generated_token_count\":1,\"input_token_count\":29,\"stop_reason\":\"max_tokens\",\"input_tokens\":[{\"text\":\"Class\",\"logprob\":null},{\"text\":\"ify\",\"logprob\":-5.9921875,\"rank\":37},{\"text\":\"\u0120the\",\"logprob\":-2.6015625,\"rank\":2},{\"text\":\"\u0120following\",\"logprob\":-1.3134765625,\"rank\":1},{\"text\":\"\u0120tweet\",\"logprob\":-13.1640625,\"rank\":10122},{\"text\":\":\",\"logprob\":-2.064453125,\"rank\":2},{\"text\":\"\u0120\",\"logprob\":-2.048828125,\"rank\":2},{\"text\":\"'\",\"logprob\":-4.15625,\"rank\":8},{\"text\":\"No\",\"logprob\":-5.609375,\"rank\":35},{\"text\":\"\u0120this\",\"logprob\":-7.58203125,\"rank\":179},{\"text\":\"\u0120is\",\"logprob\":-0.5751953125,\"rank\":1},{\"text\":\"\u0120my\",\"logprob\":-4.05859375,\"rank\":6},{\"text\":\"\u0120first\",\"logprob\":-3.880859375,\"rank\":4},{\"text\":\"\u0120job\",\"logprob\":-4.578125,\"rank\":11},{\"text\":\"'\",\"logprob\":-2.646484375,\"rank\":4},{\"text\":\"\u0120into\",\"logprob\":-4.51171875,\"rank\":5},{\"text\":\"\u0120\",\"logprob\":-2.5859375,\"rank\":5},{\"text\":\"'\",\"logprob\":-0.82421875,\"rank\":1},{\"text\":\"compl\",\"logprob\":-5.7109375,\"rank\":34},{\"text\":\"aint\",\"logprob\":-0.1875,\"rank\":1},{\"text\":\"'\",\"logprob\":-0.6171875,\"rank\":1},{\"text\":\"\u0120or\",\"logprob\":-0.316162109375,\"rank\":1},{\"text\":\"\u0120\",\"logprob\":-0.039520263671875,\"rank\":1},{\"text\":\"'\",\"logprob\":-0.00476837158203125,\"rank\":1},{\"text\":\"no\",\"logprob\":-3.275390625,\"rank\":7},{\"text\":\"\u0120complaint\",\"logprob\":-1.0419921875,\"rank\":1},{\"text\":\"':\",\"logprob\":-4.83984375,\"rank\":6},{\"text\":\"no\",\"logprob\":-9.5234375,\"rank\":546},{\"text\":\"\u0120complaint\",\"logprob\":-1.6064453125,\"rank\":2}],\"seed\":3788404351}]}" + headers: + content-length: + - '1640' + content-type: + - application/json; charset=utf-8 + content-version: + - '2024-03-19' + date: + - Tue, 02 Apr 2024 16:30:52 GMT + keep-alive: + - timeout=72 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"input": "Classify the following tweet: ''Please just give me my money + back.'' into ''complaint'' or ''no complaint'':complaint", "model_id": "tiiuae/falcon-40b", + "parameters": {"max_new_tokens": 1, "return_options": {"generated_tokens": false, + "input_text": false, "input_tokens": true, "token_logprobs": true, "token_ranks": + true}}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '329' + content-type: + - application/json + method: POST + uri: https://api.com/v2/text/generation?version=2024-03-19 + response: + body: + string: "{\"id\":\"f2e9a973-fd18-4b21-8a8f-a48481d2ccc3\",\"model_id\":\"tiiuae/falcon-40b\",\"created_at\":\"2024-04-02T16:30:52.753Z\",\"results\":[{\"generated_text\":\"\\n\",\"generated_token_count\":1,\"input_token_count\":30,\"stop_reason\":\"max_tokens\",\"input_tokens\":[{\"text\":\"Class\",\"logprob\":null},{\"text\":\"ify\",\"logprob\":-5.9921875,\"rank\":37},{\"text\":\"\u0120the\",\"logprob\":-2.6015625,\"rank\":2},{\"text\":\"\u0120following\",\"logprob\":-1.310546875,\"rank\":1},{\"text\":\"\u0120tweet\",\"logprob\":-13.1640625,\"rank\":10123},{\"text\":\":\",\"logprob\":-2.064453125,\"rank\":2},{\"text\":\"\u0120\",\"logprob\":-2.048828125,\"rank\":2},{\"text\":\"'\",\"logprob\":-4.15625,\"rank\":7},{\"text\":\"Please\",\"logprob\":-6.17578125,\"rank\":72},{\"text\":\"\u0120just\",\"logprob\":-7.22265625,\"rank\":146},{\"text\":\"\u0120give\",\"logprob\":-3.197265625,\"rank\":6},{\"text\":\"\u0120me\",\"logprob\":-0.4345703125,\"rank\":1},{\"text\":\"\u0120my\",\"logprob\":-3.30078125,\"rank\":6},{\"text\":\"\u0120money\",\"logprob\":-2.970703125,\"rank\":1},{\"text\":\"\u0120back\",\"logprob\":-0.470703125,\"rank\":1},{\"text\":\".'\",\"logprob\":-3.076171875,\"rank\":7},{\"text\":\"\u0120into\",\"logprob\":-7.015625,\"rank\":59},{\"text\":\"\u0120\",\"logprob\":-2.826171875,\"rank\":4},{\"text\":\"'\",\"logprob\":-0.8671875,\"rank\":1},{\"text\":\"compl\",\"logprob\":-3.986328125,\"rank\":8},{\"text\":\"aint\",\"logprob\":-0.08642578125,\"rank\":1},{\"text\":\"'\",\"logprob\":-0.6650390625,\"rank\":1},{\"text\":\"\u0120or\",\"logprob\":-0.314697265625,\"rank\":1},{\"text\":\"\u0120\",\"logprob\":-0.0396728515625,\"rank\":1},{\"text\":\"'\",\"logprob\":-0.0052337646484375,\"rank\":1},{\"text\":\"no\",\"logprob\":-3.81640625,\"rank\":6},{\"text\":\"\u0120complaint\",\"logprob\":-0.7177734375,\"rank\":1},{\"text\":\"':\",\"logprob\":-4.4453125,\"rank\":6},{\"text\":\"compl\",\"logprob\":-10.25,\"rank\":999},{\"text\":\"aint\",\"logprob\":-0.07476806640625,\"rank\":1}],\"seed\":4272011226}]}" + headers: + content-length: + - '1701' + content-type: + - application/json; charset=utf-8 + content-version: + - '2024-03-19' + date: + - Tue, 02 Apr 2024 16:30:52 GMT + keep-alive: + - timeout=72 + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_loglikelihood_raises_for_invalid_tokenization.yaml b/tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_loglikelihood_raises_for_invalid_tokenization.yaml new file mode 100644 index 00000000..4220e5ff --- /dev/null +++ b/tests/integration/extensions/cassettes/test_lm_eval/TestLMEval.test_loglikelihood_raises_for_invalid_tokenization.yaml @@ -0,0 +1,184 @@ +interactions: +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + method: GET + uri: https://api.com/v2/text/generation/limits?version=2023-11-22 + response: + body: + string: '{"result":{"concurrency":{"limit":10,"remaining":10}}}' + headers: + cache-control: + - private + content-length: + - '54' + content-type: + - application/json; charset=utf-8 + content-version: + - '2023-11-22' + date: + - Tue, 02 Apr 2024 16:48:38 GMT + keep-alive: + - timeout=72 + set-cookie: + - 2eef5f4c257f6bca76e8da5586743beb=f9a71fbb30e151859f32220315fc2bbc; path=/; + HttpOnly; Secure; SameSite=None + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"input": "The best ice cream flavor is:", "model_id": "tiiuae/falcon-40b", + "parameters": {"max_new_tokens": 1, "return_options": {"generated_tokens": false, + "input_text": false, "input_tokens": true, "token_logprobs": true, "token_ranks": + true}}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '247' + content-type: + - application/json + method: POST + uri: https://api.com/v2/text/generation?version=2024-03-19 + response: + body: + string: "{\"id\":\"4e3ada69-b108-4566-872d-a68173f99600\",\"model_id\":\"tiiuae/falcon-40b\",\"created_at\":\"2024-04-02T16:48:38.535Z\",\"results\":[{\"generated_text\":\"\\n\",\"generated_token_count\":1,\"input_token_count\":7,\"stop_reason\":\"max_tokens\",\"input_tokens\":[{\"text\":\"The\",\"logprob\":null},{\"text\":\"\u0120best\",\"logprob\":-5.19140625,\"rank\":7},{\"text\":\"\u0120ice\",\"logprob\":-7.53125,\"rank\":193},{\"text\":\"\u0120cream\",\"logprob\":-0.77978515625,\"rank\":1},{\"text\":\"\u0120flavor\",\"logprob\":-5.53515625,\"rank\":35},{\"text\":\"\u0120is\",\"logprob\":-1.68359375,\"rank\":2},{\"text\":\":\",\"logprob\":-3.53515625,\"rank\":8}],\"seed\":2666283366}]}" + headers: + content-length: + - '584' + content-type: + - application/json; charset=utf-8 + content-version: + - '2024-03-19' + date: + - Tue, 02 Apr 2024 16:48:38 GMT + keep-alive: + - timeout=72 + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"input": ["test str"], "model_id": "tiiuae/falcon-40b", "parameters": + {"return_options": {"tokens": true}}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '108' + content-type: + - application/json + method: POST + uri: https://api.com/v2/text/tokenization?version=2024-01-10 + response: + body: + string: "{\"model_id\":\"tiiuae/falcon-40b\",\"created_at\":\"2024-04-02T16:48:38.976Z\",\"results\":[{\"token_count\":2,\"tokens\":[\"test\",\"\u0120str\"]}]}" + headers: + content-length: + - '128' + content-type: + - application/json; charset=utf-8 + content-version: + - '2024-01-10' + date: + - Tue, 02 Apr 2024 16:48:38 GMT + keep-alive: + - timeout=72 + set-cookie: + - 2eef5f4c257f6bca76e8da5586743beb=e7012ef98dc1d6cddd80c399165de22f; path=/; + HttpOnly; Secure; SameSite=None + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + method: GET + uri: https://api.com/v2/text/generation/limits?version=2023-11-22 + response: + body: + string: '{"result":{"concurrency":{"limit":10,"remaining":10}}}' + headers: + cache-control: + - private + content-length: + - '54' + content-type: + - application/json; charset=utf-8 + content-version: + - '2023-11-22' + date: + - Tue, 02 Apr 2024 16:48:39 GMT + keep-alive: + - timeout=72 + set-cookie: + - 2eef5f4c257f6bca76e8da5586743beb=e7012ef98dc1d6cddd80c399165de22f; path=/; + HttpOnly; Secure; SameSite=None + vary: + - accept-encoding + status: + code: 200 + message: OK +- request: + body: '{"input": "test string", "model_id": "tiiuae/falcon-40b", "parameters": + {"max_new_tokens": 1, "return_options": {"generated_tokens": false, "input_text": + false, "input_tokens": true, "token_logprobs": true, "token_ranks": true}}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '229' + content-type: + - application/json + method: POST + uri: https://api.com/v2/text/generation?version=2024-03-19 + response: + body: + string: "{\"id\":\"b4d05ff1-f220-4af2-9b7b-53d35db60ea2\",\"model_id\":\"tiiuae/falcon-40b\",\"created_at\":\"2024-04-02T16:48:39.525Z\",\"results\":[{\"generated_text\":\" + \",\"generated_token_count\":1,\"input_token_count\":2,\"stop_reason\":\"max_tokens\",\"input_tokens\":[{\"text\":\"test\",\"logprob\":null},{\"text\":\"\u0120string\",\"logprob\":-10.2578125,\"rank\":1262}],\"seed\":3494838183}]}" + headers: + content-length: + - '346' + content-type: + - application/json; charset=utf-8 + content-version: + - '2024-03-19' + date: + - Tue, 02 Apr 2024 16:48:39 GMT + keep-alive: + - timeout=72 + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/tests/integration/extensions/test_lm_eval.py b/tests/integration/extensions/test_lm_eval.py new file mode 100644 index 00000000..d8d42e0e --- /dev/null +++ b/tests/integration/extensions/test_lm_eval.py @@ -0,0 +1,87 @@ +import pytest +from dotenv import load_dotenv +from lm_eval.api.instance import Instance + +from genai.extensions.lm_eval.model import IBMGenAILMEval +from genai.schema import DecodingMethod, TextGenerationParameters + + +@pytest.mark.integration +class TestLMEval: + @pytest.fixture(autouse=True) + def load_credentials(self): + load_dotenv() + + def test_create_from_arg_string_raises_without_model_id(self): + with pytest.raises(ValueError, match="'model_id' is required"): + IBMGenAILMEval.create_from_arg_string("temperature=0") + + def test_create_from_arg_string(self): + model = IBMGenAILMEval.create_from_arg_string( + "model_id=google/flan-t5-xl,temperature=0,top_k=10,decoding_method=greedy,max_new_tokens=42" + ) + assert model._parameters == TextGenerationParameters( + temperature=0, top_k=10, decoding_method=DecodingMethod.GREEDY, max_new_tokens=42 + ) + + @pytest.mark.vcr + def test_loglikelihood_raises_for_invalid_tokenization(self): + """Test loglikelihood of part of token is invalid""" + lm = IBMGenAILMEval(model_id="tiiuae/falcon-40b") + with pytest.raises(RuntimeError, match=r".*ends with a token that is substring of the continuation token"): + requests = [ + Instance(request_type="loglikelihood", doc=args, arguments=args, idx=i) + for i, args in enumerate([("test str", "ing")]) + ] + lm.loglikelihood(requests) + + @pytest.mark.vcr + def test_loglikelihood(self): + """Test loglikelihood of part of token is invalid""" + lm = IBMGenAILMEval(model_id="tiiuae/falcon-40b") + requests = [ + Instance(request_type="loglikelihood", doc=args, arguments=args, idx=i) + for i, args in enumerate( + [ + ( + "Classify the following tweet: 'No this is my first job' " + "into 'complaint' or 'no complaint':", + "no complaint", + ), + ( + "Classify the following tweet: 'Please just give me my money back.' " + "into 'complaint' or 'no complaint':", + "complaint", + ), + ] + ) + ] + results = lm.loglikelihood(requests) + assert len(results) == 2 + assert results[0].log_likelihood < 0, results[1].log_likelihood < 0 + + @pytest.mark.vcr + def test_generate_until(self): + lm = IBMGenAILMEval(model_id="google/flan-t5-xl") + requests = [ + Instance(request_type="loglikelihood", doc=args, arguments=args, idx=i) + for i, args in enumerate( + [ + ( + "Here are three sentences. My favorite color is ", + {"temperature": 1, "max_gen_toks": 1000, "until": "."}, + ), + ( + "Here are three sentences. I'm happy because ", + {"temperature": 0, "max_gen_toks": 1000, "until": "."}, + ), + ( + "Here are three sentences. When I'm bored, I ", + {"temperature": 1, "max_gen_toks": 1000, "until": "."}, + ), + ] + ) + ] + results = lm.generate_until(requests) + assert len(results) == 3 + assert {result[-1] for result in results} == {"."}