diff --git a/backend/handler/filesystem/roms_handler.py b/backend/handler/filesystem/roms_handler.py index 8fd0629d5..f53e32c84 100644 --- a/backend/handler/filesystem/roms_handler.py +++ b/backend/handler/filesystem/roms_handler.py @@ -112,9 +112,17 @@ def read_gz_file(file_path: Path) -> Iterator[bytes]: def read_7z_file(file_path: Path) -> Iterator[bytes]: try: with py7zr.SevenZipFile(file_path, "r") as f: - for _name, bio in f.readall().items(): - while chunk := bio.read(FILE_READ_CHUNK_SIZE): - yield chunk + for name in f.namelist(): + # TODO: This `read` call still reads the member file for this iteration into memory + # (but not the whole 7zip archive). This is because `py7zr` does not support + # streaming decompression yet. + # Related issue: https://github.com/miurahr/py7zr/issues/579 + for bio in f.read([name]).values(): + while chunk := bio.read(FILE_READ_CHUNK_SIZE): + yield chunk + # Extracting each file separately requires resetting file pointer and decompressor + # between `read` operations. + f.reset() except ( Bad7zFile, DecompressionError, diff --git a/poetry.lock b/poetry.lock index 98e5ffdf8..fdfa7c449 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1538,13 +1538,13 @@ tests = ["pytest"] [[package]] name = "py7zr" -version = "0.21.1" +version = "0.22.0" description = "Pure python 7-zip library" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "py7zr-0.21.1-py3-none-any.whl", hash = "sha256:57e5be6fafaa417fe93fa9c81f7f01bb579d3cfe1631f535a3e641200ac87dc2"}, - {file = "py7zr-0.21.1.tar.gz", hash = "sha256:dede8ed8b7b32b3586ac476da3a482b69dd433229420bf0f62c495404b72c799"}, + {file = "py7zr-0.22.0-py3-none-any.whl", hash = "sha256:993b951b313500697d71113da2681386589b7b74f12e48ba13cc12beca79d078"}, + {file = "py7zr-0.22.0.tar.gz", hash = "sha256:c6c7aea5913535184003b73938490f9a4d8418598e533f9ca991d3b8e45a139e"}, ] [package.dependencies] @@ -1563,7 +1563,7 @@ texttable = "*" check = ["black (>=23.1.0)", "check-manifest", "flake8 (<8)", "flake8-black (>=0.3.6)", "flake8-deprecated", "flake8-isort", "isort (>=5.0.3)", "lxml", "mypy (>=0.940)", "mypy-extensions (>=0.4.1)", "pygments", "readme-renderer", "twine", "types-psutil"] debug = ["pytest", "pytest-leaks", "pytest-profiling"] docs = ["docutils", "sphinx (>=5.0)", "sphinx-a4doc", "sphinx-py3doc-enhanced-theme"] -test = ["coverage[toml] (>=5.2)", "coveralls (>=2.1.1)", "py-cpuinfo", "pyannotate", "pytest", "pytest-benchmark", "pytest-cov", "pytest-remotedata", "pytest-timeout"] +test = ["coverage[toml] (>=5.2)", "coveralls (>=2.1.1)", "py-cpuinfo", "pytest", "pytest-benchmark", "pytest-cov", "pytest-remotedata", "pytest-timeout"] test-compat = ["libarchive-c"] [[package]] @@ -2880,7 +2880,6 @@ description = "Automatically mock your HTTP interactions to simplify and speed u optional = false python-versions = ">=3.8" files = [ - {file = "vcrpy-6.0.1-py2.py3-none-any.whl", hash = "sha256:621c3fb2d6bd8aa9f87532c688e4575bcbbde0c0afeb5ebdb7e14cac409edfdd"}, {file = "vcrpy-6.0.1.tar.gz", hash = "sha256:9e023fee7f892baa0bbda2f7da7c8ac51165c1c6e38ff8688683a12a4bde9278"}, ] @@ -3253,4 +3252,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "68a20b03af6c0f1bdf77a7f110c3b7cb331abf28ea90572bc6e24da85ae58825" +content-hash = "2893404f282f59d33a162c4d8410669f71a0bd125fb5c5c821d00f7db8627476" diff --git a/pyproject.toml b/pyproject.toml index 3a5977e53..dffc6f7a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ joserfc = "^0.9.0" pillow = "^10.3.0" certifi = "2024.07.04" python-magic = "^0.4.27" -py7zr = "^0.21.1" +py7zr = "^0.22" streaming-form-data = "^1.16.0" zipfile-deflate64 = "^0.2.0"