Merge pull request #1226 from rommapp/fix/improve-memory-usage-7z-dec…

…ompression

fix: Improve memory usage during 7zip decompression

backend/handler/filesystem/roms_handler.py

@@ -112,9 +112,17 @@ def read_gz_file(file_path: Path) -> Iterator[bytes]:
 def read_7z_file(file_path: Path) -> Iterator[bytes]:
     try:
         with py7zr.SevenZipFile(file_path, "r") as f:
-            for _name, bio in f.readall().items():
-                while chunk := bio.read(FILE_READ_CHUNK_SIZE):
-                    yield chunk
+            for name in f.namelist():
+                # TODO: This `read` call still reads the member file for this iteration into memory
+                #       (but not the whole 7zip archive). This is because `py7zr` does not support
+                #       streaming decompression yet.
+                #       Related issue: https://github.com/miurahr/py7zr/issues/579
+                for bio in f.read([name]).values():
+                    while chunk := bio.read(FILE_READ_CHUNK_SIZE):
+                        yield chunk
+                # Extracting each file separately requires resetting file pointer and decompressor
+                # between `read` operations.
+                f.reset()
     except (
         Bad7zFile,
         DecompressionError,

pyproject.toml

@@ -41,7 +41,7 @@ joserfc = "^0.9.0"
 pillow = "^10.3.0"
 certifi = "2024.07.04"
 python-magic = "^0.4.27"
-py7zr = "^0.21.1"
+py7zr = "^0.22"
 streaming-form-data = "^1.16.0"
 zipfile-deflate64 = "^0.2.0"