From cf087b3a07cb5fa7a89420adeaf3c9f7596d10af Mon Sep 17 00:00:00 2001 From: Michael Manganiello Date: Sun, 22 Sep 2024 16:16:17 -0300 Subject: [PATCH] feat: Add RAHasher and util to calculate RetroAchievements hashes Build and include the `RAHasher` binary in the Docker image, to calculate hashes for RetroAchievements. Also, add a service to run `RAHasher` from Python. Example usage: ```python from adapters.services.rahasher import RAHasherError, RAHasherService rahasher = RAHasherService() try: hash = await rahasher.calculate_hash("nes", Path("path/to/rom.nes")) except RAHasherError: # Handle error hash = None ``` --- backend/adapters/__init__.py | 0 backend/adapters/services/__init__.py | 0 backend/adapters/services/rahasher.py | 121 ++++++++++++++++++++++++++ docker/Dockerfile | 46 ++++++++-- 4 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 backend/adapters/__init__.py create mode 100644 backend/adapters/services/__init__.py create mode 100644 backend/adapters/services/rahasher.py diff --git a/backend/adapters/__init__.py b/backend/adapters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/adapters/services/__init__.py b/backend/adapters/services/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/adapters/services/rahasher.py b/backend/adapters/services/rahasher.py new file mode 100644 index 000000000..87a1ff467 --- /dev/null +++ b/backend/adapters/services/rahasher.py @@ -0,0 +1,121 @@ +import asyncio +import re +from pathlib import Path + +from logger.logger import log + +RAHASHER_VALID_HASH_REGEX = re.compile(r"^[0-9a-f]{32}$") + +# TODO: Centralize standarized platform slugs using StrEnum. +PLATFORM_SLUG_TO_RETROACHIEVEMENTS_ID: dict[str, int] = { + "3do": 43, + "cpc": 37, + "acpc": 37, + "apple2": 38, + "appleii": 38, + "arcade": 27, + "arcadia-2001": 73, + "arduboy": 71, + "atari-2600": 25, + "atari2600": 25, + "atari-7800": 51, + "atari7800": 51, + "atari-jaguar-cd": 77, + "colecovision": 44, + "dreamcast": 40, + "dc": 40, + "gameboy": 4, + "gb": 4, + "gameboy-advance": 5, + "gba": 5, + "gameboy-color": 6, + "gbc": 6, + "game-gear": 15, + "gamegear": 15, + "gamecube": 16, + "ngc": 14, + "genesis": 1, + "genesis-slash-megadrive": 16, + "intellivision": 45, + "jaguar": 17, + "lynx": 13, + "msx": 29, + "mega-duck-slash-cougar-boy": 69, + "nes": 7, + "famicom": 7, + "neo-geo-cd": 56, + "neo-geo-pocket": 14, + "neo-geo-pocket-color": 14, + "n64": 2, + "nintendo-ds": 18, + "nds": 18, + "nintendo-dsi": 78, + "odyssey-2": 23, + "pc-8000": 47, + "pc-8800-series": 47, + "pc-fx": 49, + "psp": 41, + "playstation": 12, + "ps": 12, + "ps2": 21, + "pokemon-mini": 24, + "saturn": 39, + "sega-32x": 10, + "sega32": 10, + "sega-cd": 9, + "segacd": 9, + "sega-master-system": 11, + "sms": 11, + "sg-1000": 33, + "snes": 3, + "turbografx-cd": 76, + "turbografx-16-slash-pc-engine-cd": 76, + "turbo-grafx": 8, + "turbografx16--1": 8, + "vectrex": 26, + "virtual-boy": 28, + "virtualboy": 28, + "watara-slash-quickshot-supervision": 63, + "wonderswan": 53, + "wonderswan-color": 53, +} + + +class RAHasherError(Exception): ... + + +class RAHasherService: + """Service to calculate RetroAchievements hashes using RAHasher.""" + + async def calculate_hash(self, platform_slug: str, file_path: Path) -> str: + platform_id = PLATFORM_SLUG_TO_RETROACHIEVEMENTS_ID.get(platform_slug) + if not platform_id: + raise ValueError( + f"Platform not supported by RetroAchievements. {platform_slug=}" + ) + + args = (str(platform_id), str(file_path)) + log.debug("Executing RAHasher with args: %s", args) + + proc = await asyncio.create_subprocess_exec( + "RAHasher", + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + return_code = await proc.wait() + if return_code != 1: + stderr = (await proc.stderr.read()).decode("utf-8") + raise RAHasherError(f"RAHasher failed with code {return_code}. {stderr=}") + + file_hash = (await proc.stdout.read()).decode("utf-8").strip() + if not file_hash: + raise RAHasherError( + f"RAHasher returned an empty hash. {platform_id=}, {file_path=}" + ) + if not RAHASHER_VALID_HASH_REGEX.match(file_hash): + raise RAHasherError( + f"RAHasher returned an invalid hash: {file_hash=}, {platform_id=}, {file_path=}" + ) + + return file_hash diff --git a/docker/Dockerfile b/docker/Dockerfile index 2ee544689..ffadd8c3f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,9 +1,18 @@ +# Stages: +# - front-build-stage: Build frontend +# - backend-build: Build backend environment +# - rahasher-build: Build RAHasher +# - nginx-build: Build nginx modules +# - production-stage: Setup frontend and backend +# - final-stage: Move everything to final stage + +# Versions: ARG ALPINE_VERSION=3.20 ARG NGINX_VERSION=1.27.1 ARG NODE_VERSION=20.16 ARG PYTHON_VERSION=3.12 -# Build frontend + FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} AS front-build-stage WORKDIR /front @@ -13,7 +22,7 @@ RUN npm ci COPY ./frontend ./ RUN npm run build -# Build backend environment + FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} AS backend-build # libffi-dev is needed to fix poetry dependencies for >= v1.8 on arm64 @@ -31,9 +40,33 @@ ENV POETRY_NO_INTERACTION=1 \ WORKDIR /src COPY ./pyproject.toml ./poetry.lock /src/ -RUN poetry install --no-ansi --no-cache --only main +RUN poetry install --no-ansi --no-cache + + +# TODO: Upgrade Alpine to the same version as the other stages, when RAHasher is updated to work +# with it (seems like Alpine 3.18's g++ v12 is the latest version that works with RAHasher, +# while g++ v13 fails to compile it). +FROM alpine:3.18 AS rahasher-build + +RUN apk add --no-cache \ + g++ \ + git \ + linux-headers \ + make \ + zlib-dev + +# TODO: Change to a tagged version, once v1.7.2 or newer is released. +# Current pinned commit is needed to enable building RAHasher on ARM64. +ARG RALIBRETRO_SHA=5b60a1d6d067238ece378b6250ae1ae8aeb90904 + +# TODO: Remove `sed` command when RAHasher can be compiled without it. +RUN git clone --recursive https://github.com/RetroAchievements/RALibretro.git && \ + cd ./RALibretro && \ + git checkout "${RALIBRETRO_SHA}" && \ + sed -i '22a #include ' ./src/Util.h && \ + make HAVE_CHD=1 -f ./Makefile.RAHasher + -# Build nginx modules FROM alpine:${ALPINE_VERSION} AS nginx-build RUN apk add --no-cache \ @@ -63,10 +96,11 @@ RUN git clone https://github.com/evanmiller/mod_zip.git && \ make -f ./objs/Makefile modules && \ chmod 644 ./objs/ngx_http_zip_module.so -# Setup frontend and backend + FROM nginx:${NGINX_VERSION}-alpine${ALPINE_VERSION} AS production-stage ARG WEBSERVER_FOLDER=/var/www/html +COPY --from=rahasher-build /RALibretro/bin64/RAHasher /usr/bin/RAHasher COPY --from=nginx-build ./nginx/objs/ngx_http_zip_module.so /usr/lib/nginx/modules/ COPY --from=front-build-stage /front/dist ${WEBSERVER_FOLDER} @@ -101,7 +135,7 @@ COPY ./docker/nginx/default.conf /etc/nginx/nginx.conf RUN addgroup -g 1000 -S romm && adduser -u 1000 -D -S -G romm romm && \ mkdir /romm /redis-data && chown romm:romm /romm /redis-data -# Move everything to final stage + FROM scratch AS final-stage COPY --from=production-stage / /