Skip to content

Commit

Permalink
feat: Add RAHasher and util to calculate RetroAchievements hashes
Browse files Browse the repository at this point in the history
Build and include the `RAHasher` binary in the Docker image, to
calculate hashes for RetroAchievements. Also, add a service to
run `RAHasher` from Python.

Example usage:

```python
from adapters.services.rahasher import RAHasherError, RAHasherService

rahasher = RAHasherService()
try:
    hash = await rahasher.calculate_hash("nes", Path("path/to/rom.nes"))
except RAHasherError:
    # Handle error
    hash = None
```
  • Loading branch information
adamantike committed Sep 22, 2024
1 parent 8a13611 commit cf087b3
Show file tree
Hide file tree
Showing 4 changed files with 161 additions and 6 deletions.
Empty file added backend/adapters/__init__.py
Empty file.
Empty file.
121 changes: 121 additions & 0 deletions backend/adapters/services/rahasher.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import asyncio
import re
from pathlib import Path

from logger.logger import log

RAHASHER_VALID_HASH_REGEX = re.compile(r"^[0-9a-f]{32}$")

# TODO: Centralize standarized platform slugs using StrEnum.
PLATFORM_SLUG_TO_RETROACHIEVEMENTS_ID: dict[str, int] = {
"3do": 43,
"cpc": 37,
"acpc": 37,
"apple2": 38,
"appleii": 38,
"arcade": 27,
"arcadia-2001": 73,
"arduboy": 71,
"atari-2600": 25,
"atari2600": 25,
"atari-7800": 51,
"atari7800": 51,
"atari-jaguar-cd": 77,
"colecovision": 44,
"dreamcast": 40,
"dc": 40,
"gameboy": 4,
"gb": 4,
"gameboy-advance": 5,
"gba": 5,
"gameboy-color": 6,
"gbc": 6,
"game-gear": 15,
"gamegear": 15,
"gamecube": 16,
"ngc": 14,
"genesis": 1,
"genesis-slash-megadrive": 16,
"intellivision": 45,
"jaguar": 17,
"lynx": 13,
"msx": 29,
"mega-duck-slash-cougar-boy": 69,
"nes": 7,
"famicom": 7,
"neo-geo-cd": 56,
"neo-geo-pocket": 14,
"neo-geo-pocket-color": 14,
"n64": 2,
"nintendo-ds": 18,
"nds": 18,
"nintendo-dsi": 78,
"odyssey-2": 23,
"pc-8000": 47,
"pc-8800-series": 47,
"pc-fx": 49,
"psp": 41,
"playstation": 12,
"ps": 12,
"ps2": 21,
"pokemon-mini": 24,
"saturn": 39,
"sega-32x": 10,
"sega32": 10,
"sega-cd": 9,
"segacd": 9,
"sega-master-system": 11,
"sms": 11,
"sg-1000": 33,
"snes": 3,
"turbografx-cd": 76,
"turbografx-16-slash-pc-engine-cd": 76,
"turbo-grafx": 8,
"turbografx16--1": 8,
"vectrex": 26,
"virtual-boy": 28,
"virtualboy": 28,
"watara-slash-quickshot-supervision": 63,
"wonderswan": 53,
"wonderswan-color": 53,
}


class RAHasherError(Exception): ...


class RAHasherService:
"""Service to calculate RetroAchievements hashes using RAHasher."""

async def calculate_hash(self, platform_slug: str, file_path: Path) -> str:
platform_id = PLATFORM_SLUG_TO_RETROACHIEVEMENTS_ID.get(platform_slug)
if not platform_id:
raise ValueError(
f"Platform not supported by RetroAchievements. {platform_slug=}"
)

args = (str(platform_id), str(file_path))
log.debug("Executing RAHasher with args: %s", args)

proc = await asyncio.create_subprocess_exec(
"RAHasher",
*args,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
return_code = await proc.wait()
if return_code != 1:
stderr = (await proc.stderr.read()).decode("utf-8")

Check failure on line 108 in backend/adapters/services/rahasher.py

View check run for this annotation

Trunk.io / Trunk Check

mypy(union-attr)

[new] Item "None" of "StreamReader | None" has no attribute "read"

Check failure on line 108 in backend/adapters/services/rahasher.py

View workflow job for this annotation

GitHub Actions / Trunk Check

mypy(union-attr)

[new] Item "None" of "StreamReader | None" has no attribute "read"
raise RAHasherError(f"RAHasher failed with code {return_code}. {stderr=}")

file_hash = (await proc.stdout.read()).decode("utf-8").strip()

Check failure on line 111 in backend/adapters/services/rahasher.py

View check run for this annotation

Trunk.io / Trunk Check

mypy(union-attr)

[new] Item "None" of "StreamReader | None" has no attribute "read"

Check failure on line 111 in backend/adapters/services/rahasher.py

View workflow job for this annotation

GitHub Actions / Trunk Check

mypy(union-attr)

[new] Item "None" of "StreamReader | None" has no attribute "read"
if not file_hash:
raise RAHasherError(
f"RAHasher returned an empty hash. {platform_id=}, {file_path=}"
)
if not RAHASHER_VALID_HASH_REGEX.match(file_hash):
raise RAHasherError(
f"RAHasher returned an invalid hash: {file_hash=}, {platform_id=}, {file_path=}"
)

return file_hash
46 changes: 40 additions & 6 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,9 +1,18 @@
# Stages:
# - front-build-stage: Build frontend
# - backend-build: Build backend environment
# - rahasher-build: Build RAHasher
# - nginx-build: Build nginx modules
# - production-stage: Setup frontend and backend
# - final-stage: Move everything to final stage

# Versions:
ARG ALPINE_VERSION=3.20
ARG NGINX_VERSION=1.27.1
ARG NODE_VERSION=20.16
ARG PYTHON_VERSION=3.12

# Build frontend

FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} AS front-build-stage
WORKDIR /front

Expand All @@ -13,7 +22,7 @@ RUN npm ci
COPY ./frontend ./
RUN npm run build

# Build backend environment

FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} AS backend-build

# libffi-dev is needed to fix poetry dependencies for >= v1.8 on arm64
Expand All @@ -31,9 +40,33 @@ ENV POETRY_NO_INTERACTION=1 \
WORKDIR /src

COPY ./pyproject.toml ./poetry.lock /src/
RUN poetry install --no-ansi --no-cache --only main
RUN poetry install --no-ansi --no-cache


# TODO: Upgrade Alpine to the same version as the other stages, when RAHasher is updated to work
# with it (seems like Alpine 3.18's g++ v12 is the latest version that works with RAHasher,
# while g++ v13 fails to compile it).
FROM alpine:3.18 AS rahasher-build

RUN apk add --no-cache \
g++ \
git \
linux-headers \
make \
zlib-dev

# TODO: Change to a tagged version, once v1.7.2 or newer is released.
# Current pinned commit is needed to enable building RAHasher on ARM64.
ARG RALIBRETRO_SHA=5b60a1d6d067238ece378b6250ae1ae8aeb90904

# TODO: Remove `sed` command when RAHasher can be compiled without it.
RUN git clone --recursive https://github.com/RetroAchievements/RALibretro.git && \
cd ./RALibretro && \
git checkout "${RALIBRETRO_SHA}" && \
sed -i '22a #include <ctime>' ./src/Util.h && \
make HAVE_CHD=1 -f ./Makefile.RAHasher


# Build nginx modules
FROM alpine:${ALPINE_VERSION} AS nginx-build

RUN apk add --no-cache \
Expand Down Expand Up @@ -63,10 +96,11 @@ RUN git clone https://github.com/evanmiller/mod_zip.git && \
make -f ./objs/Makefile modules && \
chmod 644 ./objs/ngx_http_zip_module.so

# Setup frontend and backend

FROM nginx:${NGINX_VERSION}-alpine${ALPINE_VERSION} AS production-stage
ARG WEBSERVER_FOLDER=/var/www/html

COPY --from=rahasher-build /RALibretro/bin64/RAHasher /usr/bin/RAHasher
COPY --from=nginx-build ./nginx/objs/ngx_http_zip_module.so /usr/lib/nginx/modules/

COPY --from=front-build-stage /front/dist ${WEBSERVER_FOLDER}
Expand Down Expand Up @@ -101,7 +135,7 @@ COPY ./docker/nginx/default.conf /etc/nginx/nginx.conf
RUN addgroup -g 1000 -S romm && adduser -u 1000 -D -S -G romm romm && \
mkdir /romm /redis-data && chown romm:romm /romm /redis-data

# Move everything to final stage

FROM scratch AS final-stage

COPY --from=production-stage / /
Expand Down

0 comments on commit cf087b3

Please sign in to comment.