Merge pull request #1140 from rommapp/master

v3.5.0

.github/FUNDING.yml

@@ -1 +1 @@
-buy_me_a_coffee: zurdi15
+open_collective: romm

.github/workflows/build.yml

@@ -47,6 +47,7 @@ jobs:
           tags: |
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
           labels: |
             org.opencontainers.image.version={{version}}
             org.opencontainers.image.title="rommapp/romm"

.github/workflows/pytest.yml

@@ -42,10 +42,10 @@ jobs:
         run: |
           pipx install poetry
 
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           cache: "poetry"
 
       - name: Install dependencies

.gitignore

@@ -18,10 +18,13 @@ coverage
 /cypress/screenshots/
 
 # Editor directories and files
-.vscode
+.vscode/*
 !.vscode/extensions.json
 !.vscode/settings.json
 !.vscode/tasks.json
+.zed/*
+!.zed/settings.json
+pyrightconfig.json
 .idea
 *.suo
 *.ntvs*

.python-version

@@ -1 +1 @@
-3.11
+3.12

.trunk/trunk.yaml

@@ -2,53 +2,73 @@
 # To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
 version: 0.1
 cli:
-  version: 1.22.2
+  version: 1.22.3
 # Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
 plugins:
   sources:
     - id: trunk
-      ref: v1.6.0
+      ref: v1.6.1
       uri: https://github.com/trunk-io/plugins
 # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
 runtimes:
   enabled:
     - go@1.21.0
     - node@18.12.1
-    - python@3.11.6
+    - python@3.12.2
 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
 lint:
   enabled:
     - markdownlint@0.41.0
-    - eslint@9.6.0
+    - eslint@9.9.0
     - actionlint@1.7.1
     - bandit@1.7.9
-    - black@24.4.2
-    - checkov@3.2.178
+    - black@24.8.0
+    - checkov@3.2.228
     - git-diff-check
     - isort@5.13.2
-    - mypy@1.10.1
-    - osv-scanner@1.8.1
-    - oxipng@9.1.1
-    - prettier@3.3.2
-    - ruff@0.5.1
+    - mypy@1.11.1
+    - osv-scanner@1.8.3
+    - oxipng@9.1.2
+    - prettier@3.3.3
+    - ruff@0.6.0
     - shellcheck@0.10.0
     - shfmt@3.6.0
     - svgo@3.3.2
-    - taplo@0.8.1
-    - trivy@0.52.2
-    - trufflehog@3.79.0
+    - taplo@0.9.3
+    - trivy@0.54.1
+    - trufflehog@3.81.9
     - yamllint@1.35.1
   ignore:
     - linters: [ALL]
       paths:
         - frontend/src/__generated__/**
         - docker/Dockerfile
+        - docker/nginx/js/**
+  files:
+    - name: vue
+      extensions: [vue]
   definitions:
     - name: eslint
-      files: [typescript, javascript]
+      files:
+        - javascript
+        - typescript
       commands:
         - name: lint
           run_from: ${root_or_parent_with_any_config}
+    - name: prettier
+      files:
+        - typescript
+        - yaml
+        - css
+        - postcss
+        - sass
+        - html
+        - markdown
+        - json
+        - javascript
+        - graphql
+        - vue
+        - prettier_supported_configs
 actions:
   disabled:
     - trunk-check-pre-push

.zed/settings.json

@@ -0,0 +1,16 @@
+{
+  "languages": {
+    "Python": {
+      "tab_size": 4
+    },
+    "Vue.js": {
+      "tab_size": 2,
+      "formatter": {
+        "external": {
+          "command": "prettier",
+          "arguments": ["--stdin-filepath", "{buffer_path}"]
+        }
+      }
+    }
+  }
+}

backend/alembic/versions/0024_sibling_roms_db_view.py

@@ -0,0 +1,68 @@
+"""empty message
+
+Revision ID: 0024_sibling_roms_db_view
+Revises: 0023_make_columns_non_nullable
+Create Date: 2024-08-08 12:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0024_sibling_roms_db_view"
+down_revision = "0023_make_columns_non_nullable"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("roms", schema=None) as batch_op:
+        batch_op.create_index("idx_roms_igdb_id", ["igdb_id"])
+        batch_op.create_index("idx_roms_moby_id", ["moby_id"])
+
+    connection = op.get_bind()
+
+    connection.execute(
+        sa.text(
+            """
+            CREATE VIEW sibling_roms AS
+            SELECT
+                r1.id AS rom_id,
+                r2.id AS sibling_rom_id,
+                r1.platform_id AS platform_id,
+                NOW() AS created_at,
+                NOW() AS updated_at,
+                CASE WHEN r1.igdb_id <=> r2.igdb_id THEN r1.igdb_id END AS igdb_id,
+                CASE WHEN r1.moby_id <=> r2.moby_id THEN r1.moby_id END AS moby_id
+            FROM
+                roms r1
+            JOIN 
+                roms r2 
+            ON 
+                r1.platform_id = r2.platform_id
+                AND r1.id != r2.id
+                AND (
+                    (r1.igdb_id = r2.igdb_id AND r1.igdb_id IS NOT NULL AND r1.igdb_id != '')
+                    OR
+                    (r1.moby_id = r2.moby_id AND r1.moby_id IS NOT NULL AND r1.moby_id != '')
+                );
+            """
+        ),
+    )
+
+
+def downgrade() -> None:
+    connection = op.get_bind()
+
+    connection.execute(
+        sa.text(
+            """
+            DROP VIEW sibling_roms;
+            """
+        ),
+    )
+
+    with op.batch_alter_table("roms", schema=None) as batch_op:
+        batch_op.drop_index("idx_roms_igdb_id")
+        batch_op.drop_index("idx_roms_moby_id")

backend/alembic/versions/0025_roms_hashes.py

@@ -0,0 +1,46 @@
+"""empty message
+
+Revision ID: 0025_roms_hashes
+Revises: 0024_sibling_roms_db_view
+Create Date: 2024-08-11 21:50:53.301352
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from config import IS_PYTEST_RUN, SCAN_TIMEOUT
+from endpoints.sockets.scan import scan_platforms
+from handler.redis_handler import high_prio_queue
+from handler.scan_handler import ScanType
+
+# revision identifiers, used by Alembic.
+revision = "0025_roms_hashes"
+down_revision = "0024_sibling_roms_db_view"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("roms", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("crc_hash", sa.String(length=100), nullable=True))
+        batch_op.add_column(sa.Column("md5_hash", sa.String(length=100), nullable=True))
+        batch_op.add_column(
+            sa.Column("sha1_hash", sa.String(length=100), nullable=True)
+        )
+
+    # Run a no-scan in the background on migrate
+    if not IS_PYTEST_RUN:
+        high_prio_queue.enqueue(
+            scan_platforms, [], ScanType.QUICK, [], [], job_timeout=SCAN_TIMEOUT
+        )
+
+        high_prio_queue.enqueue(
+            scan_platforms, [], ScanType.HASHES, [], [], job_timeout=SCAN_TIMEOUT
+        )
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("roms", schema=None) as batch_op:
+        batch_op.drop_column("sha1_hash")
+        batch_op.drop_column("md5_hash")
+        batch_op.drop_column("crc_hash")

backend/config/__init__.py

@@ -6,10 +6,14 @@
 
 load_dotenv()
 
+
+def str_to_bool(value: str) -> bool:
+    return value.lower() in ("true", "1")
+
+
 # GUNICORN
 DEV_PORT: Final = int(os.environ.get("VITE_BACKEND_DEV_PORT", "5000"))
 DEV_HOST: Final = "127.0.0.1"
-ROMM_HOST: Final = os.environ.get("ROMM_HOST", DEV_HOST)
 GUNICORN_WORKERS: Final = int(os.environ.get("GUNICORN_WORKERS", 2))
 
 # PATHS
@@ -54,35 +58,41 @@
 ROMM_AUTH_SECRET_KEY: Final = os.environ.get(
     "ROMM_AUTH_SECRET_KEY", secrets.token_hex(32)
 )
-DISABLE_CSRF_PROTECTION = os.environ.get("DISABLE_CSRF_PROTECTION", "false") == "true"
-DISABLE_DOWNLOAD_ENDPOINT_AUTH = (
-    os.environ.get("DISABLE_DOWNLOAD_ENDPOINT_AUTH", "false") == "true"
+DISABLE_CSRF_PROTECTION = str_to_bool(
+    os.environ.get("DISABLE_CSRF_PROTECTION", "false")
+)
+DISABLE_DOWNLOAD_ENDPOINT_AUTH = str_to_bool(
+    os.environ.get("DISABLE_DOWNLOAD_ENDPOINT_AUTH", "false")
 )
 
 # SCANS
 SCAN_TIMEOUT: Final = int(os.environ.get("SCAN_TIMEOUT", 60 * 60 * 4))  # 4 hours
 
 # TASKS
-ENABLE_RESCAN_ON_FILESYSTEM_CHANGE: Final = (
-    os.environ.get("ENABLE_RESCAN_ON_FILESYSTEM_CHANGE", "false") == "true"
+ENABLE_RESCAN_ON_FILESYSTEM_CHANGE: Final = str_to_bool(
+    os.environ.get("ENABLE_RESCAN_ON_FILESYSTEM_CHANGE", "false")
 )
 RESCAN_ON_FILESYSTEM_CHANGE_DELAY: Final = int(
     os.environ.get("RESCAN_ON_FILESYSTEM_CHANGE_DELAY", 5)  # 5 minutes
 )
-ENABLE_SCHEDULED_RESCAN: Final = (
-    os.environ.get("ENABLE_SCHEDULED_RESCAN", "false") == "true"
+ENABLE_SCHEDULED_RESCAN: Final = str_to_bool(
+    os.environ.get("ENABLE_SCHEDULED_RESCAN", "false")
 )
 SCHEDULED_RESCAN_CRON: Final = os.environ.get(
     "SCHEDULED_RESCAN_CRON",
     "0 3 * * *",  # At 3:00 AM every day
 )
-ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: Final = (
-    os.environ.get("ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB", "false") == "true"
+ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: Final = str_to_bool(
+    os.environ.get("ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB", "false")
 )
 SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: Final = os.environ.get(
     "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON",
     "0 4 * * *",  # At 4:00 AM every day
 )
 
+# EMULATION
+DISABLE_EMULATOR_JS = str_to_bool(os.environ.get("DISABLE_EMULATOR_JS", "false"))
+DISABLE_RUFFLE_RS = str_to_bool(os.environ.get("DISABLE_RUFFLE_RS", "false"))
+
 # TESTING
 IS_PYTEST_RUN: Final = bool(os.environ.get("PYTEST_VERSION", False))

backend/config/config_manager.py

@@ -2,7 +2,6 @@
 import sys
 from pathlib import Path
 from typing import Final
-from urllib.parse import quote_plus
 
 import pydash
 import yaml
@@ -21,6 +20,7 @@
     ConfigNotWritableException,
 )
 from logger.logger import log
+from sqlalchemy import URL
 from yaml.loader import SafeLoader
 
 ROMM_USER_CONFIG_PATH: Final = f"{ROMM_BASE_PATH}/config"
@@ -76,7 +76,7 @@ def __init__(self, config_file: str = ROMM_USER_CONFIG_FILE):
             sys.exit(5)
 
     @staticmethod
-    def get_db_engine() -> str:
+    def get_db_engine() -> URL:
         """Builds the database connection string depending on the defined database in the config.yml file
 
         Returns:
@@ -90,9 +90,13 @@ def get_db_engine() -> str:
                 )
                 sys.exit(3)
 
-            return (
-                f"mariadb+mariadbconnector://{DB_USER}:%s@{DB_HOST}:{DB_PORT}/{DB_NAME}"
-                % quote_plus(DB_PASSWD)
+            return URL.create(
+                drivername="mariadb+mariadbconnector",
+                username=DB_USER,
+                password=DB_PASSWD,
+                host=DB_HOST,
+                port=DB_PORT,
+                database=DB_NAME,
             )
 
         # DEPRECATED

backend/decorators/database.py

@@ -1,6 +1,7 @@
 import functools
 
 from fastapi import HTTPException, status
+from handler.database.base_handler import sync_session
 from logger.logger import log
 from sqlalchemy.exc import ProgrammingError
 
@@ -12,7 +13,7 @@ def wrapper(*args, **kwargs):
             return func(*args, **kwargs)
 
         try:
-            with args[0].session.begin() as s:
+            with sync_session.begin() as s:
                 kwargs["session"] = s
                 return func(*args, **kwargs)
         except ProgrammingError as exc: