Improve speed of fetching siblings for roms

backend/alembic/versions/0024_sibling_roms_db_view.py

@@ -0,0 +1,68 @@
+"""empty message
+
+Revision ID: 0024_sibling_roms_db_view
+Revises: 0023_make_columns_non_nullable
+Create Date: 2024-08-08 12:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0024_sibling_roms_db_view"
+down_revision = "0023_make_columns_non_nullable"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("roms", schema=None) as batch_op:
+        batch_op.create_index("idx_roms_igdb_id", ["igdb_id"])
+        batch_op.create_index("idx_roms_moby_id", ["moby_id"])
+
+    connection = op.get_bind()
+
+    connection.execute(
+        sa.text(
+            """
+            CREATE VIEW sibling_roms AS
+            SELECT
+                r1.id AS rom_id,
+                r2.id AS sibling_rom_id,
+                r1.platform_id AS platform_id,
+                COALESCE(r1.igdb_id, r2.igdb_id) AS igdb_id,
+                COALESCE(r1.moby_id, r2.moby_id) AS moby_id,
+                NOW() AS created_at,
+                NOW() AS updated_at
+            FROM
+                roms r1
+            JOIN 
+                roms r2 
+            ON 
+                r1.platform_id = r2.platform_id
+                AND r1.id != r2.id
+                AND (
+                    (r1.igdb_id = r2.igdb_id AND r1.igdb_id IS NOT NULL AND r1.igdb_id != '')
+                    OR
+                    (r1.moby_id = r2.moby_id AND r1.moby_id IS NOT NULL AND r1.moby_id != '')
+                );
+            """
+        ),
+    )
+
+
+def downgrade() -> None:
+    connection = op.get_bind()
+
+    connection.execute(
+        sa.text(
+            """
+            DROP VIEW sibling_roms;
+            """
+        ),
+    )
+
+    with op.batch_alter_table("roms", schema=None) as batch_op:
+        batch_op.drop_index("idx_roms_igdb_id")
+        batch_op.drop_index("idx_roms_moby_id")

backend/endpoints/responses/rom.py

@@ -113,24 +113,9 @@ class RomSchema(BaseModel):
     created_at: datetime
     updated_at: datetime
 
-    rom_user: RomUserSchema | None = Field(default=None)
-    sibling_roms: list[RomSchema] = Field(default_factory=list)
-
     class Config:
         from_attributes = True
 
-    @classmethod
-    def from_orm_with_request(cls, db_rom: Rom, request: Request) -> RomSchema:
-        rom = cls.model_validate(db_rom)
-        user_id = request.user.id
-
-        rom.rom_user = RomUserSchema.for_user(user_id, db_rom)
-        rom.sibling_roms = [
-            RomSchema.model_validate(r) for r in db_rom.get_sibling_roms()
-        ]
-
-        return rom
-
     @computed_field  # type: ignore
     @property
     def sort_comparator(self) -> str:
@@ -144,10 +129,24 @@ def sort_comparator(self) -> str:
         )
 
 
+class SimpleRomSchema(RomSchema):
+    sibling_roms: list[RomSchema] = Field(default_factory=list)
+    rom_user: RomUserSchema | None = Field(default=None)
+
+    @classmethod
+    def from_orm_with_request(cls, db_rom: Rom, request: Request) -> SimpleRomSchema:
+        rom = cls.model_validate(db_rom)
+        user_id = request.user.id
+
+        rom.rom_user = RomUserSchema.for_user(user_id, db_rom)
+
+        return rom
+
+
 class DetailedRomSchema(RomSchema):
     merged_screenshots: list[str]
-    rom_user: RomUserSchema | None = Field(default=None)
     sibling_roms: list[RomSchema] = Field(default_factory=list)
+    rom_user: RomUserSchema | None = Field(default=None)
     user_saves: list[SaveSchema] = Field(default_factory=list)
     user_states: list[StateSchema] = Field(default_factory=list)
     user_screenshots: list[ScreenshotSchema] = Field(default_factory=list)
@@ -161,9 +160,6 @@ def from_orm_with_request(cls, db_rom: Rom, request: Request) -> DetailedRomSche
 
         rom.rom_user = RomUserSchema.for_user(user_id, db_rom)
         rom.user_notes = RomUserSchema.notes_for_user(user_id, db_rom)
-        rom.sibling_roms = [
-            RomSchema.model_validate(r) for r in db_rom.get_sibling_roms()
-        ]
         rom.user_saves = [
             SaveSchema.model_validate(s) for s in db_rom.saves if s.user_id == user_id
         ]

backend/endpoints/rom.py

@@ -17,8 +17,8 @@
     AddRomsResponse,
     CustomStreamingResponse,
     DetailedRomSchema,
-    RomSchema,
     RomUserSchema,
+    SimpleRomSchema,
 )
 from exceptions.endpoint_exceptions import RomNotFoundInDatabaseException
 from exceptions.fs_exceptions import RomAlreadyExistsException
@@ -102,15 +102,15 @@ def get_roms(
     limit: int | None = None,
     order_by: str = "name",
     order_dir: str = "asc",
-) -> list[RomSchema]:
+) -> list[SimpleRomSchema]:
     """Get roms endpoint
 
     Args:
         request (Request): Fastapi Request object
         id (int, optional): Rom internal id
 
     Returns:
-        list[RomSchema]: List of roms stored in the database
+        list[SimpleRomSchema]: List of roms stored in the database
     """
 
     roms = db_rom_handler.get_roms(
@@ -122,7 +122,7 @@ def get_roms(
         limit=limit,
     )
 
-    return [RomSchema.from_orm_with_request(rom, request) for rom in roms]
+    return [SimpleRomSchema.from_orm_with_request(rom, request) for rom in roms]
 
 
 @protected_route(

backend/endpoints/sockets/scan.py

@@ -7,7 +7,7 @@
 import socketio  # type: ignore
 from config import SCAN_TIMEOUT
 from endpoints.responses.platform import PlatformSchema
-from endpoints.responses.rom import RomSchema
+from endpoints.responses.rom import SimpleRomSchema
 from exceptions.fs_exceptions import (
     FirmwareNotFoundException,
     FolderStructureNotMatchException,
@@ -361,7 +361,7 @@ async def _identify_rom(
         {
             "platform_name": platform.name,
             "platform_slug": platform.slug,
-            **RomSchema.model_validate(_added_rom).model_dump(
+            **SimpleRomSchema.model_validate(_added_rom).model_dump(
                 exclude={"created_at", "updated_at", "rom_user"}
             ),
         },

backend/handler/database/roms_handler.py

@@ -23,6 +23,7 @@ def wrapper(*args, **kwargs):
             selectinload(Rom.states),
             selectinload(Rom.screenshots),
             selectinload(Rom.rom_users),
+            selectinload(Rom.sibling_roms),
         )
         return func(*args, **kwargs)
 
@@ -38,7 +39,9 @@ def wrapper(*args, **kwargs):
                 f"{func} is missing required kwarg 'session' with type 'Session'"
             )
 
-        kwargs["query"] = select(Rom).options(selectinload(Rom.rom_users))
+        kwargs["query"] = select(Rom).options(
+            selectinload(Rom.rom_users), selectinload(Rom.sibling_roms)
+        )
         return func(*args, **kwargs)
 
     return wrapper
@@ -153,32 +156,6 @@ def get_rom_by_filename_no_ext(
             query.filter_by(file_name_no_ext=file_name_no_ext).limit(1)
         )
 
-    @begin_session
-    @with_simple
-    def get_sibling_roms(
-        self, rom: Rom, query: Query = None, session: Session = None
-    ) -> list[Rom]:
-        return session.scalars(
-            query.where(
-                and_(
-                    Rom.platform_id == rom.platform_id,
-                    Rom.id != rom.id,
-                    or_(
-                        and_(
-                            Rom.igdb_id == rom.igdb_id,
-                            Rom.igdb_id.isnot(None),
-                            Rom.igdb_id != "",
-                        ),
-                        and_(
-                            Rom.moby_id == rom.moby_id,
-                            Rom.moby_id.isnot(None),
-                            Rom.moby_id != "",
-                        ),
-                    ),
-                )
-            )
-        ).all()
-
     @begin_session
     def get_rom_collections(
         self, rom: Rom, session: Session = None

backend/models/rom.py

@@ -5,7 +5,15 @@
 
 from config import FRONTEND_RESOURCES_PATH
 from models.base import BaseModel
-from sqlalchemy import JSON, BigInteger, ForeignKey, String, Text, UniqueConstraint
+from sqlalchemy import (
+    JSON,
+    BigInteger,
+    ForeignKey,
+    Integer,
+    String,
+    Text,
+    UniqueConstraint,
+)
 from sqlalchemy.dialects.mysql.json import JSON as MySQLJSON
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
@@ -66,6 +74,11 @@ class Rom(BaseModel):
     )
 
     platform: Mapped[Platform] = relationship(lazy="immediate")
+    sibling_roms: Mapped[list[Rom]] = relationship(
+        secondary="sibling_roms",
+        primaryjoin="Rom.id == SiblingRom.rom_id",
+        secondaryjoin="Rom.id == SiblingRom.sibling_rom_id",
+    )
 
     saves: Mapped[list[Save]] = relationship(back_populates="rom")
     states: Mapped[list[State]] = relationship(back_populates="rom")
@@ -98,12 +111,6 @@ def merged_screenshots(self) -> list[str]:
             f"{FRONTEND_RESOURCES_PATH}/{s}" for s in self.path_screenshots
         ]
 
-    # This is an expensive operation so don't call it on a list of roms
-    def get_sibling_roms(self) -> list[Rom]:
-        from handler.database import db_rom_handler
-
-        return db_rom_handler.get_sibling_roms(self)
-
     def get_collections(self) -> list[Collection]:
         from handler.database import db_rom_handler
 
@@ -176,3 +183,14 @@ class RomUser(BaseModel):
     @property
     def user__username(self) -> str:
         return self.user.username
+
+
+class SiblingRom(BaseModel):
+    __tablename__ = "sibling_roms"
+
+    rom_id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    sibling_rom_id: Mapped[int] = mapped_column(Integer, primary_key=True)
+
+    __table_args__ = (
+        UniqueConstraint("rom_id", "sibling_rom_id", name="unique_sibling_roms"),
+    )