Skip to content

Commit

Permalink
Merge pull request #1128 from rommapp/large-lib-performance-fixes
Browse files Browse the repository at this point in the history 
Better performance for large collections
  • Loading branch information
@gantoine
gantoine authored Aug 28, 2024
2 parents f6197d4 + 307d906 commit c00725f
Show file tree
Hide file tree
Showing 2 changed files with 110 additions and 87 deletions.
53 changes: 20 additions & 33 deletions backend/handler/database/platforms_handler.py
View file
Original file line number Diff line number Diff line change
@@ -1,78 +1,65 @@
import functools

from decorators.database import begin_session
from models.platform import Platform
from models.rom import Rom
from sqlalchemy import Select, delete, or_, select
from sqlalchemy.orm import Query, Session, selectinload
from sqlalchemy.orm import Session

from .base_handler import DBBaseHandler


def with_roms(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
session = kwargs.get("session")
if session is None:
raise ValueError("session is required")

kwargs["query"] = select(Platform).options(
selectinload(Platform.roms).load_only(Rom.id)
)
return func(*args, **kwargs)

return wrapper


class DBPlatformsHandler(DBBaseHandler):
@begin_session
@with_roms
def add_platform(
self, platform: Platform, query: Query = None, session: Session = None
self,
platform: Platform,
session: Session,
) -> Platform:
platform = session.merge(platform)
session.flush()

return session.scalar(query.filter_by(id=platform.id).limit(1))
new_platform = session.scalar(
select(Platform).filter_by(id=platform.id).limit(1)
)
if not new_platform:
raise ValueError("Could not find newlyewly created platform")

return new_platform

@begin_session
@with_roms
def get_platform(
self, id: int, *, query: Query = None, session: Session = None
) -> Platform | None:
return session.scalar(query.filter_by(id=id).limit(1))
def get_platform(self, id: int, *, session: Session) -> Platform | None:
return session.scalar(select(Platform).filter_by(id=id).limit(1))

@begin_session
def get_platforms(self, *, session: Session = None) -> Select[tuple[Platform]]:
def get_platforms(self, *, session: Session) -> Select[tuple[Platform]]:
return (
session.scalars(select(Platform).order_by(Platform.name.asc())) # type: ignore[attr-defined]
.unique()
.all()
)

@begin_session
@with_roms
def get_platform_by_fs_slug(
self, fs_slug: str, query: Query = None, session: Session = None
self, fs_slug: str, session: Session
) -> Platform | None:
return session.scalar(query.filter_by(fs_slug=fs_slug).limit(1))
return session.scalar(select(Platform).filter_by(fs_slug=fs_slug).limit(1))

@begin_session
def delete_platform(self, id: int, session: Session = None) -> int:
def delete_platform(self, id: int, session: Session) -> None:
# Remove all roms from that platforms first
session.execute(
delete(Rom)
.where(Rom.platform_id == id)
.execution_options(synchronize_session="evaluate")
)
return session.execute(

session.execute(
delete(Platform)
.where(Platform.id == id)
.execution_options(synchronize_session="evaluate")
)

@begin_session
def purge_platforms(self, fs_platforms: list[str], session: Session = None) -> int:
def purge_platforms(self, fs_platforms: list[str], session: Session) -> int:
return session.execute(
delete(Platform)
.where(or_(Platform.fs_slug.not_in(fs_platforms), Platform.slug.is_(None))) # type: ignore[attr-defined]
Expand Down
144 changes: 90 additions & 54 deletions frontend/src/stores/roms.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ export default defineStore("roms", {
currentRom: null as DetailedRom | null,
allRoms: [] as SimpleRom[],
_grouped: [] as SimpleRom[],
_filteredIDs: [] as number[],
_selectedIDs: [] as number[],
_filteredIDs: new Set<number>(),
_selectedIDs: new Set<number>(),
recentRoms: [] as SimpleRom[],
lastSelectedIndex: -1,
selecting: false,
Expand All @@ -34,9 +34,9 @@ export default defineStore("roms", {

getters: {
filteredRoms: (state) =>
state._grouped.filter((rom) => state._filteredIDs.includes(rom.id)),
state._grouped.filter((rom) => state._filteredIDs.has(rom.id)),
selectedRoms: (state) =>
state._grouped.filter((rom) => state._selectedIDs.includes(rom.id)),
state._grouped.filter((rom) => state._selectedIDs.has(rom.id)),
},

actions: {
Expand Down Expand Up @@ -115,22 +115,18 @@ export default defineStore("roms", {
return rom.id === value.id;
});
});
this._filteredIDs = this._filteredIDs.filter((value) => {
return !roms.find((rom) => {
return rom.id === value;
});
});
roms.forEach((rom) => this._filteredIDs.delete(rom.id));
},
reset() {
this.allRoms = [];
this._grouped = [];
this._filteredIDs = [];
this._selectedIDs = [];
this._filteredIDs = new Set<number>();
this._selectedIDs = new Set<number>();
this.lastSelectedIndex = -1;
},
// Filter roms by gallery filter store state
setFiltered(roms: SimpleRom[], galleryFilter: GalleryFilterStore) {
this._filteredIDs = roms.map((rom) => rom.id);
this._filteredIDs = new Set(roms.map((rom) => rom.id));
if (galleryFilter.filterSearch) {
this._filterSearch(galleryFilter.filterSearch);
}
Expand All @@ -157,68 +153,108 @@ export default defineStore("roms", {
}
},
_filterSearch(searchFilter: string) {
this._filteredIDs = this.filteredRoms
.filter(
(rom) =>
rom.name?.toLowerCase().includes(searchFilter.toLowerCase()) ||
rom.file_name?.toLowerCase().includes(searchFilter.toLowerCase()),
)
.map((roms) => roms.id);
const bySearch = new Set(
this.filteredRoms
.filter(
(rom) =>
rom.name?.toLowerCase().includes(searchFilter.toLowerCase()) ||
rom.file_name?.toLowerCase().includes(searchFilter.toLowerCase()),
)
.map((roms) => roms.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = bySearch.intersection(this._filteredIDs);
},
_filterUnmatched() {
this._filteredIDs = this.filteredRoms
.filter((rom) => !rom.igdb_id && !rom.moby_id)
.map((roms) => roms.id);
const byUnmatched = new Set(
this.filteredRoms
.filter((rom) => !rom.igdb_id && !rom.moby_id)
.map((roms) => roms.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byUnmatched.intersection(this._filteredIDs);
},
_filterFavourites() {
this._filteredIDs = this.filteredRoms
.filter((rom) => collectionStore.favCollection?.roms?.includes(rom.id))
.map((roms) => roms.id);
const byFavourites = new Set(
this.filteredRoms
.filter((rom) =>
collectionStore.favCollection?.roms?.includes(rom.id),
)
.map((roms) => roms.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byFavourites.intersection(this._filteredIDs);
},
_filterDuplicates() {
this._filteredIDs = this.filteredRoms
.filter((rom) => rom.sibling_roms?.length)
.map((rom) => rom.id);
const byDuplicates = new Set(
this.filteredRoms
.filter((rom) => rom.sibling_roms?.length)
.map((rom) => rom.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byDuplicates.intersection(this._filteredIDs);
},
_filterGenre(genreToFilter: string) {
this._filteredIDs = this.filteredRoms
.filter((rom) => rom.genres.some((genre) => genre === genreToFilter))
.map((rom) => rom.id);
const byGenre = new Set(
this.filteredRoms
.filter((rom) => rom.genres.some((genre) => genre === genreToFilter))
.map((rom) => rom.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byGenre.intersection(this._filteredIDs);
},
_filterFranchise(franchiseToFilter: string) {
this._filteredIDs = this.filteredRoms
.filter((rom) =>
rom.franchises.some((franchise) => franchise === franchiseToFilter),
)
.map((rom) => rom.id);
const byFranchise = new Set(
this.filteredRoms
.filter((rom) =>
rom.franchises.some((franchise) => franchise === franchiseToFilter),
)
.map((rom) => rom.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byFranchise.intersection(this._filteredIDs);
},
_filterCollection(collectionToFilter: string) {
this._filteredIDs = this.filteredRoms
.filter((rom) =>
rom.collections.some(
(collection) => collection === collectionToFilter,
),
)
.map((rom) => rom.id);
const byCollection = new Set(
this.filteredRoms
.filter((rom) =>
rom.collections.some(
(collection) => collection === collectionToFilter,
),
)
.map((rom) => rom.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byCollection.intersection(this._filteredIDs);
},
_filterCompany(companyToFilter: string) {
this._filteredIDs = this.filteredRoms
.filter((rom) =>
rom.companies.some((company) => company === companyToFilter),
)
.map((rom) => rom.id);
const byCompany = new Set(
this.filteredRoms
.filter((rom) =>
rom.companies.some((company) => company === companyToFilter),
)
.map((rom) => rom.id),
);

// @ts-expect-error intersection is recently defined on Set
this._filteredIDs = byCompany.intersection(this._filteredIDs);
},
// Selected roms
setSelection(roms: SimpleRom[]) {
this._selectedIDs = roms.map((rom) => rom.id);
this._selectedIDs = new Set(roms.map((rom) => rom.id));
},
addToSelection(rom: SimpleRom) {
this._selectedIDs.push(rom.id);
this._selectedIDs.add(rom.id);
},
removeFromSelection(rom: SimpleRom) {
this._selectedIDs = this._selectedIDs.filter((id) => {
return id !== rom.id;
});
this._selectedIDs.delete(rom.id);
},
updateLastSelected(index: number) {
this.lastSelectedIndex = index;
Expand All @@ -227,7 +263,7 @@ export default defineStore("roms", {
this.selecting = !this.selecting;
},
resetSelection() {
this._selectedIDs = [];
this._selectedIDs = new Set<number>();
this.lastSelectedIndex = -1;
},
isSimpleRom(rom: SimpleRom | SearchRomSchema): rom is SimpleRom {
Expand Down

0 comments on commit c00725f

Please sign in to comment.