From 440e8bad21e9bc300cd8c24627ca183ac875f53f Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Fri, 31 Mar 2023 12:28:38 +0200 Subject: [PATCH 1/5] search term to use in IGDB search engime normalized --- backend/requirements.txt | 3 ++- backend/src/handler/igdb_handler.py | 10 ++++++---- frontend/src/components/RomDetails.vue | 25 +++++++++++++++++++++---- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index d932e3c95..536cde608 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -3,4 +3,5 @@ fastapi==0.92.0 uvicorn==0.20.0 mariadb==1.1.6 SQLAlchemy==2.0.7 -PyYAML==6.0 \ No newline at end of file +PyYAML==6.0 +Unidecode==1.3.6 \ No newline at end of file diff --git a/backend/src/handler/igdb_handler.py b/backend/src/handler/igdb_handler.py index bca3762cb..5ff4b2646 100644 --- a/backend/src/handler/igdb_handler.py +++ b/backend/src/handler/igdb_handler.py @@ -1,6 +1,7 @@ import sys import functools import re +import unidecode from time import time import requests @@ -64,11 +65,12 @@ def get_rom_details(self, file_name: str, p_igdb_id: int, r_igdb_id_search: str) pass else: + search_term: str = unidecode.unidecode(file_name_no_tags) if p_igdb_id: try: res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, - data=f"search \"{file_name_no_tags}\";fields id, slug, name, summary; where platforms=[{p_igdb_id}] & category=0;").json()[0] + data=f"search \"{search_term}\";fields id, slug, name, summary; where platforms=[{p_igdb_id}] & category=0;").json()[0] r_igdb_id = res_details['id'] slug = res_details['slug'] name = res_details['name'] @@ -79,7 +81,7 @@ def get_rom_details(self, file_name: str, p_igdb_id: int, r_igdb_id_search: str) except IndexError: try: res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, - data=f"search \"{file_name_no_tags}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}] & category=10;").json()[0] + data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}] & category=10;").json()[0] r_igdb_id = res_details['id'] slug = res_details['slug'] name = res_details['name'] @@ -90,7 +92,7 @@ def get_rom_details(self, file_name: str, p_igdb_id: int, r_igdb_id_search: str) except IndexError: try: res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, - data=f"search \"{file_name_no_tags}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}];").json()[0] + data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}];").json()[0] r_igdb_id = res_details['id'] slug = res_details['slug'] name = res_details['name'] @@ -113,7 +115,7 @@ def get_rom_details(self, file_name: str, p_igdb_id: int, r_igdb_id_search: str) @check_twitch_token def get_matched_roms(self, file_name: str, p_igdb_id: int) -> list: - search_term: str = re.sub('[\(\[].*?[\)\]]', '', file_name.split('.')[0]) + search_term: str = unidecode.unidecode(re.sub('[\(\[].*?[\)\]]', '', file_name.split('.')[0])) matched_roms: list = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}];").json() log.info(f"Matched roms for {file_name}: {matched_roms}") diff --git a/frontend/src/components/RomDetails.vue b/frontend/src/components/RomDetails.vue index 3ad0964bf..39616c441 100644 --- a/frontend/src/components/RomDetails.vue +++ b/frontend/src/components/RomDetails.vue @@ -11,6 +11,7 @@ const searching = ref(false) const matchedRoms = ref([]) const updating = ref(false) const editedRomName = ref(rom.value.file_name) +const renameAsIGDB = ref(false) const dialogSearchRom = ref(false) const dialogEditRom = ref(false) const dialogDeleteRom = ref(false) @@ -49,7 +50,14 @@ async function searchRomIGDB() { async function updateRom(updatedRom=Object.assign({},rom.value), newName=rom.value.file_name) { updating.value = true dialogSearchRom.value = false - updatedRom.file_name = newName + if (renameAsIGDB.value) { + updatedRom.file_name = rom.value.file_name.replace(rom.value.file_name_no_tags.trim(), updatedRom.name) + editedRomName.value = updatedRom.file_name + renameAsIGDB.value = false + } + else{ + updatedRom.file_name = newName + } console.log(rom.value) await axios.patch('/api/platforms/'+rom.value.p_slug+'/roms', { rom: rom.value, @@ -164,7 +172,7 @@ async function deleteRom() { Results found mdi-close - +
@@ -182,6 +190,9 @@ async function deleteRom() { + + + @@ -219,9 +230,15 @@ async function deleteRom() { Cancel
- +
- \ No newline at end of file + + + \ No newline at end of file From c5099563e2ff418c1743be857b205b401f6e44a5 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Fri, 31 Mar 2023 12:37:11 +0200 Subject: [PATCH 2/5] feature/rename_file_as_IGDB --- changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.md b/changelog.md index 4d1db9581..3c0294dcf 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,8 @@ +# v1.6 (_XX-04-2023_) + +## Added + - Now game files can be renamed after the name matched in IGDB, keeping the tags. + # v1.5.1 (_31-03-2023_) ## Fix From b2bc8833e138641cfa55cfd951efcd7426dde4b2 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Fri, 31 Mar 2023 13:47:19 +0200 Subject: [PATCH 3/5] smart scan added --- backend/src/config/__init__.py | 1 + backend/src/handler/igdb_handler.py | 2 +- backend/src/main.py | 6 ++-- backend/src/utils/fastapi.py | 2 +- backend/src/utils/fs.py | 49 +++++++++++++------------- frontend/src/components/Navigation.vue | 21 +++++++---- frontend/src/components/RomDetails.vue | 4 +-- 7 files changed, 47 insertions(+), 38 deletions(-) diff --git a/backend/src/config/__init__.py b/backend/src/config/__init__.py index 5d3d7f6b0..44f94a14b 100644 --- a/backend/src/config/__init__.py +++ b/backend/src/config/__init__.py @@ -10,6 +10,7 @@ # PATHS LIBRARY_BASE_PATH: str = f"{pathlib.Path(__file__).parent.parent.parent.parent.resolve()}/library" +HIGH_PRIO_STRUCTURE_PATH: str = f"{LIBRARY_BASE_PATH}/roms" ROMM_USER_CONFIG_PATH: str = f"{pathlib.Path(__file__).parent.parent.parent.parent.resolve()}/romm/config.yml" # ROMM RESERVED FOLDERS diff --git a/backend/src/handler/igdb_handler.py b/backend/src/handler/igdb_handler.py index 5ff4b2646..bb1283a71 100644 --- a/backend/src/handler/igdb_handler.py +++ b/backend/src/handler/igdb_handler.py @@ -64,7 +64,7 @@ def get_rom_details(self, file_name: str, p_igdb_id: int, r_igdb_id_search: str) except KeyError: pass - else: + else: #TODO: improve API calls to make only one search_term: str = unidecode.unidecode(file_name_no_tags) if p_igdb_id: try: diff --git a/backend/src/main.py b/backend/src/main.py index 45ca3d2c7..86685396c 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -75,7 +75,7 @@ async def platforms() -> dict: @app.put("/scan") -async def scan(req: Request, overwrite: bool=False) -> dict: +async def scan(req: Request, full_scan: bool=False, overwrite: bool=False) -> dict: """Scan platforms and roms and write them in database.""" log.info("complete scaning...") @@ -84,10 +84,10 @@ async def scan(req: Request, overwrite: bool=False) -> dict: platforms: list[str] = data['platforms'] if data['platforms'] else fs.get_platforms() for p_slug in platforms: platform: Platform = fastapi.scan_platform(p_slug) - roms: list[dict] = fs.get_roms(p_slug) + roms: list[dict] = fs.get_roms(p_slug, full_scan) for rom in roms: fastapi.scan_rom(platform, rom) - dbh.purge_roms(p_slug, roms) + dbh.purge_roms(p_slug, fs.get_roms(p_slug, True)) dbh.purge_platforms(fs.get_platforms()) return {'msg': 'success'} diff --git a/backend/src/utils/fastapi.py b/backend/src/utils/fastapi.py index 43140ffa0..b359b18f1 100644 --- a/backend/src/utils/fastapi.py +++ b/backend/src/utils/fastapi.py @@ -31,7 +31,7 @@ def scan_platform(p_slug: str) -> Platform: platform_attrs: dict = igdbh.get_platform_details(p_slug) platform_attrs['slug'] = p_slug platform_attrs['logo_path'] = '' - platform_attrs['n_roms'] = fs.get_roms(p_slug, only_amount=True) + platform_attrs['n_roms'] = fs.get_roms(p_slug, True, only_amount=True) log.info(f"Platform n_roms: {platform_attrs['n_roms']}") platform = Platform(**platform_attrs) dbh.add_platform(platform) diff --git a/backend/src/utils/fs.py b/backend/src/utils/fs.py index b72cbc9a9..a50351e10 100644 --- a/backend/src/utils/fs.py +++ b/backend/src/utils/fs.py @@ -6,7 +6,10 @@ import requests from fastapi import HTTPException -from config import user_config, LIBRARY_BASE_PATH, RESERVED_FOLDERS, DEFAULT_URL_COVER_L, DEFAULT_PATH_COVER_L, DEFAULT_URL_COVER_S, DEFAULT_PATH_COVER_S +from config import user_config, LIBRARY_BASE_PATH, HIGH_PRIO_STRUCTURE_PATH, RESERVED_FOLDERS, DEFAULT_URL_COVER_L, DEFAULT_PATH_COVER_L, DEFAULT_URL_COVER_S, DEFAULT_PATH_COVER_S +from models.platform import Platform +from models.rom import Rom +from handler import dbh from logger.logger import log @@ -52,12 +55,11 @@ def get_platforms() -> list[str]: # ========= Roms utils ========= def _check_folder_structure(p_slug) -> tuple: - if os.path.exists(f"{LIBRARY_BASE_PATH}/roms"): - roms_path: str = f"{LIBRARY_BASE_PATH}/roms/{p_slug}" - roms_files = list(os.walk(f"{LIBRARY_BASE_PATH}/roms/{p_slug}"))[0][2] - else: - roms_path: str = f"{LIBRARY_BASE_PATH}/{p_slug}/roms" - roms_files = list(os.walk(f"{LIBRARY_BASE_PATH}/{p_slug}/roms"))[0][2] + roms_path: str = f"{HIGH_PRIO_STRUCTURE_PATH}/{p_slug}" if os.path.exists(HIGH_PRIO_STRUCTURE_PATH) else f"{LIBRARY_BASE_PATH}/{p_slug}/roms" + try: + roms_files = list(os.walk(roms_path))[0][2] + except IndexError: + roms_files = [] return roms_path, roms_files @@ -91,7 +93,7 @@ def parse_tags(file_name: str) -> tuple: return reg, rev, other_tags -def get_roms(p_slug: str, only_amount: bool = False) -> list[dict]: +def get_roms(p_slug: str, full_scan: bool, only_amount: bool = False) -> list[dict]: """Gets all filesystem roms for a platform Args: @@ -99,22 +101,21 @@ def get_roms(p_slug: str, only_amount: bool = False) -> list[dict]: only_amount: flag to return only amount of roms instead of all info Returns: list with all the filesystem roms for a platform found in the LIBRARY_BASE_PATH. Just the amount of them if only_amount=True """ - try: - roms: list[dict] = [] - roms_path, roms_files = _check_folder_structure(p_slug) - roms_files = _exclude_files(roms_files) - - if only_amount: return len(roms_files) - - for rom in roms_files: - file_size: str = str(round(os.stat(f"{roms_path}/{rom}").st_size / (1024 * 1024), 2)) - file_extension: str = rom.split('.')[-1] if '.' in rom else "" - reg, rev, other_tags = parse_tags(rom) - roms.append({'file_name': rom, 'file_path': roms_path, 'file_size': file_size, 'file_extension': file_extension, - 'region': reg, 'revision': rev, 'tags': other_tags}) - log.info(f"Roms found for {p_slug}: {roms}") - except IndexError: - log.warning(f"Roms not found for {p_slug}") + roms: list[dict] = [] + roms_path, roms_files = _check_folder_structure(p_slug) + roms_files = _exclude_files(roms_files) + + if only_amount: return len(roms_files) + + excluded_roms: list[str] = [rom.file_name for rom in dbh.get_roms(p_slug)] + for rom in roms_files: + if rom in excluded_roms and not full_scan: continue + file_size: str = str(round(os.stat(f"{roms_path}/{rom}").st_size / (1024 * 1024), 2)) + file_extension: str = rom.split('.')[-1] if '.' in rom else "" + reg, rev, other_tags = parse_tags(rom) + roms.append({'file_name': rom, 'file_path': roms_path, 'file_size': file_size, 'file_extension': file_extension, + 'region': reg, 'revision': rev, 'tags': other_tags}) + log.info(f"Roms found for {p_slug}: {roms}") if only_amount: return 0 return roms diff --git a/frontend/src/components/Navigation.vue b/frontend/src/components/Navigation.vue index e047f766f..0bf5ceffa 100644 --- a/frontend/src/components/Navigation.vue +++ b/frontend/src/components/Navigation.vue @@ -10,6 +10,7 @@ const currentPlatform = ref(JSON.parse(localStorage.getItem('currentPlatform')) const platformsToScan = ref([]) const scanning = ref(false) const scanOverwrite = ref(false) +const fullScan = ref(false) const gettingRomsFlag = ref(false) const filter = ref('') const drawer = ref(null) @@ -33,7 +34,7 @@ async function scan() { toRaw(platformsToScan)._rawValue.forEach(p => {platforms.push(toRaw(p.slug))}) console.log(platforms) - await axios.put('/api/scan?overwrite='+scanOverwrite.value,{ + await axios.put('/api/scan?overwrite='+scanOverwrite.value+'&full_scan='+fullScan.value,{ platforms: platforms }).then((response) => { console.log("scan completed") @@ -103,12 +104,18 @@ getPlatforms() - - -

Scan

-

Scanning

- - + + + + +

Scan

+ + + + + + + diff --git a/frontend/src/components/RomDetails.vue b/frontend/src/components/RomDetails.vue index 39616c441..68e128b59 100644 --- a/frontend/src/components/RomDetails.vue +++ b/frontend/src/components/RomDetails.vue @@ -191,7 +191,7 @@ async function deleteRom() { - + @@ -230,7 +230,7 @@ async function deleteRom() { Cancel
- +
From 998187524bbc7f6fa09a10630529cc2aadf7cff5 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Fri, 31 Mar 2023 14:09:10 +0200 Subject: [PATCH 4/5] smart scan added --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index 3c0294dcf..c19f93a22 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,7 @@ # v1.6 (_XX-04-2023_) ## Added + - Smart scan: now RomM will only scan the changes in the filesystem, making the scan process too much faster. Added an option to force a full scan. - Now game files can be renamed after the name matched in IGDB, keeping the tags. # v1.5.1 (_31-03-2023_) From ad4070115f002d8acd2975b79ac58df27d4e15bb Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Fri, 31 Mar 2023 14:12:10 +0200 Subject: [PATCH 5/5] date added to v1.6 --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index c19f93a22..e42f686dd 100644 --- a/changelog.md +++ b/changelog.md @@ -1,4 +1,4 @@ -# v1.6 (_XX-04-2023_) +# v1.6 (_10-04-2023_) ## Added - Smart scan: now RomM will only scan the changes in the filesystem, making the scan process too much faster. Added an option to force a full scan.