Merge pull request #99 from zurdi15/develop

v1.6

backend/requirements.txt

@@ -3,4 +3,5 @@ fastapi==0.92.0
 uvicorn==0.20.0
 mariadb==1.1.6
 SQLAlchemy==2.0.7
-PyYAML==6.0
+PyYAML==6.0
+Unidecode==1.3.6

backend/src/config/__init__.py

@@ -10,6 +10,7 @@
 
 # PATHS
 LIBRARY_BASE_PATH: str = f"{pathlib.Path(__file__).parent.parent.parent.parent.resolve()}/library"
+HIGH_PRIO_STRUCTURE_PATH: str = f"{LIBRARY_BASE_PATH}/roms"
 ROMM_USER_CONFIG_PATH: str = f"{pathlib.Path(__file__).parent.parent.parent.parent.resolve()}/romm/config.yml"
 
 # ROMM RESERVED FOLDERS

backend/src/handler/igdb_handler.py

@@ -1,6 +1,7 @@
 import sys
 import functools
 import re
+import unidecode
 from time import time
 
 import requests
@@ -63,12 +64,13 @@ def get_rom_details(self, file_name: str, p_igdb_id: int, r_igdb_id_search: str)
             except KeyError:
                 pass            
 
-        else:
+        else: #TODO: improve API calls to make only one
+            search_term: str = unidecode.unidecode(file_name_no_tags)
             if p_igdb_id:
                 try:
 
                     res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers,
-                                                      data=f"search \"{file_name_no_tags}\";fields id, slug, name, summary; where platforms=[{p_igdb_id}] & category=0;").json()[0]
+                                                      data=f"search \"{search_term}\";fields id, slug, name, summary; where platforms=[{p_igdb_id}] & category=0;").json()[0]
                     r_igdb_id = res_details['id']
                     slug = res_details['slug']
                     name = res_details['name']
@@ -79,7 +81,7 @@ def get_rom_details(self, file_name: str, p_igdb_id: int, r_igdb_id_search: str)
                 except IndexError:
                     try:
                         res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers,
-                                                          data=f"search \"{file_name_no_tags}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}] & category=10;").json()[0]
+                                                          data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}] & category=10;").json()[0]
                         r_igdb_id = res_details['id']
                         slug = res_details['slug']
                         name = res_details['name']
@@ -90,7 +92,7 @@ def get_rom_details(self, file_name: str, p_igdb_id: int, r_igdb_id_search: str)
                     except IndexError:
                         try:
                             res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers,
-                                                              data=f"search \"{file_name_no_tags}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}];").json()[0]
+                                                              data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}];").json()[0]
                             r_igdb_id = res_details['id']
                             slug = res_details['slug']
                             name = res_details['name']
@@ -113,7 +115,7 @@ def get_rom_details(self, file_name: str, p_igdb_id: int, r_igdb_id_search: str)
 
     @check_twitch_token
     def get_matched_roms(self, file_name: str, p_igdb_id: int) -> list:
-        search_term: str = re.sub('[\(\[].*?[\)\]]', '', file_name.split('.')[0])
+        search_term: str = unidecode.unidecode(re.sub('[\(\[].*?[\)\]]', '', file_name.split('.')[0]))
         matched_roms: list = requests.post("https://api.igdb.com/v4/games/", headers=self.headers,
                                            data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}];").json()
         log.info(f"Matched roms for {file_name}: {matched_roms}")

backend/src/main.py

@@ -75,7 +75,7 @@ async def platforms() -> dict:
 
 
 @app.put("/scan")
-async def scan(req: Request, overwrite: bool=False) -> dict:
+async def scan(req: Request, full_scan: bool=False, overwrite: bool=False) -> dict:
     """Scan platforms and roms and write them in database."""
 
     log.info("complete scaning...")
@@ -84,10 +84,10 @@ async def scan(req: Request, overwrite: bool=False) -> dict:
     platforms: list[str] = data['platforms'] if data['platforms'] else fs.get_platforms()
     for p_slug in platforms:
         platform: Platform = fastapi.scan_platform(p_slug)
-        roms: list[dict] = fs.get_roms(p_slug)
+        roms: list[dict] = fs.get_roms(p_slug, full_scan)
         for rom in roms:
             fastapi.scan_rom(platform, rom)
-        dbh.purge_roms(p_slug, roms)
+        dbh.purge_roms(p_slug, fs.get_roms(p_slug, True))
     dbh.purge_platforms(fs.get_platforms())
     return {'msg': 'success'}
 

backend/src/utils/fastapi.py

@@ -31,7 +31,7 @@ def scan_platform(p_slug: str) -> Platform:
     platform_attrs: dict = igdbh.get_platform_details(p_slug)
     platform_attrs['slug'] = p_slug
     platform_attrs['logo_path'] = ''
-    platform_attrs['n_roms'] = fs.get_roms(p_slug, only_amount=True)
+    platform_attrs['n_roms'] = fs.get_roms(p_slug, True, only_amount=True)
     log.info(f"Platform n_roms: {platform_attrs['n_roms']}")
     platform = Platform(**platform_attrs)
     dbh.add_platform(platform)

backend/src/utils/fs.py

@@ -6,7 +6,10 @@
 import requests
 from fastapi import HTTPException
 
-from config import user_config, LIBRARY_BASE_PATH, RESERVED_FOLDERS, DEFAULT_URL_COVER_L, DEFAULT_PATH_COVER_L, DEFAULT_URL_COVER_S, DEFAULT_PATH_COVER_S
+from config import user_config, LIBRARY_BASE_PATH, HIGH_PRIO_STRUCTURE_PATH, RESERVED_FOLDERS, DEFAULT_URL_COVER_L, DEFAULT_PATH_COVER_L, DEFAULT_URL_COVER_S, DEFAULT_PATH_COVER_S
+from models.platform import Platform
+from models.rom import Rom
+from handler import dbh
 from logger.logger import log
 
 
@@ -52,12 +55,11 @@ def get_platforms() -> list[str]:
 
 # ========= Roms utils =========
 def _check_folder_structure(p_slug) -> tuple:
-    if os.path.exists(f"{LIBRARY_BASE_PATH}/roms"):
-        roms_path: str = f"{LIBRARY_BASE_PATH}/roms/{p_slug}"
-        roms_files = list(os.walk(f"{LIBRARY_BASE_PATH}/roms/{p_slug}"))[0][2]
-    else:
-        roms_path: str = f"{LIBRARY_BASE_PATH}/{p_slug}/roms"
-        roms_files = list(os.walk(f"{LIBRARY_BASE_PATH}/{p_slug}/roms"))[0][2]
+    roms_path: str = f"{HIGH_PRIO_STRUCTURE_PATH}/{p_slug}" if os.path.exists(HIGH_PRIO_STRUCTURE_PATH) else f"{LIBRARY_BASE_PATH}/{p_slug}/roms"
+    try:
+        roms_files = list(os.walk(roms_path))[0][2]
+    except IndexError:
+        roms_files = []
     return roms_path, roms_files
 
 
@@ -91,30 +93,29 @@ def parse_tags(file_name: str) -> tuple:
     return reg, rev, other_tags
 
 
-def get_roms(p_slug: str, only_amount: bool = False) -> list[dict]:
+def get_roms(p_slug: str, full_scan: bool, only_amount: bool = False) -> list[dict]:
     """Gets all filesystem roms for a platform
 
     Args:
         p_slug: short name of the platform
         only_amount: flag to return only amount of roms instead of all info
     Returns: list with all the filesystem roms for a platform found in the LIBRARY_BASE_PATH. Just the amount of them if only_amount=True
     """
-    try:
-        roms: list[dict] = []
-        roms_path, roms_files = _check_folder_structure(p_slug)
-        roms_files = _exclude_files(roms_files)
-
-        if only_amount: return len(roms_files)
-
-        for rom in roms_files:
-            file_size: str = str(round(os.stat(f"{roms_path}/{rom}").st_size / (1024 * 1024), 2))
-            file_extension: str = rom.split('.')[-1] if '.' in rom else ""
-            reg, rev, other_tags = parse_tags(rom)
-            roms.append({'file_name': rom, 'file_path': roms_path, 'file_size': file_size, 'file_extension': file_extension,
-                         'region': reg, 'revision': rev, 'tags': other_tags})
-        log.info(f"Roms found for {p_slug}: {roms}")
-    except IndexError:
-        log.warning(f"Roms not found for {p_slug}")
+    roms: list[dict] = []
+    roms_path, roms_files = _check_folder_structure(p_slug)
+    roms_files = _exclude_files(roms_files)
+
+    if only_amount: return len(roms_files)
+
+    excluded_roms: list[str] = [rom.file_name for rom in dbh.get_roms(p_slug)]
+    for rom in roms_files:
+        if rom in excluded_roms and not full_scan: continue
+        file_size: str = str(round(os.stat(f"{roms_path}/{rom}").st_size / (1024 * 1024), 2))
+        file_extension: str = rom.split('.')[-1] if '.' in rom else ""
+        reg, rev, other_tags = parse_tags(rom)
+        roms.append({'file_name': rom, 'file_path': roms_path, 'file_size': file_size, 'file_extension': file_extension,
+                        'region': reg, 'revision': rev, 'tags': other_tags})
+    log.info(f"Roms found for {p_slug}: {roms}")
     if only_amount: return 0
     return roms
 

changelog.md

@@ -1,3 +1,9 @@
+# v1.6 (_10-04-2023_)
+
+## Added
+ - Smart scan: now RomM will only scan the changes in the filesystem, making the scan process too much faster. Added an option to force a full scan.
+ - Now game files can be renamed after the name matched in IGDB, keeping the tags.
+
 # v1.5.1 (_31-03-2023_)
 
 ## Fix

frontend/src/components/Navigation.vue

@@ -10,6 +10,7 @@ const currentPlatform = ref(JSON.parse(localStorage.getItem('currentPlatform'))
 const platformsToScan = ref([])
 const scanning = ref(false)
 const scanOverwrite = ref(false)
+const fullScan = ref(false)
 const gettingRomsFlag = ref(false)
 const filter = ref('')
 const drawer = ref(null)
@@ -33,7 +34,7 @@ async function scan() {
     toRaw(platformsToScan)._rawValue.forEach(p => {platforms.push(toRaw(p.slug))})
     console.log(platforms)
 
-    await axios.put('/api/scan?overwrite='+scanOverwrite.value,{
+    await axios.put('/api/scan?overwrite='+scanOverwrite.value+'&full_scan='+fullScan.value,{
         platforms: platforms
     }).then((response) => {
         console.log("scan completed")
@@ -103,12 +104,18 @@ getPlatforms()
         <v-list>
             <!-- Settings drawer - scan button -->
             <v-select label="Platforms" item-title="name" v-model="platformsToScan" :items="platforms" class="pl-5 pr-5 mt-2" density="comfortable" variant="outlined" multiple return-object clearable/>
-            <v-list-item class="d-flex align-center justify-center mb-2 pt-0">
-                <v-btn title="scan" @click="scan()" :disabled="scanning" prepend-icon="mdi-magnify-scan" color="secondary" rounded="0" inset>
-                    <p v-if="!scanning">Scan</p>
-                    <p v-if="scanning">Scanning</p>
-                    <v-progress-circular v-show="scanning" class="ml-2" :width="2" :size="20" indeterminate/>
-                </v-btn>
+            <v-list-item class="pa-0">
+                <v-row class="align-center">
+                    <v-col class="d-flex justify-center">
+                        <v-btn title="scan" @click="scan()" :disabled="scanning" prepend-icon="mdi-magnify-scan" class="ml-7" color="secondary" rounded="0" inset>
+                            <p v-if="!scanning">Scan</p>
+                            <v-progress-circular v-show="scanning" class="ml-2" :width="2" :size="20" indeterminate/>
+                        </v-btn>
+                    </v-col>
+                    <v-col class="mr-4">
+                        <v-checkbox v-model="fullScan" label="Full scan" hide-details="true"/>
+                    </v-col>
+                </v-row>
             </v-list-item>
         </v-list>
         <!-- Settings drawer - theme toggle -->

frontend/src/components/RomDetails.vue

@@ -11,6 +11,7 @@ const searching = ref(false)
 const matchedRoms = ref([])
 const updating = ref(false)
 const editedRomName = ref(rom.value.file_name)
+const renameAsIGDB = ref(false)
 const dialogSearchRom = ref(false)
 const dialogEditRom = ref(false)
 const dialogDeleteRom = ref(false)
@@ -49,7 +50,14 @@ async function searchRomIGDB() {
 async function updateRom(updatedRom=Object.assign({},rom.value), newName=rom.value.file_name) {
     updating.value = true
     dialogSearchRom.value = false
-    updatedRom.file_name = newName
+    if (renameAsIGDB.value) {
+        updatedRom.file_name = rom.value.file_name.replace(rom.value.file_name_no_tags.trim(), updatedRom.name)
+        editedRomName.value = updatedRom.file_name
+        renameAsIGDB.value = false
+    }
+    else{
+        updatedRom.file_name = newName
+    }
     console.log(rom.value)
     await axios.patch('/api/platforms/'+rom.value.p_slug+'/roms', {
         rom: rom.value,
@@ -164,7 +172,7 @@ async function deleteRom() {
                 <v-toolbar-title>Results found</v-toolbar-title>
                 <v-btn icon @click="dialogSearchRom=false" class="ml-1" rounded="0"><v-icon>mdi-close</v-icon></v-btn>
             </v-toolbar>
-            <v-card-text rounded="0" class="pa-3">
+            <v-card-text rounded="0" class="pa-3 scroll">
                 <div class="d-flex justify-center">
                     <v-progress-circular v-show="searching" :width="2" :size="40" class="pa-3 ma-3" indeterminate/>
                 </div>
@@ -182,6 +190,9 @@ async function deleteRom() {
                     </v-col>
                 </v-row>
             </v-card-text>
+            <v-card-actions v-show="!searching">
+                <v-checkbox v-model="renameAsIGDB" label="Rename file" class="pl-3" hide-details="true"/>
+            </v-card-actions>
         </v-card>
     </v-dialog>
 
@@ -219,9 +230,15 @@ async function deleteRom() {
                 <v-btn @click="dialogDeleteRom=false" variant="tonal">Cancel</v-btn>
             </v-card-actions>
             <div class="pl-8">
-                <v-checkbox v-model="deleteFromFs" label="Delete from filesystem"></v-checkbox>
+                <v-checkbox v-model="deleteFromFs" label="Delete from filesystem" hide-details="true"/>
             </div>
         </v-card>
     </v-dialog>
 
-</template>
+</template>
+
+<style scoped>
+.scroll {
+   overflow-y: scroll
+}
+</style>