- fix jimmejardine#145 :: When v82beta created a Qiqqa.library file n…

…ext to an already existing S3DB file, delete it. We explicitly delete it to the RecycleBin so the user MAY recover the database file on the off-chance that this was the wrong choice.

- fix jimmejardine#144 :: Do not try to create a Qiqqa.library DB when the directory already has an S3DB database file
- some work done to find out how jimmejardine#142 came about.
- `WebLibraryDetails_WorkingWebLibraries_All`: make sure all 'Guest' libraries are added to the sync set: under very particular circumstances you CAN have multiple guest libraries, e.g. when manually recovering multiple Qiqqa libraries you extracted from your backups of previous qiqqa runs/releases.
- performance: speed up the sync metadata info collect action by NOT calculating the precise storage size of each library as that entails a HUGE I/O load as each library document's file system metadata would be queried for its filesize -- which is only used in the UI in an overview table and is deemed non-essential right now.

Qiqqa/DocumentLibrary/WebLibraryStuff/WebLibraryManager.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Text.RegularExpressions;
 using Qiqqa.Common.Configuration;
 using Qiqqa.DocumentLibrary.BundleLibrary;
@@ -55,7 +56,12 @@ private WebLibraryManager()
             // Quick hack to swiftly profile this slow crap:
             SafeThreadPool.QueueUserWorkItem(o =>
             {
-                Qiqqa.Synchronisation.BusinessLogic.LibrarySyncManager.GenerateGlobalSyncDetail();
+                //
+                // Explicitly instruct the sync info collector to perform a swift scan, which DOES NOT include
+                // collecting the precise size of every document in every Qiqqa library (which itself is a *significant*
+                // file system load when you have any reasonably large libraries like I do.          [GHo]
+                //
+                Qiqqa.Synchronisation.BusinessLogic.LibrarySyncManager.GenerateGlobalSyncDetail(tally_library_storage_size: false);
             });
         }
 
@@ -126,6 +132,24 @@ private void AddLegacyWebLibrariesThatCanBeFoundOnDisk()
                     if (File.Exists(db_syncref_path))
                     {
                         Logging.Info("Skip the Qiqqa Internet/Intranet Sync directory and the sync DB ccontained therein: '{0}'", db_syncref_path);
+
+                        // https://github.com/jimmejardine/qiqqa-open-source/issues/145 :: delete lib file when it is very small and was illegally
+                        // constructed by a previous v82beta Qiqqa release:
+                        if (File.Exists(database_file))
+                        {
+                            long s3length = File.GetSize(database_file);
+                            if (6 * 1024 > s3length)
+                            {
+                                Logging.Warn("DELETE the wrongfully created DB file '{0}' in the Qiqqa Internet/Intranet Sync directory and the sync DB ccontained therein: '{1}', which has precedence!", database_file, db_syncref_path);
+
+                                FileTools.DeleteToRecycleBin(database_file);
+                            } 
+                            else 
+                            {
+                                Logging.Error("Inspect the Library DB file '{0}' in the Qiqqa Internet/Intranet Sync directory and the sync DB ccontained therein: '{1}', which MAY have precedence. Delete one of these manually to clean up your system as Qiqqa heuristics cannot tell which is the prevalent metadata database here!", database_file, db_syncref_path);
+                            }
+                        }
+
                         continue;
                     }
                     if (File.Exists(database_file))
@@ -259,10 +283,10 @@ public List<WebLibraryDetail> WebLibraryDetails_WorkingWebLibraries_All
         {
             get
             {
-                List<WebLibraryDetail> details = new List<WebLibraryDetail>();
+                HashSet<WebLibraryDetail> details = new HashSet<WebLibraryDetail>();
                 foreach (WebLibraryDetail wld in WebLibraryDetails_All_IncludingDeleted)
                 {
-                    if (!wld.Deleted && !wld.IsLocalGuestLibrary)
+                    if (!wld.Deleted)
                     {
                         details.Add(wld);
                     }
@@ -271,7 +295,7 @@ public List<WebLibraryDetail> WebLibraryDetails_WorkingWebLibraries_All
                 // Always add the guest library
                 details.Add(WebLibraryDetails_Guest);
 
-                return details;
+                return new List<WebLibraryDetail>(details);
             }
         }
 

Qiqqa/Documents/PDF/PDFDocument.cs

@@ -777,18 +777,20 @@ public bool DocumentExists
 
         [NonSerialized]
         private long document_size = 0;
-        public long DocumentSizeInBytes
+        public long GetDocumentSizeInBytes(long uncached_document_storage_size_override = -1)
         {
-            get
-            {
-                // When the document does not exist, the size is reported as ZERO.
-                // When we do not know yet whether the document exists, we'll have to go and check and find its size anyhow.
-                if (!DocumentExists) return 0;
-                if (document_size > 0) return document_size;
+            // When the document does not exist, the size is reported as ZERO.
+            // When we do not know yet whether the document exists, we'll have to go and check and find its size anyhow,
+            // unless the override value is sensible, i.e. **non-negative**.
+            if (!DocumentExists) return 0;
+            if (document_size > 0) return document_size;
 
-                document_size = File.GetSize(DocumentPath);
-                return document_size;
-            }
+            // Note: do NOT cache the override value!
+            if (uncached_document_storage_size_override >= 0) return uncached_document_storage_size_override;
+
+            // Execute file system query and cache its result:
+            document_size = File.GetSize(DocumentPath);
+            return document_size;
         }
 
         public bool IsVanillaReference => String.Compare(FileType, Constants.VanillaReferenceFileType, StringComparison.OrdinalIgnoreCase) == 0;
@@ -2102,14 +2104,11 @@ public bool DocumentExists
             }
         }
 
-        public long DocumentSizeInBytes
+        public long GetDocumentSizeInBytes(long uncached_document_storage_size_override = -1)
         {
-            get
+            lock (access_lock)
             {
-                lock (access_lock)
-                {
-                    return doc.DocumentSizeInBytes;
-                }
+                return doc.GetDocumentSizeInBytes(uncached_document_storage_size_override);
             }
         }
 

Qiqqa/Synchronisation/BusinessLogic/LibrarySyncManager.cs

@@ -66,7 +66,7 @@ internal void RefreshSyncControl(SyncControlGridItemSet scgis_previous, SyncCont
 
             SafeThreadPool.QueueUserWorkItem(o =>
             {
-                GlobalSyncDetail global_sync_detail = GenerateGlobalSyncDetail();
+                GlobalSyncDetail global_sync_detail = GenerateGlobalSyncDetail(tally_library_storage_size: true);
                 WPFDoEvents.InvokeInUIThread(() =>
                 {
                     WPFDoEvents.ResetHourglassCursor();
@@ -86,7 +86,7 @@ public void RequestSync(SyncRequest sync_request)
 
             SafeThreadPool.QueueUserWorkItem(o =>
             {
-                GlobalSyncDetail global_sync_detail = GenerateGlobalSyncDetail();
+                GlobalSyncDetail global_sync_detail = GenerateGlobalSyncDetail(tally_library_storage_size: true);
                 WPFDoEvents.InvokeInUIThread(() =>
                 {
                     WPFDoEvents.ResetHourglassCursor();
@@ -108,7 +108,7 @@ public void RequestSync(SyncRequest sync_request)
             });
         }
 
-        internal static GlobalSyncDetail GenerateGlobalSyncDetail()
+        internal static GlobalSyncDetail GenerateGlobalSyncDetail(bool tally_library_storage_size)
         {
             WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
 
@@ -136,7 +136,7 @@ internal static GlobalSyncDetail GenerateGlobalSyncDetail()
                     {
                         LibrarySyncDetail library_sync_detail = global_sync_detail.library_sync_details[i];
                         StatusManager.Instance.UpdateStatusBusy(StatusCodes.SYNC_META_GLOBAL, String.Format("Getting the local library details for {0}", library_sync_detail.web_library_detail.Title), i, global_sync_detail.library_sync_details.Count);
-                        library_sync_detail.local_library_sync_detail = GetLocalLibrarySyncDetail(library_sync_detail.web_library_detail.library);
+                        library_sync_detail.local_library_sync_detail = GetLocalLibrarySyncDetail(library_sync_detail.web_library_detail.library, tally_library_storage_size);
                     }
                 }
 
@@ -198,7 +198,7 @@ internal static GlobalSyncDetail GenerateGlobalSyncDetail()
         }
 
 
-        private static LibrarySyncDetail.LocalLibrarySyncDetail GetLocalLibrarySyncDetail(Library library)
+        private static LibrarySyncDetail.LocalLibrarySyncDetail GetLocalLibrarySyncDetail(Library library, bool tally_library_storage_size)
         {
             WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
 
@@ -220,8 +220,18 @@ private static LibrarySyncDetail.LocalLibrarySyncDetail GetLocalLibrarySyncDetai
                         continue;
                     }
 
-                    // We can only really tally up the documents that exist locally
-                    local_library_sync_detail.total_library_size += pdf_document.DocumentSizeInBytes;
+                    if (tally_library_storage_size)
+                    {
+                        // We can only really tally up the documents that exist locally
+                        local_library_sync_detail.total_library_size += pdf_document.GetDocumentSizeInBytes();
+                    }
+                    else
+                    {
+                        // fake it: take about 10KB per document, unless we already determined (and cached) the document size before.
+                        // This spares us the large overhead of querying the file system for every document in the 
+                        // (possibly huge) library.
+                        local_library_sync_detail.total_library_size += pdf_document.GetDocumentSizeInBytes(uncached_document_storage_size_override: 10 * 1024);
+                    }
                 }
                 catch (Exception ex)
                 {

Qiqqa/Synchronisation/GUI/SyncControlGridItem.cs

@@ -26,7 +26,7 @@ internal SyncControlGridItem(LibrarySyncDetail library_sync_detail)
 
         public string LibraryTitle => library_sync_detail.web_library_detail.Title;
 
-        public string SizeLocalString => String.Format("{0:N1}", SizeLocal / 1024.0 / 1024.0);
+        public string SizeLocalString => String.Format("{0:N1}", SizeLocal / (1024.0 * 1024.0));
 
         public long SizeLocal => library_sync_detail.local_library_sync_detail.total_library_size;
     }

Utilities/Files/FileTools.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Diagnostics;
+using System.Runtime.InteropServices;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
 using File = Alphaleonis.Win32.Filesystem.File;
 using Path = Alphaleonis.Win32.Filesystem.Path;
@@ -24,6 +25,58 @@ public static void Delete(string filename)
             }
         }
 
+        // https://stackoverflow.com/questions/17612/how-do-you-place-a-file-in-recycle-bin-instead-of-delete
+        private static class Recycle
+        {
+            private const int FO_DELETE = 3;
+            private const int FOF_ALLOWUNDO = 0x40;
+            private const int FOF_NOCONFIRMATION = 0x0010;
+
+            [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto, Pack = 1)]
+            public struct SHFILEOPSTRUCT
+            {
+                public IntPtr hwnd;
+                [MarshalAs(UnmanagedType.U4)]
+                public int wFunc;
+                public string pFrom;
+                public string pTo;
+                public short fFlags;
+                [MarshalAs(UnmanagedType.Bool)]
+                public bool fAnyOperationsAborted;
+                public IntPtr hNameMappings;
+                public string lpszProgressTitle;
+            }
+
+            [DllImport("shell32.dll", CharSet = CharSet.Auto)]
+            static extern int SHFileOperation(ref SHFILEOPSTRUCT FileOp);
+
+            public static void DeleteFileOperation(string filePath)
+            {
+                SHFILEOPSTRUCT fileop = new SHFILEOPSTRUCT();
+                fileop.wFunc = FO_DELETE;
+                fileop.pFrom = filePath + '\0' + '\0';
+                fileop.fFlags = FOF_ALLOWUNDO | FOF_NOCONFIRMATION;
+
+                SHFileOperation(ref fileop);
+            }
+        }
+
+        // Delete file to RecycleBin, i.e. ensure file is recoverable after delete, if user desires such.
+        public static void DeleteToRecycleBin(string filename)
+        {
+            try
+            {
+                if (File.Exists(filename))
+                {
+                    Recycle.DeleteFileOperation(filename);
+                }
+            }
+            catch (Exception ex)
+            {
+                Logging.Warn(ex, "There was a problem deleting file {0} to the Recycle Bin.", filename);
+            }
+        }
+
         public static bool MoveSafelyWithoutOverwriting(string source, string target)
         {
             if (File.Exists(source))

Utilities/Internet/GoogleScholar/GoogleScholarScrapePaperSet.cs

@@ -12,7 +12,9 @@ public enum PaperSetSource
         }
 
         // The inputs to the query - each is optional
+#pragma warning disable CS0414 // The field 'paper_set_source' is assigned a value which is never used
         private PaperSetSource paper_set_source;
+#pragma warning enable CS0414  // The field 'paper_set_source' is assigned a value which is never used
         private string search_query;
         private GoogleScholarScrapePaper related_gssp;