Rework journal abbreviation caching (#6304)

* Rework journal abbreviation caching * Add ADR-0010 * Remove IEEE abbreviations * Convert abbreviations as part of build process * Fix tests * Update according to review feedback * Fix bug * Include feedback * Fix test * Fix localization Co-authored-by: Oliver Kopp <kopp.dev@gmail.com>
JabRef · May 1, 2020 · a123eb1 · a123eb1
1 parent 5758f8b
commit a123eb1
Show file tree

Hide file tree

Showing 70 changed files with 529 additions and 1,202 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -51,6 +51,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 
 - We removed the obsolete `External programs / Open PDF` section in the preferences, as the default application to open PDFs is now set in the `Manage external file types` dialog. [#6130](https://github.com/JabRef/jabref/pull/6130)
 - We removed the option to configure whether a `.bib.bak` file should be generated upon save. It is now always enabled. Documentation at <https://docs.jabref.org/general/autosave>. [#6092](https://github.com/JabRef/jabref/issues/6092)
+- We removed the built-in list of IEEE journal abbreviations using BibTeX strings. If you still want to use them, you have to download them separately from <https://abbrv.jabref.org>.
 
 ## [5.0] – 2020-03-06
 

diff --git a/build.gradle b/build.gradle
@@ -1,5 +1,6 @@
 import groovy.json.JsonSlurper
 import org.gradle.internal.os.OperatingSystem
+import org.jabref.build.JournalAbbreviationConverter
 import org.jabref.build.antlr.JabRefAntlrPlugin
 import org.jabref.build.localization.LocalizationPlugin
 import org.jabref.build.xjc.XjcPlugin
@@ -8,13 +9,6 @@ import org.jabref.build.xjc.XjcTask
 // to update the gradle wrapper, execute
 // ./gradlew wrapper --gradle-version=6.0 --distribution-type=bin
 
-buildscript {
-    repositories {
-        mavenLocal()
-        jcenter()
-    }
-}
-
 plugins {
     id 'application'
     id "com.simonharrer.modernizer" version '1.8.0-1'
@@ -116,6 +110,7 @@ dependencies {
     implementation 'org.apache.pdfbox:xmpbox:2.0.19'
 
     implementation group: 'org.apache.commons', name: 'commons-csv', version: '1.8'
+    implementation 'com.h2database:h2-mvstore:1.4.200'
 
     implementation group: 'org.apache.tika', name: 'tika-core', version: '1.24'
 
@@ -297,9 +292,10 @@ task generateSource(dependsOn: ["generateBstGrammarSource",
                                 "generateMedlineSource",
                                 "generateBibtexmlSource",
                                 "generateEndnoteSource",
-                                "generateModsSource"]) {
+                                "generateModsSource",
+                                "generateJournalAbbreviationList"]) {
     group = 'JabRef'
-    description 'Generates all Java source files.'
+    description 'Generates all necessary (Java) source files.'
 }
 
 task generateBstGrammarSource(type: org.jabref.build.antlr.AntlrTask) {
@@ -359,6 +355,14 @@ task generateModsSource(type: XjcTask) {
     arguments = '-npa'
 }
 
+task generateJournalAbbreviationList(type: JournalAbbreviationConverter) {
+    group = 'JabRef'
+    description = "Converts the comma-separated journal abbreviation file to a H2 MVStore."
+
+    inputDir = file("src/main/resources/journals")
+    outputDir = file("src/main/resources/journals")
+}
+
 tasks.withType(JavaCompile) {
     options.encoding = 'UTF-8'
 }

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -0,0 +1,19 @@
+apply plugin: 'java'
+
+repositories {
+    jcenter()
+}
+
+dependencies {
+    compile 'com.h2database:h2-mvstore:1.4.200'
+    compile 'org.apache.commons:commons-csv:1.8'
+    compile 'org.slf4j:slf4j-api:2.0.0-alpha1'
+}
+
+sourceSets{
+    main {
+        java {
+            srcDir "${project.rootDir}/../src/main/java/org/jabref/logic/journals"
+        }
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/jabref/build/JournalAbbreviationConverter.groovy b/buildSrc/src/main/groovy/org/jabref/build/JournalAbbreviationConverter.groovy
@@ -0,0 +1,43 @@
+package org.jabref.build
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.tasks.*
+import org.h2.mvstore.MVMap
+import org.h2.mvstore.MVStore
+import org.jabref.logic.journals.JournalAbbreviationLoader
+
+import java.util.stream.Collectors
+
+abstract class JournalAbbreviationConverter extends DefaultTask {
+    @PathSensitive(PathSensitivity.NAME_ONLY)
+    @InputDirectory
+    abstract DirectoryProperty getInputDir()
+
+    @OutputDirectory
+    abstract DirectoryProperty getOutputDir()
+
+    @TaskAction
+    def convert() {
+        def targetFile = outputDir.file("journalList.mv").get().asFile
+        targetFile.delete()
+        MVStore.open(targetFile.toString()).withCloseable { store ->
+            MVMap<String, String> fullToAbbreviation = store.openMap("FullToAbbreviation")
+            MVMap<String, String> abbreviationToFull = store.openMap("AbbreviationToFull")
+
+            inputDir.getAsFileTree().filter({ File f -> f.name.endsWith(".csv") }).getFiles().each { file ->
+                def abbreviations = JournalAbbreviationLoader.readJournalListFromFile(file.toPath())
+                fullToAbbreviation.putAll(
+                        abbreviations
+                                .stream()
+                                .collect(Collectors.toMap({ abbreviation -> abbreviation.getName() }, { abbreviation -> abbreviation.getAbbreviation() }))
+                )
+
+                abbreviations
+                        .forEach({ abbreviation ->
+                            abbreviationToFull.putIfAbsent(abbreviation.getAbbreviation(), abbreviation.getName())
+                        })
+            }
+        }
+    }
+}
diff --git a/docs/adr/0010-use-h2-as-internal-database.md b/docs/adr/0010-use-h2-as-internal-database.md
@@ -0,0 +1,24 @@
+# Use H2 as Internal SQL Database
+
+## Context and Problem Statement
+
+We need to store data internally in a structured way to gain performance.
+
+## Decision Drivers
+
+* Easy to integrate
+* Easy to use
+* Common technology
+
+## Considered Options
+
+* [H2 Database Engine](http://www.h2database.com/html/main.html)
+* [SQLite](https://www.sqlite.org/index.html)
+
+## Decision Outcome
+
+Chosen option: "H2 Database Engine", because it was straight-forward to use.
+
+## Links
+
+* [Comparison at SQL Workbench](https://www.sql-workbench.eu/dbms_comparison.html)
diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java
@@ -88,4 +88,5 @@
     requires flexmark.ext.gfm.tasklist;
     requires flexmark.util.ast;
     requires flexmark.util.data;
+    requires com.h2database.mvstore;
 }
diff --git a/src/main/java/org/jabref/Globals.java b/src/main/java/org/jabref/Globals.java
@@ -16,7 +16,7 @@
 import org.jabref.gui.util.ThemeLoader;
 import org.jabref.logic.exporter.ExporterFactory;
 import org.jabref.logic.importer.ImportFormatReader;
-import org.jabref.logic.journals.JournalAbbreviationLoader;
+import org.jabref.logic.journals.JournalAbbreviationRepository;
 import org.jabref.logic.protectedterms.ProtectedTermsLoader;
 import org.jabref.logic.remote.server.RemoteListenerServerLifecycle;
 import org.jabref.logic.util.BuildInfo;
@@ -56,7 +56,7 @@ public class Globals {
      * This field is initialized upon startup.
      * Only GUI code is allowed to access it, logic code should use dependency injection.
      */
-    public static JournalAbbreviationLoader journalAbbreviationLoader;
+    public static JournalAbbreviationRepository journalAbbreviationRepository;
 
     /**
      * This field is initialized upon startup.

diff --git a/src/main/java/org/jabref/JabRefMain.java b/src/main/java/org/jabref/JabRefMain.java
@@ -161,14 +161,14 @@ private static boolean handleMultipleAppInstances(String[] args) {
 
     private static void applyPreferences(JabRefPreferences preferences) {
         // Read list(s) of journal names and abbreviations
-        Globals.journalAbbreviationLoader = new JournalAbbreviationLoader();
+        Globals.journalAbbreviationRepository = JournalAbbreviationLoader.loadRepository(Globals.prefs.getJournalAbbreviationPreferences());
 
         // Build list of Import and Export formats
         Globals.IMPORT_FORMAT_READER.resetImportFormats(Globals.prefs.getImportFormatPreferences(),
                                                         Globals.prefs.getXMPPreferences(), Globals.getFileUpdateMonitor());
         Globals.entryTypesManager.addCustomOrModifiedTypes(preferences.loadBibEntryTypes(BibDatabaseMode.BIBTEX),
                 preferences.loadBibEntryTypes(BibDatabaseMode.BIBLATEX));
-        Globals.exportFactory = Globals.prefs.getExporterFactory(Globals.journalAbbreviationLoader);
+        Globals.exportFactory = Globals.prefs.getExporterFactory(Globals.journalAbbreviationRepository);
 
         // Initialize protected terms loader
         Globals.protectedTermsLoader = new ProtectedTermsLoader(Globals.prefs.getProtectedTermsPreferences());

diff --git a/src/main/java/org/jabref/cli/ArgumentProcessor.java b/src/main/java/org/jabref/cli/ArgumentProcessor.java
@@ -457,9 +457,9 @@ private void importPreferences() {
             Globals.prefs.importPreferences(cli.getPreferencesImport());
             Globals.entryTypesManager.addCustomOrModifiedTypes(Globals.prefs.loadBibEntryTypes(BibDatabaseMode.BIBTEX),
                     Globals.prefs.loadBibEntryTypes(BibDatabaseMode.BIBLATEX));
-            List<TemplateExporter> customExporters = Globals.prefs.getCustomExportFormats(Globals.journalAbbreviationLoader);
+            List<TemplateExporter> customExporters = Globals.prefs.getCustomExportFormats(Globals.journalAbbreviationRepository);
             LayoutFormatterPreferences layoutPreferences = Globals.prefs
-                    .getLayoutFormatterPreferences(Globals.journalAbbreviationLoader);
+                    .getLayoutFormatterPreferences(Globals.journalAbbreviationRepository);
             SavePreferences savePreferences = Globals.prefs.loadForExportFromPreferences();
             XmpPreferences xmpPreferences = Globals.prefs.getXMPPreferences();
             Globals.exportFactory = ExporterFactory.create(customExporters, layoutPreferences, savePreferences, xmpPreferences);

diff --git a/src/main/java/org/jabref/gui/BasePanel.java b/src/main/java/org/jabref/gui/BasePanel.java
@@ -399,7 +399,7 @@ public void setupMainPanel() {
     private void setupAutoCompletion() {
         AutoCompletePreferences autoCompletePreferences = preferences.getAutoCompletePreferences();
         if (autoCompletePreferences.shouldAutoComplete()) {
-            suggestionProviders = new SuggestionProviders(getDatabase(), autoCompletePreferences, Globals.journalAbbreviationLoader);
+            suggestionProviders = new SuggestionProviders(getDatabase(), Globals.journalAbbreviationRepository);
         } else {
             // Create empty suggestion providers if auto completion is deactivated
             suggestionProviders = new SuggestionProviders();

diff --git a/src/main/java/org/jabref/gui/DefaultInjector.java b/src/main/java/org/jabref/gui/DefaultInjector.java
@@ -8,7 +8,7 @@
 import org.jabref.JabRefGUI;
 import org.jabref.gui.keyboard.KeyBindingRepository;
 import org.jabref.gui.util.TaskExecutor;
-import org.jabref.logic.journals.JournalAbbreviationLoader;
+import org.jabref.logic.journals.JournalAbbreviationRepository;
 import org.jabref.logic.protectedterms.ProtectedTermsLoader;
 import org.jabref.model.util.FileUpdateMonitor;
 import org.jabref.preferences.PreferencesService;
@@ -36,8 +36,8 @@ private static Object createDependency(Class<?> clazz) {
             return Globals.prefs;
         } else if (clazz == KeyBindingRepository.class) {
             return Globals.getKeyPrefs();
-        } else if (clazz == JournalAbbreviationLoader.class) {
-            return Globals.journalAbbreviationLoader;
+        } else if (clazz == JournalAbbreviationRepository.class) {
+            return Globals.journalAbbreviationRepository;
         } else if (clazz == StateManager.class) {
             return Globals.stateManager;
         } else if (clazz == FileUpdateMonitor.class) {

diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java
@@ -771,11 +771,11 @@ private MenuBar createMenu() {
                 new SeparatorMenuItem(),
 
                 factory.createSubMenu(StandardActions.ABBREVIATE,
-                        factory.createMenuItem(StandardActions.ABBREVIATE_DEFAULT, new AbbreviateAction(StandardActions.ABBREVIATE_DEFAULT, this, dialogService, stateManager, prefs)),
-                        factory.createMenuItem(StandardActions.ABBREVIATE_MEDLINE, new AbbreviateAction(StandardActions.ABBREVIATE_MEDLINE, this, dialogService, stateManager, prefs)),
-                        factory.createMenuItem(StandardActions.ABBREVIATE_SHORTEST_UNIQUE, new AbbreviateAction(StandardActions.ABBREVIATE_SHORTEST_UNIQUE, this, dialogService, stateManager, prefs))),
+                        factory.createMenuItem(StandardActions.ABBREVIATE_DEFAULT, new AbbreviateAction(StandardActions.ABBREVIATE_DEFAULT, this, dialogService, stateManager)),
+                        factory.createMenuItem(StandardActions.ABBREVIATE_MEDLINE, new AbbreviateAction(StandardActions.ABBREVIATE_MEDLINE, this, dialogService, stateManager)),
+                        factory.createMenuItem(StandardActions.ABBREVIATE_SHORTEST_UNIQUE, new AbbreviateAction(StandardActions.ABBREVIATE_SHORTEST_UNIQUE, this, dialogService, stateManager))),
 
-                factory.createMenuItem(StandardActions.UNABBREVIATE, new AbbreviateAction(StandardActions.UNABBREVIATE, this, dialogService, stateManager, prefs))
+                factory.createMenuItem(StandardActions.UNABBREVIATE, new AbbreviateAction(StandardActions.UNABBREVIATE, this, dialogService, stateManager))
         );
 
         Menu lookupIdentifiers = factory.createSubMenu(StandardActions.LOOKUP_DOC_IDENTIFIER);

diff --git a/src/main/java/org/jabref/gui/autocompleter/JournalsSuggestionProvider.java b/src/main/java/org/jabref/gui/autocompleter/JournalsSuggestionProvider.java
@@ -1,24 +1,25 @@
 package org.jabref.gui.autocompleter;
 
-import org.jabref.logic.journals.JournalAbbreviationLoader;
+import java.util.stream.Stream;
+
+import org.jabref.logic.journals.JournalAbbreviationRepository;
 import org.jabref.model.database.BibDatabase;
 import org.jabref.model.entry.field.Field;
 
+import com.google.common.collect.Streams;
+
 public class JournalsSuggestionProvider extends FieldValueSuggestionProvider {
 
-    JournalsSuggestionProvider(Field field, BibDatabase database, AutoCompletePreferences preferences,
-                               JournalAbbreviationLoader abbreviationLoader) {
+    private final JournalAbbreviationRepository repository;
+
+    JournalsSuggestionProvider(Field field, BibDatabase database, JournalAbbreviationRepository repository) {
         super(field, database);
 
-        /*
-        TODO: Reenable
+        this.repository = repository;
+    }
 
-        JournalAbbreviationPreferences journalAbbreviationPreferences = preferences.getJournalAbbreviationPreferences();
-        List<String> journals = abbreviationLoader.getRepository(journalAbbreviationPreferences)
-                .getAbbreviations().stream()
-                .map(Abbreviation::getName)
-                .collect(Collectors.toList());
-        addPossibleSuggestions(journals);
-         */
+    @Override
+    public Stream<String> getSource() {
+        return Streams.concat(super.getSource(), repository.getFullNames().stream());
     }
 }
diff --git a/src/main/java/org/jabref/gui/autocompleter/SuggestionProviders.java b/src/main/java/org/jabref/gui/autocompleter/SuggestionProviders.java
@@ -1,9 +1,8 @@
 package org.jabref.gui.autocompleter;
 
-import java.util.Objects;
 import java.util.Set;
 
-import org.jabref.logic.journals.JournalAbbreviationLoader;
+import org.jabref.logic.journals.JournalAbbreviationRepository;
 import org.jabref.model.database.BibDatabase;
 import org.jabref.model.entry.field.Field;
 import org.jabref.model.entry.field.FieldProperty;
@@ -13,13 +12,11 @@ public class SuggestionProviders {
 
     private final boolean isEmpty;
     private BibDatabase database;
-    private AutoCompletePreferences preferences;
-    private JournalAbbreviationLoader abbreviationLoader;
+    private JournalAbbreviationRepository abbreviationRepository;
 
-    public SuggestionProviders(BibDatabase database, AutoCompletePreferences preferences, JournalAbbreviationLoader abbreviationLoader) {
+    public SuggestionProviders(BibDatabase database, JournalAbbreviationRepository abbreviationRepository) {
         this.database = database;
-        this.preferences = Objects.requireNonNull(preferences);
-        this.abbreviationLoader = abbreviationLoader;
+        this.abbreviationRepository = abbreviationRepository;
         this.isEmpty = false;
     }
 
@@ -39,7 +36,7 @@ public SuggestionProvider<?> getForField(Field field) {
             return new BibEntrySuggestionProvider(database);
         } else if (fieldProperties.contains(FieldProperty.JOURNAL_NAME)
                 || StandardField.PUBLISHER.equals(field)) {
-            return new JournalsSuggestionProvider(field, database, preferences, abbreviationLoader);
+            return new JournalsSuggestionProvider(field, database, abbreviationRepository);
         } else {
             return new WordSuggestionProvider(field, database);
         }

diff --git a/src/main/java/org/jabref/gui/cleanup/CleanupAction.java b/src/main/java/org/jabref/gui/cleanup/CleanupAction.java
@@ -92,7 +92,7 @@ private void doCleanup(BibDatabaseContext databaseContext, CleanupPreset preset,
         // Create and run cleaner
         CleanupWorker cleaner = new CleanupWorker(
                 databaseContext,
-                preferences.getCleanupPreferences(Globals.journalAbbreviationLoader));
+                preferences.getCleanupPreferences(Globals.journalAbbreviationRepository));
 
         List<FieldChange> changes = cleaner.cleanup(preset, entry);
 

diff --git a/src/main/java/org/jabref/gui/edit/CopyMoreAction.java b/src/main/java/org/jabref/gui/edit/CopyMoreAction.java
@@ -164,7 +164,7 @@ private void copyKeyAndTitle() {
         StringReader layoutString = new StringReader("\\bibtexkey - \\begin{title}\\format[RemoveBrackets]{\\title}\\end{title}\n");
         Layout layout;
         try {
-            layout = new LayoutHelper(layoutString, preferencesService.getLayoutFormatterPreferences(Globals.journalAbbreviationLoader)).getLayoutFromText();
+            layout = new LayoutHelper(layoutString, preferencesService.getLayoutFormatterPreferences(Globals.journalAbbreviationRepository)).getLayoutFromText();
         } catch (IOException e) {
             LOGGER.info("Could not get layout.", e);
             return;

diff --git a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java
@@ -16,7 +16,7 @@
 import org.jabref.gui.externalfiletype.ExternalFileTypes;
 import org.jabref.gui.icon.IconTheme;
 import org.jabref.gui.util.TaskExecutor;
-import org.jabref.logic.journals.JournalAbbreviationLoader;
+import org.jabref.logic.journals.JournalAbbreviationRepository;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.entry.BibEntry;
@@ -29,8 +29,8 @@ public class DeprecatedFieldsTab extends FieldsEditorTab {
 
     private final BibEntryTypesManager entryTypesManager;
 
-    public DeprecatedFieldsTab(BibDatabaseContext databaseContext, SuggestionProviders suggestionProviders, UndoManager undoManager, DialogService dialogService, JabRefPreferences preferences, BibEntryTypesManager entryTypesManager, ExternalFileTypes externalFileTypes, TaskExecutor taskExecutor, JournalAbbreviationLoader journalAbbreviationLoader) {
-        super(false, databaseContext, suggestionProviders, undoManager, dialogService, preferences, externalFileTypes, taskExecutor, journalAbbreviationLoader);
+    public DeprecatedFieldsTab(BibDatabaseContext databaseContext, SuggestionProviders suggestionProviders, UndoManager undoManager, DialogService dialogService, JabRefPreferences preferences, BibEntryTypesManager entryTypesManager, ExternalFileTypes externalFileTypes, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository) {
+        super(false, databaseContext, suggestionProviders, undoManager, dialogService, preferences, externalFileTypes, taskExecutor, journalAbbreviationRepository);
         this.entryTypesManager = entryTypesManager;
 
         setText(Localization.lang("Deprecated fields"));