Skip to content

Commit

Permalink
[WIP] Initial work on DBMSProcessor batch entry insertion into ENTRY …
Browse files Browse the repository at this point in the history 
…table (#5814)

* Initial work on DBMSProcessor entry insertion into ENTRY table

* Change syntax for Oracle multi-row insert SQL statement

* Run tests also when source files changed

* Add to comment about Oracle

* Assume ResultSet is in order for setting shared IDs

* Add insertEntry for DBMSProcessor tests and fix PostgresSQLProcessor

* Fix SQL typo

* Separate table drops in Oracle tests

* Remove CI tests that were added in branch

* Work on unit test for DBMSProcessor insertEntries

* Fix bug in DBMSProcessorTest and simplify DBMSProcessor.FilterForBibEntryExistence

* Remove Oracle connection bug with wrong port

* Add Oracle insertIntoEntryTable

* Oracle connection fix - taken from fix_fields_sql branch

* Fix typo bug

* Clean up code

* Remove commented blocks

* Remove comment about needing a test that probably isn't necessary

* Manually merge fix_fields_sql OracleProcessor (just add method)

* Emphasize todo

* setSharedID into OracleProcessor entry table method

* Add shared id to preparedEntryStatement

* Make Oracle insertIntoEntryTable iterative - pasted from master - not yet tested

* Add fields to fields table in parallel

* Reset test trace length

* Fix checkStyle

* Revert port setting

Co-authored-by: Tobias Diez <tobiasdiez@gmx.de>
  • Loading branch information
@abepolk @tobiasdiez
abepolk and tobiasdiez committed Feb 19, 2020
1 parent e560220 commit 93196ee
Show file tree
Hide file tree
Showing 6 changed files with 182 additions and 96 deletions.
141 changes: 88 additions & 53 deletions src/main/java/org/jabref/logic/shared/DBMSProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,14 @@
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;

import org.jabref.logic.shared.exception.OfflineLockException;
import org.jabref.model.database.shared.DBMSType;
import org.jabref.model.database.shared.DatabaseConnection;
import org.jabref.model.database.shared.DatabaseConnectionProperties;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.SharedBibEntryData;
import org.jabref.model.entry.event.EntriesEventSource;
import org.jabref.model.entry.field.Field;
import org.jabref.model.entry.field.FieldFactory;
Expand Down Expand Up @@ -131,40 +133,61 @@ public void setupSharedDatabase() throws SQLException {
abstract String escape(String expression);

/**
* Inserts the given bibEntry into shared database.
* For use in test only. Inserts the BibEntry into the shared database.
*
* @param bibEntry {@link BibEntry} to be inserted
* @param bibEntry {@link BibEntry} to be inserted.
* */
public void insertEntry(BibEntry bibEntry) {
insertEntries(Collections.singletonList(bibEntry));
}

/**
* Inserts the List of BibEntry into the shared database.
*
* @param bibEntries List of {@link BibEntry} to be inserted
*/
public void insertEntry(BibEntry bibEntry) {
if (!checkForBibEntryExistence(bibEntry)) {
insertIntoEntryTable(bibEntry);
insertIntoFieldTable(bibEntry);
public void insertEntries(List<BibEntry> bibEntries) {
List<BibEntry> notYetExistingEntries = getNotYetExistingEntries(bibEntries);
if (notYetExistingEntries.isEmpty()) {
return;
}
insertIntoEntryTable(notYetExistingEntries);
insertIntoFieldTable(notYetExistingEntries);
}

/**
* Inserts the given bibEntry into ENTRY table.
* Inserts the given List of BibEntry into the ENTRY table.
*
* @param bibEntry {@link BibEntry} to be inserted
* @param bibEntries List of {@link BibEntry} to be inserted
*/
protected void insertIntoEntryTable(BibEntry bibEntry) {
// This is the only method to get generated keys which is accepted by MySQL, PostgreSQL and Oracle.
String insertIntoEntryQuery =
"INSERT INTO " +
escape("ENTRY") +
"(" +
escape("TYPE") +
") VALUES(?)";

try (PreparedStatement preparedEntryStatement = connection.prepareStatement(insertIntoEntryQuery,
new String[]{"SHARED_ID"})) {
protected void insertIntoEntryTable(List<BibEntry> bibEntries) {
StringBuilder insertIntoEntryQuery = new StringBuilder()
.append("INSERT INTO ")
.append(escape("ENTRY"))
.append("(")
.append(escape("TYPE"))
.append(") VALUES(?)");
// Number of commas is bibEntries.size() - 1
for (int i = 0; i < bibEntries.size() - 1; i++) {
insertIntoEntryQuery.append(", (?)");
}

preparedEntryStatement.setString(1, bibEntry.getType().getName());
try (PreparedStatement preparedEntryStatement = connection.prepareStatement(insertIntoEntryQuery.toString(),
new String[]{"SHARED_ID"})) {
for (int i = 0; i < bibEntries.size(); i++) {
preparedEntryStatement.setString(i + 1, bibEntries.get(i).getType().getName());
}
preparedEntryStatement.executeUpdate();

try (ResultSet generatedKeys = preparedEntryStatement.getGeneratedKeys()) {
// The following assumes that we get the generated keys in the order the entries were inserted
// This should be the case
for (BibEntry bibEntry : bibEntries) {
generatedKeys.next();
bibEntry.getSharedBibEntryData().setSharedID(generatedKeys.getInt(1));
}
if (generatedKeys.next()) {
bibEntry.getSharedBibEntryData().setSharedID(generatedKeys.getInt(1)); // set generated ID locally
LOGGER.error("Error: Some shared IDs left unassigned");
}
}
} catch (SQLException e) {
Expand All @@ -173,48 +196,52 @@ protected void insertIntoEntryTable(BibEntry bibEntry) {
}

/**
* Checks whether the given bibEntry already exists on shared database.
* Filters a list of BibEntry to and returns those which do not exist in the database
*
* @param bibEntry {@link BibEntry} to be checked
* @param bibEntries {@link BibEntry} to be checked
* @return <code>true</code> if existent, else <code>false</code>
*/
private boolean checkForBibEntryExistence(BibEntry bibEntry) {
private List<BibEntry> getNotYetExistingEntries(List<BibEntry> bibEntries) {

List<Integer> remoteIds = new ArrayList<>();
List<Integer> localIds = bibEntries.stream()
.map(BibEntry::getSharedBibEntryData)
.map(SharedBibEntryData::getSharedID)
.filter((id) -> id != -1)
.collect(Collectors.toList());
if (localIds.isEmpty()) {
return bibEntries;
}
try {
// Check if already exists
int sharedID = bibEntry.getSharedBibEntryData().getSharedID();
if (sharedID != -1) {
String selectQuery =
"SELECT * FROM " +
escape("ENTRY") +
" WHERE " +
escape("SHARED_ID") +
" = ?";

try (PreparedStatement preparedSelectStatement = connection.prepareStatement(selectQuery)) {
preparedSelectStatement.setInt(1, sharedID);
try (ResultSet resultSet = preparedSelectStatement.executeQuery()) {
if (resultSet.next()) {
return true;
}
}
StringBuilder selectQuery = new StringBuilder()
.append("SELECT * FROM ")
.append(escape("ENTRY"));

try (ResultSet resultSet = connection.createStatement().executeQuery(selectQuery.toString())) {
while (resultSet.next()) {
int id = resultSet.getInt("SHARED_ID");
remoteIds.add(id);
}
}
} catch (SQLException e) {
LOGGER.error("SQL Error: ", e);
}
return false;
}
return bibEntries.stream().filter((entry) ->
!remoteIds.contains(entry.getSharedBibEntryData().getSharedID()))
.collect(Collectors.toList());
}

/**
* Inserts the given bibEntry into FIELD table.
* Inserts the given list of BibEntry into FIELD table.
*
* @param bibEntry {@link BibEntry} to be inserted
* @param bibEntries {@link BibEntry} to be inserted
*/
protected void insertIntoFieldTable(BibEntry bibEntry) {
protected void insertIntoFieldTable(List<BibEntry> bibEntries) {
try {
// Inserting into FIELD table
// Coerce to ArrayList in order to use List.get()
List<Field> fields = new ArrayList<>(bibEntry.getFields());
List<List<Field>> fields = bibEntries.stream().map(bibEntry -> new ArrayList<>(bibEntry.getFields()))
.collect(Collectors.toList());
StringBuilder insertFieldQuery = new StringBuilder()
.append("INSERT INTO ")
.append(escape("FIELD"))
Expand All @@ -225,16 +252,24 @@ protected void insertIntoFieldTable(BibEntry bibEntry) {
.append(", ")
.append(escape("VALUE"))
.append(") VALUES(?, ?, ?)");
int numFields = 0;
for (List<Field> entryFields : fields) {
numFields += entryFields.size();
}
// Number of commas is fields.size() - 1
for (int i = 0; i < fields.size() - 1; i++) {
for (int i = 0; i < numFields - 1; i++) {
insertFieldQuery.append(", (?, ?, ?)");
}
try (PreparedStatement preparedFieldStatement = connection.prepareStatement(insertFieldQuery.toString())) {
for (int i = 0; i < fields.size(); i++) {
// columnIndex starts with 1
preparedFieldStatement.setInt((3 * i) + 1, bibEntry.getSharedBibEntryData().getSharedID());
preparedFieldStatement.setString((3 * i) + 2, fields.get(i).getName());
preparedFieldStatement.setString((3 * i) + 3, bibEntry.getField(fields.get(i)).get());
int fieldsCompleted = 0;
for (int entryIndex = 0; entryIndex < fields.size(); entryIndex++) {
for (int entryFieldsIndex = 0; entryFieldsIndex < fields.get(entryIndex).size(); entryFieldsIndex++) {
// columnIndex starts with 1
preparedFieldStatement.setInt((3 * fieldsCompleted) + 1, bibEntries.get(entryIndex).getSharedBibEntryData().getSharedID());
preparedFieldStatement.setString((3 * fieldsCompleted) + 2, fields.get(entryIndex).get(entryFieldsIndex).getName());
preparedFieldStatement.setString((3 * fieldsCompleted) + 3, bibEntries.get(entryIndex).getField(fields.get(entryIndex).get(entryFieldsIndex)).get());
fieldsCompleted += 1;
}
}
preparedFieldStatement.executeUpdate();
}
Expand Down
5 changes: 1 addition & 4 deletions src/main/java/org/jabref/logic/shared/DBMSSynchronizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,12 +82,9 @@ public void listen(EntriesAddedEvent event) {
if (isEventSourceAccepted(event) && checkCurrentConnection()) {
synchronizeLocalMetaData();
synchronizeLocalDatabase(); // Pull changes for the case that there were some
List<BibEntry> entries = event.getBibEntries();
for (BibEntry entry : entries) {
dbmsProcessor.insertEntry(entry);
dbmsProcessor.insertEntries(event.getBibEntries());
}
}
}

/**
* Listening method. Updates an existing shared {@link BibEntry}.
Expand Down
59 changes: 49 additions & 10 deletions src/main/java/org/jabref/logic/shared/OracleProcessor.java
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
package org.jabref.logic.shared;

import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;

import org.jabref.logic.shared.listener.OracleNotificationListener;
import org.jabref.model.database.shared.DatabaseConnection;
Expand Down Expand Up @@ -103,14 +105,48 @@ public void startNotificationListener(DBMSSynchronizer dbmsSynchronizer) {
}

@Override
protected void insertIntoFieldTable(BibEntry bibEntry) {
protected void insertIntoEntryTable(List<BibEntry> entries) {
try {
for (BibEntry entry : entries) {
String insertIntoEntryQuery =
"INSERT INTO " +
escape("ENTRY") +
"(" +
escape("TYPE") +
") VALUES(?)";

try (PreparedStatement preparedEntryStatement = connection.prepareStatement(insertIntoEntryQuery,
new String[]{"SHARED_ID"})) {

preparedEntryStatement.setString(1, entry.getType().getName());
preparedEntryStatement.executeUpdate();

try (ResultSet generatedKeys = preparedEntryStatement.getGeneratedKeys()) {
if (generatedKeys.next()) {
entry.getSharedBibEntryData().setSharedID(generatedKeys.getInt(1)); // set generated ID locally
}
}
}
}
} catch (SQLException e) {
LOGGER.error("SQL Error: ", e);
}
}

@Override
protected void insertIntoFieldTable(List<BibEntry> bibEntries) {
try {
// Inserting into FIELD table
// Coerce to ArrayList in order to use List.get()
List<Field> fields = new ArrayList<>(bibEntry.getFields());
List<List<Field>> fields = bibEntries.stream().map(entry -> new ArrayList<>(entry.getFields()))
.collect(Collectors.toList());
StringBuilder insertFieldQuery = new StringBuilder()
.append("INSERT ALL");
for (Field field : fields) {
int numFields = 0;
for (List<Field> entryFields : fields) {
numFields += entryFields.size();
}
for (int i = 0; i < numFields; i++) {
insertFieldQuery.append(" INTO ")
.append(escape("FIELD"))
.append(" (")
Expand All @@ -123,14 +159,17 @@ protected void insertIntoFieldTable(BibEntry bibEntry) {
}
insertFieldQuery.append(" SELECT * FROM DUAL");
try (PreparedStatement preparedFieldStatement = connection.prepareStatement(insertFieldQuery.toString())) {
for (int i = 0; i < fields.size(); i++) {
// columnIndex starts with 1
preparedFieldStatement.setInt((3 * i) + 1, bibEntry.getSharedBibEntryData().getSharedID());
preparedFieldStatement.setString((3 * i) + 2, fields.get(i).getName());
preparedFieldStatement.setString((3 * i) + 3, bibEntry.getField(fields.get(i)).get());
int fieldsCompleted = 0;
for (int entryIndex = 0; entryIndex < fields.size(); entryIndex++) {
for (int entryFieldsIndex = 0; entryFieldsIndex < fields.get(entryIndex).size(); entryFieldsIndex++) {
// columnIndex starts with 1
preparedFieldStatement.setInt((3 * fieldsCompleted) + 1, bibEntries.get(entryIndex).getSharedBibEntryData().getSharedID());
preparedFieldStatement.setString((3 * fieldsCompleted) + 2, fields.get(entryIndex).get(entryFieldsIndex).getName());
preparedFieldStatement.setString((3 * fieldsCompleted) + 3, bibEntries.get(entryIndex).getField(fields.get(entryIndex).get(entryFieldsIndex)).get());
fieldsCompleted += 1;
}
}
preparedFieldStatement.executeUpdate();
}
preparedFieldStatement.executeUpdate(); }
} catch (SQLException e) {
LOGGER.error("SQL Error: ", e);
}
Expand Down
35 changes: 22 additions & 13 deletions src/main/java/org/jabref/logic/shared/PostgreSQLProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;

import org.jabref.JabRefExecutorService;
import org.jabref.logic.shared.listener.PostgresSQLNotificationListener;
Expand Down Expand Up @@ -49,25 +50,33 @@ public void setUp() throws SQLException {
}

@Override
protected void insertIntoEntryTable(BibEntry bibEntry) {
// Inserting into ENTRY table
protected void insertIntoEntryTable(List<BibEntry> bibEntries) {
StringBuilder insertIntoEntryQuery = new StringBuilder()
.append("INSERT INTO ")
.append(escape("ENTRY"))
.append("(")
.append(escape("TYPE"))
.append(") VALUES(?)");

// This is the only method to get generated keys which is accepted by MySQL, PostgreSQL and Oracle.
.append("INSERT INTO ")
.append(escape("ENTRY"))
.append("(")
.append(escape("TYPE"))
.append(") VALUES(?)");
// Number of commas is bibEntries.size() - 1
for (int i = 0; i < bibEntries.size() - 1; i++) {
insertIntoEntryQuery.append(", (?)");
}
try (PreparedStatement preparedEntryStatement = connection.prepareStatement(insertIntoEntryQuery.toString(),
Statement.RETURN_GENERATED_KEYS)) {

preparedEntryStatement.setString(1, bibEntry.getType().getName());
Statement.RETURN_GENERATED_KEYS)) {
for (int i = 0; i < bibEntries.size(); i++) {
preparedEntryStatement.setString(i + 1, bibEntries.get(i).getType().getName());
}
preparedEntryStatement.executeUpdate();

try (ResultSet generatedKeys = preparedEntryStatement.getGeneratedKeys()) {
// The following assumes that we get the generated keys in the order the entries were inserted
// This should be the case
for (BibEntry bibEntry : bibEntries) {
generatedKeys.next();
bibEntry.getSharedBibEntryData().setSharedID(generatedKeys.getInt(1));
}
if (generatedKeys.next()) {
bibEntry.getSharedBibEntryData().setSharedID(generatedKeys.getInt(1)); // set generated ID locally
LOGGER.error("Error: Some shared IDs left unassigned");
}
}
} catch (SQLException e) {
Expand Down
Loading

0 comments on commit 93196ee

Please sign in to comment.