Commit index updates in batch

To avoid commit all together during rebuilding index, especially for Cassandra backend, which has batch limit 65535 fixed: #144 implemented: #82 Change-Id: I88ff4bc878bc24122f0bb6ecf9964246a083b9ab
apache · Oct 30, 2018 · c02ed71 · c02ed71
1 parent 15e9fc6
commit c02ed71
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 0 deletions.
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/HugeGraph.java
@@ -550,6 +550,12 @@ public boolean closed() {
             return this.refs.get() == 0;
         }
 
+        public void commitIfGtSize(int size) {
+            // Only committing graph transaction data if reaching batch size
+            // is OK, bacause schema transaction is auto committed.
+            this.graphTransaction().commitIfGtSize(size);
+        }
+
         @Override
         public void commit() {
             try {

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/Transaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/Transaction.java
@@ -23,6 +23,8 @@ public interface Transaction {
 
     public void commit() throws BackendException;
 
+    public void commitIfGtSize(int size) throws BackendException;
+
     public void rollback() throws BackendException;
 
     public boolean autoCommit();

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/AbstractTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/AbstractTransaction.java
@@ -170,6 +170,12 @@ public void commit() throws BackendException {
         }
     }
 
+    public void commitIfGtSize(int size) throws BackendException {
+        if (this.mutationSize() >= size) {
+            this.commit();
+        }
+    }
+
     @Watched(prefix = "tx")
     @Override
     public void rollback() throws BackendException {

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/tx/GraphIndexTransaction.java
@@ -77,6 +77,7 @@ public class GraphIndexTransaction extends AbstractTransaction {
 
     private static final String INDEX_EMPTY_SYM = "\u0000";
     private static final Query EMPTY_QUERY = new ConditionQuery(null);
+    private static final int REBUILD_COMMIT_BATCH = 1000;
 
     private final Analyzer textAnalyzer;
 
@@ -1111,6 +1112,9 @@ public void rebuildIndex(HugeType type, Id label,
                     HugeVertex vertex = (HugeVertex) itor.next();
                     for (Id id : indexLabelIds) {
                         this.updateIndex(id, vertex, false);
+                        // Commit per small batch to avoid too much data
+                        // in single commit, especially for Cassandra backend.
+                        this.commitIfGtSize(REBUILD_COMMIT_BATCH);
                     }
                 }
             } else {
@@ -1123,6 +1127,9 @@ public void rebuildIndex(HugeType type, Id label,
                     HugeEdge edge = (HugeEdge) itor.next();
                     for (Id id : indexLabelIds) {
                         this.updateIndex(id, edge, false);
+                        // Commit per small batch to avoid too much data
+                        // in single commit, especially for Cassandra backend.
+                        this.commitIfGtSize(REBUILD_COMMIT_BATCH);
                     }
                 }
             }