From a495eeab694cb0864082cc0ab316ba4f05133bea Mon Sep 17 00:00:00 2001 From: Jermy Li Date: Wed, 3 Mar 2021 19:46:57 +0800 Subject: [PATCH] avoid deleteRange() as possible for performance (#1375) Change-Id: I1667e062b8a6a99c52d534a314a0af4bfecb59a0 --- .../store/rocksdb/RocksDBSessions.java | 6 +++-- .../store/rocksdb/RocksDBStdSessions.java | 19 ++++++++++++--- .../backend/store/rocksdb/RocksDBTable.java | 2 +- .../backend/store/rocksdb/RocksDBTables.java | 23 +++++++++++++++---- .../store/rocksdbsst/RocksDBSstSessions.java | 21 ++++++++++++----- .../unit/rocksdb/BaseRocksDBUnitTest.java | 3 ++- .../unit/rocksdb/RocksDBPerfTest.java | 2 +- .../unit/rocksdb/RocksDBSessionsTest.java | 20 ++++++++-------- 8 files changed, 68 insertions(+), 28 deletions(-) diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBSessions.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBSessions.java index 6a7245cbfe..1889344ed5 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBSessions.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBSessions.java @@ -79,9 +79,11 @@ public static abstract class Session extends AbstractBackendSession { public abstract void merge(String table, byte[] key, byte[] value); public abstract void increase(String table, byte[] key, byte[] value); - public abstract void remove(String table, byte[] key); - public abstract void delete(String table, byte[] keyFrom, byte[] keyTo); public abstract void delete(String table, byte[] key); + public abstract void deleteSingle(String table, byte[] key); + public abstract void deletePrefix(String table, byte[] key); + public abstract void deleteRange(String table, + byte[] keyFrom, byte[] keyTo); public abstract byte[] get(String table, byte[] key); diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index f6788d7de0..3bbfd9237d 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -840,7 +840,20 @@ public void increase(String table, byte[] key, byte[] value) { * Delete a record by key from a table */ @Override - public void remove(String table, byte[] key) { + public void delete(String table, byte[] key) { + try (CFHandle cf = cf(table)) { + this.batch.delete(cf.get(), key); + } catch (RocksDBException e) { + throw new BackendException(e); + } + } + + /** + * Delete the only one version of a record by key from a table + * NOTE: requires that the key exists and was not overwritten. + */ + @Override + public void deleteSingle(String table, byte[] key) { try (CFHandle cf = cf(table)) { this.batch.singleDelete(cf.get(), key); } catch (RocksDBException e) { @@ -852,7 +865,7 @@ public void remove(String table, byte[] key) { * Delete a record by key(or prefix with key) from a table */ @Override - public void delete(String table, byte[] key) { + public void deletePrefix(String table, byte[] key) { byte[] keyFrom = key; byte[] keyTo = Arrays.copyOf(key, key.length); keyTo = BinarySerializer.increaseOne(keyTo); @@ -867,7 +880,7 @@ public void delete(String table, byte[] key) { * Delete a range of keys from a table */ @Override - public void delete(String table, byte[] keyFrom, byte[] keyTo) { + public void deleteRange(String table, byte[] keyFrom, byte[] keyTo) { try (CFHandle cf = cf(table)) { this.batch.deleteRange(cf.get(), keyFrom, keyTo); } catch (RocksDBException e) { diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java index 6dde279b67..ba52f7e56c 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -103,7 +103,7 @@ public void delete(Session session, BackendEntry entry) { } else { for (BackendColumn col : entry.columns()) { assert entry.belongToMe(col) : entry; - session.remove(this.table(), col.name); + session.delete(this.table(), col.name); } } } diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java index 6a18edb87e..5971bcb638 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -75,7 +75,20 @@ private static long l(byte[] bytes) { } } - public static class VertexLabel extends RocksDBTable { + public static class SchemaTable extends RocksDBTable { + + public SchemaTable(String database, String table) { + super(database, table); + } + + @Override + public void delete(Session session, BackendEntry entry) { + assert entry.columns().isEmpty(); + session.deletePrefix(this.table(), entry.id().asBytes()); + } + } + + public static class VertexLabel extends SchemaTable { public static final String TABLE = HugeType.VERTEX_LABEL.string(); @@ -84,7 +97,7 @@ public VertexLabel(String database) { } } - public static class EdgeLabel extends RocksDBTable { + public static class EdgeLabel extends SchemaTable { public static final String TABLE = HugeType.EDGE_LABEL.string(); @@ -93,7 +106,7 @@ public EdgeLabel(String database) { } } - public static class PropertyKey extends RocksDBTable { + public static class PropertyKey extends SchemaTable { public static final String TABLE = HugeType.PROPERTY_KEY.string(); @@ -102,7 +115,7 @@ public PropertyKey(String database) { } } - public static class IndexLabel extends RocksDBTable { + public static class IndexLabel extends SchemaTable { public static final String TABLE = HugeType.INDEX_LABEL.string(); @@ -168,7 +181,7 @@ public void delete(Session session, BackendEntry entry) { */ for (BackendEntry.BackendColumn column : entry.columns()) { // Don't assert entry.belongToMe(column), length-prefix is 1* - session.delete(this.table(), column.name); + session.deletePrefix(this.table(), column.name); } } } diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java index d84d21ea68..1e5cce927e 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdbsst/RocksDBSstSessions.java @@ -325,24 +325,33 @@ public void increase(String table, byte[] key, byte[] value) { * Delete a record by key from a table */ @Override - public void remove(String table, byte[] key) { - throw new NotSupportException("RocksDBSstStore remove()"); + public void delete(String table, byte[] key) { + throw new NotSupportException("RocksDBSstStore delete()"); + } + + /** + * Delete the only one version of a record by key from a table + * NOTE: requires that the key exists and was not overwritten. + */ + @Override + public void deleteSingle(String table, byte[] key) { + throw new NotSupportException("RocksDBSstStore deleteSingle()"); } /** * Delete a record by key(or prefix with key) from a table */ @Override - public void delete(String table, byte[] key) { - throw new NotSupportException("RocksDBSstStore delete()"); + public void deletePrefix(String table, byte[] key) { + throw new NotSupportException("RocksDBSstStore deletePrefix()"); } /** * Delete a range of keys from a table */ @Override - public void delete(String table, byte[] keyFrom, byte[] keyTo) { - throw new NotSupportException("RocksDBSstStore delete()"); + public void deleteRange(String table, byte[] keyFrom, byte[] keyTo) { + throw new NotSupportException("RocksDBSstStore deleteRange()"); } /** diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java index 907ff97dff..97ff4ef0f4 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java @@ -77,7 +77,8 @@ protected String get(String key) throws RocksDBException { protected void clearData() throws RocksDBException { for (String table : new ArrayList<>(this.rocks.openedTables())) { - this.rocks.session().delete(table, new byte[]{0}, new byte[]{-1}); + this.rocks.session().deleteRange(table, + new byte[]{0}, new byte[]{-1}); } this.commit(); } diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/RocksDBPerfTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/RocksDBPerfTest.java index f083c08885..1f2d9204b5 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/RocksDBPerfTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/RocksDBPerfTest.java @@ -141,7 +141,7 @@ public void testUpdate() throws RocksDBException { for (int i = 0; i < n; i++) { int value = comms.get(i); String old = String.format("index:%3d:%d", i, value); - session.remove(TABLE, b(old)); + session.delete(TABLE, b(old)); value = r.nextInt(n); // TODO: aggregate value = i + 1; diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/RocksDBSessionsTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/RocksDBSessionsTest.java index 5f9063af61..e93378ee80 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/RocksDBSessionsTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/unit/rocksdb/RocksDBSessionsTest.java @@ -285,7 +285,7 @@ public void testDeleteByKey() throws RocksDBException { Assert.assertEquals("19", get("person:1gage")); Assert.assertEquals("Beijing", get("person:1gcity")); - this.rocks.session().remove(TABLE, b("person:1gage")); + this.rocks.session().delete(TABLE, b("person:1gage")); this.commit(); Assert.assertEquals("James", get("person:1gname")); @@ -303,7 +303,7 @@ public void testDeleteByKeyButNotExist() throws RocksDBException { Assert.assertEquals("19", get("person:1gage")); Assert.assertEquals("Beijing", get("person:1gcity")); - this.rocks.session().remove(TABLE, b("person:1")); + this.rocks.session().delete(TABLE, b("person:1")); this.commit(); Assert.assertEquals("James", get("person:1gname")); @@ -325,7 +325,7 @@ public void testDeleteByPrefix() throws RocksDBException { Assert.assertEquals("19", get("person:1gage")); Assert.assertEquals("Beijing", get("person:1gcity")); - this.rocks.session().delete(TABLE, b("person:1")); + this.rocks.session().deletePrefix(TABLE, b("person:1")); this.commit(); Assert.assertEquals(null, get("person:1gname")); @@ -353,7 +353,7 @@ public void testDeleteByRange() throws RocksDBException { Assert.assertEquals("Lisa", get("person:2gname")); Assert.assertEquals("Hebe", get("person:3gname")); - this.rocks.session().delete(TABLE, b("person:1"), b("person:3")); + this.rocks.session().deleteRange(TABLE, b("person:1"), b("person:3")); this.commit(); Assert.assertEquals(null, get("person:1gname")); @@ -385,7 +385,7 @@ public void testDeleteByRangeWithBytes() throws RocksDBException { byte[] value21 = b("value-2-1"); session.put(TABLE, key21, value21); - session.delete(TABLE, key11, new byte[]{1, 3}); + session.deleteRange(TABLE, key11, new byte[]{1, 3}); this.commit(); Assert.assertArrayEquals(null, session.get(TABLE, key11)); @@ -409,14 +409,14 @@ public void testDeleteByRangeWithSignedBytes() throws RocksDBException { byte[] value21 = b("value-2-1"); session.put(TABLE, key21, value21); - session.delete(TABLE, new byte[]{1, -3}, new byte[]{1, 3}); + session.deleteRange(TABLE, new byte[]{1, -3}, new byte[]{1, 3}); this.commit(); Assert.assertArrayEquals(value11, session.get(TABLE, key11)); Assert.assertArrayEquals(value12, session.get(TABLE, key12)); Assert.assertArrayEquals(value21, session.get(TABLE, key21)); - session.delete(TABLE, new byte[]{1, 1}, new byte[]{1, -1}); + session.deleteRange(TABLE, new byte[]{1, 1}, new byte[]{1, -1}); this.commit(); Assert.assertArrayEquals(null, session.get(TABLE, key11)); @@ -448,7 +448,8 @@ public void testDeleteByRangeWithMinMaxByteValue() throws RocksDBException { byte[] value20 = b("value-2-0"); session.put(TABLE, key20, value20); - session.delete(TABLE, new byte[]{1, 0}, new byte[]{1, (byte) 0xff}); + session.deleteRange(TABLE, + new byte[]{1, 0}, new byte[]{1, (byte) 0xff}); this.commit(); Assert.assertArrayEquals(null, session.get(TABLE, key11)); @@ -457,7 +458,8 @@ public void testDeleteByRangeWithMinMaxByteValue() throws RocksDBException { Assert.assertArrayEquals(value14, session.get(TABLE, key14)); Assert.assertArrayEquals(value20, session.get(TABLE, key20)); - session.delete(TABLE, new byte[]{1, (byte) 0xff}, new byte[]{2, 0}); + session.deleteRange(TABLE, + new byte[]{1, (byte) 0xff}, new byte[]{2, 0}); this.commit(); Assert.assertArrayEquals(null, session.get(TABLE, key11));