Buffered serialization

src/catalog/catalog.cpp

@@ -3,7 +3,6 @@
 #include "catalog/node_table_schema.h"
 #include "catalog/rel_table_group_schema.h"
 #include "catalog/rel_table_schema.h"
-#include "common/ser_deser.h"
 #include "storage/wal/wal.h"
 #include "transaction/transaction_action.h"
 

src/catalog/catalog_content.cpp

@@ -6,7 +6,9 @@
 #include "catalog/rel_table_schema.h"
 #include "common/exception/catalog.h"
 #include "common/exception/runtime.h"
-#include "common/ser_deser.h"
+#include "common/serializer/buffered_file.h"
+#include "common/serializer/deserializer.h"
+#include "common/serializer/serializer.h"
 #include "common/string_format.h"
 #include "common/string_utils.h"
 #include "storage/storage_utils.h"
@@ -154,39 +156,39 @@ void CatalogContent::renameTable(table_id_t tableID, const std::string& newName)
 
 void CatalogContent::saveToFile(const std::string& directory, DBFileType dbFileType) {
     auto catalogPath = StorageUtils::getCatalogFilePath(directory, dbFileType);
-    auto fileInfo = FileUtils::openFile(catalogPath, O_WRONLY | O_CREAT);
-    uint64_t offset = 0;
-    writeMagicBytes(fileInfo.get(), offset);
-    SerDeser::serializeValue(StorageVersionInfo::getStorageVersion(), fileInfo.get(), offset);
-    SerDeser::serializeValue(tableSchemas.size(), fileInfo.get(), offset);
+    Serializer serializer(
+        std::make_unique<BufferedFileWriter>(FileUtils::openFile(catalogPath, O_WRONLY | O_CREAT)));
+    writeMagicBytes(serializer);
+    serializer.serializeValue(StorageVersionInfo::getStorageVersion());
+    serializer.serializeValue(tableSchemas.size());
     for (auto& [tableID, tableSchema] : tableSchemas) {
-        SerDeser::serializeValue(tableID, fileInfo.get(), offset);
-        tableSchema->serialize(fileInfo.get(), offset);
+        serializer.serializeValue(tableID);
+        tableSchema->serialize(serializer);
     }
-    SerDeser::serializeValue(nextTableID, fileInfo.get(), offset);
-    SerDeser::serializeUnorderedMap(macros, fileInfo.get(), offset);
+    serializer.serializeValue(nextTableID);
+    serializer.serializeUnorderedMap(macros);
 }
 
 void CatalogContent::readFromFile(const std::string& directory, DBFileType dbFileType) {
     auto catalogPath = StorageUtils::getCatalogFilePath(directory, dbFileType);
-    auto fileInfo = FileUtils::openFile(catalogPath, O_RDONLY);
-    uint64_t offset = 0;
-    validateMagicBytes(fileInfo.get(), offset);
+    Deserializer deserializer(
+        std::make_unique<BufferedFileReader>(FileUtils::openFile(catalogPath, O_RDONLY)));
+    validateMagicBytes(deserializer);
     storage_version_t savedStorageVersion;
-    SerDeser::deserializeValue(savedStorageVersion, fileInfo.get(), offset);
+    deserializer.deserializeValue(savedStorageVersion);
     validateStorageVersion(savedStorageVersion);
     uint64_t numTables;
-    SerDeser::deserializeValue(numTables, fileInfo.get(), offset);
+    deserializer.deserializeValue(numTables);
     table_id_t tableID;
     for (auto i = 0u; i < numTables; i++) {
-        SerDeser::deserializeValue(tableID, fileInfo.get(), offset);
-        tableSchemas[tableID] = TableSchema::deserialize(fileInfo.get(), offset);
+        deserializer.deserializeValue(tableID);
+        tableSchemas[tableID] = TableSchema::deserialize(deserializer);
     }
     for (auto& [tableID_, tableSchema] : tableSchemas) {
         tableNameToIDMap[tableSchema->tableName] = tableID_;
     }
-    SerDeser::deserializeValue(nextTableID, fileInfo.get(), offset);
-    SerDeser::deserializeUnorderedMap(macros, fileInfo.get(), offset);
+    deserializer.deserializeValue(nextTableID);
+    deserializer.deserializeUnorderedMap(macros);
 }
 
 ExpressionType CatalogContent::getFunctionType(const std::string& name) const {
@@ -239,22 +241,22 @@ void CatalogContent::validateStorageVersion(storage_version_t savedStorageVersio
     }
 }
 
-void CatalogContent::validateMagicBytes(FileInfo* fileInfo, offset_t& offset) {
+void CatalogContent::validateMagicBytes(Deserializer& deserializer) {
     auto numMagicBytes = strlen(StorageVersionInfo::MAGIC_BYTES);
     uint8_t magicBytes[4];
     for (auto i = 0u; i < numMagicBytes; i++) {
-        SerDeser::deserializeValue<uint8_t>(magicBytes[i], fileInfo, offset);
+        deserializer.deserializeValue<uint8_t>(magicBytes[i]);
     }
     if (memcmp(magicBytes, StorageVersionInfo::MAGIC_BYTES, numMagicBytes) != 0) {
         throw RuntimeException(
             "This is not a valid Kuzu database directory for the current version of Kuzu.");
     }
 }
 
-void CatalogContent::writeMagicBytes(FileInfo* fileInfo, offset_t& offset) {
+void CatalogContent::writeMagicBytes(Serializer& serializer) {
     auto numMagicBytes = strlen(StorageVersionInfo::MAGIC_BYTES);
     for (auto i = 0u; i < numMagicBytes; i++) {
-        SerDeser::serializeValue<uint8_t>(StorageVersionInfo::MAGIC_BYTES[i], fileInfo, offset);
+        serializer.serializeValue<uint8_t>(StorageVersionInfo::MAGIC_BYTES[i]);
     }
 }
 

src/catalog/node_table_schema.cpp

@@ -1,26 +1,26 @@
 #include "catalog/node_table_schema.h"
 
-#include "common/ser_deser.h"
+#include "common/serializer/deserializer.h"
+#include "common/serializer/serializer.h"
 
 using namespace kuzu::common;
 
 namespace kuzu {
 namespace catalog {
 
-void NodeTableSchema::serializeInternal(FileInfo* fileInfo, uint64_t& offset) {
-    SerDeser::serializeValue(primaryKeyPropertyID, fileInfo, offset);
-    SerDeser::serializeUnorderedSet(fwdRelTableIDSet, fileInfo, offset);
-    SerDeser::serializeUnorderedSet(bwdRelTableIDSet, fileInfo, offset);
+void NodeTableSchema::serializeInternal(Serializer& serializer) {
+    serializer.serializeValue(primaryKeyPropertyID);
+    serializer.serializeUnorderedSet(fwdRelTableIDSet);
+    serializer.serializeUnorderedSet(bwdRelTableIDSet);
 }
 
-std::unique_ptr<NodeTableSchema> NodeTableSchema::deserialize(
-    FileInfo* fileInfo, uint64_t& offset) {
+std::unique_ptr<NodeTableSchema> NodeTableSchema::deserialize(Deserializer& deserializer) {
     property_id_t primaryKeyPropertyID;
     std::unordered_set<table_id_t> fwdRelTableIDSet;
     std::unordered_set<table_id_t> bwdRelTableIDSet;
-    SerDeser::deserializeValue(primaryKeyPropertyID, fileInfo, offset);
-    SerDeser::deserializeUnorderedSet(fwdRelTableIDSet, fileInfo, offset);
-    SerDeser::deserializeUnorderedSet(bwdRelTableIDSet, fileInfo, offset);
+    deserializer.deserializeValue(primaryKeyPropertyID);
+    deserializer.deserializeUnorderedSet(fwdRelTableIDSet);
+    deserializer.deserializeUnorderedSet(bwdRelTableIDSet);
     return std::make_unique<NodeTableSchema>(
         primaryKeyPropertyID, fwdRelTableIDSet, bwdRelTableIDSet);
 }

src/catalog/property.cpp

@@ -1,27 +1,28 @@
 #include "catalog/property.h"
 
-#include "common/ser_deser.h"
+#include "common/serializer/deserializer.h"
+#include "common/serializer/serializer.h"
 
 using namespace kuzu::common;
 
 namespace kuzu {
 namespace catalog {
 
-void Property::serialize(FileInfo* fileInfo, uint64_t& offset) const {
-    SerDeser::serializeValue(name, fileInfo, offset);
-    dataType->serialize(fileInfo, offset);
-    SerDeser::serializeValue(propertyID, fileInfo, offset);
-    SerDeser::serializeValue(tableID, fileInfo, offset);
+void Property::serialize(Serializer& serializer) const {
+    serializer.serializeValue(name);
+    dataType->serialize(serializer);
+    serializer.serializeValue(propertyID);
+    serializer.serializeValue(tableID);
 }
 
-std::unique_ptr<Property> Property::deserialize(FileInfo* fileInfo, uint64_t& offset) {
+std::unique_ptr<Property> Property::deserialize(Deserializer& deserializer) {
     std::string name;
     property_id_t propertyID;
     table_id_t tableID;
-    SerDeser::deserializeValue(name, fileInfo, offset);
-    auto dataType = LogicalType::deserialize(fileInfo, offset);
-    SerDeser::deserializeValue(propertyID, fileInfo, offset);
-    SerDeser::deserializeValue(tableID, fileInfo, offset);
+    deserializer.deserializeValue(name);
+    auto dataType = LogicalType::deserialize(deserializer);
+    deserializer.deserializeValue(propertyID);
+    deserializer.deserializeValue(tableID);
     return std::make_unique<Property>(name, std::move(dataType), propertyID, tableID);
 }
 

src/catalog/rdf_graph_schema.cpp

@@ -1,22 +1,23 @@
 #include "catalog/rdf_graph_schema.h"
 
-#include "common/ser_deser.h"
+#include "common/serializer/deserializer.h"
+#include "common/serializer/serializer.h"
 
 using namespace kuzu::common;
 
 namespace kuzu {
 namespace catalog {
 
-void RdfGraphSchema::serializeInternal(FileInfo* fileInfo, uint64_t& offset) {
-    SerDeser::serializeValue(nodeTableID, fileInfo, offset);
-    SerDeser::serializeValue(relTableID, fileInfo, offset);
+void RdfGraphSchema::serializeInternal(Serializer& serializer) {
+    serializer.serializeValue(nodeTableID);
+    serializer.serializeValue(relTableID);
 }
 
-std::unique_ptr<RdfGraphSchema> RdfGraphSchema::deserialize(FileInfo* fileInfo, uint64_t& offset) {
+std::unique_ptr<RdfGraphSchema> RdfGraphSchema::deserialize(Deserializer& deserializer) {
     table_id_t nodeTableID;
     table_id_t relTableID;
-    SerDeser::deserializeValue(nodeTableID, fileInfo, offset);
-    SerDeser::deserializeValue(relTableID, fileInfo, offset);
+    deserializer.deserializeValue(nodeTableID);
+    deserializer.deserializeValue(relTableID);
     return std::make_unique<RdfGraphSchema>(nodeTableID, relTableID);
 }
 

src/catalog/rel_table_group_schema.cpp

@@ -1,20 +1,20 @@
 #include "catalog/rel_table_group_schema.h"
 
-#include "common/ser_deser.h"
+#include "common/serializer/deserializer.h"
+#include "common/serializer/serializer.h"
 
 using namespace kuzu::common;
 
 namespace kuzu {
 namespace catalog {
 
-void RelTableGroupSchema::serializeInternal(FileInfo* fileInfo, uint64_t& offset) {
-    SerDeser::serializeVector(relTableIDs, fileInfo, offset);
+void RelTableGroupSchema::serializeInternal(Serializer& serializer) {
+    serializer.serializeVector(relTableIDs);
 }
 
-std::unique_ptr<RelTableGroupSchema> RelTableGroupSchema::deserialize(
-    FileInfo* fileInfo, uint64_t& offset) {
+std::unique_ptr<RelTableGroupSchema> RelTableGroupSchema::deserialize(Deserializer& deserializer) {
     std::vector<table_id_t> relTableIDs;
-    SerDeser::deserializeVector(relTableIDs, fileInfo, offset);
+    deserializer.deserializeVector(relTableIDs);
     return std::make_unique<RelTableGroupSchema>(std::move(relTableIDs));
 }
 

src/catalog/rel_table_schema.cpp

@@ -1,7 +1,8 @@
 #include "catalog/rel_table_schema.h"
 
 #include "common/exception/catalog.h"
-#include "common/ser_deser.h"
+#include "common/serializer/deserializer.h"
+#include "common/serializer/serializer.h"
 
 using namespace kuzu::common;
 
@@ -40,19 +41,19 @@ std::string getRelMultiplicityAsString(RelMultiplicity relMultiplicity) {
     }
 }
 
-void RelTableSchema::serializeInternal(FileInfo* fileInfo, uint64_t& offset) {
-    SerDeser::serializeValue(relMultiplicity, fileInfo, offset);
-    SerDeser::serializeValue(srcTableID, fileInfo, offset);
-    SerDeser::serializeValue(dstTableID, fileInfo, offset);
+void RelTableSchema::serializeInternal(Serializer& serializer) {
+    serializer.serializeValue(relMultiplicity);
+    serializer.serializeValue(srcTableID);
+    serializer.serializeValue(dstTableID);
 }
 
-std::unique_ptr<RelTableSchema> RelTableSchema::deserialize(FileInfo* fileInfo, uint64_t& offset) {
+std::unique_ptr<RelTableSchema> RelTableSchema::deserialize(Deserializer& deserializer) {
     RelMultiplicity relMultiplicity;
     table_id_t srcTableID;
     table_id_t dstTableID;
-    SerDeser::deserializeValue(relMultiplicity, fileInfo, offset);
-    SerDeser::deserializeValue(srcTableID, fileInfo, offset);
-    SerDeser::deserializeValue(dstTableID, fileInfo, offset);
+    deserializer.deserializeValue(relMultiplicity);
+    deserializer.deserializeValue(srcTableID);
+    deserializer.deserializeValue(dstTableID);
     return std::make_unique<RelTableSchema>(relMultiplicity, srcTableID, dstTableID);
 }
 

src/catalog/table_schema.cpp

@@ -8,7 +8,8 @@
 #include "common/exception/internal.h"
 #include "common/exception/not_implemented.h"
 #include "common/exception/runtime.h"
-#include "common/ser_deser.h"
+#include "common/serializer/deserializer.h"
+#include "common/serializer/serializer.h"
 #include "common/string_format.h"
 #include "common/string_utils.h"
 
@@ -76,42 +77,42 @@ void TableSchema::renameProperty(property_id_t propertyID, const std::string& ne
     // LCOV_EXCL_END
 }
 
-void TableSchema::serialize(FileInfo* fileInfo, uint64_t& offset) {
-    SerDeser::serializeValue(tableName, fileInfo, offset);
-    SerDeser::serializeValue(tableID, fileInfo, offset);
-    SerDeser::serializeValue(tableType, fileInfo, offset);
-    SerDeser::serializeVectorOfPtrs(properties, fileInfo, offset);
-    SerDeser::serializeValue(comment, fileInfo, offset);
-    SerDeser::serializeValue(nextPropertyID, fileInfo, offset);
-    serializeInternal(fileInfo, offset);
+void TableSchema::serialize(Serializer& serializer) {
+    serializer.serializeValue(tableName);
+    serializer.serializeValue(tableID);
+    serializer.serializeValue(tableType);
+    serializer.serializeVectorOfPtrs(properties);
+    serializer.serializeValue(comment);
+    serializer.serializeValue(nextPropertyID);
+    serializeInternal(serializer);
 }
 
-std::unique_ptr<TableSchema> TableSchema::deserialize(FileInfo* fileInfo, uint64_t& offset) {
+std::unique_ptr<TableSchema> TableSchema::deserialize(Deserializer& deserializer) {
     std::string tableName;
     table_id_t tableID;
     TableType tableType;
     std::vector<std::unique_ptr<Property>> properties;
     std::string comment;
     property_id_t nextPropertyID;
-    SerDeser::deserializeValue(tableName, fileInfo, offset);
-    SerDeser::deserializeValue(tableID, fileInfo, offset);
-    SerDeser::deserializeValue(tableType, fileInfo, offset);
-    SerDeser::deserializeVectorOfPtrs(properties, fileInfo, offset);
-    SerDeser::deserializeValue(comment, fileInfo, offset);
-    SerDeser::deserializeValue(nextPropertyID, fileInfo, offset);
+    deserializer.deserializeValue(tableName);
+    deserializer.deserializeValue(tableID);
+    deserializer.deserializeValue(tableType);
+    deserializer.deserializeVectorOfPtrs(properties);
+    deserializer.deserializeValue(comment);
+    deserializer.deserializeValue(nextPropertyID);
     std::unique_ptr<TableSchema> result;
     switch (tableType) {
     case TableType::NODE: {
-        result = NodeTableSchema::deserialize(fileInfo, offset);
+        result = NodeTableSchema::deserialize(deserializer);
     } break;
     case TableType::REL: {
-        result = RelTableSchema::deserialize(fileInfo, offset);
+        result = RelTableSchema::deserialize(deserializer);
     } break;
     case TableType::REL_GROUP: {
-        result = RelTableGroupSchema::deserialize(fileInfo, offset);
+        result = RelTableGroupSchema::deserialize(deserializer);
     } break;
     case TableType::RDF: {
-        result = RdfGraphSchema::deserialize(fileInfo, offset);
+        result = RdfGraphSchema::deserialize(deserializer);
     } break;
     default: {
         // LCOV_EXCL_START

src/common/CMakeLists.txt

@@ -2,6 +2,7 @@ add_subdirectory(arrow)
 add_subdirectory(copier_config)
 add_subdirectory(data_chunk)
 add_subdirectory(exception)
+add_subdirectory(serializer)
 add_subdirectory(task_system)
 add_subdirectory(types)
 add_subdirectory(vector)
@@ -20,8 +21,7 @@ add_library(kuzu_common
         type_utils.cpp
         utils.cpp
         string_utils.cpp
-        table_type.cpp
-        ser_deser.cpp)
+        table_type.cpp)
 
 target_link_libraries(kuzu_common Glob)