Merge branch 'master' into ashleyhx/add-uint

src/binder/bind/bind_copy.cpp

@@ -1,6 +1,7 @@
 #include "binder/binder.h"
 #include "binder/copy/bound_copy_from.h"
 #include "binder/copy/bound_copy_to.h"
+#include "catalog/node_table_schema.h"
 #include "catalog/rel_table_schema.h"
 #include "common/exception/binder.h"
 #include "common/exception/message.h"
@@ -182,10 +183,10 @@ static bool skipPropertyInFile(const Property& property) {
 }
 
 expression_vector Binder::bindExpectedNodeFileColumns(
-    TableSchema* tableSchema, common::ReaderConfig& readerConfig) {
+    TableSchema* tableSchema, ReaderConfig& readerConfig) {
     expression_vector columns;
     switch (readerConfig.fileType) {
-    case common::FileType::TURTLE: {
+    case FileType::TURTLE: {
         auto stringType = LogicalType{LogicalTypeID::STRING};
         auto columnNames = std::vector<std::string>{
             std::string(RDF_SUBJECT), std::string(RDF_PREDICATE), std::string(RDF_OBJECT)};
@@ -195,7 +196,7 @@ expression_vector Binder::bindExpectedNodeFileColumns(
             columns.push_back(createVariable(columnName, stringType));
         }
     } break;
-    case common::FileType::CSV: {
+    case FileType::CSV: {
         for (auto& property : tableSchema->properties) {
             if (skipPropertyInFile(*property)) {
                 continue;
@@ -205,8 +206,8 @@ expression_vector Binder::bindExpectedNodeFileColumns(
             columns.push_back(createVariable(property->getName(), *property->getDataType()));
         }
     } break;
-    case common::FileType::NPY:
-    case common::FileType::PARQUET: {
+    case FileType::NPY:
+    case FileType::PARQUET: {
         for (auto& property : tableSchema->properties) {
             if (skipPropertyInFile(*property)) {
                 continue;
@@ -225,11 +226,11 @@ expression_vector Binder::bindExpectedNodeFileColumns(
 }
 
 expression_vector Binder::bindExpectedRelFileColumns(
-    TableSchema* tableSchema, common::ReaderConfig& readerConfig) {
+    TableSchema* tableSchema, ReaderConfig& readerConfig) {
     auto relTableSchema = reinterpret_cast<RelTableSchema*>(tableSchema);
     expression_vector columns;
     switch (readerConfig.fileType) {
-    case common::FileType::TURTLE: {
+    case FileType::TURTLE: {
         auto stringType = LogicalType{LogicalTypeID::STRING};
         auto columnNames = std::vector<std::string>{
             std::string(RDF_SUBJECT), std::string(RDF_PREDICATE), std::string(RDF_OBJECT)};
@@ -239,16 +240,24 @@ expression_vector Binder::bindExpectedRelFileColumns(
             columns.push_back(createVariable(columnName, stringType));
         }
     } break;
-    case common::FileType::CSV:
-    case common::FileType::PARQUET:
-    case common::FileType::NPY: {
+    case FileType::CSV:
+    case FileType::PARQUET:
+    case FileType::NPY: {
         auto arrowColumnType = LogicalType{LogicalTypeID::ARROW_COLUMN};
         auto srcColumnName = std::string(Property::REL_FROM_PROPERTY_NAME);
         auto dstColumnName = std::string(Property::REL_TO_PROPERTY_NAME);
         readerConfig.columnNames.push_back(srcColumnName);
         readerConfig.columnNames.push_back(dstColumnName);
-        readerConfig.columnTypes.push_back(relTableSchema->getSrcPKDataType()->copy());
-        readerConfig.columnTypes.push_back(relTableSchema->getDstPKDataType()->copy());
+        auto srcTable =
+            catalog.getReadOnlyVersion()->getTableSchema(relTableSchema->getSrcTableID());
+        assert(srcTable->tableType == TableType::NODE);
+        auto dstTable =
+            catalog.getReadOnlyVersion()->getTableSchema(relTableSchema->getDstTableID());
+        assert(dstTable->tableType == TableType::NODE);
+        readerConfig.columnTypes.push_back(
+            reinterpret_cast<NodeTableSchema*>(srcTable)->getPrimaryKey()->getDataType()->copy());
+        readerConfig.columnTypes.push_back(
+            reinterpret_cast<NodeTableSchema*>(dstTable)->getPrimaryKey()->getDataType()->copy());
         columns.push_back(createVariable(srcColumnName, arrowColumnType));
         columns.push_back(createVariable(dstColumnName, arrowColumnType));
         for (auto& property : tableSchema->properties) {

src/binder/bind/bind_ddl.cpp

@@ -148,16 +148,9 @@ std::unique_ptr<BoundCreateTableInfo> Binder::bindCreateRelTableInfo(const Creat
     auto extraInfo = (ExtraCreateRelTableInfo*)info->extraInfo.get();
     auto relMultiplicity = getRelMultiplicityFromString(extraInfo->relMultiplicity);
     auto srcTableID = bindNodeTableID(extraInfo->srcTableName);
-    auto srcTableSchema =
-        (NodeTableSchema*)catalog.getReadOnlyVersion()->getTableSchema(srcTableID);
-    auto srcPkDataType = srcTableSchema->getPrimaryKey()->getDataType();
     auto dstTableID = bindNodeTableID(extraInfo->dstTableName);
-    auto dstTableSchema =
-        (NodeTableSchema*)catalog.getReadOnlyVersion()->getTableSchema(dstTableID);
-    auto dstPkDataType = dstTableSchema->getPrimaryKey()->getDataType();
-    auto boundExtraInfo =
-        std::make_unique<BoundExtraCreateRelTableInfo>(relMultiplicity, srcTableID, dstTableID,
-            srcPkDataType->copy(), dstPkDataType->copy(), std::move(boundProperties));
+    auto boundExtraInfo = std::make_unique<BoundExtraCreateRelTableInfo>(
+        relMultiplicity, srcTableID, dstTableID, std::move(boundProperties));
     return std::make_unique<BoundCreateTableInfo>(
         TableType::REL, info->tableName, std::move(boundExtraInfo));
 }
@@ -211,9 +204,8 @@ std::unique_ptr<BoundCreateTableInfo> Binder::bindCreateRdfGraphInfo(const Creat
     std::vector<std::unique_ptr<Property>> relProperties;
     relProperties.push_back(std::make_unique<Property>(
         RDF_PREDICT_ID, std::make_unique<LogicalType>(LogicalTypeID::INTERNAL_ID)));
-    auto boundRelExtraInfo =
-        std::make_unique<BoundExtraCreateRelTableInfo>(RelMultiplicity::MANY_MANY, INVALID_TABLE_ID,
-            INVALID_TABLE_ID, stringType->copy(), stringType->copy(), std::move(relProperties));
+    auto boundRelExtraInfo = std::make_unique<BoundExtraCreateRelTableInfo>(
+        RelMultiplicity::MANY_MANY, INVALID_TABLE_ID, INVALID_TABLE_ID, std::move(relProperties));
     auto boundRelCreateInfo = std::make_unique<BoundCreateTableInfo>(
         TableType::REL, relTableName, std::move(boundRelExtraInfo));
     auto boundExtraInfo = std::make_unique<BoundExtraCreateRdfGraphInfo>(

src/catalog/catalog.cpp

@@ -5,7 +5,6 @@
 #include "catalog/rel_table_group_schema.h"
 #include "catalog/rel_table_schema.h"
 #include "common/ser_deser.h"
-#include "common/string_utils.h"
 #include "storage/wal/wal.h"
 #include "transaction/transaction_action.h"
 

src/catalog/catalog_content.cpp

@@ -66,9 +66,9 @@ table_id_t CatalogContent::addRelTableSchema(const BoundCreateTableInfo& info) {
         reinterpret_cast<NodeTableSchema*>(getTableSchema(extraInfo->dstTableID));
     srcNodeTableSchema->addFwdRelTableID(tableID);
     dstNodeTableSchema->addBwdRelTableID(tableID);
-    auto relTableSchema = std::make_unique<RelTableSchema>(info.tableName, tableID,
-        extraInfo->relMultiplicity, std::move(properties), extraInfo->srcTableID,
-        extraInfo->dstTableID, extraInfo->srcPkDataType->copy(), extraInfo->dstPkDataType->copy());
+    auto relTableSchema =
+        std::make_unique<RelTableSchema>(info.tableName, tableID, extraInfo->relMultiplicity,
+            std::move(properties), extraInfo->srcTableID, extraInfo->dstTableID);
     tableNameToIDMap.emplace(relTableSchema->tableName, tableID);
     tableSchemas.emplace(tableID, std::move(relTableSchema));
     return tableID;

src/catalog/rel_table_schema.cpp

@@ -44,8 +44,6 @@ void RelTableSchema::serializeInternal(FileInfo* fileInfo, uint64_t& offset) {
     SerDeser::serializeValue(relMultiplicity, fileInfo, offset);
     SerDeser::serializeValue(srcTableID, fileInfo, offset);
     SerDeser::serializeValue(dstTableID, fileInfo, offset);
-    srcPKDataType->serialize(fileInfo, offset);
-    dstPKDataType->serialize(fileInfo, offset);
 }
 
 std::unique_ptr<RelTableSchema> RelTableSchema::deserialize(FileInfo* fileInfo, uint64_t& offset) {
@@ -55,10 +53,7 @@ std::unique_ptr<RelTableSchema> RelTableSchema::deserialize(FileInfo* fileInfo,
     SerDeser::deserializeValue(relMultiplicity, fileInfo, offset);
     SerDeser::deserializeValue(srcTableID, fileInfo, offset);
     SerDeser::deserializeValue(dstTableID, fileInfo, offset);
-    auto srcPKDataType = LogicalType::deserialize(fileInfo, offset);
-    auto dstPKDataType = LogicalType::deserialize(fileInfo, offset);
-    return std::make_unique<RelTableSchema>(relMultiplicity, srcTableID, dstTableID,
-        std::move(srcPKDataType), std::move(dstPKDataType));
+    return std::make_unique<RelTableSchema>(relMultiplicity, srcTableID, dstTableID);
 }
 
 } // namespace catalog

src/include/binder/ddl/bound_create_table_info.h

@@ -52,23 +52,16 @@ struct BoundExtraCreateRelTableInfo : public BoundExtraCreateTableInfo {
     catalog::RelMultiplicity relMultiplicity;
     common::table_id_t srcTableID;
     common::table_id_t dstTableID;
-    std::unique_ptr<common::LogicalType> srcPkDataType;
-    std::unique_ptr<common::LogicalType> dstPkDataType;
     std::vector<std::unique_ptr<catalog::Property>> properties;
 
     BoundExtraCreateRelTableInfo(catalog::RelMultiplicity relMultiplicity,
         common::table_id_t srcTableID, common::table_id_t dstTableID,
-        std::unique_ptr<common::LogicalType> srcPkDataType,
-        std::unique_ptr<common::LogicalType> dstPkDataType,
         std::vector<std::unique_ptr<catalog::Property>> properties)
         : relMultiplicity{relMultiplicity}, srcTableID{srcTableID}, dstTableID{dstTableID},
-          srcPkDataType{std::move(srcPkDataType)}, dstPkDataType{std::move(dstPkDataType)},
           properties{std::move(properties)} {}
     BoundExtraCreateRelTableInfo(const BoundExtraCreateRelTableInfo& other)
         : relMultiplicity{other.relMultiplicity}, srcTableID{other.srcTableID},
-          dstTableID{other.dstTableID}, srcPkDataType{other.srcPkDataType->copy()},
-          dstPkDataType{other.dstPkDataType->copy()}, properties{catalog::Property::copy(
-                                                          other.properties)} {}
+          dstTableID{other.dstTableID}, properties{catalog::Property::copy(other.properties)} {}
 
     inline std::unique_ptr<BoundExtraCreateTableInfo> copy() const final {
         return std::make_unique<BoundExtraCreateRelTableInfo>(*this);

src/include/catalog/rel_table_schema.h

@@ -15,30 +15,22 @@ class RelTableSchema : public TableSchema {
     static constexpr uint64_t INTERNAL_REL_ID_PROPERTY_ID = 0;
 
     RelTableSchema(RelMultiplicity relMultiplicity, common::table_id_t srcTableID,
-        common::table_id_t dstTableID, std::unique_ptr<common::LogicalType> srcPKDataType,
-        std::unique_ptr<common::LogicalType> dstPKDataType)
+        common::table_id_t dstTableID)
         : TableSchema{common::InternalKeyword::ANONYMOUS, common::INVALID_TABLE_ID,
               common::TableType::REL, {} /* properties */},
-          relMultiplicity{relMultiplicity}, srcTableID{srcTableID}, dstTableID{dstTableID},
-          srcPKDataType{std::move(srcPKDataType)}, dstPKDataType{std::move(dstPKDataType)} {}
+          relMultiplicity{relMultiplicity}, srcTableID{srcTableID}, dstTableID{dstTableID} {}
     RelTableSchema(std::string tableName, common::table_id_t tableID,
         RelMultiplicity relMultiplicity, std::vector<std::unique_ptr<Property>> properties,
-        common::table_id_t srcTableID, common::table_id_t dstTableID,
-        std::unique_ptr<common::LogicalType> srcPKDataType,
-        std::unique_ptr<common::LogicalType> dstPKDataType)
+        common::table_id_t srcTableID, common::table_id_t dstTableID)
         : TableSchema{std::move(tableName), tableID, common::TableType::REL, std::move(properties)},
-          relMultiplicity{relMultiplicity}, srcTableID{srcTableID}, dstTableID{dstTableID},
-          srcPKDataType{std::move(srcPKDataType)}, dstPKDataType{std::move(dstPKDataType)} {}
+          relMultiplicity{relMultiplicity}, srcTableID{srcTableID}, dstTableID{dstTableID} {}
     RelTableSchema(std::string tableName, common::table_id_t tableID,
         std::vector<std::unique_ptr<Property>> properties, std::string comment,
         common::property_id_t nextPropertyID, RelMultiplicity relMultiplicity,
-        common::table_id_t srcTableID, common::table_id_t dstTableID,
-        std::unique_ptr<common::LogicalType> srcPKDataType,
-        std::unique_ptr<common::LogicalType> dstPKDataType)
+        common::table_id_t srcTableID, common::table_id_t dstTableID)
         : TableSchema{common::TableType::REL, std::move(tableName), tableID, std::move(properties),
               std::move(comment), nextPropertyID},
-          relMultiplicity{relMultiplicity}, srcTableID{srcTableID}, dstTableID{dstTableID},
-          srcPKDataType{std::move(srcPKDataType)}, dstPKDataType{std::move(dstPKDataType)} {}
+          relMultiplicity{relMultiplicity}, srcTableID{srcTableID}, dstTableID{dstTableID} {}
 
     inline bool isSingleMultiplicityInDirection(common::RelDataDirection direction) const {
         return relMultiplicity == RelMultiplicity::ONE_ONE ||
@@ -65,17 +57,12 @@ class RelTableSchema : public TableSchema {
 
     inline common::table_id_t getDstTableID() const { return dstTableID; }
 
-    inline common::LogicalType* getSrcPKDataType() const { return srcPKDataType.get(); }
-
-    inline common::LogicalType* getDstPKDataType() const { return dstPKDataType.get(); }
-
     static std::unique_ptr<RelTableSchema> deserialize(
         common::FileInfo* fileInfo, uint64_t& offset);
 
     inline std::unique_ptr<TableSchema> copy() const override {
         return std::make_unique<RelTableSchema>(tableName, tableID, Property::copy(properties),
-            comment, nextPropertyID, relMultiplicity, srcTableID, dstTableID, srcPKDataType->copy(),
-            dstPKDataType->copy());
+            comment, nextPropertyID, relMultiplicity, srcTableID, dstTableID);
     }
 
 private:
@@ -85,8 +72,6 @@ class RelTableSchema : public TableSchema {
     RelMultiplicity relMultiplicity;
     common::table_id_t srcTableID;
     common::table_id_t dstTableID;
-    std::unique_ptr<common::LogicalType> srcPKDataType;
-    std::unique_ptr<common::LogicalType> dstPKDataType;
 };
 
 } // namespace catalog

src/include/planner/join_order/cardinality_estimator.h

@@ -2,8 +2,8 @@
 
 #include "binder/query/query_graph.h"
 #include "planner/operator/logical_plan.h"
-#include "storage/store/nodes_statistics_and_deleted_ids.h"
-#include "storage/store/rels_statistics.h"
+#include "storage/stats/nodes_statistics_and_deleted_ids.h"
+#include "storage/stats/rels_statistics.h"
 
 namespace kuzu {
 namespace planner {

src/include/processor/operator/ddl/create_node_table.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "processor/operator/ddl/ddl.h"
-#include "storage/store/nodes_statistics_and_deleted_ids.h"
+#include "storage/stats/nodes_statistics_and_deleted_ids.h"
 #include "storage/store/nodes_store.h"
 
 namespace kuzu {

src/include/processor/operator/ddl/create_rel_table.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "processor/operator/ddl/ddl.h"
-#include "storage/store/rels_statistics.h"
+#include "storage/stats/rels_statistics.h"
 
 namespace kuzu {
 namespace processor {

src/include/processor/operator/ddl/create_rel_table_group.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "processor/operator/ddl/ddl.h"
-#include "storage/store/rels_statistics.h"
+#include "storage/stats/rels_statistics.h"
 
 namespace kuzu {
 namespace processor {

src/include/processor/operator/persistent/copy_node.h

@@ -2,7 +2,7 @@
 
 #include "common/copier_config/copier_config.h"
 #include "processor/operator/sink.h"
-#include "storage/copier/node_group.h"
+#include "storage/store/node_group.h"
 #include "storage/store/node_table.h"
 
 namespace kuzu {

src/include/processor/operator/persistent/copy_rel.h

@@ -6,7 +6,7 @@
 #include "storage/in_mem_storage_structure/in_mem_column.h"
 #include "storage/in_mem_storage_structure/in_mem_column_chunk.h"
 #include "storage/in_mem_storage_structure/in_mem_lists.h"
-#include "storage/store/rels_statistics.h"
+#include "storage/stats/rels_statistics.h"
 
 namespace kuzu {
 namespace processor {

src/include/processor/operator/persistent/copy_rel_columns.h

@@ -3,7 +3,7 @@
 #include "processor/operator/persistent/copy_rel.h"
 #include "processor/operator/sink.h"
 #include "storage/index/hash_index.h"
-#include "storage/store/rels_statistics.h"
+#include "storage/stats/rels_statistics.h"
 
 namespace kuzu {
 namespace processor {

src/include/storage/copier/npy_reader.h → src/include/processor/operator/persistent/reader/npy_reader.h

@@ -11,7 +11,7 @@
 #include <arrow/record_batch.h>
 
 namespace kuzu {
-namespace storage {
+namespace processor {
 
 class NpyReader {
 public:
@@ -60,5 +60,5 @@ class NpyMultiFileReader {
     std::vector<std::unique_ptr<NpyReader>> fileReaders;
 };
 
-} // namespace storage
+} // namespace processor
 } // namespace kuzu

src/include/processor/operator/persistent/reader/rdf_reader.h

@@ -5,7 +5,7 @@
 #include "serd.h"
 
 namespace kuzu {
-namespace storage {
+namespace processor {
 
 class RDFReader {
 public:
@@ -33,5 +33,5 @@ class RDFReader {
     common::ValueVector* objectVector;
 };
 
-} // namespace storage
+} // namespace processor
 } // namespace kuzu

src/include/processor/operator/persistent/reader_functions.h

@@ -1,9 +1,11 @@
 #pragma once
 
 #include "processor/operator/persistent/reader/csv_reader.h"
+#include "processor/operator/persistent/reader/npy_reader.h"
 #include "processor/operator/persistent/reader/rdf_reader.h"
-#include "storage/copier/npy_reader.h"
-#include "storage/copier/table_copy_utils.h"
+#include "storage/store/table_copy_utils.h"
+#include <arrow/csv/reader.h>
+#include <parquet/arrow/reader.h>
 
 namespace kuzu {
 namespace processor {
@@ -28,11 +30,11 @@ struct ParquetReaderFunctionData : public ReaderFunctionData {
 };
 
 struct NPYReaderFunctionData : public ReaderFunctionData {
-    std::unique_ptr<storage::NpyMultiFileReader> reader = nullptr;
+    std::unique_ptr<NpyMultiFileReader> reader = nullptr;
 };
 
 struct RDFReaderFunctionData : public ReaderFunctionData {
-    std::unique_ptr<storage::RDFReader> reader = nullptr;
+    std::unique_ptr<RDFReader> reader = nullptr;
 };
 
 struct FileBlocksInfo {

src/include/processor/operator/scan/scan_rel_table_columns.h

@@ -2,7 +2,7 @@
 
 #include "processor/operator/filtering_operator.h"
 #include "processor/operator/scan/scan_rel_table.h"
-#include "storage/store/rels_statistics.h"
+#include "storage/stats/rels_statistics.h"
 
 namespace kuzu {
 namespace processor {

src/include/processor/plan_mapper.h

@@ -6,8 +6,8 @@
 #include "planner/operator/logical_plan.h"
 #include "processor/operator/result_collector.h"
 #include "processor/physical_plan.h"
+#include "storage/stats/nodes_statistics_and_deleted_ids.h"
 #include "storage/storage_manager.h"
-#include "storage/store/nodes_statistics_and_deleted_ids.h"
 
 namespace kuzu {
 namespace planner {