Merge pull request #2075 from kuzudb/refactor-storage

Reorganize stat files
kuzudb · Sep 24, 2023 · c22396f · c22396f
2 parents 57d37b2 + 819b0ca
commit c22396f
Show file tree

Hide file tree

Showing 47 changed files with 816 additions and 726 deletions.
diff --git a/src/include/planner/join_order/cardinality_estimator.h b/src/include/planner/join_order/cardinality_estimator.h
@@ -2,16 +2,16 @@
 
 #include "binder/query/query_graph.h"
 #include "planner/operator/logical_plan.h"
-#include "storage/stats/nodes_statistics_and_deleted_ids.h"
-#include "storage/stats/rels_statistics.h"
+#include "storage/stats/nodes_store_statistics.h"
+#include "storage/stats/rels_store_statistics.h"
 
 namespace kuzu {
 namespace planner {
 
 class CardinalityEstimator {
 public:
-    CardinalityEstimator(const storage::NodesStatisticsAndDeletedIDs& nodesStatistics,
-        const storage::RelsStatistics& relsStatistics)
+    CardinalityEstimator(const storage::NodesStoreStatsAndDeletedIDs& nodesStatistics,
+        const storage::RelsStoreStats& relsStatistics)
         : nodesStatistics{nodesStatistics}, relsStatistics{relsStatistics} {}
 
     void initNodeIDDom(binder::QueryGraph* queryGraph);
@@ -37,7 +37,7 @@ class CardinalityEstimator {
             nodeIDName2dom.insert({key, getNumNodes(node)});
         }
     }
-    uint64_t getNodeIDDom(const std::string& nodeIDName) {
+    inline uint64_t getNodeIDDom(const std::string& nodeIDName) {
         assert(nodeIDName2dom.contains(nodeIDName));
         return nodeIDName2dom.at(nodeIDName);
     }
@@ -46,8 +46,8 @@ class CardinalityEstimator {
     uint64_t getNumRels(const binder::RelExpression& rel);
 
 private:
-    const storage::NodesStatisticsAndDeletedIDs& nodesStatistics;
-    const storage::RelsStatistics& relsStatistics;
+    const storage::NodesStoreStatsAndDeletedIDs& nodesStatistics;
+    const storage::RelsStoreStats& relsStatistics;
     // The domain of nodeID is defined as the number of unique value of nodeID, i.e. num nodes.
     std::unordered_map<std::string, uint64_t> nodeIDName2dom;
 };

diff --git a/src/include/planner/planner.h b/src/include/planner/planner.h
@@ -8,12 +8,12 @@ namespace planner {
 class Planner {
 public:
     static std::unique_ptr<LogicalPlan> getBestPlan(const catalog::Catalog& catalog,
-        const storage::NodesStatisticsAndDeletedIDs& nodesStatistics,
-        const storage::RelsStatistics& relsStatistics, const BoundStatement& statement);
+        const storage::NodesStoreStatsAndDeletedIDs& nodesStatistics,
+        const storage::RelsStoreStats& relsStatistics, const BoundStatement& statement);
 
     static std::vector<std::unique_ptr<LogicalPlan>> getAllPlans(const catalog::Catalog& catalog,
-        const storage::NodesStatisticsAndDeletedIDs& nodesStatistics,
-        const storage::RelsStatistics& relsStatistics, const BoundStatement& statement);
+        const storage::NodesStoreStatsAndDeletedIDs& nodesStatistics,
+        const storage::RelsStoreStats& relsStatistics, const BoundStatement& statement);
 
 private:
     static std::unique_ptr<LogicalPlan> planCreateTable(const BoundStatement& statement);
@@ -33,26 +33,26 @@ class Planner {
     static std::unique_ptr<LogicalPlan> planCommentOn(const BoundStatement& statement);
 
     static std::unique_ptr<LogicalPlan> planExplain(const catalog::Catalog& catalog,
-        const storage::NodesStatisticsAndDeletedIDs& nodesStatistics,
-        const storage::RelsStatistics& relsStatistics, const BoundStatement& statement);
+        const storage::NodesStoreStatsAndDeletedIDs& nodesStatistics,
+        const storage::RelsStoreStats& relsStatistics, const BoundStatement& statement);
 
     static std::unique_ptr<LogicalPlan> planCreateMacro(const BoundStatement& statement);
 
     static std::unique_ptr<LogicalPlan> planTransaction(const BoundStatement& statement);
 
     static std::vector<std::unique_ptr<LogicalPlan>> getAllQueryPlans(
         const catalog::Catalog& catalog,
-        const storage::NodesStatisticsAndDeletedIDs& nodesStatistics,
-        const storage::RelsStatistics& relsStatistics, const BoundStatement& statement);
+        const storage::NodesStoreStatsAndDeletedIDs& nodesStatistics,
+        const storage::RelsStoreStats& relsStatistics, const BoundStatement& statement);
 
     static std::vector<std::unique_ptr<LogicalPlan>> getAllExplainPlans(
         const catalog::Catalog& catalog,
-        const storage::NodesStatisticsAndDeletedIDs& nodesStatistics,
-        const storage::RelsStatistics& relsStatistics, const BoundStatement& statement);
+        const storage::NodesStoreStatsAndDeletedIDs& nodesStatistics,
+        const storage::RelsStoreStats& relsStatistics, const BoundStatement& statement);
 
     static std::unique_ptr<LogicalPlan> planCopyTo(const catalog::Catalog& catalog,
-        const storage::NodesStatisticsAndDeletedIDs& nodesStatistics,
-        const storage::RelsStatistics& relsStatistics, const BoundStatement& statement);
+        const storage::NodesStoreStatsAndDeletedIDs& nodesStatistics,
+        const storage::RelsStoreStats& relsStatistics, const BoundStatement& statement);
 
     static std::unique_ptr<LogicalPlan> planCopyFrom(const BoundStatement& statement);
 

diff --git a/src/include/planner/query_planner.h b/src/include/planner/query_planner.h
@@ -26,8 +26,8 @@ class QueryPlanner {
 
 public:
     QueryPlanner(const catalog::Catalog& catalog,
-        const storage::NodesStatisticsAndDeletedIDs& nodesStatistics,
-        const storage::RelsStatistics& relsStatistics)
+        const storage::NodesStoreStatsAndDeletedIDs& nodesStatistics,
+        const storage::RelsStoreStats& relsStatistics)
         : catalog{catalog} {
         cardinalityEstimator =
             std::make_unique<CardinalityEstimator>(nodesStatistics, relsStatistics);

diff --git a/src/include/processor/operator/ddl/create_node_table.h b/src/include/processor/operator/ddl/create_node_table.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include "processor/operator/ddl/ddl.h"
-#include "storage/stats/nodes_statistics_and_deleted_ids.h"
+#include "storage/stats/nodes_store_statistics.h"
 #include "storage/store/nodes_store.h"
 
 namespace kuzu {
@@ -10,7 +10,7 @@ namespace processor {
 class CreateNodeTable : public DDL {
 public:
     CreateNodeTable(catalog::Catalog* catalog, storage::StorageManager* storageManager,
-        storage::NodesStatisticsAndDeletedIDs* nodesStatistics,
+        storage::NodesStoreStatsAndDeletedIDs* nodesStatistics,
         std::unique_ptr<binder::BoundCreateTableInfo> info, const DataPos& outputPos, uint32_t id,
         const std::string& paramsString)
         : DDL{PhysicalOperatorType::CREATE_NODE_TABLE, catalog, outputPos, id, paramsString},
@@ -27,7 +27,7 @@ class CreateNodeTable : public DDL {
 
 private:
     storage::StorageManager* storageManager;
-    storage::NodesStatisticsAndDeletedIDs* nodesStatistics;
+    storage::NodesStoreStatsAndDeletedIDs* nodesStatistics;
     std::unique_ptr<binder::BoundCreateTableInfo> info;
 };
 

diff --git a/src/include/processor/operator/ddl/create_rdf_graph.h b/src/include/processor/operator/ddl/create_rdf_graph.h
@@ -9,8 +9,8 @@ namespace processor {
 class CreateRdfGraph : public DDL {
 public:
     CreateRdfGraph(catalog::Catalog* catalog, storage::StorageManager* storageManager,
-        storage::NodesStatisticsAndDeletedIDs* nodesStatistics,
-        storage::RelsStatistics* relsStatistics, std::unique_ptr<binder::BoundCreateTableInfo> info,
+        storage::NodesStoreStatsAndDeletedIDs* nodesStatistics,
+        storage::RelsStoreStats* relsStatistics, std::unique_ptr<binder::BoundCreateTableInfo> info,
         const DataPos& outputPos, uint32_t id, const std::string& paramsString)
         : DDL{PhysicalOperatorType::CREATE_RDF_GRAPH, catalog, outputPos, id, paramsString},
           storageManager{storageManager}, nodesStatistics{nodesStatistics},
@@ -27,8 +27,8 @@ class CreateRdfGraph : public DDL {
 
 private:
     storage::StorageManager* storageManager;
-    storage::NodesStatisticsAndDeletedIDs* nodesStatistics;
-    storage::RelsStatistics* relsStatistics;
+    storage::NodesStoreStatsAndDeletedIDs* nodesStatistics;
+    storage::RelsStoreStats* relsStatistics;
     std::unique_ptr<binder::BoundCreateTableInfo> info;
 };
 

diff --git a/src/include/processor/operator/ddl/create_rel_table.h b/src/include/processor/operator/ddl/create_rel_table.h
@@ -1,14 +1,14 @@
 #pragma once
 
 #include "processor/operator/ddl/ddl.h"
-#include "storage/stats/rels_statistics.h"
+#include "storage/stats/rels_store_statistics.h"
 
 namespace kuzu {
 namespace processor {
 
 class CreateRelTable : public DDL {
 public:
-    CreateRelTable(catalog::Catalog* catalog, storage::RelsStatistics* relsStatistics,
+    CreateRelTable(catalog::Catalog* catalog, storage::RelsStoreStats* relsStatistics,
         std::unique_ptr<binder::BoundCreateTableInfo> info, const DataPos& outputPos, uint32_t id,
         const std::string& paramsString)
         : DDL{PhysicalOperatorType::CREATE_REL_TABLE, catalog, outputPos, id, paramsString},
@@ -24,7 +24,7 @@ class CreateRelTable : public DDL {
     }
 
 private:
-    storage::RelsStatistics* relsStatistics;
+    storage::RelsStoreStats* relsStatistics;
     std::unique_ptr<binder::BoundCreateTableInfo> info;
 };
 

diff --git a/src/include/processor/operator/ddl/create_rel_table_group.h b/src/include/processor/operator/ddl/create_rel_table_group.h
@@ -1,14 +1,14 @@
 #pragma once
 
 #include "processor/operator/ddl/ddl.h"
-#include "storage/stats/rels_statistics.h"
+#include "storage/stats/rels_store_statistics.h"
 
 namespace kuzu {
 namespace processor {
 
 class CreateRelTableGroup : public DDL {
 public:
-    CreateRelTableGroup(catalog::Catalog* catalog, storage::RelsStatistics* relsStatistics,
+    CreateRelTableGroup(catalog::Catalog* catalog, storage::RelsStoreStats* relsStatistics,
         std::unique_ptr<binder::BoundCreateTableInfo> info, const DataPos& outputPos, uint32_t id,
         const std::string& paramsString)
         : DDL{PhysicalOperatorType::CREATE_REL_TABLE, catalog, outputPos, id, paramsString},
@@ -24,7 +24,7 @@ class CreateRelTableGroup : public DDL {
     }
 
 private:
-    storage::RelsStatistics* relsStatistics;
+    storage::RelsStoreStats* relsStatistics;
     std::unique_ptr<binder::BoundCreateTableInfo> info;
 };
 

diff --git a/src/include/processor/operator/persistent/copy_node.h b/src/include/processor/operator/persistent/copy_node.h
@@ -103,11 +103,7 @@ class CopyNode : public Sink {
         storage::NodeTable* table, storage::NodeGroup* nodeGroup, bool isCopyTurtle);
 
 private:
-    inline bool isCopyAllowed() const {
-        auto nodesStatistics = copyNodeInfo.table->getNodeStatisticsAndDeletedIDs();
-        return nodesStatistics->getNodeStatisticsAndDeletedIDs(copyNodeInfo.table->getTableID())
-                   ->getNumTuples() == 0;
-    }
+    bool isCopyAllowed() const;
 
     static void populatePKIndex(storage::PrimaryKeyIndexBuilder* pkIndex,
         storage::ColumnChunk* chunk, common::offset_t startNodeOffset, common::offset_t numNodes);

diff --git a/src/include/processor/operator/persistent/copy_rel.h b/src/include/processor/operator/persistent/copy_rel.h
@@ -6,7 +6,7 @@
 #include "storage/in_mem_storage_structure/in_mem_column.h"
 #include "storage/in_mem_storage_structure/in_mem_column_chunk.h"
 #include "storage/in_mem_storage_structure/in_mem_lists.h"
-#include "storage/stats/rels_statistics.h"
+#include "storage/stats/rels_store_statistics.h"
 
 namespace kuzu {
 namespace processor {
@@ -70,7 +70,7 @@ class CopyRelSharedState {
     friend class CopyRelLists;
 
 public:
-    CopyRelSharedState(common::table_id_t tableID, storage::RelsStatistics* relsStatistics,
+    CopyRelSharedState(common::table_id_t tableID, storage::RelsStoreStats* relsStatistics,
         std::unique_ptr<DirectedInMemRelData> fwdRelData,
         std::unique_ptr<DirectedInMemRelData> bwdRelData, storage::MemoryManager* memoryManager);
 
@@ -85,7 +85,7 @@ class CopyRelSharedState {
 private:
     std::mutex mtx;
     common::table_id_t tableID;
-    storage::RelsStatistics* relsStatistics;
+    storage::RelsStoreStats* relsStatistics;
     std::unique_ptr<DirectedInMemRelData> fwdRelData;
     std::unique_ptr<DirectedInMemRelData> bwdRelData;
     std::shared_ptr<FactorizedTable> fTable;

diff --git a/src/include/processor/operator/persistent/copy_rel_columns.h b/src/include/processor/operator/persistent/copy_rel_columns.h
@@ -3,7 +3,7 @@
 #include "processor/operator/persistent/copy_rel.h"
 #include "processor/operator/sink.h"
 #include "storage/index/hash_index.h"
-#include "storage/stats/rels_statistics.h"
+#include "storage/stats/rels_store_statistics.h"
 
 namespace kuzu {
 namespace processor {

diff --git a/src/include/processor/operator/persistent/delete_executor.h b/src/include/processor/operator/persistent/delete_executor.h
@@ -88,7 +88,7 @@ class RelDeleteExecutor {
 
 class SingleLabelRelDeleteExecutor : public RelDeleteExecutor {
 public:
-    SingleLabelRelDeleteExecutor(storage::RelsStatistics* relsStatistic, storage::RelTable* table,
+    SingleLabelRelDeleteExecutor(storage::RelsStoreStats* relsStatistic, storage::RelTable* table,
         const DataPos& srcNodeIDPos, const DataPos& dstNodeIDPos, const DataPos& relIDPos)
         : RelDeleteExecutor(srcNodeIDPos, dstNodeIDPos, relIDPos),
           relsStatistic{relsStatistic}, table{table} {}
@@ -103,12 +103,12 @@ class SingleLabelRelDeleteExecutor : public RelDeleteExecutor {
     }
 
 private:
-    storage::RelsStatistics* relsStatistic;
+    storage::RelsStoreStats* relsStatistic;
     storage::RelTable* table;
 };
 
 class MultiLabelRelDeleteExecutor : public RelDeleteExecutor {
-    using rel_table_statistic_pair = std::pair<storage::RelTable*, storage::RelsStatistics*>;
+    using rel_table_statistic_pair = std::pair<storage::RelTable*, storage::RelsStoreStats*>;
 
 public:
     MultiLabelRelDeleteExecutor(

diff --git a/src/include/processor/operator/persistent/insert_executor.h b/src/include/processor/operator/persistent/insert_executor.h
@@ -43,7 +43,7 @@ class NodeInsertExecutor {
 
 class RelInsertExecutor {
 public:
-    RelInsertExecutor(storage::RelsStatistics& relsStatistics, storage::RelTable* table,
+    RelInsertExecutor(storage::RelsStoreStats& relsStatistics, storage::RelTable* table,
         const DataPos& srcNodePos, const DataPos& dstNodePos,
         std::vector<DataPos> propertyLhsPositions,
         std::vector<std::unique_ptr<evaluator::ExpressionEvaluator>> propertyRhsEvaluators)
@@ -65,7 +65,7 @@ class RelInsertExecutor {
         const std::vector<std::unique_ptr<RelInsertExecutor>>& executors);
 
 private:
-    storage::RelsStatistics& relsStatistics;
+    storage::RelsStoreStats& relsStatistics;
     storage::RelTable* table;
     DataPos srcNodePos;
     DataPos dstNodePos;

diff --git a/src/include/processor/operator/scan/scan_rel_table_columns.h b/src/include/processor/operator/scan/scan_rel_table_columns.h
@@ -2,7 +2,7 @@
 
 #include "processor/operator/filtering_operator.h"
 #include "processor/operator/scan/scan_rel_table.h"
-#include "storage/stats/rels_statistics.h"
+#include "storage/stats/rels_store_statistics.h"
 
 namespace kuzu {
 namespace processor {

diff --git a/src/include/processor/plan_mapper.h b/src/include/processor/plan_mapper.h
@@ -6,7 +6,7 @@
 #include "planner/operator/logical_plan.h"
 #include "processor/operator/result_collector.h"
 #include "processor/physical_plan.h"
-#include "storage/stats/nodes_statistics_and_deleted_ids.h"
+#include "storage/stats/node_table_statistics.h"
 #include "storage/storage_manager.h"
 
 namespace kuzu {

diff --git a/src/include/storage/stats/metadata_dah_info.h b/src/include/storage/stats/metadata_dah_info.h
@@ -0,0 +1,28 @@
+#pragma once
+
+#include "common/types/types.h"
+
+namespace kuzu {
+namespace storage {
+
+// DAH is the abbreviation for Disk Array Header.
+struct MetadataDAHInfo {
+    common::page_idx_t dataDAHPageIdx = common::INVALID_PAGE_IDX;
+    common::page_idx_t nullDAHPageIdx = common::INVALID_PAGE_IDX;
+    std::vector<std::unique_ptr<MetadataDAHInfo>> childrenInfos;
+
+    MetadataDAHInfo() : MetadataDAHInfo{common::INVALID_PAGE_IDX, common::INVALID_PAGE_IDX} {}
+    MetadataDAHInfo(common::page_idx_t dataDAHPageIdx)
+        : MetadataDAHInfo{dataDAHPageIdx, common::INVALID_PAGE_IDX} {}
+    MetadataDAHInfo(common::page_idx_t dataDAHPageIdx, common::page_idx_t nullDAHPageIdx)
+        : dataDAHPageIdx{dataDAHPageIdx}, nullDAHPageIdx{nullDAHPageIdx} {}
+
+    std::unique_ptr<MetadataDAHInfo> copy();
+
+    void serialize(common::FileInfo* fileInfo, uint64_t& offset) const;
+    static std::unique_ptr<MetadataDAHInfo> deserialize(
+        common::FileInfo* fileInfo, uint64_t& offset);
+};
+
+} // namespace storage
+} // namespace kuzu