Merge remote-tracking branch 'origin/master' into change-benchmark-th…

…reads

README.md

@@ -6,7 +6,7 @@
   <a href="https://github.com/kuzudb/kuzu/actions">
     <img src="https://github.com/kuzudb/kuzu/actions/workflows/ci-workflow.yml/badge.svg?branch=master" alt="Github Actions Badge">
   </a>
-  <a href="https://join.slack.com/t/kuzudb/shared_invite/zt-1n67h736q-E3AFGSI4w~ljlFMYr3_Sjg">
+  <a href="https://join.slack.com/t/kuzudb/shared_invite/zt-1qgxnn8ed-9LL7rfKozijOtvw5HyWDlQ">
     <img src="https://img.shields.io/badge/Slack-chat%20with%20us-informational?logo=slack" alt="slack" />
   </a>
   <a href="https://twitter.com/kuzudb">

benchmark/queries/ldbc-sf100/aggregation/q28.benchmark

@@ -1,9 +1,9 @@
 -NAME q28
 -COMPARE_RESULT 1
--QUERY MATCH(comment:Comment) RETURN comment.ID, count(comment.length) limit 5
+-QUERY MATCH(comment:Comment) WHERE comment.ID < 33980465466560 RETURN comment.ID as ID, count(comment.length) order by ID LIMIT 5;
 ---- 5
-39582418599937|1
-39582418599938|1
-65970697666565|1
-65970697666566|1
-65970697666567|1
+4196|1
+4197|1
+4198|1
+4199|1
+4200|1

dataset/tinysnb/eMarries.csv

@@ -1,3 +1,3 @@
 0,2,"[toronto]","[4,5]",
 3,5,,"[2,5]","long long long string"
-7,8,"[vancouver]","[3,9]","short str"
+7,8,"[]","[3,9]","short str"

src/common/in_mem_overflow_buffer_utils.cpp

@@ -25,6 +25,10 @@ void InMemOverflowBufferUtils::copyListNonRecursive(const uint8_t* srcValues, ku
 void InMemOverflowBufferUtils::copyListRecursiveIfNested(const ku_list_t& src, ku_list_t& dest,
     const DataType& dataType, InMemOverflowBuffer& inMemOverflowBuffer, uint32_t srcStartIdx,
     uint32_t srcEndIdx) {
+    if (src.size == 0) {
+        dest.size = 0;
+        return;
+    }
     if (srcEndIdx == UINT32_MAX) {
         srcEndIdx = src.size - 1;
     }

src/common/types/value.cpp

@@ -281,8 +281,11 @@ std::string Value::toString() const {
         std::string result = "[";
         for (auto i = 0u; i < listVal.size(); ++i) {
             result += listVal[i]->toString();
-            result += (i == listVal.size() - 1 ? "]" : ",");
+            if (i != listVal.size() - 1) {
+                result += ",";
+            }
         }
+        result += "]";
         return result;
     }
     case NODE:

src/include/common/constants.h

@@ -45,7 +45,7 @@ struct BufferPoolConstants {
 struct StorageConstants {
     // The default amount of memory pre-allocated to both the default and large pages buffer pool.
     static constexpr uint64_t DEFAULT_BUFFER_POOL_SIZE = 1ull << 30;             // (1GB)
-    static constexpr uint64_t DEFAULT_BUFFER_POOL_SIZE_FOR_TESTING = 1ull << 26; // (64MB)
+    static constexpr uint64_t DEFAULT_BUFFER_POOL_SIZE_FOR_TESTING = 1ull << 27; // (128MB)
     // The default ratio of system memory allocated to buffer pools (including default and large).
     static constexpr double DEFAULT_BUFFER_POOL_RATIO = 0.8;
     // The default ratio of buffer allocated to default and large pages.

src/include/main/connection.h

@@ -16,6 +16,7 @@ class Connection {
     friend class kuzu::testing::ApiTest;
     friend class kuzu::testing::BaseGraphTest;
     friend class kuzu::testing::TestHelper;
+    friend class kuzu::benchmark::Benchmark;
 
 public:
     /**
@@ -72,6 +73,7 @@ class Connection {
      * @return the maximum number of threads to use for execution in the current connection.
      */
     KUZU_API uint64_t getMaxNumThreadForExec();
+
     /**
      * @brief Executes the given query and returns the result.
      * @param query The query to execute.
@@ -125,10 +127,15 @@ class Connection {
      */
     KUZU_API std::string getRelPropertyNames(const std::string& relTableName);
 
+    // Temporary patching for C-style APIs.
+    // TODO(Change): move this to C-header once we have C-APIs.
+    KUZU_API std::unique_ptr<QueryResult> kuzu_query(const char* queryString);
+
 protected:
     ConnectionTransactionMode getTransactionMode();
     void setTransactionModeNoLock(ConnectionTransactionMode newTransactionMode);
 
+    std::unique_ptr<QueryResult> query(const std::string& query, const std::string& encodedJoin);
     // Note: This is only added for testing recovery algorithms in unit tests. Do not use
     // this otherwise.
     void commitButSkipCheckpointingForTestingRecovery();
@@ -151,8 +158,8 @@ class Connection {
 
     std::unique_ptr<QueryResult> queryResultWithError(std::string& errMsg);
 
-    std::unique_ptr<PreparedStatement> prepareNoLock(
-        const std::string& query, bool enumerateAllPlans = false);
+    std::unique_ptr<PreparedStatement> prepareNoLock(const std::string& query,
+        bool enumerateAllPlans = false, std::string joinOrder = std::string{});
 
     template<typename T, typename... Args>
     std::unique_ptr<QueryResult> executeWithParams(PreparedStatement* preparedStatement,

src/include/main/database.h

@@ -38,7 +38,6 @@ KUZU_API struct SystemConfig {
 class Database {
     friend class EmbeddedShell;
     friend class Connection;
-    friend class JOConnection;
     friend class kuzu::testing::BaseGraphTest;
 
 public:
@@ -73,6 +72,11 @@ class Database {
      */
     KUZU_API void resizeBufferManager(uint64_t newSize);
 
+    // Temporary patching for C-style APIs.
+    // TODO(Change): move this to C-header once we have C-APIs.
+    KUZU_API explicit Database(const char* databasePath);
+    KUZU_API Database(const char* databasePath, SystemConfig systemConfig);
+
 private:
     // Commits and checkpoints a write transaction or rolls that transaction back. This involves
     // either replaying the WAL and either redoing or undoing and in either case at the end WAL is

src/include/main/prepared_statement.h

@@ -13,7 +13,6 @@ namespace main {
  */
 class PreparedStatement {
     friend class Connection;
-    friend class JOConnection;
     friend class kuzu::testing::TestHelper;
     friend class kuzu::transaction::TinySnbDDLTest;
     friend class kuzu::transaction::TinySnbCopyCSVTransactionTest;

src/include/optimizer/projection_push_down_optimizer.h

@@ -37,6 +37,9 @@ class ProjectionPushDownOptimizer : public LogicalOperatorVisitor {
 
     binder::expression_vector pruneExpressions(const binder::expression_vector& expressions);
 
+    void preAppendProjection(
+        planner::LogicalOperator* op, uint32_t childIdx, binder::expression_vector expressions);
+
 private:
     binder::expression_set propertiesInUse;
 };

src/include/planner/logical_plan/logical_operator/base_logical_operator.h

@@ -77,8 +77,8 @@ class LogicalOperator {
     inline LogicalOperatorType getOperatorType() const { return operatorType; }
 
     inline Schema* getSchema() const { return schema.get(); }
-    void computeSchemaRecursive();
-    virtual void computeSchema() = 0;
+    virtual void computeFactorizedSchema() = 0;
+    virtual void computeFlatSchema() = 0;
 
     virtual std::string getExpressionsForPrinting() const = 0;
 

src/include/planner/logical_plan/logical_operator/logical_accumulate.h

@@ -7,28 +7,21 @@ namespace planner {
 
 class LogicalAccumulate : public LogicalOperator {
 public:
-    LogicalAccumulate(binder::expression_vector expressions, std::shared_ptr<LogicalOperator> child)
-        : LogicalOperator{LogicalOperatorType::ACCUMULATE, std::move(child)}, expressions{std::move(
-                                                                                  expressions)} {}
+    LogicalAccumulate(std::shared_ptr<LogicalOperator> child)
+        : LogicalOperator{LogicalOperatorType::ACCUMULATE, std::move(child)} {}
 
-    void computeSchema() override;
+    void computeFactorizedSchema() override;
+    void computeFlatSchema() override;
 
-    inline std::string getExpressionsForPrinting() const override {
-        return binder::ExpressionUtil::toString(expressions);
-    }
+    inline std::string getExpressionsForPrinting() const override { return std::string{}; }
 
-    inline void setExpressions(binder::expression_vector expressions_) {
-        expressions = std::move(expressions_);
+    inline binder::expression_vector getExpressions() const {
+        return children[0]->getSchema()->getExpressionsInScope();
     }
-    inline binder::expression_vector getExpressions() const { return expressions; }
-    inline Schema* getSchemaBeforeSink() const { return children[0]->getSchema(); }
 
     inline std::unique_ptr<LogicalOperator> copy() override {
-        return make_unique<LogicalAccumulate>(expressions, children[0]->copy());
+        return make_unique<LogicalAccumulate>(children[0]->copy());
     }
-
-private:
-    binder::expression_vector expressions;
 };
 
 } // namespace planner

src/include/planner/logical_plan/logical_operator/logical_aggregate.h

@@ -13,11 +13,12 @@ class LogicalAggregate : public LogicalOperator {
           expressionsToGroupBy{std::move(expressionsToGroupBy)}, expressionsToAggregate{std::move(
                                                                      expressionsToAggregate)} {}
 
+    void computeFactorizedSchema() override;
+    void computeFlatSchema() override;
+
     f_group_pos_set getGroupsPosToFlattenForGroupBy();
     f_group_pos_set getGroupsPosToFlattenForAggregate();
 
-    void computeSchema() override;
-
     std::string getExpressionsForPrinting() const override;
 
     inline bool hasExpressionsToGroupBy() const { return !expressionsToGroupBy.empty(); }

src/include/planner/logical_plan/logical_operator/logical_copy.h

@@ -15,7 +15,8 @@ class LogicalCopy : public LogicalOperator {
         : LogicalOperator{LogicalOperatorType::COPY_CSV},
           copyDescription{copyDescription}, tableID{tableID}, tableName{std::move(tableName)} {}
 
-    inline void computeSchema() override { createEmptySchema(); }
+    inline void computeFactorizedSchema() override { createEmptySchema(); }
+    inline void computeFlatSchema() override { createEmptySchema(); }
 
     inline std::string getExpressionsForPrinting() const override { return tableName; }
 

src/include/planner/logical_plan/logical_operator/logical_create.h

@@ -14,7 +14,8 @@ class LogicalCreateNode : public LogicalUpdateNode {
           primaryKeys{std::move(primaryKeys)} {}
     ~LogicalCreateNode() override = default;
 
-    void computeSchema() override;
+    void computeFactorizedSchema() override;
+    void computeFlatSchema() override;
 
     inline f_group_pos_set getGroupsPosToFlatten() {
         // Flatten all inputs. E.g. MATCH (a) CREATE (b). We need to create b for each tuple in the
@@ -45,6 +46,9 @@ class LogicalCreateRel : public LogicalUpdateRel {
           setItemsPerRel{std::move(setItemsPerRel)} {}
     ~LogicalCreateRel() override = default;
 
+    inline void computeFactorizedSchema() override { copyChildSchema(0); }
+    inline void computeFlatSchema() override { copyChildSchema(0); }
+
     inline f_group_pos_set getGroupsPosToFlatten() {
         auto childSchema = children[0]->getSchema();
         return factorization::FlattenAll::getGroupsPosToFlatten(

src/include/planner/logical_plan/logical_operator/logical_cross_product.h

@@ -13,7 +13,8 @@ class LogicalCrossProduct : public LogicalOperator {
         : LogicalOperator{LogicalOperatorType::CROSS_PRODUCT, std::move(probeSideChild),
               std::move(buildSideChild)} {}
 
-    void computeSchema() override;
+    void computeFactorizedSchema() override;
+    void computeFlatSchema() override;
 
     inline std::string getExpressionsForPrinting() const override { return std::string(); }
 

src/include/planner/logical_plan/logical_operator/logical_ddl.h

@@ -12,20 +12,16 @@ class LogicalDDL : public LogicalOperator {
         : LogicalOperator{operatorType}, tableName{std::move(tableName)},
           outputExpression{std::move(outputExpression)} {}
 
+    void computeFactorizedSchema() override;
+    void computeFlatSchema() override;
+
     inline std::string getTableName() const { return tableName; }
     inline std::shared_ptr<binder::Expression> getOutputExpression() const {
         return outputExpression;
     }
 
     inline std::string getExpressionsForPrinting() const override { return tableName; }
 
-    inline void computeSchema() override {
-        schema = std::make_unique<Schema>();
-        auto groupPos = schema->createGroup();
-        schema->insertToGroupAndScope(outputExpression, groupPos);
-        schema->setGroupAsSingleState(groupPos);
-    }
-
 protected:
     std::string tableName;
     std::shared_ptr<binder::Expression> outputExpression;

src/include/planner/logical_plan/logical_operator/logical_delete.h

@@ -14,6 +14,9 @@ class LogicalDeleteNode : public LogicalUpdateNode {
           primaryKeys{std::move(primaryKeys)} {}
     ~LogicalDeleteNode() override = default;
 
+    inline void computeFactorizedSchema() override { copyChildSchema(0); }
+    inline void computeFlatSchema() override { copyChildSchema(0); }
+
     inline std::shared_ptr<binder::Expression> getPrimaryKey(size_t idx) const {
         return primaryKeys[idx];
     }
@@ -33,6 +36,9 @@ class LogicalDeleteRel : public LogicalUpdateRel {
         : LogicalUpdateRel{LogicalOperatorType::DELETE_REL, std::move(rels), std::move(child)} {}
     ~LogicalDeleteRel() override = default;
 
+    inline void computeFactorizedSchema() override { copyChildSchema(0); }
+    inline void computeFlatSchema() override { copyChildSchema(0); }
+
     inline f_group_pos_set getGroupsPosToFlatten(uint32_t relIdx) {
         f_group_pos_set result;
         auto rel = rels[relIdx];

src/include/planner/logical_plan/logical_operator/logical_distinct.h

@@ -13,9 +13,10 @@ class LogicalDistinct : public LogicalOperator {
         : LogicalOperator{LogicalOperatorType::DISTINCT, std::move(child)},
           expressionsToDistinct{std::move(expressionsToDistinct)} {}
 
-    f_group_pos_set getGroupsPosToFlatten();
+    void computeFactorizedSchema() override;
+    void computeFlatSchema() override;
 
-    void computeSchema() override;
+    f_group_pos_set getGroupsPosToFlatten();
 
     std::string getExpressionsForPrinting() const override;
 

src/include/planner/logical_plan/logical_operator/logical_expressions_scan.h

@@ -14,7 +14,8 @@ class LogicalExpressionsScan : public LogicalOperator {
         : LogicalOperator{LogicalOperatorType::EXPRESSIONS_SCAN}, expressions{
                                                                       std::move(expressions)} {}
 
-    void computeSchema() override;
+    void computeFactorizedSchema() override;
+    void computeFlatSchema() override;
 
     inline std::string getExpressionsForPrinting() const override {
         return binder::ExpressionUtil::toString(expressions);

src/include/planner/logical_plan/logical_operator/logical_extend.h

@@ -19,7 +19,8 @@ class LogicalExtend : public LogicalOperator {
 
     f_group_pos_set getGroupsPosToFlatten();
 
-    void computeSchema() override;
+    void computeFactorizedSchema() override;
+    void computeFlatSchema() override;
 
     inline std::string getExpressionsForPrinting() const override {
         return boundNode->toString() + (direction == common::RelDirection::FWD ? "->" : "<-") +

src/include/planner/logical_plan/logical_operator/logical_filter.h

@@ -13,9 +13,10 @@ class LogicalFilter : public LogicalOperator {
         : LogicalOperator{LogicalOperatorType::FILTER, std::move(child)}, expression{std::move(
                                                                               expression)} {}
 
-    f_group_pos_set getGroupsPosToFlatten();
+    inline void computeFactorizedSchema() override { copyChildSchema(0); }
+    inline void computeFlatSchema() override { copyChildSchema(0); }
 
-    inline void computeSchema() override { copyChildSchema(0); }
+    f_group_pos_set getGroupsPosToFlatten();
 
     inline std::string getExpressionsForPrinting() const override { return expression->toString(); }
 

src/include/planner/logical_plan/logical_operator/logical_flatten.h

@@ -11,7 +11,8 @@ class LogicalFlatten : public LogicalOperator {
     LogicalFlatten(f_group_pos groupPos, std::shared_ptr<LogicalOperator> child)
         : LogicalOperator{LogicalOperatorType::FLATTEN, std::move(child)}, groupPos{groupPos} {}
 
-    void computeSchema() override;
+    void computeFactorizedSchema() override;
+    void computeFlatSchema() override;
 
     inline std::string getExpressionsForPrinting() const override { return std::string{}; }
 

src/include/planner/logical_plan/logical_operator/logical_ftable_scan.h

@@ -13,7 +13,8 @@ class LogicalFTableScan : public LogicalOperator {
           expressionsToScan{std::move(expressionsToScan)}, schemaToScanFrom{
                                                                std::move(schemaToScanFrom)} {}
 
-    void computeSchema() override;
+    void computeFactorizedSchema() override;
+    void computeFlatSchema() override;
 
     inline std::string getExpressionsForPrinting() const override {
         return binder::ExpressionUtil::toString(expressionsToScan);