Merge pull request #1747 from kuzudb/explain-pipeline

Refactor explain pipeline

src/binder/bind/CMakeLists.txt

@@ -4,6 +4,7 @@ add_library(
         bind_standalone_call.cpp
         bind_copy.cpp
         bind_ddl.cpp
+        bind_explain.cpp
         bind_graph_pattern.cpp
         bind_projection_clause.cpp
         bind_query.cpp

src/binder/bind/bind_explain.cpp

@@ -0,0 +1,15 @@
+#include "binder/binder.h"
+#include "binder/bound_explain.h"
+#include "parser/explain_statement.h"
+
+namespace kuzu {
+namespace binder {
+
+std::unique_ptr<BoundStatement> Binder::bindExplain(const parser::Statement& statement) {
+    auto& explainStatement = (parser::ExplainStatement&)statement;
+    auto boundStatementToExplain = bind(*explainStatement.getStatementToExplain());
+    return std::make_unique<BoundExplain>(std::move(boundStatementToExplain));
+}
+
+} // namespace binder
+} // namespace kuzu

src/binder/binder.cpp

@@ -39,9 +39,12 @@ std::unique_ptr<BoundStatement> Binder::bind(const Statement& statement) {
     case StatementType::QUERY: {
         return bindQuery((const RegularQuery&)statement);
     }
-    case StatementType::StandaloneCall: {
+    case StatementType::STANDALONE_CALL: {
         return bindStandaloneCall(statement);
     }
+    case StatementType::EXPLAIN: {
+        return bindExplain(statement);
+    }
     default:
         assert(false);
     }

src/binder/bound_statement_visitor.cpp

@@ -1,5 +1,7 @@
 #include "binder/bound_statement_visitor.h"
 
+#include "binder/bound_explain.h"
+
 using namespace kuzu::common;
 
 namespace kuzu {
@@ -34,9 +36,12 @@ void BoundStatementVisitor::visit(const kuzu::binder::BoundStatement& statement)
     case StatementType::COPY: {
         visitCopy(statement);
     } break;
-    case StatementType::StandaloneCall: {
+    case StatementType::STANDALONE_CALL: {
         visitStandaloneCall(statement);
     } break;
+    case StatementType::EXPLAIN: {
+        visitExplain(statement);
+    } break;
     default:
         throw NotImplementedException("BoundStatementVisitor::visit");
     }
@@ -69,6 +74,10 @@ void BoundStatementVisitor::visitQueryPart(const NormalizedQueryPart& queryPart)
     }
 }
 
+void BoundStatementVisitor::visitExplain(const BoundStatement& statement) {
+    visit(*((const BoundExplain&)statement).getStatementToExplain());
+}
+
 void BoundStatementVisitor::visitReadingClause(const BoundReadingClause& readingClause) {
     switch (readingClause.getClauseType()) {
     case common::ClauseType::MATCH: {

src/include/binder/binder.h

@@ -111,6 +111,9 @@ class Binder {
     /*** bind call ***/
     std::unique_ptr<BoundStatement> bindStandaloneCall(const parser::Statement& statement);
 
+    /*** bind explain ***/
+    std::unique_ptr<BoundStatement> bindExplain(const parser::Statement& statement);
+
     /*** bind reading clause ***/
     std::unique_ptr<BoundReadingClause> bindReadingClause(
         const parser::ReadingClause& readingClause);

src/include/binder/bound_explain.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "binder/bound_statement.h"
+
+namespace kuzu {
+namespace binder {
+
+class BoundExplain : public BoundStatement {
+public:
+    explicit BoundExplain(std::unique_ptr<BoundStatement> statementToExplain)
+        : BoundStatement{common::StatementType::EXPLAIN,
+              BoundStatementResult::createSingleStringColumnResult()},
+          statementToExplain{std::move(statementToExplain)} {}
+
+    inline BoundStatement* getStatementToExplain() const { return statementToExplain.get(); }
+
+private:
+    std::unique_ptr<BoundStatement> statementToExplain;
+};
+
+} // namespace binder
+} // namespace kuzu

src/include/binder/bound_statement_visitor.h

@@ -27,6 +27,7 @@ class BoundStatementVisitor {
     virtual void visitRenameProperty(const BoundStatement& statement) {}
     virtual void visitCopy(const BoundStatement& statement) {}
     virtual void visitStandaloneCall(const BoundStatement& statement) {}
+    virtual void visitExplain(const BoundStatement& statement);
 
     void visitReadingClause(const BoundReadingClause& readingClause);
     virtual void visitMatch(const BoundReadingClause& readingClause) {}

src/include/binder/call/bound_standalone_call.h

@@ -9,7 +9,7 @@ namespace binder {
 class BoundStandaloneCall : public BoundStatement {
 public:
     BoundStandaloneCall(main::ConfigurationOption option, std::shared_ptr<Expression> optionValue)
-        : BoundStatement{common::StatementType::StandaloneCall,
+        : BoundStatement{common::StatementType::STANDALONE_CALL,
               BoundStatementResult::createEmptyResult()},
           option{option}, optionValue{optionValue} {}
 

src/include/common/statement_type.h

@@ -15,7 +15,8 @@ enum class StatementType : uint8_t {
     DROP_PROPERTY = 7,
     RENAME_PROPERTY = 8,
     COPY = 20,
-    StandaloneCall = 21,
+    STANDALONE_CALL = 21,
+    EXPLAIN = 22,
 };
 
 class StatementTypeUtils {

src/include/main/query_summary.h

@@ -13,6 +13,7 @@ namespace main {
  */
 struct PreparedSummary {
     double compilingTime = 0;
+    // Only used for printing by shell.
     bool isExplain = false;
     bool isProfile = false;
 };
@@ -33,7 +34,6 @@ class QuerySummary {
      * @return query execution time in milliseconds.
      */
     KUZU_API double getExecutionTime() const;
-    bool getIsExplain() const;
     bool getIsProfile() const;
     std::ostringstream& getPlanAsOstream();
     /**
@@ -42,12 +42,19 @@ class QuerySummary {
     KUZU_API std::string getPlan();
     void setPreparedSummary(PreparedSummary preparedSummary_);
 
+    /**
+     * @return true if the query is executed with EXPLAIN.
+     */
+    inline bool isExplain() const { return preparedSummary.isExplain; }
+
 private:
     nlohmann::json& printPlanToJson();
 
 private:
     double executionTime = 0;
     PreparedSummary preparedSummary;
+    // Remove these two field once we have refactored the profiler using the existing pipeline
+    // design.
     std::unique_ptr<nlohmann::json> planInJson;
     std::ostringstream planInOstream;
 };

src/include/parser/call/standalone_call.h

@@ -9,7 +9,7 @@ namespace parser {
 class StandaloneCall : public Statement {
 public:
     explicit StandaloneCall(std::string optionName, std::unique_ptr<ParsedExpression> optionValue)
-        : Statement{common::StatementType::StandaloneCall}, optionName{std::move(optionName)},
+        : Statement{common::StatementType::STANDALONE_CALL}, optionName{std::move(optionName)},
           optionValue{std::move(optionValue)} {}
 
     inline std::string getOptionName() const { return optionName; }

src/include/parser/explain_statement.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "parser/statement.h"
+
+namespace kuzu {
+namespace parser {
+
+class ExplainStatement : public Statement {
+public:
+    explicit ExplainStatement(std::unique_ptr<Statement> statementToExplain)
+        : Statement{common::StatementType::EXPLAIN}, statementToExplain{
+                                                         std::move(statementToExplain)} {}
+
+    inline Statement* getStatementToExplain() const { return statementToExplain.get(); }
+
+private:
+    std::unique_ptr<Statement> statementToExplain;
+};
+
+} // namespace parser
+} // namespace kuzu

src/include/parser/statement.h

@@ -13,16 +13,11 @@ class Statement {
 
     inline common::StatementType getStatementType() const { return statementType; }
 
-    inline void enableExplain() { explain = true; }
-    inline bool isExplain() const { return explain; }
-
     inline void enableProfile() { profile = true; }
     inline bool isProfile() const { return profile; }
 
 private:
     common::StatementType statementType;
-    // If explain is enabled, we do not execute query but return physical plan only.
-    bool explain = false;
     bool profile = false;
 };
 

src/include/planner/logical_plan/logical_operator/base_logical_operator.h

@@ -23,6 +23,7 @@ enum class LogicalOperatorType : uint8_t {
     DROP_PROPERTY,
     DROP_TABLE,
     EXPRESSIONS_SCAN,
+    EXPLAIN,
     EXTEND,
     FILTER,
     FLATTEN,

src/include/planner/logical_plan/logical_operator/logical_explain.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "base_logical_operator.h"
+
+namespace kuzu {
+namespace planner {
+
+class LogicalExplain : public LogicalOperator {
+public:
+    LogicalExplain(std::shared_ptr<LogicalOperator> child,
+        std::shared_ptr<binder::Expression> outputExpression)
+        : LogicalOperator{LogicalOperatorType::EXPLAIN, child}, outputExpression{
+                                                                    std::move(outputExpression)} {}
+
+    void computeFactorizedSchema() override;
+    void computeFlatSchema() override;
+
+    inline std::shared_ptr<binder::Expression> getOutputExpression() const {
+        return outputExpression;
+    }
+
+    inline std::string getExpressionsForPrinting() const override { return "Explain"; }
+
+    inline std::unique_ptr<LogicalOperator> copy() override {
+        return std::make_unique<LogicalExplain>(children[0], outputExpression);
+    }
+
+protected:
+    std::shared_ptr<binder::Expression> outputExpression;
+};
+
+} // namespace planner
+} // namespace kuzu

src/include/planner/planner.h

@@ -34,6 +34,10 @@ class Planner {
         const catalog::Catalog& catalog, const BoundStatement& statement);
 
     static std::unique_ptr<LogicalPlan> planStandaloneCall(const BoundStatement& statement);
+
+    static std::unique_ptr<LogicalPlan> planExplain(const catalog::Catalog& catalog,
+        const storage::NodesStatisticsAndDeletedIDs& nodesStatistics,
+        const storage::RelsStatistics& relsStatistics, const BoundStatement& statement);
 };
 
 } // namespace planner

src/include/processor/mapper/plan_mapper.h

@@ -114,6 +114,8 @@ class PlanMapper {
         planner::LogicalOperator* logicalOperator);
     std::unique_ptr<PhysicalOperator> mapLogicalInQueryCallToPhysical(
         planner::LogicalOperator* logicalOperator);
+    std::unique_ptr<PhysicalOperator> mapLogicalExplainToPhysical(
+        planner::LogicalOperator* logicalOperator);
     std::unique_ptr<ResultCollector> appendResultCollector(
         const binder::expression_vector& expressionsToCollect, planner::Schema* schema,
         std::unique_ptr<PhysicalOperator> prevOperator);

src/main/connection.cpp

@@ -160,8 +160,9 @@ std::unique_ptr<PreparedStatement> Connection::prepareNoLock(
     try {
         // parsing
         auto statement = Parser::parseQuery(query);
-        preparedStatement->preparedSummary.isExplain = statement->isExplain();
         preparedStatement->preparedSummary.isProfile = statement->isProfile();
+        preparedStatement->preparedSummary.isExplain =
+            statement->getStatementType() == StatementType::EXPLAIN;
         // binding
         auto binder = Binder(*database->catalog);
         auto boundStatement = binder.bind(*statement);
@@ -354,27 +355,23 @@ std::unique_ptr<QueryResult> Connection::executeAndAutoCommitIfNecessaryNoLock(
     auto executionContext =
         std::make_unique<ExecutionContext>(clientContext->numThreadsForExecution, profiler.get(),
             database->memoryManager.get(), database->bufferManager.get(), clientContext.get());
-    // Execute query if EXPLAIN is not enabled.
-    if (!preparedStatement->preparedSummary.isExplain) {
-        profiler->enabled = preparedStatement->preparedSummary.isProfile;
-        auto executingTimer = TimeMetric(true /* enable */);
-        executingTimer.start();
-        std::shared_ptr<FactorizedTable> resultFT;
-        try {
-            beginTransactionIfAutoCommit(preparedStatement);
-            executionContext->transaction = activeTransaction.get();
-            resultFT =
-                database->queryProcessor->execute(physicalPlan.get(), executionContext.get());
-            if (ConnectionTransactionMode::AUTO_COMMIT == transactionMode) {
-                commitNoLock();
-            }
-        } catch (Exception& exception) { return getQueryResultWithError(exception.what()); }
-        executingTimer.stop();
-        queryResult->querySummary->executionTime = executingTimer.getElapsedTimeMS();
-        queryResult->initResultTableAndIterator(std::move(resultFT),
-            preparedStatement->statementResult->getColumns(),
-            preparedStatement->statementResult->getExpressionsToCollectPerColumn());
-    }
+    profiler->enabled = preparedStatement->preparedSummary.isProfile;
+    auto executingTimer = TimeMetric(true /* enable */);
+    executingTimer.start();
+    std::shared_ptr<FactorizedTable> resultFT;
+    try {
+        beginTransactionIfAutoCommit(preparedStatement);
+        executionContext->transaction = activeTransaction.get();
+        resultFT = database->queryProcessor->execute(physicalPlan.get(), executionContext.get());
+        if (ConnectionTransactionMode::AUTO_COMMIT == transactionMode) {
+            commitNoLock();
+        }
+    } catch (Exception& exception) { return getQueryResultWithError(exception.what()); }
+    executingTimer.stop();
+    queryResult->querySummary->executionTime = executingTimer.getElapsedTimeMS();
+    queryResult->initResultTableAndIterator(std::move(resultFT),
+        preparedStatement->statementResult->getColumns(),
+        preparedStatement->statementResult->getExpressionsToCollectPerColumn());
     auto planPrinter = std::make_unique<PlanPrinter>(physicalPlan.get(), std::move(profiler));
     queryResult->querySummary->planInJson =
         std::make_unique<nlohmann::json>(planPrinter->printPlanToJson());

src/main/query_result.cpp

@@ -66,7 +66,7 @@ std::vector<common::LogicalType> QueryResult::getColumnDataTypes() const {
 }
 
 uint64_t QueryResult::getNumTuples() const {
-    return querySummary->getIsExplain() ? 0 : factorizedTable->getTotalNumFlatTuples();
+    return factorizedTable->getTotalNumFlatTuples();
 }
 
 QuerySummary* QueryResult::getQuerySummary() const {
@@ -185,7 +185,6 @@ void QueryResult::initResultTableAndIterator(
 
 bool QueryResult::hasNext() const {
     validateQuerySucceed();
-    assert(querySummary->getIsExplain() == false);
     return iterator->hasNextFlatTuple();
 }
 

src/main/query_summary.cpp

@@ -13,10 +13,6 @@ double QuerySummary::getExecutionTime() const {
     return executionTime;
 }
 
-bool QuerySummary::getIsExplain() const {
-    return preparedSummary.isExplain;
-}
-
 bool QuerySummary::getIsProfile() const {
     return preparedSummary.isProfile;
 }

src/parser/transformer.cpp

@@ -12,6 +12,7 @@
 #include "parser/ddl/drop_table.h"
 #include "parser/ddl/rename_property.h"
 #include "parser/ddl/rename_table.h"
+#include "parser/explain_statement.h"
 #include "parser/expression/parsed_case_expression.h"
 #include "parser/expression/parsed_function_expression.h"
 #include "parser/expression/parsed_literal_expression.h"
@@ -32,7 +33,7 @@ std::unique_ptr<Statement> Transformer::transform() {
     if (root.oC_AnyCypherOption()) {
         auto cypherOption = root.oC_AnyCypherOption();
         if (cypherOption->oC_Explain()) {
-            statement->enableExplain();
+            return std::make_unique<ExplainStatement>(std::move(statement));
         }
         if (cypherOption->oC_Profile()) {
             statement->enableProfile();

src/planner/operator/CMakeLists.txt

@@ -11,6 +11,7 @@ add_library(kuzu_planner_operator
         logical_cross_product.cpp
         logical_ddl.cpp
         logical_distinct.cpp
+        logical_explain.cpp
         logical_expressions_scan.cpp
         logical_extend.cpp
         logical_filter.cpp

src/planner/operator/base_logical_operator.cpp

@@ -61,6 +61,9 @@ std::string LogicalOperatorUtils::logicalOperatorTypeToString(LogicalOperatorTyp
     case LogicalOperatorType::EXTEND: {
         return "EXTEND";
     }
+    case LogicalOperatorType::EXPLAIN: {
+        return "EXPLAIN";
+    }
     case LogicalOperatorType::FILTER: {
         return "FILTER";
     }