Testing framework v2 (#1548)

Support multiple statements, statement blocks and exceptions.

* Move the logic from TestCase to TestParser
* Converted current test files to the new framework
* Separate TestRunner from TestHelper
* Removed CMakefiles and syntax adjustments
* Cleanup functions not being used anymore
* Cleanup unused variables
* Changes after review
* Remove using namespaces from headers
* bfs_sssp test adjustment

CMakeLists.txt

@@ -71,7 +71,7 @@ endfunction()
 function(add_kuzu_test TEST_NAME)
     set(SRCS ${ARGN})
     add_executable(${TEST_NAME} ${SRCS})
-    target_link_libraries(${TEST_NAME} PRIVATE test_helper graph_test)
+    target_link_libraries(${TEST_NAME} PRIVATE test_helper test_runner graph_test)
     target_include_directories(${TEST_NAME} PRIVATE ${PROJECT_SOURCE_DIR}/test/include)
     include(GoogleTest)
     gtest_discover_tests(${TEST_NAME})

src/common/file_utils.cpp

@@ -127,16 +127,5 @@ std::vector<std::string> FileUtils::globFilePath(const std::string& path) {
     return result;
 }
 
-std::vector<std::string> FileUtils::findAllDirectories(const std::string& path) {
-    std::vector<std::string> directories;
-    directories.push_back(path);
-    for (const auto& entry : std::filesystem::recursive_directory_iterator(path)) {
-        if (entry.is_directory()) {
-            directories.push_back(entry.path().string());
-        }
-    }
-    return directories;
-}
-
 } // namespace common
 } // namespace kuzu

src/common/string_utils.cpp

@@ -22,5 +22,14 @@ std::vector<std::string> StringUtils::split(
     return result;
 }
 
+std::vector<std::string> StringUtils::splitBySpace(const std::string& input) {
+    std::istringstream iss(input);
+    std::vector<std::string> result;
+    std::string token;
+    while (iss >> token)
+        result.push_back(token);
+    return result;
+}
+
 } // namespace common
 } // namespace kuzu

src/include/common/file_utils.h

@@ -65,16 +65,6 @@ class FileUtils {
 
     static std::vector<std::string> globFilePath(const std::string& path);
 
-    static std::vector<std::string> findAllDirectories(const std::string& path);
-
-    static inline std::string getParentPath(const std::filesystem::path& path) {
-        return path.parent_path().string();
-    }
-
-    static inline std::string getParentPathStem(const std::filesystem::path& path) {
-        return path.parent_path().stem().string();
-    }
-
     static inline std::string getFileExtension(const std::filesystem::path& path) {
         return path.extension().string();
     }

src/include/common/string_utils.h

@@ -22,6 +22,8 @@ class StringUtils {
     static std::vector<std::string> split(
         const std::string& input, const std::string& delimiter, bool ignoreEmptyStringParts = true);
 
+    static std::vector<std::string> splitBySpace(const std::string& input);
+
     static void toUpper(std::string& input) {
         std::transform(input.begin(), input.end(), input.begin(), ::toupper);
     }
@@ -41,6 +43,13 @@ class StringUtils {
         return string_format("Maximum length of strings is {}. Input string's length is {}.",
             maxAllowedStrSize, strlen(strToInsert), strToInsert);
     }
+
+    static inline std::string ltrim(const std::string& input) {
+        auto s = input;
+        s.erase(
+            s.begin(), find_if(s.begin(), s.end(), [](unsigned char ch) { return !isspace(ch); }));
+        return s;
+    }
 };
 
 } // namespace common

src/include/main/connection.h

@@ -16,6 +16,7 @@ class Connection {
     friend class kuzu::testing::ApiTest;
     friend class kuzu::testing::BaseGraphTest;
     friend class kuzu::testing::TestHelper;
+    friend class kuzu::testing::TestRunner;
     friend class kuzu::benchmark::Benchmark;
 
 public:

src/include/main/kuzu_fwd.h

@@ -6,6 +6,7 @@ namespace testing {
 class ApiTest;
 class BaseGraphTest;
 class TestHelper;
+class TestRunner;
 class TinySnbDDLTest;
 class TinySnbCopyCSVTransactionTest;
 } // namespace testing

src/include/main/prepared_statement.h

@@ -14,6 +14,7 @@ namespace main {
 class PreparedStatement {
     friend class Connection;
     friend class testing::TestHelper;
+    friend class testing::TestRunner;
     friend class testing::TinySnbDDLTest;
     friend class testing::TinySnbCopyCSVTransactionTest;
 

test/CMakeLists.txt

@@ -12,6 +12,7 @@ target_link_libraries(GTest::GTest INTERFACE gmock_main)
 
 enable_testing()
 add_subdirectory(test_helper)
+add_subdirectory(test_runner)
 add_subdirectory(graph_test)
 add_subdirectory(binder)
 add_subdirectory(c_api)

test/common/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_kuzu_test(types_test
         date_test.cpp
         interval_test.cpp
+        string_test.cpp
         time_test.cpp
         timestamp_test.cpp
         types_test.cpp)

test/common/string_test.cpp

@@ -0,0 +1,22 @@
+#include <string>
+
+#include "common/types/types_include.h"
+#include "gtest/gtest.h"
+
+using namespace kuzu::common;
+
+TEST(StringTest, splitBySpace) {
+    std::string str = " a b  c\td ";
+    std::vector<std::string> result = StringUtils::splitBySpace(str);
+    EXPECT_EQ(result.size(), 4);
+    EXPECT_EQ(result[0], "a");
+    EXPECT_EQ(result[1], "b");
+    EXPECT_EQ(result[2], "c");
+    EXPECT_EQ(result[3], "d");
+}
+
+TEST(StringTest, leftTrim) {
+    std::string str = "    command ";
+    std::string result = StringUtils::ltrim(str);
+    EXPECT_EQ(result, "command ");
+}

test/include/graph_test/graph_test.h

@@ -9,6 +9,7 @@
 #include "planner/logical_plan/logical_plan_util.h"
 #include "planner/planner.h"
 #include "test_helper/test_helper.h"
+#include "test_runner/test_runner.h"
 
 using ::testing::Test;
 
@@ -155,6 +156,11 @@ class DBTest : public BaseGraphTest {
         initGraph();
     }
 
+    inline void runTest(const std::vector<std::unique_ptr<TestStatement>>& statements) {
+        TestRunner::runTest(statements, *conn);
+    }
+
+    // Deprecated
     inline void runTest(const std::string& queryFile) {
         auto queryConfigs = TestHelper::parseTestFile(queryFile);
         ASSERT_TRUE(TestHelper::testQueries(queryConfigs, *conn));

test/include/test_helper/test_helper.h

@@ -13,18 +13,6 @@ using namespace kuzu::main;
 namespace kuzu {
 namespace testing {
 
-struct TestConfig {
-    std::string testGroup;
-    std::string testName;
-    std::string dataset;
-    bool checkOrder = false;
-    std::vector<std::string> files;
-
-    bool isValid() const {
-        return !testGroup.empty() && !testName.empty() && !dataset.empty() && !files.empty();
-    }
-};
-
 struct TestQueryConfig {
     std::string name;
     std::string query;
@@ -38,8 +26,6 @@ struct TestQueryConfig {
 
 class TestHelper {
 public:
-    static TestConfig parseGroupFile(const std::string& path);
-
     static std::vector<std::unique_ptr<TestQueryConfig>> parseTestFile(
         const std::string& path, bool checkOutputOrder = false);
 
@@ -66,11 +52,6 @@ class TestHelper {
 private:
     static void initializeConnection(TestQueryConfig* config, Connection& conn);
     static bool testQuery(TestQueryConfig* config, Connection& conn);
-    static void setConfigValue(
-        const std::string& line, std::string& configItem, const std::string& configKey);
-    static void setConfigValue(
-        const std::string& line, bool& configItem, const std::string& configKey);
-    static std::string extractConfigValue(const std::string& line, const std::string& configKey);
 };
 
 } // namespace testing

test/include/test_runner/test_case.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "main/kuzu.h"
+
+namespace kuzu {
+namespace testing {
+
+struct TestStatement {
+    std::string name;
+    std::string query;
+    uint64_t numThreads = 4;
+    std::string encodedJoin;
+    uint64_t expectedNumTuples = 0;
+    bool expectedError = false;
+    bool expectedOk = false;
+    std::vector<std::string> expectedTuples;
+    std::string errorMessage;
+    bool enumerate = false;
+    bool checkOutputOrder = false;
+};
+
+struct TestCase {
+    std::string group;
+    std::string name;
+    std::string dataset;
+    std::vector<std::unique_ptr<TestStatement>> statements;
+    std::unordered_map<std::string, std::vector<std::unique_ptr<TestStatement>>> variableStatements;
+
+    bool skipTest = false;
+
+    bool isValid() const { return !group.empty() && !name.empty() && !dataset.empty(); }
+
+    bool hasStatements() const { return !statements.empty(); }
+};
+
+} // namespace testing
+} // namespace kuzu

test/include/test_runner/test_parser.h

@@ -0,0 +1,93 @@
+#include <cstring>
+
+#include "common/file_utils.h"
+#include "test_runner/test_case.h"
+
+namespace kuzu {
+namespace testing {
+
+enum class TokenType {
+    GROUP,
+    DATASET,
+    TEST,
+    CASE,
+    CHECK_ORDER,
+    DEFINE_STATEMENT_BLOCK,
+    EMPTY,
+    ENCODED_JOIN,
+    END_OF_STATEMENT_BLOCK,
+    ENUMERATE,
+    NAME,
+    PARALLELISM,
+    QUERY,
+    READ_ONLY,
+    RESULT,
+    SEPARATOR,
+    SKIP,
+    STATEMENT,
+    STATEMENT_BLOCK
+};
+
+const std::unordered_map<std::string, TokenType> tokenMap = {{"-GROUP", TokenType::GROUP},
+    {"-TEST", TokenType::TEST}, {"-DATASET", TokenType::DATASET}, {"-CASE", TokenType::CASE},
+    {"-CHECK_ORDER", TokenType::CHECK_ORDER}, {"-ENCODED_JOIN", TokenType::ENCODED_JOIN},
+    {"-DEFINE_STATEMENT_BLOCK", TokenType::DEFINE_STATEMENT_BLOCK},
+    {"-ENUMERATE", TokenType::ENUMERATE}, {"-NAME", TokenType::NAME},
+    {"-PARALLELISM", TokenType::PARALLELISM}, {"-QUERY", TokenType::QUERY},
+    {"-READ_ONLY", TokenType::READ_ONLY}, {"-SKIP", TokenType::SKIP},
+    {"-STATEMENT", TokenType::STATEMENT}, {"-STATEMENT_BLOCK", TokenType::STATEMENT_BLOCK},
+    {"]", TokenType::END_OF_STATEMENT_BLOCK}, {"----", TokenType::RESULT},
+    {"--", TokenType::SEPARATOR}, {"#", TokenType::EMPTY}};
+
+class LogicToken {
+public:
+    TokenType type;
+    std::vector<std::string> params;
+};
+
+class TestParser {
+public:
+    TestParser() : testCase(std::make_unique<TestCase>()) {}
+
+    std::unique_ptr<TestCase> parseTestFile(const std::string& path);
+
+private:
+    std::ifstream fileStream;
+    std::string line;
+    std::string name;
+    LogicToken currentToken;
+    std::unique_ptr<TestCase> testCase;
+
+    std::string paramsToString();
+
+    void openFile(const std::string& path);
+
+    void tokenize();
+
+    void parseHeader();
+
+    void parseBody();
+
+    void extractExpectedResult(TestStatement* currentStatement);
+
+    void extractStatementBlock();
+
+    void addStatementBlock(const std::string& blockName);
+
+    inline bool endOfFile() { return fileStream.eof(); }
+
+    inline bool nextLine() { return static_cast<bool>(getline(fileStream, line)); }
+
+    inline void checkMinimumParams(int minimumParams) {
+        if (currentToken.params.size() < minimumParams) {
+            throw common::Exception("Invalid number of parameters for statement [" + line + "]");
+        }
+    }
+
+    TestStatement* extractStatement(TestStatement* currentStatement);
+
+    TestStatement* addNewStatement();
+};
+
+} // namespace testing
+} // namespace kuzu

test/include/test_runner/test_runner.h

@@ -0,0 +1,29 @@
+#include "common/file_utils.h"
+#include "gtest/gtest.h"
+#include "parser/parser.h"
+#include "planner/logical_plan/logical_plan_util.h"
+#include "planner/planner.h"
+#include "test_runner/test_case.h"
+
+namespace kuzu {
+namespace testing {
+
+class TestRunner {
+public:
+    static void runTest(
+        const std::vector<std::unique_ptr<TestStatement>>& statements, main::Connection& conn);
+
+    static std::unique_ptr<planner::LogicalPlan> getLogicalPlan(
+        const std::string& query, main::Connection& conn);
+
+private:
+    static void initializeConnection(TestStatement* statement, main::Connection& conn);
+    static bool testStatement(TestStatement* statement, main::Connection& conn);
+    static bool checkLogicalPlans(std::unique_ptr<main::PreparedStatement>& preparedStatement,
+        TestStatement* statement, main::Connection& conn);
+    static std::vector<std::string> convertResultToString(
+        main::QueryResult& queryResult, bool checkOutputOrder = false);
+};
+
+} // namespace testing
+} // namespace kuzu