Auto-parse end2end tests (#1507)

There is no need to write `TEST_F` to create a new end2end read test anymore. 
To create a new end2end read test:
- create a new folder inside `test_files` (can be subfolders as well)
- add all `.test` files related to that group
- create a `test.group` file with the configuration parameters

src/common/file_utils.cpp

@@ -127,5 +127,15 @@ std::vector<std::string> FileUtils::globFilePath(const std::string& path) {
     return result;
 }
 
+std::vector<std::string> FileUtils::findAllDirectories(const std::string& path) {
+    std::vector<std::string> directories;
+    for (const auto& entry : std::filesystem::recursive_directory_iterator(path)) {
+        if (entry.is_directory()) {
+            directories.push_back(entry.path().string());
+        }
+    }
+    return directories;
+}
+
 } // namespace common
 } // namespace kuzu

src/include/common/file_utils.h

@@ -64,6 +64,20 @@ class FileUtils {
     }
 
     static std::vector<std::string> globFilePath(const std::string& path);
+
+    static std::vector<std::string> findAllDirectories(const std::string& path);
+
+    static inline std::string getParentPath(const std::filesystem::path& path) {
+        return path.parent_path().string();
+    }
+
+    static inline std::string getParentPathStem(const std::filesystem::path& path) {
+        return path.parent_path().stem().string();
+    }
+
+    static inline std::string getFileExtension(const std::filesystem::path& path) {
+        return path.extension().string();
+    }
 };
 
 } // namespace common

test/include/test_helper/test_helper.h

@@ -13,6 +13,18 @@ using namespace kuzu::main;
 namespace kuzu {
 namespace testing {
 
+struct TestConfig {
+    std::string testGroup;
+    std::string testName;
+    std::string dataset;
+    bool checkOrder = false;
+    std::vector<std::string> files;
+
+    bool isValid() const {
+        return !testGroup.empty() && !testName.empty() && !dataset.empty() && !files.empty();
+    }
+};
+
 struct TestQueryConfig {
     std::string name;
     std::string query;
@@ -26,6 +38,8 @@ struct TestQueryConfig {
 
 class TestHelper {
 public:
+    static TestConfig parseGroupFile(const std::string& path);
+
     static std::vector<std::unique_ptr<TestQueryConfig>> parseTestFile(
         const std::string& path, bool checkOutputOrder = false);
 
@@ -52,6 +66,11 @@ class TestHelper {
 private:
     static void initializeConnection(TestQueryConfig* config, Connection& conn);
     static bool testQuery(TestQueryConfig* config, Connection& conn);
+    static void setConfigValue(
+        const std::string& line, std::string& configValue, const std::string& configKey);
+    static void setConfigValue(
+        const std::string& line, bool& configValue, const std::string& configKey);
+    static std::string extractConfigValue(const std::string& line, const std::string& configKey);
 };
 
 } // namespace testing

test/runner/e2e_read_test.cpp

@@ -2,160 +2,59 @@
 
 using ::testing::Test;
 using namespace kuzu::testing;
+using namespace kuzu::common;
 
-class LongStringPKTest : public DBTest {
-    std::string getInputDir() override {
-        return TestHelper::appendKuzuRootPath("dataset/long-string-pk-tests/");
-    }
-};
-
-class TinySnbReadTest : public DBTest {
+class EndToEndReadTest : public DBTest {
 public:
+    EndToEndReadTest(TestConfig testConfig) : testConfig(std::move(testConfig)) {}
     std::string getInputDir() override {
-        return TestHelper::appendKuzuRootPath("dataset/tinysnb/");
+        return TestHelper::appendKuzuRootPath("dataset/" + testConfig.dataset + "/");
+    }
+    void TestBody() override {
+        for (auto& file : testConfig.files) {
+            if (testConfig.checkOrder) {
+                runTestAndCheckOrder(file);
+            } else {
+                runTest(file);
+            }
+        }
     }
-};
 
-class OneDimNpyReadTest : public DBTest {
-public:
-    std::string getInputDir() override { return TestHelper::appendKuzuRootPath("dataset/npy-1d/"); }
+private:
+    TestConfig testConfig;
 };
 
-TEST_F(LongStringPKTest, LongStringPKTest) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/long_string_pk/long_string_pk.test"));
-}
-
-TEST_F(TinySnbReadTest, Match) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/node.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/one_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/two_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/three_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/four_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/multi_query_part.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/multi_label.test"));
-}
-
-TEST_F(TinySnbReadTest, Filter) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/node.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/one_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/two_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/four_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/five_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/multi_query_part.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/multi_label.test"));
-}
-
-TEST_F(TinySnbReadTest, AccHJ) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/acc/acc_hj.test"));
-}
-
-TEST_F(TinySnbReadTest, Function) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/function/offset.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/function/date.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/function/timestamp.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/function/interval.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/function/list.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/function/arithmetic.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/function/boolean.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/function/string.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/function/cast.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/function/case.test"));
-}
-
-TEST_F(TinySnbReadTest, Agg) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/agg/simple.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/agg/hash.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/agg/distinct_agg.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/agg/multi_query_part.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/agg/multi_label.test"));
-}
-
-TEST_F(TinySnbReadTest, Cyclic) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/cyclic/single_label.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/cyclic/multi_label.test"));
-}
-
-TEST_F(TinySnbReadTest, Projection) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/projection/single_label.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/projection/skip_limit.test"));
-    runTest(
-        TestHelper::appendKuzuRootPath("test/test_files/tinysnb/projection/multi_query_part.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/projection/multi_label.test"));
-}
-
-TEST_F(TinySnbReadTest, Subquery) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/subquery/exists.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/subquery/multi_label.test"));
-}
-
-TEST_F(TinySnbReadTest, OptionalMatch) {
-    runTest(TestHelper::appendKuzuRootPath(
-        "test/test_files/tinysnb/optional_match/optional_match.test"));
-    runTest(
-        TestHelper::appendKuzuRootPath("test/test_files/tinysnb/optional_match/multi_label.test"));
-}
-
-TEST_F(TinySnbReadTest, OrderBy) {
-    runTestAndCheckOrder(
-        TestHelper::appendKuzuRootPath("test/test_files/tinysnb/order_by/single_label.test"));
-    runTestAndCheckOrder(
-        TestHelper::appendKuzuRootPath("test/test_files/tinysnb/order_by/multi_label.test"));
-}
-
-TEST_F(TinySnbReadTest, Union) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/union/union.test"));
-}
-
-TEST_F(TinySnbReadTest, Unwind) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/unwind/unwind.test"));
-}
-
-TEST_F(TinySnbReadTest, VarLengthExtendTests) {
-    runTest(TestHelper::appendKuzuRootPath(
-        "test/test_files/tinysnb/var_length_extend/var_length_adj_list_extend.test"));
-    runTest(TestHelper::appendKuzuRootPath(
-        "test/test_files/tinysnb/var_length_extend/var_length_column_extend.test"));
-}
-
-TEST_F(OneDimNpyReadTest, Match) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/npy-1d/match.test"));
+void registerTests(const std::vector<TestConfig> testConfig) {
+    for (auto testItem : testConfig) {
+        testing::RegisterTest(testItem.testGroup.c_str(), testItem.testName.c_str(), nullptr,
+            nullptr, __FILE__, __LINE__,
+            [=]() -> DBTest* { return new EndToEndReadTest(testItem); });
+    }
 }
 
-class SingleSourceShortestPathTest : public DBTest {
-public:
-    std::string getInputDir() override {
-        return TestHelper::appendKuzuRootPath("dataset/shortest-path-tests/");
+TestConfig parseTestGroup(const std::string& path) {
+    auto testConfig = TestHelper::parseGroupFile(path + "/test.group");
+    for (const auto& entry : std::filesystem::directory_iterator(path)) {
+        if (entry.is_regular_file() && FileUtils::getFileExtension(entry) == ".test") {
+            testConfig.files.push_back(entry.path().string());
+        }
     }
-};
-
-TEST_F(SingleSourceShortestPathTest, BFS_SSSP) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/shortest_path/bfs_sssp.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/shortest_path/bfs_sssp_large.test"));
+    return testConfig;
 }
 
-class TinySnbSerialReadTest : public DBTest {
-public:
-    std::string getInputDir() override {
-        return TestHelper::appendKuzuRootPath("dataset/tinysnb-serial/");
+void scanTestFiles(const std::string& path, std::vector<TestConfig>& configs) {
+    for (const auto& directory : FileUtils::findAllDirectories(path)) {
+        if (FileUtils::fileOrPathExists(directory + "/test.group")) {
+            TestConfig config = parseTestGroup(directory);
+            configs.push_back(config);
+        }
     }
-};
-
-TEST_F(TinySnbSerialReadTest, Match) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/node.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/one_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/two_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/three_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/four_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/multi_query_part.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/match/multi_label.test"));
 }
 
-TEST_F(TinySnbSerialReadTest, Filter) {
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/node.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/one_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/two_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/four_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/five_hop.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/multi_query_part.test"));
-    runTest(TestHelper::appendKuzuRootPath("test/test_files/tinysnb/filter/multi_label.test"));
+int main(int argc, char** argv) {
+    testing::InitGoogleTest(&argc, argv);
+    std::vector<TestConfig> configs;
+    scanTestFiles(TestHelper::appendKuzuRootPath("test/test_files"), configs);
+    registerTests(configs);
+    return RUN_ALL_TESTS();
 }

test/test_files/long_string_pk/test.group

@@ -0,0 +1,4 @@
+-GROUP LongStringPKTest
+-TEST LongStringPKTest
+-DATASET long-string-pk-tests
+-READ_ONLY TRUE

test/test_files/npy-1d/test.group

@@ -0,0 +1,4 @@
+-GROUP OneDimNpyReadTest
+-TEST Match
+-DATASET npy-1d
+-READ_ONLY TRUE

test/test_files/shortest_path/test.group

@@ -0,0 +1,4 @@
+-GROUP SingleSourceShortestPathTest
+-TEST BFS_SSSP
+-DATASET shortest-path-tests
+-READ_ONLY TRUE

test/test_files/tinysnb/acc/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST Acc
+-DATASET tinysnb
+-READ_ONLY TRUE

test/test_files/tinysnb/agg/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST Agg
+-DATASET tinysnb
+-READ_ONLY TRUE

test/test_files/tinysnb/cyclic/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST Cyclic
+-DATASET tinysnb
+-READ_ONLY TRUE

test/test_files/tinysnb/filter/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST Filter
+-DATASET tinysnb
+-READ_ONLY TRUE

test/test_files/tinysnb/function/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST Function
+-DATASET tinysnb
+-READ_ONLY TRUE

test/test_files/tinysnb/match/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST Match
+-DATASET tinysnb
+-READ_ONLY TRUE

test/test_files/tinysnb/optional_match/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST OptionalMatch
+-DATASET tinysnb
+-READ_ONLY TRUE

test/test_files/tinysnb/order_by/test.group

@@ -0,0 +1,5 @@
+-GROUP TinySnbReadTest
+-TEST OrderBy
+-DATASET tinysnb
+-READ_ONLY TRUE
+-CHECK_ORDER TRUE

test/test_files/tinysnb/projection/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST Projection
+-DATASET tinysnb
+-READ_ONLY TRUE

test/test_files/tinysnb/subquery/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST Subquery
+-DATASET tinysnb
+-READ_ONLY TRUE

test/test_files/tinysnb/union/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST Union
+-DATASET tinysnb
+-READ_ONLY TRUE

test/test_files/tinysnb/unwind/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST Unwind
+-DATASET tinysnb
+-READ_ONLY TRUE

test/test_files/tinysnb/var_length_extend/test.group

@@ -0,0 +1,4 @@
+-GROUP TinySnbReadTest
+-TEST VarLengthExtendTests
+-DATASET tinysnb
+-READ_ONLY TRUE