Enable multiple test cases inside one test file (kuzudb#1567)

This implementation enables using multiple -CASE inside the same test file so GoogleTest registers multiple test cases based on this parameter.
benjaminwinger · May 26, 2023 · facf6fd · facf6fd
1 parent 0b076bc
commit facf6fd
Show file tree

Hide file tree

Showing 64 changed files with 183 additions and 126 deletions.
diff --git a/test/include/test_runner/test_case.h → test/include/test_runner/test_group.h b/test/include/test_runner/test_case.h → test/include/test_runner/test_group.h
@@ -19,18 +19,22 @@ struct TestStatement {
     bool checkOutputOrder = false;
 };
 
-struct TestCase {
+// Test group is a collection of test cases in a single file.
+// Each test case can have multiple statements (TestStatement).
+// In GoogleTest, the behaviour would be equivalent to:
+// TEST(TestGroup.group, TestGroup.testCases[name])
+struct TestGroup {
     std::string group;
-    std::string name;
     std::string dataset;
-    std::vector<std::unique_ptr<TestStatement>> statements;
-    std::unordered_map<std::string, std::vector<std::unique_ptr<TestStatement>>> variableStatements;
+    std::unordered_map<std::string, std::vector<std::unique_ptr<TestStatement>>> testCases;
+    std::unordered_map<std::string, std::vector<std::unique_ptr<TestStatement>>>
+        testCasesStatementBlocks;
 
     bool skipTest = false;
 
-    bool isValid() const { return !group.empty() && !name.empty() && !dataset.empty(); }
+    bool isValid() const { return !group.empty() && !dataset.empty(); }
 
-    bool hasStatements() const { return !statements.empty(); }
+    bool hasStatements() const { return !testCases.empty(); }
 };
 
 } // namespace testing

diff --git a/test/include/test_runner/test_parser.h b/test/include/test_runner/test_parser.h
@@ -1,7 +1,7 @@
 #include <cstring>
 
 #include "common/file_utils.h"
-#include "test_runner/test_case.h"
+#include "test_runner/test_group.h"
 
 namespace kuzu {
 namespace testing {
@@ -47,16 +47,16 @@ class LogicToken {
 
 class TestParser {
 public:
-    TestParser() : testCase(std::make_unique<TestCase>()) {}
+    TestParser() : testGroup(std::make_unique<TestGroup>()) {}
 
-    std::unique_ptr<TestCase> parseTestFile(const std::string& path);
+    std::unique_ptr<TestGroup> parseTestFile(const std::string& path);
 
 private:
     std::ifstream fileStream;
     std::string line;
     std::string name;
     LogicToken currentToken;
-    std::unique_ptr<TestCase> testCase;
+    std::unique_ptr<TestGroup> testGroup;
 
     std::string paramsToString();
 
@@ -72,7 +72,7 @@ class TestParser {
 
     void extractStatementBlock();
 
-    void addStatementBlock(const std::string& blockName);
+    void addStatementBlock(const std::string& blockName, const std::string& testGroupName);
 
     inline bool endOfFile() { return fileStream.eof(); }
 
@@ -86,7 +86,7 @@ class TestParser {
 
     TestStatement* extractStatement(TestStatement* currentStatement);
 
-    TestStatement* addNewStatement();
+    TestStatement* addNewStatement(std::string& name);
 };
 
 } // namespace testing

diff --git a/test/include/test_runner/test_runner.h b/test/include/test_runner/test_runner.h
@@ -3,7 +3,7 @@
 #include "parser/parser.h"
 #include "planner/logical_plan/logical_plan_util.h"
 #include "planner/planner.h"
-#include "test_runner/test_case.h"
+#include "test_runner/test_group.h"
 
 namespace kuzu {
 namespace testing {

diff --git a/test/runner/e2e_read_test.cpp b/test/runner/e2e_read_test.cpp
@@ -8,41 +8,44 @@ using namespace kuzu::common;
 class EndToEndReadTest : public DBTest {
 public:
     explicit EndToEndReadTest(
-        std::string dataset, std::vector<std::unique_ptr<TestStatement>> statements)
-        : dataset{std::move(dataset)}, statements{std::move(statements)} {}
+        std::string dataset, std::vector<std::unique_ptr<TestStatement>> testStatements)
+        : dataset{dataset}, testStatements{std::move(testStatements)} {}
 
     std::string getInputDir() override {
         return TestHelper::appendKuzuRootPath("dataset/" + dataset + "/");
     }
-    void TestBody() override { runTest(statements); }
+    void TestBody() override { runTest(testStatements); }
 
 private:
     std::string dataset;
-    std::vector<std::unique_ptr<TestStatement>> statements;
+    std::vector<std::unique_ptr<TestStatement>> testStatements;
 };
 
-void parseAndRegisterTestCase(const std::string& path) {
+void parseAndRegisterTestGroup(const std::string& path) {
     auto testParser = std::make_unique<TestParser>();
-    auto testCase = std::move(testParser->parseTestFile(path));
-    if (testCase->isValid() && testCase->hasStatements()) {
-        auto dataset = testCase->dataset;
-        auto statements = std::move(testCase->statements);
-        testing::RegisterTest(testCase->group.c_str(), testCase->name.c_str(), nullptr, nullptr,
-            __FILE__, __LINE__,
-            [dataset = std::move(dataset), statements = std::move(statements)]() mutable
-            -> DBTest* { return new EndToEndReadTest(std::move(dataset), std::move(statements)); });
+    auto testGroup = std::move(testParser->parseTestFile(path));
+    if (testGroup->isValid() && testGroup->hasStatements()) {
+        auto dataset = testGroup->dataset;
+        auto testCases = std::move(testGroup->testCases);
+        for (auto& [testCaseName, testStatements] : testCases) {
+            testing::RegisterTest(testGroup->group.c_str(), testCaseName.c_str(), nullptr, nullptr,
+                __FILE__, __LINE__,
+                [dataset, testStatements = std::move(testStatements)]() mutable -> DBTest* {
+                    return new EndToEndReadTest(dataset, std::move(testStatements));
+                });
+        }
     }
 }
 
 void scanTestFiles(const std::string& path) {
     if (std::filesystem::is_regular_file(path)) {
-        parseAndRegisterTestCase(path);
+        parseAndRegisterTestGroup(path);
         return;
     }
     for (const auto& entry : std::filesystem::recursive_directory_iterator(path)) {
         if (!entry.is_regular_file() || FileUtils::getFileExtension(entry) != ".test")
             continue;
-        parseAndRegisterTestCase(entry.path().string());
+        parseAndRegisterTestGroup(entry.path().string());
     }
 }
 

diff --git a/test/test_files/copy/csv/copy_node.test b/test/test_files/copy/csv/copy_node.test
@@ -1,9 +1,10 @@
 -GROUP CopyNodeFromCSVTest
--TEST CopyNodeTest
 -DATASET copy-test/node/csv
 
 --
 
+-CASE CopyNodeTest
+
 -NAME SubsetTest
 -QUERY MATCH (row:tableOfTypes) WHERE row.id >= 20 AND row.id <= 24 RETURN *;
 ---- 5

diff --git a/test/test_files/copy/parquet/copy_node.test b/test/test_files/copy/parquet/copy_node.test
@@ -1,9 +1,10 @@
 -GROUP CopyNodeFromParquetTest
--TEST CopyNodeTest
 -DATASET copy-test/node/parquet
 
 --
 
+-CASE CopyNodeTest
+
 -NAME SubsetTest
 -QUERY MATCH (row:tableOfTypes) WHERE row.id >= 20 AND row.id <= 24 RETURN *;
 ---- 5

diff --git a/test/test_files/long_string_pk/long_string_pk.test b/test/test_files/long_string_pk/long_string_pk.test
@@ -1,9 +1,10 @@
 -GROUP LongStringPKTest
--TEST LongStringPKTest
 -DATASET long-string-pk-tests
 
 --
 
+-CASE LongStringPKTest
+
 -NAME LongStringPKTest
 -QUERY MATCH (a:Person)-[e:Knows]->(b:Person) WHERE a.name = "AAAAAAAAAAAAAAAAAAAA" RETURN COUNT(*)
 ---- 1

diff --git a/test/test_files/npy-1d/match.test b/test/test_files/npy-1d/match.test
@@ -1,9 +1,10 @@
 -GROUP OneDimNpyReadTest
--TEST Match
 -DATASET npy-1d
 
 --
 
+-CASE MatchNpy-1d
+
 -NAME count
 -QUERY MATCH (n:npytable) RETURN COUNT(*)
 ---- 1

diff --git a/test/test_files/order_by/order_by.test b/test/test_files/order_by/order_by.test
@@ -9,11 +9,12 @@
 # In total: we need 23 pages for this test case
 
 -GROUP OrderByTests
--TEST OrderByLargeDatasetTest
 -DATASET order-by-tests
 
 --
 
+-CASE OrderByLargeDatasetTest
+
 -NAME OrderByWithLimitTest
 -QUERY MATCH (p:person) RETURN p.balance ORDER BY p.balance limit 25
 -PARALLELISM 6

diff --git a/test/test_files/read_list/large_adj_list.test b/test/test_files/read_list/large_adj_list.test
@@ -1,9 +1,10 @@
 -GROUP EndToEndReadLargeListsTest
--TEST LargeAdjListTest
 -DATASET read-list-tests/large-list
 
 --
 
+-CASE LargeAdjListTest
+
 -NAME NodeP0
 -QUERY MATCH (a:person)-[:knows]->(b:person) WHERE a.ID = 0 RETURN COUNT(*)
 -ENUMERATE

diff --git a/test/test_files/read_list/large_list_sub_query.test b/test/test_files/read_list/large_list_sub_query.test
@@ -1,9 +1,10 @@
 -GROUP EndToEndReadListsSubQueryTest
--TEST LargeListSubQueryTest
 -DATASET read-list-tests/large-list-sub-query-tests
 
 --
 
+-CASE LargeListSubQueryTest
+
 -NAME KnowsVeryLargeAdjListSubQueryTest
 -QUERY MATCH (a:person) WHERE EXISTS { MATCH (a)-[:knows]->(b:person)-[:knows]->(c:person) } RETURN count(*)
 -PARALLELISM 1

diff --git a/test/test_files/read_list/small_large_property_list_reading.test b/test/test_files/read_list/small_large_property_list_reading.test
@@ -1,9 +1,10 @@
 -GROUP EndToEndReadLargeListsTest
--TEST PropLists4BytesPerEdgeTest
 -DATASET read-list-tests/large-list
 
 --
 
+-CASE PropLists4BytesPerEdgeTest
+
 # Result is 3 and 0 is omitted because 0->0 edge's int64Prop is null.
 -NAME ReadingLargeListOneHop
 -QUERY MATCH (a:person)-[e1:knows]->(b:person) WHERE a.ID = 0 AND e1.int64Prop < 4  RETURN e1.int64Prop

diff --git a/test/test_files/read_list/var_length_large_adj_list_extend.test b/test/test_files/read_list/var_length_large_adj_list_extend.test
@@ -10,11 +10,12 @@
 # So the nth level will contain 5K*(n+1) + 1 many number of nodes.
 
 -GROUP EndToEndReadLargeListsTest
--TEST VarLengthExtendLargeAdjListTest
 -DATASET read-list-tests/large-list
 
 --
 
+-CASE VarLengthExtendLargeAdjListTest
+
 -NAME KnowsVeryLargeAdjListTest
 -QUERY MATCH (a:person)-[:knows*2..4]->(b:person) RETURN COUNT(*)
 ---- 1

diff --git a/test/test_files/shortest_path/bfs_sssp.test b/test/test_files/shortest_path/bfs_sssp.test
@@ -1,9 +1,10 @@
 -GROUP SingleSourceShortestPathTest
--TEST BFS_SSSP
 -DATASET shortest-path-tests
 
 --
 
+-CASE BFS_SSSP
+
 -NAME SingleSourceAllDestinationsSSP
 -QUERY MATCH (a:person)-[r:knows* SHORTEST 1..30]->(b:person) WHERE a.fName = 'Alice' RETURN a.fName, b.fName, r
 ---- 7

diff --git a/test/test_files/shortest_path/bfs_sssp_large.test b/test/test_files/shortest_path/bfs_sssp_large.test
@@ -5,11 +5,12 @@
 # Distance between Alice11 and Alice100 is 9 because 11 -> 11,20,30,40,50,60,70,80,90,100
 
 -GROUP SingleSourceShortestPathTest
--TEST BFS_SSSP
 -DATASET shortest-path-tests
 
 --
 
+-CASE BFS_SSSP
+
 -NAME SingleSrcAllDstQueryLarge
 -QUERY MATCH (a:person)-[r:knows* SHORTEST 1..30]->(b:person) WHERE a.fName = 'Alice11' RETURN a.fName, b.fName, length(r)
 ---- 300

diff --git a/test/test_files/tinysnb/acc/acc_hj.test b/test/test_files/tinysnb/acc/acc_hj.test
@@ -1,9 +1,10 @@
 -GROUP TinySnbReadTest
--TEST Acc
 -DATASET tinysnb
 
 --
 
+-CASE Acc
+
 -NAME AspBasic
 -QUERY MATCH (a:person)-[e1:knows]->(b:person) WHERE a.age > 35 RETURN b.fName
 -ENCODED_JOIN HJ(b._id){E(b)S(a)}{S(b)}

diff --git a/test/test_files/tinysnb/agg/distinct_agg.test b/test/test_files/tinysnb/agg/distinct_agg.test
@@ -1,9 +1,10 @@
 -GROUP TinySnbReadTest
--TEST Agg
 -DATASET tinysnb
 
 --
 
+-CASE AggDistinct
+
 -NAME SingleNodeDistinctAggTest1
 -QUERY MATCH (a:person) RETURN COUNT(DISTINCT a.gender), COUNT(DISTINCT a.age)
 ---- 1

diff --git a/test/test_files/tinysnb/agg/hash.test b/test/test_files/tinysnb/agg/hash.test
@@ -1,9 +1,10 @@
 -GROUP TinySnbReadTest
--TEST AggHash
 -DATASET tinysnb
 
 --
 
+-CASE AggHash
+
 -NAME SingleNodeAggTest
 -QUERY MATCH (a:person) RETURN a.age, MIN(a.ID), AVG(a.eyeSight), COUNT(*)
 ---- 7

diff --git a/test/test_files/tinysnb/agg/multi_label.test b/test/test_files/tinysnb/agg/multi_label.test
@@ -1,9 +1,10 @@
 -GROUP TinySnbReadTest
--TEST AggMultiLabel
 -DATASET tinysnb
 
 --
 
+-CASE AggMultiLabel
+
 -NAME MultiLabelAggTest1
 -QUERY MATCH (a:person)<-[e1:marries|studyAt]-(b:person)-[e2:knows]->(c:person) WHERE b.ID = 7 RETURN ID(e1), COUNT(*), MIN(e2.date)
 ---- 1

diff --git a/test/test_files/tinysnb/agg/multi_query_part.test b/test/test_files/tinysnb/agg/multi_query_part.test
@@ -1,9 +1,10 @@
 -GROUP TinySnbReadTest
--TEST Agg
 -DATASET tinysnb
 
 --
 
+-CASE AggMultiQueryPart
+
 -NAME SimpleAvgWithFilterMultiQueryTest
 -QUERY MATCH (a:person) WHERE a.birthdate = date('1980-10-26') WITH AVG(a.age) AS avgAge, AVG(a.eyeSight) AS avgEyeSight RETURN avgAge > avgEyeSight
 ---- 1

diff --git a/test/test_files/tinysnb/agg/simple.test b/test/test_files/tinysnb/agg/simple.test
@@ -1,9 +1,10 @@
 -GROUP TinySnbReadTest
--TEST Agg
 -DATASET tinysnb
 
 --
 
+-CASE AggSimple
+
 -NAME OneHopSimpleAggTest
 -QUERY MATCH (a:person)-[:knows]->(b:person)-[:knows]->(c:person) RETURN COUNT(a.ID), MIN(a.fName), MAX(c.ID)
 -PARALLELISM 8

diff --git a/test/test_files/tinysnb/cyclic/multi_label.test b/test/test_files/tinysnb/cyclic/multi_label.test
@@ -1,9 +1,10 @@
 -GROUP TinySnbReadTest
--TEST Cyclic
 -DATASET tinysnb
 
 --
 
+-CASE CyclicMultiLabel
+
 -NAME MultiLabelCyclicTest
 -QUERY MATCH (a:person)-[:workAt|:knows|:studyAt]->(b:organisation), (a)-[:workAt]->(b) RETURN COUNT(*)
 ---- 1

diff --git a/test/test_files/tinysnb/cyclic/single_label.test b/test/test_files/tinysnb/cyclic/single_label.test
@@ -1,9 +1,10 @@
 -GROUP TinySnbReadTest
--TEST Cyclic
 -DATASET tinysnb
 
 --
 
+-CASE CyclicSingleLabel
+
 -NAME TwoNodeCycleTest
 -QUERY MATCH (a:person)-[:knows]->(b:person), (b)-[:knows]->(a) RETURN COUNT(*)
 -ENUMERATE

diff --git a/test/test_files/tinysnb/filter/five_hop.test → ...t_files/tinysnb/filter/four_five_hop.test b/test/test_files/tinysnb/filter/five_hop.test → ...t_files/tinysnb/filter/four_five_hop.test
@@ -1,11 +1,19 @@
 -GROUP TinySnbReadTest
--TEST Filter
 -DATASET tinysnb
 
 --
 
+-CASE FourHopFilter
+-NAME FourHopKnowsFilterTest
+-QUERY MATCH (a:person)-[e1:knows]->(b:person)-[e2:knows]->(c:person)-[e3:knows]->(d:person)-[e4:knows]->(e:person) WHERE a.fName = 'Alice' OR e.fName = 'Alice' RETURN COUNT(*)
+-ENUMERATE
+---- 1
+141
+
+-CASE FiveHopFilter
 -NAME FiveHopKnowsFilterTest
 -QUERY MATCH (a:person)-[e1:knows]->(b:person)-[e2:knows]->(c:person)-[e3:knows]->(d:person)-[e4:knows]->(e:person)-[:knows]->(f:person) WHERE a.age = 35 AND f.age = a.age  RETURN COUNT(*)
 -ENUMERATE
 ---- 1
 60
+